1//===- RISCVInsertVSETVLI.cpp - Insert VSETVLI instructions ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a function pass that inserts VSETVLI instructions where
10// needed and expands the vl outputs of VLEFF/VLSEGFF to PseudoReadVL
11// instructions.
12//
13// This pass consists of 3 phases:
14//
15// Phase 1 collects how each basic block affects VL/VTYPE.
16//
17// Phase 2 uses the information from phase 1 to do a data flow analysis to
18// propagate the VL/VTYPE changes through the function. This gives us the
19// VL/VTYPE at the start of each basic block.
20//
21// Phase 3 inserts VSETVLI instructions in each basic block. Information from
22// phase 2 is used to prevent inserting a VSETVLI before the first vector
23// instruction in the block if possible.
24//
25//===----------------------------------------------------------------------===//
26
27#include "RISCV.h"
28#include "RISCVSubtarget.h"
29#include "RISCVVSETVLIInfoAnalysis.h"
30#include "llvm/ADT/PostOrderIterator.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/CodeGen/LiveDebugVariables.h"
33#include "llvm/CodeGen/LiveIntervals.h"
34#include "llvm/CodeGen/LiveStacks.h"
35#include "llvm/CodeGen/MachineFunctionPass.h"
36#include "llvm/CodeGen/RegisterClassInfo.h"
37#include <queue>
38using namespace llvm;
39using namespace RISCV;
40
41#define DEBUG_TYPE "riscv-insert-vsetvli"
42#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
43
44STATISTIC(NumInsertedVSETVL, "Number of VSETVL inst inserted");
45STATISTIC(NumCoalescedVSETVL, "Number of VSETVL inst coalesced");
46
47static cl::opt<bool> EnsureWholeVectorRegisterMoveValidVTYPE(
48 DEBUG_TYPE "-whole-vector-register-move-valid-vtype", cl::Hidden,
49 cl::desc("Insert vsetvlis before vmvNr.vs to ensure vtype is valid and "
50 "vill is cleared"),
51 cl::init(Val: true));
52
53namespace {
54
55/// Given a virtual register \p Reg, return the corresponding VNInfo for it.
56/// This will return nullptr if the virtual register is an implicit_def or
57/// if LiveIntervals is not available.
58static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
59 const LiveIntervals *LIS) {
60 assert(Reg.isVirtual());
61 if (!LIS)
62 return nullptr;
63 auto &LI = LIS->getInterval(Reg);
64 SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
65 return LI.getVNInfoBefore(Idx: SI);
66}
67
68static MachineOperand &getVLOp(MachineInstr &MI) {
69 return MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
70}
71
72struct BlockData {
73 // The VSETVLIInfo that represents the VL/VTYPE settings on exit from this
74 // block. Calculated in Phase 2.
75 VSETVLIInfo Exit;
76
77 // The VSETVLIInfo that represents the VL/VTYPE settings from all predecessor
78 // blocks. Calculated in Phase 2, and used by Phase 3.
79 VSETVLIInfo Pred;
80
81 // Keeps track of whether the block is already in the queue.
82 bool InQueue = false;
83
84 BlockData() = default;
85};
86
87enum TKTMMode {
88 VSETTK = 0,
89 VSETTM = 1,
90};
91
92class RISCVInsertVSETVLI : public MachineFunctionPass {
93 const RISCVSubtarget *ST;
94 const TargetInstrInfo *TII;
95 MachineRegisterInfo *MRI;
96 // Possibly null!
97 LiveIntervals *LIS;
98 RISCVVSETVLIInfoAnalysis VIA;
99
100 std::vector<BlockData> BlockInfo;
101 std::queue<const MachineBasicBlock *> WorkList;
102
103public:
104 static char ID;
105
106 RISCVInsertVSETVLI() : MachineFunctionPass(ID) {}
107 bool runOnMachineFunction(MachineFunction &MF) override;
108
109 void getAnalysisUsage(AnalysisUsage &AU) const override {
110 AU.setPreservesCFG();
111
112 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
113 AU.addPreserved<LiveIntervalsWrapperPass>();
114 AU.addPreserved<SlotIndexesWrapperPass>();
115 AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
116 AU.addPreserved<LiveStacksWrapperLegacy>();
117
118 MachineFunctionPass::getAnalysisUsage(AU);
119 }
120
121 StringRef getPassName() const override { return RISCV_INSERT_VSETVLI_NAME; }
122
123private:
124 bool needVSETVLI(const DemandedFields &Used, const VSETVLIInfo &Require,
125 const VSETVLIInfo &CurInfo) const;
126 bool needVSETVLIPHI(const VSETVLIInfo &Require,
127 const MachineBasicBlock &MBB) const;
128 void insertVSETVLI(MachineBasicBlock &MBB,
129 MachineBasicBlock::iterator InsertPt, DebugLoc DL,
130 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);
131
132 void transferBefore(VSETVLIInfo &Info, const MachineInstr &MI) const;
133 void transferAfter(VSETVLIInfo &Info, const MachineInstr &MI) const;
134 bool computeVLVTYPEChanges(const MachineBasicBlock &MBB,
135 VSETVLIInfo &Info) const;
136 void computeIncomingVLVTYPE(const MachineBasicBlock &MBB);
137 void emitVSETVLIs(MachineBasicBlock &MBB);
138 void doPRE(MachineBasicBlock &MBB);
139 void insertReadVL(MachineBasicBlock &MBB);
140
141 bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI,
142 const DemandedFields &Used,
143 MachineInstr *&AVLDefToMove) const;
144 void coalesceVSETVLIs(MachineBasicBlock &MBB) const;
145 bool canMutatePriorConfigWithTWiden(const MachineInstr &PrevMI,
146 const MachineInstr &MI) const;
147 void coalesceVSETVLIsForTWiden(MachineBasicBlock &MBB) const;
148 bool insertVSETMTK(MachineBasicBlock &MBB, TKTMMode Mode) const;
149};
150
151} // end anonymous namespace
152
153char RISCVInsertVSETVLI::ID = 0;
154char &llvm::RISCVInsertVSETVLIID = RISCVInsertVSETVLI::ID;
155
156INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME,
157 false, false)
158
159void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
160 MachineBasicBlock::iterator InsertPt,
161 DebugLoc DL, const VSETVLIInfo &Info,
162 const VSETVLIInfo &PrevInfo) {
163 ++NumInsertedVSETVL;
164
165 if (PrevInfo.isKnown()) {
166 // Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same
167 // VLMAX.
168 if (Info.hasSameAVL(Other: PrevInfo) && Info.hasSameVLMAX(Other: PrevInfo)) {
169 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
170 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0X0
171 : RISCV::PseudoVSETVLIX0X0))
172 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
173 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
174 .addImm(Val: Info.encodeVTYPE())
175 .addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
176 if (LIS)
177 LIS->InsertMachineInstrInMaps(MI&: *MI);
178 return;
179 }
180
181 // If our AVL is a virtual register, it might be defined by a VSET(I)VLI. If
182 // it has the same VLMAX we want and the last VL/VTYPE we observed is the
183 // same, we can use the X0, X0 form.
184 if (Info.hasSameVLMAX(Other: PrevInfo) && Info.hasAVLReg()) {
185 if (const MachineInstr *DefMI = Info.getAVLDefMI(LIS);
186 DefMI && RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI)) {
187 VSETVLIInfo DefInfo = VIA.getInfoForVSETVLI(MI: *DefMI);
188 if (DefInfo.hasSameAVL(Other: PrevInfo) && DefInfo.hasSameVLMAX(Other: PrevInfo)) {
189 auto MI =
190 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
191 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0X0
192 : RISCV::PseudoVSETVLIX0X0))
193 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
194 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
195 .addImm(Val: Info.encodeVTYPE())
196 .addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
197 if (LIS)
198 LIS->InsertMachineInstrInMaps(MI&: *MI);
199 return;
200 }
201 }
202 }
203 }
204
205 if (Info.hasAVLImm()) {
206 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoVSETIVLI))
207 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
208 .addImm(Val: Info.getAVLImm())
209 .addImm(Val: Info.encodeVTYPE());
210 if (LIS)
211 LIS->InsertMachineInstrInMaps(MI&: *MI);
212 return;
213 }
214
215 if (Info.hasAVLVLMAX()) {
216 Register DestReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
217 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
218 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0
219 : RISCV::PseudoVSETVLIX0))
220 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Dead)
221 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
222 .addImm(Val: Info.encodeVTYPE());
223 if (LIS) {
224 LIS->InsertMachineInstrInMaps(MI&: *MI);
225 LIS->createAndComputeVirtRegInterval(Reg: DestReg);
226 }
227 return;
228 }
229
230 Register AVLReg = Info.getAVLReg();
231 MRI->constrainRegClass(Reg: AVLReg, RC: &RISCV::GPRNoX0RegClass);
232 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
233 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNT
234 : RISCV::PseudoVSETVLI))
235 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
236 .addReg(RegNo: AVLReg)
237 .addImm(Val: Info.encodeVTYPE());
238 if (LIS) {
239 LIS->InsertMachineInstrInMaps(MI&: *MI);
240 LiveInterval &LI = LIS->getInterval(Reg: AVLReg);
241 SlotIndex SI = LIS->getInstructionIndex(Instr: *MI).getRegSlot();
242 const VNInfo *CurVNI = Info.getAVLVNInfo();
243 // If the AVL value isn't live at MI, do a quick check to see if it's easily
244 // extendable. Otherwise, we need to copy it.
245 if (LI.getVNInfoBefore(Idx: SI) != CurVNI) {
246 if (!LI.liveAt(index: SI) && LI.containsOneValue())
247 LIS->extendToIndices(LR&: LI, Indices: SI);
248 else {
249 Register AVLCopyReg =
250 MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
251 MachineBasicBlock *MBB = LIS->getMBBFromIndex(index: CurVNI->def);
252 MachineBasicBlock::iterator II;
253 if (CurVNI->isPHIDef())
254 II = MBB->getFirstNonPHI();
255 else {
256 II = LIS->getInstructionFromIndex(index: CurVNI->def);
257 II = std::next(x: II);
258 }
259 assert(II.isValid());
260 auto AVLCopy = BuildMI(BB&: *MBB, I: II, MIMD: DL, MCID: TII->get(Opcode: RISCV::COPY), DestReg: AVLCopyReg)
261 .addReg(RegNo: AVLReg);
262 LIS->InsertMachineInstrInMaps(MI&: *AVLCopy);
263 MI->getOperand(i: 1).setReg(AVLCopyReg);
264 LIS->createAndComputeVirtRegInterval(Reg: AVLCopyReg);
265 }
266 }
267 }
268}
269
270/// Return true if a VSETVLI is required to transition from CurInfo to Require
271/// given a set of DemandedFields \p Used.
272bool RISCVInsertVSETVLI::needVSETVLI(const DemandedFields &Used,
273 const VSETVLIInfo &Require,
274 const VSETVLIInfo &CurInfo) const {
275 if (!CurInfo.isKnown() || CurInfo.hasSEWLMULRatioOnly())
276 return true;
277
278 if (CurInfo.isCompatible(Used, Require, LIS))
279 return false;
280
281 return true;
282}
283
284// If we don't use LMUL or the SEW/LMUL ratio, then adjust LMUL so that we
285// maintain the SEW/LMUL ratio. This allows us to eliminate VL toggles in more
286// places.
287static VSETVLIInfo adjustIncoming(const VSETVLIInfo &PrevInfo,
288 const VSETVLIInfo &NewInfo,
289 DemandedFields &Demanded) {
290 VSETVLIInfo Info = NewInfo;
291
292 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isKnown()) {
293 if (auto NewVLMul = RISCVVType::getSameRatioLMUL(Ratio: PrevInfo.getSEWLMULRatio(),
294 EEW: Info.getSEW()))
295 Info.setVLMul(*NewVLMul);
296 Demanded.LMUL = DemandedFields::LMULEqual;
297 }
298
299 return Info;
300}
301
302// Given an incoming state reaching MI, minimally modifies that state so that it
303// is compatible with MI. The resulting state is guaranteed to be semantically
304// legal for MI, but may not be the state requested by MI.
305void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
306 const MachineInstr &MI) const {
307 if (EnsureWholeVectorRegisterMoveValidVTYPE &&
308 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI) &&
309 (!Info.isKnown() || Info.hasSEWLMULRatioOnly())) {
310 // Use an arbitrary but valid AVL and VTYPE so vill will be cleared. It may
311 // be coalesced into another vsetvli since we won't demand any fields.
312 VSETVLIInfo NewInfo; // Need a new VSETVLIInfo to clear SEWLMULRatioOnly
313 NewInfo.setAVLImm(1);
314 NewInfo.setVTYPE(L: RISCVVType::LMUL_1, /*sew*/ S: 8, /*ta*/ TA: true, /*ma*/ MA: true,
315 /*AltFmt*/ Altfmt: false, /*W*/ 0);
316 Info = NewInfo;
317 return;
318 }
319
320 if (!RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags))
321 return;
322
323 DemandedFields Demanded = getDemanded(MI, ST);
324
325 const VSETVLIInfo NewInfo = VIA.computeInfoForInstr(MI);
326 assert(NewInfo.isKnown());
327 if (Info.isValid() && !needVSETVLI(Used: Demanded, Require: NewInfo, CurInfo: Info))
328 return;
329
330 const VSETVLIInfo PrevInfo = Info;
331 if (!Info.isKnown())
332 Info = NewInfo;
333
334 const VSETVLIInfo IncomingInfo = adjustIncoming(PrevInfo, NewInfo, Demanded);
335
336 // If MI only demands that VL has the same zeroness, we only need to set the
337 // AVL if the zeroness differs. This removes a vsetvli entirely if the types
338 // match or allows use of cheaper avl preserving variant if VLMAX doesn't
339 // change. If VLMAX might change, we couldn't use the 'vsetvli x0, x0, vtype"
340 // variant, so we avoid the transform to prevent extending live range of an
341 // avl register operand.
342 // TODO: We can probably relax this for immediates.
343 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(Other: PrevInfo, LIS) &&
344 IncomingInfo.hasSameVLMAX(Other: PrevInfo);
345 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
346 Info.setAVL(IncomingInfo);
347
348 // If we only knew the sew/lmul ratio previously, replace the VTYPE.
349 if (Info.hasSEWLMULRatioOnly()) {
350 VSETVLIInfo RatiolessInfo = IncomingInfo;
351 RatiolessInfo.setAVL(Info);
352 Info = RatiolessInfo;
353 } else {
354 unsigned SEW =
355 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
356 .getSEW();
357 Info.setVTYPE(
358 L: ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
359 .getVLMUL(),
360 S: SEW,
361 // Prefer tail/mask agnostic since it can be relaxed to undisturbed
362 // later if needed.
363 TA: (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
364 IncomingInfo.getTailAgnostic(),
365 MA: (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
366 IncomingInfo.getMaskAgnostic(),
367 // AltFmt requires SEW < 32.
368 Altfmt: (Demanded.AltFmt ? IncomingInfo : Info).getAltFmt() && SEW < 32,
369 W: Demanded.TWiden ? IncomingInfo.getTWiden() : 0);
370 }
371}
372
373// Given a state with which we evaluated MI (see transferBefore above for why
374// this might be different that the state MI requested), modify the state to
375// reflect the changes MI might make.
376void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
377 const MachineInstr &MI) const {
378 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {
379 Info = VIA.getInfoForVSETVLI(MI);
380 return;
381 }
382
383 // SETTM/TK will modify VTYPE, but it only affects the TM/TK bits.
384 // It is safe for other RVV operations.
385 // The TM/TK value will be maintained in insertVSETMTK.
386 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI))
387 return;
388
389 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
390 // Update AVL to vl-output of the fault first load.
391 assert(MI.getOperand(1).getReg().isVirtual());
392 if (LIS) {
393 auto &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
394 SlotIndex SI =
395 LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();
396 VNInfo *VNI = LI.getVNInfoAt(Idx: SI);
397 Info.setAVLRegDef(VNInfo: VNI, AVLReg: MI.getOperand(i: 1).getReg());
398 } else
399 Info.setAVLRegDef(VNInfo: nullptr, AVLReg: MI.getOperand(i: 1).getReg());
400 return;
401 }
402
403 // If this is something that updates VL/VTYPE that we don't know about, set
404 // the state to unknown.
405 if (MI.isCall() || MI.isInlineAsm() ||
406 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
407 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
408 Info = VSETVLIInfo::getUnknown();
409}
410
411bool RISCVInsertVSETVLI::computeVLVTYPEChanges(const MachineBasicBlock &MBB,
412 VSETVLIInfo &Info) const {
413 bool HadVectorOp = false;
414
415 Info = BlockInfo[MBB.getNumber()].Pred;
416 for (const MachineInstr &MI : MBB) {
417 transferBefore(Info, MI);
418
419 if (RISCVInstrInfo::isVectorConfigInstr(MI) ||
420 RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags) ||
421 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI) ||
422 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI))
423 HadVectorOp = true;
424
425 transferAfter(Info, MI);
426 }
427
428 return HadVectorOp;
429}
430
431void RISCVInsertVSETVLI::computeIncomingVLVTYPE(const MachineBasicBlock &MBB) {
432
433 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
434
435 BBInfo.InQueue = false;
436
437 // Start with the previous entry so that we keep the most conservative state
438 // we have ever found.
439 VSETVLIInfo InInfo = BBInfo.Pred;
440 if (MBB.pred_empty()) {
441 // There are no predecessors, so use the default starting status.
442 InInfo.setUnknown();
443 } else {
444 for (MachineBasicBlock *P : MBB.predecessors())
445 InInfo = InInfo.intersect(Other: BlockInfo[P->getNumber()].Exit);
446 }
447
448 // If we don't have any valid predecessor value, wait until we do.
449 if (!InInfo.isValid())
450 return;
451
452 // If no change, no need to rerun block
453 if (InInfo == BBInfo.Pred)
454 return;
455
456 BBInfo.Pred = InInfo;
457 LLVM_DEBUG(dbgs() << "Entry state of " << printMBBReference(MBB)
458 << " changed to " << BBInfo.Pred << "\n");
459
460 // Note: It's tempting to cache the state changes here, but due to the
461 // compatibility checks performed a blocks output state can change based on
462 // the input state. To cache, we'd have to add logic for finding
463 // never-compatible state changes.
464 VSETVLIInfo TmpStatus;
465 computeVLVTYPEChanges(MBB, Info&: TmpStatus);
466
467 // If the new exit value matches the old exit value, we don't need to revisit
468 // any blocks.
469 if (BBInfo.Exit == TmpStatus)
470 return;
471
472 BBInfo.Exit = TmpStatus;
473 LLVM_DEBUG(dbgs() << "Exit state of " << printMBBReference(MBB)
474 << " changed to " << BBInfo.Exit << "\n");
475
476 // Add the successors to the work list so we can propagate the changed exit
477 // status.
478 for (MachineBasicBlock *S : MBB.successors())
479 if (!BlockInfo[S->getNumber()].InQueue) {
480 BlockInfo[S->getNumber()].InQueue = true;
481 WorkList.push(x: S);
482 }
483}
484
485// If we weren't able to prove a vsetvli was directly unneeded, it might still
486// be unneeded if the AVL was a phi node where all incoming values are VL
487// outputs from the last VSETVLI in their respective basic blocks.
488bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
489 const MachineBasicBlock &MBB) const {
490 if (!Require.hasAVLReg())
491 return true;
492
493 if (!LIS)
494 return true;
495
496 // We need the AVL to have been produced by a PHI node in this basic block.
497 const VNInfo *Valno = Require.getAVLVNInfo();
498 if (!Valno->isPHIDef() || LIS->getMBBFromIndex(index: Valno->def) != &MBB)
499 return true;
500
501 const LiveRange &LR = LIS->getInterval(Reg: Require.getAVLReg());
502
503 for (auto *PBB : MBB.predecessors()) {
504 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
505
506 // We need the PHI input to the be the output of a VSET(I)VLI.
507 const VNInfo *Value = LR.getVNInfoBefore(Idx: LIS->getMBBEndIdx(mbb: PBB));
508 if (!Value)
509 return true;
510 MachineInstr *DefMI = LIS->getInstructionFromIndex(index: Value->def);
511 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI))
512 return true;
513
514 // We found a VSET(I)VLI make sure it matches the output of the
515 // predecessor block.
516 VSETVLIInfo DefInfo = VIA.getInfoForVSETVLI(MI: *DefMI);
517 if (DefInfo != PBBExit)
518 return true;
519
520 // Require has the same VL as PBBExit, so if the exit from the
521 // predecessor has the VTYPE we are looking for we might be able
522 // to avoid a VSETVLI.
523 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Other: Require))
524 return true;
525 }
526
527 // If all the incoming values to the PHI checked out, we don't need
528 // to insert a VSETVLI.
529 return false;
530}
531
532void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
533 VSETVLIInfo CurInfo = BlockInfo[MBB.getNumber()].Pred;
534 // Track whether the prefix of the block we've scanned is transparent
535 // (meaning has not yet changed the abstract state).
536 bool PrefixTransparent = true;
537 for (MachineInstr &MI : MBB) {
538 const VSETVLIInfo PrevInfo = CurInfo;
539 transferBefore(Info&: CurInfo, MI);
540
541 // If this is an explicit VSETVLI or VSETIVLI, update our state.
542 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {
543 // Conservatively, mark the VL and VTYPE as live.
544 assert(MI.getOperand(3).getReg() == RISCV::VL &&
545 MI.getOperand(4).getReg() == RISCV::VTYPE &&
546 "Unexpected operands where VL and VTYPE should be");
547 MI.getOperand(i: 3).setIsDead(false);
548 MI.getOperand(i: 4).setIsDead(false);
549 PrefixTransparent = false;
550 }
551
552 if (EnsureWholeVectorRegisterMoveValidVTYPE &&
553 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI)) {
554 if (!PrevInfo.isCompatible(Used: DemandedFields::all(), Require: CurInfo, LIS)) {
555 insertVSETVLI(MBB, InsertPt: MI, DL: MI.getDebugLoc(), Info: CurInfo, PrevInfo);
556 PrefixTransparent = false;
557 }
558 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ false,
559 /*isImp*/ true));
560 }
561
562 uint64_t TSFlags = MI.getDesc().TSFlags;
563 if (RISCVII::hasSEWOp(TSFlags)) {
564 if (!PrevInfo.isCompatible(Used: DemandedFields::all(), Require: CurInfo, LIS)) {
565 // If this is the first implicit state change, and the state change
566 // requested can be proven to produce the same register contents, we
567 // can skip emitting the actual state change and continue as if we
568 // had since we know the GPR result of the implicit state change
569 // wouldn't be used and VL/VTYPE registers are correct. Note that
570 // we *do* need to model the state as if it changed as while the
571 // register contents are unchanged, the abstract model can change.
572 if (!PrefixTransparent || needVSETVLIPHI(Require: CurInfo, MBB))
573 insertVSETVLI(MBB, InsertPt: MI, DL: MI.getDebugLoc(), Info: CurInfo, PrevInfo);
574 PrefixTransparent = false;
575 }
576
577 if (RISCVII::hasVLOp(TSFlags)) {
578 MachineOperand &VLOp = getVLOp(MI);
579 if (VLOp.isReg()) {
580 Register Reg = VLOp.getReg();
581
582 // Erase the AVL operand from the instruction.
583 VLOp.setReg(Register());
584 VLOp.setIsKill(false);
585 if (LIS) {
586 LiveInterval &LI = LIS->getInterval(Reg);
587 SmallVector<MachineInstr *> DeadMIs;
588 LIS->shrinkToUses(li: &LI, dead: &DeadMIs);
589 // We might have separate components that need split due to
590 // needVSETVLIPHI causing us to skip inserting a new VL def.
591 SmallVector<LiveInterval *> SplitLIs;
592 LIS->splitSeparateComponents(LI, SplitLIs);
593
594 // If the AVL was an immediate > 31, then it would have been emitted
595 // as an ADDI. However, the ADDI might not have been used in the
596 // vsetvli, or a vsetvli might not have been emitted, so it may be
597 // dead now.
598 for (MachineInstr *DeadMI : DeadMIs) {
599 if (!TII->isAddImmediate(MI: *DeadMI, Reg))
600 continue;
601 LIS->RemoveMachineInstrFromMaps(MI&: *DeadMI);
602 Register AddReg = DeadMI->getOperand(i: 1).getReg();
603 DeadMI->eraseFromParent();
604 if (AddReg.isVirtual())
605 LIS->shrinkToUses(li: &LIS->getInterval(Reg: AddReg));
606 }
607 }
608 }
609 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VL, /*isDef*/ false,
610 /*isImp*/ true));
611 }
612 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ false,
613 /*isImp*/ true));
614 }
615
616 if (MI.isInlineAsm()) {
617 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VL, /*isDef*/ true,
618 /*isImp*/ true));
619 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ true,
620 /*isImp*/ true));
621 }
622
623 if (MI.isCall() || MI.isInlineAsm() ||
624 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
625 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
626 PrefixTransparent = false;
627
628 transferAfter(Info&: CurInfo, MI);
629 }
630
631 const auto &Info = BlockInfo[MBB.getNumber()];
632 if (CurInfo != Info.Exit) {
633 LLVM_DEBUG(dbgs() << "in block " << printMBBReference(MBB) << "\n");
634 LLVM_DEBUG(dbgs() << " begin state: " << Info.Pred << "\n");
635 LLVM_DEBUG(dbgs() << " expected end state: " << Info.Exit << "\n");
636 LLVM_DEBUG(dbgs() << " actual end state: " << CurInfo << "\n");
637 }
638 assert(CurInfo == Info.Exit && "InsertVSETVLI dataflow invariant violated");
639}
640
641/// Perform simple partial redundancy elimination of the VSETVLI instructions
642/// we're about to insert by looking for cases where we can PRE from the
643/// beginning of one block to the end of one of its predecessors. Specifically,
644/// this is geared to catch the common case of a fixed length vsetvl in a single
645/// block loop when it could execute once in the preheader instead.
646void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {
647 if (!BlockInfo[MBB.getNumber()].Pred.isUnknown())
648 return;
649
650 MachineBasicBlock *UnavailablePred = nullptr;
651 VSETVLIInfo AvailableInfo;
652 for (MachineBasicBlock *P : MBB.predecessors()) {
653 const VSETVLIInfo &PredInfo = BlockInfo[P->getNumber()].Exit;
654 if (PredInfo.isUnknown()) {
655 if (UnavailablePred)
656 return;
657 UnavailablePred = P;
658 } else if (!AvailableInfo.isValid()) {
659 AvailableInfo = PredInfo;
660 } else if (AvailableInfo != PredInfo) {
661 return;
662 }
663 }
664
665 // Unreachable, single pred, or full redundancy. Note that FRE is handled by
666 // phase 3.
667 if (!UnavailablePred || !AvailableInfo.isValid())
668 return;
669
670 if (!LIS)
671 return;
672
673 // If we don't know the exact VTYPE, we can't copy the vsetvli to the exit of
674 // the unavailable pred.
675 if (AvailableInfo.hasSEWLMULRatioOnly())
676 return;
677
678 // Critical edge - TODO: consider splitting?
679 if (UnavailablePred->succ_size() != 1)
680 return;
681
682 // If the AVL value is a register (other than our VLMAX sentinel),
683 // we need to prove the value is available at the point we're going
684 // to insert the vsetvli at.
685 if (AvailableInfo.hasAVLReg()) {
686 SlotIndex SI = AvailableInfo.getAVLVNInfo()->def;
687 // This is an inline dominance check which covers the case of
688 // UnavailablePred being the preheader of a loop.
689 if (LIS->getMBBFromIndex(index: SI) != UnavailablePred)
690 return;
691 if (!UnavailablePred->terminators().empty() &&
692 SI >= LIS->getInstructionIndex(Instr: *UnavailablePred->getFirstTerminator()))
693 return;
694 }
695
696 // Model the effect of changing the input state of the block MBB to
697 // AvailableInfo. We're looking for two issues here; one legality,
698 // one profitability.
699 // 1) If the block doesn't use some of the fields from VL or VTYPE, we
700 // may hit the end of the block with a different end state. We can
701 // not make this change without reflowing later blocks as well.
702 // 2) If we don't actually remove a transition, inserting a vsetvli
703 // into the predecessor block would be correct, but unprofitable.
704 VSETVLIInfo OldInfo = BlockInfo[MBB.getNumber()].Pred;
705 VSETVLIInfo CurInfo = AvailableInfo;
706 int TransitionsRemoved = 0;
707 for (const MachineInstr &MI : MBB) {
708 const VSETVLIInfo LastInfo = CurInfo;
709 const VSETVLIInfo LastOldInfo = OldInfo;
710 transferBefore(Info&: CurInfo, MI);
711 transferBefore(Info&: OldInfo, MI);
712 if (CurInfo == LastInfo)
713 TransitionsRemoved++;
714 if (LastOldInfo == OldInfo)
715 TransitionsRemoved--;
716 transferAfter(Info&: CurInfo, MI);
717 transferAfter(Info&: OldInfo, MI);
718 if (CurInfo == OldInfo)
719 // Convergence. All transitions after this must match by construction.
720 break;
721 }
722 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
723 // Issues 1 and 2 above
724 return;
725
726 // Finally, update both data flow state and insert the actual vsetvli.
727 // Doing both keeps the code in sync with the dataflow results, which
728 // is critical for correctness of phase 3.
729 auto OldExit = BlockInfo[UnavailablePred->getNumber()].Exit;
730 LLVM_DEBUG(dbgs() << "PRE VSETVLI from " << MBB.getName() << " to "
731 << UnavailablePred->getName() << " with state "
732 << AvailableInfo << "\n");
733 BlockInfo[UnavailablePred->getNumber()].Exit = AvailableInfo;
734 BlockInfo[MBB.getNumber()].Pred = AvailableInfo;
735
736 // Note there's an implicit assumption here that terminators never use
737 // or modify VL or VTYPE. Also, fallthrough will return end().
738 auto InsertPt = UnavailablePred->getFirstInstrTerminator();
739 insertVSETVLI(MBB&: *UnavailablePred, InsertPt,
740 DL: UnavailablePred->findDebugLoc(MBBI: InsertPt),
741 Info: AvailableInfo, PrevInfo: OldExit);
742}
743
744// Return true if we can mutate PrevMI to match MI without changing any the
745// fields which would be observed.
746// If AVLDefToMove is non-null after the call, it points to an ADDI
747// instruction that needs to be moved before PrevMI.
748bool RISCVInsertVSETVLI::canMutatePriorConfig(
749 const MachineInstr &PrevMI, const MachineInstr &MI,
750 const DemandedFields &Used, MachineInstr *&AVLDefToMove) const {
751 AVLDefToMove = nullptr;
752 // If the VL values aren't equal, return false if either a) the former is
753 // demanded, or b) we can't rewrite the former to be the later for
754 // implementation reasons.
755 if (!RISCVInstrInfo::isVLPreservingConfig(MI)) {
756 if (Used.VLAny)
757 return false;
758
759 if (Used.VLZeroness) {
760 if (RISCVInstrInfo::isVLPreservingConfig(MI: PrevMI))
761 return false;
762 if (!VIA.getInfoForVSETVLI(MI: PrevMI).hasEquallyZeroAVL(
763 Other: VIA.getInfoForVSETVLI(MI), LIS))
764 return false;
765 }
766
767 auto &AVL = MI.getOperand(i: 1);
768
769 // If the AVL is a register, we need to make sure its definition is the same
770 // at PrevMI as it was at MI.
771 if (AVL.isReg() && AVL.getReg() != RISCV::X0) {
772 VNInfo *VNI = getVNInfoFromReg(Reg: AVL.getReg(), MI, LIS);
773 VNInfo *PrevVNI = getVNInfoFromReg(Reg: AVL.getReg(), MI: PrevMI, LIS);
774 if (!VNI || !PrevVNI || VNI != PrevVNI) {
775 // If LIS is null, we were not able to get the VNInfo so we don't know
776 // if the AVL def needs to be moved.
777 if (!LIS)
778 return false;
779 // If the AVL is defined by a load immediate instruction (ADDI x0, imm),
780 // it can be moved earlier since it has no register dependencies.
781 if (!AVL.getReg().isVirtual())
782 return false;
783
784 MachineInstr *DefMI = MRI->getUniqueVRegDef(Reg: AVL.getReg());
785 if (!DefMI || !RISCVInstrInfo::isLoadImmediate(MI: *DefMI) ||
786 DefMI->getParent() != PrevMI.getParent()) {
787 return false;
788 }
789 // Mark that this ADDI needs to be moved.
790 AVLDefToMove = DefMI;
791 }
792 }
793
794 // If we define VL and need to move the definition up, check we can extend
795 // the live interval upwards from MI to PrevMI.
796 Register VL = MI.getOperand(i: 0).getReg();
797 if (VL.isVirtual() && LIS &&
798 LIS->getInterval(Reg: VL).overlaps(Start: LIS->getInstructionIndex(Instr: PrevMI),
799 End: LIS->getInstructionIndex(Instr: MI)))
800 return false;
801 }
802
803 assert(PrevMI.getOperand(2).isImm() && MI.getOperand(2).isImm());
804 auto PriorVType = PrevMI.getOperand(i: 2).getImm();
805 auto VType = MI.getOperand(i: 2).getImm();
806 return areCompatibleVTYPEs(CurVType: PriorVType, NewVType: VType, Used);
807}
808
809void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {
810 MachineInstr *NextMI = nullptr;
811 // We can have arbitrary code in successors, so VL and VTYPE
812 // must be considered demanded.
813 DemandedFields Used;
814 Used.demandVL();
815 Used.demandVTYPE();
816 SmallVector<MachineInstr*> ToDelete;
817
818 auto dropAVLUse = [&](MachineOperand &MO) {
819 if (!MO.isReg() || !MO.getReg().isVirtual())
820 return;
821 Register OldVLReg = MO.getReg();
822 MO.setReg(Register());
823
824 if (LIS)
825 LIS->shrinkToUses(li: &LIS->getInterval(Reg: OldVLReg));
826
827 MachineInstr *VLOpDef = MRI->getUniqueVRegDef(Reg: OldVLReg);
828 if (VLOpDef && TII->isAddImmediate(MI: *VLOpDef, Reg: OldVLReg) &&
829 MRI->use_nodbg_empty(RegNo: OldVLReg))
830 ToDelete.push_back(Elt: VLOpDef);
831 };
832
833 for (MachineInstr &MI : make_early_inc_range(Range: reverse(C&: MBB))) {
834 // TODO: Support XSfmm.
835 if (RISCVII::hasTWidenOp(TSFlags: MI.getDesc().TSFlags) ||
836 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI)) {
837 NextMI = nullptr;
838 continue;
839 }
840
841 if (!RISCVInstrInfo::isVectorConfigInstr(MI)) {
842 Used.doUnion(B: getDemanded(MI, ST));
843 if (MI.isCall() || MI.isInlineAsm() ||
844 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
845 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
846 NextMI = nullptr;
847 continue;
848 }
849
850 if (!MI.getOperand(i: 0).isDead())
851 Used.demandVL();
852
853 if (NextMI) {
854 if (!Used.usedVL() && !Used.usedVTYPE()) {
855 dropAVLUse(MI.getOperand(i: 1));
856 if (LIS)
857 LIS->RemoveMachineInstrFromMaps(MI);
858 MI.eraseFromParent();
859 NumCoalescedVSETVL++;
860 // Leave NextMI unchanged
861 continue;
862 }
863
864 MachineInstr *AVLDefToMove = nullptr;
865 if (canMutatePriorConfig(PrevMI: MI, MI: *NextMI, Used, AVLDefToMove)) {
866 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *NextMI)) {
867 Register DefReg = NextMI->getOperand(i: 0).getReg();
868
869 MI.getOperand(i: 0).setReg(DefReg);
870 MI.getOperand(i: 0).setIsDead(false);
871
872 // Move the AVL from NextMI to MI
873 dropAVLUse(MI.getOperand(i: 1));
874 if (NextMI->getOperand(i: 1).isImm())
875 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: NextMI->getOperand(i: 1).getImm());
876 else {
877 MI.getOperand(i: 1).ChangeToRegister(Reg: NextMI->getOperand(i: 1).getReg(),
878 isDef: false);
879
880 // If canMutatePriorConfig indicated that an ADDI needs to be moved,
881 // move it now.
882 if (AVLDefToMove) {
883 AVLDefToMove->moveBefore(MovePos: &MI);
884 if (LIS)
885 LIS->handleMove(MI&: *AVLDefToMove);
886 }
887 }
888 dropAVLUse(NextMI->getOperand(i: 1));
889
890 // The def of DefReg moved to MI, so extend the LiveInterval up to
891 // it.
892 if (DefReg.isVirtual() && LIS) {
893 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
894 SlotIndex MISlot = LIS->getInstructionIndex(Instr: MI).getRegSlot();
895 SlotIndex NextMISlot =
896 LIS->getInstructionIndex(Instr: *NextMI).getRegSlot();
897 VNInfo *DefVNI = DefLI.getVNInfoAt(Idx: NextMISlot);
898 LiveInterval::Segment S(MISlot, NextMISlot, DefVNI);
899 DefLI.addSegment(S);
900 DefVNI->def = MISlot;
901 // Mark DefLI as spillable if it was previously unspillable
902 DefLI.setWeight(0);
903
904 // DefReg may have had no uses, in which case we need to shrink
905 // the LiveInterval up to MI.
906 LIS->shrinkToUses(li: &DefLI);
907 }
908
909 MI.setDesc(NextMI->getDesc());
910 }
911 MI.getOperand(i: 2).setImm(NextMI->getOperand(i: 2).getImm());
912
913 dropAVLUse(NextMI->getOperand(i: 1));
914 if (LIS)
915 LIS->RemoveMachineInstrFromMaps(MI&: *NextMI);
916 NextMI->eraseFromParent();
917 NumCoalescedVSETVL++;
918 // fallthrough
919 }
920 }
921 NextMI = &MI;
922 Used = getDemanded(MI, ST);
923 }
924
925 // Loop over the dead AVL values, and delete them now. This has
926 // to be outside the above loop to avoid invalidating iterators.
927 for (auto *MI : ToDelete) {
928 assert(MI->getOpcode() == RISCV::ADDI);
929 Register AddReg = MI->getOperand(i: 1).getReg();
930 if (LIS) {
931 LIS->removeInterval(Reg: MI->getOperand(i: 0).getReg());
932 LIS->RemoveMachineInstrFromMaps(MI&: *MI);
933 }
934 MI->eraseFromParent();
935 if (LIS && AddReg.isVirtual())
936 LIS->shrinkToUses(li: &LIS->getInterval(Reg: AddReg));
937 }
938}
939
940// When twiden != 0, LMUL, tail policy, and mask policy from the user are
941// ignored. The tail policy and mask policy are always treated as agnostic. The
942// normal RVV instruction will ignore the twiden parameter. This observation
943// could allow the RVV instruction and xsfmm instruction to share the same
944// configuration instruction.
945//
946// We need to make sure the AVL, SEW, and AltFmt is same between VSETVL and
947// VSETVLTN.
948//
949// For example:
950//
951// %avl = SETTM or SETTK
952// ...
953// VSETVL %avl, type1
954// VSETVLTNT %avl, type2
955//
956// ->
957//
958// %avl = SETTM or SETTK
959// ...
960// VSETVLTNT %avl, type2
961//
962bool RISCVInsertVSETVLI::canMutatePriorConfigWithTWiden(
963 const MachineInstr &PrevMI, const MachineInstr &MI) const {
964
965 if (PrevMI.getOpcode() != RISCV::PseudoVSETVLI)
966 return false;
967
968 if (MI.getOpcode() != RISCV::PseudoSF_VSETTNT)
969 return false;
970
971 auto PrevInfo = VIA.getInfoForVSETVLI(MI: PrevMI);
972 auto CurrInfo = VIA.getInfoForVSETVLI(MI);
973
974 assert(CurrInfo.hasAVLReg() && "Invalid PseudoSF_VSETTNT without an AVLReg.");
975
976 auto AVLReg = CurrInfo.getAVLReg();
977
978 auto *AVLRegDefMI = MRI->getUniqueVRegDef(Reg: AVLReg);
979
980 if (!AVLRegDefMI)
981 return false;
982
983 if (!RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI: *AVLRegDefMI))
984 return false;
985
986 auto AVLRegDefMIInfo = VIA.computeInfoForInstr(MI: *AVLRegDefMI);
987 if (AVLRegDefMIInfo.getTWiden() != CurrInfo.getTWiden())
988 return false;
989
990 if (AVLRegDefMIInfo.getSEW() != PrevInfo.getSEW())
991 return false;
992
993 // CurrInfo twiden != 0, so TailAgnostic and MaskAgnostic bit default to 1
994 if (!PrevInfo.getTailAgnostic() || !PrevInfo.getMaskAgnostic())
995 return false;
996
997 if (!PrevInfo.hasSameAVL(Other: CurrInfo))
998 return false;
999
1000 if (PrevInfo.getSEW() != CurrInfo.getSEW())
1001 return false;
1002
1003 if (PrevInfo.getAltFmt() != CurrInfo.getAltFmt())
1004 return false;
1005
1006 // The PrevMI's LMUL should be at least 8/KMAX; otherwise, converting it to a
1007 // tile-widening version could result in a VLMAX smaller than what AVLRegDefMI
1008 // expects, causing the LMUL information from PrevMI to be lost.
1009 auto [LMul, Fractional] = decodeVLMUL(VLMul: PrevInfo.getVLMUL());
1010 unsigned KMAX = (CurrInfo.getSEW() >= 32) ? 1 : (32 / CurrInfo.getSEW());
1011
1012 if (Fractional || LMul < (8 / KMAX))
1013 return false;
1014
1015 return true;
1016}
1017
1018void RISCVInsertVSETVLI::coalesceVSETVLIsForTWiden(
1019 MachineBasicBlock &MBB) const {
1020 MachineInstr *NextMI = nullptr;
1021
1022 for (MachineInstr &MI : make_early_inc_range(Range: reverse(C&: MBB))) {
1023
1024 if (!RISCVInstrInfo::isVectorConfigInstr(MI))
1025 continue;
1026
1027 if (NextMI) {
1028 // If only TWiden different. Update the MI and drop the NextMI.
1029 if (canMutatePriorConfigWithTWiden(PrevMI: MI, MI: *NextMI)) {
1030
1031 auto NextInfo = VIA.getInfoForVSETVLI(MI: *NextMI);
1032 MI.getOperand(i: 2).setImm(NextInfo.encodeVTYPE());
1033
1034 if (LIS)
1035 LIS->RemoveMachineInstrFromMaps(MI&: *NextMI);
1036 NextMI->eraseFromParent();
1037 }
1038 }
1039 NextMI = &MI;
1040 }
1041}
1042
1043void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
1044 for (auto I = MBB.begin(), E = MBB.end(); I != E;) {
1045 MachineInstr &MI = *I++;
1046 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
1047 Register VLOutput = MI.getOperand(i: 1).getReg();
1048 assert(VLOutput.isVirtual());
1049 if (!MI.getOperand(i: 1).isDead()) {
1050 auto ReadVLMI = BuildMI(BB&: MBB, I, MIMD: MI.getDebugLoc(),
1051 MCID: TII->get(Opcode: RISCV::PseudoReadVL), DestReg: VLOutput);
1052 // Move the LiveInterval's definition down to PseudoReadVL.
1053 if (LIS) {
1054 SlotIndex NewDefSI =
1055 LIS->InsertMachineInstrInMaps(MI&: *ReadVLMI).getRegSlot();
1056 LiveInterval &DefLI = LIS->getInterval(Reg: VLOutput);
1057 LiveRange::Segment *DefSeg = DefLI.getSegmentContaining(Idx: NewDefSI);
1058 VNInfo *DefVNI = DefLI.getVNInfoAt(Idx: DefSeg->start);
1059 DefLI.removeSegment(Start: DefSeg->start, End: NewDefSI);
1060 DefVNI->def = NewDefSI;
1061 }
1062 }
1063 // We don't use the vl output of the VLEFF/VLSEGFF anymore.
1064 MI.getOperand(i: 1).setReg(RISCV::X0);
1065 MI.addRegisterDefined(Reg: RISCV::VL, RegInfo: MRI->getTargetRegisterInfo());
1066 }
1067 }
1068}
1069
1070bool RISCVInsertVSETVLI::insertVSETMTK(MachineBasicBlock &MBB,
1071 TKTMMode Mode) const {
1072
1073 bool Changed = false;
1074 for (auto &MI : MBB) {
1075 uint64_t TSFlags = MI.getDesc().TSFlags;
1076 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI) ||
1077 !RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasTWidenOp(TSFlags))
1078 continue;
1079
1080 VSETVLIInfo CurrInfo = VIA.computeInfoForInstr(MI);
1081
1082 unsigned Opcode = 0, OpNum = 0;
1083 switch (Mode) {
1084 case VSETTK:
1085 if (!RISCVII::hasTKOp(TSFlags))
1086 continue;
1087 OpNum = RISCVII::getTKOpNum(Desc: MI.getDesc());
1088 Opcode = RISCV::PseudoSF_VSETTK;
1089 break;
1090 case VSETTM:
1091 if (!RISCVII::hasTMOp(TSFlags))
1092 continue;
1093 OpNum = RISCVII::getTMOpNum(Desc: MI.getDesc());
1094 Opcode = RISCV::PseudoSF_VSETTM;
1095 break;
1096 }
1097
1098 assert(OpNum && Opcode && "Invalid OpNum or Opcode");
1099
1100 MachineOperand &Op = MI.getOperand(i: OpNum);
1101
1102 auto TmpMI = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode))
1103 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
1104 .addReg(RegNo: Op.getReg())
1105 .addImm(Val: Log2_32(Value: CurrInfo.getSEW()))
1106 .addImm(Val: CurrInfo.getTWiden());
1107
1108 Changed = true;
1109 Register Reg = Op.getReg();
1110 Op.setReg(Register());
1111 Op.setIsKill(false);
1112 if (LIS) {
1113 LIS->InsertMachineInstrInMaps(MI&: *TmpMI);
1114 LiveInterval &LI = LIS->getInterval(Reg);
1115
1116 // Erase the AVL operand from the instruction.
1117 LIS->shrinkToUses(li: &LI);
1118 // TODO: Enable this once needVSETVLIPHI is supported.
1119 // SmallVector<LiveInterval *> SplitLIs;
1120 // LIS->splitSeparateComponents(LI, SplitLIs);
1121 }
1122 }
1123 return Changed;
1124}
1125
1126bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {
1127 // Skip if the vector extension is not enabled.
1128 ST = &MF.getSubtarget<RISCVSubtarget>();
1129 if (!ST->hasVInstructions())
1130 return false;
1131
1132 LLVM_DEBUG(dbgs() << "Entering InsertVSETVLI for " << MF.getName() << "\n");
1133
1134 TII = ST->getInstrInfo();
1135 MRI = &MF.getRegInfo();
1136 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
1137 LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
1138 VIA = RISCVVSETVLIInfoAnalysis(ST, LIS);
1139
1140 assert(BlockInfo.empty() && "Expect empty block infos");
1141 BlockInfo.resize(new_size: MF.getNumBlockIDs());
1142
1143 bool HaveVectorOp = false;
1144
1145 // Phase 1 - determine how VL/VTYPE are affected by the each block.
1146 for (const MachineBasicBlock &MBB : MF) {
1147 VSETVLIInfo TmpStatus;
1148 HaveVectorOp |= computeVLVTYPEChanges(MBB, Info&: TmpStatus);
1149 // Initial exit state is whatever change we found in the block.
1150 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
1151 BBInfo.Exit = TmpStatus;
1152 LLVM_DEBUG(dbgs() << "Initial exit state of " << printMBBReference(MBB)
1153 << " is " << BBInfo.Exit << "\n");
1154
1155 }
1156
1157 // If we didn't find any instructions that need VSETVLI, we're done.
1158 if (!HaveVectorOp) {
1159 BlockInfo.clear();
1160 return false;
1161 }
1162
1163 // Phase 2 - determine the exit VL/VTYPE from each block. We add all
1164 // blocks to the list here, but will also add any that need to be revisited
1165 // during Phase 2 processing.
1166 for (const MachineBasicBlock &MBB : MF) {
1167 WorkList.push(x: &MBB);
1168 BlockInfo[MBB.getNumber()].InQueue = true;
1169 }
1170 while (!WorkList.empty()) {
1171 const MachineBasicBlock &MBB = *WorkList.front();
1172 WorkList.pop();
1173 computeIncomingVLVTYPE(MBB);
1174 }
1175
1176 // Perform partial redundancy elimination of vsetvli transitions.
1177 for (MachineBasicBlock &MBB : MF)
1178 doPRE(MBB);
1179
1180 // Phase 3 - add any vsetvli instructions needed in the block. Use the
1181 // Phase 2 information to avoid adding vsetvlis before the first vector
1182 // instruction in the block if the VL/VTYPE is satisfied by its
1183 // predecessors.
1184 for (MachineBasicBlock &MBB : MF)
1185 emitVSETVLIs(MBB);
1186
1187 // Now that all vsetvlis are explicit, go through and do block local
1188 // DSE and peephole based demanded fields based transforms. Note that
1189 // this *must* be done outside the main dataflow so long as we allow
1190 // any cross block analysis within the dataflow. We can't have both
1191 // demanded fields based mutation and non-local analysis in the
1192 // dataflow at the same time without introducing inconsistencies.
1193 // We're visiting blocks from the bottom up because a VSETVLI in the
1194 // earlier block might become dead when its uses in later blocks are
1195 // optimized away.
1196 for (MachineBasicBlock *MBB : post_order(G: &MF))
1197 coalesceVSETVLIs(MBB&: *MBB);
1198
1199 if (ST->hasVendorXSfmmbase()) {
1200 for (MachineBasicBlock &MBB : MF)
1201 coalesceVSETVLIsForTWiden(MBB);
1202 }
1203
1204 // Insert PseudoReadVL after VLEFF/VLSEGFF and replace it with the vl output
1205 // of VLEFF/VLSEGFF.
1206 for (MachineBasicBlock &MBB : MF)
1207 insertReadVL(MBB);
1208
1209 if (ST->hasVendorXSfmmbase()) {
1210 for (MachineBasicBlock &MBB : MF) {
1211 insertVSETMTK(MBB, Mode: VSETTM);
1212 insertVSETMTK(MBB, Mode: VSETTK);
1213 }
1214 }
1215
1216 BlockInfo.clear();
1217 return HaveVectorOp;
1218}
1219
1220/// Returns an instance of the Insert VSETVLI pass.
1221FunctionPass *llvm::createRISCVInsertVSETVLIPass() {
1222 return new RISCVInsertVSETVLI();
1223}
1224