1//===- RISCVInsertVSETVLI.cpp - Insert VSETVLI instructions ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a function pass that inserts VSETVLI instructions where
10// needed and expands the vl outputs of VLEFF/VLSEGFF to PseudoReadVL
11// instructions.
12//
13// This pass consists of 3 phases:
14//
15// Phase 1 collects how each basic block affects VL/VTYPE.
16//
17// Phase 2 uses the information from phase 1 to do a data flow analysis to
18// propagate the VL/VTYPE changes through the function. This gives us the
19// VL/VTYPE at the start of each basic block.
20//
21// Phase 3 inserts VSETVLI instructions in each basic block. Information from
22// phase 2 is used to prevent inserting a VSETVLI before the first vector
23// instruction in the block if possible.
24//
25//===----------------------------------------------------------------------===//
26
27#include "RISCV.h"
28#include "RISCVSubtarget.h"
29#include "RISCVVSETVLIInfoAnalysis.h"
30#include "llvm/ADT/PostOrderIterator.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/CodeGen/LiveDebugVariables.h"
33#include "llvm/CodeGen/LiveIntervals.h"
34#include "llvm/CodeGen/LiveStacks.h"
35#include "llvm/CodeGen/MachineFunctionPass.h"
36#include <queue>
37using namespace llvm;
38using namespace RISCV;
39
40#define DEBUG_TYPE "riscv-insert-vsetvli"
41#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
42
43STATISTIC(NumInsertedVSETVL, "Number of VSETVL inst inserted");
44STATISTIC(NumCoalescedVSETVL, "Number of VSETVL inst coalesced");
45
46static cl::opt<bool> EnsureWholeVectorRegisterMoveValidVTYPE(
47 DEBUG_TYPE "-whole-vector-register-move-valid-vtype", cl::Hidden,
48 cl::desc("Insert vsetvlis before vmvNr.vs to ensure vtype is valid and "
49 "vill is cleared"),
50 cl::init(Val: true));
51
52namespace {
53
54/// Given a virtual register \p Reg, return the corresponding VNInfo for it.
55/// This will return nullptr if the virtual register is an implicit_def or
56/// if LiveIntervals is not available.
57static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
58 const LiveIntervals *LIS) {
59 assert(Reg.isVirtual());
60 if (!LIS)
61 return nullptr;
62 auto &LI = LIS->getInterval(Reg);
63 SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
64 return LI.getVNInfoBefore(Idx: SI);
65}
66
67static MachineOperand &getVLOp(MachineInstr &MI) {
68 return MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
69}
70
71struct BlockData {
72 // The VSETVLIInfo that represents the VL/VTYPE settings on exit from this
73 // block. Calculated in Phase 2.
74 VSETVLIInfo Exit;
75
76 // The VSETVLIInfo that represents the VL/VTYPE settings from all predecessor
77 // blocks. Calculated in Phase 2, and used by Phase 3.
78 VSETVLIInfo Pred;
79
80 // Keeps track of whether the block is already in the queue.
81 bool InQueue = false;
82
83 BlockData() = default;
84};
85
86enum TKTMMode {
87 VSETTK = 0,
88 VSETTM = 1,
89};
90
91class RISCVInsertVSETVLI : public MachineFunctionPass {
92 const RISCVSubtarget *ST;
93 const TargetInstrInfo *TII;
94 MachineRegisterInfo *MRI;
95 // Possibly null!
96 LiveIntervals *LIS;
97 RISCVVSETVLIInfoAnalysis VIA;
98
99 std::vector<BlockData> BlockInfo;
100 std::queue<const MachineBasicBlock *> WorkList;
101
102public:
103 static char ID;
104
105 RISCVInsertVSETVLI() : MachineFunctionPass(ID) {}
106 bool runOnMachineFunction(MachineFunction &MF) override;
107
108 void getAnalysisUsage(AnalysisUsage &AU) const override {
109 AU.setPreservesCFG();
110
111 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
112 AU.addPreserved<LiveIntervalsWrapperPass>();
113 AU.addPreserved<SlotIndexesWrapperPass>();
114 AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
115 AU.addPreserved<LiveStacksWrapperLegacy>();
116
117 MachineFunctionPass::getAnalysisUsage(AU);
118 }
119
120 StringRef getPassName() const override { return RISCV_INSERT_VSETVLI_NAME; }
121
122private:
123 bool needVSETVLI(const DemandedFields &Used, const VSETVLIInfo &Require,
124 const VSETVLIInfo &CurInfo) const;
125 bool needVSETVLIPHI(const VSETVLIInfo &Require,
126 const MachineBasicBlock &MBB) const;
127 void insertVSETVLI(MachineBasicBlock &MBB,
128 MachineBasicBlock::iterator InsertPt, DebugLoc DL,
129 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);
130
131 void transferBefore(VSETVLIInfo &Info, const MachineInstr &MI) const;
132 void transferAfter(VSETVLIInfo &Info, const MachineInstr &MI) const;
133 bool computeVLVTYPEChanges(const MachineBasicBlock &MBB,
134 VSETVLIInfo &Info) const;
135 void computeIncomingVLVTYPE(const MachineBasicBlock &MBB);
136 void emitVSETVLIs(MachineBasicBlock &MBB);
137 void doPRE(MachineBasicBlock &MBB);
138 void insertReadVL(MachineBasicBlock &MBB);
139
140 bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI,
141 const DemandedFields &Used,
142 MachineInstr *&AVLDefToMove) const;
143 void coalesceVSETVLIs(MachineBasicBlock &MBB) const;
144 bool insertVSETMTK(MachineBasicBlock &MBB, TKTMMode Mode) const;
145};
146
147} // end anonymous namespace
148
149char RISCVInsertVSETVLI::ID = 0;
150char &llvm::RISCVInsertVSETVLIID = RISCVInsertVSETVLI::ID;
151
152INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME,
153 false, false)
154
155void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
156 MachineBasicBlock::iterator InsertPt,
157 DebugLoc DL, const VSETVLIInfo &Info,
158 const VSETVLIInfo &PrevInfo) {
159 ++NumInsertedVSETVL;
160
161 if (PrevInfo.isKnown()) {
162 // Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same
163 // VLMAX.
164 if (Info.hasSameAVL(Other: PrevInfo) && Info.hasSameVLMAX(Other: PrevInfo)) {
165 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
166 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0X0
167 : RISCV::PseudoVSETVLIX0X0))
168 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
169 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
170 .addImm(Val: Info.encodeVTYPE())
171 .addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
172 if (LIS)
173 LIS->InsertMachineInstrInMaps(MI&: *MI);
174 return;
175 }
176
177 // If our AVL is a virtual register, it might be defined by a VSET(I)VLI. If
178 // it has the same VLMAX we want and the last VL/VTYPE we observed is the
179 // same, we can use the X0, X0 form.
180 if (Info.hasSameVLMAX(Other: PrevInfo) && Info.hasAVLReg()) {
181 if (const MachineInstr *DefMI = Info.getAVLDefMI(LIS);
182 DefMI && RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI)) {
183 VSETVLIInfo DefInfo = VIA.getInfoForVSETVLI(MI: *DefMI);
184 if (DefInfo.hasSameAVL(Other: PrevInfo) && DefInfo.hasSameVLMAX(Other: PrevInfo)) {
185 auto MI =
186 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
187 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0X0
188 : RISCV::PseudoVSETVLIX0X0))
189 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
190 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
191 .addImm(Val: Info.encodeVTYPE())
192 .addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
193 if (LIS)
194 LIS->InsertMachineInstrInMaps(MI&: *MI);
195 return;
196 }
197 }
198 }
199 }
200
201 if (Info.hasAVLImm()) {
202 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoVSETIVLI))
203 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
204 .addImm(Val: Info.getAVLImm())
205 .addImm(Val: Info.encodeVTYPE());
206 if (LIS)
207 LIS->InsertMachineInstrInMaps(MI&: *MI);
208 return;
209 }
210
211 if (Info.hasAVLVLMAX()) {
212 Register DestReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
213 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
214 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0
215 : RISCV::PseudoVSETVLIX0))
216 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Dead)
217 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
218 .addImm(Val: Info.encodeVTYPE());
219 if (LIS) {
220 LIS->InsertMachineInstrInMaps(MI&: *MI);
221 LIS->createAndComputeVirtRegInterval(Reg: DestReg);
222 }
223 return;
224 }
225
226 Register AVLReg = Info.getAVLReg();
227 MRI->constrainRegClass(Reg: AVLReg, RC: &RISCV::GPRNoX0RegClass);
228 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
229 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNT
230 : RISCV::PseudoVSETVLI))
231 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
232 .addReg(RegNo: AVLReg)
233 .addImm(Val: Info.encodeVTYPE());
234 if (LIS) {
235 LIS->InsertMachineInstrInMaps(MI&: *MI);
236 LiveInterval &LI = LIS->getInterval(Reg: AVLReg);
237 SlotIndex SI = LIS->getInstructionIndex(Instr: *MI).getRegSlot();
238 const VNInfo *CurVNI = Info.getAVLVNInfo();
239 // If the AVL value isn't live at MI, do a quick check to see if it's easily
240 // extendable. Otherwise, we need to copy it.
241 if (LI.getVNInfoBefore(Idx: SI) != CurVNI) {
242 if (!LI.liveAt(index: SI) && LI.containsOneValue())
243 LIS->extendToIndices(LR&: LI, Indices: SI);
244 else {
245 Register AVLCopyReg =
246 MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
247 MachineBasicBlock *MBB = LIS->getMBBFromIndex(index: CurVNI->def);
248 MachineBasicBlock::iterator II;
249 if (CurVNI->isPHIDef())
250 II = MBB->getFirstNonPHI();
251 else {
252 II = LIS->getInstructionFromIndex(index: CurVNI->def);
253 II = std::next(x: II);
254 }
255 assert(II.isValid());
256 auto AVLCopy = BuildMI(BB&: *MBB, I: II, MIMD: DL, MCID: TII->get(Opcode: RISCV::COPY), DestReg: AVLCopyReg)
257 .addReg(RegNo: AVLReg);
258 LIS->InsertMachineInstrInMaps(MI&: *AVLCopy);
259 MI->getOperand(i: 1).setReg(AVLCopyReg);
260 LIS->createAndComputeVirtRegInterval(Reg: AVLCopyReg);
261 }
262 }
263 }
264}
265
266/// Return true if a VSETVLI is required to transition from CurInfo to Require
267/// given a set of DemandedFields \p Used.
268bool RISCVInsertVSETVLI::needVSETVLI(const DemandedFields &Used,
269 const VSETVLIInfo &Require,
270 const VSETVLIInfo &CurInfo) const {
271 if (!CurInfo.isKnown() || CurInfo.hasSEWLMULRatioOnly())
272 return true;
273
274 if (CurInfo.isCompatible(Used, Require, LIS))
275 return false;
276
277 return true;
278}
279
280// If we don't use LMUL or the SEW/LMUL ratio, then adjust LMUL so that we
281// maintain the SEW/LMUL ratio. This allows us to eliminate VL toggles in more
282// places.
283static VSETVLIInfo adjustIncoming(const VSETVLIInfo &PrevInfo,
284 const VSETVLIInfo &NewInfo,
285 DemandedFields &Demanded) {
286 VSETVLIInfo Info = NewInfo;
287
288 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isKnown()) {
289 if (auto NewVLMul = RISCVVType::getSameRatioLMUL(Ratio: PrevInfo.getSEWLMULRatio(),
290 EEW: Info.getSEW()))
291 Info.setVLMul(*NewVLMul);
292 Demanded.LMUL = DemandedFields::LMULEqual;
293 }
294
295 return Info;
296}
297
298// Given an incoming state reaching MI, minimally modifies that state so that it
299// is compatible with MI. The resulting state is guaranteed to be semantically
300// legal for MI, but may not be the state requested by MI.
301void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
302 const MachineInstr &MI) const {
303 if (EnsureWholeVectorRegisterMoveValidVTYPE &&
304 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI) &&
305 (!Info.isKnown() || Info.hasSEWLMULRatioOnly())) {
306 // Use an arbitrary but valid AVL and VTYPE so vill will be cleared. It may
307 // be coalesced into another vsetvli since we won't demand any fields.
308 VSETVLIInfo NewInfo; // Need a new VSETVLIInfo to clear SEWLMULRatioOnly
309 NewInfo.setAVLImm(1);
310 NewInfo.setVTYPE(L: RISCVVType::LMUL_1, /*sew*/ S: 8, /*ta*/ TA: true, /*ma*/ MA: true,
311 /*AltFmt*/ Altfmt: false, /*W*/ 0);
312 Info = NewInfo;
313 return;
314 }
315
316 if (!RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags))
317 return;
318
319 DemandedFields Demanded = getDemanded(MI, ST);
320
321 const VSETVLIInfo NewInfo = VIA.computeInfoForInstr(MI);
322 assert(NewInfo.isKnown());
323 if (Info.isValid() && !needVSETVLI(Used: Demanded, Require: NewInfo, CurInfo: Info))
324 return;
325
326 const VSETVLIInfo PrevInfo = Info;
327 if (!Info.isKnown())
328 Info = NewInfo;
329
330 const VSETVLIInfo IncomingInfo = adjustIncoming(PrevInfo, NewInfo, Demanded);
331
332 // If MI only demands that VL has the same zeroness, we only need to set the
333 // AVL if the zeroness differs. This removes a vsetvli entirely if the types
334 // match or allows use of cheaper avl preserving variant if VLMAX doesn't
335 // change. If VLMAX might change, we couldn't use the 'vsetvli x0, x0, vtype"
336 // variant, so we avoid the transform to prevent extending live range of an
337 // avl register operand.
338 // TODO: We can probably relax this for immediates.
339 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(Other: PrevInfo, LIS) &&
340 IncomingInfo.hasSameVLMAX(Other: PrevInfo);
341 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
342 Info.setAVL(IncomingInfo);
343
344 // If we only knew the sew/lmul ratio previously, replace the VTYPE.
345 if (Info.hasSEWLMULRatioOnly()) {
346 VSETVLIInfo RatiolessInfo = IncomingInfo;
347 RatiolessInfo.setAVL(Info);
348 Info = RatiolessInfo;
349 } else {
350 unsigned SEW =
351 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
352 .getSEW();
353 Info.setVTYPE(
354 L: ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
355 .getVLMUL(),
356 S: SEW,
357 // Prefer tail/mask agnostic since it can be relaxed to undisturbed
358 // later if needed.
359 TA: (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
360 IncomingInfo.getTailAgnostic(),
361 MA: (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
362 IncomingInfo.getMaskAgnostic(),
363 // AltFmt requires SEW < 32.
364 Altfmt: (Demanded.AltFmt ? IncomingInfo : Info).getAltFmt() && SEW < 32,
365 W: Demanded.TWiden ? IncomingInfo.getTWiden() : 0);
366 }
367}
368
369// Given a state with which we evaluated MI (see transferBefore above for why
370// this might be different that the state MI requested), modify the state to
371// reflect the changes MI might make.
372void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
373 const MachineInstr &MI) const {
374 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {
375 Info = VIA.getInfoForVSETVLI(MI);
376 return;
377 }
378
379 // SETTM/TK will modify VTYPE, but it only affects the TM/TK bits.
380 // It is safe for other RVV operations.
381 // The TM/TK value will be maintained in insertVSETMTK.
382 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI))
383 return;
384
385 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
386 // Update AVL to vl-output of the fault first load.
387 assert(MI.getOperand(1).getReg().isVirtual());
388 if (LIS) {
389 auto &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
390 SlotIndex SI =
391 LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();
392 VNInfo *VNI = LI.getVNInfoAt(Idx: SI);
393 Info.setAVLRegDef(VNInfo: VNI, AVLReg: MI.getOperand(i: 1).getReg());
394 } else
395 Info.setAVLRegDef(VNInfo: nullptr, AVLReg: MI.getOperand(i: 1).getReg());
396 return;
397 }
398
399 // If this is something that updates VL/VTYPE that we don't know about, set
400 // the state to unknown.
401 if (MI.isCall() || MI.isInlineAsm() ||
402 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
403 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
404 Info = VSETVLIInfo::getUnknown();
405}
406
407bool RISCVInsertVSETVLI::computeVLVTYPEChanges(const MachineBasicBlock &MBB,
408 VSETVLIInfo &Info) const {
409 bool HadVectorOp = false;
410
411 Info = BlockInfo[MBB.getNumber()].Pred;
412 for (const MachineInstr &MI : MBB) {
413 transferBefore(Info, MI);
414
415 if (RISCVInstrInfo::isVectorConfigInstr(MI) ||
416 RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags) ||
417 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI) ||
418 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI))
419 HadVectorOp = true;
420
421 transferAfter(Info, MI);
422 }
423
424 return HadVectorOp;
425}
426
427void RISCVInsertVSETVLI::computeIncomingVLVTYPE(const MachineBasicBlock &MBB) {
428
429 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
430
431 BBInfo.InQueue = false;
432
433 // Start with the previous entry so that we keep the most conservative state
434 // we have ever found.
435 VSETVLIInfo InInfo = BBInfo.Pred;
436 if (MBB.pred_empty()) {
437 // There are no predecessors, so use the default starting status.
438 InInfo.setUnknown();
439 } else {
440 for (MachineBasicBlock *P : MBB.predecessors())
441 InInfo = InInfo.intersect(Other: BlockInfo[P->getNumber()].Exit);
442 }
443
444 // If we don't have any valid predecessor value, wait until we do.
445 if (!InInfo.isValid())
446 return;
447
448 // If no change, no need to rerun block
449 if (InInfo == BBInfo.Pred)
450 return;
451
452 BBInfo.Pred = InInfo;
453 LLVM_DEBUG(dbgs() << "Entry state of " << printMBBReference(MBB)
454 << " changed to " << BBInfo.Pred << "\n");
455
456 // Note: It's tempting to cache the state changes here, but due to the
457 // compatibility checks performed a blocks output state can change based on
458 // the input state. To cache, we'd have to add logic for finding
459 // never-compatible state changes.
460 VSETVLIInfo TmpStatus;
461 computeVLVTYPEChanges(MBB, Info&: TmpStatus);
462
463 // If the new exit value matches the old exit value, we don't need to revisit
464 // any blocks.
465 if (BBInfo.Exit == TmpStatus)
466 return;
467
468 BBInfo.Exit = TmpStatus;
469 LLVM_DEBUG(dbgs() << "Exit state of " << printMBBReference(MBB)
470 << " changed to " << BBInfo.Exit << "\n");
471
472 // Add the successors to the work list so we can propagate the changed exit
473 // status.
474 for (MachineBasicBlock *S : MBB.successors())
475 if (!BlockInfo[S->getNumber()].InQueue) {
476 BlockInfo[S->getNumber()].InQueue = true;
477 WorkList.push(x: S);
478 }
479}
480
481// If we weren't able to prove a vsetvli was directly unneeded, it might still
482// be unneeded if the AVL was a phi node where all incoming values are VL
483// outputs from the last VSETVLI in their respective basic blocks.
484bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
485 const MachineBasicBlock &MBB) const {
486 if (!Require.hasAVLReg())
487 return true;
488
489 if (!LIS)
490 return true;
491
492 // We need the AVL to have been produced by a PHI node in this basic block.
493 const VNInfo *Valno = Require.getAVLVNInfo();
494 if (!Valno->isPHIDef() || LIS->getMBBFromIndex(index: Valno->def) != &MBB)
495 return true;
496
497 const LiveRange &LR = LIS->getInterval(Reg: Require.getAVLReg());
498
499 for (auto *PBB : MBB.predecessors()) {
500 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
501
502 // We need the PHI input to the be the output of a VSET(I)VLI.
503 const VNInfo *Value = LR.getVNInfoBefore(Idx: LIS->getMBBEndIdx(mbb: PBB));
504 if (!Value)
505 return true;
506 MachineInstr *DefMI = LIS->getInstructionFromIndex(index: Value->def);
507 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI))
508 return true;
509
510 // We found a VSET(I)VLI make sure it matches the output of the
511 // predecessor block.
512 VSETVLIInfo DefInfo = VIA.getInfoForVSETVLI(MI: *DefMI);
513 if (DefInfo != PBBExit)
514 return true;
515
516 // Require has the same VL as PBBExit, so if the exit from the
517 // predecessor has the VTYPE we are looking for we might be able
518 // to avoid a VSETVLI.
519 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Other: Require))
520 return true;
521 }
522
523 // If all the incoming values to the PHI checked out, we don't need
524 // to insert a VSETVLI.
525 return false;
526}
527
528void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
529 VSETVLIInfo CurInfo = BlockInfo[MBB.getNumber()].Pred;
530 // Track whether the prefix of the block we've scanned is transparent
531 // (meaning has not yet changed the abstract state).
532 bool PrefixTransparent = true;
533 for (MachineInstr &MI : MBB) {
534 const VSETVLIInfo PrevInfo = CurInfo;
535 transferBefore(Info&: CurInfo, MI);
536
537 // If this is an explicit VSETVLI or VSETIVLI, update our state.
538 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {
539 // Conservatively, mark the VL and VTYPE as live.
540 assert(MI.getOperand(3).getReg() == RISCV::VL &&
541 MI.getOperand(4).getReg() == RISCV::VTYPE &&
542 "Unexpected operands where VL and VTYPE should be");
543 MI.getOperand(i: 3).setIsDead(false);
544 MI.getOperand(i: 4).setIsDead(false);
545 PrefixTransparent = false;
546 }
547
548 if (EnsureWholeVectorRegisterMoveValidVTYPE &&
549 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI)) {
550 if (!PrevInfo.isCompatible(Used: DemandedFields::all(), Require: CurInfo, LIS)) {
551 insertVSETVLI(MBB, InsertPt: MI, DL: MI.getDebugLoc(), Info: CurInfo, PrevInfo);
552 PrefixTransparent = false;
553 }
554 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ false,
555 /*isImp*/ true));
556 }
557
558 uint64_t TSFlags = MI.getDesc().TSFlags;
559 if (RISCVII::hasSEWOp(TSFlags)) {
560 if (!PrevInfo.isCompatible(Used: DemandedFields::all(), Require: CurInfo, LIS)) {
561 // If this is the first implicit state change, and the state change
562 // requested can be proven to produce the same register contents, we
563 // can skip emitting the actual state change and continue as if we
564 // had since we know the GPR result of the implicit state change
565 // wouldn't be used and VL/VTYPE registers are correct. Note that
566 // we *do* need to model the state as if it changed as while the
567 // register contents are unchanged, the abstract model can change.
568 if (!PrefixTransparent || needVSETVLIPHI(Require: CurInfo, MBB))
569 insertVSETVLI(MBB, InsertPt: MI, DL: MI.getDebugLoc(), Info: CurInfo, PrevInfo);
570 PrefixTransparent = false;
571 }
572
573 if (RISCVII::hasVLOp(TSFlags)) {
574 MachineOperand &VLOp = getVLOp(MI);
575 if (VLOp.isReg()) {
576 Register Reg = VLOp.getReg();
577
578 // Erase the AVL operand from the instruction.
579 VLOp.setReg(Register());
580 VLOp.setIsKill(false);
581 if (LIS) {
582 LiveInterval &LI = LIS->getInterval(Reg);
583 SmallVector<MachineInstr *> DeadMIs;
584 LIS->shrinkToUses(li: &LI, dead: &DeadMIs);
585 // We might have separate components that need split due to
586 // needVSETVLIPHI causing us to skip inserting a new VL def.
587 SmallVector<LiveInterval *> SplitLIs;
588 LIS->splitSeparateComponents(LI, SplitLIs);
589
590 // If the AVL was an immediate > 31, then it would have been emitted
591 // as an ADDI. However, the ADDI might not have been used in the
592 // vsetvli, or a vsetvli might not have been emitted, so it may be
593 // dead now.
594 for (MachineInstr *DeadMI : DeadMIs) {
595 if (!TII->isAddImmediate(MI: *DeadMI, Reg))
596 continue;
597 LIS->RemoveMachineInstrFromMaps(MI&: *DeadMI);
598 Register AddReg = DeadMI->getOperand(i: 1).getReg();
599 DeadMI->eraseFromParent();
600 if (AddReg.isVirtual())
601 LIS->shrinkToUses(li: &LIS->getInterval(Reg: AddReg));
602 }
603 }
604 }
605 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VL, /*isDef*/ false,
606 /*isImp*/ true));
607 }
608 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ false,
609 /*isImp*/ true));
610 }
611
612 if (MI.isInlineAsm()) {
613 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VL, /*isDef*/ true,
614 /*isImp*/ true));
615 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ true,
616 /*isImp*/ true));
617 }
618
619 if (MI.isCall() || MI.isInlineAsm() ||
620 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
621 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
622 PrefixTransparent = false;
623
624 transferAfter(Info&: CurInfo, MI);
625 }
626
627 const auto &Info = BlockInfo[MBB.getNumber()];
628 if (CurInfo != Info.Exit) {
629 LLVM_DEBUG(dbgs() << "in block " << printMBBReference(MBB) << "\n");
630 LLVM_DEBUG(dbgs() << " begin state: " << Info.Pred << "\n");
631 LLVM_DEBUG(dbgs() << " expected end state: " << Info.Exit << "\n");
632 LLVM_DEBUG(dbgs() << " actual end state: " << CurInfo << "\n");
633 }
634 assert(CurInfo == Info.Exit && "InsertVSETVLI dataflow invariant violated");
635}
636
637/// Perform simple partial redundancy elimination of the VSETVLI instructions
638/// we're about to insert by looking for cases where we can PRE from the
639/// beginning of one block to the end of one of its predecessors. Specifically,
640/// this is geared to catch the common case of a fixed length vsetvl in a single
641/// block loop when it could execute once in the preheader instead.
642void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {
643 if (!BlockInfo[MBB.getNumber()].Pred.isUnknown())
644 return;
645
646 MachineBasicBlock *UnavailablePred = nullptr;
647 VSETVLIInfo AvailableInfo;
648 for (MachineBasicBlock *P : MBB.predecessors()) {
649 const VSETVLIInfo &PredInfo = BlockInfo[P->getNumber()].Exit;
650 if (PredInfo.isUnknown()) {
651 if (UnavailablePred)
652 return;
653 UnavailablePred = P;
654 } else if (!AvailableInfo.isValid()) {
655 AvailableInfo = PredInfo;
656 } else if (AvailableInfo != PredInfo) {
657 return;
658 }
659 }
660
661 // Unreachable, single pred, or full redundancy. Note that FRE is handled by
662 // phase 3.
663 if (!UnavailablePred || !AvailableInfo.isValid())
664 return;
665
666 if (!LIS)
667 return;
668
669 // If we don't know the exact VTYPE, we can't copy the vsetvli to the exit of
670 // the unavailable pred.
671 if (AvailableInfo.hasSEWLMULRatioOnly())
672 return;
673
674 // Critical edge - TODO: consider splitting?
675 if (UnavailablePred->succ_size() != 1)
676 return;
677
678 // If the AVL value is a register (other than our VLMAX sentinel),
679 // we need to prove the value is available at the point we're going
680 // to insert the vsetvli at.
681 if (AvailableInfo.hasAVLReg()) {
682 SlotIndex SI = AvailableInfo.getAVLVNInfo()->def;
683 // This is an inline dominance check which covers the case of
684 // UnavailablePred being the preheader of a loop.
685 if (LIS->getMBBFromIndex(index: SI) != UnavailablePred)
686 return;
687 if (!UnavailablePred->terminators().empty() &&
688 SI >= LIS->getInstructionIndex(Instr: *UnavailablePred->getFirstTerminator()))
689 return;
690 }
691
692 // Model the effect of changing the input state of the block MBB to
693 // AvailableInfo. We're looking for two issues here; one legality,
694 // one profitability.
695 // 1) If the block doesn't use some of the fields from VL or VTYPE, we
696 // may hit the end of the block with a different end state. We can
697 // not make this change without reflowing later blocks as well.
698 // 2) If we don't actually remove a transition, inserting a vsetvli
699 // into the predecessor block would be correct, but unprofitable.
700 VSETVLIInfo OldInfo = BlockInfo[MBB.getNumber()].Pred;
701 VSETVLIInfo CurInfo = AvailableInfo;
702 int TransitionsRemoved = 0;
703 for (const MachineInstr &MI : MBB) {
704 const VSETVLIInfo LastInfo = CurInfo;
705 const VSETVLIInfo LastOldInfo = OldInfo;
706 transferBefore(Info&: CurInfo, MI);
707 transferBefore(Info&: OldInfo, MI);
708 if (CurInfo == LastInfo)
709 TransitionsRemoved++;
710 if (LastOldInfo == OldInfo)
711 TransitionsRemoved--;
712 transferAfter(Info&: CurInfo, MI);
713 transferAfter(Info&: OldInfo, MI);
714 if (CurInfo == OldInfo)
715 // Convergence. All transitions after this must match by construction.
716 break;
717 }
718 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
719 // Issues 1 and 2 above
720 return;
721
722 // Finally, update both data flow state and insert the actual vsetvli.
723 // Doing both keeps the code in sync with the dataflow results, which
724 // is critical for correctness of phase 3.
725 auto OldExit = BlockInfo[UnavailablePred->getNumber()].Exit;
726 LLVM_DEBUG(dbgs() << "PRE VSETVLI from " << MBB.getName() << " to "
727 << UnavailablePred->getName() << " with state "
728 << AvailableInfo << "\n");
729 BlockInfo[UnavailablePred->getNumber()].Exit = AvailableInfo;
730 BlockInfo[MBB.getNumber()].Pred = AvailableInfo;
731
732 // Note there's an implicit assumption here that terminators never use
733 // or modify VL or VTYPE. Also, fallthrough will return end().
734 auto InsertPt = UnavailablePred->getFirstInstrTerminator();
735 insertVSETVLI(MBB&: *UnavailablePred, InsertPt,
736 DL: UnavailablePred->findDebugLoc(MBBI: InsertPt),
737 Info: AvailableInfo, PrevInfo: OldExit);
738}
739
740// Return true if we can mutate PrevMI to match MI without changing any the
741// fields which would be observed.
742// If AVLDefToMove is non-null after the call, it points to an ADDI
743// instruction that needs to be moved before PrevMI.
744bool RISCVInsertVSETVLI::canMutatePriorConfig(
745 const MachineInstr &PrevMI, const MachineInstr &MI,
746 const DemandedFields &Used, MachineInstr *&AVLDefToMove) const {
747 AVLDefToMove = nullptr;
748 // If the VL values aren't equal, return false if either a) the former is
749 // demanded, or b) we can't rewrite the former to be the later for
750 // implementation reasons.
751 if (!RISCVInstrInfo::isVLPreservingConfig(MI)) {
752 if (Used.VLAny)
753 return false;
754
755 if (Used.VLZeroness) {
756 if (RISCVInstrInfo::isVLPreservingConfig(MI: PrevMI))
757 return false;
758 if (!VIA.getInfoForVSETVLI(MI: PrevMI).hasEquallyZeroAVL(
759 Other: VIA.getInfoForVSETVLI(MI), LIS))
760 return false;
761 }
762
763 auto &AVL = MI.getOperand(i: 1);
764
765 // If the AVL is a register, we need to make sure its definition is the same
766 // at PrevMI as it was at MI.
767 if (AVL.isReg() && AVL.getReg() != RISCV::X0) {
768 VNInfo *VNI = getVNInfoFromReg(Reg: AVL.getReg(), MI, LIS);
769 VNInfo *PrevVNI = getVNInfoFromReg(Reg: AVL.getReg(), MI: PrevMI, LIS);
770 if (!VNI || !PrevVNI || VNI != PrevVNI) {
771 // If LIS is null, we were not able to get the VNInfo so we don't know
772 // if the AVL def needs to be moved.
773 if (!LIS)
774 return false;
775 // If the AVL is defined by a load immediate instruction (ADDI x0, imm),
776 // it can be moved earlier since it has no register dependencies.
777 if (!AVL.getReg().isVirtual())
778 return false;
779
780 MachineInstr *DefMI = MRI->getUniqueVRegDef(Reg: AVL.getReg());
781 if (!DefMI || !RISCVInstrInfo::isLoadImmediate(MI: *DefMI) ||
782 DefMI->getParent() != PrevMI.getParent()) {
783 return false;
784 }
785 // Mark that this ADDI needs to be moved.
786 AVLDefToMove = DefMI;
787 }
788 }
789
790 // If we define VL and need to move the definition up, check we can extend
791 // the live interval upwards from MI to PrevMI.
792 Register VL = MI.getOperand(i: 0).getReg();
793 if (VL.isVirtual() && LIS &&
794 LIS->getInterval(Reg: VL).overlaps(Start: LIS->getInstructionIndex(Instr: PrevMI),
795 End: LIS->getInstructionIndex(Instr: MI)))
796 return false;
797 }
798
799 assert(PrevMI.getOperand(2).isImm() && MI.getOperand(2).isImm());
800 auto PriorVType = PrevMI.getOperand(i: 2).getImm();
801 auto VType = MI.getOperand(i: 2).getImm();
802 return areCompatibleVTYPEs(CurVType: PriorVType, NewVType: VType, Used);
803}
804
805void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {
806 MachineInstr *NextMI = nullptr;
807 // We can have arbitrary code in successors, so VL and VTYPE
808 // must be considered demanded.
809 DemandedFields Used;
810 Used.demandVL();
811 Used.demandVTYPE();
812 SmallVector<MachineInstr*> ToDelete;
813
814 auto dropAVLUse = [&](MachineOperand &MO) {
815 if (!MO.isReg() || !MO.getReg().isVirtual())
816 return;
817 Register OldVLReg = MO.getReg();
818 MO.setReg(Register());
819
820 if (LIS)
821 LIS->shrinkToUses(li: &LIS->getInterval(Reg: OldVLReg));
822
823 MachineInstr *VLOpDef = MRI->getUniqueVRegDef(Reg: OldVLReg);
824 if (VLOpDef && TII->isAddImmediate(MI: *VLOpDef, Reg: OldVLReg) &&
825 MRI->use_nodbg_empty(RegNo: OldVLReg))
826 ToDelete.push_back(Elt: VLOpDef);
827 };
828
829 for (MachineInstr &MI : make_early_inc_range(Range: reverse(C&: MBB))) {
830 // TODO: Support XSfmm.
831 if (RISCVII::hasTWidenOp(TSFlags: MI.getDesc().TSFlags) ||
832 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI)) {
833 NextMI = nullptr;
834 continue;
835 }
836
837 if (!RISCVInstrInfo::isVectorConfigInstr(MI)) {
838 Used.doUnion(B: getDemanded(MI, ST));
839 if (MI.isCall() || MI.isInlineAsm() ||
840 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
841 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
842 NextMI = nullptr;
843 continue;
844 }
845
846 if (!MI.getOperand(i: 0).isDead())
847 Used.demandVL();
848
849 if (NextMI) {
850 if (!Used.usedVL() && !Used.usedVTYPE()) {
851 dropAVLUse(MI.getOperand(i: 1));
852 if (LIS)
853 LIS->RemoveMachineInstrFromMaps(MI);
854 MI.eraseFromParent();
855 NumCoalescedVSETVL++;
856 // Leave NextMI unchanged
857 continue;
858 }
859
860 MachineInstr *AVLDefToMove = nullptr;
861 if (canMutatePriorConfig(PrevMI: MI, MI: *NextMI, Used, AVLDefToMove)) {
862 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *NextMI)) {
863 Register DefReg = NextMI->getOperand(i: 0).getReg();
864
865 MI.getOperand(i: 0).setReg(DefReg);
866 MI.getOperand(i: 0).setIsDead(false);
867
868 // Move the AVL from NextMI to MI
869 dropAVLUse(MI.getOperand(i: 1));
870 if (NextMI->getOperand(i: 1).isImm())
871 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: NextMI->getOperand(i: 1).getImm());
872 else {
873 MI.getOperand(i: 1).ChangeToRegister(Reg: NextMI->getOperand(i: 1).getReg(),
874 isDef: false);
875
876 // If canMutatePriorConfig indicated that an ADDI needs to be moved,
877 // move it now.
878 if (AVLDefToMove) {
879 AVLDefToMove->moveBefore(MovePos: &MI);
880 if (LIS)
881 LIS->handleMove(MI&: *AVLDefToMove);
882 }
883 }
884 dropAVLUse(NextMI->getOperand(i: 1));
885
886 // The def of DefReg moved to MI, so extend the LiveInterval up to
887 // it.
888 if (DefReg.isVirtual() && LIS) {
889 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
890 SlotIndex MISlot = LIS->getInstructionIndex(Instr: MI).getRegSlot();
891 SlotIndex NextMISlot =
892 LIS->getInstructionIndex(Instr: *NextMI).getRegSlot();
893 VNInfo *DefVNI = DefLI.getVNInfoAt(Idx: NextMISlot);
894 LiveInterval::Segment S(MISlot, NextMISlot, DefVNI);
895 DefLI.addSegment(S);
896 DefVNI->def = MISlot;
897 // Mark DefLI as spillable if it was previously unspillable
898 DefLI.setWeight(0);
899
900 // DefReg may have had no uses, in which case we need to shrink
901 // the LiveInterval up to MI.
902 LIS->shrinkToUses(li: &DefLI);
903 }
904
905 MI.setDesc(NextMI->getDesc());
906 }
907 MI.getOperand(i: 2).setImm(NextMI->getOperand(i: 2).getImm());
908
909 dropAVLUse(NextMI->getOperand(i: 1));
910 if (LIS)
911 LIS->RemoveMachineInstrFromMaps(MI&: *NextMI);
912 NextMI->eraseFromParent();
913 NumCoalescedVSETVL++;
914 // fallthrough
915 }
916 }
917 NextMI = &MI;
918 Used = getDemanded(MI, ST);
919 }
920
921 // Loop over the dead AVL values, and delete them now. This has
922 // to be outside the above loop to avoid invalidating iterators.
923 for (auto *MI : ToDelete) {
924 assert(MI->getOpcode() == RISCV::ADDI);
925 Register AddReg = MI->getOperand(i: 1).getReg();
926 if (LIS) {
927 LIS->removeInterval(Reg: MI->getOperand(i: 0).getReg());
928 LIS->RemoveMachineInstrFromMaps(MI&: *MI);
929 }
930 MI->eraseFromParent();
931 if (LIS && AddReg.isVirtual())
932 LIS->shrinkToUses(li: &LIS->getInterval(Reg: AddReg));
933 }
934}
935
936void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
937 for (auto I = MBB.begin(), E = MBB.end(); I != E;) {
938 MachineInstr &MI = *I++;
939 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
940 Register VLOutput = MI.getOperand(i: 1).getReg();
941 assert(VLOutput.isVirtual());
942 if (!MI.getOperand(i: 1).isDead()) {
943 auto ReadVLMI = BuildMI(BB&: MBB, I, MIMD: MI.getDebugLoc(),
944 MCID: TII->get(Opcode: RISCV::PseudoReadVL), DestReg: VLOutput);
945 // Move the LiveInterval's definition down to PseudoReadVL.
946 if (LIS) {
947 SlotIndex NewDefSI =
948 LIS->InsertMachineInstrInMaps(MI&: *ReadVLMI).getRegSlot();
949 LiveInterval &DefLI = LIS->getInterval(Reg: VLOutput);
950 LiveRange::Segment *DefSeg = DefLI.getSegmentContaining(Idx: NewDefSI);
951 VNInfo *DefVNI = DefLI.getVNInfoAt(Idx: DefSeg->start);
952 DefLI.removeSegment(Start: DefSeg->start, End: NewDefSI);
953 DefVNI->def = NewDefSI;
954 }
955 }
956 // We don't use the vl output of the VLEFF/VLSEGFF anymore.
957 MI.getOperand(i: 1).setReg(RISCV::X0);
958 MI.addRegisterDefined(Reg: RISCV::VL, RegInfo: MRI->getTargetRegisterInfo());
959 }
960 }
961}
962
963bool RISCVInsertVSETVLI::insertVSETMTK(MachineBasicBlock &MBB,
964 TKTMMode Mode) const {
965
966 bool Changed = false;
967 for (auto &MI : MBB) {
968 uint64_t TSFlags = MI.getDesc().TSFlags;
969 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI) ||
970 !RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasTWidenOp(TSFlags))
971 continue;
972
973 VSETVLIInfo CurrInfo = VIA.computeInfoForInstr(MI);
974
975 unsigned Opcode = 0, OpNum = 0;
976 switch (Mode) {
977 case VSETTK:
978 if (!RISCVII::hasTKOp(TSFlags))
979 continue;
980 OpNum = RISCVII::getTKOpNum(Desc: MI.getDesc());
981 Opcode = RISCV::PseudoSF_VSETTK;
982 break;
983 case VSETTM:
984 if (!RISCVII::hasTMOp(TSFlags))
985 continue;
986 OpNum = RISCVII::getTMOpNum(Desc: MI.getDesc());
987 Opcode = RISCV::PseudoSF_VSETTM;
988 break;
989 }
990
991 assert(OpNum && Opcode && "Invalid OpNum or Opcode");
992
993 MachineOperand &Op = MI.getOperand(i: OpNum);
994
995 auto TmpMI = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode))
996 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
997 .addReg(RegNo: Op.getReg())
998 .addImm(Val: Log2_32(Value: CurrInfo.getSEW()))
999 .addImm(Val: CurrInfo.getTWiden());
1000
1001 Changed = true;
1002 Register Reg = Op.getReg();
1003 Op.setReg(Register());
1004 Op.setIsKill(false);
1005 if (LIS) {
1006 LIS->InsertMachineInstrInMaps(MI&: *TmpMI);
1007 LiveInterval &LI = LIS->getInterval(Reg);
1008
1009 // Erase the AVL operand from the instruction.
1010 LIS->shrinkToUses(li: &LI);
1011 // TODO: Enable this once needVSETVLIPHI is supported.
1012 // SmallVector<LiveInterval *> SplitLIs;
1013 // LIS->splitSeparateComponents(LI, SplitLIs);
1014 }
1015 }
1016 return Changed;
1017}
1018
1019bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {
1020 // Skip if the vector extension is not enabled.
1021 ST = &MF.getSubtarget<RISCVSubtarget>();
1022 if (!ST->hasVInstructions())
1023 return false;
1024
1025 LLVM_DEBUG(dbgs() << "Entering InsertVSETVLI for " << MF.getName() << "\n");
1026
1027 TII = ST->getInstrInfo();
1028 MRI = &MF.getRegInfo();
1029 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
1030 LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
1031 VIA = RISCVVSETVLIInfoAnalysis(ST, LIS);
1032
1033 assert(BlockInfo.empty() && "Expect empty block infos");
1034 BlockInfo.resize(new_size: MF.getNumBlockIDs());
1035
1036 bool HaveVectorOp = false;
1037
1038 // Phase 1 - determine how VL/VTYPE are affected by the each block.
1039 for (const MachineBasicBlock &MBB : MF) {
1040 VSETVLIInfo TmpStatus;
1041 HaveVectorOp |= computeVLVTYPEChanges(MBB, Info&: TmpStatus);
1042 // Initial exit state is whatever change we found in the block.
1043 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
1044 BBInfo.Exit = TmpStatus;
1045 LLVM_DEBUG(dbgs() << "Initial exit state of " << printMBBReference(MBB)
1046 << " is " << BBInfo.Exit << "\n");
1047
1048 }
1049
1050 // If we didn't find any instructions that need VSETVLI, we're done.
1051 if (!HaveVectorOp) {
1052 BlockInfo.clear();
1053 return false;
1054 }
1055
1056 // Phase 2 - determine the exit VL/VTYPE from each block. We add all
1057 // blocks to the list here, but will also add any that need to be revisited
1058 // during Phase 2 processing.
1059 for (const MachineBasicBlock &MBB : MF) {
1060 WorkList.push(x: &MBB);
1061 BlockInfo[MBB.getNumber()].InQueue = true;
1062 }
1063 while (!WorkList.empty()) {
1064 const MachineBasicBlock &MBB = *WorkList.front();
1065 WorkList.pop();
1066 computeIncomingVLVTYPE(MBB);
1067 }
1068
1069 // Perform partial redundancy elimination of vsetvli transitions.
1070 for (MachineBasicBlock &MBB : MF)
1071 doPRE(MBB);
1072
1073 // Phase 3 - add any vsetvli instructions needed in the block. Use the
1074 // Phase 2 information to avoid adding vsetvlis before the first vector
1075 // instruction in the block if the VL/VTYPE is satisfied by its
1076 // predecessors.
1077 for (MachineBasicBlock &MBB : MF)
1078 emitVSETVLIs(MBB);
1079
1080 // Now that all vsetvlis are explicit, go through and do block local
1081 // DSE and peephole based demanded fields based transforms. Note that
1082 // this *must* be done outside the main dataflow so long as we allow
1083 // any cross block analysis within the dataflow. We can't have both
1084 // demanded fields based mutation and non-local analysis in the
1085 // dataflow at the same time without introducing inconsistencies.
1086 // We're visiting blocks from the bottom up because a VSETVLI in the
1087 // earlier block might become dead when its uses in later blocks are
1088 // optimized away.
1089 for (MachineBasicBlock *MBB : post_order(G: &MF))
1090 coalesceVSETVLIs(MBB&: *MBB);
1091
1092 // Insert PseudoReadVL after VLEFF/VLSEGFF and replace it with the vl output
1093 // of VLEFF/VLSEGFF.
1094 for (MachineBasicBlock &MBB : MF)
1095 insertReadVL(MBB);
1096
1097 if (ST->hasVendorXSfmmbase()) {
1098 for (MachineBasicBlock &MBB : MF) {
1099 insertVSETMTK(MBB, Mode: VSETTM);
1100 insertVSETMTK(MBB, Mode: VSETTK);
1101 }
1102 }
1103
1104 BlockInfo.clear();
1105 return HaveVectorOp;
1106}
1107
1108/// Returns an instance of the Insert VSETVLI pass.
1109FunctionPass *llvm::createRISCVInsertVSETVLIPass() {
1110 return new RISCVInsertVSETVLI();
1111}
1112