1//===- RISCVInsertVSETVLI.cpp - Insert VSETVLI instructions ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a function pass that inserts VSETVLI instructions where
10// needed and expands the vl outputs of VLEFF/VLSEGFF to PseudoReadVL
11// instructions.
12//
13// This pass consists of 3 phases:
14//
15// Phase 1 collects how each basic block affects VL/VTYPE.
16//
17// Phase 2 uses the information from phase 1 to do a data flow analysis to
18// propagate the VL/VTYPE changes through the function. This gives us the
19// VL/VTYPE at the start of each basic block.
20//
21// Phase 3 inserts VSETVLI instructions in each basic block. Information from
22// phase 2 is used to prevent inserting a VSETVLI before the first vector
23// instruction in the block if possible.
24//
25//===----------------------------------------------------------------------===//
26
27#include "RISCV.h"
28#include "RISCVSubtarget.h"
29#include "RISCVVSETVLIInfoAnalysis.h"
30#include "llvm/ADT/PostOrderIterator.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/CodeGen/LiveDebugVariables.h"
33#include "llvm/CodeGen/LiveIntervals.h"
34#include "llvm/CodeGen/LiveStacks.h"
35#include "llvm/CodeGen/MachineFunctionPass.h"
36#include <queue>
37using namespace llvm;
38using namespace RISCV;
39
40#define DEBUG_TYPE "riscv-insert-vsetvli"
41#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
42
43STATISTIC(NumInsertedVSETVL, "Number of VSETVL inst inserted");
44STATISTIC(NumCoalescedVSETVL, "Number of VSETVL inst coalesced");
45
46static cl::opt<bool> EnsureWholeVectorRegisterMoveValidVTYPE(
47 DEBUG_TYPE "-whole-vector-register-move-valid-vtype", cl::Hidden,
48 cl::desc("Insert vsetvlis before vmvNr.vs to ensure vtype is valid and "
49 "vill is cleared"),
50 cl::init(Val: true));
51
52namespace {
53
54/// Given a virtual register \p Reg, return the corresponding VNInfo for it.
55/// This will return nullptr if the virtual register is an implicit_def or
56/// if LiveIntervals is not available.
57static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
58 const LiveIntervals *LIS) {
59 assert(Reg.isVirtual());
60 if (!LIS)
61 return nullptr;
62 auto &LI = LIS->getInterval(Reg);
63 SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
64 return LI.getVNInfoBefore(Idx: SI);
65}
66
67static MachineOperand &getVLOp(MachineInstr &MI) {
68 return MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
69}
70
71struct BlockData {
72 // The VSETVLIInfo that represents the VL/VTYPE settings on exit from this
73 // block. Calculated in Phase 2.
74 VSETVLIInfo Exit;
75
76 // The VSETVLIInfo that represents the VL/VTYPE settings from all predecessor
77 // blocks. Calculated in Phase 2, and used by Phase 3.
78 VSETVLIInfo Pred;
79
80 // Keeps track of whether the block is already in the queue.
81 bool InQueue = false;
82
83 BlockData() = default;
84};
85
86enum TKTMMode {
87 VSETTK = 0,
88 VSETTM = 1,
89};
90
91class RISCVInsertVSETVLI : public MachineFunctionPass {
92 const RISCVSubtarget *ST;
93 const TargetInstrInfo *TII;
94 MachineRegisterInfo *MRI;
95 // Possibly null!
96 LiveIntervals *LIS;
97 RISCVVSETVLIInfoAnalysis VIA;
98
99 std::vector<BlockData> BlockInfo;
100 std::queue<const MachineBasicBlock *> WorkList;
101
102public:
103 static char ID;
104
105 RISCVInsertVSETVLI() : MachineFunctionPass(ID) {}
106 bool runOnMachineFunction(MachineFunction &MF) override;
107
108 void getAnalysisUsage(AnalysisUsage &AU) const override {
109 AU.setPreservesCFG();
110
111 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
112 AU.addPreserved<LiveIntervalsWrapperPass>();
113 AU.addPreserved<SlotIndexesWrapperPass>();
114 AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
115 AU.addPreserved<LiveStacksWrapperLegacy>();
116
117 MachineFunctionPass::getAnalysisUsage(AU);
118 }
119
120 StringRef getPassName() const override { return RISCV_INSERT_VSETVLI_NAME; }
121
122private:
123 bool needVSETVLI(const DemandedFields &Used, const VSETVLIInfo &Require,
124 const VSETVLIInfo &CurInfo) const;
125 bool needVSETVLIPHI(const VSETVLIInfo &Require,
126 const MachineBasicBlock &MBB) const;
127 void insertVSETVLI(MachineBasicBlock &MBB,
128 MachineBasicBlock::iterator InsertPt, DebugLoc DL,
129 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);
130
131 void transferBefore(VSETVLIInfo &Info, const MachineInstr &MI) const;
132 void transferAfter(VSETVLIInfo &Info, const MachineInstr &MI) const;
133 bool computeVLVTYPEChanges(const MachineBasicBlock &MBB,
134 VSETVLIInfo &Info) const;
135 void computeIncomingVLVTYPE(const MachineBasicBlock &MBB);
136 void emitVSETVLIs(MachineBasicBlock &MBB);
137 void doPRE(MachineBasicBlock &MBB);
138 void insertReadVL(MachineBasicBlock &MBB);
139
140 bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI,
141 const DemandedFields &Used,
142 MachineInstr *&AVLDefToMove) const;
143 void coalesceVSETVLIs(MachineBasicBlock &MBB) const;
144 bool insertVSETMTK(MachineBasicBlock &MBB, TKTMMode Mode) const;
145};
146
147} // end anonymous namespace
148
149char RISCVInsertVSETVLI::ID = 0;
150char &llvm::RISCVInsertVSETVLIID = RISCVInsertVSETVLI::ID;
151
152INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME,
153 false, false)
154
155void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
156 MachineBasicBlock::iterator InsertPt,
157 DebugLoc DL, const VSETVLIInfo &Info,
158 const VSETVLIInfo &PrevInfo) {
159 ++NumInsertedVSETVL;
160
161 if (Info.getTWiden()) {
162 if (Info.hasAVLVLMAX()) {
163 Register DestReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
164 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoSF_VSETTNTX0))
165 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Dead)
166 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
167 .addImm(Val: Info.encodeVTYPE());
168 if (LIS) {
169 LIS->InsertMachineInstrInMaps(MI&: *MI);
170 LIS->createAndComputeVirtRegInterval(Reg: DestReg);
171 }
172 } else {
173 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoSF_VSETTNT))
174 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
175 .addReg(RegNo: Info.getAVLReg())
176 .addImm(Val: Info.encodeVTYPE());
177 if (LIS)
178 LIS->InsertMachineInstrInMaps(MI&: *MI);
179 }
180 return;
181 }
182
183 if (PrevInfo.isValid() && !PrevInfo.isUnknown()) {
184 // Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same
185 // VLMAX.
186 if (Info.hasSameAVL(Other: PrevInfo) && Info.hasSameVLMAX(Other: PrevInfo)) {
187 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoVSETVLIX0X0))
188 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
189 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
190 .addImm(Val: Info.encodeVTYPE())
191 .addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
192 if (LIS)
193 LIS->InsertMachineInstrInMaps(MI&: *MI);
194 return;
195 }
196
197 // If our AVL is a virtual register, it might be defined by a VSET(I)VLI. If
198 // it has the same VLMAX we want and the last VL/VTYPE we observed is the
199 // same, we can use the X0, X0 form.
200 if (Info.hasSameVLMAX(Other: PrevInfo) && Info.hasAVLReg()) {
201 if (const MachineInstr *DefMI = Info.getAVLDefMI(LIS);
202 DefMI && RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI)) {
203 VSETVLIInfo DefInfo = VIA.getInfoForVSETVLI(MI: *DefMI);
204 if (DefInfo.hasSameAVL(Other: PrevInfo) && DefInfo.hasSameVLMAX(Other: PrevInfo)) {
205 auto MI =
206 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoVSETVLIX0X0))
207 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
208 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
209 .addImm(Val: Info.encodeVTYPE())
210 .addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
211 if (LIS)
212 LIS->InsertMachineInstrInMaps(MI&: *MI);
213 return;
214 }
215 }
216 }
217 }
218
219 if (Info.hasAVLImm()) {
220 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoVSETIVLI))
221 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
222 .addImm(Val: Info.getAVLImm())
223 .addImm(Val: Info.encodeVTYPE());
224 if (LIS)
225 LIS->InsertMachineInstrInMaps(MI&: *MI);
226 return;
227 }
228
229 if (Info.hasAVLVLMAX()) {
230 Register DestReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
231 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoVSETVLIX0))
232 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Dead)
233 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
234 .addImm(Val: Info.encodeVTYPE());
235 if (LIS) {
236 LIS->InsertMachineInstrInMaps(MI&: *MI);
237 LIS->createAndComputeVirtRegInterval(Reg: DestReg);
238 }
239 return;
240 }
241
242 Register AVLReg = Info.getAVLReg();
243 MRI->constrainRegClass(Reg: AVLReg, RC: &RISCV::GPRNoX0RegClass);
244 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoVSETVLI))
245 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
246 .addReg(RegNo: AVLReg)
247 .addImm(Val: Info.encodeVTYPE());
248 if (LIS) {
249 LIS->InsertMachineInstrInMaps(MI&: *MI);
250 LiveInterval &LI = LIS->getInterval(Reg: AVLReg);
251 SlotIndex SI = LIS->getInstructionIndex(Instr: *MI).getRegSlot();
252 const VNInfo *CurVNI = Info.getAVLVNInfo();
253 // If the AVL value isn't live at MI, do a quick check to see if it's easily
254 // extendable. Otherwise, we need to copy it.
255 if (LI.getVNInfoBefore(Idx: SI) != CurVNI) {
256 if (!LI.liveAt(index: SI) && LI.containsOneValue())
257 LIS->extendToIndices(LR&: LI, Indices: SI);
258 else {
259 Register AVLCopyReg =
260 MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
261 MachineBasicBlock *MBB = LIS->getMBBFromIndex(index: CurVNI->def);
262 MachineBasicBlock::iterator II;
263 if (CurVNI->isPHIDef())
264 II = MBB->getFirstNonPHI();
265 else {
266 II = LIS->getInstructionFromIndex(index: CurVNI->def);
267 II = std::next(x: II);
268 }
269 assert(II.isValid());
270 auto AVLCopy = BuildMI(BB&: *MBB, I: II, MIMD: DL, MCID: TII->get(Opcode: RISCV::COPY), DestReg: AVLCopyReg)
271 .addReg(RegNo: AVLReg);
272 LIS->InsertMachineInstrInMaps(MI&: *AVLCopy);
273 MI->getOperand(i: 1).setReg(AVLCopyReg);
274 LIS->createAndComputeVirtRegInterval(Reg: AVLCopyReg);
275 }
276 }
277 }
278}
279
280/// Return true if a VSETVLI is required to transition from CurInfo to Require
281/// given a set of DemandedFields \p Used.
282bool RISCVInsertVSETVLI::needVSETVLI(const DemandedFields &Used,
283 const VSETVLIInfo &Require,
284 const VSETVLIInfo &CurInfo) const {
285 if (!CurInfo.isValid() || CurInfo.isUnknown() || CurInfo.hasSEWLMULRatioOnly())
286 return true;
287
288 if (CurInfo.isCompatible(Used, Require, LIS))
289 return false;
290
291 return true;
292}
293
294// If we don't use LMUL or the SEW/LMUL ratio, then adjust LMUL so that we
295// maintain the SEW/LMUL ratio. This allows us to eliminate VL toggles in more
296// places.
297static VSETVLIInfo adjustIncoming(const VSETVLIInfo &PrevInfo,
298 const VSETVLIInfo &NewInfo,
299 DemandedFields &Demanded) {
300 VSETVLIInfo Info = NewInfo;
301
302 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isValid() &&
303 !PrevInfo.isUnknown()) {
304 if (auto NewVLMul = RISCVVType::getSameRatioLMUL(Ratio: PrevInfo.getSEWLMULRatio(),
305 EEW: Info.getSEW()))
306 Info.setVLMul(*NewVLMul);
307 Demanded.LMUL = DemandedFields::LMULEqual;
308 }
309
310 return Info;
311}
312
313// Given an incoming state reaching MI, minimally modifies that state so that it
314// is compatible with MI. The resulting state is guaranteed to be semantically
315// legal for MI, but may not be the state requested by MI.
316void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
317 const MachineInstr &MI) const {
318 if (EnsureWholeVectorRegisterMoveValidVTYPE &&
319 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI) &&
320 (Info.isUnknown() || !Info.isValid() || Info.hasSEWLMULRatioOnly())) {
321 // Use an arbitrary but valid AVL and VTYPE so vill will be cleared. It may
322 // be coalesced into another vsetvli since we won't demand any fields.
323 VSETVLIInfo NewInfo; // Need a new VSETVLIInfo to clear SEWLMULRatioOnly
324 NewInfo.setAVLImm(1);
325 NewInfo.setVTYPE(L: RISCVVType::LMUL_1, /*sew*/ S: 8, /*ta*/ TA: true, /*ma*/ MA: true,
326 /*AltFmt*/ Altfmt: false, /*W*/ 0);
327 Info = NewInfo;
328 return;
329 }
330
331 if (!RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags))
332 return;
333
334 DemandedFields Demanded = getDemanded(MI, ST);
335
336 const VSETVLIInfo NewInfo = VIA.computeInfoForInstr(MI);
337 assert(NewInfo.isValid() && !NewInfo.isUnknown());
338 if (Info.isValid() && !needVSETVLI(Used: Demanded, Require: NewInfo, CurInfo: Info))
339 return;
340
341 const VSETVLIInfo PrevInfo = Info;
342 if (!Info.isValid() || Info.isUnknown())
343 Info = NewInfo;
344
345 const VSETVLIInfo IncomingInfo = adjustIncoming(PrevInfo, NewInfo, Demanded);
346
347 // If MI only demands that VL has the same zeroness, we only need to set the
348 // AVL if the zeroness differs. This removes a vsetvli entirely if the types
349 // match or allows use of cheaper avl preserving variant if VLMAX doesn't
350 // change. If VLMAX might change, we couldn't use the 'vsetvli x0, x0, vtype"
351 // variant, so we avoid the transform to prevent extending live range of an
352 // avl register operand.
353 // TODO: We can probably relax this for immediates.
354 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(Other: PrevInfo, LIS) &&
355 IncomingInfo.hasSameVLMAX(Other: PrevInfo);
356 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
357 Info.setAVL(IncomingInfo);
358
359 // If we only knew the sew/lmul ratio previously, replace the VTYPE.
360 if (Info.hasSEWLMULRatioOnly()) {
361 VSETVLIInfo RatiolessInfo = IncomingInfo;
362 RatiolessInfo.setAVL(Info);
363 Info = RatiolessInfo;
364 } else {
365 Info.setVTYPE(
366 L: ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
367 .getVLMUL(),
368 S: ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
369 .getSEW(),
370 // Prefer tail/mask agnostic since it can be relaxed to undisturbed
371 // later if needed.
372 TA: (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
373 IncomingInfo.getTailAgnostic(),
374 MA: (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
375 IncomingInfo.getMaskAgnostic(),
376 Altfmt: (Demanded.AltFmt ? IncomingInfo : Info).getAltFmt(),
377 W: Demanded.TWiden ? IncomingInfo.getTWiden() : 0);
378 }
379}
380
381// Given a state with which we evaluated MI (see transferBefore above for why
382// this might be different that the state MI requested), modify the state to
383// reflect the changes MI might make.
384void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
385 const MachineInstr &MI) const {
386 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {
387 Info = VIA.getInfoForVSETVLI(MI);
388 return;
389 }
390
391 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
392 // Update AVL to vl-output of the fault first load.
393 assert(MI.getOperand(1).getReg().isVirtual());
394 if (LIS) {
395 auto &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
396 SlotIndex SI =
397 LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();
398 VNInfo *VNI = LI.getVNInfoAt(Idx: SI);
399 Info.setAVLRegDef(VNInfo: VNI, AVLReg: MI.getOperand(i: 1).getReg());
400 } else
401 Info.setAVLRegDef(VNInfo: nullptr, AVLReg: MI.getOperand(i: 1).getReg());
402 return;
403 }
404
405 // If this is something that updates VL/VTYPE that we don't know about, set
406 // the state to unknown.
407 if (MI.isCall() || MI.isInlineAsm() ||
408 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
409 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
410 Info = VSETVLIInfo::getUnknown();
411}
412
413bool RISCVInsertVSETVLI::computeVLVTYPEChanges(const MachineBasicBlock &MBB,
414 VSETVLIInfo &Info) const {
415 bool HadVectorOp = false;
416
417 Info = BlockInfo[MBB.getNumber()].Pred;
418 for (const MachineInstr &MI : MBB) {
419 transferBefore(Info, MI);
420
421 if (RISCVInstrInfo::isVectorConfigInstr(MI) ||
422 RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags) ||
423 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI) ||
424 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI))
425 HadVectorOp = true;
426
427 transferAfter(Info, MI);
428 }
429
430 return HadVectorOp;
431}
432
433void RISCVInsertVSETVLI::computeIncomingVLVTYPE(const MachineBasicBlock &MBB) {
434
435 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
436
437 BBInfo.InQueue = false;
438
439 // Start with the previous entry so that we keep the most conservative state
440 // we have ever found.
441 VSETVLIInfo InInfo = BBInfo.Pred;
442 if (MBB.pred_empty()) {
443 // There are no predecessors, so use the default starting status.
444 InInfo.setUnknown();
445 } else {
446 for (MachineBasicBlock *P : MBB.predecessors())
447 InInfo = InInfo.intersect(Other: BlockInfo[P->getNumber()].Exit);
448 }
449
450 // If we don't have any valid predecessor value, wait until we do.
451 if (!InInfo.isValid())
452 return;
453
454 // If no change, no need to rerun block
455 if (InInfo == BBInfo.Pred)
456 return;
457
458 BBInfo.Pred = InInfo;
459 LLVM_DEBUG(dbgs() << "Entry state of " << printMBBReference(MBB)
460 << " changed to " << BBInfo.Pred << "\n");
461
462 // Note: It's tempting to cache the state changes here, but due to the
463 // compatibility checks performed a blocks output state can change based on
464 // the input state. To cache, we'd have to add logic for finding
465 // never-compatible state changes.
466 VSETVLIInfo TmpStatus;
467 computeVLVTYPEChanges(MBB, Info&: TmpStatus);
468
469 // If the new exit value matches the old exit value, we don't need to revisit
470 // any blocks.
471 if (BBInfo.Exit == TmpStatus)
472 return;
473
474 BBInfo.Exit = TmpStatus;
475 LLVM_DEBUG(dbgs() << "Exit state of " << printMBBReference(MBB)
476 << " changed to " << BBInfo.Exit << "\n");
477
478 // Add the successors to the work list so we can propagate the changed exit
479 // status.
480 for (MachineBasicBlock *S : MBB.successors())
481 if (!BlockInfo[S->getNumber()].InQueue) {
482 BlockInfo[S->getNumber()].InQueue = true;
483 WorkList.push(x: S);
484 }
485}
486
487// If we weren't able to prove a vsetvli was directly unneeded, it might still
488// be unneeded if the AVL was a phi node where all incoming values are VL
489// outputs from the last VSETVLI in their respective basic blocks.
490bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
491 const MachineBasicBlock &MBB) const {
492 if (!Require.hasAVLReg())
493 return true;
494
495 if (!LIS)
496 return true;
497
498 // We need the AVL to have been produced by a PHI node in this basic block.
499 const VNInfo *Valno = Require.getAVLVNInfo();
500 if (!Valno->isPHIDef() || LIS->getMBBFromIndex(index: Valno->def) != &MBB)
501 return true;
502
503 const LiveRange &LR = LIS->getInterval(Reg: Require.getAVLReg());
504
505 for (auto *PBB : MBB.predecessors()) {
506 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
507
508 // We need the PHI input to the be the output of a VSET(I)VLI.
509 const VNInfo *Value = LR.getVNInfoBefore(Idx: LIS->getMBBEndIdx(mbb: PBB));
510 if (!Value)
511 return true;
512 MachineInstr *DefMI = LIS->getInstructionFromIndex(index: Value->def);
513 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI))
514 return true;
515
516 // We found a VSET(I)VLI make sure it matches the output of the
517 // predecessor block.
518 VSETVLIInfo DefInfo = VIA.getInfoForVSETVLI(MI: *DefMI);
519 if (DefInfo != PBBExit)
520 return true;
521
522 // Require has the same VL as PBBExit, so if the exit from the
523 // predecessor has the VTYPE we are looking for we might be able
524 // to avoid a VSETVLI.
525 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Other: Require))
526 return true;
527 }
528
529 // If all the incoming values to the PHI checked out, we don't need
530 // to insert a VSETVLI.
531 return false;
532}
533
534void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
535 VSETVLIInfo CurInfo = BlockInfo[MBB.getNumber()].Pred;
536 // Track whether the prefix of the block we've scanned is transparent
537 // (meaning has not yet changed the abstract state).
538 bool PrefixTransparent = true;
539 for (MachineInstr &MI : MBB) {
540 const VSETVLIInfo PrevInfo = CurInfo;
541 transferBefore(Info&: CurInfo, MI);
542
543 // If this is an explicit VSETVLI or VSETIVLI, update our state.
544 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {
545 // Conservatively, mark the VL and VTYPE as live.
546 assert(MI.getOperand(3).getReg() == RISCV::VL &&
547 MI.getOperand(4).getReg() == RISCV::VTYPE &&
548 "Unexpected operands where VL and VTYPE should be");
549 MI.getOperand(i: 3).setIsDead(false);
550 MI.getOperand(i: 4).setIsDead(false);
551 PrefixTransparent = false;
552 }
553
554 if (EnsureWholeVectorRegisterMoveValidVTYPE &&
555 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI)) {
556 if (!PrevInfo.isCompatible(Used: DemandedFields::all(), Require: CurInfo, LIS)) {
557 insertVSETVLI(MBB, InsertPt: MI, DL: MI.getDebugLoc(), Info: CurInfo, PrevInfo);
558 PrefixTransparent = false;
559 }
560 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ false,
561 /*isImp*/ true));
562 }
563
564 uint64_t TSFlags = MI.getDesc().TSFlags;
565 if (RISCVII::hasSEWOp(TSFlags)) {
566 if (!PrevInfo.isCompatible(Used: DemandedFields::all(), Require: CurInfo, LIS)) {
567 // If this is the first implicit state change, and the state change
568 // requested can be proven to produce the same register contents, we
569 // can skip emitting the actual state change and continue as if we
570 // had since we know the GPR result of the implicit state change
571 // wouldn't be used and VL/VTYPE registers are correct. Note that
572 // we *do* need to model the state as if it changed as while the
573 // register contents are unchanged, the abstract model can change.
574 if (!PrefixTransparent || needVSETVLIPHI(Require: CurInfo, MBB))
575 insertVSETVLI(MBB, InsertPt: MI, DL: MI.getDebugLoc(), Info: CurInfo, PrevInfo);
576 PrefixTransparent = false;
577 }
578
579 if (RISCVII::hasVLOp(TSFlags)) {
580 MachineOperand &VLOp = getVLOp(MI);
581 if (VLOp.isReg()) {
582 Register Reg = VLOp.getReg();
583
584 // Erase the AVL operand from the instruction.
585 VLOp.setReg(Register());
586 VLOp.setIsKill(false);
587 if (LIS) {
588 LiveInterval &LI = LIS->getInterval(Reg);
589 SmallVector<MachineInstr *> DeadMIs;
590 LIS->shrinkToUses(li: &LI, dead: &DeadMIs);
591 // We might have separate components that need split due to
592 // needVSETVLIPHI causing us to skip inserting a new VL def.
593 SmallVector<LiveInterval *> SplitLIs;
594 LIS->splitSeparateComponents(LI, SplitLIs);
595
596 // If the AVL was an immediate > 31, then it would have been emitted
597 // as an ADDI. However, the ADDI might not have been used in the
598 // vsetvli, or a vsetvli might not have been emitted, so it may be
599 // dead now.
600 for (MachineInstr *DeadMI : DeadMIs) {
601 if (!TII->isAddImmediate(MI: *DeadMI, Reg))
602 continue;
603 LIS->RemoveMachineInstrFromMaps(MI&: *DeadMI);
604 Register AddReg = DeadMI->getOperand(i: 1).getReg();
605 DeadMI->eraseFromParent();
606 if (AddReg.isVirtual())
607 LIS->shrinkToUses(li: &LIS->getInterval(Reg: AddReg));
608 }
609 }
610 }
611 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VL, /*isDef*/ false,
612 /*isImp*/ true));
613 }
614 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ false,
615 /*isImp*/ true));
616 }
617
618 if (MI.isInlineAsm()) {
619 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VL, /*isDef*/ true,
620 /*isImp*/ true));
621 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ true,
622 /*isImp*/ true));
623 }
624
625 if (MI.isCall() || MI.isInlineAsm() ||
626 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
627 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
628 PrefixTransparent = false;
629
630 transferAfter(Info&: CurInfo, MI);
631 }
632
633 const auto &Info = BlockInfo[MBB.getNumber()];
634 if (CurInfo != Info.Exit) {
635 LLVM_DEBUG(dbgs() << "in block " << printMBBReference(MBB) << "\n");
636 LLVM_DEBUG(dbgs() << " begin state: " << Info.Pred << "\n");
637 LLVM_DEBUG(dbgs() << " expected end state: " << Info.Exit << "\n");
638 LLVM_DEBUG(dbgs() << " actual end state: " << CurInfo << "\n");
639 }
640 assert(CurInfo == Info.Exit && "InsertVSETVLI dataflow invariant violated");
641}
642
643/// Perform simple partial redundancy elimination of the VSETVLI instructions
644/// we're about to insert by looking for cases where we can PRE from the
645/// beginning of one block to the end of one of its predecessors. Specifically,
646/// this is geared to catch the common case of a fixed length vsetvl in a single
647/// block loop when it could execute once in the preheader instead.
648void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {
649 if (!BlockInfo[MBB.getNumber()].Pred.isUnknown())
650 return;
651
652 MachineBasicBlock *UnavailablePred = nullptr;
653 VSETVLIInfo AvailableInfo;
654 for (MachineBasicBlock *P : MBB.predecessors()) {
655 const VSETVLIInfo &PredInfo = BlockInfo[P->getNumber()].Exit;
656 if (PredInfo.isUnknown()) {
657 if (UnavailablePred)
658 return;
659 UnavailablePred = P;
660 } else if (!AvailableInfo.isValid()) {
661 AvailableInfo = PredInfo;
662 } else if (AvailableInfo != PredInfo) {
663 return;
664 }
665 }
666
667 // Unreachable, single pred, or full redundancy. Note that FRE is handled by
668 // phase 3.
669 if (!UnavailablePred || !AvailableInfo.isValid())
670 return;
671
672 if (!LIS)
673 return;
674
675 // If we don't know the exact VTYPE, we can't copy the vsetvli to the exit of
676 // the unavailable pred.
677 if (AvailableInfo.hasSEWLMULRatioOnly())
678 return;
679
680 // Critical edge - TODO: consider splitting?
681 if (UnavailablePred->succ_size() != 1)
682 return;
683
684 // If the AVL value is a register (other than our VLMAX sentinel),
685 // we need to prove the value is available at the point we're going
686 // to insert the vsetvli at.
687 if (AvailableInfo.hasAVLReg()) {
688 SlotIndex SI = AvailableInfo.getAVLVNInfo()->def;
689 // This is an inline dominance check which covers the case of
690 // UnavailablePred being the preheader of a loop.
691 if (LIS->getMBBFromIndex(index: SI) != UnavailablePred)
692 return;
693 if (!UnavailablePred->terminators().empty() &&
694 SI >= LIS->getInstructionIndex(Instr: *UnavailablePred->getFirstTerminator()))
695 return;
696 }
697
698 // Model the effect of changing the input state of the block MBB to
699 // AvailableInfo. We're looking for two issues here; one legality,
700 // one profitability.
701 // 1) If the block doesn't use some of the fields from VL or VTYPE, we
702 // may hit the end of the block with a different end state. We can
703 // not make this change without reflowing later blocks as well.
704 // 2) If we don't actually remove a transition, inserting a vsetvli
705 // into the predecessor block would be correct, but unprofitable.
706 VSETVLIInfo OldInfo = BlockInfo[MBB.getNumber()].Pred;
707 VSETVLIInfo CurInfo = AvailableInfo;
708 int TransitionsRemoved = 0;
709 for (const MachineInstr &MI : MBB) {
710 const VSETVLIInfo LastInfo = CurInfo;
711 const VSETVLIInfo LastOldInfo = OldInfo;
712 transferBefore(Info&: CurInfo, MI);
713 transferBefore(Info&: OldInfo, MI);
714 if (CurInfo == LastInfo)
715 TransitionsRemoved++;
716 if (LastOldInfo == OldInfo)
717 TransitionsRemoved--;
718 transferAfter(Info&: CurInfo, MI);
719 transferAfter(Info&: OldInfo, MI);
720 if (CurInfo == OldInfo)
721 // Convergence. All transitions after this must match by construction.
722 break;
723 }
724 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
725 // Issues 1 and 2 above
726 return;
727
728 // Finally, update both data flow state and insert the actual vsetvli.
729 // Doing both keeps the code in sync with the dataflow results, which
730 // is critical for correctness of phase 3.
731 auto OldExit = BlockInfo[UnavailablePred->getNumber()].Exit;
732 LLVM_DEBUG(dbgs() << "PRE VSETVLI from " << MBB.getName() << " to "
733 << UnavailablePred->getName() << " with state "
734 << AvailableInfo << "\n");
735 BlockInfo[UnavailablePred->getNumber()].Exit = AvailableInfo;
736 BlockInfo[MBB.getNumber()].Pred = AvailableInfo;
737
738 // Note there's an implicit assumption here that terminators never use
739 // or modify VL or VTYPE. Also, fallthrough will return end().
740 auto InsertPt = UnavailablePred->getFirstInstrTerminator();
741 insertVSETVLI(MBB&: *UnavailablePred, InsertPt,
742 DL: UnavailablePred->findDebugLoc(MBBI: InsertPt),
743 Info: AvailableInfo, PrevInfo: OldExit);
744}
745
746// Return true if we can mutate PrevMI to match MI without changing any the
747// fields which would be observed.
748// If AVLDefToMove is non-null after the call, it points to an ADDI
749// instruction that needs to be moved before PrevMI.
750bool RISCVInsertVSETVLI::canMutatePriorConfig(
751 const MachineInstr &PrevMI, const MachineInstr &MI,
752 const DemandedFields &Used, MachineInstr *&AVLDefToMove) const {
753 AVLDefToMove = nullptr;
754 // If the VL values aren't equal, return false if either a) the former is
755 // demanded, or b) we can't rewrite the former to be the later for
756 // implementation reasons.
757 if (!RISCVInstrInfo::isVLPreservingConfig(MI)) {
758 if (Used.VLAny)
759 return false;
760
761 if (Used.VLZeroness) {
762 if (RISCVInstrInfo::isVLPreservingConfig(MI: PrevMI))
763 return false;
764 if (!VIA.getInfoForVSETVLI(MI: PrevMI).hasEquallyZeroAVL(
765 Other: VIA.getInfoForVSETVLI(MI), LIS))
766 return false;
767 }
768
769 auto &AVL = MI.getOperand(i: 1);
770
771 // If the AVL is a register, we need to make sure its definition is the same
772 // at PrevMI as it was at MI.
773 if (AVL.isReg() && AVL.getReg() != RISCV::X0) {
774 VNInfo *VNI = getVNInfoFromReg(Reg: AVL.getReg(), MI, LIS);
775 VNInfo *PrevVNI = getVNInfoFromReg(Reg: AVL.getReg(), MI: PrevMI, LIS);
776 if (!VNI || !PrevVNI || VNI != PrevVNI) {
777 // If the AVL is defined by a load immediate instruction (ADDI x0, imm),
778 // it can be moved earlier since it has no register dependencies.
779 if (!AVL.getReg().isVirtual())
780 return false;
781
782 MachineInstr *DefMI = MRI->getUniqueVRegDef(Reg: AVL.getReg());
783 if (!DefMI || !RISCVInstrInfo::isLoadImmediate(MI: *DefMI) ||
784 DefMI->getParent() != PrevMI.getParent()) {
785 return false;
786 }
787 // Mark that this ADDI needs to be moved.
788 AVLDefToMove = DefMI;
789 }
790 }
791
792 // If we define VL and need to move the definition up, check we can extend
793 // the live interval upwards from MI to PrevMI.
794 Register VL = MI.getOperand(i: 0).getReg();
795 if (VL.isVirtual() && LIS &&
796 LIS->getInterval(Reg: VL).overlaps(Start: LIS->getInstructionIndex(Instr: PrevMI),
797 End: LIS->getInstructionIndex(Instr: MI)))
798 return false;
799 }
800
801 assert(PrevMI.getOperand(2).isImm() && MI.getOperand(2).isImm());
802 auto PriorVType = PrevMI.getOperand(i: 2).getImm();
803 auto VType = MI.getOperand(i: 2).getImm();
804 return areCompatibleVTYPEs(CurVType: PriorVType, NewVType: VType, Used);
805}
806
807void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {
808 MachineInstr *NextMI = nullptr;
809 // We can have arbitrary code in successors, so VL and VTYPE
810 // must be considered demanded.
811 DemandedFields Used;
812 Used.demandVL();
813 Used.demandVTYPE();
814 SmallVector<MachineInstr*> ToDelete;
815
816 auto dropAVLUse = [&](MachineOperand &MO) {
817 if (!MO.isReg() || !MO.getReg().isVirtual())
818 return;
819 Register OldVLReg = MO.getReg();
820 MO.setReg(Register());
821
822 if (LIS)
823 LIS->shrinkToUses(li: &LIS->getInterval(Reg: OldVLReg));
824
825 MachineInstr *VLOpDef = MRI->getUniqueVRegDef(Reg: OldVLReg);
826 if (VLOpDef && TII->isAddImmediate(MI: *VLOpDef, Reg: OldVLReg) &&
827 MRI->use_nodbg_empty(RegNo: OldVLReg))
828 ToDelete.push_back(Elt: VLOpDef);
829 };
830
831 for (MachineInstr &MI : make_early_inc_range(Range: reverse(C&: MBB))) {
832 // TODO: Support XSfmm.
833 if (RISCVII::hasTWidenOp(TSFlags: MI.getDesc().TSFlags) ||
834 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI)) {
835 NextMI = nullptr;
836 continue;
837 }
838
839 if (!RISCVInstrInfo::isVectorConfigInstr(MI)) {
840 Used.doUnion(B: getDemanded(MI, ST));
841 if (MI.isCall() || MI.isInlineAsm() ||
842 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
843 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
844 NextMI = nullptr;
845 continue;
846 }
847
848 if (!MI.getOperand(i: 0).isDead())
849 Used.demandVL();
850
851 if (NextMI) {
852 if (!Used.usedVL() && !Used.usedVTYPE()) {
853 dropAVLUse(MI.getOperand(i: 1));
854 if (LIS)
855 LIS->RemoveMachineInstrFromMaps(MI);
856 MI.eraseFromParent();
857 NumCoalescedVSETVL++;
858 // Leave NextMI unchanged
859 continue;
860 }
861
862 MachineInstr *AVLDefToMove = nullptr;
863 if (canMutatePriorConfig(PrevMI: MI, MI: *NextMI, Used, AVLDefToMove)) {
864 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *NextMI)) {
865 Register DefReg = NextMI->getOperand(i: 0).getReg();
866
867 MI.getOperand(i: 0).setReg(DefReg);
868 MI.getOperand(i: 0).setIsDead(false);
869
870 // Move the AVL from NextMI to MI
871 dropAVLUse(MI.getOperand(i: 1));
872 if (NextMI->getOperand(i: 1).isImm())
873 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: NextMI->getOperand(i: 1).getImm());
874 else {
875 MI.getOperand(i: 1).ChangeToRegister(Reg: NextMI->getOperand(i: 1).getReg(),
876 isDef: false);
877
878 // If canMutatePriorConfig indicated that an ADDI needs to be moved,
879 // move it now.
880 if (AVLDefToMove) {
881 AVLDefToMove->moveBefore(MovePos: &MI);
882 if (LIS)
883 LIS->handleMove(MI&: *AVLDefToMove);
884 }
885 }
886 dropAVLUse(NextMI->getOperand(i: 1));
887
888 // The def of DefReg moved to MI, so extend the LiveInterval up to
889 // it.
890 if (DefReg.isVirtual() && LIS) {
891 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
892 SlotIndex MISlot = LIS->getInstructionIndex(Instr: MI).getRegSlot();
893 SlotIndex NextMISlot =
894 LIS->getInstructionIndex(Instr: *NextMI).getRegSlot();
895 VNInfo *DefVNI = DefLI.getVNInfoAt(Idx: NextMISlot);
896 LiveInterval::Segment S(MISlot, NextMISlot, DefVNI);
897 DefLI.addSegment(S);
898 DefVNI->def = MISlot;
899 // Mark DefLI as spillable if it was previously unspillable
900 DefLI.setWeight(0);
901
902 // DefReg may have had no uses, in which case we need to shrink
903 // the LiveInterval up to MI.
904 LIS->shrinkToUses(li: &DefLI);
905 }
906
907 MI.setDesc(NextMI->getDesc());
908 }
909 MI.getOperand(i: 2).setImm(NextMI->getOperand(i: 2).getImm());
910
911 dropAVLUse(NextMI->getOperand(i: 1));
912 if (LIS)
913 LIS->RemoveMachineInstrFromMaps(MI&: *NextMI);
914 NextMI->eraseFromParent();
915 NumCoalescedVSETVL++;
916 // fallthrough
917 }
918 }
919 NextMI = &MI;
920 Used = getDemanded(MI, ST);
921 }
922
923 // Loop over the dead AVL values, and delete them now. This has
924 // to be outside the above loop to avoid invalidating iterators.
925 for (auto *MI : ToDelete) {
926 assert(MI->getOpcode() == RISCV::ADDI);
927 Register AddReg = MI->getOperand(i: 1).getReg();
928 if (LIS) {
929 LIS->removeInterval(Reg: MI->getOperand(i: 0).getReg());
930 LIS->RemoveMachineInstrFromMaps(MI&: *MI);
931 }
932 MI->eraseFromParent();
933 if (LIS && AddReg.isVirtual())
934 LIS->shrinkToUses(li: &LIS->getInterval(Reg: AddReg));
935 }
936}
937
938void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
939 for (auto I = MBB.begin(), E = MBB.end(); I != E;) {
940 MachineInstr &MI = *I++;
941 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
942 Register VLOutput = MI.getOperand(i: 1).getReg();
943 assert(VLOutput.isVirtual());
944 if (!MI.getOperand(i: 1).isDead()) {
945 auto ReadVLMI = BuildMI(BB&: MBB, I, MIMD: MI.getDebugLoc(),
946 MCID: TII->get(Opcode: RISCV::PseudoReadVL), DestReg: VLOutput);
947 // Move the LiveInterval's definition down to PseudoReadVL.
948 if (LIS) {
949 SlotIndex NewDefSI =
950 LIS->InsertMachineInstrInMaps(MI&: *ReadVLMI).getRegSlot();
951 LiveInterval &DefLI = LIS->getInterval(Reg: VLOutput);
952 LiveRange::Segment *DefSeg = DefLI.getSegmentContaining(Idx: NewDefSI);
953 VNInfo *DefVNI = DefLI.getVNInfoAt(Idx: DefSeg->start);
954 DefLI.removeSegment(Start: DefSeg->start, End: NewDefSI);
955 DefVNI->def = NewDefSI;
956 }
957 }
958 // We don't use the vl output of the VLEFF/VLSEGFF anymore.
959 MI.getOperand(i: 1).setReg(RISCV::X0);
960 MI.addRegisterDefined(Reg: RISCV::VL, RegInfo: MRI->getTargetRegisterInfo());
961 }
962 }
963}
964
965bool RISCVInsertVSETVLI::insertVSETMTK(MachineBasicBlock &MBB,
966 TKTMMode Mode) const {
967
968 bool Changed = false;
969 for (auto &MI : MBB) {
970 uint64_t TSFlags = MI.getDesc().TSFlags;
971 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI) ||
972 !RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasTWidenOp(TSFlags))
973 continue;
974
975 VSETVLIInfo CurrInfo = VIA.computeInfoForInstr(MI);
976
977 unsigned Opcode = 0, OpNum = 0;
978 switch (Mode) {
979 case VSETTK:
980 if (!RISCVII::hasTKOp(TSFlags))
981 continue;
982 OpNum = RISCVII::getTKOpNum(Desc: MI.getDesc());
983 Opcode = RISCV::PseudoSF_VSETTK;
984 break;
985 case VSETTM:
986 if (!RISCVII::hasTMOp(TSFlags))
987 continue;
988 OpNum = RISCVII::getTMOpNum(Desc: MI.getDesc());
989 Opcode = RISCV::PseudoSF_VSETTM;
990 break;
991 }
992
993 assert(OpNum && Opcode && "Invalid OpNum or Opcode");
994
995 MachineOperand &Op = MI.getOperand(i: OpNum);
996
997 auto TmpMI = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode))
998 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
999 .addReg(RegNo: Op.getReg())
1000 .addImm(Val: Log2_32(Value: CurrInfo.getSEW()))
1001 .addImm(Val: CurrInfo.getTWiden());
1002
1003 Changed = true;
1004 Register Reg = Op.getReg();
1005 Op.setReg(Register());
1006 Op.setIsKill(false);
1007 if (LIS) {
1008 LIS->InsertMachineInstrInMaps(MI&: *TmpMI);
1009 LiveInterval &LI = LIS->getInterval(Reg);
1010
1011 // Erase the AVL operand from the instruction.
1012 LIS->shrinkToUses(li: &LI);
1013 // TODO: Enable this once needVSETVLIPHI is supported.
1014 // SmallVector<LiveInterval *> SplitLIs;
1015 // LIS->splitSeparateComponents(LI, SplitLIs);
1016 }
1017 }
1018 return Changed;
1019}
1020
1021bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {
1022 // Skip if the vector extension is not enabled.
1023 ST = &MF.getSubtarget<RISCVSubtarget>();
1024 if (!ST->hasVInstructions())
1025 return false;
1026
1027 LLVM_DEBUG(dbgs() << "Entering InsertVSETVLI for " << MF.getName() << "\n");
1028
1029 TII = ST->getInstrInfo();
1030 MRI = &MF.getRegInfo();
1031 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
1032 LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
1033 VIA = RISCVVSETVLIInfoAnalysis(ST, LIS);
1034
1035 assert(BlockInfo.empty() && "Expect empty block infos");
1036 BlockInfo.resize(new_size: MF.getNumBlockIDs());
1037
1038 bool HaveVectorOp = false;
1039
1040 // Phase 1 - determine how VL/VTYPE are affected by the each block.
1041 for (const MachineBasicBlock &MBB : MF) {
1042 VSETVLIInfo TmpStatus;
1043 HaveVectorOp |= computeVLVTYPEChanges(MBB, Info&: TmpStatus);
1044 // Initial exit state is whatever change we found in the block.
1045 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
1046 BBInfo.Exit = TmpStatus;
1047 LLVM_DEBUG(dbgs() << "Initial exit state of " << printMBBReference(MBB)
1048 << " is " << BBInfo.Exit << "\n");
1049
1050 }
1051
1052 // If we didn't find any instructions that need VSETVLI, we're done.
1053 if (!HaveVectorOp) {
1054 BlockInfo.clear();
1055 return false;
1056 }
1057
1058 // Phase 2 - determine the exit VL/VTYPE from each block. We add all
1059 // blocks to the list here, but will also add any that need to be revisited
1060 // during Phase 2 processing.
1061 for (const MachineBasicBlock &MBB : MF) {
1062 WorkList.push(x: &MBB);
1063 BlockInfo[MBB.getNumber()].InQueue = true;
1064 }
1065 while (!WorkList.empty()) {
1066 const MachineBasicBlock &MBB = *WorkList.front();
1067 WorkList.pop();
1068 computeIncomingVLVTYPE(MBB);
1069 }
1070
1071 // Perform partial redundancy elimination of vsetvli transitions.
1072 for (MachineBasicBlock &MBB : MF)
1073 doPRE(MBB);
1074
1075 // Phase 3 - add any vsetvli instructions needed in the block. Use the
1076 // Phase 2 information to avoid adding vsetvlis before the first vector
1077 // instruction in the block if the VL/VTYPE is satisfied by its
1078 // predecessors.
1079 for (MachineBasicBlock &MBB : MF)
1080 emitVSETVLIs(MBB);
1081
1082 // Now that all vsetvlis are explicit, go through and do block local
1083 // DSE and peephole based demanded fields based transforms. Note that
1084 // this *must* be done outside the main dataflow so long as we allow
1085 // any cross block analysis within the dataflow. We can't have both
1086 // demanded fields based mutation and non-local analysis in the
1087 // dataflow at the same time without introducing inconsistencies.
1088 // We're visiting blocks from the bottom up because a VSETVLI in the
1089 // earlier block might become dead when its uses in later blocks are
1090 // optimized away.
1091 for (MachineBasicBlock *MBB : post_order(G: &MF))
1092 coalesceVSETVLIs(MBB&: *MBB);
1093
1094 // Insert PseudoReadVL after VLEFF/VLSEGFF and replace it with the vl output
1095 // of VLEFF/VLSEGFF.
1096 for (MachineBasicBlock &MBB : MF)
1097 insertReadVL(MBB);
1098
1099 for (MachineBasicBlock &MBB : MF) {
1100 insertVSETMTK(MBB, Mode: VSETTM);
1101 insertVSETMTK(MBB, Mode: VSETTK);
1102 }
1103
1104 BlockInfo.clear();
1105 return HaveVectorOp;
1106}
1107
1108/// Returns an instance of the Insert VSETVLI pass.
1109FunctionPass *llvm::createRISCVInsertVSETVLIPass() {
1110 return new RISCVInsertVSETVLI();
1111}
1112