1//===- RISCVInsertVSETVLI.cpp - Insert VSETVLI instructions ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a function pass that inserts VSETVLI instructions where
10// needed and expands the vl outputs of VLEFF/VLSEGFF to PseudoReadVL
11// instructions.
12//
13// This pass consists of 3 phases:
14//
15// Phase 1 collects how each basic block affects VL/VTYPE.
16//
17// Phase 2 uses the information from phase 1 to do a data flow analysis to
18// propagate the VL/VTYPE changes through the function. This gives us the
19// VL/VTYPE at the start of each basic block.
20//
21// Phase 3 inserts VSETVLI instructions in each basic block. Information from
22// phase 2 is used to prevent inserting a VSETVLI before the first vector
23// instruction in the block if possible.
24//
25//===----------------------------------------------------------------------===//
26
27#include "RISCV.h"
28#include "RISCVSubtarget.h"
29#include "RISCVVSETVLIInfoAnalysis.h"
30#include "llvm/ADT/PostOrderIterator.h"
31#include "llvm/ADT/Statistic.h"
32#include "llvm/CodeGen/LiveDebugVariables.h"
33#include "llvm/CodeGen/LiveIntervals.h"
34#include "llvm/CodeGen/LiveStacks.h"
35#include "llvm/CodeGen/MachineFunctionPass.h"
36#include <queue>
37using namespace llvm;
38using namespace RISCV;
39
40#define DEBUG_TYPE "riscv-insert-vsetvli"
41#define RISCV_INSERT_VSETVLI_NAME "RISC-V Insert VSETVLI pass"
42
43STATISTIC(NumInsertedVSETVL, "Number of VSETVL inst inserted");
44STATISTIC(NumCoalescedVSETVL, "Number of VSETVL inst coalesced");
45
46static cl::opt<bool> EnsureWholeVectorRegisterMoveValidVTYPE(
47 DEBUG_TYPE "-whole-vector-register-move-valid-vtype", cl::Hidden,
48 cl::desc("Insert vsetvlis before vmvNr.vs to ensure vtype is valid and "
49 "vill is cleared"),
50 cl::init(Val: true));
51
52namespace {
53
54/// Given a virtual register \p Reg, return the corresponding VNInfo for it.
55/// This will return nullptr if the virtual register is an implicit_def or
56/// if LiveIntervals is not available.
57static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
58 const LiveIntervals *LIS) {
59 assert(Reg.isVirtual());
60 if (!LIS)
61 return nullptr;
62 auto &LI = LIS->getInterval(Reg);
63 SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
64 return LI.getVNInfoBefore(Idx: SI);
65}
66
67static MachineOperand &getVLOp(MachineInstr &MI) {
68 return MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
69}
70
71struct BlockData {
72 // The VSETVLIInfo that represents the VL/VTYPE settings on exit from this
73 // block. Calculated in Phase 2.
74 VSETVLIInfo Exit;
75
76 // The VSETVLIInfo that represents the VL/VTYPE settings from all predecessor
77 // blocks. Calculated in Phase 2, and used by Phase 3.
78 VSETVLIInfo Pred;
79
80 // Keeps track of whether the block is already in the queue.
81 bool InQueue = false;
82
83 BlockData() = default;
84};
85
86enum TKTMMode {
87 VSETTK = 0,
88 VSETTM = 1,
89};
90
91class RISCVInsertVSETVLI : public MachineFunctionPass {
92 const RISCVSubtarget *ST;
93 const TargetInstrInfo *TII;
94 MachineRegisterInfo *MRI;
95 // Possibly null!
96 LiveIntervals *LIS;
97 RISCVVSETVLIInfoAnalysis VIA;
98
99 std::vector<BlockData> BlockInfo;
100 std::queue<const MachineBasicBlock *> WorkList;
101
102public:
103 static char ID;
104
105 RISCVInsertVSETVLI() : MachineFunctionPass(ID) {}
106 bool runOnMachineFunction(MachineFunction &MF) override;
107
108 void getAnalysisUsage(AnalysisUsage &AU) const override {
109 AU.setPreservesCFG();
110
111 AU.addUsedIfAvailable<LiveIntervalsWrapperPass>();
112 AU.addPreserved<LiveIntervalsWrapperPass>();
113 AU.addPreserved<SlotIndexesWrapperPass>();
114 AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
115 AU.addPreserved<LiveStacksWrapperLegacy>();
116
117 MachineFunctionPass::getAnalysisUsage(AU);
118 }
119
120 StringRef getPassName() const override { return RISCV_INSERT_VSETVLI_NAME; }
121
122private:
123 bool needVSETVLI(const DemandedFields &Used, const VSETVLIInfo &Require,
124 const VSETVLIInfo &CurInfo) const;
125 bool needVSETVLIPHI(const VSETVLIInfo &Require,
126 const MachineBasicBlock &MBB) const;
127 void insertVSETVLI(MachineBasicBlock &MBB,
128 MachineBasicBlock::iterator InsertPt, DebugLoc DL,
129 const VSETVLIInfo &Info, const VSETVLIInfo &PrevInfo);
130
131 void transferBefore(VSETVLIInfo &Info, const MachineInstr &MI) const;
132 void transferAfter(VSETVLIInfo &Info, const MachineInstr &MI) const;
133 bool computeVLVTYPEChanges(const MachineBasicBlock &MBB,
134 VSETVLIInfo &Info) const;
135 void computeIncomingVLVTYPE(const MachineBasicBlock &MBB);
136 void emitVSETVLIs(MachineBasicBlock &MBB);
137 void doPRE(MachineBasicBlock &MBB);
138 void insertReadVL(MachineBasicBlock &MBB);
139
140 bool canMutatePriorConfig(const MachineInstr &PrevMI, const MachineInstr &MI,
141 const DemandedFields &Used,
142 MachineInstr *&AVLDefToMove) const;
143 void coalesceVSETVLIs(MachineBasicBlock &MBB) const;
144 bool canMutatePriorConfigWithTWiden(const MachineInstr &PrevMI,
145 const MachineInstr &MI) const;
146 void coalesceVSETVLIsForTWiden(MachineBasicBlock &MBB) const;
147 bool insertVSETMTK(MachineBasicBlock &MBB, TKTMMode Mode) const;
148};
149
150} // end anonymous namespace
151
152char RISCVInsertVSETVLI::ID = 0;
153char &llvm::RISCVInsertVSETVLIID = RISCVInsertVSETVLI::ID;
154
155INITIALIZE_PASS(RISCVInsertVSETVLI, DEBUG_TYPE, RISCV_INSERT_VSETVLI_NAME,
156 false, false)
157
158void RISCVInsertVSETVLI::insertVSETVLI(MachineBasicBlock &MBB,
159 MachineBasicBlock::iterator InsertPt,
160 DebugLoc DL, const VSETVLIInfo &Info,
161 const VSETVLIInfo &PrevInfo) {
162 ++NumInsertedVSETVL;
163
164 if (PrevInfo.isKnown()) {
165 // Use X0, X0 form if the AVL is the same and the SEW+LMUL gives the same
166 // VLMAX.
167 if (Info.hasSameAVL(Other: PrevInfo) && Info.hasSameVLMAX(Other: PrevInfo)) {
168 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
169 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0X0
170 : RISCV::PseudoVSETVLIX0X0))
171 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
172 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
173 .addImm(Val: Info.encodeVTYPE())
174 .addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
175 if (LIS)
176 LIS->InsertMachineInstrInMaps(MI&: *MI);
177 return;
178 }
179
180 // If our AVL is a virtual register, it might be defined by a VSET(I)VLI. If
181 // it has the same VLMAX we want and the last VL/VTYPE we observed is the
182 // same, we can use the X0, X0 form.
183 if (Info.hasSameVLMAX(Other: PrevInfo) && Info.hasAVLReg()) {
184 if (const MachineInstr *DefMI = Info.getAVLDefMI(LIS);
185 DefMI && RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI)) {
186 VSETVLIInfo DefInfo = VIA.getInfoForVSETVLI(MI: *DefMI);
187 if (DefInfo.hasSameAVL(Other: PrevInfo) && DefInfo.hasSameVLMAX(Other: PrevInfo)) {
188 auto MI =
189 BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
190 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0X0
191 : RISCV::PseudoVSETVLIX0X0))
192 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
193 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
194 .addImm(Val: Info.encodeVTYPE())
195 .addReg(RegNo: RISCV::VL, Flags: RegState::Implicit);
196 if (LIS)
197 LIS->InsertMachineInstrInMaps(MI&: *MI);
198 return;
199 }
200 }
201 }
202 }
203
204 if (Info.hasAVLImm()) {
205 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: TII->get(Opcode: RISCV::PseudoVSETIVLI))
206 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
207 .addImm(Val: Info.getAVLImm())
208 .addImm(Val: Info.encodeVTYPE());
209 if (LIS)
210 LIS->InsertMachineInstrInMaps(MI&: *MI);
211 return;
212 }
213
214 if (Info.hasAVLVLMAX()) {
215 Register DestReg = MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
216 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
217 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNTX0
218 : RISCV::PseudoVSETVLIX0))
219 .addReg(RegNo: DestReg, Flags: RegState::Define | RegState::Dead)
220 .addReg(RegNo: RISCV::X0, Flags: RegState::Kill)
221 .addImm(Val: Info.encodeVTYPE());
222 if (LIS) {
223 LIS->InsertMachineInstrInMaps(MI&: *MI);
224 LIS->createAndComputeVirtRegInterval(Reg: DestReg);
225 }
226 return;
227 }
228
229 Register AVLReg = Info.getAVLReg();
230 MRI->constrainRegClass(Reg: AVLReg, RC: &RISCV::GPRNoX0RegClass);
231 auto MI = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL,
232 MCID: TII->get(Opcode: Info.getTWiden() ? RISCV::PseudoSF_VSETTNT
233 : RISCV::PseudoVSETVLI))
234 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
235 .addReg(RegNo: AVLReg)
236 .addImm(Val: Info.encodeVTYPE());
237 if (LIS) {
238 LIS->InsertMachineInstrInMaps(MI&: *MI);
239 LiveInterval &LI = LIS->getInterval(Reg: AVLReg);
240 SlotIndex SI = LIS->getInstructionIndex(Instr: *MI).getRegSlot();
241 const VNInfo *CurVNI = Info.getAVLVNInfo();
242 // If the AVL value isn't live at MI, do a quick check to see if it's easily
243 // extendable. Otherwise, we need to copy it.
244 if (LI.getVNInfoBefore(Idx: SI) != CurVNI) {
245 if (!LI.liveAt(index: SI) && LI.containsOneValue())
246 LIS->extendToIndices(LR&: LI, Indices: SI);
247 else {
248 Register AVLCopyReg =
249 MRI->createVirtualRegister(RegClass: &RISCV::GPRNoX0RegClass);
250 MachineBasicBlock *MBB = LIS->getMBBFromIndex(index: CurVNI->def);
251 MachineBasicBlock::iterator II;
252 if (CurVNI->isPHIDef())
253 II = MBB->getFirstNonPHI();
254 else {
255 II = LIS->getInstructionFromIndex(index: CurVNI->def);
256 II = std::next(x: II);
257 }
258 assert(II.isValid());
259 auto AVLCopy = BuildMI(BB&: *MBB, I: II, MIMD: DL, MCID: TII->get(Opcode: RISCV::COPY), DestReg: AVLCopyReg)
260 .addReg(RegNo: AVLReg);
261 LIS->InsertMachineInstrInMaps(MI&: *AVLCopy);
262 MI->getOperand(i: 1).setReg(AVLCopyReg);
263 LIS->createAndComputeVirtRegInterval(Reg: AVLCopyReg);
264 }
265 }
266 }
267}
268
269/// Return true if a VSETVLI is required to transition from CurInfo to Require
270/// given a set of DemandedFields \p Used.
271bool RISCVInsertVSETVLI::needVSETVLI(const DemandedFields &Used,
272 const VSETVLIInfo &Require,
273 const VSETVLIInfo &CurInfo) const {
274 if (!CurInfo.isKnown() || CurInfo.hasSEWLMULRatioOnly())
275 return true;
276
277 if (CurInfo.isCompatible(Used, Require, LIS))
278 return false;
279
280 return true;
281}
282
283// If we don't use LMUL or the SEW/LMUL ratio, then adjust LMUL so that we
284// maintain the SEW/LMUL ratio. This allows us to eliminate VL toggles in more
285// places.
286static VSETVLIInfo adjustIncoming(const VSETVLIInfo &PrevInfo,
287 const VSETVLIInfo &NewInfo,
288 DemandedFields &Demanded) {
289 VSETVLIInfo Info = NewInfo;
290
291 if (!Demanded.LMUL && !Demanded.SEWLMULRatio && PrevInfo.isKnown()) {
292 if (auto NewVLMul = RISCVVType::getSameRatioLMUL(Ratio: PrevInfo.getSEWLMULRatio(),
293 EEW: Info.getSEW()))
294 Info.setVLMul(*NewVLMul);
295 Demanded.LMUL = DemandedFields::LMULEqual;
296 }
297
298 return Info;
299}
300
301// Given an incoming state reaching MI, minimally modifies that state so that it
302// is compatible with MI. The resulting state is guaranteed to be semantically
303// legal for MI, but may not be the state requested by MI.
304void RISCVInsertVSETVLI::transferBefore(VSETVLIInfo &Info,
305 const MachineInstr &MI) const {
306 if (EnsureWholeVectorRegisterMoveValidVTYPE &&
307 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI) &&
308 (!Info.isKnown() || Info.hasSEWLMULRatioOnly())) {
309 // Use an arbitrary but valid AVL and VTYPE so vill will be cleared. It may
310 // be coalesced into another vsetvli since we won't demand any fields.
311 VSETVLIInfo NewInfo; // Need a new VSETVLIInfo to clear SEWLMULRatioOnly
312 NewInfo.setAVLImm(1);
313 NewInfo.setVTYPE(L: RISCVVType::LMUL_1, /*sew*/ S: 8, /*ta*/ TA: true, /*ma*/ MA: true,
314 /*AltFmt*/ Altfmt: false, /*W*/ 0);
315 Info = NewInfo;
316 return;
317 }
318
319 if (!RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags))
320 return;
321
322 DemandedFields Demanded = getDemanded(MI, ST);
323
324 const VSETVLIInfo NewInfo = VIA.computeInfoForInstr(MI);
325 assert(NewInfo.isKnown());
326 if (Info.isValid() && !needVSETVLI(Used: Demanded, Require: NewInfo, CurInfo: Info))
327 return;
328
329 const VSETVLIInfo PrevInfo = Info;
330 if (!Info.isKnown())
331 Info = NewInfo;
332
333 const VSETVLIInfo IncomingInfo = adjustIncoming(PrevInfo, NewInfo, Demanded);
334
335 // If MI only demands that VL has the same zeroness, we only need to set the
336 // AVL if the zeroness differs. This removes a vsetvli entirely if the types
337 // match or allows use of cheaper avl preserving variant if VLMAX doesn't
338 // change. If VLMAX might change, we couldn't use the 'vsetvli x0, x0, vtype"
339 // variant, so we avoid the transform to prevent extending live range of an
340 // avl register operand.
341 // TODO: We can probably relax this for immediates.
342 bool EquallyZero = IncomingInfo.hasEquallyZeroAVL(Other: PrevInfo, LIS) &&
343 IncomingInfo.hasSameVLMAX(Other: PrevInfo);
344 if (Demanded.VLAny || (Demanded.VLZeroness && !EquallyZero))
345 Info.setAVL(IncomingInfo);
346
347 // If we only knew the sew/lmul ratio previously, replace the VTYPE.
348 if (Info.hasSEWLMULRatioOnly()) {
349 VSETVLIInfo RatiolessInfo = IncomingInfo;
350 RatiolessInfo.setAVL(Info);
351 Info = RatiolessInfo;
352 } else {
353 unsigned SEW =
354 ((Demanded.SEW || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
355 .getSEW();
356 Info.setVTYPE(
357 L: ((Demanded.LMUL || Demanded.SEWLMULRatio) ? IncomingInfo : Info)
358 .getVLMUL(),
359 S: SEW,
360 // Prefer tail/mask agnostic since it can be relaxed to undisturbed
361 // later if needed.
362 TA: (Demanded.TailPolicy ? IncomingInfo : Info).getTailAgnostic() ||
363 IncomingInfo.getTailAgnostic(),
364 MA: (Demanded.MaskPolicy ? IncomingInfo : Info).getMaskAgnostic() ||
365 IncomingInfo.getMaskAgnostic(),
366 // AltFmt requires SEW < 32.
367 Altfmt: (Demanded.AltFmt ? IncomingInfo : Info).getAltFmt() && SEW < 32,
368 W: Demanded.TWiden ? IncomingInfo.getTWiden() : 0);
369 }
370}
371
372// Given a state with which we evaluated MI (see transferBefore above for why
373// this might be different that the state MI requested), modify the state to
374// reflect the changes MI might make.
375void RISCVInsertVSETVLI::transferAfter(VSETVLIInfo &Info,
376 const MachineInstr &MI) const {
377 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {
378 Info = VIA.getInfoForVSETVLI(MI);
379 return;
380 }
381
382 // SETTM/TK will modify VTYPE, but it only affects the TM/TK bits.
383 // It is safe for other RVV operations.
384 // The TM/TK value will be maintained in insertVSETMTK.
385 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI))
386 return;
387
388 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
389 // Update AVL to vl-output of the fault first load.
390 assert(MI.getOperand(1).getReg().isVirtual());
391 if (LIS) {
392 auto &LI = LIS->getInterval(Reg: MI.getOperand(i: 1).getReg());
393 SlotIndex SI =
394 LIS->getSlotIndexes()->getInstructionIndex(MI).getRegSlot();
395 VNInfo *VNI = LI.getVNInfoAt(Idx: SI);
396 Info.setAVLRegDef(VNInfo: VNI, AVLReg: MI.getOperand(i: 1).getReg());
397 } else
398 Info.setAVLRegDef(VNInfo: nullptr, AVLReg: MI.getOperand(i: 1).getReg());
399 return;
400 }
401
402 // If this is something that updates VL/VTYPE that we don't know about, set
403 // the state to unknown.
404 if (MI.isCall() || MI.isInlineAsm() ||
405 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
406 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
407 Info = VSETVLIInfo::getUnknown();
408}
409
410bool RISCVInsertVSETVLI::computeVLVTYPEChanges(const MachineBasicBlock &MBB,
411 VSETVLIInfo &Info) const {
412 bool HadVectorOp = false;
413
414 Info = BlockInfo[MBB.getNumber()].Pred;
415 for (const MachineInstr &MI : MBB) {
416 transferBefore(Info, MI);
417
418 if (RISCVInstrInfo::isVectorConfigInstr(MI) ||
419 RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags) ||
420 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI) ||
421 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI))
422 HadVectorOp = true;
423
424 transferAfter(Info, MI);
425 }
426
427 return HadVectorOp;
428}
429
430void RISCVInsertVSETVLI::computeIncomingVLVTYPE(const MachineBasicBlock &MBB) {
431
432 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
433
434 BBInfo.InQueue = false;
435
436 // Start with the previous entry so that we keep the most conservative state
437 // we have ever found.
438 VSETVLIInfo InInfo = BBInfo.Pred;
439 if (MBB.pred_empty()) {
440 // There are no predecessors, so use the default starting status.
441 InInfo.setUnknown();
442 } else {
443 for (MachineBasicBlock *P : MBB.predecessors())
444 InInfo = InInfo.intersect(Other: BlockInfo[P->getNumber()].Exit);
445 }
446
447 // If we don't have any valid predecessor value, wait until we do.
448 if (!InInfo.isValid())
449 return;
450
451 // If no change, no need to rerun block
452 if (InInfo == BBInfo.Pred)
453 return;
454
455 BBInfo.Pred = InInfo;
456 LLVM_DEBUG(dbgs() << "Entry state of " << printMBBReference(MBB)
457 << " changed to " << BBInfo.Pred << "\n");
458
459 // Note: It's tempting to cache the state changes here, but due to the
460 // compatibility checks performed a blocks output state can change based on
461 // the input state. To cache, we'd have to add logic for finding
462 // never-compatible state changes.
463 VSETVLIInfo TmpStatus;
464 computeVLVTYPEChanges(MBB, Info&: TmpStatus);
465
466 // If the new exit value matches the old exit value, we don't need to revisit
467 // any blocks.
468 if (BBInfo.Exit == TmpStatus)
469 return;
470
471 BBInfo.Exit = TmpStatus;
472 LLVM_DEBUG(dbgs() << "Exit state of " << printMBBReference(MBB)
473 << " changed to " << BBInfo.Exit << "\n");
474
475 // Add the successors to the work list so we can propagate the changed exit
476 // status.
477 for (MachineBasicBlock *S : MBB.successors())
478 if (!BlockInfo[S->getNumber()].InQueue) {
479 BlockInfo[S->getNumber()].InQueue = true;
480 WorkList.push(x: S);
481 }
482}
483
484// If we weren't able to prove a vsetvli was directly unneeded, it might still
485// be unneeded if the AVL was a phi node where all incoming values are VL
486// outputs from the last VSETVLI in their respective basic blocks.
487bool RISCVInsertVSETVLI::needVSETVLIPHI(const VSETVLIInfo &Require,
488 const MachineBasicBlock &MBB) const {
489 if (!Require.hasAVLReg())
490 return true;
491
492 if (!LIS)
493 return true;
494
495 // We need the AVL to have been produced by a PHI node in this basic block.
496 const VNInfo *Valno = Require.getAVLVNInfo();
497 if (!Valno->isPHIDef() || LIS->getMBBFromIndex(index: Valno->def) != &MBB)
498 return true;
499
500 const LiveRange &LR = LIS->getInterval(Reg: Require.getAVLReg());
501
502 for (auto *PBB : MBB.predecessors()) {
503 const VSETVLIInfo &PBBExit = BlockInfo[PBB->getNumber()].Exit;
504
505 // We need the PHI input to the be the output of a VSET(I)VLI.
506 const VNInfo *Value = LR.getVNInfoBefore(Idx: LIS->getMBBEndIdx(mbb: PBB));
507 if (!Value)
508 return true;
509 MachineInstr *DefMI = LIS->getInstructionFromIndex(index: Value->def);
510 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI))
511 return true;
512
513 // We found a VSET(I)VLI make sure it matches the output of the
514 // predecessor block.
515 VSETVLIInfo DefInfo = VIA.getInfoForVSETVLI(MI: *DefMI);
516 if (DefInfo != PBBExit)
517 return true;
518
519 // Require has the same VL as PBBExit, so if the exit from the
520 // predecessor has the VTYPE we are looking for we might be able
521 // to avoid a VSETVLI.
522 if (PBBExit.isUnknown() || !PBBExit.hasSameVTYPE(Other: Require))
523 return true;
524 }
525
526 // If all the incoming values to the PHI checked out, we don't need
527 // to insert a VSETVLI.
528 return false;
529}
530
531void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
532 VSETVLIInfo CurInfo = BlockInfo[MBB.getNumber()].Pred;
533 // Track whether the prefix of the block we've scanned is transparent
534 // (meaning has not yet changed the abstract state).
535 bool PrefixTransparent = true;
536 for (MachineInstr &MI : MBB) {
537 const VSETVLIInfo PrevInfo = CurInfo;
538 transferBefore(Info&: CurInfo, MI);
539
540 // If this is an explicit VSETVLI or VSETIVLI, update our state.
541 if (RISCVInstrInfo::isVectorConfigInstr(MI)) {
542 // Conservatively, mark the VL and VTYPE as live.
543 assert(MI.getOperand(3).getReg() == RISCV::VL &&
544 MI.getOperand(4).getReg() == RISCV::VTYPE &&
545 "Unexpected operands where VL and VTYPE should be");
546
547 if (LIS) {
548 // Clearing a dead flag extends that def past its previous dead-def
549 // slot, so the stale VL/VTYPE range must be dropped.
550 if (MI.getOperand(i: 3).isDead())
551 LIS->removeAllRegUnitsForPhysReg(Reg: RISCV::VL);
552 if (MI.getOperand(i: 4).isDead())
553 LIS->removeAllRegUnitsForPhysReg(Reg: RISCV::VTYPE);
554 }
555
556 MI.getOperand(i: 3).setIsDead(false);
557 MI.getOperand(i: 4).setIsDead(false);
558 PrefixTransparent = false;
559 }
560
561 if (EnsureWholeVectorRegisterMoveValidVTYPE &&
562 RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI)) {
563 if (!PrevInfo.isCompatible(Used: DemandedFields::all(), Require: CurInfo, LIS)) {
564 insertVSETVLI(MBB, InsertPt: MI, DL: MI.getDebugLoc(), Info: CurInfo, PrevInfo);
565 PrefixTransparent = false;
566 }
567 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ false,
568 /*isImp*/ true));
569 }
570
571 uint64_t TSFlags = MI.getDesc().TSFlags;
572 if (RISCVII::hasSEWOp(TSFlags)) {
573 if (!PrevInfo.isCompatible(Used: DemandedFields::all(), Require: CurInfo, LIS)) {
574 // If this is the first implicit state change, and the state change
575 // requested can be proven to produce the same register contents, we
576 // can skip emitting the actual state change and continue as if we
577 // had since we know the GPR result of the implicit state change
578 // wouldn't be used and VL/VTYPE registers are correct. Note that
579 // we *do* need to model the state as if it changed as while the
580 // register contents are unchanged, the abstract model can change.
581 if (!PrefixTransparent || needVSETVLIPHI(Require: CurInfo, MBB))
582 insertVSETVLI(MBB, InsertPt: MI, DL: MI.getDebugLoc(), Info: CurInfo, PrevInfo);
583 PrefixTransparent = false;
584 }
585
586 if (RISCVII::hasVLOp(TSFlags)) {
587 MachineOperand &VLOp = getVLOp(MI);
588 if (VLOp.isReg()) {
589 Register Reg = VLOp.getReg();
590
591 // Erase the AVL operand from the instruction.
592 VLOp.setReg(Register());
593 VLOp.setIsKill(false);
594 if (LIS) {
595 LiveInterval &LI = LIS->getInterval(Reg);
596 SmallVector<MachineInstr *> DeadMIs;
597 LIS->shrinkToUses(li: &LI, dead: &DeadMIs);
598 // We might have separate components that need split due to
599 // needVSETVLIPHI causing us to skip inserting a new VL def.
600 SmallVector<LiveInterval *> SplitLIs;
601 LIS->splitSeparateComponents(LI, SplitLIs);
602
603 // If the AVL was an immediate > 31, then it would have been emitted
604 // as an ADDI. However, the ADDI might not have been used in the
605 // vsetvli, or a vsetvli might not have been emitted, so it may be
606 // dead now.
607 for (MachineInstr *DeadMI : DeadMIs) {
608 if (!TII->isAddImmediate(MI: *DeadMI, Reg))
609 continue;
610 LIS->RemoveMachineInstrFromMaps(MI&: *DeadMI);
611 Register AddReg = DeadMI->getOperand(i: 1).getReg();
612 DeadMI->eraseFromParent();
613 if (AddReg.isVirtual())
614 LIS->shrinkToUses(li: &LIS->getInterval(Reg: AddReg));
615 }
616 }
617 }
618 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VL, /*isDef*/ false,
619 /*isImp*/ true));
620 }
621 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ false,
622 /*isImp*/ true));
623 }
624
625 if (MI.isInlineAsm()) {
626 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VL, /*isDef*/ true,
627 /*isImp*/ true));
628 MI.addOperand(Op: MachineOperand::CreateReg(Reg: RISCV::VTYPE, /*isDef*/ true,
629 /*isImp*/ true));
630 }
631
632 if (MI.isCall() || MI.isInlineAsm() ||
633 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
634 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
635 PrefixTransparent = false;
636
637 transferAfter(Info&: CurInfo, MI);
638 }
639
640 const auto &Info = BlockInfo[MBB.getNumber()];
641 if (CurInfo != Info.Exit) {
642 LLVM_DEBUG(dbgs() << "in block " << printMBBReference(MBB) << "\n");
643 LLVM_DEBUG(dbgs() << " begin state: " << Info.Pred << "\n");
644 LLVM_DEBUG(dbgs() << " expected end state: " << Info.Exit << "\n");
645 LLVM_DEBUG(dbgs() << " actual end state: " << CurInfo << "\n");
646 }
647 assert(CurInfo == Info.Exit && "InsertVSETVLI dataflow invariant violated");
648}
649
650/// Perform simple partial redundancy elimination of the VSETVLI instructions
651/// we're about to insert by looking for cases where we can PRE from the
652/// beginning of one block to the end of one of its predecessors. Specifically,
653/// this is geared to catch the common case of a fixed length vsetvl in a single
654/// block loop when it could execute once in the preheader instead.
655void RISCVInsertVSETVLI::doPRE(MachineBasicBlock &MBB) {
656 if (!BlockInfo[MBB.getNumber()].Pred.isUnknown())
657 return;
658
659 MachineBasicBlock *UnavailablePred = nullptr;
660 VSETVLIInfo AvailableInfo;
661 for (MachineBasicBlock *P : MBB.predecessors()) {
662 const VSETVLIInfo &PredInfo = BlockInfo[P->getNumber()].Exit;
663 if (PredInfo.isUnknown()) {
664 if (UnavailablePred)
665 return;
666 UnavailablePred = P;
667 } else if (!AvailableInfo.isValid()) {
668 AvailableInfo = PredInfo;
669 } else if (AvailableInfo != PredInfo) {
670 return;
671 }
672 }
673
674 // Unreachable, single pred, or full redundancy. Note that FRE is handled by
675 // phase 3.
676 if (!UnavailablePred || !AvailableInfo.isValid())
677 return;
678
679 if (!LIS)
680 return;
681
682 // If we don't know the exact VTYPE, we can't copy the vsetvli to the exit of
683 // the unavailable pred.
684 if (AvailableInfo.hasSEWLMULRatioOnly())
685 return;
686
687 // Critical edge - TODO: consider splitting?
688 if (UnavailablePred->succ_size() != 1)
689 return;
690
691 // If the AVL value is a register (other than our VLMAX sentinel),
692 // we need to prove the value is available at the point we're going
693 // to insert the vsetvli at.
694 if (AvailableInfo.hasAVLReg()) {
695 SlotIndex SI = AvailableInfo.getAVLVNInfo()->def;
696 // This is an inline dominance check which covers the case of
697 // UnavailablePred being the preheader of a loop.
698 if (LIS->getMBBFromIndex(index: SI) != UnavailablePred)
699 return;
700 if (!UnavailablePred->terminators().empty() &&
701 SI >= LIS->getInstructionIndex(Instr: *UnavailablePred->getFirstTerminator()))
702 return;
703 }
704
705 // Model the effect of changing the input state of the block MBB to
706 // AvailableInfo. We're looking for two issues here; one legality,
707 // one profitability.
708 // 1) If the block doesn't use some of the fields from VL or VTYPE, we
709 // may hit the end of the block with a different end state. We can
710 // not make this change without reflowing later blocks as well.
711 // 2) If we don't actually remove a transition, inserting a vsetvli
712 // into the predecessor block would be correct, but unprofitable.
713 VSETVLIInfo OldInfo = BlockInfo[MBB.getNumber()].Pred;
714 VSETVLIInfo CurInfo = AvailableInfo;
715 int TransitionsRemoved = 0;
716 for (const MachineInstr &MI : MBB) {
717 const VSETVLIInfo LastInfo = CurInfo;
718 const VSETVLIInfo LastOldInfo = OldInfo;
719 transferBefore(Info&: CurInfo, MI);
720 transferBefore(Info&: OldInfo, MI);
721 if (CurInfo == LastInfo)
722 TransitionsRemoved++;
723 if (LastOldInfo == OldInfo)
724 TransitionsRemoved--;
725 transferAfter(Info&: CurInfo, MI);
726 transferAfter(Info&: OldInfo, MI);
727 if (CurInfo == OldInfo)
728 // Convergence. All transitions after this must match by construction.
729 break;
730 }
731 if (CurInfo != OldInfo || TransitionsRemoved <= 0)
732 // Issues 1 and 2 above
733 return;
734
735 // Finally, update both data flow state and insert the actual vsetvli.
736 // Doing both keeps the code in sync with the dataflow results, which
737 // is critical for correctness of phase 3.
738 auto OldExit = BlockInfo[UnavailablePred->getNumber()].Exit;
739 LLVM_DEBUG(dbgs() << "PRE VSETVLI from " << MBB.getName() << " to "
740 << UnavailablePred->getName() << " with state "
741 << AvailableInfo << "\n");
742 BlockInfo[UnavailablePred->getNumber()].Exit = AvailableInfo;
743 BlockInfo[MBB.getNumber()].Pred = AvailableInfo;
744
745 // Note there's an implicit assumption here that terminators never use
746 // or modify VL or VTYPE. Also, fallthrough will return end().
747 auto InsertPt = UnavailablePred->getFirstInstrTerminator();
748 insertVSETVLI(MBB&: *UnavailablePred, InsertPt,
749 DL: UnavailablePred->findDebugLoc(MBBI: InsertPt),
750 Info: AvailableInfo, PrevInfo: OldExit);
751}
752
753// Return true if we can mutate PrevMI to match MI without changing any the
754// fields which would be observed.
755// If AVLDefToMove is non-null after the call, it points to an ADDI
756// instruction that needs to be moved before PrevMI.
757bool RISCVInsertVSETVLI::canMutatePriorConfig(
758 const MachineInstr &PrevMI, const MachineInstr &MI,
759 const DemandedFields &Used, MachineInstr *&AVLDefToMove) const {
760 AVLDefToMove = nullptr;
761 // If the VL values aren't equal, return false if either a) the former is
762 // demanded, or b) we can't rewrite the former to be the later for
763 // implementation reasons.
764 if (!RISCVInstrInfo::isVLPreservingConfig(MI)) {
765 if (Used.VLAny)
766 return false;
767
768 if (Used.VLZeroness) {
769 if (RISCVInstrInfo::isVLPreservingConfig(MI: PrevMI))
770 return false;
771 if (!VIA.getInfoForVSETVLI(MI: PrevMI).hasEquallyZeroAVL(
772 Other: VIA.getInfoForVSETVLI(MI), LIS))
773 return false;
774 }
775
776 auto &AVL = MI.getOperand(i: 1);
777
778 // If the AVL is a register, we need to make sure its definition is the same
779 // at PrevMI as it was at MI.
780 if (AVL.isReg() && AVL.getReg() != RISCV::X0) {
781 VNInfo *VNI = getVNInfoFromReg(Reg: AVL.getReg(), MI, LIS);
782 VNInfo *PrevVNI = getVNInfoFromReg(Reg: AVL.getReg(), MI: PrevMI, LIS);
783 if (!VNI || !PrevVNI || VNI != PrevVNI) {
784 // If LIS is null, we were not able to get the VNInfo so we don't know
785 // if the AVL def needs to be moved.
786 if (!LIS)
787 return false;
788 // If the AVL is defined by a load immediate instruction (ADDI x0, imm),
789 // it can be moved earlier since it has no register dependencies.
790 if (!AVL.getReg().isVirtual())
791 return false;
792
793 MachineInstr *DefMI = MRI->getUniqueVRegDef(Reg: AVL.getReg());
794 if (!DefMI || !RISCVInstrInfo::isLoadImmediate(MI: *DefMI) ||
795 DefMI->getParent() != PrevMI.getParent()) {
796 return false;
797 }
798 // Mark that this ADDI needs to be moved.
799 AVLDefToMove = DefMI;
800 }
801 }
802
803 // If we define VL and need to move the definition up, check we can extend
804 // the live interval upwards from MI to PrevMI.
805 Register VL = MI.getOperand(i: 0).getReg();
806 if (VL.isVirtual() && LIS &&
807 LIS->getInterval(Reg: VL).overlaps(Start: LIS->getInstructionIndex(Instr: PrevMI),
808 End: LIS->getInstructionIndex(Instr: MI)))
809 return false;
810 }
811
812 assert(PrevMI.getOperand(2).isImm() && MI.getOperand(2).isImm());
813 auto PriorVType = PrevMI.getOperand(i: 2).getImm();
814 auto VType = MI.getOperand(i: 2).getImm();
815 return areCompatibleVTYPEs(CurVType: PriorVType, NewVType: VType, Used);
816}
817
818void RISCVInsertVSETVLI::coalesceVSETVLIs(MachineBasicBlock &MBB) const {
819 MachineInstr *NextMI = nullptr;
820 // We can have arbitrary code in successors, so VL and VTYPE
821 // must be considered demanded.
822 DemandedFields Used;
823 Used.demandVL();
824 Used.demandVTYPE();
825 SmallVector<MachineInstr*> ToDelete;
826
827 auto dropAVLUse = [&](MachineOperand &MO) {
828 if (!MO.isReg() || !MO.getReg().isVirtual())
829 return;
830 Register OldVLReg = MO.getReg();
831 MO.setReg(Register());
832
833 if (LIS)
834 LIS->shrinkToUses(li: &LIS->getInterval(Reg: OldVLReg));
835
836 MachineInstr *VLOpDef = MRI->getUniqueVRegDef(Reg: OldVLReg);
837 if (VLOpDef && TII->isAddImmediate(MI: *VLOpDef, Reg: OldVLReg) &&
838 MRI->use_nodbg_empty(RegNo: OldVLReg))
839 ToDelete.push_back(Elt: VLOpDef);
840 };
841
842 for (MachineInstr &MI : make_early_inc_range(Range: reverse(C&: MBB))) {
843 // TODO: Support XSfmm.
844 if (RISCVII::hasTWidenOp(TSFlags: MI.getDesc().TSFlags) ||
845 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI)) {
846 NextMI = nullptr;
847 continue;
848 }
849
850 if (!RISCVInstrInfo::isVectorConfigInstr(MI)) {
851 Used.doUnion(B: getDemanded(MI, ST));
852 if (MI.isCall() || MI.isInlineAsm() ||
853 MI.modifiesRegister(Reg: RISCV::VL, /*TRI=*/nullptr) ||
854 MI.modifiesRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
855 NextMI = nullptr;
856 continue;
857 }
858
859 if (!MI.getOperand(i: 0).isDead())
860 Used.demandVL();
861
862 if (NextMI) {
863 if (!Used.usedVL() && !Used.usedVTYPE()) {
864 dropAVLUse(MI.getOperand(i: 1));
865 if (LIS)
866 LIS->RemoveMachineInstrFromMaps(MI);
867 MI.eraseFromParent();
868 NumCoalescedVSETVL++;
869 // Leave NextMI unchanged
870 continue;
871 }
872
873 MachineInstr *AVLDefToMove = nullptr;
874 if (canMutatePriorConfig(PrevMI: MI, MI: *NextMI, Used, AVLDefToMove)) {
875 if (!RISCVInstrInfo::isVLPreservingConfig(MI: *NextMI)) {
876 Register DefReg = NextMI->getOperand(i: 0).getReg();
877
878 MI.getOperand(i: 0).setReg(DefReg);
879 MI.getOperand(i: 0).setIsDead(false);
880
881 // Move the AVL from NextMI to MI
882 dropAVLUse(MI.getOperand(i: 1));
883 if (NextMI->getOperand(i: 1).isImm())
884 MI.getOperand(i: 1).ChangeToImmediate(ImmVal: NextMI->getOperand(i: 1).getImm());
885 else {
886 MI.getOperand(i: 1).ChangeToRegister(Reg: NextMI->getOperand(i: 1).getReg(),
887 isDef: false);
888
889 // If canMutatePriorConfig indicated that an ADDI needs to be moved,
890 // move it now.
891 if (AVLDefToMove) {
892 AVLDefToMove->moveBefore(MovePos: &MI);
893 if (LIS)
894 LIS->handleMove(MI&: *AVLDefToMove);
895 }
896 }
897 dropAVLUse(NextMI->getOperand(i: 1));
898
899 // The def of DefReg moved to MI, so extend the LiveInterval up to
900 // it.
901 if (DefReg.isVirtual() && LIS) {
902 LiveInterval &DefLI = LIS->getInterval(Reg: DefReg);
903 SlotIndex MISlot = LIS->getInstructionIndex(Instr: MI).getRegSlot();
904 SlotIndex NextMISlot =
905 LIS->getInstructionIndex(Instr: *NextMI).getRegSlot();
906 VNInfo *DefVNI = DefLI.getVNInfoAt(Idx: NextMISlot);
907 LiveInterval::Segment S(MISlot, NextMISlot, DefVNI);
908 DefLI.addSegment(S);
909 DefVNI->def = MISlot;
910 // Mark DefLI as spillable if it was previously unspillable
911 DefLI.setWeight(0);
912
913 // DefReg may have had no uses, in which case we need to shrink
914 // the LiveInterval up to MI.
915 LIS->shrinkToUses(li: &DefLI);
916 }
917
918 MI.setDesc(NextMI->getDesc());
919 }
920 MI.getOperand(i: 2).setImm(NextMI->getOperand(i: 2).getImm());
921
922 dropAVLUse(NextMI->getOperand(i: 1));
923 if (LIS)
924 LIS->RemoveMachineInstrFromMaps(MI&: *NextMI);
925 NextMI->eraseFromParent();
926 NumCoalescedVSETVL++;
927 // fallthrough
928 }
929 }
930 NextMI = &MI;
931 Used = getDemanded(MI, ST);
932 }
933
934 // Loop over the dead AVL values, and delete them now. This has
935 // to be outside the above loop to avoid invalidating iterators.
936 for (auto *MI : ToDelete) {
937 assert(MI->getOpcode() == RISCV::ADDI);
938 Register AddReg = MI->getOperand(i: 1).getReg();
939 if (LIS) {
940 LIS->removeInterval(Reg: MI->getOperand(i: 0).getReg());
941 LIS->RemoveMachineInstrFromMaps(MI&: *MI);
942 }
943 MI->eraseFromParent();
944 if (LIS && AddReg.isVirtual())
945 LIS->shrinkToUses(li: &LIS->getInterval(Reg: AddReg));
946 }
947}
948
949// When twiden != 0, LMUL, tail policy, and mask policy from the user are
950// ignored. The tail policy and mask policy are always treated as agnostic. The
951// normal RVV instruction will ignore the twiden parameter. This observation
952// could allow the RVV instruction and xsfmm instruction to share the same
953// configuration instruction.
954//
955// We need to make sure the AVL, SEW, and AltFmt is same between VSETVL and
956// VSETVLTN.
957//
958// For example:
959//
960// %avl = SETTM or SETTK
961// ...
962// VSETVL %avl, type1
963// VSETVLTNT %avl, type2
964//
965// ->
966//
967// %avl = SETTM or SETTK
968// ...
969// VSETVLTNT %avl, type2
970//
971bool RISCVInsertVSETVLI::canMutatePriorConfigWithTWiden(
972 const MachineInstr &PrevMI, const MachineInstr &MI) const {
973
974 if (PrevMI.getOpcode() != RISCV::PseudoVSETVLI)
975 return false;
976
977 if (MI.getOpcode() != RISCV::PseudoSF_VSETTNT)
978 return false;
979
980 auto PrevInfo = VIA.getInfoForVSETVLI(MI: PrevMI);
981 auto CurrInfo = VIA.getInfoForVSETVLI(MI);
982
983 assert(CurrInfo.hasAVLReg() && "Invalid PseudoSF_VSETTNT without an AVLReg.");
984
985 auto AVLReg = CurrInfo.getAVLReg();
986
987 auto *AVLRegDefMI = MRI->getUniqueVRegDef(Reg: AVLReg);
988
989 if (!AVLRegDefMI)
990 return false;
991
992 if (!RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI: *AVLRegDefMI))
993 return false;
994
995 auto AVLRegDefMIInfo = VIA.computeInfoForInstr(MI: *AVLRegDefMI);
996 if (AVLRegDefMIInfo.getTWiden() != CurrInfo.getTWiden())
997 return false;
998
999 if (AVLRegDefMIInfo.getSEW() != PrevInfo.getSEW())
1000 return false;
1001
1002 // CurrInfo twiden != 0, so TailAgnostic and MaskAgnostic bit default to 1
1003 if (!PrevInfo.getTailAgnostic() || !PrevInfo.getMaskAgnostic())
1004 return false;
1005
1006 if (!PrevInfo.hasSameAVL(Other: CurrInfo))
1007 return false;
1008
1009 if (PrevInfo.getSEW() != CurrInfo.getSEW())
1010 return false;
1011
1012 if (PrevInfo.getAltFmt() != CurrInfo.getAltFmt())
1013 return false;
1014
1015 // The PrevMI's LMUL should be at least 8/KMAX; otherwise, converting it to a
1016 // tile-widening version could result in a VLMAX smaller than what AVLRegDefMI
1017 // expects, causing the LMUL information from PrevMI to be lost.
1018 auto [LMul, Fractional] = decodeVLMUL(VLMul: PrevInfo.getVLMUL());
1019 unsigned KMAX = (CurrInfo.getSEW() >= 32) ? 1 : (32 / CurrInfo.getSEW());
1020
1021 if (Fractional || LMul < (8 / KMAX))
1022 return false;
1023
1024 return true;
1025}
1026
1027void RISCVInsertVSETVLI::coalesceVSETVLIsForTWiden(
1028 MachineBasicBlock &MBB) const {
1029 MachineInstr *NextMI = nullptr;
1030
1031 for (MachineInstr &MI : make_early_inc_range(Range: reverse(C&: MBB))) {
1032
1033 if (!RISCVInstrInfo::isVectorConfigInstr(MI))
1034 continue;
1035
1036 if (NextMI) {
1037 // If only TWiden different. Update the MI and drop the NextMI.
1038 if (canMutatePriorConfigWithTWiden(PrevMI: MI, MI: *NextMI)) {
1039
1040 auto NextInfo = VIA.getInfoForVSETVLI(MI: *NextMI);
1041 MI.getOperand(i: 2).setImm(NextInfo.encodeVTYPE());
1042
1043 if (LIS)
1044 LIS->RemoveMachineInstrFromMaps(MI&: *NextMI);
1045 NextMI->eraseFromParent();
1046 }
1047 }
1048 NextMI = &MI;
1049 }
1050}
1051
1052void RISCVInsertVSETVLI::insertReadVL(MachineBasicBlock &MBB) {
1053 for (auto I = MBB.begin(), E = MBB.end(); I != E;) {
1054 MachineInstr &MI = *I++;
1055 if (RISCVInstrInfo::isFaultOnlyFirstLoad(MI)) {
1056 Register VLOutput = MI.getOperand(i: 1).getReg();
1057 assert(VLOutput.isVirtual());
1058 if (!MI.getOperand(i: 1).isDead()) {
1059 auto ReadVLMI = BuildMI(BB&: MBB, I, MIMD: MI.getDebugLoc(),
1060 MCID: TII->get(Opcode: RISCV::PseudoReadVL), DestReg: VLOutput);
1061 // Move the LiveInterval's definition down to PseudoReadVL.
1062 if (LIS) {
1063 SlotIndex NewDefSI =
1064 LIS->InsertMachineInstrInMaps(MI&: *ReadVLMI).getRegSlot();
1065 LiveInterval &DefLI = LIS->getInterval(Reg: VLOutput);
1066 LiveRange::Segment *DefSeg = DefLI.getSegmentContaining(Idx: NewDefSI);
1067 VNInfo *DefVNI = DefLI.getVNInfoAt(Idx: DefSeg->start);
1068 DefLI.removeSegment(Start: DefSeg->start, End: NewDefSI);
1069 DefVNI->def = NewDefSI;
1070 }
1071 }
1072 // We don't use the vl output of the VLEFF/VLSEGFF anymore.
1073 MI.getOperand(i: 1).setReg(RISCV::X0);
1074 MI.addRegisterDefined(Reg: RISCV::VL, RegInfo: MRI->getTargetRegisterInfo());
1075 }
1076 }
1077}
1078
1079bool RISCVInsertVSETVLI::insertVSETMTK(MachineBasicBlock &MBB,
1080 TKTMMode Mode) const {
1081
1082 bool Changed = false;
1083 for (auto &MI : MBB) {
1084 uint64_t TSFlags = MI.getDesc().TSFlags;
1085 if (RISCVInstrInfo::isXSfmmVectorConfigTMTKInstr(MI) ||
1086 !RISCVII::hasSEWOp(TSFlags) || !RISCVII::hasTWidenOp(TSFlags))
1087 continue;
1088
1089 VSETVLIInfo CurrInfo = VIA.computeInfoForInstr(MI);
1090
1091 unsigned Opcode = 0, OpNum = 0;
1092 switch (Mode) {
1093 case VSETTK:
1094 if (!RISCVII::hasTKOp(TSFlags))
1095 continue;
1096 OpNum = RISCVII::getTKOpNum(Desc: MI.getDesc());
1097 Opcode = RISCV::PseudoSF_VSETTK;
1098 break;
1099 case VSETTM:
1100 if (!RISCVII::hasTMOp(TSFlags))
1101 continue;
1102 OpNum = RISCVII::getTMOpNum(Desc: MI.getDesc());
1103 Opcode = RISCV::PseudoSF_VSETTM;
1104 break;
1105 }
1106
1107 assert(OpNum && Opcode && "Invalid OpNum or Opcode");
1108
1109 MachineOperand &Op = MI.getOperand(i: OpNum);
1110
1111 auto TmpMI = BuildMI(BB&: MBB, I&: MI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode))
1112 .addReg(RegNo: RISCV::X0, Flags: RegState::Define | RegState::Dead)
1113 .addReg(RegNo: Op.getReg())
1114 .addImm(Val: Log2_32(Value: CurrInfo.getSEW()))
1115 .addImm(Val: CurrInfo.getTWiden());
1116
1117 Changed = true;
1118 Register Reg = Op.getReg();
1119 Op.setReg(Register());
1120 Op.setIsKill(false);
1121 if (LIS) {
1122 LIS->InsertMachineInstrInMaps(MI&: *TmpMI);
1123 LiveInterval &LI = LIS->getInterval(Reg);
1124
1125 // Erase the AVL operand from the instruction.
1126 LIS->shrinkToUses(li: &LI);
1127 // TODO: Enable this once needVSETVLIPHI is supported.
1128 // SmallVector<LiveInterval *> SplitLIs;
1129 // LIS->splitSeparateComponents(LI, SplitLIs);
1130 }
1131 }
1132 return Changed;
1133}
1134
1135bool RISCVInsertVSETVLI::runOnMachineFunction(MachineFunction &MF) {
1136 // Skip if the vector extension is not enabled.
1137 ST = &MF.getSubtarget<RISCVSubtarget>();
1138 if (!ST->hasVInstructions())
1139 return false;
1140
1141 LLVM_DEBUG(dbgs() << "Entering InsertVSETVLI for " << MF.getName() << "\n");
1142
1143 TII = ST->getInstrInfo();
1144 MRI = &MF.getRegInfo();
1145 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
1146 LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
1147 VIA = RISCVVSETVLIInfoAnalysis(ST, LIS);
1148
1149 assert(BlockInfo.empty() && "Expect empty block infos");
1150 BlockInfo.resize(new_size: MF.getNumBlockIDs());
1151
1152 bool HaveVectorOp = false;
1153
1154 // Phase 1 - determine how VL/VTYPE are affected by the each block.
1155 for (const MachineBasicBlock &MBB : MF) {
1156 VSETVLIInfo TmpStatus;
1157 HaveVectorOp |= computeVLVTYPEChanges(MBB, Info&: TmpStatus);
1158 // Initial exit state is whatever change we found in the block.
1159 BlockData &BBInfo = BlockInfo[MBB.getNumber()];
1160 BBInfo.Exit = TmpStatus;
1161 LLVM_DEBUG(dbgs() << "Initial exit state of " << printMBBReference(MBB)
1162 << " is " << BBInfo.Exit << "\n");
1163
1164 }
1165
1166 // If we didn't find any instructions that need VSETVLI, we're done.
1167 if (!HaveVectorOp) {
1168 BlockInfo.clear();
1169 return false;
1170 }
1171
1172 // Phase 2 - determine the exit VL/VTYPE from each block. We add all
1173 // blocks to the list here, but will also add any that need to be revisited
1174 // during Phase 2 processing.
1175 for (const MachineBasicBlock &MBB : MF) {
1176 WorkList.push(x: &MBB);
1177 BlockInfo[MBB.getNumber()].InQueue = true;
1178 }
1179 while (!WorkList.empty()) {
1180 const MachineBasicBlock &MBB = *WorkList.front();
1181 WorkList.pop();
1182 computeIncomingVLVTYPE(MBB);
1183 }
1184
1185 // Perform partial redundancy elimination of vsetvli transitions.
1186 for (MachineBasicBlock &MBB : MF)
1187 doPRE(MBB);
1188
1189 // Phase 3 - add any vsetvli instructions needed in the block. Use the
1190 // Phase 2 information to avoid adding vsetvlis before the first vector
1191 // instruction in the block if the VL/VTYPE is satisfied by its
1192 // predecessors.
1193 for (MachineBasicBlock &MBB : MF)
1194 emitVSETVLIs(MBB);
1195
1196 // Now that all vsetvlis are explicit, go through and do block local
1197 // DSE and peephole based demanded fields based transforms. Note that
1198 // this *must* be done outside the main dataflow so long as we allow
1199 // any cross block analysis within the dataflow. We can't have both
1200 // demanded fields based mutation and non-local analysis in the
1201 // dataflow at the same time without introducing inconsistencies.
1202 // We're visiting blocks from the bottom up because a VSETVLI in the
1203 // earlier block might become dead when its uses in later blocks are
1204 // optimized away.
1205 for (MachineBasicBlock *MBB : post_order(G: &MF))
1206 coalesceVSETVLIs(MBB&: *MBB);
1207
1208 if (ST->hasVendorXSfmmbase()) {
1209 for (MachineBasicBlock &MBB : MF)
1210 coalesceVSETVLIsForTWiden(MBB);
1211 }
1212
1213 // Insert PseudoReadVL after VLEFF/VLSEGFF and replace it with the vl output
1214 // of VLEFF/VLSEGFF.
1215 for (MachineBasicBlock &MBB : MF)
1216 insertReadVL(MBB);
1217
1218 if (ST->hasVendorXSfmmbase()) {
1219 for (MachineBasicBlock &MBB : MF) {
1220 insertVSETMTK(MBB, Mode: VSETTM);
1221 insertVSETMTK(MBB, Mode: VSETTK);
1222 }
1223 }
1224
1225 BlockInfo.clear();
1226 return HaveVectorOp;
1227}
1228
1229/// Returns an instance of the Insert VSETVLI pass.
1230FunctionPass *llvm::createRISCVInsertVSETVLIPass() {
1231 return new RISCVInsertVSETVLI();
1232}
1233