1//===-------------- RISCVVLOptimizer.cpp - VL Optimizer -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===---------------------------------------------------------------------===//
8//
9// This pass reduces the VL where possible at the MI level, before VSETVLI
10// instructions are inserted.
11//
12// The purpose of this optimization is to make the VL argument, for instructions
13// that have a VL argument, as small as possible.
14//
15// This is split into a sparse dataflow analysis where we determine what VL is
16// demanded by each instruction first, and then afterwards try to reduce the VL
17// of each instruction if it demands less than its VL operand.
18//
19// The analysis is explained in more detail in the 2025 EuroLLVM Developers'
20// Meeting talk "Accidental Dataflow Analysis: Extending the RISC-V VL
21// Optimizer", which is available on YouTube at
22// https://www.youtube.com/watch?v=Mfb5fRSdJAc
23//
24// The slides for the talk are available at
25// https://llvm.org/devmtg/2025-04/slides/technical_talk/lau_accidental_dataflow.pdf
26//
27//===---------------------------------------------------------------------===//
28
29#include "RISCV.h"
30#include "RISCVSubtarget.h"
31#include "llvm/ADT/PostOrderIterator.h"
32#include "llvm/ADT/SetVector.h"
33#include "llvm/CodeGen/MachineDominators.h"
34#include "llvm/CodeGen/MachineFunctionPass.h"
35#include "llvm/CodeGen/RegisterClassInfo.h"
36#include "llvm/InitializePasses.h"
37
38using namespace llvm;
39
40#define DEBUG_TYPE "riscv-vl-optimizer"
41#define PASS_NAME "RISC-V VL Optimizer"
42
43namespace {
44
45/// Wrapper around MachineOperand that defaults to immediate 0.
46struct DemandedVL {
47 MachineOperand VL;
48 DemandedVL() : VL(MachineOperand::CreateImm(Val: 0)) {}
49 DemandedVL(MachineOperand VL) : VL(VL) {}
50 static DemandedVL vlmax() {
51 return DemandedVL(MachineOperand::CreateImm(Val: RISCV::VLMaxSentinel));
52 }
53 bool operator!=(const DemandedVL &Other) const {
54 return !VL.isIdenticalTo(Other: Other.VL);
55 }
56
57 DemandedVL max(const MachineRegisterInfo &MRI, const DemandedVL &X) const {
58 if (RISCV::isVLKnownLE(MRI, LHS: VL, RHS: X.VL))
59 return X;
60 if (RISCV::isVLKnownLE(MRI, LHS: X.VL, RHS: VL))
61 return *this;
62 return DemandedVL::vlmax();
63 }
64};
65
66class RISCVVLOptimizerImpl {
67 MachineRegisterInfo *MRI;
68 const MachineDominatorTree *MDT;
69 const TargetInstrInfo *TII;
70
71public:
72 RISCVVLOptimizerImpl(const MachineDominatorTree *MDT) : MDT(MDT) {}
73
74 bool run(MachineFunction &MF);
75
76private:
77 DemandedVL getMinimumVLForUser(const MachineOperand &UserOp) const;
78 /// Returns true if the users of \p MI have compatible EEWs and SEWs.
79 bool checkUsers(const MachineInstr &MI) const;
80 bool tryReduceVL(MachineInstr &MI, MachineOperand VL) const;
81 bool isSupportedInstr(const MachineInstr &MI) const;
82 bool isCandidate(const MachineInstr &MI) const;
83 void transfer(const MachineInstr &MI);
84
85 /// For a given instruction, records what elements of it are demanded by
86 /// downstream users.
87 MapVector<const MachineInstr *, DemandedVL> DemandedVLs;
88 SetVector<const MachineInstr *> Worklist;
89
90 /// \returns all vector virtual registers that \p MI uses.
91 auto virtual_vec_uses(const MachineInstr &MI) const {
92 return make_filter_range(Range: MI.uses(), Pred: [this](const MachineOperand &MO) {
93 return MO.isReg() && MO.getReg().isVirtual() &&
94 RISCVRegisterInfo::isRVVRegClass(RC: MRI->getRegClass(Reg: MO.getReg()));
95 });
96 }
97};
98
99class RISCVVLOptimizerLegacy : public MachineFunctionPass {
100public:
101 static char ID;
102
103 RISCVVLOptimizerLegacy() : MachineFunctionPass(ID) {}
104
105 bool runOnMachineFunction(MachineFunction &MF) override;
106
107 void getAnalysisUsage(AnalysisUsage &AU) const override {
108 AU.setPreservesCFG();
109 AU.addRequired<MachineDominatorTreeWrapperPass>();
110 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
111 MachineFunctionPass::getAnalysisUsage(AU);
112 }
113
114 StringRef getPassName() const override { return PASS_NAME; }
115};
116
117/// Represents the EMUL and EEW of a MachineOperand.
118struct OperandInfo {
119 // Represent as 1,2,4,8, ... and fractional indicator. This is because
120 // EMUL can take on values that don't map to RISCVVType::VLMUL values exactly.
121 // For example, a mask operand can have an EMUL less than MF8.
122 // If nullopt, then EMUL isn't used (i.e. only a single scalar is read).
123 std::optional<std::pair<unsigned, bool>> EMUL;
124
125 unsigned Log2EEW;
126
127 OperandInfo(RISCVVType::VLMUL EMUL, unsigned Log2EEW)
128 : EMUL(RISCVVType::decodeVLMUL(VLMul: EMUL)), Log2EEW(Log2EEW) {}
129
130 OperandInfo(std::pair<unsigned, bool> EMUL, unsigned Log2EEW)
131 : EMUL(EMUL), Log2EEW(Log2EEW) {}
132
133 OperandInfo(unsigned Log2EEW) : Log2EEW(Log2EEW) {}
134
135 OperandInfo() = delete;
136
137 /// Return true if the EMUL and EEW produced by \p Def is compatible with the
138 /// EMUL and EEW used by \p User.
139 static bool areCompatible(const OperandInfo &Def, const OperandInfo &User) {
140 if (Def.Log2EEW != User.Log2EEW)
141 return false;
142 if (User.EMUL && Def.EMUL != User.EMUL)
143 return false;
144 return true;
145 }
146
147 void print(raw_ostream &OS) const {
148 if (EMUL) {
149 OS << "EMUL: m";
150 if (EMUL->second)
151 OS << "f";
152 OS << EMUL->first;
153 } else
154 OS << "EMUL: none\n";
155 OS << ", EEW: " << (1 << Log2EEW);
156 }
157};
158
159} // end anonymous namespace
160
161char RISCVVLOptimizerLegacy::ID = 0;
162INITIALIZE_PASS_BEGIN(RISCVVLOptimizerLegacy, DEBUG_TYPE, PASS_NAME, false,
163 false)
164INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
165INITIALIZE_PASS_END(RISCVVLOptimizerLegacy, DEBUG_TYPE, PASS_NAME, false, false)
166
167FunctionPass *llvm::createRISCVVLOptimizerLegacyPass() {
168 return new RISCVVLOptimizerLegacy();
169}
170
171[[maybe_unused]]
172static raw_ostream &operator<<(raw_ostream &OS, const OperandInfo &OI) {
173 OI.print(OS);
174 return OS;
175}
176
177[[maybe_unused]]
178static raw_ostream &operator<<(raw_ostream &OS,
179 const std::optional<OperandInfo> &OI) {
180 if (OI)
181 OI->print(OS);
182 else
183 OS << "nullopt";
184 return OS;
185}
186
187/// Return EMUL = (EEW / SEW) * LMUL where EEW comes from Log2EEW and LMUL and
188/// SEW are from the TSFlags of MI.
189static std::pair<unsigned, bool>
190getEMULEqualsEEWDivSEWTimesLMUL(unsigned Log2EEW, const MachineInstr &MI) {
191 RISCVVType::VLMUL MIVLMUL = RISCVII::getLMul(TSFlags: MI.getDesc().TSFlags);
192 auto [MILMUL, MILMULIsFractional] = RISCVVType::decodeVLMUL(VLMul: MIVLMUL);
193 unsigned MILog2SEW =
194 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
195
196 // Mask instructions will have 0 as the SEW operand. But the LMUL of these
197 // instructions is calculated is as if the SEW operand was 3 (e8).
198 if (MILog2SEW == 0)
199 MILog2SEW = 3;
200
201 unsigned MISEW = 1 << MILog2SEW;
202
203 unsigned EEW = 1 << Log2EEW;
204 // Calculate (EEW/SEW)*LMUL preserving fractions less than 1. Use GCD
205 // to put fraction in simplest form.
206 unsigned Num = EEW, Denom = MISEW;
207 int GCD = MILMULIsFractional ? std::gcd(m: Num, n: Denom * MILMUL)
208 : std::gcd(m: Num * MILMUL, n: Denom);
209 Num = MILMULIsFractional ? Num / GCD : Num * MILMUL / GCD;
210 Denom = MILMULIsFractional ? Denom * MILMUL / GCD : Denom / GCD;
211 return std::make_pair(x&: Num > Denom ? Num : Denom, y: Denom > Num);
212}
213
214/// Dest has EEW=SEW. Source EEW=SEW/Factor (i.e. F2 => EEW/2).
215/// SEW comes from TSFlags of MI.
216static unsigned getIntegerExtensionOperandEEW(unsigned Factor,
217 const MachineInstr &MI,
218 unsigned OpIdx) {
219 unsigned MILog2SEW =
220 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
221
222 if (OpIdx == 0)
223 return MILog2SEW;
224
225 unsigned MISEW = 1 << MILog2SEW;
226 unsigned EEW = MISEW / Factor;
227 unsigned Log2EEW = Log2_32(Value: EEW);
228
229 return Log2EEW;
230}
231
232#define VSEG_CASES(Prefix, EEW) \
233 RISCV::Prefix##SEG2E##EEW##_V: \
234 case RISCV::Prefix##SEG3E##EEW##_V: \
235 case RISCV::Prefix##SEG4E##EEW##_V: \
236 case RISCV::Prefix##SEG5E##EEW##_V: \
237 case RISCV::Prefix##SEG6E##EEW##_V: \
238 case RISCV::Prefix##SEG7E##EEW##_V: \
239 case RISCV::Prefix##SEG8E##EEW##_V
240#define VSSEG_CASES(EEW) VSEG_CASES(VS, EEW)
241#define VSSSEG_CASES(EEW) VSEG_CASES(VSS, EEW)
242#define VSUXSEG_CASES(EEW) VSEG_CASES(VSUX, I##EEW)
243#define VSOXSEG_CASES(EEW) VSEG_CASES(VSOX, I##EEW)
244
245static std::optional<unsigned> getOperandLog2EEW(const MachineInstr &MI,
246 unsigned OpIdx) {
247 const MCInstrDesc &Desc = MI.getDesc();
248 const RISCVVPseudosTable::PseudoInfo *RVV =
249 RISCVVPseudosTable::getPseudoInfo(Pseudo: MI.getOpcode());
250 assert(RVV && "Could not find MI in PseudoTable");
251
252 // MI has a SEW associated with it. The RVV specification defines
253 // the EEW of each operand and definition in relation to MI.SEW.
254 unsigned MILog2SEW = MI.getOperand(i: RISCVII::getSEWOpNum(Desc)).getImm();
255
256 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc);
257 const bool IsTied = RISCVII::isTiedPseudo(TSFlags: Desc.TSFlags);
258
259 bool IsMODef =
260 OpIdx == 0 || (HasPassthru && OpIdx == MI.getNumExplicitDefs());
261
262 // All mask operands have EEW=1
263 const MCOperandInfo &Info = Desc.operands()[OpIdx];
264 if (Info.OperandType == MCOI::OPERAND_REGISTER &&
265 Info.RegClass == RISCV::VMV0RegClassID)
266 return 0;
267
268 // switch against BaseInstr to reduce number of cases that need to be
269 // considered.
270 switch (RVV->BaseInstr) {
271
272 // 6. Configuration-Setting Instructions
273 // Configuration setting instructions do not read or write vector registers
274 case RISCV::VSETIVLI:
275 case RISCV::VSETVL:
276 case RISCV::VSETVLI:
277 llvm_unreachable("Configuration setting instructions do not read or write "
278 "vector registers");
279
280 // Vector Loads and Stores
281 // Vector Unit-Stride Instructions
282 // Vector Strided Instructions
283 /// Dest EEW encoded in the instruction
284 case RISCV::VLM_V:
285 case RISCV::VSM_V:
286 return 0;
287 case RISCV::VLE8_V:
288 case RISCV::VSE8_V:
289 case RISCV::VLSE8_V:
290 case RISCV::VSSE8_V:
291 case VSSEG_CASES(8):
292 case VSSSEG_CASES(8):
293 return 3;
294 case RISCV::VLE16_V:
295 case RISCV::VSE16_V:
296 case RISCV::VLSE16_V:
297 case RISCV::VSSE16_V:
298 case VSSEG_CASES(16):
299 case VSSSEG_CASES(16):
300 return 4;
301 case RISCV::VLE32_V:
302 case RISCV::VSE32_V:
303 case RISCV::VLSE32_V:
304 case RISCV::VSSE32_V:
305 case VSSEG_CASES(32):
306 case VSSSEG_CASES(32):
307 return 5;
308 case RISCV::VLE64_V:
309 case RISCV::VSE64_V:
310 case RISCV::VLSE64_V:
311 case RISCV::VSSE64_V:
312 case VSSEG_CASES(64):
313 case VSSSEG_CASES(64):
314 return 6;
315
316 // Vector Indexed Instructions
317 // vs(o|u)xei<eew>.v
318 // Dest/Data (operand 0) EEW=SEW. Source EEW=<eew>.
319 case RISCV::VLUXEI8_V:
320 case RISCV::VLOXEI8_V:
321 case RISCV::VSUXEI8_V:
322 case RISCV::VSOXEI8_V:
323 case VSUXSEG_CASES(8):
324 case VSOXSEG_CASES(8): {
325 if (OpIdx == 0)
326 return MILog2SEW;
327 return 3;
328 }
329 case RISCV::VLUXEI16_V:
330 case RISCV::VLOXEI16_V:
331 case RISCV::VSUXEI16_V:
332 case RISCV::VSOXEI16_V:
333 case VSUXSEG_CASES(16):
334 case VSOXSEG_CASES(16): {
335 if (OpIdx == 0)
336 return MILog2SEW;
337 return 4;
338 }
339 case RISCV::VLUXEI32_V:
340 case RISCV::VLOXEI32_V:
341 case RISCV::VSUXEI32_V:
342 case RISCV::VSOXEI32_V:
343 case VSUXSEG_CASES(32):
344 case VSOXSEG_CASES(32): {
345 if (OpIdx == 0)
346 return MILog2SEW;
347 return 5;
348 }
349 case RISCV::VLUXEI64_V:
350 case RISCV::VLOXEI64_V:
351 case RISCV::VSUXEI64_V:
352 case RISCV::VSOXEI64_V:
353 case VSUXSEG_CASES(64):
354 case VSOXSEG_CASES(64): {
355 if (OpIdx == 0)
356 return MILog2SEW;
357 return 6;
358 }
359
360 // Vector Integer Arithmetic Instructions
361 // Vector Single-Width Integer Add and Subtract
362 case RISCV::VADD_VI:
363 case RISCV::VADD_VV:
364 case RISCV::VADD_VX:
365 case RISCV::VSUB_VV:
366 case RISCV::VSUB_VX:
367 case RISCV::VRSUB_VI:
368 case RISCV::VRSUB_VX:
369 // Vector Bitwise Logical Instructions
370 // Vector Single-Width Shift Instructions
371 // EEW=SEW.
372 case RISCV::VAND_VI:
373 case RISCV::VAND_VV:
374 case RISCV::VAND_VX:
375 case RISCV::VOR_VI:
376 case RISCV::VOR_VV:
377 case RISCV::VOR_VX:
378 case RISCV::VXOR_VI:
379 case RISCV::VXOR_VV:
380 case RISCV::VXOR_VX:
381 case RISCV::VSLL_VI:
382 case RISCV::VSLL_VV:
383 case RISCV::VSLL_VX:
384 case RISCV::VSRL_VI:
385 case RISCV::VSRL_VV:
386 case RISCV::VSRL_VX:
387 case RISCV::VSRA_VI:
388 case RISCV::VSRA_VV:
389 case RISCV::VSRA_VX:
390 // Vector Integer Min/Max Instructions
391 // EEW=SEW.
392 case RISCV::VMINU_VV:
393 case RISCV::VMINU_VX:
394 case RISCV::VMIN_VV:
395 case RISCV::VMIN_VX:
396 case RISCV::VMAXU_VV:
397 case RISCV::VMAXU_VX:
398 case RISCV::VMAX_VV:
399 case RISCV::VMAX_VX:
400 // Vector Single-Width Integer Multiply Instructions
401 // Source and Dest EEW=SEW.
402 case RISCV::VMUL_VV:
403 case RISCV::VMUL_VX:
404 case RISCV::VMULH_VV:
405 case RISCV::VMULH_VX:
406 case RISCV::VMULHU_VV:
407 case RISCV::VMULHU_VX:
408 case RISCV::VMULHSU_VV:
409 case RISCV::VMULHSU_VX:
410 // Vector Integer Divide Instructions
411 // EEW=SEW.
412 case RISCV::VDIVU_VV:
413 case RISCV::VDIVU_VX:
414 case RISCV::VDIV_VV:
415 case RISCV::VDIV_VX:
416 case RISCV::VREMU_VV:
417 case RISCV::VREMU_VX:
418 case RISCV::VREM_VV:
419 case RISCV::VREM_VX:
420 // Vector Single-Width Integer Multiply-Add Instructions
421 // EEW=SEW.
422 case RISCV::VMACC_VV:
423 case RISCV::VMACC_VX:
424 case RISCV::VNMSAC_VV:
425 case RISCV::VNMSAC_VX:
426 case RISCV::VMADD_VV:
427 case RISCV::VMADD_VX:
428 case RISCV::VNMSUB_VV:
429 case RISCV::VNMSUB_VX:
430 // Vector Integer Merge Instructions
431 // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
432 // EEW=SEW, except the mask operand has EEW=1. Mask operand is handled
433 // before this switch.
434 case RISCV::VMERGE_VIM:
435 case RISCV::VMERGE_VVM:
436 case RISCV::VMERGE_VXM:
437 case RISCV::VADC_VIM:
438 case RISCV::VADC_VVM:
439 case RISCV::VADC_VXM:
440 case RISCV::VSBC_VVM:
441 case RISCV::VSBC_VXM:
442 // Vector Integer Move Instructions
443 // Vector Fixed-Point Arithmetic Instructions
444 // Vector Single-Width Saturating Add and Subtract
445 // Vector Single-Width Averaging Add and Subtract
446 // EEW=SEW.
447 case RISCV::VMV_V_I:
448 case RISCV::VMV_V_V:
449 case RISCV::VMV_V_X:
450 case RISCV::VSADDU_VI:
451 case RISCV::VSADDU_VV:
452 case RISCV::VSADDU_VX:
453 case RISCV::VSADD_VI:
454 case RISCV::VSADD_VV:
455 case RISCV::VSADD_VX:
456 case RISCV::VSSUBU_VV:
457 case RISCV::VSSUBU_VX:
458 case RISCV::VSSUB_VV:
459 case RISCV::VSSUB_VX:
460 case RISCV::VAADDU_VV:
461 case RISCV::VAADDU_VX:
462 case RISCV::VAADD_VV:
463 case RISCV::VAADD_VX:
464 case RISCV::VASUBU_VV:
465 case RISCV::VASUBU_VX:
466 case RISCV::VASUB_VV:
467 case RISCV::VASUB_VX:
468 // Vector Single-Width Fractional Multiply with Rounding and Saturation
469 // EEW=SEW. The instruction produces 2*SEW product internally but
470 // saturates to fit into SEW bits.
471 case RISCV::VSMUL_VV:
472 case RISCV::VSMUL_VX:
473 // Vector Single-Width Scaling Shift Instructions
474 // EEW=SEW.
475 case RISCV::VSSRL_VI:
476 case RISCV::VSSRL_VV:
477 case RISCV::VSSRL_VX:
478 case RISCV::VSSRA_VI:
479 case RISCV::VSSRA_VV:
480 case RISCV::VSSRA_VX:
481 // Vector Permutation Instructions
482 // Integer Scalar Move Instructions
483 // Floating-Point Scalar Move Instructions
484 // EEW=SEW.
485 case RISCV::VMV_X_S:
486 case RISCV::VMV_S_X:
487 case RISCV::VFMV_F_S:
488 case RISCV::VFMV_S_F:
489 // Vector Slide Instructions
490 // EEW=SEW.
491 case RISCV::VSLIDEUP_VI:
492 case RISCV::VSLIDEUP_VX:
493 case RISCV::VSLIDEDOWN_VI:
494 case RISCV::VSLIDEDOWN_VX:
495 case RISCV::VSLIDE1UP_VX:
496 case RISCV::VFSLIDE1UP_VF:
497 case RISCV::VSLIDE1DOWN_VX:
498 case RISCV::VFSLIDE1DOWN_VF:
499 // Vector Register Gather Instructions
500 // EEW=SEW. For mask operand, EEW=1.
501 case RISCV::VRGATHER_VI:
502 case RISCV::VRGATHER_VV:
503 case RISCV::VRGATHER_VX:
504 // Vector Element Index Instruction
505 case RISCV::VID_V:
506 // Vector Single-Width Floating-Point Add/Subtract Instructions
507 case RISCV::VFADD_VF:
508 case RISCV::VFADD_VV:
509 case RISCV::VFSUB_VF:
510 case RISCV::VFSUB_VV:
511 case RISCV::VFRSUB_VF:
512 // Vector Single-Width Floating-Point Multiply/Divide Instructions
513 case RISCV::VFMUL_VF:
514 case RISCV::VFMUL_VV:
515 case RISCV::VFDIV_VF:
516 case RISCV::VFDIV_VV:
517 case RISCV::VFRDIV_VF:
518 // Vector Single-Width Floating-Point Fused Multiply-Add Instructions
519 case RISCV::VFMACC_VV:
520 case RISCV::VFMACC_VF:
521 case RISCV::VFNMACC_VV:
522 case RISCV::VFNMACC_VF:
523 case RISCV::VFMSAC_VV:
524 case RISCV::VFMSAC_VF:
525 case RISCV::VFNMSAC_VV:
526 case RISCV::VFNMSAC_VF:
527 case RISCV::VFMADD_VV:
528 case RISCV::VFMADD_VF:
529 case RISCV::VFNMADD_VV:
530 case RISCV::VFNMADD_VF:
531 case RISCV::VFMSUB_VV:
532 case RISCV::VFMSUB_VF:
533 case RISCV::VFNMSUB_VV:
534 case RISCV::VFNMSUB_VF:
535 // Vector Floating-Point Square-Root Instruction
536 case RISCV::VFSQRT_V:
537 // Vector Floating-Point Reciprocal Square-Root Estimate Instruction
538 case RISCV::VFRSQRT7_V:
539 // Vector Floating-Point Reciprocal Estimate Instruction
540 case RISCV::VFREC7_V:
541 // Vector Floating-Point MIN/MAX Instructions
542 case RISCV::VFMIN_VF:
543 case RISCV::VFMIN_VV:
544 case RISCV::VFMAX_VF:
545 case RISCV::VFMAX_VV:
546 // Vector Floating-Point Sign-Injection Instructions
547 case RISCV::VFSGNJ_VF:
548 case RISCV::VFSGNJ_VV:
549 case RISCV::VFSGNJN_VV:
550 case RISCV::VFSGNJN_VF:
551 case RISCV::VFSGNJX_VF:
552 case RISCV::VFSGNJX_VV:
553 // Vector Floating-Point Classify Instruction
554 case RISCV::VFCLASS_V:
555 // Vector Floating-Point Move Instruction
556 case RISCV::VFMV_V_F:
557 // Single-Width Floating-Point/Integer Type-Convert Instructions
558 case RISCV::VFCVT_XU_F_V:
559 case RISCV::VFCVT_X_F_V:
560 case RISCV::VFCVT_RTZ_XU_F_V:
561 case RISCV::VFCVT_RTZ_X_F_V:
562 case RISCV::VFCVT_F_XU_V:
563 case RISCV::VFCVT_F_X_V:
564 // Vector Floating-Point Merge Instruction
565 case RISCV::VFMERGE_VFM:
566 // Vector count population in mask vcpop.m
567 // vfirst find-first-set mask bit
568 case RISCV::VCPOP_M:
569 case RISCV::VFIRST_M:
570 // Vector Bit-manipulation Instructions (Zvbb)
571 // Vector And-Not
572 case RISCV::VANDN_VV:
573 case RISCV::VANDN_VX:
574 // Vector Reverse Bits in Elements
575 case RISCV::VBREV_V:
576 // Vector Reverse Bits in Bytes
577 case RISCV::VBREV8_V:
578 // Vector Reverse Bytes
579 case RISCV::VREV8_V:
580 // Vector Count Leading Zeros
581 case RISCV::VCLZ_V:
582 // Vector Count Trailing Zeros
583 case RISCV::VCTZ_V:
584 // Vector Population Count
585 case RISCV::VCPOP_V:
586 // Vector Rotate Left
587 case RISCV::VROL_VV:
588 case RISCV::VROL_VX:
589 // Vector Rotate Right
590 case RISCV::VROR_VI:
591 case RISCV::VROR_VV:
592 case RISCV::VROR_VX:
593 // Vector Carry-less Multiplication Instructions (Zvbc)
594 // Vector Carry-less Multiply
595 case RISCV::VCLMUL_VV:
596 case RISCV::VCLMUL_VX:
597 // Vector Carry-less Multiply Return High Half
598 case RISCV::VCLMULH_VV:
599 case RISCV::VCLMULH_VX:
600
601 // Zvabd
602 case RISCV::VABD_VV:
603 case RISCV::VABD_VX:
604 case RISCV::VABDU_VV:
605 case RISCV::VABDU_VX:
606 return MILog2SEW;
607
608 // Vector Widening Shift Left Logical (Zvbb)
609 case RISCV::VWSLL_VI:
610 case RISCV::VWSLL_VX:
611 case RISCV::VWSLL_VV:
612 // Vector Widening Integer Add/Subtract
613 // Def uses EEW=2*SEW . Operands use EEW=SEW.
614 case RISCV::VWADDU_VV:
615 case RISCV::VWADDU_VX:
616 case RISCV::VWSUBU_VV:
617 case RISCV::VWSUBU_VX:
618 case RISCV::VWADD_VV:
619 case RISCV::VWADD_VX:
620 case RISCV::VWSUB_VV:
621 case RISCV::VWSUB_VX:
622 // Vector Widening Integer Multiply Instructions
623 // Destination EEW=2*SEW. Source EEW=SEW.
624 case RISCV::VWMUL_VV:
625 case RISCV::VWMUL_VX:
626 case RISCV::VWMULSU_VV:
627 case RISCV::VWMULSU_VX:
628 case RISCV::VWMULU_VV:
629 case RISCV::VWMULU_VX:
630 // Vector Widening Integer Multiply-Add Instructions
631 // Destination EEW=2*SEW. Source EEW=SEW.
632 // A SEW-bit*SEW-bit multiply of the sources forms a 2*SEW-bit value, which
633 // is then added to the 2*SEW-bit Dest. These instructions never have a
634 // passthru operand.
635 case RISCV::VWMACCU_VV:
636 case RISCV::VWMACCU_VX:
637 case RISCV::VWMACC_VV:
638 case RISCV::VWMACC_VX:
639 case RISCV::VWMACCSU_VV:
640 case RISCV::VWMACCSU_VX:
641 case RISCV::VWMACCUS_VX:
642 // Vector Widening Floating-Point Fused Multiply-Add Instructions
643 case RISCV::VFWMACC_VF:
644 case RISCV::VFWMACC_VV:
645 case RISCV::VFWNMACC_VF:
646 case RISCV::VFWNMACC_VV:
647 case RISCV::VFWMSAC_VF:
648 case RISCV::VFWMSAC_VV:
649 case RISCV::VFWNMSAC_VF:
650 case RISCV::VFWNMSAC_VV:
651 case RISCV::VFWMACCBF16_VV:
652 case RISCV::VFWMACCBF16_VF:
653 // Vector Widening Floating-Point Add/Subtract Instructions
654 // Dest EEW=2*SEW. Source EEW=SEW.
655 case RISCV::VFWADD_VV:
656 case RISCV::VFWADD_VF:
657 case RISCV::VFWSUB_VV:
658 case RISCV::VFWSUB_VF:
659 // Vector Widening Floating-Point Multiply
660 case RISCV::VFWMUL_VF:
661 case RISCV::VFWMUL_VV:
662 // Widening Floating-Point/Integer Type-Convert Instructions
663 case RISCV::VFWCVT_XU_F_V:
664 case RISCV::VFWCVT_X_F_V:
665 case RISCV::VFWCVT_RTZ_XU_F_V:
666 case RISCV::VFWCVT_RTZ_X_F_V:
667 case RISCV::VFWCVT_F_XU_V:
668 case RISCV::VFWCVT_F_X_V:
669 case RISCV::VFWCVT_F_F_V:
670 case RISCV::VFWCVTBF16_F_F_V:
671 // Zvabd
672 case RISCV::VWABDA_VV:
673 case RISCV::VWABDA_VX:
674 case RISCV::VWABDAU_VV:
675 case RISCV::VWABDAU_VX:
676 return IsMODef ? MILog2SEW + 1 : MILog2SEW;
677
678 // Def and Op1 uses EEW=2*SEW. Op2 uses EEW=SEW.
679 case RISCV::VWADDU_WV:
680 case RISCV::VWADDU_WX:
681 case RISCV::VWSUBU_WV:
682 case RISCV::VWSUBU_WX:
683 case RISCV::VWADD_WV:
684 case RISCV::VWADD_WX:
685 case RISCV::VWSUB_WV:
686 case RISCV::VWSUB_WX:
687 // Vector Widening Floating-Point Add/Subtract Instructions
688 case RISCV::VFWADD_WF:
689 case RISCV::VFWADD_WV:
690 case RISCV::VFWSUB_WF:
691 case RISCV::VFWSUB_WV: {
692 bool IsOp1 = (HasPassthru && !IsTied) ? OpIdx == 2 : OpIdx == 1;
693 bool TwoTimes = IsMODef || IsOp1;
694 return TwoTimes ? MILog2SEW + 1 : MILog2SEW;
695 }
696
697 // Vector Integer Extension
698 case RISCV::VZEXT_VF2:
699 case RISCV::VSEXT_VF2:
700 return getIntegerExtensionOperandEEW(Factor: 2, MI, OpIdx);
701 case RISCV::VZEXT_VF4:
702 case RISCV::VSEXT_VF4:
703 return getIntegerExtensionOperandEEW(Factor: 4, MI, OpIdx);
704 case RISCV::VZEXT_VF8:
705 case RISCV::VSEXT_VF8:
706 return getIntegerExtensionOperandEEW(Factor: 8, MI, OpIdx);
707
708 // Vector Narrowing Integer Right Shift Instructions
709 // Destination EEW=SEW, Op 1 has EEW=2*SEW. Op2 has EEW=SEW
710 case RISCV::VNSRL_WX:
711 case RISCV::VNSRL_WI:
712 case RISCV::VNSRL_WV:
713 case RISCV::VNSRA_WI:
714 case RISCV::VNSRA_WV:
715 case RISCV::VNSRA_WX:
716 // Vector Narrowing Fixed-Point Clip Instructions
717 // Destination and Op1 EEW=SEW. Op2 EEW=2*SEW.
718 case RISCV::VNCLIPU_WI:
719 case RISCV::VNCLIPU_WV:
720 case RISCV::VNCLIPU_WX:
721 case RISCV::VNCLIP_WI:
722 case RISCV::VNCLIP_WV:
723 case RISCV::VNCLIP_WX:
724 // Narrowing Floating-Point/Integer Type-Convert Instructions
725 case RISCV::VFNCVT_XU_F_W:
726 case RISCV::VFNCVT_X_F_W:
727 case RISCV::VFNCVT_RTZ_XU_F_W:
728 case RISCV::VFNCVT_RTZ_X_F_W:
729 case RISCV::VFNCVT_F_XU_W:
730 case RISCV::VFNCVT_F_X_W:
731 case RISCV::VFNCVT_F_F_W:
732 case RISCV::VFNCVT_ROD_F_F_W:
733 case RISCV::VFNCVTBF16_F_F_W: {
734 assert(!IsTied);
735 bool IsOp1 = HasPassthru ? OpIdx == 2 : OpIdx == 1;
736 bool TwoTimes = IsOp1;
737 return TwoTimes ? MILog2SEW + 1 : MILog2SEW;
738 }
739
740 // Vector Mask Instructions
741 // Vector Mask-Register Logical Instructions
742 // vmsbf.m set-before-first mask bit
743 // vmsif.m set-including-first mask bit
744 // vmsof.m set-only-first mask bit
745 // EEW=1
746 // We handle the cases when operand is a v0 mask operand above the switch,
747 // but these instructions may use non-v0 mask operands and need to be handled
748 // specifically.
749 case RISCV::VMAND_MM:
750 case RISCV::VMNAND_MM:
751 case RISCV::VMANDN_MM:
752 case RISCV::VMXOR_MM:
753 case RISCV::VMOR_MM:
754 case RISCV::VMNOR_MM:
755 case RISCV::VMORN_MM:
756 case RISCV::VMXNOR_MM:
757 case RISCV::VMSBF_M:
758 case RISCV::VMSIF_M:
759 case RISCV::VMSOF_M: {
760 return MILog2SEW;
761 }
762
763 // Vector Compress Instruction
764 // EEW=SEW, except the mask operand has EEW=1. Mask operand is not handled
765 // before this switch.
766 case RISCV::VCOMPRESS_VM:
767 return OpIdx == 3 ? 0 : MILog2SEW;
768
769 // Vector Iota Instruction
770 // EEW=SEW, except the mask operand has EEW=1. Mask operand is not handled
771 // before this switch.
772 case RISCV::VIOTA_M: {
773 if (IsMODef || OpIdx == 1)
774 return MILog2SEW;
775 return 0;
776 }
777
778 // Vector Integer Compare Instructions
779 // Dest EEW=1. Source EEW=SEW.
780 case RISCV::VMSEQ_VI:
781 case RISCV::VMSEQ_VV:
782 case RISCV::VMSEQ_VX:
783 case RISCV::VMSNE_VI:
784 case RISCV::VMSNE_VV:
785 case RISCV::VMSNE_VX:
786 case RISCV::VMSLTU_VV:
787 case RISCV::VMSLTU_VX:
788 case RISCV::VMSLT_VV:
789 case RISCV::VMSLT_VX:
790 case RISCV::VMSLEU_VV:
791 case RISCV::VMSLEU_VI:
792 case RISCV::VMSLEU_VX:
793 case RISCV::VMSLE_VV:
794 case RISCV::VMSLE_VI:
795 case RISCV::VMSLE_VX:
796 case RISCV::VMSGTU_VI:
797 case RISCV::VMSGTU_VX:
798 case RISCV::VMSGT_VI:
799 case RISCV::VMSGT_VX:
800 // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
801 // Dest EEW=1. Source EEW=SEW. Mask source operand handled above this switch.
802 case RISCV::VMADC_VIM:
803 case RISCV::VMADC_VVM:
804 case RISCV::VMADC_VXM:
805 case RISCV::VMSBC_VVM:
806 case RISCV::VMSBC_VXM:
807 // Dest EEW=1. Source EEW=SEW.
808 case RISCV::VMADC_VV:
809 case RISCV::VMADC_VI:
810 case RISCV::VMADC_VX:
811 case RISCV::VMSBC_VV:
812 case RISCV::VMSBC_VX:
813 // 13.13. Vector Floating-Point Compare Instructions
814 // Dest EEW=1. Source EEW=SEW
815 case RISCV::VMFEQ_VF:
816 case RISCV::VMFEQ_VV:
817 case RISCV::VMFNE_VF:
818 case RISCV::VMFNE_VV:
819 case RISCV::VMFLT_VF:
820 case RISCV::VMFLT_VV:
821 case RISCV::VMFLE_VF:
822 case RISCV::VMFLE_VV:
823 case RISCV::VMFGT_VF:
824 case RISCV::VMFGE_VF: {
825 if (IsMODef)
826 return 0;
827 return MILog2SEW;
828 }
829
830 // Vector Reduction Operations
831 // Vector Single-Width Integer Reduction Instructions
832 case RISCV::VREDAND_VS:
833 case RISCV::VREDMAX_VS:
834 case RISCV::VREDMAXU_VS:
835 case RISCV::VREDMIN_VS:
836 case RISCV::VREDMINU_VS:
837 case RISCV::VREDOR_VS:
838 case RISCV::VREDSUM_VS:
839 case RISCV::VREDXOR_VS:
840 // Vector Single-Width Floating-Point Reduction Instructions
841 case RISCV::VFREDMAX_VS:
842 case RISCV::VFREDMIN_VS:
843 case RISCV::VFREDOSUM_VS:
844 case RISCV::VFREDUSUM_VS: {
845 return MILog2SEW;
846 }
847
848 // Vector Widening Integer Reduction Instructions
849 // The Dest and VS1 read only element 0 for the vector register. Return
850 // 2*EEW for these. VS2 has EEW=SEW and EMUL=LMUL.
851 case RISCV::VWREDSUM_VS:
852 case RISCV::VWREDSUMU_VS:
853 // Vector Widening Floating-Point Reduction Instructions
854 case RISCV::VFWREDOSUM_VS:
855 case RISCV::VFWREDUSUM_VS: {
856 bool TwoTimes = IsMODef || OpIdx == 3;
857 return TwoTimes ? MILog2SEW + 1 : MILog2SEW;
858 }
859
860 // Vector Register Gather with 16-bit Index Elements Instruction
861 // Dest and source data EEW=SEW. Index vector EEW=16.
862 case RISCV::VRGATHEREI16_VV: {
863 if (OpIdx == 2)
864 return 4;
865 return MILog2SEW;
866 }
867
868 default:
869 return std::nullopt;
870 }
871}
872
873static std::optional<OperandInfo> getOperandInfo(const MachineInstr &MI,
874 unsigned OpIdx) {
875 const RISCVVPseudosTable::PseudoInfo *RVV =
876 RISCVVPseudosTable::getPseudoInfo(Pseudo: MI.getOpcode());
877 assert(RVV && "Could not find MI in PseudoTable");
878
879 std::optional<unsigned> Log2EEW = getOperandLog2EEW(MI, OpIdx);
880 if (!Log2EEW)
881 return std::nullopt;
882
883 switch (RVV->BaseInstr) {
884 // Vector Reduction Operations
885 // Vector Single-Width Integer Reduction Instructions
886 // Vector Widening Integer Reduction Instructions
887 // Vector Widening Floating-Point Reduction Instructions
888 // The Dest and VS1 only read element 0 of the vector register. Return just
889 // the EEW for these.
890 case RISCV::VREDAND_VS:
891 case RISCV::VREDMAX_VS:
892 case RISCV::VREDMAXU_VS:
893 case RISCV::VREDMIN_VS:
894 case RISCV::VREDMINU_VS:
895 case RISCV::VREDOR_VS:
896 case RISCV::VREDSUM_VS:
897 case RISCV::VREDXOR_VS:
898 case RISCV::VWREDSUM_VS:
899 case RISCV::VWREDSUMU_VS:
900 case RISCV::VFWREDOSUM_VS:
901 case RISCV::VFWREDUSUM_VS:
902 if (OpIdx != 2)
903 return OperandInfo(*Log2EEW);
904 break;
905 };
906
907 // All others have EMUL=EEW/SEW*LMUL
908 return OperandInfo(getEMULEqualsEEWDivSEWTimesLMUL(Log2EEW: *Log2EEW, MI), *Log2EEW);
909}
910
911static bool isTupleInsertInstr(const MachineInstr &MI);
912
913/// Return true if we can reason about demanded VLs elementwise for \p MI.
914bool RISCVVLOptimizerImpl::isSupportedInstr(const MachineInstr &MI) const {
915 if (MI.isPHI() || MI.isFullCopy() || isTupleInsertInstr(MI))
916 return true;
917
918 unsigned RVVOpc = RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode());
919 if (!RVVOpc)
920 return false;
921
922 assert(!(MI.getNumExplicitDefs() == 0 && !MI.mayStore() &&
923 !RISCVII::elementsDependOnVL(TII->get(RVVOpc).TSFlags)) &&
924 "No defs but elements don't depend on VL?");
925
926 // TODO: Reduce vl for vmv.s.x and vfmv.s.f. Currently this introduces more vl
927 // toggles, we need to extend PRE in RISCVInsertVSETVLI first.
928 if (RVVOpc == RISCV::VMV_S_X || RVVOpc == RISCV::VFMV_S_F)
929 return false;
930
931 if (RISCVII::elementsDependOnVL(TSFlags: TII->get(Opcode: RVVOpc).TSFlags))
932 return false;
933
934 if (MI.mayStore())
935 return false;
936
937 return true;
938}
939
940/// Return true if operand \p OpIdx of \p MI is a vector operand but is used as
941/// a scalar operand.
942static bool isVectorOpUsedAsScalarOp(const MachineInstr &MI, unsigned OpIdx) {
943 const RISCVVPseudosTable::PseudoInfo *RVV =
944 RISCVVPseudosTable::getPseudoInfo(Pseudo: MI.getOpcode());
945
946 if (!RVV)
947 return false;
948
949 switch (RVV->BaseInstr) {
950 // Reductions only use vs1[0] of vs1
951 case RISCV::VREDAND_VS:
952 case RISCV::VREDMAX_VS:
953 case RISCV::VREDMAXU_VS:
954 case RISCV::VREDMIN_VS:
955 case RISCV::VREDMINU_VS:
956 case RISCV::VREDOR_VS:
957 case RISCV::VREDSUM_VS:
958 case RISCV::VREDXOR_VS:
959 case RISCV::VWREDSUM_VS:
960 case RISCV::VWREDSUMU_VS:
961 case RISCV::VFREDMAX_VS:
962 case RISCV::VFREDMIN_VS:
963 case RISCV::VFREDOSUM_VS:
964 case RISCV::VFREDUSUM_VS:
965 case RISCV::VFWREDOSUM_VS:
966 case RISCV::VFWREDUSUM_VS:
967 return OpIdx == 3;
968 case RISCV::VMV_X_S:
969 case RISCV::VFMV_F_S:
970 return OpIdx == 1;
971 default:
972 return false;
973 }
974}
975
976bool RISCVVLOptimizerImpl::isCandidate(const MachineInstr &MI) const {
977 const MCInstrDesc &Desc = MI.getDesc();
978 if (!RISCVII::hasVLOp(TSFlags: Desc.TSFlags) || !RISCVII::hasSEWOp(TSFlags: Desc.TSFlags))
979 return false;
980
981 if (MI.getNumExplicitDefs() != 1)
982 return false;
983
984 // Some instructions have implicit defs e.g. $vxsat. If they might be read
985 // later then we can't reduce VL.
986 if (!MI.allImplicitDefsAreDead()) {
987 LLVM_DEBUG(dbgs() << "Not a candidate because has non-dead implicit def\n");
988 return false;
989 }
990
991 if (MI.mayRaiseFPException()) {
992 LLVM_DEBUG(dbgs() << "Not a candidate because may raise FP exception\n");
993 return false;
994 }
995
996 for (const MachineMemOperand *MMO : MI.memoperands()) {
997 if (MMO->isVolatile()) {
998 LLVM_DEBUG(dbgs() << "Not a candidate because contains volatile MMO\n");
999 return false;
1000 }
1001 }
1002
1003 if (!isSupportedInstr(MI)) {
1004 LLVM_DEBUG(dbgs() << "Not a candidate due to unsupported instruction: "
1005 << MI);
1006 return false;
1007 }
1008
1009 assert(!RISCVII::elementsDependOnVL(
1010 TII->get(RISCV::getRVVMCOpcode(MI.getOpcode())).TSFlags) &&
1011 "Instruction shouldn't be supported if elements depend on VL");
1012
1013 assert(RISCVRI::isVRegClass(
1014 MRI->getRegClass(MI.getOperand(0).getReg())->TSFlags) &&
1015 "All supported instructions produce a vector register result");
1016
1017 LLVM_DEBUG(dbgs() << "Found a candidate for VL reduction: " << MI << "\n");
1018 return true;
1019}
1020
1021/// Given a vslidedown.vx like:
1022///
1023/// %slideamt = ADDI %x, -1
1024/// %v = PseudoVSLIDEDOWN_VX %passthru, %src, %slideamt, avl=1
1025///
1026/// %v will only read the first %slideamt + 1 lanes of %src, which = %x.
1027/// This is a common case when lowering extractelement.
1028///
1029/// Note that if %x is 0, %slideamt will be all ones. In this case %src will be
1030/// completely slid down and none of its lanes will be read (since %slideamt is
1031/// greater than the largest VLMAX of 65536) so we can demand any minimum VL.
1032static std::optional<DemandedVL>
1033getMinimumVLForVSLIDEDOWN_VX(const MachineOperand &UserOp,
1034 const MachineRegisterInfo *MRI) {
1035 const MachineInstr &MI = *UserOp.getParent();
1036 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode()) != RISCV::VSLIDEDOWN_VX)
1037 return std::nullopt;
1038 // We're looking at what lanes are used from the src operand.
1039 if (UserOp.getOperandNo() != 2)
1040 return std::nullopt;
1041 // For now, the AVL must be 1.
1042 const MachineOperand &AVL = MI.getOperand(i: 4);
1043 if (!AVL.isImm() || AVL.getImm() != 1)
1044 return std::nullopt;
1045 // The slide amount must be %x - 1.
1046 const MachineOperand &SlideAmt = MI.getOperand(i: 3);
1047 if (!SlideAmt.getReg().isVirtual())
1048 return std::nullopt;
1049 MachineInstr *SlideAmtDef = MRI->getVRegDef(Reg: SlideAmt.getReg());
1050 if (!SlideAmtDef || SlideAmtDef->getOpcode() != RISCV::ADDI ||
1051 SlideAmtDef->getOperand(i: 2).getImm() != -AVL.getImm() ||
1052 !SlideAmtDef->getOperand(i: 1).getReg().isVirtual())
1053 return std::nullopt;
1054 return SlideAmtDef->getOperand(i: 1);
1055}
1056
1057DemandedVL
1058RISCVVLOptimizerImpl::getMinimumVLForUser(const MachineOperand &UserOp) const {
1059 const MachineInstr &UserMI = *UserOp.getParent();
1060 const MCInstrDesc &Desc = UserMI.getDesc();
1061
1062 if (UserMI.isPHI() || UserMI.isFullCopy() || isTupleInsertInstr(MI: UserMI))
1063 return DemandedVLs.lookup(Key: &UserMI);
1064
1065 if (!RISCVII::hasVLOp(TSFlags: Desc.TSFlags) || !RISCVII::hasSEWOp(TSFlags: Desc.TSFlags)) {
1066 LLVM_DEBUG(dbgs() << " Abort due to lack of VL, assume that"
1067 " use VLMAX\n");
1068 return DemandedVL::vlmax();
1069 }
1070
1071 if (auto VL = getMinimumVLForVSLIDEDOWN_VX(UserOp, MRI))
1072 return *VL;
1073
1074 if (RISCVII::readsPastVL(
1075 TSFlags: TII->get(Opcode: RISCV::getRVVMCOpcode(RVVPseudoOpcode: UserMI.getOpcode())).TSFlags)) {
1076 LLVM_DEBUG(dbgs() << " Abort because used by unsafe instruction\n");
1077 return DemandedVL::vlmax();
1078 }
1079
1080 unsigned VLOpNum = RISCVII::getVLOpNum(Desc);
1081 const MachineOperand &VLOp = UserMI.getOperand(i: VLOpNum);
1082 // Looking for an immediate or a register VL that isn't X0.
1083 assert((!VLOp.isReg() || VLOp.getReg() != RISCV::X0) &&
1084 "Did not expect X0 VL");
1085
1086 // If the user is a passthru it will read the elements past VL, so
1087 // abort if any of the elements past VL are demanded.
1088 if (UserOp.isTied()) {
1089 assert(UserOp.getOperandNo() == UserMI.getNumExplicitDefs() &&
1090 RISCVII::isFirstDefTiedToFirstUse(UserMI.getDesc()));
1091 if (!RISCV::isVLKnownLE(MRI: *MRI, LHS: DemandedVLs.lookup(Key: &UserMI).VL, RHS: VLOp)) {
1092 LLVM_DEBUG(dbgs() << " Abort because user is passthru in "
1093 "instruction with demanded tail\n");
1094 return DemandedVL::vlmax();
1095 }
1096 }
1097
1098 // Instructions like reductions may use a vector register as a scalar
1099 // register. In this case, we should treat it as only reading the first lane.
1100 if (isVectorOpUsedAsScalarOp(MI: UserMI, OpIdx: UserMI.getOperandNo(I: &UserOp))) {
1101 LLVM_DEBUG(dbgs() << " Used this operand as a scalar operand\n");
1102 return MachineOperand::CreateImm(Val: 1);
1103 }
1104
1105 // If we know the demanded VL of UserMI, then we can reduce the VL it
1106 // requires.
1107 if (RISCV::isVLKnownLE(MRI: *MRI, LHS: DemandedVLs.lookup(Key: &UserMI).VL, RHS: VLOp))
1108 return DemandedVLs.lookup(Key: &UserMI);
1109
1110 return VLOp;
1111}
1112
1113/// Return true if MI is an instruction used for assembling registers
1114/// for segmented store instructions, namely, RISCVISD::TUPLE_INSERT.
1115/// Currently it's lowered to INSERT_SUBREG.
1116static bool isTupleInsertInstr(const MachineInstr &MI) {
1117 if (!MI.isInsertSubreg())
1118 return false;
1119
1120 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1121 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: MI.getOperand(i: 0).getReg());
1122 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
1123 if (!RISCVRI::isVRegClass(TSFlags: DstRC->TSFlags))
1124 return false;
1125 unsigned NF = RISCVRI::getNF(TSFlags: DstRC->TSFlags);
1126 if (NF < 2)
1127 return false;
1128
1129 // Check whether INSERT_SUBREG has the correct subreg index for tuple inserts.
1130 auto VLMul = RISCVRI::getLMul(TSFlags: DstRC->TSFlags);
1131 unsigned SubRegIdx = MI.getOperand(i: 3).getImm();
1132 [[maybe_unused]] auto [LMul, IsFractional] = RISCVVType::decodeVLMUL(VLMul);
1133 assert(!IsFractional && "unexpected LMUL for tuple register classes");
1134 return TRI->getSubRegIdxSize(Idx: SubRegIdx) == RISCV::RVVBitsPerBlock * LMul;
1135}
1136
1137static bool isSegmentedStoreInstr(const MachineInstr &MI) {
1138 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
1139 case VSSEG_CASES(8):
1140 case VSSSEG_CASES(8):
1141 case VSUXSEG_CASES(8):
1142 case VSOXSEG_CASES(8):
1143 case VSSEG_CASES(16):
1144 case VSSSEG_CASES(16):
1145 case VSUXSEG_CASES(16):
1146 case VSOXSEG_CASES(16):
1147 case VSSEG_CASES(32):
1148 case VSSSEG_CASES(32):
1149 case VSUXSEG_CASES(32):
1150 case VSOXSEG_CASES(32):
1151 case VSSEG_CASES(64):
1152 case VSSSEG_CASES(64):
1153 case VSUXSEG_CASES(64):
1154 case VSOXSEG_CASES(64):
1155 return true;
1156 default:
1157 return false;
1158 }
1159}
1160
1161bool RISCVVLOptimizerImpl::checkUsers(const MachineInstr &MI) const {
1162 if (MI.isPHI() || MI.isFullCopy() || isTupleInsertInstr(MI))
1163 return true;
1164
1165 SmallSetVector<MachineOperand *, 8> OpWorklist;
1166 SmallPtrSet<const MachineInstr *, 4> PHISeen;
1167 for (auto &UserOp : MRI->use_operands(Reg: MI.getOperand(i: 0).getReg()))
1168 OpWorklist.insert(X: &UserOp);
1169
1170 while (!OpWorklist.empty()) {
1171 MachineOperand &UserOp = *OpWorklist.pop_back_val();
1172 const MachineInstr &UserMI = *UserOp.getParent();
1173 LLVM_DEBUG(dbgs() << " Checking user: " << UserMI << "\n");
1174
1175 if (UserMI.isFullCopy() && UserMI.getOperand(i: 0).getReg().isVirtual()) {
1176 LLVM_DEBUG(dbgs() << " Peeking through uses of COPY\n");
1177 OpWorklist.insert_range(R: llvm::make_pointer_range(
1178 Range: MRI->use_operands(Reg: UserMI.getOperand(i: 0).getReg())));
1179 continue;
1180 }
1181
1182 if (isTupleInsertInstr(MI: UserMI)) {
1183 LLVM_DEBUG(dbgs().indent(4) << "Peeking through uses of INSERT_SUBREG\n");
1184 for (MachineOperand &UseOp :
1185 MRI->use_operands(Reg: UserMI.getOperand(i: 0).getReg())) {
1186 const MachineInstr &CandidateMI = *UseOp.getParent();
1187 // We should not propagate the VL if the user is not a segmented store
1188 // or another INSERT_SUBREG, since VL just works differently
1189 // between segmented operations (per-field) v.s. other RVV ops (on the
1190 // whole register group).
1191 if (!isTupleInsertInstr(MI: CandidateMI) &&
1192 !isSegmentedStoreInstr(MI: CandidateMI))
1193 return false;
1194 OpWorklist.insert(X: &UseOp);
1195 }
1196 continue;
1197 }
1198
1199 if (UserMI.isPHI()) {
1200 // Don't follow PHI cycles
1201 if (!PHISeen.insert(Ptr: &UserMI).second)
1202 continue;
1203 LLVM_DEBUG(dbgs() << " Peeking through uses of PHI\n");
1204 OpWorklist.insert_range(R: llvm::make_pointer_range(
1205 Range: MRI->use_operands(Reg: UserMI.getOperand(i: 0).getReg())));
1206 continue;
1207 }
1208
1209 if (!RISCVII::hasSEWOp(TSFlags: UserMI.getDesc().TSFlags)) {
1210 LLVM_DEBUG(dbgs() << " Abort due to lack of SEW operand\n");
1211 return false;
1212 }
1213
1214 std::optional<OperandInfo> ConsumerInfo =
1215 getOperandInfo(MI: UserMI, OpIdx: UserMI.getOperandNo(I: &UserOp));
1216 std::optional<OperandInfo> ProducerInfo = getOperandInfo(MI, OpIdx: 0);
1217 if (!ConsumerInfo || !ProducerInfo) {
1218 LLVM_DEBUG(dbgs() << " Abort due to unknown operand information.\n");
1219 LLVM_DEBUG(dbgs() << " ConsumerInfo is: " << ConsumerInfo << "\n");
1220 LLVM_DEBUG(dbgs() << " ProducerInfo is: " << ProducerInfo << "\n");
1221 return false;
1222 }
1223
1224 if (!OperandInfo::areCompatible(Def: *ProducerInfo, User: *ConsumerInfo)) {
1225 LLVM_DEBUG(
1226 dbgs()
1227 << " Abort due to incompatible information for EMUL or EEW.\n");
1228 LLVM_DEBUG(dbgs() << " ConsumerInfo is: " << ConsumerInfo << "\n");
1229 LLVM_DEBUG(dbgs() << " ProducerInfo is: " << ProducerInfo << "\n");
1230 return false;
1231 }
1232 }
1233
1234 return true;
1235}
1236
1237bool RISCVVLOptimizerImpl::tryReduceVL(MachineInstr &MI,
1238 MachineOperand CommonVL) const {
1239 LLVM_DEBUG(dbgs() << "Trying to reduce VL for " << MI);
1240
1241 unsigned VLOpNum = RISCVII::getVLOpNum(Desc: MI.getDesc());
1242 MachineOperand &VLOp = MI.getOperand(i: VLOpNum);
1243
1244 assert((CommonVL.isImm() || CommonVL.getReg().isVirtual()) &&
1245 "Expected VL to be an Imm or virtual Reg");
1246
1247 // If the VL is defined by a vleff that doesn't dominate MI, try using the
1248 // vleff's AVL. It will be greater than or equal to the output VL.
1249 if (CommonVL.isReg()) {
1250 const MachineInstr *VLMI = MRI->getVRegDef(Reg: CommonVL.getReg());
1251 if (VLMI && RISCVInstrInfo::isFaultOnlyFirstLoad(MI: *VLMI) &&
1252 !MDT->dominates(A: VLMI, B: &MI))
1253 CommonVL = VLMI->getOperand(i: RISCVII::getVLOpNum(Desc: VLMI->getDesc()));
1254 }
1255
1256 if (!RISCV::isVLKnownLE(MRI: *MRI, LHS: CommonVL, RHS: VLOp)) {
1257 LLVM_DEBUG(dbgs() << " Abort due to CommonVL not <= VLOp.\n");
1258 return false;
1259 }
1260
1261 if (CommonVL.isIdenticalTo(Other: VLOp)) {
1262 LLVM_DEBUG(
1263 dbgs() << " Abort due to CommonVL == VLOp, no point in reducing.\n");
1264 return false;
1265 }
1266
1267 if (CommonVL.isImm()) {
1268 LLVM_DEBUG(dbgs() << " Reduce VL from " << VLOp << " to "
1269 << CommonVL.getImm() << " for " << MI << "\n");
1270 VLOp.ChangeToImmediate(ImmVal: CommonVL.getImm());
1271 return true;
1272 }
1273 MachineInstr *VLMI = MRI->getVRegDef(Reg: CommonVL.getReg());
1274 if (!VLMI)
1275 return false;
1276
1277 auto VLDominates = [this, &VLMI](const MachineInstr &MI) {
1278 return MDT->dominates(A: VLMI, B: &MI);
1279 };
1280 if (!VLDominates(MI)) {
1281 assert(MI.getNumExplicitDefs() == 1);
1282 auto Uses = MRI->use_instructions(Reg: MI.getOperand(i: 0).getReg());
1283 auto UsesSameBB = make_filter_range(Range&: Uses, Pred: [&MI](const MachineInstr &Use) {
1284 return Use.getParent() == MI.getParent();
1285 });
1286 if (VLMI->getParent() == MI.getParent() &&
1287 all_of(Range&: UsesSameBB, P: VLDominates) &&
1288 RISCVInstrInfo::isSafeToMove(From: MI, To: std::next(x: VLMI->getIterator()))) {
1289 VLMI->getParent()->splice(Where: std::next(x: VLMI->getIterator()), Other: MI.getParent(),
1290 From: MI.getIterator());
1291 } else {
1292 LLVM_DEBUG(dbgs() << " Abort due to VL not dominating.\n");
1293 return false;
1294 }
1295 }
1296 LLVM_DEBUG(dbgs() << " Reduce VL from " << VLOp << " to "
1297 << printReg(CommonVL.getReg(), MRI->getTargetRegisterInfo())
1298 << " for " << MI << "\n");
1299
1300 // All our checks passed. We can reduce VL.
1301 VLOp.ChangeToRegister(Reg: CommonVL.getReg(), isDef: false);
1302 MRI->constrainRegClass(Reg: CommonVL.getReg(), RC: &RISCV::GPRNoX0RegClass);
1303 return true;
1304}
1305
1306static bool isPhysical(const MachineOperand &MO) {
1307 return MO.isReg() && MO.getReg().isPhysical();
1308}
1309
1310/// Look through \p MI's operands and propagate what it demands to its uses.
1311void RISCVVLOptimizerImpl::transfer(const MachineInstr &MI) {
1312 if (!isSupportedInstr(MI) || !checkUsers(MI) || any_of(Range: MI.defs(), P: isPhysical))
1313 DemandedVLs[&MI] = DemandedVL::vlmax();
1314
1315 for (const MachineOperand &MO : virtual_vec_uses(MI)) {
1316 const MachineInstr *Def = MRI->getVRegDef(Reg: MO.getReg());
1317 DemandedVL Prev = DemandedVLs[Def];
1318 DemandedVLs[Def] = DemandedVLs[Def].max(MRI: *MRI, X: getMinimumVLForUser(UserOp: MO));
1319 if (DemandedVLs[Def] != Prev)
1320 Worklist.insert(X: Def);
1321 }
1322}
1323
1324bool RISCVVLOptimizerImpl::run(MachineFunction &MF) {
1325 MRI = &MF.getRegInfo();
1326
1327 const RISCVSubtarget &ST = MF.getSubtarget<RISCVSubtarget>();
1328 if (!ST.hasVInstructions())
1329 return false;
1330
1331 TII = ST.getInstrInfo();
1332
1333 assert(DemandedVLs.empty());
1334
1335 // For each instruction that defines a vector, propagate the VL it
1336 // uses to its inputs.
1337 for (MachineBasicBlock *MBB : post_order(G: &MF)) {
1338 assert(MDT->isReachableFromEntry(MBB));
1339 for (MachineInstr &MI : reverse(C&: *MBB))
1340 if (!MI.isDebugInstr())
1341 Worklist.insert(X: &MI);
1342 }
1343
1344 while (!Worklist.empty()) {
1345 const MachineInstr *MI = Worklist.front();
1346 Worklist.remove(X: MI);
1347 transfer(MI: *MI);
1348 }
1349
1350 // Then go through and see if we can reduce the VL of any instructions to
1351 // only what's demanded.
1352 bool MadeChange = false;
1353 for (auto &[MI, VL] : DemandedVLs) {
1354 assert(MDT->isReachableFromEntry(MI->getParent()));
1355 if (!isCandidate(MI: *MI))
1356 continue;
1357 if (!tryReduceVL(MI&: *const_cast<MachineInstr *>(MI), CommonVL: VL.VL))
1358 continue;
1359 MadeChange = true;
1360 }
1361
1362 DemandedVLs.clear();
1363 return MadeChange;
1364}
1365
1366bool RISCVVLOptimizerLegacy::runOnMachineFunction(MachineFunction &MF) {
1367 if (skipFunction(F: MF.getFunction()))
1368 return false;
1369
1370 auto *MDT = &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
1371 return RISCVVLOptimizerImpl(MDT).run(MF);
1372}
1373
1374PreservedAnalyses
1375RISCVVLOptimizerPass::run(MachineFunction &MF,
1376 MachineFunctionAnalysisManager &MFAM) {
1377 auto *MDT = &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
1378 bool Changed = RISCVVLOptimizerImpl(MDT).run(MF);
1379 if (!Changed)
1380 return PreservedAnalyses::all();
1381
1382 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1383 PA.preserveSet<CFGAnalyses>();
1384 PA.preserve<MachineRegisterClassAnalysis>();
1385 return PA;
1386}
1387