1//===-------------- RISCVVLOptimizer.cpp - VL Optimizer -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===---------------------------------------------------------------------===//
8//
9// This pass reduces the VL where possible at the MI level, before VSETVLI
10// instructions are inserted.
11//
12// The purpose of this optimization is to make the VL argument, for instructions
13// that have a VL argument, as small as possible.
14//
15// This is split into a sparse dataflow analysis where we determine what VL is
16// demanded by each instruction first, and then afterwards try to reduce the VL
17// of each instruction if it demands less than its VL operand.
18//
19// The analysis is explained in more detail in the 2025 EuroLLVM Developers'
20// Meeting talk "Accidental Dataflow Analysis: Extending the RISC-V VL
21// Optimizer", which is available on YouTube at
22// https://www.youtube.com/watch?v=Mfb5fRSdJAc
23//
24// The slides for the talk are available at
25// https://llvm.org/devmtg/2025-04/slides/technical_talk/lau_accidental_dataflow.pdf
26//
27//===---------------------------------------------------------------------===//
28
29#include "RISCV.h"
30#include "RISCVSubtarget.h"
31#include "llvm/ADT/PostOrderIterator.h"
32#include "llvm/ADT/SetVector.h"
33#include "llvm/CodeGen/MachineDominators.h"
34#include "llvm/CodeGen/MachineFunctionPass.h"
35#include "llvm/CodeGen/RegisterClassInfo.h"
36#include "llvm/InitializePasses.h"
37
38using namespace llvm;
39
40#define DEBUG_TYPE "riscv-vl-optimizer"
41#define PASS_NAME "RISC-V VL Optimizer"
42
43namespace {
44
45/// Wrapper around MachineOperand that defaults to immediate 0.
46struct DemandedVL {
47 MachineOperand VL;
48 DemandedVL() : VL(MachineOperand::CreateImm(Val: 0)) {}
49 DemandedVL(MachineOperand VL) : VL(VL) {}
50 static DemandedVL vlmax() {
51 return DemandedVL(MachineOperand::CreateImm(Val: RISCV::VLMaxSentinel));
52 }
53 bool operator!=(const DemandedVL &Other) const {
54 return !VL.isIdenticalTo(Other: Other.VL);
55 }
56
57 DemandedVL max(const MachineRegisterInfo &MRI, const DemandedVL &X) const {
58 if (RISCV::isVLKnownLE(MRI, LHS: VL, RHS: X.VL))
59 return X;
60 if (RISCV::isVLKnownLE(MRI, LHS: X.VL, RHS: VL))
61 return *this;
62 return DemandedVL::vlmax();
63 }
64};
65
66class RISCVVLOptimizerImpl {
67 MachineRegisterInfo *MRI;
68 function_ref<const MachineDominatorTree &()> GetMDT;
69 const MachineDominatorTree *MDT = nullptr;
70 const TargetInstrInfo *TII;
71
72public:
73 RISCVVLOptimizerImpl(function_ref<const MachineDominatorTree &()> GetMDT)
74 : GetMDT(GetMDT) {}
75
76 bool run(MachineFunction &MF);
77
78private:
79 DemandedVL getMinimumVLForUser(const MachineInstr &UserMI,
80 unsigned OpIdx) const;
81 /// Returns true if the users of \p MI have compatible EEWs and SEWs.
82 bool checkUsers(const MachineInstr &MI) const;
83 bool tryReduceVL(MachineInstr &MI, MachineOperand VL);
84 bool isSupportedInstr(const MachineInstr &MI) const;
85 bool isCandidate(const MachineInstr &MI) const;
86 void transfer(const MachineInstr &MI);
87
88 /// For a given instruction, records what elements of it are demanded by
89 /// downstream users.
90 MapVector<const MachineInstr *, DemandedVL> DemandedVLs;
91 SetVector<const MachineInstr *> Worklist;
92
93 /// \returns all vector virtual registers that \p MI uses.
94 auto virtual_vec_uses(const MachineInstr &MI) const {
95 return make_filter_range(Range: MI.uses(), Pred: [this](const MachineOperand &MO) {
96 return MO.isReg() && MO.getReg().isVirtual() &&
97 RISCVRegisterInfo::isRVVRegClass(RC: MRI->getRegClass(Reg: MO.getReg()));
98 });
99 }
100
101 /// \returns all vector virtual registers that \p MI defines.
102 auto virtual_vec_defs(const MachineInstr &MI) const {
103 return make_filter_range(Range: MI.all_defs(), Pred: [this](const MachineOperand &MO) {
104 return MO.getReg().isVirtual() &&
105 RISCVRegisterInfo::isRVVRegClass(RC: MRI->getRegClass(Reg: MO.getReg()));
106 });
107 }
108};
109
110class RISCVVLOptimizerLegacy : public MachineFunctionPass {
111public:
112 static char ID;
113
114 RISCVVLOptimizerLegacy() : MachineFunctionPass(ID) {}
115
116 bool runOnMachineFunction(MachineFunction &MF) override;
117
118 void getAnalysisUsage(AnalysisUsage &AU) const override {
119 AU.setPreservesCFG();
120 AU.addRequired<MachineDominatorTreeWrapperPass>();
121 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
122 MachineFunctionPass::getAnalysisUsage(AU);
123 }
124
125 StringRef getPassName() const override { return PASS_NAME; }
126};
127
128/// Represents the EMUL and EEW of a MachineOperand.
129struct OperandInfo {
130 // Represent as 1,2,4,8, ... and fractional indicator. This is because
131 // EMUL can take on values that don't map to RISCVVType::VLMUL values exactly.
132 // For example, a mask operand can have an EMUL less than MF8.
133 // If nullopt, then EMUL isn't used (i.e. only a single scalar is read).
134 std::optional<std::pair<unsigned, bool>> EMUL;
135
136 unsigned Log2EEW;
137
138 OperandInfo(RISCVVType::VLMUL EMUL, unsigned Log2EEW)
139 : EMUL(RISCVVType::decodeVLMUL(VLMul: EMUL)), Log2EEW(Log2EEW) {}
140
141 OperandInfo(std::pair<unsigned, bool> EMUL, unsigned Log2EEW)
142 : EMUL(EMUL), Log2EEW(Log2EEW) {}
143
144 OperandInfo(unsigned Log2EEW) : Log2EEW(Log2EEW) {}
145
146 OperandInfo() = delete;
147
148 /// Return true if the EMUL and EEW produced by \p Def is compatible with the
149 /// EMUL and EEW used by \p User.
150 static bool areCompatible(const OperandInfo &Def, const OperandInfo &User) {
151 if (Def.Log2EEW != User.Log2EEW)
152 return false;
153 if (User.EMUL && Def.EMUL != User.EMUL)
154 return false;
155 return true;
156 }
157
158 void print(raw_ostream &OS) const {
159 if (EMUL) {
160 OS << "EMUL: m";
161 if (EMUL->second)
162 OS << "f";
163 OS << EMUL->first;
164 } else
165 OS << "EMUL: none\n";
166 OS << ", EEW: " << (1 << Log2EEW);
167 }
168};
169
170} // end anonymous namespace
171
172char RISCVVLOptimizerLegacy::ID = 0;
173INITIALIZE_PASS_BEGIN(RISCVVLOptimizerLegacy, DEBUG_TYPE, PASS_NAME, false,
174 false)
175INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
176INITIALIZE_PASS_END(RISCVVLOptimizerLegacy, DEBUG_TYPE, PASS_NAME, false, false)
177
178FunctionPass *llvm::createRISCVVLOptimizerLegacyPass() {
179 return new RISCVVLOptimizerLegacy();
180}
181
182[[maybe_unused]]
183static raw_ostream &operator<<(raw_ostream &OS, const OperandInfo &OI) {
184 OI.print(OS);
185 return OS;
186}
187
188[[maybe_unused]]
189static raw_ostream &operator<<(raw_ostream &OS,
190 const std::optional<OperandInfo> &OI) {
191 if (OI)
192 OI->print(OS);
193 else
194 OS << "nullopt";
195 return OS;
196}
197
198/// Return EMUL = (EEW / SEW) * LMUL where EEW comes from Log2EEW and LMUL and
199/// SEW are from the TSFlags of MI.
200static std::pair<unsigned, bool>
201getEMULEqualsEEWDivSEWTimesLMUL(unsigned Log2EEW, const MachineInstr &MI) {
202 RISCVVType::VLMUL MIVLMUL = RISCVII::getLMul(TSFlags: MI.getDesc().TSFlags);
203 auto [MILMUL, MILMULIsFractional] = RISCVVType::decodeVLMUL(VLMul: MIVLMUL);
204 unsigned MILog2SEW =
205 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
206
207 // Mask instructions will have 0 as the SEW operand. But the LMUL of these
208 // instructions is calculated is as if the SEW operand was 3 (e8).
209 if (MILog2SEW == 0)
210 MILog2SEW = 3;
211
212 unsigned MISEW = 1 << MILog2SEW;
213
214 unsigned EEW = 1 << Log2EEW;
215 // Calculate (EEW/SEW)*LMUL preserving fractions less than 1. Use GCD
216 // to put fraction in simplest form.
217 unsigned Num = EEW, Denom = MISEW;
218 int GCD = MILMULIsFractional ? std::gcd(m: Num, n: Denom * MILMUL)
219 : std::gcd(m: Num * MILMUL, n: Denom);
220 Num = MILMULIsFractional ? Num / GCD : Num * MILMUL / GCD;
221 Denom = MILMULIsFractional ? Denom * MILMUL / GCD : Denom / GCD;
222 return std::make_pair(x&: Num > Denom ? Num : Denom, y: Denom > Num);
223}
224
225static DemandedVL doubleVL(DemandedVL MinimumVL) {
226 if (!MinimumVL.VL.isImm())
227 return DemandedVL::vlmax();
228
229 int64_t VL = MinimumVL.VL.getImm();
230 if (!isUInt<4>(x: VL))
231 return DemandedVL::vlmax();
232 return MachineOperand::CreateImm(Val: VL * 2);
233}
234
235static DemandedVL halfVL(DemandedVL MinimumVL, bool Ceil = false) {
236 if (!MinimumVL.VL.isImm())
237 return DemandedVL::vlmax();
238
239 int64_t VL = MinimumVL.VL.getImm();
240 if (!isUInt<5>(x: VL))
241 return DemandedVL::vlmax();
242 return MachineOperand::CreateImm(Val: (VL + Ceil) / 2);
243}
244
245static std::pair<unsigned, bool> doubleEMUL(std::pair<unsigned, bool> EMUL) {
246 auto [Num, IsFractional] = EMUL;
247 if (IsFractional)
248 return std::make_pair(x: Num / 2, y: Num > 2);
249 return std::make_pair(x: Num * 2, y: false);
250}
251
252static std::pair<unsigned, bool> halfEMUL(std::pair<unsigned, bool> EMUL) {
253 auto [Num, IsFractional] = EMUL;
254 if (IsFractional || Num == 1)
255 return std::make_pair(x: Num * 2, y: true);
256 return std::make_pair(x: Num / 2, y: false);
257}
258
259/// Dest has EEW=SEW. Source EEW=SEW/Factor (i.e. F2 => EEW/2).
260/// SEW comes from TSFlags of MI.
261static unsigned getIntegerExtensionOperandEEW(unsigned Factor,
262 const MachineInstr &MI,
263 unsigned OpIdx) {
264 unsigned MILog2SEW =
265 MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
266
267 if (OpIdx == 0)
268 return MILog2SEW;
269
270 unsigned MISEW = 1 << MILog2SEW;
271 unsigned EEW = MISEW / Factor;
272 unsigned Log2EEW = Log2_32(Value: EEW);
273
274 return Log2EEW;
275}
276
277#define VSEG_CASES(Prefix, EEW) \
278 RISCV::Prefix##SEG2E##EEW##_V: \
279 case RISCV::Prefix##SEG3E##EEW##_V: \
280 case RISCV::Prefix##SEG4E##EEW##_V: \
281 case RISCV::Prefix##SEG5E##EEW##_V: \
282 case RISCV::Prefix##SEG6E##EEW##_V: \
283 case RISCV::Prefix##SEG7E##EEW##_V: \
284 case RISCV::Prefix##SEG8E##EEW##_V
285#define VSSEG_CASES(EEW) VSEG_CASES(VS, EEW)
286#define VSSSEG_CASES(EEW) VSEG_CASES(VSS, EEW)
287#define VSUXSEG_CASES(EEW) VSEG_CASES(VSUX, I##EEW)
288#define VSOXSEG_CASES(EEW) VSEG_CASES(VSOX, I##EEW)
289
290static std::optional<unsigned> getOperandLog2EEW(const MachineInstr &MI,
291 unsigned OpIdx) {
292 const MCInstrDesc &Desc = MI.getDesc();
293 const RISCVVPseudosTable::PseudoInfo *RVV =
294 RISCVVPseudosTable::getPseudoInfo(Pseudo: MI.getOpcode());
295 assert(RVV && "Could not find MI in PseudoTable");
296
297 // MI has a SEW associated with it. The RVV specification defines
298 // the EEW of each operand and definition in relation to MI.SEW.
299 unsigned MILog2SEW = MI.getOperand(i: RISCVII::getSEWOpNum(Desc)).getImm();
300
301 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc);
302 const bool IsTied = RISCVII::isTiedPseudo(TSFlags: Desc.TSFlags);
303
304 bool IsMODef =
305 OpIdx == 0 || (HasPassthru && OpIdx == MI.getNumExplicitDefs());
306
307 // All mask operands have EEW=1
308 const MCOperandInfo &Info = Desc.operands()[OpIdx];
309 if (Info.OperandType == MCOI::OPERAND_REGISTER &&
310 Info.RegClass == RISCV::VMV0RegClassID)
311 return 0;
312
313 // switch against BaseInstr to reduce number of cases that need to be
314 // considered.
315 switch (RVV->BaseInstr) {
316
317 // 6. Configuration-Setting Instructions
318 // Configuration setting instructions do not read or write vector registers
319 case RISCV::VSETIVLI:
320 case RISCV::VSETVL:
321 case RISCV::VSETVLI:
322 llvm_unreachable("Configuration setting instructions do not read or write "
323 "vector registers");
324
325 // Vector Loads and Stores
326 // Vector Unit-Stride Instructions
327 // Vector Strided Instructions
328 /// Dest EEW encoded in the instruction
329 case RISCV::VLM_V:
330 case RISCV::VSM_V:
331 return 0;
332 case RISCV::VLE8_V:
333 case RISCV::VSE8_V:
334 case RISCV::VLSE8_V:
335 case RISCV::VSSE8_V:
336 case VSSEG_CASES(8):
337 case VSSSEG_CASES(8):
338 return 3;
339 case RISCV::VLE16_V:
340 case RISCV::VSE16_V:
341 case RISCV::VLSE16_V:
342 case RISCV::VSSE16_V:
343 case VSSEG_CASES(16):
344 case VSSSEG_CASES(16):
345 return 4;
346 case RISCV::VLE32_V:
347 case RISCV::VSE32_V:
348 case RISCV::VLSE32_V:
349 case RISCV::VSSE32_V:
350 case VSSEG_CASES(32):
351 case VSSSEG_CASES(32):
352 return 5;
353 case RISCV::VLE64_V:
354 case RISCV::VSE64_V:
355 case RISCV::VLSE64_V:
356 case RISCV::VSSE64_V:
357 case VSSEG_CASES(64):
358 case VSSSEG_CASES(64):
359 return 6;
360
361 // Vector Indexed Instructions
362 // vs(o|u)xei<eew>.v
363 // Dest/Data (operand 0) EEW=SEW. Source EEW=<eew>.
364 case RISCV::VLUXEI8_V:
365 case RISCV::VLOXEI8_V:
366 case RISCV::VSUXEI8_V:
367 case RISCV::VSOXEI8_V:
368 case VSUXSEG_CASES(8):
369 case VSOXSEG_CASES(8): {
370 if (OpIdx == 0)
371 return MILog2SEW;
372 return 3;
373 }
374 case RISCV::VLUXEI16_V:
375 case RISCV::VLOXEI16_V:
376 case RISCV::VSUXEI16_V:
377 case RISCV::VSOXEI16_V:
378 case VSUXSEG_CASES(16):
379 case VSOXSEG_CASES(16): {
380 if (OpIdx == 0)
381 return MILog2SEW;
382 return 4;
383 }
384 case RISCV::VLUXEI32_V:
385 case RISCV::VLOXEI32_V:
386 case RISCV::VSUXEI32_V:
387 case RISCV::VSOXEI32_V:
388 case VSUXSEG_CASES(32):
389 case VSOXSEG_CASES(32): {
390 if (OpIdx == 0)
391 return MILog2SEW;
392 return 5;
393 }
394 case RISCV::VLUXEI64_V:
395 case RISCV::VLOXEI64_V:
396 case RISCV::VSUXEI64_V:
397 case RISCV::VSOXEI64_V:
398 case VSUXSEG_CASES(64):
399 case VSOXSEG_CASES(64): {
400 if (OpIdx == 0)
401 return MILog2SEW;
402 return 6;
403 }
404
405 // Vector Integer Arithmetic Instructions
406 // Vector Single-Width Integer Add and Subtract
407 case RISCV::VADD_VI:
408 case RISCV::VADD_VV:
409 case RISCV::VADD_VX:
410 case RISCV::VSUB_VV:
411 case RISCV::VSUB_VX:
412 case RISCV::VRSUB_VI:
413 case RISCV::VRSUB_VX:
414 // Vector Bitwise Logical Instructions
415 // Vector Single-Width Shift Instructions
416 // EEW=SEW.
417 case RISCV::VAND_VI:
418 case RISCV::VAND_VV:
419 case RISCV::VAND_VX:
420 case RISCV::VOR_VI:
421 case RISCV::VOR_VV:
422 case RISCV::VOR_VX:
423 case RISCV::VXOR_VI:
424 case RISCV::VXOR_VV:
425 case RISCV::VXOR_VX:
426 case RISCV::VSLL_VI:
427 case RISCV::VSLL_VV:
428 case RISCV::VSLL_VX:
429 case RISCV::VSRL_VI:
430 case RISCV::VSRL_VV:
431 case RISCV::VSRL_VX:
432 case RISCV::VSRA_VI:
433 case RISCV::VSRA_VV:
434 case RISCV::VSRA_VX:
435 // Vector Integer Min/Max Instructions
436 // EEW=SEW.
437 case RISCV::VMINU_VV:
438 case RISCV::VMINU_VX:
439 case RISCV::VMIN_VV:
440 case RISCV::VMIN_VX:
441 case RISCV::VMAXU_VV:
442 case RISCV::VMAXU_VX:
443 case RISCV::VMAX_VV:
444 case RISCV::VMAX_VX:
445 // Vector Single-Width Integer Multiply Instructions
446 // Source and Dest EEW=SEW.
447 case RISCV::VMUL_VV:
448 case RISCV::VMUL_VX:
449 case RISCV::VMULH_VV:
450 case RISCV::VMULH_VX:
451 case RISCV::VMULHU_VV:
452 case RISCV::VMULHU_VX:
453 case RISCV::VMULHSU_VV:
454 case RISCV::VMULHSU_VX:
455 // Vector Integer Divide Instructions
456 // EEW=SEW.
457 case RISCV::VDIVU_VV:
458 case RISCV::VDIVU_VX:
459 case RISCV::VDIV_VV:
460 case RISCV::VDIV_VX:
461 case RISCV::VREMU_VV:
462 case RISCV::VREMU_VX:
463 case RISCV::VREM_VV:
464 case RISCV::VREM_VX:
465 // Vector Single-Width Integer Multiply-Add Instructions
466 // EEW=SEW.
467 case RISCV::VMACC_VV:
468 case RISCV::VMACC_VX:
469 case RISCV::VNMSAC_VV:
470 case RISCV::VNMSAC_VX:
471 case RISCV::VMADD_VV:
472 case RISCV::VMADD_VX:
473 case RISCV::VNMSUB_VV:
474 case RISCV::VNMSUB_VX:
475 // Vector Integer Merge Instructions
476 // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
477 // EEW=SEW, except the mask operand has EEW=1. Mask operand is handled
478 // before this switch.
479 case RISCV::VMERGE_VIM:
480 case RISCV::VMERGE_VVM:
481 case RISCV::VMERGE_VXM:
482 case RISCV::VADC_VIM:
483 case RISCV::VADC_VVM:
484 case RISCV::VADC_VXM:
485 case RISCV::VSBC_VVM:
486 case RISCV::VSBC_VXM:
487 // Vector Integer Move Instructions
488 // Vector Fixed-Point Arithmetic Instructions
489 // Vector Single-Width Saturating Add and Subtract
490 // Vector Single-Width Averaging Add and Subtract
491 // EEW=SEW.
492 case RISCV::VMV_V_I:
493 case RISCV::VMV_V_V:
494 case RISCV::VMV_V_X:
495 case RISCV::VSADDU_VI:
496 case RISCV::VSADDU_VV:
497 case RISCV::VSADDU_VX:
498 case RISCV::VSADD_VI:
499 case RISCV::VSADD_VV:
500 case RISCV::VSADD_VX:
501 case RISCV::VSSUBU_VV:
502 case RISCV::VSSUBU_VX:
503 case RISCV::VSSUB_VV:
504 case RISCV::VSSUB_VX:
505 case RISCV::VAADDU_VV:
506 case RISCV::VAADDU_VX:
507 case RISCV::VAADD_VV:
508 case RISCV::VAADD_VX:
509 case RISCV::VASUBU_VV:
510 case RISCV::VASUBU_VX:
511 case RISCV::VASUB_VV:
512 case RISCV::VASUB_VX:
513 // Vector Single-Width Fractional Multiply with Rounding and Saturation
514 // EEW=SEW. The instruction produces 2*SEW product internally but
515 // saturates to fit into SEW bits.
516 case RISCV::VSMUL_VV:
517 case RISCV::VSMUL_VX:
518 // Vector Single-Width Scaling Shift Instructions
519 // EEW=SEW.
520 case RISCV::VSSRL_VI:
521 case RISCV::VSSRL_VV:
522 case RISCV::VSSRL_VX:
523 case RISCV::VSSRA_VI:
524 case RISCV::VSSRA_VV:
525 case RISCV::VSSRA_VX:
526 // Vector Permutation Instructions
527 // Integer Scalar Move Instructions
528 // Floating-Point Scalar Move Instructions
529 // EEW=SEW.
530 case RISCV::VMV_X_S:
531 case RISCV::VMV_S_X:
532 case RISCV::VFMV_F_S:
533 case RISCV::VFMV_S_F:
534 // Vector Slide Instructions
535 // EEW=SEW.
536 case RISCV::VSLIDEUP_VI:
537 case RISCV::VSLIDEUP_VX:
538 case RISCV::VSLIDEDOWN_VI:
539 case RISCV::VSLIDEDOWN_VX:
540 case RISCV::VSLIDE1UP_VX:
541 case RISCV::VFSLIDE1UP_VF:
542 case RISCV::VSLIDE1DOWN_VX:
543 case RISCV::VFSLIDE1DOWN_VF:
544 // Vector Register Gather Instructions
545 // EEW=SEW. For mask operand, EEW=1.
546 case RISCV::VRGATHER_VI:
547 case RISCV::VRGATHER_VV:
548 case RISCV::VRGATHER_VX:
549 // Vector Element Index Instruction
550 case RISCV::VID_V:
551 // Vector Single-Width Floating-Point Add/Subtract Instructions
552 case RISCV::VFADD_VF:
553 case RISCV::VFADD_VV:
554 case RISCV::VFSUB_VF:
555 case RISCV::VFSUB_VV:
556 case RISCV::VFRSUB_VF:
557 // Vector Single-Width Floating-Point Multiply/Divide Instructions
558 case RISCV::VFMUL_VF:
559 case RISCV::VFMUL_VV:
560 case RISCV::VFDIV_VF:
561 case RISCV::VFDIV_VV:
562 case RISCV::VFRDIV_VF:
563 // Vector Single-Width Floating-Point Fused Multiply-Add Instructions
564 case RISCV::VFMACC_VV:
565 case RISCV::VFMACC_VF:
566 case RISCV::VFNMACC_VV:
567 case RISCV::VFNMACC_VF:
568 case RISCV::VFMSAC_VV:
569 case RISCV::VFMSAC_VF:
570 case RISCV::VFNMSAC_VV:
571 case RISCV::VFNMSAC_VF:
572 case RISCV::VFMADD_VV:
573 case RISCV::VFMADD_VF:
574 case RISCV::VFNMADD_VV:
575 case RISCV::VFNMADD_VF:
576 case RISCV::VFMSUB_VV:
577 case RISCV::VFMSUB_VF:
578 case RISCV::VFNMSUB_VV:
579 case RISCV::VFNMSUB_VF:
580 // Vector Floating-Point Square-Root Instruction
581 case RISCV::VFSQRT_V:
582 // Vector Floating-Point Reciprocal Square-Root Estimate Instruction
583 case RISCV::VFRSQRT7_V:
584 // Vector Floating-Point Reciprocal Estimate Instruction
585 case RISCV::VFREC7_V:
586 // Vector Floating-Point MIN/MAX Instructions
587 case RISCV::VFMIN_VF:
588 case RISCV::VFMIN_VV:
589 case RISCV::VFMAX_VF:
590 case RISCV::VFMAX_VV:
591 // Vector Floating-Point Sign-Injection Instructions
592 case RISCV::VFSGNJ_VF:
593 case RISCV::VFSGNJ_VV:
594 case RISCV::VFSGNJN_VV:
595 case RISCV::VFSGNJN_VF:
596 case RISCV::VFSGNJX_VF:
597 case RISCV::VFSGNJX_VV:
598 // Vector Floating-Point Classify Instruction
599 case RISCV::VFCLASS_V:
600 // Vector Floating-Point Move Instruction
601 case RISCV::VFMV_V_F:
602 // Single-Width Floating-Point/Integer Type-Convert Instructions
603 case RISCV::VFCVT_XU_F_V:
604 case RISCV::VFCVT_X_F_V:
605 case RISCV::VFCVT_RTZ_XU_F_V:
606 case RISCV::VFCVT_RTZ_X_F_V:
607 case RISCV::VFCVT_F_XU_V:
608 case RISCV::VFCVT_F_X_V:
609 // Vector Floating-Point Merge Instruction
610 case RISCV::VFMERGE_VFM:
611 // Vector count population in mask vcpop.m
612 // vfirst find-first-set mask bit
613 case RISCV::VCPOP_M:
614 case RISCV::VFIRST_M:
615 // Vector Bit-manipulation Instructions (Zvbb)
616 // Vector And-Not
617 case RISCV::VANDN_VV:
618 case RISCV::VANDN_VX:
619 // Vector Reverse Bits in Elements
620 case RISCV::VBREV_V:
621 // Vector Reverse Bits in Bytes
622 case RISCV::VBREV8_V:
623 // Vector Reverse Bytes
624 case RISCV::VREV8_V:
625 // Vector Count Leading Zeros
626 case RISCV::VCLZ_V:
627 // Vector Count Trailing Zeros
628 case RISCV::VCTZ_V:
629 // Vector Population Count
630 case RISCV::VCPOP_V:
631 // Vector Rotate Left
632 case RISCV::VROL_VV:
633 case RISCV::VROL_VX:
634 // Vector Rotate Right
635 case RISCV::VROR_VI:
636 case RISCV::VROR_VV:
637 case RISCV::VROR_VX:
638 // Vector Carry-less Multiplication Instructions (Zvbc)
639 // Vector Carry-less Multiply
640 case RISCV::VCLMUL_VV:
641 case RISCV::VCLMUL_VX:
642 // Vector Carry-less Multiply Return High Half
643 case RISCV::VCLMULH_VV:
644 case RISCV::VCLMULH_VX:
645
646 // Zvabd
647 case RISCV::VABD_VV:
648 case RISCV::VABD_VX:
649 case RISCV::VABDU_VV:
650 case RISCV::VABDU_VX:
651
652 // Zvzip
653 case RISCV::VZIP_VV:
654 case RISCV::VUNZIPE_V:
655 case RISCV::VUNZIPO_V:
656 case RISCV::VPAIRE_VV:
657 case RISCV::VPAIRO_VV:
658 return MILog2SEW;
659
660 // Vector Widening Shift Left Logical (Zvbb)
661 case RISCV::VWSLL_VI:
662 case RISCV::VWSLL_VX:
663 case RISCV::VWSLL_VV:
664 // Vector Widening Integer Add/Subtract
665 // Def uses EEW=2*SEW . Operands use EEW=SEW.
666 case RISCV::VWADDU_VV:
667 case RISCV::VWADDU_VX:
668 case RISCV::VWSUBU_VV:
669 case RISCV::VWSUBU_VX:
670 case RISCV::VWADD_VV:
671 case RISCV::VWADD_VX:
672 case RISCV::VWSUB_VV:
673 case RISCV::VWSUB_VX:
674 // Vector Widening Integer Multiply Instructions
675 // Destination EEW=2*SEW. Source EEW=SEW.
676 case RISCV::VWMUL_VV:
677 case RISCV::VWMUL_VX:
678 case RISCV::VWMULSU_VV:
679 case RISCV::VWMULSU_VX:
680 case RISCV::VWMULU_VV:
681 case RISCV::VWMULU_VX:
682 // Vector Widening Integer Multiply-Add Instructions
683 // Destination EEW=2*SEW. Source EEW=SEW.
684 // A SEW-bit*SEW-bit multiply of the sources forms a 2*SEW-bit value, which
685 // is then added to the 2*SEW-bit Dest. These instructions never have a
686 // passthru operand.
687 case RISCV::VWMACCU_VV:
688 case RISCV::VWMACCU_VX:
689 case RISCV::VWMACC_VV:
690 case RISCV::VWMACC_VX:
691 case RISCV::VWMACCSU_VV:
692 case RISCV::VWMACCSU_VX:
693 case RISCV::VWMACCUS_VX:
694 // Vector Widening Floating-Point Fused Multiply-Add Instructions
695 case RISCV::VFWMACC_VF:
696 case RISCV::VFWMACC_VV:
697 case RISCV::VFWNMACC_VF:
698 case RISCV::VFWNMACC_VV:
699 case RISCV::VFWMSAC_VF:
700 case RISCV::VFWMSAC_VV:
701 case RISCV::VFWNMSAC_VF:
702 case RISCV::VFWNMSAC_VV:
703 case RISCV::VFWMACCBF16_VV:
704 case RISCV::VFWMACCBF16_VF:
705 // Vector Widening Floating-Point Add/Subtract Instructions
706 // Dest EEW=2*SEW. Source EEW=SEW.
707 case RISCV::VFWADD_VV:
708 case RISCV::VFWADD_VF:
709 case RISCV::VFWSUB_VV:
710 case RISCV::VFWSUB_VF:
711 // Vector Widening Floating-Point Multiply
712 case RISCV::VFWMUL_VF:
713 case RISCV::VFWMUL_VV:
714 // Widening Floating-Point/Integer Type-Convert Instructions
715 case RISCV::VFWCVT_XU_F_V:
716 case RISCV::VFWCVT_X_F_V:
717 case RISCV::VFWCVT_RTZ_XU_F_V:
718 case RISCV::VFWCVT_RTZ_X_F_V:
719 case RISCV::VFWCVT_F_XU_V:
720 case RISCV::VFWCVT_F_X_V:
721 case RISCV::VFWCVT_F_F_V:
722 case RISCV::VFWCVTBF16_F_F_V:
723 // Zvabd
724 case RISCV::VWABDA_VV:
725 case RISCV::VWABDA_VX:
726 case RISCV::VWABDAU_VV:
727 case RISCV::VWABDAU_VX:
728 return IsMODef ? MILog2SEW + 1 : MILog2SEW;
729
730 // Def and Op1 uses EEW=2*SEW. Op2 uses EEW=SEW.
731 case RISCV::VWADDU_WV:
732 case RISCV::VWADDU_WX:
733 case RISCV::VWSUBU_WV:
734 case RISCV::VWSUBU_WX:
735 case RISCV::VWADD_WV:
736 case RISCV::VWADD_WX:
737 case RISCV::VWSUB_WV:
738 case RISCV::VWSUB_WX:
739 // Vector Widening Floating-Point Add/Subtract Instructions
740 case RISCV::VFWADD_WF:
741 case RISCV::VFWADD_WV:
742 case RISCV::VFWSUB_WF:
743 case RISCV::VFWSUB_WV: {
744 bool IsOp1 = (HasPassthru && !IsTied) ? OpIdx == 2 : OpIdx == 1;
745 bool TwoTimes = IsMODef || IsOp1;
746 return TwoTimes ? MILog2SEW + 1 : MILog2SEW;
747 }
748
749 // Vector Integer Extension
750 case RISCV::VZEXT_VF2:
751 case RISCV::VSEXT_VF2:
752 return getIntegerExtensionOperandEEW(Factor: 2, MI, OpIdx);
753 case RISCV::VZEXT_VF4:
754 case RISCV::VSEXT_VF4:
755 return getIntegerExtensionOperandEEW(Factor: 4, MI, OpIdx);
756 case RISCV::VZEXT_VF8:
757 case RISCV::VSEXT_VF8:
758 return getIntegerExtensionOperandEEW(Factor: 8, MI, OpIdx);
759
760 // Vector Narrowing Integer Right Shift Instructions
761 // Destination EEW=SEW, Op 1 has EEW=2*SEW. Op2 has EEW=SEW
762 case RISCV::VNSRL_WX:
763 case RISCV::VNSRL_WI:
764 case RISCV::VNSRL_WV:
765 case RISCV::VNSRA_WI:
766 case RISCV::VNSRA_WV:
767 case RISCV::VNSRA_WX:
768 // Vector Narrowing Fixed-Point Clip Instructions
769 // Destination and Op1 EEW=SEW. Op2 EEW=2*SEW.
770 case RISCV::VNCLIPU_WI:
771 case RISCV::VNCLIPU_WV:
772 case RISCV::VNCLIPU_WX:
773 case RISCV::VNCLIP_WI:
774 case RISCV::VNCLIP_WV:
775 case RISCV::VNCLIP_WX:
776 // Narrowing Floating-Point/Integer Type-Convert Instructions
777 case RISCV::VFNCVT_XU_F_W:
778 case RISCV::VFNCVT_X_F_W:
779 case RISCV::VFNCVT_RTZ_XU_F_W:
780 case RISCV::VFNCVT_RTZ_X_F_W:
781 case RISCV::VFNCVT_F_XU_W:
782 case RISCV::VFNCVT_F_X_W:
783 case RISCV::VFNCVT_F_F_W:
784 case RISCV::VFNCVT_ROD_F_F_W:
785 case RISCV::VFNCVTBF16_F_F_W: {
786 assert(!IsTied);
787 bool IsOp1 = HasPassthru ? OpIdx == 2 : OpIdx == 1;
788 bool TwoTimes = IsOp1;
789 return TwoTimes ? MILog2SEW + 1 : MILog2SEW;
790 }
791
792 // Vector Mask Instructions
793 // Vector Mask-Register Logical Instructions
794 // vmsbf.m set-before-first mask bit
795 // vmsif.m set-including-first mask bit
796 // vmsof.m set-only-first mask bit
797 // EEW=1
798 // We handle the cases when operand is a v0 mask operand above the switch,
799 // but these instructions may use non-v0 mask operands and need to be handled
800 // specifically.
801 case RISCV::VMAND_MM:
802 case RISCV::VMNAND_MM:
803 case RISCV::VMANDN_MM:
804 case RISCV::VMXOR_MM:
805 case RISCV::VMOR_MM:
806 case RISCV::VMNOR_MM:
807 case RISCV::VMORN_MM:
808 case RISCV::VMXNOR_MM:
809 case RISCV::VMSBF_M:
810 case RISCV::VMSIF_M:
811 case RISCV::VMSOF_M: {
812 return MILog2SEW;
813 }
814
815 // Vector Compress Instruction
816 // EEW=SEW, except the mask operand has EEW=1. Mask operand is not handled
817 // before this switch.
818 case RISCV::VCOMPRESS_VM:
819 return OpIdx == 3 ? 0 : MILog2SEW;
820
821 // Vector Iota Instruction
822 // EEW=SEW, except the mask operand has EEW=1. Mask operand is not handled
823 // before this switch.
824 case RISCV::VIOTA_M: {
825 if (IsMODef || OpIdx == 1)
826 return MILog2SEW;
827 return 0;
828 }
829
830 // Vector Integer Compare Instructions
831 // Dest EEW=1. Source EEW=SEW.
832 case RISCV::VMSEQ_VI:
833 case RISCV::VMSEQ_VV:
834 case RISCV::VMSEQ_VX:
835 case RISCV::VMSNE_VI:
836 case RISCV::VMSNE_VV:
837 case RISCV::VMSNE_VX:
838 case RISCV::VMSLTU_VV:
839 case RISCV::VMSLTU_VX:
840 case RISCV::VMSLT_VV:
841 case RISCV::VMSLT_VX:
842 case RISCV::VMSLEU_VV:
843 case RISCV::VMSLEU_VI:
844 case RISCV::VMSLEU_VX:
845 case RISCV::VMSLE_VV:
846 case RISCV::VMSLE_VI:
847 case RISCV::VMSLE_VX:
848 case RISCV::VMSGTU_VI:
849 case RISCV::VMSGTU_VX:
850 case RISCV::VMSGT_VI:
851 case RISCV::VMSGT_VX:
852 // Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions
853 // Dest EEW=1. Source EEW=SEW. Mask source operand handled above this switch.
854 case RISCV::VMADC_VIM:
855 case RISCV::VMADC_VVM:
856 case RISCV::VMADC_VXM:
857 case RISCV::VMSBC_VVM:
858 case RISCV::VMSBC_VXM:
859 // Dest EEW=1. Source EEW=SEW.
860 case RISCV::VMADC_VV:
861 case RISCV::VMADC_VI:
862 case RISCV::VMADC_VX:
863 case RISCV::VMSBC_VV:
864 case RISCV::VMSBC_VX:
865 // 13.13. Vector Floating-Point Compare Instructions
866 // Dest EEW=1. Source EEW=SEW
867 case RISCV::VMFEQ_VF:
868 case RISCV::VMFEQ_VV:
869 case RISCV::VMFNE_VF:
870 case RISCV::VMFNE_VV:
871 case RISCV::VMFLT_VF:
872 case RISCV::VMFLT_VV:
873 case RISCV::VMFLE_VF:
874 case RISCV::VMFLE_VV:
875 case RISCV::VMFGT_VF:
876 case RISCV::VMFGE_VF: {
877 if (IsMODef)
878 return 0;
879 return MILog2SEW;
880 }
881
882 // Vector Reduction Operations
883 // Vector Single-Width Integer Reduction Instructions
884 case RISCV::VREDAND_VS:
885 case RISCV::VREDMAX_VS:
886 case RISCV::VREDMAXU_VS:
887 case RISCV::VREDMIN_VS:
888 case RISCV::VREDMINU_VS:
889 case RISCV::VREDOR_VS:
890 case RISCV::VREDSUM_VS:
891 case RISCV::VREDXOR_VS:
892 // Vector Single-Width Floating-Point Reduction Instructions
893 case RISCV::VFREDMAX_VS:
894 case RISCV::VFREDMIN_VS:
895 case RISCV::VFREDOSUM_VS:
896 case RISCV::VFREDUSUM_VS: {
897 return MILog2SEW;
898 }
899
900 // Vector Widening Integer Reduction Instructions
901 // The Dest and VS1 read only element 0 for the vector register. Return
902 // 2*EEW for these. VS2 has EEW=SEW and EMUL=LMUL.
903 case RISCV::VWREDSUM_VS:
904 case RISCV::VWREDSUMU_VS:
905 // Vector Widening Floating-Point Reduction Instructions
906 case RISCV::VFWREDOSUM_VS:
907 case RISCV::VFWREDUSUM_VS: {
908 bool TwoTimes = IsMODef || OpIdx == 3;
909 return TwoTimes ? MILog2SEW + 1 : MILog2SEW;
910 }
911
912 // Vector Register Gather with 16-bit Index Elements Instruction
913 // Dest and source data EEW=SEW. Index vector EEW=16.
914 case RISCV::VRGATHEREI16_VV: {
915 if (OpIdx == 2)
916 return 4;
917 return MILog2SEW;
918 }
919
920 default:
921 return std::nullopt;
922 }
923}
924
925static std::optional<OperandInfo> getOperandInfo(const MachineInstr &MI,
926 unsigned OpIdx) {
927 const RISCVVPseudosTable::PseudoInfo *RVV =
928 RISCVVPseudosTable::getPseudoInfo(Pseudo: MI.getOpcode());
929 assert(RVV && "Could not find MI in PseudoTable");
930
931 std::optional<unsigned> Log2EEW = getOperandLog2EEW(MI, OpIdx);
932 if (!Log2EEW)
933 return std::nullopt;
934
935 switch (RVV->BaseInstr) {
936 // Vector Reduction Operations
937 // Vector Single-Width Integer Reduction Instructions
938 // Vector Widening Integer Reduction Instructions
939 // Vector Widening Floating-Point Reduction Instructions
940 // The Dest and VS1 only read element 0 of the vector register. Return just
941 // the EEW for these.
942 case RISCV::VREDAND_VS:
943 case RISCV::VREDMAX_VS:
944 case RISCV::VREDMAXU_VS:
945 case RISCV::VREDMIN_VS:
946 case RISCV::VREDMINU_VS:
947 case RISCV::VREDOR_VS:
948 case RISCV::VREDSUM_VS:
949 case RISCV::VREDXOR_VS:
950 case RISCV::VWREDSUM_VS:
951 case RISCV::VWREDSUMU_VS:
952 case RISCV::VFWREDOSUM_VS:
953 case RISCV::VFWREDUSUM_VS:
954 if (OpIdx != 2)
955 return OperandInfo(*Log2EEW);
956 break;
957
958 // Zvzip - vzip.vv interleaves two half-LMUL vectors into an LMUL result with
959 // the same SEW. The vtype LMUL describes the result, so only the two source
960 // operands have half the instruction's EMUL.
961 case RISCV::VZIP_VV: {
962 auto EMUL = getEMULEqualsEEWDivSEWTimesLMUL(Log2EEW: *Log2EEW, MI);
963 if (OpIdx == 2 || OpIdx == 3)
964 EMUL = halfEMUL(EMUL);
965 return OperandInfo(EMUL, *Log2EEW);
966 }
967 // Zvzip - vunzipe.v / vunzipo.v split a 2*LMUL vector into LMUL even/odd
968 // elements with the same SEW. The source (and passthru tied to dest which is
969 // also LMUL sized - so only the vs2 source) has 2 * EMUL.
970 case RISCV::VUNZIPE_V:
971 case RISCV::VUNZIPO_V: {
972 auto EMUL = getEMULEqualsEEWDivSEWTimesLMUL(Log2EEW: *Log2EEW, MI);
973 if (OpIdx == 2)
974 EMUL = doubleEMUL(EMUL);
975 return OperandInfo(EMUL, *Log2EEW);
976 }
977 };
978
979 // All others have EMUL=EEW/SEW*LMUL
980 return OperandInfo(getEMULEqualsEEWDivSEWTimesLMUL(Log2EEW: *Log2EEW, MI), *Log2EEW);
981}
982
983static bool isTupleInsertInstr(const MachineInstr &MI);
984
985/// Return true if we can reason about demanded VLs elementwise for \p MI.
986bool RISCVVLOptimizerImpl::isSupportedInstr(const MachineInstr &MI) const {
987 if (MI.isPHI() || MI.isFullCopy() || isTupleInsertInstr(MI))
988 return true;
989
990 unsigned RVVOpc = RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode());
991 if (!RVVOpc)
992 return false;
993
994 assert(!(MI.getNumExplicitDefs() == 0 && !MI.mayStore() &&
995 !RISCVII::elementsDependOnVL(TII->get(RVVOpc).TSFlags)) &&
996 "No defs but elements don't depend on VL?");
997
998 // TODO: Reduce vl for vmv.s.x and vfmv.s.f. Currently this introduces more vl
999 // toggles, we need to extend PRE in RISCVInsertVSETVLI first.
1000 if (RVVOpc == RISCV::VMV_S_X || RVVOpc == RISCV::VFMV_S_F)
1001 return false;
1002
1003 if (RISCVII::elementsDependOnVL(TSFlags: TII->get(Opcode: RVVOpc).TSFlags))
1004 return false;
1005
1006 if (MI.mayStore())
1007 return false;
1008
1009 return true;
1010}
1011
1012/// Return true if operand \p OpIdx of \p MI is a vector operand but is used as
1013/// a scalar operand.
1014static bool isVectorOpUsedAsScalarOp(const MachineInstr &MI, unsigned OpIdx) {
1015 const RISCVVPseudosTable::PseudoInfo *RVV =
1016 RISCVVPseudosTable::getPseudoInfo(Pseudo: MI.getOpcode());
1017
1018 if (!RVV)
1019 return false;
1020
1021 switch (RVV->BaseInstr) {
1022 // Reductions only use vs1[0] of vs1
1023 case RISCV::VREDAND_VS:
1024 case RISCV::VREDMAX_VS:
1025 case RISCV::VREDMAXU_VS:
1026 case RISCV::VREDMIN_VS:
1027 case RISCV::VREDMINU_VS:
1028 case RISCV::VREDOR_VS:
1029 case RISCV::VREDSUM_VS:
1030 case RISCV::VREDXOR_VS:
1031 case RISCV::VWREDSUM_VS:
1032 case RISCV::VWREDSUMU_VS:
1033 case RISCV::VFREDMAX_VS:
1034 case RISCV::VFREDMIN_VS:
1035 case RISCV::VFREDOSUM_VS:
1036 case RISCV::VFREDUSUM_VS:
1037 case RISCV::VFWREDOSUM_VS:
1038 case RISCV::VFWREDUSUM_VS:
1039 return OpIdx == 3;
1040 case RISCV::VMV_X_S:
1041 case RISCV::VFMV_F_S:
1042 return OpIdx == 1;
1043 default:
1044 return false;
1045 }
1046}
1047
1048bool RISCVVLOptimizerImpl::isCandidate(const MachineInstr &MI) const {
1049 const MCInstrDesc &Desc = MI.getDesc();
1050 if (!RISCVII::hasVLOp(TSFlags: Desc.TSFlags) || !RISCVII::hasSEWOp(TSFlags: Desc.TSFlags))
1051 return false;
1052
1053 if (MI.getNumExplicitDefs() != 1)
1054 return false;
1055
1056 // Some instructions have implicit defs e.g. $vxsat. If they might be read
1057 // later then we can't reduce VL.
1058 if (!MI.allImplicitDefsAreDead()) {
1059 LLVM_DEBUG(dbgs() << "Not a candidate because has non-dead implicit def\n");
1060 return false;
1061 }
1062
1063 if (MI.mayRaiseFPException()) {
1064 LLVM_DEBUG(dbgs() << "Not a candidate because may raise FP exception\n");
1065 return false;
1066 }
1067
1068 for (const MachineMemOperand *MMO : MI.memoperands()) {
1069 if (MMO->isVolatile()) {
1070 LLVM_DEBUG(dbgs() << "Not a candidate because contains volatile MMO\n");
1071 return false;
1072 }
1073 }
1074
1075 if (!isSupportedInstr(MI)) {
1076 LLVM_DEBUG(dbgs() << "Not a candidate due to unsupported instruction: "
1077 << MI);
1078 return false;
1079 }
1080
1081 assert(!RISCVII::elementsDependOnVL(
1082 TII->get(RISCV::getRVVMCOpcode(MI.getOpcode())).TSFlags) &&
1083 "Instruction shouldn't be supported if elements depend on VL");
1084
1085 assert(RISCVRI::isVRegClass(
1086 MRI->getRegClass(MI.getOperand(0).getReg())->TSFlags) &&
1087 "All supported instructions produce a vector register result");
1088
1089 LLVM_DEBUG(dbgs() << "Found a candidate for VL reduction: " << MI << "\n");
1090 return true;
1091}
1092
1093/// Given a vslidedown.vx like:
1094///
1095/// %slideamt = ADDI %x, -1
1096/// %v = PseudoVSLIDEDOWN_VX %passthru, %src, %slideamt, avl=1
1097///
1098/// %v will only read the first %slideamt + 1 lanes of %src, which = %x.
1099/// This is a common case when lowering extractelement.
1100///
1101/// Note that if %x is 0, %slideamt will be all ones. In this case %src will be
1102/// completely slid down and none of its lanes will be read (since %slideamt is
1103/// greater than the largest VLMAX of 65536) so we can demand any minimum VL.
1104static std::optional<DemandedVL>
1105getMinimumVLForVSLIDEDOWN_VX(const MachineInstr &MI, unsigned OpIdx,
1106 const MachineRegisterInfo *MRI) {
1107 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode()) != RISCV::VSLIDEDOWN_VX)
1108 return std::nullopt;
1109 // We're looking at what lanes are used from the src operand.
1110 if (OpIdx != 2)
1111 return std::nullopt;
1112 // For now, the AVL must be 1.
1113 const MachineOperand &AVL = MI.getOperand(i: 4);
1114 if (!AVL.isImm() || AVL.getImm() != 1)
1115 return std::nullopt;
1116 // The slide amount must be %x - 1.
1117 const MachineOperand &SlideAmt = MI.getOperand(i: 3);
1118 if (!SlideAmt.getReg().isVirtual())
1119 return std::nullopt;
1120 MachineInstr *SlideAmtDef = MRI->getVRegDef(Reg: SlideAmt.getReg());
1121 if (!SlideAmtDef || SlideAmtDef->getOpcode() != RISCV::ADDI ||
1122 !SlideAmtDef->getOperand(i: 1).isReg() ||
1123 SlideAmtDef->getOperand(i: 2).getImm() != -AVL.getImm() ||
1124 !SlideAmtDef->getOperand(i: 1).getReg().isVirtual())
1125 return std::nullopt;
1126 return SlideAmtDef->getOperand(i: 1);
1127}
1128
1129DemandedVL RISCVVLOptimizerImpl::getMinimumVLForUser(const MachineInstr &UserMI,
1130 unsigned OpIdx) const {
1131 const MachineOperand &UserOp = UserMI.getOperand(i: OpIdx);
1132 const MCInstrDesc &Desc = UserMI.getDesc();
1133
1134 if (UserMI.isPHI() || UserMI.isFullCopy() || isTupleInsertInstr(MI: UserMI))
1135 return DemandedVLs.lookup(Key: &UserMI);
1136
1137 if (!RISCVII::hasVLOp(TSFlags: Desc.TSFlags) || !RISCVII::hasSEWOp(TSFlags: Desc.TSFlags)) {
1138 LLVM_DEBUG(dbgs() << " Abort due to lack of VL, assume that"
1139 " use VLMAX\n");
1140 return DemandedVL::vlmax();
1141 }
1142
1143 if (auto VL = getMinimumVLForVSLIDEDOWN_VX(MI: UserMI, OpIdx, MRI))
1144 return *VL;
1145
1146 unsigned RVVOpc = RISCV::getRVVMCOpcode(RVVPseudoOpcode: UserMI.getOpcode());
1147 bool IsVUNZIP = RVVOpc == RISCV::VUNZIPE_V || RVVOpc == RISCV::VUNZIPO_V;
1148 bool IsVZIP = RVVOpc == RISCV::VZIP_VV;
1149 if (!IsVUNZIP && RISCVII::readsPastVL(TSFlags: TII->get(Opcode: RVVOpc).TSFlags)) {
1150 LLVM_DEBUG(dbgs() << " Abort because used by unsafe instruction\n");
1151 return DemandedVL::vlmax();
1152 }
1153
1154 unsigned VLOpNum = RISCVII::getVLOpNum(Desc);
1155 const MachineOperand &VLOp = UserMI.getOperand(i: VLOpNum);
1156 // Looking for an immediate or a register VL that isn't X0.
1157 assert((!VLOp.isReg() || VLOp.getReg() != RISCV::X0) &&
1158 "Did not expect X0 VL");
1159
1160 // If the user is a passthru it will read the elements past VL, so
1161 // abort if any of the elements past VL are demanded.
1162 if (UserOp.isTied()) {
1163 assert(OpIdx == UserMI.getNumExplicitDefs() &&
1164 RISCVII::isFirstDefTiedToFirstUse(UserMI.getDesc()));
1165 if (!RISCV::isVLKnownLE(MRI: *MRI, LHS: DemandedVLs.lookup(Key: &UserMI).VL, RHS: VLOp)) {
1166 LLVM_DEBUG(dbgs() << " Abort because user is passthru in "
1167 "instruction with demanded tail\n");
1168 return DemandedVL::vlmax();
1169 }
1170 }
1171
1172 // Instructions like reductions may use a vector register as a scalar
1173 // register. In this case, we should treat it as only reading the first lane.
1174 if (isVectorOpUsedAsScalarOp(MI: UserMI, OpIdx)) {
1175 LLVM_DEBUG(dbgs() << " Used this operand as a scalar operand\n");
1176 return MachineOperand::CreateImm(Val: 1);
1177 }
1178
1179 // If we know the demanded VL of UserMI, then we can reduce the VL it
1180 // requires.
1181 DemandedVL MinimumVL = VLOp;
1182 if (RISCV::isVLKnownLE(MRI: *MRI, LHS: DemandedVLs.lookup(Key: &UserMI).VL, RHS: VLOp))
1183 MinimumVL = DemandedVLs.lookup(Key: &UserMI);
1184
1185 if (IsVUNZIP && OpIdx == 2)
1186 MinimumVL = doubleVL(MinimumVL);
1187 if (IsVZIP && (OpIdx == 2 || OpIdx == 3))
1188 MinimumVL = halfVL(MinimumVL, Ceil: OpIdx == 2);
1189
1190 return MinimumVL;
1191}
1192
1193/// Return true if MI is an instruction used for assembling registers
1194/// for segmented store instructions, namely, RISCVISD::TUPLE_INSERT.
1195/// Currently it's lowered to INSERT_SUBREG.
1196static bool isTupleInsertInstr(const MachineInstr &MI) {
1197 if (!MI.isInsertSubreg())
1198 return false;
1199
1200 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1201 const TargetRegisterClass *DstRC = MRI.getRegClass(Reg: MI.getOperand(i: 0).getReg());
1202 const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo();
1203 if (!RISCVRI::isVRegClass(TSFlags: DstRC->TSFlags))
1204 return false;
1205 unsigned NF = RISCVRI::getNF(TSFlags: DstRC->TSFlags);
1206 if (NF < 2)
1207 return false;
1208
1209 // Check whether INSERT_SUBREG has the correct subreg index for tuple inserts.
1210 auto VLMul = RISCVRI::getLMul(TSFlags: DstRC->TSFlags);
1211 unsigned SubRegIdx = MI.getOperand(i: 3).getImm();
1212 [[maybe_unused]] auto [LMul, IsFractional] = RISCVVType::decodeVLMUL(VLMul);
1213 assert(!IsFractional && "unexpected LMUL for tuple register classes");
1214 return TRI->getSubRegIdxSize(Idx: SubRegIdx) == RISCV::RVVBitsPerBlock * LMul;
1215}
1216
1217static bool isSegmentedStoreInstr(const MachineInstr &MI) {
1218 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
1219 case VSSEG_CASES(8):
1220 case VSSSEG_CASES(8):
1221 case VSUXSEG_CASES(8):
1222 case VSOXSEG_CASES(8):
1223 case VSSEG_CASES(16):
1224 case VSSSEG_CASES(16):
1225 case VSUXSEG_CASES(16):
1226 case VSOXSEG_CASES(16):
1227 case VSSEG_CASES(32):
1228 case VSSSEG_CASES(32):
1229 case VSUXSEG_CASES(32):
1230 case VSOXSEG_CASES(32):
1231 case VSSEG_CASES(64):
1232 case VSSSEG_CASES(64):
1233 case VSUXSEG_CASES(64):
1234 case VSOXSEG_CASES(64):
1235 return true;
1236 default:
1237 return false;
1238 }
1239}
1240
1241bool RISCVVLOptimizerImpl::checkUsers(const MachineInstr &MI) const {
1242 if (MI.isPHI() || MI.isFullCopy() || isTupleInsertInstr(MI))
1243 return true;
1244
1245 SmallSetVector<MachineOperand *, 8> OpWorklist;
1246 SmallPtrSet<const MachineInstr *, 4> PHISeen;
1247 for (auto &UserOp : MRI->use_operands(Reg: MI.getOperand(i: 0).getReg()))
1248 OpWorklist.insert(X: &UserOp);
1249
1250 while (!OpWorklist.empty()) {
1251 MachineOperand &UserOp = *OpWorklist.pop_back_val();
1252 const MachineInstr &UserMI = *UserOp.getParent();
1253 LLVM_DEBUG(dbgs() << " Checking user: " << UserMI << "\n");
1254
1255 if (UserMI.isFullCopy() && UserMI.getOperand(i: 0).getReg().isVirtual()) {
1256 LLVM_DEBUG(dbgs() << " Peeking through uses of COPY\n");
1257 OpWorklist.insert_range(R: llvm::make_pointer_range(
1258 Range: MRI->use_operands(Reg: UserMI.getOperand(i: 0).getReg())));
1259 continue;
1260 }
1261
1262 if (isTupleInsertInstr(MI: UserMI)) {
1263 LLVM_DEBUG(dbgs().indent(4) << "Peeking through uses of INSERT_SUBREG\n");
1264 for (MachineOperand &UseOp :
1265 MRI->use_operands(Reg: UserMI.getOperand(i: 0).getReg())) {
1266 const MachineInstr &CandidateMI = *UseOp.getParent();
1267 // We should not propagate the VL if the user is not a segmented store
1268 // or another INSERT_SUBREG, since VL just works differently
1269 // between segmented operations (per-field) v.s. other RVV ops (on the
1270 // whole register group).
1271 if (!isTupleInsertInstr(MI: CandidateMI) &&
1272 !isSegmentedStoreInstr(MI: CandidateMI))
1273 return false;
1274 OpWorklist.insert(X: &UseOp);
1275 }
1276 continue;
1277 }
1278
1279 if (UserMI.isPHI()) {
1280 // Don't follow PHI cycles
1281 if (!PHISeen.insert(Ptr: &UserMI).second)
1282 continue;
1283 LLVM_DEBUG(dbgs() << " Peeking through uses of PHI\n");
1284 OpWorklist.insert_range(R: llvm::make_pointer_range(
1285 Range: MRI->use_operands(Reg: UserMI.getOperand(i: 0).getReg())));
1286 continue;
1287 }
1288
1289 if (!RISCVII::hasSEWOp(TSFlags: UserMI.getDesc().TSFlags)) {
1290 LLVM_DEBUG(dbgs() << " Abort due to lack of SEW operand\n");
1291 return false;
1292 }
1293
1294 std::optional<OperandInfo> ConsumerInfo =
1295 getOperandInfo(MI: UserMI, OpIdx: UserMI.getOperandNo(I: &UserOp));
1296 std::optional<OperandInfo> ProducerInfo = getOperandInfo(MI, OpIdx: 0);
1297 if (!ConsumerInfo || !ProducerInfo) {
1298 LLVM_DEBUG(dbgs() << " Abort due to unknown operand information.\n");
1299 LLVM_DEBUG(dbgs() << " ConsumerInfo is: " << ConsumerInfo << "\n");
1300 LLVM_DEBUG(dbgs() << " ProducerInfo is: " << ProducerInfo << "\n");
1301 return false;
1302 }
1303
1304 if (!OperandInfo::areCompatible(Def: *ProducerInfo, User: *ConsumerInfo)) {
1305 LLVM_DEBUG(
1306 dbgs()
1307 << " Abort due to incompatible information for EMUL or EEW.\n");
1308 LLVM_DEBUG(dbgs() << " ConsumerInfo is: " << ConsumerInfo << "\n");
1309 LLVM_DEBUG(dbgs() << " ProducerInfo is: " << ProducerInfo << "\n");
1310 return false;
1311 }
1312 }
1313
1314 return true;
1315}
1316
1317bool RISCVVLOptimizerImpl::tryReduceVL(MachineInstr &MI,
1318 MachineOperand CommonVL) {
1319 LLVM_DEBUG(dbgs() << "Trying to reduce VL for " << MI);
1320
1321 unsigned VLOpNum = RISCVII::getVLOpNum(Desc: MI.getDesc());
1322 MachineOperand &VLOp = MI.getOperand(i: VLOpNum);
1323
1324 assert((CommonVL.isImm() || CommonVL.getReg().isVirtual()) &&
1325 "Expected VL to be an Imm or virtual Reg");
1326
1327 // If the VL is defined by a vleff that doesn't dominate MI, try using the
1328 // vleff's AVL. It will be greater than or equal to the output VL.
1329 if (CommonVL.isReg()) {
1330 const MachineInstr *VLMI = MRI->getVRegDef(Reg: CommonVL.getReg());
1331 if (VLMI && RISCVInstrInfo::isFaultOnlyFirstLoad(MI: *VLMI)) {
1332 if (!MDT)
1333 MDT = &GetMDT();
1334 if (!MDT->dominates(A: VLMI, B: &MI))
1335 CommonVL = VLMI->getOperand(i: RISCVII::getVLOpNum(Desc: VLMI->getDesc()));
1336 }
1337 }
1338
1339 if (!RISCV::isVLKnownLE(MRI: *MRI, LHS: CommonVL, RHS: VLOp)) {
1340 LLVM_DEBUG(dbgs() << " Abort due to CommonVL not <= VLOp.\n");
1341 return false;
1342 }
1343
1344 if (CommonVL.isIdenticalTo(Other: VLOp)) {
1345 LLVM_DEBUG(
1346 dbgs() << " Abort due to CommonVL == VLOp, no point in reducing.\n");
1347 return false;
1348 }
1349
1350 if (CommonVL.isImm()) {
1351 LLVM_DEBUG(dbgs() << " Reduce VL from " << VLOp << " to "
1352 << CommonVL.getImm() << " for " << MI << "\n");
1353 VLOp.ChangeToImmediate(ImmVal: CommonVL.getImm());
1354 return true;
1355 }
1356 MachineInstr *VLMI = MRI->getVRegDef(Reg: CommonVL.getReg());
1357 if (!VLMI)
1358 return false;
1359
1360 if (!MDT)
1361 MDT = &GetMDT();
1362 auto VLDominates = [this, &VLMI](const MachineInstr &MI) {
1363 return MDT->dominates(A: VLMI, B: &MI);
1364 };
1365 if (!VLDominates(MI)) {
1366 assert(MI.getNumExplicitDefs() == 1);
1367 auto Uses = MRI->use_instructions(Reg: MI.getOperand(i: 0).getReg());
1368 auto UsesSameBB = make_filter_range(Range&: Uses, Pred: [&MI](const MachineInstr &Use) {
1369 return Use.getParent() == MI.getParent();
1370 });
1371 if (VLMI->getParent() == MI.getParent() &&
1372 all_of(Range&: UsesSameBB, P: VLDominates) &&
1373 RISCVInstrInfo::isSafeToMove(From: MI, To: std::next(x: VLMI->getIterator()))) {
1374 VLMI->getParent()->splice(Where: std::next(x: VLMI->getIterator()), Other: MI.getParent(),
1375 From: MI.getIterator());
1376 } else {
1377 LLVM_DEBUG(dbgs() << " Abort due to VL not dominating.\n");
1378 return false;
1379 }
1380 }
1381 LLVM_DEBUG(dbgs() << " Reduce VL from " << VLOp << " to "
1382 << printReg(CommonVL.getReg(), MRI->getTargetRegisterInfo())
1383 << " for " << MI << "\n");
1384
1385 // All our checks passed. We can reduce VL.
1386 VLOp.ChangeToRegister(Reg: CommonVL.getReg(), isDef: false);
1387 MRI->constrainRegClass(Reg: CommonVL.getReg(), RC: &RISCV::GPRNoX0RegClass);
1388 return true;
1389}
1390
1391static bool isPhysical(const MachineOperand &MO) {
1392 return MO.isReg() && MO.getReg().isPhysical();
1393}
1394
1395/// Look through \p MI's operands and propagate what it demands to its uses.
1396void RISCVVLOptimizerImpl::transfer(const MachineInstr &MI) {
1397 if (!isSupportedInstr(MI) || !checkUsers(MI) || any_of(Range: MI.defs(), P: isPhysical))
1398 DemandedVLs[&MI] = DemandedVL::vlmax();
1399
1400 for (const MachineOperand &MO : virtual_vec_uses(MI)) {
1401 const MachineInstr *Def = MRI->getVRegDef(Reg: MO.getReg());
1402 DemandedVL Prev = DemandedVLs[Def];
1403 DemandedVLs[Def] = DemandedVLs[Def].max(
1404 MRI: *MRI, X: getMinimumVLForUser(UserMI: MI, OpIdx: MI.getOperandNo(I: &MO)));
1405 if (DemandedVLs[Def] != Prev)
1406 Worklist.insert(X: Def);
1407 }
1408}
1409
1410bool RISCVVLOptimizerImpl::run(MachineFunction &MF) {
1411 MRI = &MF.getRegInfo();
1412
1413 const RISCVSubtarget &ST = MF.getSubtarget<RISCVSubtarget>();
1414 if (!ST.hasVInstructions())
1415 return false;
1416
1417 TII = ST.getInstrInfo();
1418
1419 assert(DemandedVLs.empty());
1420
1421 // For each instruction that defines or uses a vector, propagate the VL it
1422 // uses to its inputs.
1423 for (MachineBasicBlock *MBB : post_order(G: &MF)) {
1424 for (MachineInstr &MI : reverse(C&: *MBB)) {
1425 if (MI.isDebugInstr())
1426 continue;
1427 if (virtual_vec_defs(MI).empty() && virtual_vec_uses(MI).empty())
1428 continue;
1429 Worklist.insert(X: &MI);
1430 }
1431 }
1432
1433 while (!Worklist.empty()) {
1434 const MachineInstr *MI = Worklist.front();
1435 Worklist.remove(X: MI);
1436 transfer(MI: *MI);
1437 }
1438
1439 // Then go through and see if we can reduce the VL of any instructions to
1440 // only what's demanded.
1441 bool MadeChange = false;
1442 for (auto &[MI, VL] : DemandedVLs) {
1443 if (!isCandidate(MI: *MI))
1444 continue;
1445 if (!tryReduceVL(MI&: *const_cast<MachineInstr *>(MI), CommonVL: VL.VL))
1446 continue;
1447 MadeChange = true;
1448 }
1449
1450 DemandedVLs.clear();
1451 return MadeChange;
1452}
1453
1454bool RISCVVLOptimizerLegacy::runOnMachineFunction(MachineFunction &MF) {
1455 if (skipFunction(F: MF.getFunction()))
1456 return false;
1457
1458 auto GetMDT = [&]() -> const MachineDominatorTree & {
1459 return getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
1460 };
1461 return RISCVVLOptimizerImpl(GetMDT).run(MF);
1462}
1463
1464PreservedAnalyses
1465RISCVVLOptimizerPass::run(MachineFunction &MF,
1466 MachineFunctionAnalysisManager &MFAM) {
1467 auto GetMDT = [&]() -> const MachineDominatorTree & {
1468 return MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
1469 };
1470 bool Changed = RISCVVLOptimizerImpl(GetMDT).run(MF);
1471 if (!Changed)
1472 return PreservedAnalyses::all();
1473
1474 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1475 PA.preserveSet<CFGAnalyses>();
1476 PA.preserve<MachineRegisterClassAnalysis>();
1477 return PA;
1478}
1479