1//===- RISCVVectorPeephole.cpp - MI Vector Pseudo Peepholes ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass performs various vector pseudo peephole optimisations after
10// instruction selection.
11//
12// Currently it converts vmerge.vvm to vmv.v.v
13// PseudoVMERGE_VVM %false, %false, %true, %allonesmask, %vl, %sew
14// ->
15// PseudoVMV_V_V %false, %true, %vl, %sew
16//
17// And masked pseudos to unmasked pseudos
18// PseudoVADD_V_V_MASK %passthru, %a, %b, %allonesmask, %vl, sew, policy
19// ->
20// PseudoVADD_V_V %passthru %a, %b, %vl, sew, policy
21//
22// It also converts AVLs to VLMAX where possible
23// %vl = VLENB * something
24// PseudoVADD_V_V %passthru, %a, %b, %vl, sew, policy
25// ->
26// PseudoVADD_V_V %passthru, %a, %b, -1, sew, policy
27//
28//===----------------------------------------------------------------------===//
29
30#include "RISCV.h"
31#include "RISCVSubtarget.h"
32#include "llvm/CodeGen/MachineFunctionPass.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/RegisterClassInfo.h"
35#include "llvm/CodeGen/TargetInstrInfo.h"
36#include "llvm/CodeGen/TargetRegisterInfo.h"
37
38using namespace llvm;
39
40#define DEBUG_TYPE "riscv-vector-peephole"
41
42namespace {
43
44class RISCVVectorPeepholeImpl {
45public:
46 bool run(MachineFunction &MF);
47
48private:
49 const TargetInstrInfo *TII;
50 MachineRegisterInfo *MRI;
51 const TargetRegisterInfo *TRI;
52 const RISCVSubtarget *ST;
53 bool convertToVLMAX(MachineInstr &MI) const;
54 bool convertToWholeRegister(MachineInstr &MI) const;
55 bool convertToUnmasked(MachineInstr &MI) const;
56 bool convertAllOnesVMergeToVMv(MachineInstr &MI) const;
57 bool convertSameMaskVMergeToVMv(MachineInstr &MI);
58 bool foldUndefPassthruVMV_V_V(MachineInstr &MI);
59 bool foldVMV_V_V(MachineInstr &MI);
60 bool foldVMergeToMask(MachineInstr &MI) const;
61 bool foldVMANDToMaskedCompare(MachineInstr &MI) const;
62
63 bool hasSameEEW(const MachineInstr &User, const MachineInstr &Src) const;
64 bool isAllOnesMask(const MachineInstr *MaskDef) const;
65 std::optional<unsigned> getConstant(const MachineOperand &VL) const;
66 bool ensureDominates(ArrayRef<const MachineOperand *> Defs,
67 MachineInstr &Use) const;
68 Register
69 lookThruCopies(Register Reg, bool OneUseOnly = false,
70 SmallVectorImpl<MachineInstr *> *Copies = nullptr) const;
71};
72
73class RISCVVectorPeepholeLegacy : public MachineFunctionPass {
74public:
75 static char ID;
76
77 RISCVVectorPeepholeLegacy() : MachineFunctionPass(ID) {}
78
79 bool runOnMachineFunction(MachineFunction &MF) override;
80 MachineFunctionProperties getRequiredProperties() const override {
81 return MachineFunctionProperties().setIsSSA();
82 }
83
84 StringRef getPassName() const override {
85 return "RISC-V Vector Peephole Optimization";
86 }
87
88 void getAnalysisUsage(AnalysisUsage &AU) const override {
89 AU.setPreservesCFG();
90 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
91 MachineFunctionPass::getAnalysisUsage(AU);
92 }
93};
94
95} // namespace
96
97char RISCVVectorPeepholeLegacy::ID = 0;
98
99INITIALIZE_PASS(RISCVVectorPeepholeLegacy, DEBUG_TYPE, "RISC-V Fold Masks",
100 false, false)
101
102/// Given \p User that has an input operand with EEW=SEW, which uses the dest
103/// operand of \p Src with an unknown EEW, return true if their EEWs match.
104bool RISCVVectorPeepholeImpl::hasSameEEW(const MachineInstr &User,
105 const MachineInstr &Src) const {
106 unsigned UserLog2SEW =
107 User.getOperand(i: RISCVII::getSEWOpNum(Desc: User.getDesc())).getImm();
108 unsigned SrcLog2SEW =
109 Src.getOperand(i: RISCVII::getSEWOpNum(Desc: Src.getDesc())).getImm();
110 unsigned SrcLog2EEW = RISCV::getDestLog2EEW(
111 Desc: TII->get(Opcode: RISCV::getRVVMCOpcode(RVVPseudoOpcode: Src.getOpcode())), Log2SEW: SrcLog2SEW);
112 return SrcLog2EEW == UserLog2SEW;
113}
114
115/// Check if an operand is an immediate or a materialized ADDI $x0, imm.
116std::optional<unsigned>
117RISCVVectorPeepholeImpl::getConstant(const MachineOperand &VL) const {
118 if (VL.isImm())
119 return VL.getImm();
120
121 if (!VL.getReg().isVirtual())
122 return std::nullopt;
123 MachineInstr *Def = MRI->getVRegDef(Reg: VL.getReg());
124 if (!Def || Def->getOpcode() != RISCV::ADDI || !Def->getOperand(i: 1).isReg() ||
125 Def->getOperand(i: 1).getReg() != RISCV::X0)
126 return std::nullopt;
127 return Def->getOperand(i: 2).getImm();
128}
129
130/// Convert AVLs that are known to be VLMAX to the VLMAX sentinel.
131bool RISCVVectorPeepholeImpl::convertToVLMAX(MachineInstr &MI) const {
132 if (!RISCVII::hasVLOp(TSFlags: MI.getDesc().TSFlags) ||
133 !RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags))
134 return false;
135
136 auto LMUL = RISCVVType::decodeVLMUL(VLMul: RISCVII::getLMul(TSFlags: MI.getDesc().TSFlags));
137 // Fixed-point value, denominator=8
138 unsigned LMULFixed = LMUL.second ? (8 / LMUL.first) : 8 * LMUL.first;
139 unsigned Log2SEW = MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
140 // A Log2SEW of 0 is an operation on mask registers only
141 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
142 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
143 assert(8 * LMULFixed / SEW > 0);
144
145 // If the exact VLEN is known then we know VLMAX, check if the AVL == VLMAX.
146 MachineOperand &VL = MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
147 if (auto VLen = ST->getRealVLen(), AVL = getConstant(VL);
148 VLen && AVL && (*VLen * LMULFixed) / SEW == *AVL * 8) {
149 VL.ChangeToImmediate(ImmVal: RISCV::VLMaxSentinel);
150 return true;
151 }
152
153 // If an AVL is a VLENB that's possibly scaled to be equal to VLMAX, convert
154 // it to the VLMAX sentinel value.
155 if (!VL.isReg())
156 return false;
157 MachineInstr *Def = MRI->getVRegDef(Reg: VL.getReg());
158 if (!Def)
159 return false;
160
161 // Fixed-point value, denominator=8
162 uint64_t ScaleFixed = 8;
163 // Check if the VLENB was potentially scaled with slli/srli
164 if (Def->getOpcode() == RISCV::SLLI) {
165 assert(Def->getOperand(2).getImm() < 64);
166 ScaleFixed <<= Def->getOperand(i: 2).getImm();
167 Def = MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg());
168 } else if (Def->getOpcode() == RISCV::SRLI) {
169 assert(Def->getOperand(2).getImm() < 64);
170 ScaleFixed >>= Def->getOperand(i: 2).getImm();
171 Def = MRI->getVRegDef(Reg: Def->getOperand(i: 1).getReg());
172 }
173
174 if (!Def || Def->getOpcode() != RISCV::PseudoReadVLENB)
175 return false;
176
177 // AVL = (VLENB * Scale)
178 //
179 // VLMAX = (VLENB * 8 * LMUL) / SEW
180 //
181 // AVL == VLMAX
182 // -> VLENB * Scale == (VLENB * 8 * LMUL) / SEW
183 // -> Scale == (8 * LMUL) / SEW
184 if (ScaleFixed != 8 * LMULFixed / SEW)
185 return false;
186
187 VL.ChangeToImmediate(ImmVal: RISCV::VLMaxSentinel);
188
189 return true;
190}
191
192bool RISCVVectorPeepholeImpl::isAllOnesMask(const MachineInstr *MaskDef) const {
193 while (MaskDef->isCopy() && MaskDef->getOperand(i: 1).getReg().isVirtual())
194 MaskDef = MRI->getVRegDef(Reg: MaskDef->getOperand(i: 1).getReg());
195
196 // TODO: Check that the VMSET is the expected bitwidth? The pseudo has
197 // undefined behaviour if it's the wrong bitwidth, so we could choose to
198 // assume that it's all-ones? Same applies to its VL.
199 switch (MaskDef->getOpcode()) {
200 case RISCV::PseudoVMSET_M_B1:
201 case RISCV::PseudoVMSET_M_B2:
202 case RISCV::PseudoVMSET_M_B4:
203 case RISCV::PseudoVMSET_M_B8:
204 case RISCV::PseudoVMSET_M_B16:
205 case RISCV::PseudoVMSET_M_B32:
206 case RISCV::PseudoVMSET_M_B64:
207 return true;
208 default:
209 return false;
210 }
211}
212
213/// Convert unit strided unmasked loads and stores to whole-register equivalents
214/// to avoid the dependency on $vl and $vtype.
215///
216/// %x = PseudoVLE8_V_M1 %passthru, %ptr, %vlmax, policy
217/// PseudoVSE8_V_M1 %v, %ptr, %vlmax
218///
219/// ->
220///
221/// %x = VL1RE8_V %ptr
222/// VS1R_V %v, %ptr
223bool RISCVVectorPeepholeImpl::convertToWholeRegister(MachineInstr &MI) const {
224#define CASE_WHOLE_REGISTER_LMUL_SEW(lmul, sew) \
225 case RISCV::PseudoVLE##sew##_V_M##lmul: \
226 NewOpc = RISCV::VL##lmul##RE##sew##_V; \
227 break; \
228 case RISCV::PseudoVSE##sew##_V_M##lmul: \
229 NewOpc = RISCV::VS##lmul##R_V; \
230 break;
231#define CASE_WHOLE_REGISTER_LMUL(lmul) \
232 CASE_WHOLE_REGISTER_LMUL_SEW(lmul, 8) \
233 CASE_WHOLE_REGISTER_LMUL_SEW(lmul, 16) \
234 CASE_WHOLE_REGISTER_LMUL_SEW(lmul, 32) \
235 CASE_WHOLE_REGISTER_LMUL_SEW(lmul, 64)
236
237 unsigned NewOpc;
238 switch (MI.getOpcode()) {
239 CASE_WHOLE_REGISTER_LMUL(1)
240 CASE_WHOLE_REGISTER_LMUL(2)
241 CASE_WHOLE_REGISTER_LMUL(4)
242 CASE_WHOLE_REGISTER_LMUL(8)
243 default:
244 return false;
245 }
246
247 MachineOperand &VLOp = MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
248 if (!VLOp.isImm() || VLOp.getImm() != RISCV::VLMaxSentinel)
249 return false;
250
251 // Whole register instructions aren't pseudos so they don't have
252 // policy/SEW/AVL ops, and they don't have passthrus.
253 if (RISCVII::hasVecPolicyOp(TSFlags: MI.getDesc().TSFlags))
254 MI.removeOperand(OpNo: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc()));
255 MI.removeOperand(OpNo: RISCVII::getSEWOpNum(Desc: MI.getDesc()));
256 MI.removeOperand(OpNo: RISCVII::getVLOpNum(Desc: MI.getDesc()));
257 if (RISCVII::isFirstDefTiedToFirstUse(Desc: MI.getDesc()))
258 MI.removeOperand(OpNo: 1);
259
260 MI.setDesc(TII->get(Opcode: NewOpc));
261
262 return true;
263}
264
265static unsigned getVMV_V_VOpcodeForVMERGE_VVM(const MachineInstr &MI) {
266#define CASE_VMERGE_TO_VMV(lmul) \
267 case RISCV::PseudoVMERGE_VVM_##lmul: \
268 return RISCV::PseudoVMV_V_V_##lmul;
269 switch (MI.getOpcode()) {
270 default:
271 return 0;
272 CASE_VMERGE_TO_VMV(MF8)
273 CASE_VMERGE_TO_VMV(MF4)
274 CASE_VMERGE_TO_VMV(MF2)
275 CASE_VMERGE_TO_VMV(M1)
276 CASE_VMERGE_TO_VMV(M2)
277 CASE_VMERGE_TO_VMV(M4)
278 CASE_VMERGE_TO_VMV(M8)
279 }
280}
281
282/// Convert a PseudoVMERGE_VVM with an all ones mask to a PseudoVMV_V_V.
283///
284/// %x = PseudoVMERGE_VVM %passthru, %false, %true, %allones, sew, vl
285/// ->
286/// %x = PseudoVMV_V_V %passthru, %true, vl, sew, tu_mu
287bool RISCVVectorPeepholeImpl::convertAllOnesVMergeToVMv(
288 MachineInstr &MI) const {
289 unsigned NewOpc = getVMV_V_VOpcodeForVMERGE_VVM(MI);
290 if (!NewOpc)
291 return false;
292 if (!isAllOnesMask(MaskDef: MRI->getVRegDef(Reg: MI.getOperand(i: 4).getReg())))
293 return false;
294
295 MI.setDesc(TII->get(Opcode: NewOpc));
296 MI.removeOperand(OpNo: 2); // False operand
297 MI.removeOperand(OpNo: 3); // Mask operand
298 MI.addOperand(
299 Op: MachineOperand::CreateImm(Val: RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED));
300
301 // vmv.v.v doesn't have a mask operand, so we may be able to inflate the
302 // register class for the destination and passthru operands e.g. VRNoV0 -> VR
303 MRI->recomputeRegClass(Reg: MI.getOperand(i: 0).getReg());
304 if (MI.getOperand(i: 1).getReg().isValid())
305 MRI->recomputeRegClass(Reg: MI.getOperand(i: 1).getReg());
306 return true;
307}
308
309// If \p Reg is defined by one or more COPYs of virtual registers, traverses
310// the chain and returns the root non-COPY source.
311Register RISCVVectorPeepholeImpl::lookThruCopies(
312 Register Reg, bool OneUseOnly,
313 SmallVectorImpl<MachineInstr *> *Copies) const {
314 while (MachineInstr *Def = MRI->getUniqueVRegDef(Reg)) {
315 if (!Def->isFullCopy())
316 break;
317 Register Src = Def->getOperand(i: 1).getReg();
318 if (!Src.isVirtual())
319 break;
320 if (OneUseOnly && !MRI->hasOneNonDBGUse(RegNo: Reg))
321 break;
322 if (Copies)
323 Copies->push_back(Elt: Def);
324 Reg = Src;
325 }
326 return Reg;
327}
328
329/// If a PseudoVMERGE_VVM's true operand is a masked pseudo and both have the
330/// same mask, and the masked pseudo's passthru is the same as the false
331/// operand, we can convert the PseudoVMERGE_VVM to a PseudoVMV_V_V.
332///
333/// %true = PseudoVADD_VV_M1_MASK %false, %x, %y, %mask, vl1, sew, policy
334/// %x = PseudoVMERGE_VVM %passthru, %false, %true, %mask, vl2, sew
335/// ->
336/// %true = PseudoVADD_VV_M1_MASK %false, %x, %y, %mask, vl1, sew, policy
337/// %x = PseudoVMV_V_V %passthru, %true, vl2, sew, tu_mu
338bool RISCVVectorPeepholeImpl::convertSameMaskVMergeToVMv(MachineInstr &MI) {
339 unsigned NewOpc = getVMV_V_VOpcodeForVMERGE_VVM(MI);
340 if (!NewOpc)
341 return false;
342 MachineInstr *True = MRI->getVRegDef(Reg: MI.getOperand(i: 3).getReg());
343
344 if (!True || True->getParent() != MI.getParent())
345 return false;
346
347 auto *TrueMaskedInfo = RISCV::getMaskedPseudoInfo(MaskedPseudo: True->getOpcode());
348 if (!TrueMaskedInfo || !hasSameEEW(User: MI, Src: *True))
349 return false;
350
351 Register TrueMaskReg = lookThruCopies(
352 Reg: True->getOperand(i: TrueMaskedInfo->MaskOpIdx + True->getNumExplicitDefs())
353 .getReg());
354 Register MIMaskReg = lookThruCopies(Reg: MI.getOperand(i: 4).getReg());
355 if (!TrueMaskReg.isVirtual() || TrueMaskReg != MIMaskReg)
356 return false;
357
358 // Masked off lanes past TrueVL will come from False, and converting to vmv
359 // will lose these lanes unless MIVL <= TrueVL.
360 // We can relax this when False == Passthru and True's tail policy is TU,
361 // because True's tail lanes will preserve its passthru (= False = Passthru).
362 const MachineOperand &MIVL = MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
363 const MachineOperand &TrueVL =
364 True->getOperand(i: RISCVII::getVLOpNum(Desc: True->getDesc()));
365 Register FalseReg = MI.getOperand(i: 2).getReg();
366 if (!RISCV::isVLKnownLE(MRI: *MRI, LHS: MIVL, RHS: TrueVL)) {
367 Register PassthruReg = MI.getOperand(i: 1).getReg();
368 if (FalseReg.isValid() && FalseReg != PassthruReg)
369 return false;
370 if (!RISCVII::hasVecPolicyOp(TSFlags: True->getDesc().TSFlags))
371 return false;
372 uint64_t TruePolicy =
373 True->getOperand(i: RISCVII::getVecPolicyOpNum(Desc: True->getDesc())).getImm();
374 if (TruePolicy & RISCVVType::TAIL_AGNOSTIC)
375 return false;
376 }
377
378 // True's passthru needs to be equivalent to False
379 Register TruePassthruReg = True->getOperand(i: 1).getReg();
380 if (TruePassthruReg != FalseReg) {
381 // If True's passthru is undef see if we can change it to False
382 if (TruePassthruReg.isValid() ||
383 !MRI->hasOneUse(RegNo: MI.getOperand(i: 3).getReg()) ||
384 !ensureDominates(Defs: &MI.getOperand(i: 2), Use&: *True))
385 return false;
386 True->getOperand(i: 1).setReg(MI.getOperand(i: 2).getReg());
387 // If True is masked then its passthru needs to be in VRNoV0.
388 MRI->constrainRegClass(Reg: True->getOperand(i: 1).getReg(),
389 RC: TII->getRegClass(MCID: True->getDesc(), OpNum: 1));
390 }
391
392 // If True is mask agnostic, we need to make it mask undisturbed.
393 if (RISCVII::hasVecPolicyOp(TSFlags: True->getDesc().TSFlags)) {
394 MachineOperand &PolicyOp =
395 True->getOperand(i: RISCVII::getVecPolicyOpNum(Desc: True->getDesc()));
396 PolicyOp.setImm(PolicyOp.getImm() & ~RISCVVType::MASK_AGNOSTIC);
397 }
398
399 MI.setDesc(TII->get(Opcode: NewOpc));
400 MI.removeOperand(OpNo: 2); // False operand
401 MI.removeOperand(OpNo: 3); // Mask operand
402 MI.addOperand(
403 Op: MachineOperand::CreateImm(Val: RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED));
404
405 // vmv.v.v doesn't have a mask operand, so we may be able to inflate the
406 // register class for the destination and passthru operands e.g. VRNoV0 -> VR
407 MRI->recomputeRegClass(Reg: MI.getOperand(i: 0).getReg());
408 if (MI.getOperand(i: 1).getReg().isValid())
409 MRI->recomputeRegClass(Reg: MI.getOperand(i: 1).getReg());
410 return true;
411}
412
413bool RISCVVectorPeepholeImpl::convertToUnmasked(MachineInstr &MI) const {
414 const RISCV::RISCVMaskedPseudoInfo *I =
415 RISCV::getMaskedPseudoInfo(MaskedPseudo: MI.getOpcode());
416 if (!I)
417 return false;
418
419 if (!isAllOnesMask(MaskDef: MRI->getVRegDef(
420 Reg: MI.getOperand(i: I->MaskOpIdx + MI.getNumExplicitDefs()).getReg())))
421 return false;
422
423 // There are two classes of pseudos in the table - compares and
424 // everything else. See the comment on RISCVMaskedPseudo for details.
425 const unsigned Opc = I->UnmaskedPseudo;
426 const MCInstrDesc &MCID = TII->get(Opcode: Opc);
427 [[maybe_unused]] const bool HasPolicyOp =
428 RISCVII::hasVecPolicyOp(TSFlags: MCID.TSFlags);
429 const bool HasPassthru = RISCVII::isFirstDefTiedToFirstUse(Desc: MCID);
430 const MCInstrDesc &MaskedMCID = TII->get(Opcode: MI.getOpcode());
431 assert((RISCVII::hasVecPolicyOp(MaskedMCID.TSFlags) ||
432 !RISCVII::hasVecPolicyOp(MCID.TSFlags)) &&
433 "Unmasked pseudo has policy but masked pseudo doesn't?");
434 assert(HasPolicyOp == HasPassthru && "Unexpected pseudo structure");
435 assert(!(HasPassthru && !RISCVII::isFirstDefTiedToFirstUse(MaskedMCID)) &&
436 "Unmasked with passthru but masked with no passthru?");
437 (void)HasPolicyOp;
438
439 MI.setDesc(MCID);
440
441 // Drop the policy operand if unmasked doesn't need it.
442 if (RISCVII::hasVecPolicyOp(TSFlags: MaskedMCID.TSFlags) &&
443 !RISCVII::hasVecPolicyOp(TSFlags: MCID.TSFlags))
444 MI.removeOperand(OpNo: RISCVII::getVecPolicyOpNum(Desc: MaskedMCID));
445
446 // TODO: Increment all MaskOpIdxs in tablegen by num of explicit defs?
447 unsigned MaskOpIdx = I->MaskOpIdx + MI.getNumExplicitDefs();
448 MI.removeOperand(OpNo: MaskOpIdx);
449
450 // The unmasked pseudo will no longer be constrained to the vrnov0 reg class,
451 // so try and relax it to vr.
452 MRI->recomputeRegClass(Reg: MI.getOperand(i: 0).getReg());
453
454 // If the original masked pseudo had a passthru, relax it or remove it.
455 if (RISCVII::isFirstDefTiedToFirstUse(Desc: MaskedMCID)) {
456 unsigned PassthruOpIdx = MI.getNumExplicitDefs();
457 if (HasPassthru) {
458 if (MI.getOperand(i: PassthruOpIdx).getReg())
459 MRI->recomputeRegClass(Reg: MI.getOperand(i: PassthruOpIdx).getReg());
460 } else
461 MI.removeOperand(OpNo: PassthruOpIdx);
462 }
463
464 return true;
465}
466
467/// Given A and B are in the same MBB, returns true if A comes before B.
468static bool strictlyDominates(MachineBasicBlock::const_iterator A,
469 MachineBasicBlock::const_iterator B) {
470 assert(A->getParent() == B->getParent());
471 if (A == B)
472 return false;
473 const MachineBasicBlock *MBB = A->getParent();
474 auto MBBEnd = MBB->end();
475 if (B == MBBEnd)
476 return true;
477
478 MachineBasicBlock::const_iterator I = MBB->begin();
479 for (; &*I != A && &*I != B; ++I)
480 ;
481
482 return &*I == A;
483}
484
485/// If a register in \p Defs doesn't dominate \p Use, try to move Use so it
486/// does. Returns false if any def doesn't dominate and we can't move Use. Each
487/// def must be in the same block as Use.
488bool RISCVVectorPeepholeImpl::ensureDominates(
489 ArrayRef<const MachineOperand *> Defs, MachineInstr &Use) const {
490 MachineInstr *Dest = &Use;
491
492 for (const MachineOperand *MO : Defs) {
493 assert(MO->getParent()->getParent() == Use.getParent());
494 if (!MO->isReg() || !MO->getReg().isValid())
495 continue;
496
497 MachineInstr *Def = MRI->getVRegDef(Reg: MO->getReg());
498 if (Def->getParent() == Dest->getParent() &&
499 !strictlyDominates(A: Def, B: *Dest)) {
500 if (!RISCVInstrInfo::isSafeToMove(From: *Dest, To: *Def->getNextNode()))
501 return false;
502 Dest = Def->getNextNode();
503 }
504 }
505
506 if (Dest != &Use)
507 Use.moveBefore(MovePos: Dest);
508
509 return true;
510}
511
512/// If a PseudoVMV_V_V's passthru is undef then we can replace it with its input
513bool RISCVVectorPeepholeImpl::foldUndefPassthruVMV_V_V(MachineInstr &MI) {
514 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode()) != RISCV::VMV_V_V)
515 return false;
516 if (MI.getOperand(i: 1).getReg().isValid())
517 return false;
518
519 // If the input was a pseudo with a policy operand, we can give it a tail
520 // agnostic policy if MI's undef tail subsumes the input's.
521 MachineInstr *Src = MRI->getVRegDef(Reg: MI.getOperand(i: 2).getReg());
522 if (Src && !Src->hasUnmodeledSideEffects() &&
523 MRI->hasOneUse(RegNo: MI.getOperand(i: 2).getReg()) &&
524 RISCVII::hasVLOp(TSFlags: Src->getDesc().TSFlags) &&
525 RISCVII::hasVecPolicyOp(TSFlags: Src->getDesc().TSFlags) && hasSameEEW(User: MI, Src: *Src)) {
526 const MachineOperand &MIVL = MI.getOperand(i: 3);
527 const MachineOperand &SrcVL =
528 Src->getOperand(i: RISCVII::getVLOpNum(Desc: Src->getDesc()));
529
530 MachineOperand &SrcPolicy =
531 Src->getOperand(i: RISCVII::getVecPolicyOpNum(Desc: Src->getDesc()));
532
533 if (RISCV::isVLKnownLE(MRI: *MRI, LHS: MIVL, RHS: SrcVL))
534 SrcPolicy.setImm(SrcPolicy.getImm() | RISCVVType::TAIL_AGNOSTIC);
535 }
536
537 MRI->constrainRegClass(Reg: MI.getOperand(i: 2).getReg(),
538 RC: MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg()));
539 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: MI.getOperand(i: 2).getReg());
540 MRI->clearKillFlags(Reg: MI.getOperand(i: 2).getReg());
541 MI.eraseFromParent();
542 return true;
543}
544
545/// If a PseudoVMV_V_V is the only user of its input, fold its passthru and VL
546/// into it.
547///
548/// %x = PseudoVADD_V_V_M1 %passthru, %a, %b, %vl1, sew, policy
549/// %y = PseudoVMV_V_V_M1 %passthru, %x, %vl2, sew, policy
550/// (where %vl1 <= %vl2)
551///
552/// ->
553///
554/// %y = PseudoVADD_V_V_M1 %passthru, %a, %b, vl1, sew, policy
555bool RISCVVectorPeepholeImpl::foldVMV_V_V(MachineInstr &MI) {
556 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode()) != RISCV::VMV_V_V)
557 return false;
558
559 MachineOperand &Passthru = MI.getOperand(i: 1);
560
561 if (!MRI->hasOneUse(RegNo: MI.getOperand(i: 2).getReg()))
562 return false;
563
564 MachineInstr *Src = MRI->getVRegDef(Reg: MI.getOperand(i: 2).getReg());
565 if (!Src || Src->hasUnmodeledSideEffects() ||
566 Src->getParent() != MI.getParent() ||
567 !RISCVII::isFirstDefTiedToFirstUse(Desc: Src->getDesc()) ||
568 !RISCVII::hasVLOp(TSFlags: Src->getDesc().TSFlags))
569 return false;
570
571 // Src's dest needs to have the same EEW as MI's input.
572 if (!hasSameEEW(User: MI, Src: *Src))
573 return false;
574
575 std::optional<std::pair<unsigned, unsigned>> NeedsCommute;
576
577 // Src needs to have the same passthru as VMV_V_V
578 MachineOperand &SrcPassthru = Src->getOperand(i: Src->getNumExplicitDefs());
579 if (SrcPassthru.getReg().isValid() &&
580 SrcPassthru.getReg() != Passthru.getReg()) {
581 // If Src's passthru != Passthru, check if it uses Passthru in another
582 // operand and try to commute it.
583 int OtherIdx = Src->findRegisterUseOperandIdx(Reg: Passthru.getReg(), TRI);
584 if (OtherIdx == -1)
585 return false;
586 unsigned OpIdx1 = OtherIdx;
587 unsigned OpIdx2 = Src->getNumExplicitDefs();
588 if (!TII->findCommutedOpIndices(MI: *Src, SrcOpIdx1&: OpIdx1, SrcOpIdx2&: OpIdx2))
589 return false;
590 NeedsCommute = {OpIdx1, OpIdx2};
591 }
592
593 // Src VL will have already been reduced if legal by RISCVVLOptimizer,
594 // so we don't need to handle a smaller source VL here. However, the
595 // user's VL may be larger
596 MachineOperand &SrcVL = Src->getOperand(i: RISCVII::getVLOpNum(Desc: Src->getDesc()));
597 if (!RISCV::isVLKnownLE(MRI: *MRI, LHS: SrcVL, RHS: MI.getOperand(i: 3)))
598 return false;
599
600 // If the new passthru doesn't dominate Src, try to move Src so it does.
601 if (!ensureDominates(Defs: &Passthru, Use&: *Src))
602 return false;
603
604 if (NeedsCommute) {
605 auto [OpIdx1, OpIdx2] = *NeedsCommute;
606 [[maybe_unused]] bool Commuted =
607 TII->commuteInstruction(MI&: *Src, /*NewMI=*/false, OpIdx1, OpIdx2);
608 assert(Commuted && "Failed to commute Src?");
609 }
610
611 if (SrcPassthru.getReg() != Passthru.getReg()) {
612 SrcPassthru.setReg(Passthru.getReg());
613 // If Src is masked then its passthru needs to be in VRNoV0.
614 if (Passthru.getReg().isValid())
615 MRI->constrainRegClass(
616 Reg: Passthru.getReg(),
617 RC: TII->getRegClass(MCID: Src->getDesc(), OpNum: SrcPassthru.getOperandNo()));
618 }
619
620 if (RISCVII::hasVecPolicyOp(TSFlags: Src->getDesc().TSFlags)) {
621 // If MI was tail agnostic and the VL didn't increase, preserve it.
622 int64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
623 if ((MI.getOperand(i: 5).getImm() & RISCVVType::TAIL_AGNOSTIC) &&
624 RISCV::isVLKnownLE(MRI: *MRI, LHS: MI.getOperand(i: 3), RHS: SrcVL))
625 Policy |= RISCVVType::TAIL_AGNOSTIC;
626 Src->getOperand(i: RISCVII::getVecPolicyOpNum(Desc: Src->getDesc())).setImm(Policy);
627 }
628
629 MRI->constrainRegClass(Reg: Src->getOperand(i: 0).getReg(),
630 RC: MRI->getRegClass(Reg: MI.getOperand(i: 0).getReg()));
631 MRI->replaceRegWith(FromReg: MI.getOperand(i: 0).getReg(), ToReg: Src->getOperand(i: 0).getReg());
632 MI.eraseFromParent();
633
634 return true;
635}
636
637/// Try to fold away VMERGE_VVM instructions into their operands:
638///
639/// %true = PseudoVADD_VV ...
640/// %x = PseudoVMERGE_VVM_M1 %false, %false, %true, %mask
641/// ->
642/// %x = PseudoVADD_VV_M1_MASK %false, ..., %mask
643///
644/// We can only fold if vmerge's passthru operand, vmerge's false operand and
645/// %true's passthru operand (if it has one) are the same. This is because we
646/// have to consolidate them into one passthru operand in the result.
647///
648/// If %true is masked, then we can use its mask instead of vmerge's if vmerge's
649/// mask is all ones.
650///
651/// The resulting VL is the minimum of the two VLs.
652///
653/// The resulting policy is the effective policy the vmerge would have had,
654/// i.e. whether or not it's passthru operand was implicit-def.
655bool RISCVVectorPeepholeImpl::foldVMergeToMask(MachineInstr &MI) const {
656 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode()) != RISCV::VMERGE_VVM)
657 return false;
658
659 // Collect chain of COPYs on True's result for later cleanup.
660 SmallVector<MachineInstr *, 4> TrueCopies;
661 Register PassthruReg = lookThruCopies(Reg: MI.getOperand(i: 1).getReg());
662 const MachineOperand &FalseOp = MI.getOperand(i: 2);
663 Register FalseReg = lookThruCopies(Reg: FalseOp.getReg());
664 Register TrueReg = lookThruCopies(Reg: MI.getOperand(i: 3).getReg(),
665 /*OneUseOnly=*/true, Copies: &TrueCopies);
666 if (!TrueReg.isVirtual() || !MRI->hasOneUse(RegNo: TrueReg))
667 return false;
668 MachineInstr *TrueDef = MRI->getVRegDef(Reg: TrueReg);
669 if (!TrueDef)
670 return false;
671 MachineInstr &True = *TrueDef;
672 if (True.getParent() != MI.getParent())
673 return false;
674 const MachineOperand &MaskOp = MI.getOperand(i: 4);
675 MachineInstr *Mask = MRI->getUniqueVRegDef(Reg: MaskOp.getReg());
676 assert(Mask);
677
678 const RISCV::RISCVMaskedPseudoInfo *Info =
679 RISCV::lookupMaskedIntrinsicByUnmasked(UnmaskedPseudo: True.getOpcode());
680 if (!Info)
681 return false;
682
683 // If the EEW of True is different from vmerge's SEW, then we can't fold.
684 if (!hasSameEEW(User: MI, Src: True))
685 return false;
686
687 // We require that either passthru and false are the same, or that passthru
688 // is undefined.
689 if (PassthruReg && !(PassthruReg.isVirtual() && PassthruReg == FalseReg))
690 return false;
691
692 std::optional<std::pair<unsigned, unsigned>> NeedsCommute;
693
694 // If True has a passthru operand then it needs to be the same as vmerge's
695 // False, since False will be used for the result's passthru operand.
696 Register TruePassthru;
697 if (RISCVII::isFirstDefTiedToFirstUse(Desc: True.getDesc()))
698 TruePassthru =
699 lookThruCopies(Reg: True.getOperand(i: True.getNumExplicitDefs()).getReg());
700 if (TruePassthru && !(TruePassthru.isVirtual() && TruePassthru == FalseReg)) {
701 // If True's passthru != False, check if it uses False in another operand
702 // and try to commute it.
703 int OtherIdx = True.findRegisterUseOperandIdx(Reg: FalseReg, TRI);
704 if (OtherIdx == -1)
705 return false;
706 unsigned OpIdx1 = OtherIdx;
707 unsigned OpIdx2 = True.getNumExplicitDefs();
708 if (!TII->findCommutedOpIndices(MI: True, SrcOpIdx1&: OpIdx1, SrcOpIdx2&: OpIdx2))
709 return false;
710 NeedsCommute = {OpIdx1, OpIdx2};
711 }
712
713 // Make sure it doesn't raise any observable fp exceptions, since changing the
714 // active elements will affect how fflags is set.
715 if (True.hasUnmodeledSideEffects() || True.mayRaiseFPException())
716 return false;
717
718 const MachineOperand &VMergeVL =
719 MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
720 const MachineOperand &TrueVL =
721 True.getOperand(i: RISCVII::getVLOpNum(Desc: True.getDesc()));
722
723 MachineOperand MinVL = MachineOperand::CreateImm(Val: 0);
724 if (RISCV::isVLKnownLE(MRI: *MRI, LHS: TrueVL, RHS: VMergeVL))
725 MinVL = TrueVL;
726 else if (RISCV::isVLKnownLE(MRI: *MRI, LHS: VMergeVL, RHS: TrueVL))
727 MinVL = VMergeVL;
728 else if (!TruePassthru && !True.mayLoadOrStore())
729 // If True's passthru is undef, we can use vmerge's vl.
730 MinVL = VMergeVL;
731 else
732 return false;
733
734 unsigned RVVTSFlags =
735 TII->get(Opcode: RISCV::getRVVMCOpcode(RVVPseudoOpcode: True.getOpcode())).TSFlags;
736 if (RISCVII::elementsDependOnVL(TSFlags: RVVTSFlags) && !TrueVL.isIdenticalTo(Other: MinVL))
737 return false;
738 if (RISCVII::elementsDependOnMask(TSFlags: RVVTSFlags) && !isAllOnesMask(MaskDef: Mask))
739 return false;
740
741 // Use a tumu policy, relaxing it to tail agnostic provided that the passthru
742 // operand is undefined.
743 //
744 // However, if the VL became smaller than what the vmerge had originally, then
745 // elements past VL that were previously in the vmerge's body will have moved
746 // to the tail. In that case we always need to use tail undisturbed to
747 // preserve them.
748 uint64_t Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
749 if (!PassthruReg && RISCV::isVLKnownLE(MRI: *MRI, LHS: VMergeVL, RHS: MinVL))
750 Policy |= RISCVVType::TAIL_AGNOSTIC;
751
752 assert(RISCVII::hasVecPolicyOp(True.getDesc().TSFlags) &&
753 "Foldable unmasked pseudo should have a policy op already");
754
755 // Make sure Mask, False and MinVL dominate True and its copies, otherwise
756 // move down True so it does.
757 if (!ensureDominates(Defs: {&MaskOp, &FalseOp, &MinVL}, Use&: True))
758 return false;
759
760 if (NeedsCommute) {
761 auto [OpIdx1, OpIdx2] = *NeedsCommute;
762 [[maybe_unused]] bool Commuted =
763 TII->commuteInstruction(MI&: True, /*NewMI=*/false, OpIdx1, OpIdx2);
764 assert(Commuted && "Failed to commute True?");
765 Info = RISCV::lookupMaskedIntrinsicByUnmasked(UnmaskedPseudo: True.getOpcode());
766 }
767
768 True.setDesc(TII->get(Opcode: Info->MaskedPseudo));
769
770 // Insert the mask operand.
771 // TODO: Increment MaskOpIdx by number of explicit defs?
772 True.insert(InsertBefore: True.operands_begin() + Info->MaskOpIdx +
773 True.getNumExplicitDefs(),
774 Ops: MachineOperand::CreateReg(Reg: MaskOp.getReg(), isDef: false));
775
776 // Update the passthru, AVL and policy.
777 True.getOperand(i: True.getNumExplicitDefs()).setReg(FalseReg);
778 True.removeOperand(OpNo: RISCVII::getVLOpNum(Desc: True.getDesc()));
779 True.insert(InsertBefore: True.operands_begin() + RISCVII::getVLOpNum(Desc: True.getDesc()),
780 Ops: MinVL);
781 True.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: True.getDesc())).setImm(Policy);
782
783 MRI->replaceRegWith(FromReg: True.getOperand(i: 0).getReg(), ToReg: MI.getOperand(i: 0).getReg());
784 // Now that True is masked, constrain its operands from vr -> vrnov0.
785 for (MachineOperand &MO : True.explicit_operands()) {
786 if (!MO.isReg() || !MO.getReg().isVirtual())
787 continue;
788 MRI->constrainRegClass(
789 Reg: MO.getReg(), RC: True.getRegClassConstraint(OpIdx: MO.getOperandNo(), TII, TRI));
790 }
791 // We should clear the IsKill flag since we have a new use now.
792 MRI->clearKillFlags(Reg: FalseReg);
793 MI.eraseFromParent();
794
795 // Cleanup all the COPYs on True's value. We have to manually do this because
796 // sometimes sinking True causes these COPY to be invalid (use before define).
797 for (MachineInstr *TrueCopy : TrueCopies)
798 TrueCopy->eraseFromParent();
799
800 return true;
801}
802
803/// Fold a mask-register AND of a mask comparison into a mask-undisturbed
804/// masked comparison, saving an instruction:
805///
806/// %cmp1 = PseudoVMSLT_VV_M1 %a, %b, %vl, %sew
807/// %cmp2 = PseudoVMSLT_VV_M1 %c, %d, %vl, %sew
808/// %and = PseudoVMAND_MM %cmp1, %cmp2, %vl, 0
809/// ->
810/// %cmp1 = PseudoVMSLT_VV_M1 %a, %b, %vl, %sew
811/// %and = PseudoVMSLT_VV_M1_MASK %cmp1, %c, %d, %cmp1, %vl, %sew, mu
812///
813/// This works because for a mask-undisturbed masked compare whose passthru is
814/// the same register as its mask %m, the result is %m[i] ? (c cmp d)[i] :
815/// %m[i], which is exactly %m[i] & (c cmp d)[i], i.e. vmand(%m, vmscmp(c, d)).
816///
817/// Since vmand is commutative it's enough for either operand to be a foldable
818/// comparison; the other operand becomes both the mask and the passthru.
819bool RISCVVectorPeepholeImpl::foldVMANDToMaskedCompare(MachineInstr &MI) const {
820 if (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode()) != RISCV::VMAND_MM)
821 return false;
822
823 // The masked comparison we create needs its mask (and passthru) in v0, which
824 // the original vmand did not require. If the vmand's result has more than one
825 // use then it is an interior mask value rather than a final result feeding
826 // v0, and introducing the v0 requirement tends to add vmv1r.v moves. Only
827 // fold single-use results, where the value coalesces onto v0 for free.
828 if (!MRI->hasOneUse(RegNo: MI.getOperand(i: 0).getReg()))
829 return false;
830
831 // Try each operand as the comparison to be masked; the other becomes the
832 // mask/passthru.
833 for (unsigned CmpIdx : {1, 2}) {
834 unsigned MaskIdx = CmpIdx == 1 ? 2 : 1;
835
836 // The comparison must be single use so that folding it into MI doesn't
837 // leave an extra unmasked comparison behind.
838 SmallVector<MachineInstr *, 4> CmpCopies;
839 Register CmpReg = lookThruCopies(Reg: MI.getOperand(i: CmpIdx).getReg(),
840 /*OneUseOnly=*/true, Copies: &CmpCopies);
841 if (!CmpReg.isVirtual() || !MRI->hasOneUse(RegNo: CmpReg))
842 continue;
843 MachineInstr &Cmp = *MRI->getUniqueVRegDef(Reg: CmpReg);
844 if (Cmp.getParent() != MI.getParent())
845 continue;
846 if (!RISCVInstrInfo::isRVVCompare(MI: Cmp))
847 continue;
848
849 // Find the masked pseudo corresponding to the unmasked comparison.
850 const RISCV::RISCVMaskedPseudoInfo *Info =
851 RISCV::lookupMaskedIntrinsicByUnmasked(UnmaskedPseudo: Cmp.getOpcode());
852 if (!Info)
853 continue;
854
855 // The EEW of the comparison's dest must match vmand's SEW.
856 if (!hasSameEEW(User: MI, Src: Cmp))
857 continue;
858
859 // Masking restricts the comparison to the mask's active elements, so any FP
860 // exceptions raised on inactive elements would be lost.
861 if (Cmp.hasUnmodeledSideEffects() || Cmp.mayRaiseFPException())
862 continue;
863
864 // All active elements of vmand must also be active in the comparison. If
865 // the comparison's VL were smaller, elements in between the two VLs would
866 // become tail elements of the masked comparison and could not be preserved
867 // from the mask because mask results are always tail agnostic.
868 const MachineOperand &CmpVL =
869 Cmp.getOperand(i: RISCVII::getVLOpNum(Desc: Cmp.getDesc()));
870 const MachineOperand &MIVL =
871 MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
872 if (!RISCV::isVLKnownLE(MRI: *MRI, LHS: MIVL, RHS: CmpVL))
873 continue;
874
875 const MachineOperand &MaskOp = MI.getOperand(i: MaskIdx);
876 Register MaskReg = MaskOp.getReg();
877
878 unsigned MaskedOpc = Info->MaskedPseudo;
879 const MCInstrDesc &MaskedDesc = TII->get(Opcode: MaskedOpc);
880 unsigned SEW = Cmp.getOperand(i: RISCVII::getSEWOpNum(Desc: Cmp.getDesc())).getImm();
881
882 // Only fold if the masked comparison's dest can live in v0. Its mask
883 // operand must be v0, and we reuse the mask as the passthru, so if the dest
884 // can also be v0 the whole thing coalesces onto v0 and we save the vmand
885 // for free. For LMUL >= 2 the dest is earlyclobbered into vrnov0, which
886 // would force extra vmv1r.v moves for the mask and result and make this a
887 // regression, so bail out in that case. This check must happen before we
888 // mutate any instructions below.
889 if (!TII->getRegClass(MCID: MaskedDesc, OpNum: 0)->contains(Reg: RISCV::V0))
890 continue;
891
892 // Make sure the mask and VL dominate the comparison, sinking it if needed.
893 if (!ensureDominates(Defs: {&MaskOp, &MIVL}, Use&: Cmp))
894 continue;
895
896 // The masked comparison's mask operand lives in the VMV0 (v0) class, and
897 // its passthru operand shares the dest's class. Copy the vmand mask into
898 // both; the coalescer collapses these back onto v0, matching the
899 // two-instruction ideal.
900 Register MaskV0Reg = MRI->createVirtualRegister(RegClass: &RISCV::VMV0RegClass);
901 BuildMI(BB&: *MI.getParent(), I&: Cmp, MIMD: Cmp.getDebugLoc(),
902 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: MaskV0Reg)
903 .addReg(RegNo: MaskReg);
904 // The passthru shares the dest's class, which the V0 check above restricts
905 // to LMUL <= 1, so it is always a single vector register.
906 Register PassthruReg = MRI->createVirtualRegister(RegClass: &RISCV::VRRegClass);
907 BuildMI(BB&: *MI.getParent(), I&: Cmp, MIMD: Cmp.getDebugLoc(),
908 MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: PassthruReg)
909 .addReg(RegNo: MaskReg);
910
911 // Build the masked comparison. Its dest reuses vmand's dest; the passthru
912 // (tied to the dest) and mask are both the other vmand operand. Preserve
913 // the source comparison's MI flags (e.g. nofpexcept), which still hold
914 // since the masked comparison operates on a subset of the original active
915 // elements.
916 Register DestReg = MI.getOperand(i: 0).getReg();
917 MachineInstr *Masked =
918 BuildMI(BB&: *MI.getParent(), I&: Cmp, MIMD: MIMetadata(Cmp), MCID: MaskedDesc, DestReg)
919 .addReg(RegNo: PassthruReg)
920 .add(MO: Cmp.getOperand(i: 1))
921 .add(MO: Cmp.getOperand(i: 2))
922 .addReg(RegNo: MaskV0Reg)
923 .add(MO: MIVL)
924 .addImm(Val: SEW)
925 // The result is a mask register, whose tail is always agnostic, so
926 // we only need mask-undisturbed (MASK_AGNOSTIC clear) to preserve
927 // the inactive elements from the mask/passthru.
928 .addImm(Val: RISCVVType::TAIL_AGNOSTIC)
929 .setMIFlags(Cmp.getFlags());
930
931 // Now that the comparison is masked, constrain its operands to the masked
932 // pseudo's register classes (e.g. vr -> vrnov0 for LMUL >= 2).
933 for (MachineOperand &MO : Masked->explicit_operands()) {
934 if (!MO.isReg() || !MO.getReg().isVirtual())
935 continue;
936 if (const TargetRegisterClass *RC =
937 Masked->getRegClassConstraint(OpIdx: MO.getOperandNo(), TII, TRI))
938 MRI->constrainRegClass(Reg: MO.getReg(), RC);
939 }
940 MRI->clearKillFlags(Reg: MaskReg);
941 MI.eraseFromParent();
942 Cmp.eraseFromParent();
943 for (MachineInstr *CmpCopy : CmpCopies)
944 CmpCopy->eraseFromParent();
945
946 return true;
947 }
948
949 return false;
950}
951
952bool RISCVVectorPeepholeImpl::run(MachineFunction &MF) {
953 // Skip if the vector extension is not enabled.
954 ST = &MF.getSubtarget<RISCVSubtarget>();
955 if (!ST->hasVInstructions())
956 return false;
957
958 TII = ST->getInstrInfo();
959 MRI = &MF.getRegInfo();
960 TRI = MRI->getTargetRegisterInfo();
961
962 bool Changed = false;
963
964 for (MachineBasicBlock &MBB : MF) {
965 for (MachineInstr &MI : make_early_inc_range(Range&: MBB))
966 Changed |= foldVMergeToMask(MI);
967
968 for (MachineInstr &MI : make_early_inc_range(Range&: MBB))
969 Changed |= foldVMANDToMaskedCompare(MI);
970
971 for (MachineInstr &MI : make_early_inc_range(Range&: MBB)) {
972 Changed |= convertToVLMAX(MI);
973 Changed |= convertToUnmasked(MI);
974 Changed |= convertToWholeRegister(MI);
975 Changed |= convertAllOnesVMergeToVMv(MI);
976 Changed |= convertSameMaskVMergeToVMv(MI);
977 if (foldUndefPassthruVMV_V_V(MI)) {
978 Changed |= true;
979 continue; // MI is erased
980 }
981 Changed |= foldVMV_V_V(MI);
982 }
983 }
984
985 return Changed;
986}
987
988bool RISCVVectorPeepholeLegacy::runOnMachineFunction(MachineFunction &MF) {
989 if (skipFunction(F: MF.getFunction()))
990 return false;
991 return RISCVVectorPeepholeImpl().run(MF);
992}
993
994PreservedAnalyses
995RISCVVectorPeepholePass::run(MachineFunction &MF,
996 MachineFunctionAnalysisManager &MFAM) {
997 MFPropsModifier _(*this, MF);
998 bool Changed = RISCVVectorPeepholeImpl().run(MF);
999 if (!Changed)
1000 return PreservedAnalyses::all();
1001
1002 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1003 PA.preserveSet<CFGAnalyses>();
1004 PA.preserve<MachineRegisterClassAnalysis>();
1005 return PA;
1006}
1007
1008FunctionPass *llvm::createRISCVVectorPeepholeLegacyPass() {
1009 return new RISCVVectorPeepholeLegacy();
1010}
1011