1//===- RISCVVSETVLIInfoAnalysis.cpp - VSETVLI Info Analysis ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements an analysis of the vtype/vl information that is needed
10// by RISCVInsertVSETVLI pass and others.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVVSETVLIInfoAnalysis.h"
15#include "RISCVSubtarget.h"
16#include "llvm/CodeGen/LiveIntervals.h"
17
18namespace llvm {
19namespace RISCV {
20
21/// Given a virtual register \p Reg, return the corresponding VNInfo for it.
22/// This will return nullptr if the virtual register is an implicit_def or
23/// if LiveIntervals is not available.
24static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
25 const LiveIntervals *LIS) {
26 assert(Reg.isVirtual());
27 if (!LIS)
28 return nullptr;
29 auto &LI = LIS->getInterval(Reg);
30 SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
31 return LI.getVNInfoBefore(Idx: SI);
32}
33
34static unsigned getVLOpNum(const MachineInstr &MI) {
35 return RISCVII::getVLOpNum(Desc: MI.getDesc());
36}
37
38static unsigned getSEWOpNum(const MachineInstr &MI) {
39 return RISCVII::getSEWOpNum(Desc: MI.getDesc());
40}
41
42static unsigned getVecPolicyOpNum(const MachineInstr &MI) {
43 return RISCVII::getVecPolicyOpNum(Desc: MI.getDesc());
44}
45
46/// Get the EEW for a load or store instruction. Return std::nullopt if MI is
47/// not a load or store which ignores SEW.
48static std::optional<unsigned> getEEWForLoadStore(const MachineInstr &MI) {
49 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
50 default:
51 return std::nullopt;
52 case RISCV::VLE8_V:
53 case RISCV::VLSE8_V:
54 case RISCV::VSE8_V:
55 case RISCV::VSSE8_V:
56 return 8;
57 case RISCV::VLE16_V:
58 case RISCV::VLSE16_V:
59 case RISCV::VSE16_V:
60 case RISCV::VSSE16_V:
61 return 16;
62 case RISCV::VLE32_V:
63 case RISCV::VLSE32_V:
64 case RISCV::VSE32_V:
65 case RISCV::VSSE32_V:
66 return 32;
67 case RISCV::VLE64_V:
68 case RISCV::VLSE64_V:
69 case RISCV::VSE64_V:
70 case RISCV::VSSE64_V:
71 return 64;
72 }
73}
74
75/// Return true if this is an operation on mask registers. Note that
76/// this includes both arithmetic/logical ops and load/store (vlm/vsm).
77static bool isMaskRegOp(const MachineInstr &MI) {
78 if (!RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags))
79 return false;
80 const unsigned Log2SEW = MI.getOperand(i: getSEWOpNum(MI)).getImm();
81 // A Log2SEW of 0 is an operation on mask registers only.
82 return Log2SEW == 0;
83}
84
85/// Return true if the inactive elements in the result are entirely undefined.
86/// Note that this is different from "agnostic" as defined by the vector
87/// specification. Agnostic requires each lane to either be undisturbed, or
88/// take the value -1; no other value is allowed.
89static bool hasUndefinedPassthru(const MachineInstr &MI) {
90 unsigned UseOpIdx;
91 if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx))
92 // If there is no passthrough operand, then the pass through
93 // lanes are undefined.
94 return true;
95
96 // All undefined passthrus should be $noreg: see
97 // RISCVDAGToDAGISel::doPeepholeNoRegPassThru
98 const MachineOperand &UseMO = MI.getOperand(i: UseOpIdx);
99 return !UseMO.getReg().isValid() || UseMO.isUndef();
100}
101
102static bool isLMUL1OrSmaller(RISCVVType::VLMUL LMUL) {
103 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul: LMUL);
104 return Fractional || LMul == 1;
105}
106
107/// Return true if moving from CurVType to NewVType is
108/// indistinguishable from the perspective of an instruction (or set
109/// of instructions) which use only the Used subfields and properties.
110bool areCompatibleVTYPEs(uint64_t CurVType, uint64_t NewVType,
111 const DemandedFields &Used) {
112 switch (Used.SEW) {
113 case DemandedFields::SEWNone:
114 break;
115 case DemandedFields::SEWEqual:
116 if (RISCVVType::getSEW(VType: CurVType) != RISCVVType::getSEW(VType: NewVType))
117 return false;
118 break;
119 case DemandedFields::SEWGreaterThanOrEqual:
120 if (RISCVVType::getSEW(VType: NewVType) < RISCVVType::getSEW(VType: CurVType))
121 return false;
122 break;
123 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
124 if (RISCVVType::getSEW(VType: NewVType) < RISCVVType::getSEW(VType: CurVType) ||
125 RISCVVType::getSEW(VType: NewVType) >= 64)
126 return false;
127 break;
128 }
129
130 switch (Used.LMUL) {
131 case DemandedFields::LMULNone:
132 break;
133 case DemandedFields::LMULEqual:
134 if (RISCVVType::getVLMUL(VType: CurVType) != RISCVVType::getVLMUL(VType: NewVType))
135 return false;
136 break;
137 case DemandedFields::LMULLessThanOrEqualToM1:
138 if (!isLMUL1OrSmaller(LMUL: RISCVVType::getVLMUL(VType: NewVType)))
139 return false;
140 break;
141 }
142
143 if (Used.SEWLMULRatio) {
144 auto Ratio1 = RISCVVType::getSEWLMULRatio(SEW: RISCVVType::getSEW(VType: CurVType),
145 VLMul: RISCVVType::getVLMUL(VType: CurVType));
146 auto Ratio2 = RISCVVType::getSEWLMULRatio(SEW: RISCVVType::getSEW(VType: NewVType),
147 VLMul: RISCVVType::getVLMUL(VType: NewVType));
148 if (Ratio1 != Ratio2)
149 return false;
150 }
151
152 if (Used.TailPolicy && RISCVVType::isTailAgnostic(VType: CurVType) !=
153 RISCVVType::isTailAgnostic(VType: NewVType))
154 return false;
155 if (Used.MaskPolicy && RISCVVType::isMaskAgnostic(VType: CurVType) !=
156 RISCVVType::isMaskAgnostic(VType: NewVType))
157 return false;
158 if (Used.TWiden && (RISCVVType::hasXSfmmWiden(VType: CurVType) !=
159 RISCVVType::hasXSfmmWiden(VType: NewVType) ||
160 (RISCVVType::hasXSfmmWiden(VType: CurVType) &&
161 RISCVVType::getXSfmmWiden(VType: CurVType) !=
162 RISCVVType::getXSfmmWiden(VType: NewVType))))
163 return false;
164 if (Used.AltFmt &&
165 RISCVVType::isAltFmt(VType: CurVType) != RISCVVType::isAltFmt(VType: NewVType))
166 return false;
167 return true;
168}
169
170/// Return the fields and properties demanded by the provided instruction.
171DemandedFields getDemanded(const MachineInstr &MI, const RISCVSubtarget *ST) {
172 // This function works in coalesceVSETVLI too. We can still use the value of a
173 // SEW, VL, or Policy operand even though it might not be the exact value in
174 // the VL or VTYPE, since we only care about what the instruction originally
175 // demanded.
176
177 // Most instructions don't use any of these subfeilds.
178 DemandedFields Res;
179 // Start conservative if registers are used
180 if (MI.isCall() || MI.isInlineAsm() ||
181 MI.readsRegister(Reg: RISCV::VL, /*TRI=*/nullptr))
182 Res.demandVL();
183 if (MI.isCall() || MI.isInlineAsm() ||
184 MI.readsRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
185 Res.demandVTYPE();
186 // Start conservative on the unlowered form too
187 uint64_t TSFlags = MI.getDesc().TSFlags;
188 if (RISCVII::hasSEWOp(TSFlags)) {
189 Res.demandVTYPE();
190 if (RISCVII::hasVLOp(TSFlags))
191 if (const MachineOperand &VLOp = MI.getOperand(i: getVLOpNum(MI));
192 !VLOp.isReg() || !VLOp.isUndef())
193 Res.demandVL();
194
195 // Behavior is independent of mask policy.
196 if (!RISCVII::usesMaskPolicy(TSFlags))
197 Res.MaskPolicy = false;
198 }
199
200 // Loads and stores with implicit EEW do not demand SEW or LMUL directly.
201 // They instead demand the ratio of the two which is used in computing
202 // EMUL, but which allows us the flexibility to change SEW and LMUL
203 // provided we don't change the ratio.
204 // Note: We assume that the instructions initial SEW is the EEW encoded
205 // in the opcode. This is asserted when constructing the VSETVLIInfo.
206 if (RISCV::getEEWForLoadStore(MI)) {
207 Res.SEW = DemandedFields::SEWNone;
208 Res.LMUL = DemandedFields::LMULNone;
209 }
210
211 // Store instructions don't use the policy fields.
212 if (RISCVII::hasSEWOp(TSFlags) && MI.getNumExplicitDefs() == 0) {
213 Res.TailPolicy = false;
214 Res.MaskPolicy = false;
215 }
216
217 // If this is a mask reg operation, it only cares about VLMAX.
218 // TODO: Possible extensions to this logic
219 // * Probably ok if available VLMax is larger than demanded
220 // * The policy bits can probably be ignored..
221 if (isMaskRegOp(MI)) {
222 Res.SEW = DemandedFields::SEWNone;
223 Res.LMUL = DemandedFields::LMULNone;
224 }
225
226 // For vmv.s.x and vfmv.s.f, there are only two behaviors, VL = 0 and VL > 0.
227 if (RISCVInstrInfo::isScalarInsertInstr(MI)) {
228 Res.LMUL = DemandedFields::LMULNone;
229 Res.SEWLMULRatio = false;
230 Res.VLAny = false;
231 // For vmv.s.x and vfmv.s.f, if the passthru is *undefined*, we don't
232 // need to preserve any other bits and are thus compatible with any larger,
233 // etype and can disregard policy bits. Warning: It's tempting to try doing
234 // this for any tail agnostic operation, but we can't as TA requires
235 // tail lanes to either be the original value or -1. We are writing
236 // unknown bits to the lanes here.
237 if (RISCV::hasUndefinedPassthru(MI)) {
238 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&
239 !ST->hasVInstructionsF64())
240 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
241 else
242 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
243 Res.TailPolicy = false;
244 }
245 }
246
247 // vmv.x.s, and vfmv.f.s are unconditional and ignore everything except SEW.
248 if (RISCVInstrInfo::isScalarExtractInstr(MI)) {
249 assert(!RISCVII::hasVLOp(TSFlags));
250 Res.LMUL = DemandedFields::LMULNone;
251 Res.SEWLMULRatio = false;
252 Res.TailPolicy = false;
253 Res.MaskPolicy = false;
254 }
255
256 if (RISCVII::hasVLOp(TSFlags: MI.getDesc().TSFlags)) {
257 const MachineOperand &VLOp = MI.getOperand(i: getVLOpNum(MI));
258 // A slidedown/slideup with an *undefined* passthru can freely clobber
259 // elements not copied from the source vector (e.g. masked off, tail, or
260 // slideup's prefix). Notes:
261 // * We can't modify SEW here since the slide amount is in units of SEW.
262 // * VL=1 is special only because we have existing support for zero vs
263 // non-zero VL. We could generalize this if we had a VL > C predicate.
264 // * The LMUL1 restriction is for machines whose latency may depend on LMUL.
265 // * As above, this is only legal for tail "undefined" not "agnostic".
266 // * We avoid increasing vl if the subtarget has +vl-dependent-latency
267 if (RISCVInstrInfo::isVSlideInstr(MI) && VLOp.isImm() &&
268 VLOp.getImm() == 1 && RISCV::hasUndefinedPassthru(MI) &&
269 !ST->hasVLDependentLatency()) {
270 Res.VLAny = false;
271 Res.VLZeroness = true;
272 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;
273 Res.TailPolicy = false;
274 }
275
276 // A tail undefined vmv.v.i/x or vfmv.v.f with VL=1 can be treated in the
277 // same semantically as vmv.s.x. This is particularly useful since we don't
278 // have an immediate form of vmv.s.x, and thus frequently use vmv.v.i in
279 // it's place. Since a splat is non-constant time in LMUL, we do need to be
280 // careful to not increase the number of active vector registers (unlike for
281 // vmv.s.x.)
282 if (RISCVInstrInfo::isScalarSplatInstr(MI) && VLOp.isImm() &&
283 VLOp.getImm() == 1 && RISCV::hasUndefinedPassthru(MI) &&
284 !ST->hasVLDependentLatency()) {
285 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;
286 Res.SEWLMULRatio = false;
287 Res.VLAny = false;
288 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&
289 !ST->hasVInstructionsF64())
290 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
291 else
292 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
293 Res.TailPolicy = false;
294 }
295 }
296
297 // In §32.16.6, whole vector register moves have a dependency on SEW. At the
298 // MIR level though we don't encode the element type, and it gives the same
299 // result whatever the SEW may be.
300 //
301 // However it does need valid SEW, i.e. vill must be cleared. The entry to a
302 // function, calls and inline assembly may all set it, so make sure we clear
303 // it for whole register copies. Do this by leaving VILL demanded.
304 if (RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI)) {
305 Res.LMUL = DemandedFields::LMULNone;
306 Res.SEW = DemandedFields::SEWNone;
307 Res.SEWLMULRatio = false;
308 Res.TailPolicy = false;
309 Res.MaskPolicy = false;
310 }
311
312 if (RISCVInstrInfo::isVExtractInstr(MI)) {
313 assert(!RISCVII::hasVLOp(TSFlags));
314 // TODO: LMUL can be any larger value (without cost)
315 Res.TailPolicy = false;
316 }
317
318 Res.AltFmt = RISCVII::getAltFmtType(TSFlags: MI.getDesc().TSFlags) !=
319 RISCVII::AltFmtType::DontCare;
320 Res.TWiden = RISCVII::hasTWidenOp(TSFlags: MI.getDesc().TSFlags) ||
321 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI);
322
323 return Res;
324}
325
326bool VSETVLIInfo::hasCompatibleVTYPE(const DemandedFields &Used,
327 const VSETVLIInfo &Require) const {
328 return areCompatibleVTYPEs(CurVType: Require.encodeVTYPE(), NewVType: encodeVTYPE(), Used);
329}
330
331// If the AVL is defined by a vsetvli's output vl with the same VLMAX, we can
332// replace the AVL operand with the AVL of the defining vsetvli. E.g.
333//
334// %vl = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
335// $x0 = PseudoVSETVLI %vl:gpr, SEW=32, LMUL=M1
336// ->
337// %vl = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
338// $x0 = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
339void RISCVVSETVLIInfoAnalysis::forwardVSETVLIAVL(VSETVLIInfo &Info) const {
340 if (!Info.hasAVLReg())
341 return;
342 const MachineInstr *DefMI = Info.getAVLDefMI(LIS);
343 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI))
344 return;
345 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(MI: *DefMI);
346 if (!DefInstrInfo.hasSameVLMAX(Other: Info))
347 return;
348 Info.setAVL(DefInstrInfo);
349}
350
351// Return a VSETVLIInfo representing the changes made by this VSETVLI or
352// VSETIVLI instruction.
353VSETVLIInfo
354RISCVVSETVLIInfoAnalysis::getInfoForVSETVLI(const MachineInstr &MI) const {
355 VSETVLIInfo NewInfo;
356 if (MI.getOpcode() == RISCV::PseudoVSETIVLI) {
357 NewInfo.setAVLImm(MI.getOperand(i: 1).getImm());
358 } else if (RISCVInstrInfo::isXSfmmVectorConfigTNInstr(MI)) {
359 assert(MI.getOpcode() == RISCV::PseudoSF_VSETTNT ||
360 MI.getOpcode() == RISCV::PseudoSF_VSETTNTX0);
361 switch (MI.getOpcode()) {
362 case RISCV::PseudoSF_VSETTNTX0:
363 NewInfo.setAVLVLMAX();
364 break;
365 case RISCV::PseudoSF_VSETTNT:
366 Register ATNReg = MI.getOperand(i: 1).getReg();
367 NewInfo.setAVLRegDef(VNInfo: getVNInfoFromReg(Reg: ATNReg, MI, LIS), AVLReg: ATNReg);
368 break;
369 }
370 } else {
371 assert(MI.getOpcode() == RISCV::PseudoVSETVLI ||
372 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
373 if (MI.getOpcode() == RISCV::PseudoVSETVLIX0)
374 NewInfo.setAVLVLMAX();
375 else if (MI.getOperand(i: 1).isUndef())
376 // Otherwise use an AVL of 1 to avoid depending on previous vl.
377 NewInfo.setAVLImm(1);
378 else {
379 Register AVLReg = MI.getOperand(i: 1).getReg();
380 VNInfo *VNI = getVNInfoFromReg(Reg: AVLReg, MI, LIS);
381 NewInfo.setAVLRegDef(VNInfo: VNI, AVLReg);
382 }
383 }
384 NewInfo.setVTYPE(MI.getOperand(i: 2).getImm());
385
386 forwardVSETVLIAVL(Info&: NewInfo);
387
388 return NewInfo;
389}
390
391static unsigned computeVLMAX(unsigned VLEN, unsigned SEW,
392 RISCVVType::VLMUL VLMul) {
393 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul);
394 if (Fractional)
395 VLEN = VLEN / LMul;
396 else
397 VLEN = VLEN * LMul;
398 return VLEN / SEW;
399}
400
401VSETVLIInfo
402RISCVVSETVLIInfoAnalysis::computeInfoForInstr(const MachineInstr &MI) const {
403 VSETVLIInfo InstrInfo;
404 const uint64_t TSFlags = MI.getDesc().TSFlags;
405
406 bool TailAgnostic = true;
407 bool MaskAgnostic = true;
408 if (!RISCV::hasUndefinedPassthru(MI)) {
409 // Start with undisturbed.
410 TailAgnostic = false;
411 MaskAgnostic = false;
412
413 // If there is a policy operand, use it.
414 if (RISCVII::hasVecPolicyOp(TSFlags)) {
415 const MachineOperand &Op = MI.getOperand(i: getVecPolicyOpNum(MI));
416 uint64_t Policy = Op.getImm();
417 assert(Policy <=
418 (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&
419 "Invalid Policy Value");
420 TailAgnostic = Policy & RISCVVType::TAIL_AGNOSTIC;
421 MaskAgnostic = Policy & RISCVVType::MASK_AGNOSTIC;
422 }
423
424 if (!RISCVII::usesMaskPolicy(TSFlags))
425 MaskAgnostic = true;
426 }
427
428 RISCVVType::VLMUL VLMul = RISCVII::getLMul(TSFlags);
429
430 bool AltFmt = RISCVII::getAltFmtType(TSFlags) == RISCVII::AltFmtType::AltFmt;
431 InstrInfo.setAltFmt(AltFmt);
432
433 unsigned Log2SEW = MI.getOperand(i: getSEWOpNum(MI)).getImm();
434 // A Log2SEW of 0 is an operation on mask registers only.
435 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
436 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
437
438 if (RISCVII::hasTWidenOp(TSFlags)) {
439 const MachineOperand &TWidenOp =
440 MI.getOperand(i: MI.getNumExplicitOperands() - 1);
441 unsigned TWiden = TWidenOp.getImm();
442
443 InstrInfo.setAVLVLMAX();
444 if (RISCVII::hasVLOp(TSFlags)) {
445 const MachineOperand &TNOp =
446 MI.getOperand(i: RISCVII::getTNOpNum(Desc: MI.getDesc()));
447
448 if (TNOp.getReg().isVirtual())
449 InstrInfo.setAVLRegDef(VNInfo: getVNInfoFromReg(Reg: TNOp.getReg(), MI, LIS),
450 AVLReg: TNOp.getReg());
451 }
452
453 InstrInfo.setVTYPE(L: VLMul, S: SEW, TA: TailAgnostic, MA: MaskAgnostic, Altfmt: AltFmt, W: TWiden);
454
455 return InstrInfo;
456 }
457
458 if (RISCVII::hasVLOp(TSFlags)) {
459 const MachineOperand &VLOp = MI.getOperand(i: getVLOpNum(MI));
460 if (VLOp.isImm()) {
461 int64_t Imm = VLOp.getImm();
462 // Convert the VLMax sentintel to X0 register.
463 if (Imm == RISCV::VLMaxSentinel) {
464 // If we know the exact VLEN, see if we can use the constant encoding
465 // for the VLMAX instead. This reduces register pressure slightly.
466 const unsigned VLMAX = computeVLMAX(VLEN: ST->getRealMaxVLen(), SEW, VLMul);
467 if (ST->getRealMinVLen() == ST->getRealMaxVLen() && VLMAX <= 31)
468 InstrInfo.setAVLImm(VLMAX);
469 else
470 InstrInfo.setAVLVLMAX();
471 } else
472 InstrInfo.setAVLImm(Imm);
473 } else if (VLOp.isUndef()) {
474 // Otherwise use an AVL of 1 to avoid depending on previous vl.
475 InstrInfo.setAVLImm(1);
476 } else {
477 VNInfo *VNI = getVNInfoFromReg(Reg: VLOp.getReg(), MI, LIS);
478 InstrInfo.setAVLRegDef(VNInfo: VNI, AVLReg: VLOp.getReg());
479 }
480 } else {
481 assert(RISCVInstrInfo::isScalarExtractInstr(MI) ||
482 RISCVInstrInfo::isVExtractInstr(MI));
483 // Pick a random value for state tracking purposes, will be ignored via
484 // the demanded fields mechanism
485 InstrInfo.setAVLImm(1);
486 }
487#ifndef NDEBUG
488 if (std::optional<unsigned> EEW = RISCV::getEEWForLoadStore(MI)) {
489 assert(SEW == EEW && "Initial SEW doesn't match expected EEW");
490 }
491#endif
492 // TODO: Propagate the twiden from previous vtype for potential reuse.
493 InstrInfo.setVTYPE(L: VLMul, S: SEW, TA: TailAgnostic, MA: MaskAgnostic, Altfmt: AltFmt,
494 /*TWiden*/ W: 0);
495
496 forwardVSETVLIAVL(Info&: InstrInfo);
497
498 return InstrInfo;
499}
500} // namespace RISCV
501} // namespace llvm
502