1//===- RISCVVSETVLIInfoAnalysis.cpp - VSETVLI Info Analysis ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements an analysis of the vtype/vl information that is needed
10// by RISCVInsertVSETVLI pass and others.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVVSETVLIInfoAnalysis.h"
15#include "RISCVSubtarget.h"
16#include "llvm/CodeGen/LiveIntervals.h"
17
18namespace llvm {
19namespace RISCV {
20
21/// Given a virtual register \p Reg, return the corresponding VNInfo for it.
22/// This will return nullptr if the virtual register is an implicit_def or
23/// if LiveIntervals is not available.
24static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
25 const LiveIntervals *LIS) {
26 assert(Reg.isVirtual());
27 if (!LIS)
28 return nullptr;
29 auto &LI = LIS->getInterval(Reg);
30 SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
31 return LI.getVNInfoBefore(Idx: SI);
32}
33
34static const MachineOperand &getVLOp(const MachineInstr &MI) {
35 return MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
36}
37
38static const MachineOperand &getSEWOp(const MachineInstr &MI) {
39 return MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc()));
40}
41
42static const MachineOperand &getVecPolicyOp(const MachineInstr &MI) {
43 return MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc()));
44}
45
46static const MachineOperand &getTWidenOp(const MachineInstr &MI) {
47 return MI.getOperand(i: RISCVII::getTWidenOpNum(Desc: MI.getDesc()));
48}
49
50/// Get the EEW for a load or store instruction. Return std::nullopt if MI is
51/// not a load or store which ignores SEW.
52static std::optional<unsigned> getEEWForLoadStore(const MachineInstr &MI) {
53 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
54 default:
55 return std::nullopt;
56 case RISCV::VLE8_V:
57 case RISCV::VLSE8_V:
58 case RISCV::VSE8_V:
59 case RISCV::VSSE8_V:
60 return 8;
61 case RISCV::VLE16_V:
62 case RISCV::VLSE16_V:
63 case RISCV::VSE16_V:
64 case RISCV::VSSE16_V:
65 return 16;
66 case RISCV::VLE32_V:
67 case RISCV::VLSE32_V:
68 case RISCV::VSE32_V:
69 case RISCV::VSSE32_V:
70 return 32;
71 case RISCV::VLE64_V:
72 case RISCV::VLSE64_V:
73 case RISCV::VSE64_V:
74 case RISCV::VSSE64_V:
75 return 64;
76 }
77}
78
79/// Return true if this is an operation on mask registers. Note that
80/// this includes both arithmetic/logical ops and load/store (vlm/vsm).
81static bool isMaskRegOp(const MachineInstr &MI) {
82 if (!RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags))
83 return false;
84 const unsigned Log2SEW = getSEWOp(MI).getImm();
85 // A Log2SEW of 0 is an operation on mask registers only.
86 return Log2SEW == 0;
87}
88
89/// Return true if the inactive elements in the result are entirely undefined.
90/// Note that this is different from "agnostic" as defined by the vector
91/// specification. Agnostic requires each lane to either be undisturbed, or
92/// take the value -1; no other value is allowed.
93static bool hasUndefinedPassthru(const MachineInstr &MI) {
94 unsigned UseOpIdx;
95 if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx))
96 // If there is no passthrough operand, then the pass through
97 // lanes are undefined.
98 return true;
99
100 // All undefined passthrus should be $noreg: see
101 // RISCVDAGToDAGISel::doPeepholeNoRegPassThru
102 const MachineOperand &UseMO = MI.getOperand(i: UseOpIdx);
103 return !UseMO.getReg().isValid() || UseMO.isUndef();
104}
105
106static bool isLMUL1OrSmaller(RISCVVType::VLMUL LMUL) {
107 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul: LMUL);
108 return Fractional || LMul == 1;
109}
110
111/// Return true if moving from CurVType to NewVType is
112/// indistinguishable from the perspective of an instruction (or set
113/// of instructions) which use only the Used subfields and properties.
114bool areCompatibleVTYPEs(uint64_t CurVType, uint64_t NewVType,
115 const DemandedFields &Used) {
116 switch (Used.SEW) {
117 case DemandedFields::SEWNone:
118 break;
119 case DemandedFields::SEWEqual:
120 if (RISCVVType::getSEW(VType: CurVType) != RISCVVType::getSEW(VType: NewVType))
121 return false;
122 break;
123 case DemandedFields::SEWGreaterThanOrEqual:
124 if (RISCVVType::getSEW(VType: NewVType) < RISCVVType::getSEW(VType: CurVType))
125 return false;
126 break;
127 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
128 if (RISCVVType::getSEW(VType: NewVType) < RISCVVType::getSEW(VType: CurVType) ||
129 RISCVVType::getSEW(VType: NewVType) >= 64)
130 return false;
131 break;
132 }
133
134 switch (Used.LMUL) {
135 case DemandedFields::LMULNone:
136 break;
137 case DemandedFields::LMULEqual:
138 if (RISCVVType::getVLMUL(VType: CurVType) != RISCVVType::getVLMUL(VType: NewVType))
139 return false;
140 break;
141 case DemandedFields::LMULLessThanOrEqualToM1:
142 if (!isLMUL1OrSmaller(LMUL: RISCVVType::getVLMUL(VType: NewVType)))
143 return false;
144 break;
145 }
146
147 if (Used.SEWLMULRatio) {
148 auto Ratio1 = RISCVVType::getSEWLMULRatio(SEW: RISCVVType::getSEW(VType: CurVType),
149 VLMul: RISCVVType::getVLMUL(VType: CurVType));
150 auto Ratio2 = RISCVVType::getSEWLMULRatio(SEW: RISCVVType::getSEW(VType: NewVType),
151 VLMul: RISCVVType::getVLMUL(VType: NewVType));
152 if (Ratio1 != Ratio2)
153 return false;
154 }
155
156 if (Used.TailPolicy && RISCVVType::isTailAgnostic(VType: CurVType) !=
157 RISCVVType::isTailAgnostic(VType: NewVType))
158 return false;
159 if (Used.MaskPolicy && RISCVVType::isMaskAgnostic(VType: CurVType) !=
160 RISCVVType::isMaskAgnostic(VType: NewVType))
161 return false;
162 if (Used.TWiden && (RISCVVType::hasXSfmmWiden(VType: CurVType) !=
163 RISCVVType::hasXSfmmWiden(VType: NewVType) ||
164 (RISCVVType::hasXSfmmWiden(VType: CurVType) &&
165 RISCVVType::getXSfmmWiden(VType: CurVType) !=
166 RISCVVType::getXSfmmWiden(VType: NewVType))))
167 return false;
168 if (Used.AltFmt &&
169 RISCVVType::isAltFmt(VType: CurVType) != RISCVVType::isAltFmt(VType: NewVType))
170 return false;
171 return true;
172}
173
174/// Return the fields and properties demanded by the provided instruction.
175DemandedFields getDemanded(const MachineInstr &MI, const RISCVSubtarget *ST) {
176 // This function works in coalesceVSETVLI too. We can still use the value of a
177 // SEW, VL, or Policy operand even though it might not be the exact value in
178 // the VL or VTYPE, since we only care about what the instruction originally
179 // demanded.
180
181 // Most instructions don't use any of these subfeilds.
182 DemandedFields Res;
183 // Start conservative if registers are used
184 if (MI.isCall() || MI.isInlineAsm() ||
185 MI.readsRegister(Reg: RISCV::VL, /*TRI=*/nullptr))
186 Res.demandVL();
187 if (MI.isCall() || MI.isInlineAsm() ||
188 MI.readsRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
189 Res.demandVTYPE();
190 // Start conservative on the unlowered form too
191 uint64_t TSFlags = MI.getDesc().TSFlags;
192 if (RISCVII::hasSEWOp(TSFlags)) {
193 Res.demandVTYPE();
194 if (RISCVII::hasVLOp(TSFlags))
195 if (const MachineOperand &VLOp = getVLOp(MI);
196 !VLOp.isReg() || !VLOp.isUndef())
197 Res.demandVL();
198
199 // Behavior is independent of mask policy.
200 if (!RISCVII::usesMaskPolicy(TSFlags))
201 Res.MaskPolicy = false;
202 }
203
204 // Loads and stores with implicit EEW do not demand SEW or LMUL directly.
205 // They instead demand the ratio of the two which is used in computing
206 // EMUL, but which allows us the flexibility to change SEW and LMUL
207 // provided we don't change the ratio.
208 // Note: We assume that the instructions initial SEW is the EEW encoded
209 // in the opcode. This is asserted when constructing the VSETVLIInfo.
210 if (RISCV::getEEWForLoadStore(MI)) {
211 Res.SEW = DemandedFields::SEWNone;
212 Res.LMUL = DemandedFields::LMULNone;
213 }
214
215 // Store instructions don't use the policy fields.
216 if (RISCVII::hasSEWOp(TSFlags) && MI.getNumExplicitDefs() == 0) {
217 Res.TailPolicy = false;
218 Res.MaskPolicy = false;
219 }
220
221 // If this is a mask reg operation, it only cares about VLMAX.
222 // TODO: Possible extensions to this logic
223 // * Probably ok if available VLMax is larger than demanded
224 // * The policy bits can probably be ignored..
225 if (isMaskRegOp(MI)) {
226 Res.SEW = DemandedFields::SEWNone;
227 Res.LMUL = DemandedFields::LMULNone;
228 }
229
230 // For vmv.s.x and vfmv.s.f, there are only two behaviors, VL = 0 and VL > 0.
231 if (RISCVInstrInfo::isScalarInsertInstr(MI)) {
232 Res.LMUL = DemandedFields::LMULNone;
233 Res.SEWLMULRatio = false;
234 Res.VLAny = false;
235 // For vmv.s.x and vfmv.s.f, if the passthru is *undefined*, we don't
236 // need to preserve any other bits and are thus compatible with any larger,
237 // etype and can disregard policy bits. Warning: It's tempting to try doing
238 // this for any tail agnostic operation, but we can't as TA requires
239 // tail lanes to either be the original value or -1. We are writing
240 // unknown bits to the lanes here.
241 if (RISCV::hasUndefinedPassthru(MI)) {
242 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&
243 !ST->hasVInstructionsF64())
244 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
245 else
246 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
247 Res.TailPolicy = false;
248 }
249 }
250
251 // vmv.x.s, and vfmv.f.s are unconditional and ignore everything except SEW.
252 if (RISCVInstrInfo::isScalarExtractInstr(MI)) {
253 assert(!RISCVII::hasVLOp(TSFlags));
254 Res.LMUL = DemandedFields::LMULNone;
255 Res.SEWLMULRatio = false;
256 Res.TailPolicy = false;
257 Res.MaskPolicy = false;
258 }
259
260 if (RISCVII::hasVLOp(TSFlags: MI.getDesc().TSFlags)) {
261 const MachineOperand &VLOp = getVLOp(MI);
262 // A slidedown/slideup with an *undefined* passthru can freely clobber
263 // elements not copied from the source vector (e.g. masked off, tail, or
264 // slideup's prefix). Notes:
265 // * We can't modify SEW here since the slide amount is in units of SEW.
266 // * VL=1 is special only because we have existing support for zero vs
267 // non-zero VL. We could generalize this if we had a VL > C predicate.
268 // * The LMUL1 restriction is for machines whose latency may depend on LMUL.
269 // * As above, this is only legal for tail "undefined" not "agnostic".
270 // * We avoid increasing vl if the subtarget has +vl-dependent-latency
271 if (RISCVInstrInfo::isVSlideInstr(MI) && VLOp.isImm() &&
272 VLOp.getImm() == 1 && RISCV::hasUndefinedPassthru(MI) &&
273 !ST->hasVLDependentLatency()) {
274 Res.VLAny = false;
275 Res.VLZeroness = true;
276 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;
277 Res.TailPolicy = false;
278 }
279
280 // A tail undefined vmv.v.i/x or vfmv.v.f with VL=1 can be treated in the
281 // same semantically as vmv.s.x. This is particularly useful since we don't
282 // have an immediate form of vmv.s.x, and thus frequently use vmv.v.i in
283 // it's place. Since a splat is non-constant time in LMUL, we do need to be
284 // careful to not increase the number of active vector registers (unlike for
285 // vmv.s.x.)
286 if (RISCVInstrInfo::isScalarSplatInstr(MI) && VLOp.isImm() &&
287 VLOp.getImm() == 1 && RISCV::hasUndefinedPassthru(MI) &&
288 !ST->hasVLDependentLatency()) {
289 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;
290 Res.SEWLMULRatio = false;
291 Res.VLAny = false;
292 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&
293 !ST->hasVInstructionsF64())
294 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
295 else
296 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
297 Res.TailPolicy = false;
298 }
299 }
300
301 // In §32.16.6, whole vector register moves have a dependency on SEW. At the
302 // MIR level though we don't encode the element type, and it gives the same
303 // result whatever the SEW may be.
304 //
305 // However it does need valid SEW, i.e. vill must be cleared. The entry to a
306 // function, calls and inline assembly may all set it, so make sure we clear
307 // it for whole register copies. Do this by leaving VILL demanded.
308 if (RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI)) {
309 Res.LMUL = DemandedFields::LMULNone;
310 Res.SEW = DemandedFields::SEWNone;
311 Res.SEWLMULRatio = false;
312 Res.TailPolicy = false;
313 Res.MaskPolicy = false;
314 }
315
316 Res.AltFmt = RISCVII::getAltFmtType(TSFlags: MI.getDesc().TSFlags) !=
317 RISCVII::AltFmtType::DontCare;
318 Res.TWiden = RISCVII::hasTWidenOp(TSFlags: MI.getDesc().TSFlags) ||
319 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI);
320
321 return Res;
322}
323
324bool VSETVLIInfo::hasCompatibleVTYPE(const DemandedFields &Used,
325 const VSETVLIInfo &Require) const {
326 return areCompatibleVTYPEs(CurVType: Require.encodeVTYPE(), NewVType: encodeVTYPE(), Used);
327}
328
329// If the AVL is defined by a vsetvli's output vl with the same VLMAX, we can
330// replace the AVL operand with the AVL of the defining vsetvli. E.g.
331//
332// %vl = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
333// $x0 = PseudoVSETVLI %vl:gpr, SEW=32, LMUL=M1
334// ->
335// %vl = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
336// $x0 = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
337void RISCVVSETVLIInfoAnalysis::forwardVSETVLIAVL(VSETVLIInfo &Info) const {
338 if (!Info.hasAVLReg())
339 return;
340 const MachineInstr *DefMI = Info.getAVLDefMI(LIS);
341 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI))
342 return;
343 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(MI: *DefMI);
344 if (!DefInstrInfo.hasSameVLMAX(Other: Info))
345 return;
346 Info.setAVL(DefInstrInfo);
347}
348
349// Return a VSETVLIInfo representing the changes made by this VSETVLI or
350// VSETIVLI instruction.
351VSETVLIInfo
352RISCVVSETVLIInfoAnalysis::getInfoForVSETVLI(const MachineInstr &MI) const {
353 VSETVLIInfo NewInfo;
354 if (MI.getOpcode() == RISCV::PseudoVSETIVLI) {
355 NewInfo.setAVLImm(MI.getOperand(i: 1).getImm());
356 } else if (RISCVInstrInfo::isXSfmmVectorConfigTNInstr(MI)) {
357 switch (MI.getOpcode()) {
358 default:
359 llvm_unreachable("Unexpected opcode");
360 case RISCV::PseudoSF_VSETTNTX0:
361 NewInfo.setAVLVLMAX();
362 break;
363 case RISCV::PseudoSF_VSETTNT:
364 Register ATNReg = MI.getOperand(i: 1).getReg();
365 NewInfo.setAVLRegDef(VNInfo: getVNInfoFromReg(Reg: ATNReg, MI, LIS), AVLReg: ATNReg);
366 break;
367 }
368 } else {
369 assert(MI.getOpcode() == RISCV::PseudoVSETVLI ||
370 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
371 if (MI.getOpcode() == RISCV::PseudoVSETVLIX0)
372 NewInfo.setAVLVLMAX();
373 else if (MI.getOperand(i: 1).isUndef())
374 // Otherwise use an AVL of 1 to avoid depending on previous vl.
375 NewInfo.setAVLImm(1);
376 else {
377 Register AVLReg = MI.getOperand(i: 1).getReg();
378 VNInfo *VNI = getVNInfoFromReg(Reg: AVLReg, MI, LIS);
379 NewInfo.setAVLRegDef(VNInfo: VNI, AVLReg);
380 }
381 }
382 NewInfo.setVTYPE(MI.getOperand(i: 2).getImm());
383
384 forwardVSETVLIAVL(Info&: NewInfo);
385
386 return NewInfo;
387}
388
389static unsigned computeVLMAX(unsigned VLEN, unsigned SEW,
390 RISCVVType::VLMUL VLMul) {
391 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul);
392 if (Fractional)
393 VLEN = VLEN / LMul;
394 else
395 VLEN = VLEN * LMul;
396 return VLEN / SEW;
397}
398
399VSETVLIInfo
400RISCVVSETVLIInfoAnalysis::computeInfoForInstr(const MachineInstr &MI) const {
401 VSETVLIInfo InstrInfo;
402 const uint64_t TSFlags = MI.getDesc().TSFlags;
403
404 bool TailAgnostic = true;
405 bool MaskAgnostic = true;
406 if (!RISCV::hasUndefinedPassthru(MI)) {
407 // Start with undisturbed.
408 TailAgnostic = false;
409 MaskAgnostic = false;
410
411 // If there is a policy operand, use it.
412 if (RISCVII::hasVecPolicyOp(TSFlags)) {
413 const MachineOperand &Op = getVecPolicyOp(MI);
414 uint64_t Policy = Op.getImm();
415 assert(Policy <=
416 (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&
417 "Invalid Policy Value");
418 TailAgnostic = Policy & RISCVVType::TAIL_AGNOSTIC;
419 MaskAgnostic = Policy & RISCVVType::MASK_AGNOSTIC;
420 }
421
422 if (!RISCVII::usesMaskPolicy(TSFlags))
423 MaskAgnostic = true;
424 }
425
426 RISCVVType::VLMUL VLMul = RISCVII::getLMul(TSFlags);
427
428 bool AltFmt = RISCVII::getAltFmtType(TSFlags) == RISCVII::AltFmtType::AltFmt;
429 InstrInfo.setAltFmt(AltFmt);
430
431 unsigned Log2SEW = getSEWOp(MI).getImm();
432 // A Log2SEW of 0 is an operation on mask registers only.
433 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
434 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
435
436 if (RISCVII::hasTWidenOp(TSFlags)) {
437 const MachineOperand &TWidenOp = getTWidenOp(MI);
438 unsigned TWiden = TWidenOp.getImm();
439
440 InstrInfo.setAVLVLMAX();
441 if (RISCVII::hasVLOp(TSFlags)) {
442 const MachineOperand &TNOp =
443 MI.getOperand(i: RISCVII::getTNOpNum(Desc: MI.getDesc()));
444
445 if (TNOp.getReg().isVirtual())
446 InstrInfo.setAVLRegDef(VNInfo: getVNInfoFromReg(Reg: TNOp.getReg(), MI, LIS),
447 AVLReg: TNOp.getReg());
448 }
449
450 InstrInfo.setVTYPE(L: VLMul, S: SEW, TA: TailAgnostic, MA: MaskAgnostic, Altfmt: AltFmt, W: TWiden);
451
452 return InstrInfo;
453 }
454
455 if (RISCVII::hasVLOp(TSFlags)) {
456 const MachineOperand &VLOp = getVLOp(MI);
457 if (VLOp.isImm()) {
458 int64_t Imm = VLOp.getImm();
459 // Convert the VLMax sentintel to X0 register.
460 if (Imm == RISCV::VLMaxSentinel) {
461 // If we know the exact VLEN, see if we can use the constant encoding
462 // for the VLMAX instead. This reduces register pressure slightly.
463 const unsigned VLMAX = computeVLMAX(VLEN: ST->getRealMaxVLen(), SEW, VLMul);
464 if (ST->getRealMinVLen() == ST->getRealMaxVLen() && VLMAX <= 31)
465 InstrInfo.setAVLImm(VLMAX);
466 else
467 InstrInfo.setAVLVLMAX();
468 } else
469 InstrInfo.setAVLImm(Imm);
470 } else if (VLOp.isUndef()) {
471 // Otherwise use an AVL of 1 to avoid depending on previous vl.
472 InstrInfo.setAVLImm(1);
473 } else {
474 VNInfo *VNI = getVNInfoFromReg(Reg: VLOp.getReg(), MI, LIS);
475 InstrInfo.setAVLRegDef(VNInfo: VNI, AVLReg: VLOp.getReg());
476 }
477 } else {
478 assert(RISCVInstrInfo::isScalarExtractInstr(MI));
479 // Pick a random value for state tracking purposes, will be ignored via
480 // the demanded fields mechanism
481 InstrInfo.setAVLImm(1);
482 }
483#ifndef NDEBUG
484 if (std::optional<unsigned> EEW = RISCV::getEEWForLoadStore(MI)) {
485 assert(SEW == EEW && "Initial SEW doesn't match expected EEW");
486 }
487#endif
488 // TODO: Propagate the twiden from previous vtype for potential reuse.
489 InstrInfo.setVTYPE(L: VLMul, S: SEW, TA: TailAgnostic, MA: MaskAgnostic, Altfmt: AltFmt,
490 /*TWiden*/ W: 0);
491
492 forwardVSETVLIAVL(Info&: InstrInfo);
493
494 return InstrInfo;
495}
496} // namespace RISCV
497} // namespace llvm
498