1//===- RISCVVSETVLIInfoAnalysis.cpp - VSETVLI Info Analysis ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements an analysis of the vtype/vl information that is needed
10// by RISCVInsertVSETVLI pass and others.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVVSETVLIInfoAnalysis.h"
15#include "RISCVSubtarget.h"
16#include "llvm/CodeGen/LiveIntervals.h"
17
18namespace llvm {
19namespace RISCV {
20
21/// Given a virtual register \p Reg, return the corresponding VNInfo for it.
22/// This will return nullptr if the virtual register is an implicit_def or
23/// if LiveIntervals is not available.
24static VNInfo *getVNInfoFromReg(Register Reg, const MachineInstr &MI,
25 const LiveIntervals *LIS) {
26 assert(Reg.isVirtual());
27 if (!LIS)
28 return nullptr;
29 auto &LI = LIS->getInterval(Reg);
30 SlotIndex SI = LIS->getSlotIndexes()->getInstructionIndex(MI);
31 return LI.getVNInfoBefore(Idx: SI);
32}
33
34static const MachineOperand &getVLOp(const MachineInstr &MI) {
35 return MI.getOperand(i: RISCVII::getVLOpNum(Desc: MI.getDesc()));
36}
37
38static const MachineOperand &getSEWOp(const MachineInstr &MI) {
39 return MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc()));
40}
41
42static const MachineOperand &getVecPolicyOp(const MachineInstr &MI) {
43 return MI.getOperand(i: RISCVII::getVecPolicyOpNum(Desc: MI.getDesc()));
44}
45
46static const MachineOperand &getTWidenOp(const MachineInstr &MI) {
47 return MI.getOperand(i: RISCVII::getTWidenOpNum(Desc: MI.getDesc()));
48}
49
50/// Get the EEW for a load or store instruction. Return std::nullopt if MI is
51/// not a load or store which ignores SEW.
52static std::optional<unsigned> getEEWForLoadStore(const MachineInstr &MI) {
53 switch (RISCV::getRVVMCOpcode(RVVPseudoOpcode: MI.getOpcode())) {
54 default:
55 return std::nullopt;
56 case RISCV::VLE8_V:
57 case RISCV::VLSE8_V:
58 case RISCV::VSE8_V:
59 case RISCV::VSSE8_V:
60 return 8;
61 case RISCV::VLE16_V:
62 case RISCV::VLSE16_V:
63 case RISCV::VSE16_V:
64 case RISCV::VSSE16_V:
65 return 16;
66 case RISCV::VLE32_V:
67 case RISCV::VLSE32_V:
68 case RISCV::VSE32_V:
69 case RISCV::VSSE32_V:
70 return 32;
71 case RISCV::VLE64_V:
72 case RISCV::VLSE64_V:
73 case RISCV::VSE64_V:
74 case RISCV::VSSE64_V:
75 return 64;
76 }
77}
78
79/// Return true if this is an operation on mask registers. Note that
80/// this includes both arithmetic/logical ops and load/store (vlm/vsm).
81static bool isMaskRegOp(const MachineInstr &MI) {
82 if (!RISCVII::hasSEWOp(TSFlags: MI.getDesc().TSFlags))
83 return false;
84 const unsigned Log2SEW = getSEWOp(MI).getImm();
85 // A Log2SEW of 0 is an operation on mask registers only.
86 return Log2SEW == 0;
87}
88
89/// Return true if the inactive elements in the result are entirely undefined.
90/// Note that this is different from "agnostic" as defined by the vector
91/// specification. Agnostic requires each lane to either be undisturbed, or
92/// take the value -1; no other value is allowed.
93static bool hasUndefinedPassthru(const MachineInstr &MI) {
94 unsigned UseOpIdx;
95 if (!MI.isRegTiedToUseOperand(DefOpIdx: 0, UseOpIdx: &UseOpIdx))
96 // If there is no passthrough operand, then the pass through
97 // lanes are undefined.
98 return true;
99
100 // All undefined passthrus should be $noreg: see
101 // RISCVDAGToDAGISel::doPeepholeNoRegPassThru
102 const MachineOperand &UseMO = MI.getOperand(i: UseOpIdx);
103 return !UseMO.getReg().isValid() || UseMO.isUndef();
104}
105
106static bool isLMUL1OrSmaller(RISCVVType::VLMUL LMUL) {
107 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul: LMUL);
108 return Fractional || LMul == 1;
109}
110
111/// Return true if moving from CurVType to NewVType is
112/// indistinguishable from the perspective of an instruction (or set
113/// of instructions) which use only the Used subfields and properties.
114bool areCompatibleVTYPEs(uint64_t CurVType, uint64_t NewVType,
115 const DemandedFields &Used) {
116 switch (Used.SEW) {
117 case DemandedFields::SEWNone:
118 break;
119 case DemandedFields::SEWEqual:
120 if (RISCVVType::getSEW(VType: CurVType) != RISCVVType::getSEW(VType: NewVType))
121 return false;
122 break;
123 case DemandedFields::SEWGreaterThanOrEqual:
124 if (RISCVVType::getSEW(VType: NewVType) < RISCVVType::getSEW(VType: CurVType))
125 return false;
126 break;
127 case DemandedFields::SEWGreaterThanOrEqualAndLessThan64:
128 if (RISCVVType::getSEW(VType: NewVType) < RISCVVType::getSEW(VType: CurVType) ||
129 RISCVVType::getSEW(VType: NewVType) >= 64)
130 return false;
131 break;
132 }
133
134 switch (Used.LMUL) {
135 case DemandedFields::LMULNone:
136 break;
137 case DemandedFields::LMULEqual:
138 if (RISCVVType::getVLMUL(VType: CurVType) != RISCVVType::getVLMUL(VType: NewVType))
139 return false;
140 break;
141 case DemandedFields::LMULLessThanOrEqualToM1:
142 if (!isLMUL1OrSmaller(LMUL: RISCVVType::getVLMUL(VType: NewVType)))
143 return false;
144 break;
145 }
146
147 if (Used.SEWLMULRatio) {
148 auto Ratio1 = RISCVVType::getSEWLMULRatio(SEW: RISCVVType::getSEW(VType: CurVType),
149 VLMul: RISCVVType::getVLMUL(VType: CurVType));
150 auto Ratio2 = RISCVVType::getSEWLMULRatio(SEW: RISCVVType::getSEW(VType: NewVType),
151 VLMul: RISCVVType::getVLMUL(VType: NewVType));
152 if (Ratio1 != Ratio2)
153 return false;
154 }
155
156 if (Used.TailPolicy && RISCVVType::isTailAgnostic(VType: CurVType) !=
157 RISCVVType::isTailAgnostic(VType: NewVType))
158 return false;
159 if (Used.MaskPolicy && RISCVVType::isMaskAgnostic(VType: CurVType) !=
160 RISCVVType::isMaskAgnostic(VType: NewVType))
161 return false;
162 if (Used.TWiden && (RISCVVType::hasXSfmmWiden(VType: CurVType) !=
163 RISCVVType::hasXSfmmWiden(VType: NewVType) ||
164 (RISCVVType::hasXSfmmWiden(VType: CurVType) &&
165 RISCVVType::getXSfmmWiden(VType: CurVType) !=
166 RISCVVType::getXSfmmWiden(VType: NewVType))))
167 return false;
168 if (Used.AltFmt &&
169 RISCVVType::isAltFmt(VType: CurVType) != RISCVVType::isAltFmt(VType: NewVType))
170 return false;
171 return true;
172}
173
174/// Return the fields and properties demanded by the provided instruction.
175DemandedFields getDemanded(const MachineInstr &MI, const RISCVSubtarget *ST) {
176 // This function works in coalesceVSETVLI too. We can still use the value of a
177 // SEW, VL, or Policy operand even though it might not be the exact value in
178 // the VL or VTYPE, since we only care about what the instruction originally
179 // demanded.
180
181 // Most instructions don't use any of these subfeilds.
182 DemandedFields Res;
183 // Start conservative if registers are used
184 if (MI.isCall() || MI.isInlineAsm() ||
185 MI.readsRegister(Reg: RISCV::VL, /*TRI=*/nullptr))
186 Res.demandVL();
187 if (MI.isCall() || MI.isInlineAsm() ||
188 MI.readsRegister(Reg: RISCV::VTYPE, /*TRI=*/nullptr))
189 Res.demandVTYPE();
190 // Start conservative on the unlowered form too
191 uint64_t TSFlags = MI.getDesc().TSFlags;
192 if (RISCVII::hasSEWOp(TSFlags)) {
193 Res.demandVTYPE();
194 if (RISCVII::hasVLOp(TSFlags))
195 if (const MachineOperand &VLOp = getVLOp(MI);
196 !VLOp.isReg() || !VLOp.isUndef())
197 Res.demandVL();
198
199 // Behavior is independent of mask policy.
200 if (!RISCVII::usesMaskPolicy(TSFlags))
201 Res.MaskPolicy = false;
202 }
203
204 // Loads and stores with implicit EEW do not demand SEW or LMUL directly.
205 // They instead demand the ratio of the two which is used in computing
206 // EMUL, but which allows us the flexibility to change SEW and LMUL
207 // provided we don't change the ratio.
208 // Note: We assume that the instructions initial SEW is the EEW encoded
209 // in the opcode. This is asserted when constructing the VSETVLIInfo.
210 if (RISCV::getEEWForLoadStore(MI)) {
211 Res.SEW = DemandedFields::SEWNone;
212 Res.LMUL = DemandedFields::LMULNone;
213 }
214
215 // Store instructions don't use the policy fields.
216 if (RISCVII::hasSEWOp(TSFlags) && MI.getNumExplicitDefs() == 0) {
217 Res.TailPolicy = false;
218 Res.MaskPolicy = false;
219 }
220
221 // If this is a mask reg operation, it only cares about VLMAX.
222 // TODO: Possible extensions to this logic
223 // * Probably ok if available VLMax is larger than demanded
224 // * The policy bits can probably be ignored..
225 if (isMaskRegOp(MI)) {
226 Res.SEW = DemandedFields::SEWNone;
227 Res.LMUL = DemandedFields::LMULNone;
228 }
229
230 // For vmv.s.x and vfmv.s.f, there are only two behaviors, VL = 0 and VL > 0.
231 if (RISCVInstrInfo::isScalarInsertInstr(MI)) {
232 Res.LMUL = DemandedFields::LMULNone;
233 Res.SEWLMULRatio = false;
234 Res.VLAny = false;
235 // For vmv.s.x and vfmv.s.f, if the passthru is *undefined*, we don't
236 // need to preserve any other bits and are thus compatible with any larger,
237 // etype and can disregard policy bits. Warning: It's tempting to try doing
238 // this for any tail agnostic operation, but we can't as TA requires
239 // tail lanes to either be the original value or -1. We are writing
240 // unknown bits to the lanes here.
241 if (RISCV::hasUndefinedPassthru(MI)) {
242 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&
243 !ST->hasVInstructionsF64())
244 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
245 else
246 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
247 Res.TailPolicy = false;
248 }
249 }
250
251 // vmv.x.s, and vfmv.f.s are unconditional and ignore everything except SEW.
252 if (RISCVInstrInfo::isScalarExtractInstr(MI)) {
253 assert(!RISCVII::hasVLOp(TSFlags));
254 Res.LMUL = DemandedFields::LMULNone;
255 Res.SEWLMULRatio = false;
256 Res.TailPolicy = false;
257 Res.MaskPolicy = false;
258 }
259
260 if (RISCVII::hasVLOp(TSFlags: MI.getDesc().TSFlags)) {
261 const MachineOperand &VLOp = getVLOp(MI);
262 // A slidedown/slideup with an *undefined* passthru can freely clobber
263 // elements not copied from the source vector (e.g. masked off, tail, or
264 // slideup's prefix). Notes:
265 // * We can't modify SEW here since the slide amount is in units of SEW.
266 // * VL=1 is special only because we have existing support for zero vs
267 // non-zero VL. We could generalize this if we had a VL > C predicate.
268 // * The LMUL1 restriction is for machines whose latency may depend on LMUL.
269 // * As above, this is only legal for tail "undefined" not "agnostic".
270 // * We avoid increasing vl if the subtarget has +vl-dependent-latency
271 if (RISCVInstrInfo::isVSlideInstr(MI) && VLOp.isImm() &&
272 VLOp.getImm() == 1 && RISCV::hasUndefinedPassthru(MI) &&
273 !ST->hasVLDependentLatency()) {
274 Res.VLAny = false;
275 Res.VLZeroness = true;
276 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;
277 Res.TailPolicy = false;
278 }
279
280 // A tail undefined vmv.v.i/x or vfmv.v.f with VL=1 can be treated in the
281 // same semantically as vmv.s.x. This is particularly useful since we don't
282 // have an immediate form of vmv.s.x, and thus frequently use vmv.v.i in
283 // it's place. Since a splat is non-constant time in LMUL, we do need to be
284 // careful to not increase the number of active vector registers (unlike for
285 // vmv.s.x.)
286 if (RISCVInstrInfo::isScalarSplatInstr(MI) && VLOp.isImm() &&
287 VLOp.getImm() == 1 && RISCV::hasUndefinedPassthru(MI) &&
288 !ST->hasVLDependentLatency()) {
289 Res.LMUL = DemandedFields::LMULLessThanOrEqualToM1;
290 Res.SEWLMULRatio = false;
291 Res.VLAny = false;
292 if (RISCVInstrInfo::isFloatScalarMoveOrScalarSplatInstr(MI) &&
293 !ST->hasVInstructionsF64())
294 Res.SEW = DemandedFields::SEWGreaterThanOrEqualAndLessThan64;
295 else
296 Res.SEW = DemandedFields::SEWGreaterThanOrEqual;
297 Res.TailPolicy = false;
298 }
299 }
300
301 // In §32.16.6, whole vector register moves have a dependency on SEW. At the
302 // MIR level though we don't encode the element type, and it gives the same
303 // result whatever the SEW may be.
304 //
305 // However it does need valid SEW, i.e. vill must be cleared. The entry to a
306 // function, calls and inline assembly may all set it, so make sure we clear
307 // it for whole register copies. Do this by leaving VILL demanded.
308 if (RISCV::isVectorCopy(TRI: ST->getRegisterInfo(), MI)) {
309 Res.LMUL = DemandedFields::LMULNone;
310 Res.SEW = DemandedFields::SEWNone;
311 Res.SEWLMULRatio = false;
312 Res.TailPolicy = false;
313 Res.MaskPolicy = false;
314 }
315
316 if (RISCVInstrInfo::isVExtractInstr(MI)) {
317 assert(!RISCVII::hasVLOp(TSFlags));
318 // TODO: LMUL can be any larger value (without cost)
319 Res.TailPolicy = false;
320 }
321
322 Res.AltFmt = RISCVII::getAltFmtType(TSFlags: MI.getDesc().TSFlags) !=
323 RISCVII::AltFmtType::DontCare;
324 Res.TWiden = RISCVII::hasTWidenOp(TSFlags: MI.getDesc().TSFlags) ||
325 RISCVInstrInfo::isXSfmmVectorConfigInstr(MI);
326
327 return Res;
328}
329
330bool VSETVLIInfo::hasCompatibleVTYPE(const DemandedFields &Used,
331 const VSETVLIInfo &Require) const {
332 return areCompatibleVTYPEs(CurVType: Require.encodeVTYPE(), NewVType: encodeVTYPE(), Used);
333}
334
335// If the AVL is defined by a vsetvli's output vl with the same VLMAX, we can
336// replace the AVL operand with the AVL of the defining vsetvli. E.g.
337//
338// %vl = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
339// $x0 = PseudoVSETVLI %vl:gpr, SEW=32, LMUL=M1
340// ->
341// %vl = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
342// $x0 = PseudoVSETVLI %avl:gpr, SEW=32, LMUL=M1
343void RISCVVSETVLIInfoAnalysis::forwardVSETVLIAVL(VSETVLIInfo &Info) const {
344 if (!Info.hasAVLReg())
345 return;
346 const MachineInstr *DefMI = Info.getAVLDefMI(LIS);
347 if (!DefMI || !RISCVInstrInfo::isVectorConfigInstr(MI: *DefMI))
348 return;
349 VSETVLIInfo DefInstrInfo = getInfoForVSETVLI(MI: *DefMI);
350 if (!DefInstrInfo.hasSameVLMAX(Other: Info))
351 return;
352 Info.setAVL(DefInstrInfo);
353}
354
355// Return a VSETVLIInfo representing the changes made by this VSETVLI or
356// VSETIVLI instruction.
357VSETVLIInfo
358RISCVVSETVLIInfoAnalysis::getInfoForVSETVLI(const MachineInstr &MI) const {
359 VSETVLIInfo NewInfo;
360 if (MI.getOpcode() == RISCV::PseudoVSETIVLI) {
361 NewInfo.setAVLImm(MI.getOperand(i: 1).getImm());
362 } else if (RISCVInstrInfo::isXSfmmVectorConfigTNInstr(MI)) {
363 assert(MI.getOpcode() == RISCV::PseudoSF_VSETTNT ||
364 MI.getOpcode() == RISCV::PseudoSF_VSETTNTX0);
365 switch (MI.getOpcode()) {
366 case RISCV::PseudoSF_VSETTNTX0:
367 NewInfo.setAVLVLMAX();
368 break;
369 case RISCV::PseudoSF_VSETTNT:
370 Register ATNReg = MI.getOperand(i: 1).getReg();
371 NewInfo.setAVLRegDef(VNInfo: getVNInfoFromReg(Reg: ATNReg, MI, LIS), AVLReg: ATNReg);
372 break;
373 }
374 } else {
375 assert(MI.getOpcode() == RISCV::PseudoVSETVLI ||
376 MI.getOpcode() == RISCV::PseudoVSETVLIX0);
377 if (MI.getOpcode() == RISCV::PseudoVSETVLIX0)
378 NewInfo.setAVLVLMAX();
379 else if (MI.getOperand(i: 1).isUndef())
380 // Otherwise use an AVL of 1 to avoid depending on previous vl.
381 NewInfo.setAVLImm(1);
382 else {
383 Register AVLReg = MI.getOperand(i: 1).getReg();
384 VNInfo *VNI = getVNInfoFromReg(Reg: AVLReg, MI, LIS);
385 NewInfo.setAVLRegDef(VNInfo: VNI, AVLReg);
386 }
387 }
388 NewInfo.setVTYPE(MI.getOperand(i: 2).getImm());
389
390 forwardVSETVLIAVL(Info&: NewInfo);
391
392 return NewInfo;
393}
394
395static unsigned computeVLMAX(unsigned VLEN, unsigned SEW,
396 RISCVVType::VLMUL VLMul) {
397 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul);
398 if (Fractional)
399 VLEN = VLEN / LMul;
400 else
401 VLEN = VLEN * LMul;
402 return VLEN / SEW;
403}
404
405VSETVLIInfo
406RISCVVSETVLIInfoAnalysis::computeInfoForInstr(const MachineInstr &MI) const {
407 VSETVLIInfo InstrInfo;
408 const uint64_t TSFlags = MI.getDesc().TSFlags;
409
410 bool TailAgnostic = true;
411 bool MaskAgnostic = true;
412 if (!RISCV::hasUndefinedPassthru(MI)) {
413 // Start with undisturbed.
414 TailAgnostic = false;
415 MaskAgnostic = false;
416
417 // If there is a policy operand, use it.
418 if (RISCVII::hasVecPolicyOp(TSFlags)) {
419 const MachineOperand &Op = getVecPolicyOp(MI);
420 uint64_t Policy = Op.getImm();
421 assert(Policy <=
422 (RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC) &&
423 "Invalid Policy Value");
424 TailAgnostic = Policy & RISCVVType::TAIL_AGNOSTIC;
425 MaskAgnostic = Policy & RISCVVType::MASK_AGNOSTIC;
426 }
427
428 if (!RISCVII::usesMaskPolicy(TSFlags))
429 MaskAgnostic = true;
430 }
431
432 RISCVVType::VLMUL VLMul = RISCVII::getLMul(TSFlags);
433
434 bool AltFmt = RISCVII::getAltFmtType(TSFlags) == RISCVII::AltFmtType::AltFmt;
435 InstrInfo.setAltFmt(AltFmt);
436
437 unsigned Log2SEW = getSEWOp(MI).getImm();
438 // A Log2SEW of 0 is an operation on mask registers only.
439 unsigned SEW = Log2SEW ? 1 << Log2SEW : 8;
440 assert(RISCVVType::isValidSEW(SEW) && "Unexpected SEW");
441
442 if (RISCVII::hasTWidenOp(TSFlags)) {
443 const MachineOperand &TWidenOp = getTWidenOp(MI);
444 unsigned TWiden = TWidenOp.getImm();
445
446 InstrInfo.setAVLVLMAX();
447 if (RISCVII::hasVLOp(TSFlags)) {
448 const MachineOperand &TNOp =
449 MI.getOperand(i: RISCVII::getTNOpNum(Desc: MI.getDesc()));
450
451 if (TNOp.getReg().isVirtual())
452 InstrInfo.setAVLRegDef(VNInfo: getVNInfoFromReg(Reg: TNOp.getReg(), MI, LIS),
453 AVLReg: TNOp.getReg());
454 }
455
456 InstrInfo.setVTYPE(L: VLMul, S: SEW, TA: TailAgnostic, MA: MaskAgnostic, Altfmt: AltFmt, W: TWiden);
457
458 return InstrInfo;
459 }
460
461 if (RISCVII::hasVLOp(TSFlags)) {
462 const MachineOperand &VLOp = getVLOp(MI);
463 if (VLOp.isImm()) {
464 int64_t Imm = VLOp.getImm();
465 // Convert the VLMax sentintel to X0 register.
466 if (Imm == RISCV::VLMaxSentinel) {
467 // If we know the exact VLEN, see if we can use the constant encoding
468 // for the VLMAX instead. This reduces register pressure slightly.
469 const unsigned VLMAX = computeVLMAX(VLEN: ST->getRealMaxVLen(), SEW, VLMul);
470 if (ST->getRealMinVLen() == ST->getRealMaxVLen() && VLMAX <= 31)
471 InstrInfo.setAVLImm(VLMAX);
472 else
473 InstrInfo.setAVLVLMAX();
474 } else
475 InstrInfo.setAVLImm(Imm);
476 } else if (VLOp.isUndef()) {
477 // Otherwise use an AVL of 1 to avoid depending on previous vl.
478 InstrInfo.setAVLImm(1);
479 } else {
480 VNInfo *VNI = getVNInfoFromReg(Reg: VLOp.getReg(), MI, LIS);
481 InstrInfo.setAVLRegDef(VNInfo: VNI, AVLReg: VLOp.getReg());
482 }
483 } else {
484 assert(RISCVInstrInfo::isScalarExtractInstr(MI) ||
485 RISCVInstrInfo::isVExtractInstr(MI));
486 // Pick a random value for state tracking purposes, will be ignored via
487 // the demanded fields mechanism
488 InstrInfo.setAVLImm(1);
489 }
490#ifndef NDEBUG
491 if (std::optional<unsigned> EEW = RISCV::getEEWForLoadStore(MI)) {
492 assert(SEW == EEW && "Initial SEW doesn't match expected EEW");
493 }
494#endif
495 // TODO: Propagate the twiden from previous vtype for potential reuse.
496 InstrInfo.setVTYPE(L: VLMul, S: SEW, TA: TailAgnostic, MA: MaskAgnostic, Altfmt: AltFmt,
497 /*TWiden*/ W: 0);
498
499 forwardVSETVLIAVL(Info&: InstrInfo);
500
501 return InstrInfo;
502}
503} // namespace RISCV
504} // namespace llvm
505