1//===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "RISCVTargetTransformInfo.h"
10#include "MCTargetDesc/RISCVMatInt.h"
11#include "RISCVVectorUtils.h"
12#include "llvm/ADT/STLExtras.h"
13#include "llvm/Analysis/TargetTransformInfo.h"
14#include "llvm/CodeGen/BasicTTIImpl.h"
15#include "llvm/CodeGen/CostTable.h"
16#include "llvm/CodeGen/TargetLowering.h"
17#include "llvm/CodeGen/ValueTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/IntrinsicsRISCV.h"
20#include "llvm/IR/PatternMatch.h"
21#include "llvm/Transforms/InstCombine/InstCombiner.h"
22#include <cmath>
23#include <limits>
24#include <optional>
25using namespace llvm;
26using namespace llvm::PatternMatch;
27
28#define DEBUG_TYPE "riscvtti"
29
30InstructionCost
31RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
32 TTI::TargetCostKind CostKind) const {
33 // Check if the type is valid for all CostKind
34 if (!VT.isVector())
35 return InstructionCost::getInvalid();
36 size_t NumInstr = OpCodes.size();
37 if (CostKind == TTI::TCK_CodeSize)
38 return NumInstr;
39 InstructionCost LMULCost = TLI->getLMULCost(VT);
40 if ((CostKind != TTI::TCK_RecipThroughput) && (CostKind != TTI::TCK_Latency))
41 return LMULCost * NumInstr;
42 InstructionCost Cost = 0;
43 for (auto Op : OpCodes) {
44 switch (Op) {
45 case RISCV::VRGATHER_VI:
46 Cost += TLI->getVRGatherVICost(VT);
47 break;
48 case RISCV::VRGATHER_VV:
49 Cost += TLI->getVRGatherVVCost(VT);
50 break;
51 case RISCV::VSLIDEUP_VI:
52 case RISCV::VSLIDEDOWN_VI:
53 Cost += TLI->getVSlideVICost(VT);
54 break;
55 case RISCV::VSLIDEUP_VX:
56 case RISCV::VSLIDEDOWN_VX:
57 Cost += TLI->getVSlideVXCost(VT);
58 break;
59 case RISCV::VREDMAX_VS:
60 case RISCV::VREDMIN_VS:
61 case RISCV::VREDMAXU_VS:
62 case RISCV::VREDMINU_VS:
63 case RISCV::VREDSUM_VS:
64 case RISCV::VREDAND_VS:
65 case RISCV::VREDOR_VS:
66 case RISCV::VREDXOR_VS:
67 case RISCV::VFREDMAX_VS:
68 case RISCV::VFREDMIN_VS:
69 case RISCV::VFREDUSUM_VS: {
70 unsigned VL = VT.getVectorMinNumElements();
71 if (!VT.isFixedLengthVector())
72 VL *= *getVScaleForTuning();
73 Cost += Log2_32_Ceil(Value: VL);
74 break;
75 }
76 case RISCV::VFREDOSUM_VS: {
77 unsigned VL = VT.getVectorMinNumElements();
78 if (!VT.isFixedLengthVector())
79 VL *= *getVScaleForTuning();
80 Cost += VL;
81 break;
82 }
83 case RISCV::VMV_X_S:
84 case RISCV::VFMV_F_S:
85 // Domain crossings from vector -> scalar are usually more expensive.
86 Cost += 2;
87 break;
88 case RISCV::VMV_S_X:
89 case RISCV::VFMV_S_F:
90 case RISCV::VMOR_MM:
91 case RISCV::VMXOR_MM:
92 case RISCV::VMAND_MM:
93 case RISCV::VMANDN_MM:
94 case RISCV::VMNAND_MM:
95 case RISCV::VCPOP_M:
96 case RISCV::VFIRST_M:
97 Cost += 1;
98 break;
99 case RISCV::VDIV_VV:
100 case RISCV::VREM_VV:
101 Cost += LMULCost * TTI::TCC_Expensive;
102 break;
103 default:
104 Cost += LMULCost;
105 }
106 }
107 return Cost;
108}
109
110static InstructionCost getIntImmCostImpl(const DataLayout &DL,
111 const RISCVSubtarget *ST,
112 const APInt &Imm, Type *Ty,
113 TTI::TargetCostKind CostKind,
114 bool FreeZeroes) {
115 assert(Ty->isIntegerTy() &&
116 "getIntImmCost can only estimate cost of materialising integers");
117
118 // We have a Zero register, so 0 is always free.
119 if (Imm == 0)
120 return TTI::TCC_Free;
121
122 // Otherwise, we check how many instructions it will take to materialise.
123 return RISCVMatInt::getIntMatCost(Val: Imm, Size: DL.getTypeSizeInBits(Ty), STI: *ST,
124 /*CompressionCost=*/false, FreeZeroes);
125}
126
127InstructionCost
128RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
129 TTI::TargetCostKind CostKind) const {
130 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind, FreeZeroes: false);
131}
132
133// Look for patterns of shift followed by AND that can be turned into a pair of
134// shifts. We won't need to materialize an immediate for the AND so these can
135// be considered free.
136static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) {
137 uint64_t Mask = Imm.getZExtValue();
138 auto *BO = dyn_cast<BinaryOperator>(Val: Inst->getOperand(i: 0));
139 if (!BO || !BO->hasOneUse())
140 return false;
141
142 if (BO->getOpcode() != Instruction::Shl)
143 return false;
144
145 if (!isa<ConstantInt>(Val: BO->getOperand(i_nocapture: 1)))
146 return false;
147
148 unsigned ShAmt = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1))->getZExtValue();
149 // (and (shl x, c2), c1) will be matched to (srli (slli x, c2+c3), c3) if c1
150 // is a mask shifted by c2 bits with c3 leading zeros.
151 if (isShiftedMask_64(Value: Mask)) {
152 unsigned Trailing = llvm::countr_zero(Val: Mask);
153 if (ShAmt == Trailing)
154 return true;
155 }
156
157 return false;
158}
159
160// If this is i64 AND is part of (X & -(1 << C1) & 0xffffffff) == C2 << C1),
161// DAGCombiner can convert this to (sraiw X, C1) == sext(C2) for RV64. On RV32,
162// the type will be split so only the lower 32 bits need to be compared using
163// (srai/srli X, C) == C2.
164static bool canUseShiftCmp(Instruction *Inst, const APInt &Imm) {
165 if (!Inst->hasOneUse())
166 return false;
167
168 // Look for equality comparison.
169 auto *Cmp = dyn_cast<ICmpInst>(Val: *Inst->user_begin());
170 if (!Cmp || !Cmp->isEquality())
171 return false;
172
173 // Right hand side of comparison should be a constant.
174 auto *C = dyn_cast<ConstantInt>(Val: Cmp->getOperand(i_nocapture: 1));
175 if (!C)
176 return false;
177
178 uint64_t Mask = Imm.getZExtValue();
179
180 // Mask should be of the form -(1 << C) in the lower 32 bits.
181 if (!isUInt<32>(x: Mask) || !isPowerOf2_32(Value: -uint32_t(Mask)))
182 return false;
183
184 // Comparison constant should be a subset of Mask.
185 uint64_t CmpC = C->getZExtValue();
186 if ((CmpC & Mask) != CmpC)
187 return false;
188
189 // We'll need to sign extend the comparison constant and shift it right. Make
190 // sure the new constant can use addi/xori+seqz/snez.
191 unsigned ShiftBits = llvm::countr_zero(Val: Mask);
192 int64_t NewCmpC = SignExtend64<32>(x: CmpC) >> ShiftBits;
193 return NewCmpC >= -2048 && NewCmpC <= 2048;
194}
195
196InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
197 const APInt &Imm, Type *Ty,
198 TTI::TargetCostKind CostKind,
199 Instruction *Inst) const {
200 assert(Ty->isIntegerTy() &&
201 "getIntImmCost can only estimate cost of materialising integers");
202
203 // We have a Zero register, so 0 is always free.
204 if (Imm == 0)
205 return TTI::TCC_Free;
206
207 // Some instructions in RISC-V can take a 12-bit immediate. Some of these are
208 // commutative, in others the immediate comes from a specific argument index.
209 bool Takes12BitImm = false;
210 unsigned ImmArgIdx = ~0U;
211
212 switch (Opcode) {
213 case Instruction::GetElementPtr:
214 // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will
215 // split up large offsets in GEP into better parts than ConstantHoisting
216 // can.
217 return TTI::TCC_Free;
218 case Instruction::Store: {
219 // Use the materialization cost regardless of if it's the address or the
220 // value that is constant, except for if the store is misaligned and
221 // misaligned accesses are not legal (experience shows constant hoisting
222 // can sometimes be harmful in such cases).
223 if (Idx == 1 || !Inst)
224 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
225 /*FreeZeroes=*/true);
226
227 StoreInst *ST = cast<StoreInst>(Val: Inst);
228 if (!getTLI()->allowsMemoryAccessForAlignment(
229 Context&: Ty->getContext(), DL, VT: getTLI()->getValueType(DL, Ty),
230 AddrSpace: ST->getPointerAddressSpace(), Alignment: ST->getAlign()))
231 return TTI::TCC_Free;
232
233 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
234 /*FreeZeroes=*/true);
235 }
236 case Instruction::Load:
237 // If the address is a constant, use the materialization cost.
238 return getIntImmCost(Imm, Ty, CostKind);
239 case Instruction::And:
240 // zext.h
241 if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())
242 return TTI::TCC_Free;
243 // zext.w
244 if (Imm == UINT64_C(0xffffffff) && (!ST->is64Bit() || ST->hasStdExtZba()))
245 return TTI::TCC_Free;
246 // bclri
247 if (ST->hasStdExtZbs() && (~Imm).isPowerOf2())
248 return TTI::TCC_Free;
249 if (Inst && Idx == 1 && Imm.getBitWidth() <= ST->getXLen() &&
250 canUseShiftPair(Inst, Imm))
251 return TTI::TCC_Free;
252 if (Inst && Idx == 1 && Imm.getBitWidth() == 64 &&
253 canUseShiftCmp(Inst, Imm))
254 return TTI::TCC_Free;
255 Takes12BitImm = true;
256 break;
257 case Instruction::Add:
258 Takes12BitImm = true;
259 break;
260 case Instruction::Or:
261 case Instruction::Xor:
262 // bseti/binvi
263 if (ST->hasStdExtZbs() && Imm.isPowerOf2())
264 return TTI::TCC_Free;
265 Takes12BitImm = true;
266 break;
267 case Instruction::Mul:
268 // Power of 2 is a shift. Negated power of 2 is a shift and a negate.
269 if (Imm.isPowerOf2() || Imm.isNegatedPowerOf2())
270 return TTI::TCC_Free;
271 // One more or less than a power of 2 can use SLLI+ADD/SUB.
272 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2())
273 return TTI::TCC_Free;
274 // FIXME: There is no MULI instruction.
275 Takes12BitImm = true;
276 break;
277 case Instruction::Sub:
278 case Instruction::Shl:
279 case Instruction::LShr:
280 case Instruction::AShr:
281 Takes12BitImm = true;
282 ImmArgIdx = 1;
283 break;
284 default:
285 break;
286 }
287
288 if (Takes12BitImm) {
289 // Check immediate is the correct argument...
290 if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {
291 // ... and fits into the 12-bit immediate.
292 if (Imm.getSignificantBits() <= 64 &&
293 getTLI()->isLegalAddImmediate(Imm: Imm.getSExtValue())) {
294 return TTI::TCC_Free;
295 }
296 }
297
298 // Otherwise, use the full materialisation cost.
299 return getIntImmCost(Imm, Ty, CostKind);
300 }
301
302 // By default, prevent hoisting.
303 return TTI::TCC_Free;
304}
305
306InstructionCost
307RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
308 const APInt &Imm, Type *Ty,
309 TTI::TargetCostKind CostKind) const {
310 // Prevent hoisting in unknown cases.
311 return TTI::TCC_Free;
312}
313
314bool RISCVTTIImpl::hasActiveVectorLength() const {
315 return ST->hasVInstructions();
316}
317
318TargetTransformInfo::PopcntSupportKind
319RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {
320 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
321 return ST->hasCPOPLike() ? TTI::PSK_FastHardware : TTI::PSK_Software;
322}
323
324InstructionCost RISCVTTIImpl::getPartialReductionCost(
325 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
326 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
327 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
328 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
329 if (Opcode == Instruction::FAdd)
330 return InstructionCost::getInvalid();
331
332 // zve32x is broken for partial_reduce_umla, but let's make sure we
333 // don't generate them.
334 // vdot4a* reduces four i8 products into an i32 result; an i64 accumulator is
335 // additionally supported by widening the i32 partial sums to i64 (see
336 // lowerPARTIAL_REDUCE_MLA). VF is the number of i8 input elements, so the
337 // reduction factor is AccumBits / 8 (4 for i32, 8 for i64).
338 if (!ST->hasStdExtZvdot4a8i() || ST->getELen() < 64 ||
339 Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||
340 InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(BitWidth: 8) ||
341 (!AccumType->isIntegerTy(BitWidth: 32) && !AccumType->isIntegerTy(BitWidth: 64)))
342 return InstructionCost::getInvalid();
343
344 unsigned Ratio = AccumType->getScalarSizeInBits() / 8;
345 if (!VF.isKnownMultipleOf(RHS: Ratio))
346 return InstructionCost::getInvalid();
347
348 // Cost of the vdot4a* itself, which operates on the i32 intermediate type
349 // holding VF/4 elements.
350 Type *DotTp = VectorType::get(ElementType: Type::getInt32Ty(C&: AccumType->getContext()),
351 EC: VF.divideCoefficientBy(RHS: 4));
352 std::pair<InstructionCost, MVT> DotLT = getTypeLegalizationCost(Ty: DotTp);
353 // Note: Asuming all vdot4a* variants are equal cost
354 InstructionCost Cost =
355 DotLT.first *
356 getRISCVInstructionCost(OpCodes: RISCV::VDOT4A_VV, VT: DotLT.second, CostKind);
357
358 // Account for reducing the i32 partial sums down to the i64 accumulator's
359 // element count and accumulating into it (see lowerPARTIAL_REDUCE_MLA), which
360 // has two shapes depending on the accumulator's LMUL.
361 if (AccumType->isIntegerTy(BitWidth: 64)) {
362 LLVMContext &Ctx = AccumType->getContext();
363 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
364 ElementCount AccVF = VF.divideCoefficientBy(RHS: Ratio);
365 std::pair<InstructionCost, MVT> AccLT =
366 getTypeLegalizationCost(Ty: VectorType::get(ElementType: AccumType, EC: AccVF));
367
368 // When the i32 subvectors of a single-vector scalable accumulator are a
369 // fractional LMUL, extracting the high subvector would need a vslidedown,
370 // so instead the i32 dot result is widened to i64 first (vsext.vf2 /
371 // vzext.vf2) and then reduced and accumulated with register-aligned i64
372 // vadd.vv.
373 bool WidenFirst = false;
374 if (VF.isScalable() && AccLT.second.isScalableVector()) {
375 MVT NarrowMVT = AccLT.second.changeVectorElementType(EltVT: MVT::i32);
376 WidenFirst =
377 RISCVVType::decodeVLMUL(VLMul: RISCVTargetLowering::getLMUL(VT: NarrowMVT))
378 .second;
379 }
380
381 if (WidenFirst) {
382 // The widened i64 dot result has VF/4 elements, i.e. twice the
383 // accumulator's element count, so the reduction plus the accumulate are
384 // two i64 vadd.vv.
385 std::pair<InstructionCost, MVT> WideLT = getTypeLegalizationCost(
386 Ty: VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: 4)));
387 Cost +=
388 WideLT.first * getRISCVInstructionCost(OpCodes: RISCV::VSEXT_VF2,
389 VT: WideLT.second, CostKind) +
390 2 * AccLT.first *
391 getRISCVInstructionCost(OpCodes: RISCV::VADD_VV, VT: AccLT.second, CostKind);
392 } else {
393 // Otherwise the scale-4 i32 sums are halved with a single i32 vadd.vv,
394 // then widened and added into the i64 result with a vwadd.wv.
395 std::pair<InstructionCost, MVT> RedLT =
396 getTypeLegalizationCost(Ty: VectorType::get(ElementType: I32Ty, EC: AccVF));
397 Cost += RedLT.first * getRISCVInstructionCost(OpCodes: RISCV::VADD_VV,
398 VT: RedLT.second, CostKind) +
399 AccLT.first * getRISCVInstructionCost(OpCodes: RISCV::VWADD_WV,
400 VT: AccLT.second, CostKind);
401 // Fixed-length vectors extract the high i32 subvector with a vslidedown.
402 if (VF.isFixed())
403 Cost += DotLT.first * getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI,
404 VT: DotLT.second, CostKind);
405 }
406 }
407
408 return Cost;
409}
410
411bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
412 // Currently, the ExpandReductions pass can't expand scalable-vector
413 // reductions, but we still request expansion as RVV doesn't support certain
414 // reductions and the SelectionDAG can't legalize them either.
415 switch (II->getIntrinsicID()) {
416 default:
417 return false;
418 // These reductions have no equivalent in RVV
419 case Intrinsic::vector_reduce_mul:
420 case Intrinsic::vector_reduce_fmul:
421 return true;
422 }
423}
424
425std::optional<unsigned> RISCVTTIImpl::getVScaleForTuning() const {
426 if (ST->hasVInstructions())
427 if (unsigned MinVLen = ST->getRealMinVLen();
428 MinVLen >= RISCV::RVVBitsPerBlock)
429 return MinVLen / RISCV::RVVBitsPerBlock;
430 return BaseT::getVScaleForTuning();
431}
432
433TypeSize
434RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
435 unsigned LMUL = llvm::bit_floor(
436 Value: std::clamp<unsigned>(val: ST->getCLOpts().v_register_bit_width_lmul, lo: 1, hi: 8));
437 switch (K) {
438 case TargetTransformInfo::RGK_Scalar:
439 return TypeSize::getFixed(ExactSize: ST->getXLen());
440 case TargetTransformInfo::RGK_FixedWidthVector:
441 return TypeSize::getFixed(
442 ExactSize: ST->useRVVForFixedLengthVectors() ? LMUL * ST->getRealMinVLen() : 0);
443 case TargetTransformInfo::RGK_ScalableVector:
444 return TypeSize::getScalable(
445 MinimumSize: (ST->hasVInstructions() &&
446 ST->getRealMinVLen() >= RISCV::RVVBitsPerBlock)
447 ? LMUL * RISCV::RVVBitsPerBlock
448 : 0);
449 }
450
451 llvm_unreachable("Unsupported register kind");
452}
453
454InstructionCost RISCVTTIImpl::getStaticDataAddrGenerationCost(
455 const TTI::TargetCostKind CostKind) const {
456 switch (CostKind) {
457 case TTI::TCK_CodeSize:
458 case TTI::TCK_SizeAndLatency:
459 // Always 2 instructions
460 return 2;
461 case TTI::TCK_Latency:
462 case TTI::TCK_RecipThroughput:
463 // Depending on the memory model the address generation will
464 // require AUIPC + ADDI (medany) or LUI + ADDI (medlow). Don't
465 // have a way of getting this information here, so conservatively
466 // require both.
467 // In practice, these are generally implemented together.
468 return (ST->hasAUIPCADDIFusion() && ST->hasLUIADDIFusion()) ? 1 : 2;
469 }
470 llvm_unreachable("Unsupported cost kind");
471}
472
473InstructionCost
474RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,
475 TTI::TargetCostKind CostKind) const {
476 // Add a cost of address generation + the cost of the load. The address
477 // is expected to be a PC relative offset to a constant pool entry
478 // using auipc/addi.
479 InstructionCost Cost = 0;
480 Cost = getStaticDataAddrGenerationCost(CostKind) +
481 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: DL.getABITypeAlign(Ty),
482 /*AddressSpace=*/0, CostKind);
483 // Estimate the amount of 4 byte instructions that could fit
484 // instead of the constant pool, ignoring any extra padding.
485 if (CostKind == TTI::TCK_CodeSize)
486 Cost += ((InstructionCost)DL.getTypeAllocSize(Ty)) / 4;
487 return Cost;
488}
489
490static bool isRepeatedConcatMask(ArrayRef<int> Mask, int &SubVectorSize) {
491 unsigned Size = Mask.size();
492 if (!isPowerOf2_32(Value: Size))
493 return false;
494 for (unsigned I = 0; I != Size; ++I) {
495 if (static_cast<unsigned>(Mask[I]) == I)
496 continue;
497 if (Mask[I] != 0)
498 return false;
499 if (Size % I != 0)
500 return false;
501 for (unsigned J = I + 1; J != Size; ++J)
502 // Check the pattern is repeated.
503 if (static_cast<unsigned>(Mask[J]) != J % I)
504 return false;
505 SubVectorSize = I;
506 return true;
507 }
508 // That means Mask is <0, 1, 2, 3>. This is not a concatenation.
509 return false;
510}
511
512static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,
513 LLVMContext &C) {
514 assert((DataVT.getScalarSizeInBits() != 8 ||
515 DataVT.getVectorNumElements() <= 256) && "unhandled case in lowering");
516 MVT IndexVT = DataVT.changeTypeToInteger();
517 if (IndexVT.getScalarType().bitsGT(VT: ST.getXLenVT()))
518 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
519 return cast<VectorType>(Val: EVT(IndexVT).getTypeForEVT(Context&: C));
520}
521
522/// Attempt to approximate the cost of a shuffle which will require splitting
523/// during legalization. Note that processShuffleMasks is not an exact proxy
524/// for the algorithm used in LegalizeVectorTypes, but hopefully it's a
525/// reasonably close upperbound.
526static InstructionCost costShuffleViaSplitting(const RISCVTTIImpl &TTI,
527 MVT LegalVT, VectorType *Tp,
528 ArrayRef<int> Mask,
529 TTI::TargetCostKind CostKind) {
530 assert(LegalVT.isFixedLengthVector() && !Mask.empty() &&
531 "Expected fixed vector type and non-empty mask");
532 unsigned LegalNumElts = LegalVT.getVectorNumElements();
533 // Number of destination vectors after legalization:
534 unsigned NumOfDests = divideCeil(Numerator: Mask.size(), Denominator: LegalNumElts);
535 // We are going to permute multiple sources and the result will be in
536 // multiple destinations. Providing an accurate cost only for splits where
537 // the element type remains the same.
538 if (NumOfDests <= 1 ||
539 LegalVT.getVectorElementType().getSizeInBits() !=
540 Tp->getElementType()->getPrimitiveSizeInBits() ||
541 LegalNumElts >= Tp->getElementCount().getFixedValue())
542 return InstructionCost::getInvalid();
543
544 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
545 unsigned LegalVTSize = LegalVT.getStoreSize();
546 // Number of source vectors after legalization:
547 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
548
549 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: LegalNumElts);
550
551 unsigned NormalizedVF = LegalNumElts * std::max(a: NumOfSrcs, b: NumOfDests);
552 unsigned NumOfSrcRegs = NormalizedVF / LegalNumElts;
553 unsigned NumOfDestRegs = NormalizedVF / LegalNumElts;
554 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
555 assert(NormalizedVF >= Mask.size() &&
556 "Normalized mask expected to be not shorter than original mask.");
557 copy(Range&: Mask, Out: NormalizedMask.begin());
558 InstructionCost Cost = 0;
559 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
560 processShuffleMasks(
561 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
562 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
563 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
564 return;
565 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
566 .second)
567 return;
568 Cost += TTI.getShuffleCost(
569 Kind: TTI::SK_PermuteSingleSrc,
570 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
571 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
572 },
573 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
574 Cost += TTI.getShuffleCost(
575 Kind: TTI::SK_PermuteTwoSrc,
576 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
577 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
578 });
579 return Cost;
580}
581
582/// Try to perform better estimation of the permutation.
583/// 1. Split the source/destination vectors into real registers.
584/// 2. Do the mask analysis to identify which real registers are
585/// permuted. If more than 1 source registers are used for the
586/// destination register building, the cost for this destination register
587/// is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
588/// source register is used, build mask and calculate the cost as a cost
589/// of PermuteSingleSrc.
590/// Also, for the single register permute we try to identify if the
591/// destination register is just a copy of the source register or the
592/// copy of the previous destination register (the cost is
593/// TTI::TCC_Basic). If the source register is just reused, the cost for
594/// this operation is 0.
595static InstructionCost
596costShuffleViaVRegSplitting(const RISCVTTIImpl &TTI, MVT LegalVT,
597 std::optional<unsigned> VLen, VectorType *Tp,
598 ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {
599 assert(LegalVT.isFixedLengthVector());
600 if (!VLen || Mask.empty())
601 return InstructionCost::getInvalid();
602 MVT ElemVT = LegalVT.getVectorElementType();
603 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
604 LegalVT = TTI.getTypeLegalizationCost(
605 Ty: FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: ElemsPerVReg))
606 .second;
607 // Number of destination vectors after legalization:
608 InstructionCost NumOfDests =
609 divideCeil(Numerator: Mask.size(), Denominator: LegalVT.getVectorNumElements());
610 if (NumOfDests <= 1 ||
611 LegalVT.getVectorElementType().getSizeInBits() !=
612 Tp->getElementType()->getPrimitiveSizeInBits() ||
613 LegalVT.getVectorNumElements() >= Tp->getElementCount().getFixedValue())
614 return InstructionCost::getInvalid();
615
616 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
617 unsigned LegalVTSize = LegalVT.getStoreSize();
618 // Number of source vectors after legalization:
619 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
620
621 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(),
622 NumElts: LegalVT.getVectorNumElements());
623
624 unsigned E = NumOfDests.getValue();
625 unsigned NormalizedVF =
626 LegalVT.getVectorNumElements() * std::max(a: NumOfSrcs, b: E);
627 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
628 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
629 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
630 assert(NormalizedVF >= Mask.size() &&
631 "Normalized mask expected to be not shorter than original mask.");
632 copy(Range&: Mask, Out: NormalizedMask.begin());
633 InstructionCost Cost = 0;
634 int NumShuffles = 0;
635 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
636 processShuffleMasks(
637 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
638 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
639 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
640 return;
641 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
642 .second)
643 return;
644 ++NumShuffles;
645 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleOpTy,
646 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
647 },
648 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
649 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
650 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
651 NumShuffles += 2;
652 });
653 // Note: check that we do not emit too many shuffles here to prevent code
654 // size explosion.
655 // TODO: investigate, if it can be improved by extra analysis of the masks
656 // to check if the code is more profitable.
657 if ((NumOfDestRegs > 2 && NumShuffles <= static_cast<int>(NumOfDestRegs)) ||
658 (NumOfDestRegs <= 2 && NumShuffles < 4))
659 return Cost;
660 return InstructionCost::getInvalid();
661}
662
663InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
664 ArrayRef<int> Mask,
665 TTI::TargetCostKind CostKind) const {
666 // Avoid missing masks and length changing shuffles
667 if (Mask.size() <= 2 || Mask.size() != Tp->getNumElements())
668 return InstructionCost::getInvalid();
669
670 int NumElts = Tp->getNumElements();
671 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Tp);
672 // Avoid scalarization cases
673 if (!LT.second.isFixedLengthVector())
674 return InstructionCost::getInvalid();
675
676 // Requires moving elements between parts, which requires additional
677 // unmodeled instructions.
678 if (LT.first != 1)
679 return InstructionCost::getInvalid();
680
681 auto GetSlideOpcode = [&](int SlideAmt) {
682 assert(SlideAmt != 0);
683 bool IsVI = isUInt<5>(x: std::abs(x: SlideAmt));
684 if (SlideAmt < 0)
685 return IsVI ? RISCV::VSLIDEDOWN_VI : RISCV::VSLIDEDOWN_VX;
686 return IsVI ? RISCV::VSLIDEUP_VI : RISCV::VSLIDEUP_VX;
687 };
688
689 std::array<std::pair<int, int>, 2> SrcInfo;
690 if (!isMaskedSlidePair(Mask, NumElts, SrcInfo))
691 return InstructionCost::getInvalid();
692
693 if (SrcInfo[1].second == 0)
694 std::swap(x&: SrcInfo[0], y&: SrcInfo[1]);
695
696 if (ST->hasStdExtZvzip() && LT.second.getScalarSizeInBits() != 1) {
697 unsigned Factor;
698 if (isPairEven(SrcInfo, Mask, Factor) && Factor == 1)
699 return getRISCVInstructionCost(OpCodes: RISCV::VPAIRE_VV, VT: LT.second, CostKind);
700 if (isPairOdd(SrcInfo, Mask, Factor) && Factor == 1)
701 return getRISCVInstructionCost(OpCodes: RISCV::VPAIRO_VV, VT: LT.second, CostKind);
702 }
703
704 InstructionCost FirstSlideCost = 0;
705 if (SrcInfo[0].second != 0) {
706 unsigned Opcode = GetSlideOpcode(SrcInfo[0].second);
707 FirstSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
708 }
709
710 if (SrcInfo[1].first == -1)
711 return FirstSlideCost;
712
713 InstructionCost SecondSlideCost = 0;
714 if (SrcInfo[1].second != 0) {
715 unsigned Opcode = GetSlideOpcode(SrcInfo[1].second);
716 SecondSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
717 } else {
718 SecondSlideCost =
719 getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second, CostKind);
720 }
721
722 auto EC = Tp->getElementCount();
723 VectorType *MaskTy =
724 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: Tp->getContext()), EC);
725 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
726 return FirstSlideCost + SecondSlideCost + MaskCost;
727}
728
729MVT RISCVTTIImpl::getZvzipVZIPCostVT(MVT InterleavedVT) const {
730 if (!InterleavedVT.getVectorElementCount().isKnownEven())
731 return MVT::INVALID_SIMPLE_VALUE_TYPE;
732
733 MVT CostVT = InterleavedVT;
734 if (InterleavedVT.isFixedLengthVector()) {
735 MVT SourceVT = InterleavedVT.getHalfNumVectorElementsVT();
736 CostVT = TLI->getContainerForFixedLengthVector(VT: SourceVT)
737 .getDoubleNumVectorElementsVT();
738 }
739
740 unsigned EltBits = CostVT.getScalarSizeInBits();
741 unsigned MinSize = CostVT.getSizeInBits().getKnownMinValue();
742 unsigned LMULOctuple = MinSize / (RISCV::RVVBitsPerBlock / 8);
743 // Perform the 2 * SEW <= LMUL * min(ELEN, VLEN) check.
744 if (EltBits * 16 >
745 LMULOctuple * std::min(a: ST->getELen(), b: ST->getRealMinVLen()))
746 return MVT::INVALID_SIMPLE_VALUE_TYPE;
747 return CostVT;
748}
749
750MVT RISCVTTIImpl::getZvzipVUNZIPCostVT(MVT InterleavedVT) const {
751 if (!InterleavedVT.getVectorElementCount().isKnownEven())
752 return MVT::INVALID_SIMPLE_VALUE_TYPE;
753
754 MVT CostVT = InterleavedVT;
755 // lowerZvzipVUNZIP widens the source container if halving it would produce
756 // an illegal result type. Apply the same rule here so the cost uses the
757 // source LMUL selected by ISel.
758 if (InterleavedVT.isFixedLengthVector()) {
759 CostVT = TLI->getContainerForFixedLengthVector(VT: InterleavedVT);
760 if (CostVT.getVectorMinNumElements() == 1 ||
761 !TLI->isTypeLegal(VT: CostVT.getHalfNumVectorElementsVT()))
762 CostVT = CostVT.getDoubleNumVectorElementsVT();
763 }
764
765 MVT DeinterleavedVT = CostVT.getHalfNumVectorElementsVT();
766 if (RISCVTargetLowering::getLMUL(VT: DeinterleavedVT) == RISCVVType::LMUL_8)
767 return MVT::INVALID_SIMPLE_VALUE_TYPE;
768 return CostVT;
769}
770
771InstructionCost RISCVTTIImpl::getVZIPCost(TTI::ShuffleKind Kind,
772 VectorType *DstTy, VectorType *SrcTy,
773 ArrayRef<int> Mask,
774 TTI::TargetCostKind CostKind) const {
775 assert(isa<FixedVectorType>(SrcTy) && "Expected fixed-length vector types");
776
777 if (!ST->hasStdExtZvzip() || Mask.size() < 2 ||
778 DstTy->getScalarSizeInBits() == 1)
779 return InstructionCost::getInvalid();
780
781 unsigned NumSrcElts = SrcTy->getElementCount().getKnownMinValue();
782 unsigned NumDstElts = DstTy->getElementCount().getKnownMinValue();
783 unsigned NumInputElts;
784
785 if (Kind == TTI::SK_PermuteSingleSrc) {
786 // A single-source vzip shuffle interleaves two halves of the same vector,
787 // so its source and destination must have the same number of elements.
788 if (NumSrcElts != NumDstElts)
789 return InstructionCost::getInvalid();
790 NumInputElts = NumSrcElts;
791 } else if (Kind == TTI::SK_PermuteTwoSrc) {
792 // Accept both forms that can be lowered with vzip.vv:
793 //
794 // 1. The destination has twice as many elements as each source, e.g.
795 // %r = shufflevector <4 x i32> %a, <4 x i32> %b,
796 // <8 x i32> <0, 4, 1, 5, 2, 6, 3, 7>
797 // The two operands are already the half-sized inputs of vzip.vv.
798 //
799 // 2. Each source has the same number of elements as the destination, e.g.
800 // %r = shufflevector <4 x i32> %a, <4 x i32> %b,
801 // <4 x i32> <0, 4, 1, 5>
802 // ISel first extracts a <2 x i32> half from each source and then uses
803 // those halves as the inputs of vzip.vv.
804 if (NumSrcElts != NumDstElts && NumSrcElts * 2 != NumDstElts)
805 return InstructionCost::getInvalid();
806 NumInputElts = NumSrcElts * 2;
807 } else {
808 return InstructionCost::getInvalid();
809 }
810
811 // ISel deliberately excludes the two-element single-source identity case.
812 if (Kind == TTI::SK_PermuteSingleSrc && Mask.size() == 2 &&
813 ShuffleVectorInst::isSingleSourceMask(Mask, NumSrcElts: Mask.size()))
814 return InstructionCost::getInvalid();
815
816 // A deinterleave shuffle for an interleaved load may have a mask such as
817 // %even = shufflevector <4 x i32> %wide, <4 x i32> poison,
818 // <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
819 // Because isInterleaveMask ignores poison lanes, this mask may also satisfy
820 // the interleave matcher. Do not cost it as vzip.vv.
821 unsigned DeinterleaveIndex;
822 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2,
823 Index&: DeinterleaveIndex) &&
824 count_if(Range&: Mask, P: [](int Idx) { return Idx >= 0; }) > 1)
825 return InstructionCost::getInvalid();
826
827 // Require the two start indices found in the mask to be aligned to
828 // half-vector boundaries, with at least one equal to zero.
829 SmallVector<unsigned, 2> StartIndexes;
830 if (!ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts, StartIndexes))
831 return InstructionCost::getInvalid();
832
833 unsigned HalfNumElts = Mask.size() / 2;
834 if ((StartIndexes[0] != 0 && StartIndexes[1] != 0) ||
835 StartIndexes[0] % HalfNumElts != 0 || StartIndexes[1] % HalfNumElts != 0)
836 return InstructionCost::getInvalid();
837
838 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: DstTy);
839 MVT CostVT = getZvzipVZIPCostVT(InterleavedVT: DstLT.second);
840 if (!CostVT.isValid())
841 return InstructionCost::getInvalid();
842
843 InstructionCost Cost =
844 DstLT.first * getRISCVInstructionCost(OpCodes: RISCV::VZIP_VV, VT: CostVT, CostKind);
845
846 // For a shuffle such as
847 // %r = shufflevector <4 x i32> %v, <4 x i32> poison,
848 // <4 x i32> <i32 0, i32 2, i32 1, i32 3>
849 // vzip.vv needs two <2 x i32> inputs. Include the cost of extracting each
850 // upper half; extracting a low half is free.
851 if (NumSrcElts == NumDstElts) {
852 auto *HalfSrcTy = FixedVectorType::getHalfElementsVectorType(
853 VTy: cast<FixedVectorType>(Val: SrcTy));
854 for (unsigned Start : StartIndexes) {
855 unsigned ExtractIndex = Start % NumSrcElts;
856 if (ExtractIndex != 0)
857 Cost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: HalfSrcTy, SrcTy,
858 CostKind, Mask: {}, Index: ExtractIndex, SubTp: HalfSrcTy);
859 }
860 }
861
862 return Cost;
863}
864
865InstructionCost RISCVTTIImpl::getShuffleCost(
866 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
867 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
868 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
869 TTI::VectorInstrContext VIC) const {
870 assert((Mask.empty() || DstTy->isScalableTy() ||
871 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
872 "Expected the Mask to match the return size if given");
873 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
874 "Expected the same scalar types");
875
876 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
877 if (VIC == TTI::VectorInstrContext::SplatOpFolded &&
878 ST->sinkSplatOperands() && Kind == TTI::SK_Broadcast)
879 return TTI::TCC_Free;
880
881 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
882 // For now, skip all fixed vector cost analysis when P extension is available
883 // to avoid crashes in getMinRVVVectorSizeInBits()
884 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: SrcTy))
885 return 1;
886
887 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
888
889 // First, handle cases where having a fixed length vector enables us to
890 // give a more accurate cost than falling back to generic scalable codegen.
891 // TODO: Each of these cases hints at a modeling gap around scalable vectors.
892 if (auto *FVTp = dyn_cast<FixedVectorType>(Val: SrcTy);
893 FVTp && ST->hasVInstructions() && LT.second.isFixedLengthVector()) {
894 InstructionCost VRegSplittingCost = costShuffleViaVRegSplitting(
895 TTI: *this, LegalVT: LT.second, VLen: ST->getRealVLen(),
896 Tp: Kind == TTI::SK_InsertSubvector ? DstTy : SrcTy, Mask, CostKind);
897 if (VRegSplittingCost.isValid())
898 return VRegSplittingCost;
899 switch (Kind) {
900 default:
901 break;
902 case TTI::SK_PermuteSingleSrc: {
903 if (Mask.size() >= 2) {
904 MVT EltTp = LT.second.getVectorElementType();
905
906 // Try the vzip.vv cost first, otherwise model the shuffle as a
907 // vwaddu.vv/vwmaccu.vx sequence below if the element size < ELEN.
908 if (InstructionCost VZipCost =
909 getVZIPCost(Kind, DstTy, SrcTy, Mask, CostKind);
910 VZipCost.isValid())
911 return VZipCost;
912
913 unsigned Index;
914 if (ST->hasStdExtZvzip() && EltTp.getScalarSizeInBits() != 1 &&
915 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2, Index)) {
916 unsigned Opcode = Index == 0 ? RISCV::VUNZIPE_V : RISCV::VUNZIPO_V;
917 if (MVT CostVT = getZvzipVUNZIPCostVT(InterleavedVT: LT.second); CostVT.isValid())
918 return LT.first * getRISCVInstructionCost(OpCodes: Opcode, VT: CostVT, CostKind);
919 }
920
921 // If the size of the element is < ELEN then shuffles of interleaves and
922 // deinterleaves of 2 vectors can be lowered into the following
923 // sequences
924 if (EltTp.getScalarSizeInBits() < ST->getELen()) {
925 // Example sequence:
926 // vsetivli zero, 4, e8, mf4, ta, ma (ignored)
927 // vwaddu.vv v10, v8, v9
928 // li a0, -1 (ignored)
929 // vwmaccu.vx v10, a0, v9
930 if (ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Mask.size()))
931 return 2 * LT.first * TLI->getLMULCost(VT: LT.second);
932
933 if (Mask[0] == 0 || Mask[0] == 1) {
934 auto DeinterleaveMask = createStrideMask(Start: Mask[0], Stride: 2, VF: Mask.size());
935 // Example sequence:
936 // vnsrl.wi v10, v8, 0
937 if (equal(LRange&: DeinterleaveMask, RRange&: Mask))
938 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VNSRL_WI,
939 VT: LT.second, CostKind);
940 }
941 }
942
943 int SubVectorSize;
944 if (LT.second.getScalarSizeInBits() != 1 &&
945 isRepeatedConcatMask(Mask, SubVectorSize)) {
946 InstructionCost Cost = 0;
947 unsigned NumSlides = Log2_32(Value: Mask.size() / SubVectorSize);
948 // The cost of extraction from a subvector is 0 if the index is 0.
949 for (unsigned I = 0; I != NumSlides; ++I) {
950 unsigned InsertIndex = SubVectorSize * (1 << I);
951 FixedVectorType *SubTp =
952 FixedVectorType::get(ElementType: SrcTy->getElementType(), NumElts: InsertIndex);
953 FixedVectorType *DestTp =
954 FixedVectorType::getDoubleElementsVectorType(VTy: SubTp);
955 std::pair<InstructionCost, MVT> DestLT =
956 getTypeLegalizationCost(Ty: DestTp);
957 // Add the cost of whole vector register move because the
958 // destination vector register group for vslideup cannot overlap the
959 // source.
960 Cost += DestLT.first * TLI->getLMULCost(VT: DestLT.second);
961 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: DestTp, SrcTy: DestTp,
962 CostKind, Mask: {}, Index: InsertIndex, SubTp);
963 }
964 return Cost;
965 }
966 }
967
968 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
969 SlideCost.isValid())
970 return SlideCost;
971
972 // vrgather + cost of generating the mask constant.
973 // We model this for an unknown mask with a single vrgather.
974 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
975 LT.second.getVectorNumElements() <= 256)) {
976 VectorType *IdxTy =
977 getVRGatherIndexType(DataVT: LT.second, ST: *ST, C&: SrcTy->getContext());
978 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
979 return IndexCost +
980 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VV, VT: LT.second, CostKind);
981 }
982 break;
983 }
984 case TTI::SK_Transpose:
985 case TTI::SK_PermuteTwoSrc: {
986
987 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
988 SlideCost.isValid())
989 return SlideCost;
990
991 if (InstructionCost VZipCost =
992 getVZIPCost(Kind, DstTy, SrcTy, Mask, CostKind);
993 VZipCost.isValid())
994 return VZipCost;
995 // 2 x (vrgather + cost of generating the mask constant) + cost of mask
996 // register for the second vrgather. We model this for an unknown
997 // (shuffle) mask.
998 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
999 LT.second.getVectorNumElements() <= 256)) {
1000 auto &C = SrcTy->getContext();
1001 auto EC = SrcTy->getElementCount();
1002 VectorType *IdxTy = getVRGatherIndexType(DataVT: LT.second, ST: *ST, C);
1003 VectorType *MaskTy = VectorType::get(ElementType: IntegerType::getInt1Ty(C), EC);
1004 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
1005 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
1006 return 2 * IndexCost +
1007 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV, RISCV::VRGATHER_VV},
1008 VT: LT.second, CostKind) +
1009 MaskCost;
1010 }
1011 break;
1012 }
1013 }
1014
1015 auto shouldSplit = [](TTI::ShuffleKind Kind) {
1016 switch (Kind) {
1017 default:
1018 return false;
1019 case TTI::SK_PermuteSingleSrc:
1020 case TTI::SK_Transpose:
1021 case TTI::SK_PermuteTwoSrc:
1022 return true;
1023 }
1024 };
1025
1026 if (!Mask.empty() && LT.first.isValid() && LT.first != 1 &&
1027 shouldSplit(Kind)) {
1028 InstructionCost SplitCost =
1029 costShuffleViaSplitting(TTI: *this, LegalVT: LT.second, Tp: FVTp, Mask, CostKind);
1030 if (SplitCost.isValid())
1031 return SplitCost;
1032 }
1033 }
1034
1035 // Handle scalable vectors (and fixed vectors legalized to scalable vectors).
1036 switch (Kind) {
1037 default:
1038 // Fallthrough to generic handling.
1039 // TODO: Most of these cases will return getInvalid in generic code, and
1040 // must be implemented here.
1041 break;
1042 case TTI::SK_ExtractSubvector:
1043 // Extract at zero is always a subregister extract
1044 if (Index == 0)
1045 return TTI::TCC_Free;
1046
1047 // If we're extracting a subvector of at most m1 size at a sub-register
1048 // boundary - which unfortunately we need exact vlen to identify - this is
1049 // a subregister extract at worst and thus won't require a vslidedown.
1050 // TODO: Extend for aligned m2, m4 subvector extracts
1051 // TODO: Extend for misalgined (but contained) extracts
1052 // TODO: Extend for scalable subvector types
1053 if (std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
1054 SubLT.second.isValid() && SubLT.second.isFixedLengthVector()) {
1055 if (std::optional<unsigned> VLen = ST->getRealVLen();
1056 VLen && SubLT.second.getScalarSizeInBits() * Index % *VLen == 0 &&
1057 SubLT.second.getSizeInBits() <= *VLen)
1058 return TTI::TCC_Free;
1059 }
1060
1061 // Example sequence:
1062 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
1063 // vslidedown.vi v8, v9, 2
1064 return LT.first *
1065 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI, VT: LT.second, CostKind);
1066 case TTI::SK_InsertSubvector:
1067 // Example sequence:
1068 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
1069 // vslideup.vi v8, v9, 2
1070 LT = getTypeLegalizationCost(Ty: DstTy);
1071 return LT.first *
1072 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEUP_VI, VT: LT.second, CostKind);
1073 case TTI::SK_Select: {
1074 // Example sequence:
1075 // li a0, 90
1076 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
1077 // vmv.s.x v0, a0
1078 // vmerge.vvm v8, v9, v8, v0
1079 // We use 2 for the cost of the mask materialization as this is the true
1080 // cost for small masks and most shuffles are small. At worst, this cost
1081 // should be a very small constant for the constant pool load. As such,
1082 // we may bias towards large selects slightly more than truly warranted.
1083 return LT.first *
1084 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_S_X, RISCV::VMERGE_VVM},
1085 VT: LT.second, CostKind));
1086 }
1087 case TTI::SK_Broadcast: {
1088 // Check for broadcast loads, which are synthesized by optimized zero-stride
1089 // loads (this is checked in RISCVTTIImpl::isLegalBroadcastLoad).
1090 bool IsLoad = !Args.empty() && isa<LoadInst>(Val: Args[0]);
1091 if (IsLoad && LT.second.isVector() &&
1092 isLegalBroadcastLoad(ElementTy: SrcTy->getElementType(),
1093 NumElements: LT.second.getVectorElementCount()))
1094 return 0;
1095
1096 bool HasScalar = (Args.size() > 0) && (Operator::getOpcode(V: Args[0]) ==
1097 Instruction::InsertElement);
1098 if (LT.second.getScalarSizeInBits() == 1) {
1099 if (HasScalar) {
1100 // Example sequence:
1101 // andi a0, a0, 1
1102 // vsetivli zero, 2, e8, mf8, ta, ma (ignored)
1103 // vmv.v.x v8, a0
1104 // vmsne.vi v0, v8, 0
1105 return LT.first *
1106 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
1107 VT: LT.second, CostKind));
1108 }
1109 // Example sequence:
1110 // vsetivli zero, 2, e8, mf8, ta, mu (ignored)
1111 // vmv.v.i v8, 0
1112 // vmerge.vim v8, v8, 1, v0
1113 // vmv.x.s a0, v8
1114 // andi a0, a0, 1
1115 // vmv.v.x v8, a0
1116 // vmsne.vi v0, v8, 0
1117
1118 return LT.first *
1119 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_I, RISCV::VMERGE_VIM,
1120 RISCV::VMV_X_S, RISCV::VMV_V_X,
1121 RISCV::VMSNE_VI},
1122 VT: LT.second, CostKind));
1123 }
1124
1125 if (HasScalar) {
1126 // Example sequence:
1127 // vmv.v.x v8, a0
1128 return LT.first *
1129 getRISCVInstructionCost(OpCodes: RISCV::VMV_V_X, VT: LT.second, CostKind);
1130 }
1131
1132 // Example sequence:
1133 // vrgather.vi v9, v8, 0
1134 return LT.first *
1135 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VI, VT: LT.second, CostKind);
1136 }
1137 case TTI::SK_Splice: {
1138 // vslidedown+vslideup.
1139 // TODO: Multiplying by LT.first implies this legalizes into multiple copies
1140 // of similar code, but I think we expand through memory.
1141 unsigned Opcodes[2] = {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX};
1142 if (Index >= 0 && Index < 32)
1143 Opcodes[0] = RISCV::VSLIDEDOWN_VI;
1144 else if (Index < 0 && Index > -32)
1145 Opcodes[1] = RISCV::VSLIDEUP_VI;
1146 return LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1147 }
1148 case TTI::SK_Reverse: {
1149
1150 if (!LT.second.isVector())
1151 return InstructionCost::getInvalid();
1152
1153 // TODO: Cases to improve here:
1154 // * Illegal vector types
1155 // * i64 on RV32
1156 if (SrcTy->getElementType()->isIntegerTy(BitWidth: 1)) {
1157 VectorType *WideTy =
1158 VectorType::get(ElementType: IntegerType::get(C&: SrcTy->getContext(), NumBits: 8),
1159 EC: cast<VectorType>(Val: SrcTy)->getElementCount());
1160 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: SrcTy,
1161 CCH: TTI::CastContextHint::None, CostKind) +
1162 getShuffleCost(Kind: TTI::SK_Reverse, DstTy: WideTy, SrcTy: WideTy, CostKind, Mask: {}, Index: 0,
1163 SubTp: nullptr) +
1164 getCastInstrCost(Opcode: Instruction::Trunc, Dst: SrcTy, Src: WideTy,
1165 CCH: TTI::CastContextHint::None, CostKind);
1166 }
1167
1168 MVT ContainerVT = LT.second;
1169 if (LT.second.isFixedLengthVector())
1170 ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1171 MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
1172 if (ContainerVT.bitsLE(VT: M1VT)) {
1173 // Example sequence:
1174 // csrr a0, vlenb
1175 // srli a0, a0, 3
1176 // addi a0, a0, -1
1177 // vsetvli a1, zero, e8, mf8, ta, mu (ignored)
1178 // vid.v v9
1179 // vrsub.vx v10, v9, a0
1180 // vrgather.vv v9, v8, v10
1181 InstructionCost LenCost = 3;
1182 if (LT.second.isFixedLengthVector())
1183 // vrsub.vi has a 5 bit immediate field, otherwise an li suffices
1184 LenCost = isInt<5>(x: LT.second.getVectorNumElements() - 1) ? 0 : 1;
1185 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX, RISCV::VRGATHER_VV};
1186 if (LT.second.isFixedLengthVector() &&
1187 isInt<5>(x: LT.second.getVectorNumElements() - 1))
1188 Opcodes[1] = RISCV::VRSUB_VI;
1189 InstructionCost GatherCost =
1190 getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1191 return LT.first * (LenCost + GatherCost);
1192 }
1193
1194 // At high LMUL, we split into a series of M1 reverses (see
1195 // lowerVECTOR_REVERSE) and then do a single slide at the end to eliminate
1196 // the resulting gap at the bottom (for fixed vectors only). The important
1197 // bit is that the cost scales linearly, not quadratically with LMUL.
1198 unsigned M1Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX};
1199 InstructionCost FixedCost =
1200 getRISCVInstructionCost(OpCodes: M1Opcodes, VT: M1VT, CostKind) + 3;
1201 unsigned Ratio =
1202 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
1203 InstructionCost GatherCost =
1204 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV}, VT: M1VT, CostKind) * Ratio;
1205 InstructionCost SlideCost = !LT.second.isFixedLengthVector() ? 0 :
1206 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX}, VT: LT.second, CostKind);
1207 return FixedCost + LT.first * (GatherCost + SlideCost);
1208 }
1209 }
1210 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1211 SubTp);
1212}
1213
1214static unsigned isM1OrSmaller(MVT VT) {
1215 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1216 return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
1217 LMUL == RISCVVType::VLMUL::LMUL_F4 ||
1218 LMUL == RISCVVType::VLMUL::LMUL_F2 ||
1219 LMUL == RISCVVType::VLMUL::LMUL_1);
1220}
1221
1222InstructionCost RISCVTTIImpl::getScalarizationOverhead(
1223 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
1224 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
1225 TTI::VectorInstrContext VIC) const {
1226 if (isa<ScalableVectorType>(Val: Ty))
1227 return InstructionCost::getInvalid();
1228
1229 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
1230 // For now, skip all fixed vector cost analysis when P extension is available
1231 // to avoid crashes in getMinRVVVectorSizeInBits()
1232 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: Ty)) {
1233 return 1; // Treat as single instruction cost for now
1234 }
1235
1236 // A build_vector (which is m1 sized or smaller) can be done in no
1237 // worse than one vslide1down.vx per element in the type. We could
1238 // in theory do an explode_vector in the inverse manner, but our
1239 // lowering today does not have a first class node for this pattern.
1240 InstructionCost Cost = BaseT::getScalarizationOverhead(
1241 InTy: Ty, DemandedElts, Insert, Extract, CostKind);
1242 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1243 if (Insert && !Extract && LT.first.isValid() && LT.second.isVector()) {
1244 if (Ty->getScalarSizeInBits() == 1) {
1245 auto *WideVecTy = cast<VectorType>(Val: Ty->getWithNewBitWidth(NewBitWidth: 8));
1246 // Note: Implicit scalar anyextend is assumed to be free since the i1
1247 // must be stored in a GPR.
1248 return getScalarizationOverhead(Ty: WideVecTy, DemandedElts, Insert, Extract,
1249 CostKind) +
1250 getCastInstrCost(Opcode: Instruction::Trunc, Dst: Ty, Src: WideVecTy,
1251 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
1252 }
1253
1254 assert(LT.second.isFixedLengthVector());
1255 MVT ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1256 if (isM1OrSmaller(VT: ContainerVT)) {
1257 InstructionCost BV =
1258 cast<FixedVectorType>(Val: Ty)->getNumElements() *
1259 getRISCVInstructionCost(OpCodes: RISCV::VSLIDE1DOWN_VX, VT: LT.second, CostKind);
1260 if (BV < Cost)
1261 Cost = BV;
1262 }
1263 }
1264 return Cost;
1265}
1266
1267InstructionCost
1268RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1269 TTI::TargetCostKind CostKind) const {
1270 Type *DataTy = MICA.getDataType();
1271 Align Alignment = MICA.getAlignment();
1272 switch (MICA.getID()) {
1273 case Intrinsic::vp_load_ff: {
1274 EVT DataTypeVT = TLI->getValueType(DL, Ty: DataTy);
1275 if (!TLI->isLegalFirstFaultLoad(DataType: DataTypeVT, Alignment))
1276 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1277
1278 unsigned AS = MICA.getAddressSpace();
1279 return getMemoryOpCost(Opcode: Instruction::Load, Src: DataTy, Alignment, AddressSpace: AS, CostKind,
1280 OpdInfo: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I: nullptr);
1281 }
1282 case Intrinsic::experimental_vp_strided_load:
1283 case Intrinsic::experimental_vp_strided_store:
1284 return getStridedMemoryOpCost(MICA, CostKind);
1285 case Intrinsic::masked_compressstore:
1286 case Intrinsic::masked_expandload:
1287 return getExpandCompressMemoryOpCost(MICA, CostKind);
1288 case Intrinsic::vp_scatter:
1289 case Intrinsic::vp_gather:
1290 case Intrinsic::masked_scatter:
1291 case Intrinsic::masked_gather:
1292 return getGatherScatterOpCost(MICA, CostKind);
1293 case Intrinsic::vp_load:
1294 case Intrinsic::vp_store:
1295 case Intrinsic::masked_load:
1296 case Intrinsic::masked_store:
1297 return getMaskedMemoryOpCost(MICA, CostKind);
1298 }
1299 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1300}
1301
1302InstructionCost
1303RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1304 TTI::TargetCostKind CostKind) const {
1305 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
1306 : Instruction::Store;
1307 Type *Src = MICA.getDataType();
1308 Align Alignment = MICA.getAlignment();
1309 unsigned AddressSpace = MICA.getAddressSpace();
1310
1311 if (!isLegalMaskedLoadStore(DataType: Src, Alignment) ||
1312 CostKind != TTI::TCK_RecipThroughput)
1313 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1314
1315 // Splitting involves additional evl arithmetic and vl toggles.
1316 InstructionCost SplitCost = 0;
1317 if (MICA.getID() == Intrinsic::vp_load ||
1318 MICA.getID() == Intrinsic::vp_store) {
1319 auto LT = getTypeLegalizationCost(Ty: Src);
1320 if (LT.first > 1)
1321 SplitCost += LT.first * TTI::TCC_Expensive;
1322 }
1323
1324 return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
1325}
1326
1327InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
1328 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1329 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1330 bool UseMaskForCond, bool UseMaskForGaps) const {
1331
1332 // The interleaved memory access pass will lower (de)interleave ops combined
1333 // with an adjacent appropriate memory to vlseg/vsseg intrinsics. vlseg/vsseg
1334 // only support masking per-iteration (i.e. condition), not per-segment (i.e.
1335 // gap).
1336 if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {
1337 auto *VTy = cast<VectorType>(Val: VecTy);
1338 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VTy);
1339 // Need to make sure type has't been scalarized
1340 if (LT.second.isVector()) {
1341 if (CostKind == TTI::TCK_CodeSize)
1342 return LT.first * TTI::TCC_Basic;
1343
1344 auto *SubVecTy =
1345 VectorType::get(ElementType: VTy->getElementType(),
1346 EC: VTy->getElementCount().divideCoefficientBy(RHS: Factor));
1347 if (VTy->getElementCount().isKnownMultipleOf(RHS: Factor) &&
1348 TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor, Alignment,
1349 AddrSpace: AddressSpace, DL)) {
1350
1351 // Some processors optimize segment loads/stores as N * DLEN sized
1352 // load ops + Factor * LMUL shuffle ops.
1353 if (ST->hasOptimizedSegmentLoadStore(NF: Factor)) {
1354 unsigned VecSizeInBits =
1355 getEstimatedVLFor(Ty: VTy) * VTy->getScalarSizeInBits();
1356 unsigned VLENForTuning =
1357 *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
1358 unsigned DLENForTuning = VLENForTuning / ST->getDLenFactor();
1359 InstructionCost Cost = divideCeil(Numerator: VecSizeInBits, Denominator: DLENForTuning);
1360 MVT SubVecVT = getTLI()->getValueType(DL, Ty: SubVecTy).getSimpleVT();
1361 Cost += Factor * TLI->getLMULCost(VT: SubVecVT);
1362 return Cost;
1363 }
1364
1365 // Otherwise, the cost is proportional to the number of elements (VL *
1366 // Factor ops).
1367 unsigned NumLoads = getEstimatedVLFor(Ty: VTy);
1368 return NumLoads * TTI::TCC_Basic;
1369 }
1370 }
1371 }
1372
1373 // TODO: Return the cost of interleaved accesses for scalable vector when
1374 // unable to convert to segment accesses instructions.
1375 if (isa<ScalableVectorType>(Val: VecTy))
1376 return InstructionCost::getInvalid();
1377
1378 auto *FVTy = cast<FixedVectorType>(Val: VecTy);
1379 // When gaps are only at the tail, for interleaved load, we can emit a wide
1380 // masked load and shufflevectors. For interleaved store, we can emit
1381 // shufflevectors and a wide masked store. The interleaved memory access pass
1382 // will lower them into vlsseg/vssseg intrinsics.
1383 if (UseMaskForGaps) {
1384 assert(llvm::is_sorted(Indices) && "Indices must be sorted");
1385 assert(llvm::adjacent_find(Indices) == Indices.end() &&
1386 "Indices should not contain duplicate elements");
1387 unsigned NumOfFields = Indices.size();
1388 bool IsTailGapOnly = NumOfFields > 1 && (NumOfFields == Indices.back() + 1);
1389 if (IsTailGapOnly &&
1390 NumOfFields <= TLI->getMaxSupportedInterleaveFactor()) {
1391 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: FVTy);
1392 if (LT.second.isVector() &&
1393 FVTy->getElementCount().isKnownMultipleOf(RHS: Factor)) {
1394 auto *SubVecTy = VectorType::get(
1395 ElementType: FVTy->getElementType(),
1396 EC: FVTy->getElementCount().divideCoefficientBy(RHS: Factor));
1397 if (TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor: NumOfFields, Alignment,
1398 AddrSpace: AddressSpace, DL)) {
1399 // The cost is proportional to the total number of element accesses.
1400 unsigned NumAccesses = getEstimatedVLFor(Ty: FVTy);
1401 return NumAccesses * TTI::TCC_Basic;
1402 }
1403 }
1404 }
1405 }
1406
1407 InstructionCost MemCost =
1408 getMemoryOpCost(Opcode, Src: VecTy, Alignment, AddressSpace, CostKind);
1409 unsigned VF = FVTy->getNumElements() / Factor;
1410
1411 // An interleaved load will look like this for Factor=3:
1412 // %wide.vec = load <12 x i32>, ptr %3, align 4
1413 // %strided.vec = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1414 // %strided.vec1 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1415 // %strided.vec2 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1416 if (Opcode == Instruction::Load) {
1417 InstructionCost Cost = MemCost;
1418 for (unsigned Index : Indices) {
1419 FixedVectorType *VecTy =
1420 FixedVectorType::get(ElementType: FVTy->getElementType(), NumElts: VF * Factor);
1421 auto Mask = createStrideMask(Start: Index, Stride: Factor, VF);
1422 Mask.resize(N: VF * Factor, NV: -1);
1423 InstructionCost ShuffleCost =
1424 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: VecTy, SrcTy: VecTy,
1425 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {});
1426 Cost += ShuffleCost;
1427 }
1428 return Cost;
1429 }
1430
1431 // TODO: Model for NF > 2
1432 // We'll need to enhance getShuffleCost to model shuffles that are just
1433 // inserts and extracts into subvectors, since they won't have the full cost
1434 // of a vrgather.
1435 // An interleaved store for 3 vectors of 4 lanes will look like
1436 // %11 = shufflevector <4 x i32> %4, <4 x i32> %6, <8 x i32> <0...7>
1437 // %12 = shufflevector <4 x i32> %9, <4 x i32> poison, <8 x i32> <0...3>
1438 // %13 = shufflevector <8 x i32> %11, <8 x i32> %12, <12 x i32> <0...11>
1439 // %interleaved.vec = shufflevector %13, poison, <12 x i32> <interleave mask>
1440 // store <12 x i32> %interleaved.vec, ptr %10, align 4
1441 if (Factor != 2)
1442 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1443 Alignment, AddressSpace, CostKind,
1444 UseMaskForCond, UseMaskForGaps);
1445
1446 assert(Opcode == Instruction::Store && "Opcode must be a store");
1447 // For an interleaving store of 2 vectors, we perform one large interleaving
1448 // shuffle that goes into the wide store
1449 auto Mask = createInterleaveMask(VF, NumVecs: Factor);
1450 InstructionCost ShuffleCost =
1451 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: FVTy, SrcTy: FVTy,
1452 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {});
1453 return MemCost + ShuffleCost;
1454}
1455
1456InstructionCost
1457RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1458 TTI::TargetCostKind CostKind) const {
1459
1460 bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
1461 MICA.getID() == Intrinsic::vp_gather;
1462 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
1463 Type *DataTy = MICA.getDataType();
1464 Type *PtrTy = DataTy->getWithNewType(
1465 EltTy: DL.getAddressType(C&: DataTy->getContext(), AddressSpace: MICA.getAddressSpace()));
1466 Align Alignment = MICA.getAlignment();
1467 if (CostKind != TTI::TCK_RecipThroughput)
1468 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1469
1470 if ((Opcode == Instruction::Load &&
1471 !isLegalMaskedGather(DataType: DataTy, Alignment: Align(Alignment))) ||
1472 (Opcode == Instruction::Store &&
1473 !isLegalMaskedScatter(DataType: DataTy, Alignment: Align(Alignment))))
1474 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1475
1476 // Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
1477 InstructionCost SplitCost = 0;
1478 if (MICA.getID() == Intrinsic::vp_gather ||
1479 MICA.getID() == Intrinsic::vp_scatter) {
1480 auto DataLT = getTypeLegalizationCost(Ty: DataTy);
1481 auto PtrLT = getTypeLegalizationCost(Ty: PtrTy);
1482 if (DataLT.first > 1)
1483 SplitCost += DataLT.first * TTI::TCC_Expensive;
1484 if (PtrLT.first > 1)
1485 SplitCost += PtrLT.first * TTI::TCC_Expensive;
1486 }
1487
1488 // Cost is proportional to the number of memory operations implied. For
1489 // scalable vectors, we use an estimate on that number since we don't
1490 // know exactly what VL will be.
1491 auto &VTy = *cast<VectorType>(Val: DataTy);
1492 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1493 return SplitCost + NumLoads * TTI::TCC_Basic;
1494}
1495
1496InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
1497 const MemIntrinsicCostAttributes &MICA,
1498 TTI::TargetCostKind CostKind) const {
1499 unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
1500 ? Instruction::Load
1501 : Instruction::Store;
1502 Type *DataTy = MICA.getDataType();
1503 bool VariableMask = MICA.getVariableMask();
1504 Align Alignment = MICA.getAlignment();
1505 bool IsLegal = (Opcode == Instruction::Store &&
1506 isLegalMaskedCompressStore(DataTy, Alignment)) ||
1507 (Opcode == Instruction::Load &&
1508 isLegalMaskedExpandLoad(DataType: DataTy, Alignment));
1509 if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
1510 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1511 // Example compressstore sequence:
1512 // vsetivli zero, 8, e32, m2, ta, ma (ignored)
1513 // vcompress.vm v10, v8, v0
1514 // vcpop.m a1, v0
1515 // vsetvli zero, a1, e32, m2, ta, ma
1516 // vse32.v v10, (a0)
1517 // Example expandload sequence:
1518 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
1519 // vcpop.m a1, v0
1520 // vsetvli zero, a1, e32, m2, ta, ma
1521 // vle32.v v10, (a0)
1522 // vsetivli zero, 8, e32, m2, ta, ma
1523 // viota.m v12, v0
1524 // vrgather.vv v8, v10, v12, v0.t
1525 auto MemOpCost =
1526 getMemoryOpCost(Opcode, Src: DataTy, Alignment, /*AddressSpace*/ 0, CostKind);
1527 auto LT = getTypeLegalizationCost(Ty: DataTy);
1528 SmallVector<unsigned, 4> Opcodes{RISCV::VSETVLI};
1529 if (VariableMask)
1530 Opcodes.push_back(Elt: RISCV::VCPOP_M);
1531 if (Opcode == Instruction::Store)
1532 Opcodes.append(IL: {RISCV::VCOMPRESS_VM});
1533 else
1534 Opcodes.append(IL: {RISCV::VSETIVLI, RISCV::VIOTA_M, RISCV::VRGATHER_VV});
1535 return MemOpCost +
1536 LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1537}
1538
1539InstructionCost
1540RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1541 TTI::TargetCostKind CostKind) const {
1542 Type *DataTy = MICA.getDataType();
1543 Align Alignment = MICA.getAlignment();
1544
1545 if (!isLegalStridedLoadStore(DataType: DataTy, Alignment))
1546 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1547
1548 if (CostKind == TTI::TCK_CodeSize)
1549 return TTI::TCC_Basic;
1550
1551 // Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
1552 InstructionCost SplitCost = 0;
1553 auto LT = getTypeLegalizationCost(Ty: DataTy);
1554 if (LT.first > 1)
1555 SplitCost += LT.first * TTI::TCC_Expensive;
1556
1557 // Cost is proportional to the number of memory operations implied. For
1558 // scalable vectors, we use an estimate on that number since we don't
1559 // know exactly what VL will be.
1560 auto &VTy = *cast<VectorType>(Val: DataTy);
1561 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1562 // Performant implementations of the vector extension will coalesce
1563 // elements if they fall on the same cache line
1564 uint64_t CacheLineBytes = ST->getCacheLineSize();
1565 if (!CacheLineBytes) // If no value, use default value of 64
1566 CacheLineBytes = 64;
1567 if (const ConstantInt *StrideCI =
1568 dyn_cast_or_null<ConstantInt>(Val: MICA.getStrideVal())) {
1569 int64_t Stride = StrideCI->getSExtValue();
1570 // Bail early to avoid UB with std:abs() call
1571 if (Stride != std::numeric_limits<int64_t>::min() && Stride != 0) {
1572 uint64_t AbsStride = (uint64_t)std::abs(i: Stride);
1573 if (AbsStride < CacheLineBytes) {
1574 uint64_t MaxCombines = ST->getMaxVectorCoalesceElts();
1575 if ((CacheLineBytes / AbsStride) >= MaxCombines)
1576 NumLoads = divideCeil(Numerator: NumLoads, Denominator: MaxCombines);
1577 else
1578 // If we were to calculate CacheLineBytes / AbsStride first, would
1579 // lose accuracy
1580 NumLoads = divideCeil(Numerator: (NumLoads * AbsStride), Denominator: CacheLineBytes);
1581 }
1582 }
1583 }
1584 return SplitCost + NumLoads * TTI::TCC_Basic;
1585}
1586
1587InstructionCost
1588RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1589 // FIXME: This is a property of the default vector convention, not
1590 // all possible calling conventions. Fixing that will require
1591 // some TTI API and SLP rework.
1592 InstructionCost Cost = 0;
1593 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
1594 for (auto *Ty : Tys) {
1595 if (!Ty->isVectorTy())
1596 continue;
1597 Align A = DL.getPrefTypeAlign(Ty);
1598 Cost += getMemoryOpCost(Opcode: Instruction::Store, Src: Ty, Alignment: A, AddressSpace: 0, CostKind) +
1599 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: A, AddressSpace: 0, CostKind);
1600 }
1601 return Cost;
1602}
1603
1604// Currently, these represent both throughput and codesize costs
1605// for the respective intrinsics. The costs in this table are simply
1606// instruction counts with the following adjustments made:
1607// * One vsetvli is considered free.
1608static const CostTblEntry VectorIntrinsicCostTable[]{
1609 {.ISD: Intrinsic::floor, .Type: MVT::f32, .Cost: 9},
1610 {.ISD: Intrinsic::floor, .Type: MVT::f64, .Cost: 9},
1611 {.ISD: Intrinsic::ceil, .Type: MVT::f32, .Cost: 9},
1612 {.ISD: Intrinsic::ceil, .Type: MVT::f64, .Cost: 9},
1613 {.ISD: Intrinsic::trunc, .Type: MVT::f32, .Cost: 7},
1614 {.ISD: Intrinsic::trunc, .Type: MVT::f64, .Cost: 7},
1615 {.ISD: Intrinsic::round, .Type: MVT::f32, .Cost: 9},
1616 {.ISD: Intrinsic::round, .Type: MVT::f64, .Cost: 9},
1617 {.ISD: Intrinsic::roundeven, .Type: MVT::f32, .Cost: 9},
1618 {.ISD: Intrinsic::roundeven, .Type: MVT::f64, .Cost: 9},
1619 {.ISD: Intrinsic::rint, .Type: MVT::f32, .Cost: 7},
1620 {.ISD: Intrinsic::rint, .Type: MVT::f64, .Cost: 7},
1621 {.ISD: Intrinsic::nearbyint, .Type: MVT::f32, .Cost: 9},
1622 {.ISD: Intrinsic::nearbyint, .Type: MVT::f64, .Cost: 9},
1623 {.ISD: Intrinsic::bswap, .Type: MVT::i16, .Cost: 3},
1624 {.ISD: Intrinsic::bswap, .Type: MVT::i32, .Cost: 12},
1625 {.ISD: Intrinsic::bswap, .Type: MVT::i64, .Cost: 31},
1626 {.ISD: Intrinsic::bitreverse, .Type: MVT::i8, .Cost: 17},
1627 {.ISD: Intrinsic::bitreverse, .Type: MVT::i16, .Cost: 24},
1628 {.ISD: Intrinsic::bitreverse, .Type: MVT::i32, .Cost: 33},
1629 {.ISD: Intrinsic::bitreverse, .Type: MVT::i64, .Cost: 52},
1630 {.ISD: Intrinsic::ctpop, .Type: MVT::i8, .Cost: 12},
1631 {.ISD: Intrinsic::ctpop, .Type: MVT::i16, .Cost: 19},
1632 {.ISD: Intrinsic::ctpop, .Type: MVT::i32, .Cost: 20},
1633 {.ISD: Intrinsic::ctpop, .Type: MVT::i64, .Cost: 21},
1634 {.ISD: Intrinsic::ctlz, .Type: MVT::i8, .Cost: 19},
1635 {.ISD: Intrinsic::ctlz, .Type: MVT::i16, .Cost: 28},
1636 {.ISD: Intrinsic::ctlz, .Type: MVT::i32, .Cost: 31},
1637 {.ISD: Intrinsic::ctlz, .Type: MVT::i64, .Cost: 35},
1638 {.ISD: Intrinsic::cttz, .Type: MVT::i8, .Cost: 16},
1639 {.ISD: Intrinsic::cttz, .Type: MVT::i16, .Cost: 23},
1640 {.ISD: Intrinsic::cttz, .Type: MVT::i32, .Cost: 24},
1641 {.ISD: Intrinsic::cttz, .Type: MVT::i64, .Cost: 25},
1642};
1643
1644InstructionCost
1645RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1646 TTI::TargetCostKind CostKind) const {
1647 auto *RetTy = ICA.getReturnType();
1648 switch (ICA.getID()) {
1649 case Intrinsic::lrint:
1650 case Intrinsic::llrint:
1651 case Intrinsic::lround:
1652 case Intrinsic::llround: {
1653 auto LT = getTypeLegalizationCost(Ty: RetTy);
1654 Type *SrcTy = ICA.getArgTypes().front();
1655 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
1656 if (ST->hasVInstructions() && LT.second.isVector()) {
1657 SmallVector<unsigned, 2> Ops;
1658 unsigned SrcEltSz = DL.getTypeSizeInBits(Ty: SrcTy->getScalarType());
1659 unsigned DstEltSz = DL.getTypeSizeInBits(Ty: RetTy->getScalarType());
1660 if (LT.second.getVectorElementType() == MVT::bf16) {
1661 if (!ST->hasVInstructionsBF16Minimal())
1662 return InstructionCost::getInvalid();
1663 if (DstEltSz == 32)
1664 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFCVT_X_F_V};
1665 else
1666 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVT_X_F_V};
1667 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1668 !ST->hasVInstructionsF16()) {
1669 if (!ST->hasVInstructionsF16Minimal())
1670 return InstructionCost::getInvalid();
1671 if (DstEltSz == 32)
1672 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFCVT_X_F_V};
1673 else
1674 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_X_F_V};
1675
1676 } else if (SrcEltSz > DstEltSz) {
1677 Ops = {RISCV::VFNCVT_X_F_W};
1678 } else if (SrcEltSz < DstEltSz) {
1679 Ops = {RISCV::VFWCVT_X_F_V};
1680 } else {
1681 Ops = {RISCV::VFCVT_X_F_V};
1682 }
1683
1684 // We need to use the source LMUL in the case of a narrowing op, and the
1685 // destination LMUL otherwise.
1686 if (SrcEltSz > DstEltSz)
1687 return SrcLT.first *
1688 getRISCVInstructionCost(OpCodes: Ops, VT: SrcLT.second, CostKind);
1689 return LT.first * getRISCVInstructionCost(OpCodes: Ops, VT: LT.second, CostKind);
1690 }
1691 break;
1692 }
1693 case Intrinsic::ceil:
1694 case Intrinsic::floor:
1695 case Intrinsic::trunc:
1696 case Intrinsic::rint:
1697 case Intrinsic::round:
1698 case Intrinsic::roundeven: {
1699 // These all use the same code.
1700 auto LT = getTypeLegalizationCost(Ty: RetTy);
1701 if (!LT.second.isVector() && TLI->isOperationCustom(Op: ISD::FCEIL, VT: LT.second))
1702 return LT.first * 8;
1703 break;
1704 }
1705 case Intrinsic::umin:
1706 case Intrinsic::umax:
1707 case Intrinsic::smin:
1708 case Intrinsic::smax: {
1709 auto LT = getTypeLegalizationCost(Ty: RetTy);
1710 if (LT.second.isScalarInteger() && ST->hasStdExtZbb())
1711 return LT.first;
1712
1713 if (ST->hasVInstructions() && LT.second.isVector()) {
1714 unsigned Op;
1715 switch (ICA.getID()) {
1716 case Intrinsic::umin:
1717 Op = RISCV::VMINU_VV;
1718 break;
1719 case Intrinsic::umax:
1720 Op = RISCV::VMAXU_VV;
1721 break;
1722 case Intrinsic::smin:
1723 Op = RISCV::VMIN_VV;
1724 break;
1725 case Intrinsic::smax:
1726 Op = RISCV::VMAX_VV;
1727 break;
1728 }
1729 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1730 }
1731 break;
1732 }
1733 case Intrinsic::sadd_sat:
1734 case Intrinsic::ssub_sat:
1735 case Intrinsic::uadd_sat:
1736 case Intrinsic::usub_sat: {
1737 auto LT = getTypeLegalizationCost(Ty: RetTy);
1738 if (ST->hasVInstructions() && LT.second.isVector()) {
1739 unsigned Op;
1740 switch (ICA.getID()) {
1741 case Intrinsic::sadd_sat:
1742 Op = RISCV::VSADD_VV;
1743 break;
1744 case Intrinsic::ssub_sat:
1745 Op = RISCV::VSSUB_VV;
1746 break;
1747 case Intrinsic::uadd_sat:
1748 Op = RISCV::VSADDU_VV;
1749 break;
1750 case Intrinsic::usub_sat:
1751 Op = RISCV::VSSUBU_VV;
1752 break;
1753 }
1754 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1755 }
1756 break;
1757 }
1758 case Intrinsic::fma:
1759 case Intrinsic::fmuladd: {
1760 // TODO: handle promotion with f16/bf16 with zvfhmin/zvfbfmin
1761 auto LT = getTypeLegalizationCost(Ty: RetTy);
1762 if (ST->hasVInstructions() && LT.second.isVector())
1763 return LT.first *
1764 getRISCVInstructionCost(OpCodes: RISCV::VFMADD_VV, VT: LT.second, CostKind);
1765 break;
1766 }
1767 case Intrinsic::fabs: {
1768 auto LT = getTypeLegalizationCost(Ty: RetTy);
1769 if (ST->hasVInstructions() && LT.second.isVector()) {
1770 // lui a0, 8
1771 // addi a0, a0, -1
1772 // vsetvli a1, zero, e16, m1, ta, ma
1773 // vand.vx v8, v8, a0
1774 // f16 with zvfhmin and bf16 with zvfhbmin
1775 if (LT.second.getVectorElementType() == MVT::bf16 ||
1776 (LT.second.getVectorElementType() == MVT::f16 &&
1777 !ST->hasVInstructionsF16()))
1778 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VAND_VX, VT: LT.second,
1779 CostKind) +
1780 2;
1781 else
1782 return LT.first *
1783 getRISCVInstructionCost(OpCodes: RISCV::VFSGNJX_VV, VT: LT.second, CostKind);
1784 }
1785 break;
1786 }
1787 case Intrinsic::sqrt: {
1788 auto LT = getTypeLegalizationCost(Ty: RetTy);
1789 if (ST->hasVInstructions() && LT.second.isVector()) {
1790 SmallVector<unsigned, 4> ConvOp;
1791 SmallVector<unsigned, 2> FsqrtOp;
1792 MVT ConvType = LT.second;
1793 MVT FsqrtType = LT.second;
1794 // f16 with zvfhmin and bf16 with zvfbfmin and the type of nxv32[b]f16
1795 // will be spilt.
1796 if (LT.second.getVectorElementType() == MVT::bf16) {
1797 if (LT.second == MVT::nxv32bf16) {
1798 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVTBF16_F_F_V,
1799 RISCV::VFNCVTBF16_F_F_W, RISCV::VFNCVTBF16_F_F_W};
1800 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1801 ConvType = MVT::nxv16f16;
1802 FsqrtType = MVT::nxv16f32;
1803 } else {
1804 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFNCVTBF16_F_F_W};
1805 FsqrtOp = {RISCV::VFSQRT_V};
1806 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1807 }
1808 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1809 !ST->hasVInstructionsF16()) {
1810 if (LT.second == MVT::nxv32f16) {
1811 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_F_F_V,
1812 RISCV::VFNCVT_F_F_W, RISCV::VFNCVT_F_F_W};
1813 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1814 ConvType = MVT::nxv16f16;
1815 FsqrtType = MVT::nxv16f32;
1816 } else {
1817 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFNCVT_F_F_W};
1818 FsqrtOp = {RISCV::VFSQRT_V};
1819 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1820 }
1821 } else {
1822 FsqrtOp = {RISCV::VFSQRT_V};
1823 }
1824
1825 return LT.first * (getRISCVInstructionCost(OpCodes: FsqrtOp, VT: FsqrtType, CostKind) +
1826 getRISCVInstructionCost(OpCodes: ConvOp, VT: ConvType, CostKind));
1827 }
1828 break;
1829 }
1830 case Intrinsic::cttz:
1831 case Intrinsic::ctlz:
1832 case Intrinsic::ctpop: {
1833 auto LT = getTypeLegalizationCost(Ty: RetTy);
1834 if (ST->hasStdExtZvbb() && LT.second.isVector()) {
1835 unsigned Op;
1836 switch (ICA.getID()) {
1837 case Intrinsic::cttz:
1838 Op = RISCV::VCTZ_V;
1839 break;
1840 case Intrinsic::ctlz:
1841 Op = RISCV::VCLZ_V;
1842 break;
1843 case Intrinsic::ctpop:
1844 Op = RISCV::VCPOP_V;
1845 break;
1846 }
1847 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1848 }
1849 break;
1850 }
1851 case Intrinsic::abs: {
1852 auto LT = getTypeLegalizationCost(Ty: RetTy);
1853 if (ST->hasVInstructions() && LT.second.isVector()) {
1854 // vabs.v v10, v8 (alias for vabd.vx v10, v8, zero)
1855 if (ST->hasStdExtZvabd())
1856 return LT.first *
1857 getRISCVInstructionCost(OpCodes: {RISCV::VABD_VX}, VT: LT.second, CostKind);
1858
1859 // vrsub.vi v10, v8, 0
1860 // vmax.vv v8, v8, v10
1861 return LT.first *
1862 getRISCVInstructionCost(OpCodes: {RISCV::VRSUB_VI, RISCV::VMAX_VV},
1863 VT: LT.second, CostKind);
1864 }
1865 break;
1866 }
1867 case Intrinsic::fshl:
1868 case Intrinsic::fshr: {
1869 if (ICA.getArgs().empty())
1870 break;
1871
1872 // Funnel-shifts are ROTL/ROTR when the first and second operand are equal.
1873 // When Zbb/Zbkb is enabled we can use a single ROL(W)/ROR(I)(W)
1874 // instruction.
1875 if ((ST->hasStdExtZbb() || ST->hasStdExtZbkb()) && RetTy->isIntegerTy() &&
1876 ICA.getArgs()[0] == ICA.getArgs()[1] &&
1877 (RetTy->getIntegerBitWidth() == 32 ||
1878 RetTy->getIntegerBitWidth() == 64) &&
1879 RetTy->getIntegerBitWidth() <= ST->getXLen()) {
1880 return 1;
1881 }
1882 break;
1883 }
1884 case Intrinsic::clmul: {
1885 auto LT = getTypeLegalizationCost(Ty: RetTy);
1886 if (!LT.second.isVector() && ST->hasStdExtZvbc() && !ST->hasStdExtZbkc()) {
1887 // TODO: Once custom lowering in this case for RV32 is added, this guard
1888 // should be removed and the cost model should be updated.
1889 if (!ST->is64Bit() || LT.second != MVT::i64)
1890 break;
1891 // vmv.s.x v8, a0
1892 // vclmul.vx v8, v8, a1
1893 // vmv.x.s a0, v8
1894 MVT VecVT = MVT::getScalableVectorVT(VT: LT.second, NumElements: 1);
1895 return LT.first * getRISCVInstructionCost(
1896 OpCodes: {RISCV::VMV_S_X, RISCV::VCLMUL_VX, RISCV::VMV_X_S},
1897 VT: VecVT, CostKind);
1898 }
1899 break;
1900 }
1901 case Intrinsic::masked_udiv:
1902 return getArithmeticInstrCost(Opcode: Instruction::UDiv, Ty: ICA.getReturnType(),
1903 CostKind);
1904 case Intrinsic::masked_sdiv:
1905 return getArithmeticInstrCost(Opcode: Instruction::SDiv, Ty: ICA.getReturnType(),
1906 CostKind);
1907 case Intrinsic::masked_urem:
1908 return getArithmeticInstrCost(Opcode: Instruction::URem, Ty: ICA.getReturnType(),
1909 CostKind);
1910 case Intrinsic::masked_srem:
1911 return getArithmeticInstrCost(Opcode: Instruction::SRem, Ty: ICA.getReturnType(),
1912 CostKind);
1913 case Intrinsic::get_active_lane_mask: {
1914 if (ST->hasVInstructions()) {
1915 Type *ExpRetTy = VectorType::get(
1916 ElementType: ICA.getArgTypes()[0], EC: cast<VectorType>(Val: RetTy)->getElementCount());
1917 auto LT = getTypeLegalizationCost(Ty: ExpRetTy);
1918
1919 // vid.v v8 // considered hoisted
1920 // vsaddu.vx v8, v8, a0
1921 // vmsltu.vx v0, v8, a1
1922 return LT.first *
1923 getRISCVInstructionCost(OpCodes: {RISCV::VSADDU_VX, RISCV::VMSLTU_VX},
1924 VT: LT.second, CostKind);
1925 }
1926 break;
1927 }
1928 // TODO: add more intrinsic
1929 case Intrinsic::stepvector: {
1930 auto LT = getTypeLegalizationCost(Ty: RetTy);
1931 // Legalisation of illegal types involves an `index' instruction plus
1932 // (LT.first - 1) vector adds.
1933 if (ST->hasVInstructions())
1934 return getRISCVInstructionCost(OpCodes: RISCV::VID_V, VT: LT.second, CostKind) +
1935 (LT.first - 1) *
1936 getRISCVInstructionCost(OpCodes: RISCV::VADD_VX, VT: LT.second, CostKind);
1937 return 1 + (LT.first - 1);
1938 }
1939 case Intrinsic::vector_splice_left:
1940 case Intrinsic::vector_splice_right: {
1941 auto LT = getTypeLegalizationCost(Ty: RetTy);
1942 // Constant offsets fall through to getShuffleCost.
1943 if (!ICA.isTypeBasedOnly() && isa<ConstantInt>(Val: ICA.getArgs()[2]))
1944 break;
1945 if (ST->hasVInstructions() && LT.second.isVector()) {
1946 return LT.first *
1947 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX},
1948 VT: LT.second, CostKind);
1949 }
1950 break;
1951 }
1952 case Intrinsic::experimental_cttz_elts: {
1953 if (!ST->hasVInstructions())
1954 break;
1955 InstructionCost Cost = 0;
1956 Type *ArgTy = ICA.getArgTypes()[0];
1957 auto LT = getTypeLegalizationCost(Ty: ArgTy);
1958 if (!LT.second.isVector())
1959 break;
1960
1961 // If the element type is not i1, do a comparison with all-zeros.
1962 if (LT.second.getVectorElementType() != MVT::i1)
1963 Cost += getRISCVInstructionCost(OpCodes: RISCV::VMSNE_VI, VT: LT.second, CostKind);
1964
1965 Cost += getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind);
1966
1967 // If zero_is_poison is false, then we will generate additional
1968 // cmp + select instructions to convert -1 to EVL.
1969 Type *BoolTy = Type::getInt1Ty(C&: RetTy->getContext());
1970 if (ICA.getArgs().size() > 1 &&
1971 cast<ConstantInt>(Val: ICA.getArgs()[1])->isZero())
1972 Cost += getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: BoolTy, CondTy: RetTy,
1973 VecPred: CmpInst::ICMP_SLT, CostKind) +
1974 getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: RetTy, CondTy: BoolTy,
1975 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1976
1977 return LT.first * Cost;
1978 }
1979 case Intrinsic::experimental_vp_splice: {
1980 // To support type-based query from vectorizer, set the index to 0.
1981 // Note that index only change the cost from vslide.vx to vslide.vi and in
1982 // current implementations they have same costs.
1983 return getShuffleCost(Kind: TTI::SK_Splice, DstTy: cast<VectorType>(Val: ICA.getReturnType()),
1984 SrcTy: cast<VectorType>(Val: ICA.getArgTypes()[0]), CostKind, Mask: {},
1985 Index: 0, SubTp: cast<VectorType>(Val: ICA.getReturnType()));
1986 }
1987 case Intrinsic::vp_merge: {
1988 // If an operand is a binary op and the type is legal, RISCVVectorPeephole
1989 // will likely fold the resulting vmerge.vvm away.
1990 if (ICA.getVectorInstrContext() == VectorInstrContext::BinaryOp &&
1991 getTypeLegalizationCost(Ty: RetTy).first == 1)
1992 return TTI::TCC_Free;
1993 break;
1994 }
1995 case Intrinsic::fptoui_sat:
1996 case Intrinsic::fptosi_sat: {
1997 InstructionCost Cost = 0;
1998 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
1999 Type *SrcTy = ICA.getArgTypes()[0];
2000
2001 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
2002 auto DstLT = getTypeLegalizationCost(Ty: RetTy);
2003 if (!SrcTy->isVectorTy())
2004 break;
2005
2006 if (!SrcLT.first.isValid() || !DstLT.first.isValid())
2007 return InstructionCost::getInvalid();
2008
2009 Cost +=
2010 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
2011 Dst: RetTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
2012
2013 // Handle NaN.
2014 // vmfne v0, v8, v8 # If v8[i] is NaN set v0[i] to 1.
2015 // vmerge.vim v8, v8, 0, v0 # Convert NaN to 0.
2016 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2017 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: SrcTy, CondTy,
2018 VecPred: CmpInst::FCMP_UNO, CostKind);
2019 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
2020 VecPred: CmpInst::FCMP_UNO, CostKind);
2021 return Cost;
2022 }
2023 case Intrinsic::experimental_vector_extract_last_active: {
2024 auto *ValTy = cast<VectorType>(Val: ICA.getArgTypes()[0]);
2025 auto *MaskTy = cast<VectorType>(Val: ICA.getArgTypes()[1]);
2026
2027 auto ValLT = getTypeLegalizationCost(Ty: ValTy);
2028 auto MaskLT = getTypeLegalizationCost(Ty: MaskTy);
2029
2030 // TODO: Return cheaper cost when the entire lane is inactive.
2031 // The expected asm sequence is:
2032 // vcpop.m a0, v0
2033 // beqz a0, exit # Return passthru when the entire lane is inactive.
2034 // vid v10, v0.t
2035 // vredmaxu.vs v10, v10, v10
2036 // vmv.x.s a0, v10
2037 // zext.b a0, a0
2038 // vslidedown.vx v8, v8, a0
2039 // vmv.x.s a0, v8
2040 // exit:
2041 // ...
2042
2043 // Find a suitable type for a stepvector.
2044 ConstantRange VScaleRange(APInt(64, 1), APInt::getZero(numBits: 64));
2045 unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
2046 RetVT: TLI->getVectorIdxTy(DL: getDataLayout()), EC: MaskTy->getElementCount(),
2047 /*ZeroIsPoison=*/true, VScaleRange: &VScaleRange);
2048 EltWidth = std::max(a: EltWidth, b: MaskTy->getScalarSizeInBits());
2049 Type *StepTy = Type::getIntNTy(C&: MaskTy->getContext(), N: EltWidth);
2050 auto *StepVecTy = VectorType::get(ElementType: StepTy, EC: ValTy->getElementCount());
2051 auto StepLT = getTypeLegalizationCost(Ty: StepVecTy);
2052
2053 // Currently expandVectorFindLastActive cannot handle step vector split.
2054 // So return invalid when the type needs split.
2055 // FIXME: Remove this if expandVectorFindLastActive supports split vector.
2056 if (StepLT.first > 1)
2057 return InstructionCost::getInvalid();
2058
2059 InstructionCost Cost = 0;
2060 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
2061
2062 Cost += MaskLT.first *
2063 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: MaskLT.second, CostKind);
2064 Cost += getCFInstrCost(Opcode: Instruction::CondBr, CostKind, I: nullptr);
2065 Cost += StepLT.first *
2066 getRISCVInstructionCost(OpCodes: Opcodes, VT: StepLT.second, CostKind);
2067 Cost += getCastInstrCost(Opcode: Instruction::ZExt,
2068 Dst: Type::getInt64Ty(C&: ValTy->getContext()), Src: StepTy,
2069 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
2070 Cost += ValLT.first *
2071 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VI, RISCV::VMV_X_S},
2072 VT: ValLT.second, CostKind);
2073 return Cost;
2074 }
2075 case Intrinsic::vector_interleave2:
2076 case Intrinsic::vector_deinterleave2: {
2077 if (!ST->hasStdExtZvzip())
2078 break;
2079
2080 bool IsInterleave = ICA.getID() == Intrinsic::vector_interleave2;
2081 Type *InterleavedTy = IsInterleave ? RetTy : ICA.getArgTypes().front();
2082 // ISel does not select vzip.vv if either interleave2 input is undef.
2083 if (IsInterleave && !ICA.isTypeBasedOnly() &&
2084 any_of(Range: ICA.getArgs(),
2085 P: [](const Value *Arg) { return isa<UndefValue>(Val: Arg); }))
2086 break;
2087 if (InterleavedTy->getScalarSizeInBits() == 1)
2088 break;
2089
2090 if (auto *FVT = dyn_cast<FixedVectorType>(Val: InterleavedTy)) {
2091 auto *HalfFVT = FixedVectorType::getHalfElementsVectorType(VTy: FVT);
2092 unsigned HalfVF = HalfFVT->getNumElements();
2093 if (IsInterleave)
2094 return getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: FVT, SrcTy: HalfFVT, CostKind,
2095 Mask: createInterleaveMask(VF: HalfVF, NumVecs: 2), Index: 0, SubTp: nullptr);
2096 InstructionCost Cost = 0;
2097 for (unsigned Start = 0; Start != 2; ++Start)
2098 Cost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: HalfFVT, SrcTy: FVT, CostKind,
2099 Mask: createStrideMask(Start, Stride: 2, VF: HalfVF), Index: 0, SubTp: nullptr);
2100 return Cost;
2101 }
2102
2103 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: InterleavedTy);
2104 if (!LT.second.isScalableVector())
2105 break;
2106 if (IsInterleave) {
2107 if (MVT CostVT = getZvzipVZIPCostVT(InterleavedVT: LT.second); CostVT.isValid())
2108 return LT.first *
2109 getRISCVInstructionCost(OpCodes: RISCV::VZIP_VV, VT: CostVT, CostKind);
2110 } else if (MVT CostVT = getZvzipVUNZIPCostVT(InterleavedVT: LT.second); CostVT.isValid()) {
2111 return LT.first *
2112 getRISCVInstructionCost(OpCodes: {RISCV::VUNZIPE_V, RISCV::VUNZIPO_V},
2113 VT: CostVT, CostKind);
2114 }
2115 break;
2116 }
2117 }
2118
2119 if (ST->hasVInstructions() && RetTy->isVectorTy()) {
2120 if (auto LT = getTypeLegalizationCost(Ty: RetTy);
2121 LT.second.isVector()) {
2122 MVT EltTy = LT.second.getVectorElementType();
2123 if (const auto *Entry = CostTableLookup(Table: VectorIntrinsicCostTable,
2124 ISD: ICA.getID(), Ty: EltTy))
2125 return LT.first * Entry->Cost;
2126 }
2127 }
2128
2129 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
2130}
2131
2132InstructionCost
2133RISCVTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
2134 const SCEV *Ptr,
2135 TTI::TargetCostKind CostKind) const {
2136 // Address computations for vector indexed load/store likely require an offset
2137 // and/or scaling.
2138 if (ST->hasVInstructions() && PtrTy->isVectorTy())
2139 return getArithmeticInstrCost(Opcode: Instruction::Add, Ty: PtrTy, CostKind);
2140
2141 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
2142}
2143
2144InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
2145 Type *Src,
2146 TTI::CastContextHint CCH,
2147 TTI::TargetCostKind CostKind,
2148 const Instruction *I) const {
2149 bool IsVectorType = isa<VectorType>(Val: Dst) && isa<VectorType>(Val: Src);
2150 if (!IsVectorType)
2151 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2152
2153 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
2154 // For now, skip all fixed vector cost analysis when P extension is available
2155 // to avoid crashes in getMinRVVVectorSizeInBits()
2156 if (ST->hasStdExtP() &&
2157 (isa<FixedVectorType>(Val: Dst) || isa<FixedVectorType>(Val: Src))) {
2158 return 1; // Treat as single instruction cost for now
2159 }
2160
2161 // FIXME: Need to compute legalizing cost for illegal types. The current
2162 // code handles only legal types and those which can be trivially
2163 // promoted to legal.
2164 if (!ST->hasVInstructions() || Src->getScalarSizeInBits() > ST->getELen() ||
2165 Dst->getScalarSizeInBits() > ST->getELen())
2166 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2167
2168 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2169 assert(ISD && "Invalid opcode");
2170 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: Src);
2171 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: Dst);
2172
2173 // Handle i1 source and dest cases *before* calling logic in BasicTTI.
2174 // The shared implementation doesn't model vector widening during legalization
2175 // and instead assumes scalarization. In order to scalarize an <N x i1>
2176 // vector, we need to extend/trunc to/from i8. If we don't special case
2177 // this, we can get an infinite recursion cycle.
2178 switch (ISD) {
2179 default:
2180 break;
2181 case ISD::SIGN_EXTEND:
2182 case ISD::ZERO_EXTEND:
2183 if (Src->getScalarSizeInBits() == 1) {
2184 // We do not use vsext/vzext to extend from mask vector.
2185 // Instead we use the following instructions to extend from mask vector:
2186 // vmv.v.i v8, 0
2187 // vmerge.vim v8, v8, -1, v0 (repeated per split)
2188 return getRISCVInstructionCost(OpCodes: RISCV::VMV_V_I, VT: DstLT.second, CostKind) +
2189 DstLT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VIM,
2190 VT: DstLT.second, CostKind) +
2191 DstLT.first - 1;
2192 }
2193 break;
2194 case ISD::TRUNCATE:
2195 if (Dst->getScalarSizeInBits() == 1) {
2196 // We do not use several vncvt to truncate to mask vector. So we could
2197 // not use PowDiff to calculate it.
2198 // Instead we use the following instructions to truncate to mask vector:
2199 // vand.vi v8, v8, 1
2200 // vmsne.vi v0, v8, 0
2201 return SrcLT.first *
2202 getRISCVInstructionCost(OpCodes: {RISCV::VAND_VI, RISCV::VMSNE_VI},
2203 VT: SrcLT.second, CostKind) +
2204 SrcLT.first - 1;
2205 }
2206 break;
2207 };
2208
2209 // Our actual lowering for the case where a wider legal type is available
2210 // uses promotion to the wider type. This is reflected in the result of
2211 // getTypeLegalizationCost, but BasicTTI assumes the widened cases are
2212 // scalarized if the legalized Src and Dst are not equal sized.
2213 const DataLayout &DL = this->getDataLayout();
2214 if (!SrcLT.second.isVector() || !DstLT.second.isVector() ||
2215 !SrcLT.first.isValid() || !DstLT.first.isValid() ||
2216 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Src),
2217 RHS: SrcLT.second.getSizeInBits()) ||
2218 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Dst),
2219 RHS: DstLT.second.getSizeInBits()) ||
2220 SrcLT.first > 1 || DstLT.first > 1)
2221 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2222
2223 // The split cost is handled by the base getCastInstrCost
2224 assert((SrcLT.first == 1) && (DstLT.first == 1) && "Illegal type");
2225
2226 int PowDiff = (int)Log2_32(Value: DstLT.second.getScalarSizeInBits()) -
2227 (int)Log2_32(Value: SrcLT.second.getScalarSizeInBits());
2228 switch (ISD) {
2229 case ISD::SIGN_EXTEND:
2230 case ISD::ZERO_EXTEND: {
2231 if ((PowDiff < 1) || (PowDiff > 3))
2232 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2233 unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
2234 unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
2235 unsigned Op =
2236 (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
2237 return getRISCVInstructionCost(OpCodes: Op, VT: DstLT.second, CostKind);
2238 }
2239 case ISD::TRUNCATE:
2240 case ISD::FP_EXTEND:
2241 case ISD::FP_ROUND: {
2242 // Counts of narrow/widen instructions.
2243 unsigned SrcEltSize = SrcLT.second.getScalarSizeInBits();
2244 unsigned DstEltSize = DstLT.second.getScalarSizeInBits();
2245
2246 unsigned Op = (ISD == ISD::TRUNCATE) ? RISCV::VNSRL_WI
2247 : (ISD == ISD::FP_EXTEND) ? RISCV::VFWCVT_F_F_V
2248 : RISCV::VFNCVT_F_F_W;
2249 InstructionCost Cost = 0;
2250 for (; SrcEltSize != DstEltSize;) {
2251 MVT ElementMVT = (ISD == ISD::TRUNCATE)
2252 ? MVT::getIntegerVT(BitWidth: DstEltSize)
2253 : MVT::getFloatingPointVT(BitWidth: DstEltSize);
2254 MVT DstMVT = DstLT.second.changeVectorElementType(EltVT: ElementMVT);
2255 DstEltSize =
2256 (DstEltSize > SrcEltSize) ? DstEltSize >> 1 : DstEltSize << 1;
2257 Cost += getRISCVInstructionCost(OpCodes: Op, VT: DstMVT, CostKind);
2258 }
2259 return Cost;
2260 }
2261 case ISD::FP_TO_SINT:
2262 case ISD::FP_TO_UINT: {
2263 unsigned IsSigned = ISD == ISD::FP_TO_SINT;
2264 unsigned FCVT = IsSigned ? RISCV::VFCVT_RTZ_X_F_V : RISCV::VFCVT_RTZ_XU_F_V;
2265 unsigned FWCVT =
2266 IsSigned ? RISCV::VFWCVT_RTZ_X_F_V : RISCV::VFWCVT_RTZ_XU_F_V;
2267 unsigned FNCVT =
2268 IsSigned ? RISCV::VFNCVT_RTZ_X_F_W : RISCV::VFNCVT_RTZ_XU_F_W;
2269 unsigned SrcEltSize = Src->getScalarSizeInBits();
2270 unsigned DstEltSize = Dst->getScalarSizeInBits();
2271 InstructionCost Cost = 0;
2272 if ((SrcEltSize == 16) &&
2273 (!ST->hasVInstructionsF16() || ((DstEltSize / 2) > SrcEltSize))) {
2274 // If the target only supports zvfhmin or it is fp16-to-i64 conversion
2275 // pre-widening to f32 and then convert f32 to integer
2276 VectorType *VecF32Ty =
2277 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2278 EC: cast<VectorType>(Val: Dst)->getElementCount());
2279 std::pair<InstructionCost, MVT> VecF32LT =
2280 getTypeLegalizationCost(Ty: VecF32Ty);
2281 Cost +=
2282 VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFWCVT_F_F_V,
2283 VT: VecF32LT.second, CostKind);
2284 Cost += getCastInstrCost(Opcode, Dst, Src: VecF32Ty, CCH, CostKind, I);
2285 return Cost;
2286 }
2287 if (DstEltSize == SrcEltSize)
2288 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2289 else if (DstEltSize > SrcEltSize)
2290 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2291 else { // (SrcEltSize > DstEltSize)
2292 // First do a narrowing conversion to an integer half the size, then
2293 // truncate if needed.
2294 MVT ElementVT = MVT::getIntegerVT(BitWidth: SrcEltSize / 2);
2295 MVT VecVT = DstLT.second.changeVectorElementType(EltVT: ElementVT);
2296 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: VecVT, CostKind);
2297 if ((SrcEltSize / 2) > DstEltSize) {
2298 Type *VecTy = EVT(VecVT).getTypeForEVT(Context&: Dst->getContext());
2299 Cost +=
2300 getCastInstrCost(Opcode: Instruction::Trunc, Dst, Src: VecTy, CCH, CostKind, I);
2301 }
2302 }
2303 return Cost;
2304 }
2305 case ISD::SINT_TO_FP:
2306 case ISD::UINT_TO_FP: {
2307 unsigned IsSigned = ISD == ISD::SINT_TO_FP;
2308 unsigned FCVT = IsSigned ? RISCV::VFCVT_F_X_V : RISCV::VFCVT_F_XU_V;
2309 unsigned FWCVT = IsSigned ? RISCV::VFWCVT_F_X_V : RISCV::VFWCVT_F_XU_V;
2310 unsigned FNCVT = IsSigned ? RISCV::VFNCVT_F_X_W : RISCV::VFNCVT_F_XU_W;
2311 unsigned SrcEltSize = Src->getScalarSizeInBits();
2312 unsigned DstEltSize = Dst->getScalarSizeInBits();
2313
2314 InstructionCost Cost = 0;
2315 if ((DstEltSize == 16) &&
2316 (!ST->hasVInstructionsF16() || ((SrcEltSize / 2) > DstEltSize))) {
2317 // If the target only supports zvfhmin or it is i64-to-fp16 conversion
2318 // it is converted to f32 and then converted to f16
2319 VectorType *VecF32Ty =
2320 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2321 EC: cast<VectorType>(Val: Dst)->getElementCount());
2322 std::pair<InstructionCost, MVT> VecF32LT =
2323 getTypeLegalizationCost(Ty: VecF32Ty);
2324 Cost += getCastInstrCost(Opcode, Dst: VecF32Ty, Src, CCH, CostKind, I);
2325 Cost += VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFNCVT_F_F_W,
2326 VT: DstLT.second, CostKind);
2327 return Cost;
2328 }
2329
2330 if (DstEltSize == SrcEltSize)
2331 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2332 else if (DstEltSize > SrcEltSize) {
2333 if ((DstEltSize / 2) > SrcEltSize) {
2334 VectorType *VecTy =
2335 VectorType::get(ElementType: IntegerType::get(C&: Dst->getContext(), NumBits: DstEltSize / 2),
2336 EC: cast<VectorType>(Val: Dst)->getElementCount());
2337 unsigned Op = IsSigned ? Instruction::SExt : Instruction::ZExt;
2338 Cost += getCastInstrCost(Opcode: Op, Dst: VecTy, Src, CCH, CostKind, I);
2339 }
2340 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2341 } else
2342 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: DstLT.second, CostKind);
2343 return Cost;
2344 }
2345 }
2346 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2347}
2348
2349unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) const {
2350 if (isa<ScalableVectorType>(Val: Ty)) {
2351 const unsigned EltSize = DL.getTypeSizeInBits(Ty: Ty->getElementType());
2352 const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue();
2353 const unsigned VectorBits = *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
2354 return RISCVTargetLowering::computeVLMAX(VectorBits, EltSize, MinSize);
2355 }
2356 return cast<FixedVectorType>(Val: Ty)->getNumElements();
2357}
2358
2359InstructionCost
2360RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
2361 FastMathFlags FMF,
2362 TTI::TargetCostKind CostKind) const {
2363 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2364 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2365
2366 // Skip if scalar size of Ty is bigger than ELEN.
2367 if (Ty->getScalarSizeInBits() > ST->getELen())
2368 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2369
2370 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2371 if (Ty->getElementType()->isIntegerTy(BitWidth: 1)) {
2372 // SelectionDAGBuilder does following transforms:
2373 // vector_reduce_{smin,umax}(<n x i1>) --> vector_reduce_or(<n x i1>)
2374 // vector_reduce_{smax,umin}(<n x i1>) --> vector_reduce_and(<n x i1>)
2375 if (IID == Intrinsic::umax || IID == Intrinsic::smin)
2376 return getArithmeticReductionCost(Opcode: Instruction::Or, Ty, FMF, CostKind);
2377 else
2378 return getArithmeticReductionCost(Opcode: Instruction::And, Ty, FMF, CostKind);
2379 }
2380
2381 if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) {
2382 SmallVector<unsigned, 3> Opcodes;
2383 InstructionCost ExtraCost = 0;
2384 switch (IID) {
2385 case Intrinsic::maximum:
2386 if (FMF.noNaNs()) {
2387 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2388 } else {
2389 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS,
2390 RISCV::VFMV_F_S};
2391 // Cost of Canonical Nan + branch
2392 // lui a0, 523264
2393 // fmv.w.x fa0, a0
2394 Type *DstTy = Ty->getScalarType();
2395 const unsigned EltTyBits = DstTy->getScalarSizeInBits();
2396 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2397 ExtraCost = 1 +
2398 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2399 CCH: TTI::CastContextHint::None, CostKind) +
2400 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2401 }
2402 break;
2403
2404 case Intrinsic::minimum:
2405 if (FMF.noNaNs()) {
2406 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2407 } else {
2408 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS,
2409 RISCV::VFMV_F_S};
2410 // Cost of Canonical Nan + branch
2411 // lui a0, 523264
2412 // fmv.w.x fa0, a0
2413 Type *DstTy = Ty->getScalarType();
2414 const unsigned EltTyBits = DL.getTypeSizeInBits(Ty: DstTy);
2415 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2416 ExtraCost = 1 +
2417 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2418 CCH: TTI::CastContextHint::None, CostKind) +
2419 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2420 }
2421 break;
2422 }
2423 return ExtraCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2424 }
2425
2426 // IR Reduction is composed by one rvv reduction instruction and vmv
2427 unsigned SplitOp;
2428 SmallVector<unsigned, 3> Opcodes;
2429 switch (IID) {
2430 default:
2431 llvm_unreachable("Unsupported intrinsic");
2432 case Intrinsic::smax:
2433 SplitOp = RISCV::VMAX_VV;
2434 Opcodes = {RISCV::VREDMAX_VS, RISCV::VMV_X_S};
2435 break;
2436 case Intrinsic::smin:
2437 SplitOp = RISCV::VMIN_VV;
2438 Opcodes = {RISCV::VREDMIN_VS, RISCV::VMV_X_S};
2439 break;
2440 case Intrinsic::umax:
2441 SplitOp = RISCV::VMAXU_VV;
2442 Opcodes = {RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
2443 break;
2444 case Intrinsic::umin:
2445 SplitOp = RISCV::VMINU_VV;
2446 Opcodes = {RISCV::VREDMINU_VS, RISCV::VMV_X_S};
2447 break;
2448 case Intrinsic::maxnum:
2449 SplitOp = RISCV::VFMAX_VV;
2450 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2451 break;
2452 case Intrinsic::minnum:
2453 SplitOp = RISCV::VFMIN_VV;
2454 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2455 break;
2456 }
2457 // Add a cost for data larger than LMUL8
2458 InstructionCost SplitCost =
2459 (LT.first > 1) ? (LT.first - 1) *
2460 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2461 : 0;
2462 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2463}
2464
2465InstructionCost
2466RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
2467 std::optional<FastMathFlags> FMF,
2468 TTI::TargetCostKind CostKind) const {
2469 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2470 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2471
2472 // Skip if scalar size of Ty is bigger than ELEN.
2473 if (Ty->getScalarSizeInBits() > ST->getELen())
2474 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2475
2476 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2477 assert(ISD && "Invalid opcode");
2478
2479 if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&
2480 ISD != ISD::FADD)
2481 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2482
2483 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2484 Type *ElementTy = Ty->getElementType();
2485 if (ElementTy->isIntegerTy(BitWidth: 1)) {
2486 // Example sequences:
2487 // vfirst.m a0, v0
2488 // seqz a0, a0
2489 if (LT.second == MVT::v1i1)
2490 return getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind) +
2491 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2492 VecPred: CmpInst::ICMP_EQ, CostKind);
2493
2494 if (ISD == ISD::AND) {
2495 // Example sequences:
2496 // vmand.mm v8, v9, v8 ; needed every time type is split
2497 // vmnot.m v8, v0 ; alias for vmnand
2498 // vcpop.m a0, v8
2499 // seqz a0, a0
2500
2501 // See the discussion: https://github.com/llvm/llvm-project/pull/119160
2502 // For LMUL <= 8, there is no splitting,
2503 // the sequences are vmnot, vcpop and seqz.
2504 // When LMUL > 8 and split = 1,
2505 // the sequences are vmnand, vcpop and seqz.
2506 // When LMUL > 8 and split > 1,
2507 // the sequences are (LT.first-2) * vmand, vmnand, vcpop and seqz.
2508 return ((LT.first > 2) ? (LT.first - 2) : 0) *
2509 getRISCVInstructionCost(OpCodes: RISCV::VMAND_MM, VT: LT.second, CostKind) +
2510 getRISCVInstructionCost(OpCodes: RISCV::VMNAND_MM, VT: LT.second, CostKind) +
2511 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2512 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2513 VecPred: CmpInst::ICMP_EQ, CostKind);
2514 } else if (ISD == ISD::XOR || ISD == ISD::ADD) {
2515 // Example sequences:
2516 // vsetvli a0, zero, e8, mf8, ta, ma
2517 // vmxor.mm v8, v0, v8 ; needed every time type is split
2518 // vcpop.m a0, v8
2519 // andi a0, a0, 1
2520 return (LT.first - 1) *
2521 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind) +
2522 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) + 1;
2523 } else {
2524 assert(ISD == ISD::OR);
2525 // Example sequences:
2526 // vsetvli a0, zero, e8, mf8, ta, ma
2527 // vmor.mm v8, v9, v8 ; needed every time type is split
2528 // vcpop.m a0, v0
2529 // snez a0, a0
2530 return (LT.first - 1) *
2531 getRISCVInstructionCost(OpCodes: RISCV::VMOR_MM, VT: LT.second, CostKind) +
2532 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2533 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2534 VecPred: CmpInst::ICMP_NE, CostKind);
2535 }
2536 }
2537
2538 // IR Reduction of or/and is composed by one vmv and one rvv reduction
2539 // instruction, and others is composed by two vmv and one rvv reduction
2540 // instruction
2541 unsigned SplitOp;
2542 SmallVector<unsigned, 3> Opcodes;
2543 switch (ISD) {
2544 case ISD::ADD:
2545 SplitOp = RISCV::VADD_VV;
2546 Opcodes = {RISCV::VMV_S_X, RISCV::VREDSUM_VS, RISCV::VMV_X_S};
2547 break;
2548 case ISD::OR:
2549 SplitOp = RISCV::VOR_VV;
2550 Opcodes = {RISCV::VREDOR_VS, RISCV::VMV_X_S};
2551 break;
2552 case ISD::XOR:
2553 SplitOp = RISCV::VXOR_VV;
2554 Opcodes = {RISCV::VMV_S_X, RISCV::VREDXOR_VS, RISCV::VMV_X_S};
2555 break;
2556 case ISD::AND:
2557 SplitOp = RISCV::VAND_VV;
2558 Opcodes = {RISCV::VREDAND_VS, RISCV::VMV_X_S};
2559 break;
2560 case ISD::FADD:
2561 // We can't promote f16/bf16 fadd reductions.
2562 if ((LT.second.getScalarType() == MVT::f16 && !ST->hasVInstructionsF16()) ||
2563 LT.second.getScalarType() == MVT::bf16)
2564 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2565 if (TTI::requiresOrderedReduction(FMF)) {
2566 Opcodes.push_back(Elt: RISCV::VFMV_S_F);
2567 for (unsigned i = 0; i < LT.first.getValue(); i++)
2568 Opcodes.push_back(Elt: RISCV::VFREDOSUM_VS);
2569 Opcodes.push_back(Elt: RISCV::VFMV_F_S);
2570 return getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2571 }
2572 SplitOp = RISCV::VFADD_VV;
2573 Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDUSUM_VS, RISCV::VFMV_F_S};
2574 break;
2575 }
2576 // Add a cost for data larger than LMUL8
2577 InstructionCost SplitCost =
2578 (LT.first > 1) ? (LT.first - 1) *
2579 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2580 : 0;
2581 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2582}
2583
2584InstructionCost RISCVTTIImpl::getExtendedReductionCost(
2585 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
2586 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
2587 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2588 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2589 FMF, CostKind);
2590
2591 // Skip if scalar size of ResTy is bigger than ELEN.
2592 if (ResTy->getScalarSizeInBits() > ST->getELen())
2593 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2594 FMF, CostKind);
2595
2596 if (Opcode != Instruction::Add && Opcode != Instruction::FAdd)
2597 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2598 FMF, CostKind);
2599
2600 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2601
2602 if (IsUnsigned && Opcode == Instruction::Add &&
2603 LT.second.isFixedLengthVectorOf(EltVT: MVT::i1)) {
2604 // Represent vector_reduce_add(ZExt(<n x i1>)) as
2605 // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
2606 return LT.first *
2607 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind);
2608 }
2609
2610 if (ResTy->getScalarSizeInBits() != 2 * LT.second.getScalarSizeInBits())
2611 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2612 FMF, CostKind);
2613
2614 return (LT.first - 1) +
2615 getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
2616}
2617
2618InstructionCost
2619RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,
2620 TTI::TargetCostKind CostKind) const {
2621 assert(OpInfo.isConstant() && "non constant operand?");
2622 if (!isa<VectorType>(Val: Ty))
2623 // FIXME: We need to account for immediate materialization here, but doing
2624 // a decent job requires more knowledge about the immediate than we
2625 // currently have here.
2626 return 0;
2627
2628 if (OpInfo.isUniform())
2629 // vmv.v.i, vmv.v.x, or vfmv.v.f
2630 // We ignore the cost of the scalar constant materialization to be consistent
2631 // with how we treat scalar constants themselves just above.
2632 return 1;
2633
2634 return getConstantPoolLoadCost(Ty, CostKind);
2635}
2636
2637InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
2638 Align Alignment,
2639 unsigned AddressSpace,
2640 TTI::TargetCostKind CostKind,
2641 TTI::OperandValueInfo OpInfo,
2642 const Instruction *I) const {
2643 EVT VT = TLI->getValueType(DL, Ty: Src, AllowUnknown: true);
2644 // Type legalization can't handle structs, and load latency isn't handled here
2645 if (VT == MVT::Other ||
2646 (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency))
2647 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2648 CostKind, OpInfo, I);
2649
2650 InstructionCost Cost = 0;
2651 if (Opcode == Instruction::Store && OpInfo.isConstant())
2652 Cost += getStoreImmCost(Ty: Src, OpInfo, CostKind);
2653
2654 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
2655
2656 InstructionCost BaseCost = [&]() {
2657 InstructionCost Cost = LT.first;
2658 if (CostKind != TTI::TCK_RecipThroughput)
2659 return Cost;
2660
2661 // Our actual lowering for the case where a wider legal type is available
2662 // uses the a VL predicated load on the wider type. This is reflected in
2663 // the result of getTypeLegalizationCost, but BasicTTI assumes the
2664 // widened cases are scalarized.
2665 const DataLayout &DL = this->getDataLayout();
2666 if (Src->isVectorTy() && LT.second.isVector() &&
2667 TypeSize::isKnownLT(LHS: DL.getTypeStoreSizeInBits(Ty: Src),
2668 RHS: LT.second.getSizeInBits()))
2669 return Cost;
2670
2671 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2672 CostKind, OpInfo, I);
2673 }();
2674
2675 // Assume memory ops cost scale with the number of vector registers
2676 // possible accessed by the instruction. Note that BasicTTI already
2677 // handles the LT.first term for us.
2678 if (ST->hasVInstructions() && LT.second.isVector() &&
2679 CostKind != TTI::TCK_CodeSize)
2680 BaseCost *= TLI->getLMULCost(VT: LT.second);
2681 return Cost + BaseCost;
2682}
2683
2684InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
2685 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
2686 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
2687 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
2688 if (CostKind != TTI::TCK_RecipThroughput)
2689 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2690 Op1Info, Op2Info, I);
2691
2692 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2693 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2694 Op1Info, Op2Info, I);
2695
2696 // Skip if scalar size of ValTy is bigger than ELEN.
2697 if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen())
2698 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2699 Op1Info, Op2Info, I);
2700
2701 auto GetConstantMatCost =
2702 [&](TTI::OperandValueInfo OpInfo) -> InstructionCost {
2703 if (OpInfo.isUniform())
2704 // We return 0 we currently ignore the cost of materializing scalar
2705 // constants in GPRs.
2706 return 0;
2707
2708 return getConstantPoolLoadCost(Ty: ValTy, CostKind);
2709 };
2710
2711 InstructionCost ConstantMatCost;
2712 if (Op1Info.isConstant())
2713 ConstantMatCost += GetConstantMatCost(Op1Info);
2714 if (Op2Info.isConstant())
2715 ConstantMatCost += GetConstantMatCost(Op2Info);
2716
2717 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2718 if (Opcode == Instruction::Select && LT.second.isVector()) {
2719 if (CondTy->isVectorTy()) {
2720 if (ValTy->getScalarSizeInBits() == 1) {
2721 // vmandn.mm v8, v8, v9
2722 // vmand.mm v9, v0, v9
2723 // vmor.mm v0, v9, v8
2724 return ConstantMatCost +
2725 LT.first *
2726 getRISCVInstructionCost(
2727 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2728 VT: LT.second, CostKind);
2729 }
2730 // vselect and max/min are supported natively.
2731 return ConstantMatCost +
2732 LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second,
2733 CostKind);
2734 }
2735
2736 if (ValTy->getScalarSizeInBits() == 1) {
2737 // vmv.v.x v9, a0
2738 // vmsne.vi v9, v9, 0
2739 // vmandn.mm v8, v8, v9
2740 // vmand.mm v9, v0, v9
2741 // vmor.mm v0, v9, v8
2742 MVT InterimVT = LT.second.changeVectorElementType(EltVT: MVT::i8);
2743 return ConstantMatCost +
2744 LT.first *
2745 getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
2746 VT: InterimVT, CostKind) +
2747 LT.first * getRISCVInstructionCost(
2748 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2749 VT: LT.second, CostKind);
2750 }
2751
2752 // vmv.v.x v10, a0
2753 // vmsne.vi v0, v10, 0
2754 // vmerge.vvm v8, v9, v8, v0
2755 return ConstantMatCost +
2756 LT.first * getRISCVInstructionCost(
2757 OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI, RISCV::VMERGE_VVM},
2758 VT: LT.second, CostKind);
2759 }
2760
2761 if ((Opcode == Instruction::ICmp) && ValTy->isVectorTy() &&
2762 CmpInst::isIntPredicate(P: VecPred)) {
2763 // Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE
2764 // provided they incur the same cost across all implementations
2765 return ConstantMatCost + LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMSLT_VV,
2766 VT: LT.second,
2767 CostKind);
2768 }
2769
2770 if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() &&
2771 CmpInst::isFPPredicate(P: VecPred)) {
2772
2773 // Use VMXOR_MM and VMXNOR_MM to generate all true/false mask
2774 if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE))
2775 return ConstantMatCost +
2776 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind);
2777
2778 // If we do not support the input floating point vector type, use the base
2779 // one which will calculate as:
2780 // ScalarizeCost + Num * Cost for fixed vector,
2781 // InvalidCost for scalable vector.
2782 if ((ValTy->getScalarSizeInBits() == 16 && !ST->hasVInstructionsF16()) ||
2783 (ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) ||
2784 (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64()))
2785 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2786 Op1Info, Op2Info, I);
2787
2788 // Assuming vector fp compare and mask instructions are all the same cost
2789 // until a need arises to differentiate them.
2790 switch (VecPred) {
2791 case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm
2792 case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm
2793 case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm
2794 case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm
2795 return ConstantMatCost +
2796 LT.first * getRISCVInstructionCost(
2797 OpCodes: {RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM},
2798 VT: LT.second, CostKind);
2799
2800 case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m
2801 case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m
2802 case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m
2803 case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m
2804 return ConstantMatCost +
2805 LT.first *
2806 getRISCVInstructionCost(OpCodes: {RISCV::VMFLT_VV, RISCV::VMNAND_MM},
2807 VT: LT.second, CostKind);
2808
2809 case CmpInst::FCMP_OEQ: // vmfeq.vv
2810 case CmpInst::FCMP_OGT: // vmflt.vv
2811 case CmpInst::FCMP_OGE: // vmfle.vv
2812 case CmpInst::FCMP_OLT: // vmflt.vv
2813 case CmpInst::FCMP_OLE: // vmfle.vv
2814 case CmpInst::FCMP_UNE: // vmfne.vv
2815 return ConstantMatCost +
2816 LT.first *
2817 getRISCVInstructionCost(OpCodes: RISCV::VMFLT_VV, VT: LT.second, CostKind);
2818 default:
2819 break;
2820 }
2821 }
2822
2823 // With ShortForwardBranchOpt or ConditionalMoveFusion, scalar icmp + select
2824 // instructions will lower to SELECT_CC and lower to PseudoCCMOVGPR which will
2825 // generate a conditional branch + mv. The cost of scalar (icmp + select) will
2826 // be (0 + select instr cost).
2827 if (ST->hasConditionalMoveFusion() && I && isa<ICmpInst>(Val: I) &&
2828 ValTy->isIntegerTy() && !I->user_empty()) {
2829 if (all_of(Range: I->users(), P: [&](const User *U) {
2830 return match(V: U, P: m_Select(C: m_Specific(V: I), L: m_Value(), R: m_Value())) &&
2831 U->getType()->isIntegerTy() &&
2832 !isa<ConstantData>(Val: U->getOperand(i: 1)) &&
2833 !isa<ConstantData>(Val: U->getOperand(i: 2));
2834 }))
2835 return 0;
2836 }
2837
2838 // TODO: Add cost for scalar type.
2839
2840 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2841 Op1Info, Op2Info, I);
2842}
2843
2844InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
2845 TTI::TargetCostKind CostKind,
2846 const Instruction *I) const {
2847 if (CostKind != TTI::TCK_RecipThroughput)
2848 return Opcode == Instruction::PHI ? 0 : 1;
2849 // Branches are assumed to be predicted.
2850 return 0;
2851}
2852
2853InstructionCost RISCVTTIImpl::getVectorInstrCost(
2854 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
2855 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
2856 assert(Val->isVectorTy() && "This must be a vector type");
2857
2858 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
2859 // For now, skip all fixed vector cost analysis when P extension is available
2860 // to avoid crashes in getMinRVVVectorSizeInBits()
2861 if (ST->hasStdExtP() && isa<FixedVectorType>(Val)) {
2862 return 1; // Treat as single instruction cost for now
2863 }
2864
2865 if (Opcode != Instruction::ExtractElement &&
2866 Opcode != Instruction::InsertElement)
2867 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
2868 VIC);
2869
2870 // Scalar splat operand can be folded for vector ops that support splatting
2871 // the scalar operand, so the explicit insertelement is free in this context.
2872 if (Opcode == Instruction::InsertElement &&
2873 VIC == TTI::VectorInstrContext::SplatOpFolded &&
2874 ST->sinkSplatOperands() && Index == 0)
2875 return TTI::TCC_Free;
2876
2877 // Legalize the type.
2878 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Val);
2879
2880 // This type is legalized to a scalar type.
2881 if (!LT.second.isVector()) {
2882 auto *FixedVecTy = cast<FixedVectorType>(Val);
2883 // If Index is a known constant, cost is zero.
2884 if (Index != -1U)
2885 return 0;
2886 // Extract/InsertElement with non-constant index is very costly when
2887 // scalarized; estimate cost of loads/stores sequence via the stack:
2888 // ExtractElement cost: store vector to stack, load scalar;
2889 // InsertElement cost: store vector to stack, store scalar, load vector.
2890 Type *ElemTy = FixedVecTy->getElementType();
2891 auto NumElems = FixedVecTy->getNumElements();
2892 auto Align = DL.getPrefTypeAlign(Ty: ElemTy);
2893 InstructionCost LoadCost =
2894 getMemoryOpCost(Opcode: Instruction::Load, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2895 InstructionCost StoreCost =
2896 getMemoryOpCost(Opcode: Instruction::Store, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2897 return Opcode == Instruction::ExtractElement
2898 ? StoreCost * NumElems + LoadCost
2899 : (StoreCost + LoadCost) * NumElems + StoreCost;
2900 }
2901
2902 // For unsupported scalable vector.
2903 if (LT.second.isScalableVector() && !LT.first.isValid())
2904 return LT.first;
2905
2906 // Mask vector extract/insert is expanded via e8.
2907 if (Val->getScalarSizeInBits() == 1) {
2908 VectorType *WideTy =
2909 VectorType::get(ElementType: IntegerType::get(C&: Val->getContext(), NumBits: 8),
2910 EC: cast<VectorType>(Val)->getElementCount());
2911 if (Opcode == Instruction::ExtractElement) {
2912 InstructionCost ExtendCost
2913 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2914 CCH: TTI::CastContextHint::None, CostKind);
2915 InstructionCost ExtractCost
2916 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2917 return ExtendCost + ExtractCost;
2918 }
2919 InstructionCost ExtendCost
2920 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2921 CCH: TTI::CastContextHint::None, CostKind);
2922 InstructionCost InsertCost
2923 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2924 InstructionCost TruncCost
2925 = getCastInstrCost(Opcode: Instruction::Trunc, Dst: Val, Src: WideTy,
2926 CCH: TTI::CastContextHint::None, CostKind);
2927 return ExtendCost + InsertCost + TruncCost;
2928 }
2929
2930
2931 // In RVV, we could use vslidedown + vmv.x.s to extract element from vector
2932 // and vslideup + vmv.s.x to insert element to vector.
2933 unsigned MoveOpc;
2934 if (LT.second.isFloatingPoint())
2935 MoveOpc = Opcode == Instruction::InsertElement ? RISCV::VFMV_S_F
2936 : RISCV::VFMV_F_S;
2937 else
2938 MoveOpc =
2939 Opcode == Instruction::InsertElement ? RISCV::VMV_S_X : RISCV::VMV_X_S;
2940 InstructionCost BaseCost =
2941 getRISCVInstructionCost(OpCodes: MoveOpc, VT: LT.second, CostKind);
2942 // When insertelement we should add the index with 1 as the input of vslideup.
2943 InstructionCost SlideCost = Opcode == Instruction::InsertElement ? 2 : 1;
2944
2945 if (Index != -1U) {
2946 // The type may be split. For fixed-width vectors we can normalize the
2947 // index to the new type.
2948 if (LT.second.isFixedLengthVector()) {
2949 unsigned Width = LT.second.getVectorNumElements();
2950 Index = Index % Width;
2951 }
2952
2953 // If exact VLEN is known, we will insert/extract into the appropriate
2954 // subvector with no additional subvector insert/extract cost.
2955 if (auto VLEN = ST->getRealVLen()) {
2956 unsigned EltSize = LT.second.getScalarSizeInBits();
2957 unsigned M1Max = *VLEN / EltSize;
2958 Index = Index % M1Max;
2959 }
2960
2961 if (Index == 0)
2962 // We can extract/insert the first element without vslidedown/vslideup.
2963 SlideCost = 0;
2964 else if (Opcode == Instruction::InsertElement)
2965 SlideCost = 1; // With a constant index, we do not need to use addi.
2966 }
2967
2968 // When the vector needs to split into multiple register groups and the index
2969 // exceeds single vector register group, we need to insert/extract the element
2970 // via stack.
2971 if (LT.first > 1 &&
2972 ((Index == -1U) || (Index >= LT.second.getVectorMinNumElements() &&
2973 LT.second.isScalableVector()))) {
2974 Type *ScalarType = Val->getScalarType();
2975 Align VecAlign = DL.getPrefTypeAlign(Ty: Val);
2976 Align SclAlign = DL.getPrefTypeAlign(Ty: ScalarType);
2977 // Extra addi for unknown index.
2978 InstructionCost IdxCost = Index == -1U ? 1 : 0;
2979
2980 // Store all split vectors into stack and load the target element.
2981 if (Opcode == Instruction::ExtractElement)
2982 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2983 getMemoryOpCost(Opcode: Instruction::Load, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
2984 CostKind) +
2985 IdxCost;
2986
2987 // Store all split vectors into stack and store the target element and load
2988 // vectors back.
2989 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2990 getMemoryOpCost(Opcode: Instruction::Load, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2991 getMemoryOpCost(Opcode: Instruction::Store, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
2992 CostKind) +
2993 IdxCost;
2994 }
2995
2996 // Extract i64 in the target that has XLEN=32 need more instruction.
2997 if (Val->getScalarType()->isIntegerTy() &&
2998 ST->getXLen() < Val->getScalarSizeInBits()) {
2999 // For extractelement, we need the following instructions:
3000 // vsetivli zero, 1, e64, m1, ta, mu (not count)
3001 // vslidedown.vx v8, v8, a0
3002 // vmv.x.s a0, v8
3003 // li a1, 32
3004 // vsrl.vx v8, v8, a1
3005 // vmv.x.s a1, v8
3006
3007 // For insertelement, we need the following instructions:
3008 // vsetivli zero, 2, e32, m4, ta, ma (don't count)
3009 // vslide1down.vx v12, v8, a0
3010 // vslide1down.vx v12, v12, a1
3011 // addi a0, a2, 1
3012 // vsetvli zero, a0, e64, m4, tu, ma (don't count)
3013 // vslideup.vx v8, v12, a2
3014
3015 // TODO: should we count these special vsetvlis?
3016 BaseCost =
3017 Opcode == Instruction::InsertElement
3018 ? getRISCVInstructionCost(OpCodes: {RISCV::VSLIDE1DOWN_VX,
3019 RISCV::VSLIDE1DOWN_VX,
3020 RISCV::VSLIDEUP_VX},
3021 VT: LT.second, CostKind)
3022 : getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VMV_X_S,
3023 RISCV::VSRL_VX, RISCV::VMV_X_S},
3024 VT: LT.second, CostKind);
3025 }
3026 return BaseCost + SlideCost;
3027}
3028
3029InstructionCost
3030RISCVTTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
3031 TTI::TargetCostKind CostKind,
3032 unsigned Index) const {
3033 if (isa<FixedVectorType>(Val))
3034 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
3035 Index);
3036
3037 // TODO: This code replicates what LoopVectorize.cpp used to do when asking
3038 // for the cost of extracting the last lane of a scalable vector. It probably
3039 // needs a more accurate cost.
3040 ElementCount EC = cast<VectorType>(Val)->getElementCount();
3041 assert(Index < EC.getKnownMinValue() && "Unexpected reverse index");
3042 return getVectorInstrCost(Opcode, Val, CostKind,
3043 Index: EC.getKnownMinValue() - 1 - Index, Op0: nullptr,
3044 Op1: nullptr);
3045}
3046
3047/// Check to see if this instruction is expected to be combined to a simpler
3048/// operation during/before lowering. If so return the cost of the combined
3049/// operation rather than provided one. For instance, `udiv i16 %X, 2` is likely
3050/// to be combined to `lshr i16 %X, 1`, so return the cost of a `lshr` rather
3051/// than the cost of a `udiv`
3052std::optional<InstructionCost>
3053RISCVTTIImpl::getCombinedArithmeticInstructionCost(
3054 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
3055 TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
3056 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
3057 // Vector unsigned division/remainder will be simplified to shifts/masks.
3058 if ((Opcode == Instruction::UDiv || Opcode == Instruction::URem) &&
3059 Opd2Info.isConstant() && Opd2Info.isPowerOf2()) {
3060 if (Opcode == Instruction::UDiv)
3061 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind, Op1Info: Opd1Info,
3062 Op2Info: Opd2Info.getNoProps());
3063 // UREM
3064 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind, Op1Info: Opd1Info,
3065 Op2Info: Opd2Info.getNoProps());
3066 }
3067 return std::nullopt;
3068}
3069
3070InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
3071 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
3072 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
3073 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
3074
3075 // TODO: Handle more cost kinds.
3076 if (CostKind != TTI::TCK_RecipThroughput)
3077 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3078 Args, CtxI);
3079
3080 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
3081 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3082 Args, CtxI);
3083
3084 // Skip if scalar size of Ty is bigger than ELEN.
3085 if (isa<VectorType>(Val: Ty) && Ty->getScalarSizeInBits() > ST->getELen())
3086 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3087 Args, CtxI);
3088
3089 if (std::optional<InstructionCost> CombinedCost =
3090 getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
3091 Opd2Info: Op2Info, Args, CtxI))
3092 return *CombinedCost;
3093
3094 // Legalize the type.
3095 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
3096 unsigned ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
3097
3098 // TODO: Handle scalar type.
3099 if (!LT.second.isVector()) {
3100 static const CostTblEntry DivTbl[]{
3101 {.ISD: ISD::UDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
3102 {.ISD: ISD::UDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
3103 {.ISD: ISD::SDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
3104 {.ISD: ISD::SDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
3105 {.ISD: ISD::UREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
3106 {.ISD: ISD::UREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
3107 {.ISD: ISD::SREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
3108 {.ISD: ISD::SREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive}};
3109 if (TLI->isOperationLegalOrPromote(Op: ISDOpcode, VT: LT.second))
3110 if (const auto *Entry = CostTableLookup(Table: DivTbl, ISD: ISDOpcode, Ty: LT.second))
3111 return Entry->Cost * LT.first;
3112
3113 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3114 Args, CtxI);
3115 }
3116
3117 // f16 with zvfhmin and bf16 will be promoted to f32.
3118 // FIXME: nxv32[b]f16 will be custom lowered and split.
3119 InstructionCost CastCost = 0;
3120 if ((LT.second.getVectorElementType() == MVT::f16 ||
3121 LT.second.getVectorElementType() == MVT::bf16) &&
3122 TLI->getOperationAction(Op: ISDOpcode, VT: LT.second) ==
3123 TargetLoweringBase::LegalizeAction::Promote) {
3124 MVT PromotedVT = TLI->getTypeToPromoteTo(Op: ISDOpcode, VT: LT.second);
3125 Type *PromotedTy = EVT(PromotedVT).getTypeForEVT(Context&: Ty->getContext());
3126 Type *LegalTy = EVT(LT.second).getTypeForEVT(Context&: Ty->getContext());
3127 // Add cost of extending arguments
3128 CastCost += LT.first * Args.size() *
3129 getCastInstrCost(Opcode: Instruction::FPExt, Dst: PromotedTy, Src: LegalTy,
3130 CCH: TTI::CastContextHint::None, CostKind);
3131 // Add cost of truncating result
3132 CastCost +=
3133 LT.first * getCastInstrCost(Opcode: Instruction::FPTrunc, Dst: LegalTy, Src: PromotedTy,
3134 CCH: TTI::CastContextHint::None, CostKind);
3135 // Compute cost of op in promoted type
3136 LT.second = PromotedVT;
3137 }
3138
3139 auto getConstantMatCost =
3140 [&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost {
3141 if (OpInfo.isUniform() &&
3142 canSplatOperand(Opcode, Operand, IntrinsicID: Intrinsic::not_intrinsic))
3143 // Two sub-cases:
3144 // * Has a 5 bit immediate operand which can be splatted.
3145 // * Has a larger immediate which must be materialized in scalar register
3146 // We return 0 for both as we currently ignore the cost of materializing
3147 // scalar constants in GPRs.
3148 return 0;
3149
3150 return getConstantPoolLoadCost(Ty, CostKind);
3151 };
3152
3153 // Add the cost of materializing any constant vectors required.
3154 InstructionCost ConstantMatCost = 0;
3155 if (Op1Info.isConstant())
3156 ConstantMatCost += getConstantMatCost(0, Op1Info);
3157 if (Op2Info.isConstant())
3158 ConstantMatCost += getConstantMatCost(1, Op2Info);
3159
3160 unsigned Op;
3161 switch (ISDOpcode) {
3162 case ISD::ADD:
3163 case ISD::SUB:
3164 Op = RISCV::VADD_VV;
3165 break;
3166 case ISD::SHL:
3167 case ISD::SRL:
3168 case ISD::SRA:
3169 Op = RISCV::VSLL_VV;
3170 break;
3171 case ISD::AND:
3172 case ISD::OR:
3173 case ISD::XOR:
3174 Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
3175 break;
3176 case ISD::MUL:
3177 case ISD::MULHS:
3178 case ISD::MULHU:
3179 Op = RISCV::VMUL_VV;
3180 break;
3181 case ISD::SDIV:
3182 case ISD::UDIV:
3183 Op = RISCV::VDIV_VV;
3184 break;
3185 case ISD::SREM:
3186 case ISD::UREM:
3187 Op = RISCV::VREM_VV;
3188 break;
3189 case ISD::FADD:
3190 case ISD::FSUB:
3191 Op = RISCV::VFADD_VV;
3192 break;
3193 case ISD::FMUL:
3194 Op = RISCV::VFMUL_VV;
3195 break;
3196 case ISD::FDIV:
3197 Op = RISCV::VFDIV_VV;
3198 break;
3199 case ISD::FNEG:
3200 Op = RISCV::VFSGNJN_VV;
3201 break;
3202 default:
3203 // Assuming all other instructions have the same cost until a need arises to
3204 // differentiate them.
3205 return CastCost + ConstantMatCost +
3206 BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3207 Args, CtxI);
3208 }
3209
3210 InstructionCost InstrCost = getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
3211 // We use BasicTTIImpl to calculate scalar costs, which assumes floating point
3212 // ops are twice as expensive as integer ops. Do the same for vectors so
3213 // scalar floating point ops aren't cheaper than their vector equivalents.
3214 if (Ty->isFPOrFPVectorTy())
3215 InstrCost *= 2;
3216 return CastCost + ConstantMatCost + LT.first * InstrCost;
3217}
3218
3219// TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
3220InstructionCost RISCVTTIImpl::getPointersChainCost(
3221 ArrayRef<const Value *> Ptrs, const Value *Base,
3222 const TTI::PointersChainInfo &Info, Type *AccessTy,
3223 const TTI::TargetCostKind CostKind) const {
3224 InstructionCost Cost = TTI::TCC_Free;
3225 // In the basic model we take into account GEP instructions only
3226 // (although here can come alloca instruction, a value, constants and/or
3227 // constant expressions, PHIs, bitcasts ... whatever allowed to be used as a
3228 // pointer). Typically, if Base is a not a GEP-instruction and all the
3229 // pointers are relative to the same base address, all the rest are
3230 // either GEP instructions, PHIs, bitcasts or constants. When we have same
3231 // base, we just calculate cost of each non-Base GEP as an ADD operation if
3232 // any their index is a non-const.
3233 // If no known dependencies between the pointers cost is calculated as a sum
3234 // of costs of GEP instructions.
3235 for (auto [I, V] : enumerate(First&: Ptrs)) {
3236 const auto *GEP = dyn_cast<GetElementPtrInst>(Val: V);
3237 if (!GEP)
3238 continue;
3239 if (Info.isSameBase() && V != Base) {
3240 if (GEP->hasAllConstantIndices())
3241 continue;
3242 // If the chain is unit-stride and BaseReg + stride*i is a legal
3243 // addressing mode, then presume the base GEP is sitting around in a
3244 // register somewhere and check if we can fold the offset relative to
3245 // it.
3246 unsigned Stride = DL.getTypeStoreSize(Ty: AccessTy);
3247 if (Info.isUnitStride() &&
3248 isLegalAddressingMode(Ty: AccessTy,
3249 /* BaseGV */ nullptr,
3250 /* BaseOffset */ Stride * I,
3251 /* HasBaseReg */ true,
3252 /* Scale */ 0,
3253 AddrSpace: GEP->getType()->getPointerAddressSpace()))
3254 continue;
3255 Cost += getArithmeticInstrCost(Opcode: Instruction::Add, Ty: GEP->getType(), CostKind,
3256 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
3257 Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Args: {});
3258 } else {
3259 SmallVector<const Value *> Indices(GEP->indices());
3260 Cost += getGEPCost(PointeeType: GEP->getSourceElementType(), Ptr: GEP->getPointerOperand(),
3261 Operands: Indices, CostKind, AccessType: AccessTy);
3262 }
3263 }
3264 return Cost;
3265}
3266
3267void RISCVTTIImpl::getUnrollingPreferences(
3268 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
3269 OptimizationRemarkEmitter *ORE) const {
3270 // TODO: More tuning on benchmarks and metrics with changes as needed
3271 // would apply to all settings below to enable performance.
3272
3273
3274 if (ST->enableDefaultUnroll())
3275 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
3276
3277 // Enable Upper bound unrolling universally, not dependent upon the conditions
3278 // below.
3279 UP.UpperBound = true;
3280
3281 // Disable loop unrolling for Oz and Os.
3282 UP.OptSizeThreshold = 0;
3283 UP.PartialOptSizeThreshold = 0;
3284 if (L->getHeader()->getParent()->hasOptSize())
3285 return;
3286
3287 SmallVector<BasicBlock *, 4> ExitingBlocks;
3288 L->getExitingBlocks(ExitingBlocks);
3289 LLVM_DEBUG(dbgs() << "Loop has:\n"
3290 << "Blocks: " << L->getNumBlocks() << "\n"
3291 << "Exit blocks: " << ExitingBlocks.size() << "\n");
3292
3293 // Only allow another exit other than the latch. This acts as an early exit
3294 // as it mirrors the profitability calculation of the runtime unroller.
3295 if (ExitingBlocks.size() > 2)
3296 return;
3297
3298 // Limit the CFG of the loop body for targets with a branch predictor.
3299 // Allowing 4 blocks permits if-then-else diamonds in the body.
3300 if (L->getNumBlocks() > 4)
3301 return;
3302
3303 // Scan the loop: don't unroll loops with calls as this could prevent
3304 // inlining. Don't unroll auto-vectorized loops either, though do allow
3305 // unrolling of the scalar remainder.
3306 bool IsVectorized = getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized");
3307 InstructionCost Cost = 0;
3308 for (auto *BB : L->getBlocks()) {
3309 for (auto &I : *BB) {
3310 // Both auto-vectorized loops and the scalar remainder have the
3311 // isvectorized attribute, so differentiate between them by the presence
3312 // of vector instructions.
3313 if (IsVectorized && (I.getType()->isVectorTy() ||
3314 llvm::any_of(Range: I.operand_values(), P: [](Value *V) {
3315 return V->getType()->isVectorTy();
3316 })))
3317 return;
3318
3319 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
3320 const Function *F = cast<CallBase>(Val&: I).getCalledFunction();
3321 if (!F || isLoweredToCall(F))
3322 return;
3323 }
3324
3325 SmallVector<const Value *> Operands(I.operand_values());
3326 Cost += getInstructionCost(U: &I, Operands,
3327 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
3328 }
3329 }
3330
3331 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
3332
3333 UP.Partial = true;
3334 UP.Runtime = true;
3335 UP.UnrollRemainder = true;
3336 UP.UnrollAndJam = true;
3337
3338 // Force unrolling small loops can be very useful because of the branch
3339 // taken cost of the backedge.
3340 if (Cost < 12)
3341 UP.Force = true;
3342}
3343
3344void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
3345 TTI::PeelingPreferences &PP) const {
3346 BaseT::getPeelingPreferences(L, SE, PP);
3347}
3348
3349bool RISCVTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
3350 MemIntrinsicInfo &Info) const {
3351 const DataLayout &DL = getDataLayout();
3352 Intrinsic::ID IID = Inst->getIntrinsicID();
3353 LLVMContext &C = Inst->getContext();
3354 bool HasMask = false;
3355
3356 auto getSegNum = [](const IntrinsicInst *II, unsigned PtrOperandNo,
3357 bool IsWrite) -> int64_t {
3358 if (auto *TarExtTy =
3359 dyn_cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType()))
3360 return TarExtTy->getIntParameter(i: 0);
3361
3362 return 1;
3363 };
3364
3365 switch (IID) {
3366 case Intrinsic::riscv_vle_mask:
3367 case Intrinsic::riscv_vse_mask:
3368 case Intrinsic::riscv_vlseg2_mask:
3369 case Intrinsic::riscv_vlseg3_mask:
3370 case Intrinsic::riscv_vlseg4_mask:
3371 case Intrinsic::riscv_vlseg5_mask:
3372 case Intrinsic::riscv_vlseg6_mask:
3373 case Intrinsic::riscv_vlseg7_mask:
3374 case Intrinsic::riscv_vlseg8_mask:
3375 case Intrinsic::riscv_vsseg2_mask:
3376 case Intrinsic::riscv_vsseg3_mask:
3377 case Intrinsic::riscv_vsseg4_mask:
3378 case Intrinsic::riscv_vsseg5_mask:
3379 case Intrinsic::riscv_vsseg6_mask:
3380 case Intrinsic::riscv_vsseg7_mask:
3381 case Intrinsic::riscv_vsseg8_mask:
3382 HasMask = true;
3383 [[fallthrough]];
3384 case Intrinsic::riscv_vle:
3385 case Intrinsic::riscv_vse:
3386 case Intrinsic::riscv_vlseg2:
3387 case Intrinsic::riscv_vlseg3:
3388 case Intrinsic::riscv_vlseg4:
3389 case Intrinsic::riscv_vlseg5:
3390 case Intrinsic::riscv_vlseg6:
3391 case Intrinsic::riscv_vlseg7:
3392 case Intrinsic::riscv_vlseg8:
3393 case Intrinsic::riscv_vsseg2:
3394 case Intrinsic::riscv_vsseg3:
3395 case Intrinsic::riscv_vsseg4:
3396 case Intrinsic::riscv_vsseg5:
3397 case Intrinsic::riscv_vsseg6:
3398 case Intrinsic::riscv_vsseg7:
3399 case Intrinsic::riscv_vsseg8: {
3400 // Intrinsic interface:
3401 // riscv_vle(merge, ptr, vl)
3402 // riscv_vle_mask(merge, ptr, mask, vl, policy)
3403 // riscv_vse(val, ptr, vl)
3404 // riscv_vse_mask(val, ptr, mask, vl, policy)
3405 // riscv_vlseg#(merge, ptr, vl, sew)
3406 // riscv_vlseg#_mask(merge, ptr, mask, vl, policy, sew)
3407 // riscv_vsseg#(val, ptr, vl, sew)
3408 // riscv_vsseg#_mask(val, ptr, mask, vl, sew)
3409 bool IsWrite = Inst->getType()->isVoidTy();
3410 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3411 // The results of segment loads are TargetExtType.
3412 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3413 unsigned SEW =
3414 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3415 ->getZExtValue();
3416 Ty = TarExtTy->getTypeParameter(i: 0U);
3417 Ty = ScalableVectorType::get(
3418 ElementType: IntegerType::get(C, NumBits: SEW),
3419 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3420 }
3421 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3422 unsigned VLIndex = RVVIInfo->VLOperand;
3423 unsigned PtrOperandNo = VLIndex - 1 - HasMask;
3424 MaybeAlign Alignment =
3425 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3426 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3427 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3428 if (HasMask)
3429 Mask = Inst->getArgOperand(i: VLIndex - 1);
3430 Value *EVL = Inst->getArgOperand(i: VLIndex);
3431 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3432 // RVV uses contiguous elements as a segment.
3433 if (SegNum > 1) {
3434 unsigned ElemSize = Ty->getScalarSizeInBits();
3435 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3436 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3437 }
3438 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3439 Args&: Alignment, Args&: Mask, Args&: EVL);
3440 return true;
3441 }
3442 case Intrinsic::riscv_vlse_mask:
3443 case Intrinsic::riscv_vsse_mask:
3444 case Intrinsic::riscv_vlsseg2_mask:
3445 case Intrinsic::riscv_vlsseg3_mask:
3446 case Intrinsic::riscv_vlsseg4_mask:
3447 case Intrinsic::riscv_vlsseg5_mask:
3448 case Intrinsic::riscv_vlsseg6_mask:
3449 case Intrinsic::riscv_vlsseg7_mask:
3450 case Intrinsic::riscv_vlsseg8_mask:
3451 case Intrinsic::riscv_vssseg2_mask:
3452 case Intrinsic::riscv_vssseg3_mask:
3453 case Intrinsic::riscv_vssseg4_mask:
3454 case Intrinsic::riscv_vssseg5_mask:
3455 case Intrinsic::riscv_vssseg6_mask:
3456 case Intrinsic::riscv_vssseg7_mask:
3457 case Intrinsic::riscv_vssseg8_mask:
3458 HasMask = true;
3459 [[fallthrough]];
3460 case Intrinsic::riscv_vlse:
3461 case Intrinsic::riscv_vsse:
3462 case Intrinsic::riscv_vlsseg2:
3463 case Intrinsic::riscv_vlsseg3:
3464 case Intrinsic::riscv_vlsseg4:
3465 case Intrinsic::riscv_vlsseg5:
3466 case Intrinsic::riscv_vlsseg6:
3467 case Intrinsic::riscv_vlsseg7:
3468 case Intrinsic::riscv_vlsseg8:
3469 case Intrinsic::riscv_vssseg2:
3470 case Intrinsic::riscv_vssseg3:
3471 case Intrinsic::riscv_vssseg4:
3472 case Intrinsic::riscv_vssseg5:
3473 case Intrinsic::riscv_vssseg6:
3474 case Intrinsic::riscv_vssseg7:
3475 case Intrinsic::riscv_vssseg8: {
3476 // Intrinsic interface:
3477 // riscv_vlse(merge, ptr, stride, vl)
3478 // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
3479 // riscv_vsse(val, ptr, stride, vl)
3480 // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
3481 // riscv_vlsseg#(merge, ptr, offset, vl, sew)
3482 // riscv_vlsseg#_mask(merge, ptr, offset, mask, vl, policy, sew)
3483 // riscv_vssseg#(val, ptr, offset, vl, sew)
3484 // riscv_vssseg#_mask(val, ptr, offset, mask, vl, sew)
3485 bool IsWrite = Inst->getType()->isVoidTy();
3486 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3487 // The results of segment loads are TargetExtType.
3488 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3489 unsigned SEW =
3490 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3491 ->getZExtValue();
3492 Ty = TarExtTy->getTypeParameter(i: 0U);
3493 Ty = ScalableVectorType::get(
3494 ElementType: IntegerType::get(C, NumBits: SEW),
3495 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3496 }
3497 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3498 unsigned VLIndex = RVVIInfo->VLOperand;
3499 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3500 MaybeAlign Alignment =
3501 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3502
3503 Value *Stride = Inst->getArgOperand(i: PtrOperandNo + 1);
3504 // Use the pointer alignment as the element alignment if the stride is a
3505 // multiple of the pointer alignment. Otherwise, the element alignment
3506 // should be the greatest common divisor of pointer alignment and stride.
3507 // For simplicity, just consider unalignment for elements.
3508 unsigned PointerAlign = Alignment.valueOrOne().value();
3509 if (!isa<ConstantInt>(Val: Stride) ||
3510 cast<ConstantInt>(Val: Stride)->getZExtValue() % PointerAlign != 0)
3511 Alignment = Align(1);
3512
3513 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3514 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3515 if (HasMask)
3516 Mask = Inst->getArgOperand(i: VLIndex - 1);
3517 Value *EVL = Inst->getArgOperand(i: VLIndex);
3518 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3519 // RVV uses contiguous elements as a segment.
3520 if (SegNum > 1) {
3521 unsigned ElemSize = Ty->getScalarSizeInBits();
3522 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3523 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3524 }
3525 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3526 Args&: Alignment, Args&: Mask, Args&: EVL, Args&: Stride);
3527 return true;
3528 }
3529 case Intrinsic::riscv_vloxei_mask:
3530 case Intrinsic::riscv_vluxei_mask:
3531 case Intrinsic::riscv_vsoxei_mask:
3532 case Intrinsic::riscv_vsuxei_mask:
3533 case Intrinsic::riscv_vloxseg2_mask:
3534 case Intrinsic::riscv_vloxseg3_mask:
3535 case Intrinsic::riscv_vloxseg4_mask:
3536 case Intrinsic::riscv_vloxseg5_mask:
3537 case Intrinsic::riscv_vloxseg6_mask:
3538 case Intrinsic::riscv_vloxseg7_mask:
3539 case Intrinsic::riscv_vloxseg8_mask:
3540 case Intrinsic::riscv_vluxseg2_mask:
3541 case Intrinsic::riscv_vluxseg3_mask:
3542 case Intrinsic::riscv_vluxseg4_mask:
3543 case Intrinsic::riscv_vluxseg5_mask:
3544 case Intrinsic::riscv_vluxseg6_mask:
3545 case Intrinsic::riscv_vluxseg7_mask:
3546 case Intrinsic::riscv_vluxseg8_mask:
3547 case Intrinsic::riscv_vsoxseg2_mask:
3548 case Intrinsic::riscv_vsoxseg3_mask:
3549 case Intrinsic::riscv_vsoxseg4_mask:
3550 case Intrinsic::riscv_vsoxseg5_mask:
3551 case Intrinsic::riscv_vsoxseg6_mask:
3552 case Intrinsic::riscv_vsoxseg7_mask:
3553 case Intrinsic::riscv_vsoxseg8_mask:
3554 case Intrinsic::riscv_vsuxseg2_mask:
3555 case Intrinsic::riscv_vsuxseg3_mask:
3556 case Intrinsic::riscv_vsuxseg4_mask:
3557 case Intrinsic::riscv_vsuxseg5_mask:
3558 case Intrinsic::riscv_vsuxseg6_mask:
3559 case Intrinsic::riscv_vsuxseg7_mask:
3560 case Intrinsic::riscv_vsuxseg8_mask:
3561 HasMask = true;
3562 [[fallthrough]];
3563 case Intrinsic::riscv_vloxei:
3564 case Intrinsic::riscv_vluxei:
3565 case Intrinsic::riscv_vsoxei:
3566 case Intrinsic::riscv_vsuxei:
3567 case Intrinsic::riscv_vloxseg2:
3568 case Intrinsic::riscv_vloxseg3:
3569 case Intrinsic::riscv_vloxseg4:
3570 case Intrinsic::riscv_vloxseg5:
3571 case Intrinsic::riscv_vloxseg6:
3572 case Intrinsic::riscv_vloxseg7:
3573 case Intrinsic::riscv_vloxseg8:
3574 case Intrinsic::riscv_vluxseg2:
3575 case Intrinsic::riscv_vluxseg3:
3576 case Intrinsic::riscv_vluxseg4:
3577 case Intrinsic::riscv_vluxseg5:
3578 case Intrinsic::riscv_vluxseg6:
3579 case Intrinsic::riscv_vluxseg7:
3580 case Intrinsic::riscv_vluxseg8:
3581 case Intrinsic::riscv_vsoxseg2:
3582 case Intrinsic::riscv_vsoxseg3:
3583 case Intrinsic::riscv_vsoxseg4:
3584 case Intrinsic::riscv_vsoxseg5:
3585 case Intrinsic::riscv_vsoxseg6:
3586 case Intrinsic::riscv_vsoxseg7:
3587 case Intrinsic::riscv_vsoxseg8:
3588 case Intrinsic::riscv_vsuxseg2:
3589 case Intrinsic::riscv_vsuxseg3:
3590 case Intrinsic::riscv_vsuxseg4:
3591 case Intrinsic::riscv_vsuxseg5:
3592 case Intrinsic::riscv_vsuxseg6:
3593 case Intrinsic::riscv_vsuxseg7:
3594 case Intrinsic::riscv_vsuxseg8: {
3595 // Intrinsic interface (only listed ordered version):
3596 // riscv_vloxei(merge, ptr, index, vl)
3597 // riscv_vloxei_mask(merge, ptr, index, mask, vl, policy)
3598 // riscv_vsoxei(val, ptr, index, vl)
3599 // riscv_vsoxei_mask(val, ptr, index, mask, vl, policy)
3600 // riscv_vloxseg#(merge, ptr, index, vl, sew)
3601 // riscv_vloxseg#_mask(merge, ptr, index, mask, vl, policy, sew)
3602 // riscv_vsoxseg#(val, ptr, index, vl, sew)
3603 // riscv_vsoxseg#_mask(val, ptr, index, mask, vl, sew)
3604 bool IsWrite = Inst->getType()->isVoidTy();
3605 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3606 // The results of segment loads are TargetExtType.
3607 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3608 unsigned SEW =
3609 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3610 ->getZExtValue();
3611 Ty = TarExtTy->getTypeParameter(i: 0U);
3612 Ty = ScalableVectorType::get(
3613 ElementType: IntegerType::get(C, NumBits: SEW),
3614 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3615 }
3616 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3617 unsigned VLIndex = RVVIInfo->VLOperand;
3618 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3619 Value *Mask;
3620 if (HasMask) {
3621 Mask = Inst->getArgOperand(i: VLIndex - 1);
3622 } else {
3623 // Mask cannot be nullptr here: vector GEP produces <vscale x N x ptr>,
3624 // and casting that to scalar i64 triggers a vector/scalar mismatch
3625 // assertion in CreatePointerCast. Use an all-true mask so ASan lowers it
3626 // via extractelement instead.
3627 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3628 Mask = ConstantInt::getTrue(Ty: MaskType);
3629 }
3630 Value *EVL = Inst->getArgOperand(i: VLIndex);
3631 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3632 // RVV uses contiguous elements as a segment.
3633 if (SegNum > 1) {
3634 unsigned ElemSize = Ty->getScalarSizeInBits();
3635 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3636 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3637 }
3638 Value *OffsetOp = Inst->getArgOperand(i: PtrOperandNo + 1);
3639 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3640 Args: Align(1), Args&: Mask, Args&: EVL,
3641 /* Stride */ Args: nullptr, Args&: OffsetOp);
3642 return true;
3643 }
3644 }
3645 return false;
3646}
3647
3648unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) const {
3649 if (Ty->isVectorTy()) {
3650 // f16 with only zvfhmin and bf16 will be promoted to f32
3651 Type *EltTy = cast<VectorType>(Val: Ty)->getElementType();
3652 if ((EltTy->isHalfTy() && !ST->hasVInstructionsF16()) ||
3653 EltTy->isBFloatTy())
3654 Ty = VectorType::get(ElementType: Type::getFloatTy(C&: Ty->getContext()),
3655 Other: cast<VectorType>(Val: Ty));
3656
3657 TypeSize Size = DL.getTypeSizeInBits(Ty);
3658 if (Size.isScalable() && ST->hasVInstructions())
3659 return divideCeil(Numerator: Size.getKnownMinValue(), Denominator: RISCV::RVVBitsPerBlock);
3660
3661 if (ST->useRVVForFixedLengthVectors())
3662 return divideCeil(Numerator: Size, Denominator: ST->getRealMinVLen());
3663 }
3664
3665 return BaseT::getRegUsageForType(Ty);
3666}
3667
3668unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
3669 if (std::optional<unsigned> MaxVF = ST->getCLOpts().v_slp_max_vf)
3670 return *MaxVF;
3671
3672 // Return how many elements can fit in getRegisterBitwidth. This is the
3673 // same routine as used in LoopVectorizer. We should probably be
3674 // accounting for whether we actually have instructions with the right
3675 // lane type, but we don't have enough information to do that without
3676 // some additional plumbing which hasn't been justified yet.
3677 TypeSize RegWidth =
3678 getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
3679 // If no vector registers, or absurd element widths, disable
3680 // vectorization by returning 1.
3681 return std::max<unsigned>(a: 1U, b: RegWidth.getFixedValue() / ElemWidth);
3682}
3683
3684unsigned RISCVTTIImpl::getMinTripCountTailFoldingThreshold() const {
3685 return ST->getCLOpts().v_min_trip_count;
3686}
3687
3688bool RISCVTTIImpl::preferAlternateOpcodeVectorization() const {
3689 return ST->enableUnalignedVectorMem();
3690}
3691
3692TTI::AddressingModeKind
3693RISCVTTIImpl::getPreferredAddressingMode(const Loop *L,
3694 ScalarEvolution *SE) const {
3695 if (ST->hasVendorXCVmem() && !ST->is64Bit())
3696 return TTI::AMK_PostIndexed;
3697
3698 return BasicTTIImplBase::getPreferredAddressingMode(L, SE);
3699}
3700
3701bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
3702 const TargetTransformInfo::LSRCost &C2) const {
3703 // RISC-V specific here are "instruction number 1st priority".
3704 // If we need to emit adds inside the loop to add up base registers, then
3705 // we need at least one extra temporary register.
3706 unsigned C1NumRegs = C1.NumRegs + (C1.NumBaseAdds != 0);
3707 unsigned C2NumRegs = C2.NumRegs + (C2.NumBaseAdds != 0);
3708 return std::tie(args: C1.Insns, args&: C1NumRegs, args: C1.AddRecCost,
3709 args: C1.NumIVMuls, args: C1.NumBaseAdds,
3710 args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
3711 std::tie(args: C2.Insns, args&: C2NumRegs, args: C2.AddRecCost,
3712 args: C2.NumIVMuls, args: C2.NumBaseAdds,
3713 args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
3714}
3715
3716bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
3717 Align Alignment) const {
3718 auto *VTy = dyn_cast<VectorType>(Val: DataTy);
3719 if (!VTy)
3720 return false;
3721
3722 if (!isLegalMaskedLoadStore(DataType: DataTy, Alignment))
3723 return false;
3724
3725 // FIXME: If it is an i8 vector and the element count exceeds 256, we should
3726 // scalarize these types with LMUL >= maximum fixed-length LMUL.
3727 if (VTy->getElementType()->isIntegerTy(BitWidth: 8)) {
3728 uint64_t MaxEltCount = VTy->getElementCount().getKnownMinValue();
3729 if (VTy->isScalableTy())
3730 MaxEltCount *= ST->getRealMaxVLen() / RISCV::RVVBitsPerBlock;
3731 // We can't yet split any widened indices type.
3732 if (MaxEltCount > 256)
3733 return getTypeLegalizationCost(
3734 Ty: VTy->getWithNewType(EltTy: Type::getInt16Ty(C&: VTy->getContext())))
3735 .first == 1;
3736 }
3737 return true;
3738}
3739
3740bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,
3741 Align Alignment) const {
3742 return isLegalMaskedLoadStore(DataType: DataTy, Alignment);
3743}
3744
3745bool RISCVTTIImpl::isLegalBroadcastLoad(Type *ElementTy,
3746 ElementCount NumElements) const {
3747 // Optimized zero-stride loads can be treated as broadcasts.
3748 if (!ST->hasVInstructions() || !ST->hasOptimizedZeroStrideLoad())
3749 return false;
3750
3751 return TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: ElementTy));
3752}
3753
3754/// See if \p I should be considered for address type promotion. We check if \p
3755/// I is a sext with right type and used in memory accesses. If it used in a
3756/// "complex" getelementptr, we allow it to be promoted without finding other
3757/// sext instructions that sign extended the same initial value. A getelementptr
3758/// is considered as "complex" if it has more than 2 operands.
3759bool RISCVTTIImpl::shouldConsiderAddressTypePromotion(
3760 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
3761 bool Considerable = false;
3762 AllowPromotionWithoutCommonHeader = false;
3763 if (!isa<SExtInst>(Val: &I))
3764 return false;
3765 Type *ConsideredSExtType =
3766 Type::getInt64Ty(C&: I.getParent()->getParent()->getContext());
3767 if (I.getType() != ConsideredSExtType)
3768 return false;
3769 // See if the sext is the one with the right type and used in at least one
3770 // GetElementPtrInst.
3771 for (const User *U : I.users()) {
3772 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(Val: U)) {
3773 Considerable = true;
3774 // A getelementptr is considered as "complex" if it has more than 2
3775 // operands. We will promote a SExt used in such complex GEP as we
3776 // expect some computation to be merged if they are done on 64 bits.
3777 if (GEPInst->getNumOperands() > 2) {
3778 AllowPromotionWithoutCommonHeader = true;
3779 break;
3780 }
3781 }
3782 }
3783 return Considerable;
3784}
3785
3786bool RISCVTTIImpl::canSplatOperand(unsigned Opcode, int Operand,
3787 Intrinsic::ID IntrinsicID) const {
3788 switch (Opcode) {
3789 case Instruction::Add:
3790 case Instruction::Sub:
3791 case Instruction::Mul:
3792 case Instruction::And:
3793 case Instruction::Or:
3794 case Instruction::Xor:
3795 case Instruction::FAdd:
3796 case Instruction::FSub:
3797 case Instruction::FMul:
3798 case Instruction::FDiv:
3799 case Instruction::ICmp:
3800 case Instruction::FCmp:
3801 return true;
3802 case Instruction::Shl:
3803 case Instruction::LShr:
3804 case Instruction::AShr:
3805 case Instruction::UDiv:
3806 case Instruction::SDiv:
3807 case Instruction::URem:
3808 case Instruction::SRem:
3809 case Instruction::Select:
3810 return Operand == 1;
3811 case Instruction::Call:
3812 switch (IntrinsicID) {
3813 case Intrinsic::fma:
3814 case Intrinsic::fmuladd:
3815 return Operand == 0 || Operand == 1;
3816 case Intrinsic::vp_udiv:
3817 case Intrinsic::vp_sdiv:
3818 case Intrinsic::vp_urem:
3819 case Intrinsic::vp_srem:
3820 case Intrinsic::ssub_sat:
3821 case Intrinsic::usub_sat:
3822 return Operand == 1;
3823 // These intrinsics are commutative.
3824 case Intrinsic::smin:
3825 case Intrinsic::umin:
3826 case Intrinsic::smax:
3827 case Intrinsic::umax:
3828 case Intrinsic::sadd_sat:
3829 case Intrinsic::uadd_sat:
3830 return Operand == 0 || Operand == 1;
3831 default:
3832 return false;
3833 }
3834 default:
3835 return false;
3836 }
3837}
3838
3839bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {
3840 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3841 return false;
3842
3843 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3844 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
3845 if (II)
3846 IntrinsicID = II->getIntrinsicID();
3847
3848 return canSplatOperand(Opcode: I->getOpcode(), Operand, IntrinsicID);
3849}
3850
3851TargetTransformInfo::VectorInstrContext RISCVTTIImpl::getBuildVectorContextHint(
3852 ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
3853 function_ref<bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)>
3854 GatherUseOps) const {
3855 if (Scalars.empty() || !ST->hasVInstructions() || !ST->sinkSplatOperands() ||
3856 !ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts: Mask.size()))
3857 return VectorInstrContext::None;
3858
3859 const auto *SplatIt = find_if_not(Range&: Scalars, P: IsaPred<UndefValue>);
3860 if (SplatIt == Scalars.end() || (*SplatIt)->getType()->isIntegerTy(BitWidth: 1) ||
3861 isa<VectorType>(Val: (*SplatIt)->getType()) ||
3862 isa<ExtractElementInst>(Val: *SplatIt))
3863 return VectorInstrContext::None;
3864
3865 SmallVector<TargetTransformInfo::BuildVectorUseOp, 4> UserOps;
3866 if (!GatherUseOps(UserOps) || UserOps.empty())
3867 return VectorInstrContext::None;
3868
3869 if (all_of(Range&: UserOps,
3870 P: [this](const TargetTransformInfo::BuildVectorUseOp &UserOp) {
3871 return canSplatOperand(Opcode: UserOp.Opcode, Operand: UserOp.OperandIndex,
3872 IntrinsicID: UserOp.IntrinsicID);
3873 }))
3874 return VectorInstrContext::SplatOpFolded;
3875
3876 return VectorInstrContext::None;
3877}
3878
3879/// Check if sinking \p I's operands to I's basic block is profitable, because
3880/// the operands can be folded into a target instruction, e.g.
3881/// splats of scalars can fold into vector instructions.
3882bool RISCVTTIImpl::isProfitableToSinkOperands(
3883 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
3884 using namespace llvm::PatternMatch;
3885
3886 if (I->isBitwiseLogicOp()) {
3887 if (!I->getType()->isVectorTy()) {
3888 if (ST->hasStdExtZbb() || ST->hasStdExtZbkb()) {
3889 for (auto &Op : I->operands()) {
3890 // (and/or/xor X, (not Y)) -> (andn/orn/xnor X, Y)
3891 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3892 Ops.push_back(Elt: &Op);
3893 return true;
3894 }
3895 }
3896 }
3897 } else if (I->getOpcode() == Instruction::And && ST->hasStdExtZvkb()) {
3898 for (auto &Op : I->operands()) {
3899 // (and X, (not Y)) -> (vandn.vv X, Y)
3900 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3901 Ops.push_back(Elt: &Op);
3902 return true;
3903 }
3904 // (and X, (splat (not Y))) -> (vandn.vx X, Y)
3905 if (match(V: Op.get(), P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()),
3906 Idx: m_ZeroInt()),
3907 v2: m_Value(), mask: m_ZeroMask()))) {
3908 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
3909 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
3910 Ops.push_back(Elt: &Not);
3911 Ops.push_back(Elt: &InsertElt);
3912 Ops.push_back(Elt: &Op);
3913 return true;
3914 }
3915 }
3916 }
3917 }
3918
3919 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3920 return false;
3921
3922 // Don't sink splat operands if the target prefers it. Some targets requires
3923 // S2V transfer buffers and we can run out of them copying the same value
3924 // repeatedly.
3925 // FIXME: It could still be worth doing if it would improve vector register
3926 // pressure and prevent a vector spill.
3927 if (!ST->sinkSplatOperands())
3928 return false;
3929
3930 for (auto OpIdx : enumerate(First: I->operands())) {
3931 if (!canSplatOperand(I, Operand: OpIdx.index()))
3932 continue;
3933
3934 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
3935 // Make sure we are not already sinking this operand
3936 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
3937 continue;
3938
3939 // We are looking for a splat that can be sunk.
3940 if (!match(V: Op, P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ZeroInt()),
3941 v2: m_Value(), mask: m_ZeroMask())))
3942 continue;
3943
3944 // Don't sink i1 splats.
3945 if (cast<VectorType>(Val: Op->getType())->getElementType()->isIntegerTy(BitWidth: 1))
3946 continue;
3947
3948 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
3949 // and vector registers
3950 for (Use &U : Op->uses()) {
3951 Instruction *Insn = cast<Instruction>(Val: U.getUser());
3952 if (!canSplatOperand(I: Insn, Operand: U.getOperandNo()))
3953 return false;
3954 }
3955
3956 // Sink any fpexts since they might be used in a widening fp pattern.
3957 Use *InsertEltUse = &Op->getOperandUse(i: 0);
3958 auto *InsertElt = cast<InsertElementInst>(Val: InsertEltUse);
3959 if (isa<FPExtInst>(Val: InsertElt->getOperand(i_nocapture: 1)))
3960 Ops.push_back(Elt: &InsertElt->getOperandUse(i: 1));
3961 Ops.push_back(Elt: InsertEltUse);
3962 Ops.push_back(Elt: &OpIdx.value());
3963 }
3964 return true;
3965}
3966
3967RISCVTTIImpl::TTI::MemCmpExpansionOptions
3968RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
3969 TTI::MemCmpExpansionOptions Options;
3970
3971 if (!ST->hasStdExtZbb() && !ST->hasStdExtZbkb() && !IsZeroCmp)
3972 return Options;
3973
3974 Options.AllowOverlappingLoads = true;
3975 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
3976 Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
3977 if (ST->is64Bit()) {
3978 Options.LoadSizes = {8, 4, 2, 1};
3979 Options.AllowedTailExpansions = {3, 5, 6};
3980 } else {
3981 Options.LoadSizes = {4, 2, 1};
3982 Options.AllowedTailExpansions = {3};
3983 }
3984
3985 if (IsZeroCmp && ST->hasVInstructions()) {
3986 unsigned VLenB = ST->getRealMinVLen() / 8;
3987 // The minimum size should be `XLen / 8 + 1`, and the maxinum size should be
3988 // `VLenB * MaxLMUL` so that it fits in a single register group.
3989 unsigned MinSize = ST->getXLen() / 8 + 1;
3990 unsigned MaxSize = VLenB * 8;
3991 for (unsigned Size = MinSize; Size <= MaxSize; Size++)
3992 Options.LoadSizes.insert(I: Options.LoadSizes.begin(), Elt: Size);
3993 }
3994 return Options;
3995}
3996
3997bool RISCVTTIImpl::shouldTreatInstructionLikeSelect(
3998 const Instruction *I) const {
3999 if (ST->getCLOpts().or_like_select) {
4000 // For the binary operators (e.g. or) we need to be more careful than
4001 // selects, here we only transform them if they are already at a natural
4002 // break point in the code - the end of a block with an unconditional
4003 // terminator.
4004 if (I->getOpcode() == Instruction::Or &&
4005 isa<UncondBrInst>(Val: I->getNextNode()))
4006 return true;
4007
4008 if (I->getOpcode() == Instruction::Add ||
4009 I->getOpcode() == Instruction::Sub)
4010 return true;
4011 }
4012 return BaseT::shouldTreatInstructionLikeSelect(I);
4013}
4014
4015bool RISCVTTIImpl::shouldCopyAttributeWhenOutliningFrom(
4016 const Function *Caller, const Attribute &Attr) const {
4017 // "interrupt" controls the prolog/epilog of interrupt handlers (and includes
4018 // restrictions on their signatures). We can outline from the bodies of these
4019 // handlers, but when we do we need to make sure we don't mark the outlined
4020 // function as an interrupt handler too.
4021 if (Attr.isStringAttribute() && Attr.getKindAsString() == "interrupt")
4022 return false;
4023
4024 return BaseT::shouldCopyAttributeWhenOutliningFrom(Caller, Attr);
4025}
4026
4027std::optional<Instruction *>
4028RISCVTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
4029 // Attach a range return attribute describing the result of vsetvli/vsetvlimax
4030 // so generic value analyses can reason about it. The verifier guarantees an
4031 // XLen result and constant VSEW/VLMUL encoding a valid vtype, so no defensive
4032 // validation is needed here.
4033 if (is_contained(Set: {Intrinsic::riscv_vsetvli, Intrinsic::riscv_vsetvlimax},
4034 Element: II.getIntrinsicID())) {
4035 // These intrinsics require the V extension; without it the VLEN queries
4036 // below would assert. Such IR would fail isel anyway, so just bail out.
4037 if (!ST->hasVInstructions())
4038 return {};
4039
4040 bool HasAVL = II.getIntrinsicID() == Intrinsic::riscv_vsetvli;
4041 unsigned Offset = HasAVL ? 1 : 0;
4042 unsigned BitWidth = II.getType()->getIntegerBitWidth();
4043 ConstantRange VLenRange(APInt(BitWidth, ST->getRealMinVLen()),
4044 APInt(BitWidth, ST->getRealMaxVLen()) + 1);
4045
4046 uint64_t VSEW = cast<ConstantInt>(Val: II.getArgOperand(i: Offset))->getZExtValue();
4047 auto VLMUL = static_cast<RISCVVType::VLMUL>(
4048 cast<ConstantInt>(Val: II.getArgOperand(i: Offset + 1))->getZExtValue());
4049 unsigned SEW = RISCVVType::decodeVSEW(VSEW);
4050 unsigned Ratio = RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
4051
4052 // VLMAX = VLEN / (SEW / LMUL), clamped to >= 1 for any usable vtype.
4053 ConstantRange VLMAXRange =
4054 VLenRange.udiv(Other: ConstantRange(APInt(BitWidth, Ratio)))
4055 .umax(Other: ConstantRange(APInt(BitWidth, 1)));
4056
4057 // vsetvlimax returns exactly VLMAX; vsetvli returns vl with
4058 // 0 <= vl <= min(AVL, VLMAX). vl == AVL only when AVL <= the smallest
4059 // possible VLMAX; otherwise vl can shrink below VLMAX (to 0 at runtime), so
4060 // only the VLMAX upper bound is sound.
4061 ConstantRange VLRange = VLMAXRange;
4062 if (HasAVL) {
4063 // vl ≤ VLMAX
4064 VLRange =
4065 ConstantRange::makeAllowedICmpRegion(Pred: CmpInst::ICMP_ULE, Other: VLMAXRange);
4066
4067 Value *AVL = II.getArgOperand(i: 0);
4068 ConstantRange AVLRange = computeConstantRangeIncludingKnownBits(
4069 V: AVL, /*ForSigned=*/false,
4070 SQ: IC.getSimplifyQuery().getWithInstruction(I: &II));
4071
4072 // vl = AVL if AVL ≤ VLMAX
4073 if (AVLRange.icmp(Pred: CmpInst::ICMP_ULE, Other: VLMAXRange))
4074 return IC.replaceInstUsesWith(I&: II, V: AVL);
4075
4076 // vl ≤ AVL
4077 VLRange = VLRange.umin(Other: AVLRange.getUnsignedMax());
4078
4079 // vl > 0 if AVL > 0
4080 if (AVLRange.icmp(Pred: CmpInst::ICMP_UGT, Other: APInt::getZero(numBits: BitWidth)))
4081 VLRange = VLRange.umax(Other: APInt(BitWidth, 1));
4082
4083 // vl = VLMAX if AVL ≥ (2 * VLMAX)
4084 ConstantRange TwoVLMAX = VLMAXRange.multiply(Other: APInt(BitWidth, 2));
4085 if (AVLRange.icmp(Pred: CmpInst::ICMP_UGE, Other: TwoVLMAX))
4086 VLRange = VLRange.intersectWith(CR: VLMAXRange);
4087
4088 // ceil(AVL / 2) ≤ vl ≤ VLMAX if AVL < (2 * VLMAX)
4089 if (AVLRange.icmp(Pred: CmpInst::ICMP_ULT, Other: TwoVLMAX))
4090 VLRange = VLRange.umax(Other: APIntOps::RoundingUDiv(A: AVLRange.getUnsignedMin(),
4091 B: APInt(BitWidth, 2),
4092 RM: APInt::Rounding::UP));
4093 }
4094
4095 ConstantRange OldRange =
4096 II.getRange().value_or(u: ConstantRange::getFull(BitWidth));
4097 ConstantRange NewRange = VLRange.intersectWith(CR: OldRange);
4098 if (NewRange != OldRange) {
4099 II.addRangeRetAttr(CR: NewRange);
4100 return &II;
4101 }
4102 return {};
4103 }
4104
4105 // If all operands of a vmv.v.x are constant, fold a bitcast(vmv.v.x) to scale
4106 // the vmv.v.x, enabling removal of the bitcast. The transform helps avoid
4107 // creating redundant masks.
4108 const DataLayout &DL = IC.getDataLayout();
4109 if (II.user_empty())
4110 return {};
4111 auto *TargetVecTy = dyn_cast<ScalableVectorType>(Val: II.user_back()->getType());
4112 if (!TargetVecTy)
4113 return {};
4114 const APInt *Scalar;
4115 uint64_t VL;
4116 if (!match(V: &II, P: m_Intrinsic<Intrinsic::riscv_vmv_v_x>(
4117 Ops: m_Poison(), Ops: m_APInt(Res&: Scalar), Ops: m_ConstantInt(V&: VL))) ||
4118 !all_of(Range: II.users(), P: [TargetVecTy](User *U) {
4119 return U->getType() == TargetVecTy && match(V: U, P: m_BitCast(Op: m_Value()));
4120 }))
4121 return {};
4122 auto *SourceVecTy = cast<ScalableVectorType>(Val: II.getType());
4123 unsigned TargetEltBW = DL.getTypeSizeInBits(Ty: TargetVecTy->getElementType());
4124 unsigned SourceEltBW = DL.getTypeSizeInBits(Ty: SourceVecTy->getElementType());
4125 if (TargetEltBW % SourceEltBW)
4126 return {};
4127 unsigned TargetScale = TargetEltBW / SourceEltBW;
4128 if (VL % TargetScale || TargetScale == 1)
4129 return {};
4130 Type *VLTy = II.getOperand(i_nocapture: 2)->getType();
4131 ElementCount SourceEC = SourceVecTy->getElementCount();
4132 unsigned NewEltBW = SourceEltBW * TargetScale;
4133 if (!SourceEC.isKnownMultipleOf(RHS: TargetScale) ||
4134 !DL.fitsInLegalInteger(Width: NewEltBW))
4135 return {};
4136 auto *NewEltTy = IntegerType::get(C&: II.getContext(), NumBits: NewEltBW);
4137 if (!TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: NewEltTy)))
4138 return {};
4139 ElementCount NewEC = SourceEC.divideCoefficientBy(RHS: TargetScale);
4140 Type *RetTy = VectorType::get(ElementType: NewEltTy, EC: NewEC);
4141 assert(SourceVecTy->canLosslesslyBitCastTo(RetTy) &&
4142 "Lossless bitcast between types expected");
4143 APInt NewScalar = APInt::getSplat(NewLen: NewEltBW, V: *Scalar);
4144 return IC.replaceInstUsesWith(
4145 I&: II,
4146 V: IC.Builder.CreateBitCast(
4147 V: IC.Builder.CreateIntrinsic(
4148 RetTy, ID: Intrinsic::riscv_vmv_v_x,
4149 Args: {PoisonValue::get(T: RetTy), ConstantInt::get(Ty: NewEltTy, V: NewScalar),
4150 ConstantInt::get(Ty: VLTy, V: VL / TargetScale)}),
4151 DestTy: SourceVecTy));
4152}
4153