1//===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "RISCVTargetTransformInfo.h"
10#include "MCTargetDesc/RISCVMatInt.h"
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Analysis/TargetTransformInfo.h"
13#include "llvm/CodeGen/BasicTTIImpl.h"
14#include "llvm/CodeGen/CostTable.h"
15#include "llvm/CodeGen/TargetLowering.h"
16#include "llvm/CodeGen/ValueTypes.h"
17#include "llvm/IR/Instructions.h"
18#include "llvm/IR/IntrinsicsRISCV.h"
19#include "llvm/IR/PatternMatch.h"
20#include "llvm/Transforms/InstCombine/InstCombiner.h"
21#include <cmath>
22#include <optional>
23using namespace llvm;
24using namespace llvm::PatternMatch;
25
26#define DEBUG_TYPE "riscvtti"
27
28static cl::opt<unsigned> RVVRegisterWidthLMUL(
29 "riscv-v-register-bit-width-lmul",
30 cl::desc(
31 "The LMUL to use for getRegisterBitWidth queries. Affects LMUL used "
32 "by autovectorized code. Fractional LMULs are not supported."),
33 cl::init(Val: 2), cl::Hidden);
34
35static cl::opt<unsigned> SLPMaxVF(
36 "riscv-v-slp-max-vf",
37 cl::desc(
38 "Overrides result used for getMaximumVF query which is used "
39 "exclusively by SLP vectorizer."),
40 cl::Hidden);
41
42static cl::opt<unsigned>
43 RVVMinTripCount("riscv-v-min-trip-count",
44 cl::desc("Set the lower bound of a trip count to decide on "
45 "vectorization while tail-folding."),
46 cl::init(Val: 5), cl::Hidden);
47
48static cl::opt<bool> EnableOrLikeSelectOpt("enable-riscv-or-like-select",
49 cl::init(Val: true), cl::Hidden);
50
51InstructionCost
52RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
53 TTI::TargetCostKind CostKind) const {
54 // Check if the type is valid for all CostKind
55 if (!VT.isVector())
56 return InstructionCost::getInvalid();
57 size_t NumInstr = OpCodes.size();
58 if (CostKind == TTI::TCK_CodeSize)
59 return NumInstr;
60 InstructionCost LMULCost = TLI->getLMULCost(VT);
61 if ((CostKind != TTI::TCK_RecipThroughput) && (CostKind != TTI::TCK_Latency))
62 return LMULCost * NumInstr;
63 InstructionCost Cost = 0;
64 for (auto Op : OpCodes) {
65 switch (Op) {
66 case RISCV::VRGATHER_VI:
67 Cost += TLI->getVRGatherVICost(VT);
68 break;
69 case RISCV::VRGATHER_VV:
70 Cost += TLI->getVRGatherVVCost(VT);
71 break;
72 case RISCV::VSLIDEUP_VI:
73 case RISCV::VSLIDEDOWN_VI:
74 Cost += TLI->getVSlideVICost(VT);
75 break;
76 case RISCV::VSLIDEUP_VX:
77 case RISCV::VSLIDEDOWN_VX:
78 Cost += TLI->getVSlideVXCost(VT);
79 break;
80 case RISCV::VREDMAX_VS:
81 case RISCV::VREDMIN_VS:
82 case RISCV::VREDMAXU_VS:
83 case RISCV::VREDMINU_VS:
84 case RISCV::VREDSUM_VS:
85 case RISCV::VREDAND_VS:
86 case RISCV::VREDOR_VS:
87 case RISCV::VREDXOR_VS:
88 case RISCV::VFREDMAX_VS:
89 case RISCV::VFREDMIN_VS:
90 case RISCV::VFREDUSUM_VS: {
91 unsigned VL = VT.getVectorMinNumElements();
92 if (!VT.isFixedLengthVector())
93 VL *= *getVScaleForTuning();
94 Cost += Log2_32_Ceil(Value: VL);
95 break;
96 }
97 case RISCV::VFREDOSUM_VS: {
98 unsigned VL = VT.getVectorMinNumElements();
99 if (!VT.isFixedLengthVector())
100 VL *= *getVScaleForTuning();
101 Cost += VL;
102 break;
103 }
104 case RISCV::VMV_X_S:
105 case RISCV::VFMV_F_S:
106 // Domain crossings from vector -> scalar are usually more expensive.
107 Cost += 2;
108 break;
109 case RISCV::VMV_S_X:
110 case RISCV::VFMV_S_F:
111 case RISCV::VMOR_MM:
112 case RISCV::VMXOR_MM:
113 case RISCV::VMAND_MM:
114 case RISCV::VMANDN_MM:
115 case RISCV::VMNAND_MM:
116 case RISCV::VCPOP_M:
117 case RISCV::VFIRST_M:
118 Cost += 1;
119 break;
120 case RISCV::VDIV_VV:
121 case RISCV::VREM_VV:
122 Cost += LMULCost * TTI::TCC_Expensive;
123 break;
124 default:
125 Cost += LMULCost;
126 }
127 }
128 return Cost;
129}
130
131static InstructionCost getIntImmCostImpl(const DataLayout &DL,
132 const RISCVSubtarget *ST,
133 const APInt &Imm, Type *Ty,
134 TTI::TargetCostKind CostKind,
135 bool FreeZeroes) {
136 assert(Ty->isIntegerTy() &&
137 "getIntImmCost can only estimate cost of materialising integers");
138
139 // We have a Zero register, so 0 is always free.
140 if (Imm == 0)
141 return TTI::TCC_Free;
142
143 // Otherwise, we check how many instructions it will take to materialise.
144 return RISCVMatInt::getIntMatCost(Val: Imm, Size: DL.getTypeSizeInBits(Ty), STI: *ST,
145 /*CompressionCost=*/false, FreeZeroes);
146}
147
148InstructionCost
149RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
150 TTI::TargetCostKind CostKind) const {
151 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind, FreeZeroes: false);
152}
153
154// Look for patterns of shift followed by AND that can be turned into a pair of
155// shifts. We won't need to materialize an immediate for the AND so these can
156// be considered free.
157static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) {
158 uint64_t Mask = Imm.getZExtValue();
159 auto *BO = dyn_cast<BinaryOperator>(Val: Inst->getOperand(i: 0));
160 if (!BO || !BO->hasOneUse())
161 return false;
162
163 if (BO->getOpcode() != Instruction::Shl)
164 return false;
165
166 if (!isa<ConstantInt>(Val: BO->getOperand(i_nocapture: 1)))
167 return false;
168
169 unsigned ShAmt = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1))->getZExtValue();
170 // (and (shl x, c2), c1) will be matched to (srli (slli x, c2+c3), c3) if c1
171 // is a mask shifted by c2 bits with c3 leading zeros.
172 if (isShiftedMask_64(Value: Mask)) {
173 unsigned Trailing = llvm::countr_zero(Val: Mask);
174 if (ShAmt == Trailing)
175 return true;
176 }
177
178 return false;
179}
180
181// If this is i64 AND is part of (X & -(1 << C1) & 0xffffffff) == C2 << C1),
182// DAGCombiner can convert this to (sraiw X, C1) == sext(C2) for RV64. On RV32,
183// the type will be split so only the lower 32 bits need to be compared using
184// (srai/srli X, C) == C2.
185static bool canUseShiftCmp(Instruction *Inst, const APInt &Imm) {
186 if (!Inst->hasOneUse())
187 return false;
188
189 // Look for equality comparison.
190 auto *Cmp = dyn_cast<ICmpInst>(Val: *Inst->user_begin());
191 if (!Cmp || !Cmp->isEquality())
192 return false;
193
194 // Right hand side of comparison should be a constant.
195 auto *C = dyn_cast<ConstantInt>(Val: Cmp->getOperand(i_nocapture: 1));
196 if (!C)
197 return false;
198
199 uint64_t Mask = Imm.getZExtValue();
200
201 // Mask should be of the form -(1 << C) in the lower 32 bits.
202 if (!isUInt<32>(x: Mask) || !isPowerOf2_32(Value: -uint32_t(Mask)))
203 return false;
204
205 // Comparison constant should be a subset of Mask.
206 uint64_t CmpC = C->getZExtValue();
207 if ((CmpC & Mask) != CmpC)
208 return false;
209
210 // We'll need to sign extend the comparison constant and shift it right. Make
211 // sure the new constant can use addi/xori+seqz/snez.
212 unsigned ShiftBits = llvm::countr_zero(Val: Mask);
213 int64_t NewCmpC = SignExtend64<32>(x: CmpC) >> ShiftBits;
214 return NewCmpC >= -2048 && NewCmpC <= 2048;
215}
216
217InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
218 const APInt &Imm, Type *Ty,
219 TTI::TargetCostKind CostKind,
220 Instruction *Inst) const {
221 assert(Ty->isIntegerTy() &&
222 "getIntImmCost can only estimate cost of materialising integers");
223
224 // We have a Zero register, so 0 is always free.
225 if (Imm == 0)
226 return TTI::TCC_Free;
227
228 // Some instructions in RISC-V can take a 12-bit immediate. Some of these are
229 // commutative, in others the immediate comes from a specific argument index.
230 bool Takes12BitImm = false;
231 unsigned ImmArgIdx = ~0U;
232
233 switch (Opcode) {
234 case Instruction::GetElementPtr:
235 // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will
236 // split up large offsets in GEP into better parts than ConstantHoisting
237 // can.
238 return TTI::TCC_Free;
239 case Instruction::Store: {
240 // Use the materialization cost regardless of if it's the address or the
241 // value that is constant, except for if the store is misaligned and
242 // misaligned accesses are not legal (experience shows constant hoisting
243 // can sometimes be harmful in such cases).
244 if (Idx == 1 || !Inst)
245 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
246 /*FreeZeroes=*/true);
247
248 StoreInst *ST = cast<StoreInst>(Val: Inst);
249 if (!getTLI()->allowsMemoryAccessForAlignment(
250 Context&: Ty->getContext(), DL, VT: getTLI()->getValueType(DL, Ty),
251 AddrSpace: ST->getPointerAddressSpace(), Alignment: ST->getAlign()))
252 return TTI::TCC_Free;
253
254 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
255 /*FreeZeroes=*/true);
256 }
257 case Instruction::Load:
258 // If the address is a constant, use the materialization cost.
259 return getIntImmCost(Imm, Ty, CostKind);
260 case Instruction::And:
261 // zext.h
262 if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())
263 return TTI::TCC_Free;
264 // zext.w
265 if (Imm == UINT64_C(0xffffffff) && (!ST->is64Bit() || ST->hasStdExtZba()))
266 return TTI::TCC_Free;
267 // bclri
268 if (ST->hasStdExtZbs() && (~Imm).isPowerOf2())
269 return TTI::TCC_Free;
270 if (Inst && Idx == 1 && Imm.getBitWidth() <= ST->getXLen() &&
271 canUseShiftPair(Inst, Imm))
272 return TTI::TCC_Free;
273 if (Inst && Idx == 1 && Imm.getBitWidth() == 64 &&
274 canUseShiftCmp(Inst, Imm))
275 return TTI::TCC_Free;
276 Takes12BitImm = true;
277 break;
278 case Instruction::Add:
279 Takes12BitImm = true;
280 break;
281 case Instruction::Or:
282 case Instruction::Xor:
283 // bseti/binvi
284 if (ST->hasStdExtZbs() && Imm.isPowerOf2())
285 return TTI::TCC_Free;
286 Takes12BitImm = true;
287 break;
288 case Instruction::Mul:
289 // Power of 2 is a shift. Negated power of 2 is a shift and a negate.
290 if (Imm.isPowerOf2() || Imm.isNegatedPowerOf2())
291 return TTI::TCC_Free;
292 // One more or less than a power of 2 can use SLLI+ADD/SUB.
293 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2())
294 return TTI::TCC_Free;
295 // FIXME: There is no MULI instruction.
296 Takes12BitImm = true;
297 break;
298 case Instruction::Sub:
299 case Instruction::Shl:
300 case Instruction::LShr:
301 case Instruction::AShr:
302 Takes12BitImm = true;
303 ImmArgIdx = 1;
304 break;
305 default:
306 break;
307 }
308
309 if (Takes12BitImm) {
310 // Check immediate is the correct argument...
311 if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {
312 // ... and fits into the 12-bit immediate.
313 if (Imm.getSignificantBits() <= 64 &&
314 getTLI()->isLegalAddImmediate(Imm: Imm.getSExtValue())) {
315 return TTI::TCC_Free;
316 }
317 }
318
319 // Otherwise, use the full materialisation cost.
320 return getIntImmCost(Imm, Ty, CostKind);
321 }
322
323 // By default, prevent hoisting.
324 return TTI::TCC_Free;
325}
326
327InstructionCost
328RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
329 const APInt &Imm, Type *Ty,
330 TTI::TargetCostKind CostKind) const {
331 // Prevent hoisting in unknown cases.
332 return TTI::TCC_Free;
333}
334
335bool RISCVTTIImpl::hasActiveVectorLength() const {
336 return ST->hasVInstructions();
337}
338
339TargetTransformInfo::PopcntSupportKind
340RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {
341 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
342 return ST->hasCPOPLike() ? TTI::PSK_FastHardware : TTI::PSK_Software;
343}
344
345InstructionCost RISCVTTIImpl::getPartialReductionCost(
346 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
347 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
348 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
349 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
350 if (Opcode == Instruction::FAdd)
351 return InstructionCost::getInvalid();
352
353 // zve32x is broken for partial_reduce_umla, but let's make sure we
354 // don't generate them.
355 // vdot4a* reduces four i8 products into an i32 result; an i64 accumulator is
356 // additionally supported by widening the i32 partial sums to i64 (see
357 // lowerPARTIAL_REDUCE_MLA). VF is the number of i8 input elements, so the
358 // reduction factor is AccumBits / 8 (4 for i32, 8 for i64).
359 if (!ST->hasStdExtZvdot4a8i() || ST->getELen() < 64 ||
360 Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||
361 InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(BitWidth: 8) ||
362 (!AccumType->isIntegerTy(BitWidth: 32) && !AccumType->isIntegerTy(BitWidth: 64)))
363 return InstructionCost::getInvalid();
364
365 unsigned Ratio = AccumType->getScalarSizeInBits() / 8;
366 if (!VF.isKnownMultipleOf(RHS: Ratio))
367 return InstructionCost::getInvalid();
368
369 // Cost of the vdot4a* itself, which operates on the i32 intermediate type
370 // holding VF/4 elements.
371 Type *DotTp = VectorType::get(ElementType: Type::getInt32Ty(C&: AccumType->getContext()),
372 EC: VF.divideCoefficientBy(RHS: 4));
373 std::pair<InstructionCost, MVT> DotLT = getTypeLegalizationCost(Ty: DotTp);
374 // Note: Asuming all vdot4a* variants are equal cost
375 InstructionCost Cost =
376 DotLT.first *
377 getRISCVInstructionCost(OpCodes: RISCV::VDOT4A_VV, VT: DotLT.second, CostKind);
378
379 // Account for reducing the i32 partial sums down to the i64 accumulator's
380 // element count and accumulating into it (see lowerPARTIAL_REDUCE_MLA), which
381 // has two shapes depending on the accumulator's LMUL.
382 if (AccumType->isIntegerTy(BitWidth: 64)) {
383 LLVMContext &Ctx = AccumType->getContext();
384 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
385 ElementCount AccVF = VF.divideCoefficientBy(RHS: Ratio);
386 std::pair<InstructionCost, MVT> AccLT =
387 getTypeLegalizationCost(Ty: VectorType::get(ElementType: AccumType, EC: AccVF));
388
389 // When the i32 subvectors of a single-vector scalable accumulator are a
390 // fractional LMUL, extracting the high subvector would need a vslidedown,
391 // so instead the i32 dot result is widened to i64 first (vsext.vf2 /
392 // vzext.vf2) and then reduced and accumulated with register-aligned i64
393 // vadd.vv.
394 bool WidenFirst = false;
395 if (VF.isScalable() && AccLT.second.isScalableVector()) {
396 MVT NarrowMVT = AccLT.second.changeVectorElementType(EltVT: MVT::i32);
397 WidenFirst =
398 RISCVVType::decodeVLMUL(VLMul: RISCVTargetLowering::getLMUL(VT: NarrowMVT))
399 .second;
400 }
401
402 if (WidenFirst) {
403 // The widened i64 dot result has VF/4 elements, i.e. twice the
404 // accumulator's element count, so the reduction plus the accumulate are
405 // two i64 vadd.vv.
406 std::pair<InstructionCost, MVT> WideLT = getTypeLegalizationCost(
407 Ty: VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: 4)));
408 Cost +=
409 WideLT.first * getRISCVInstructionCost(OpCodes: RISCV::VSEXT_VF2,
410 VT: WideLT.second, CostKind) +
411 2 * AccLT.first *
412 getRISCVInstructionCost(OpCodes: RISCV::VADD_VV, VT: AccLT.second, CostKind);
413 } else {
414 // Otherwise the scale-4 i32 sums are halved with a single i32 vadd.vv,
415 // then widened and added into the i64 result with a vwadd.wv.
416 std::pair<InstructionCost, MVT> RedLT =
417 getTypeLegalizationCost(Ty: VectorType::get(ElementType: I32Ty, EC: AccVF));
418 Cost += RedLT.first * getRISCVInstructionCost(OpCodes: RISCV::VADD_VV,
419 VT: RedLT.second, CostKind) +
420 AccLT.first * getRISCVInstructionCost(OpCodes: RISCV::VWADD_WV,
421 VT: AccLT.second, CostKind);
422 // Fixed-length vectors extract the high i32 subvector with a vslidedown.
423 if (VF.isFixed())
424 Cost += DotLT.first * getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI,
425 VT: DotLT.second, CostKind);
426 }
427 }
428
429 return Cost;
430}
431
432bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
433 // Currently, the ExpandReductions pass can't expand scalable-vector
434 // reductions, but we still request expansion as RVV doesn't support certain
435 // reductions and the SelectionDAG can't legalize them either.
436 switch (II->getIntrinsicID()) {
437 default:
438 return false;
439 // These reductions have no equivalent in RVV
440 case Intrinsic::vector_reduce_mul:
441 case Intrinsic::vector_reduce_fmul:
442 return true;
443 }
444}
445
446std::optional<unsigned> RISCVTTIImpl::getVScaleForTuning() const {
447 if (ST->hasVInstructions())
448 if (unsigned MinVLen = ST->getRealMinVLen();
449 MinVLen >= RISCV::RVVBitsPerBlock)
450 return MinVLen / RISCV::RVVBitsPerBlock;
451 return BaseT::getVScaleForTuning();
452}
453
454TypeSize
455RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
456 unsigned LMUL =
457 llvm::bit_floor(Value: std::clamp<unsigned>(val: RVVRegisterWidthLMUL, lo: 1, hi: 8));
458 switch (K) {
459 case TargetTransformInfo::RGK_Scalar:
460 return TypeSize::getFixed(ExactSize: ST->getXLen());
461 case TargetTransformInfo::RGK_FixedWidthVector:
462 return TypeSize::getFixed(
463 ExactSize: ST->useRVVForFixedLengthVectors() ? LMUL * ST->getRealMinVLen() : 0);
464 case TargetTransformInfo::RGK_ScalableVector:
465 return TypeSize::getScalable(
466 MinimumSize: (ST->hasVInstructions() &&
467 ST->getRealMinVLen() >= RISCV::RVVBitsPerBlock)
468 ? LMUL * RISCV::RVVBitsPerBlock
469 : 0);
470 }
471
472 llvm_unreachable("Unsupported register kind");
473}
474
475InstructionCost RISCVTTIImpl::getStaticDataAddrGenerationCost(
476 const TTI::TargetCostKind CostKind) const {
477 switch (CostKind) {
478 case TTI::TCK_CodeSize:
479 case TTI::TCK_SizeAndLatency:
480 // Always 2 instructions
481 return 2;
482 case TTI::TCK_Latency:
483 case TTI::TCK_RecipThroughput:
484 // Depending on the memory model the address generation will
485 // require AUIPC + ADDI (medany) or LUI + ADDI (medlow). Don't
486 // have a way of getting this information here, so conservatively
487 // require both.
488 // In practice, these are generally implemented together.
489 return (ST->hasAUIPCADDIFusion() && ST->hasLUIADDIFusion()) ? 1 : 2;
490 }
491 llvm_unreachable("Unsupported cost kind");
492}
493
494InstructionCost
495RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,
496 TTI::TargetCostKind CostKind) const {
497 // Add a cost of address generation + the cost of the load. The address
498 // is expected to be a PC relative offset to a constant pool entry
499 // using auipc/addi.
500 return getStaticDataAddrGenerationCost(CostKind) +
501 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: DL.getABITypeAlign(Ty),
502 /*AddressSpace=*/0, CostKind);
503}
504
505static bool isRepeatedConcatMask(ArrayRef<int> Mask, int &SubVectorSize) {
506 unsigned Size = Mask.size();
507 if (!isPowerOf2_32(Value: Size))
508 return false;
509 for (unsigned I = 0; I != Size; ++I) {
510 if (static_cast<unsigned>(Mask[I]) == I)
511 continue;
512 if (Mask[I] != 0)
513 return false;
514 if (Size % I != 0)
515 return false;
516 for (unsigned J = I + 1; J != Size; ++J)
517 // Check the pattern is repeated.
518 if (static_cast<unsigned>(Mask[J]) != J % I)
519 return false;
520 SubVectorSize = I;
521 return true;
522 }
523 // That means Mask is <0, 1, 2, 3>. This is not a concatenation.
524 return false;
525}
526
527static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,
528 LLVMContext &C) {
529 assert((DataVT.getScalarSizeInBits() != 8 ||
530 DataVT.getVectorNumElements() <= 256) && "unhandled case in lowering");
531 MVT IndexVT = DataVT.changeTypeToInteger();
532 if (IndexVT.getScalarType().bitsGT(VT: ST.getXLenVT()))
533 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
534 return cast<VectorType>(Val: EVT(IndexVT).getTypeForEVT(Context&: C));
535}
536
537/// Attempt to approximate the cost of a shuffle which will require splitting
538/// during legalization. Note that processShuffleMasks is not an exact proxy
539/// for the algorithm used in LegalizeVectorTypes, but hopefully it's a
540/// reasonably close upperbound.
541static InstructionCost costShuffleViaSplitting(const RISCVTTIImpl &TTI,
542 MVT LegalVT, VectorType *Tp,
543 ArrayRef<int> Mask,
544 TTI::TargetCostKind CostKind) {
545 assert(LegalVT.isFixedLengthVector() && !Mask.empty() &&
546 "Expected fixed vector type and non-empty mask");
547 unsigned LegalNumElts = LegalVT.getVectorNumElements();
548 // Number of destination vectors after legalization:
549 unsigned NumOfDests = divideCeil(Numerator: Mask.size(), Denominator: LegalNumElts);
550 // We are going to permute multiple sources and the result will be in
551 // multiple destinations. Providing an accurate cost only for splits where
552 // the element type remains the same.
553 if (NumOfDests <= 1 ||
554 LegalVT.getVectorElementType().getSizeInBits() !=
555 Tp->getElementType()->getPrimitiveSizeInBits() ||
556 LegalNumElts >= Tp->getElementCount().getFixedValue())
557 return InstructionCost::getInvalid();
558
559 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
560 unsigned LegalVTSize = LegalVT.getStoreSize();
561 // Number of source vectors after legalization:
562 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
563
564 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: LegalNumElts);
565
566 unsigned NormalizedVF = LegalNumElts * std::max(a: NumOfSrcs, b: NumOfDests);
567 unsigned NumOfSrcRegs = NormalizedVF / LegalNumElts;
568 unsigned NumOfDestRegs = NormalizedVF / LegalNumElts;
569 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
570 assert(NormalizedVF >= Mask.size() &&
571 "Normalized mask expected to be not shorter than original mask.");
572 copy(Range&: Mask, Out: NormalizedMask.begin());
573 InstructionCost Cost = 0;
574 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
575 processShuffleMasks(
576 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
577 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
578 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
579 return;
580 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
581 .second)
582 return;
583 Cost += TTI.getShuffleCost(
584 Kind: TTI::SK_PermuteSingleSrc,
585 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
586 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
587 },
588 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
589 Cost += TTI.getShuffleCost(
590 Kind: TTI::SK_PermuteTwoSrc,
591 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
592 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
593 });
594 return Cost;
595}
596
597/// Try to perform better estimation of the permutation.
598/// 1. Split the source/destination vectors into real registers.
599/// 2. Do the mask analysis to identify which real registers are
600/// permuted. If more than 1 source registers are used for the
601/// destination register building, the cost for this destination register
602/// is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
603/// source register is used, build mask and calculate the cost as a cost
604/// of PermuteSingleSrc.
605/// Also, for the single register permute we try to identify if the
606/// destination register is just a copy of the source register or the
607/// copy of the previous destination register (the cost is
608/// TTI::TCC_Basic). If the source register is just reused, the cost for
609/// this operation is 0.
610static InstructionCost
611costShuffleViaVRegSplitting(const RISCVTTIImpl &TTI, MVT LegalVT,
612 std::optional<unsigned> VLen, VectorType *Tp,
613 ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {
614 assert(LegalVT.isFixedLengthVector());
615 if (!VLen || Mask.empty())
616 return InstructionCost::getInvalid();
617 MVT ElemVT = LegalVT.getVectorElementType();
618 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
619 LegalVT = TTI.getTypeLegalizationCost(
620 Ty: FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: ElemsPerVReg))
621 .second;
622 // Number of destination vectors after legalization:
623 InstructionCost NumOfDests =
624 divideCeil(Numerator: Mask.size(), Denominator: LegalVT.getVectorNumElements());
625 if (NumOfDests <= 1 ||
626 LegalVT.getVectorElementType().getSizeInBits() !=
627 Tp->getElementType()->getPrimitiveSizeInBits() ||
628 LegalVT.getVectorNumElements() >= Tp->getElementCount().getFixedValue())
629 return InstructionCost::getInvalid();
630
631 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
632 unsigned LegalVTSize = LegalVT.getStoreSize();
633 // Number of source vectors after legalization:
634 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
635
636 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(),
637 NumElts: LegalVT.getVectorNumElements());
638
639 unsigned E = NumOfDests.getValue();
640 unsigned NormalizedVF =
641 LegalVT.getVectorNumElements() * std::max(a: NumOfSrcs, b: E);
642 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
643 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
644 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
645 assert(NormalizedVF >= Mask.size() &&
646 "Normalized mask expected to be not shorter than original mask.");
647 copy(Range&: Mask, Out: NormalizedMask.begin());
648 InstructionCost Cost = 0;
649 int NumShuffles = 0;
650 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
651 processShuffleMasks(
652 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
653 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
654 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
655 return;
656 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
657 .second)
658 return;
659 ++NumShuffles;
660 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleOpTy,
661 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
662 },
663 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
664 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
665 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
666 NumShuffles += 2;
667 });
668 // Note: check that we do not emit too many shuffles here to prevent code
669 // size explosion.
670 // TODO: investigate, if it can be improved by extra analysis of the masks
671 // to check if the code is more profitable.
672 if ((NumOfDestRegs > 2 && NumShuffles <= static_cast<int>(NumOfDestRegs)) ||
673 (NumOfDestRegs <= 2 && NumShuffles < 4))
674 return Cost;
675 return InstructionCost::getInvalid();
676}
677
678InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
679 ArrayRef<int> Mask,
680 TTI::TargetCostKind CostKind) const {
681 // Avoid missing masks and length changing shuffles
682 if (Mask.size() <= 2 || Mask.size() != Tp->getNumElements())
683 return InstructionCost::getInvalid();
684
685 int NumElts = Tp->getNumElements();
686 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Tp);
687 // Avoid scalarization cases
688 if (!LT.second.isFixedLengthVector())
689 return InstructionCost::getInvalid();
690
691 // Requires moving elements between parts, which requires additional
692 // unmodeled instructions.
693 if (LT.first != 1)
694 return InstructionCost::getInvalid();
695
696 auto GetSlideOpcode = [&](int SlideAmt) {
697 assert(SlideAmt != 0);
698 bool IsVI = isUInt<5>(x: std::abs(x: SlideAmt));
699 if (SlideAmt < 0)
700 return IsVI ? RISCV::VSLIDEDOWN_VI : RISCV::VSLIDEDOWN_VX;
701 return IsVI ? RISCV::VSLIDEUP_VI : RISCV::VSLIDEUP_VX;
702 };
703
704 std::array<std::pair<int, int>, 2> SrcInfo;
705 if (!isMaskedSlidePair(Mask, NumElts, SrcInfo))
706 return InstructionCost::getInvalid();
707
708 if (SrcInfo[1].second == 0)
709 std::swap(x&: SrcInfo[0], y&: SrcInfo[1]);
710
711 InstructionCost FirstSlideCost = 0;
712 if (SrcInfo[0].second != 0) {
713 unsigned Opcode = GetSlideOpcode(SrcInfo[0].second);
714 FirstSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
715 }
716
717 if (SrcInfo[1].first == -1)
718 return FirstSlideCost;
719
720 InstructionCost SecondSlideCost = 0;
721 if (SrcInfo[1].second != 0) {
722 unsigned Opcode = GetSlideOpcode(SrcInfo[1].second);
723 SecondSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
724 } else {
725 SecondSlideCost =
726 getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second, CostKind);
727 }
728
729 auto EC = Tp->getElementCount();
730 VectorType *MaskTy =
731 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: Tp->getContext()), EC);
732 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
733 return FirstSlideCost + SecondSlideCost + MaskCost;
734}
735
736InstructionCost
737RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
738 VectorType *SrcTy, TTI::TargetCostKind CostKind,
739 ArrayRef<int> Mask, int Index, VectorType *SubTp,
740 ArrayRef<const Value *> Args,
741 const Instruction *CxtI) const {
742 assert((Mask.empty() || DstTy->isScalableTy() ||
743 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
744 "Expected the Mask to match the return size if given");
745 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
746 "Expected the same scalar types");
747
748 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
749
750 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
751 // For now, skip all fixed vector cost analysis when P extension is available
752 // to avoid crashes in getMinRVVVectorSizeInBits()
753 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: SrcTy))
754 return 1;
755
756 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
757
758 // First, handle cases where having a fixed length vector enables us to
759 // give a more accurate cost than falling back to generic scalable codegen.
760 // TODO: Each of these cases hints at a modeling gap around scalable vectors.
761 if (auto *FVTp = dyn_cast<FixedVectorType>(Val: SrcTy);
762 FVTp && ST->hasVInstructions() && LT.second.isFixedLengthVector()) {
763 InstructionCost VRegSplittingCost = costShuffleViaVRegSplitting(
764 TTI: *this, LegalVT: LT.second, VLen: ST->getRealVLen(),
765 Tp: Kind == TTI::SK_InsertSubvector ? DstTy : SrcTy, Mask, CostKind);
766 if (VRegSplittingCost.isValid())
767 return VRegSplittingCost;
768 switch (Kind) {
769 default:
770 break;
771 case TTI::SK_PermuteSingleSrc: {
772 if (Mask.size() >= 2) {
773 MVT EltTp = LT.second.getVectorElementType();
774 // If the size of the element is < ELEN then shuffles of interleaves and
775 // deinterleaves of 2 vectors can be lowered into the following
776 // sequences
777 if (EltTp.getScalarSizeInBits() < ST->getELen()) {
778 // Example sequence:
779 // vsetivli zero, 4, e8, mf4, ta, ma (ignored)
780 // vwaddu.vv v10, v8, v9
781 // li a0, -1 (ignored)
782 // vwmaccu.vx v10, a0, v9
783 if (ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Mask.size()))
784 return 2 * LT.first * TLI->getLMULCost(VT: LT.second);
785
786 if (Mask[0] == 0 || Mask[0] == 1) {
787 auto DeinterleaveMask = createStrideMask(Start: Mask[0], Stride: 2, VF: Mask.size());
788 // Example sequence:
789 // vnsrl.wi v10, v8, 0
790 if (equal(LRange&: DeinterleaveMask, RRange&: Mask))
791 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VNSRL_WI,
792 VT: LT.second, CostKind);
793 }
794 }
795 int SubVectorSize;
796 if (LT.second.getScalarSizeInBits() != 1 &&
797 isRepeatedConcatMask(Mask, SubVectorSize)) {
798 InstructionCost Cost = 0;
799 unsigned NumSlides = Log2_32(Value: Mask.size() / SubVectorSize);
800 // The cost of extraction from a subvector is 0 if the index is 0.
801 for (unsigned I = 0; I != NumSlides; ++I) {
802 unsigned InsertIndex = SubVectorSize * (1 << I);
803 FixedVectorType *SubTp =
804 FixedVectorType::get(ElementType: SrcTy->getElementType(), NumElts: InsertIndex);
805 FixedVectorType *DestTp =
806 FixedVectorType::getDoubleElementsVectorType(VTy: SubTp);
807 std::pair<InstructionCost, MVT> DestLT =
808 getTypeLegalizationCost(Ty: DestTp);
809 // Add the cost of whole vector register move because the
810 // destination vector register group for vslideup cannot overlap the
811 // source.
812 Cost += DestLT.first * TLI->getLMULCost(VT: DestLT.second);
813 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: DestTp, SrcTy: DestTp,
814 CostKind, Mask: {}, Index: InsertIndex, SubTp);
815 }
816 return Cost;
817 }
818 }
819
820 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
821 SlideCost.isValid())
822 return SlideCost;
823
824 // vrgather + cost of generating the mask constant.
825 // We model this for an unknown mask with a single vrgather.
826 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
827 LT.second.getVectorNumElements() <= 256)) {
828 VectorType *IdxTy =
829 getVRGatherIndexType(DataVT: LT.second, ST: *ST, C&: SrcTy->getContext());
830 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
831 return IndexCost +
832 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VV, VT: LT.second, CostKind);
833 }
834 break;
835 }
836 case TTI::SK_Transpose:
837 case TTI::SK_PermuteTwoSrc: {
838
839 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
840 SlideCost.isValid())
841 return SlideCost;
842
843 // 2 x (vrgather + cost of generating the mask constant) + cost of mask
844 // register for the second vrgather. We model this for an unknown
845 // (shuffle) mask.
846 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
847 LT.second.getVectorNumElements() <= 256)) {
848 auto &C = SrcTy->getContext();
849 auto EC = SrcTy->getElementCount();
850 VectorType *IdxTy = getVRGatherIndexType(DataVT: LT.second, ST: *ST, C);
851 VectorType *MaskTy = VectorType::get(ElementType: IntegerType::getInt1Ty(C), EC);
852 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
853 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
854 return 2 * IndexCost +
855 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV, RISCV::VRGATHER_VV},
856 VT: LT.second, CostKind) +
857 MaskCost;
858 }
859 break;
860 }
861 }
862
863 auto shouldSplit = [](TTI::ShuffleKind Kind) {
864 switch (Kind) {
865 default:
866 return false;
867 case TTI::SK_PermuteSingleSrc:
868 case TTI::SK_Transpose:
869 case TTI::SK_PermuteTwoSrc:
870 return true;
871 }
872 };
873
874 if (!Mask.empty() && LT.first.isValid() && LT.first != 1 &&
875 shouldSplit(Kind)) {
876 InstructionCost SplitCost =
877 costShuffleViaSplitting(TTI: *this, LegalVT: LT.second, Tp: FVTp, Mask, CostKind);
878 if (SplitCost.isValid())
879 return SplitCost;
880 }
881 }
882
883 // Handle scalable vectors (and fixed vectors legalized to scalable vectors).
884 switch (Kind) {
885 default:
886 // Fallthrough to generic handling.
887 // TODO: Most of these cases will return getInvalid in generic code, and
888 // must be implemented here.
889 break;
890 case TTI::SK_ExtractSubvector:
891 // Extract at zero is always a subregister extract
892 if (Index == 0)
893 return TTI::TCC_Free;
894
895 // If we're extracting a subvector of at most m1 size at a sub-register
896 // boundary - which unfortunately we need exact vlen to identify - this is
897 // a subregister extract at worst and thus won't require a vslidedown.
898 // TODO: Extend for aligned m2, m4 subvector extracts
899 // TODO: Extend for misalgined (but contained) extracts
900 // TODO: Extend for scalable subvector types
901 if (std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
902 SubLT.second.isValid() && SubLT.second.isFixedLengthVector()) {
903 if (std::optional<unsigned> VLen = ST->getRealVLen();
904 VLen && SubLT.second.getScalarSizeInBits() * Index % *VLen == 0 &&
905 SubLT.second.getSizeInBits() <= *VLen)
906 return TTI::TCC_Free;
907 }
908
909 // Example sequence:
910 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
911 // vslidedown.vi v8, v9, 2
912 return LT.first *
913 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI, VT: LT.second, CostKind);
914 case TTI::SK_InsertSubvector:
915 // Example sequence:
916 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
917 // vslideup.vi v8, v9, 2
918 LT = getTypeLegalizationCost(Ty: DstTy);
919 return LT.first *
920 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEUP_VI, VT: LT.second, CostKind);
921 case TTI::SK_Select: {
922 // Example sequence:
923 // li a0, 90
924 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
925 // vmv.s.x v0, a0
926 // vmerge.vvm v8, v9, v8, v0
927 // We use 2 for the cost of the mask materialization as this is the true
928 // cost for small masks and most shuffles are small. At worst, this cost
929 // should be a very small constant for the constant pool load. As such,
930 // we may bias towards large selects slightly more than truly warranted.
931 return LT.first *
932 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_S_X, RISCV::VMERGE_VVM},
933 VT: LT.second, CostKind));
934 }
935 case TTI::SK_Broadcast: {
936 // Check for broadcast loads, which are synthesized by optimized zero-stride
937 // loads (this is checked in RISCVTTIImpl::isLegalBroadcastLoad).
938 bool IsLoad = !Args.empty() && isa<LoadInst>(Val: Args[0]);
939 if (IsLoad && LT.second.isVector() &&
940 isLegalBroadcastLoad(ElementTy: SrcTy->getElementType(),
941 NumElements: LT.second.getVectorElementCount()))
942 return 0;
943
944 bool HasScalar = (Args.size() > 0) && (Operator::getOpcode(V: Args[0]) ==
945 Instruction::InsertElement);
946 if (LT.second.getScalarSizeInBits() == 1) {
947 if (HasScalar) {
948 // Example sequence:
949 // andi a0, a0, 1
950 // vsetivli zero, 2, e8, mf8, ta, ma (ignored)
951 // vmv.v.x v8, a0
952 // vmsne.vi v0, v8, 0
953 return LT.first *
954 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
955 VT: LT.second, CostKind));
956 }
957 // Example sequence:
958 // vsetivli zero, 2, e8, mf8, ta, mu (ignored)
959 // vmv.v.i v8, 0
960 // vmerge.vim v8, v8, 1, v0
961 // vmv.x.s a0, v8
962 // andi a0, a0, 1
963 // vmv.v.x v8, a0
964 // vmsne.vi v0, v8, 0
965
966 return LT.first *
967 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_I, RISCV::VMERGE_VIM,
968 RISCV::VMV_X_S, RISCV::VMV_V_X,
969 RISCV::VMSNE_VI},
970 VT: LT.second, CostKind));
971 }
972
973 if (HasScalar) {
974 // Example sequence:
975 // vmv.v.x v8, a0
976 return LT.first *
977 getRISCVInstructionCost(OpCodes: RISCV::VMV_V_X, VT: LT.second, CostKind);
978 }
979
980 // Example sequence:
981 // vrgather.vi v9, v8, 0
982 return LT.first *
983 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VI, VT: LT.second, CostKind);
984 }
985 case TTI::SK_Splice: {
986 // vslidedown+vslideup.
987 // TODO: Multiplying by LT.first implies this legalizes into multiple copies
988 // of similar code, but I think we expand through memory.
989 unsigned Opcodes[2] = {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX};
990 if (Index >= 0 && Index < 32)
991 Opcodes[0] = RISCV::VSLIDEDOWN_VI;
992 else if (Index < 0 && Index > -32)
993 Opcodes[1] = RISCV::VSLIDEUP_VI;
994 return LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
995 }
996 case TTI::SK_Reverse: {
997
998 if (!LT.second.isVector())
999 return InstructionCost::getInvalid();
1000
1001 // TODO: Cases to improve here:
1002 // * Illegal vector types
1003 // * i64 on RV32
1004 if (SrcTy->getElementType()->isIntegerTy(BitWidth: 1)) {
1005 VectorType *WideTy =
1006 VectorType::get(ElementType: IntegerType::get(C&: SrcTy->getContext(), NumBits: 8),
1007 EC: cast<VectorType>(Val: SrcTy)->getElementCount());
1008 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: SrcTy,
1009 CCH: TTI::CastContextHint::None, CostKind) +
1010 getShuffleCost(Kind: TTI::SK_Reverse, DstTy: WideTy, SrcTy: WideTy, CostKind, Mask: {}, Index: 0,
1011 SubTp: nullptr) +
1012 getCastInstrCost(Opcode: Instruction::Trunc, Dst: SrcTy, Src: WideTy,
1013 CCH: TTI::CastContextHint::None, CostKind);
1014 }
1015
1016 MVT ContainerVT = LT.second;
1017 if (LT.second.isFixedLengthVector())
1018 ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1019 MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
1020 if (ContainerVT.bitsLE(VT: M1VT)) {
1021 // Example sequence:
1022 // csrr a0, vlenb
1023 // srli a0, a0, 3
1024 // addi a0, a0, -1
1025 // vsetvli a1, zero, e8, mf8, ta, mu (ignored)
1026 // vid.v v9
1027 // vrsub.vx v10, v9, a0
1028 // vrgather.vv v9, v8, v10
1029 InstructionCost LenCost = 3;
1030 if (LT.second.isFixedLengthVector())
1031 // vrsub.vi has a 5 bit immediate field, otherwise an li suffices
1032 LenCost = isInt<5>(x: LT.second.getVectorNumElements() - 1) ? 0 : 1;
1033 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX, RISCV::VRGATHER_VV};
1034 if (LT.second.isFixedLengthVector() &&
1035 isInt<5>(x: LT.second.getVectorNumElements() - 1))
1036 Opcodes[1] = RISCV::VRSUB_VI;
1037 InstructionCost GatherCost =
1038 getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1039 return LT.first * (LenCost + GatherCost);
1040 }
1041
1042 // At high LMUL, we split into a series of M1 reverses (see
1043 // lowerVECTOR_REVERSE) and then do a single slide at the end to eliminate
1044 // the resulting gap at the bottom (for fixed vectors only). The important
1045 // bit is that the cost scales linearly, not quadratically with LMUL.
1046 unsigned M1Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX};
1047 InstructionCost FixedCost =
1048 getRISCVInstructionCost(OpCodes: M1Opcodes, VT: M1VT, CostKind) + 3;
1049 unsigned Ratio =
1050 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
1051 InstructionCost GatherCost =
1052 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV}, VT: M1VT, CostKind) * Ratio;
1053 InstructionCost SlideCost = !LT.second.isFixedLengthVector() ? 0 :
1054 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX}, VT: LT.second, CostKind);
1055 return FixedCost + LT.first * (GatherCost + SlideCost);
1056 }
1057 }
1058 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1059 SubTp);
1060}
1061
1062static unsigned isM1OrSmaller(MVT VT) {
1063 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1064 return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
1065 LMUL == RISCVVType::VLMUL::LMUL_F4 ||
1066 LMUL == RISCVVType::VLMUL::LMUL_F2 ||
1067 LMUL == RISCVVType::VLMUL::LMUL_1);
1068}
1069
1070InstructionCost RISCVTTIImpl::getScalarizationOverhead(
1071 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
1072 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
1073 TTI::VectorInstrContext VIC) const {
1074 if (isa<ScalableVectorType>(Val: Ty))
1075 return InstructionCost::getInvalid();
1076
1077 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
1078 // For now, skip all fixed vector cost analysis when P extension is available
1079 // to avoid crashes in getMinRVVVectorSizeInBits()
1080 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: Ty)) {
1081 return 1; // Treat as single instruction cost for now
1082 }
1083
1084 // A build_vector (which is m1 sized or smaller) can be done in no
1085 // worse than one vslide1down.vx per element in the type. We could
1086 // in theory do an explode_vector in the inverse manner, but our
1087 // lowering today does not have a first class node for this pattern.
1088 InstructionCost Cost = BaseT::getScalarizationOverhead(
1089 InTy: Ty, DemandedElts, Insert, Extract, CostKind);
1090 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1091 if (Insert && !Extract && LT.first.isValid() && LT.second.isVector()) {
1092 if (Ty->getScalarSizeInBits() == 1) {
1093 auto *WideVecTy = cast<VectorType>(Val: Ty->getWithNewBitWidth(NewBitWidth: 8));
1094 // Note: Implicit scalar anyextend is assumed to be free since the i1
1095 // must be stored in a GPR.
1096 return getScalarizationOverhead(Ty: WideVecTy, DemandedElts, Insert, Extract,
1097 CostKind) +
1098 getCastInstrCost(Opcode: Instruction::Trunc, Dst: Ty, Src: WideVecTy,
1099 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
1100 }
1101
1102 assert(LT.second.isFixedLengthVector());
1103 MVT ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1104 if (isM1OrSmaller(VT: ContainerVT)) {
1105 InstructionCost BV =
1106 cast<FixedVectorType>(Val: Ty)->getNumElements() *
1107 getRISCVInstructionCost(OpCodes: RISCV::VSLIDE1DOWN_VX, VT: LT.second, CostKind);
1108 if (BV < Cost)
1109 Cost = BV;
1110 }
1111 }
1112 return Cost;
1113}
1114
1115InstructionCost
1116RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1117 TTI::TargetCostKind CostKind) const {
1118 Type *DataTy = MICA.getDataType();
1119 Align Alignment = MICA.getAlignment();
1120 switch (MICA.getID()) {
1121 case Intrinsic::vp_load_ff: {
1122 EVT DataTypeVT = TLI->getValueType(DL, Ty: DataTy);
1123 if (!TLI->isLegalFirstFaultLoad(DataType: DataTypeVT, Alignment))
1124 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1125
1126 unsigned AS = MICA.getAddressSpace();
1127 return getMemoryOpCost(Opcode: Instruction::Load, Src: DataTy, Alignment, AddressSpace: AS, CostKind,
1128 OpdInfo: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I: nullptr);
1129 }
1130 case Intrinsic::experimental_vp_strided_load:
1131 case Intrinsic::experimental_vp_strided_store:
1132 return getStridedMemoryOpCost(MICA, CostKind);
1133 case Intrinsic::masked_compressstore:
1134 case Intrinsic::masked_expandload:
1135 return getExpandCompressMemoryOpCost(MICA, CostKind);
1136 case Intrinsic::vp_scatter:
1137 case Intrinsic::vp_gather:
1138 case Intrinsic::masked_scatter:
1139 case Intrinsic::masked_gather:
1140 return getGatherScatterOpCost(MICA, CostKind);
1141 case Intrinsic::vp_load:
1142 case Intrinsic::vp_store:
1143 case Intrinsic::masked_load:
1144 case Intrinsic::masked_store:
1145 return getMaskedMemoryOpCost(MICA, CostKind);
1146 }
1147 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1148}
1149
1150InstructionCost
1151RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1152 TTI::TargetCostKind CostKind) const {
1153 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
1154 : Instruction::Store;
1155 Type *Src = MICA.getDataType();
1156 Align Alignment = MICA.getAlignment();
1157 unsigned AddressSpace = MICA.getAddressSpace();
1158
1159 if (!isLegalMaskedLoadStore(DataType: Src, Alignment) ||
1160 CostKind != TTI::TCK_RecipThroughput)
1161 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1162
1163 // Splitting involves additional evl arithmetic and vl toggles.
1164 InstructionCost SplitCost = 0;
1165 if (MICA.getID() == Intrinsic::vp_load ||
1166 MICA.getID() == Intrinsic::vp_store) {
1167 auto LT = getTypeLegalizationCost(Ty: Src);
1168 if (LT.first > 1)
1169 SplitCost += LT.first * TTI::TCC_Expensive;
1170 }
1171
1172 return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
1173}
1174
1175InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
1176 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1177 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1178 bool UseMaskForCond, bool UseMaskForGaps) const {
1179
1180 // The interleaved memory access pass will lower (de)interleave ops combined
1181 // with an adjacent appropriate memory to vlseg/vsseg intrinsics. vlseg/vsseg
1182 // only support masking per-iteration (i.e. condition), not per-segment (i.e.
1183 // gap).
1184 if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {
1185 auto *VTy = cast<VectorType>(Val: VecTy);
1186 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VTy);
1187 // Need to make sure type has't been scalarized
1188 if (LT.second.isVector()) {
1189 if (CostKind == TTI::TCK_CodeSize)
1190 return LT.first * TTI::TCC_Basic;
1191
1192 auto *SubVecTy =
1193 VectorType::get(ElementType: VTy->getElementType(),
1194 EC: VTy->getElementCount().divideCoefficientBy(RHS: Factor));
1195 if (VTy->getElementCount().isKnownMultipleOf(RHS: Factor) &&
1196 TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor, Alignment,
1197 AddrSpace: AddressSpace, DL)) {
1198
1199 // Some processors optimize segment loads/stores as N * DLEN sized
1200 // load ops + Factor * LMUL shuffle ops.
1201 if (ST->hasOptimizedSegmentLoadStore(NF: Factor)) {
1202 unsigned VecSizeInBits =
1203 getEstimatedVLFor(Ty: VTy) * VTy->getScalarSizeInBits();
1204 unsigned VLENForTuning =
1205 *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
1206 unsigned DLENForTuning = VLENForTuning / ST->getDLenFactor();
1207 InstructionCost Cost = divideCeil(Numerator: VecSizeInBits, Denominator: DLENForTuning);
1208 MVT SubVecVT = getTLI()->getValueType(DL, Ty: SubVecTy).getSimpleVT();
1209 Cost += Factor * TLI->getLMULCost(VT: SubVecVT);
1210 return Cost;
1211 }
1212
1213 // Otherwise, the cost is proportional to the number of elements (VL *
1214 // Factor ops).
1215 unsigned NumLoads = getEstimatedVLFor(Ty: VTy);
1216 return NumLoads * TTI::TCC_Basic;
1217 }
1218 }
1219 }
1220
1221 // TODO: Return the cost of interleaved accesses for scalable vector when
1222 // unable to convert to segment accesses instructions.
1223 if (isa<ScalableVectorType>(Val: VecTy))
1224 return InstructionCost::getInvalid();
1225
1226 auto *FVTy = cast<FixedVectorType>(Val: VecTy);
1227 // When gaps are only at the tail, for interleaved load, we can emit a wide
1228 // masked load and shufflevectors. For interleaved store, we can emit
1229 // shufflevectors and a wide masked store. The interleaved memory access pass
1230 // will lower them into vlsseg/vssseg intrinsics.
1231 if (UseMaskForGaps) {
1232 assert(llvm::is_sorted(Indices) && "Indices must be sorted");
1233 assert(llvm::adjacent_find(Indices) == Indices.end() &&
1234 "Indices should not contain duplicate elements");
1235 unsigned NumOfFields = Indices.size();
1236 bool IsTailGapOnly = NumOfFields > 1 && (NumOfFields == Indices.back() + 1);
1237 if (IsTailGapOnly &&
1238 NumOfFields <= TLI->getMaxSupportedInterleaveFactor()) {
1239 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: FVTy);
1240 if (LT.second.isVector() &&
1241 FVTy->getElementCount().isKnownMultipleOf(RHS: Factor)) {
1242 auto *SubVecTy = VectorType::get(
1243 ElementType: FVTy->getElementType(),
1244 EC: FVTy->getElementCount().divideCoefficientBy(RHS: Factor));
1245 if (TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor: NumOfFields, Alignment,
1246 AddrSpace: AddressSpace, DL)) {
1247 // The cost is proportional to the total number of element accesses.
1248 unsigned NumAccesses = getEstimatedVLFor(Ty: FVTy);
1249 return NumAccesses * TTI::TCC_Basic;
1250 }
1251 }
1252 }
1253 }
1254
1255 InstructionCost MemCost =
1256 getMemoryOpCost(Opcode, Src: VecTy, Alignment, AddressSpace, CostKind);
1257 unsigned VF = FVTy->getNumElements() / Factor;
1258
1259 // An interleaved load will look like this for Factor=3:
1260 // %wide.vec = load <12 x i32>, ptr %3, align 4
1261 // %strided.vec = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1262 // %strided.vec1 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1263 // %strided.vec2 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1264 if (Opcode == Instruction::Load) {
1265 InstructionCost Cost = MemCost;
1266 for (unsigned Index : Indices) {
1267 FixedVectorType *VecTy =
1268 FixedVectorType::get(ElementType: FVTy->getElementType(), NumElts: VF * Factor);
1269 auto Mask = createStrideMask(Start: Index, Stride: Factor, VF);
1270 Mask.resize(N: VF * Factor, NV: -1);
1271 InstructionCost ShuffleCost =
1272 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: VecTy, SrcTy: VecTy,
1273 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {});
1274 Cost += ShuffleCost;
1275 }
1276 return Cost;
1277 }
1278
1279 // TODO: Model for NF > 2
1280 // We'll need to enhance getShuffleCost to model shuffles that are just
1281 // inserts and extracts into subvectors, since they won't have the full cost
1282 // of a vrgather.
1283 // An interleaved store for 3 vectors of 4 lanes will look like
1284 // %11 = shufflevector <4 x i32> %4, <4 x i32> %6, <8 x i32> <0...7>
1285 // %12 = shufflevector <4 x i32> %9, <4 x i32> poison, <8 x i32> <0...3>
1286 // %13 = shufflevector <8 x i32> %11, <8 x i32> %12, <12 x i32> <0...11>
1287 // %interleaved.vec = shufflevector %13, poison, <12 x i32> <interleave mask>
1288 // store <12 x i32> %interleaved.vec, ptr %10, align 4
1289 if (Factor != 2)
1290 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1291 Alignment, AddressSpace, CostKind,
1292 UseMaskForCond, UseMaskForGaps);
1293
1294 assert(Opcode == Instruction::Store && "Opcode must be a store");
1295 // For an interleaving store of 2 vectors, we perform one large interleaving
1296 // shuffle that goes into the wide store
1297 auto Mask = createInterleaveMask(VF, NumVecs: Factor);
1298 InstructionCost ShuffleCost =
1299 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: FVTy, SrcTy: FVTy,
1300 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {});
1301 return MemCost + ShuffleCost;
1302}
1303
1304InstructionCost
1305RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1306 TTI::TargetCostKind CostKind) const {
1307
1308 bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
1309 MICA.getID() == Intrinsic::vp_gather;
1310 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
1311 Type *DataTy = MICA.getDataType();
1312 Type *PtrTy = DataTy->getWithNewType(
1313 EltTy: DL.getAddressType(C&: DataTy->getContext(), AddressSpace: MICA.getAddressSpace()));
1314 Align Alignment = MICA.getAlignment();
1315 if (CostKind != TTI::TCK_RecipThroughput)
1316 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1317
1318 if ((Opcode == Instruction::Load &&
1319 !isLegalMaskedGather(DataType: DataTy, Alignment: Align(Alignment))) ||
1320 (Opcode == Instruction::Store &&
1321 !isLegalMaskedScatter(DataType: DataTy, Alignment: Align(Alignment))))
1322 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1323
1324 // Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
1325 InstructionCost SplitCost = 0;
1326 if (MICA.getID() == Intrinsic::vp_gather ||
1327 MICA.getID() == Intrinsic::vp_scatter) {
1328 auto DataLT = getTypeLegalizationCost(Ty: DataTy);
1329 auto PtrLT = getTypeLegalizationCost(Ty: PtrTy);
1330 if (DataLT.first > 1)
1331 SplitCost += DataLT.first * TTI::TCC_Expensive;
1332 if (PtrLT.first > 1)
1333 SplitCost += PtrLT.first * TTI::TCC_Expensive;
1334 }
1335
1336 // Cost is proportional to the number of memory operations implied. For
1337 // scalable vectors, we use an estimate on that number since we don't
1338 // know exactly what VL will be.
1339 auto &VTy = *cast<VectorType>(Val: DataTy);
1340 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1341 return SplitCost + NumLoads * TTI::TCC_Basic;
1342}
1343
1344InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
1345 const MemIntrinsicCostAttributes &MICA,
1346 TTI::TargetCostKind CostKind) const {
1347 unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
1348 ? Instruction::Load
1349 : Instruction::Store;
1350 Type *DataTy = MICA.getDataType();
1351 bool VariableMask = MICA.getVariableMask();
1352 Align Alignment = MICA.getAlignment();
1353 bool IsLegal = (Opcode == Instruction::Store &&
1354 isLegalMaskedCompressStore(DataTy, Alignment)) ||
1355 (Opcode == Instruction::Load &&
1356 isLegalMaskedExpandLoad(DataType: DataTy, Alignment));
1357 if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
1358 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1359 // Example compressstore sequence:
1360 // vsetivli zero, 8, e32, m2, ta, ma (ignored)
1361 // vcompress.vm v10, v8, v0
1362 // vcpop.m a1, v0
1363 // vsetvli zero, a1, e32, m2, ta, ma
1364 // vse32.v v10, (a0)
1365 // Example expandload sequence:
1366 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
1367 // vcpop.m a1, v0
1368 // vsetvli zero, a1, e32, m2, ta, ma
1369 // vle32.v v10, (a0)
1370 // vsetivli zero, 8, e32, m2, ta, ma
1371 // viota.m v12, v0
1372 // vrgather.vv v8, v10, v12, v0.t
1373 auto MemOpCost =
1374 getMemoryOpCost(Opcode, Src: DataTy, Alignment, /*AddressSpace*/ 0, CostKind);
1375 auto LT = getTypeLegalizationCost(Ty: DataTy);
1376 SmallVector<unsigned, 4> Opcodes{RISCV::VSETVLI};
1377 if (VariableMask)
1378 Opcodes.push_back(Elt: RISCV::VCPOP_M);
1379 if (Opcode == Instruction::Store)
1380 Opcodes.append(IL: {RISCV::VCOMPRESS_VM});
1381 else
1382 Opcodes.append(IL: {RISCV::VSETIVLI, RISCV::VIOTA_M, RISCV::VRGATHER_VV});
1383 return MemOpCost +
1384 LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1385}
1386
1387InstructionCost
1388RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1389 TTI::TargetCostKind CostKind) const {
1390 Type *DataTy = MICA.getDataType();
1391 Align Alignment = MICA.getAlignment();
1392
1393 if (!isLegalStridedLoadStore(DataType: DataTy, Alignment))
1394 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1395
1396 if (CostKind == TTI::TCK_CodeSize)
1397 return TTI::TCC_Basic;
1398
1399 // Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
1400 InstructionCost SplitCost = 0;
1401 auto LT = getTypeLegalizationCost(Ty: DataTy);
1402 if (LT.first > 1)
1403 SplitCost += LT.first * TTI::TCC_Expensive;
1404
1405 // Cost is proportional to the number of memory operations implied. For
1406 // scalable vectors, we use an estimate on that number since we don't
1407 // know exactly what VL will be.
1408 auto &VTy = *cast<VectorType>(Val: DataTy);
1409 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1410 return SplitCost + NumLoads * TTI::TCC_Basic;
1411}
1412
1413InstructionCost
1414RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1415 // FIXME: This is a property of the default vector convention, not
1416 // all possible calling conventions. Fixing that will require
1417 // some TTI API and SLP rework.
1418 InstructionCost Cost = 0;
1419 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
1420 for (auto *Ty : Tys) {
1421 if (!Ty->isVectorTy())
1422 continue;
1423 Align A = DL.getPrefTypeAlign(Ty);
1424 Cost += getMemoryOpCost(Opcode: Instruction::Store, Src: Ty, Alignment: A, AddressSpace: 0, CostKind) +
1425 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: A, AddressSpace: 0, CostKind);
1426 }
1427 return Cost;
1428}
1429
1430// Currently, these represent both throughput and codesize costs
1431// for the respective intrinsics. The costs in this table are simply
1432// instruction counts with the following adjustments made:
1433// * One vsetvli is considered free.
1434static const CostTblEntry VectorIntrinsicCostTable[]{
1435 {.ISD: Intrinsic::floor, .Type: MVT::f32, .Cost: 9},
1436 {.ISD: Intrinsic::floor, .Type: MVT::f64, .Cost: 9},
1437 {.ISD: Intrinsic::ceil, .Type: MVT::f32, .Cost: 9},
1438 {.ISD: Intrinsic::ceil, .Type: MVT::f64, .Cost: 9},
1439 {.ISD: Intrinsic::trunc, .Type: MVT::f32, .Cost: 7},
1440 {.ISD: Intrinsic::trunc, .Type: MVT::f64, .Cost: 7},
1441 {.ISD: Intrinsic::round, .Type: MVT::f32, .Cost: 9},
1442 {.ISD: Intrinsic::round, .Type: MVT::f64, .Cost: 9},
1443 {.ISD: Intrinsic::roundeven, .Type: MVT::f32, .Cost: 9},
1444 {.ISD: Intrinsic::roundeven, .Type: MVT::f64, .Cost: 9},
1445 {.ISD: Intrinsic::rint, .Type: MVT::f32, .Cost: 7},
1446 {.ISD: Intrinsic::rint, .Type: MVT::f64, .Cost: 7},
1447 {.ISD: Intrinsic::nearbyint, .Type: MVT::f32, .Cost: 9},
1448 {.ISD: Intrinsic::nearbyint, .Type: MVT::f64, .Cost: 9},
1449 {.ISD: Intrinsic::bswap, .Type: MVT::i16, .Cost: 3},
1450 {.ISD: Intrinsic::bswap, .Type: MVT::i32, .Cost: 12},
1451 {.ISD: Intrinsic::bswap, .Type: MVT::i64, .Cost: 31},
1452 {.ISD: Intrinsic::bitreverse, .Type: MVT::i8, .Cost: 17},
1453 {.ISD: Intrinsic::bitreverse, .Type: MVT::i16, .Cost: 24},
1454 {.ISD: Intrinsic::bitreverse, .Type: MVT::i32, .Cost: 33},
1455 {.ISD: Intrinsic::bitreverse, .Type: MVT::i64, .Cost: 52},
1456 {.ISD: Intrinsic::ctpop, .Type: MVT::i8, .Cost: 12},
1457 {.ISD: Intrinsic::ctpop, .Type: MVT::i16, .Cost: 19},
1458 {.ISD: Intrinsic::ctpop, .Type: MVT::i32, .Cost: 20},
1459 {.ISD: Intrinsic::ctpop, .Type: MVT::i64, .Cost: 21},
1460 {.ISD: Intrinsic::ctlz, .Type: MVT::i8, .Cost: 19},
1461 {.ISD: Intrinsic::ctlz, .Type: MVT::i16, .Cost: 28},
1462 {.ISD: Intrinsic::ctlz, .Type: MVT::i32, .Cost: 31},
1463 {.ISD: Intrinsic::ctlz, .Type: MVT::i64, .Cost: 35},
1464 {.ISD: Intrinsic::cttz, .Type: MVT::i8, .Cost: 16},
1465 {.ISD: Intrinsic::cttz, .Type: MVT::i16, .Cost: 23},
1466 {.ISD: Intrinsic::cttz, .Type: MVT::i32, .Cost: 24},
1467 {.ISD: Intrinsic::cttz, .Type: MVT::i64, .Cost: 25},
1468};
1469
1470InstructionCost
1471RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1472 TTI::TargetCostKind CostKind) const {
1473 auto *RetTy = ICA.getReturnType();
1474 switch (ICA.getID()) {
1475 case Intrinsic::lrint:
1476 case Intrinsic::llrint:
1477 case Intrinsic::lround:
1478 case Intrinsic::llround: {
1479 auto LT = getTypeLegalizationCost(Ty: RetTy);
1480 Type *SrcTy = ICA.getArgTypes().front();
1481 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
1482 if (ST->hasVInstructions() && LT.second.isVector()) {
1483 SmallVector<unsigned, 2> Ops;
1484 unsigned SrcEltSz = DL.getTypeSizeInBits(Ty: SrcTy->getScalarType());
1485 unsigned DstEltSz = DL.getTypeSizeInBits(Ty: RetTy->getScalarType());
1486 if (LT.second.getVectorElementType() == MVT::bf16) {
1487 if (!ST->hasVInstructionsBF16Minimal())
1488 return InstructionCost::getInvalid();
1489 if (DstEltSz == 32)
1490 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFCVT_X_F_V};
1491 else
1492 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVT_X_F_V};
1493 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1494 !ST->hasVInstructionsF16()) {
1495 if (!ST->hasVInstructionsF16Minimal())
1496 return InstructionCost::getInvalid();
1497 if (DstEltSz == 32)
1498 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFCVT_X_F_V};
1499 else
1500 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_X_F_V};
1501
1502 } else if (SrcEltSz > DstEltSz) {
1503 Ops = {RISCV::VFNCVT_X_F_W};
1504 } else if (SrcEltSz < DstEltSz) {
1505 Ops = {RISCV::VFWCVT_X_F_V};
1506 } else {
1507 Ops = {RISCV::VFCVT_X_F_V};
1508 }
1509
1510 // We need to use the source LMUL in the case of a narrowing op, and the
1511 // destination LMUL otherwise.
1512 if (SrcEltSz > DstEltSz)
1513 return SrcLT.first *
1514 getRISCVInstructionCost(OpCodes: Ops, VT: SrcLT.second, CostKind);
1515 return LT.first * getRISCVInstructionCost(OpCodes: Ops, VT: LT.second, CostKind);
1516 }
1517 break;
1518 }
1519 case Intrinsic::ceil:
1520 case Intrinsic::floor:
1521 case Intrinsic::trunc:
1522 case Intrinsic::rint:
1523 case Intrinsic::round:
1524 case Intrinsic::roundeven: {
1525 // These all use the same code.
1526 auto LT = getTypeLegalizationCost(Ty: RetTy);
1527 if (!LT.second.isVector() && TLI->isOperationCustom(Op: ISD::FCEIL, VT: LT.second))
1528 return LT.first * 8;
1529 break;
1530 }
1531 case Intrinsic::umin:
1532 case Intrinsic::umax:
1533 case Intrinsic::smin:
1534 case Intrinsic::smax: {
1535 auto LT = getTypeLegalizationCost(Ty: RetTy);
1536 if (LT.second.isScalarInteger() && ST->hasStdExtZbb())
1537 return LT.first;
1538
1539 if (ST->hasVInstructions() && LT.second.isVector()) {
1540 unsigned Op;
1541 switch (ICA.getID()) {
1542 case Intrinsic::umin:
1543 Op = RISCV::VMINU_VV;
1544 break;
1545 case Intrinsic::umax:
1546 Op = RISCV::VMAXU_VV;
1547 break;
1548 case Intrinsic::smin:
1549 Op = RISCV::VMIN_VV;
1550 break;
1551 case Intrinsic::smax:
1552 Op = RISCV::VMAX_VV;
1553 break;
1554 }
1555 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1556 }
1557 break;
1558 }
1559 case Intrinsic::sadd_sat:
1560 case Intrinsic::ssub_sat:
1561 case Intrinsic::uadd_sat:
1562 case Intrinsic::usub_sat: {
1563 auto LT = getTypeLegalizationCost(Ty: RetTy);
1564 if (ST->hasVInstructions() && LT.second.isVector()) {
1565 unsigned Op;
1566 switch (ICA.getID()) {
1567 case Intrinsic::sadd_sat:
1568 Op = RISCV::VSADD_VV;
1569 break;
1570 case Intrinsic::ssub_sat:
1571 Op = RISCV::VSSUB_VV;
1572 break;
1573 case Intrinsic::uadd_sat:
1574 Op = RISCV::VSADDU_VV;
1575 break;
1576 case Intrinsic::usub_sat:
1577 Op = RISCV::VSSUBU_VV;
1578 break;
1579 }
1580 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1581 }
1582 break;
1583 }
1584 case Intrinsic::fma:
1585 case Intrinsic::fmuladd: {
1586 // TODO: handle promotion with f16/bf16 with zvfhmin/zvfbfmin
1587 auto LT = getTypeLegalizationCost(Ty: RetTy);
1588 if (ST->hasVInstructions() && LT.second.isVector())
1589 return LT.first *
1590 getRISCVInstructionCost(OpCodes: RISCV::VFMADD_VV, VT: LT.second, CostKind);
1591 break;
1592 }
1593 case Intrinsic::fabs: {
1594 auto LT = getTypeLegalizationCost(Ty: RetTy);
1595 if (ST->hasVInstructions() && LT.second.isVector()) {
1596 // lui a0, 8
1597 // addi a0, a0, -1
1598 // vsetvli a1, zero, e16, m1, ta, ma
1599 // vand.vx v8, v8, a0
1600 // f16 with zvfhmin and bf16 with zvfhbmin
1601 if (LT.second.getVectorElementType() == MVT::bf16 ||
1602 (LT.second.getVectorElementType() == MVT::f16 &&
1603 !ST->hasVInstructionsF16()))
1604 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VAND_VX, VT: LT.second,
1605 CostKind) +
1606 2;
1607 else
1608 return LT.first *
1609 getRISCVInstructionCost(OpCodes: RISCV::VFSGNJX_VV, VT: LT.second, CostKind);
1610 }
1611 break;
1612 }
1613 case Intrinsic::sqrt: {
1614 auto LT = getTypeLegalizationCost(Ty: RetTy);
1615 if (ST->hasVInstructions() && LT.second.isVector()) {
1616 SmallVector<unsigned, 4> ConvOp;
1617 SmallVector<unsigned, 2> FsqrtOp;
1618 MVT ConvType = LT.second;
1619 MVT FsqrtType = LT.second;
1620 // f16 with zvfhmin and bf16 with zvfbfmin and the type of nxv32[b]f16
1621 // will be spilt.
1622 if (LT.second.getVectorElementType() == MVT::bf16) {
1623 if (LT.second == MVT::nxv32bf16) {
1624 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVTBF16_F_F_V,
1625 RISCV::VFNCVTBF16_F_F_W, RISCV::VFNCVTBF16_F_F_W};
1626 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1627 ConvType = MVT::nxv16f16;
1628 FsqrtType = MVT::nxv16f32;
1629 } else {
1630 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFNCVTBF16_F_F_W};
1631 FsqrtOp = {RISCV::VFSQRT_V};
1632 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1633 }
1634 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1635 !ST->hasVInstructionsF16()) {
1636 if (LT.second == MVT::nxv32f16) {
1637 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_F_F_V,
1638 RISCV::VFNCVT_F_F_W, RISCV::VFNCVT_F_F_W};
1639 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1640 ConvType = MVT::nxv16f16;
1641 FsqrtType = MVT::nxv16f32;
1642 } else {
1643 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFNCVT_F_F_W};
1644 FsqrtOp = {RISCV::VFSQRT_V};
1645 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1646 }
1647 } else {
1648 FsqrtOp = {RISCV::VFSQRT_V};
1649 }
1650
1651 return LT.first * (getRISCVInstructionCost(OpCodes: FsqrtOp, VT: FsqrtType, CostKind) +
1652 getRISCVInstructionCost(OpCodes: ConvOp, VT: ConvType, CostKind));
1653 }
1654 break;
1655 }
1656 case Intrinsic::cttz:
1657 case Intrinsic::ctlz:
1658 case Intrinsic::ctpop: {
1659 auto LT = getTypeLegalizationCost(Ty: RetTy);
1660 if (ST->hasStdExtZvbb() && LT.second.isVector()) {
1661 unsigned Op;
1662 switch (ICA.getID()) {
1663 case Intrinsic::cttz:
1664 Op = RISCV::VCTZ_V;
1665 break;
1666 case Intrinsic::ctlz:
1667 Op = RISCV::VCLZ_V;
1668 break;
1669 case Intrinsic::ctpop:
1670 Op = RISCV::VCPOP_V;
1671 break;
1672 }
1673 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1674 }
1675 break;
1676 }
1677 case Intrinsic::abs: {
1678 auto LT = getTypeLegalizationCost(Ty: RetTy);
1679 if (ST->hasVInstructions() && LT.second.isVector()) {
1680 // vabs.v v10, v8 (alias for vabd.vx v10, v8, zero)
1681 if (ST->hasStdExtZvabd())
1682 return LT.first *
1683 getRISCVInstructionCost(OpCodes: {RISCV::VABD_VX}, VT: LT.second, CostKind);
1684
1685 // vrsub.vi v10, v8, 0
1686 // vmax.vv v8, v8, v10
1687 return LT.first *
1688 getRISCVInstructionCost(OpCodes: {RISCV::VRSUB_VI, RISCV::VMAX_VV},
1689 VT: LT.second, CostKind);
1690 }
1691 break;
1692 }
1693 case Intrinsic::fshl:
1694 case Intrinsic::fshr: {
1695 if (ICA.getArgs().empty())
1696 break;
1697
1698 // Funnel-shifts are ROTL/ROTR when the first and second operand are equal.
1699 // When Zbb/Zbkb is enabled we can use a single ROL(W)/ROR(I)(W)
1700 // instruction.
1701 if ((ST->hasStdExtZbb() || ST->hasStdExtZbkb()) && RetTy->isIntegerTy() &&
1702 ICA.getArgs()[0] == ICA.getArgs()[1] &&
1703 (RetTy->getIntegerBitWidth() == 32 ||
1704 RetTy->getIntegerBitWidth() == 64) &&
1705 RetTy->getIntegerBitWidth() <= ST->getXLen()) {
1706 return 1;
1707 }
1708 break;
1709 }
1710 case Intrinsic::clmul: {
1711 auto LT = getTypeLegalizationCost(Ty: RetTy);
1712 if (!LT.second.isVector() && ST->hasStdExtZvbc() && !ST->hasStdExtZbc() &&
1713 !ST->hasStdExtZbkc()) {
1714 // TODO: Once custom lowering in this case for RV32 is added, this guard
1715 // should be removed and the cost model should be updated.
1716 if (!ST->is64Bit() || LT.second != MVT::i64)
1717 break;
1718 // vmv.s.x v8, a0
1719 // vclmul.vx v8, v8, a1
1720 // vmv.x.s a0, v8
1721 MVT VecVT = MVT::getScalableVectorVT(VT: LT.second, NumElements: 1);
1722 return LT.first * getRISCVInstructionCost(
1723 OpCodes: {RISCV::VMV_S_X, RISCV::VCLMUL_VX, RISCV::VMV_X_S},
1724 VT: VecVT, CostKind);
1725 }
1726 break;
1727 }
1728 case Intrinsic::masked_udiv:
1729 return getArithmeticInstrCost(Opcode: Instruction::UDiv, Ty: ICA.getReturnType(),
1730 CostKind);
1731 case Intrinsic::masked_sdiv:
1732 return getArithmeticInstrCost(Opcode: Instruction::SDiv, Ty: ICA.getReturnType(),
1733 CostKind);
1734 case Intrinsic::masked_urem:
1735 return getArithmeticInstrCost(Opcode: Instruction::URem, Ty: ICA.getReturnType(),
1736 CostKind);
1737 case Intrinsic::masked_srem:
1738 return getArithmeticInstrCost(Opcode: Instruction::SRem, Ty: ICA.getReturnType(),
1739 CostKind);
1740 case Intrinsic::get_active_lane_mask: {
1741 if (ST->hasVInstructions()) {
1742 Type *ExpRetTy = VectorType::get(
1743 ElementType: ICA.getArgTypes()[0], EC: cast<VectorType>(Val: RetTy)->getElementCount());
1744 auto LT = getTypeLegalizationCost(Ty: ExpRetTy);
1745
1746 // vid.v v8 // considered hoisted
1747 // vsaddu.vx v8, v8, a0
1748 // vmsltu.vx v0, v8, a1
1749 return LT.first *
1750 getRISCVInstructionCost(OpCodes: {RISCV::VSADDU_VX, RISCV::VMSLTU_VX},
1751 VT: LT.second, CostKind);
1752 }
1753 break;
1754 }
1755 // TODO: add more intrinsic
1756 case Intrinsic::stepvector: {
1757 auto LT = getTypeLegalizationCost(Ty: RetTy);
1758 // Legalisation of illegal types involves an `index' instruction plus
1759 // (LT.first - 1) vector adds.
1760 if (ST->hasVInstructions())
1761 return getRISCVInstructionCost(OpCodes: RISCV::VID_V, VT: LT.second, CostKind) +
1762 (LT.first - 1) *
1763 getRISCVInstructionCost(OpCodes: RISCV::VADD_VX, VT: LT.second, CostKind);
1764 return 1 + (LT.first - 1);
1765 }
1766 case Intrinsic::vector_splice_left:
1767 case Intrinsic::vector_splice_right: {
1768 auto LT = getTypeLegalizationCost(Ty: RetTy);
1769 // Constant offsets fall through to getShuffleCost.
1770 if (!ICA.isTypeBasedOnly() && isa<ConstantInt>(Val: ICA.getArgs()[2]))
1771 break;
1772 if (ST->hasVInstructions() && LT.second.isVector()) {
1773 return LT.first *
1774 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX},
1775 VT: LT.second, CostKind);
1776 }
1777 break;
1778 }
1779 case Intrinsic::experimental_cttz_elts: {
1780 if (!ST->hasVInstructions())
1781 break;
1782 InstructionCost Cost = 0;
1783 Type *ArgTy = ICA.getArgTypes()[0];
1784 auto LT = getTypeLegalizationCost(Ty: ArgTy);
1785
1786 // If the element type is not i1, do a comparison with all-zeros.
1787 if (LT.second.getVectorElementType() != MVT::i1)
1788 Cost += getRISCVInstructionCost(OpCodes: RISCV::VMSNE_VI, VT: LT.second, CostKind);
1789
1790 Cost += getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind);
1791
1792 // If zero_is_poison is false, then we will generate additional
1793 // cmp + select instructions to convert -1 to EVL.
1794 Type *BoolTy = Type::getInt1Ty(C&: RetTy->getContext());
1795 if (ICA.getArgs().size() > 1 &&
1796 cast<ConstantInt>(Val: ICA.getArgs()[1])->isZero())
1797 Cost += getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: BoolTy, CondTy: RetTy,
1798 VecPred: CmpInst::ICMP_SLT, CostKind) +
1799 getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: RetTy, CondTy: BoolTy,
1800 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1801
1802 return LT.first * Cost;
1803 }
1804 case Intrinsic::experimental_vp_splice: {
1805 // To support type-based query from vectorizer, set the index to 0.
1806 // Note that index only change the cost from vslide.vx to vslide.vi and in
1807 // current implementations they have same costs.
1808 return getShuffleCost(Kind: TTI::SK_Splice, DstTy: cast<VectorType>(Val: ICA.getReturnType()),
1809 SrcTy: cast<VectorType>(Val: ICA.getArgTypes()[0]), CostKind, Mask: {},
1810 Index: 0, SubTp: cast<VectorType>(Val: ICA.getReturnType()));
1811 }
1812 case Intrinsic::vp_merge: {
1813 // If an operand is a binary op and the type is legal, RISCVVectorPeephole
1814 // will likely fold the resulting vmerge.vvm away.
1815 if (ICA.getVectorInstrContext() == VectorInstrContext::BinaryOp &&
1816 getTypeLegalizationCost(Ty: RetTy).first == 1)
1817 return TTI::TCC_Free;
1818 break;
1819 }
1820 case Intrinsic::fptoui_sat:
1821 case Intrinsic::fptosi_sat: {
1822 InstructionCost Cost = 0;
1823 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
1824 Type *SrcTy = ICA.getArgTypes()[0];
1825
1826 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
1827 auto DstLT = getTypeLegalizationCost(Ty: RetTy);
1828 if (!SrcTy->isVectorTy())
1829 break;
1830
1831 if (!SrcLT.first.isValid() || !DstLT.first.isValid())
1832 return InstructionCost::getInvalid();
1833
1834 Cost +=
1835 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
1836 Dst: RetTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
1837
1838 // Handle NaN.
1839 // vmfne v0, v8, v8 # If v8[i] is NaN set v0[i] to 1.
1840 // vmerge.vim v8, v8, 0, v0 # Convert NaN to 0.
1841 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
1842 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: SrcTy, CondTy,
1843 VecPred: CmpInst::FCMP_UNO, CostKind);
1844 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
1845 VecPred: CmpInst::FCMP_UNO, CostKind);
1846 return Cost;
1847 }
1848 case Intrinsic::experimental_vector_extract_last_active: {
1849 auto *ValTy = cast<VectorType>(Val: ICA.getArgTypes()[0]);
1850 auto *MaskTy = cast<VectorType>(Val: ICA.getArgTypes()[1]);
1851
1852 auto ValLT = getTypeLegalizationCost(Ty: ValTy);
1853 auto MaskLT = getTypeLegalizationCost(Ty: MaskTy);
1854
1855 // TODO: Return cheaper cost when the entire lane is inactive.
1856 // The expected asm sequence is:
1857 // vcpop.m a0, v0
1858 // beqz a0, exit # Return passthru when the entire lane is inactive.
1859 // vid v10, v0.t
1860 // vredmaxu.vs v10, v10, v10
1861 // vmv.x.s a0, v10
1862 // zext.b a0, a0
1863 // vslidedown.vx v8, v8, a0
1864 // vmv.x.s a0, v8
1865 // exit:
1866 // ...
1867
1868 // Find a suitable type for a stepvector.
1869 ConstantRange VScaleRange(APInt(64, 1), APInt::getZero(numBits: 64));
1870 unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
1871 RetVT: TLI->getVectorIdxTy(DL: getDataLayout()), EC: MaskTy->getElementCount(),
1872 /*ZeroIsPoison=*/true, VScaleRange: &VScaleRange);
1873 EltWidth = std::max(a: EltWidth, b: MaskTy->getScalarSizeInBits());
1874 Type *StepTy = Type::getIntNTy(C&: MaskTy->getContext(), N: EltWidth);
1875 auto *StepVecTy = VectorType::get(ElementType: StepTy, EC: ValTy->getElementCount());
1876 auto StepLT = getTypeLegalizationCost(Ty: StepVecTy);
1877
1878 // Currently expandVectorFindLastActive cannot handle step vector split.
1879 // So return invalid when the type needs split.
1880 // FIXME: Remove this if expandVectorFindLastActive supports split vector.
1881 if (StepLT.first > 1)
1882 return InstructionCost::getInvalid();
1883
1884 InstructionCost Cost = 0;
1885 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
1886
1887 Cost += MaskLT.first *
1888 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: MaskLT.second, CostKind);
1889 Cost += getCFInstrCost(Opcode: Instruction::CondBr, CostKind, I: nullptr);
1890 Cost += StepLT.first *
1891 getRISCVInstructionCost(OpCodes: Opcodes, VT: StepLT.second, CostKind);
1892 Cost += getCastInstrCost(Opcode: Instruction::ZExt,
1893 Dst: Type::getInt64Ty(C&: ValTy->getContext()), Src: StepTy,
1894 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
1895 Cost += ValLT.first *
1896 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VI, RISCV::VMV_X_S},
1897 VT: ValLT.second, CostKind);
1898 return Cost;
1899 }
1900 }
1901
1902 if (ST->hasVInstructions() && RetTy->isVectorTy()) {
1903 if (auto LT = getTypeLegalizationCost(Ty: RetTy);
1904 LT.second.isVector()) {
1905 MVT EltTy = LT.second.getVectorElementType();
1906 if (const auto *Entry = CostTableLookup(Table: VectorIntrinsicCostTable,
1907 ISD: ICA.getID(), Ty: EltTy))
1908 return LT.first * Entry->Cost;
1909 }
1910 }
1911
1912 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1913}
1914
1915InstructionCost
1916RISCVTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
1917 const SCEV *Ptr,
1918 TTI::TargetCostKind CostKind) const {
1919 // Address computations for vector indexed load/store likely require an offset
1920 // and/or scaling.
1921 if (ST->hasVInstructions() && PtrTy->isVectorTy())
1922 return getArithmeticInstrCost(Opcode: Instruction::Add, Ty: PtrTy, CostKind);
1923
1924 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
1925}
1926
1927InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
1928 Type *Src,
1929 TTI::CastContextHint CCH,
1930 TTI::TargetCostKind CostKind,
1931 const Instruction *I) const {
1932 bool IsVectorType = isa<VectorType>(Val: Dst) && isa<VectorType>(Val: Src);
1933 if (!IsVectorType)
1934 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1935
1936 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
1937 // For now, skip all fixed vector cost analysis when P extension is available
1938 // to avoid crashes in getMinRVVVectorSizeInBits()
1939 if (ST->hasStdExtP() &&
1940 (isa<FixedVectorType>(Val: Dst) || isa<FixedVectorType>(Val: Src))) {
1941 return 1; // Treat as single instruction cost for now
1942 }
1943
1944 // FIXME: Need to compute legalizing cost for illegal types. The current
1945 // code handles only legal types and those which can be trivially
1946 // promoted to legal.
1947 if (!ST->hasVInstructions() || Src->getScalarSizeInBits() > ST->getELen() ||
1948 Dst->getScalarSizeInBits() > ST->getELen())
1949 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1950
1951 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1952 assert(ISD && "Invalid opcode");
1953 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: Src);
1954 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: Dst);
1955
1956 // Handle i1 source and dest cases *before* calling logic in BasicTTI.
1957 // The shared implementation doesn't model vector widening during legalization
1958 // and instead assumes scalarization. In order to scalarize an <N x i1>
1959 // vector, we need to extend/trunc to/from i8. If we don't special case
1960 // this, we can get an infinite recursion cycle.
1961 switch (ISD) {
1962 default:
1963 break;
1964 case ISD::SIGN_EXTEND:
1965 case ISD::ZERO_EXTEND:
1966 if (Src->getScalarSizeInBits() == 1) {
1967 // We do not use vsext/vzext to extend from mask vector.
1968 // Instead we use the following instructions to extend from mask vector:
1969 // vmv.v.i v8, 0
1970 // vmerge.vim v8, v8, -1, v0 (repeated per split)
1971 return getRISCVInstructionCost(OpCodes: RISCV::VMV_V_I, VT: DstLT.second, CostKind) +
1972 DstLT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VIM,
1973 VT: DstLT.second, CostKind) +
1974 DstLT.first - 1;
1975 }
1976 break;
1977 case ISD::TRUNCATE:
1978 if (Dst->getScalarSizeInBits() == 1) {
1979 // We do not use several vncvt to truncate to mask vector. So we could
1980 // not use PowDiff to calculate it.
1981 // Instead we use the following instructions to truncate to mask vector:
1982 // vand.vi v8, v8, 1
1983 // vmsne.vi v0, v8, 0
1984 return SrcLT.first *
1985 getRISCVInstructionCost(OpCodes: {RISCV::VAND_VI, RISCV::VMSNE_VI},
1986 VT: SrcLT.second, CostKind) +
1987 SrcLT.first - 1;
1988 }
1989 break;
1990 };
1991
1992 // Our actual lowering for the case where a wider legal type is available
1993 // uses promotion to the wider type. This is reflected in the result of
1994 // getTypeLegalizationCost, but BasicTTI assumes the widened cases are
1995 // scalarized if the legalized Src and Dst are not equal sized.
1996 const DataLayout &DL = this->getDataLayout();
1997 if (!SrcLT.second.isVector() || !DstLT.second.isVector() ||
1998 !SrcLT.first.isValid() || !DstLT.first.isValid() ||
1999 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Src),
2000 RHS: SrcLT.second.getSizeInBits()) ||
2001 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Dst),
2002 RHS: DstLT.second.getSizeInBits()) ||
2003 SrcLT.first > 1 || DstLT.first > 1)
2004 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2005
2006 // The split cost is handled by the base getCastInstrCost
2007 assert((SrcLT.first == 1) && (DstLT.first == 1) && "Illegal type");
2008
2009 int PowDiff = (int)Log2_32(Value: DstLT.second.getScalarSizeInBits()) -
2010 (int)Log2_32(Value: SrcLT.second.getScalarSizeInBits());
2011 switch (ISD) {
2012 case ISD::SIGN_EXTEND:
2013 case ISD::ZERO_EXTEND: {
2014 if ((PowDiff < 1) || (PowDiff > 3))
2015 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2016 unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
2017 unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
2018 unsigned Op =
2019 (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
2020 return getRISCVInstructionCost(OpCodes: Op, VT: DstLT.second, CostKind);
2021 }
2022 case ISD::TRUNCATE:
2023 case ISD::FP_EXTEND:
2024 case ISD::FP_ROUND: {
2025 // Counts of narrow/widen instructions.
2026 unsigned SrcEltSize = SrcLT.second.getScalarSizeInBits();
2027 unsigned DstEltSize = DstLT.second.getScalarSizeInBits();
2028
2029 unsigned Op = (ISD == ISD::TRUNCATE) ? RISCV::VNSRL_WI
2030 : (ISD == ISD::FP_EXTEND) ? RISCV::VFWCVT_F_F_V
2031 : RISCV::VFNCVT_F_F_W;
2032 InstructionCost Cost = 0;
2033 for (; SrcEltSize != DstEltSize;) {
2034 MVT ElementMVT = (ISD == ISD::TRUNCATE)
2035 ? MVT::getIntegerVT(BitWidth: DstEltSize)
2036 : MVT::getFloatingPointVT(BitWidth: DstEltSize);
2037 MVT DstMVT = DstLT.second.changeVectorElementType(EltVT: ElementMVT);
2038 DstEltSize =
2039 (DstEltSize > SrcEltSize) ? DstEltSize >> 1 : DstEltSize << 1;
2040 Cost += getRISCVInstructionCost(OpCodes: Op, VT: DstMVT, CostKind);
2041 }
2042 return Cost;
2043 }
2044 case ISD::FP_TO_SINT:
2045 case ISD::FP_TO_UINT: {
2046 unsigned IsSigned = ISD == ISD::FP_TO_SINT;
2047 unsigned FCVT = IsSigned ? RISCV::VFCVT_RTZ_X_F_V : RISCV::VFCVT_RTZ_XU_F_V;
2048 unsigned FWCVT =
2049 IsSigned ? RISCV::VFWCVT_RTZ_X_F_V : RISCV::VFWCVT_RTZ_XU_F_V;
2050 unsigned FNCVT =
2051 IsSigned ? RISCV::VFNCVT_RTZ_X_F_W : RISCV::VFNCVT_RTZ_XU_F_W;
2052 unsigned SrcEltSize = Src->getScalarSizeInBits();
2053 unsigned DstEltSize = Dst->getScalarSizeInBits();
2054 InstructionCost Cost = 0;
2055 if ((SrcEltSize == 16) &&
2056 (!ST->hasVInstructionsF16() || ((DstEltSize / 2) > SrcEltSize))) {
2057 // If the target only supports zvfhmin or it is fp16-to-i64 conversion
2058 // pre-widening to f32 and then convert f32 to integer
2059 VectorType *VecF32Ty =
2060 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2061 EC: cast<VectorType>(Val: Dst)->getElementCount());
2062 std::pair<InstructionCost, MVT> VecF32LT =
2063 getTypeLegalizationCost(Ty: VecF32Ty);
2064 Cost +=
2065 VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFWCVT_F_F_V,
2066 VT: VecF32LT.second, CostKind);
2067 Cost += getCastInstrCost(Opcode, Dst, Src: VecF32Ty, CCH, CostKind, I);
2068 return Cost;
2069 }
2070 if (DstEltSize == SrcEltSize)
2071 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2072 else if (DstEltSize > SrcEltSize)
2073 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2074 else { // (SrcEltSize > DstEltSize)
2075 // First do a narrowing conversion to an integer half the size, then
2076 // truncate if needed.
2077 MVT ElementVT = MVT::getIntegerVT(BitWidth: SrcEltSize / 2);
2078 MVT VecVT = DstLT.second.changeVectorElementType(EltVT: ElementVT);
2079 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: VecVT, CostKind);
2080 if ((SrcEltSize / 2) > DstEltSize) {
2081 Type *VecTy = EVT(VecVT).getTypeForEVT(Context&: Dst->getContext());
2082 Cost +=
2083 getCastInstrCost(Opcode: Instruction::Trunc, Dst, Src: VecTy, CCH, CostKind, I);
2084 }
2085 }
2086 return Cost;
2087 }
2088 case ISD::SINT_TO_FP:
2089 case ISD::UINT_TO_FP: {
2090 unsigned IsSigned = ISD == ISD::SINT_TO_FP;
2091 unsigned FCVT = IsSigned ? RISCV::VFCVT_F_X_V : RISCV::VFCVT_F_XU_V;
2092 unsigned FWCVT = IsSigned ? RISCV::VFWCVT_F_X_V : RISCV::VFWCVT_F_XU_V;
2093 unsigned FNCVT = IsSigned ? RISCV::VFNCVT_F_X_W : RISCV::VFNCVT_F_XU_W;
2094 unsigned SrcEltSize = Src->getScalarSizeInBits();
2095 unsigned DstEltSize = Dst->getScalarSizeInBits();
2096
2097 InstructionCost Cost = 0;
2098 if ((DstEltSize == 16) &&
2099 (!ST->hasVInstructionsF16() || ((SrcEltSize / 2) > DstEltSize))) {
2100 // If the target only supports zvfhmin or it is i64-to-fp16 conversion
2101 // it is converted to f32 and then converted to f16
2102 VectorType *VecF32Ty =
2103 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2104 EC: cast<VectorType>(Val: Dst)->getElementCount());
2105 std::pair<InstructionCost, MVT> VecF32LT =
2106 getTypeLegalizationCost(Ty: VecF32Ty);
2107 Cost += getCastInstrCost(Opcode, Dst: VecF32Ty, Src, CCH, CostKind, I);
2108 Cost += VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFNCVT_F_F_W,
2109 VT: DstLT.second, CostKind);
2110 return Cost;
2111 }
2112
2113 if (DstEltSize == SrcEltSize)
2114 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2115 else if (DstEltSize > SrcEltSize) {
2116 if ((DstEltSize / 2) > SrcEltSize) {
2117 VectorType *VecTy =
2118 VectorType::get(ElementType: IntegerType::get(C&: Dst->getContext(), NumBits: DstEltSize / 2),
2119 EC: cast<VectorType>(Val: Dst)->getElementCount());
2120 unsigned Op = IsSigned ? Instruction::SExt : Instruction::ZExt;
2121 Cost += getCastInstrCost(Opcode: Op, Dst: VecTy, Src, CCH, CostKind, I);
2122 }
2123 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2124 } else
2125 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: DstLT.second, CostKind);
2126 return Cost;
2127 }
2128 }
2129 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2130}
2131
2132unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) const {
2133 if (isa<ScalableVectorType>(Val: Ty)) {
2134 const unsigned EltSize = DL.getTypeSizeInBits(Ty: Ty->getElementType());
2135 const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue();
2136 const unsigned VectorBits = *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
2137 return RISCVTargetLowering::computeVLMAX(VectorBits, EltSize, MinSize);
2138 }
2139 return cast<FixedVectorType>(Val: Ty)->getNumElements();
2140}
2141
2142InstructionCost
2143RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
2144 FastMathFlags FMF,
2145 TTI::TargetCostKind CostKind) const {
2146 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2147 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2148
2149 // Skip if scalar size of Ty is bigger than ELEN.
2150 if (Ty->getScalarSizeInBits() > ST->getELen())
2151 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2152
2153 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2154 if (Ty->getElementType()->isIntegerTy(BitWidth: 1)) {
2155 // SelectionDAGBuilder does following transforms:
2156 // vector_reduce_{smin,umax}(<n x i1>) --> vector_reduce_or(<n x i1>)
2157 // vector_reduce_{smax,umin}(<n x i1>) --> vector_reduce_and(<n x i1>)
2158 if (IID == Intrinsic::umax || IID == Intrinsic::smin)
2159 return getArithmeticReductionCost(Opcode: Instruction::Or, Ty, FMF, CostKind);
2160 else
2161 return getArithmeticReductionCost(Opcode: Instruction::And, Ty, FMF, CostKind);
2162 }
2163
2164 if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) {
2165 SmallVector<unsigned, 3> Opcodes;
2166 InstructionCost ExtraCost = 0;
2167 switch (IID) {
2168 case Intrinsic::maximum:
2169 if (FMF.noNaNs()) {
2170 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2171 } else {
2172 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS,
2173 RISCV::VFMV_F_S};
2174 // Cost of Canonical Nan + branch
2175 // lui a0, 523264
2176 // fmv.w.x fa0, a0
2177 Type *DstTy = Ty->getScalarType();
2178 const unsigned EltTyBits = DstTy->getScalarSizeInBits();
2179 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2180 ExtraCost = 1 +
2181 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2182 CCH: TTI::CastContextHint::None, CostKind) +
2183 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2184 }
2185 break;
2186
2187 case Intrinsic::minimum:
2188 if (FMF.noNaNs()) {
2189 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2190 } else {
2191 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS,
2192 RISCV::VFMV_F_S};
2193 // Cost of Canonical Nan + branch
2194 // lui a0, 523264
2195 // fmv.w.x fa0, a0
2196 Type *DstTy = Ty->getScalarType();
2197 const unsigned EltTyBits = DL.getTypeSizeInBits(Ty: DstTy);
2198 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2199 ExtraCost = 1 +
2200 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2201 CCH: TTI::CastContextHint::None, CostKind) +
2202 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2203 }
2204 break;
2205 }
2206 return ExtraCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2207 }
2208
2209 // IR Reduction is composed by one rvv reduction instruction and vmv
2210 unsigned SplitOp;
2211 SmallVector<unsigned, 3> Opcodes;
2212 switch (IID) {
2213 default:
2214 llvm_unreachable("Unsupported intrinsic");
2215 case Intrinsic::smax:
2216 SplitOp = RISCV::VMAX_VV;
2217 Opcodes = {RISCV::VREDMAX_VS, RISCV::VMV_X_S};
2218 break;
2219 case Intrinsic::smin:
2220 SplitOp = RISCV::VMIN_VV;
2221 Opcodes = {RISCV::VREDMIN_VS, RISCV::VMV_X_S};
2222 break;
2223 case Intrinsic::umax:
2224 SplitOp = RISCV::VMAXU_VV;
2225 Opcodes = {RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
2226 break;
2227 case Intrinsic::umin:
2228 SplitOp = RISCV::VMINU_VV;
2229 Opcodes = {RISCV::VREDMINU_VS, RISCV::VMV_X_S};
2230 break;
2231 case Intrinsic::maxnum:
2232 SplitOp = RISCV::VFMAX_VV;
2233 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2234 break;
2235 case Intrinsic::minnum:
2236 SplitOp = RISCV::VFMIN_VV;
2237 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2238 break;
2239 }
2240 // Add a cost for data larger than LMUL8
2241 InstructionCost SplitCost =
2242 (LT.first > 1) ? (LT.first - 1) *
2243 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2244 : 0;
2245 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2246}
2247
2248InstructionCost
2249RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
2250 std::optional<FastMathFlags> FMF,
2251 TTI::TargetCostKind CostKind) const {
2252 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2253 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2254
2255 // Skip if scalar size of Ty is bigger than ELEN.
2256 if (Ty->getScalarSizeInBits() > ST->getELen())
2257 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2258
2259 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2260 assert(ISD && "Invalid opcode");
2261
2262 if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&
2263 ISD != ISD::FADD)
2264 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2265
2266 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2267 Type *ElementTy = Ty->getElementType();
2268 if (ElementTy->isIntegerTy(BitWidth: 1)) {
2269 // Example sequences:
2270 // vfirst.m a0, v0
2271 // seqz a0, a0
2272 if (LT.second == MVT::v1i1)
2273 return getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind) +
2274 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2275 VecPred: CmpInst::ICMP_EQ, CostKind);
2276
2277 if (ISD == ISD::AND) {
2278 // Example sequences:
2279 // vmand.mm v8, v9, v8 ; needed every time type is split
2280 // vmnot.m v8, v0 ; alias for vmnand
2281 // vcpop.m a0, v8
2282 // seqz a0, a0
2283
2284 // See the discussion: https://github.com/llvm/llvm-project/pull/119160
2285 // For LMUL <= 8, there is no splitting,
2286 // the sequences are vmnot, vcpop and seqz.
2287 // When LMUL > 8 and split = 1,
2288 // the sequences are vmnand, vcpop and seqz.
2289 // When LMUL > 8 and split > 1,
2290 // the sequences are (LT.first-2) * vmand, vmnand, vcpop and seqz.
2291 return ((LT.first > 2) ? (LT.first - 2) : 0) *
2292 getRISCVInstructionCost(OpCodes: RISCV::VMAND_MM, VT: LT.second, CostKind) +
2293 getRISCVInstructionCost(OpCodes: RISCV::VMNAND_MM, VT: LT.second, CostKind) +
2294 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2295 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2296 VecPred: CmpInst::ICMP_EQ, CostKind);
2297 } else if (ISD == ISD::XOR || ISD == ISD::ADD) {
2298 // Example sequences:
2299 // vsetvli a0, zero, e8, mf8, ta, ma
2300 // vmxor.mm v8, v0, v8 ; needed every time type is split
2301 // vcpop.m a0, v8
2302 // andi a0, a0, 1
2303 return (LT.first - 1) *
2304 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind) +
2305 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) + 1;
2306 } else {
2307 assert(ISD == ISD::OR);
2308 // Example sequences:
2309 // vsetvli a0, zero, e8, mf8, ta, ma
2310 // vmor.mm v8, v9, v8 ; needed every time type is split
2311 // vcpop.m a0, v0
2312 // snez a0, a0
2313 return (LT.first - 1) *
2314 getRISCVInstructionCost(OpCodes: RISCV::VMOR_MM, VT: LT.second, CostKind) +
2315 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2316 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2317 VecPred: CmpInst::ICMP_NE, CostKind);
2318 }
2319 }
2320
2321 // IR Reduction of or/and is composed by one vmv and one rvv reduction
2322 // instruction, and others is composed by two vmv and one rvv reduction
2323 // instruction
2324 unsigned SplitOp;
2325 SmallVector<unsigned, 3> Opcodes;
2326 switch (ISD) {
2327 case ISD::ADD:
2328 SplitOp = RISCV::VADD_VV;
2329 Opcodes = {RISCV::VMV_S_X, RISCV::VREDSUM_VS, RISCV::VMV_X_S};
2330 break;
2331 case ISD::OR:
2332 SplitOp = RISCV::VOR_VV;
2333 Opcodes = {RISCV::VREDOR_VS, RISCV::VMV_X_S};
2334 break;
2335 case ISD::XOR:
2336 SplitOp = RISCV::VXOR_VV;
2337 Opcodes = {RISCV::VMV_S_X, RISCV::VREDXOR_VS, RISCV::VMV_X_S};
2338 break;
2339 case ISD::AND:
2340 SplitOp = RISCV::VAND_VV;
2341 Opcodes = {RISCV::VREDAND_VS, RISCV::VMV_X_S};
2342 break;
2343 case ISD::FADD:
2344 // We can't promote f16/bf16 fadd reductions.
2345 if ((LT.second.getScalarType() == MVT::f16 && !ST->hasVInstructionsF16()) ||
2346 LT.second.getScalarType() == MVT::bf16)
2347 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2348 if (TTI::requiresOrderedReduction(FMF)) {
2349 Opcodes.push_back(Elt: RISCV::VFMV_S_F);
2350 for (unsigned i = 0; i < LT.first.getValue(); i++)
2351 Opcodes.push_back(Elt: RISCV::VFREDOSUM_VS);
2352 Opcodes.push_back(Elt: RISCV::VFMV_F_S);
2353 return getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2354 }
2355 SplitOp = RISCV::VFADD_VV;
2356 Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDUSUM_VS, RISCV::VFMV_F_S};
2357 break;
2358 }
2359 // Add a cost for data larger than LMUL8
2360 InstructionCost SplitCost =
2361 (LT.first > 1) ? (LT.first - 1) *
2362 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2363 : 0;
2364 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2365}
2366
2367InstructionCost RISCVTTIImpl::getExtendedReductionCost(
2368 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
2369 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
2370 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2371 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2372 FMF, CostKind);
2373
2374 // Skip if scalar size of ResTy is bigger than ELEN.
2375 if (ResTy->getScalarSizeInBits() > ST->getELen())
2376 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2377 FMF, CostKind);
2378
2379 if (Opcode != Instruction::Add && Opcode != Instruction::FAdd)
2380 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2381 FMF, CostKind);
2382
2383 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2384
2385 if (IsUnsigned && Opcode == Instruction::Add &&
2386 LT.second.isFixedLengthVectorOf(EltVT: MVT::i1)) {
2387 // Represent vector_reduce_add(ZExt(<n x i1>)) as
2388 // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
2389 return LT.first *
2390 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind);
2391 }
2392
2393 if (ResTy->getScalarSizeInBits() != 2 * LT.second.getScalarSizeInBits())
2394 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2395 FMF, CostKind);
2396
2397 return (LT.first - 1) +
2398 getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
2399}
2400
2401InstructionCost
2402RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,
2403 TTI::TargetCostKind CostKind) const {
2404 assert(OpInfo.isConstant() && "non constant operand?");
2405 if (!isa<VectorType>(Val: Ty))
2406 // FIXME: We need to account for immediate materialization here, but doing
2407 // a decent job requires more knowledge about the immediate than we
2408 // currently have here.
2409 return 0;
2410
2411 if (OpInfo.isUniform())
2412 // vmv.v.i, vmv.v.x, or vfmv.v.f
2413 // We ignore the cost of the scalar constant materialization to be consistent
2414 // with how we treat scalar constants themselves just above.
2415 return 1;
2416
2417 return getConstantPoolLoadCost(Ty, CostKind);
2418}
2419
2420InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
2421 Align Alignment,
2422 unsigned AddressSpace,
2423 TTI::TargetCostKind CostKind,
2424 TTI::OperandValueInfo OpInfo,
2425 const Instruction *I) const {
2426 EVT VT = TLI->getValueType(DL, Ty: Src, AllowUnknown: true);
2427 // Type legalization can't handle structs, and load latency isn't handled here
2428 if (VT == MVT::Other ||
2429 (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency))
2430 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2431 CostKind, OpInfo, I);
2432
2433 InstructionCost Cost = 0;
2434 if (Opcode == Instruction::Store && OpInfo.isConstant())
2435 Cost += getStoreImmCost(Ty: Src, OpInfo, CostKind);
2436
2437 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
2438
2439 InstructionCost BaseCost = [&]() {
2440 InstructionCost Cost = LT.first;
2441 if (CostKind != TTI::TCK_RecipThroughput)
2442 return Cost;
2443
2444 // Our actual lowering for the case where a wider legal type is available
2445 // uses the a VL predicated load on the wider type. This is reflected in
2446 // the result of getTypeLegalizationCost, but BasicTTI assumes the
2447 // widened cases are scalarized.
2448 const DataLayout &DL = this->getDataLayout();
2449 if (Src->isVectorTy() && LT.second.isVector() &&
2450 TypeSize::isKnownLT(LHS: DL.getTypeStoreSizeInBits(Ty: Src),
2451 RHS: LT.second.getSizeInBits()))
2452 return Cost;
2453
2454 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2455 CostKind, OpInfo, I);
2456 }();
2457
2458 // Assume memory ops cost scale with the number of vector registers
2459 // possible accessed by the instruction. Note that BasicTTI already
2460 // handles the LT.first term for us.
2461 if (ST->hasVInstructions() && LT.second.isVector() &&
2462 CostKind != TTI::TCK_CodeSize)
2463 BaseCost *= TLI->getLMULCost(VT: LT.second);
2464 return Cost + BaseCost;
2465}
2466
2467InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
2468 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
2469 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
2470 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
2471 if (CostKind != TTI::TCK_RecipThroughput)
2472 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2473 Op1Info, Op2Info, I);
2474
2475 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2476 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2477 Op1Info, Op2Info, I);
2478
2479 // Skip if scalar size of ValTy is bigger than ELEN.
2480 if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen())
2481 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2482 Op1Info, Op2Info, I);
2483
2484 auto GetConstantMatCost =
2485 [&](TTI::OperandValueInfo OpInfo) -> InstructionCost {
2486 if (OpInfo.isUniform())
2487 // We return 0 we currently ignore the cost of materializing scalar
2488 // constants in GPRs.
2489 return 0;
2490
2491 return getConstantPoolLoadCost(Ty: ValTy, CostKind);
2492 };
2493
2494 InstructionCost ConstantMatCost;
2495 if (Op1Info.isConstant())
2496 ConstantMatCost += GetConstantMatCost(Op1Info);
2497 if (Op2Info.isConstant())
2498 ConstantMatCost += GetConstantMatCost(Op2Info);
2499
2500 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2501 if (Opcode == Instruction::Select && LT.second.isVector()) {
2502 if (CondTy->isVectorTy()) {
2503 if (ValTy->getScalarSizeInBits() == 1) {
2504 // vmandn.mm v8, v8, v9
2505 // vmand.mm v9, v0, v9
2506 // vmor.mm v0, v9, v8
2507 return ConstantMatCost +
2508 LT.first *
2509 getRISCVInstructionCost(
2510 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2511 VT: LT.second, CostKind);
2512 }
2513 // vselect and max/min are supported natively.
2514 return ConstantMatCost +
2515 LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second,
2516 CostKind);
2517 }
2518
2519 if (ValTy->getScalarSizeInBits() == 1) {
2520 // vmv.v.x v9, a0
2521 // vmsne.vi v9, v9, 0
2522 // vmandn.mm v8, v8, v9
2523 // vmand.mm v9, v0, v9
2524 // vmor.mm v0, v9, v8
2525 MVT InterimVT = LT.second.changeVectorElementType(EltVT: MVT::i8);
2526 return ConstantMatCost +
2527 LT.first *
2528 getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
2529 VT: InterimVT, CostKind) +
2530 LT.first * getRISCVInstructionCost(
2531 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2532 VT: LT.second, CostKind);
2533 }
2534
2535 // vmv.v.x v10, a0
2536 // vmsne.vi v0, v10, 0
2537 // vmerge.vvm v8, v9, v8, v0
2538 return ConstantMatCost +
2539 LT.first * getRISCVInstructionCost(
2540 OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI, RISCV::VMERGE_VVM},
2541 VT: LT.second, CostKind);
2542 }
2543
2544 if ((Opcode == Instruction::ICmp) && ValTy->isVectorTy() &&
2545 CmpInst::isIntPredicate(P: VecPred)) {
2546 // Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE
2547 // provided they incur the same cost across all implementations
2548 return ConstantMatCost + LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMSLT_VV,
2549 VT: LT.second,
2550 CostKind);
2551 }
2552
2553 if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() &&
2554 CmpInst::isFPPredicate(P: VecPred)) {
2555
2556 // Use VMXOR_MM and VMXNOR_MM to generate all true/false mask
2557 if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE))
2558 return ConstantMatCost +
2559 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind);
2560
2561 // If we do not support the input floating point vector type, use the base
2562 // one which will calculate as:
2563 // ScalarizeCost + Num * Cost for fixed vector,
2564 // InvalidCost for scalable vector.
2565 if ((ValTy->getScalarSizeInBits() == 16 && !ST->hasVInstructionsF16()) ||
2566 (ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) ||
2567 (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64()))
2568 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2569 Op1Info, Op2Info, I);
2570
2571 // Assuming vector fp compare and mask instructions are all the same cost
2572 // until a need arises to differentiate them.
2573 switch (VecPred) {
2574 case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm
2575 case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm
2576 case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm
2577 case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm
2578 return ConstantMatCost +
2579 LT.first * getRISCVInstructionCost(
2580 OpCodes: {RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM},
2581 VT: LT.second, CostKind);
2582
2583 case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m
2584 case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m
2585 case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m
2586 case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m
2587 return ConstantMatCost +
2588 LT.first *
2589 getRISCVInstructionCost(OpCodes: {RISCV::VMFLT_VV, RISCV::VMNAND_MM},
2590 VT: LT.second, CostKind);
2591
2592 case CmpInst::FCMP_OEQ: // vmfeq.vv
2593 case CmpInst::FCMP_OGT: // vmflt.vv
2594 case CmpInst::FCMP_OGE: // vmfle.vv
2595 case CmpInst::FCMP_OLT: // vmflt.vv
2596 case CmpInst::FCMP_OLE: // vmfle.vv
2597 case CmpInst::FCMP_UNE: // vmfne.vv
2598 return ConstantMatCost +
2599 LT.first *
2600 getRISCVInstructionCost(OpCodes: RISCV::VMFLT_VV, VT: LT.second, CostKind);
2601 default:
2602 break;
2603 }
2604 }
2605
2606 // With ShortForwardBranchOpt or ConditionalMoveFusion, scalar icmp + select
2607 // instructions will lower to SELECT_CC and lower to PseudoCCMOVGPR which will
2608 // generate a conditional branch + mv. The cost of scalar (icmp + select) will
2609 // be (0 + select instr cost).
2610 if (ST->hasConditionalMoveFusion() && I && isa<ICmpInst>(Val: I) &&
2611 ValTy->isIntegerTy() && !I->user_empty()) {
2612 if (all_of(Range: I->users(), P: [&](const User *U) {
2613 return match(V: U, P: m_Select(C: m_Specific(V: I), L: m_Value(), R: m_Value())) &&
2614 U->getType()->isIntegerTy() &&
2615 !isa<ConstantData>(Val: U->getOperand(i: 1)) &&
2616 !isa<ConstantData>(Val: U->getOperand(i: 2));
2617 }))
2618 return 0;
2619 }
2620
2621 // TODO: Add cost for scalar type.
2622
2623 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2624 Op1Info, Op2Info, I);
2625}
2626
2627InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
2628 TTI::TargetCostKind CostKind,
2629 const Instruction *I) const {
2630 if (CostKind != TTI::TCK_RecipThroughput)
2631 return Opcode == Instruction::PHI ? 0 : 1;
2632 // Branches are assumed to be predicted.
2633 return 0;
2634}
2635
2636InstructionCost RISCVTTIImpl::getVectorInstrCost(
2637 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
2638 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
2639 assert(Val->isVectorTy() && "This must be a vector type");
2640
2641 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
2642 // For now, skip all fixed vector cost analysis when P extension is available
2643 // to avoid crashes in getMinRVVVectorSizeInBits()
2644 if (ST->hasStdExtP() && isa<FixedVectorType>(Val)) {
2645 return 1; // Treat as single instruction cost for now
2646 }
2647
2648 if (Opcode != Instruction::ExtractElement &&
2649 Opcode != Instruction::InsertElement)
2650 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
2651 VIC);
2652
2653 // Legalize the type.
2654 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Val);
2655
2656 // This type is legalized to a scalar type.
2657 if (!LT.second.isVector()) {
2658 auto *FixedVecTy = cast<FixedVectorType>(Val);
2659 // If Index is a known constant, cost is zero.
2660 if (Index != -1U)
2661 return 0;
2662 // Extract/InsertElement with non-constant index is very costly when
2663 // scalarized; estimate cost of loads/stores sequence via the stack:
2664 // ExtractElement cost: store vector to stack, load scalar;
2665 // InsertElement cost: store vector to stack, store scalar, load vector.
2666 Type *ElemTy = FixedVecTy->getElementType();
2667 auto NumElems = FixedVecTy->getNumElements();
2668 auto Align = DL.getPrefTypeAlign(Ty: ElemTy);
2669 InstructionCost LoadCost =
2670 getMemoryOpCost(Opcode: Instruction::Load, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2671 InstructionCost StoreCost =
2672 getMemoryOpCost(Opcode: Instruction::Store, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2673 return Opcode == Instruction::ExtractElement
2674 ? StoreCost * NumElems + LoadCost
2675 : (StoreCost + LoadCost) * NumElems + StoreCost;
2676 }
2677
2678 // For unsupported scalable vector.
2679 if (LT.second.isScalableVector() && !LT.first.isValid())
2680 return LT.first;
2681
2682 // Mask vector extract/insert is expanded via e8.
2683 if (Val->getScalarSizeInBits() == 1) {
2684 VectorType *WideTy =
2685 VectorType::get(ElementType: IntegerType::get(C&: Val->getContext(), NumBits: 8),
2686 EC: cast<VectorType>(Val)->getElementCount());
2687 if (Opcode == Instruction::ExtractElement) {
2688 InstructionCost ExtendCost
2689 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2690 CCH: TTI::CastContextHint::None, CostKind);
2691 InstructionCost ExtractCost
2692 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2693 return ExtendCost + ExtractCost;
2694 }
2695 InstructionCost ExtendCost
2696 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2697 CCH: TTI::CastContextHint::None, CostKind);
2698 InstructionCost InsertCost
2699 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2700 InstructionCost TruncCost
2701 = getCastInstrCost(Opcode: Instruction::Trunc, Dst: Val, Src: WideTy,
2702 CCH: TTI::CastContextHint::None, CostKind);
2703 return ExtendCost + InsertCost + TruncCost;
2704 }
2705
2706
2707 // In RVV, we could use vslidedown + vmv.x.s to extract element from vector
2708 // and vslideup + vmv.s.x to insert element to vector.
2709 unsigned MoveOpc;
2710 if (LT.second.isFloatingPoint())
2711 MoveOpc = Opcode == Instruction::InsertElement ? RISCV::VFMV_S_F
2712 : RISCV::VFMV_F_S;
2713 else
2714 MoveOpc =
2715 Opcode == Instruction::InsertElement ? RISCV::VMV_S_X : RISCV::VMV_X_S;
2716 InstructionCost BaseCost =
2717 getRISCVInstructionCost(OpCodes: MoveOpc, VT: LT.second, CostKind);
2718 // When insertelement we should add the index with 1 as the input of vslideup.
2719 InstructionCost SlideCost = Opcode == Instruction::InsertElement ? 2 : 1;
2720
2721 if (Index != -1U) {
2722 // The type may be split. For fixed-width vectors we can normalize the
2723 // index to the new type.
2724 if (LT.second.isFixedLengthVector()) {
2725 unsigned Width = LT.second.getVectorNumElements();
2726 Index = Index % Width;
2727 }
2728
2729 // If exact VLEN is known, we will insert/extract into the appropriate
2730 // subvector with no additional subvector insert/extract cost.
2731 if (auto VLEN = ST->getRealVLen()) {
2732 unsigned EltSize = LT.second.getScalarSizeInBits();
2733 unsigned M1Max = *VLEN / EltSize;
2734 Index = Index % M1Max;
2735 }
2736
2737 if (Index == 0)
2738 // We can extract/insert the first element without vslidedown/vslideup.
2739 SlideCost = 0;
2740 else if (Opcode == Instruction::InsertElement)
2741 SlideCost = 1; // With a constant index, we do not need to use addi.
2742 }
2743
2744 // When the vector needs to split into multiple register groups and the index
2745 // exceeds single vector register group, we need to insert/extract the element
2746 // via stack.
2747 if (LT.first > 1 &&
2748 ((Index == -1U) || (Index >= LT.second.getVectorMinNumElements() &&
2749 LT.second.isScalableVector()))) {
2750 Type *ScalarType = Val->getScalarType();
2751 Align VecAlign = DL.getPrefTypeAlign(Ty: Val);
2752 Align SclAlign = DL.getPrefTypeAlign(Ty: ScalarType);
2753 // Extra addi for unknown index.
2754 InstructionCost IdxCost = Index == -1U ? 1 : 0;
2755
2756 // Store all split vectors into stack and load the target element.
2757 if (Opcode == Instruction::ExtractElement)
2758 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2759 getMemoryOpCost(Opcode: Instruction::Load, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
2760 CostKind) +
2761 IdxCost;
2762
2763 // Store all split vectors into stack and store the target element and load
2764 // vectors back.
2765 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2766 getMemoryOpCost(Opcode: Instruction::Load, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2767 getMemoryOpCost(Opcode: Instruction::Store, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
2768 CostKind) +
2769 IdxCost;
2770 }
2771
2772 // Extract i64 in the target that has XLEN=32 need more instruction.
2773 if (Val->getScalarType()->isIntegerTy() &&
2774 ST->getXLen() < Val->getScalarSizeInBits()) {
2775 // For extractelement, we need the following instructions:
2776 // vsetivli zero, 1, e64, m1, ta, mu (not count)
2777 // vslidedown.vx v8, v8, a0
2778 // vmv.x.s a0, v8
2779 // li a1, 32
2780 // vsrl.vx v8, v8, a1
2781 // vmv.x.s a1, v8
2782
2783 // For insertelement, we need the following instructions:
2784 // vsetivli zero, 2, e32, m4, ta, ma (don't count)
2785 // vslide1down.vx v12, v8, a0
2786 // vslide1down.vx v12, v12, a1
2787 // addi a0, a2, 1
2788 // vsetvli zero, a0, e64, m4, tu, ma (don't count)
2789 // vslideup.vx v8, v12, a2
2790
2791 // TODO: should we count these special vsetvlis?
2792 BaseCost =
2793 Opcode == Instruction::InsertElement
2794 ? getRISCVInstructionCost(OpCodes: {RISCV::VSLIDE1DOWN_VX,
2795 RISCV::VSLIDE1DOWN_VX,
2796 RISCV::VSLIDEUP_VX},
2797 VT: LT.second, CostKind)
2798 : getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VMV_X_S,
2799 RISCV::VSRL_VX, RISCV::VMV_X_S},
2800 VT: LT.second, CostKind);
2801 }
2802 return BaseCost + SlideCost;
2803}
2804
2805InstructionCost
2806RISCVTTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
2807 TTI::TargetCostKind CostKind,
2808 unsigned Index) const {
2809 if (isa<FixedVectorType>(Val))
2810 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
2811 Index);
2812
2813 // TODO: This code replicates what LoopVectorize.cpp used to do when asking
2814 // for the cost of extracting the last lane of a scalable vector. It probably
2815 // needs a more accurate cost.
2816 ElementCount EC = cast<VectorType>(Val)->getElementCount();
2817 assert(Index < EC.getKnownMinValue() && "Unexpected reverse index");
2818 return getVectorInstrCost(Opcode, Val, CostKind,
2819 Index: EC.getKnownMinValue() - 1 - Index, Op0: nullptr,
2820 Op1: nullptr);
2821}
2822
2823/// Check to see if this instruction is expected to be combined to a simpler
2824/// operation during/before lowering. If so return the cost of the combined
2825/// operation rather than provided one. For instance, `udiv i16 %X, 2` is likely
2826/// to be combined to `lshr i16 %X, 1`, so return the cost of a `lshr` rather
2827/// than the cost of a `udiv`
2828std::optional<InstructionCost>
2829RISCVTTIImpl::getCombinedArithmeticInstructionCost(
2830 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
2831 TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
2832 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
2833 // Vector unsigned division/remainder will be simplified to shifts/masks.
2834 if ((Opcode == Instruction::UDiv || Opcode == Instruction::URem) &&
2835 Opd2Info.isConstant() && Opd2Info.isPowerOf2()) {
2836 if (Opcode == Instruction::UDiv)
2837 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind, Op1Info: Opd1Info,
2838 Op2Info: Opd2Info.getNoProps());
2839 // UREM
2840 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind, Op1Info: Opd1Info,
2841 Op2Info: Opd2Info.getNoProps());
2842 }
2843 return std::nullopt;
2844}
2845
2846InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
2847 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
2848 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
2849 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
2850
2851 // TODO: Handle more cost kinds.
2852 if (CostKind != TTI::TCK_RecipThroughput)
2853 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2854 Args, CxtI);
2855
2856 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2857 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2858 Args, CxtI);
2859
2860 // Skip if scalar size of Ty is bigger than ELEN.
2861 if (isa<VectorType>(Val: Ty) && Ty->getScalarSizeInBits() > ST->getELen())
2862 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2863 Args, CxtI);
2864
2865 if (std::optional<InstructionCost> CombinedCost =
2866 getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
2867 Opd2Info: Op2Info, Args, CxtI))
2868 return *CombinedCost;
2869
2870 // Legalize the type.
2871 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2872 unsigned ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
2873
2874 // TODO: Handle scalar type.
2875 if (!LT.second.isVector()) {
2876 static const CostTblEntry DivTbl[]{
2877 {.ISD: ISD::UDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2878 {.ISD: ISD::UDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2879 {.ISD: ISD::SDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2880 {.ISD: ISD::SDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2881 {.ISD: ISD::UREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2882 {.ISD: ISD::UREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2883 {.ISD: ISD::SREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2884 {.ISD: ISD::SREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive}};
2885 if (TLI->isOperationLegalOrPromote(Op: ISDOpcode, VT: LT.second))
2886 if (const auto *Entry = CostTableLookup(Table: DivTbl, ISD: ISDOpcode, Ty: LT.second))
2887 return Entry->Cost * LT.first;
2888
2889 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2890 Args, CxtI);
2891 }
2892
2893 // f16 with zvfhmin and bf16 will be promoted to f32.
2894 // FIXME: nxv32[b]f16 will be custom lowered and split.
2895 InstructionCost CastCost = 0;
2896 if ((LT.second.getVectorElementType() == MVT::f16 ||
2897 LT.second.getVectorElementType() == MVT::bf16) &&
2898 TLI->getOperationAction(Op: ISDOpcode, VT: LT.second) ==
2899 TargetLoweringBase::LegalizeAction::Promote) {
2900 MVT PromotedVT = TLI->getTypeToPromoteTo(Op: ISDOpcode, VT: LT.second);
2901 Type *PromotedTy = EVT(PromotedVT).getTypeForEVT(Context&: Ty->getContext());
2902 Type *LegalTy = EVT(LT.second).getTypeForEVT(Context&: Ty->getContext());
2903 // Add cost of extending arguments
2904 CastCost += LT.first * Args.size() *
2905 getCastInstrCost(Opcode: Instruction::FPExt, Dst: PromotedTy, Src: LegalTy,
2906 CCH: TTI::CastContextHint::None, CostKind);
2907 // Add cost of truncating result
2908 CastCost +=
2909 LT.first * getCastInstrCost(Opcode: Instruction::FPTrunc, Dst: LegalTy, Src: PromotedTy,
2910 CCH: TTI::CastContextHint::None, CostKind);
2911 // Compute cost of op in promoted type
2912 LT.second = PromotedVT;
2913 }
2914
2915 auto getConstantMatCost =
2916 [&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost {
2917 if (OpInfo.isUniform() && canSplatOperand(Opcode, Operand))
2918 // Two sub-cases:
2919 // * Has a 5 bit immediate operand which can be splatted.
2920 // * Has a larger immediate which must be materialized in scalar register
2921 // We return 0 for both as we currently ignore the cost of materializing
2922 // scalar constants in GPRs.
2923 return 0;
2924
2925 return getConstantPoolLoadCost(Ty, CostKind);
2926 };
2927
2928 // Add the cost of materializing any constant vectors required.
2929 InstructionCost ConstantMatCost = 0;
2930 if (Op1Info.isConstant())
2931 ConstantMatCost += getConstantMatCost(0, Op1Info);
2932 if (Op2Info.isConstant())
2933 ConstantMatCost += getConstantMatCost(1, Op2Info);
2934
2935 unsigned Op;
2936 switch (ISDOpcode) {
2937 case ISD::ADD:
2938 case ISD::SUB:
2939 Op = RISCV::VADD_VV;
2940 break;
2941 case ISD::SHL:
2942 case ISD::SRL:
2943 case ISD::SRA:
2944 Op = RISCV::VSLL_VV;
2945 break;
2946 case ISD::AND:
2947 case ISD::OR:
2948 case ISD::XOR:
2949 Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
2950 break;
2951 case ISD::MUL:
2952 case ISD::MULHS:
2953 case ISD::MULHU:
2954 Op = RISCV::VMUL_VV;
2955 break;
2956 case ISD::SDIV:
2957 case ISD::UDIV:
2958 Op = RISCV::VDIV_VV;
2959 break;
2960 case ISD::SREM:
2961 case ISD::UREM:
2962 Op = RISCV::VREM_VV;
2963 break;
2964 case ISD::FADD:
2965 case ISD::FSUB:
2966 Op = RISCV::VFADD_VV;
2967 break;
2968 case ISD::FMUL:
2969 Op = RISCV::VFMUL_VV;
2970 break;
2971 case ISD::FDIV:
2972 Op = RISCV::VFDIV_VV;
2973 break;
2974 case ISD::FNEG:
2975 Op = RISCV::VFSGNJN_VV;
2976 break;
2977 default:
2978 // Assuming all other instructions have the same cost until a need arises to
2979 // differentiate them.
2980 return CastCost + ConstantMatCost +
2981 BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2982 Args, CxtI);
2983 }
2984
2985 InstructionCost InstrCost = getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
2986 // We use BasicTTIImpl to calculate scalar costs, which assumes floating point
2987 // ops are twice as expensive as integer ops. Do the same for vectors so
2988 // scalar floating point ops aren't cheaper than their vector equivalents.
2989 if (Ty->isFPOrFPVectorTy())
2990 InstrCost *= 2;
2991 return CastCost + ConstantMatCost + LT.first * InstrCost;
2992}
2993
2994// TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
2995InstructionCost RISCVTTIImpl::getPointersChainCost(
2996 ArrayRef<const Value *> Ptrs, const Value *Base,
2997 const TTI::PointersChainInfo &Info, Type *AccessTy,
2998 const TTI::TargetCostKind CostKind) const {
2999 InstructionCost Cost = TTI::TCC_Free;
3000 // In the basic model we take into account GEP instructions only
3001 // (although here can come alloca instruction, a value, constants and/or
3002 // constant expressions, PHIs, bitcasts ... whatever allowed to be used as a
3003 // pointer). Typically, if Base is a not a GEP-instruction and all the
3004 // pointers are relative to the same base address, all the rest are
3005 // either GEP instructions, PHIs, bitcasts or constants. When we have same
3006 // base, we just calculate cost of each non-Base GEP as an ADD operation if
3007 // any their index is a non-const.
3008 // If no known dependencies between the pointers cost is calculated as a sum
3009 // of costs of GEP instructions.
3010 for (auto [I, V] : enumerate(First&: Ptrs)) {
3011 const auto *GEP = dyn_cast<GetElementPtrInst>(Val: V);
3012 if (!GEP)
3013 continue;
3014 if (Info.isSameBase() && V != Base) {
3015 if (GEP->hasAllConstantIndices())
3016 continue;
3017 // If the chain is unit-stride and BaseReg + stride*i is a legal
3018 // addressing mode, then presume the base GEP is sitting around in a
3019 // register somewhere and check if we can fold the offset relative to
3020 // it.
3021 unsigned Stride = DL.getTypeStoreSize(Ty: AccessTy);
3022 if (Info.isUnitStride() &&
3023 isLegalAddressingMode(Ty: AccessTy,
3024 /* BaseGV */ nullptr,
3025 /* BaseOffset */ Stride * I,
3026 /* HasBaseReg */ true,
3027 /* Scale */ 0,
3028 AddrSpace: GEP->getType()->getPointerAddressSpace()))
3029 continue;
3030 Cost += getArithmeticInstrCost(Opcode: Instruction::Add, Ty: GEP->getType(), CostKind,
3031 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
3032 Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Args: {});
3033 } else {
3034 SmallVector<const Value *> Indices(GEP->indices());
3035 Cost += getGEPCost(PointeeType: GEP->getSourceElementType(), Ptr: GEP->getPointerOperand(),
3036 Operands: Indices, CostKind, AccessType: AccessTy);
3037 }
3038 }
3039 return Cost;
3040}
3041
3042void RISCVTTIImpl::getUnrollingPreferences(
3043 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
3044 OptimizationRemarkEmitter *ORE) const {
3045 // TODO: More tuning on benchmarks and metrics with changes as needed
3046 // would apply to all settings below to enable performance.
3047
3048
3049 if (ST->enableDefaultUnroll())
3050 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
3051
3052 // Enable Upper bound unrolling universally, not dependent upon the conditions
3053 // below.
3054 UP.UpperBound = true;
3055
3056 // Disable loop unrolling for Oz and Os.
3057 UP.OptSizeThreshold = 0;
3058 UP.PartialOptSizeThreshold = 0;
3059 if (L->getHeader()->getParent()->hasOptSize())
3060 return;
3061
3062 SmallVector<BasicBlock *, 4> ExitingBlocks;
3063 L->getExitingBlocks(ExitingBlocks);
3064 LLVM_DEBUG(dbgs() << "Loop has:\n"
3065 << "Blocks: " << L->getNumBlocks() << "\n"
3066 << "Exit blocks: " << ExitingBlocks.size() << "\n");
3067
3068 // Only allow another exit other than the latch. This acts as an early exit
3069 // as it mirrors the profitability calculation of the runtime unroller.
3070 if (ExitingBlocks.size() > 2)
3071 return;
3072
3073 // Limit the CFG of the loop body for targets with a branch predictor.
3074 // Allowing 4 blocks permits if-then-else diamonds in the body.
3075 if (L->getNumBlocks() > 4)
3076 return;
3077
3078 // Scan the loop: don't unroll loops with calls as this could prevent
3079 // inlining. Don't unroll auto-vectorized loops either, though do allow
3080 // unrolling of the scalar remainder.
3081 bool IsVectorized = getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized");
3082 InstructionCost Cost = 0;
3083 for (auto *BB : L->getBlocks()) {
3084 for (auto &I : *BB) {
3085 // Both auto-vectorized loops and the scalar remainder have the
3086 // isvectorized attribute, so differentiate between them by the presence
3087 // of vector instructions.
3088 if (IsVectorized && (I.getType()->isVectorTy() ||
3089 llvm::any_of(Range: I.operand_values(), P: [](Value *V) {
3090 return V->getType()->isVectorTy();
3091 })))
3092 return;
3093
3094 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
3095 const Function *F = cast<CallBase>(Val&: I).getCalledFunction();
3096 if (!F || isLoweredToCall(F))
3097 return;
3098 }
3099
3100 SmallVector<const Value *> Operands(I.operand_values());
3101 Cost += getInstructionCost(U: &I, Operands,
3102 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
3103 }
3104 }
3105
3106 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
3107
3108 UP.Partial = true;
3109 UP.Runtime = true;
3110 UP.UnrollRemainder = true;
3111 UP.UnrollAndJam = true;
3112
3113 // Force unrolling small loops can be very useful because of the branch
3114 // taken cost of the backedge.
3115 if (Cost < 12)
3116 UP.Force = true;
3117}
3118
3119void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
3120 TTI::PeelingPreferences &PP) const {
3121 BaseT::getPeelingPreferences(L, SE, PP);
3122}
3123
3124bool RISCVTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
3125 MemIntrinsicInfo &Info) const {
3126 const DataLayout &DL = getDataLayout();
3127 Intrinsic::ID IID = Inst->getIntrinsicID();
3128 LLVMContext &C = Inst->getContext();
3129 bool HasMask = false;
3130
3131 auto getSegNum = [](const IntrinsicInst *II, unsigned PtrOperandNo,
3132 bool IsWrite) -> int64_t {
3133 if (auto *TarExtTy =
3134 dyn_cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType()))
3135 return TarExtTy->getIntParameter(i: 0);
3136
3137 return 1;
3138 };
3139
3140 switch (IID) {
3141 case Intrinsic::riscv_vle_mask:
3142 case Intrinsic::riscv_vse_mask:
3143 case Intrinsic::riscv_vlseg2_mask:
3144 case Intrinsic::riscv_vlseg3_mask:
3145 case Intrinsic::riscv_vlseg4_mask:
3146 case Intrinsic::riscv_vlseg5_mask:
3147 case Intrinsic::riscv_vlseg6_mask:
3148 case Intrinsic::riscv_vlseg7_mask:
3149 case Intrinsic::riscv_vlseg8_mask:
3150 case Intrinsic::riscv_vsseg2_mask:
3151 case Intrinsic::riscv_vsseg3_mask:
3152 case Intrinsic::riscv_vsseg4_mask:
3153 case Intrinsic::riscv_vsseg5_mask:
3154 case Intrinsic::riscv_vsseg6_mask:
3155 case Intrinsic::riscv_vsseg7_mask:
3156 case Intrinsic::riscv_vsseg8_mask:
3157 HasMask = true;
3158 [[fallthrough]];
3159 case Intrinsic::riscv_vle:
3160 case Intrinsic::riscv_vse:
3161 case Intrinsic::riscv_vlseg2:
3162 case Intrinsic::riscv_vlseg3:
3163 case Intrinsic::riscv_vlseg4:
3164 case Intrinsic::riscv_vlseg5:
3165 case Intrinsic::riscv_vlseg6:
3166 case Intrinsic::riscv_vlseg7:
3167 case Intrinsic::riscv_vlseg8:
3168 case Intrinsic::riscv_vsseg2:
3169 case Intrinsic::riscv_vsseg3:
3170 case Intrinsic::riscv_vsseg4:
3171 case Intrinsic::riscv_vsseg5:
3172 case Intrinsic::riscv_vsseg6:
3173 case Intrinsic::riscv_vsseg7:
3174 case Intrinsic::riscv_vsseg8: {
3175 // Intrinsic interface:
3176 // riscv_vle(merge, ptr, vl)
3177 // riscv_vle_mask(merge, ptr, mask, vl, policy)
3178 // riscv_vse(val, ptr, vl)
3179 // riscv_vse_mask(val, ptr, mask, vl, policy)
3180 // riscv_vlseg#(merge, ptr, vl, sew)
3181 // riscv_vlseg#_mask(merge, ptr, mask, vl, policy, sew)
3182 // riscv_vsseg#(val, ptr, vl, sew)
3183 // riscv_vsseg#_mask(val, ptr, mask, vl, sew)
3184 bool IsWrite = Inst->getType()->isVoidTy();
3185 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3186 // The results of segment loads are TargetExtType.
3187 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3188 unsigned SEW =
3189 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3190 ->getZExtValue();
3191 Ty = TarExtTy->getTypeParameter(i: 0U);
3192 Ty = ScalableVectorType::get(
3193 ElementType: IntegerType::get(C, NumBits: SEW),
3194 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3195 }
3196 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3197 unsigned VLIndex = RVVIInfo->VLOperand;
3198 unsigned PtrOperandNo = VLIndex - 1 - HasMask;
3199 MaybeAlign Alignment =
3200 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3201 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3202 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3203 if (HasMask)
3204 Mask = Inst->getArgOperand(i: VLIndex - 1);
3205 Value *EVL = Inst->getArgOperand(i: VLIndex);
3206 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3207 // RVV uses contiguous elements as a segment.
3208 if (SegNum > 1) {
3209 unsigned ElemSize = Ty->getScalarSizeInBits();
3210 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3211 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3212 }
3213 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3214 Args&: Alignment, Args&: Mask, Args&: EVL);
3215 return true;
3216 }
3217 case Intrinsic::riscv_vlse_mask:
3218 case Intrinsic::riscv_vsse_mask:
3219 case Intrinsic::riscv_vlsseg2_mask:
3220 case Intrinsic::riscv_vlsseg3_mask:
3221 case Intrinsic::riscv_vlsseg4_mask:
3222 case Intrinsic::riscv_vlsseg5_mask:
3223 case Intrinsic::riscv_vlsseg6_mask:
3224 case Intrinsic::riscv_vlsseg7_mask:
3225 case Intrinsic::riscv_vlsseg8_mask:
3226 case Intrinsic::riscv_vssseg2_mask:
3227 case Intrinsic::riscv_vssseg3_mask:
3228 case Intrinsic::riscv_vssseg4_mask:
3229 case Intrinsic::riscv_vssseg5_mask:
3230 case Intrinsic::riscv_vssseg6_mask:
3231 case Intrinsic::riscv_vssseg7_mask:
3232 case Intrinsic::riscv_vssseg8_mask:
3233 HasMask = true;
3234 [[fallthrough]];
3235 case Intrinsic::riscv_vlse:
3236 case Intrinsic::riscv_vsse:
3237 case Intrinsic::riscv_vlsseg2:
3238 case Intrinsic::riscv_vlsseg3:
3239 case Intrinsic::riscv_vlsseg4:
3240 case Intrinsic::riscv_vlsseg5:
3241 case Intrinsic::riscv_vlsseg6:
3242 case Intrinsic::riscv_vlsseg7:
3243 case Intrinsic::riscv_vlsseg8:
3244 case Intrinsic::riscv_vssseg2:
3245 case Intrinsic::riscv_vssseg3:
3246 case Intrinsic::riscv_vssseg4:
3247 case Intrinsic::riscv_vssseg5:
3248 case Intrinsic::riscv_vssseg6:
3249 case Intrinsic::riscv_vssseg7:
3250 case Intrinsic::riscv_vssseg8: {
3251 // Intrinsic interface:
3252 // riscv_vlse(merge, ptr, stride, vl)
3253 // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
3254 // riscv_vsse(val, ptr, stride, vl)
3255 // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
3256 // riscv_vlsseg#(merge, ptr, offset, vl, sew)
3257 // riscv_vlsseg#_mask(merge, ptr, offset, mask, vl, policy, sew)
3258 // riscv_vssseg#(val, ptr, offset, vl, sew)
3259 // riscv_vssseg#_mask(val, ptr, offset, mask, vl, sew)
3260 bool IsWrite = Inst->getType()->isVoidTy();
3261 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3262 // The results of segment loads are TargetExtType.
3263 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3264 unsigned SEW =
3265 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3266 ->getZExtValue();
3267 Ty = TarExtTy->getTypeParameter(i: 0U);
3268 Ty = ScalableVectorType::get(
3269 ElementType: IntegerType::get(C, NumBits: SEW),
3270 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3271 }
3272 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3273 unsigned VLIndex = RVVIInfo->VLOperand;
3274 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3275 MaybeAlign Alignment =
3276 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3277
3278 Value *Stride = Inst->getArgOperand(i: PtrOperandNo + 1);
3279 // Use the pointer alignment as the element alignment if the stride is a
3280 // multiple of the pointer alignment. Otherwise, the element alignment
3281 // should be the greatest common divisor of pointer alignment and stride.
3282 // For simplicity, just consider unalignment for elements.
3283 unsigned PointerAlign = Alignment.valueOrOne().value();
3284 if (!isa<ConstantInt>(Val: Stride) ||
3285 cast<ConstantInt>(Val: Stride)->getZExtValue() % PointerAlign != 0)
3286 Alignment = Align(1);
3287
3288 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3289 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3290 if (HasMask)
3291 Mask = Inst->getArgOperand(i: VLIndex - 1);
3292 Value *EVL = Inst->getArgOperand(i: VLIndex);
3293 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3294 // RVV uses contiguous elements as a segment.
3295 if (SegNum > 1) {
3296 unsigned ElemSize = Ty->getScalarSizeInBits();
3297 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3298 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3299 }
3300 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3301 Args&: Alignment, Args&: Mask, Args&: EVL, Args&: Stride);
3302 return true;
3303 }
3304 case Intrinsic::riscv_vloxei_mask:
3305 case Intrinsic::riscv_vluxei_mask:
3306 case Intrinsic::riscv_vsoxei_mask:
3307 case Intrinsic::riscv_vsuxei_mask:
3308 case Intrinsic::riscv_vloxseg2_mask:
3309 case Intrinsic::riscv_vloxseg3_mask:
3310 case Intrinsic::riscv_vloxseg4_mask:
3311 case Intrinsic::riscv_vloxseg5_mask:
3312 case Intrinsic::riscv_vloxseg6_mask:
3313 case Intrinsic::riscv_vloxseg7_mask:
3314 case Intrinsic::riscv_vloxseg8_mask:
3315 case Intrinsic::riscv_vluxseg2_mask:
3316 case Intrinsic::riscv_vluxseg3_mask:
3317 case Intrinsic::riscv_vluxseg4_mask:
3318 case Intrinsic::riscv_vluxseg5_mask:
3319 case Intrinsic::riscv_vluxseg6_mask:
3320 case Intrinsic::riscv_vluxseg7_mask:
3321 case Intrinsic::riscv_vluxseg8_mask:
3322 case Intrinsic::riscv_vsoxseg2_mask:
3323 case Intrinsic::riscv_vsoxseg3_mask:
3324 case Intrinsic::riscv_vsoxseg4_mask:
3325 case Intrinsic::riscv_vsoxseg5_mask:
3326 case Intrinsic::riscv_vsoxseg6_mask:
3327 case Intrinsic::riscv_vsoxseg7_mask:
3328 case Intrinsic::riscv_vsoxseg8_mask:
3329 case Intrinsic::riscv_vsuxseg2_mask:
3330 case Intrinsic::riscv_vsuxseg3_mask:
3331 case Intrinsic::riscv_vsuxseg4_mask:
3332 case Intrinsic::riscv_vsuxseg5_mask:
3333 case Intrinsic::riscv_vsuxseg6_mask:
3334 case Intrinsic::riscv_vsuxseg7_mask:
3335 case Intrinsic::riscv_vsuxseg8_mask:
3336 HasMask = true;
3337 [[fallthrough]];
3338 case Intrinsic::riscv_vloxei:
3339 case Intrinsic::riscv_vluxei:
3340 case Intrinsic::riscv_vsoxei:
3341 case Intrinsic::riscv_vsuxei:
3342 case Intrinsic::riscv_vloxseg2:
3343 case Intrinsic::riscv_vloxseg3:
3344 case Intrinsic::riscv_vloxseg4:
3345 case Intrinsic::riscv_vloxseg5:
3346 case Intrinsic::riscv_vloxseg6:
3347 case Intrinsic::riscv_vloxseg7:
3348 case Intrinsic::riscv_vloxseg8:
3349 case Intrinsic::riscv_vluxseg2:
3350 case Intrinsic::riscv_vluxseg3:
3351 case Intrinsic::riscv_vluxseg4:
3352 case Intrinsic::riscv_vluxseg5:
3353 case Intrinsic::riscv_vluxseg6:
3354 case Intrinsic::riscv_vluxseg7:
3355 case Intrinsic::riscv_vluxseg8:
3356 case Intrinsic::riscv_vsoxseg2:
3357 case Intrinsic::riscv_vsoxseg3:
3358 case Intrinsic::riscv_vsoxseg4:
3359 case Intrinsic::riscv_vsoxseg5:
3360 case Intrinsic::riscv_vsoxseg6:
3361 case Intrinsic::riscv_vsoxseg7:
3362 case Intrinsic::riscv_vsoxseg8:
3363 case Intrinsic::riscv_vsuxseg2:
3364 case Intrinsic::riscv_vsuxseg3:
3365 case Intrinsic::riscv_vsuxseg4:
3366 case Intrinsic::riscv_vsuxseg5:
3367 case Intrinsic::riscv_vsuxseg6:
3368 case Intrinsic::riscv_vsuxseg7:
3369 case Intrinsic::riscv_vsuxseg8: {
3370 // Intrinsic interface (only listed ordered version):
3371 // riscv_vloxei(merge, ptr, index, vl)
3372 // riscv_vloxei_mask(merge, ptr, index, mask, vl, policy)
3373 // riscv_vsoxei(val, ptr, index, vl)
3374 // riscv_vsoxei_mask(val, ptr, index, mask, vl, policy)
3375 // riscv_vloxseg#(merge, ptr, index, vl, sew)
3376 // riscv_vloxseg#_mask(merge, ptr, index, mask, vl, policy, sew)
3377 // riscv_vsoxseg#(val, ptr, index, vl, sew)
3378 // riscv_vsoxseg#_mask(val, ptr, index, mask, vl, sew)
3379 bool IsWrite = Inst->getType()->isVoidTy();
3380 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3381 // The results of segment loads are TargetExtType.
3382 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3383 unsigned SEW =
3384 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3385 ->getZExtValue();
3386 Ty = TarExtTy->getTypeParameter(i: 0U);
3387 Ty = ScalableVectorType::get(
3388 ElementType: IntegerType::get(C, NumBits: SEW),
3389 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3390 }
3391 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3392 unsigned VLIndex = RVVIInfo->VLOperand;
3393 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3394 Value *Mask;
3395 if (HasMask) {
3396 Mask = Inst->getArgOperand(i: VLIndex - 1);
3397 } else {
3398 // Mask cannot be nullptr here: vector GEP produces <vscale x N x ptr>,
3399 // and casting that to scalar i64 triggers a vector/scalar mismatch
3400 // assertion in CreatePointerCast. Use an all-true mask so ASan lowers it
3401 // via extractelement instead.
3402 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3403 Mask = ConstantInt::getTrue(Ty: MaskType);
3404 }
3405 Value *EVL = Inst->getArgOperand(i: VLIndex);
3406 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3407 // RVV uses contiguous elements as a segment.
3408 if (SegNum > 1) {
3409 unsigned ElemSize = Ty->getScalarSizeInBits();
3410 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3411 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3412 }
3413 Value *OffsetOp = Inst->getArgOperand(i: PtrOperandNo + 1);
3414 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3415 Args: Align(1), Args&: Mask, Args&: EVL,
3416 /* Stride */ Args: nullptr, Args&: OffsetOp);
3417 return true;
3418 }
3419 }
3420 return false;
3421}
3422
3423unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) const {
3424 if (Ty->isVectorTy()) {
3425 // f16 with only zvfhmin and bf16 will be promoted to f32
3426 Type *EltTy = cast<VectorType>(Val: Ty)->getElementType();
3427 if ((EltTy->isHalfTy() && !ST->hasVInstructionsF16()) ||
3428 EltTy->isBFloatTy())
3429 Ty = VectorType::get(ElementType: Type::getFloatTy(C&: Ty->getContext()),
3430 Other: cast<VectorType>(Val: Ty));
3431
3432 TypeSize Size = DL.getTypeSizeInBits(Ty);
3433 if (Size.isScalable() && ST->hasVInstructions())
3434 return divideCeil(Numerator: Size.getKnownMinValue(), Denominator: RISCV::RVVBitsPerBlock);
3435
3436 if (ST->useRVVForFixedLengthVectors())
3437 return divideCeil(Numerator: Size, Denominator: ST->getRealMinVLen());
3438 }
3439
3440 return BaseT::getRegUsageForType(Ty);
3441}
3442
3443unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
3444 if (SLPMaxVF.getNumOccurrences())
3445 return SLPMaxVF;
3446
3447 // Return how many elements can fit in getRegisterBitwidth. This is the
3448 // same routine as used in LoopVectorizer. We should probably be
3449 // accounting for whether we actually have instructions with the right
3450 // lane type, but we don't have enough information to do that without
3451 // some additional plumbing which hasn't been justified yet.
3452 TypeSize RegWidth =
3453 getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
3454 // If no vector registers, or absurd element widths, disable
3455 // vectorization by returning 1.
3456 return std::max<unsigned>(a: 1U, b: RegWidth.getFixedValue() / ElemWidth);
3457}
3458
3459unsigned RISCVTTIImpl::getMinTripCountTailFoldingThreshold() const {
3460 return RVVMinTripCount;
3461}
3462
3463bool RISCVTTIImpl::preferAlternateOpcodeVectorization() const {
3464 return ST->enableUnalignedVectorMem();
3465}
3466
3467TTI::AddressingModeKind
3468RISCVTTIImpl::getPreferredAddressingMode(const Loop *L,
3469 ScalarEvolution *SE) const {
3470 if (ST->hasVendorXCVmem() && !ST->is64Bit())
3471 return TTI::AMK_PostIndexed;
3472
3473 return BasicTTIImplBase::getPreferredAddressingMode(L, SE);
3474}
3475
3476bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
3477 const TargetTransformInfo::LSRCost &C2) const {
3478 // RISC-V specific here are "instruction number 1st priority".
3479 // If we need to emit adds inside the loop to add up base registers, then
3480 // we need at least one extra temporary register.
3481 unsigned C1NumRegs = C1.NumRegs + (C1.NumBaseAdds != 0);
3482 unsigned C2NumRegs = C2.NumRegs + (C2.NumBaseAdds != 0);
3483 return std::tie(args: C1.Insns, args&: C1NumRegs, args: C1.AddRecCost,
3484 args: C1.NumIVMuls, args: C1.NumBaseAdds,
3485 args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
3486 std::tie(args: C2.Insns, args&: C2NumRegs, args: C2.AddRecCost,
3487 args: C2.NumIVMuls, args: C2.NumBaseAdds,
3488 args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
3489}
3490
3491bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
3492 Align Alignment) const {
3493 auto *VTy = dyn_cast<VectorType>(Val: DataTy);
3494 if (!VTy || VTy->isScalableTy())
3495 return false;
3496
3497 if (!isLegalMaskedLoadStore(DataType: DataTy, Alignment))
3498 return false;
3499
3500 // FIXME: If it is an i8 vector and the element count exceeds 256, we should
3501 // scalarize these types with LMUL >= maximum fixed-length LMUL.
3502 if (VTy->getElementType()->isIntegerTy(BitWidth: 8))
3503 if (VTy->getElementCount().getFixedValue() > 256)
3504 return VTy->getPrimitiveSizeInBits() / ST->getRealMinVLen() <
3505 ST->getMaxLMULForFixedLengthVectors();
3506 return true;
3507}
3508
3509bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,
3510 Align Alignment) const {
3511 auto *VTy = dyn_cast<VectorType>(Val: DataTy);
3512 if (!VTy || VTy->isScalableTy())
3513 return false;
3514
3515 if (!isLegalMaskedLoadStore(DataType: DataTy, Alignment))
3516 return false;
3517 return true;
3518}
3519
3520bool RISCVTTIImpl::isLegalBroadcastLoad(Type *ElementTy,
3521 ElementCount NumElements) const {
3522 // Optimized zero-stride loads can be treated as broadcasts.
3523 if (!ST->hasVInstructions() || !ST->hasOptimizedZeroStrideLoad())
3524 return false;
3525
3526 return TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: ElementTy));
3527}
3528
3529/// See if \p I should be considered for address type promotion. We check if \p
3530/// I is a sext with right type and used in memory accesses. If it used in a
3531/// "complex" getelementptr, we allow it to be promoted without finding other
3532/// sext instructions that sign extended the same initial value. A getelementptr
3533/// is considered as "complex" if it has more than 2 operands.
3534bool RISCVTTIImpl::shouldConsiderAddressTypePromotion(
3535 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
3536 bool Considerable = false;
3537 AllowPromotionWithoutCommonHeader = false;
3538 if (!isa<SExtInst>(Val: &I))
3539 return false;
3540 Type *ConsideredSExtType =
3541 Type::getInt64Ty(C&: I.getParent()->getParent()->getContext());
3542 if (I.getType() != ConsideredSExtType)
3543 return false;
3544 // See if the sext is the one with the right type and used in at least one
3545 // GetElementPtrInst.
3546 for (const User *U : I.users()) {
3547 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(Val: U)) {
3548 Considerable = true;
3549 // A getelementptr is considered as "complex" if it has more than 2
3550 // operands. We will promote a SExt used in such complex GEP as we
3551 // expect some computation to be merged if they are done on 64 bits.
3552 if (GEPInst->getNumOperands() > 2) {
3553 AllowPromotionWithoutCommonHeader = true;
3554 break;
3555 }
3556 }
3557 }
3558 return Considerable;
3559}
3560
3561bool RISCVTTIImpl::canSplatOperand(unsigned Opcode, int Operand) const {
3562 switch (Opcode) {
3563 case Instruction::Add:
3564 case Instruction::Sub:
3565 case Instruction::Mul:
3566 case Instruction::And:
3567 case Instruction::Or:
3568 case Instruction::Xor:
3569 case Instruction::FAdd:
3570 case Instruction::FSub:
3571 case Instruction::FMul:
3572 case Instruction::FDiv:
3573 case Instruction::ICmp:
3574 case Instruction::FCmp:
3575 return true;
3576 case Instruction::Shl:
3577 case Instruction::LShr:
3578 case Instruction::AShr:
3579 case Instruction::UDiv:
3580 case Instruction::SDiv:
3581 case Instruction::URem:
3582 case Instruction::SRem:
3583 case Instruction::Select:
3584 return Operand == 1;
3585 default:
3586 return false;
3587 }
3588}
3589
3590bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {
3591 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3592 return false;
3593
3594 if (canSplatOperand(Opcode: I->getOpcode(), Operand))
3595 return true;
3596
3597 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3598 if (!II)
3599 return false;
3600
3601 switch (II->getIntrinsicID()) {
3602 case Intrinsic::fma:
3603 case Intrinsic::fmuladd:
3604 return Operand == 0 || Operand == 1;
3605 case Intrinsic::vp_udiv:
3606 case Intrinsic::vp_sdiv:
3607 case Intrinsic::vp_urem:
3608 case Intrinsic::vp_srem:
3609 case Intrinsic::ssub_sat:
3610 case Intrinsic::usub_sat:
3611 return Operand == 1;
3612 // These intrinsics are commutative.
3613 case Intrinsic::smin:
3614 case Intrinsic::umin:
3615 case Intrinsic::smax:
3616 case Intrinsic::umax:
3617 case Intrinsic::sadd_sat:
3618 case Intrinsic::uadd_sat:
3619 return Operand == 0 || Operand == 1;
3620 default:
3621 return false;
3622 }
3623}
3624
3625/// Check if sinking \p I's operands to I's basic block is profitable, because
3626/// the operands can be folded into a target instruction, e.g.
3627/// splats of scalars can fold into vector instructions.
3628bool RISCVTTIImpl::isProfitableToSinkOperands(
3629 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
3630 using namespace llvm::PatternMatch;
3631
3632 if (I->isBitwiseLogicOp()) {
3633 if (!I->getType()->isVectorTy()) {
3634 if (ST->hasStdExtZbb() || ST->hasStdExtZbkb()) {
3635 for (auto &Op : I->operands()) {
3636 // (and/or/xor X, (not Y)) -> (andn/orn/xnor X, Y)
3637 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3638 Ops.push_back(Elt: &Op);
3639 return true;
3640 }
3641 }
3642 }
3643 } else if (I->getOpcode() == Instruction::And && ST->hasStdExtZvkb()) {
3644 for (auto &Op : I->operands()) {
3645 // (and X, (not Y)) -> (vandn.vv X, Y)
3646 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3647 Ops.push_back(Elt: &Op);
3648 return true;
3649 }
3650 // (and X, (splat (not Y))) -> (vandn.vx X, Y)
3651 if (match(V: Op.get(), P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()),
3652 Idx: m_ZeroInt()),
3653 v2: m_Value(), mask: m_ZeroMask()))) {
3654 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
3655 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
3656 Ops.push_back(Elt: &Not);
3657 Ops.push_back(Elt: &InsertElt);
3658 Ops.push_back(Elt: &Op);
3659 return true;
3660 }
3661 }
3662 }
3663 }
3664
3665 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3666 return false;
3667
3668 // Don't sink splat operands if the target prefers it. Some targets requires
3669 // S2V transfer buffers and we can run out of them copying the same value
3670 // repeatedly.
3671 // FIXME: It could still be worth doing if it would improve vector register
3672 // pressure and prevent a vector spill.
3673 if (!ST->sinkSplatOperands())
3674 return false;
3675
3676 for (auto OpIdx : enumerate(First: I->operands())) {
3677 if (!canSplatOperand(I, Operand: OpIdx.index()))
3678 continue;
3679
3680 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
3681 // Make sure we are not already sinking this operand
3682 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
3683 continue;
3684
3685 // We are looking for a splat that can be sunk.
3686 if (!match(V: Op, P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ZeroInt()),
3687 v2: m_Value(), mask: m_ZeroMask())))
3688 continue;
3689
3690 // Don't sink i1 splats.
3691 if (cast<VectorType>(Val: Op->getType())->getElementType()->isIntegerTy(BitWidth: 1))
3692 continue;
3693
3694 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
3695 // and vector registers
3696 for (Use &U : Op->uses()) {
3697 Instruction *Insn = cast<Instruction>(Val: U.getUser());
3698 if (!canSplatOperand(I: Insn, Operand: U.getOperandNo()))
3699 return false;
3700 }
3701
3702 // Sink any fpexts since they might be used in a widening fp pattern.
3703 Use *InsertEltUse = &Op->getOperandUse(i: 0);
3704 auto *InsertElt = cast<InsertElementInst>(Val: InsertEltUse);
3705 if (isa<FPExtInst>(Val: InsertElt->getOperand(i_nocapture: 1)))
3706 Ops.push_back(Elt: &InsertElt->getOperandUse(i: 1));
3707 Ops.push_back(Elt: InsertEltUse);
3708 Ops.push_back(Elt: &OpIdx.value());
3709 }
3710 return true;
3711}
3712
3713RISCVTTIImpl::TTI::MemCmpExpansionOptions
3714RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
3715 TTI::MemCmpExpansionOptions Options;
3716
3717 if (!ST->hasStdExtZbb() && !ST->hasStdExtZbkb() && !IsZeroCmp)
3718 return Options;
3719
3720 Options.AllowOverlappingLoads = true;
3721 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
3722 Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
3723 if (ST->is64Bit()) {
3724 Options.LoadSizes = {8, 4, 2, 1};
3725 Options.AllowedTailExpansions = {3, 5, 6};
3726 } else {
3727 Options.LoadSizes = {4, 2, 1};
3728 Options.AllowedTailExpansions = {3};
3729 }
3730
3731 if (IsZeroCmp && ST->hasVInstructions()) {
3732 unsigned VLenB = ST->getRealMinVLen() / 8;
3733 // The minimum size should be `XLen / 8 + 1`, and the maxinum size should be
3734 // `VLenB * MaxLMUL` so that it fits in a single register group.
3735 unsigned MinSize = ST->getXLen() / 8 + 1;
3736 unsigned MaxSize = VLenB * ST->getMaxLMULForFixedLengthVectors();
3737 for (unsigned Size = MinSize; Size <= MaxSize; Size++)
3738 Options.LoadSizes.insert(I: Options.LoadSizes.begin(), Elt: Size);
3739 }
3740 return Options;
3741}
3742
3743bool RISCVTTIImpl::shouldTreatInstructionLikeSelect(
3744 const Instruction *I) const {
3745 if (EnableOrLikeSelectOpt) {
3746 // For the binary operators (e.g. or) we need to be more careful than
3747 // selects, here we only transform them if they are already at a natural
3748 // break point in the code - the end of a block with an unconditional
3749 // terminator.
3750 if (I->getOpcode() == Instruction::Or &&
3751 isa<UncondBrInst>(Val: I->getNextNode()))
3752 return true;
3753
3754 if (I->getOpcode() == Instruction::Add ||
3755 I->getOpcode() == Instruction::Sub)
3756 return true;
3757 }
3758 return BaseT::shouldTreatInstructionLikeSelect(I);
3759}
3760
3761bool RISCVTTIImpl::shouldCopyAttributeWhenOutliningFrom(
3762 const Function *Caller, const Attribute &Attr) const {
3763 // "interrupt" controls the prolog/epilog of interrupt handlers (and includes
3764 // restrictions on their signatures). We can outline from the bodies of these
3765 // handlers, but when we do we need to make sure we don't mark the outlined
3766 // function as an interrupt handler too.
3767 if (Attr.isStringAttribute() && Attr.getKindAsString() == "interrupt")
3768 return false;
3769
3770 return BaseT::shouldCopyAttributeWhenOutliningFrom(Caller, Attr);
3771}
3772
3773std::optional<Instruction *>
3774RISCVTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
3775 // Attach a range return attribute describing the result of vsetvli/vsetvlimax
3776 // so generic value analyses can reason about it. The verifier guarantees an
3777 // XLen result and constant VSEW/VLMUL encoding a valid vtype, so no defensive
3778 // validation is needed here.
3779 if (is_contained(Set: {Intrinsic::riscv_vsetvli, Intrinsic::riscv_vsetvlimax},
3780 Element: II.getIntrinsicID())) {
3781 // These intrinsics require the V extension; without it the VLEN queries
3782 // below would assert. Such IR would fail isel anyway, so just bail out.
3783 if (!ST->hasVInstructions())
3784 return {};
3785
3786 bool HasAVL = II.getIntrinsicID() == Intrinsic::riscv_vsetvli;
3787 unsigned Offset = HasAVL ? 1 : 0;
3788 unsigned BitWidth = II.getType()->getIntegerBitWidth();
3789 ConstantRange VLenRange(APInt(BitWidth, ST->getRealMinVLen()),
3790 APInt(BitWidth, ST->getRealMaxVLen()) + 1);
3791
3792 uint64_t VSEW = cast<ConstantInt>(Val: II.getArgOperand(i: Offset))->getZExtValue();
3793 auto VLMUL = static_cast<RISCVVType::VLMUL>(
3794 cast<ConstantInt>(Val: II.getArgOperand(i: Offset + 1))->getZExtValue());
3795 unsigned SEW = RISCVVType::decodeVSEW(VSEW);
3796 unsigned Ratio = RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
3797
3798 // VLMAX = VLEN / (SEW / LMUL), clamped to >= 1 for any usable vtype.
3799 ConstantRange VLMAXRange =
3800 VLenRange.udiv(Other: ConstantRange(APInt(BitWidth, Ratio)))
3801 .umax(Other: ConstantRange(APInt(BitWidth, 1)));
3802
3803 // vsetvlimax returns exactly VLMAX; vsetvli returns vl with
3804 // 0 <= vl <= min(AVL, VLMAX). vl == AVL only when AVL <= the smallest
3805 // possible VLMAX; otherwise vl can shrink below VLMAX (to 0 at runtime), so
3806 // only the VLMAX upper bound is sound.
3807 ConstantRange VLRange = VLMAXRange;
3808 if (HasAVL) {
3809 // vl ≤ VLMAX
3810 VLRange =
3811 ConstantRange::makeAllowedICmpRegion(Pred: CmpInst::ICMP_ULE, Other: VLMAXRange);
3812
3813 Value *AVL = II.getArgOperand(i: 0);
3814 ConstantRange AVLRange = computeConstantRangeIncludingKnownBits(
3815 V: AVL, /*ForSigned=*/false,
3816 SQ: IC.getSimplifyQuery().getWithInstruction(I: &II));
3817
3818 // vl = AVL if AVL ≤ VLMAX
3819 if (AVLRange.icmp(Pred: CmpInst::ICMP_ULE, Other: VLMAXRange))
3820 return IC.replaceInstUsesWith(I&: II, V: AVL);
3821
3822 // vl ≤ AVL
3823 VLRange = VLRange.umin(Other: AVLRange.getUnsignedMax());
3824
3825 // vl > 0 if AVL > 0
3826 if (AVLRange.icmp(Pred: CmpInst::ICMP_UGT, Other: APInt::getZero(numBits: BitWidth)))
3827 VLRange = VLRange.umax(Other: APInt(BitWidth, 1));
3828
3829 // vl = VLMAX if AVL ≥ (2 * VLMAX)
3830 ConstantRange TwoVLMAX = VLMAXRange.multiply(Other: APInt(BitWidth, 2));
3831 if (AVLRange.icmp(Pred: CmpInst::ICMP_UGE, Other: TwoVLMAX))
3832 VLRange = VLRange.intersectWith(CR: VLMAXRange);
3833
3834 // ceil(AVL / 2) ≤ vl ≤ VLMAX if AVL < (2 * VLMAX)
3835 if (AVLRange.icmp(Pred: CmpInst::ICMP_ULT, Other: TwoVLMAX))
3836 VLRange = VLRange.umax(Other: APIntOps::RoundingUDiv(A: AVLRange.getUnsignedMin(),
3837 B: APInt(BitWidth, 2),
3838 RM: APInt::Rounding::UP));
3839 }
3840
3841 ConstantRange OldRange =
3842 II.getRange().value_or(u: ConstantRange::getFull(BitWidth));
3843 ConstantRange NewRange = VLRange.intersectWith(CR: OldRange);
3844 if (NewRange != OldRange) {
3845 II.addRangeRetAttr(CR: NewRange);
3846 return &II;
3847 }
3848 return {};
3849 }
3850
3851 // If all operands of a vmv.v.x are constant, fold a bitcast(vmv.v.x) to scale
3852 // the vmv.v.x, enabling removal of the bitcast. The transform helps avoid
3853 // creating redundant masks.
3854 const DataLayout &DL = IC.getDataLayout();
3855 if (II.user_empty())
3856 return {};
3857 auto *TargetVecTy = dyn_cast<ScalableVectorType>(Val: II.user_back()->getType());
3858 if (!TargetVecTy)
3859 return {};
3860 const APInt *Scalar;
3861 uint64_t VL;
3862 if (!match(V: &II, P: m_Intrinsic<Intrinsic::riscv_vmv_v_x>(
3863 Ops: m_Poison(), Ops: m_APInt(Res&: Scalar), Ops: m_ConstantInt(V&: VL))) ||
3864 !all_of(Range: II.users(), P: [TargetVecTy](User *U) {
3865 return U->getType() == TargetVecTy && match(V: U, P: m_BitCast(Op: m_Value()));
3866 }))
3867 return {};
3868 auto *SourceVecTy = cast<ScalableVectorType>(Val: II.getType());
3869 unsigned TargetEltBW = DL.getTypeSizeInBits(Ty: TargetVecTy->getElementType());
3870 unsigned SourceEltBW = DL.getTypeSizeInBits(Ty: SourceVecTy->getElementType());
3871 if (TargetEltBW % SourceEltBW)
3872 return {};
3873 unsigned TargetScale = TargetEltBW / SourceEltBW;
3874 if (VL % TargetScale || TargetScale == 1)
3875 return {};
3876 Type *VLTy = II.getOperand(i_nocapture: 2)->getType();
3877 ElementCount SourceEC = SourceVecTy->getElementCount();
3878 unsigned NewEltBW = SourceEltBW * TargetScale;
3879 if (!SourceEC.isKnownMultipleOf(RHS: TargetScale) ||
3880 !DL.fitsInLegalInteger(Width: NewEltBW))
3881 return {};
3882 auto *NewEltTy = IntegerType::get(C&: II.getContext(), NumBits: NewEltBW);
3883 if (!TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: NewEltTy)))
3884 return {};
3885 ElementCount NewEC = SourceEC.divideCoefficientBy(RHS: TargetScale);
3886 Type *RetTy = VectorType::get(ElementType: NewEltTy, EC: NewEC);
3887 assert(SourceVecTy->canLosslesslyBitCastTo(RetTy) &&
3888 "Lossless bitcast between types expected");
3889 APInt NewScalar = APInt::getSplat(NewLen: NewEltBW, V: *Scalar);
3890 return IC.replaceInstUsesWith(
3891 I&: II,
3892 V: IC.Builder.CreateBitCast(
3893 V: IC.Builder.CreateIntrinsic(
3894 RetTy, ID: Intrinsic::riscv_vmv_v_x,
3895 Args: {PoisonValue::get(T: RetTy), ConstantInt::get(Ty: NewEltTy, V: NewScalar),
3896 ConstantInt::get(Ty: VLTy, V: VL / TargetScale)}),
3897 DestTy: SourceVecTy));
3898}
3899