1//===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "RISCVTargetTransformInfo.h"
10#include "MCTargetDesc/RISCVMatInt.h"
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Analysis/TargetTransformInfo.h"
13#include "llvm/CodeGen/BasicTTIImpl.h"
14#include "llvm/CodeGen/CostTable.h"
15#include "llvm/CodeGen/TargetLowering.h"
16#include "llvm/CodeGen/ValueTypes.h"
17#include "llvm/IR/Instructions.h"
18#include "llvm/IR/IntrinsicsRISCV.h"
19#include "llvm/IR/PatternMatch.h"
20#include "llvm/Transforms/InstCombine/InstCombiner.h"
21#include <cmath>
22#include <optional>
23using namespace llvm;
24using namespace llvm::PatternMatch;
25
26#define DEBUG_TYPE "riscvtti"
27
28static cl::opt<unsigned> RVVRegisterWidthLMUL(
29 "riscv-v-register-bit-width-lmul",
30 cl::desc(
31 "The LMUL to use for getRegisterBitWidth queries. Affects LMUL used "
32 "by autovectorized code. Fractional LMULs are not supported."),
33 cl::init(Val: 2), cl::Hidden);
34
35static cl::opt<unsigned> SLPMaxVF(
36 "riscv-v-slp-max-vf",
37 cl::desc(
38 "Overrides result used for getMaximumVF query which is used "
39 "exclusively by SLP vectorizer."),
40 cl::Hidden);
41
42static cl::opt<unsigned>
43 RVVMinTripCount("riscv-v-min-trip-count",
44 cl::desc("Set the lower bound of a trip count to decide on "
45 "vectorization while tail-folding."),
46 cl::init(Val: 5), cl::Hidden);
47
48static cl::opt<bool> EnableOrLikeSelectOpt("enable-riscv-or-like-select",
49 cl::init(Val: true), cl::Hidden);
50
51InstructionCost
52RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
53 TTI::TargetCostKind CostKind) const {
54 // Check if the type is valid for all CostKind
55 if (!VT.isVector())
56 return InstructionCost::getInvalid();
57 size_t NumInstr = OpCodes.size();
58 if (CostKind == TTI::TCK_CodeSize)
59 return NumInstr;
60 InstructionCost LMULCost = TLI->getLMULCost(VT);
61 if ((CostKind != TTI::TCK_RecipThroughput) && (CostKind != TTI::TCK_Latency))
62 return LMULCost * NumInstr;
63 InstructionCost Cost = 0;
64 for (auto Op : OpCodes) {
65 switch (Op) {
66 case RISCV::VRGATHER_VI:
67 Cost += TLI->getVRGatherVICost(VT);
68 break;
69 case RISCV::VRGATHER_VV:
70 Cost += TLI->getVRGatherVVCost(VT);
71 break;
72 case RISCV::VSLIDEUP_VI:
73 case RISCV::VSLIDEDOWN_VI:
74 Cost += TLI->getVSlideVICost(VT);
75 break;
76 case RISCV::VSLIDEUP_VX:
77 case RISCV::VSLIDEDOWN_VX:
78 Cost += TLI->getVSlideVXCost(VT);
79 break;
80 case RISCV::VREDMAX_VS:
81 case RISCV::VREDMIN_VS:
82 case RISCV::VREDMAXU_VS:
83 case RISCV::VREDMINU_VS:
84 case RISCV::VREDSUM_VS:
85 case RISCV::VREDAND_VS:
86 case RISCV::VREDOR_VS:
87 case RISCV::VREDXOR_VS:
88 case RISCV::VFREDMAX_VS:
89 case RISCV::VFREDMIN_VS:
90 case RISCV::VFREDUSUM_VS: {
91 unsigned VL = VT.getVectorMinNumElements();
92 if (!VT.isFixedLengthVector())
93 VL *= *getVScaleForTuning();
94 Cost += Log2_32_Ceil(Value: VL);
95 break;
96 }
97 case RISCV::VFREDOSUM_VS: {
98 unsigned VL = VT.getVectorMinNumElements();
99 if (!VT.isFixedLengthVector())
100 VL *= *getVScaleForTuning();
101 Cost += VL;
102 break;
103 }
104 case RISCV::VMV_X_S:
105 case RISCV::VFMV_F_S:
106 // Domain crossings from vector -> scalar are usually more expensive.
107 Cost += 2;
108 break;
109 case RISCV::VMV_S_X:
110 case RISCV::VFMV_S_F:
111 case RISCV::VMOR_MM:
112 case RISCV::VMXOR_MM:
113 case RISCV::VMAND_MM:
114 case RISCV::VMANDN_MM:
115 case RISCV::VMNAND_MM:
116 case RISCV::VCPOP_M:
117 case RISCV::VFIRST_M:
118 Cost += 1;
119 break;
120 case RISCV::VDIV_VV:
121 case RISCV::VREM_VV:
122 Cost += LMULCost * TTI::TCC_Expensive;
123 break;
124 default:
125 Cost += LMULCost;
126 }
127 }
128 return Cost;
129}
130
131static InstructionCost getIntImmCostImpl(const DataLayout &DL,
132 const RISCVSubtarget *ST,
133 const APInt &Imm, Type *Ty,
134 TTI::TargetCostKind CostKind,
135 bool FreeZeroes) {
136 assert(Ty->isIntegerTy() &&
137 "getIntImmCost can only estimate cost of materialising integers");
138
139 // We have a Zero register, so 0 is always free.
140 if (Imm == 0)
141 return TTI::TCC_Free;
142
143 // Otherwise, we check how many instructions it will take to materialise.
144 return RISCVMatInt::getIntMatCost(Val: Imm, Size: DL.getTypeSizeInBits(Ty), STI: *ST,
145 /*CompressionCost=*/false, FreeZeroes);
146}
147
148InstructionCost
149RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
150 TTI::TargetCostKind CostKind) const {
151 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind, FreeZeroes: false);
152}
153
154// Look for patterns of shift followed by AND that can be turned into a pair of
155// shifts. We won't need to materialize an immediate for the AND so these can
156// be considered free.
157static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) {
158 uint64_t Mask = Imm.getZExtValue();
159 auto *BO = dyn_cast<BinaryOperator>(Val: Inst->getOperand(i: 0));
160 if (!BO || !BO->hasOneUse())
161 return false;
162
163 if (BO->getOpcode() != Instruction::Shl)
164 return false;
165
166 if (!isa<ConstantInt>(Val: BO->getOperand(i_nocapture: 1)))
167 return false;
168
169 unsigned ShAmt = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1))->getZExtValue();
170 // (and (shl x, c2), c1) will be matched to (srli (slli x, c2+c3), c3) if c1
171 // is a mask shifted by c2 bits with c3 leading zeros.
172 if (isShiftedMask_64(Value: Mask)) {
173 unsigned Trailing = llvm::countr_zero(Val: Mask);
174 if (ShAmt == Trailing)
175 return true;
176 }
177
178 return false;
179}
180
181// If this is i64 AND is part of (X & -(1 << C1) & 0xffffffff) == C2 << C1),
182// DAGCombiner can convert this to (sraiw X, C1) == sext(C2) for RV64. On RV32,
183// the type will be split so only the lower 32 bits need to be compared using
184// (srai/srli X, C) == C2.
185static bool canUseShiftCmp(Instruction *Inst, const APInt &Imm) {
186 if (!Inst->hasOneUse())
187 return false;
188
189 // Look for equality comparison.
190 auto *Cmp = dyn_cast<ICmpInst>(Val: *Inst->user_begin());
191 if (!Cmp || !Cmp->isEquality())
192 return false;
193
194 // Right hand side of comparison should be a constant.
195 auto *C = dyn_cast<ConstantInt>(Val: Cmp->getOperand(i_nocapture: 1));
196 if (!C)
197 return false;
198
199 uint64_t Mask = Imm.getZExtValue();
200
201 // Mask should be of the form -(1 << C) in the lower 32 bits.
202 if (!isUInt<32>(x: Mask) || !isPowerOf2_32(Value: -uint32_t(Mask)))
203 return false;
204
205 // Comparison constant should be a subset of Mask.
206 uint64_t CmpC = C->getZExtValue();
207 if ((CmpC & Mask) != CmpC)
208 return false;
209
210 // We'll need to sign extend the comparison constant and shift it right. Make
211 // sure the new constant can use addi/xori+seqz/snez.
212 unsigned ShiftBits = llvm::countr_zero(Val: Mask);
213 int64_t NewCmpC = SignExtend64<32>(x: CmpC) >> ShiftBits;
214 return NewCmpC >= -2048 && NewCmpC <= 2048;
215}
216
217InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
218 const APInt &Imm, Type *Ty,
219 TTI::TargetCostKind CostKind,
220 Instruction *Inst) const {
221 assert(Ty->isIntegerTy() &&
222 "getIntImmCost can only estimate cost of materialising integers");
223
224 // We have a Zero register, so 0 is always free.
225 if (Imm == 0)
226 return TTI::TCC_Free;
227
228 // Some instructions in RISC-V can take a 12-bit immediate. Some of these are
229 // commutative, in others the immediate comes from a specific argument index.
230 bool Takes12BitImm = false;
231 unsigned ImmArgIdx = ~0U;
232
233 switch (Opcode) {
234 case Instruction::GetElementPtr:
235 // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will
236 // split up large offsets in GEP into better parts than ConstantHoisting
237 // can.
238 return TTI::TCC_Free;
239 case Instruction::Store: {
240 // Use the materialization cost regardless of if it's the address or the
241 // value that is constant, except for if the store is misaligned and
242 // misaligned accesses are not legal (experience shows constant hoisting
243 // can sometimes be harmful in such cases).
244 if (Idx == 1 || !Inst)
245 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
246 /*FreeZeroes=*/true);
247
248 StoreInst *ST = cast<StoreInst>(Val: Inst);
249 if (!getTLI()->allowsMemoryAccessForAlignment(
250 Context&: Ty->getContext(), DL, VT: getTLI()->getValueType(DL, Ty),
251 AddrSpace: ST->getPointerAddressSpace(), Alignment: ST->getAlign()))
252 return TTI::TCC_Free;
253
254 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
255 /*FreeZeroes=*/true);
256 }
257 case Instruction::Load:
258 // If the address is a constant, use the materialization cost.
259 return getIntImmCost(Imm, Ty, CostKind);
260 case Instruction::And:
261 // zext.h
262 if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())
263 return TTI::TCC_Free;
264 // zext.w
265 if (Imm == UINT64_C(0xffffffff) &&
266 ((ST->hasStdExtZba() && ST->isRV64()) || ST->isRV32()))
267 return TTI::TCC_Free;
268 // bclri
269 if (ST->hasStdExtZbs() && (~Imm).isPowerOf2())
270 return TTI::TCC_Free;
271 if (Inst && Idx == 1 && Imm.getBitWidth() <= ST->getXLen() &&
272 canUseShiftPair(Inst, Imm))
273 return TTI::TCC_Free;
274 if (Inst && Idx == 1 && Imm.getBitWidth() == 64 &&
275 canUseShiftCmp(Inst, Imm))
276 return TTI::TCC_Free;
277 Takes12BitImm = true;
278 break;
279 case Instruction::Add:
280 Takes12BitImm = true;
281 break;
282 case Instruction::Or:
283 case Instruction::Xor:
284 // bseti/binvi
285 if (ST->hasStdExtZbs() && Imm.isPowerOf2())
286 return TTI::TCC_Free;
287 Takes12BitImm = true;
288 break;
289 case Instruction::Mul:
290 // Power of 2 is a shift. Negated power of 2 is a shift and a negate.
291 if (Imm.isPowerOf2() || Imm.isNegatedPowerOf2())
292 return TTI::TCC_Free;
293 // One more or less than a power of 2 can use SLLI+ADD/SUB.
294 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2())
295 return TTI::TCC_Free;
296 // FIXME: There is no MULI instruction.
297 Takes12BitImm = true;
298 break;
299 case Instruction::Sub:
300 case Instruction::Shl:
301 case Instruction::LShr:
302 case Instruction::AShr:
303 Takes12BitImm = true;
304 ImmArgIdx = 1;
305 break;
306 default:
307 break;
308 }
309
310 if (Takes12BitImm) {
311 // Check immediate is the correct argument...
312 if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {
313 // ... and fits into the 12-bit immediate.
314 if (Imm.getSignificantBits() <= 64 &&
315 getTLI()->isLegalAddImmediate(Imm: Imm.getSExtValue())) {
316 return TTI::TCC_Free;
317 }
318 }
319
320 // Otherwise, use the full materialisation cost.
321 return getIntImmCost(Imm, Ty, CostKind);
322 }
323
324 // By default, prevent hoisting.
325 return TTI::TCC_Free;
326}
327
328InstructionCost
329RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
330 const APInt &Imm, Type *Ty,
331 TTI::TargetCostKind CostKind) const {
332 // Prevent hoisting in unknown cases.
333 return TTI::TCC_Free;
334}
335
336bool RISCVTTIImpl::hasActiveVectorLength() const {
337 return ST->hasVInstructions();
338}
339
340TargetTransformInfo::PopcntSupportKind
341RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {
342 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
343 return ST->hasCPOPLike() ? TTI::PSK_FastHardware : TTI::PSK_Software;
344}
345
346InstructionCost RISCVTTIImpl::getPartialReductionCost(
347 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
348 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
349 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
350 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
351 if (Opcode == Instruction::FAdd)
352 return InstructionCost::getInvalid();
353
354 // zve32x is broken for partial_reduce_umla, but let's make sure we
355 // don't generate them.
356 // vdot4a* reduces four i8 products into an i32 result; an i64 accumulator is
357 // additionally supported by widening the i32 partial sums to i64 (see
358 // lowerPARTIAL_REDUCE_MLA). VF is the number of i8 input elements, so the
359 // reduction factor is AccumBits / 8 (4 for i32, 8 for i64).
360 if (!ST->hasStdExtZvdot4a8i() || ST->getELen() < 64 ||
361 Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||
362 InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(BitWidth: 8) ||
363 (!AccumType->isIntegerTy(BitWidth: 32) && !AccumType->isIntegerTy(BitWidth: 64)))
364 return InstructionCost::getInvalid();
365
366 unsigned Ratio = AccumType->getScalarSizeInBits() / 8;
367 if (!VF.isKnownMultipleOf(RHS: Ratio))
368 return InstructionCost::getInvalid();
369
370 // Cost of the vdot4a* itself, which operates on the i32 intermediate type
371 // holding VF/4 elements.
372 Type *DotTp = VectorType::get(ElementType: Type::getInt32Ty(C&: AccumType->getContext()),
373 EC: VF.divideCoefficientBy(RHS: 4));
374 std::pair<InstructionCost, MVT> DotLT = getTypeLegalizationCost(Ty: DotTp);
375 // Note: Asuming all vdot4a* variants are equal cost
376 InstructionCost Cost =
377 DotLT.first *
378 getRISCVInstructionCost(OpCodes: RISCV::VDOT4A_VV, VT: DotLT.second, CostKind);
379
380 // Account for reducing the i32 partial sums down to the i64 accumulator's
381 // element count and accumulating into it (see lowerPARTIAL_REDUCE_MLA), which
382 // has two shapes depending on the accumulator's LMUL.
383 if (AccumType->isIntegerTy(BitWidth: 64)) {
384 LLVMContext &Ctx = AccumType->getContext();
385 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
386 ElementCount AccVF = VF.divideCoefficientBy(RHS: Ratio);
387 std::pair<InstructionCost, MVT> AccLT =
388 getTypeLegalizationCost(Ty: VectorType::get(ElementType: AccumType, EC: AccVF));
389
390 // When the i32 subvectors of a single-vector scalable accumulator are a
391 // fractional LMUL, extracting the high subvector would need a vslidedown,
392 // so instead the i32 dot result is widened to i64 first (vsext.vf2 /
393 // vzext.vf2) and then reduced and accumulated with register-aligned i64
394 // vadd.vv.
395 bool WidenFirst = false;
396 if (VF.isScalable() && AccLT.second.isScalableVector()) {
397 MVT NarrowMVT = AccLT.second.changeVectorElementType(EltVT: MVT::i32);
398 WidenFirst =
399 RISCVVType::decodeVLMUL(VLMul: RISCVTargetLowering::getLMUL(VT: NarrowMVT))
400 .second;
401 }
402
403 if (WidenFirst) {
404 // The widened i64 dot result has VF/4 elements, i.e. twice the
405 // accumulator's element count, so the reduction plus the accumulate are
406 // two i64 vadd.vv.
407 std::pair<InstructionCost, MVT> WideLT = getTypeLegalizationCost(
408 Ty: VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: 4)));
409 Cost +=
410 WideLT.first * getRISCVInstructionCost(OpCodes: RISCV::VSEXT_VF2,
411 VT: WideLT.second, CostKind) +
412 2 * AccLT.first *
413 getRISCVInstructionCost(OpCodes: RISCV::VADD_VV, VT: AccLT.second, CostKind);
414 } else {
415 // Otherwise the scale-4 i32 sums are halved with a single i32 vadd.vv,
416 // then widened and added into the i64 result with a vwadd.wv.
417 std::pair<InstructionCost, MVT> RedLT =
418 getTypeLegalizationCost(Ty: VectorType::get(ElementType: I32Ty, EC: AccVF));
419 Cost += RedLT.first * getRISCVInstructionCost(OpCodes: RISCV::VADD_VV,
420 VT: RedLT.second, CostKind) +
421 AccLT.first * getRISCVInstructionCost(OpCodes: RISCV::VWADD_WV,
422 VT: AccLT.second, CostKind);
423 // Fixed-length vectors extract the high i32 subvector with a vslidedown.
424 if (VF.isFixed())
425 Cost += DotLT.first * getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI,
426 VT: DotLT.second, CostKind);
427 }
428 }
429
430 return Cost;
431}
432
433bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
434 // Currently, the ExpandReductions pass can't expand scalable-vector
435 // reductions, but we still request expansion as RVV doesn't support certain
436 // reductions and the SelectionDAG can't legalize them either.
437 switch (II->getIntrinsicID()) {
438 default:
439 return false;
440 // These reductions have no equivalent in RVV
441 case Intrinsic::vector_reduce_mul:
442 case Intrinsic::vector_reduce_fmul:
443 return true;
444 }
445}
446
447std::optional<unsigned> RISCVTTIImpl::getMaxVScale() const {
448 if (ST->hasVInstructions())
449 return ST->getRealMaxVLen() / RISCV::RVVBitsPerBlock;
450 return BaseT::getMaxVScale();
451}
452
453std::optional<unsigned> RISCVTTIImpl::getVScaleForTuning() const {
454 if (ST->hasVInstructions())
455 if (unsigned MinVLen = ST->getRealMinVLen();
456 MinVLen >= RISCV::RVVBitsPerBlock)
457 return MinVLen / RISCV::RVVBitsPerBlock;
458 return BaseT::getVScaleForTuning();
459}
460
461TypeSize
462RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
463 unsigned LMUL =
464 llvm::bit_floor(Value: std::clamp<unsigned>(val: RVVRegisterWidthLMUL, lo: 1, hi: 8));
465 switch (K) {
466 case TargetTransformInfo::RGK_Scalar:
467 return TypeSize::getFixed(ExactSize: ST->getXLen());
468 case TargetTransformInfo::RGK_FixedWidthVector:
469 return TypeSize::getFixed(
470 ExactSize: ST->useRVVForFixedLengthVectors() ? LMUL * ST->getRealMinVLen() : 0);
471 case TargetTransformInfo::RGK_ScalableVector:
472 return TypeSize::getScalable(
473 MinimumSize: (ST->hasVInstructions() &&
474 ST->getRealMinVLen() >= RISCV::RVVBitsPerBlock)
475 ? LMUL * RISCV::RVVBitsPerBlock
476 : 0);
477 }
478
479 llvm_unreachable("Unsupported register kind");
480}
481
482InstructionCost RISCVTTIImpl::getStaticDataAddrGenerationCost(
483 const TTI::TargetCostKind CostKind) const {
484 switch (CostKind) {
485 case TTI::TCK_CodeSize:
486 case TTI::TCK_SizeAndLatency:
487 // Always 2 instructions
488 return 2;
489 case TTI::TCK_Latency:
490 case TTI::TCK_RecipThroughput:
491 // Depending on the memory model the address generation will
492 // require AUIPC + ADDI (medany) or LUI + ADDI (medlow). Don't
493 // have a way of getting this information here, so conservatively
494 // require both.
495 // In practice, these are generally implemented together.
496 return (ST->hasAUIPCADDIFusion() && ST->hasLUIADDIFusion()) ? 1 : 2;
497 }
498 llvm_unreachable("Unsupported cost kind");
499}
500
501InstructionCost
502RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,
503 TTI::TargetCostKind CostKind) const {
504 // Add a cost of address generation + the cost of the load. The address
505 // is expected to be a PC relative offset to a constant pool entry
506 // using auipc/addi.
507 return getStaticDataAddrGenerationCost(CostKind) +
508 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: DL.getABITypeAlign(Ty),
509 /*AddressSpace=*/0, CostKind);
510}
511
512static bool isRepeatedConcatMask(ArrayRef<int> Mask, int &SubVectorSize) {
513 unsigned Size = Mask.size();
514 if (!isPowerOf2_32(Value: Size))
515 return false;
516 for (unsigned I = 0; I != Size; ++I) {
517 if (static_cast<unsigned>(Mask[I]) == I)
518 continue;
519 if (Mask[I] != 0)
520 return false;
521 if (Size % I != 0)
522 return false;
523 for (unsigned J = I + 1; J != Size; ++J)
524 // Check the pattern is repeated.
525 if (static_cast<unsigned>(Mask[J]) != J % I)
526 return false;
527 SubVectorSize = I;
528 return true;
529 }
530 // That means Mask is <0, 1, 2, 3>. This is not a concatenation.
531 return false;
532}
533
534static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,
535 LLVMContext &C) {
536 assert((DataVT.getScalarSizeInBits() != 8 ||
537 DataVT.getVectorNumElements() <= 256) && "unhandled case in lowering");
538 MVT IndexVT = DataVT.changeTypeToInteger();
539 if (IndexVT.getScalarType().bitsGT(VT: ST.getXLenVT()))
540 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
541 return cast<VectorType>(Val: EVT(IndexVT).getTypeForEVT(Context&: C));
542}
543
544/// Attempt to approximate the cost of a shuffle which will require splitting
545/// during legalization. Note that processShuffleMasks is not an exact proxy
546/// for the algorithm used in LegalizeVectorTypes, but hopefully it's a
547/// reasonably close upperbound.
548static InstructionCost costShuffleViaSplitting(const RISCVTTIImpl &TTI,
549 MVT LegalVT, VectorType *Tp,
550 ArrayRef<int> Mask,
551 TTI::TargetCostKind CostKind) {
552 assert(LegalVT.isFixedLengthVector() && !Mask.empty() &&
553 "Expected fixed vector type and non-empty mask");
554 unsigned LegalNumElts = LegalVT.getVectorNumElements();
555 // Number of destination vectors after legalization:
556 unsigned NumOfDests = divideCeil(Numerator: Mask.size(), Denominator: LegalNumElts);
557 // We are going to permute multiple sources and the result will be in
558 // multiple destinations. Providing an accurate cost only for splits where
559 // the element type remains the same.
560 if (NumOfDests <= 1 ||
561 LegalVT.getVectorElementType().getSizeInBits() !=
562 Tp->getElementType()->getPrimitiveSizeInBits() ||
563 LegalNumElts >= Tp->getElementCount().getFixedValue())
564 return InstructionCost::getInvalid();
565
566 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
567 unsigned LegalVTSize = LegalVT.getStoreSize();
568 // Number of source vectors after legalization:
569 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
570
571 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: LegalNumElts);
572
573 unsigned NormalizedVF = LegalNumElts * std::max(a: NumOfSrcs, b: NumOfDests);
574 unsigned NumOfSrcRegs = NormalizedVF / LegalNumElts;
575 unsigned NumOfDestRegs = NormalizedVF / LegalNumElts;
576 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
577 assert(NormalizedVF >= Mask.size() &&
578 "Normalized mask expected to be not shorter than original mask.");
579 copy(Range&: Mask, Out: NormalizedMask.begin());
580 InstructionCost Cost = 0;
581 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
582 processShuffleMasks(
583 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
584 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
585 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
586 return;
587 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
588 .second)
589 return;
590 Cost += TTI.getShuffleCost(
591 Kind: TTI::SK_PermuteSingleSrc,
592 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
593 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
594 },
595 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
596 Cost += TTI.getShuffleCost(
597 Kind: TTI::SK_PermuteTwoSrc,
598 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
599 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
600 });
601 return Cost;
602}
603
604/// Try to perform better estimation of the permutation.
605/// 1. Split the source/destination vectors into real registers.
606/// 2. Do the mask analysis to identify which real registers are
607/// permuted. If more than 1 source registers are used for the
608/// destination register building, the cost for this destination register
609/// is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
610/// source register is used, build mask and calculate the cost as a cost
611/// of PermuteSingleSrc.
612/// Also, for the single register permute we try to identify if the
613/// destination register is just a copy of the source register or the
614/// copy of the previous destination register (the cost is
615/// TTI::TCC_Basic). If the source register is just reused, the cost for
616/// this operation is 0.
617static InstructionCost
618costShuffleViaVRegSplitting(const RISCVTTIImpl &TTI, MVT LegalVT,
619 std::optional<unsigned> VLen, VectorType *Tp,
620 ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {
621 assert(LegalVT.isFixedLengthVector());
622 if (!VLen || Mask.empty())
623 return InstructionCost::getInvalid();
624 MVT ElemVT = LegalVT.getVectorElementType();
625 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
626 LegalVT = TTI.getTypeLegalizationCost(
627 Ty: FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: ElemsPerVReg))
628 .second;
629 // Number of destination vectors after legalization:
630 InstructionCost NumOfDests =
631 divideCeil(Numerator: Mask.size(), Denominator: LegalVT.getVectorNumElements());
632 if (NumOfDests <= 1 ||
633 LegalVT.getVectorElementType().getSizeInBits() !=
634 Tp->getElementType()->getPrimitiveSizeInBits() ||
635 LegalVT.getVectorNumElements() >= Tp->getElementCount().getFixedValue())
636 return InstructionCost::getInvalid();
637
638 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
639 unsigned LegalVTSize = LegalVT.getStoreSize();
640 // Number of source vectors after legalization:
641 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
642
643 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(),
644 NumElts: LegalVT.getVectorNumElements());
645
646 unsigned E = NumOfDests.getValue();
647 unsigned NormalizedVF =
648 LegalVT.getVectorNumElements() * std::max(a: NumOfSrcs, b: E);
649 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
650 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
651 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
652 assert(NormalizedVF >= Mask.size() &&
653 "Normalized mask expected to be not shorter than original mask.");
654 copy(Range&: Mask, Out: NormalizedMask.begin());
655 InstructionCost Cost = 0;
656 int NumShuffles = 0;
657 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
658 processShuffleMasks(
659 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
660 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
661 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
662 return;
663 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
664 .second)
665 return;
666 ++NumShuffles;
667 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleOpTy,
668 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
669 },
670 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
671 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
672 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
673 NumShuffles += 2;
674 });
675 // Note: check that we do not emit too many shuffles here to prevent code
676 // size explosion.
677 // TODO: investigate, if it can be improved by extra analysis of the masks
678 // to check if the code is more profitable.
679 if ((NumOfDestRegs > 2 && NumShuffles <= static_cast<int>(NumOfDestRegs)) ||
680 (NumOfDestRegs <= 2 && NumShuffles < 4))
681 return Cost;
682 return InstructionCost::getInvalid();
683}
684
685InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
686 ArrayRef<int> Mask,
687 TTI::TargetCostKind CostKind) const {
688 // Avoid missing masks and length changing shuffles
689 if (Mask.size() <= 2 || Mask.size() != Tp->getNumElements())
690 return InstructionCost::getInvalid();
691
692 int NumElts = Tp->getNumElements();
693 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Tp);
694 // Avoid scalarization cases
695 if (!LT.second.isFixedLengthVector())
696 return InstructionCost::getInvalid();
697
698 // Requires moving elements between parts, which requires additional
699 // unmodeled instructions.
700 if (LT.first != 1)
701 return InstructionCost::getInvalid();
702
703 auto GetSlideOpcode = [&](int SlideAmt) {
704 assert(SlideAmt != 0);
705 bool IsVI = isUInt<5>(x: std::abs(x: SlideAmt));
706 if (SlideAmt < 0)
707 return IsVI ? RISCV::VSLIDEDOWN_VI : RISCV::VSLIDEDOWN_VX;
708 return IsVI ? RISCV::VSLIDEUP_VI : RISCV::VSLIDEUP_VX;
709 };
710
711 std::array<std::pair<int, int>, 2> SrcInfo;
712 if (!isMaskedSlidePair(Mask, NumElts, SrcInfo))
713 return InstructionCost::getInvalid();
714
715 if (SrcInfo[1].second == 0)
716 std::swap(x&: SrcInfo[0], y&: SrcInfo[1]);
717
718 InstructionCost FirstSlideCost = 0;
719 if (SrcInfo[0].second != 0) {
720 unsigned Opcode = GetSlideOpcode(SrcInfo[0].second);
721 FirstSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
722 }
723
724 if (SrcInfo[1].first == -1)
725 return FirstSlideCost;
726
727 InstructionCost SecondSlideCost = 0;
728 if (SrcInfo[1].second != 0) {
729 unsigned Opcode = GetSlideOpcode(SrcInfo[1].second);
730 SecondSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
731 } else {
732 SecondSlideCost =
733 getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second, CostKind);
734 }
735
736 auto EC = Tp->getElementCount();
737 VectorType *MaskTy =
738 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: Tp->getContext()), EC);
739 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
740 return FirstSlideCost + SecondSlideCost + MaskCost;
741}
742
743InstructionCost
744RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
745 VectorType *SrcTy, TTI::TargetCostKind CostKind,
746 ArrayRef<int> Mask, int Index, VectorType *SubTp,
747 ArrayRef<const Value *> Args,
748 const Instruction *CxtI) const {
749 assert((Mask.empty() || DstTy->isScalableTy() ||
750 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
751 "Expected the Mask to match the return size if given");
752 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
753 "Expected the same scalar types");
754
755 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
756
757 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
758 // For now, skip all fixed vector cost analysis when P extension is available
759 // to avoid crashes in getMinRVVVectorSizeInBits()
760 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: SrcTy))
761 return 1;
762
763 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
764
765 // First, handle cases where having a fixed length vector enables us to
766 // give a more accurate cost than falling back to generic scalable codegen.
767 // TODO: Each of these cases hints at a modeling gap around scalable vectors.
768 if (auto *FVTp = dyn_cast<FixedVectorType>(Val: SrcTy);
769 FVTp && ST->hasVInstructions() && LT.second.isFixedLengthVector()) {
770 InstructionCost VRegSplittingCost = costShuffleViaVRegSplitting(
771 TTI: *this, LegalVT: LT.second, VLen: ST->getRealVLen(),
772 Tp: Kind == TTI::SK_InsertSubvector ? DstTy : SrcTy, Mask, CostKind);
773 if (VRegSplittingCost.isValid())
774 return VRegSplittingCost;
775 switch (Kind) {
776 default:
777 break;
778 case TTI::SK_PermuteSingleSrc: {
779 if (Mask.size() >= 2) {
780 MVT EltTp = LT.second.getVectorElementType();
781 // If the size of the element is < ELEN then shuffles of interleaves and
782 // deinterleaves of 2 vectors can be lowered into the following
783 // sequences
784 if (EltTp.getScalarSizeInBits() < ST->getELen()) {
785 // Example sequence:
786 // vsetivli zero, 4, e8, mf4, ta, ma (ignored)
787 // vwaddu.vv v10, v8, v9
788 // li a0, -1 (ignored)
789 // vwmaccu.vx v10, a0, v9
790 if (ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Mask.size()))
791 return 2 * LT.first * TLI->getLMULCost(VT: LT.second);
792
793 if (Mask[0] == 0 || Mask[0] == 1) {
794 auto DeinterleaveMask = createStrideMask(Start: Mask[0], Stride: 2, VF: Mask.size());
795 // Example sequence:
796 // vnsrl.wi v10, v8, 0
797 if (equal(LRange&: DeinterleaveMask, RRange&: Mask))
798 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VNSRL_WI,
799 VT: LT.second, CostKind);
800 }
801 }
802 int SubVectorSize;
803 if (LT.second.getScalarSizeInBits() != 1 &&
804 isRepeatedConcatMask(Mask, SubVectorSize)) {
805 InstructionCost Cost = 0;
806 unsigned NumSlides = Log2_32(Value: Mask.size() / SubVectorSize);
807 // The cost of extraction from a subvector is 0 if the index is 0.
808 for (unsigned I = 0; I != NumSlides; ++I) {
809 unsigned InsertIndex = SubVectorSize * (1 << I);
810 FixedVectorType *SubTp =
811 FixedVectorType::get(ElementType: SrcTy->getElementType(), NumElts: InsertIndex);
812 FixedVectorType *DestTp =
813 FixedVectorType::getDoubleElementsVectorType(VTy: SubTp);
814 std::pair<InstructionCost, MVT> DestLT =
815 getTypeLegalizationCost(Ty: DestTp);
816 // Add the cost of whole vector register move because the
817 // destination vector register group for vslideup cannot overlap the
818 // source.
819 Cost += DestLT.first * TLI->getLMULCost(VT: DestLT.second);
820 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: DestTp, SrcTy: DestTp,
821 CostKind, Mask: {}, Index: InsertIndex, SubTp);
822 }
823 return Cost;
824 }
825 }
826
827 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
828 SlideCost.isValid())
829 return SlideCost;
830
831 // vrgather + cost of generating the mask constant.
832 // We model this for an unknown mask with a single vrgather.
833 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
834 LT.second.getVectorNumElements() <= 256)) {
835 VectorType *IdxTy =
836 getVRGatherIndexType(DataVT: LT.second, ST: *ST, C&: SrcTy->getContext());
837 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
838 return IndexCost +
839 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VV, VT: LT.second, CostKind);
840 }
841 break;
842 }
843 case TTI::SK_Transpose:
844 case TTI::SK_PermuteTwoSrc: {
845
846 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
847 SlideCost.isValid())
848 return SlideCost;
849
850 // 2 x (vrgather + cost of generating the mask constant) + cost of mask
851 // register for the second vrgather. We model this for an unknown
852 // (shuffle) mask.
853 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
854 LT.second.getVectorNumElements() <= 256)) {
855 auto &C = SrcTy->getContext();
856 auto EC = SrcTy->getElementCount();
857 VectorType *IdxTy = getVRGatherIndexType(DataVT: LT.second, ST: *ST, C);
858 VectorType *MaskTy = VectorType::get(ElementType: IntegerType::getInt1Ty(C), EC);
859 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
860 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
861 return 2 * IndexCost +
862 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV, RISCV::VRGATHER_VV},
863 VT: LT.second, CostKind) +
864 MaskCost;
865 }
866 break;
867 }
868 }
869
870 auto shouldSplit = [](TTI::ShuffleKind Kind) {
871 switch (Kind) {
872 default:
873 return false;
874 case TTI::SK_PermuteSingleSrc:
875 case TTI::SK_Transpose:
876 case TTI::SK_PermuteTwoSrc:
877 return true;
878 }
879 };
880
881 if (!Mask.empty() && LT.first.isValid() && LT.first != 1 &&
882 shouldSplit(Kind)) {
883 InstructionCost SplitCost =
884 costShuffleViaSplitting(TTI: *this, LegalVT: LT.second, Tp: FVTp, Mask, CostKind);
885 if (SplitCost.isValid())
886 return SplitCost;
887 }
888 }
889
890 // Handle scalable vectors (and fixed vectors legalized to scalable vectors).
891 switch (Kind) {
892 default:
893 // Fallthrough to generic handling.
894 // TODO: Most of these cases will return getInvalid in generic code, and
895 // must be implemented here.
896 break;
897 case TTI::SK_ExtractSubvector:
898 // Extract at zero is always a subregister extract
899 if (Index == 0)
900 return TTI::TCC_Free;
901
902 // If we're extracting a subvector of at most m1 size at a sub-register
903 // boundary - which unfortunately we need exact vlen to identify - this is
904 // a subregister extract at worst and thus won't require a vslidedown.
905 // TODO: Extend for aligned m2, m4 subvector extracts
906 // TODO: Extend for misalgined (but contained) extracts
907 // TODO: Extend for scalable subvector types
908 if (std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
909 SubLT.second.isValid() && SubLT.second.isFixedLengthVector()) {
910 if (std::optional<unsigned> VLen = ST->getRealVLen();
911 VLen && SubLT.second.getScalarSizeInBits() * Index % *VLen == 0 &&
912 SubLT.second.getSizeInBits() <= *VLen)
913 return TTI::TCC_Free;
914 }
915
916 // Example sequence:
917 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
918 // vslidedown.vi v8, v9, 2
919 return LT.first *
920 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI, VT: LT.second, CostKind);
921 case TTI::SK_InsertSubvector:
922 // Example sequence:
923 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
924 // vslideup.vi v8, v9, 2
925 LT = getTypeLegalizationCost(Ty: DstTy);
926 return LT.first *
927 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEUP_VI, VT: LT.second, CostKind);
928 case TTI::SK_Select: {
929 // Example sequence:
930 // li a0, 90
931 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
932 // vmv.s.x v0, a0
933 // vmerge.vvm v8, v9, v8, v0
934 // We use 2 for the cost of the mask materialization as this is the true
935 // cost for small masks and most shuffles are small. At worst, this cost
936 // should be a very small constant for the constant pool load. As such,
937 // we may bias towards large selects slightly more than truly warranted.
938 return LT.first *
939 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_S_X, RISCV::VMERGE_VVM},
940 VT: LT.second, CostKind));
941 }
942 case TTI::SK_Broadcast: {
943 // Check for broadcast loads, which are synthesized by optimized zero-stride
944 // loads (this is checked in RISCVTTIImpl::isLegalBroadcastLoad).
945 bool IsLoad = !Args.empty() && isa<LoadInst>(Val: Args[0]);
946 if (IsLoad && LT.second.isVector() &&
947 isLegalBroadcastLoad(ElementTy: SrcTy->getElementType(),
948 NumElements: LT.second.getVectorElementCount()))
949 return 0;
950
951 bool HasScalar = (Args.size() > 0) && (Operator::getOpcode(V: Args[0]) ==
952 Instruction::InsertElement);
953 if (LT.second.getScalarSizeInBits() == 1) {
954 if (HasScalar) {
955 // Example sequence:
956 // andi a0, a0, 1
957 // vsetivli zero, 2, e8, mf8, ta, ma (ignored)
958 // vmv.v.x v8, a0
959 // vmsne.vi v0, v8, 0
960 return LT.first *
961 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
962 VT: LT.second, CostKind));
963 }
964 // Example sequence:
965 // vsetivli zero, 2, e8, mf8, ta, mu (ignored)
966 // vmv.v.i v8, 0
967 // vmerge.vim v8, v8, 1, v0
968 // vmv.x.s a0, v8
969 // andi a0, a0, 1
970 // vmv.v.x v8, a0
971 // vmsne.vi v0, v8, 0
972
973 return LT.first *
974 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_I, RISCV::VMERGE_VIM,
975 RISCV::VMV_X_S, RISCV::VMV_V_X,
976 RISCV::VMSNE_VI},
977 VT: LT.second, CostKind));
978 }
979
980 if (HasScalar) {
981 // Example sequence:
982 // vmv.v.x v8, a0
983 return LT.first *
984 getRISCVInstructionCost(OpCodes: RISCV::VMV_V_X, VT: LT.second, CostKind);
985 }
986
987 // Example sequence:
988 // vrgather.vi v9, v8, 0
989 return LT.first *
990 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VI, VT: LT.second, CostKind);
991 }
992 case TTI::SK_Splice: {
993 // vslidedown+vslideup.
994 // TODO: Multiplying by LT.first implies this legalizes into multiple copies
995 // of similar code, but I think we expand through memory.
996 unsigned Opcodes[2] = {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX};
997 if (Index >= 0 && Index < 32)
998 Opcodes[0] = RISCV::VSLIDEDOWN_VI;
999 else if (Index < 0 && Index > -32)
1000 Opcodes[1] = RISCV::VSLIDEUP_VI;
1001 return LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1002 }
1003 case TTI::SK_Reverse: {
1004
1005 if (!LT.second.isVector())
1006 return InstructionCost::getInvalid();
1007
1008 // TODO: Cases to improve here:
1009 // * Illegal vector types
1010 // * i64 on RV32
1011 if (SrcTy->getElementType()->isIntegerTy(BitWidth: 1)) {
1012 VectorType *WideTy =
1013 VectorType::get(ElementType: IntegerType::get(C&: SrcTy->getContext(), NumBits: 8),
1014 EC: cast<VectorType>(Val: SrcTy)->getElementCount());
1015 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: SrcTy,
1016 CCH: TTI::CastContextHint::None, CostKind) +
1017 getShuffleCost(Kind: TTI::SK_Reverse, DstTy: WideTy, SrcTy: WideTy, CostKind, Mask: {}, Index: 0,
1018 SubTp: nullptr) +
1019 getCastInstrCost(Opcode: Instruction::Trunc, Dst: SrcTy, Src: WideTy,
1020 CCH: TTI::CastContextHint::None, CostKind);
1021 }
1022
1023 MVT ContainerVT = LT.second;
1024 if (LT.second.isFixedLengthVector())
1025 ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1026 MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
1027 if (ContainerVT.bitsLE(VT: M1VT)) {
1028 // Example sequence:
1029 // csrr a0, vlenb
1030 // srli a0, a0, 3
1031 // addi a0, a0, -1
1032 // vsetvli a1, zero, e8, mf8, ta, mu (ignored)
1033 // vid.v v9
1034 // vrsub.vx v10, v9, a0
1035 // vrgather.vv v9, v8, v10
1036 InstructionCost LenCost = 3;
1037 if (LT.second.isFixedLengthVector())
1038 // vrsub.vi has a 5 bit immediate field, otherwise an li suffices
1039 LenCost = isInt<5>(x: LT.second.getVectorNumElements() - 1) ? 0 : 1;
1040 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX, RISCV::VRGATHER_VV};
1041 if (LT.second.isFixedLengthVector() &&
1042 isInt<5>(x: LT.second.getVectorNumElements() - 1))
1043 Opcodes[1] = RISCV::VRSUB_VI;
1044 InstructionCost GatherCost =
1045 getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1046 return LT.first * (LenCost + GatherCost);
1047 }
1048
1049 // At high LMUL, we split into a series of M1 reverses (see
1050 // lowerVECTOR_REVERSE) and then do a single slide at the end to eliminate
1051 // the resulting gap at the bottom (for fixed vectors only). The important
1052 // bit is that the cost scales linearly, not quadratically with LMUL.
1053 unsigned M1Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX};
1054 InstructionCost FixedCost =
1055 getRISCVInstructionCost(OpCodes: M1Opcodes, VT: M1VT, CostKind) + 3;
1056 unsigned Ratio =
1057 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
1058 InstructionCost GatherCost =
1059 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV}, VT: M1VT, CostKind) * Ratio;
1060 InstructionCost SlideCost = !LT.second.isFixedLengthVector() ? 0 :
1061 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX}, VT: LT.second, CostKind);
1062 return FixedCost + LT.first * (GatherCost + SlideCost);
1063 }
1064 }
1065 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1066 SubTp);
1067}
1068
1069static unsigned isM1OrSmaller(MVT VT) {
1070 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1071 return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
1072 LMUL == RISCVVType::VLMUL::LMUL_F4 ||
1073 LMUL == RISCVVType::VLMUL::LMUL_F2 ||
1074 LMUL == RISCVVType::VLMUL::LMUL_1);
1075}
1076
1077InstructionCost RISCVTTIImpl::getScalarizationOverhead(
1078 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
1079 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
1080 TTI::VectorInstrContext VIC) const {
1081 if (isa<ScalableVectorType>(Val: Ty))
1082 return InstructionCost::getInvalid();
1083
1084 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
1085 // For now, skip all fixed vector cost analysis when P extension is available
1086 // to avoid crashes in getMinRVVVectorSizeInBits()
1087 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: Ty)) {
1088 return 1; // Treat as single instruction cost for now
1089 }
1090
1091 // A build_vector (which is m1 sized or smaller) can be done in no
1092 // worse than one vslide1down.vx per element in the type. We could
1093 // in theory do an explode_vector in the inverse manner, but our
1094 // lowering today does not have a first class node for this pattern.
1095 InstructionCost Cost = BaseT::getScalarizationOverhead(
1096 InTy: Ty, DemandedElts, Insert, Extract, CostKind);
1097 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1098 if (Insert && !Extract && LT.first.isValid() && LT.second.isVector()) {
1099 if (Ty->getScalarSizeInBits() == 1) {
1100 auto *WideVecTy = cast<VectorType>(Val: Ty->getWithNewBitWidth(NewBitWidth: 8));
1101 // Note: Implicit scalar anyextend is assumed to be free since the i1
1102 // must be stored in a GPR.
1103 return getScalarizationOverhead(Ty: WideVecTy, DemandedElts, Insert, Extract,
1104 CostKind) +
1105 getCastInstrCost(Opcode: Instruction::Trunc, Dst: Ty, Src: WideVecTy,
1106 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
1107 }
1108
1109 assert(LT.second.isFixedLengthVector());
1110 MVT ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1111 if (isM1OrSmaller(VT: ContainerVT)) {
1112 InstructionCost BV =
1113 cast<FixedVectorType>(Val: Ty)->getNumElements() *
1114 getRISCVInstructionCost(OpCodes: RISCV::VSLIDE1DOWN_VX, VT: LT.second, CostKind);
1115 if (BV < Cost)
1116 Cost = BV;
1117 }
1118 }
1119 return Cost;
1120}
1121
1122InstructionCost
1123RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1124 TTI::TargetCostKind CostKind) const {
1125 Type *DataTy = MICA.getDataType();
1126 Align Alignment = MICA.getAlignment();
1127 switch (MICA.getID()) {
1128 case Intrinsic::vp_load_ff: {
1129 EVT DataTypeVT = TLI->getValueType(DL, Ty: DataTy);
1130 if (!TLI->isLegalFirstFaultLoad(DataType: DataTypeVT, Alignment))
1131 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1132
1133 unsigned AS = MICA.getAddressSpace();
1134 return getMemoryOpCost(Opcode: Instruction::Load, Src: DataTy, Alignment, AddressSpace: AS, CostKind,
1135 OpdInfo: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I: nullptr);
1136 }
1137 case Intrinsic::experimental_vp_strided_load:
1138 case Intrinsic::experimental_vp_strided_store:
1139 return getStridedMemoryOpCost(MICA, CostKind);
1140 case Intrinsic::masked_compressstore:
1141 case Intrinsic::masked_expandload:
1142 return getExpandCompressMemoryOpCost(MICA, CostKind);
1143 case Intrinsic::vp_scatter:
1144 case Intrinsic::vp_gather:
1145 case Intrinsic::masked_scatter:
1146 case Intrinsic::masked_gather:
1147 return getGatherScatterOpCost(MICA, CostKind);
1148 case Intrinsic::vp_load:
1149 case Intrinsic::vp_store:
1150 case Intrinsic::masked_load:
1151 case Intrinsic::masked_store:
1152 return getMaskedMemoryOpCost(MICA, CostKind);
1153 }
1154 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1155}
1156
1157InstructionCost
1158RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1159 TTI::TargetCostKind CostKind) const {
1160 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
1161 : Instruction::Store;
1162 Type *Src = MICA.getDataType();
1163 Align Alignment = MICA.getAlignment();
1164 unsigned AddressSpace = MICA.getAddressSpace();
1165
1166 if (!isLegalMaskedLoadStore(DataType: Src, Alignment) ||
1167 CostKind != TTI::TCK_RecipThroughput)
1168 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1169
1170 return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
1171}
1172
1173InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
1174 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1175 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1176 bool UseMaskForCond, bool UseMaskForGaps) const {
1177
1178 // The interleaved memory access pass will lower (de)interleave ops combined
1179 // with an adjacent appropriate memory to vlseg/vsseg intrinsics. vlseg/vsseg
1180 // only support masking per-iteration (i.e. condition), not per-segment (i.e.
1181 // gap).
1182 if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {
1183 auto *VTy = cast<VectorType>(Val: VecTy);
1184 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VTy);
1185 // Need to make sure type has't been scalarized
1186 if (LT.second.isVector()) {
1187 if (CostKind == TTI::TCK_CodeSize)
1188 return LT.first * TTI::TCC_Basic;
1189
1190 auto *SubVecTy =
1191 VectorType::get(ElementType: VTy->getElementType(),
1192 EC: VTy->getElementCount().divideCoefficientBy(RHS: Factor));
1193 if (VTy->getElementCount().isKnownMultipleOf(RHS: Factor) &&
1194 TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor, Alignment,
1195 AddrSpace: AddressSpace, DL)) {
1196
1197 // Some processors optimize segment loads/stores as N * DLEN sized
1198 // load ops + Factor * LMUL shuffle ops.
1199 if (ST->hasOptimizedSegmentLoadStore(NF: Factor)) {
1200 unsigned VecSizeInBits =
1201 getEstimatedVLFor(Ty: VTy) * VTy->getScalarSizeInBits();
1202 unsigned VLENForTuning =
1203 *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
1204 unsigned DLENForTuning = VLENForTuning / ST->getDLenFactor();
1205 InstructionCost Cost = divideCeil(Numerator: VecSizeInBits, Denominator: DLENForTuning);
1206 MVT SubVecVT = getTLI()->getValueType(DL, Ty: SubVecTy).getSimpleVT();
1207 Cost += Factor * TLI->getLMULCost(VT: SubVecVT);
1208 return Cost;
1209 }
1210
1211 // Otherwise, the cost is proportional to the number of elements (VL *
1212 // Factor ops).
1213 unsigned NumLoads = getEstimatedVLFor(Ty: VTy);
1214 return NumLoads * TTI::TCC_Basic;
1215 }
1216 }
1217 }
1218
1219 // TODO: Return the cost of interleaved accesses for scalable vector when
1220 // unable to convert to segment accesses instructions.
1221 if (isa<ScalableVectorType>(Val: VecTy))
1222 return InstructionCost::getInvalid();
1223
1224 auto *FVTy = cast<FixedVectorType>(Val: VecTy);
1225 // When gaps are only at the tail, for interleaved load, we can emit a wide
1226 // masked load and shufflevectors. For interleaved store, we can emit
1227 // shufflevectors and a wide masked store. The interleaved memory access pass
1228 // will lower them into vlsseg/vssseg intrinsics.
1229 if (UseMaskForGaps) {
1230 assert(llvm::is_sorted(Indices) && "Indices must be sorted");
1231 assert(llvm::adjacent_find(Indices) == Indices.end() &&
1232 "Indices should not contain duplicate elements");
1233 unsigned NumOfFields = Indices.size();
1234 bool IsTailGapOnly = NumOfFields > 1 && (NumOfFields == Indices.back() + 1);
1235 if (IsTailGapOnly &&
1236 NumOfFields <= TLI->getMaxSupportedInterleaveFactor()) {
1237 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: FVTy);
1238 if (LT.second.isVector() &&
1239 FVTy->getElementCount().isKnownMultipleOf(RHS: Factor)) {
1240 auto *SubVecTy = VectorType::get(
1241 ElementType: FVTy->getElementType(),
1242 EC: FVTy->getElementCount().divideCoefficientBy(RHS: Factor));
1243 if (TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor: NumOfFields, Alignment,
1244 AddrSpace: AddressSpace, DL)) {
1245 // The cost is proportional to the total number of element accesses.
1246 unsigned NumAccesses = getEstimatedVLFor(Ty: FVTy);
1247 return NumAccesses * TTI::TCC_Basic;
1248 }
1249 }
1250 }
1251 }
1252
1253 InstructionCost MemCost =
1254 getMemoryOpCost(Opcode, Src: VecTy, Alignment, AddressSpace, CostKind);
1255 unsigned VF = FVTy->getNumElements() / Factor;
1256
1257 // An interleaved load will look like this for Factor=3:
1258 // %wide.vec = load <12 x i32>, ptr %3, align 4
1259 // %strided.vec = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1260 // %strided.vec1 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1261 // %strided.vec2 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1262 if (Opcode == Instruction::Load) {
1263 InstructionCost Cost = MemCost;
1264 for (unsigned Index : Indices) {
1265 FixedVectorType *VecTy =
1266 FixedVectorType::get(ElementType: FVTy->getElementType(), NumElts: VF * Factor);
1267 auto Mask = createStrideMask(Start: Index, Stride: Factor, VF);
1268 Mask.resize(N: VF * Factor, NV: -1);
1269 InstructionCost ShuffleCost =
1270 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: VecTy, SrcTy: VecTy,
1271 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {});
1272 Cost += ShuffleCost;
1273 }
1274 return Cost;
1275 }
1276
1277 // TODO: Model for NF > 2
1278 // We'll need to enhance getShuffleCost to model shuffles that are just
1279 // inserts and extracts into subvectors, since they won't have the full cost
1280 // of a vrgather.
1281 // An interleaved store for 3 vectors of 4 lanes will look like
1282 // %11 = shufflevector <4 x i32> %4, <4 x i32> %6, <8 x i32> <0...7>
1283 // %12 = shufflevector <4 x i32> %9, <4 x i32> poison, <8 x i32> <0...3>
1284 // %13 = shufflevector <8 x i32> %11, <8 x i32> %12, <12 x i32> <0...11>
1285 // %interleaved.vec = shufflevector %13, poison, <12 x i32> <interleave mask>
1286 // store <12 x i32> %interleaved.vec, ptr %10, align 4
1287 if (Factor != 2)
1288 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1289 Alignment, AddressSpace, CostKind,
1290 UseMaskForCond, UseMaskForGaps);
1291
1292 assert(Opcode == Instruction::Store && "Opcode must be a store");
1293 // For an interleaving store of 2 vectors, we perform one large interleaving
1294 // shuffle that goes into the wide store
1295 auto Mask = createInterleaveMask(VF, NumVecs: Factor);
1296 InstructionCost ShuffleCost =
1297 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: FVTy, SrcTy: FVTy,
1298 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {});
1299 return MemCost + ShuffleCost;
1300}
1301
1302InstructionCost
1303RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1304 TTI::TargetCostKind CostKind) const {
1305
1306 bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
1307 MICA.getID() == Intrinsic::vp_gather;
1308 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
1309 Type *DataTy = MICA.getDataType();
1310 Align Alignment = MICA.getAlignment();
1311 if (CostKind != TTI::TCK_RecipThroughput)
1312 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1313
1314 if ((Opcode == Instruction::Load &&
1315 !isLegalMaskedGather(DataType: DataTy, Alignment: Align(Alignment))) ||
1316 (Opcode == Instruction::Store &&
1317 !isLegalMaskedScatter(DataType: DataTy, Alignment: Align(Alignment))))
1318 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1319
1320 // Cost is proportional to the number of memory operations implied. For
1321 // scalable vectors, we use an estimate on that number since we don't
1322 // know exactly what VL will be.
1323 auto &VTy = *cast<VectorType>(Val: DataTy);
1324 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1325 return NumLoads * TTI::TCC_Basic;
1326}
1327
1328InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
1329 const MemIntrinsicCostAttributes &MICA,
1330 TTI::TargetCostKind CostKind) const {
1331 unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
1332 ? Instruction::Load
1333 : Instruction::Store;
1334 Type *DataTy = MICA.getDataType();
1335 bool VariableMask = MICA.getVariableMask();
1336 Align Alignment = MICA.getAlignment();
1337 bool IsLegal = (Opcode == Instruction::Store &&
1338 isLegalMaskedCompressStore(DataTy, Alignment)) ||
1339 (Opcode == Instruction::Load &&
1340 isLegalMaskedExpandLoad(DataType: DataTy, Alignment));
1341 if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
1342 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1343 // Example compressstore sequence:
1344 // vsetivli zero, 8, e32, m2, ta, ma (ignored)
1345 // vcompress.vm v10, v8, v0
1346 // vcpop.m a1, v0
1347 // vsetvli zero, a1, e32, m2, ta, ma
1348 // vse32.v v10, (a0)
1349 // Example expandload sequence:
1350 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
1351 // vcpop.m a1, v0
1352 // vsetvli zero, a1, e32, m2, ta, ma
1353 // vle32.v v10, (a0)
1354 // vsetivli zero, 8, e32, m2, ta, ma
1355 // viota.m v12, v0
1356 // vrgather.vv v8, v10, v12, v0.t
1357 auto MemOpCost =
1358 getMemoryOpCost(Opcode, Src: DataTy, Alignment, /*AddressSpace*/ 0, CostKind);
1359 auto LT = getTypeLegalizationCost(Ty: DataTy);
1360 SmallVector<unsigned, 4> Opcodes{RISCV::VSETVLI};
1361 if (VariableMask)
1362 Opcodes.push_back(Elt: RISCV::VCPOP_M);
1363 if (Opcode == Instruction::Store)
1364 Opcodes.append(IL: {RISCV::VCOMPRESS_VM});
1365 else
1366 Opcodes.append(IL: {RISCV::VSETIVLI, RISCV::VIOTA_M, RISCV::VRGATHER_VV});
1367 return MemOpCost +
1368 LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1369}
1370
1371InstructionCost
1372RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1373 TTI::TargetCostKind CostKind) const {
1374 Type *DataTy = MICA.getDataType();
1375 Align Alignment = MICA.getAlignment();
1376
1377 if (!isLegalStridedLoadStore(DataType: DataTy, Alignment))
1378 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1379
1380 if (CostKind == TTI::TCK_CodeSize)
1381 return TTI::TCC_Basic;
1382
1383 // Cost is proportional to the number of memory operations implied. For
1384 // scalable vectors, we use an estimate on that number since we don't
1385 // know exactly what VL will be.
1386 auto &VTy = *cast<VectorType>(Val: DataTy);
1387 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1388 return NumLoads * TTI::TCC_Basic;
1389}
1390
1391InstructionCost
1392RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1393 // FIXME: This is a property of the default vector convention, not
1394 // all possible calling conventions. Fixing that will require
1395 // some TTI API and SLP rework.
1396 InstructionCost Cost = 0;
1397 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
1398 for (auto *Ty : Tys) {
1399 if (!Ty->isVectorTy())
1400 continue;
1401 Align A = DL.getPrefTypeAlign(Ty);
1402 Cost += getMemoryOpCost(Opcode: Instruction::Store, Src: Ty, Alignment: A, AddressSpace: 0, CostKind) +
1403 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: A, AddressSpace: 0, CostKind);
1404 }
1405 return Cost;
1406}
1407
1408// Currently, these represent both throughput and codesize costs
1409// for the respective intrinsics. The costs in this table are simply
1410// instruction counts with the following adjustments made:
1411// * One vsetvli is considered free.
1412static const CostTblEntry VectorIntrinsicCostTable[]{
1413 {.ISD: Intrinsic::floor, .Type: MVT::f32, .Cost: 9},
1414 {.ISD: Intrinsic::floor, .Type: MVT::f64, .Cost: 9},
1415 {.ISD: Intrinsic::ceil, .Type: MVT::f32, .Cost: 9},
1416 {.ISD: Intrinsic::ceil, .Type: MVT::f64, .Cost: 9},
1417 {.ISD: Intrinsic::trunc, .Type: MVT::f32, .Cost: 7},
1418 {.ISD: Intrinsic::trunc, .Type: MVT::f64, .Cost: 7},
1419 {.ISD: Intrinsic::round, .Type: MVT::f32, .Cost: 9},
1420 {.ISD: Intrinsic::round, .Type: MVT::f64, .Cost: 9},
1421 {.ISD: Intrinsic::roundeven, .Type: MVT::f32, .Cost: 9},
1422 {.ISD: Intrinsic::roundeven, .Type: MVT::f64, .Cost: 9},
1423 {.ISD: Intrinsic::rint, .Type: MVT::f32, .Cost: 7},
1424 {.ISD: Intrinsic::rint, .Type: MVT::f64, .Cost: 7},
1425 {.ISD: Intrinsic::nearbyint, .Type: MVT::f32, .Cost: 9},
1426 {.ISD: Intrinsic::nearbyint, .Type: MVT::f64, .Cost: 9},
1427 {.ISD: Intrinsic::bswap, .Type: MVT::i16, .Cost: 3},
1428 {.ISD: Intrinsic::bswap, .Type: MVT::i32, .Cost: 12},
1429 {.ISD: Intrinsic::bswap, .Type: MVT::i64, .Cost: 31},
1430 {.ISD: Intrinsic::bitreverse, .Type: MVT::i8, .Cost: 17},
1431 {.ISD: Intrinsic::bitreverse, .Type: MVT::i16, .Cost: 24},
1432 {.ISD: Intrinsic::bitreverse, .Type: MVT::i32, .Cost: 33},
1433 {.ISD: Intrinsic::bitreverse, .Type: MVT::i64, .Cost: 52},
1434 {.ISD: Intrinsic::ctpop, .Type: MVT::i8, .Cost: 12},
1435 {.ISD: Intrinsic::ctpop, .Type: MVT::i16, .Cost: 19},
1436 {.ISD: Intrinsic::ctpop, .Type: MVT::i32, .Cost: 20},
1437 {.ISD: Intrinsic::ctpop, .Type: MVT::i64, .Cost: 21},
1438 {.ISD: Intrinsic::ctlz, .Type: MVT::i8, .Cost: 19},
1439 {.ISD: Intrinsic::ctlz, .Type: MVT::i16, .Cost: 28},
1440 {.ISD: Intrinsic::ctlz, .Type: MVT::i32, .Cost: 31},
1441 {.ISD: Intrinsic::ctlz, .Type: MVT::i64, .Cost: 35},
1442 {.ISD: Intrinsic::cttz, .Type: MVT::i8, .Cost: 16},
1443 {.ISD: Intrinsic::cttz, .Type: MVT::i16, .Cost: 23},
1444 {.ISD: Intrinsic::cttz, .Type: MVT::i32, .Cost: 24},
1445 {.ISD: Intrinsic::cttz, .Type: MVT::i64, .Cost: 25},
1446};
1447
1448InstructionCost
1449RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1450 TTI::TargetCostKind CostKind) const {
1451 auto *RetTy = ICA.getReturnType();
1452 switch (ICA.getID()) {
1453 case Intrinsic::lrint:
1454 case Intrinsic::llrint:
1455 case Intrinsic::lround:
1456 case Intrinsic::llround: {
1457 auto LT = getTypeLegalizationCost(Ty: RetTy);
1458 Type *SrcTy = ICA.getArgTypes().front();
1459 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
1460 if (ST->hasVInstructions() && LT.second.isVector()) {
1461 SmallVector<unsigned, 2> Ops;
1462 unsigned SrcEltSz = DL.getTypeSizeInBits(Ty: SrcTy->getScalarType());
1463 unsigned DstEltSz = DL.getTypeSizeInBits(Ty: RetTy->getScalarType());
1464 if (LT.second.getVectorElementType() == MVT::bf16) {
1465 if (!ST->hasVInstructionsBF16Minimal())
1466 return InstructionCost::getInvalid();
1467 if (DstEltSz == 32)
1468 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFCVT_X_F_V};
1469 else
1470 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVT_X_F_V};
1471 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1472 !ST->hasVInstructionsF16()) {
1473 if (!ST->hasVInstructionsF16Minimal())
1474 return InstructionCost::getInvalid();
1475 if (DstEltSz == 32)
1476 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFCVT_X_F_V};
1477 else
1478 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_X_F_V};
1479
1480 } else if (SrcEltSz > DstEltSz) {
1481 Ops = {RISCV::VFNCVT_X_F_W};
1482 } else if (SrcEltSz < DstEltSz) {
1483 Ops = {RISCV::VFWCVT_X_F_V};
1484 } else {
1485 Ops = {RISCV::VFCVT_X_F_V};
1486 }
1487
1488 // We need to use the source LMUL in the case of a narrowing op, and the
1489 // destination LMUL otherwise.
1490 if (SrcEltSz > DstEltSz)
1491 return SrcLT.first *
1492 getRISCVInstructionCost(OpCodes: Ops, VT: SrcLT.second, CostKind);
1493 return LT.first * getRISCVInstructionCost(OpCodes: Ops, VT: LT.second, CostKind);
1494 }
1495 break;
1496 }
1497 case Intrinsic::ceil:
1498 case Intrinsic::floor:
1499 case Intrinsic::trunc:
1500 case Intrinsic::rint:
1501 case Intrinsic::round:
1502 case Intrinsic::roundeven: {
1503 // These all use the same code.
1504 auto LT = getTypeLegalizationCost(Ty: RetTy);
1505 if (!LT.second.isVector() && TLI->isOperationCustom(Op: ISD::FCEIL, VT: LT.second))
1506 return LT.first * 8;
1507 break;
1508 }
1509 case Intrinsic::umin:
1510 case Intrinsic::umax:
1511 case Intrinsic::smin:
1512 case Intrinsic::smax: {
1513 auto LT = getTypeLegalizationCost(Ty: RetTy);
1514 if (LT.second.isScalarInteger() && ST->hasStdExtZbb())
1515 return LT.first;
1516
1517 if (ST->hasVInstructions() && LT.second.isVector()) {
1518 unsigned Op;
1519 switch (ICA.getID()) {
1520 case Intrinsic::umin:
1521 Op = RISCV::VMINU_VV;
1522 break;
1523 case Intrinsic::umax:
1524 Op = RISCV::VMAXU_VV;
1525 break;
1526 case Intrinsic::smin:
1527 Op = RISCV::VMIN_VV;
1528 break;
1529 case Intrinsic::smax:
1530 Op = RISCV::VMAX_VV;
1531 break;
1532 }
1533 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1534 }
1535 break;
1536 }
1537 case Intrinsic::sadd_sat:
1538 case Intrinsic::ssub_sat:
1539 case Intrinsic::uadd_sat:
1540 case Intrinsic::usub_sat: {
1541 auto LT = getTypeLegalizationCost(Ty: RetTy);
1542 if (ST->hasVInstructions() && LT.second.isVector()) {
1543 unsigned Op;
1544 switch (ICA.getID()) {
1545 case Intrinsic::sadd_sat:
1546 Op = RISCV::VSADD_VV;
1547 break;
1548 case Intrinsic::ssub_sat:
1549 Op = RISCV::VSSUB_VV;
1550 break;
1551 case Intrinsic::uadd_sat:
1552 Op = RISCV::VSADDU_VV;
1553 break;
1554 case Intrinsic::usub_sat:
1555 Op = RISCV::VSSUBU_VV;
1556 break;
1557 }
1558 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1559 }
1560 break;
1561 }
1562 case Intrinsic::fma:
1563 case Intrinsic::fmuladd: {
1564 // TODO: handle promotion with f16/bf16 with zvfhmin/zvfbfmin
1565 auto LT = getTypeLegalizationCost(Ty: RetTy);
1566 if (ST->hasVInstructions() && LT.second.isVector())
1567 return LT.first *
1568 getRISCVInstructionCost(OpCodes: RISCV::VFMADD_VV, VT: LT.second, CostKind);
1569 break;
1570 }
1571 case Intrinsic::fabs: {
1572 auto LT = getTypeLegalizationCost(Ty: RetTy);
1573 if (ST->hasVInstructions() && LT.second.isVector()) {
1574 // lui a0, 8
1575 // addi a0, a0, -1
1576 // vsetvli a1, zero, e16, m1, ta, ma
1577 // vand.vx v8, v8, a0
1578 // f16 with zvfhmin and bf16 with zvfhbmin
1579 if (LT.second.getVectorElementType() == MVT::bf16 ||
1580 (LT.second.getVectorElementType() == MVT::f16 &&
1581 !ST->hasVInstructionsF16()))
1582 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VAND_VX, VT: LT.second,
1583 CostKind) +
1584 2;
1585 else
1586 return LT.first *
1587 getRISCVInstructionCost(OpCodes: RISCV::VFSGNJX_VV, VT: LT.second, CostKind);
1588 }
1589 break;
1590 }
1591 case Intrinsic::sqrt: {
1592 auto LT = getTypeLegalizationCost(Ty: RetTy);
1593 if (ST->hasVInstructions() && LT.second.isVector()) {
1594 SmallVector<unsigned, 4> ConvOp;
1595 SmallVector<unsigned, 2> FsqrtOp;
1596 MVT ConvType = LT.second;
1597 MVT FsqrtType = LT.second;
1598 // f16 with zvfhmin and bf16 with zvfbfmin and the type of nxv32[b]f16
1599 // will be spilt.
1600 if (LT.second.getVectorElementType() == MVT::bf16) {
1601 if (LT.second == MVT::nxv32bf16) {
1602 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVTBF16_F_F_V,
1603 RISCV::VFNCVTBF16_F_F_W, RISCV::VFNCVTBF16_F_F_W};
1604 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1605 ConvType = MVT::nxv16f16;
1606 FsqrtType = MVT::nxv16f32;
1607 } else {
1608 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFNCVTBF16_F_F_W};
1609 FsqrtOp = {RISCV::VFSQRT_V};
1610 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1611 }
1612 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1613 !ST->hasVInstructionsF16()) {
1614 if (LT.second == MVT::nxv32f16) {
1615 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_F_F_V,
1616 RISCV::VFNCVT_F_F_W, RISCV::VFNCVT_F_F_W};
1617 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1618 ConvType = MVT::nxv16f16;
1619 FsqrtType = MVT::nxv16f32;
1620 } else {
1621 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFNCVT_F_F_W};
1622 FsqrtOp = {RISCV::VFSQRT_V};
1623 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1624 }
1625 } else {
1626 FsqrtOp = {RISCV::VFSQRT_V};
1627 }
1628
1629 return LT.first * (getRISCVInstructionCost(OpCodes: FsqrtOp, VT: FsqrtType, CostKind) +
1630 getRISCVInstructionCost(OpCodes: ConvOp, VT: ConvType, CostKind));
1631 }
1632 break;
1633 }
1634 case Intrinsic::cttz:
1635 case Intrinsic::ctlz:
1636 case Intrinsic::ctpop: {
1637 auto LT = getTypeLegalizationCost(Ty: RetTy);
1638 if (ST->hasStdExtZvbb() && LT.second.isVector()) {
1639 unsigned Op;
1640 switch (ICA.getID()) {
1641 case Intrinsic::cttz:
1642 Op = RISCV::VCTZ_V;
1643 break;
1644 case Intrinsic::ctlz:
1645 Op = RISCV::VCLZ_V;
1646 break;
1647 case Intrinsic::ctpop:
1648 Op = RISCV::VCPOP_V;
1649 break;
1650 }
1651 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1652 }
1653 break;
1654 }
1655 case Intrinsic::abs: {
1656 auto LT = getTypeLegalizationCost(Ty: RetTy);
1657 if (ST->hasVInstructions() && LT.second.isVector()) {
1658 // vabs.v v10, v8 (alias for vabd.vx v10, v8, zero)
1659 if (ST->hasStdExtZvabd())
1660 return LT.first *
1661 getRISCVInstructionCost(OpCodes: {RISCV::VABD_VX}, VT: LT.second, CostKind);
1662
1663 // vrsub.vi v10, v8, 0
1664 // vmax.vv v8, v8, v10
1665 return LT.first *
1666 getRISCVInstructionCost(OpCodes: {RISCV::VRSUB_VI, RISCV::VMAX_VV},
1667 VT: LT.second, CostKind);
1668 }
1669 break;
1670 }
1671 case Intrinsic::fshl:
1672 case Intrinsic::fshr: {
1673 if (ICA.getArgs().empty())
1674 break;
1675
1676 // Funnel-shifts are ROTL/ROTR when the first and second operand are equal.
1677 // When Zbb/Zbkb is enabled we can use a single ROL(W)/ROR(I)(W)
1678 // instruction.
1679 if ((ST->hasStdExtZbb() || ST->hasStdExtZbkb()) && RetTy->isIntegerTy() &&
1680 ICA.getArgs()[0] == ICA.getArgs()[1] &&
1681 (RetTy->getIntegerBitWidth() == 32 ||
1682 RetTy->getIntegerBitWidth() == 64) &&
1683 RetTy->getIntegerBitWidth() <= ST->getXLen()) {
1684 return 1;
1685 }
1686 break;
1687 }
1688 case Intrinsic::clmul: {
1689 auto LT = getTypeLegalizationCost(Ty: RetTy);
1690 if (!LT.second.isVector() && ST->hasStdExtZvbc() && !ST->hasStdExtZbc() &&
1691 !ST->hasStdExtZbkc()) {
1692 // TODO: Once custom lowering in this case for RV32 is added, this guard
1693 // should be removed and the cost model should be updated.
1694 if (!ST->is64Bit() || LT.second != MVT::i64)
1695 break;
1696 // vmv.s.x v8, a0
1697 // vclmul.vx v8, v8, a1
1698 // vmv.x.s a0, v8
1699 MVT VecVT = MVT::getScalableVectorVT(VT: LT.second, NumElements: 1);
1700 return LT.first * getRISCVInstructionCost(
1701 OpCodes: {RISCV::VMV_S_X, RISCV::VCLMUL_VX, RISCV::VMV_X_S},
1702 VT: VecVT, CostKind);
1703 }
1704 break;
1705 }
1706 case Intrinsic::masked_udiv:
1707 return getArithmeticInstrCost(Opcode: Instruction::UDiv, Ty: ICA.getReturnType(),
1708 CostKind);
1709 case Intrinsic::masked_sdiv:
1710 return getArithmeticInstrCost(Opcode: Instruction::SDiv, Ty: ICA.getReturnType(),
1711 CostKind);
1712 case Intrinsic::masked_urem:
1713 return getArithmeticInstrCost(Opcode: Instruction::URem, Ty: ICA.getReturnType(),
1714 CostKind);
1715 case Intrinsic::masked_srem:
1716 return getArithmeticInstrCost(Opcode: Instruction::SRem, Ty: ICA.getReturnType(),
1717 CostKind);
1718 case Intrinsic::get_active_lane_mask: {
1719 if (ST->hasVInstructions()) {
1720 Type *ExpRetTy = VectorType::get(
1721 ElementType: ICA.getArgTypes()[0], EC: cast<VectorType>(Val: RetTy)->getElementCount());
1722 auto LT = getTypeLegalizationCost(Ty: ExpRetTy);
1723
1724 // vid.v v8 // considered hoisted
1725 // vsaddu.vx v8, v8, a0
1726 // vmsltu.vx v0, v8, a1
1727 return LT.first *
1728 getRISCVInstructionCost(OpCodes: {RISCV::VSADDU_VX, RISCV::VMSLTU_VX},
1729 VT: LT.second, CostKind);
1730 }
1731 break;
1732 }
1733 // TODO: add more intrinsic
1734 case Intrinsic::stepvector: {
1735 auto LT = getTypeLegalizationCost(Ty: RetTy);
1736 // Legalisation of illegal types involves an `index' instruction plus
1737 // (LT.first - 1) vector adds.
1738 if (ST->hasVInstructions())
1739 return getRISCVInstructionCost(OpCodes: RISCV::VID_V, VT: LT.second, CostKind) +
1740 (LT.first - 1) *
1741 getRISCVInstructionCost(OpCodes: RISCV::VADD_VX, VT: LT.second, CostKind);
1742 return 1 + (LT.first - 1);
1743 }
1744 case Intrinsic::vector_splice_left:
1745 case Intrinsic::vector_splice_right: {
1746 auto LT = getTypeLegalizationCost(Ty: RetTy);
1747 // Constant offsets fall through to getShuffleCost.
1748 if (!ICA.isTypeBasedOnly() && isa<ConstantInt>(Val: ICA.getArgs()[2]))
1749 break;
1750 if (ST->hasVInstructions() && LT.second.isVector()) {
1751 return LT.first *
1752 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX},
1753 VT: LT.second, CostKind);
1754 }
1755 break;
1756 }
1757 case Intrinsic::experimental_cttz_elts: {
1758 if (!ST->hasVInstructions())
1759 break;
1760 InstructionCost Cost = 0;
1761 Type *ArgTy = ICA.getArgTypes()[0];
1762 auto LT = getTypeLegalizationCost(Ty: ArgTy);
1763
1764 // If the element type is not i1, do a comparison with all-zeros.
1765 if (LT.second.getVectorElementType() != MVT::i1)
1766 Cost += getRISCVInstructionCost(OpCodes: RISCV::VMSNE_VI, VT: LT.second, CostKind);
1767
1768 Cost += getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind);
1769
1770 // If zero_is_poison is false, then we will generate additional
1771 // cmp + select instructions to convert -1 to EVL.
1772 Type *BoolTy = Type::getInt1Ty(C&: RetTy->getContext());
1773 if (ICA.getArgs().size() > 1 &&
1774 cast<ConstantInt>(Val: ICA.getArgs()[1])->isZero())
1775 Cost += getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: BoolTy, CondTy: RetTy,
1776 VecPred: CmpInst::ICMP_SLT, CostKind) +
1777 getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: RetTy, CondTy: BoolTy,
1778 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1779
1780 return LT.first * Cost;
1781 }
1782 case Intrinsic::experimental_vp_splice: {
1783 // To support type-based query from vectorizer, set the index to 0.
1784 // Note that index only change the cost from vslide.vx to vslide.vi and in
1785 // current implementations they have same costs.
1786 return getShuffleCost(Kind: TTI::SK_Splice, DstTy: cast<VectorType>(Val: ICA.getReturnType()),
1787 SrcTy: cast<VectorType>(Val: ICA.getArgTypes()[0]), CostKind, Mask: {},
1788 Index: 0, SubTp: cast<VectorType>(Val: ICA.getReturnType()));
1789 }
1790 case Intrinsic::vp_merge: {
1791 // If an operand is a binary op and the type is legal, RISCVVectorPeephole
1792 // will likely fold the resulting vmerge.vvm away.
1793 if (ICA.getVectorInstrContext() == VectorInstrContext::BinaryOp &&
1794 getTypeLegalizationCost(Ty: RetTy).first == 1)
1795 return TTI::TCC_Free;
1796 break;
1797 }
1798 case Intrinsic::fptoui_sat:
1799 case Intrinsic::fptosi_sat: {
1800 InstructionCost Cost = 0;
1801 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
1802 Type *SrcTy = ICA.getArgTypes()[0];
1803
1804 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
1805 auto DstLT = getTypeLegalizationCost(Ty: RetTy);
1806 if (!SrcTy->isVectorTy())
1807 break;
1808
1809 if (!SrcLT.first.isValid() || !DstLT.first.isValid())
1810 return InstructionCost::getInvalid();
1811
1812 Cost +=
1813 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
1814 Dst: RetTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
1815
1816 // Handle NaN.
1817 // vmfne v0, v8, v8 # If v8[i] is NaN set v0[i] to 1.
1818 // vmerge.vim v8, v8, 0, v0 # Convert NaN to 0.
1819 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
1820 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: SrcTy, CondTy,
1821 VecPred: CmpInst::FCMP_UNO, CostKind);
1822 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
1823 VecPred: CmpInst::FCMP_UNO, CostKind);
1824 return Cost;
1825 }
1826 case Intrinsic::experimental_vector_extract_last_active: {
1827 auto *ValTy = cast<VectorType>(Val: ICA.getArgTypes()[0]);
1828 auto *MaskTy = cast<VectorType>(Val: ICA.getArgTypes()[1]);
1829
1830 auto ValLT = getTypeLegalizationCost(Ty: ValTy);
1831 auto MaskLT = getTypeLegalizationCost(Ty: MaskTy);
1832
1833 // TODO: Return cheaper cost when the entire lane is inactive.
1834 // The expected asm sequence is:
1835 // vcpop.m a0, v0
1836 // beqz a0, exit # Return passthru when the entire lane is inactive.
1837 // vid v10, v0.t
1838 // vredmaxu.vs v10, v10, v10
1839 // vmv.x.s a0, v10
1840 // zext.b a0, a0
1841 // vslidedown.vx v8, v8, a0
1842 // vmv.x.s a0, v8
1843 // exit:
1844 // ...
1845
1846 // Find a suitable type for a stepvector.
1847 ConstantRange VScaleRange(APInt(64, 1), APInt::getZero(numBits: 64));
1848 unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
1849 RetVT: TLI->getVectorIdxTy(DL: getDataLayout()), EC: MaskTy->getElementCount(),
1850 /*ZeroIsPoison=*/true, VScaleRange: &VScaleRange);
1851 EltWidth = std::max(a: EltWidth, b: MaskTy->getScalarSizeInBits());
1852 Type *StepTy = Type::getIntNTy(C&: MaskTy->getContext(), N: EltWidth);
1853 auto *StepVecTy = VectorType::get(ElementType: StepTy, EC: ValTy->getElementCount());
1854 auto StepLT = getTypeLegalizationCost(Ty: StepVecTy);
1855
1856 // Currently expandVectorFindLastActive cannot handle step vector split.
1857 // So return invalid when the type needs split.
1858 // FIXME: Remove this if expandVectorFindLastActive supports split vector.
1859 if (StepLT.first > 1)
1860 return InstructionCost::getInvalid();
1861
1862 InstructionCost Cost = 0;
1863 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
1864
1865 Cost += MaskLT.first *
1866 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: MaskLT.second, CostKind);
1867 Cost += getCFInstrCost(Opcode: Instruction::CondBr, CostKind, I: nullptr);
1868 Cost += StepLT.first *
1869 getRISCVInstructionCost(OpCodes: Opcodes, VT: StepLT.second, CostKind);
1870 Cost += getCastInstrCost(Opcode: Instruction::ZExt,
1871 Dst: Type::getInt64Ty(C&: ValTy->getContext()), Src: StepTy,
1872 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
1873 Cost += ValLT.first *
1874 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VI, RISCV::VMV_X_S},
1875 VT: ValLT.second, CostKind);
1876 return Cost;
1877 }
1878 }
1879
1880 if (ST->hasVInstructions() && RetTy->isVectorTy()) {
1881 if (auto LT = getTypeLegalizationCost(Ty: RetTy);
1882 LT.second.isVector()) {
1883 MVT EltTy = LT.second.getVectorElementType();
1884 if (const auto *Entry = CostTableLookup(Table: VectorIntrinsicCostTable,
1885 ISD: ICA.getID(), Ty: EltTy))
1886 return LT.first * Entry->Cost;
1887 }
1888 }
1889
1890 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1891}
1892
1893InstructionCost
1894RISCVTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
1895 const SCEV *Ptr,
1896 TTI::TargetCostKind CostKind) const {
1897 // Address computations for vector indexed load/store likely require an offset
1898 // and/or scaling.
1899 if (ST->hasVInstructions() && PtrTy->isVectorTy())
1900 return getArithmeticInstrCost(Opcode: Instruction::Add, Ty: PtrTy, CostKind);
1901
1902 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
1903}
1904
1905InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
1906 Type *Src,
1907 TTI::CastContextHint CCH,
1908 TTI::TargetCostKind CostKind,
1909 const Instruction *I) const {
1910 bool IsVectorType = isa<VectorType>(Val: Dst) && isa<VectorType>(Val: Src);
1911 if (!IsVectorType)
1912 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1913
1914 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
1915 // For now, skip all fixed vector cost analysis when P extension is available
1916 // to avoid crashes in getMinRVVVectorSizeInBits()
1917 if (ST->hasStdExtP() &&
1918 (isa<FixedVectorType>(Val: Dst) || isa<FixedVectorType>(Val: Src))) {
1919 return 1; // Treat as single instruction cost for now
1920 }
1921
1922 // FIXME: Need to compute legalizing cost for illegal types. The current
1923 // code handles only legal types and those which can be trivially
1924 // promoted to legal.
1925 if (!ST->hasVInstructions() || Src->getScalarSizeInBits() > ST->getELen() ||
1926 Dst->getScalarSizeInBits() > ST->getELen())
1927 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1928
1929 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1930 assert(ISD && "Invalid opcode");
1931 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: Src);
1932 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: Dst);
1933
1934 // Handle i1 source and dest cases *before* calling logic in BasicTTI.
1935 // The shared implementation doesn't model vector widening during legalization
1936 // and instead assumes scalarization. In order to scalarize an <N x i1>
1937 // vector, we need to extend/trunc to/from i8. If we don't special case
1938 // this, we can get an infinite recursion cycle.
1939 switch (ISD) {
1940 default:
1941 break;
1942 case ISD::SIGN_EXTEND:
1943 case ISD::ZERO_EXTEND:
1944 if (Src->getScalarSizeInBits() == 1) {
1945 // We do not use vsext/vzext to extend from mask vector.
1946 // Instead we use the following instructions to extend from mask vector:
1947 // vmv.v.i v8, 0
1948 // vmerge.vim v8, v8, -1, v0 (repeated per split)
1949 return getRISCVInstructionCost(OpCodes: RISCV::VMV_V_I, VT: DstLT.second, CostKind) +
1950 DstLT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VIM,
1951 VT: DstLT.second, CostKind) +
1952 DstLT.first - 1;
1953 }
1954 break;
1955 case ISD::TRUNCATE:
1956 if (Dst->getScalarSizeInBits() == 1) {
1957 // We do not use several vncvt to truncate to mask vector. So we could
1958 // not use PowDiff to calculate it.
1959 // Instead we use the following instructions to truncate to mask vector:
1960 // vand.vi v8, v8, 1
1961 // vmsne.vi v0, v8, 0
1962 return SrcLT.first *
1963 getRISCVInstructionCost(OpCodes: {RISCV::VAND_VI, RISCV::VMSNE_VI},
1964 VT: SrcLT.second, CostKind) +
1965 SrcLT.first - 1;
1966 }
1967 break;
1968 };
1969
1970 // Our actual lowering for the case where a wider legal type is available
1971 // uses promotion to the wider type. This is reflected in the result of
1972 // getTypeLegalizationCost, but BasicTTI assumes the widened cases are
1973 // scalarized if the legalized Src and Dst are not equal sized.
1974 const DataLayout &DL = this->getDataLayout();
1975 if (!SrcLT.second.isVector() || !DstLT.second.isVector() ||
1976 !SrcLT.first.isValid() || !DstLT.first.isValid() ||
1977 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Src),
1978 RHS: SrcLT.second.getSizeInBits()) ||
1979 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Dst),
1980 RHS: DstLT.second.getSizeInBits()) ||
1981 SrcLT.first > 1 || DstLT.first > 1)
1982 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1983
1984 // The split cost is handled by the base getCastInstrCost
1985 assert((SrcLT.first == 1) && (DstLT.first == 1) && "Illegal type");
1986
1987 int PowDiff = (int)Log2_32(Value: DstLT.second.getScalarSizeInBits()) -
1988 (int)Log2_32(Value: SrcLT.second.getScalarSizeInBits());
1989 switch (ISD) {
1990 case ISD::SIGN_EXTEND:
1991 case ISD::ZERO_EXTEND: {
1992 if ((PowDiff < 1) || (PowDiff > 3))
1993 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1994 unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
1995 unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
1996 unsigned Op =
1997 (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
1998 return getRISCVInstructionCost(OpCodes: Op, VT: DstLT.second, CostKind);
1999 }
2000 case ISD::TRUNCATE:
2001 case ISD::FP_EXTEND:
2002 case ISD::FP_ROUND: {
2003 // Counts of narrow/widen instructions.
2004 unsigned SrcEltSize = SrcLT.second.getScalarSizeInBits();
2005 unsigned DstEltSize = DstLT.second.getScalarSizeInBits();
2006
2007 unsigned Op = (ISD == ISD::TRUNCATE) ? RISCV::VNSRL_WI
2008 : (ISD == ISD::FP_EXTEND) ? RISCV::VFWCVT_F_F_V
2009 : RISCV::VFNCVT_F_F_W;
2010 InstructionCost Cost = 0;
2011 for (; SrcEltSize != DstEltSize;) {
2012 MVT ElementMVT = (ISD == ISD::TRUNCATE)
2013 ? MVT::getIntegerVT(BitWidth: DstEltSize)
2014 : MVT::getFloatingPointVT(BitWidth: DstEltSize);
2015 MVT DstMVT = DstLT.second.changeVectorElementType(EltVT: ElementMVT);
2016 DstEltSize =
2017 (DstEltSize > SrcEltSize) ? DstEltSize >> 1 : DstEltSize << 1;
2018 Cost += getRISCVInstructionCost(OpCodes: Op, VT: DstMVT, CostKind);
2019 }
2020 return Cost;
2021 }
2022 case ISD::FP_TO_SINT:
2023 case ISD::FP_TO_UINT: {
2024 unsigned IsSigned = ISD == ISD::FP_TO_SINT;
2025 unsigned FCVT = IsSigned ? RISCV::VFCVT_RTZ_X_F_V : RISCV::VFCVT_RTZ_XU_F_V;
2026 unsigned FWCVT =
2027 IsSigned ? RISCV::VFWCVT_RTZ_X_F_V : RISCV::VFWCVT_RTZ_XU_F_V;
2028 unsigned FNCVT =
2029 IsSigned ? RISCV::VFNCVT_RTZ_X_F_W : RISCV::VFNCVT_RTZ_XU_F_W;
2030 unsigned SrcEltSize = Src->getScalarSizeInBits();
2031 unsigned DstEltSize = Dst->getScalarSizeInBits();
2032 InstructionCost Cost = 0;
2033 if ((SrcEltSize == 16) &&
2034 (!ST->hasVInstructionsF16() || ((DstEltSize / 2) > SrcEltSize))) {
2035 // If the target only supports zvfhmin or it is fp16-to-i64 conversion
2036 // pre-widening to f32 and then convert f32 to integer
2037 VectorType *VecF32Ty =
2038 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2039 EC: cast<VectorType>(Val: Dst)->getElementCount());
2040 std::pair<InstructionCost, MVT> VecF32LT =
2041 getTypeLegalizationCost(Ty: VecF32Ty);
2042 Cost +=
2043 VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFWCVT_F_F_V,
2044 VT: VecF32LT.second, CostKind);
2045 Cost += getCastInstrCost(Opcode, Dst, Src: VecF32Ty, CCH, CostKind, I);
2046 return Cost;
2047 }
2048 if (DstEltSize == SrcEltSize)
2049 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2050 else if (DstEltSize > SrcEltSize)
2051 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2052 else { // (SrcEltSize > DstEltSize)
2053 // First do a narrowing conversion to an integer half the size, then
2054 // truncate if needed.
2055 MVT ElementVT = MVT::getIntegerVT(BitWidth: SrcEltSize / 2);
2056 MVT VecVT = DstLT.second.changeVectorElementType(EltVT: ElementVT);
2057 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: VecVT, CostKind);
2058 if ((SrcEltSize / 2) > DstEltSize) {
2059 Type *VecTy = EVT(VecVT).getTypeForEVT(Context&: Dst->getContext());
2060 Cost +=
2061 getCastInstrCost(Opcode: Instruction::Trunc, Dst, Src: VecTy, CCH, CostKind, I);
2062 }
2063 }
2064 return Cost;
2065 }
2066 case ISD::SINT_TO_FP:
2067 case ISD::UINT_TO_FP: {
2068 unsigned IsSigned = ISD == ISD::SINT_TO_FP;
2069 unsigned FCVT = IsSigned ? RISCV::VFCVT_F_X_V : RISCV::VFCVT_F_XU_V;
2070 unsigned FWCVT = IsSigned ? RISCV::VFWCVT_F_X_V : RISCV::VFWCVT_F_XU_V;
2071 unsigned FNCVT = IsSigned ? RISCV::VFNCVT_F_X_W : RISCV::VFNCVT_F_XU_W;
2072 unsigned SrcEltSize = Src->getScalarSizeInBits();
2073 unsigned DstEltSize = Dst->getScalarSizeInBits();
2074
2075 InstructionCost Cost = 0;
2076 if ((DstEltSize == 16) &&
2077 (!ST->hasVInstructionsF16() || ((SrcEltSize / 2) > DstEltSize))) {
2078 // If the target only supports zvfhmin or it is i64-to-fp16 conversion
2079 // it is converted to f32 and then converted to f16
2080 VectorType *VecF32Ty =
2081 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2082 EC: cast<VectorType>(Val: Dst)->getElementCount());
2083 std::pair<InstructionCost, MVT> VecF32LT =
2084 getTypeLegalizationCost(Ty: VecF32Ty);
2085 Cost += getCastInstrCost(Opcode, Dst: VecF32Ty, Src, CCH, CostKind, I);
2086 Cost += VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFNCVT_F_F_W,
2087 VT: DstLT.second, CostKind);
2088 return Cost;
2089 }
2090
2091 if (DstEltSize == SrcEltSize)
2092 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2093 else if (DstEltSize > SrcEltSize) {
2094 if ((DstEltSize / 2) > SrcEltSize) {
2095 VectorType *VecTy =
2096 VectorType::get(ElementType: IntegerType::get(C&: Dst->getContext(), NumBits: DstEltSize / 2),
2097 EC: cast<VectorType>(Val: Dst)->getElementCount());
2098 unsigned Op = IsSigned ? Instruction::SExt : Instruction::ZExt;
2099 Cost += getCastInstrCost(Opcode: Op, Dst: VecTy, Src, CCH, CostKind, I);
2100 }
2101 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2102 } else
2103 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: DstLT.second, CostKind);
2104 return Cost;
2105 }
2106 }
2107 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2108}
2109
2110unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) const {
2111 if (isa<ScalableVectorType>(Val: Ty)) {
2112 const unsigned EltSize = DL.getTypeSizeInBits(Ty: Ty->getElementType());
2113 const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue();
2114 const unsigned VectorBits = *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
2115 return RISCVTargetLowering::computeVLMAX(VectorBits, EltSize, MinSize);
2116 }
2117 return cast<FixedVectorType>(Val: Ty)->getNumElements();
2118}
2119
2120InstructionCost
2121RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
2122 FastMathFlags FMF,
2123 TTI::TargetCostKind CostKind) const {
2124 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2125 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2126
2127 // Skip if scalar size of Ty is bigger than ELEN.
2128 if (Ty->getScalarSizeInBits() > ST->getELen())
2129 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2130
2131 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2132 if (Ty->getElementType()->isIntegerTy(BitWidth: 1)) {
2133 // SelectionDAGBuilder does following transforms:
2134 // vector_reduce_{smin,umax}(<n x i1>) --> vector_reduce_or(<n x i1>)
2135 // vector_reduce_{smax,umin}(<n x i1>) --> vector_reduce_and(<n x i1>)
2136 if (IID == Intrinsic::umax || IID == Intrinsic::smin)
2137 return getArithmeticReductionCost(Opcode: Instruction::Or, Ty, FMF, CostKind);
2138 else
2139 return getArithmeticReductionCost(Opcode: Instruction::And, Ty, FMF, CostKind);
2140 }
2141
2142 if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) {
2143 SmallVector<unsigned, 3> Opcodes;
2144 InstructionCost ExtraCost = 0;
2145 switch (IID) {
2146 case Intrinsic::maximum:
2147 if (FMF.noNaNs()) {
2148 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2149 } else {
2150 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS,
2151 RISCV::VFMV_F_S};
2152 // Cost of Canonical Nan + branch
2153 // lui a0, 523264
2154 // fmv.w.x fa0, a0
2155 Type *DstTy = Ty->getScalarType();
2156 const unsigned EltTyBits = DstTy->getScalarSizeInBits();
2157 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2158 ExtraCost = 1 +
2159 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2160 CCH: TTI::CastContextHint::None, CostKind) +
2161 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2162 }
2163 break;
2164
2165 case Intrinsic::minimum:
2166 if (FMF.noNaNs()) {
2167 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2168 } else {
2169 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS,
2170 RISCV::VFMV_F_S};
2171 // Cost of Canonical Nan + branch
2172 // lui a0, 523264
2173 // fmv.w.x fa0, a0
2174 Type *DstTy = Ty->getScalarType();
2175 const unsigned EltTyBits = DL.getTypeSizeInBits(Ty: DstTy);
2176 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2177 ExtraCost = 1 +
2178 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2179 CCH: TTI::CastContextHint::None, CostKind) +
2180 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2181 }
2182 break;
2183 }
2184 return ExtraCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2185 }
2186
2187 // IR Reduction is composed by one rvv reduction instruction and vmv
2188 unsigned SplitOp;
2189 SmallVector<unsigned, 3> Opcodes;
2190 switch (IID) {
2191 default:
2192 llvm_unreachable("Unsupported intrinsic");
2193 case Intrinsic::smax:
2194 SplitOp = RISCV::VMAX_VV;
2195 Opcodes = {RISCV::VREDMAX_VS, RISCV::VMV_X_S};
2196 break;
2197 case Intrinsic::smin:
2198 SplitOp = RISCV::VMIN_VV;
2199 Opcodes = {RISCV::VREDMIN_VS, RISCV::VMV_X_S};
2200 break;
2201 case Intrinsic::umax:
2202 SplitOp = RISCV::VMAXU_VV;
2203 Opcodes = {RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
2204 break;
2205 case Intrinsic::umin:
2206 SplitOp = RISCV::VMINU_VV;
2207 Opcodes = {RISCV::VREDMINU_VS, RISCV::VMV_X_S};
2208 break;
2209 case Intrinsic::maxnum:
2210 SplitOp = RISCV::VFMAX_VV;
2211 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2212 break;
2213 case Intrinsic::minnum:
2214 SplitOp = RISCV::VFMIN_VV;
2215 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2216 break;
2217 }
2218 // Add a cost for data larger than LMUL8
2219 InstructionCost SplitCost =
2220 (LT.first > 1) ? (LT.first - 1) *
2221 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2222 : 0;
2223 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2224}
2225
2226InstructionCost
2227RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
2228 std::optional<FastMathFlags> FMF,
2229 TTI::TargetCostKind CostKind) const {
2230 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2231 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2232
2233 // Skip if scalar size of Ty is bigger than ELEN.
2234 if (Ty->getScalarSizeInBits() > ST->getELen())
2235 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2236
2237 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2238 assert(ISD && "Invalid opcode");
2239
2240 if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&
2241 ISD != ISD::FADD)
2242 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2243
2244 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2245 Type *ElementTy = Ty->getElementType();
2246 if (ElementTy->isIntegerTy(BitWidth: 1)) {
2247 // Example sequences:
2248 // vfirst.m a0, v0
2249 // seqz a0, a0
2250 if (LT.second == MVT::v1i1)
2251 return getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind) +
2252 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2253 VecPred: CmpInst::ICMP_EQ, CostKind);
2254
2255 if (ISD == ISD::AND) {
2256 // Example sequences:
2257 // vmand.mm v8, v9, v8 ; needed every time type is split
2258 // vmnot.m v8, v0 ; alias for vmnand
2259 // vcpop.m a0, v8
2260 // seqz a0, a0
2261
2262 // See the discussion: https://github.com/llvm/llvm-project/pull/119160
2263 // For LMUL <= 8, there is no splitting,
2264 // the sequences are vmnot, vcpop and seqz.
2265 // When LMUL > 8 and split = 1,
2266 // the sequences are vmnand, vcpop and seqz.
2267 // When LMUL > 8 and split > 1,
2268 // the sequences are (LT.first-2) * vmand, vmnand, vcpop and seqz.
2269 return ((LT.first > 2) ? (LT.first - 2) : 0) *
2270 getRISCVInstructionCost(OpCodes: RISCV::VMAND_MM, VT: LT.second, CostKind) +
2271 getRISCVInstructionCost(OpCodes: RISCV::VMNAND_MM, VT: LT.second, CostKind) +
2272 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2273 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2274 VecPred: CmpInst::ICMP_EQ, CostKind);
2275 } else if (ISD == ISD::XOR || ISD == ISD::ADD) {
2276 // Example sequences:
2277 // vsetvli a0, zero, e8, mf8, ta, ma
2278 // vmxor.mm v8, v0, v8 ; needed every time type is split
2279 // vcpop.m a0, v8
2280 // andi a0, a0, 1
2281 return (LT.first - 1) *
2282 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind) +
2283 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) + 1;
2284 } else {
2285 assert(ISD == ISD::OR);
2286 // Example sequences:
2287 // vsetvli a0, zero, e8, mf8, ta, ma
2288 // vmor.mm v8, v9, v8 ; needed every time type is split
2289 // vcpop.m a0, v0
2290 // snez a0, a0
2291 return (LT.first - 1) *
2292 getRISCVInstructionCost(OpCodes: RISCV::VMOR_MM, VT: LT.second, CostKind) +
2293 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2294 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2295 VecPred: CmpInst::ICMP_NE, CostKind);
2296 }
2297 }
2298
2299 // IR Reduction of or/and is composed by one vmv and one rvv reduction
2300 // instruction, and others is composed by two vmv and one rvv reduction
2301 // instruction
2302 unsigned SplitOp;
2303 SmallVector<unsigned, 3> Opcodes;
2304 switch (ISD) {
2305 case ISD::ADD:
2306 SplitOp = RISCV::VADD_VV;
2307 Opcodes = {RISCV::VMV_S_X, RISCV::VREDSUM_VS, RISCV::VMV_X_S};
2308 break;
2309 case ISD::OR:
2310 SplitOp = RISCV::VOR_VV;
2311 Opcodes = {RISCV::VREDOR_VS, RISCV::VMV_X_S};
2312 break;
2313 case ISD::XOR:
2314 SplitOp = RISCV::VXOR_VV;
2315 Opcodes = {RISCV::VMV_S_X, RISCV::VREDXOR_VS, RISCV::VMV_X_S};
2316 break;
2317 case ISD::AND:
2318 SplitOp = RISCV::VAND_VV;
2319 Opcodes = {RISCV::VREDAND_VS, RISCV::VMV_X_S};
2320 break;
2321 case ISD::FADD:
2322 // We can't promote f16/bf16 fadd reductions.
2323 if ((LT.second.getScalarType() == MVT::f16 && !ST->hasVInstructionsF16()) ||
2324 LT.second.getScalarType() == MVT::bf16)
2325 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2326 if (TTI::requiresOrderedReduction(FMF)) {
2327 Opcodes.push_back(Elt: RISCV::VFMV_S_F);
2328 for (unsigned i = 0; i < LT.first.getValue(); i++)
2329 Opcodes.push_back(Elt: RISCV::VFREDOSUM_VS);
2330 Opcodes.push_back(Elt: RISCV::VFMV_F_S);
2331 return getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2332 }
2333 SplitOp = RISCV::VFADD_VV;
2334 Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDUSUM_VS, RISCV::VFMV_F_S};
2335 break;
2336 }
2337 // Add a cost for data larger than LMUL8
2338 InstructionCost SplitCost =
2339 (LT.first > 1) ? (LT.first - 1) *
2340 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2341 : 0;
2342 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2343}
2344
2345InstructionCost RISCVTTIImpl::getExtendedReductionCost(
2346 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
2347 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
2348 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2349 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2350 FMF, CostKind);
2351
2352 // Skip if scalar size of ResTy is bigger than ELEN.
2353 if (ResTy->getScalarSizeInBits() > ST->getELen())
2354 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2355 FMF, CostKind);
2356
2357 if (Opcode != Instruction::Add && Opcode != Instruction::FAdd)
2358 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2359 FMF, CostKind);
2360
2361 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2362
2363 if (IsUnsigned && Opcode == Instruction::Add &&
2364 LT.second.isFixedLengthVectorOf(EltVT: MVT::i1)) {
2365 // Represent vector_reduce_add(ZExt(<n x i1>)) as
2366 // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
2367 return LT.first *
2368 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind);
2369 }
2370
2371 if (ResTy->getScalarSizeInBits() != 2 * LT.second.getScalarSizeInBits())
2372 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2373 FMF, CostKind);
2374
2375 return (LT.first - 1) +
2376 getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
2377}
2378
2379InstructionCost
2380RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,
2381 TTI::TargetCostKind CostKind) const {
2382 assert(OpInfo.isConstant() && "non constant operand?");
2383 if (!isa<VectorType>(Val: Ty))
2384 // FIXME: We need to account for immediate materialization here, but doing
2385 // a decent job requires more knowledge about the immediate than we
2386 // currently have here.
2387 return 0;
2388
2389 if (OpInfo.isUniform())
2390 // vmv.v.i, vmv.v.x, or vfmv.v.f
2391 // We ignore the cost of the scalar constant materialization to be consistent
2392 // with how we treat scalar constants themselves just above.
2393 return 1;
2394
2395 return getConstantPoolLoadCost(Ty, CostKind);
2396}
2397
2398InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
2399 Align Alignment,
2400 unsigned AddressSpace,
2401 TTI::TargetCostKind CostKind,
2402 TTI::OperandValueInfo OpInfo,
2403 const Instruction *I) const {
2404 EVT VT = TLI->getValueType(DL, Ty: Src, AllowUnknown: true);
2405 // Type legalization can't handle structs, and load latency isn't handled here
2406 if (VT == MVT::Other ||
2407 (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency))
2408 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2409 CostKind, OpInfo, I);
2410
2411 InstructionCost Cost = 0;
2412 if (Opcode == Instruction::Store && OpInfo.isConstant())
2413 Cost += getStoreImmCost(Ty: Src, OpInfo, CostKind);
2414
2415 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
2416
2417 InstructionCost BaseCost = [&]() {
2418 InstructionCost Cost = LT.first;
2419 if (CostKind != TTI::TCK_RecipThroughput)
2420 return Cost;
2421
2422 // Our actual lowering for the case where a wider legal type is available
2423 // uses the a VL predicated load on the wider type. This is reflected in
2424 // the result of getTypeLegalizationCost, but BasicTTI assumes the
2425 // widened cases are scalarized.
2426 const DataLayout &DL = this->getDataLayout();
2427 if (Src->isVectorTy() && LT.second.isVector() &&
2428 TypeSize::isKnownLT(LHS: DL.getTypeStoreSizeInBits(Ty: Src),
2429 RHS: LT.second.getSizeInBits()))
2430 return Cost;
2431
2432 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2433 CostKind, OpInfo, I);
2434 }();
2435
2436 // Assume memory ops cost scale with the number of vector registers
2437 // possible accessed by the instruction. Note that BasicTTI already
2438 // handles the LT.first term for us.
2439 if (ST->hasVInstructions() && LT.second.isVector() &&
2440 CostKind != TTI::TCK_CodeSize)
2441 BaseCost *= TLI->getLMULCost(VT: LT.second);
2442 return Cost + BaseCost;
2443}
2444
2445InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
2446 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
2447 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
2448 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
2449 if (CostKind != TTI::TCK_RecipThroughput)
2450 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2451 Op1Info, Op2Info, I);
2452
2453 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2454 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2455 Op1Info, Op2Info, I);
2456
2457 // Skip if scalar size of ValTy is bigger than ELEN.
2458 if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen())
2459 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2460 Op1Info, Op2Info, I);
2461
2462 auto GetConstantMatCost =
2463 [&](TTI::OperandValueInfo OpInfo) -> InstructionCost {
2464 if (OpInfo.isUniform())
2465 // We return 0 we currently ignore the cost of materializing scalar
2466 // constants in GPRs.
2467 return 0;
2468
2469 return getConstantPoolLoadCost(Ty: ValTy, CostKind);
2470 };
2471
2472 InstructionCost ConstantMatCost;
2473 if (Op1Info.isConstant())
2474 ConstantMatCost += GetConstantMatCost(Op1Info);
2475 if (Op2Info.isConstant())
2476 ConstantMatCost += GetConstantMatCost(Op2Info);
2477
2478 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2479 if (Opcode == Instruction::Select && LT.second.isVector()) {
2480 if (CondTy->isVectorTy()) {
2481 if (ValTy->getScalarSizeInBits() == 1) {
2482 // vmandn.mm v8, v8, v9
2483 // vmand.mm v9, v0, v9
2484 // vmor.mm v0, v9, v8
2485 return ConstantMatCost +
2486 LT.first *
2487 getRISCVInstructionCost(
2488 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2489 VT: LT.second, CostKind);
2490 }
2491 // vselect and max/min are supported natively.
2492 return ConstantMatCost +
2493 LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second,
2494 CostKind);
2495 }
2496
2497 if (ValTy->getScalarSizeInBits() == 1) {
2498 // vmv.v.x v9, a0
2499 // vmsne.vi v9, v9, 0
2500 // vmandn.mm v8, v8, v9
2501 // vmand.mm v9, v0, v9
2502 // vmor.mm v0, v9, v8
2503 MVT InterimVT = LT.second.changeVectorElementType(EltVT: MVT::i8);
2504 return ConstantMatCost +
2505 LT.first *
2506 getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
2507 VT: InterimVT, CostKind) +
2508 LT.first * getRISCVInstructionCost(
2509 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2510 VT: LT.second, CostKind);
2511 }
2512
2513 // vmv.v.x v10, a0
2514 // vmsne.vi v0, v10, 0
2515 // vmerge.vvm v8, v9, v8, v0
2516 return ConstantMatCost +
2517 LT.first * getRISCVInstructionCost(
2518 OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI, RISCV::VMERGE_VVM},
2519 VT: LT.second, CostKind);
2520 }
2521
2522 if ((Opcode == Instruction::ICmp) && ValTy->isVectorTy() &&
2523 CmpInst::isIntPredicate(P: VecPred)) {
2524 // Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE
2525 // provided they incur the same cost across all implementations
2526 return ConstantMatCost + LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMSLT_VV,
2527 VT: LT.second,
2528 CostKind);
2529 }
2530
2531 if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() &&
2532 CmpInst::isFPPredicate(P: VecPred)) {
2533
2534 // Use VMXOR_MM and VMXNOR_MM to generate all true/false mask
2535 if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE))
2536 return ConstantMatCost +
2537 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind);
2538
2539 // If we do not support the input floating point vector type, use the base
2540 // one which will calculate as:
2541 // ScalarizeCost + Num * Cost for fixed vector,
2542 // InvalidCost for scalable vector.
2543 if ((ValTy->getScalarSizeInBits() == 16 && !ST->hasVInstructionsF16()) ||
2544 (ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) ||
2545 (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64()))
2546 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2547 Op1Info, Op2Info, I);
2548
2549 // Assuming vector fp compare and mask instructions are all the same cost
2550 // until a need arises to differentiate them.
2551 switch (VecPred) {
2552 case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm
2553 case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm
2554 case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm
2555 case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm
2556 return ConstantMatCost +
2557 LT.first * getRISCVInstructionCost(
2558 OpCodes: {RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM},
2559 VT: LT.second, CostKind);
2560
2561 case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m
2562 case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m
2563 case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m
2564 case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m
2565 return ConstantMatCost +
2566 LT.first *
2567 getRISCVInstructionCost(OpCodes: {RISCV::VMFLT_VV, RISCV::VMNAND_MM},
2568 VT: LT.second, CostKind);
2569
2570 case CmpInst::FCMP_OEQ: // vmfeq.vv
2571 case CmpInst::FCMP_OGT: // vmflt.vv
2572 case CmpInst::FCMP_OGE: // vmfle.vv
2573 case CmpInst::FCMP_OLT: // vmflt.vv
2574 case CmpInst::FCMP_OLE: // vmfle.vv
2575 case CmpInst::FCMP_UNE: // vmfne.vv
2576 return ConstantMatCost +
2577 LT.first *
2578 getRISCVInstructionCost(OpCodes: RISCV::VMFLT_VV, VT: LT.second, CostKind);
2579 default:
2580 break;
2581 }
2582 }
2583
2584 // With ShortForwardBranchOpt or ConditionalMoveFusion, scalar icmp + select
2585 // instructions will lower to SELECT_CC and lower to PseudoCCMOVGPR which will
2586 // generate a conditional branch + mv. The cost of scalar (icmp + select) will
2587 // be (0 + select instr cost).
2588 if (ST->hasConditionalMoveFusion() && I && isa<ICmpInst>(Val: I) &&
2589 ValTy->isIntegerTy() && !I->user_empty()) {
2590 if (all_of(Range: I->users(), P: [&](const User *U) {
2591 return match(V: U, P: m_Select(C: m_Specific(V: I), L: m_Value(), R: m_Value())) &&
2592 U->getType()->isIntegerTy() &&
2593 !isa<ConstantData>(Val: U->getOperand(i: 1)) &&
2594 !isa<ConstantData>(Val: U->getOperand(i: 2));
2595 }))
2596 return 0;
2597 }
2598
2599 // TODO: Add cost for scalar type.
2600
2601 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2602 Op1Info, Op2Info, I);
2603}
2604
2605InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
2606 TTI::TargetCostKind CostKind,
2607 const Instruction *I) const {
2608 if (CostKind != TTI::TCK_RecipThroughput)
2609 return Opcode == Instruction::PHI ? 0 : 1;
2610 // Branches are assumed to be predicted.
2611 return 0;
2612}
2613
2614InstructionCost RISCVTTIImpl::getVectorInstrCost(
2615 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
2616 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
2617 assert(Val->isVectorTy() && "This must be a vector type");
2618
2619 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
2620 // For now, skip all fixed vector cost analysis when P extension is available
2621 // to avoid crashes in getMinRVVVectorSizeInBits()
2622 if (ST->hasStdExtP() && isa<FixedVectorType>(Val)) {
2623 return 1; // Treat as single instruction cost for now
2624 }
2625
2626 if (Opcode != Instruction::ExtractElement &&
2627 Opcode != Instruction::InsertElement)
2628 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
2629 VIC);
2630
2631 // Legalize the type.
2632 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Val);
2633
2634 // This type is legalized to a scalar type.
2635 if (!LT.second.isVector()) {
2636 auto *FixedVecTy = cast<FixedVectorType>(Val);
2637 // If Index is a known constant, cost is zero.
2638 if (Index != -1U)
2639 return 0;
2640 // Extract/InsertElement with non-constant index is very costly when
2641 // scalarized; estimate cost of loads/stores sequence via the stack:
2642 // ExtractElement cost: store vector to stack, load scalar;
2643 // InsertElement cost: store vector to stack, store scalar, load vector.
2644 Type *ElemTy = FixedVecTy->getElementType();
2645 auto NumElems = FixedVecTy->getNumElements();
2646 auto Align = DL.getPrefTypeAlign(Ty: ElemTy);
2647 InstructionCost LoadCost =
2648 getMemoryOpCost(Opcode: Instruction::Load, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2649 InstructionCost StoreCost =
2650 getMemoryOpCost(Opcode: Instruction::Store, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2651 return Opcode == Instruction::ExtractElement
2652 ? StoreCost * NumElems + LoadCost
2653 : (StoreCost + LoadCost) * NumElems + StoreCost;
2654 }
2655
2656 // For unsupported scalable vector.
2657 if (LT.second.isScalableVector() && !LT.first.isValid())
2658 return LT.first;
2659
2660 // Mask vector extract/insert is expanded via e8.
2661 if (Val->getScalarSizeInBits() == 1) {
2662 VectorType *WideTy =
2663 VectorType::get(ElementType: IntegerType::get(C&: Val->getContext(), NumBits: 8),
2664 EC: cast<VectorType>(Val)->getElementCount());
2665 if (Opcode == Instruction::ExtractElement) {
2666 InstructionCost ExtendCost
2667 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2668 CCH: TTI::CastContextHint::None, CostKind);
2669 InstructionCost ExtractCost
2670 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2671 return ExtendCost + ExtractCost;
2672 }
2673 InstructionCost ExtendCost
2674 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2675 CCH: TTI::CastContextHint::None, CostKind);
2676 InstructionCost InsertCost
2677 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2678 InstructionCost TruncCost
2679 = getCastInstrCost(Opcode: Instruction::Trunc, Dst: Val, Src: WideTy,
2680 CCH: TTI::CastContextHint::None, CostKind);
2681 return ExtendCost + InsertCost + TruncCost;
2682 }
2683
2684
2685 // In RVV, we could use vslidedown + vmv.x.s to extract element from vector
2686 // and vslideup + vmv.s.x to insert element to vector.
2687 unsigned MoveOpc;
2688 if (LT.second.isFloatingPoint())
2689 MoveOpc = Opcode == Instruction::InsertElement ? RISCV::VFMV_S_F
2690 : RISCV::VFMV_F_S;
2691 else
2692 MoveOpc =
2693 Opcode == Instruction::InsertElement ? RISCV::VMV_S_X : RISCV::VMV_X_S;
2694 InstructionCost BaseCost =
2695 getRISCVInstructionCost(OpCodes: MoveOpc, VT: LT.second, CostKind);
2696 // When insertelement we should add the index with 1 as the input of vslideup.
2697 InstructionCost SlideCost = Opcode == Instruction::InsertElement ? 2 : 1;
2698
2699 if (Index != -1U) {
2700 // The type may be split. For fixed-width vectors we can normalize the
2701 // index to the new type.
2702 if (LT.second.isFixedLengthVector()) {
2703 unsigned Width = LT.second.getVectorNumElements();
2704 Index = Index % Width;
2705 }
2706
2707 // If exact VLEN is known, we will insert/extract into the appropriate
2708 // subvector with no additional subvector insert/extract cost.
2709 if (auto VLEN = ST->getRealVLen()) {
2710 unsigned EltSize = LT.second.getScalarSizeInBits();
2711 unsigned M1Max = *VLEN / EltSize;
2712 Index = Index % M1Max;
2713 }
2714
2715 if (Index == 0)
2716 // We can extract/insert the first element without vslidedown/vslideup.
2717 SlideCost = 0;
2718 else if (Opcode == Instruction::InsertElement)
2719 SlideCost = 1; // With a constant index, we do not need to use addi.
2720 }
2721
2722 // When the vector needs to split into multiple register groups and the index
2723 // exceeds single vector register group, we need to insert/extract the element
2724 // via stack.
2725 if (LT.first > 1 &&
2726 ((Index == -1U) || (Index >= LT.second.getVectorMinNumElements() &&
2727 LT.second.isScalableVector()))) {
2728 Type *ScalarType = Val->getScalarType();
2729 Align VecAlign = DL.getPrefTypeAlign(Ty: Val);
2730 Align SclAlign = DL.getPrefTypeAlign(Ty: ScalarType);
2731 // Extra addi for unknown index.
2732 InstructionCost IdxCost = Index == -1U ? 1 : 0;
2733
2734 // Store all split vectors into stack and load the target element.
2735 if (Opcode == Instruction::ExtractElement)
2736 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2737 getMemoryOpCost(Opcode: Instruction::Load, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
2738 CostKind) +
2739 IdxCost;
2740
2741 // Store all split vectors into stack and store the target element and load
2742 // vectors back.
2743 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2744 getMemoryOpCost(Opcode: Instruction::Load, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2745 getMemoryOpCost(Opcode: Instruction::Store, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
2746 CostKind) +
2747 IdxCost;
2748 }
2749
2750 // Extract i64 in the target that has XLEN=32 need more instruction.
2751 if (Val->getScalarType()->isIntegerTy() &&
2752 ST->getXLen() < Val->getScalarSizeInBits()) {
2753 // For extractelement, we need the following instructions:
2754 // vsetivli zero, 1, e64, m1, ta, mu (not count)
2755 // vslidedown.vx v8, v8, a0
2756 // vmv.x.s a0, v8
2757 // li a1, 32
2758 // vsrl.vx v8, v8, a1
2759 // vmv.x.s a1, v8
2760
2761 // For insertelement, we need the following instructions:
2762 // vsetivli zero, 2, e32, m4, ta, ma (don't count)
2763 // vslide1down.vx v12, v8, a0
2764 // vslide1down.vx v12, v12, a1
2765 // addi a0, a2, 1
2766 // vsetvli zero, a0, e64, m4, tu, ma (don't count)
2767 // vslideup.vx v8, v12, a2
2768
2769 // TODO: should we count these special vsetvlis?
2770 BaseCost =
2771 Opcode == Instruction::InsertElement
2772 ? getRISCVInstructionCost(OpCodes: {RISCV::VSLIDE1DOWN_VX,
2773 RISCV::VSLIDE1DOWN_VX,
2774 RISCV::VSLIDEUP_VX},
2775 VT: LT.second, CostKind)
2776 : getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VMV_X_S,
2777 RISCV::VSRL_VX, RISCV::VMV_X_S},
2778 VT: LT.second, CostKind);
2779 }
2780 return BaseCost + SlideCost;
2781}
2782
2783InstructionCost
2784RISCVTTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
2785 TTI::TargetCostKind CostKind,
2786 unsigned Index) const {
2787 if (isa<FixedVectorType>(Val))
2788 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
2789 Index);
2790
2791 // TODO: This code replicates what LoopVectorize.cpp used to do when asking
2792 // for the cost of extracting the last lane of a scalable vector. It probably
2793 // needs a more accurate cost.
2794 ElementCount EC = cast<VectorType>(Val)->getElementCount();
2795 assert(Index < EC.getKnownMinValue() && "Unexpected reverse index");
2796 return getVectorInstrCost(Opcode, Val, CostKind,
2797 Index: EC.getKnownMinValue() - 1 - Index, Op0: nullptr,
2798 Op1: nullptr);
2799}
2800
2801/// Check to see if this instruction is expected to be combined to a simpler
2802/// operation during/before lowering. If so return the cost of the combined
2803/// operation rather than provided one. For instance, `udiv i16 %X, 2` is likely
2804/// to be combined to `lshr i16 %X, 1`, so return the cost of a `lshr` rather
2805/// than the cost of a `udiv`
2806std::optional<InstructionCost>
2807RISCVTTIImpl::getCombinedArithmeticInstructionCost(
2808 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
2809 TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
2810 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
2811 // Vector unsigned division/remainder will be simplified to shifts/masks.
2812 if ((Opcode == Instruction::UDiv || Opcode == Instruction::URem) &&
2813 Opd2Info.isConstant() && Opd2Info.isPowerOf2()) {
2814 if (Opcode == Instruction::UDiv)
2815 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind, Op1Info: Opd1Info,
2816 Op2Info: Opd2Info.getNoProps());
2817 // UREM
2818 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind, Op1Info: Opd1Info,
2819 Op2Info: Opd2Info.getNoProps());
2820 }
2821 return std::nullopt;
2822}
2823
2824InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
2825 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
2826 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
2827 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
2828
2829 // TODO: Handle more cost kinds.
2830 if (CostKind != TTI::TCK_RecipThroughput)
2831 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2832 Args, CxtI);
2833
2834 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2835 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2836 Args, CxtI);
2837
2838 // Skip if scalar size of Ty is bigger than ELEN.
2839 if (isa<VectorType>(Val: Ty) && Ty->getScalarSizeInBits() > ST->getELen())
2840 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2841 Args, CxtI);
2842
2843 if (std::optional<InstructionCost> CombinedCost =
2844 getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
2845 Opd2Info: Op2Info, Args, CxtI))
2846 return *CombinedCost;
2847
2848 // Legalize the type.
2849 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2850 unsigned ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
2851
2852 // TODO: Handle scalar type.
2853 if (!LT.second.isVector()) {
2854 static const CostTblEntry DivTbl[]{
2855 {.ISD: ISD::UDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2856 {.ISD: ISD::UDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2857 {.ISD: ISD::SDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2858 {.ISD: ISD::SDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2859 {.ISD: ISD::UREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2860 {.ISD: ISD::UREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2861 {.ISD: ISD::SREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2862 {.ISD: ISD::SREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive}};
2863 if (TLI->isOperationLegalOrPromote(Op: ISDOpcode, VT: LT.second))
2864 if (const auto *Entry = CostTableLookup(Table: DivTbl, ISD: ISDOpcode, Ty: LT.second))
2865 return Entry->Cost * LT.first;
2866
2867 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2868 Args, CxtI);
2869 }
2870
2871 // f16 with zvfhmin and bf16 will be promoted to f32.
2872 // FIXME: nxv32[b]f16 will be custom lowered and split.
2873 InstructionCost CastCost = 0;
2874 if ((LT.second.getVectorElementType() == MVT::f16 ||
2875 LT.second.getVectorElementType() == MVT::bf16) &&
2876 TLI->getOperationAction(Op: ISDOpcode, VT: LT.second) ==
2877 TargetLoweringBase::LegalizeAction::Promote) {
2878 MVT PromotedVT = TLI->getTypeToPromoteTo(Op: ISDOpcode, VT: LT.second);
2879 Type *PromotedTy = EVT(PromotedVT).getTypeForEVT(Context&: Ty->getContext());
2880 Type *LegalTy = EVT(LT.second).getTypeForEVT(Context&: Ty->getContext());
2881 // Add cost of extending arguments
2882 CastCost += LT.first * Args.size() *
2883 getCastInstrCost(Opcode: Instruction::FPExt, Dst: PromotedTy, Src: LegalTy,
2884 CCH: TTI::CastContextHint::None, CostKind);
2885 // Add cost of truncating result
2886 CastCost +=
2887 LT.first * getCastInstrCost(Opcode: Instruction::FPTrunc, Dst: LegalTy, Src: PromotedTy,
2888 CCH: TTI::CastContextHint::None, CostKind);
2889 // Compute cost of op in promoted type
2890 LT.second = PromotedVT;
2891 }
2892
2893 auto getConstantMatCost =
2894 [&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost {
2895 if (OpInfo.isUniform() && canSplatOperand(Opcode, Operand))
2896 // Two sub-cases:
2897 // * Has a 5 bit immediate operand which can be splatted.
2898 // * Has a larger immediate which must be materialized in scalar register
2899 // We return 0 for both as we currently ignore the cost of materializing
2900 // scalar constants in GPRs.
2901 return 0;
2902
2903 return getConstantPoolLoadCost(Ty, CostKind);
2904 };
2905
2906 // Add the cost of materializing any constant vectors required.
2907 InstructionCost ConstantMatCost = 0;
2908 if (Op1Info.isConstant())
2909 ConstantMatCost += getConstantMatCost(0, Op1Info);
2910 if (Op2Info.isConstant())
2911 ConstantMatCost += getConstantMatCost(1, Op2Info);
2912
2913 unsigned Op;
2914 switch (ISDOpcode) {
2915 case ISD::ADD:
2916 case ISD::SUB:
2917 Op = RISCV::VADD_VV;
2918 break;
2919 case ISD::SHL:
2920 case ISD::SRL:
2921 case ISD::SRA:
2922 Op = RISCV::VSLL_VV;
2923 break;
2924 case ISD::AND:
2925 case ISD::OR:
2926 case ISD::XOR:
2927 Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
2928 break;
2929 case ISD::MUL:
2930 case ISD::MULHS:
2931 case ISD::MULHU:
2932 Op = RISCV::VMUL_VV;
2933 break;
2934 case ISD::SDIV:
2935 case ISD::UDIV:
2936 Op = RISCV::VDIV_VV;
2937 break;
2938 case ISD::SREM:
2939 case ISD::UREM:
2940 Op = RISCV::VREM_VV;
2941 break;
2942 case ISD::FADD:
2943 case ISD::FSUB:
2944 Op = RISCV::VFADD_VV;
2945 break;
2946 case ISD::FMUL:
2947 Op = RISCV::VFMUL_VV;
2948 break;
2949 case ISD::FDIV:
2950 Op = RISCV::VFDIV_VV;
2951 break;
2952 case ISD::FNEG:
2953 Op = RISCV::VFSGNJN_VV;
2954 break;
2955 default:
2956 // Assuming all other instructions have the same cost until a need arises to
2957 // differentiate them.
2958 return CastCost + ConstantMatCost +
2959 BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2960 Args, CxtI);
2961 }
2962
2963 InstructionCost InstrCost = getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
2964 // We use BasicTTIImpl to calculate scalar costs, which assumes floating point
2965 // ops are twice as expensive as integer ops. Do the same for vectors so
2966 // scalar floating point ops aren't cheaper than their vector equivalents.
2967 if (Ty->isFPOrFPVectorTy())
2968 InstrCost *= 2;
2969 return CastCost + ConstantMatCost + LT.first * InstrCost;
2970}
2971
2972// TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
2973InstructionCost RISCVTTIImpl::getPointersChainCost(
2974 ArrayRef<const Value *> Ptrs, const Value *Base,
2975 const TTI::PointersChainInfo &Info, Type *AccessTy,
2976 const TTI::TargetCostKind CostKind) const {
2977 InstructionCost Cost = TTI::TCC_Free;
2978 // In the basic model we take into account GEP instructions only
2979 // (although here can come alloca instruction, a value, constants and/or
2980 // constant expressions, PHIs, bitcasts ... whatever allowed to be used as a
2981 // pointer). Typically, if Base is a not a GEP-instruction and all the
2982 // pointers are relative to the same base address, all the rest are
2983 // either GEP instructions, PHIs, bitcasts or constants. When we have same
2984 // base, we just calculate cost of each non-Base GEP as an ADD operation if
2985 // any their index is a non-const.
2986 // If no known dependencies between the pointers cost is calculated as a sum
2987 // of costs of GEP instructions.
2988 for (auto [I, V] : enumerate(First&: Ptrs)) {
2989 const auto *GEP = dyn_cast<GetElementPtrInst>(Val: V);
2990 if (!GEP)
2991 continue;
2992 if (Info.isSameBase() && V != Base) {
2993 if (GEP->hasAllConstantIndices())
2994 continue;
2995 // If the chain is unit-stride and BaseReg + stride*i is a legal
2996 // addressing mode, then presume the base GEP is sitting around in a
2997 // register somewhere and check if we can fold the offset relative to
2998 // it.
2999 unsigned Stride = DL.getTypeStoreSize(Ty: AccessTy);
3000 if (Info.isUnitStride() &&
3001 isLegalAddressingMode(Ty: AccessTy,
3002 /* BaseGV */ nullptr,
3003 /* BaseOffset */ Stride * I,
3004 /* HasBaseReg */ true,
3005 /* Scale */ 0,
3006 AddrSpace: GEP->getType()->getPointerAddressSpace()))
3007 continue;
3008 Cost += getArithmeticInstrCost(Opcode: Instruction::Add, Ty: GEP->getType(), CostKind,
3009 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
3010 Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Args: {});
3011 } else {
3012 SmallVector<const Value *> Indices(GEP->indices());
3013 Cost += getGEPCost(PointeeType: GEP->getSourceElementType(), Ptr: GEP->getPointerOperand(),
3014 Operands: Indices, CostKind, AccessType: AccessTy);
3015 }
3016 }
3017 return Cost;
3018}
3019
3020void RISCVTTIImpl::getUnrollingPreferences(
3021 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
3022 OptimizationRemarkEmitter *ORE) const {
3023 // TODO: More tuning on benchmarks and metrics with changes as needed
3024 // would apply to all settings below to enable performance.
3025
3026
3027 if (ST->enableDefaultUnroll())
3028 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
3029
3030 // Enable Upper bound unrolling universally, not dependent upon the conditions
3031 // below.
3032 UP.UpperBound = true;
3033
3034 // Disable loop unrolling for Oz and Os.
3035 UP.OptSizeThreshold = 0;
3036 UP.PartialOptSizeThreshold = 0;
3037 if (L->getHeader()->getParent()->hasOptSize())
3038 return;
3039
3040 SmallVector<BasicBlock *, 4> ExitingBlocks;
3041 L->getExitingBlocks(ExitingBlocks);
3042 LLVM_DEBUG(dbgs() << "Loop has:\n"
3043 << "Blocks: " << L->getNumBlocks() << "\n"
3044 << "Exit blocks: " << ExitingBlocks.size() << "\n");
3045
3046 // Only allow another exit other than the latch. This acts as an early exit
3047 // as it mirrors the profitability calculation of the runtime unroller.
3048 if (ExitingBlocks.size() > 2)
3049 return;
3050
3051 // Limit the CFG of the loop body for targets with a branch predictor.
3052 // Allowing 4 blocks permits if-then-else diamonds in the body.
3053 if (L->getNumBlocks() > 4)
3054 return;
3055
3056 // Scan the loop: don't unroll loops with calls as this could prevent
3057 // inlining. Don't unroll auto-vectorized loops either, though do allow
3058 // unrolling of the scalar remainder.
3059 bool IsVectorized = getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized");
3060 InstructionCost Cost = 0;
3061 for (auto *BB : L->getBlocks()) {
3062 for (auto &I : *BB) {
3063 // Both auto-vectorized loops and the scalar remainder have the
3064 // isvectorized attribute, so differentiate between them by the presence
3065 // of vector instructions.
3066 if (IsVectorized && (I.getType()->isVectorTy() ||
3067 llvm::any_of(Range: I.operand_values(), P: [](Value *V) {
3068 return V->getType()->isVectorTy();
3069 })))
3070 return;
3071
3072 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
3073 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction()) {
3074 if (!isLoweredToCall(F))
3075 continue;
3076 }
3077 return;
3078 }
3079
3080 SmallVector<const Value *> Operands(I.operand_values());
3081 Cost += getInstructionCost(U: &I, Operands,
3082 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
3083 }
3084 }
3085
3086 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
3087
3088 UP.Partial = true;
3089 UP.Runtime = true;
3090 UP.UnrollRemainder = true;
3091 UP.UnrollAndJam = true;
3092
3093 // Force unrolling small loops can be very useful because of the branch
3094 // taken cost of the backedge.
3095 if (Cost < 12)
3096 UP.Force = true;
3097}
3098
3099void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
3100 TTI::PeelingPreferences &PP) const {
3101 BaseT::getPeelingPreferences(L, SE, PP);
3102}
3103
3104bool RISCVTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
3105 MemIntrinsicInfo &Info) const {
3106 const DataLayout &DL = getDataLayout();
3107 Intrinsic::ID IID = Inst->getIntrinsicID();
3108 LLVMContext &C = Inst->getContext();
3109 bool HasMask = false;
3110
3111 auto getSegNum = [](const IntrinsicInst *II, unsigned PtrOperandNo,
3112 bool IsWrite) -> int64_t {
3113 if (auto *TarExtTy =
3114 dyn_cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType()))
3115 return TarExtTy->getIntParameter(i: 0);
3116
3117 return 1;
3118 };
3119
3120 switch (IID) {
3121 case Intrinsic::riscv_vle_mask:
3122 case Intrinsic::riscv_vse_mask:
3123 case Intrinsic::riscv_vlseg2_mask:
3124 case Intrinsic::riscv_vlseg3_mask:
3125 case Intrinsic::riscv_vlseg4_mask:
3126 case Intrinsic::riscv_vlseg5_mask:
3127 case Intrinsic::riscv_vlseg6_mask:
3128 case Intrinsic::riscv_vlseg7_mask:
3129 case Intrinsic::riscv_vlseg8_mask:
3130 case Intrinsic::riscv_vsseg2_mask:
3131 case Intrinsic::riscv_vsseg3_mask:
3132 case Intrinsic::riscv_vsseg4_mask:
3133 case Intrinsic::riscv_vsseg5_mask:
3134 case Intrinsic::riscv_vsseg6_mask:
3135 case Intrinsic::riscv_vsseg7_mask:
3136 case Intrinsic::riscv_vsseg8_mask:
3137 HasMask = true;
3138 [[fallthrough]];
3139 case Intrinsic::riscv_vle:
3140 case Intrinsic::riscv_vse:
3141 case Intrinsic::riscv_vlseg2:
3142 case Intrinsic::riscv_vlseg3:
3143 case Intrinsic::riscv_vlseg4:
3144 case Intrinsic::riscv_vlseg5:
3145 case Intrinsic::riscv_vlseg6:
3146 case Intrinsic::riscv_vlseg7:
3147 case Intrinsic::riscv_vlseg8:
3148 case Intrinsic::riscv_vsseg2:
3149 case Intrinsic::riscv_vsseg3:
3150 case Intrinsic::riscv_vsseg4:
3151 case Intrinsic::riscv_vsseg5:
3152 case Intrinsic::riscv_vsseg6:
3153 case Intrinsic::riscv_vsseg7:
3154 case Intrinsic::riscv_vsseg8: {
3155 // Intrinsic interface:
3156 // riscv_vle(merge, ptr, vl)
3157 // riscv_vle_mask(merge, ptr, mask, vl, policy)
3158 // riscv_vse(val, ptr, vl)
3159 // riscv_vse_mask(val, ptr, mask, vl, policy)
3160 // riscv_vlseg#(merge, ptr, vl, sew)
3161 // riscv_vlseg#_mask(merge, ptr, mask, vl, policy, sew)
3162 // riscv_vsseg#(val, ptr, vl, sew)
3163 // riscv_vsseg#_mask(val, ptr, mask, vl, sew)
3164 bool IsWrite = Inst->getType()->isVoidTy();
3165 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3166 // The results of segment loads are TargetExtType.
3167 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3168 unsigned SEW =
3169 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3170 ->getZExtValue();
3171 Ty = TarExtTy->getTypeParameter(i: 0U);
3172 Ty = ScalableVectorType::get(
3173 ElementType: IntegerType::get(C, NumBits: SEW),
3174 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3175 }
3176 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3177 unsigned VLIndex = RVVIInfo->VLOperand;
3178 unsigned PtrOperandNo = VLIndex - 1 - HasMask;
3179 MaybeAlign Alignment =
3180 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3181 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3182 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3183 if (HasMask)
3184 Mask = Inst->getArgOperand(i: VLIndex - 1);
3185 Value *EVL = Inst->getArgOperand(i: VLIndex);
3186 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3187 // RVV uses contiguous elements as a segment.
3188 if (SegNum > 1) {
3189 unsigned ElemSize = Ty->getScalarSizeInBits();
3190 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3191 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3192 }
3193 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3194 Args&: Alignment, Args&: Mask, Args&: EVL);
3195 return true;
3196 }
3197 case Intrinsic::riscv_vlse_mask:
3198 case Intrinsic::riscv_vsse_mask:
3199 case Intrinsic::riscv_vlsseg2_mask:
3200 case Intrinsic::riscv_vlsseg3_mask:
3201 case Intrinsic::riscv_vlsseg4_mask:
3202 case Intrinsic::riscv_vlsseg5_mask:
3203 case Intrinsic::riscv_vlsseg6_mask:
3204 case Intrinsic::riscv_vlsseg7_mask:
3205 case Intrinsic::riscv_vlsseg8_mask:
3206 case Intrinsic::riscv_vssseg2_mask:
3207 case Intrinsic::riscv_vssseg3_mask:
3208 case Intrinsic::riscv_vssseg4_mask:
3209 case Intrinsic::riscv_vssseg5_mask:
3210 case Intrinsic::riscv_vssseg6_mask:
3211 case Intrinsic::riscv_vssseg7_mask:
3212 case Intrinsic::riscv_vssseg8_mask:
3213 HasMask = true;
3214 [[fallthrough]];
3215 case Intrinsic::riscv_vlse:
3216 case Intrinsic::riscv_vsse:
3217 case Intrinsic::riscv_vlsseg2:
3218 case Intrinsic::riscv_vlsseg3:
3219 case Intrinsic::riscv_vlsseg4:
3220 case Intrinsic::riscv_vlsseg5:
3221 case Intrinsic::riscv_vlsseg6:
3222 case Intrinsic::riscv_vlsseg7:
3223 case Intrinsic::riscv_vlsseg8:
3224 case Intrinsic::riscv_vssseg2:
3225 case Intrinsic::riscv_vssseg3:
3226 case Intrinsic::riscv_vssseg4:
3227 case Intrinsic::riscv_vssseg5:
3228 case Intrinsic::riscv_vssseg6:
3229 case Intrinsic::riscv_vssseg7:
3230 case Intrinsic::riscv_vssseg8: {
3231 // Intrinsic interface:
3232 // riscv_vlse(merge, ptr, stride, vl)
3233 // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
3234 // riscv_vsse(val, ptr, stride, vl)
3235 // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
3236 // riscv_vlsseg#(merge, ptr, offset, vl, sew)
3237 // riscv_vlsseg#_mask(merge, ptr, offset, mask, vl, policy, sew)
3238 // riscv_vssseg#(val, ptr, offset, vl, sew)
3239 // riscv_vssseg#_mask(val, ptr, offset, mask, vl, sew)
3240 bool IsWrite = Inst->getType()->isVoidTy();
3241 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3242 // The results of segment loads are TargetExtType.
3243 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3244 unsigned SEW =
3245 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3246 ->getZExtValue();
3247 Ty = TarExtTy->getTypeParameter(i: 0U);
3248 Ty = ScalableVectorType::get(
3249 ElementType: IntegerType::get(C, NumBits: SEW),
3250 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3251 }
3252 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3253 unsigned VLIndex = RVVIInfo->VLOperand;
3254 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3255 MaybeAlign Alignment =
3256 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3257
3258 Value *Stride = Inst->getArgOperand(i: PtrOperandNo + 1);
3259 // Use the pointer alignment as the element alignment if the stride is a
3260 // multiple of the pointer alignment. Otherwise, the element alignment
3261 // should be the greatest common divisor of pointer alignment and stride.
3262 // For simplicity, just consider unalignment for elements.
3263 unsigned PointerAlign = Alignment.valueOrOne().value();
3264 if (!isa<ConstantInt>(Val: Stride) ||
3265 cast<ConstantInt>(Val: Stride)->getZExtValue() % PointerAlign != 0)
3266 Alignment = Align(1);
3267
3268 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3269 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3270 if (HasMask)
3271 Mask = Inst->getArgOperand(i: VLIndex - 1);
3272 Value *EVL = Inst->getArgOperand(i: VLIndex);
3273 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3274 // RVV uses contiguous elements as a segment.
3275 if (SegNum > 1) {
3276 unsigned ElemSize = Ty->getScalarSizeInBits();
3277 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3278 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3279 }
3280 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3281 Args&: Alignment, Args&: Mask, Args&: EVL, Args&: Stride);
3282 return true;
3283 }
3284 case Intrinsic::riscv_vloxei_mask:
3285 case Intrinsic::riscv_vluxei_mask:
3286 case Intrinsic::riscv_vsoxei_mask:
3287 case Intrinsic::riscv_vsuxei_mask:
3288 case Intrinsic::riscv_vloxseg2_mask:
3289 case Intrinsic::riscv_vloxseg3_mask:
3290 case Intrinsic::riscv_vloxseg4_mask:
3291 case Intrinsic::riscv_vloxseg5_mask:
3292 case Intrinsic::riscv_vloxseg6_mask:
3293 case Intrinsic::riscv_vloxseg7_mask:
3294 case Intrinsic::riscv_vloxseg8_mask:
3295 case Intrinsic::riscv_vluxseg2_mask:
3296 case Intrinsic::riscv_vluxseg3_mask:
3297 case Intrinsic::riscv_vluxseg4_mask:
3298 case Intrinsic::riscv_vluxseg5_mask:
3299 case Intrinsic::riscv_vluxseg6_mask:
3300 case Intrinsic::riscv_vluxseg7_mask:
3301 case Intrinsic::riscv_vluxseg8_mask:
3302 case Intrinsic::riscv_vsoxseg2_mask:
3303 case Intrinsic::riscv_vsoxseg3_mask:
3304 case Intrinsic::riscv_vsoxseg4_mask:
3305 case Intrinsic::riscv_vsoxseg5_mask:
3306 case Intrinsic::riscv_vsoxseg6_mask:
3307 case Intrinsic::riscv_vsoxseg7_mask:
3308 case Intrinsic::riscv_vsoxseg8_mask:
3309 case Intrinsic::riscv_vsuxseg2_mask:
3310 case Intrinsic::riscv_vsuxseg3_mask:
3311 case Intrinsic::riscv_vsuxseg4_mask:
3312 case Intrinsic::riscv_vsuxseg5_mask:
3313 case Intrinsic::riscv_vsuxseg6_mask:
3314 case Intrinsic::riscv_vsuxseg7_mask:
3315 case Intrinsic::riscv_vsuxseg8_mask:
3316 HasMask = true;
3317 [[fallthrough]];
3318 case Intrinsic::riscv_vloxei:
3319 case Intrinsic::riscv_vluxei:
3320 case Intrinsic::riscv_vsoxei:
3321 case Intrinsic::riscv_vsuxei:
3322 case Intrinsic::riscv_vloxseg2:
3323 case Intrinsic::riscv_vloxseg3:
3324 case Intrinsic::riscv_vloxseg4:
3325 case Intrinsic::riscv_vloxseg5:
3326 case Intrinsic::riscv_vloxseg6:
3327 case Intrinsic::riscv_vloxseg7:
3328 case Intrinsic::riscv_vloxseg8:
3329 case Intrinsic::riscv_vluxseg2:
3330 case Intrinsic::riscv_vluxseg3:
3331 case Intrinsic::riscv_vluxseg4:
3332 case Intrinsic::riscv_vluxseg5:
3333 case Intrinsic::riscv_vluxseg6:
3334 case Intrinsic::riscv_vluxseg7:
3335 case Intrinsic::riscv_vluxseg8:
3336 case Intrinsic::riscv_vsoxseg2:
3337 case Intrinsic::riscv_vsoxseg3:
3338 case Intrinsic::riscv_vsoxseg4:
3339 case Intrinsic::riscv_vsoxseg5:
3340 case Intrinsic::riscv_vsoxseg6:
3341 case Intrinsic::riscv_vsoxseg7:
3342 case Intrinsic::riscv_vsoxseg8:
3343 case Intrinsic::riscv_vsuxseg2:
3344 case Intrinsic::riscv_vsuxseg3:
3345 case Intrinsic::riscv_vsuxseg4:
3346 case Intrinsic::riscv_vsuxseg5:
3347 case Intrinsic::riscv_vsuxseg6:
3348 case Intrinsic::riscv_vsuxseg7:
3349 case Intrinsic::riscv_vsuxseg8: {
3350 // Intrinsic interface (only listed ordered version):
3351 // riscv_vloxei(merge, ptr, index, vl)
3352 // riscv_vloxei_mask(merge, ptr, index, mask, vl, policy)
3353 // riscv_vsoxei(val, ptr, index, vl)
3354 // riscv_vsoxei_mask(val, ptr, index, mask, vl, policy)
3355 // riscv_vloxseg#(merge, ptr, index, vl, sew)
3356 // riscv_vloxseg#_mask(merge, ptr, index, mask, vl, policy, sew)
3357 // riscv_vsoxseg#(val, ptr, index, vl, sew)
3358 // riscv_vsoxseg#_mask(val, ptr, index, mask, vl, sew)
3359 bool IsWrite = Inst->getType()->isVoidTy();
3360 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3361 // The results of segment loads are TargetExtType.
3362 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3363 unsigned SEW =
3364 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3365 ->getZExtValue();
3366 Ty = TarExtTy->getTypeParameter(i: 0U);
3367 Ty = ScalableVectorType::get(
3368 ElementType: IntegerType::get(C, NumBits: SEW),
3369 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3370 }
3371 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3372 unsigned VLIndex = RVVIInfo->VLOperand;
3373 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3374 Value *Mask;
3375 if (HasMask) {
3376 Mask = Inst->getArgOperand(i: VLIndex - 1);
3377 } else {
3378 // Mask cannot be nullptr here: vector GEP produces <vscale x N x ptr>,
3379 // and casting that to scalar i64 triggers a vector/scalar mismatch
3380 // assertion in CreatePointerCast. Use an all-true mask so ASan lowers it
3381 // via extractelement instead.
3382 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3383 Mask = ConstantInt::getTrue(Ty: MaskType);
3384 }
3385 Value *EVL = Inst->getArgOperand(i: VLIndex);
3386 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3387 // RVV uses contiguous elements as a segment.
3388 if (SegNum > 1) {
3389 unsigned ElemSize = Ty->getScalarSizeInBits();
3390 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3391 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3392 }
3393 Value *OffsetOp = Inst->getArgOperand(i: PtrOperandNo + 1);
3394 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3395 Args: Align(1), Args&: Mask, Args&: EVL,
3396 /* Stride */ Args: nullptr, Args&: OffsetOp);
3397 return true;
3398 }
3399 }
3400 return false;
3401}
3402
3403unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) const {
3404 if (Ty->isVectorTy()) {
3405 // f16 with only zvfhmin and bf16 will be promoted to f32
3406 Type *EltTy = cast<VectorType>(Val: Ty)->getElementType();
3407 if ((EltTy->isHalfTy() && !ST->hasVInstructionsF16()) ||
3408 EltTy->isBFloatTy())
3409 Ty = VectorType::get(ElementType: Type::getFloatTy(C&: Ty->getContext()),
3410 Other: cast<VectorType>(Val: Ty));
3411
3412 TypeSize Size = DL.getTypeSizeInBits(Ty);
3413 if (Size.isScalable() && ST->hasVInstructions())
3414 return divideCeil(Numerator: Size.getKnownMinValue(), Denominator: RISCV::RVVBitsPerBlock);
3415
3416 if (ST->useRVVForFixedLengthVectors())
3417 return divideCeil(Numerator: Size, Denominator: ST->getRealMinVLen());
3418 }
3419
3420 return BaseT::getRegUsageForType(Ty);
3421}
3422
3423unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
3424 if (SLPMaxVF.getNumOccurrences())
3425 return SLPMaxVF;
3426
3427 // Return how many elements can fit in getRegisterBitwidth. This is the
3428 // same routine as used in LoopVectorizer. We should probably be
3429 // accounting for whether we actually have instructions with the right
3430 // lane type, but we don't have enough information to do that without
3431 // some additional plumbing which hasn't been justified yet.
3432 TypeSize RegWidth =
3433 getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
3434 // If no vector registers, or absurd element widths, disable
3435 // vectorization by returning 1.
3436 return std::max<unsigned>(a: 1U, b: RegWidth.getFixedValue() / ElemWidth);
3437}
3438
3439unsigned RISCVTTIImpl::getMinTripCountTailFoldingThreshold() const {
3440 return RVVMinTripCount;
3441}
3442
3443bool RISCVTTIImpl::preferAlternateOpcodeVectorization() const {
3444 return ST->enableUnalignedVectorMem();
3445}
3446
3447TTI::AddressingModeKind
3448RISCVTTIImpl::getPreferredAddressingMode(const Loop *L,
3449 ScalarEvolution *SE) const {
3450 if (ST->hasVendorXCVmem() && !ST->is64Bit())
3451 return TTI::AMK_PostIndexed;
3452
3453 return BasicTTIImplBase::getPreferredAddressingMode(L, SE);
3454}
3455
3456bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
3457 const TargetTransformInfo::LSRCost &C2) const {
3458 // RISC-V specific here are "instruction number 1st priority".
3459 // If we need to emit adds inside the loop to add up base registers, then
3460 // we need at least one extra temporary register.
3461 unsigned C1NumRegs = C1.NumRegs + (C1.NumBaseAdds != 0);
3462 unsigned C2NumRegs = C2.NumRegs + (C2.NumBaseAdds != 0);
3463 return std::tie(args: C1.Insns, args&: C1NumRegs, args: C1.AddRecCost,
3464 args: C1.NumIVMuls, args: C1.NumBaseAdds,
3465 args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
3466 std::tie(args: C2.Insns, args&: C2NumRegs, args: C2.AddRecCost,
3467 args: C2.NumIVMuls, args: C2.NumBaseAdds,
3468 args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
3469}
3470
3471bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
3472 Align Alignment) const {
3473 auto *VTy = dyn_cast<VectorType>(Val: DataTy);
3474 if (!VTy || VTy->isScalableTy())
3475 return false;
3476
3477 if (!isLegalMaskedLoadStore(DataType: DataTy, Alignment))
3478 return false;
3479
3480 // FIXME: If it is an i8 vector and the element count exceeds 256, we should
3481 // scalarize these types with LMUL >= maximum fixed-length LMUL.
3482 if (VTy->getElementType()->isIntegerTy(BitWidth: 8))
3483 if (VTy->getElementCount().getFixedValue() > 256)
3484 return VTy->getPrimitiveSizeInBits() / ST->getRealMinVLen() <
3485 ST->getMaxLMULForFixedLengthVectors();
3486 return true;
3487}
3488
3489bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,
3490 Align Alignment) const {
3491 auto *VTy = dyn_cast<VectorType>(Val: DataTy);
3492 if (!VTy || VTy->isScalableTy())
3493 return false;
3494
3495 if (!isLegalMaskedLoadStore(DataType: DataTy, Alignment))
3496 return false;
3497 return true;
3498}
3499
3500bool RISCVTTIImpl::isLegalBroadcastLoad(Type *ElementTy,
3501 ElementCount NumElements) const {
3502 // Optimized zero-stride loads can be treated as broadcasts.
3503 if (!ST->hasVInstructions() || !ST->hasOptimizedZeroStrideLoad())
3504 return false;
3505
3506 return TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: ElementTy));
3507}
3508
3509/// See if \p I should be considered for address type promotion. We check if \p
3510/// I is a sext with right type and used in memory accesses. If it used in a
3511/// "complex" getelementptr, we allow it to be promoted without finding other
3512/// sext instructions that sign extended the same initial value. A getelementptr
3513/// is considered as "complex" if it has more than 2 operands.
3514bool RISCVTTIImpl::shouldConsiderAddressTypePromotion(
3515 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
3516 bool Considerable = false;
3517 AllowPromotionWithoutCommonHeader = false;
3518 if (!isa<SExtInst>(Val: &I))
3519 return false;
3520 Type *ConsideredSExtType =
3521 Type::getInt64Ty(C&: I.getParent()->getParent()->getContext());
3522 if (I.getType() != ConsideredSExtType)
3523 return false;
3524 // See if the sext is the one with the right type and used in at least one
3525 // GetElementPtrInst.
3526 for (const User *U : I.users()) {
3527 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(Val: U)) {
3528 Considerable = true;
3529 // A getelementptr is considered as "complex" if it has more than 2
3530 // operands. We will promote a SExt used in such complex GEP as we
3531 // expect some computation to be merged if they are done on 64 bits.
3532 if (GEPInst->getNumOperands() > 2) {
3533 AllowPromotionWithoutCommonHeader = true;
3534 break;
3535 }
3536 }
3537 }
3538 return Considerable;
3539}
3540
3541bool RISCVTTIImpl::canSplatOperand(unsigned Opcode, int Operand) const {
3542 switch (Opcode) {
3543 case Instruction::Add:
3544 case Instruction::Sub:
3545 case Instruction::Mul:
3546 case Instruction::And:
3547 case Instruction::Or:
3548 case Instruction::Xor:
3549 case Instruction::FAdd:
3550 case Instruction::FSub:
3551 case Instruction::FMul:
3552 case Instruction::FDiv:
3553 case Instruction::ICmp:
3554 case Instruction::FCmp:
3555 return true;
3556 case Instruction::Shl:
3557 case Instruction::LShr:
3558 case Instruction::AShr:
3559 case Instruction::UDiv:
3560 case Instruction::SDiv:
3561 case Instruction::URem:
3562 case Instruction::SRem:
3563 case Instruction::Select:
3564 return Operand == 1;
3565 default:
3566 return false;
3567 }
3568}
3569
3570bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {
3571 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3572 return false;
3573
3574 if (canSplatOperand(Opcode: I->getOpcode(), Operand))
3575 return true;
3576
3577 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3578 if (!II)
3579 return false;
3580
3581 switch (II->getIntrinsicID()) {
3582 case Intrinsic::fma:
3583 case Intrinsic::fmuladd:
3584 return Operand == 0 || Operand == 1;
3585 case Intrinsic::vp_udiv:
3586 case Intrinsic::vp_sdiv:
3587 case Intrinsic::vp_urem:
3588 case Intrinsic::vp_srem:
3589 case Intrinsic::ssub_sat:
3590 case Intrinsic::usub_sat:
3591 return Operand == 1;
3592 // These intrinsics are commutative.
3593 case Intrinsic::smin:
3594 case Intrinsic::umin:
3595 case Intrinsic::smax:
3596 case Intrinsic::umax:
3597 case Intrinsic::sadd_sat:
3598 case Intrinsic::uadd_sat:
3599 return Operand == 0 || Operand == 1;
3600 default:
3601 return false;
3602 }
3603}
3604
3605/// Check if sinking \p I's operands to I's basic block is profitable, because
3606/// the operands can be folded into a target instruction, e.g.
3607/// splats of scalars can fold into vector instructions.
3608bool RISCVTTIImpl::isProfitableToSinkOperands(
3609 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
3610 using namespace llvm::PatternMatch;
3611
3612 if (I->isBitwiseLogicOp()) {
3613 if (!I->getType()->isVectorTy()) {
3614 if (ST->hasStdExtZbb() || ST->hasStdExtZbkb()) {
3615 for (auto &Op : I->operands()) {
3616 // (and/or/xor X, (not Y)) -> (andn/orn/xnor X, Y)
3617 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3618 Ops.push_back(Elt: &Op);
3619 return true;
3620 }
3621 }
3622 }
3623 } else if (I->getOpcode() == Instruction::And && ST->hasStdExtZvkb()) {
3624 for (auto &Op : I->operands()) {
3625 // (and X, (not Y)) -> (vandn.vv X, Y)
3626 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3627 Ops.push_back(Elt: &Op);
3628 return true;
3629 }
3630 // (and X, (splat (not Y))) -> (vandn.vx X, Y)
3631 if (match(V: Op.get(), P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()),
3632 Idx: m_ZeroInt()),
3633 v2: m_Value(), mask: m_ZeroMask()))) {
3634 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
3635 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
3636 Ops.push_back(Elt: &Not);
3637 Ops.push_back(Elt: &InsertElt);
3638 Ops.push_back(Elt: &Op);
3639 return true;
3640 }
3641 }
3642 }
3643 }
3644
3645 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3646 return false;
3647
3648 // Don't sink splat operands if the target prefers it. Some targets requires
3649 // S2V transfer buffers and we can run out of them copying the same value
3650 // repeatedly.
3651 // FIXME: It could still be worth doing if it would improve vector register
3652 // pressure and prevent a vector spill.
3653 if (!ST->sinkSplatOperands())
3654 return false;
3655
3656 for (auto OpIdx : enumerate(First: I->operands())) {
3657 if (!canSplatOperand(I, Operand: OpIdx.index()))
3658 continue;
3659
3660 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
3661 // Make sure we are not already sinking this operand
3662 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
3663 continue;
3664
3665 // We are looking for a splat that can be sunk.
3666 if (!match(V: Op, P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ZeroInt()),
3667 v2: m_Value(), mask: m_ZeroMask())))
3668 continue;
3669
3670 // Don't sink i1 splats.
3671 if (cast<VectorType>(Val: Op->getType())->getElementType()->isIntegerTy(BitWidth: 1))
3672 continue;
3673
3674 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
3675 // and vector registers
3676 for (Use &U : Op->uses()) {
3677 Instruction *Insn = cast<Instruction>(Val: U.getUser());
3678 if (!canSplatOperand(I: Insn, Operand: U.getOperandNo()))
3679 return false;
3680 }
3681
3682 // Sink any fpexts since they might be used in a widening fp pattern.
3683 Use *InsertEltUse = &Op->getOperandUse(i: 0);
3684 auto *InsertElt = cast<InsertElementInst>(Val: InsertEltUse);
3685 if (isa<FPExtInst>(Val: InsertElt->getOperand(i_nocapture: 1)))
3686 Ops.push_back(Elt: &InsertElt->getOperandUse(i: 1));
3687 Ops.push_back(Elt: InsertEltUse);
3688 Ops.push_back(Elt: &OpIdx.value());
3689 }
3690 return true;
3691}
3692
3693RISCVTTIImpl::TTI::MemCmpExpansionOptions
3694RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
3695 TTI::MemCmpExpansionOptions Options;
3696
3697 if (!ST->hasStdExtZbb() && !ST->hasStdExtZbkb() && !IsZeroCmp)
3698 return Options;
3699
3700 Options.AllowOverlappingLoads = true;
3701 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
3702 Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
3703 if (ST->is64Bit()) {
3704 Options.LoadSizes = {8, 4, 2, 1};
3705 Options.AllowedTailExpansions = {3, 5, 6};
3706 } else {
3707 Options.LoadSizes = {4, 2, 1};
3708 Options.AllowedTailExpansions = {3};
3709 }
3710
3711 if (IsZeroCmp && ST->hasVInstructions()) {
3712 unsigned VLenB = ST->getRealMinVLen() / 8;
3713 // The minimum size should be `XLen / 8 + 1`, and the maxinum size should be
3714 // `VLenB * MaxLMUL` so that it fits in a single register group.
3715 unsigned MinSize = ST->getXLen() / 8 + 1;
3716 unsigned MaxSize = VLenB * ST->getMaxLMULForFixedLengthVectors();
3717 for (unsigned Size = MinSize; Size <= MaxSize; Size++)
3718 Options.LoadSizes.insert(I: Options.LoadSizes.begin(), Elt: Size);
3719 }
3720 return Options;
3721}
3722
3723bool RISCVTTIImpl::shouldTreatInstructionLikeSelect(
3724 const Instruction *I) const {
3725 if (EnableOrLikeSelectOpt) {
3726 // For the binary operators (e.g. or) we need to be more careful than
3727 // selects, here we only transform them if they are already at a natural
3728 // break point in the code - the end of a block with an unconditional
3729 // terminator.
3730 if (I->getOpcode() == Instruction::Or &&
3731 isa<UncondBrInst>(Val: I->getNextNode()))
3732 return true;
3733
3734 if (I->getOpcode() == Instruction::Add ||
3735 I->getOpcode() == Instruction::Sub)
3736 return true;
3737 }
3738 return BaseT::shouldTreatInstructionLikeSelect(I);
3739}
3740
3741bool RISCVTTIImpl::shouldCopyAttributeWhenOutliningFrom(
3742 const Function *Caller, const Attribute &Attr) const {
3743 // "interrupt" controls the prolog/epilog of interrupt handlers (and includes
3744 // restrictions on their signatures). We can outline from the bodies of these
3745 // handlers, but when we do we need to make sure we don't mark the outlined
3746 // function as an interrupt handler too.
3747 if (Attr.isStringAttribute() && Attr.getKindAsString() == "interrupt")
3748 return false;
3749
3750 return BaseT::shouldCopyAttributeWhenOutliningFrom(Caller, Attr);
3751}
3752
3753std::optional<Instruction *>
3754RISCVTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
3755 // Attach a range return attribute describing the result of vsetvli/vsetvlimax
3756 // so generic value analyses can reason about it. The verifier guarantees an
3757 // XLen result and constant VSEW/VLMUL encoding a valid vtype, so no defensive
3758 // validation is needed here.
3759 if (is_contained(Set: {Intrinsic::riscv_vsetvli, Intrinsic::riscv_vsetvlimax},
3760 Element: II.getIntrinsicID())) {
3761 // These intrinsics require the V extension; without it the VLEN queries
3762 // below would assert. Such IR would fail isel anyway, so just bail out.
3763 if (!ST->hasVInstructions())
3764 return {};
3765
3766 bool HasAVL = II.getIntrinsicID() == Intrinsic::riscv_vsetvli;
3767 unsigned Offset = HasAVL ? 1 : 0;
3768 unsigned BitWidth = II.getType()->getIntegerBitWidth();
3769 ConstantRange VLenRange(APInt(BitWidth, ST->getRealMinVLen()),
3770 APInt(BitWidth, ST->getRealMaxVLen()) + 1);
3771
3772 uint64_t VSEW = cast<ConstantInt>(Val: II.getArgOperand(i: Offset))->getZExtValue();
3773 auto VLMUL = static_cast<RISCVVType::VLMUL>(
3774 cast<ConstantInt>(Val: II.getArgOperand(i: Offset + 1))->getZExtValue());
3775 unsigned SEW = RISCVVType::decodeVSEW(VSEW);
3776 unsigned Ratio = RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
3777
3778 // VLMAX = VLEN / (SEW / LMUL), clamped to >= 1 for any usable vtype.
3779 ConstantRange VLMAXRange =
3780 VLenRange.udiv(Other: ConstantRange(APInt(BitWidth, Ratio)))
3781 .umax(Other: ConstantRange(APInt(BitWidth, 1)));
3782
3783 // vsetvlimax returns exactly VLMAX; vsetvli returns vl with
3784 // 0 <= vl <= min(AVL, VLMAX). vl == AVL only when AVL <= the smallest
3785 // possible VLMAX; otherwise vl can shrink below VLMAX (to 0 at runtime), so
3786 // only the VLMAX upper bound is sound.
3787 ConstantRange VLRange = VLMAXRange;
3788 if (HasAVL) {
3789 APInt MaxVL = VLMAXRange.getUnsignedMax();
3790 Value *AVL = II.getArgOperand(i: 0);
3791 // vl > 0 if AVL > 0
3792 APInt MinVL = APInt(BitWidth, isKnownNonZero(V: AVL, Q: DL) ? 1 : 0);
3793 if (auto *AVLC = dyn_cast<ConstantInt>(Val: AVL)) {
3794 const APInt &C = AVLC->getValue();
3795 // A constant AVL not exceeding the smallest possible VLMAX means vl is
3796 // exactly AVL, so replace the intrinsic with that constant.
3797 if (C.ule(RHS: VLMAXRange.getUnsignedMin()))
3798 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::get(Ty: II.getType(), V: C));
3799 VLRange =
3800 ConstantRange::getNonEmpty(Lower: MinVL, Upper: APIntOps::umin(A: C, B: MaxVL) + 1);
3801 } else {
3802 VLRange = ConstantRange::getNonEmpty(Lower: MinVL, Upper: MaxVL + 1);
3803 }
3804 }
3805
3806 ConstantRange OldRange =
3807 II.getRange().value_or(u: ConstantRange::getFull(BitWidth));
3808 ConstantRange NewRange = VLRange.intersectWith(CR: OldRange);
3809 if (NewRange != OldRange) {
3810 II.addRangeRetAttr(CR: NewRange);
3811 return &II;
3812 }
3813 return {};
3814 }
3815
3816 // If all operands of a vmv.v.x are constant, fold a bitcast(vmv.v.x) to scale
3817 // the vmv.v.x, enabling removal of the bitcast. The transform helps avoid
3818 // creating redundant masks.
3819 const DataLayout &DL = IC.getDataLayout();
3820 if (II.user_empty())
3821 return {};
3822 auto *TargetVecTy = dyn_cast<ScalableVectorType>(Val: II.user_back()->getType());
3823 if (!TargetVecTy)
3824 return {};
3825 const APInt *Scalar;
3826 uint64_t VL;
3827 if (!match(V: &II, P: m_Intrinsic<Intrinsic::riscv_vmv_v_x>(
3828 Ops: m_Poison(), Ops: m_APInt(Res&: Scalar), Ops: m_ConstantInt(V&: VL))) ||
3829 !all_of(Range: II.users(), P: [TargetVecTy](User *U) {
3830 return U->getType() == TargetVecTy && match(V: U, P: m_BitCast(Op: m_Value()));
3831 }))
3832 return {};
3833 auto *SourceVecTy = cast<ScalableVectorType>(Val: II.getType());
3834 unsigned TargetEltBW = DL.getTypeSizeInBits(Ty: TargetVecTy->getElementType());
3835 unsigned SourceEltBW = DL.getTypeSizeInBits(Ty: SourceVecTy->getElementType());
3836 if (TargetEltBW % SourceEltBW)
3837 return {};
3838 unsigned TargetScale = TargetEltBW / SourceEltBW;
3839 if (VL % TargetScale || TargetScale == 1)
3840 return {};
3841 Type *VLTy = II.getOperand(i_nocapture: 2)->getType();
3842 ElementCount SourceEC = SourceVecTy->getElementCount();
3843 unsigned NewEltBW = SourceEltBW * TargetScale;
3844 if (!SourceEC.isKnownMultipleOf(RHS: TargetScale) ||
3845 !DL.fitsInLegalInteger(Width: NewEltBW))
3846 return {};
3847 auto *NewEltTy = IntegerType::get(C&: II.getContext(), NumBits: NewEltBW);
3848 if (!TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: NewEltTy)))
3849 return {};
3850 ElementCount NewEC = SourceEC.divideCoefficientBy(RHS: TargetScale);
3851 Type *RetTy = VectorType::get(ElementType: NewEltTy, EC: NewEC);
3852 assert(SourceVecTy->canLosslesslyBitCastTo(RetTy) &&
3853 "Lossless bitcast between types expected");
3854 APInt NewScalar = APInt::getSplat(NewLen: NewEltBW, V: *Scalar);
3855 return IC.replaceInstUsesWith(
3856 I&: II,
3857 V: IC.Builder.CreateBitCast(
3858 V: IC.Builder.CreateIntrinsic(
3859 RetTy, ID: Intrinsic::riscv_vmv_v_x,
3860 Args: {PoisonValue::get(T: RetTy), ConstantInt::get(Ty: NewEltTy, V: NewScalar),
3861 ConstantInt::get(Ty: VLTy, V: VL / TargetScale)}),
3862 DestTy: SourceVecTy));
3863}
3864