1//===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "RISCVTargetTransformInfo.h"
10#include "MCTargetDesc/RISCVMatInt.h"
11#include "llvm/ADT/STLExtras.h"
12#include "llvm/Analysis/TargetTransformInfo.h"
13#include "llvm/CodeGen/BasicTTIImpl.h"
14#include "llvm/CodeGen/CostTable.h"
15#include "llvm/CodeGen/TargetLowering.h"
16#include "llvm/CodeGen/ValueTypes.h"
17#include "llvm/IR/Instructions.h"
18#include "llvm/IR/IntrinsicsRISCV.h"
19#include "llvm/IR/PatternMatch.h"
20#include "llvm/Transforms/InstCombine/InstCombiner.h"
21#include <cmath>
22#include <optional>
23using namespace llvm;
24using namespace llvm::PatternMatch;
25
26#define DEBUG_TYPE "riscvtti"
27
28static cl::opt<unsigned> RVVRegisterWidthLMUL(
29 "riscv-v-register-bit-width-lmul",
30 cl::desc(
31 "The LMUL to use for getRegisterBitWidth queries. Affects LMUL used "
32 "by autovectorized code. Fractional LMULs are not supported."),
33 cl::init(Val: 2), cl::Hidden);
34
35static cl::opt<unsigned> SLPMaxVF(
36 "riscv-v-slp-max-vf",
37 cl::desc(
38 "Overrides result used for getMaximumVF query which is used "
39 "exclusively by SLP vectorizer."),
40 cl::Hidden);
41
42static cl::opt<unsigned>
43 RVVMinTripCount("riscv-v-min-trip-count",
44 cl::desc("Set the lower bound of a trip count to decide on "
45 "vectorization while tail-folding."),
46 cl::init(Val: 5), cl::Hidden);
47
48static cl::opt<bool> EnableOrLikeSelectOpt("enable-riscv-or-like-select",
49 cl::init(Val: true), cl::Hidden);
50
51InstructionCost
52RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
53 TTI::TargetCostKind CostKind) const {
54 // Check if the type is valid for all CostKind
55 if (!VT.isVector())
56 return InstructionCost::getInvalid();
57 size_t NumInstr = OpCodes.size();
58 if (CostKind == TTI::TCK_CodeSize)
59 return NumInstr;
60 InstructionCost LMULCost = TLI->getLMULCost(VT);
61 if ((CostKind != TTI::TCK_RecipThroughput) && (CostKind != TTI::TCK_Latency))
62 return LMULCost * NumInstr;
63 InstructionCost Cost = 0;
64 for (auto Op : OpCodes) {
65 switch (Op) {
66 case RISCV::VRGATHER_VI:
67 Cost += TLI->getVRGatherVICost(VT);
68 break;
69 case RISCV::VRGATHER_VV:
70 Cost += TLI->getVRGatherVVCost(VT);
71 break;
72 case RISCV::VSLIDEUP_VI:
73 case RISCV::VSLIDEDOWN_VI:
74 Cost += TLI->getVSlideVICost(VT);
75 break;
76 case RISCV::VSLIDEUP_VX:
77 case RISCV::VSLIDEDOWN_VX:
78 Cost += TLI->getVSlideVXCost(VT);
79 break;
80 case RISCV::VREDMAX_VS:
81 case RISCV::VREDMIN_VS:
82 case RISCV::VREDMAXU_VS:
83 case RISCV::VREDMINU_VS:
84 case RISCV::VREDSUM_VS:
85 case RISCV::VREDAND_VS:
86 case RISCV::VREDOR_VS:
87 case RISCV::VREDXOR_VS:
88 case RISCV::VFREDMAX_VS:
89 case RISCV::VFREDMIN_VS:
90 case RISCV::VFREDUSUM_VS: {
91 unsigned VL = VT.getVectorMinNumElements();
92 if (!VT.isFixedLengthVector())
93 VL *= *getVScaleForTuning();
94 Cost += Log2_32_Ceil(Value: VL);
95 break;
96 }
97 case RISCV::VFREDOSUM_VS: {
98 unsigned VL = VT.getVectorMinNumElements();
99 if (!VT.isFixedLengthVector())
100 VL *= *getVScaleForTuning();
101 Cost += VL;
102 break;
103 }
104 case RISCV::VMV_X_S:
105 case RISCV::VMV_S_X:
106 case RISCV::VFMV_F_S:
107 case RISCV::VFMV_S_F:
108 case RISCV::VMOR_MM:
109 case RISCV::VMXOR_MM:
110 case RISCV::VMAND_MM:
111 case RISCV::VMANDN_MM:
112 case RISCV::VMNAND_MM:
113 case RISCV::VCPOP_M:
114 case RISCV::VFIRST_M:
115 Cost += 1;
116 break;
117 case RISCV::VDIV_VV:
118 case RISCV::VREM_VV:
119 Cost += LMULCost * TTI::TCC_Expensive;
120 break;
121 default:
122 Cost += LMULCost;
123 }
124 }
125 return Cost;
126}
127
128static InstructionCost getIntImmCostImpl(const DataLayout &DL,
129 const RISCVSubtarget *ST,
130 const APInt &Imm, Type *Ty,
131 TTI::TargetCostKind CostKind,
132 bool FreeZeroes) {
133 assert(Ty->isIntegerTy() &&
134 "getIntImmCost can only estimate cost of materialising integers");
135
136 // We have a Zero register, so 0 is always free.
137 if (Imm == 0)
138 return TTI::TCC_Free;
139
140 // Otherwise, we check how many instructions it will take to materialise.
141 return RISCVMatInt::getIntMatCost(Val: Imm, Size: DL.getTypeSizeInBits(Ty), STI: *ST,
142 /*CompressionCost=*/false, FreeZeroes);
143}
144
145InstructionCost
146RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
147 TTI::TargetCostKind CostKind) const {
148 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind, FreeZeroes: false);
149}
150
151// Look for patterns of shift followed by AND that can be turned into a pair of
152// shifts. We won't need to materialize an immediate for the AND so these can
153// be considered free.
154static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) {
155 uint64_t Mask = Imm.getZExtValue();
156 auto *BO = dyn_cast<BinaryOperator>(Val: Inst->getOperand(i: 0));
157 if (!BO || !BO->hasOneUse())
158 return false;
159
160 if (BO->getOpcode() != Instruction::Shl)
161 return false;
162
163 if (!isa<ConstantInt>(Val: BO->getOperand(i_nocapture: 1)))
164 return false;
165
166 unsigned ShAmt = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1))->getZExtValue();
167 // (and (shl x, c2), c1) will be matched to (srli (slli x, c2+c3), c3) if c1
168 // is a mask shifted by c2 bits with c3 leading zeros.
169 if (isShiftedMask_64(Value: Mask)) {
170 unsigned Trailing = llvm::countr_zero(Val: Mask);
171 if (ShAmt == Trailing)
172 return true;
173 }
174
175 return false;
176}
177
178// If this is i64 AND is part of (X & -(1 << C1) & 0xffffffff) == C2 << C1),
179// DAGCombiner can convert this to (sraiw X, C1) == sext(C2) for RV64. On RV32,
180// the type will be split so only the lower 32 bits need to be compared using
181// (srai/srli X, C) == C2.
182static bool canUseShiftCmp(Instruction *Inst, const APInt &Imm) {
183 if (!Inst->hasOneUse())
184 return false;
185
186 // Look for equality comparison.
187 auto *Cmp = dyn_cast<ICmpInst>(Val: *Inst->user_begin());
188 if (!Cmp || !Cmp->isEquality())
189 return false;
190
191 // Right hand side of comparison should be a constant.
192 auto *C = dyn_cast<ConstantInt>(Val: Cmp->getOperand(i_nocapture: 1));
193 if (!C)
194 return false;
195
196 uint64_t Mask = Imm.getZExtValue();
197
198 // Mask should be of the form -(1 << C) in the lower 32 bits.
199 if (!isUInt<32>(x: Mask) || !isPowerOf2_32(Value: -uint32_t(Mask)))
200 return false;
201
202 // Comparison constant should be a subset of Mask.
203 uint64_t CmpC = C->getZExtValue();
204 if ((CmpC & Mask) != CmpC)
205 return false;
206
207 // We'll need to sign extend the comparison constant and shift it right. Make
208 // sure the new constant can use addi/xori+seqz/snez.
209 unsigned ShiftBits = llvm::countr_zero(Val: Mask);
210 int64_t NewCmpC = SignExtend64<32>(x: CmpC) >> ShiftBits;
211 return NewCmpC >= -2048 && NewCmpC <= 2048;
212}
213
214InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
215 const APInt &Imm, Type *Ty,
216 TTI::TargetCostKind CostKind,
217 Instruction *Inst) const {
218 assert(Ty->isIntegerTy() &&
219 "getIntImmCost can only estimate cost of materialising integers");
220
221 // We have a Zero register, so 0 is always free.
222 if (Imm == 0)
223 return TTI::TCC_Free;
224
225 // Some instructions in RISC-V can take a 12-bit immediate. Some of these are
226 // commutative, in others the immediate comes from a specific argument index.
227 bool Takes12BitImm = false;
228 unsigned ImmArgIdx = ~0U;
229
230 switch (Opcode) {
231 case Instruction::GetElementPtr:
232 // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will
233 // split up large offsets in GEP into better parts than ConstantHoisting
234 // can.
235 return TTI::TCC_Free;
236 case Instruction::Store: {
237 // Use the materialization cost regardless of if it's the address or the
238 // value that is constant, except for if the store is misaligned and
239 // misaligned accesses are not legal (experience shows constant hoisting
240 // can sometimes be harmful in such cases).
241 if (Idx == 1 || !Inst)
242 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
243 /*FreeZeroes=*/true);
244
245 StoreInst *ST = cast<StoreInst>(Val: Inst);
246 if (!getTLI()->allowsMemoryAccessForAlignment(
247 Context&: Ty->getContext(), DL, VT: getTLI()->getValueType(DL, Ty),
248 AddrSpace: ST->getPointerAddressSpace(), Alignment: ST->getAlign()))
249 return TTI::TCC_Free;
250
251 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
252 /*FreeZeroes=*/true);
253 }
254 case Instruction::Load:
255 // If the address is a constant, use the materialization cost.
256 return getIntImmCost(Imm, Ty, CostKind);
257 case Instruction::And:
258 // zext.h
259 if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())
260 return TTI::TCC_Free;
261 // zext.w
262 if (Imm == UINT64_C(0xffffffff) &&
263 ((ST->hasStdExtZba() && ST->isRV64()) || ST->isRV32()))
264 return TTI::TCC_Free;
265 // bclri
266 if (ST->hasStdExtZbs() && (~Imm).isPowerOf2())
267 return TTI::TCC_Free;
268 if (Inst && Idx == 1 && Imm.getBitWidth() <= ST->getXLen() &&
269 canUseShiftPair(Inst, Imm))
270 return TTI::TCC_Free;
271 if (Inst && Idx == 1 && Imm.getBitWidth() == 64 &&
272 canUseShiftCmp(Inst, Imm))
273 return TTI::TCC_Free;
274 Takes12BitImm = true;
275 break;
276 case Instruction::Add:
277 Takes12BitImm = true;
278 break;
279 case Instruction::Or:
280 case Instruction::Xor:
281 // bseti/binvi
282 if (ST->hasStdExtZbs() && Imm.isPowerOf2())
283 return TTI::TCC_Free;
284 Takes12BitImm = true;
285 break;
286 case Instruction::Mul:
287 // Power of 2 is a shift. Negated power of 2 is a shift and a negate.
288 if (Imm.isPowerOf2() || Imm.isNegatedPowerOf2())
289 return TTI::TCC_Free;
290 // One more or less than a power of 2 can use SLLI+ADD/SUB.
291 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2())
292 return TTI::TCC_Free;
293 // FIXME: There is no MULI instruction.
294 Takes12BitImm = true;
295 break;
296 case Instruction::Sub:
297 case Instruction::Shl:
298 case Instruction::LShr:
299 case Instruction::AShr:
300 Takes12BitImm = true;
301 ImmArgIdx = 1;
302 break;
303 default:
304 break;
305 }
306
307 if (Takes12BitImm) {
308 // Check immediate is the correct argument...
309 if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {
310 // ... and fits into the 12-bit immediate.
311 if (Imm.getSignificantBits() <= 64 &&
312 getTLI()->isLegalAddImmediate(Imm: Imm.getSExtValue())) {
313 return TTI::TCC_Free;
314 }
315 }
316
317 // Otherwise, use the full materialisation cost.
318 return getIntImmCost(Imm, Ty, CostKind);
319 }
320
321 // By default, prevent hoisting.
322 return TTI::TCC_Free;
323}
324
325InstructionCost
326RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
327 const APInt &Imm, Type *Ty,
328 TTI::TargetCostKind CostKind) const {
329 // Prevent hoisting in unknown cases.
330 return TTI::TCC_Free;
331}
332
333bool RISCVTTIImpl::hasActiveVectorLength() const {
334 return ST->hasVInstructions();
335}
336
337TargetTransformInfo::PopcntSupportKind
338RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {
339 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
340 return ST->hasCPOPLike() ? TTI::PSK_FastHardware : TTI::PSK_Software;
341}
342
343InstructionCost RISCVTTIImpl::getPartialReductionCost(
344 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
345 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
346 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
347 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
348 if (Opcode == Instruction::FAdd)
349 return InstructionCost::getInvalid();
350
351 // zve32x is broken for partial_reduce_umla, but let's make sure we
352 // don't generate them.
353 if (!ST->hasStdExtZvdot4a8i() || ST->getELen() < 64 ||
354 Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||
355 InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(BitWidth: 8) ||
356 !AccumType->isIntegerTy(BitWidth: 32) || !VF.isKnownMultipleOf(RHS: 4))
357 return InstructionCost::getInvalid();
358
359 Type *Tp = VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: 4));
360 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Tp);
361 // Note: Asuming all vdot4a* variants are equal cost
362 return LT.first *
363 getRISCVInstructionCost(OpCodes: RISCV::VDOT4A_VV, VT: LT.second, CostKind);
364}
365
366bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
367 // Currently, the ExpandReductions pass can't expand scalable-vector
368 // reductions, but we still request expansion as RVV doesn't support certain
369 // reductions and the SelectionDAG can't legalize them either.
370 switch (II->getIntrinsicID()) {
371 default:
372 return false;
373 // These reductions have no equivalent in RVV
374 case Intrinsic::vector_reduce_mul:
375 case Intrinsic::vector_reduce_fmul:
376 return true;
377 }
378}
379
380std::optional<unsigned> RISCVTTIImpl::getMaxVScale() const {
381 if (ST->hasVInstructions())
382 return ST->getRealMaxVLen() / RISCV::RVVBitsPerBlock;
383 return BaseT::getMaxVScale();
384}
385
386std::optional<unsigned> RISCVTTIImpl::getVScaleForTuning() const {
387 if (ST->hasVInstructions())
388 if (unsigned MinVLen = ST->getRealMinVLen();
389 MinVLen >= RISCV::RVVBitsPerBlock)
390 return MinVLen / RISCV::RVVBitsPerBlock;
391 return BaseT::getVScaleForTuning();
392}
393
394TypeSize
395RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
396 unsigned LMUL =
397 llvm::bit_floor(Value: std::clamp<unsigned>(val: RVVRegisterWidthLMUL, lo: 1, hi: 8));
398 switch (K) {
399 case TargetTransformInfo::RGK_Scalar:
400 return TypeSize::getFixed(ExactSize: ST->getXLen());
401 case TargetTransformInfo::RGK_FixedWidthVector:
402 return TypeSize::getFixed(
403 ExactSize: ST->useRVVForFixedLengthVectors() ? LMUL * ST->getRealMinVLen() : 0);
404 case TargetTransformInfo::RGK_ScalableVector:
405 return TypeSize::getScalable(
406 MinimumSize: (ST->hasVInstructions() &&
407 ST->getRealMinVLen() >= RISCV::RVVBitsPerBlock)
408 ? LMUL * RISCV::RVVBitsPerBlock
409 : 0);
410 }
411
412 llvm_unreachable("Unsupported register kind");
413}
414
415InstructionCost RISCVTTIImpl::getStaticDataAddrGenerationCost(
416 const TTI::TargetCostKind CostKind) const {
417 switch (CostKind) {
418 case TTI::TCK_CodeSize:
419 case TTI::TCK_SizeAndLatency:
420 // Always 2 instructions
421 return 2;
422 case TTI::TCK_Latency:
423 case TTI::TCK_RecipThroughput:
424 // Depending on the memory model the address generation will
425 // require AUIPC + ADDI (medany) or LUI + ADDI (medlow). Don't
426 // have a way of getting this information here, so conservatively
427 // require both.
428 // In practice, these are generally implemented together.
429 return (ST->hasAUIPCADDIFusion() && ST->hasLUIADDIFusion()) ? 1 : 2;
430 }
431 llvm_unreachable("Unsupported cost kind");
432}
433
434InstructionCost
435RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,
436 TTI::TargetCostKind CostKind) const {
437 // Add a cost of address generation + the cost of the load. The address
438 // is expected to be a PC relative offset to a constant pool entry
439 // using auipc/addi.
440 return getStaticDataAddrGenerationCost(CostKind) +
441 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: DL.getABITypeAlign(Ty),
442 /*AddressSpace=*/0, CostKind);
443}
444
445static bool isRepeatedConcatMask(ArrayRef<int> Mask, int &SubVectorSize) {
446 unsigned Size = Mask.size();
447 if (!isPowerOf2_32(Value: Size))
448 return false;
449 for (unsigned I = 0; I != Size; ++I) {
450 if (static_cast<unsigned>(Mask[I]) == I)
451 continue;
452 if (Mask[I] != 0)
453 return false;
454 if (Size % I != 0)
455 return false;
456 for (unsigned J = I + 1; J != Size; ++J)
457 // Check the pattern is repeated.
458 if (static_cast<unsigned>(Mask[J]) != J % I)
459 return false;
460 SubVectorSize = I;
461 return true;
462 }
463 // That means Mask is <0, 1, 2, 3>. This is not a concatenation.
464 return false;
465}
466
467static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,
468 LLVMContext &C) {
469 assert((DataVT.getScalarSizeInBits() != 8 ||
470 DataVT.getVectorNumElements() <= 256) && "unhandled case in lowering");
471 MVT IndexVT = DataVT.changeTypeToInteger();
472 if (IndexVT.getScalarType().bitsGT(VT: ST.getXLenVT()))
473 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
474 return cast<VectorType>(Val: EVT(IndexVT).getTypeForEVT(Context&: C));
475}
476
477/// Attempt to approximate the cost of a shuffle which will require splitting
478/// during legalization. Note that processShuffleMasks is not an exact proxy
479/// for the algorithm used in LegalizeVectorTypes, but hopefully it's a
480/// reasonably close upperbound.
481static InstructionCost costShuffleViaSplitting(const RISCVTTIImpl &TTI,
482 MVT LegalVT, VectorType *Tp,
483 ArrayRef<int> Mask,
484 TTI::TargetCostKind CostKind) {
485 assert(LegalVT.isFixedLengthVector() && !Mask.empty() &&
486 "Expected fixed vector type and non-empty mask");
487 unsigned LegalNumElts = LegalVT.getVectorNumElements();
488 // Number of destination vectors after legalization:
489 unsigned NumOfDests = divideCeil(Numerator: Mask.size(), Denominator: LegalNumElts);
490 // We are going to permute multiple sources and the result will be in
491 // multiple destinations. Providing an accurate cost only for splits where
492 // the element type remains the same.
493 if (NumOfDests <= 1 ||
494 LegalVT.getVectorElementType().getSizeInBits() !=
495 Tp->getElementType()->getPrimitiveSizeInBits() ||
496 LegalNumElts >= Tp->getElementCount().getFixedValue())
497 return InstructionCost::getInvalid();
498
499 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
500 unsigned LegalVTSize = LegalVT.getStoreSize();
501 // Number of source vectors after legalization:
502 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
503
504 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: LegalNumElts);
505
506 unsigned NormalizedVF = LegalNumElts * std::max(a: NumOfSrcs, b: NumOfDests);
507 unsigned NumOfSrcRegs = NormalizedVF / LegalNumElts;
508 unsigned NumOfDestRegs = NormalizedVF / LegalNumElts;
509 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
510 assert(NormalizedVF >= Mask.size() &&
511 "Normalized mask expected to be not shorter than original mask.");
512 copy(Range&: Mask, Out: NormalizedMask.begin());
513 InstructionCost Cost = 0;
514 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
515 processShuffleMasks(
516 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
517 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
518 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
519 return;
520 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
521 .second)
522 return;
523 Cost += TTI.getShuffleCost(
524 Kind: TTI::SK_PermuteSingleSrc,
525 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
526 SrcTy: SingleOpTy, Mask: RegMask, CostKind, Index: 0, SubTp: nullptr);
527 },
528 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
529 Cost += TTI.getShuffleCost(
530 Kind: TTI::SK_PermuteTwoSrc,
531 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
532 SrcTy: SingleOpTy, Mask: RegMask, CostKind, Index: 0, SubTp: nullptr);
533 });
534 return Cost;
535}
536
537/// Try to perform better estimation of the permutation.
538/// 1. Split the source/destination vectors into real registers.
539/// 2. Do the mask analysis to identify which real registers are
540/// permuted. If more than 1 source registers are used for the
541/// destination register building, the cost for this destination register
542/// is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
543/// source register is used, build mask and calculate the cost as a cost
544/// of PermuteSingleSrc.
545/// Also, for the single register permute we try to identify if the
546/// destination register is just a copy of the source register or the
547/// copy of the previous destination register (the cost is
548/// TTI::TCC_Basic). If the source register is just reused, the cost for
549/// this operation is 0.
550static InstructionCost
551costShuffleViaVRegSplitting(const RISCVTTIImpl &TTI, MVT LegalVT,
552 std::optional<unsigned> VLen, VectorType *Tp,
553 ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {
554 assert(LegalVT.isFixedLengthVector());
555 if (!VLen || Mask.empty())
556 return InstructionCost::getInvalid();
557 MVT ElemVT = LegalVT.getVectorElementType();
558 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
559 LegalVT = TTI.getTypeLegalizationCost(
560 Ty: FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: ElemsPerVReg))
561 .second;
562 // Number of destination vectors after legalization:
563 InstructionCost NumOfDests =
564 divideCeil(Numerator: Mask.size(), Denominator: LegalVT.getVectorNumElements());
565 if (NumOfDests <= 1 ||
566 LegalVT.getVectorElementType().getSizeInBits() !=
567 Tp->getElementType()->getPrimitiveSizeInBits() ||
568 LegalVT.getVectorNumElements() >= Tp->getElementCount().getFixedValue())
569 return InstructionCost::getInvalid();
570
571 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
572 unsigned LegalVTSize = LegalVT.getStoreSize();
573 // Number of source vectors after legalization:
574 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
575
576 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(),
577 NumElts: LegalVT.getVectorNumElements());
578
579 unsigned E = NumOfDests.getValue();
580 unsigned NormalizedVF =
581 LegalVT.getVectorNumElements() * std::max(a: NumOfSrcs, b: E);
582 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
583 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
584 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
585 assert(NormalizedVF >= Mask.size() &&
586 "Normalized mask expected to be not shorter than original mask.");
587 copy(Range&: Mask, Out: NormalizedMask.begin());
588 InstructionCost Cost = 0;
589 int NumShuffles = 0;
590 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
591 processShuffleMasks(
592 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
593 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
594 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
595 return;
596 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
597 .second)
598 return;
599 ++NumShuffles;
600 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleOpTy,
601 SrcTy: SingleOpTy, Mask: RegMask, CostKind, Index: 0, SubTp: nullptr);
602 },
603 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
604 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
605 SrcTy: SingleOpTy, Mask: RegMask, CostKind, Index: 0, SubTp: nullptr);
606 NumShuffles += 2;
607 });
608 // Note: check that we do not emit too many shuffles here to prevent code
609 // size explosion.
610 // TODO: investigate, if it can be improved by extra analysis of the masks
611 // to check if the code is more profitable.
612 if ((NumOfDestRegs > 2 && NumShuffles <= static_cast<int>(NumOfDestRegs)) ||
613 (NumOfDestRegs <= 2 && NumShuffles < 4))
614 return Cost;
615 return InstructionCost::getInvalid();
616}
617
618InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
619 ArrayRef<int> Mask,
620 TTI::TargetCostKind CostKind) const {
621 // Avoid missing masks and length changing shuffles
622 if (Mask.size() <= 2 || Mask.size() != Tp->getNumElements())
623 return InstructionCost::getInvalid();
624
625 int NumElts = Tp->getNumElements();
626 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Tp);
627 // Avoid scalarization cases
628 if (!LT.second.isFixedLengthVector())
629 return InstructionCost::getInvalid();
630
631 // Requires moving elements between parts, which requires additional
632 // unmodeled instructions.
633 if (LT.first != 1)
634 return InstructionCost::getInvalid();
635
636 auto GetSlideOpcode = [&](int SlideAmt) {
637 assert(SlideAmt != 0);
638 bool IsVI = isUInt<5>(x: std::abs(x: SlideAmt));
639 if (SlideAmt < 0)
640 return IsVI ? RISCV::VSLIDEDOWN_VI : RISCV::VSLIDEDOWN_VX;
641 return IsVI ? RISCV::VSLIDEUP_VI : RISCV::VSLIDEUP_VX;
642 };
643
644 std::array<std::pair<int, int>, 2> SrcInfo;
645 if (!isMaskedSlidePair(Mask, NumElts, SrcInfo))
646 return InstructionCost::getInvalid();
647
648 if (SrcInfo[1].second == 0)
649 std::swap(x&: SrcInfo[0], y&: SrcInfo[1]);
650
651 InstructionCost FirstSlideCost = 0;
652 if (SrcInfo[0].second != 0) {
653 unsigned Opcode = GetSlideOpcode(SrcInfo[0].second);
654 FirstSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
655 }
656
657 if (SrcInfo[1].first == -1)
658 return FirstSlideCost;
659
660 InstructionCost SecondSlideCost = 0;
661 if (SrcInfo[1].second != 0) {
662 unsigned Opcode = GetSlideOpcode(SrcInfo[1].second);
663 SecondSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
664 } else {
665 SecondSlideCost =
666 getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second, CostKind);
667 }
668
669 auto EC = Tp->getElementCount();
670 VectorType *MaskTy =
671 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: Tp->getContext()), EC);
672 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
673 return FirstSlideCost + SecondSlideCost + MaskCost;
674}
675
676InstructionCost
677RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
678 VectorType *SrcTy, ArrayRef<int> Mask,
679 TTI::TargetCostKind CostKind, int Index,
680 VectorType *SubTp, ArrayRef<const Value *> Args,
681 const Instruction *CxtI) const {
682 assert((Mask.empty() || DstTy->isScalableTy() ||
683 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
684 "Expected the Mask to match the return size if given");
685 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
686 "Expected the same scalar types");
687
688 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
689
690 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
691 // For now, skip all fixed vector cost analysis when P extension is available
692 // to avoid crashes in getMinRVVVectorSizeInBits()
693 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: SrcTy))
694 return 1;
695
696 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
697
698 // First, handle cases where having a fixed length vector enables us to
699 // give a more accurate cost than falling back to generic scalable codegen.
700 // TODO: Each of these cases hints at a modeling gap around scalable vectors.
701 if (auto *FVTp = dyn_cast<FixedVectorType>(Val: SrcTy);
702 FVTp && ST->hasVInstructions() && LT.second.isFixedLengthVector()) {
703 InstructionCost VRegSplittingCost = costShuffleViaVRegSplitting(
704 TTI: *this, LegalVT: LT.second, VLen: ST->getRealVLen(),
705 Tp: Kind == TTI::SK_InsertSubvector ? DstTy : SrcTy, Mask, CostKind);
706 if (VRegSplittingCost.isValid())
707 return VRegSplittingCost;
708 switch (Kind) {
709 default:
710 break;
711 case TTI::SK_PermuteSingleSrc: {
712 if (Mask.size() >= 2) {
713 MVT EltTp = LT.second.getVectorElementType();
714 // If the size of the element is < ELEN then shuffles of interleaves and
715 // deinterleaves of 2 vectors can be lowered into the following
716 // sequences
717 if (EltTp.getScalarSizeInBits() < ST->getELen()) {
718 // Example sequence:
719 // vsetivli zero, 4, e8, mf4, ta, ma (ignored)
720 // vwaddu.vv v10, v8, v9
721 // li a0, -1 (ignored)
722 // vwmaccu.vx v10, a0, v9
723 if (ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Mask.size()))
724 return 2 * LT.first * TLI->getLMULCost(VT: LT.second);
725
726 if (Mask[0] == 0 || Mask[0] == 1) {
727 auto DeinterleaveMask = createStrideMask(Start: Mask[0], Stride: 2, VF: Mask.size());
728 // Example sequence:
729 // vnsrl.wi v10, v8, 0
730 if (equal(LRange&: DeinterleaveMask, RRange&: Mask))
731 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VNSRL_WI,
732 VT: LT.second, CostKind);
733 }
734 }
735 int SubVectorSize;
736 if (LT.second.getScalarSizeInBits() != 1 &&
737 isRepeatedConcatMask(Mask, SubVectorSize)) {
738 InstructionCost Cost = 0;
739 unsigned NumSlides = Log2_32(Value: Mask.size() / SubVectorSize);
740 // The cost of extraction from a subvector is 0 if the index is 0.
741 for (unsigned I = 0; I != NumSlides; ++I) {
742 unsigned InsertIndex = SubVectorSize * (1 << I);
743 FixedVectorType *SubTp =
744 FixedVectorType::get(ElementType: SrcTy->getElementType(), NumElts: InsertIndex);
745 FixedVectorType *DestTp =
746 FixedVectorType::getDoubleElementsVectorType(VTy: SubTp);
747 std::pair<InstructionCost, MVT> DestLT =
748 getTypeLegalizationCost(Ty: DestTp);
749 // Add the cost of whole vector register move because the
750 // destination vector register group for vslideup cannot overlap the
751 // source.
752 Cost += DestLT.first * TLI->getLMULCost(VT: DestLT.second);
753 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: DestTp, SrcTy: DestTp, Mask: {},
754 CostKind, Index: InsertIndex, SubTp);
755 }
756 return Cost;
757 }
758 }
759
760 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
761 SlideCost.isValid())
762 return SlideCost;
763
764 // vrgather + cost of generating the mask constant.
765 // We model this for an unknown mask with a single vrgather.
766 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
767 LT.second.getVectorNumElements() <= 256)) {
768 VectorType *IdxTy =
769 getVRGatherIndexType(DataVT: LT.second, ST: *ST, C&: SrcTy->getContext());
770 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
771 return IndexCost +
772 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VV, VT: LT.second, CostKind);
773 }
774 break;
775 }
776 case TTI::SK_Transpose:
777 case TTI::SK_PermuteTwoSrc: {
778
779 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
780 SlideCost.isValid())
781 return SlideCost;
782
783 // 2 x (vrgather + cost of generating the mask constant) + cost of mask
784 // register for the second vrgather. We model this for an unknown
785 // (shuffle) mask.
786 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
787 LT.second.getVectorNumElements() <= 256)) {
788 auto &C = SrcTy->getContext();
789 auto EC = SrcTy->getElementCount();
790 VectorType *IdxTy = getVRGatherIndexType(DataVT: LT.second, ST: *ST, C);
791 VectorType *MaskTy = VectorType::get(ElementType: IntegerType::getInt1Ty(C), EC);
792 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
793 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
794 return 2 * IndexCost +
795 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV, RISCV::VRGATHER_VV},
796 VT: LT.second, CostKind) +
797 MaskCost;
798 }
799 break;
800 }
801 }
802
803 auto shouldSplit = [](TTI::ShuffleKind Kind) {
804 switch (Kind) {
805 default:
806 return false;
807 case TTI::SK_PermuteSingleSrc:
808 case TTI::SK_Transpose:
809 case TTI::SK_PermuteTwoSrc:
810 return true;
811 }
812 };
813
814 if (!Mask.empty() && LT.first.isValid() && LT.first != 1 &&
815 shouldSplit(Kind)) {
816 InstructionCost SplitCost =
817 costShuffleViaSplitting(TTI: *this, LegalVT: LT.second, Tp: FVTp, Mask, CostKind);
818 if (SplitCost.isValid())
819 return SplitCost;
820 }
821 }
822
823 // Handle scalable vectors (and fixed vectors legalized to scalable vectors).
824 switch (Kind) {
825 default:
826 // Fallthrough to generic handling.
827 // TODO: Most of these cases will return getInvalid in generic code, and
828 // must be implemented here.
829 break;
830 case TTI::SK_ExtractSubvector:
831 // Extract at zero is always a subregister extract
832 if (Index == 0)
833 return TTI::TCC_Free;
834
835 // If we're extracting a subvector of at most m1 size at a sub-register
836 // boundary - which unfortunately we need exact vlen to identify - this is
837 // a subregister extract at worst and thus won't require a vslidedown.
838 // TODO: Extend for aligned m2, m4 subvector extracts
839 // TODO: Extend for misalgined (but contained) extracts
840 // TODO: Extend for scalable subvector types
841 if (std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
842 SubLT.second.isValid() && SubLT.second.isFixedLengthVector()) {
843 if (std::optional<unsigned> VLen = ST->getRealVLen();
844 VLen && SubLT.second.getScalarSizeInBits() * Index % *VLen == 0 &&
845 SubLT.second.getSizeInBits() <= *VLen)
846 return TTI::TCC_Free;
847 }
848
849 // Example sequence:
850 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
851 // vslidedown.vi v8, v9, 2
852 return LT.first *
853 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI, VT: LT.second, CostKind);
854 case TTI::SK_InsertSubvector:
855 // Example sequence:
856 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
857 // vslideup.vi v8, v9, 2
858 LT = getTypeLegalizationCost(Ty: DstTy);
859 return LT.first *
860 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEUP_VI, VT: LT.second, CostKind);
861 case TTI::SK_Select: {
862 // Example sequence:
863 // li a0, 90
864 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
865 // vmv.s.x v0, a0
866 // vmerge.vvm v8, v9, v8, v0
867 // We use 2 for the cost of the mask materialization as this is the true
868 // cost for small masks and most shuffles are small. At worst, this cost
869 // should be a very small constant for the constant pool load. As such,
870 // we may bias towards large selects slightly more than truly warranted.
871 return LT.first *
872 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_S_X, RISCV::VMERGE_VVM},
873 VT: LT.second, CostKind));
874 }
875 case TTI::SK_Broadcast: {
876 // Check for broadcast loads, which are synthesized by optimized zero-stride
877 // loads (this is checked in RISCVTTIImpl::isLegalBroadcastLoad).
878 bool IsLoad = !Args.empty() && isa<LoadInst>(Val: Args[0]);
879 if (IsLoad && LT.second.isVector() &&
880 isLegalBroadcastLoad(ElementTy: SrcTy->getElementType(),
881 NumElements: LT.second.getVectorElementCount()))
882 return 0;
883
884 bool HasScalar = (Args.size() > 0) && (Operator::getOpcode(V: Args[0]) ==
885 Instruction::InsertElement);
886 if (LT.second.getScalarSizeInBits() == 1) {
887 if (HasScalar) {
888 // Example sequence:
889 // andi a0, a0, 1
890 // vsetivli zero, 2, e8, mf8, ta, ma (ignored)
891 // vmv.v.x v8, a0
892 // vmsne.vi v0, v8, 0
893 return LT.first *
894 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
895 VT: LT.second, CostKind));
896 }
897 // Example sequence:
898 // vsetivli zero, 2, e8, mf8, ta, mu (ignored)
899 // vmv.v.i v8, 0
900 // vmerge.vim v8, v8, 1, v0
901 // vmv.x.s a0, v8
902 // andi a0, a0, 1
903 // vmv.v.x v8, a0
904 // vmsne.vi v0, v8, 0
905
906 return LT.first *
907 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_I, RISCV::VMERGE_VIM,
908 RISCV::VMV_X_S, RISCV::VMV_V_X,
909 RISCV::VMSNE_VI},
910 VT: LT.second, CostKind));
911 }
912
913 if (HasScalar) {
914 // Example sequence:
915 // vmv.v.x v8, a0
916 return LT.first *
917 getRISCVInstructionCost(OpCodes: RISCV::VMV_V_X, VT: LT.second, CostKind);
918 }
919
920 // Example sequence:
921 // vrgather.vi v9, v8, 0
922 return LT.first *
923 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VI, VT: LT.second, CostKind);
924 }
925 case TTI::SK_Splice: {
926 // vslidedown+vslideup.
927 // TODO: Multiplying by LT.first implies this legalizes into multiple copies
928 // of similar code, but I think we expand through memory.
929 unsigned Opcodes[2] = {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX};
930 if (Index >= 0 && Index < 32)
931 Opcodes[0] = RISCV::VSLIDEDOWN_VI;
932 else if (Index < 0 && Index > -32)
933 Opcodes[1] = RISCV::VSLIDEUP_VI;
934 return LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
935 }
936 case TTI::SK_Reverse: {
937
938 if (!LT.second.isVector())
939 return InstructionCost::getInvalid();
940
941 // TODO: Cases to improve here:
942 // * Illegal vector types
943 // * i64 on RV32
944 if (SrcTy->getElementType()->isIntegerTy(BitWidth: 1)) {
945 VectorType *WideTy =
946 VectorType::get(ElementType: IntegerType::get(C&: SrcTy->getContext(), NumBits: 8),
947 EC: cast<VectorType>(Val: SrcTy)->getElementCount());
948 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: SrcTy,
949 CCH: TTI::CastContextHint::None, CostKind) +
950 getShuffleCost(Kind: TTI::SK_Reverse, DstTy: WideTy, SrcTy: WideTy, Mask: {}, CostKind, Index: 0,
951 SubTp: nullptr) +
952 getCastInstrCost(Opcode: Instruction::Trunc, Dst: SrcTy, Src: WideTy,
953 CCH: TTI::CastContextHint::None, CostKind);
954 }
955
956 MVT ContainerVT = LT.second;
957 if (LT.second.isFixedLengthVector())
958 ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
959 MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
960 if (ContainerVT.bitsLE(VT: M1VT)) {
961 // Example sequence:
962 // csrr a0, vlenb
963 // srli a0, a0, 3
964 // addi a0, a0, -1
965 // vsetvli a1, zero, e8, mf8, ta, mu (ignored)
966 // vid.v v9
967 // vrsub.vx v10, v9, a0
968 // vrgather.vv v9, v8, v10
969 InstructionCost LenCost = 3;
970 if (LT.second.isFixedLengthVector())
971 // vrsub.vi has a 5 bit immediate field, otherwise an li suffices
972 LenCost = isInt<5>(x: LT.second.getVectorNumElements() - 1) ? 0 : 1;
973 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX, RISCV::VRGATHER_VV};
974 if (LT.second.isFixedLengthVector() &&
975 isInt<5>(x: LT.second.getVectorNumElements() - 1))
976 Opcodes[1] = RISCV::VRSUB_VI;
977 InstructionCost GatherCost =
978 getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
979 return LT.first * (LenCost + GatherCost);
980 }
981
982 // At high LMUL, we split into a series of M1 reverses (see
983 // lowerVECTOR_REVERSE) and then do a single slide at the end to eliminate
984 // the resulting gap at the bottom (for fixed vectors only). The important
985 // bit is that the cost scales linearly, not quadratically with LMUL.
986 unsigned M1Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX};
987 InstructionCost FixedCost =
988 getRISCVInstructionCost(OpCodes: M1Opcodes, VT: M1VT, CostKind) + 3;
989 unsigned Ratio =
990 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
991 InstructionCost GatherCost =
992 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV}, VT: M1VT, CostKind) * Ratio;
993 InstructionCost SlideCost = !LT.second.isFixedLengthVector() ? 0 :
994 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX}, VT: LT.second, CostKind);
995 return FixedCost + LT.first * (GatherCost + SlideCost);
996 }
997 }
998 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index,
999 SubTp);
1000}
1001
1002static unsigned isM1OrSmaller(MVT VT) {
1003 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1004 return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
1005 LMUL == RISCVVType::VLMUL::LMUL_F4 ||
1006 LMUL == RISCVVType::VLMUL::LMUL_F2 ||
1007 LMUL == RISCVVType::VLMUL::LMUL_1);
1008}
1009
1010InstructionCost RISCVTTIImpl::getScalarizationOverhead(
1011 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
1012 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
1013 TTI::VectorInstrContext VIC) const {
1014 if (isa<ScalableVectorType>(Val: Ty))
1015 return InstructionCost::getInvalid();
1016
1017 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
1018 // For now, skip all fixed vector cost analysis when P extension is available
1019 // to avoid crashes in getMinRVVVectorSizeInBits()
1020 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: Ty)) {
1021 return 1; // Treat as single instruction cost for now
1022 }
1023
1024 // A build_vector (which is m1 sized or smaller) can be done in no
1025 // worse than one vslide1down.vx per element in the type. We could
1026 // in theory do an explode_vector in the inverse manner, but our
1027 // lowering today does not have a first class node for this pattern.
1028 InstructionCost Cost = BaseT::getScalarizationOverhead(
1029 InTy: Ty, DemandedElts, Insert, Extract, CostKind);
1030 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1031 if (Insert && !Extract && LT.first.isValid() && LT.second.isVector()) {
1032 if (Ty->getScalarSizeInBits() == 1) {
1033 auto *WideVecTy = cast<VectorType>(Val: Ty->getWithNewBitWidth(NewBitWidth: 8));
1034 // Note: Implicit scalar anyextend is assumed to be free since the i1
1035 // must be stored in a GPR.
1036 return getScalarizationOverhead(Ty: WideVecTy, DemandedElts, Insert, Extract,
1037 CostKind) +
1038 getCastInstrCost(Opcode: Instruction::Trunc, Dst: Ty, Src: WideVecTy,
1039 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
1040 }
1041
1042 assert(LT.second.isFixedLengthVector());
1043 MVT ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1044 if (isM1OrSmaller(VT: ContainerVT)) {
1045 InstructionCost BV =
1046 cast<FixedVectorType>(Val: Ty)->getNumElements() *
1047 getRISCVInstructionCost(OpCodes: RISCV::VSLIDE1DOWN_VX, VT: LT.second, CostKind);
1048 if (BV < Cost)
1049 Cost = BV;
1050 }
1051 }
1052 return Cost;
1053}
1054
1055InstructionCost
1056RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1057 TTI::TargetCostKind CostKind) const {
1058 Type *DataTy = MICA.getDataType();
1059 Align Alignment = MICA.getAlignment();
1060 switch (MICA.getID()) {
1061 case Intrinsic::vp_load_ff: {
1062 EVT DataTypeVT = TLI->getValueType(DL, Ty: DataTy);
1063 if (!TLI->isLegalFirstFaultLoad(DataType: DataTypeVT, Alignment))
1064 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1065
1066 unsigned AS = MICA.getAddressSpace();
1067 return getMemoryOpCost(Opcode: Instruction::Load, Src: DataTy, Alignment, AddressSpace: AS, CostKind,
1068 OpdInfo: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I: nullptr);
1069 }
1070 case Intrinsic::experimental_vp_strided_load:
1071 case Intrinsic::experimental_vp_strided_store:
1072 return getStridedMemoryOpCost(MICA, CostKind);
1073 case Intrinsic::masked_compressstore:
1074 case Intrinsic::masked_expandload:
1075 return getExpandCompressMemoryOpCost(MICA, CostKind);
1076 case Intrinsic::vp_scatter:
1077 case Intrinsic::vp_gather:
1078 case Intrinsic::masked_scatter:
1079 case Intrinsic::masked_gather:
1080 return getGatherScatterOpCost(MICA, CostKind);
1081 case Intrinsic::vp_load:
1082 case Intrinsic::vp_store:
1083 case Intrinsic::masked_load:
1084 case Intrinsic::masked_store:
1085 return getMaskedMemoryOpCost(MICA, CostKind);
1086 }
1087 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1088}
1089
1090InstructionCost
1091RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1092 TTI::TargetCostKind CostKind) const {
1093 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
1094 : Instruction::Store;
1095 Type *Src = MICA.getDataType();
1096 Align Alignment = MICA.getAlignment();
1097 unsigned AddressSpace = MICA.getAddressSpace();
1098
1099 if (!isLegalMaskedLoadStore(DataType: Src, Alignment) ||
1100 CostKind != TTI::TCK_RecipThroughput)
1101 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1102
1103 return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
1104}
1105
1106InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
1107 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1108 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1109 bool UseMaskForCond, bool UseMaskForGaps) const {
1110
1111 // The interleaved memory access pass will lower (de)interleave ops combined
1112 // with an adjacent appropriate memory to vlseg/vsseg intrinsics. vlseg/vsseg
1113 // only support masking per-iteration (i.e. condition), not per-segment (i.e.
1114 // gap).
1115 if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {
1116 auto *VTy = cast<VectorType>(Val: VecTy);
1117 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VTy);
1118 // Need to make sure type has't been scalarized
1119 if (LT.second.isVector()) {
1120 if (CostKind == TTI::TCK_CodeSize)
1121 return LT.first * TTI::TCC_Basic;
1122
1123 auto *SubVecTy =
1124 VectorType::get(ElementType: VTy->getElementType(),
1125 EC: VTy->getElementCount().divideCoefficientBy(RHS: Factor));
1126 if (VTy->getElementCount().isKnownMultipleOf(RHS: Factor) &&
1127 TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor, Alignment,
1128 AddrSpace: AddressSpace, DL)) {
1129
1130 // Some processors optimize segment loads/stores as one wide memory op +
1131 // Factor * LMUL shuffle ops.
1132 if (ST->hasOptimizedSegmentLoadStore(NF: Factor)) {
1133 InstructionCost Cost =
1134 getMemoryOpCost(Opcode, Src: VTy, Alignment, AddressSpace, CostKind);
1135 MVT SubVecVT = getTLI()->getValueType(DL, Ty: SubVecTy).getSimpleVT();
1136 Cost += Factor * TLI->getLMULCost(VT: SubVecVT);
1137 return LT.first * Cost;
1138 }
1139
1140 // Otherwise, the cost is proportional to the number of elements (VL *
1141 // Factor ops).
1142 unsigned NumLoads = getEstimatedVLFor(Ty: VTy);
1143 return NumLoads * TTI::TCC_Basic;
1144 }
1145 }
1146 }
1147
1148 // TODO: Return the cost of interleaved accesses for scalable vector when
1149 // unable to convert to segment accesses instructions.
1150 if (isa<ScalableVectorType>(Val: VecTy))
1151 return InstructionCost::getInvalid();
1152
1153 auto *FVTy = cast<FixedVectorType>(Val: VecTy);
1154 // When gaps are only at the tail, for interleaved load, we can emit a wide
1155 // masked load and shufflevectors. For interleaved store, we can emit
1156 // shufflevectors and a wide masked store. The interleaved memory access pass
1157 // will lower them into vlsseg/vssseg intrinsics.
1158 if (UseMaskForGaps) {
1159 assert(llvm::is_sorted(Indices) && "Indices must be sorted");
1160 assert(llvm::adjacent_find(Indices) == Indices.end() &&
1161 "Indices should not contain duplicate elements");
1162 unsigned NumOfFields = Indices.size();
1163 bool IsTailGapOnly = NumOfFields > 1 && (NumOfFields == Indices.back() + 1);
1164 if (IsTailGapOnly &&
1165 NumOfFields <= TLI->getMaxSupportedInterleaveFactor()) {
1166 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: FVTy);
1167 if (LT.second.isVector() &&
1168 FVTy->getElementCount().isKnownMultipleOf(RHS: Factor)) {
1169 auto *SubVecTy = VectorType::get(
1170 ElementType: FVTy->getElementType(),
1171 EC: FVTy->getElementCount().divideCoefficientBy(RHS: Factor));
1172 if (TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor: NumOfFields, Alignment,
1173 AddrSpace: AddressSpace, DL)) {
1174 // The cost is proportional to the total number of element accesses.
1175 unsigned NumAccesses = getEstimatedVLFor(Ty: FVTy);
1176 return NumAccesses * TTI::TCC_Basic;
1177 }
1178 }
1179 }
1180 }
1181
1182 InstructionCost MemCost =
1183 getMemoryOpCost(Opcode, Src: VecTy, Alignment, AddressSpace, CostKind);
1184 unsigned VF = FVTy->getNumElements() / Factor;
1185
1186 // An interleaved load will look like this for Factor=3:
1187 // %wide.vec = load <12 x i32>, ptr %3, align 4
1188 // %strided.vec = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1189 // %strided.vec1 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1190 // %strided.vec2 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1191 if (Opcode == Instruction::Load) {
1192 InstructionCost Cost = MemCost;
1193 for (unsigned Index : Indices) {
1194 FixedVectorType *VecTy =
1195 FixedVectorType::get(ElementType: FVTy->getElementType(), NumElts: VF * Factor);
1196 auto Mask = createStrideMask(Start: Index, Stride: Factor, VF);
1197 Mask.resize(N: VF * Factor, NV: -1);
1198 InstructionCost ShuffleCost =
1199 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: VecTy, SrcTy: VecTy,
1200 Mask, CostKind, Index: 0, SubTp: nullptr, Args: {});
1201 Cost += ShuffleCost;
1202 }
1203 return Cost;
1204 }
1205
1206 // TODO: Model for NF > 2
1207 // We'll need to enhance getShuffleCost to model shuffles that are just
1208 // inserts and extracts into subvectors, since they won't have the full cost
1209 // of a vrgather.
1210 // An interleaved store for 3 vectors of 4 lanes will look like
1211 // %11 = shufflevector <4 x i32> %4, <4 x i32> %6, <8 x i32> <0...7>
1212 // %12 = shufflevector <4 x i32> %9, <4 x i32> poison, <8 x i32> <0...3>
1213 // %13 = shufflevector <8 x i32> %11, <8 x i32> %12, <12 x i32> <0...11>
1214 // %interleaved.vec = shufflevector %13, poison, <12 x i32> <interleave mask>
1215 // store <12 x i32> %interleaved.vec, ptr %10, align 4
1216 if (Factor != 2)
1217 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1218 Alignment, AddressSpace, CostKind,
1219 UseMaskForCond, UseMaskForGaps);
1220
1221 assert(Opcode == Instruction::Store && "Opcode must be a store");
1222 // For an interleaving store of 2 vectors, we perform one large interleaving
1223 // shuffle that goes into the wide store
1224 auto Mask = createInterleaveMask(VF, NumVecs: Factor);
1225 InstructionCost ShuffleCost =
1226 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: FVTy, SrcTy: FVTy, Mask,
1227 CostKind, Index: 0, SubTp: nullptr, Args: {});
1228 return MemCost + ShuffleCost;
1229}
1230
1231InstructionCost
1232RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1233 TTI::TargetCostKind CostKind) const {
1234
1235 bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
1236 MICA.getID() == Intrinsic::vp_gather;
1237 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
1238 Type *DataTy = MICA.getDataType();
1239 Align Alignment = MICA.getAlignment();
1240 if (CostKind != TTI::TCK_RecipThroughput)
1241 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1242
1243 if ((Opcode == Instruction::Load &&
1244 !isLegalMaskedGather(DataType: DataTy, Alignment: Align(Alignment))) ||
1245 (Opcode == Instruction::Store &&
1246 !isLegalMaskedScatter(DataType: DataTy, Alignment: Align(Alignment))))
1247 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1248
1249 // Cost is proportional to the number of memory operations implied. For
1250 // scalable vectors, we use an estimate on that number since we don't
1251 // know exactly what VL will be.
1252 auto &VTy = *cast<VectorType>(Val: DataTy);
1253 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1254 return NumLoads * TTI::TCC_Basic;
1255}
1256
1257InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
1258 const MemIntrinsicCostAttributes &MICA,
1259 TTI::TargetCostKind CostKind) const {
1260 unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
1261 ? Instruction::Load
1262 : Instruction::Store;
1263 Type *DataTy = MICA.getDataType();
1264 bool VariableMask = MICA.getVariableMask();
1265 Align Alignment = MICA.getAlignment();
1266 bool IsLegal = (Opcode == Instruction::Store &&
1267 isLegalMaskedCompressStore(DataTy, Alignment)) ||
1268 (Opcode == Instruction::Load &&
1269 isLegalMaskedExpandLoad(DataType: DataTy, Alignment));
1270 if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
1271 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1272 // Example compressstore sequence:
1273 // vsetivli zero, 8, e32, m2, ta, ma (ignored)
1274 // vcompress.vm v10, v8, v0
1275 // vcpop.m a1, v0
1276 // vsetvli zero, a1, e32, m2, ta, ma
1277 // vse32.v v10, (a0)
1278 // Example expandload sequence:
1279 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
1280 // vcpop.m a1, v0
1281 // vsetvli zero, a1, e32, m2, ta, ma
1282 // vle32.v v10, (a0)
1283 // vsetivli zero, 8, e32, m2, ta, ma
1284 // viota.m v12, v0
1285 // vrgather.vv v8, v10, v12, v0.t
1286 auto MemOpCost =
1287 getMemoryOpCost(Opcode, Src: DataTy, Alignment, /*AddressSpace*/ 0, CostKind);
1288 auto LT = getTypeLegalizationCost(Ty: DataTy);
1289 SmallVector<unsigned, 4> Opcodes{RISCV::VSETVLI};
1290 if (VariableMask)
1291 Opcodes.push_back(Elt: RISCV::VCPOP_M);
1292 if (Opcode == Instruction::Store)
1293 Opcodes.append(IL: {RISCV::VCOMPRESS_VM});
1294 else
1295 Opcodes.append(IL: {RISCV::VSETIVLI, RISCV::VIOTA_M, RISCV::VRGATHER_VV});
1296 return MemOpCost +
1297 LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1298}
1299
1300InstructionCost
1301RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1302 TTI::TargetCostKind CostKind) const {
1303 Type *DataTy = MICA.getDataType();
1304 Align Alignment = MICA.getAlignment();
1305
1306 if (!isLegalStridedLoadStore(DataType: DataTy, Alignment))
1307 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1308
1309 if (CostKind == TTI::TCK_CodeSize)
1310 return TTI::TCC_Basic;
1311
1312 // Cost is proportional to the number of memory operations implied. For
1313 // scalable vectors, we use an estimate on that number since we don't
1314 // know exactly what VL will be.
1315 auto &VTy = *cast<VectorType>(Val: DataTy);
1316 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1317 return NumLoads * TTI::TCC_Basic;
1318}
1319
1320InstructionCost
1321RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1322 // FIXME: This is a property of the default vector convention, not
1323 // all possible calling conventions. Fixing that will require
1324 // some TTI API and SLP rework.
1325 InstructionCost Cost = 0;
1326 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
1327 for (auto *Ty : Tys) {
1328 if (!Ty->isVectorTy())
1329 continue;
1330 Align A = DL.getPrefTypeAlign(Ty);
1331 Cost += getMemoryOpCost(Opcode: Instruction::Store, Src: Ty, Alignment: A, AddressSpace: 0, CostKind) +
1332 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: A, AddressSpace: 0, CostKind);
1333 }
1334 return Cost;
1335}
1336
1337// Currently, these represent both throughput and codesize costs
1338// for the respective intrinsics. The costs in this table are simply
1339// instruction counts with the following adjustments made:
1340// * One vsetvli is considered free.
1341static const CostTblEntry VectorIntrinsicCostTable[]{
1342 {.ISD: Intrinsic::floor, .Type: MVT::f32, .Cost: 9},
1343 {.ISD: Intrinsic::floor, .Type: MVT::f64, .Cost: 9},
1344 {.ISD: Intrinsic::ceil, .Type: MVT::f32, .Cost: 9},
1345 {.ISD: Intrinsic::ceil, .Type: MVT::f64, .Cost: 9},
1346 {.ISD: Intrinsic::trunc, .Type: MVT::f32, .Cost: 7},
1347 {.ISD: Intrinsic::trunc, .Type: MVT::f64, .Cost: 7},
1348 {.ISD: Intrinsic::round, .Type: MVT::f32, .Cost: 9},
1349 {.ISD: Intrinsic::round, .Type: MVT::f64, .Cost: 9},
1350 {.ISD: Intrinsic::roundeven, .Type: MVT::f32, .Cost: 9},
1351 {.ISD: Intrinsic::roundeven, .Type: MVT::f64, .Cost: 9},
1352 {.ISD: Intrinsic::rint, .Type: MVT::f32, .Cost: 7},
1353 {.ISD: Intrinsic::rint, .Type: MVT::f64, .Cost: 7},
1354 {.ISD: Intrinsic::nearbyint, .Type: MVT::f32, .Cost: 9},
1355 {.ISD: Intrinsic::nearbyint, .Type: MVT::f64, .Cost: 9},
1356 {.ISD: Intrinsic::bswap, .Type: MVT::i16, .Cost: 3},
1357 {.ISD: Intrinsic::bswap, .Type: MVT::i32, .Cost: 12},
1358 {.ISD: Intrinsic::bswap, .Type: MVT::i64, .Cost: 31},
1359 {.ISD: Intrinsic::vp_bswap, .Type: MVT::i16, .Cost: 3},
1360 {.ISD: Intrinsic::vp_bswap, .Type: MVT::i32, .Cost: 12},
1361 {.ISD: Intrinsic::vp_bswap, .Type: MVT::i64, .Cost: 31},
1362 {.ISD: Intrinsic::vp_fshl, .Type: MVT::i8, .Cost: 7},
1363 {.ISD: Intrinsic::vp_fshl, .Type: MVT::i16, .Cost: 7},
1364 {.ISD: Intrinsic::vp_fshl, .Type: MVT::i32, .Cost: 7},
1365 {.ISD: Intrinsic::vp_fshl, .Type: MVT::i64, .Cost: 7},
1366 {.ISD: Intrinsic::vp_fshr, .Type: MVT::i8, .Cost: 7},
1367 {.ISD: Intrinsic::vp_fshr, .Type: MVT::i16, .Cost: 7},
1368 {.ISD: Intrinsic::vp_fshr, .Type: MVT::i32, .Cost: 7},
1369 {.ISD: Intrinsic::vp_fshr, .Type: MVT::i64, .Cost: 7},
1370 {.ISD: Intrinsic::bitreverse, .Type: MVT::i8, .Cost: 17},
1371 {.ISD: Intrinsic::bitreverse, .Type: MVT::i16, .Cost: 24},
1372 {.ISD: Intrinsic::bitreverse, .Type: MVT::i32, .Cost: 33},
1373 {.ISD: Intrinsic::bitreverse, .Type: MVT::i64, .Cost: 52},
1374 {.ISD: Intrinsic::vp_bitreverse, .Type: MVT::i8, .Cost: 17},
1375 {.ISD: Intrinsic::vp_bitreverse, .Type: MVT::i16, .Cost: 24},
1376 {.ISD: Intrinsic::vp_bitreverse, .Type: MVT::i32, .Cost: 33},
1377 {.ISD: Intrinsic::vp_bitreverse, .Type: MVT::i64, .Cost: 52},
1378 {.ISD: Intrinsic::ctpop, .Type: MVT::i8, .Cost: 12},
1379 {.ISD: Intrinsic::ctpop, .Type: MVT::i16, .Cost: 19},
1380 {.ISD: Intrinsic::ctpop, .Type: MVT::i32, .Cost: 20},
1381 {.ISD: Intrinsic::ctpop, .Type: MVT::i64, .Cost: 21},
1382 {.ISD: Intrinsic::ctlz, .Type: MVT::i8, .Cost: 19},
1383 {.ISD: Intrinsic::ctlz, .Type: MVT::i16, .Cost: 28},
1384 {.ISD: Intrinsic::ctlz, .Type: MVT::i32, .Cost: 31},
1385 {.ISD: Intrinsic::ctlz, .Type: MVT::i64, .Cost: 35},
1386 {.ISD: Intrinsic::cttz, .Type: MVT::i8, .Cost: 16},
1387 {.ISD: Intrinsic::cttz, .Type: MVT::i16, .Cost: 23},
1388 {.ISD: Intrinsic::cttz, .Type: MVT::i32, .Cost: 24},
1389 {.ISD: Intrinsic::cttz, .Type: MVT::i64, .Cost: 25},
1390 {.ISD: Intrinsic::vp_ctpop, .Type: MVT::i8, .Cost: 12},
1391 {.ISD: Intrinsic::vp_ctpop, .Type: MVT::i16, .Cost: 19},
1392 {.ISD: Intrinsic::vp_ctpop, .Type: MVT::i32, .Cost: 20},
1393 {.ISD: Intrinsic::vp_ctpop, .Type: MVT::i64, .Cost: 21},
1394 {.ISD: Intrinsic::vp_ctlz, .Type: MVT::i8, .Cost: 19},
1395 {.ISD: Intrinsic::vp_ctlz, .Type: MVT::i16, .Cost: 28},
1396 {.ISD: Intrinsic::vp_ctlz, .Type: MVT::i32, .Cost: 31},
1397 {.ISD: Intrinsic::vp_ctlz, .Type: MVT::i64, .Cost: 35},
1398 {.ISD: Intrinsic::vp_cttz, .Type: MVT::i8, .Cost: 16},
1399 {.ISD: Intrinsic::vp_cttz, .Type: MVT::i16, .Cost: 23},
1400 {.ISD: Intrinsic::vp_cttz, .Type: MVT::i32, .Cost: 24},
1401 {.ISD: Intrinsic::vp_cttz, .Type: MVT::i64, .Cost: 25},
1402};
1403
1404InstructionCost
1405RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1406 TTI::TargetCostKind CostKind) const {
1407 auto *RetTy = ICA.getReturnType();
1408 switch (ICA.getID()) {
1409 case Intrinsic::lrint:
1410 case Intrinsic::llrint:
1411 case Intrinsic::lround:
1412 case Intrinsic::llround: {
1413 auto LT = getTypeLegalizationCost(Ty: RetTy);
1414 Type *SrcTy = ICA.getArgTypes().front();
1415 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
1416 if (ST->hasVInstructions() && LT.second.isVector()) {
1417 SmallVector<unsigned, 2> Ops;
1418 unsigned SrcEltSz = DL.getTypeSizeInBits(Ty: SrcTy->getScalarType());
1419 unsigned DstEltSz = DL.getTypeSizeInBits(Ty: RetTy->getScalarType());
1420 if (LT.second.getVectorElementType() == MVT::bf16) {
1421 if (!ST->hasVInstructionsBF16Minimal())
1422 return InstructionCost::getInvalid();
1423 if (DstEltSz == 32)
1424 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFCVT_X_F_V};
1425 else
1426 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVT_X_F_V};
1427 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1428 !ST->hasVInstructionsF16()) {
1429 if (!ST->hasVInstructionsF16Minimal())
1430 return InstructionCost::getInvalid();
1431 if (DstEltSz == 32)
1432 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFCVT_X_F_V};
1433 else
1434 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_X_F_V};
1435
1436 } else if (SrcEltSz > DstEltSz) {
1437 Ops = {RISCV::VFNCVT_X_F_W};
1438 } else if (SrcEltSz < DstEltSz) {
1439 Ops = {RISCV::VFWCVT_X_F_V};
1440 } else {
1441 Ops = {RISCV::VFCVT_X_F_V};
1442 }
1443
1444 // We need to use the source LMUL in the case of a narrowing op, and the
1445 // destination LMUL otherwise.
1446 if (SrcEltSz > DstEltSz)
1447 return SrcLT.first *
1448 getRISCVInstructionCost(OpCodes: Ops, VT: SrcLT.second, CostKind);
1449 return LT.first * getRISCVInstructionCost(OpCodes: Ops, VT: LT.second, CostKind);
1450 }
1451 break;
1452 }
1453 case Intrinsic::ceil:
1454 case Intrinsic::floor:
1455 case Intrinsic::trunc:
1456 case Intrinsic::rint:
1457 case Intrinsic::round:
1458 case Intrinsic::roundeven: {
1459 // These all use the same code.
1460 auto LT = getTypeLegalizationCost(Ty: RetTy);
1461 if (!LT.second.isVector() && TLI->isOperationCustom(Op: ISD::FCEIL, VT: LT.second))
1462 return LT.first * 8;
1463 break;
1464 }
1465 case Intrinsic::umin:
1466 case Intrinsic::umax:
1467 case Intrinsic::smin:
1468 case Intrinsic::smax: {
1469 auto LT = getTypeLegalizationCost(Ty: RetTy);
1470 if (LT.second.isScalarInteger() && ST->hasStdExtZbb())
1471 return LT.first;
1472
1473 if (ST->hasVInstructions() && LT.second.isVector()) {
1474 unsigned Op;
1475 switch (ICA.getID()) {
1476 case Intrinsic::umin:
1477 Op = RISCV::VMINU_VV;
1478 break;
1479 case Intrinsic::umax:
1480 Op = RISCV::VMAXU_VV;
1481 break;
1482 case Intrinsic::smin:
1483 Op = RISCV::VMIN_VV;
1484 break;
1485 case Intrinsic::smax:
1486 Op = RISCV::VMAX_VV;
1487 break;
1488 }
1489 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1490 }
1491 break;
1492 }
1493 case Intrinsic::sadd_sat:
1494 case Intrinsic::ssub_sat:
1495 case Intrinsic::uadd_sat:
1496 case Intrinsic::usub_sat: {
1497 auto LT = getTypeLegalizationCost(Ty: RetTy);
1498 if (ST->hasVInstructions() && LT.second.isVector()) {
1499 unsigned Op;
1500 switch (ICA.getID()) {
1501 case Intrinsic::sadd_sat:
1502 Op = RISCV::VSADD_VV;
1503 break;
1504 case Intrinsic::ssub_sat:
1505 Op = RISCV::VSSUB_VV;
1506 break;
1507 case Intrinsic::uadd_sat:
1508 Op = RISCV::VSADDU_VV;
1509 break;
1510 case Intrinsic::usub_sat:
1511 Op = RISCV::VSSUBU_VV;
1512 break;
1513 }
1514 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1515 }
1516 break;
1517 }
1518 case Intrinsic::fma:
1519 case Intrinsic::fmuladd: {
1520 // TODO: handle promotion with f16/bf16 with zvfhmin/zvfbfmin
1521 auto LT = getTypeLegalizationCost(Ty: RetTy);
1522 if (ST->hasVInstructions() && LT.second.isVector())
1523 return LT.first *
1524 getRISCVInstructionCost(OpCodes: RISCV::VFMADD_VV, VT: LT.second, CostKind);
1525 break;
1526 }
1527 case Intrinsic::fabs: {
1528 auto LT = getTypeLegalizationCost(Ty: RetTy);
1529 if (ST->hasVInstructions() && LT.second.isVector()) {
1530 // lui a0, 8
1531 // addi a0, a0, -1
1532 // vsetvli a1, zero, e16, m1, ta, ma
1533 // vand.vx v8, v8, a0
1534 // f16 with zvfhmin and bf16 with zvfhbmin
1535 if (LT.second.getVectorElementType() == MVT::bf16 ||
1536 (LT.second.getVectorElementType() == MVT::f16 &&
1537 !ST->hasVInstructionsF16()))
1538 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VAND_VX, VT: LT.second,
1539 CostKind) +
1540 2;
1541 else
1542 return LT.first *
1543 getRISCVInstructionCost(OpCodes: RISCV::VFSGNJX_VV, VT: LT.second, CostKind);
1544 }
1545 break;
1546 }
1547 case Intrinsic::sqrt: {
1548 auto LT = getTypeLegalizationCost(Ty: RetTy);
1549 if (ST->hasVInstructions() && LT.second.isVector()) {
1550 SmallVector<unsigned, 4> ConvOp;
1551 SmallVector<unsigned, 2> FsqrtOp;
1552 MVT ConvType = LT.second;
1553 MVT FsqrtType = LT.second;
1554 // f16 with zvfhmin and bf16 with zvfbfmin and the type of nxv32[b]f16
1555 // will be spilt.
1556 if (LT.second.getVectorElementType() == MVT::bf16) {
1557 if (LT.second == MVT::nxv32bf16) {
1558 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVTBF16_F_F_V,
1559 RISCV::VFNCVTBF16_F_F_W, RISCV::VFNCVTBF16_F_F_W};
1560 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1561 ConvType = MVT::nxv16f16;
1562 FsqrtType = MVT::nxv16f32;
1563 } else {
1564 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFNCVTBF16_F_F_W};
1565 FsqrtOp = {RISCV::VFSQRT_V};
1566 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1567 }
1568 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1569 !ST->hasVInstructionsF16()) {
1570 if (LT.second == MVT::nxv32f16) {
1571 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_F_F_V,
1572 RISCV::VFNCVT_F_F_W, RISCV::VFNCVT_F_F_W};
1573 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1574 ConvType = MVT::nxv16f16;
1575 FsqrtType = MVT::nxv16f32;
1576 } else {
1577 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFNCVT_F_F_W};
1578 FsqrtOp = {RISCV::VFSQRT_V};
1579 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1580 }
1581 } else {
1582 FsqrtOp = {RISCV::VFSQRT_V};
1583 }
1584
1585 return LT.first * (getRISCVInstructionCost(OpCodes: FsqrtOp, VT: FsqrtType, CostKind) +
1586 getRISCVInstructionCost(OpCodes: ConvOp, VT: ConvType, CostKind));
1587 }
1588 break;
1589 }
1590 case Intrinsic::cttz:
1591 case Intrinsic::ctlz:
1592 case Intrinsic::ctpop: {
1593 auto LT = getTypeLegalizationCost(Ty: RetTy);
1594 if (ST->hasStdExtZvbb() && LT.second.isVector()) {
1595 unsigned Op;
1596 switch (ICA.getID()) {
1597 case Intrinsic::cttz:
1598 Op = RISCV::VCTZ_V;
1599 break;
1600 case Intrinsic::ctlz:
1601 Op = RISCV::VCLZ_V;
1602 break;
1603 case Intrinsic::ctpop:
1604 Op = RISCV::VCPOP_V;
1605 break;
1606 }
1607 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1608 }
1609 break;
1610 }
1611 case Intrinsic::abs: {
1612 auto LT = getTypeLegalizationCost(Ty: RetTy);
1613 if (ST->hasVInstructions() && LT.second.isVector()) {
1614 // vabs.v v10, v8
1615 if (ST->hasStdExtZvabd())
1616 return LT.first *
1617 getRISCVInstructionCost(OpCodes: {RISCV::VABS_V}, VT: LT.second, CostKind);
1618
1619 // vrsub.vi v10, v8, 0
1620 // vmax.vv v8, v8, v10
1621 return LT.first *
1622 getRISCVInstructionCost(OpCodes: {RISCV::VRSUB_VI, RISCV::VMAX_VV},
1623 VT: LT.second, CostKind);
1624 }
1625 break;
1626 }
1627 case Intrinsic::fshl:
1628 case Intrinsic::fshr: {
1629 if (ICA.getArgs().empty())
1630 break;
1631
1632 // Funnel-shifts are ROTL/ROTR when the first and second operand are equal.
1633 // When Zbb/Zbkb is enabled we can use a single ROL(W)/ROR(I)(W)
1634 // instruction.
1635 if ((ST->hasStdExtZbb() || ST->hasStdExtZbkb()) && RetTy->isIntegerTy() &&
1636 ICA.getArgs()[0] == ICA.getArgs()[1] &&
1637 (RetTy->getIntegerBitWidth() == 32 ||
1638 RetTy->getIntegerBitWidth() == 64) &&
1639 RetTy->getIntegerBitWidth() <= ST->getXLen()) {
1640 return 1;
1641 }
1642 break;
1643 }
1644 case Intrinsic::masked_udiv:
1645 return getArithmeticInstrCost(Opcode: Instruction::UDiv, Ty: ICA.getReturnType(),
1646 CostKind);
1647 case Intrinsic::masked_sdiv:
1648 return getArithmeticInstrCost(Opcode: Instruction::SDiv, Ty: ICA.getReturnType(),
1649 CostKind);
1650 case Intrinsic::masked_urem:
1651 return getArithmeticInstrCost(Opcode: Instruction::URem, Ty: ICA.getReturnType(),
1652 CostKind);
1653 case Intrinsic::masked_srem:
1654 return getArithmeticInstrCost(Opcode: Instruction::SRem, Ty: ICA.getReturnType(),
1655 CostKind);
1656 case Intrinsic::get_active_lane_mask: {
1657 if (ST->hasVInstructions()) {
1658 Type *ExpRetTy = VectorType::get(
1659 ElementType: ICA.getArgTypes()[0], EC: cast<VectorType>(Val: RetTy)->getElementCount());
1660 auto LT = getTypeLegalizationCost(Ty: ExpRetTy);
1661
1662 // vid.v v8 // considered hoisted
1663 // vsaddu.vx v8, v8, a0
1664 // vmsltu.vx v0, v8, a1
1665 return LT.first *
1666 getRISCVInstructionCost(OpCodes: {RISCV::VSADDU_VX, RISCV::VMSLTU_VX},
1667 VT: LT.second, CostKind);
1668 }
1669 break;
1670 }
1671 // TODO: add more intrinsic
1672 case Intrinsic::stepvector: {
1673 auto LT = getTypeLegalizationCost(Ty: RetTy);
1674 // Legalisation of illegal types involves an `index' instruction plus
1675 // (LT.first - 1) vector adds.
1676 if (ST->hasVInstructions())
1677 return getRISCVInstructionCost(OpCodes: RISCV::VID_V, VT: LT.second, CostKind) +
1678 (LT.first - 1) *
1679 getRISCVInstructionCost(OpCodes: RISCV::VADD_VX, VT: LT.second, CostKind);
1680 return 1 + (LT.first - 1);
1681 }
1682 case Intrinsic::vector_splice_left:
1683 case Intrinsic::vector_splice_right: {
1684 auto LT = getTypeLegalizationCost(Ty: RetTy);
1685 // Constant offsets fall through to getShuffleCost.
1686 if (!ICA.isTypeBasedOnly() && isa<ConstantInt>(Val: ICA.getArgs()[2]))
1687 break;
1688 if (ST->hasVInstructions() && LT.second.isVector()) {
1689 return LT.first *
1690 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX},
1691 VT: LT.second, CostKind);
1692 }
1693 break;
1694 }
1695 case Intrinsic::experimental_cttz_elts: {
1696 Type *ArgTy = ICA.getArgTypes()[0];
1697 EVT ArgType = TLI->getValueType(DL, Ty: ArgTy, AllowUnknown: true);
1698 if (getTLI()->shouldExpandCttzElements(VT: ArgType))
1699 break;
1700 InstructionCost Cost = getRISCVInstructionCost(
1701 OpCodes: RISCV::VFIRST_M, VT: getTypeLegalizationCost(Ty: ArgTy).second, CostKind);
1702
1703 // If zero_is_poison is false, then we will generate additional
1704 // cmp + select instructions to convert -1 to EVL.
1705 Type *BoolTy = Type::getInt1Ty(C&: RetTy->getContext());
1706 if (ICA.getArgs().size() > 1 &&
1707 cast<ConstantInt>(Val: ICA.getArgs()[1])->isZero())
1708 Cost += getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: BoolTy, CondTy: RetTy,
1709 VecPred: CmpInst::ICMP_SLT, CostKind) +
1710 getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: RetTy, CondTy: BoolTy,
1711 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1712
1713 return Cost;
1714 }
1715 case Intrinsic::experimental_vp_splice: {
1716 // To support type-based query from vectorizer, set the index to 0.
1717 // Note that index only change the cost from vslide.vx to vslide.vi and in
1718 // current implementations they have same costs.
1719 return getShuffleCost(Kind: TTI::SK_Splice, DstTy: cast<VectorType>(Val: ICA.getReturnType()),
1720 SrcTy: cast<VectorType>(Val: ICA.getArgTypes()[0]), Mask: {}, CostKind,
1721 Index: 0, SubTp: cast<VectorType>(Val: ICA.getReturnType()));
1722 }
1723 case Intrinsic::vp_merge: {
1724 // If an operand is a binary op and the type is legal, RISCVVectorPeephole
1725 // will likely fold the resulting vmerge.vvm away.
1726 if (ICA.getVectorInstrContext() == VectorInstrContext::BinaryOp &&
1727 getTypeLegalizationCost(Ty: RetTy).first == 1)
1728 return TTI::TCC_Free;
1729 break;
1730 }
1731 case Intrinsic::fptoui_sat:
1732 case Intrinsic::fptosi_sat: {
1733 InstructionCost Cost = 0;
1734 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
1735 Type *SrcTy = ICA.getArgTypes()[0];
1736
1737 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
1738 auto DstLT = getTypeLegalizationCost(Ty: RetTy);
1739 if (!SrcTy->isVectorTy())
1740 break;
1741
1742 if (!SrcLT.first.isValid() || !DstLT.first.isValid())
1743 return InstructionCost::getInvalid();
1744
1745 Cost +=
1746 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
1747 Dst: RetTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
1748
1749 // Handle NaN.
1750 // vmfne v0, v8, v8 # If v8[i] is NaN set v0[i] to 1.
1751 // vmerge.vim v8, v8, 0, v0 # Convert NaN to 0.
1752 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
1753 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: SrcTy, CondTy,
1754 VecPred: CmpInst::FCMP_UNO, CostKind);
1755 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
1756 VecPred: CmpInst::FCMP_UNO, CostKind);
1757 return Cost;
1758 }
1759 case Intrinsic::experimental_vector_extract_last_active: {
1760 auto *ValTy = cast<VectorType>(Val: ICA.getArgTypes()[0]);
1761 auto *MaskTy = cast<VectorType>(Val: ICA.getArgTypes()[1]);
1762
1763 auto ValLT = getTypeLegalizationCost(Ty: ValTy);
1764 auto MaskLT = getTypeLegalizationCost(Ty: MaskTy);
1765
1766 // TODO: Return cheaper cost when the entire lane is inactive.
1767 // The expected asm sequence is:
1768 // vcpop.m a0, v0
1769 // beqz a0, exit # Return passthru when the entire lane is inactive.
1770 // vid v10, v0.t
1771 // vredmaxu.vs v10, v10, v10
1772 // vmv.x.s a0, v10
1773 // zext.b a0, a0
1774 // vslidedown.vx v8, v8, a0
1775 // vmv.x.s a0, v8
1776 // exit:
1777 // ...
1778
1779 // Find a suitable type for a stepvector.
1780 ConstantRange VScaleRange(APInt(64, 1), APInt::getZero(numBits: 64));
1781 unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
1782 RetVT: TLI->getVectorIdxTy(DL: getDataLayout()), EC: MaskTy->getElementCount(),
1783 /*ZeroIsPoison=*/true, VScaleRange: &VScaleRange);
1784 EltWidth = std::max(a: EltWidth, b: MaskTy->getScalarSizeInBits());
1785 Type *StepTy = Type::getIntNTy(C&: MaskTy->getContext(), N: EltWidth);
1786 auto *StepVecTy = VectorType::get(ElementType: StepTy, EC: ValTy->getElementCount());
1787 auto StepLT = getTypeLegalizationCost(Ty: StepVecTy);
1788
1789 // Currently expandVectorFindLastActive cannot handle step vector split.
1790 // So return invalid when the type needs split.
1791 // FIXME: Remove this if expandVectorFindLastActive supports split vector.
1792 if (StepLT.first > 1)
1793 return InstructionCost::getInvalid();
1794
1795 InstructionCost Cost = 0;
1796 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
1797
1798 Cost += MaskLT.first *
1799 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: MaskLT.second, CostKind);
1800 Cost += getCFInstrCost(Opcode: Instruction::CondBr, CostKind, I: nullptr);
1801 Cost += StepLT.first *
1802 getRISCVInstructionCost(OpCodes: Opcodes, VT: StepLT.second, CostKind);
1803 Cost += getCastInstrCost(Opcode: Instruction::ZExt,
1804 Dst: Type::getInt64Ty(C&: ValTy->getContext()), Src: StepTy,
1805 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
1806 Cost += ValLT.first *
1807 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VI, RISCV::VMV_X_S},
1808 VT: ValLT.second, CostKind);
1809 return Cost;
1810 }
1811 }
1812
1813 if (ST->hasVInstructions() && RetTy->isVectorTy()) {
1814 if (auto LT = getTypeLegalizationCost(Ty: RetTy);
1815 LT.second.isVector()) {
1816 MVT EltTy = LT.second.getVectorElementType();
1817 if (const auto *Entry = CostTableLookup(Table: VectorIntrinsicCostTable,
1818 ISD: ICA.getID(), Ty: EltTy))
1819 return LT.first * Entry->Cost;
1820 }
1821 }
1822
1823 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1824}
1825
1826InstructionCost
1827RISCVTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
1828 const SCEV *Ptr,
1829 TTI::TargetCostKind CostKind) const {
1830 // Address computations for vector indexed load/store likely require an offset
1831 // and/or scaling.
1832 if (ST->hasVInstructions() && PtrTy->isVectorTy())
1833 return getArithmeticInstrCost(Opcode: Instruction::Add, Ty: PtrTy, CostKind);
1834
1835 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
1836}
1837
1838InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
1839 Type *Src,
1840 TTI::CastContextHint CCH,
1841 TTI::TargetCostKind CostKind,
1842 const Instruction *I) const {
1843 bool IsVectorType = isa<VectorType>(Val: Dst) && isa<VectorType>(Val: Src);
1844 if (!IsVectorType)
1845 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1846
1847 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
1848 // For now, skip all fixed vector cost analysis when P extension is available
1849 // to avoid crashes in getMinRVVVectorSizeInBits()
1850 if (ST->hasStdExtP() &&
1851 (isa<FixedVectorType>(Val: Dst) || isa<FixedVectorType>(Val: Src))) {
1852 return 1; // Treat as single instruction cost for now
1853 }
1854
1855 // FIXME: Need to compute legalizing cost for illegal types. The current
1856 // code handles only legal types and those which can be trivially
1857 // promoted to legal.
1858 if (!ST->hasVInstructions() || Src->getScalarSizeInBits() > ST->getELen() ||
1859 Dst->getScalarSizeInBits() > ST->getELen())
1860 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1861
1862 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1863 assert(ISD && "Invalid opcode");
1864 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: Src);
1865 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: Dst);
1866
1867 // Handle i1 source and dest cases *before* calling logic in BasicTTI.
1868 // The shared implementation doesn't model vector widening during legalization
1869 // and instead assumes scalarization. In order to scalarize an <N x i1>
1870 // vector, we need to extend/trunc to/from i8. If we don't special case
1871 // this, we can get an infinite recursion cycle.
1872 switch (ISD) {
1873 default:
1874 break;
1875 case ISD::SIGN_EXTEND:
1876 case ISD::ZERO_EXTEND:
1877 if (Src->getScalarSizeInBits() == 1) {
1878 // We do not use vsext/vzext to extend from mask vector.
1879 // Instead we use the following instructions to extend from mask vector:
1880 // vmv.v.i v8, 0
1881 // vmerge.vim v8, v8, -1, v0 (repeated per split)
1882 return getRISCVInstructionCost(OpCodes: RISCV::VMV_V_I, VT: DstLT.second, CostKind) +
1883 DstLT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VIM,
1884 VT: DstLT.second, CostKind) +
1885 DstLT.first - 1;
1886 }
1887 break;
1888 case ISD::TRUNCATE:
1889 if (Dst->getScalarSizeInBits() == 1) {
1890 // We do not use several vncvt to truncate to mask vector. So we could
1891 // not use PowDiff to calculate it.
1892 // Instead we use the following instructions to truncate to mask vector:
1893 // vand.vi v8, v8, 1
1894 // vmsne.vi v0, v8, 0
1895 return SrcLT.first *
1896 getRISCVInstructionCost(OpCodes: {RISCV::VAND_VI, RISCV::VMSNE_VI},
1897 VT: SrcLT.second, CostKind) +
1898 SrcLT.first - 1;
1899 }
1900 break;
1901 };
1902
1903 // Our actual lowering for the case where a wider legal type is available
1904 // uses promotion to the wider type. This is reflected in the result of
1905 // getTypeLegalizationCost, but BasicTTI assumes the widened cases are
1906 // scalarized if the legalized Src and Dst are not equal sized.
1907 const DataLayout &DL = this->getDataLayout();
1908 if (!SrcLT.second.isVector() || !DstLT.second.isVector() ||
1909 !SrcLT.first.isValid() || !DstLT.first.isValid() ||
1910 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Src),
1911 RHS: SrcLT.second.getSizeInBits()) ||
1912 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Dst),
1913 RHS: DstLT.second.getSizeInBits()) ||
1914 SrcLT.first > 1 || DstLT.first > 1)
1915 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1916
1917 // The split cost is handled by the base getCastInstrCost
1918 assert((SrcLT.first == 1) && (DstLT.first == 1) && "Illegal type");
1919
1920 int PowDiff = (int)Log2_32(Value: DstLT.second.getScalarSizeInBits()) -
1921 (int)Log2_32(Value: SrcLT.second.getScalarSizeInBits());
1922 switch (ISD) {
1923 case ISD::SIGN_EXTEND:
1924 case ISD::ZERO_EXTEND: {
1925 if ((PowDiff < 1) || (PowDiff > 3))
1926 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
1927 unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
1928 unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
1929 unsigned Op =
1930 (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
1931 return getRISCVInstructionCost(OpCodes: Op, VT: DstLT.second, CostKind);
1932 }
1933 case ISD::TRUNCATE:
1934 case ISD::FP_EXTEND:
1935 case ISD::FP_ROUND: {
1936 // Counts of narrow/widen instructions.
1937 unsigned SrcEltSize = SrcLT.second.getScalarSizeInBits();
1938 unsigned DstEltSize = DstLT.second.getScalarSizeInBits();
1939
1940 unsigned Op = (ISD == ISD::TRUNCATE) ? RISCV::VNSRL_WI
1941 : (ISD == ISD::FP_EXTEND) ? RISCV::VFWCVT_F_F_V
1942 : RISCV::VFNCVT_F_F_W;
1943 InstructionCost Cost = 0;
1944 for (; SrcEltSize != DstEltSize;) {
1945 MVT ElementMVT = (ISD == ISD::TRUNCATE)
1946 ? MVT::getIntegerVT(BitWidth: DstEltSize)
1947 : MVT::getFloatingPointVT(BitWidth: DstEltSize);
1948 MVT DstMVT = DstLT.second.changeVectorElementType(EltVT: ElementMVT);
1949 DstEltSize =
1950 (DstEltSize > SrcEltSize) ? DstEltSize >> 1 : DstEltSize << 1;
1951 Cost += getRISCVInstructionCost(OpCodes: Op, VT: DstMVT, CostKind);
1952 }
1953 return Cost;
1954 }
1955 case ISD::FP_TO_SINT:
1956 case ISD::FP_TO_UINT: {
1957 unsigned IsSigned = ISD == ISD::FP_TO_SINT;
1958 unsigned FCVT = IsSigned ? RISCV::VFCVT_RTZ_X_F_V : RISCV::VFCVT_RTZ_XU_F_V;
1959 unsigned FWCVT =
1960 IsSigned ? RISCV::VFWCVT_RTZ_X_F_V : RISCV::VFWCVT_RTZ_XU_F_V;
1961 unsigned FNCVT =
1962 IsSigned ? RISCV::VFNCVT_RTZ_X_F_W : RISCV::VFNCVT_RTZ_XU_F_W;
1963 unsigned SrcEltSize = Src->getScalarSizeInBits();
1964 unsigned DstEltSize = Dst->getScalarSizeInBits();
1965 InstructionCost Cost = 0;
1966 if ((SrcEltSize == 16) &&
1967 (!ST->hasVInstructionsF16() || ((DstEltSize / 2) > SrcEltSize))) {
1968 // If the target only supports zvfhmin or it is fp16-to-i64 conversion
1969 // pre-widening to f32 and then convert f32 to integer
1970 VectorType *VecF32Ty =
1971 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
1972 EC: cast<VectorType>(Val: Dst)->getElementCount());
1973 std::pair<InstructionCost, MVT> VecF32LT =
1974 getTypeLegalizationCost(Ty: VecF32Ty);
1975 Cost +=
1976 VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFWCVT_F_F_V,
1977 VT: VecF32LT.second, CostKind);
1978 Cost += getCastInstrCost(Opcode, Dst, Src: VecF32Ty, CCH, CostKind, I);
1979 return Cost;
1980 }
1981 if (DstEltSize == SrcEltSize)
1982 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
1983 else if (DstEltSize > SrcEltSize)
1984 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
1985 else { // (SrcEltSize > DstEltSize)
1986 // First do a narrowing conversion to an integer half the size, then
1987 // truncate if needed.
1988 MVT ElementVT = MVT::getIntegerVT(BitWidth: SrcEltSize / 2);
1989 MVT VecVT = DstLT.second.changeVectorElementType(EltVT: ElementVT);
1990 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: VecVT, CostKind);
1991 if ((SrcEltSize / 2) > DstEltSize) {
1992 Type *VecTy = EVT(VecVT).getTypeForEVT(Context&: Dst->getContext());
1993 Cost +=
1994 getCastInstrCost(Opcode: Instruction::Trunc, Dst, Src: VecTy, CCH, CostKind, I);
1995 }
1996 }
1997 return Cost;
1998 }
1999 case ISD::SINT_TO_FP:
2000 case ISD::UINT_TO_FP: {
2001 unsigned IsSigned = ISD == ISD::SINT_TO_FP;
2002 unsigned FCVT = IsSigned ? RISCV::VFCVT_F_X_V : RISCV::VFCVT_F_XU_V;
2003 unsigned FWCVT = IsSigned ? RISCV::VFWCVT_F_X_V : RISCV::VFWCVT_F_XU_V;
2004 unsigned FNCVT = IsSigned ? RISCV::VFNCVT_F_X_W : RISCV::VFNCVT_F_XU_W;
2005 unsigned SrcEltSize = Src->getScalarSizeInBits();
2006 unsigned DstEltSize = Dst->getScalarSizeInBits();
2007
2008 InstructionCost Cost = 0;
2009 if ((DstEltSize == 16) &&
2010 (!ST->hasVInstructionsF16() || ((SrcEltSize / 2) > DstEltSize))) {
2011 // If the target only supports zvfhmin or it is i64-to-fp16 conversion
2012 // it is converted to f32 and then converted to f16
2013 VectorType *VecF32Ty =
2014 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2015 EC: cast<VectorType>(Val: Dst)->getElementCount());
2016 std::pair<InstructionCost, MVT> VecF32LT =
2017 getTypeLegalizationCost(Ty: VecF32Ty);
2018 Cost += getCastInstrCost(Opcode, Dst: VecF32Ty, Src, CCH, CostKind, I);
2019 Cost += VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFNCVT_F_F_W,
2020 VT: DstLT.second, CostKind);
2021 return Cost;
2022 }
2023
2024 if (DstEltSize == SrcEltSize)
2025 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2026 else if (DstEltSize > SrcEltSize) {
2027 if ((DstEltSize / 2) > SrcEltSize) {
2028 VectorType *VecTy =
2029 VectorType::get(ElementType: IntegerType::get(C&: Dst->getContext(), NumBits: DstEltSize / 2),
2030 EC: cast<VectorType>(Val: Dst)->getElementCount());
2031 unsigned Op = IsSigned ? Instruction::SExt : Instruction::ZExt;
2032 Cost += getCastInstrCost(Opcode: Op, Dst: VecTy, Src, CCH, CostKind, I);
2033 }
2034 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2035 } else
2036 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: DstLT.second, CostKind);
2037 return Cost;
2038 }
2039 }
2040 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2041}
2042
2043unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) const {
2044 if (isa<ScalableVectorType>(Val: Ty)) {
2045 const unsigned EltSize = DL.getTypeSizeInBits(Ty: Ty->getElementType());
2046 const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue();
2047 const unsigned VectorBits = *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
2048 return RISCVTargetLowering::computeVLMAX(VectorBits, EltSize, MinSize);
2049 }
2050 return cast<FixedVectorType>(Val: Ty)->getNumElements();
2051}
2052
2053InstructionCost
2054RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
2055 FastMathFlags FMF,
2056 TTI::TargetCostKind CostKind) const {
2057 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2058 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2059
2060 // Skip if scalar size of Ty is bigger than ELEN.
2061 if (Ty->getScalarSizeInBits() > ST->getELen())
2062 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2063
2064 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2065 if (Ty->getElementType()->isIntegerTy(BitWidth: 1)) {
2066 // SelectionDAGBuilder does following transforms:
2067 // vector_reduce_{smin,umax}(<n x i1>) --> vector_reduce_or(<n x i1>)
2068 // vector_reduce_{smax,umin}(<n x i1>) --> vector_reduce_and(<n x i1>)
2069 if (IID == Intrinsic::umax || IID == Intrinsic::smin)
2070 return getArithmeticReductionCost(Opcode: Instruction::Or, Ty, FMF, CostKind);
2071 else
2072 return getArithmeticReductionCost(Opcode: Instruction::And, Ty, FMF, CostKind);
2073 }
2074
2075 if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) {
2076 SmallVector<unsigned, 3> Opcodes;
2077 InstructionCost ExtraCost = 0;
2078 switch (IID) {
2079 case Intrinsic::maximum:
2080 if (FMF.noNaNs()) {
2081 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2082 } else {
2083 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS,
2084 RISCV::VFMV_F_S};
2085 // Cost of Canonical Nan + branch
2086 // lui a0, 523264
2087 // fmv.w.x fa0, a0
2088 Type *DstTy = Ty->getScalarType();
2089 const unsigned EltTyBits = DstTy->getScalarSizeInBits();
2090 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2091 ExtraCost = 1 +
2092 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2093 CCH: TTI::CastContextHint::None, CostKind) +
2094 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2095 }
2096 break;
2097
2098 case Intrinsic::minimum:
2099 if (FMF.noNaNs()) {
2100 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2101 } else {
2102 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS,
2103 RISCV::VFMV_F_S};
2104 // Cost of Canonical Nan + branch
2105 // lui a0, 523264
2106 // fmv.w.x fa0, a0
2107 Type *DstTy = Ty->getScalarType();
2108 const unsigned EltTyBits = DL.getTypeSizeInBits(Ty: DstTy);
2109 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2110 ExtraCost = 1 +
2111 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2112 CCH: TTI::CastContextHint::None, CostKind) +
2113 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2114 }
2115 break;
2116 }
2117 return ExtraCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2118 }
2119
2120 // IR Reduction is composed by one rvv reduction instruction and vmv
2121 unsigned SplitOp;
2122 SmallVector<unsigned, 3> Opcodes;
2123 switch (IID) {
2124 default:
2125 llvm_unreachable("Unsupported intrinsic");
2126 case Intrinsic::smax:
2127 SplitOp = RISCV::VMAX_VV;
2128 Opcodes = {RISCV::VREDMAX_VS, RISCV::VMV_X_S};
2129 break;
2130 case Intrinsic::smin:
2131 SplitOp = RISCV::VMIN_VV;
2132 Opcodes = {RISCV::VREDMIN_VS, RISCV::VMV_X_S};
2133 break;
2134 case Intrinsic::umax:
2135 SplitOp = RISCV::VMAXU_VV;
2136 Opcodes = {RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
2137 break;
2138 case Intrinsic::umin:
2139 SplitOp = RISCV::VMINU_VV;
2140 Opcodes = {RISCV::VREDMINU_VS, RISCV::VMV_X_S};
2141 break;
2142 case Intrinsic::maxnum:
2143 SplitOp = RISCV::VFMAX_VV;
2144 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2145 break;
2146 case Intrinsic::minnum:
2147 SplitOp = RISCV::VFMIN_VV;
2148 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2149 break;
2150 }
2151 // Add a cost for data larger than LMUL8
2152 InstructionCost SplitCost =
2153 (LT.first > 1) ? (LT.first - 1) *
2154 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2155 : 0;
2156 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2157}
2158
2159InstructionCost
2160RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
2161 std::optional<FastMathFlags> FMF,
2162 TTI::TargetCostKind CostKind) const {
2163 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2164 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2165
2166 // Skip if scalar size of Ty is bigger than ELEN.
2167 if (Ty->getScalarSizeInBits() > ST->getELen())
2168 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2169
2170 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2171 assert(ISD && "Invalid opcode");
2172
2173 if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&
2174 ISD != ISD::FADD)
2175 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2176
2177 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2178 Type *ElementTy = Ty->getElementType();
2179 if (ElementTy->isIntegerTy(BitWidth: 1)) {
2180 // Example sequences:
2181 // vfirst.m a0, v0
2182 // seqz a0, a0
2183 if (LT.second == MVT::v1i1)
2184 return getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind) +
2185 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2186 VecPred: CmpInst::ICMP_EQ, CostKind);
2187
2188 if (ISD == ISD::AND) {
2189 // Example sequences:
2190 // vmand.mm v8, v9, v8 ; needed every time type is split
2191 // vmnot.m v8, v0 ; alias for vmnand
2192 // vcpop.m a0, v8
2193 // seqz a0, a0
2194
2195 // See the discussion: https://github.com/llvm/llvm-project/pull/119160
2196 // For LMUL <= 8, there is no splitting,
2197 // the sequences are vmnot, vcpop and seqz.
2198 // When LMUL > 8 and split = 1,
2199 // the sequences are vmnand, vcpop and seqz.
2200 // When LMUL > 8 and split > 1,
2201 // the sequences are (LT.first-2) * vmand, vmnand, vcpop and seqz.
2202 return ((LT.first > 2) ? (LT.first - 2) : 0) *
2203 getRISCVInstructionCost(OpCodes: RISCV::VMAND_MM, VT: LT.second, CostKind) +
2204 getRISCVInstructionCost(OpCodes: RISCV::VMNAND_MM, VT: LT.second, CostKind) +
2205 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2206 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2207 VecPred: CmpInst::ICMP_EQ, CostKind);
2208 } else if (ISD == ISD::XOR || ISD == ISD::ADD) {
2209 // Example sequences:
2210 // vsetvli a0, zero, e8, mf8, ta, ma
2211 // vmxor.mm v8, v0, v8 ; needed every time type is split
2212 // vcpop.m a0, v8
2213 // andi a0, a0, 1
2214 return (LT.first - 1) *
2215 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind) +
2216 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) + 1;
2217 } else {
2218 assert(ISD == ISD::OR);
2219 // Example sequences:
2220 // vsetvli a0, zero, e8, mf8, ta, ma
2221 // vmor.mm v8, v9, v8 ; needed every time type is split
2222 // vcpop.m a0, v0
2223 // snez a0, a0
2224 return (LT.first - 1) *
2225 getRISCVInstructionCost(OpCodes: RISCV::VMOR_MM, VT: LT.second, CostKind) +
2226 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2227 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2228 VecPred: CmpInst::ICMP_NE, CostKind);
2229 }
2230 }
2231
2232 // IR Reduction of or/and is composed by one vmv and one rvv reduction
2233 // instruction, and others is composed by two vmv and one rvv reduction
2234 // instruction
2235 unsigned SplitOp;
2236 SmallVector<unsigned, 3> Opcodes;
2237 switch (ISD) {
2238 case ISD::ADD:
2239 SplitOp = RISCV::VADD_VV;
2240 Opcodes = {RISCV::VMV_S_X, RISCV::VREDSUM_VS, RISCV::VMV_X_S};
2241 break;
2242 case ISD::OR:
2243 SplitOp = RISCV::VOR_VV;
2244 Opcodes = {RISCV::VREDOR_VS, RISCV::VMV_X_S};
2245 break;
2246 case ISD::XOR:
2247 SplitOp = RISCV::VXOR_VV;
2248 Opcodes = {RISCV::VMV_S_X, RISCV::VREDXOR_VS, RISCV::VMV_X_S};
2249 break;
2250 case ISD::AND:
2251 SplitOp = RISCV::VAND_VV;
2252 Opcodes = {RISCV::VREDAND_VS, RISCV::VMV_X_S};
2253 break;
2254 case ISD::FADD:
2255 // We can't promote f16/bf16 fadd reductions.
2256 if ((LT.second.getScalarType() == MVT::f16 && !ST->hasVInstructionsF16()) ||
2257 LT.second.getScalarType() == MVT::bf16)
2258 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2259 if (TTI::requiresOrderedReduction(FMF)) {
2260 Opcodes.push_back(Elt: RISCV::VFMV_S_F);
2261 for (unsigned i = 0; i < LT.first.getValue(); i++)
2262 Opcodes.push_back(Elt: RISCV::VFREDOSUM_VS);
2263 Opcodes.push_back(Elt: RISCV::VFMV_F_S);
2264 return getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2265 }
2266 SplitOp = RISCV::VFADD_VV;
2267 Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDUSUM_VS, RISCV::VFMV_F_S};
2268 break;
2269 }
2270 // Add a cost for data larger than LMUL8
2271 InstructionCost SplitCost =
2272 (LT.first > 1) ? (LT.first - 1) *
2273 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2274 : 0;
2275 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2276}
2277
2278InstructionCost RISCVTTIImpl::getExtendedReductionCost(
2279 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
2280 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
2281 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2282 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2283 FMF, CostKind);
2284
2285 // Skip if scalar size of ResTy is bigger than ELEN.
2286 if (ResTy->getScalarSizeInBits() > ST->getELen())
2287 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2288 FMF, CostKind);
2289
2290 if (Opcode != Instruction::Add && Opcode != Instruction::FAdd)
2291 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2292 FMF, CostKind);
2293
2294 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2295
2296 if (IsUnsigned && Opcode == Instruction::Add &&
2297 LT.second.isFixedLengthVectorOf(EltVT: MVT::i1)) {
2298 // Represent vector_reduce_add(ZExt(<n x i1>)) as
2299 // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
2300 return LT.first *
2301 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind);
2302 }
2303
2304 if (ResTy->getScalarSizeInBits() != 2 * LT.second.getScalarSizeInBits())
2305 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2306 FMF, CostKind);
2307
2308 return (LT.first - 1) +
2309 getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
2310}
2311
2312InstructionCost
2313RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,
2314 TTI::TargetCostKind CostKind) const {
2315 assert(OpInfo.isConstant() && "non constant operand?");
2316 if (!isa<VectorType>(Val: Ty))
2317 // FIXME: We need to account for immediate materialization here, but doing
2318 // a decent job requires more knowledge about the immediate than we
2319 // currently have here.
2320 return 0;
2321
2322 if (OpInfo.isUniform())
2323 // vmv.v.i, vmv.v.x, or vfmv.v.f
2324 // We ignore the cost of the scalar constant materialization to be consistent
2325 // with how we treat scalar constants themselves just above.
2326 return 1;
2327
2328 return getConstantPoolLoadCost(Ty, CostKind);
2329}
2330
2331InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
2332 Align Alignment,
2333 unsigned AddressSpace,
2334 TTI::TargetCostKind CostKind,
2335 TTI::OperandValueInfo OpInfo,
2336 const Instruction *I) const {
2337 EVT VT = TLI->getValueType(DL, Ty: Src, AllowUnknown: true);
2338 // Type legalization can't handle structs, and load latency isn't handled here
2339 if (VT == MVT::Other ||
2340 (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency))
2341 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2342 CostKind, OpInfo, I);
2343
2344 InstructionCost Cost = 0;
2345 if (Opcode == Instruction::Store && OpInfo.isConstant())
2346 Cost += getStoreImmCost(Ty: Src, OpInfo, CostKind);
2347
2348 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
2349
2350 InstructionCost BaseCost = [&]() {
2351 InstructionCost Cost = LT.first;
2352 if (CostKind != TTI::TCK_RecipThroughput)
2353 return Cost;
2354
2355 // Our actual lowering for the case where a wider legal type is available
2356 // uses the a VL predicated load on the wider type. This is reflected in
2357 // the result of getTypeLegalizationCost, but BasicTTI assumes the
2358 // widened cases are scalarized.
2359 const DataLayout &DL = this->getDataLayout();
2360 if (Src->isVectorTy() && LT.second.isVector() &&
2361 TypeSize::isKnownLT(LHS: DL.getTypeStoreSizeInBits(Ty: Src),
2362 RHS: LT.second.getSizeInBits()))
2363 return Cost;
2364
2365 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2366 CostKind, OpInfo, I);
2367 }();
2368
2369 // Assume memory ops cost scale with the number of vector registers
2370 // possible accessed by the instruction. Note that BasicTTI already
2371 // handles the LT.first term for us.
2372 if (ST->hasVInstructions() && LT.second.isVector() &&
2373 CostKind != TTI::TCK_CodeSize)
2374 BaseCost *= TLI->getLMULCost(VT: LT.second);
2375 return Cost + BaseCost;
2376}
2377
2378InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
2379 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
2380 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
2381 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
2382 if (CostKind != TTI::TCK_RecipThroughput)
2383 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2384 Op1Info, Op2Info, I);
2385
2386 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2387 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2388 Op1Info, Op2Info, I);
2389
2390 // Skip if scalar size of ValTy is bigger than ELEN.
2391 if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen())
2392 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2393 Op1Info, Op2Info, I);
2394
2395 auto GetConstantMatCost =
2396 [&](TTI::OperandValueInfo OpInfo) -> InstructionCost {
2397 if (OpInfo.isUniform())
2398 // We return 0 we currently ignore the cost of materializing scalar
2399 // constants in GPRs.
2400 return 0;
2401
2402 return getConstantPoolLoadCost(Ty: ValTy, CostKind);
2403 };
2404
2405 InstructionCost ConstantMatCost;
2406 if (Op1Info.isConstant())
2407 ConstantMatCost += GetConstantMatCost(Op1Info);
2408 if (Op2Info.isConstant())
2409 ConstantMatCost += GetConstantMatCost(Op2Info);
2410
2411 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2412 if (Opcode == Instruction::Select && ValTy->isVectorTy()) {
2413 if (CondTy->isVectorTy()) {
2414 if (ValTy->getScalarSizeInBits() == 1) {
2415 // vmandn.mm v8, v8, v9
2416 // vmand.mm v9, v0, v9
2417 // vmor.mm v0, v9, v8
2418 return ConstantMatCost +
2419 LT.first *
2420 getRISCVInstructionCost(
2421 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2422 VT: LT.second, CostKind);
2423 }
2424 // vselect and max/min are supported natively.
2425 return ConstantMatCost +
2426 LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second,
2427 CostKind);
2428 }
2429
2430 if (ValTy->getScalarSizeInBits() == 1) {
2431 // vmv.v.x v9, a0
2432 // vmsne.vi v9, v9, 0
2433 // vmandn.mm v8, v8, v9
2434 // vmand.mm v9, v0, v9
2435 // vmor.mm v0, v9, v8
2436 MVT InterimVT = LT.second.changeVectorElementType(EltVT: MVT::i8);
2437 return ConstantMatCost +
2438 LT.first *
2439 getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
2440 VT: InterimVT, CostKind) +
2441 LT.first * getRISCVInstructionCost(
2442 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2443 VT: LT.second, CostKind);
2444 }
2445
2446 // vmv.v.x v10, a0
2447 // vmsne.vi v0, v10, 0
2448 // vmerge.vvm v8, v9, v8, v0
2449 return ConstantMatCost +
2450 LT.first * getRISCVInstructionCost(
2451 OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI, RISCV::VMERGE_VVM},
2452 VT: LT.second, CostKind);
2453 }
2454
2455 if ((Opcode == Instruction::ICmp) && ValTy->isVectorTy() &&
2456 CmpInst::isIntPredicate(P: VecPred)) {
2457 // Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE
2458 // provided they incur the same cost across all implementations
2459 return ConstantMatCost + LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMSLT_VV,
2460 VT: LT.second,
2461 CostKind);
2462 }
2463
2464 if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() &&
2465 CmpInst::isFPPredicate(P: VecPred)) {
2466
2467 // Use VMXOR_MM and VMXNOR_MM to generate all true/false mask
2468 if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE))
2469 return ConstantMatCost +
2470 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind);
2471
2472 // If we do not support the input floating point vector type, use the base
2473 // one which will calculate as:
2474 // ScalarizeCost + Num * Cost for fixed vector,
2475 // InvalidCost for scalable vector.
2476 if ((ValTy->getScalarSizeInBits() == 16 && !ST->hasVInstructionsF16()) ||
2477 (ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) ||
2478 (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64()))
2479 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2480 Op1Info, Op2Info, I);
2481
2482 // Assuming vector fp compare and mask instructions are all the same cost
2483 // until a need arises to differentiate them.
2484 switch (VecPred) {
2485 case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm
2486 case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm
2487 case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm
2488 case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm
2489 return ConstantMatCost +
2490 LT.first * getRISCVInstructionCost(
2491 OpCodes: {RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM},
2492 VT: LT.second, CostKind);
2493
2494 case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m
2495 case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m
2496 case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m
2497 case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m
2498 return ConstantMatCost +
2499 LT.first *
2500 getRISCVInstructionCost(OpCodes: {RISCV::VMFLT_VV, RISCV::VMNAND_MM},
2501 VT: LT.second, CostKind);
2502
2503 case CmpInst::FCMP_OEQ: // vmfeq.vv
2504 case CmpInst::FCMP_OGT: // vmflt.vv
2505 case CmpInst::FCMP_OGE: // vmfle.vv
2506 case CmpInst::FCMP_OLT: // vmflt.vv
2507 case CmpInst::FCMP_OLE: // vmfle.vv
2508 case CmpInst::FCMP_UNE: // vmfne.vv
2509 return ConstantMatCost +
2510 LT.first *
2511 getRISCVInstructionCost(OpCodes: RISCV::VMFLT_VV, VT: LT.second, CostKind);
2512 default:
2513 break;
2514 }
2515 }
2516
2517 // With ShortForwardBranchOpt or ConditionalMoveFusion, scalar icmp + select
2518 // instructions will lower to SELECT_CC and lower to PseudoCCMOVGPR which will
2519 // generate a conditional branch + mv. The cost of scalar (icmp + select) will
2520 // be (0 + select instr cost).
2521 if (ST->hasConditionalMoveFusion() && I && isa<ICmpInst>(Val: I) &&
2522 ValTy->isIntegerTy() && !I->user_empty()) {
2523 if (all_of(Range: I->users(), P: [&](const User *U) {
2524 return match(V: U, P: m_Select(C: m_Specific(V: I), L: m_Value(), R: m_Value())) &&
2525 U->getType()->isIntegerTy() &&
2526 !isa<ConstantData>(Val: U->getOperand(i: 1)) &&
2527 !isa<ConstantData>(Val: U->getOperand(i: 2));
2528 }))
2529 return 0;
2530 }
2531
2532 // TODO: Add cost for scalar type.
2533
2534 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2535 Op1Info, Op2Info, I);
2536}
2537
2538InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
2539 TTI::TargetCostKind CostKind,
2540 const Instruction *I) const {
2541 if (CostKind != TTI::TCK_RecipThroughput)
2542 return Opcode == Instruction::PHI ? 0 : 1;
2543 // Branches are assumed to be predicted.
2544 return 0;
2545}
2546
2547InstructionCost RISCVTTIImpl::getVectorInstrCost(
2548 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
2549 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
2550 assert(Val->isVectorTy() && "This must be a vector type");
2551
2552 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
2553 // For now, skip all fixed vector cost analysis when P extension is available
2554 // to avoid crashes in getMinRVVVectorSizeInBits()
2555 if (ST->hasStdExtP() && isa<FixedVectorType>(Val)) {
2556 return 1; // Treat as single instruction cost for now
2557 }
2558
2559 if (Opcode != Instruction::ExtractElement &&
2560 Opcode != Instruction::InsertElement)
2561 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
2562 VIC);
2563
2564 // Legalize the type.
2565 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Val);
2566
2567 // This type is legalized to a scalar type.
2568 if (!LT.second.isVector()) {
2569 auto *FixedVecTy = cast<FixedVectorType>(Val);
2570 // If Index is a known constant, cost is zero.
2571 if (Index != -1U)
2572 return 0;
2573 // Extract/InsertElement with non-constant index is very costly when
2574 // scalarized; estimate cost of loads/stores sequence via the stack:
2575 // ExtractElement cost: store vector to stack, load scalar;
2576 // InsertElement cost: store vector to stack, store scalar, load vector.
2577 Type *ElemTy = FixedVecTy->getElementType();
2578 auto NumElems = FixedVecTy->getNumElements();
2579 auto Align = DL.getPrefTypeAlign(Ty: ElemTy);
2580 InstructionCost LoadCost =
2581 getMemoryOpCost(Opcode: Instruction::Load, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2582 InstructionCost StoreCost =
2583 getMemoryOpCost(Opcode: Instruction::Store, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2584 return Opcode == Instruction::ExtractElement
2585 ? StoreCost * NumElems + LoadCost
2586 : (StoreCost + LoadCost) * NumElems + StoreCost;
2587 }
2588
2589 // For unsupported scalable vector.
2590 if (LT.second.isScalableVector() && !LT.first.isValid())
2591 return LT.first;
2592
2593 // Mask vector extract/insert is expanded via e8.
2594 if (Val->getScalarSizeInBits() == 1) {
2595 VectorType *WideTy =
2596 VectorType::get(ElementType: IntegerType::get(C&: Val->getContext(), NumBits: 8),
2597 EC: cast<VectorType>(Val)->getElementCount());
2598 if (Opcode == Instruction::ExtractElement) {
2599 InstructionCost ExtendCost
2600 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2601 CCH: TTI::CastContextHint::None, CostKind);
2602 InstructionCost ExtractCost
2603 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2604 return ExtendCost + ExtractCost;
2605 }
2606 InstructionCost ExtendCost
2607 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2608 CCH: TTI::CastContextHint::None, CostKind);
2609 InstructionCost InsertCost
2610 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2611 InstructionCost TruncCost
2612 = getCastInstrCost(Opcode: Instruction::Trunc, Dst: Val, Src: WideTy,
2613 CCH: TTI::CastContextHint::None, CostKind);
2614 return ExtendCost + InsertCost + TruncCost;
2615 }
2616
2617
2618 // In RVV, we could use vslidedown + vmv.x.s to extract element from vector
2619 // and vslideup + vmv.s.x to insert element to vector.
2620 unsigned BaseCost = 1;
2621 // When insertelement we should add the index with 1 as the input of vslideup.
2622 unsigned SlideCost = Opcode == Instruction::InsertElement ? 2 : 1;
2623
2624 if (Index != -1U) {
2625 // The type may be split. For fixed-width vectors we can normalize the
2626 // index to the new type.
2627 if (LT.second.isFixedLengthVector()) {
2628 unsigned Width = LT.second.getVectorNumElements();
2629 Index = Index % Width;
2630 }
2631
2632 // If exact VLEN is known, we will insert/extract into the appropriate
2633 // subvector with no additional subvector insert/extract cost.
2634 if (auto VLEN = ST->getRealVLen()) {
2635 unsigned EltSize = LT.second.getScalarSizeInBits();
2636 unsigned M1Max = *VLEN / EltSize;
2637 Index = Index % M1Max;
2638 }
2639
2640 if (Index == 0)
2641 // We can extract/insert the first element without vslidedown/vslideup.
2642 SlideCost = 0;
2643 else if (Opcode == Instruction::InsertElement)
2644 SlideCost = 1; // With a constant index, we do not need to use addi.
2645 }
2646
2647 // When the vector needs to split into multiple register groups and the index
2648 // exceeds single vector register group, we need to insert/extract the element
2649 // via stack.
2650 if (LT.first > 1 &&
2651 ((Index == -1U) || (Index >= LT.second.getVectorMinNumElements() &&
2652 LT.second.isScalableVector()))) {
2653 Type *ScalarType = Val->getScalarType();
2654 Align VecAlign = DL.getPrefTypeAlign(Ty: Val);
2655 Align SclAlign = DL.getPrefTypeAlign(Ty: ScalarType);
2656 // Extra addi for unknown index.
2657 InstructionCost IdxCost = Index == -1U ? 1 : 0;
2658
2659 // Store all split vectors into stack and load the target element.
2660 if (Opcode == Instruction::ExtractElement)
2661 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2662 getMemoryOpCost(Opcode: Instruction::Load, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
2663 CostKind) +
2664 IdxCost;
2665
2666 // Store all split vectors into stack and store the target element and load
2667 // vectors back.
2668 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2669 getMemoryOpCost(Opcode: Instruction::Load, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
2670 getMemoryOpCost(Opcode: Instruction::Store, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
2671 CostKind) +
2672 IdxCost;
2673 }
2674
2675 // Extract i64 in the target that has XLEN=32 need more instruction.
2676 if (Val->getScalarType()->isIntegerTy() &&
2677 ST->getXLen() < Val->getScalarSizeInBits()) {
2678 // For extractelement, we need the following instructions:
2679 // vsetivli zero, 1, e64, m1, ta, mu (not count)
2680 // vslidedown.vx v8, v8, a0
2681 // vmv.x.s a0, v8
2682 // li a1, 32
2683 // vsrl.vx v8, v8, a1
2684 // vmv.x.s a1, v8
2685
2686 // For insertelement, we need the following instructions:
2687 // vsetivli zero, 2, e32, m4, ta, mu (not count)
2688 // vmv.v.i v12, 0
2689 // vslide1up.vx v16, v12, a1
2690 // vslide1up.vx v12, v16, a0
2691 // addi a0, a2, 1
2692 // vsetvli zero, a0, e64, m4, tu, mu (not count)
2693 // vslideup.vx v8, v12, a2
2694
2695 // TODO: should we count these special vsetvlis?
2696 BaseCost = Opcode == Instruction::InsertElement ? 3 : 4;
2697 }
2698 return BaseCost + SlideCost;
2699}
2700
2701InstructionCost
2702RISCVTTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
2703 TTI::TargetCostKind CostKind,
2704 unsigned Index) const {
2705 if (isa<FixedVectorType>(Val))
2706 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
2707 Index);
2708
2709 // TODO: This code replicates what LoopVectorize.cpp used to do when asking
2710 // for the cost of extracting the last lane of a scalable vector. It probably
2711 // needs a more accurate cost.
2712 ElementCount EC = cast<VectorType>(Val)->getElementCount();
2713 assert(Index < EC.getKnownMinValue() && "Unexpected reverse index");
2714 return getVectorInstrCost(Opcode, Val, CostKind,
2715 Index: EC.getKnownMinValue() - 1 - Index, Op0: nullptr,
2716 Op1: nullptr);
2717}
2718
2719/// Check to see if this instruction is expected to be combined to a simpler
2720/// operation during/before lowering. If so return the cost of the combined
2721/// operation rather than provided one. For instance, `udiv i16 %X, 2` is likely
2722/// to be combined to `lshr i16 %X, 1`, so return the cost of a `lshr` rather
2723/// than the cost of a `udiv`
2724std::optional<InstructionCost>
2725RISCVTTIImpl::getCombinedArithmeticInstructionCost(
2726 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
2727 TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
2728 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
2729 // Vector unsigned division/remainder will be simplified to shifts/masks.
2730 if ((Opcode == Instruction::UDiv || Opcode == Instruction::URem) &&
2731 Opd2Info.isConstant() && Opd2Info.isPowerOf2()) {
2732 if (Opcode == Instruction::UDiv)
2733 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind, Op1Info: Opd1Info,
2734 Op2Info: Opd2Info.getNoProps());
2735 // UREM
2736 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind, Op1Info: Opd1Info,
2737 Op2Info: Opd2Info.getNoProps());
2738 }
2739 return std::nullopt;
2740}
2741
2742InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
2743 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
2744 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
2745 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
2746
2747 // TODO: Handle more cost kinds.
2748 if (CostKind != TTI::TCK_RecipThroughput)
2749 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2750 Args, CxtI);
2751
2752 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2753 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2754 Args, CxtI);
2755
2756 // Skip if scalar size of Ty is bigger than ELEN.
2757 if (isa<VectorType>(Val: Ty) && Ty->getScalarSizeInBits() > ST->getELen())
2758 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2759 Args, CxtI);
2760
2761 if (std::optional<InstructionCost> CombinedCost =
2762 getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
2763 Opd2Info: Op2Info, Args, CxtI))
2764 return *CombinedCost;
2765
2766 // Legalize the type.
2767 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2768 unsigned ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
2769
2770 // TODO: Handle scalar type.
2771 if (!LT.second.isVector()) {
2772 static const CostTblEntry DivTbl[]{
2773 {.ISD: ISD::UDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2774 {.ISD: ISD::UDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2775 {.ISD: ISD::SDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2776 {.ISD: ISD::SDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2777 {.ISD: ISD::UREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2778 {.ISD: ISD::UREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
2779 {.ISD: ISD::SREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
2780 {.ISD: ISD::SREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive}};
2781 if (TLI->isOperationLegalOrPromote(Op: ISDOpcode, VT: LT.second))
2782 if (const auto *Entry = CostTableLookup(Table: DivTbl, ISD: ISDOpcode, Ty: LT.second))
2783 return Entry->Cost * LT.first;
2784
2785 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2786 Args, CxtI);
2787 }
2788
2789 // f16 with zvfhmin and bf16 will be promoted to f32.
2790 // FIXME: nxv32[b]f16 will be custom lowered and split.
2791 InstructionCost CastCost = 0;
2792 if ((LT.second.getVectorElementType() == MVT::f16 ||
2793 LT.second.getVectorElementType() == MVT::bf16) &&
2794 TLI->getOperationAction(Op: ISDOpcode, VT: LT.second) ==
2795 TargetLoweringBase::LegalizeAction::Promote) {
2796 MVT PromotedVT = TLI->getTypeToPromoteTo(Op: ISDOpcode, VT: LT.second);
2797 Type *PromotedTy = EVT(PromotedVT).getTypeForEVT(Context&: Ty->getContext());
2798 Type *LegalTy = EVT(LT.second).getTypeForEVT(Context&: Ty->getContext());
2799 // Add cost of extending arguments
2800 CastCost += LT.first * Args.size() *
2801 getCastInstrCost(Opcode: Instruction::FPExt, Dst: PromotedTy, Src: LegalTy,
2802 CCH: TTI::CastContextHint::None, CostKind);
2803 // Add cost of truncating result
2804 CastCost +=
2805 LT.first * getCastInstrCost(Opcode: Instruction::FPTrunc, Dst: LegalTy, Src: PromotedTy,
2806 CCH: TTI::CastContextHint::None, CostKind);
2807 // Compute cost of op in promoted type
2808 LT.second = PromotedVT;
2809 }
2810
2811 auto getConstantMatCost =
2812 [&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost {
2813 if (OpInfo.isUniform() && canSplatOperand(Opcode, Operand))
2814 // Two sub-cases:
2815 // * Has a 5 bit immediate operand which can be splatted.
2816 // * Has a larger immediate which must be materialized in scalar register
2817 // We return 0 for both as we currently ignore the cost of materializing
2818 // scalar constants in GPRs.
2819 return 0;
2820
2821 return getConstantPoolLoadCost(Ty, CostKind);
2822 };
2823
2824 // Add the cost of materializing any constant vectors required.
2825 InstructionCost ConstantMatCost = 0;
2826 if (Op1Info.isConstant())
2827 ConstantMatCost += getConstantMatCost(0, Op1Info);
2828 if (Op2Info.isConstant())
2829 ConstantMatCost += getConstantMatCost(1, Op2Info);
2830
2831 unsigned Op;
2832 switch (ISDOpcode) {
2833 case ISD::ADD:
2834 case ISD::SUB:
2835 Op = RISCV::VADD_VV;
2836 break;
2837 case ISD::SHL:
2838 case ISD::SRL:
2839 case ISD::SRA:
2840 Op = RISCV::VSLL_VV;
2841 break;
2842 case ISD::AND:
2843 case ISD::OR:
2844 case ISD::XOR:
2845 Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
2846 break;
2847 case ISD::MUL:
2848 case ISD::MULHS:
2849 case ISD::MULHU:
2850 Op = RISCV::VMUL_VV;
2851 break;
2852 case ISD::SDIV:
2853 case ISD::UDIV:
2854 Op = RISCV::VDIV_VV;
2855 break;
2856 case ISD::SREM:
2857 case ISD::UREM:
2858 Op = RISCV::VREM_VV;
2859 break;
2860 case ISD::FADD:
2861 case ISD::FSUB:
2862 Op = RISCV::VFADD_VV;
2863 break;
2864 case ISD::FMUL:
2865 Op = RISCV::VFMUL_VV;
2866 break;
2867 case ISD::FDIV:
2868 Op = RISCV::VFDIV_VV;
2869 break;
2870 case ISD::FNEG:
2871 Op = RISCV::VFSGNJN_VV;
2872 break;
2873 default:
2874 // Assuming all other instructions have the same cost until a need arises to
2875 // differentiate them.
2876 return CastCost + ConstantMatCost +
2877 BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
2878 Args, CxtI);
2879 }
2880
2881 InstructionCost InstrCost = getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
2882 // We use BasicTTIImpl to calculate scalar costs, which assumes floating point
2883 // ops are twice as expensive as integer ops. Do the same for vectors so
2884 // scalar floating point ops aren't cheaper than their vector equivalents.
2885 if (Ty->isFPOrFPVectorTy())
2886 InstrCost *= 2;
2887 return CastCost + ConstantMatCost + LT.first * InstrCost;
2888}
2889
2890// TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
2891InstructionCost RISCVTTIImpl::getPointersChainCost(
2892 ArrayRef<const Value *> Ptrs, const Value *Base,
2893 const TTI::PointersChainInfo &Info, Type *AccessTy,
2894 TTI::TargetCostKind CostKind) const {
2895 InstructionCost Cost = TTI::TCC_Free;
2896 // In the basic model we take into account GEP instructions only
2897 // (although here can come alloca instruction, a value, constants and/or
2898 // constant expressions, PHIs, bitcasts ... whatever allowed to be used as a
2899 // pointer). Typically, if Base is a not a GEP-instruction and all the
2900 // pointers are relative to the same base address, all the rest are
2901 // either GEP instructions, PHIs, bitcasts or constants. When we have same
2902 // base, we just calculate cost of each non-Base GEP as an ADD operation if
2903 // any their index is a non-const.
2904 // If no known dependencies between the pointers cost is calculated as a sum
2905 // of costs of GEP instructions.
2906 for (auto [I, V] : enumerate(First&: Ptrs)) {
2907 const auto *GEP = dyn_cast<GetElementPtrInst>(Val: V);
2908 if (!GEP)
2909 continue;
2910 if (Info.isSameBase() && V != Base) {
2911 if (GEP->hasAllConstantIndices())
2912 continue;
2913 // If the chain is unit-stride and BaseReg + stride*i is a legal
2914 // addressing mode, then presume the base GEP is sitting around in a
2915 // register somewhere and check if we can fold the offset relative to
2916 // it.
2917 unsigned Stride = DL.getTypeStoreSize(Ty: AccessTy);
2918 if (Info.isUnitStride() &&
2919 isLegalAddressingMode(Ty: AccessTy,
2920 /* BaseGV */ nullptr,
2921 /* BaseOffset */ Stride * I,
2922 /* HasBaseReg */ true,
2923 /* Scale */ 0,
2924 AddrSpace: GEP->getType()->getPointerAddressSpace()))
2925 continue;
2926 Cost += getArithmeticInstrCost(Opcode: Instruction::Add, Ty: GEP->getType(), CostKind,
2927 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
2928 Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Args: {});
2929 } else {
2930 SmallVector<const Value *> Indices(GEP->indices());
2931 Cost += getGEPCost(PointeeType: GEP->getSourceElementType(), Ptr: GEP->getPointerOperand(),
2932 Operands: Indices, AccessType: AccessTy, CostKind);
2933 }
2934 }
2935 return Cost;
2936}
2937
2938void RISCVTTIImpl::getUnrollingPreferences(
2939 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
2940 OptimizationRemarkEmitter *ORE) const {
2941 // TODO: More tuning on benchmarks and metrics with changes as needed
2942 // would apply to all settings below to enable performance.
2943
2944
2945 if (ST->enableDefaultUnroll())
2946 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
2947
2948 // Enable Upper bound unrolling universally, not dependent upon the conditions
2949 // below.
2950 UP.UpperBound = true;
2951
2952 // Disable loop unrolling for Oz and Os.
2953 UP.OptSizeThreshold = 0;
2954 UP.PartialOptSizeThreshold = 0;
2955 if (L->getHeader()->getParent()->hasOptSize())
2956 return;
2957
2958 SmallVector<BasicBlock *, 4> ExitingBlocks;
2959 L->getExitingBlocks(ExitingBlocks);
2960 LLVM_DEBUG(dbgs() << "Loop has:\n"
2961 << "Blocks: " << L->getNumBlocks() << "\n"
2962 << "Exit blocks: " << ExitingBlocks.size() << "\n");
2963
2964 // Only allow another exit other than the latch. This acts as an early exit
2965 // as it mirrors the profitability calculation of the runtime unroller.
2966 if (ExitingBlocks.size() > 2)
2967 return;
2968
2969 // Limit the CFG of the loop body for targets with a branch predictor.
2970 // Allowing 4 blocks permits if-then-else diamonds in the body.
2971 if (L->getNumBlocks() > 4)
2972 return;
2973
2974 // Scan the loop: don't unroll loops with calls as this could prevent
2975 // inlining. Don't unroll auto-vectorized loops either, though do allow
2976 // unrolling of the scalar remainder.
2977 bool IsVectorized = getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized");
2978 InstructionCost Cost = 0;
2979 for (auto *BB : L->getBlocks()) {
2980 for (auto &I : *BB) {
2981 // Both auto-vectorized loops and the scalar remainder have the
2982 // isvectorized attribute, so differentiate between them by the presence
2983 // of vector instructions.
2984 if (IsVectorized && (I.getType()->isVectorTy() ||
2985 llvm::any_of(Range: I.operand_values(), P: [](Value *V) {
2986 return V->getType()->isVectorTy();
2987 })))
2988 return;
2989
2990 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
2991 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction()) {
2992 if (!isLoweredToCall(F))
2993 continue;
2994 }
2995 return;
2996 }
2997
2998 SmallVector<const Value *> Operands(I.operand_values());
2999 Cost += getInstructionCost(U: &I, Operands,
3000 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
3001 }
3002 }
3003
3004 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
3005
3006 UP.Partial = true;
3007 UP.Runtime = true;
3008 UP.UnrollRemainder = true;
3009 UP.UnrollAndJam = true;
3010
3011 // Force unrolling small loops can be very useful because of the branch
3012 // taken cost of the backedge.
3013 if (Cost < 12)
3014 UP.Force = true;
3015}
3016
3017void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
3018 TTI::PeelingPreferences &PP) const {
3019 BaseT::getPeelingPreferences(L, SE, PP);
3020}
3021
3022bool RISCVTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
3023 MemIntrinsicInfo &Info) const {
3024 const DataLayout &DL = getDataLayout();
3025 Intrinsic::ID IID = Inst->getIntrinsicID();
3026 LLVMContext &C = Inst->getContext();
3027 bool HasMask = false;
3028
3029 auto getSegNum = [](const IntrinsicInst *II, unsigned PtrOperandNo,
3030 bool IsWrite) -> int64_t {
3031 if (auto *TarExtTy =
3032 dyn_cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType()))
3033 return TarExtTy->getIntParameter(i: 0);
3034
3035 return 1;
3036 };
3037
3038 switch (IID) {
3039 case Intrinsic::riscv_vle_mask:
3040 case Intrinsic::riscv_vse_mask:
3041 case Intrinsic::riscv_vlseg2_mask:
3042 case Intrinsic::riscv_vlseg3_mask:
3043 case Intrinsic::riscv_vlseg4_mask:
3044 case Intrinsic::riscv_vlseg5_mask:
3045 case Intrinsic::riscv_vlseg6_mask:
3046 case Intrinsic::riscv_vlseg7_mask:
3047 case Intrinsic::riscv_vlseg8_mask:
3048 case Intrinsic::riscv_vsseg2_mask:
3049 case Intrinsic::riscv_vsseg3_mask:
3050 case Intrinsic::riscv_vsseg4_mask:
3051 case Intrinsic::riscv_vsseg5_mask:
3052 case Intrinsic::riscv_vsseg6_mask:
3053 case Intrinsic::riscv_vsseg7_mask:
3054 case Intrinsic::riscv_vsseg8_mask:
3055 HasMask = true;
3056 [[fallthrough]];
3057 case Intrinsic::riscv_vle:
3058 case Intrinsic::riscv_vse:
3059 case Intrinsic::riscv_vlseg2:
3060 case Intrinsic::riscv_vlseg3:
3061 case Intrinsic::riscv_vlseg4:
3062 case Intrinsic::riscv_vlseg5:
3063 case Intrinsic::riscv_vlseg6:
3064 case Intrinsic::riscv_vlseg7:
3065 case Intrinsic::riscv_vlseg8:
3066 case Intrinsic::riscv_vsseg2:
3067 case Intrinsic::riscv_vsseg3:
3068 case Intrinsic::riscv_vsseg4:
3069 case Intrinsic::riscv_vsseg5:
3070 case Intrinsic::riscv_vsseg6:
3071 case Intrinsic::riscv_vsseg7:
3072 case Intrinsic::riscv_vsseg8: {
3073 // Intrinsic interface:
3074 // riscv_vle(merge, ptr, vl)
3075 // riscv_vle_mask(merge, ptr, mask, vl, policy)
3076 // riscv_vse(val, ptr, vl)
3077 // riscv_vse_mask(val, ptr, mask, vl, policy)
3078 // riscv_vlseg#(merge, ptr, vl, sew)
3079 // riscv_vlseg#_mask(merge, ptr, mask, vl, policy, sew)
3080 // riscv_vsseg#(val, ptr, vl, sew)
3081 // riscv_vsseg#_mask(val, ptr, mask, vl, sew)
3082 bool IsWrite = Inst->getType()->isVoidTy();
3083 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3084 // The results of segment loads are TargetExtType.
3085 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3086 unsigned SEW =
3087 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3088 ->getZExtValue();
3089 Ty = TarExtTy->getTypeParameter(i: 0U);
3090 Ty = ScalableVectorType::get(
3091 ElementType: IntegerType::get(C, NumBits: SEW),
3092 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3093 }
3094 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3095 unsigned VLIndex = RVVIInfo->VLOperand;
3096 unsigned PtrOperandNo = VLIndex - 1 - HasMask;
3097 MaybeAlign Alignment =
3098 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3099 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3100 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3101 if (HasMask)
3102 Mask = Inst->getArgOperand(i: VLIndex - 1);
3103 Value *EVL = Inst->getArgOperand(i: VLIndex);
3104 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3105 // RVV uses contiguous elements as a segment.
3106 if (SegNum > 1) {
3107 unsigned ElemSize = Ty->getScalarSizeInBits();
3108 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3109 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3110 }
3111 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3112 Args&: Alignment, Args&: Mask, Args&: EVL);
3113 return true;
3114 }
3115 case Intrinsic::riscv_vlse_mask:
3116 case Intrinsic::riscv_vsse_mask:
3117 case Intrinsic::riscv_vlsseg2_mask:
3118 case Intrinsic::riscv_vlsseg3_mask:
3119 case Intrinsic::riscv_vlsseg4_mask:
3120 case Intrinsic::riscv_vlsseg5_mask:
3121 case Intrinsic::riscv_vlsseg6_mask:
3122 case Intrinsic::riscv_vlsseg7_mask:
3123 case Intrinsic::riscv_vlsseg8_mask:
3124 case Intrinsic::riscv_vssseg2_mask:
3125 case Intrinsic::riscv_vssseg3_mask:
3126 case Intrinsic::riscv_vssseg4_mask:
3127 case Intrinsic::riscv_vssseg5_mask:
3128 case Intrinsic::riscv_vssseg6_mask:
3129 case Intrinsic::riscv_vssseg7_mask:
3130 case Intrinsic::riscv_vssseg8_mask:
3131 HasMask = true;
3132 [[fallthrough]];
3133 case Intrinsic::riscv_vlse:
3134 case Intrinsic::riscv_vsse:
3135 case Intrinsic::riscv_vlsseg2:
3136 case Intrinsic::riscv_vlsseg3:
3137 case Intrinsic::riscv_vlsseg4:
3138 case Intrinsic::riscv_vlsseg5:
3139 case Intrinsic::riscv_vlsseg6:
3140 case Intrinsic::riscv_vlsseg7:
3141 case Intrinsic::riscv_vlsseg8:
3142 case Intrinsic::riscv_vssseg2:
3143 case Intrinsic::riscv_vssseg3:
3144 case Intrinsic::riscv_vssseg4:
3145 case Intrinsic::riscv_vssseg5:
3146 case Intrinsic::riscv_vssseg6:
3147 case Intrinsic::riscv_vssseg7:
3148 case Intrinsic::riscv_vssseg8: {
3149 // Intrinsic interface:
3150 // riscv_vlse(merge, ptr, stride, vl)
3151 // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
3152 // riscv_vsse(val, ptr, stride, vl)
3153 // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
3154 // riscv_vlsseg#(merge, ptr, offset, vl, sew)
3155 // riscv_vlsseg#_mask(merge, ptr, offset, mask, vl, policy, sew)
3156 // riscv_vssseg#(val, ptr, offset, vl, sew)
3157 // riscv_vssseg#_mask(val, ptr, offset, mask, vl, sew)
3158 bool IsWrite = Inst->getType()->isVoidTy();
3159 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3160 // The results of segment loads are TargetExtType.
3161 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3162 unsigned SEW =
3163 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3164 ->getZExtValue();
3165 Ty = TarExtTy->getTypeParameter(i: 0U);
3166 Ty = ScalableVectorType::get(
3167 ElementType: IntegerType::get(C, NumBits: SEW),
3168 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3169 }
3170 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3171 unsigned VLIndex = RVVIInfo->VLOperand;
3172 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3173 MaybeAlign Alignment =
3174 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3175
3176 Value *Stride = Inst->getArgOperand(i: PtrOperandNo + 1);
3177 // Use the pointer alignment as the element alignment if the stride is a
3178 // multiple of the pointer alignment. Otherwise, the element alignment
3179 // should be the greatest common divisor of pointer alignment and stride.
3180 // For simplicity, just consider unalignment for elements.
3181 unsigned PointerAlign = Alignment.valueOrOne().value();
3182 if (!isa<ConstantInt>(Val: Stride) ||
3183 cast<ConstantInt>(Val: Stride)->getZExtValue() % PointerAlign != 0)
3184 Alignment = Align(1);
3185
3186 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3187 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3188 if (HasMask)
3189 Mask = Inst->getArgOperand(i: VLIndex - 1);
3190 Value *EVL = Inst->getArgOperand(i: VLIndex);
3191 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3192 // RVV uses contiguous elements as a segment.
3193 if (SegNum > 1) {
3194 unsigned ElemSize = Ty->getScalarSizeInBits();
3195 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3196 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3197 }
3198 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3199 Args&: Alignment, Args&: Mask, Args&: EVL, Args&: Stride);
3200 return true;
3201 }
3202 case Intrinsic::riscv_vloxei_mask:
3203 case Intrinsic::riscv_vluxei_mask:
3204 case Intrinsic::riscv_vsoxei_mask:
3205 case Intrinsic::riscv_vsuxei_mask:
3206 case Intrinsic::riscv_vloxseg2_mask:
3207 case Intrinsic::riscv_vloxseg3_mask:
3208 case Intrinsic::riscv_vloxseg4_mask:
3209 case Intrinsic::riscv_vloxseg5_mask:
3210 case Intrinsic::riscv_vloxseg6_mask:
3211 case Intrinsic::riscv_vloxseg7_mask:
3212 case Intrinsic::riscv_vloxseg8_mask:
3213 case Intrinsic::riscv_vluxseg2_mask:
3214 case Intrinsic::riscv_vluxseg3_mask:
3215 case Intrinsic::riscv_vluxseg4_mask:
3216 case Intrinsic::riscv_vluxseg5_mask:
3217 case Intrinsic::riscv_vluxseg6_mask:
3218 case Intrinsic::riscv_vluxseg7_mask:
3219 case Intrinsic::riscv_vluxseg8_mask:
3220 case Intrinsic::riscv_vsoxseg2_mask:
3221 case Intrinsic::riscv_vsoxseg3_mask:
3222 case Intrinsic::riscv_vsoxseg4_mask:
3223 case Intrinsic::riscv_vsoxseg5_mask:
3224 case Intrinsic::riscv_vsoxseg6_mask:
3225 case Intrinsic::riscv_vsoxseg7_mask:
3226 case Intrinsic::riscv_vsoxseg8_mask:
3227 case Intrinsic::riscv_vsuxseg2_mask:
3228 case Intrinsic::riscv_vsuxseg3_mask:
3229 case Intrinsic::riscv_vsuxseg4_mask:
3230 case Intrinsic::riscv_vsuxseg5_mask:
3231 case Intrinsic::riscv_vsuxseg6_mask:
3232 case Intrinsic::riscv_vsuxseg7_mask:
3233 case Intrinsic::riscv_vsuxseg8_mask:
3234 HasMask = true;
3235 [[fallthrough]];
3236 case Intrinsic::riscv_vloxei:
3237 case Intrinsic::riscv_vluxei:
3238 case Intrinsic::riscv_vsoxei:
3239 case Intrinsic::riscv_vsuxei:
3240 case Intrinsic::riscv_vloxseg2:
3241 case Intrinsic::riscv_vloxseg3:
3242 case Intrinsic::riscv_vloxseg4:
3243 case Intrinsic::riscv_vloxseg5:
3244 case Intrinsic::riscv_vloxseg6:
3245 case Intrinsic::riscv_vloxseg7:
3246 case Intrinsic::riscv_vloxseg8:
3247 case Intrinsic::riscv_vluxseg2:
3248 case Intrinsic::riscv_vluxseg3:
3249 case Intrinsic::riscv_vluxseg4:
3250 case Intrinsic::riscv_vluxseg5:
3251 case Intrinsic::riscv_vluxseg6:
3252 case Intrinsic::riscv_vluxseg7:
3253 case Intrinsic::riscv_vluxseg8:
3254 case Intrinsic::riscv_vsoxseg2:
3255 case Intrinsic::riscv_vsoxseg3:
3256 case Intrinsic::riscv_vsoxseg4:
3257 case Intrinsic::riscv_vsoxseg5:
3258 case Intrinsic::riscv_vsoxseg6:
3259 case Intrinsic::riscv_vsoxseg7:
3260 case Intrinsic::riscv_vsoxseg8:
3261 case Intrinsic::riscv_vsuxseg2:
3262 case Intrinsic::riscv_vsuxseg3:
3263 case Intrinsic::riscv_vsuxseg4:
3264 case Intrinsic::riscv_vsuxseg5:
3265 case Intrinsic::riscv_vsuxseg6:
3266 case Intrinsic::riscv_vsuxseg7:
3267 case Intrinsic::riscv_vsuxseg8: {
3268 // Intrinsic interface (only listed ordered version):
3269 // riscv_vloxei(merge, ptr, index, vl)
3270 // riscv_vloxei_mask(merge, ptr, index, mask, vl, policy)
3271 // riscv_vsoxei(val, ptr, index, vl)
3272 // riscv_vsoxei_mask(val, ptr, index, mask, vl, policy)
3273 // riscv_vloxseg#(merge, ptr, index, vl, sew)
3274 // riscv_vloxseg#_mask(merge, ptr, index, mask, vl, policy, sew)
3275 // riscv_vsoxseg#(val, ptr, index, vl, sew)
3276 // riscv_vsoxseg#_mask(val, ptr, index, mask, vl, sew)
3277 bool IsWrite = Inst->getType()->isVoidTy();
3278 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3279 // The results of segment loads are TargetExtType.
3280 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3281 unsigned SEW =
3282 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3283 ->getZExtValue();
3284 Ty = TarExtTy->getTypeParameter(i: 0U);
3285 Ty = ScalableVectorType::get(
3286 ElementType: IntegerType::get(C, NumBits: SEW),
3287 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3288 }
3289 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3290 unsigned VLIndex = RVVIInfo->VLOperand;
3291 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3292 Value *Mask;
3293 if (HasMask) {
3294 Mask = Inst->getArgOperand(i: VLIndex - 1);
3295 } else {
3296 // Mask cannot be nullptr here: vector GEP produces <vscale x N x ptr>,
3297 // and casting that to scalar i64 triggers a vector/scalar mismatch
3298 // assertion in CreatePointerCast. Use an all-true mask so ASan lowers it
3299 // via extractelement instead.
3300 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3301 Mask = ConstantInt::getTrue(Ty: MaskType);
3302 }
3303 Value *EVL = Inst->getArgOperand(i: VLIndex);
3304 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3305 // RVV uses contiguous elements as a segment.
3306 if (SegNum > 1) {
3307 unsigned ElemSize = Ty->getScalarSizeInBits();
3308 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3309 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3310 }
3311 Value *OffsetOp = Inst->getArgOperand(i: PtrOperandNo + 1);
3312 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3313 Args: Align(1), Args&: Mask, Args&: EVL,
3314 /* Stride */ Args: nullptr, Args&: OffsetOp);
3315 return true;
3316 }
3317 }
3318 return false;
3319}
3320
3321unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) const {
3322 if (Ty->isVectorTy()) {
3323 // f16 with only zvfhmin and bf16 will be promoted to f32
3324 Type *EltTy = cast<VectorType>(Val: Ty)->getElementType();
3325 if ((EltTy->isHalfTy() && !ST->hasVInstructionsF16()) ||
3326 EltTy->isBFloatTy())
3327 Ty = VectorType::get(ElementType: Type::getFloatTy(C&: Ty->getContext()),
3328 Other: cast<VectorType>(Val: Ty));
3329
3330 TypeSize Size = DL.getTypeSizeInBits(Ty);
3331 if (Size.isScalable() && ST->hasVInstructions())
3332 return divideCeil(Numerator: Size.getKnownMinValue(), Denominator: RISCV::RVVBitsPerBlock);
3333
3334 if (ST->useRVVForFixedLengthVectors())
3335 return divideCeil(Numerator: Size, Denominator: ST->getRealMinVLen());
3336 }
3337
3338 return BaseT::getRegUsageForType(Ty);
3339}
3340
3341unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
3342 if (SLPMaxVF.getNumOccurrences())
3343 return SLPMaxVF;
3344
3345 // Return how many elements can fit in getRegisterBitwidth. This is the
3346 // same routine as used in LoopVectorizer. We should probably be
3347 // accounting for whether we actually have instructions with the right
3348 // lane type, but we don't have enough information to do that without
3349 // some additional plumbing which hasn't been justified yet.
3350 TypeSize RegWidth =
3351 getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
3352 // If no vector registers, or absurd element widths, disable
3353 // vectorization by returning 1.
3354 return std::max<unsigned>(a: 1U, b: RegWidth.getFixedValue() / ElemWidth);
3355}
3356
3357unsigned RISCVTTIImpl::getMinTripCountTailFoldingThreshold() const {
3358 return RVVMinTripCount;
3359}
3360
3361bool RISCVTTIImpl::preferAlternateOpcodeVectorization() const {
3362 return ST->enableUnalignedVectorMem();
3363}
3364
3365TTI::AddressingModeKind
3366RISCVTTIImpl::getPreferredAddressingMode(const Loop *L,
3367 ScalarEvolution *SE) const {
3368 if (ST->hasVendorXCVmem() && !ST->is64Bit())
3369 return TTI::AMK_PostIndexed;
3370
3371 return BasicTTIImplBase::getPreferredAddressingMode(L, SE);
3372}
3373
3374bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
3375 const TargetTransformInfo::LSRCost &C2) const {
3376 // RISC-V specific here are "instruction number 1st priority".
3377 // If we need to emit adds inside the loop to add up base registers, then
3378 // we need at least one extra temporary register.
3379 unsigned C1NumRegs = C1.NumRegs + (C1.NumBaseAdds != 0);
3380 unsigned C2NumRegs = C2.NumRegs + (C2.NumBaseAdds != 0);
3381 return std::tie(args: C1.Insns, args&: C1NumRegs, args: C1.AddRecCost,
3382 args: C1.NumIVMuls, args: C1.NumBaseAdds,
3383 args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
3384 std::tie(args: C2.Insns, args&: C2NumRegs, args: C2.AddRecCost,
3385 args: C2.NumIVMuls, args: C2.NumBaseAdds,
3386 args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
3387}
3388
3389bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
3390 Align Alignment) const {
3391 auto *VTy = dyn_cast<VectorType>(Val: DataTy);
3392 if (!VTy || VTy->isScalableTy())
3393 return false;
3394
3395 if (!isLegalMaskedLoadStore(DataType: DataTy, Alignment))
3396 return false;
3397
3398 // FIXME: If it is an i8 vector and the element count exceeds 256, we should
3399 // scalarize these types with LMUL >= maximum fixed-length LMUL.
3400 if (VTy->getElementType()->isIntegerTy(BitWidth: 8))
3401 if (VTy->getElementCount().getFixedValue() > 256)
3402 return VTy->getPrimitiveSizeInBits() / ST->getRealMinVLen() <
3403 ST->getMaxLMULForFixedLengthVectors();
3404 return true;
3405}
3406
3407bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,
3408 Align Alignment) const {
3409 auto *VTy = dyn_cast<VectorType>(Val: DataTy);
3410 if (!VTy || VTy->isScalableTy())
3411 return false;
3412
3413 if (!isLegalMaskedLoadStore(DataType: DataTy, Alignment))
3414 return false;
3415 return true;
3416}
3417
3418bool RISCVTTIImpl::isLegalBroadcastLoad(Type *ElementTy,
3419 ElementCount NumElements) const {
3420 // Optimized zero-stride loads can be treated as broadcasts.
3421 if (!ST->hasVInstructions() || !ST->hasOptimizedZeroStrideLoad())
3422 return false;
3423
3424 return TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: ElementTy));
3425}
3426
3427/// See if \p I should be considered for address type promotion. We check if \p
3428/// I is a sext with right type and used in memory accesses. If it used in a
3429/// "complex" getelementptr, we allow it to be promoted without finding other
3430/// sext instructions that sign extended the same initial value. A getelementptr
3431/// is considered as "complex" if it has more than 2 operands.
3432bool RISCVTTIImpl::shouldConsiderAddressTypePromotion(
3433 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
3434 bool Considerable = false;
3435 AllowPromotionWithoutCommonHeader = false;
3436 if (!isa<SExtInst>(Val: &I))
3437 return false;
3438 Type *ConsideredSExtType =
3439 Type::getInt64Ty(C&: I.getParent()->getParent()->getContext());
3440 if (I.getType() != ConsideredSExtType)
3441 return false;
3442 // See if the sext is the one with the right type and used in at least one
3443 // GetElementPtrInst.
3444 for (const User *U : I.users()) {
3445 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(Val: U)) {
3446 Considerable = true;
3447 // A getelementptr is considered as "complex" if it has more than 2
3448 // operands. We will promote a SExt used in such complex GEP as we
3449 // expect some computation to be merged if they are done on 64 bits.
3450 if (GEPInst->getNumOperands() > 2) {
3451 AllowPromotionWithoutCommonHeader = true;
3452 break;
3453 }
3454 }
3455 }
3456 return Considerable;
3457}
3458
3459bool RISCVTTIImpl::canSplatOperand(unsigned Opcode, int Operand) const {
3460 switch (Opcode) {
3461 case Instruction::Add:
3462 case Instruction::Sub:
3463 case Instruction::Mul:
3464 case Instruction::And:
3465 case Instruction::Or:
3466 case Instruction::Xor:
3467 case Instruction::FAdd:
3468 case Instruction::FSub:
3469 case Instruction::FMul:
3470 case Instruction::FDiv:
3471 case Instruction::ICmp:
3472 case Instruction::FCmp:
3473 return true;
3474 case Instruction::Shl:
3475 case Instruction::LShr:
3476 case Instruction::AShr:
3477 case Instruction::UDiv:
3478 case Instruction::SDiv:
3479 case Instruction::URem:
3480 case Instruction::SRem:
3481 case Instruction::Select:
3482 return Operand == 1;
3483 default:
3484 return false;
3485 }
3486}
3487
3488bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {
3489 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3490 return false;
3491
3492 if (canSplatOperand(Opcode: I->getOpcode(), Operand))
3493 return true;
3494
3495 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3496 if (!II)
3497 return false;
3498
3499 switch (II->getIntrinsicID()) {
3500 case Intrinsic::fma:
3501 case Intrinsic::vp_fma:
3502 case Intrinsic::fmuladd:
3503 case Intrinsic::vp_fmuladd:
3504 return Operand == 0 || Operand == 1;
3505 case Intrinsic::vp_shl:
3506 case Intrinsic::vp_lshr:
3507 case Intrinsic::vp_ashr:
3508 case Intrinsic::vp_udiv:
3509 case Intrinsic::vp_sdiv:
3510 case Intrinsic::vp_urem:
3511 case Intrinsic::vp_srem:
3512 case Intrinsic::ssub_sat:
3513 case Intrinsic::vp_ssub_sat:
3514 case Intrinsic::usub_sat:
3515 case Intrinsic::vp_usub_sat:
3516 case Intrinsic::vp_select:
3517 return Operand == 1;
3518 // These intrinsics are commutative.
3519 case Intrinsic::vp_add:
3520 case Intrinsic::vp_mul:
3521 case Intrinsic::vp_and:
3522 case Intrinsic::vp_or:
3523 case Intrinsic::vp_xor:
3524 case Intrinsic::vp_fadd:
3525 case Intrinsic::vp_fmul:
3526 case Intrinsic::vp_icmp:
3527 case Intrinsic::vp_fcmp:
3528 case Intrinsic::smin:
3529 case Intrinsic::vp_smin:
3530 case Intrinsic::umin:
3531 case Intrinsic::vp_umin:
3532 case Intrinsic::smax:
3533 case Intrinsic::vp_smax:
3534 case Intrinsic::umax:
3535 case Intrinsic::vp_umax:
3536 case Intrinsic::sadd_sat:
3537 case Intrinsic::vp_sadd_sat:
3538 case Intrinsic::uadd_sat:
3539 case Intrinsic::vp_uadd_sat:
3540 // These intrinsics have 'vr' versions.
3541 case Intrinsic::vp_sub:
3542 case Intrinsic::vp_fsub:
3543 case Intrinsic::vp_fdiv:
3544 return Operand == 0 || Operand == 1;
3545 default:
3546 return false;
3547 }
3548}
3549
3550/// Check if sinking \p I's operands to I's basic block is profitable, because
3551/// the operands can be folded into a target instruction, e.g.
3552/// splats of scalars can fold into vector instructions.
3553bool RISCVTTIImpl::isProfitableToSinkOperands(
3554 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
3555 using namespace llvm::PatternMatch;
3556
3557 if (I->isBitwiseLogicOp()) {
3558 if (!I->getType()->isVectorTy()) {
3559 if (ST->hasStdExtZbb() || ST->hasStdExtZbkb()) {
3560 for (auto &Op : I->operands()) {
3561 // (and/or/xor X, (not Y)) -> (andn/orn/xnor X, Y)
3562 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3563 Ops.push_back(Elt: &Op);
3564 return true;
3565 }
3566 }
3567 }
3568 } else if (I->getOpcode() == Instruction::And && ST->hasStdExtZvkb()) {
3569 for (auto &Op : I->operands()) {
3570 // (and X, (not Y)) -> (vandn.vv X, Y)
3571 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3572 Ops.push_back(Elt: &Op);
3573 return true;
3574 }
3575 // (and X, (splat (not Y))) -> (vandn.vx X, Y)
3576 if (match(V: Op.get(), P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()),
3577 Idx: m_ZeroInt()),
3578 v2: m_Value(), mask: m_ZeroMask()))) {
3579 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
3580 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
3581 Ops.push_back(Elt: &Not);
3582 Ops.push_back(Elt: &InsertElt);
3583 Ops.push_back(Elt: &Op);
3584 return true;
3585 }
3586 }
3587 }
3588 }
3589
3590 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3591 return false;
3592
3593 // Don't sink splat operands if the target prefers it. Some targets requires
3594 // S2V transfer buffers and we can run out of them copying the same value
3595 // repeatedly.
3596 // FIXME: It could still be worth doing if it would improve vector register
3597 // pressure and prevent a vector spill.
3598 if (!ST->sinkSplatOperands())
3599 return false;
3600
3601 for (auto OpIdx : enumerate(First: I->operands())) {
3602 if (!canSplatOperand(I, Operand: OpIdx.index()))
3603 continue;
3604
3605 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
3606 // Make sure we are not already sinking this operand
3607 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
3608 continue;
3609
3610 // We are looking for a splat that can be sunk.
3611 if (!match(V: Op, P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ZeroInt()),
3612 v2: m_Value(), mask: m_ZeroMask())))
3613 continue;
3614
3615 // Don't sink i1 splats.
3616 if (cast<VectorType>(Val: Op->getType())->getElementType()->isIntegerTy(BitWidth: 1))
3617 continue;
3618
3619 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
3620 // and vector registers
3621 for (Use &U : Op->uses()) {
3622 Instruction *Insn = cast<Instruction>(Val: U.getUser());
3623 if (!canSplatOperand(I: Insn, Operand: U.getOperandNo()))
3624 return false;
3625 }
3626
3627 // Sink any fpexts since they might be used in a widening fp pattern.
3628 Use *InsertEltUse = &Op->getOperandUse(i: 0);
3629 auto *InsertElt = cast<InsertElementInst>(Val: InsertEltUse);
3630 if (isa<FPExtInst>(Val: InsertElt->getOperand(i_nocapture: 1)))
3631 Ops.push_back(Elt: &InsertElt->getOperandUse(i: 1));
3632 Ops.push_back(Elt: InsertEltUse);
3633 Ops.push_back(Elt: &OpIdx.value());
3634 }
3635 return true;
3636}
3637
3638RISCVTTIImpl::TTI::MemCmpExpansionOptions
3639RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
3640 TTI::MemCmpExpansionOptions Options;
3641
3642 if (!ST->hasStdExtZbb() && !ST->hasStdExtZbkb() && !IsZeroCmp)
3643 return Options;
3644
3645 // Even if the target does not support unaligned scalar memory access,
3646 // expansion is still possible when both pointers are statically known to be
3647 // sufficiently aligned. ExpandMemCmp queries the target for each load size
3648 // and keeps only the ones the target can actually access at the known
3649 // per-call-site alignment, falling back to the libcall when none fits.
3650 // Overlapping loads and merged tail expansions produce accesses that need
3651 // not be naturally aligned, so they are only offered when unaligned scalar
3652 // access is supported.
3653 bool UnalignedScalar = ST->enableUnalignedScalarMem();
3654 Options.AllowOverlappingLoads = UnalignedScalar;
3655 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
3656 Options.NumLoadsPerBlock = Options.MaxNumLoads;
3657 if (ST->is64Bit()) {
3658 Options.LoadSizes = {8, 4, 2, 1};
3659 if (UnalignedScalar)
3660 Options.AllowedTailExpansions = {3, 5, 6};
3661 } else {
3662 Options.LoadSizes = {4, 2, 1};
3663 if (UnalignedScalar)
3664 Options.AllowedTailExpansions = {3};
3665 }
3666
3667 if (IsZeroCmp && ST->hasVInstructions()) {
3668 unsigned VLenB = ST->getRealMinVLen() / 8;
3669 // The minimum size should be `XLen / 8 + 1`, and the maxinum size should be
3670 // `VLenB * MaxLMUL` so that it fits in a single register group.
3671 unsigned MinSize = ST->getXLen() / 8 + 1;
3672 unsigned MaxSize = VLenB * ST->getMaxLMULForFixedLengthVectors();
3673 for (unsigned Size = MinSize; Size <= MaxSize; Size++)
3674 Options.LoadSizes.insert(I: Options.LoadSizes.begin(), Elt: Size);
3675 }
3676 return Options;
3677}
3678
3679bool RISCVTTIImpl::shouldTreatInstructionLikeSelect(
3680 const Instruction *I) const {
3681 if (EnableOrLikeSelectOpt) {
3682 // For the binary operators (e.g. or) we need to be more careful than
3683 // selects, here we only transform them if they are already at a natural
3684 // break point in the code - the end of a block with an unconditional
3685 // terminator.
3686 if (I->getOpcode() == Instruction::Or &&
3687 isa<UncondBrInst>(Val: I->getNextNode()))
3688 return true;
3689
3690 if (I->getOpcode() == Instruction::Add ||
3691 I->getOpcode() == Instruction::Sub)
3692 return true;
3693 }
3694 return BaseT::shouldTreatInstructionLikeSelect(I);
3695}
3696
3697bool RISCVTTIImpl::shouldCopyAttributeWhenOutliningFrom(
3698 const Function *Caller, const Attribute &Attr) const {
3699 // "interrupt" controls the prolog/epilog of interrupt handlers (and includes
3700 // restrictions on their signatures). We can outline from the bodies of these
3701 // handlers, but when we do we need to make sure we don't mark the outlined
3702 // function as an interrupt handler too.
3703 if (Attr.isStringAttribute() && Attr.getKindAsString() == "interrupt")
3704 return false;
3705
3706 return BaseT::shouldCopyAttributeWhenOutliningFrom(Caller, Attr);
3707}
3708
3709std::optional<Instruction *>
3710RISCVTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
3711 // If all operands of a vmv.v.x are constant, fold a bitcast(vmv.v.x) to scale
3712 // the vmv.v.x, enabling removal of the bitcast. The transform helps avoid
3713 // creating redundant masks.
3714 const DataLayout &DL = IC.getDataLayout();
3715 if (II.user_empty())
3716 return {};
3717 auto *TargetVecTy = dyn_cast<ScalableVectorType>(Val: II.user_back()->getType());
3718 if (!TargetVecTy)
3719 return {};
3720 const APInt *Scalar;
3721 uint64_t VL;
3722 if (!match(V: &II, P: m_Intrinsic<Intrinsic::riscv_vmv_v_x>(
3723 Ops: m_Poison(), Ops: m_APInt(Res&: Scalar), Ops: m_ConstantInt(V&: VL))) ||
3724 !all_of(Range: II.users(), P: [TargetVecTy](User *U) {
3725 return U->getType() == TargetVecTy && match(V: U, P: m_BitCast(Op: m_Value()));
3726 }))
3727 return {};
3728 auto *SourceVecTy = cast<ScalableVectorType>(Val: II.getType());
3729 unsigned TargetEltBW = DL.getTypeSizeInBits(Ty: TargetVecTy->getElementType());
3730 unsigned SourceEltBW = DL.getTypeSizeInBits(Ty: SourceVecTy->getElementType());
3731 if (TargetEltBW % SourceEltBW)
3732 return {};
3733 unsigned TargetScale = TargetEltBW / SourceEltBW;
3734 if (VL % TargetScale || TargetScale == 1)
3735 return {};
3736 Type *VLTy = II.getOperand(i_nocapture: 2)->getType();
3737 ElementCount SourceEC = SourceVecTy->getElementCount();
3738 unsigned NewEltBW = SourceEltBW * TargetScale;
3739 if (!SourceEC.isKnownMultipleOf(RHS: TargetScale) ||
3740 !DL.fitsInLegalInteger(Width: NewEltBW))
3741 return {};
3742 auto *NewEltTy = IntegerType::get(C&: II.getContext(), NumBits: NewEltBW);
3743 if (!TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: NewEltTy)))
3744 return {};
3745 ElementCount NewEC = SourceEC.divideCoefficientBy(RHS: TargetScale);
3746 Type *RetTy = VectorType::get(ElementType: NewEltTy, EC: NewEC);
3747 assert(SourceVecTy->canLosslesslyBitCastTo(RetTy) &&
3748 "Lossless bitcast between types expected");
3749 APInt NewScalar = APInt::getSplat(NewLen: NewEltBW, V: *Scalar);
3750 return IC.replaceInstUsesWith(
3751 I&: II,
3752 V: IC.Builder.CreateBitCast(
3753 V: IC.Builder.CreateIntrinsic(
3754 RetTy, ID: Intrinsic::riscv_vmv_v_x,
3755 Args: {PoisonValue::get(T: RetTy), ConstantInt::get(Ty: NewEltTy, V: NewScalar),
3756 ConstantInt::get(Ty: VLTy, V: VL / TargetScale)}),
3757 DestTy: SourceVecTy));
3758}
3759