1//===-- RISCVTargetTransformInfo.cpp - RISC-V specific TTI ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "RISCVTargetTransformInfo.h"
10#include "MCTargetDesc/RISCVMatInt.h"
11#include "RISCVVectorUtils.h"
12#include "llvm/ADT/STLExtras.h"
13#include "llvm/Analysis/TargetTransformInfo.h"
14#include "llvm/CodeGen/BasicTTIImpl.h"
15#include "llvm/CodeGen/CostTable.h"
16#include "llvm/CodeGen/TargetLowering.h"
17#include "llvm/CodeGen/ValueTypes.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/IntrinsicsRISCV.h"
20#include "llvm/IR/PatternMatch.h"
21#include "llvm/Transforms/InstCombine/InstCombiner.h"
22#include <cmath>
23#include <limits>
24#include <optional>
25using namespace llvm;
26using namespace llvm::PatternMatch;
27
28#define DEBUG_TYPE "riscvtti"
29
30InstructionCost
31RISCVTTIImpl::getRISCVInstructionCost(ArrayRef<unsigned> OpCodes, MVT VT,
32 TTI::TargetCostKind CostKind) const {
33 // Check if the type is valid for all CostKind
34 if (!VT.isVector())
35 return InstructionCost::getInvalid();
36 size_t NumInstr = OpCodes.size();
37 if (CostKind == TTI::TCK_CodeSize)
38 return NumInstr;
39 InstructionCost LMULCost = TLI->getLMULCost(VT);
40 if ((CostKind != TTI::TCK_RecipThroughput) && (CostKind != TTI::TCK_Latency))
41 return LMULCost * NumInstr;
42 InstructionCost Cost = 0;
43 for (auto Op : OpCodes) {
44 switch (Op) {
45 case RISCV::VRGATHER_VI:
46 Cost += TLI->getVRGatherVICost(VT);
47 break;
48 case RISCV::VRGATHER_VV:
49 Cost += TLI->getVRGatherVVCost(VT);
50 break;
51 case RISCV::VSLIDEUP_VI:
52 case RISCV::VSLIDEDOWN_VI:
53 Cost += TLI->getVSlideVICost(VT);
54 break;
55 case RISCV::VSLIDEUP_VX:
56 case RISCV::VSLIDEDOWN_VX:
57 Cost += TLI->getVSlideVXCost(VT);
58 break;
59 case RISCV::VREDMAX_VS:
60 case RISCV::VREDMIN_VS:
61 case RISCV::VREDMAXU_VS:
62 case RISCV::VREDMINU_VS:
63 case RISCV::VREDSUM_VS:
64 case RISCV::VREDAND_VS:
65 case RISCV::VREDOR_VS:
66 case RISCV::VREDXOR_VS:
67 case RISCV::VFREDMAX_VS:
68 case RISCV::VFREDMIN_VS:
69 case RISCV::VFREDUSUM_VS: {
70 unsigned VL = VT.getVectorMinNumElements();
71 if (!VT.isFixedLengthVector())
72 VL *= *getVScaleForTuning();
73 Cost += Log2_32_Ceil(Value: VL);
74 break;
75 }
76 case RISCV::VFREDOSUM_VS: {
77 unsigned VL = VT.getVectorMinNumElements();
78 if (!VT.isFixedLengthVector())
79 VL *= *getVScaleForTuning();
80 Cost += VL;
81 break;
82 }
83 case RISCV::VMV_X_S:
84 case RISCV::VFMV_F_S:
85 // Domain crossings from vector -> scalar are usually more expensive.
86 Cost += 2;
87 break;
88 case RISCV::VMV_S_X:
89 case RISCV::VFMV_S_F:
90 case RISCV::VMOR_MM:
91 case RISCV::VMXOR_MM:
92 case RISCV::VMAND_MM:
93 case RISCV::VMANDN_MM:
94 case RISCV::VMNAND_MM:
95 case RISCV::VCPOP_M:
96 case RISCV::VFIRST_M:
97 Cost += 1;
98 break;
99 case RISCV::VDIV_VV:
100 case RISCV::VREM_VV:
101 Cost += LMULCost * TTI::TCC_Expensive;
102 break;
103 default:
104 Cost += LMULCost;
105 }
106 }
107 return Cost;
108}
109
110static InstructionCost getIntImmCostImpl(const DataLayout &DL,
111 const RISCVSubtarget *ST,
112 const APInt &Imm, Type *Ty,
113 TTI::TargetCostKind CostKind,
114 bool FreeZeroes) {
115 assert(Ty->isIntegerTy() &&
116 "getIntImmCost can only estimate cost of materialising integers");
117
118 // We have a Zero register, so 0 is always free.
119 if (Imm == 0)
120 return TTI::TCC_Free;
121
122 // Otherwise, we check how many instructions it will take to materialise.
123 return RISCVMatInt::getIntMatCost(Val: Imm, Size: DL.getTypeSizeInBits(Ty), STI: *ST,
124 /*CompressionCost=*/false, FreeZeroes);
125}
126
127InstructionCost
128RISCVTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
129 TTI::TargetCostKind CostKind) const {
130 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind, FreeZeroes: false);
131}
132
133// Look for patterns of shift followed by AND that can be turned into a pair of
134// shifts. We won't need to materialize an immediate for the AND so these can
135// be considered free.
136static bool canUseShiftPair(Instruction *Inst, const APInt &Imm) {
137 uint64_t Mask = Imm.getZExtValue();
138 auto *BO = dyn_cast<BinaryOperator>(Val: Inst->getOperand(i: 0));
139 if (!BO || !BO->hasOneUse())
140 return false;
141
142 if (BO->getOpcode() != Instruction::Shl)
143 return false;
144
145 if (!isa<ConstantInt>(Val: BO->getOperand(i_nocapture: 1)))
146 return false;
147
148 unsigned ShAmt = cast<ConstantInt>(Val: BO->getOperand(i_nocapture: 1))->getZExtValue();
149 // (and (shl x, c2), c1) will be matched to (srli (slli x, c2+c3), c3) if c1
150 // is a mask shifted by c2 bits with c3 leading zeros.
151 if (isShiftedMask_64(Value: Mask)) {
152 unsigned Trailing = llvm::countr_zero(Val: Mask);
153 if (ShAmt == Trailing)
154 return true;
155 }
156
157 return false;
158}
159
160// If this is i64 AND is part of (X & -(1 << C1) & 0xffffffff) == C2 << C1),
161// DAGCombiner can convert this to (sraiw X, C1) == sext(C2) for RV64. On RV32,
162// the type will be split so only the lower 32 bits need to be compared using
163// (srai/srli X, C) == C2.
164static bool canUseShiftCmp(Instruction *Inst, const APInt &Imm) {
165 if (!Inst->hasOneUse())
166 return false;
167
168 // Look for equality comparison.
169 auto *Cmp = dyn_cast<ICmpInst>(Val: *Inst->user_begin());
170 if (!Cmp || !Cmp->isEquality())
171 return false;
172
173 // Right hand side of comparison should be a constant.
174 auto *C = dyn_cast<ConstantInt>(Val: Cmp->getOperand(i_nocapture: 1));
175 if (!C)
176 return false;
177
178 uint64_t Mask = Imm.getZExtValue();
179
180 // Mask should be of the form -(1 << C) in the lower 32 bits.
181 if (!isUInt<32>(x: Mask) || !isPowerOf2_32(Value: -uint32_t(Mask)))
182 return false;
183
184 // Comparison constant should be a subset of Mask.
185 uint64_t CmpC = C->getZExtValue();
186 if ((CmpC & Mask) != CmpC)
187 return false;
188
189 // We'll need to sign extend the comparison constant and shift it right. Make
190 // sure the new constant can use addi/xori+seqz/snez.
191 unsigned ShiftBits = llvm::countr_zero(Val: Mask);
192 int64_t NewCmpC = SignExtend64<32>(x: CmpC) >> ShiftBits;
193 return NewCmpC >= -2048 && NewCmpC <= 2048;
194}
195
196InstructionCost RISCVTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
197 const APInt &Imm, Type *Ty,
198 TTI::TargetCostKind CostKind,
199 Instruction *Inst) const {
200 assert(Ty->isIntegerTy() &&
201 "getIntImmCost can only estimate cost of materialising integers");
202
203 // We have a Zero register, so 0 is always free.
204 if (Imm == 0)
205 return TTI::TCC_Free;
206
207 // Some instructions in RISC-V can take a 12-bit immediate. Some of these are
208 // commutative, in others the immediate comes from a specific argument index.
209 bool Takes12BitImm = false;
210 unsigned ImmArgIdx = ~0U;
211
212 switch (Opcode) {
213 case Instruction::GetElementPtr:
214 // Never hoist any arguments to a GetElementPtr. CodeGenPrepare will
215 // split up large offsets in GEP into better parts than ConstantHoisting
216 // can.
217 return TTI::TCC_Free;
218 case Instruction::Store: {
219 // Use the materialization cost regardless of if it's the address or the
220 // value that is constant, except for if the store is misaligned and
221 // misaligned accesses are not legal (experience shows constant hoisting
222 // can sometimes be harmful in such cases).
223 if (Idx == 1 || !Inst)
224 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
225 /*FreeZeroes=*/true);
226
227 StoreInst *ST = cast<StoreInst>(Val: Inst);
228 if (!getTLI()->allowsMemoryAccessForAlignment(
229 Context&: Ty->getContext(), DL, VT: getTLI()->getValueType(DL, Ty),
230 AddrSpace: ST->getPointerAddressSpace(), Alignment: ST->getAlign()))
231 return TTI::TCC_Free;
232
233 return getIntImmCostImpl(DL: getDataLayout(), ST: getST(), Imm, Ty, CostKind,
234 /*FreeZeroes=*/true);
235 }
236 case Instruction::Load:
237 // If the address is a constant, use the materialization cost.
238 return getIntImmCost(Imm, Ty, CostKind);
239 case Instruction::And:
240 // zext.h
241 if (Imm == UINT64_C(0xffff) && ST->hasStdExtZbb())
242 return TTI::TCC_Free;
243 // zext.w
244 if (Imm == UINT64_C(0xffffffff) && (!ST->is64Bit() || ST->hasStdExtZba()))
245 return TTI::TCC_Free;
246 // bclri
247 if (ST->hasStdExtZbs() && (~Imm).isPowerOf2())
248 return TTI::TCC_Free;
249 if (Inst && Idx == 1 && Imm.getBitWidth() <= ST->getXLen() &&
250 canUseShiftPair(Inst, Imm))
251 return TTI::TCC_Free;
252 if (Inst && Idx == 1 && Imm.getBitWidth() == 64 &&
253 canUseShiftCmp(Inst, Imm))
254 return TTI::TCC_Free;
255 Takes12BitImm = true;
256 break;
257 case Instruction::Add:
258 Takes12BitImm = true;
259 break;
260 case Instruction::Or:
261 case Instruction::Xor:
262 // bseti/binvi
263 if (ST->hasStdExtZbs() && Imm.isPowerOf2())
264 return TTI::TCC_Free;
265 Takes12BitImm = true;
266 break;
267 case Instruction::Mul:
268 // Power of 2 is a shift. Negated power of 2 is a shift and a negate.
269 if (Imm.isPowerOf2() || Imm.isNegatedPowerOf2())
270 return TTI::TCC_Free;
271 // One more or less than a power of 2 can use SLLI+ADD/SUB.
272 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2())
273 return TTI::TCC_Free;
274 // FIXME: There is no MULI instruction.
275 Takes12BitImm = true;
276 break;
277 case Instruction::Sub:
278 case Instruction::Shl:
279 case Instruction::LShr:
280 case Instruction::AShr:
281 Takes12BitImm = true;
282 ImmArgIdx = 1;
283 break;
284 default:
285 break;
286 }
287
288 if (Takes12BitImm) {
289 // Check immediate is the correct argument...
290 if (Instruction::isCommutative(Opcode) || Idx == ImmArgIdx) {
291 // ... and fits into the 12-bit immediate.
292 if (Imm.getSignificantBits() <= 64 &&
293 getTLI()->isLegalAddImmediate(Imm: Imm.getSExtValue())) {
294 return TTI::TCC_Free;
295 }
296 }
297
298 // Otherwise, use the full materialisation cost.
299 return getIntImmCost(Imm, Ty, CostKind);
300 }
301
302 // By default, prevent hoisting.
303 return TTI::TCC_Free;
304}
305
306bool RISCVTTIImpl::preferLoopExitWithEqualityComparison(
307 const ICmpInst &Cond, const APInt &PostIncExitValue) const {
308 if (!ST->hasStdExtZibi())
309 return false;
310
311 // Relational comparisons against zero and some adjacent constants can
312 // already lower to a single branch against x0.
313 const auto Pred = Cond.getPredicate();
314 const auto *C = dyn_cast<ConstantInt>(Val: Cond.getOperand(i_nocapture: 1));
315 if (C && (C->isZero() ||
316 (C->isMinusOne() &&
317 (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE)) ||
318 (C->isOne() &&
319 (Pred == ICmpInst::ICMP_SLT || Pred == ICmpInst::ICMP_SGE ||
320 Pred == ICmpInst::ICMP_ULT || Pred == ICmpInst::ICMP_UGE))))
321 return false;
322
323 int64_t Imm = PostIncExitValue.getSExtValue();
324 return Imm == -1 || (Imm >= 1 && Imm <= 31);
325}
326
327InstructionCost
328RISCVTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
329 const APInt &Imm, Type *Ty,
330 TTI::TargetCostKind CostKind) const {
331 // Prevent hoisting in unknown cases.
332 return TTI::TCC_Free;
333}
334
335bool RISCVTTIImpl::hasActiveVectorLength() const {
336 return ST->hasVInstructions();
337}
338
339TargetTransformInfo::PopcntSupportKind
340RISCVTTIImpl::getPopcntSupport(unsigned TyWidth) const {
341 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
342 return ST->hasCPOPLike() ? TTI::PSK_FastHardware : TTI::PSK_Software;
343}
344
345InstructionCost RISCVTTIImpl::getPartialReductionCost(
346 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
347 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
348 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
349 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
350 if (Opcode == Instruction::FAdd)
351 return InstructionCost::getInvalid();
352
353 // zve32x is broken for partial_reduce_umla, but let's make sure we
354 // don't generate them.
355 // vdot4a* reduces four i8 products into an i32 result; an i64 accumulator is
356 // additionally supported by widening the i32 partial sums to i64 (see
357 // lowerPARTIAL_REDUCE_MLA). VF is the number of i8 input elements, so the
358 // reduction factor is AccumBits / 8 (4 for i32, 8 for i64).
359 if (!ST->hasStdExtZvdot4a8i() || ST->getELen() < 64 ||
360 Opcode != Instruction::Add || !BinOp || *BinOp != Instruction::Mul ||
361 InputTypeA != InputTypeB || !InputTypeA->isIntegerTy(BitWidth: 8) ||
362 (!AccumType->isIntegerTy(BitWidth: 32) && !AccumType->isIntegerTy(BitWidth: 64)))
363 return InstructionCost::getInvalid();
364
365 unsigned Ratio = AccumType->getScalarSizeInBits() / 8;
366 if (!VF.isKnownMultipleOf(RHS: Ratio))
367 return InstructionCost::getInvalid();
368
369 // Cost of the vdot4a* itself, which operates on the i32 intermediate type
370 // holding VF/4 elements.
371 Type *DotTp = VectorType::get(ElementType: Type::getInt32Ty(C&: AccumType->getContext()),
372 EC: VF.divideCoefficientBy(RHS: 4));
373 std::pair<InstructionCost, MVT> DotLT = getTypeLegalizationCost(Ty: DotTp);
374 // Note: Asuming all vdot4a* variants are equal cost
375 InstructionCost Cost =
376 DotLT.first *
377 getRISCVInstructionCost(OpCodes: RISCV::VDOT4A_VV, VT: DotLT.second, CostKind);
378
379 // Account for reducing the i32 partial sums down to the i64 accumulator's
380 // element count and accumulating into it (see lowerPARTIAL_REDUCE_MLA), which
381 // has two shapes depending on the accumulator's LMUL.
382 if (AccumType->isIntegerTy(BitWidth: 64)) {
383 LLVMContext &Ctx = AccumType->getContext();
384 Type *I32Ty = Type::getInt32Ty(C&: Ctx);
385 ElementCount AccVF = VF.divideCoefficientBy(RHS: Ratio);
386 std::pair<InstructionCost, MVT> AccLT =
387 getTypeLegalizationCost(Ty: VectorType::get(ElementType: AccumType, EC: AccVF));
388
389 // When the i32 subvectors of a single-vector scalable accumulator are a
390 // fractional LMUL, extracting the high subvector would need a vslidedown,
391 // so instead the i32 dot result is widened to i64 first (vsext.vf2 /
392 // vzext.vf2) and then reduced and accumulated with register-aligned i64
393 // vadd.vv.
394 bool WidenFirst = false;
395 if (VF.isScalable() && AccLT.second.isScalableVector()) {
396 MVT NarrowMVT = AccLT.second.changeVectorElementType(EltVT: MVT::i32);
397 WidenFirst =
398 RISCVVType::decodeVLMUL(VLMul: RISCVTargetLowering::getLMUL(VT: NarrowMVT))
399 .second;
400 }
401
402 if (WidenFirst) {
403 // The widened i64 dot result has VF/4 elements, i.e. twice the
404 // accumulator's element count, so the reduction plus the accumulate are
405 // two i64 vadd.vv.
406 std::pair<InstructionCost, MVT> WideLT = getTypeLegalizationCost(
407 Ty: VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: 4)));
408 Cost +=
409 WideLT.first * getRISCVInstructionCost(OpCodes: RISCV::VSEXT_VF2,
410 VT: WideLT.second, CostKind) +
411 2 * AccLT.first *
412 getRISCVInstructionCost(OpCodes: RISCV::VADD_VV, VT: AccLT.second, CostKind);
413 } else {
414 // Otherwise the scale-4 i32 sums are halved with a single i32 vadd.vv,
415 // then widened and added into the i64 result with a vwadd.wv.
416 std::pair<InstructionCost, MVT> RedLT =
417 getTypeLegalizationCost(Ty: VectorType::get(ElementType: I32Ty, EC: AccVF));
418 Cost += RedLT.first * getRISCVInstructionCost(OpCodes: RISCV::VADD_VV,
419 VT: RedLT.second, CostKind) +
420 AccLT.first * getRISCVInstructionCost(OpCodes: RISCV::VWADD_WV,
421 VT: AccLT.second, CostKind);
422 // Fixed-length vectors extract the high i32 subvector with a vslidedown.
423 if (VF.isFixed())
424 Cost += DotLT.first * getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI,
425 VT: DotLT.second, CostKind);
426 }
427 }
428
429 return Cost;
430}
431
432bool RISCVTTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
433 // Currently, the ExpandReductions pass can't expand scalable-vector
434 // reductions, but we still request expansion as RVV doesn't support certain
435 // reductions and the SelectionDAG can't legalize them either.
436 switch (II->getIntrinsicID()) {
437 default:
438 return false;
439 // These reductions have no equivalent in RVV
440 case Intrinsic::vector_reduce_mul:
441 case Intrinsic::vector_reduce_fmul:
442 return true;
443 }
444}
445
446std::optional<unsigned> RISCVTTIImpl::getVScaleForTuning() const {
447 if (ST->hasVInstructions())
448 if (unsigned MinVLen = ST->getRealMinVLen();
449 MinVLen >= RISCV::RVVBitsPerBlock)
450 return MinVLen / RISCV::RVVBitsPerBlock;
451 return BaseT::getVScaleForTuning();
452}
453
454TypeSize
455RISCVTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
456 unsigned LMUL = llvm::bit_floor(
457 Value: std::clamp<unsigned>(val: ST->getCLOpts().v_register_bit_width_lmul, lo: 1, hi: 8));
458 switch (K) {
459 case TargetTransformInfo::RGK_Scalar:
460 return TypeSize::getFixed(ExactSize: ST->getXLen());
461 case TargetTransformInfo::RGK_FixedWidthVector:
462 return TypeSize::getFixed(
463 ExactSize: ST->useRVVForFixedLengthVectors() ? LMUL * ST->getRealMinVLen() : 0);
464 case TargetTransformInfo::RGK_ScalableVector:
465 return TypeSize::getScalable(
466 MinimumSize: (ST->hasVInstructions() &&
467 ST->getRealMinVLen() >= RISCV::RVVBitsPerBlock)
468 ? LMUL * RISCV::RVVBitsPerBlock
469 : 0);
470 }
471
472 llvm_unreachable("Unsupported register kind");
473}
474
475InstructionCost RISCVTTIImpl::getStaticDataAddrGenerationCost(
476 const TTI::TargetCostKind CostKind) const {
477 switch (CostKind) {
478 case TTI::TCK_CodeSize:
479 case TTI::TCK_SizeAndLatency:
480 // Always 2 instructions
481 return 2;
482 case TTI::TCK_Latency:
483 case TTI::TCK_RecipThroughput:
484 // Depending on the memory model the address generation will
485 // require AUIPC + ADDI (medany) or LUI + ADDI (medlow). Don't
486 // have a way of getting this information here, so conservatively
487 // require both.
488 // In practice, these are generally implemented together.
489 return (ST->hasAUIPCADDIFusion() && ST->hasLUIADDIFusion()) ? 1 : 2;
490 }
491 llvm_unreachable("Unsupported cost kind");
492}
493
494InstructionCost
495RISCVTTIImpl::getConstantPoolLoadCost(Type *Ty,
496 TTI::TargetCostKind CostKind) const {
497 // Add a cost of address generation + the cost of the load. The address
498 // is expected to be a PC relative offset to a constant pool entry
499 // using auipc/addi.
500 InstructionCost Cost = 0;
501 Cost = getStaticDataAddrGenerationCost(CostKind) +
502 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: DL.getABITypeAlign(Ty),
503 /*AddressSpace=*/0, CostKind);
504 // Estimate the amount of 4 byte instructions that could fit
505 // instead of the constant pool, ignoring any extra padding.
506 if (CostKind == TTI::TCK_CodeSize)
507 Cost += ((InstructionCost)DL.getTypeAllocSize(Ty)) / 4;
508 return Cost;
509}
510
511static bool isRepeatedConcatMask(ArrayRef<int> Mask, int &SubVectorSize) {
512 unsigned Size = Mask.size();
513 if (!isPowerOf2_32(Value: Size))
514 return false;
515 for (unsigned I = 0; I != Size; ++I) {
516 if (static_cast<unsigned>(Mask[I]) == I)
517 continue;
518 if (Mask[I] != 0)
519 return false;
520 if (Size % I != 0)
521 return false;
522 for (unsigned J = I + 1; J != Size; ++J)
523 // Check the pattern is repeated.
524 if (static_cast<unsigned>(Mask[J]) != J % I)
525 return false;
526 SubVectorSize = I;
527 return true;
528 }
529 // That means Mask is <0, 1, 2, 3>. This is not a concatenation.
530 return false;
531}
532
533static VectorType *getVRGatherIndexType(MVT DataVT, const RISCVSubtarget &ST,
534 LLVMContext &C) {
535 assert((DataVT.getScalarSizeInBits() != 8 ||
536 DataVT.getVectorNumElements() <= 256) && "unhandled case in lowering");
537 MVT IndexVT = DataVT.changeTypeToInteger();
538 if (IndexVT.getScalarType().bitsGT(VT: ST.getXLenVT()))
539 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
540 return cast<VectorType>(Val: EVT(IndexVT).getTypeForEVT(Context&: C));
541}
542
543/// Attempt to approximate the cost of a shuffle which will require splitting
544/// during legalization. Note that processShuffleMasks is not an exact proxy
545/// for the algorithm used in LegalizeVectorTypes, but hopefully it's a
546/// reasonably close upperbound.
547static InstructionCost costShuffleViaSplitting(const RISCVTTIImpl &TTI,
548 MVT LegalVT, VectorType *Tp,
549 ArrayRef<int> Mask,
550 TTI::TargetCostKind CostKind) {
551 assert(LegalVT.isFixedLengthVector() && !Mask.empty() &&
552 "Expected fixed vector type and non-empty mask");
553 unsigned LegalNumElts = LegalVT.getVectorNumElements();
554 // Number of destination vectors after legalization:
555 unsigned NumOfDests = divideCeil(Numerator: Mask.size(), Denominator: LegalNumElts);
556 // We are going to permute multiple sources and the result will be in
557 // multiple destinations. Providing an accurate cost only for splits where
558 // the element type remains the same.
559 if (NumOfDests <= 1 ||
560 LegalVT.getVectorElementType().getSizeInBits() !=
561 Tp->getElementType()->getPrimitiveSizeInBits() ||
562 LegalNumElts >= Tp->getElementCount().getFixedValue())
563 return InstructionCost::getInvalid();
564
565 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
566 unsigned LegalVTSize = LegalVT.getStoreSize();
567 // Number of source vectors after legalization:
568 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
569
570 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: LegalNumElts);
571
572 unsigned NormalizedVF = LegalNumElts * std::max(a: NumOfSrcs, b: NumOfDests);
573 unsigned NumOfSrcRegs = NormalizedVF / LegalNumElts;
574 unsigned NumOfDestRegs = NormalizedVF / LegalNumElts;
575 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
576 assert(NormalizedVF >= Mask.size() &&
577 "Normalized mask expected to be not shorter than original mask.");
578 copy(Range&: Mask, Out: NormalizedMask.begin());
579 InstructionCost Cost = 0;
580 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
581 processShuffleMasks(
582 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
583 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
584 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
585 return;
586 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
587 .second)
588 return;
589 Cost += TTI.getShuffleCost(
590 Kind: TTI::SK_PermuteSingleSrc,
591 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
592 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
593 },
594 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
595 Cost += TTI.getShuffleCost(
596 Kind: TTI::SK_PermuteTwoSrc,
597 DstTy: FixedVectorType::get(ElementType: SingleOpTy->getElementType(), NumElts: RegMask.size()),
598 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
599 });
600 return Cost;
601}
602
603/// Try to perform better estimation of the permutation.
604/// 1. Split the source/destination vectors into real registers.
605/// 2. Do the mask analysis to identify which real registers are
606/// permuted. If more than 1 source registers are used for the
607/// destination register building, the cost for this destination register
608/// is (Number_of_source_register - 1) * Cost_PermuteTwoSrc. If only one
609/// source register is used, build mask and calculate the cost as a cost
610/// of PermuteSingleSrc.
611/// Also, for the single register permute we try to identify if the
612/// destination register is just a copy of the source register or the
613/// copy of the previous destination register (the cost is
614/// TTI::TCC_Basic). If the source register is just reused, the cost for
615/// this operation is 0.
616static InstructionCost
617costShuffleViaVRegSplitting(const RISCVTTIImpl &TTI, MVT LegalVT,
618 std::optional<unsigned> VLen, VectorType *Tp,
619 ArrayRef<int> Mask, TTI::TargetCostKind CostKind) {
620 assert(LegalVT.isFixedLengthVector());
621 if (!VLen || Mask.empty())
622 return InstructionCost::getInvalid();
623 MVT ElemVT = LegalVT.getVectorElementType();
624 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
625 LegalVT = TTI.getTypeLegalizationCost(
626 Ty: FixedVectorType::get(ElementType: Tp->getElementType(), NumElts: ElemsPerVReg))
627 .second;
628 // Number of destination vectors after legalization:
629 InstructionCost NumOfDests =
630 divideCeil(Numerator: Mask.size(), Denominator: LegalVT.getVectorNumElements());
631 if (NumOfDests <= 1 ||
632 LegalVT.getVectorElementType().getSizeInBits() !=
633 Tp->getElementType()->getPrimitiveSizeInBits() ||
634 LegalVT.getVectorNumElements() >= Tp->getElementCount().getFixedValue())
635 return InstructionCost::getInvalid();
636
637 unsigned VecTySize = TTI.getDataLayout().getTypeStoreSize(Ty: Tp);
638 unsigned LegalVTSize = LegalVT.getStoreSize();
639 // Number of source vectors after legalization:
640 unsigned NumOfSrcs = divideCeil(Numerator: VecTySize, Denominator: LegalVTSize);
641
642 auto *SingleOpTy = FixedVectorType::get(ElementType: Tp->getElementType(),
643 NumElts: LegalVT.getVectorNumElements());
644
645 unsigned E = NumOfDests.getValue();
646 unsigned NormalizedVF =
647 LegalVT.getVectorNumElements() * std::max(a: NumOfSrcs, b: E);
648 unsigned NumOfSrcRegs = NormalizedVF / LegalVT.getVectorNumElements();
649 unsigned NumOfDestRegs = NormalizedVF / LegalVT.getVectorNumElements();
650 SmallVector<int> NormalizedMask(NormalizedVF, PoisonMaskElem);
651 assert(NormalizedVF >= Mask.size() &&
652 "Normalized mask expected to be not shorter than original mask.");
653 copy(Range&: Mask, Out: NormalizedMask.begin());
654 InstructionCost Cost = 0;
655 int NumShuffles = 0;
656 SmallDenseSet<std::pair<ArrayRef<int>, unsigned>> ReusedSingleSrcShuffles;
657 processShuffleMasks(
658 Mask: NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs, NoInputAction: []() {},
659 SingleInputAction: [&](ArrayRef<int> RegMask, unsigned SrcReg, unsigned DestReg) {
660 if (ShuffleVectorInst::isIdentityMask(Mask: RegMask, NumSrcElts: RegMask.size()))
661 return;
662 if (!ReusedSingleSrcShuffles.insert(V: std::make_pair(x&: RegMask, y&: SrcReg))
663 .second)
664 return;
665 ++NumShuffles;
666 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: SingleOpTy,
667 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
668 },
669 ManyInputsAction: [&](ArrayRef<int> RegMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
670 Cost += TTI.getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: SingleOpTy,
671 SrcTy: SingleOpTy, CostKind, Mask: RegMask, Index: 0, SubTp: nullptr);
672 NumShuffles += 2;
673 });
674 // Note: check that we do not emit too many shuffles here to prevent code
675 // size explosion.
676 // TODO: investigate, if it can be improved by extra analysis of the masks
677 // to check if the code is more profitable.
678 if ((NumOfDestRegs > 2 && NumShuffles <= static_cast<int>(NumOfDestRegs)) ||
679 (NumOfDestRegs <= 2 && NumShuffles < 4))
680 return Cost;
681 return InstructionCost::getInvalid();
682}
683
684InstructionCost RISCVTTIImpl::getSlideCost(FixedVectorType *Tp,
685 ArrayRef<int> Mask,
686 TTI::TargetCostKind CostKind) const {
687 // Avoid missing masks and length changing shuffles
688 if (Mask.size() <= 2 || Mask.size() != Tp->getNumElements())
689 return InstructionCost::getInvalid();
690
691 int NumElts = Tp->getNumElements();
692 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Tp);
693 // Avoid scalarization cases
694 if (!LT.second.isFixedLengthVector())
695 return InstructionCost::getInvalid();
696
697 // Requires moving elements between parts, which requires additional
698 // unmodeled instructions.
699 if (LT.first != 1)
700 return InstructionCost::getInvalid();
701
702 auto GetSlideOpcode = [&](int SlideAmt) {
703 assert(SlideAmt != 0);
704 bool IsVI = isUInt<5>(x: std::abs(x: SlideAmt));
705 if (SlideAmt < 0)
706 return IsVI ? RISCV::VSLIDEDOWN_VI : RISCV::VSLIDEDOWN_VX;
707 return IsVI ? RISCV::VSLIDEUP_VI : RISCV::VSLIDEUP_VX;
708 };
709
710 std::array<std::pair<int, int>, 2> SrcInfo;
711 if (!isMaskedSlidePair(Mask, NumElts, SrcInfo))
712 return InstructionCost::getInvalid();
713
714 if (SrcInfo[1].second == 0)
715 std::swap(x&: SrcInfo[0], y&: SrcInfo[1]);
716
717 if (ST->hasStdExtZvzip() && LT.second.getScalarSizeInBits() != 1) {
718 unsigned Factor;
719 if (isPairEven(SrcInfo, Mask, Factor) && Factor == 1)
720 return getRISCVInstructionCost(OpCodes: RISCV::VPAIRE_VV, VT: LT.second, CostKind);
721 if (isPairOdd(SrcInfo, Mask, Factor) && Factor == 1)
722 return getRISCVInstructionCost(OpCodes: RISCV::VPAIRO_VV, VT: LT.second, CostKind);
723 }
724
725 InstructionCost FirstSlideCost = 0;
726 if (SrcInfo[0].second != 0) {
727 unsigned Opcode = GetSlideOpcode(SrcInfo[0].second);
728 FirstSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
729 }
730
731 if (SrcInfo[1].first == -1)
732 return FirstSlideCost;
733
734 InstructionCost SecondSlideCost = 0;
735 if (SrcInfo[1].second != 0) {
736 unsigned Opcode = GetSlideOpcode(SrcInfo[1].second);
737 SecondSlideCost = getRISCVInstructionCost(OpCodes: Opcode, VT: LT.second, CostKind);
738 } else {
739 SecondSlideCost =
740 getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second, CostKind);
741 }
742
743 auto EC = Tp->getElementCount();
744 VectorType *MaskTy =
745 VectorType::get(ElementType: IntegerType::getInt1Ty(C&: Tp->getContext()), EC);
746 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
747 return FirstSlideCost + SecondSlideCost + MaskCost;
748}
749
750MVT RISCVTTIImpl::getZvzipVZIPCostVT(MVT InterleavedVT) const {
751 if (!InterleavedVT.getVectorElementCount().isKnownEven())
752 return MVT::INVALID_SIMPLE_VALUE_TYPE;
753
754 MVT CostVT = InterleavedVT;
755 if (InterleavedVT.isFixedLengthVector()) {
756 MVT SourceVT = InterleavedVT.getHalfNumVectorElementsVT();
757 CostVT = TLI->getContainerForFixedLengthVector(VT: SourceVT)
758 .getDoubleNumVectorElementsVT();
759 }
760
761 unsigned EltBits = CostVT.getScalarSizeInBits();
762 unsigned MinSize = CostVT.getSizeInBits().getKnownMinValue();
763 unsigned LMULOctuple = MinSize / (RISCV::RVVBitsPerBlock / 8);
764 // Perform the 2 * SEW <= LMUL * min(ELEN, VLEN) check.
765 if (EltBits * 16 >
766 LMULOctuple * std::min(a: ST->getELen(), b: ST->getRealMinVLen()))
767 return MVT::INVALID_SIMPLE_VALUE_TYPE;
768 return CostVT;
769}
770
771MVT RISCVTTIImpl::getZvzipVUNZIPCostVT(MVT InterleavedVT) const {
772 if (!InterleavedVT.getVectorElementCount().isKnownEven())
773 return MVT::INVALID_SIMPLE_VALUE_TYPE;
774
775 MVT CostVT = InterleavedVT;
776 // lowerZvzipVUNZIP widens the source container if halving it would produce
777 // an illegal result type. Apply the same rule here so the cost uses the
778 // source LMUL selected by ISel.
779 if (InterleavedVT.isFixedLengthVector()) {
780 CostVT = TLI->getContainerForFixedLengthVector(VT: InterleavedVT);
781 if (CostVT.getVectorMinNumElements() == 1 ||
782 !TLI->isTypeLegal(VT: CostVT.getHalfNumVectorElementsVT()))
783 CostVT = CostVT.getDoubleNumVectorElementsVT();
784 }
785
786 MVT DeinterleavedVT = CostVT.getHalfNumVectorElementsVT();
787 if (RISCVTargetLowering::getLMUL(VT: DeinterleavedVT) == RISCVVType::LMUL_8)
788 return MVT::INVALID_SIMPLE_VALUE_TYPE;
789 return CostVT;
790}
791
792InstructionCost RISCVTTIImpl::getVZIPCost(TTI::ShuffleKind Kind,
793 VectorType *DstTy, VectorType *SrcTy,
794 ArrayRef<int> Mask,
795 TTI::TargetCostKind CostKind) const {
796 assert(isa<FixedVectorType>(SrcTy) && "Expected fixed-length vector types");
797
798 if (!ST->hasStdExtZvzip() || Mask.size() < 2 ||
799 DstTy->getScalarSizeInBits() == 1)
800 return InstructionCost::getInvalid();
801
802 unsigned NumSrcElts = SrcTy->getElementCount().getKnownMinValue();
803 unsigned NumDstElts = DstTy->getElementCount().getKnownMinValue();
804 unsigned NumInputElts;
805
806 if (Kind == TTI::SK_PermuteSingleSrc) {
807 // A single-source vzip shuffle interleaves two halves of the same vector,
808 // so its source and destination must have the same number of elements.
809 if (NumSrcElts != NumDstElts)
810 return InstructionCost::getInvalid();
811 NumInputElts = NumSrcElts;
812 } else if (Kind == TTI::SK_PermuteTwoSrc) {
813 // Accept both forms that can be lowered with vzip.vv:
814 //
815 // 1. The destination has twice as many elements as each source, e.g.
816 // %r = shufflevector <4 x i32> %a, <4 x i32> %b,
817 // <8 x i32> <0, 4, 1, 5, 2, 6, 3, 7>
818 // The two operands are already the half-sized inputs of vzip.vv.
819 //
820 // 2. Each source has the same number of elements as the destination, e.g.
821 // %r = shufflevector <4 x i32> %a, <4 x i32> %b,
822 // <4 x i32> <0, 4, 1, 5>
823 // ISel first extracts a <2 x i32> half from each source and then uses
824 // those halves as the inputs of vzip.vv.
825 if (NumSrcElts != NumDstElts && NumSrcElts * 2 != NumDstElts)
826 return InstructionCost::getInvalid();
827 NumInputElts = NumSrcElts * 2;
828 } else {
829 return InstructionCost::getInvalid();
830 }
831
832 // ISel deliberately excludes the two-element single-source identity case.
833 if (Kind == TTI::SK_PermuteSingleSrc && Mask.size() == 2 &&
834 ShuffleVectorInst::isSingleSourceMask(Mask, NumSrcElts: Mask.size()))
835 return InstructionCost::getInvalid();
836
837 // A deinterleave shuffle for an interleaved load may have a mask such as
838 // %even = shufflevector <4 x i32> %wide, <4 x i32> poison,
839 // <4 x i32> <i32 0, i32 2, i32 poison, i32 poison>
840 // Because isInterleaveMask ignores poison lanes, this mask may also satisfy
841 // the interleave matcher. Do not cost it as vzip.vv.
842 unsigned DeinterleaveIndex;
843 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2,
844 Index&: DeinterleaveIndex) &&
845 count_if(Range&: Mask, P: [](int Idx) { return Idx >= 0; }) > 1)
846 return InstructionCost::getInvalid();
847
848 // Require the two start indices found in the mask to be aligned to
849 // half-vector boundaries, with at least one equal to zero.
850 SmallVector<unsigned, 2> StartIndexes;
851 if (!ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts, StartIndexes))
852 return InstructionCost::getInvalid();
853
854 unsigned HalfNumElts = Mask.size() / 2;
855 if ((StartIndexes[0] != 0 && StartIndexes[1] != 0) ||
856 StartIndexes[0] % HalfNumElts != 0 || StartIndexes[1] % HalfNumElts != 0)
857 return InstructionCost::getInvalid();
858
859 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: DstTy);
860 MVT CostVT = getZvzipVZIPCostVT(InterleavedVT: DstLT.second);
861 if (!CostVT.isValid())
862 return InstructionCost::getInvalid();
863
864 InstructionCost Cost =
865 DstLT.first * getRISCVInstructionCost(OpCodes: RISCV::VZIP_VV, VT: CostVT, CostKind);
866
867 // For a shuffle such as
868 // %r = shufflevector <4 x i32> %v, <4 x i32> poison,
869 // <4 x i32> <i32 0, i32 2, i32 1, i32 3>
870 // vzip.vv needs two <2 x i32> inputs. Include the cost of extracting each
871 // upper half; extracting a low half is free.
872 if (NumSrcElts == NumDstElts) {
873 auto *HalfSrcTy = FixedVectorType::getHalfElementsVectorType(
874 VTy: cast<FixedVectorType>(Val: SrcTy));
875 for (unsigned Start : StartIndexes) {
876 unsigned ExtractIndex = Start % NumSrcElts;
877 if (ExtractIndex != 0)
878 Cost += getShuffleCost(Kind: TTI::SK_ExtractSubvector, DstTy: HalfSrcTy, SrcTy,
879 CostKind, Mask: {}, Index: ExtractIndex, SubTp: HalfSrcTy);
880 }
881 }
882
883 return Cost;
884}
885
886InstructionCost RISCVTTIImpl::getShuffleCost(
887 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
888 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
889 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
890 TTI::VectorInstrContext VIC) const {
891 assert((Mask.empty() || DstTy->isScalableTy() ||
892 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
893 "Expected the Mask to match the return size if given");
894 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
895 "Expected the same scalar types");
896
897 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
898 if (VIC == TTI::VectorInstrContext::SplatOpFolded &&
899 ST->sinkSplatOperands() && Kind == TTI::SK_Broadcast)
900 return TTI::TCC_Free;
901
902 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
903 // For now, skip all fixed vector cost analysis when P extension is available
904 // to avoid crashes in getMinRVVVectorSizeInBits()
905 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: SrcTy))
906 return 1;
907
908 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
909
910 // First, handle cases where having a fixed length vector enables us to
911 // give a more accurate cost than falling back to generic scalable codegen.
912 // TODO: Each of these cases hints at a modeling gap around scalable vectors.
913 if (auto *FVTp = dyn_cast<FixedVectorType>(Val: SrcTy);
914 FVTp && ST->hasVInstructions() && LT.second.isFixedLengthVector()) {
915 InstructionCost VRegSplittingCost = costShuffleViaVRegSplitting(
916 TTI: *this, LegalVT: LT.second, VLen: ST->getRealVLen(),
917 Tp: Kind == TTI::SK_InsertSubvector ? DstTy : SrcTy, Mask, CostKind);
918 if (VRegSplittingCost.isValid())
919 return VRegSplittingCost;
920 switch (Kind) {
921 default:
922 break;
923 case TTI::SK_PermuteSingleSrc: {
924 if (Mask.size() >= 2) {
925 MVT EltTp = LT.second.getVectorElementType();
926
927 // Try the vzip.vv cost first, otherwise model the shuffle as a
928 // vwaddu.vv/vwmaccu.vx sequence below if the element size < ELEN.
929 if (InstructionCost VZipCost =
930 getVZIPCost(Kind, DstTy, SrcTy, Mask, CostKind);
931 VZipCost.isValid())
932 return VZipCost;
933
934 unsigned Index;
935 if (ST->hasStdExtZvzip() && EltTp.getScalarSizeInBits() != 1 &&
936 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2, Index)) {
937 unsigned Opcode = Index == 0 ? RISCV::VUNZIPE_V : RISCV::VUNZIPO_V;
938 if (MVT CostVT = getZvzipVUNZIPCostVT(InterleavedVT: LT.second); CostVT.isValid())
939 return LT.first * getRISCVInstructionCost(OpCodes: Opcode, VT: CostVT, CostKind);
940 }
941
942 // If the size of the element is < ELEN then shuffles of interleaves and
943 // deinterleaves of 2 vectors can be lowered into the following
944 // sequences
945 if (EltTp.getScalarSizeInBits() < ST->getELen()) {
946 // Example sequence:
947 // vsetivli zero, 4, e8, mf4, ta, ma (ignored)
948 // vwaddu.vv v10, v8, v9
949 // li a0, -1 (ignored)
950 // vwmaccu.vx v10, a0, v9
951 if (ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Mask.size()))
952 return 2 * LT.first * TLI->getLMULCost(VT: LT.second);
953
954 if (Mask[0] == 0 || Mask[0] == 1) {
955 auto DeinterleaveMask = createStrideMask(Start: Mask[0], Stride: 2, VF: Mask.size());
956 // Example sequence:
957 // vnsrl.wi v10, v8, 0
958 if (equal(LRange&: DeinterleaveMask, RRange&: Mask))
959 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VNSRL_WI,
960 VT: LT.second, CostKind);
961 }
962 }
963
964 int SubVectorSize;
965 if (LT.second.getScalarSizeInBits() != 1 &&
966 isRepeatedConcatMask(Mask, SubVectorSize)) {
967 InstructionCost Cost = 0;
968 unsigned NumSlides = Log2_32(Value: Mask.size() / SubVectorSize);
969 // The cost of extraction from a subvector is 0 if the index is 0.
970 for (unsigned I = 0; I != NumSlides; ++I) {
971 unsigned InsertIndex = SubVectorSize * (1 << I);
972 FixedVectorType *SubTp =
973 FixedVectorType::get(ElementType: SrcTy->getElementType(), NumElts: InsertIndex);
974 FixedVectorType *DestTp =
975 FixedVectorType::getDoubleElementsVectorType(VTy: SubTp);
976 std::pair<InstructionCost, MVT> DestLT =
977 getTypeLegalizationCost(Ty: DestTp);
978 // Add the cost of whole vector register move because the
979 // destination vector register group for vslideup cannot overlap the
980 // source.
981 Cost += DestLT.first * TLI->getLMULCost(VT: DestLT.second);
982 Cost += getShuffleCost(Kind: TTI::SK_InsertSubvector, DstTy: DestTp, SrcTy: DestTp,
983 CostKind, Mask: {}, Index: InsertIndex, SubTp);
984 }
985 return Cost;
986 }
987 }
988
989 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
990 SlideCost.isValid())
991 return SlideCost;
992
993 // vrgather + cost of generating the mask constant.
994 // We model this for an unknown mask with a single vrgather.
995 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
996 LT.second.getVectorNumElements() <= 256)) {
997 VectorType *IdxTy =
998 getVRGatherIndexType(DataVT: LT.second, ST: *ST, C&: SrcTy->getContext());
999 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
1000 return IndexCost +
1001 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VV, VT: LT.second, CostKind);
1002 }
1003 break;
1004 }
1005 case TTI::SK_Transpose:
1006 case TTI::SK_PermuteTwoSrc: {
1007
1008 if (InstructionCost SlideCost = getSlideCost(Tp: FVTp, Mask, CostKind);
1009 SlideCost.isValid())
1010 return SlideCost;
1011
1012 if (InstructionCost VZipCost =
1013 getVZIPCost(Kind, DstTy, SrcTy, Mask, CostKind);
1014 VZipCost.isValid())
1015 return VZipCost;
1016 // 2 x (vrgather + cost of generating the mask constant) + cost of mask
1017 // register for the second vrgather. We model this for an unknown
1018 // (shuffle) mask.
1019 if (LT.first == 1 && (LT.second.getScalarSizeInBits() != 8 ||
1020 LT.second.getVectorNumElements() <= 256)) {
1021 auto &C = SrcTy->getContext();
1022 auto EC = SrcTy->getElementCount();
1023 VectorType *IdxTy = getVRGatherIndexType(DataVT: LT.second, ST: *ST, C);
1024 VectorType *MaskTy = VectorType::get(ElementType: IntegerType::getInt1Ty(C), EC);
1025 InstructionCost IndexCost = getConstantPoolLoadCost(Ty: IdxTy, CostKind);
1026 InstructionCost MaskCost = getConstantPoolLoadCost(Ty: MaskTy, CostKind);
1027 return 2 * IndexCost +
1028 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV, RISCV::VRGATHER_VV},
1029 VT: LT.second, CostKind) +
1030 MaskCost;
1031 }
1032 break;
1033 }
1034 }
1035
1036 auto shouldSplit = [](TTI::ShuffleKind Kind) {
1037 switch (Kind) {
1038 default:
1039 return false;
1040 case TTI::SK_PermuteSingleSrc:
1041 case TTI::SK_Transpose:
1042 case TTI::SK_PermuteTwoSrc:
1043 return true;
1044 }
1045 };
1046
1047 if (!Mask.empty() && LT.first.isValid() && LT.first != 1 &&
1048 shouldSplit(Kind)) {
1049 InstructionCost SplitCost =
1050 costShuffleViaSplitting(TTI: *this, LegalVT: LT.second, Tp: FVTp, Mask, CostKind);
1051 if (SplitCost.isValid())
1052 return SplitCost;
1053 }
1054 }
1055
1056 // Handle scalable vectors (and fixed vectors legalized to scalable vectors).
1057 switch (Kind) {
1058 default:
1059 // Fallthrough to generic handling.
1060 // TODO: Most of these cases will return getInvalid in generic code, and
1061 // must be implemented here.
1062 break;
1063 case TTI::SK_ExtractSubvector:
1064 // Extract at zero is always a subregister extract
1065 if (Index == 0)
1066 return TTI::TCC_Free;
1067
1068 // If we're extracting a subvector of at most m1 size at a sub-register
1069 // boundary - which unfortunately we need exact vlen to identify - this is
1070 // a subregister extract at worst and thus won't require a vslidedown.
1071 // TODO: Extend for aligned m2, m4 subvector extracts
1072 // TODO: Extend for misalgined (but contained) extracts
1073 // TODO: Extend for scalable subvector types
1074 if (std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
1075 SubLT.second.isValid() && SubLT.second.isFixedLengthVector()) {
1076 if (std::optional<unsigned> VLen = ST->getRealVLen();
1077 VLen && SubLT.second.getScalarSizeInBits() * Index % *VLen == 0 &&
1078 SubLT.second.getSizeInBits() <= *VLen)
1079 return TTI::TCC_Free;
1080 }
1081
1082 // Example sequence:
1083 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
1084 // vslidedown.vi v8, v9, 2
1085 return LT.first *
1086 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEDOWN_VI, VT: LT.second, CostKind);
1087 case TTI::SK_InsertSubvector:
1088 // Example sequence:
1089 // vsetivli zero, 4, e8, mf2, tu, ma (ignored)
1090 // vslideup.vi v8, v9, 2
1091 LT = getTypeLegalizationCost(Ty: DstTy);
1092 return LT.first *
1093 getRISCVInstructionCost(OpCodes: RISCV::VSLIDEUP_VI, VT: LT.second, CostKind);
1094 case TTI::SK_Select: {
1095 // Example sequence:
1096 // li a0, 90
1097 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
1098 // vmv.s.x v0, a0
1099 // vmerge.vvm v8, v9, v8, v0
1100 // We use 2 for the cost of the mask materialization as this is the true
1101 // cost for small masks and most shuffles are small. At worst, this cost
1102 // should be a very small constant for the constant pool load. As such,
1103 // we may bias towards large selects slightly more than truly warranted.
1104 return LT.first *
1105 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_S_X, RISCV::VMERGE_VVM},
1106 VT: LT.second, CostKind));
1107 }
1108 case TTI::SK_Broadcast: {
1109 // Check for broadcast loads, which are synthesized by optimized zero-stride
1110 // loads (this is checked in RISCVTTIImpl::isLegalBroadcastLoad).
1111 bool IsLoad = !Args.empty() && isa<LoadInst>(Val: Args[0]);
1112 if (IsLoad && LT.second.isVector() &&
1113 isLegalBroadcastLoad(ElementTy: SrcTy->getElementType(),
1114 NumElements: LT.second.getVectorElementCount()))
1115 return 0;
1116
1117 bool HasScalar = (Args.size() > 0) && (Operator::getOpcode(V: Args[0]) ==
1118 Instruction::InsertElement);
1119 if (LT.second.getScalarSizeInBits() == 1) {
1120 if (HasScalar) {
1121 // Example sequence:
1122 // andi a0, a0, 1
1123 // vsetivli zero, 2, e8, mf8, ta, ma (ignored)
1124 // vmv.v.x v8, a0
1125 // vmsne.vi v0, v8, 0
1126 return LT.first *
1127 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
1128 VT: LT.second, CostKind));
1129 }
1130 // Example sequence:
1131 // vsetivli zero, 2, e8, mf8, ta, mu (ignored)
1132 // vmv.v.i v8, 0
1133 // vmerge.vim v8, v8, 1, v0
1134 // vmv.x.s a0, v8
1135 // andi a0, a0, 1
1136 // vmv.v.x v8, a0
1137 // vmsne.vi v0, v8, 0
1138
1139 return LT.first *
1140 (1 + getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_I, RISCV::VMERGE_VIM,
1141 RISCV::VMV_X_S, RISCV::VMV_V_X,
1142 RISCV::VMSNE_VI},
1143 VT: LT.second, CostKind));
1144 }
1145
1146 if (HasScalar) {
1147 // Example sequence:
1148 // vmv.v.x v8, a0
1149 return LT.first *
1150 getRISCVInstructionCost(OpCodes: RISCV::VMV_V_X, VT: LT.second, CostKind);
1151 }
1152
1153 // Example sequence:
1154 // vrgather.vi v9, v8, 0
1155 return LT.first *
1156 getRISCVInstructionCost(OpCodes: RISCV::VRGATHER_VI, VT: LT.second, CostKind);
1157 }
1158 case TTI::SK_Splice: {
1159 // vslidedown+vslideup.
1160 // TODO: Multiplying by LT.first implies this legalizes into multiple copies
1161 // of similar code, but I think we expand through memory.
1162 unsigned Opcodes[2] = {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX};
1163 if (Index >= 0 && Index < 32)
1164 Opcodes[0] = RISCV::VSLIDEDOWN_VI;
1165 else if (Index < 0 && Index > -32)
1166 Opcodes[1] = RISCV::VSLIDEUP_VI;
1167 return LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1168 }
1169 case TTI::SK_Reverse: {
1170
1171 if (!LT.second.isVector())
1172 return InstructionCost::getInvalid();
1173
1174 // TODO: Cases to improve here:
1175 // * Illegal vector types
1176 // * i64 on RV32
1177 if (SrcTy->getElementType()->isIntegerTy(BitWidth: 1)) {
1178 VectorType *WideTy =
1179 VectorType::get(ElementType: IntegerType::get(C&: SrcTy->getContext(), NumBits: 8),
1180 EC: cast<VectorType>(Val: SrcTy)->getElementCount());
1181 return getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: SrcTy,
1182 CCH: TTI::CastContextHint::None, CostKind) +
1183 getShuffleCost(Kind: TTI::SK_Reverse, DstTy: WideTy, SrcTy: WideTy, CostKind, Mask: {}, Index: 0,
1184 SubTp: nullptr) +
1185 getCastInstrCost(Opcode: Instruction::Trunc, Dst: SrcTy, Src: WideTy,
1186 CCH: TTI::CastContextHint::None, CostKind);
1187 }
1188
1189 MVT ContainerVT = LT.second;
1190 if (LT.second.isFixedLengthVector())
1191 ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1192 MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
1193 if (ContainerVT.bitsLE(VT: M1VT)) {
1194 // Example sequence:
1195 // csrr a0, vlenb
1196 // srli a0, a0, 3
1197 // addi a0, a0, -1
1198 // vsetvli a1, zero, e8, mf8, ta, mu (ignored)
1199 // vid.v v9
1200 // vrsub.vx v10, v9, a0
1201 // vrgather.vv v9, v8, v10
1202 InstructionCost LenCost = 3;
1203 if (LT.second.isFixedLengthVector())
1204 // vrsub.vi has a 5 bit immediate field, otherwise an li suffices
1205 LenCost = isInt<5>(x: LT.second.getVectorNumElements() - 1) ? 0 : 1;
1206 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX, RISCV::VRGATHER_VV};
1207 if (LT.second.isFixedLengthVector() &&
1208 isInt<5>(x: LT.second.getVectorNumElements() - 1))
1209 Opcodes[1] = RISCV::VRSUB_VI;
1210 InstructionCost GatherCost =
1211 getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1212 return LT.first * (LenCost + GatherCost);
1213 }
1214
1215 // At high LMUL, we split into a series of M1 reverses (see
1216 // lowerVECTOR_REVERSE) and then do a single slide at the end to eliminate
1217 // the resulting gap at the bottom (for fixed vectors only). The important
1218 // bit is that the cost scales linearly, not quadratically with LMUL.
1219 unsigned M1Opcodes[] = {RISCV::VID_V, RISCV::VRSUB_VX};
1220 InstructionCost FixedCost =
1221 getRISCVInstructionCost(OpCodes: M1Opcodes, VT: M1VT, CostKind) + 3;
1222 unsigned Ratio =
1223 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
1224 InstructionCost GatherCost =
1225 getRISCVInstructionCost(OpCodes: {RISCV::VRGATHER_VV}, VT: M1VT, CostKind) * Ratio;
1226 InstructionCost SlideCost = !LT.second.isFixedLengthVector() ? 0 :
1227 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX}, VT: LT.second, CostKind);
1228 return FixedCost + LT.first * (GatherCost + SlideCost);
1229 }
1230 }
1231 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index,
1232 SubTp);
1233}
1234
1235static unsigned isM1OrSmaller(MVT VT) {
1236 RISCVVType::VLMUL LMUL = RISCVTargetLowering::getLMUL(VT);
1237 return (LMUL == RISCVVType::VLMUL::LMUL_F8 ||
1238 LMUL == RISCVVType::VLMUL::LMUL_F4 ||
1239 LMUL == RISCVVType::VLMUL::LMUL_F2 ||
1240 LMUL == RISCVVType::VLMUL::LMUL_1);
1241}
1242
1243InstructionCost RISCVTTIImpl::getScalarizationOverhead(
1244 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
1245 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
1246 TTI::VectorInstrContext VIC) const {
1247 if (isa<ScalableVectorType>(Val: Ty))
1248 return InstructionCost::getInvalid();
1249
1250 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
1251 // For now, skip all fixed vector cost analysis when P extension is available
1252 // to avoid crashes in getMinRVVVectorSizeInBits()
1253 if (ST->hasStdExtP() && isa<FixedVectorType>(Val: Ty)) {
1254 return 1; // Treat as single instruction cost for now
1255 }
1256
1257 // A build_vector (which is m1 sized or smaller) can be done in no
1258 // worse than one vslide1down.vx per element in the type. We could
1259 // in theory do an explode_vector in the inverse manner, but our
1260 // lowering today does not have a first class node for this pattern.
1261 InstructionCost Cost = BaseT::getScalarizationOverhead(
1262 InTy: Ty, DemandedElts, Insert, Extract, CostKind);
1263 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1264 if (Insert && !Extract && LT.first.isValid() && LT.second.isVector()) {
1265 if (Ty->getScalarSizeInBits() == 1) {
1266 auto *WideVecTy = cast<VectorType>(Val: Ty->getWithNewBitWidth(NewBitWidth: 8));
1267 // Note: Implicit scalar anyextend is assumed to be free since the i1
1268 // must be stored in a GPR.
1269 return getScalarizationOverhead(Ty: WideVecTy, DemandedElts, Insert, Extract,
1270 CostKind) +
1271 getCastInstrCost(Opcode: Instruction::Trunc, Dst: Ty, Src: WideVecTy,
1272 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
1273 }
1274
1275 assert(LT.second.isFixedLengthVector());
1276 MVT ContainerVT = TLI->getContainerForFixedLengthVector(VT: LT.second);
1277 if (isM1OrSmaller(VT: ContainerVT)) {
1278 InstructionCost BV =
1279 cast<FixedVectorType>(Val: Ty)->getNumElements() *
1280 getRISCVInstructionCost(OpCodes: RISCV::VSLIDE1DOWN_VX, VT: LT.second, CostKind);
1281 if (BV < Cost)
1282 Cost = BV;
1283 }
1284 }
1285 return Cost;
1286}
1287
1288InstructionCost
1289RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1290 TTI::TargetCostKind CostKind) const {
1291 Type *DataTy = MICA.getDataType();
1292 Align Alignment = MICA.getAlignment();
1293 switch (MICA.getID()) {
1294 case Intrinsic::vp_load_ff: {
1295 EVT DataTypeVT = TLI->getValueType(DL, Ty: DataTy);
1296 if (!TLI->isLegalFirstFaultLoad(DataType: DataTypeVT, Alignment))
1297 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1298
1299 unsigned AS = MICA.getAddressSpace();
1300 return getMemoryOpCost(Opcode: Instruction::Load, Src: DataTy, Alignment, AddressSpace: AS, CostKind,
1301 OpdInfo: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I: nullptr);
1302 }
1303 case Intrinsic::experimental_vp_strided_load:
1304 case Intrinsic::experimental_vp_strided_store:
1305 return getStridedMemoryOpCost(MICA, CostKind);
1306 case Intrinsic::masked_compressstore:
1307 case Intrinsic::masked_expandload:
1308 return getExpandCompressMemoryOpCost(MICA, CostKind);
1309 case Intrinsic::vp_scatter:
1310 case Intrinsic::vp_gather:
1311 case Intrinsic::masked_scatter:
1312 case Intrinsic::masked_gather:
1313 return getGatherScatterOpCost(MICA, CostKind);
1314 case Intrinsic::vp_load:
1315 case Intrinsic::vp_store:
1316 case Intrinsic::masked_load:
1317 case Intrinsic::masked_store:
1318 return getMaskedMemoryOpCost(MICA, CostKind);
1319 }
1320 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1321}
1322
1323InstructionCost
1324RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1325 TTI::TargetCostKind CostKind) const {
1326 unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load
1327 : Instruction::Store;
1328 Type *Src = MICA.getDataType();
1329 Align Alignment = MICA.getAlignment();
1330 unsigned AddressSpace = MICA.getAddressSpace();
1331
1332 if (!isLegalMaskedLoadStore(DataType: Src, Alignment) ||
1333 CostKind != TTI::TCK_RecipThroughput)
1334 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1335
1336 // Splitting involves additional evl arithmetic and vl toggles.
1337 InstructionCost SplitCost = 0;
1338 if (MICA.getID() == Intrinsic::vp_load ||
1339 MICA.getID() == Intrinsic::vp_store) {
1340 auto LT = getTypeLegalizationCost(Ty: Src);
1341 if (LT.first > 1)
1342 SplitCost += LT.first * TTI::TCC_Expensive;
1343 }
1344
1345 return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind);
1346}
1347
1348InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost(
1349 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1350 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1351 bool UseMaskForCond, bool UseMaskForGaps) const {
1352
1353 // The interleaved memory access pass will lower (de)interleave ops combined
1354 // with an adjacent appropriate memory to vlseg/vsseg intrinsics. vlseg/vsseg
1355 // only support masking per-iteration (i.e. condition), not per-segment (i.e.
1356 // gap).
1357 if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {
1358 auto *VTy = cast<VectorType>(Val: VecTy);
1359 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VTy);
1360 // Need to make sure type has't been scalarized
1361 if (LT.second.isVector()) {
1362 if (CostKind == TTI::TCK_CodeSize)
1363 return LT.first * TTI::TCC_Basic;
1364
1365 auto *SubVecTy =
1366 VectorType::get(ElementType: VTy->getElementType(),
1367 EC: VTy->getElementCount().divideCoefficientBy(RHS: Factor));
1368 if (VTy->getElementCount().isKnownMultipleOf(RHS: Factor) &&
1369 TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor, Alignment,
1370 AddrSpace: AddressSpace, DL)) {
1371
1372 // Some processors optimize segment loads/stores as N * DLEN sized
1373 // load ops + Factor * LMUL shuffle ops.
1374 if (ST->hasOptimizedSegmentLoadStore(NF: Factor)) {
1375 unsigned VecSizeInBits =
1376 getEstimatedVLFor(Ty: VTy) * VTy->getScalarSizeInBits();
1377 unsigned VLENForTuning =
1378 *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
1379 unsigned DLENForTuning = VLENForTuning / ST->getDLenFactor();
1380 InstructionCost Cost = divideCeil(Numerator: VecSizeInBits, Denominator: DLENForTuning);
1381 MVT SubVecVT = getTLI()->getValueType(DL, Ty: SubVecTy).getSimpleVT();
1382 Cost += Factor * TLI->getLMULCost(VT: SubVecVT);
1383 return Cost;
1384 }
1385
1386 // Otherwise, the cost is proportional to the number of elements (VL *
1387 // Factor ops).
1388 unsigned NumLoads = getEstimatedVLFor(Ty: VTy);
1389 return NumLoads * TTI::TCC_Basic;
1390 }
1391 }
1392 }
1393
1394 // TODO: Return the cost of interleaved accesses for scalable vector when
1395 // unable to convert to segment accesses instructions.
1396 if (isa<ScalableVectorType>(Val: VecTy))
1397 return InstructionCost::getInvalid();
1398
1399 auto *FVTy = cast<FixedVectorType>(Val: VecTy);
1400 // When gaps are only at the tail, for interleaved load, we can emit a wide
1401 // masked load and shufflevectors. For interleaved store, we can emit
1402 // shufflevectors and a wide masked store. The interleaved memory access pass
1403 // will lower them into vlsseg/vssseg intrinsics.
1404 if (UseMaskForGaps) {
1405 assert(llvm::is_sorted(Indices) && "Indices must be sorted");
1406 assert(llvm::adjacent_find(Indices) == Indices.end() &&
1407 "Indices should not contain duplicate elements");
1408 unsigned NumOfFields = Indices.size();
1409 bool IsTailGapOnly = NumOfFields > 1 && (NumOfFields == Indices.back() + 1);
1410 if (IsTailGapOnly &&
1411 NumOfFields <= TLI->getMaxSupportedInterleaveFactor()) {
1412 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: FVTy);
1413 if (LT.second.isVector() &&
1414 FVTy->getElementCount().isKnownMultipleOf(RHS: Factor)) {
1415 auto *SubVecTy = VectorType::get(
1416 ElementType: FVTy->getElementType(),
1417 EC: FVTy->getElementCount().divideCoefficientBy(RHS: Factor));
1418 if (TLI->isLegalInterleavedAccessType(VTy: SubVecTy, Factor: NumOfFields, Alignment,
1419 AddrSpace: AddressSpace, DL)) {
1420 // The cost is proportional to the total number of element accesses.
1421 unsigned NumAccesses = getEstimatedVLFor(Ty: FVTy);
1422 return NumAccesses * TTI::TCC_Basic;
1423 }
1424 }
1425 }
1426 }
1427
1428 InstructionCost MemCost =
1429 getMemoryOpCost(Opcode, Src: VecTy, Alignment, AddressSpace, CostKind);
1430 unsigned VF = FVTy->getNumElements() / Factor;
1431
1432 // An interleaved load will look like this for Factor=3:
1433 // %wide.vec = load <12 x i32>, ptr %3, align 4
1434 // %strided.vec = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1435 // %strided.vec1 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1436 // %strided.vec2 = shufflevector %wide.vec, poison, <4 x i32> <stride mask>
1437 if (Opcode == Instruction::Load) {
1438 InstructionCost Cost = MemCost;
1439 for (unsigned Index : Indices) {
1440 FixedVectorType *VecTy =
1441 FixedVectorType::get(ElementType: FVTy->getElementType(), NumElts: VF * Factor);
1442 auto Mask = createStrideMask(Start: Index, Stride: Factor, VF);
1443 Mask.resize(N: VF * Factor, NV: -1);
1444 InstructionCost ShuffleCost =
1445 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: VecTy, SrcTy: VecTy,
1446 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {});
1447 Cost += ShuffleCost;
1448 }
1449 return Cost;
1450 }
1451
1452 // TODO: Model for NF > 2
1453 // We'll need to enhance getShuffleCost to model shuffles that are just
1454 // inserts and extracts into subvectors, since they won't have the full cost
1455 // of a vrgather.
1456 // An interleaved store for 3 vectors of 4 lanes will look like
1457 // %11 = shufflevector <4 x i32> %4, <4 x i32> %6, <8 x i32> <0...7>
1458 // %12 = shufflevector <4 x i32> %9, <4 x i32> poison, <8 x i32> <0...3>
1459 // %13 = shufflevector <8 x i32> %11, <8 x i32> %12, <12 x i32> <0...11>
1460 // %interleaved.vec = shufflevector %13, poison, <12 x i32> <interleave mask>
1461 // store <12 x i32> %interleaved.vec, ptr %10, align 4
1462 if (Factor != 2)
1463 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1464 Alignment, AddressSpace, CostKind,
1465 UseMaskForCond, UseMaskForGaps);
1466
1467 assert(Opcode == Instruction::Store && "Opcode must be a store");
1468 // For an interleaving store of 2 vectors, we perform one large interleaving
1469 // shuffle that goes into the wide store
1470 auto Mask = createInterleaveMask(VF, NumVecs: Factor);
1471 InstructionCost ShuffleCost =
1472 getShuffleCost(Kind: TTI::ShuffleKind::SK_PermuteSingleSrc, DstTy: FVTy, SrcTy: FVTy,
1473 CostKind, Mask, Index: 0, SubTp: nullptr, Args: {});
1474 return MemCost + ShuffleCost;
1475}
1476
1477InstructionCost
1478RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1479 TTI::TargetCostKind CostKind) const {
1480
1481 bool IsLoad = MICA.getID() == Intrinsic::masked_gather ||
1482 MICA.getID() == Intrinsic::vp_gather;
1483 unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store;
1484 Type *DataTy = MICA.getDataType();
1485 Type *PtrTy = DataTy->getWithNewType(
1486 EltTy: DL.getAddressType(C&: DataTy->getContext(), AddressSpace: MICA.getAddressSpace()));
1487 Align Alignment = MICA.getAlignment();
1488 if (CostKind != TTI::TCK_RecipThroughput)
1489 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1490
1491 if ((Opcode == Instruction::Load &&
1492 !isLegalMaskedGather(DataType: DataTy, Alignment: Align(Alignment))) ||
1493 (Opcode == Instruction::Store &&
1494 !isLegalMaskedScatter(DataType: DataTy, Alignment: Align(Alignment))))
1495 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1496
1497 // Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
1498 InstructionCost SplitCost = 0;
1499 if (MICA.getID() == Intrinsic::vp_gather ||
1500 MICA.getID() == Intrinsic::vp_scatter) {
1501 auto DataLT = getTypeLegalizationCost(Ty: DataTy);
1502 auto PtrLT = getTypeLegalizationCost(Ty: PtrTy);
1503 if (DataLT.first > 1)
1504 SplitCost += DataLT.first * TTI::TCC_Expensive;
1505 if (PtrLT.first > 1)
1506 SplitCost += PtrLT.first * TTI::TCC_Expensive;
1507 }
1508
1509 // Cost is proportional to the number of memory operations implied. For
1510 // scalable vectors, we use an estimate on that number since we don't
1511 // know exactly what VL will be.
1512 auto &VTy = *cast<VectorType>(Val: DataTy);
1513 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1514 return SplitCost + NumLoads * TTI::TCC_Basic;
1515}
1516
1517InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
1518 const MemIntrinsicCostAttributes &MICA,
1519 TTI::TargetCostKind CostKind) const {
1520 unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
1521 ? Instruction::Load
1522 : Instruction::Store;
1523 Type *DataTy = MICA.getDataType();
1524 bool VariableMask = MICA.getVariableMask();
1525 Align Alignment = MICA.getAlignment();
1526 bool IsLegal = (Opcode == Instruction::Store &&
1527 isLegalMaskedCompressStore(DataTy, Alignment)) ||
1528 (Opcode == Instruction::Load &&
1529 isLegalMaskedExpandLoad(DataType: DataTy, Alignment));
1530 if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
1531 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1532 // Example compressstore sequence:
1533 // vsetivli zero, 8, e32, m2, ta, ma (ignored)
1534 // vcompress.vm v10, v8, v0
1535 // vcpop.m a1, v0
1536 // vsetvli zero, a1, e32, m2, ta, ma
1537 // vse32.v v10, (a0)
1538 // Example expandload sequence:
1539 // vsetivli zero, 8, e8, mf2, ta, ma (ignored)
1540 // vcpop.m a1, v0
1541 // vsetvli zero, a1, e32, m2, ta, ma
1542 // vle32.v v10, (a0)
1543 // vsetivli zero, 8, e32, m2, ta, ma
1544 // viota.m v12, v0
1545 // vrgather.vv v8, v10, v12, v0.t
1546 auto MemOpCost =
1547 getMemoryOpCost(Opcode, Src: DataTy, Alignment, /*AddressSpace*/ 0, CostKind);
1548 auto LT = getTypeLegalizationCost(Ty: DataTy);
1549 SmallVector<unsigned, 4> Opcodes{RISCV::VSETVLI};
1550 if (VariableMask)
1551 Opcodes.push_back(Elt: RISCV::VCPOP_M);
1552 if (Opcode == Instruction::Store)
1553 Opcodes.append(IL: {RISCV::VCOMPRESS_VM});
1554 else
1555 Opcodes.append(IL: {RISCV::VSETIVLI, RISCV::VIOTA_M, RISCV::VRGATHER_VV});
1556 return MemOpCost +
1557 LT.first * getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
1558}
1559
1560InstructionCost
1561RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1562 TTI::TargetCostKind CostKind) const {
1563 Type *DataTy = MICA.getDataType();
1564 Align Alignment = MICA.getAlignment();
1565
1566 if (!isLegalStridedLoadStore(DataType: DataTy, Alignment))
1567 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1568
1569 if (CostKind == TTI::TCK_CodeSize)
1570 return TTI::TCC_Basic;
1571
1572 // Splitting vp intrinsics involves additional evl arithmetic and vl toggles.
1573 InstructionCost SplitCost = 0;
1574 auto LT = getTypeLegalizationCost(Ty: DataTy);
1575 if (LT.first > 1)
1576 SplitCost += LT.first * TTI::TCC_Expensive;
1577
1578 // Cost is proportional to the number of memory operations implied. For
1579 // scalable vectors, we use an estimate on that number since we don't
1580 // know exactly what VL will be.
1581 auto &VTy = *cast<VectorType>(Val: DataTy);
1582 unsigned NumLoads = getEstimatedVLFor(Ty: &VTy);
1583 // Performant implementations of the vector extension will coalesce
1584 // elements if they fall on the same cache line
1585 uint64_t CacheLineBytes = ST->getCacheLineSize();
1586 if (!CacheLineBytes) // If no value, use default value of 64
1587 CacheLineBytes = 64;
1588 if (const ConstantInt *StrideCI =
1589 dyn_cast_or_null<ConstantInt>(Val: MICA.getStrideVal())) {
1590 int64_t Stride = StrideCI->getSExtValue();
1591 // Bail early to avoid UB with std:abs() call
1592 if (Stride != std::numeric_limits<int64_t>::min() && Stride != 0) {
1593 uint64_t AbsStride = (uint64_t)std::abs(i: Stride);
1594 if (AbsStride < CacheLineBytes) {
1595 uint64_t MaxCombines = ST->getMaxVectorCoalesceElts();
1596 if ((CacheLineBytes / AbsStride) >= MaxCombines)
1597 NumLoads = divideCeil(Numerator: NumLoads, Denominator: MaxCombines);
1598 else
1599 // If we were to calculate CacheLineBytes / AbsStride first, would
1600 // lose accuracy
1601 NumLoads = divideCeil(Numerator: (NumLoads * AbsStride), Denominator: CacheLineBytes);
1602 }
1603 }
1604 }
1605 return SplitCost + NumLoads * TTI::TCC_Basic;
1606}
1607
1608InstructionCost
1609RISCVTTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
1610 // FIXME: This is a property of the default vector convention, not
1611 // all possible calling conventions. Fixing that will require
1612 // some TTI API and SLP rework.
1613 InstructionCost Cost = 0;
1614 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
1615 for (auto *Ty : Tys) {
1616 if (!Ty->isVectorTy())
1617 continue;
1618 Align A = DL.getPrefTypeAlign(Ty);
1619 Cost += getMemoryOpCost(Opcode: Instruction::Store, Src: Ty, Alignment: A, AddressSpace: 0, CostKind) +
1620 getMemoryOpCost(Opcode: Instruction::Load, Src: Ty, Alignment: A, AddressSpace: 0, CostKind);
1621 }
1622 return Cost;
1623}
1624
1625// Currently, these represent both throughput and codesize costs
1626// for the respective intrinsics. The costs in this table are simply
1627// instruction counts with the following adjustments made:
1628// * One vsetvli is considered free.
1629static const CostTblEntry VectorIntrinsicCostTable[]{
1630 {.ISD: Intrinsic::floor, .Type: MVT::f32, .Cost: 9},
1631 {.ISD: Intrinsic::floor, .Type: MVT::f64, .Cost: 9},
1632 {.ISD: Intrinsic::ceil, .Type: MVT::f32, .Cost: 9},
1633 {.ISD: Intrinsic::ceil, .Type: MVT::f64, .Cost: 9},
1634 {.ISD: Intrinsic::trunc, .Type: MVT::f32, .Cost: 7},
1635 {.ISD: Intrinsic::trunc, .Type: MVT::f64, .Cost: 7},
1636 {.ISD: Intrinsic::round, .Type: MVT::f32, .Cost: 9},
1637 {.ISD: Intrinsic::round, .Type: MVT::f64, .Cost: 9},
1638 {.ISD: Intrinsic::roundeven, .Type: MVT::f32, .Cost: 9},
1639 {.ISD: Intrinsic::roundeven, .Type: MVT::f64, .Cost: 9},
1640 {.ISD: Intrinsic::rint, .Type: MVT::f32, .Cost: 7},
1641 {.ISD: Intrinsic::rint, .Type: MVT::f64, .Cost: 7},
1642 {.ISD: Intrinsic::nearbyint, .Type: MVT::f32, .Cost: 9},
1643 {.ISD: Intrinsic::nearbyint, .Type: MVT::f64, .Cost: 9},
1644 {.ISD: Intrinsic::bswap, .Type: MVT::i16, .Cost: 3},
1645 {.ISD: Intrinsic::bswap, .Type: MVT::i32, .Cost: 12},
1646 {.ISD: Intrinsic::bswap, .Type: MVT::i64, .Cost: 31},
1647 {.ISD: Intrinsic::bitreverse, .Type: MVT::i8, .Cost: 17},
1648 {.ISD: Intrinsic::bitreverse, .Type: MVT::i16, .Cost: 24},
1649 {.ISD: Intrinsic::bitreverse, .Type: MVT::i32, .Cost: 33},
1650 {.ISD: Intrinsic::bitreverse, .Type: MVT::i64, .Cost: 52},
1651 {.ISD: Intrinsic::ctpop, .Type: MVT::i8, .Cost: 12},
1652 {.ISD: Intrinsic::ctpop, .Type: MVT::i16, .Cost: 19},
1653 {.ISD: Intrinsic::ctpop, .Type: MVT::i32, .Cost: 20},
1654 {.ISD: Intrinsic::ctpop, .Type: MVT::i64, .Cost: 21},
1655 {.ISD: Intrinsic::ctlz, .Type: MVT::i8, .Cost: 19},
1656 {.ISD: Intrinsic::ctlz, .Type: MVT::i16, .Cost: 28},
1657 {.ISD: Intrinsic::ctlz, .Type: MVT::i32, .Cost: 31},
1658 {.ISD: Intrinsic::ctlz, .Type: MVT::i64, .Cost: 35},
1659 {.ISD: Intrinsic::cttz, .Type: MVT::i8, .Cost: 16},
1660 {.ISD: Intrinsic::cttz, .Type: MVT::i16, .Cost: 23},
1661 {.ISD: Intrinsic::cttz, .Type: MVT::i32, .Cost: 24},
1662 {.ISD: Intrinsic::cttz, .Type: MVT::i64, .Cost: 25},
1663};
1664
1665InstructionCost
1666RISCVTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1667 TTI::TargetCostKind CostKind) const {
1668 auto *RetTy = ICA.getReturnType();
1669 switch (ICA.getID()) {
1670 case Intrinsic::lrint:
1671 case Intrinsic::llrint:
1672 case Intrinsic::lround:
1673 case Intrinsic::llround: {
1674 auto LT = getTypeLegalizationCost(Ty: RetTy);
1675 Type *SrcTy = ICA.getArgTypes().front();
1676 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
1677 if (ST->hasVInstructions() && LT.second.isVector()) {
1678 SmallVector<unsigned, 2> Ops;
1679 unsigned SrcEltSz = DL.getTypeSizeInBits(Ty: SrcTy->getScalarType());
1680 unsigned DstEltSz = DL.getTypeSizeInBits(Ty: RetTy->getScalarType());
1681 if (LT.second.getVectorElementType() == MVT::bf16) {
1682 if (!ST->hasVInstructionsBF16Minimal())
1683 return InstructionCost::getInvalid();
1684 if (DstEltSz == 32)
1685 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFCVT_X_F_V};
1686 else
1687 Ops = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVT_X_F_V};
1688 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1689 !ST->hasVInstructionsF16()) {
1690 if (!ST->hasVInstructionsF16Minimal())
1691 return InstructionCost::getInvalid();
1692 if (DstEltSz == 32)
1693 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFCVT_X_F_V};
1694 else
1695 Ops = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_X_F_V};
1696
1697 } else if (SrcEltSz > DstEltSz) {
1698 Ops = {RISCV::VFNCVT_X_F_W};
1699 } else if (SrcEltSz < DstEltSz) {
1700 Ops = {RISCV::VFWCVT_X_F_V};
1701 } else {
1702 Ops = {RISCV::VFCVT_X_F_V};
1703 }
1704
1705 // We need to use the source LMUL in the case of a narrowing op, and the
1706 // destination LMUL otherwise.
1707 if (SrcEltSz > DstEltSz)
1708 return SrcLT.first *
1709 getRISCVInstructionCost(OpCodes: Ops, VT: SrcLT.second, CostKind);
1710 return LT.first * getRISCVInstructionCost(OpCodes: Ops, VT: LT.second, CostKind);
1711 }
1712 break;
1713 }
1714 case Intrinsic::ceil:
1715 case Intrinsic::floor:
1716 case Intrinsic::trunc:
1717 case Intrinsic::rint:
1718 case Intrinsic::round:
1719 case Intrinsic::roundeven: {
1720 // These all use the same code.
1721 auto LT = getTypeLegalizationCost(Ty: RetTy);
1722 if (!LT.second.isVector() && TLI->isOperationCustom(Op: ISD::FCEIL, VT: LT.second))
1723 return LT.first * 8;
1724 break;
1725 }
1726 case Intrinsic::umin:
1727 case Intrinsic::umax:
1728 case Intrinsic::smin:
1729 case Intrinsic::smax: {
1730 auto LT = getTypeLegalizationCost(Ty: RetTy);
1731 if (LT.second.isScalarInteger() && ST->hasStdExtZbb())
1732 return LT.first;
1733
1734 if (ST->hasVInstructions() && LT.second.isVector()) {
1735 unsigned Op;
1736 switch (ICA.getID()) {
1737 case Intrinsic::umin:
1738 Op = RISCV::VMINU_VV;
1739 break;
1740 case Intrinsic::umax:
1741 Op = RISCV::VMAXU_VV;
1742 break;
1743 case Intrinsic::smin:
1744 Op = RISCV::VMIN_VV;
1745 break;
1746 case Intrinsic::smax:
1747 Op = RISCV::VMAX_VV;
1748 break;
1749 }
1750 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1751 }
1752 break;
1753 }
1754 case Intrinsic::sadd_sat:
1755 case Intrinsic::ssub_sat:
1756 case Intrinsic::uadd_sat:
1757 case Intrinsic::usub_sat: {
1758 auto LT = getTypeLegalizationCost(Ty: RetTy);
1759 if (ST->hasVInstructions() && LT.second.isVector()) {
1760 unsigned Op;
1761 switch (ICA.getID()) {
1762 case Intrinsic::sadd_sat:
1763 Op = RISCV::VSADD_VV;
1764 break;
1765 case Intrinsic::ssub_sat:
1766 Op = RISCV::VSSUB_VV;
1767 break;
1768 case Intrinsic::uadd_sat:
1769 Op = RISCV::VSADDU_VV;
1770 break;
1771 case Intrinsic::usub_sat:
1772 Op = RISCV::VSSUBU_VV;
1773 break;
1774 }
1775 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1776 }
1777 break;
1778 }
1779 case Intrinsic::fma:
1780 case Intrinsic::fmuladd: {
1781 // TODO: handle promotion with f16/bf16 with zvfhmin/zvfbfmin
1782 auto LT = getTypeLegalizationCost(Ty: RetTy);
1783 if (ST->hasVInstructions() && LT.second.isVector())
1784 return LT.first *
1785 getRISCVInstructionCost(OpCodes: RISCV::VFMADD_VV, VT: LT.second, CostKind);
1786 break;
1787 }
1788 case Intrinsic::fabs: {
1789 auto LT = getTypeLegalizationCost(Ty: RetTy);
1790 if (ST->hasVInstructions() && LT.second.isVector()) {
1791 // lui a0, 8
1792 // addi a0, a0, -1
1793 // vsetvli a1, zero, e16, m1, ta, ma
1794 // vand.vx v8, v8, a0
1795 // f16 with zvfhmin and bf16 with zvfhbmin
1796 if (LT.second.getVectorElementType() == MVT::bf16 ||
1797 (LT.second.getVectorElementType() == MVT::f16 &&
1798 !ST->hasVInstructionsF16()))
1799 return LT.first * getRISCVInstructionCost(OpCodes: RISCV::VAND_VX, VT: LT.second,
1800 CostKind) +
1801 2;
1802 else
1803 return LT.first *
1804 getRISCVInstructionCost(OpCodes: RISCV::VFSGNJX_VV, VT: LT.second, CostKind);
1805 }
1806 break;
1807 }
1808 case Intrinsic::sqrt: {
1809 auto LT = getTypeLegalizationCost(Ty: RetTy);
1810 if (ST->hasVInstructions() && LT.second.isVector()) {
1811 SmallVector<unsigned, 4> ConvOp;
1812 SmallVector<unsigned, 2> FsqrtOp;
1813 MVT ConvType = LT.second;
1814 MVT FsqrtType = LT.second;
1815 // f16 with zvfhmin and bf16 with zvfbfmin and the type of nxv32[b]f16
1816 // will be spilt.
1817 if (LT.second.getVectorElementType() == MVT::bf16) {
1818 if (LT.second == MVT::nxv32bf16) {
1819 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFWCVTBF16_F_F_V,
1820 RISCV::VFNCVTBF16_F_F_W, RISCV::VFNCVTBF16_F_F_W};
1821 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1822 ConvType = MVT::nxv16f16;
1823 FsqrtType = MVT::nxv16f32;
1824 } else {
1825 ConvOp = {RISCV::VFWCVTBF16_F_F_V, RISCV::VFNCVTBF16_F_F_W};
1826 FsqrtOp = {RISCV::VFSQRT_V};
1827 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1828 }
1829 } else if (LT.second.getVectorElementType() == MVT::f16 &&
1830 !ST->hasVInstructionsF16()) {
1831 if (LT.second == MVT::nxv32f16) {
1832 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFWCVT_F_F_V,
1833 RISCV::VFNCVT_F_F_W, RISCV::VFNCVT_F_F_W};
1834 FsqrtOp = {RISCV::VFSQRT_V, RISCV::VFSQRT_V};
1835 ConvType = MVT::nxv16f16;
1836 FsqrtType = MVT::nxv16f32;
1837 } else {
1838 ConvOp = {RISCV::VFWCVT_F_F_V, RISCV::VFNCVT_F_F_W};
1839 FsqrtOp = {RISCV::VFSQRT_V};
1840 FsqrtType = TLI->getTypeToPromoteTo(Op: ISD::FSQRT, VT: FsqrtType);
1841 }
1842 } else {
1843 FsqrtOp = {RISCV::VFSQRT_V};
1844 }
1845
1846 return LT.first * (getRISCVInstructionCost(OpCodes: FsqrtOp, VT: FsqrtType, CostKind) +
1847 getRISCVInstructionCost(OpCodes: ConvOp, VT: ConvType, CostKind));
1848 }
1849 break;
1850 }
1851 case Intrinsic::cttz:
1852 case Intrinsic::ctlz:
1853 case Intrinsic::ctpop: {
1854 auto LT = getTypeLegalizationCost(Ty: RetTy);
1855 if (ST->hasStdExtZvbb() && LT.second.isVector()) {
1856 unsigned Op;
1857 switch (ICA.getID()) {
1858 case Intrinsic::cttz:
1859 Op = RISCV::VCTZ_V;
1860 break;
1861 case Intrinsic::ctlz:
1862 Op = RISCV::VCLZ_V;
1863 break;
1864 case Intrinsic::ctpop:
1865 Op = RISCV::VCPOP_V;
1866 break;
1867 }
1868 return LT.first * getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
1869 }
1870 break;
1871 }
1872 case Intrinsic::abs: {
1873 auto LT = getTypeLegalizationCost(Ty: RetTy);
1874 if (ST->hasVInstructions() && LT.second.isVector()) {
1875 // vabs.v v10, v8 (alias for vabd.vx v10, v8, zero)
1876 if (ST->hasStdExtZvabd())
1877 return LT.first *
1878 getRISCVInstructionCost(OpCodes: {RISCV::VABD_VX}, VT: LT.second, CostKind);
1879
1880 // vrsub.vi v10, v8, 0
1881 // vmax.vv v8, v8, v10
1882 return LT.first *
1883 getRISCVInstructionCost(OpCodes: {RISCV::VRSUB_VI, RISCV::VMAX_VV},
1884 VT: LT.second, CostKind);
1885 }
1886 break;
1887 }
1888 case Intrinsic::fshl:
1889 case Intrinsic::fshr: {
1890 if (ICA.getArgs().empty())
1891 break;
1892
1893 // Funnel-shifts are ROTL/ROTR when the first and second operand are equal.
1894 // When Zbb/Zbkb is enabled we can use a single ROL(W)/ROR(I)(W)
1895 // instruction.
1896 if ((ST->hasStdExtZbb() || ST->hasStdExtZbkb()) && RetTy->isIntegerTy() &&
1897 ICA.getArgs()[0] == ICA.getArgs()[1] &&
1898 (RetTy->getIntegerBitWidth() == 32 ||
1899 RetTy->getIntegerBitWidth() == 64) &&
1900 RetTy->getIntegerBitWidth() <= ST->getXLen()) {
1901 return 1;
1902 }
1903 break;
1904 }
1905 case Intrinsic::clmul: {
1906 auto LT = getTypeLegalizationCost(Ty: RetTy);
1907 if (!LT.second.isVector() && ST->hasStdExtZvbc() && !ST->hasStdExtZbkc()) {
1908 // TODO: Once custom lowering in this case for RV32 is added, this guard
1909 // should be removed and the cost model should be updated.
1910 if (!ST->is64Bit() || LT.second != MVT::i64)
1911 break;
1912 // vmv.s.x v8, a0
1913 // vclmul.vx v8, v8, a1
1914 // vmv.x.s a0, v8
1915 MVT VecVT = MVT::getScalableVectorVT(VT: LT.second, NumElements: 1);
1916 return LT.first * getRISCVInstructionCost(
1917 OpCodes: {RISCV::VMV_S_X, RISCV::VCLMUL_VX, RISCV::VMV_X_S},
1918 VT: VecVT, CostKind);
1919 }
1920 break;
1921 }
1922 case Intrinsic::masked_udiv:
1923 return getArithmeticInstrCost(Opcode: Instruction::UDiv, Ty: ICA.getReturnType(),
1924 CostKind);
1925 case Intrinsic::masked_sdiv:
1926 return getArithmeticInstrCost(Opcode: Instruction::SDiv, Ty: ICA.getReturnType(),
1927 CostKind);
1928 case Intrinsic::masked_urem:
1929 return getArithmeticInstrCost(Opcode: Instruction::URem, Ty: ICA.getReturnType(),
1930 CostKind);
1931 case Intrinsic::masked_srem:
1932 return getArithmeticInstrCost(Opcode: Instruction::SRem, Ty: ICA.getReturnType(),
1933 CostKind);
1934 case Intrinsic::get_active_lane_mask: {
1935 if (ST->hasVInstructions()) {
1936 Type *ExpRetTy = VectorType::get(
1937 ElementType: ICA.getArgTypes()[0], EC: cast<VectorType>(Val: RetTy)->getElementCount());
1938 auto LT = getTypeLegalizationCost(Ty: ExpRetTy);
1939
1940 // vid.v v8 // considered hoisted
1941 // vsaddu.vx v8, v8, a0
1942 // vmsltu.vx v0, v8, a1
1943 return LT.first *
1944 getRISCVInstructionCost(OpCodes: {RISCV::VSADDU_VX, RISCV::VMSLTU_VX},
1945 VT: LT.second, CostKind);
1946 }
1947 break;
1948 }
1949 // TODO: add more intrinsic
1950 case Intrinsic::stepvector: {
1951 auto LT = getTypeLegalizationCost(Ty: RetTy);
1952 // Legalisation of illegal types involves an `index' instruction plus
1953 // (LT.first - 1) vector adds.
1954 if (ST->hasVInstructions())
1955 return getRISCVInstructionCost(OpCodes: RISCV::VID_V, VT: LT.second, CostKind) +
1956 (LT.first - 1) *
1957 getRISCVInstructionCost(OpCodes: RISCV::VADD_VX, VT: LT.second, CostKind);
1958 return 1 + (LT.first - 1);
1959 }
1960 case Intrinsic::vector_splice_left:
1961 case Intrinsic::vector_splice_right: {
1962 auto LT = getTypeLegalizationCost(Ty: RetTy);
1963 // Constant offsets fall through to getShuffleCost.
1964 if (!ICA.isTypeBasedOnly() && isa<ConstantInt>(Val: ICA.getArgs()[2]))
1965 break;
1966 if (ST->hasVInstructions() && LT.second.isVector()) {
1967 return LT.first *
1968 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VSLIDEUP_VX},
1969 VT: LT.second, CostKind);
1970 }
1971 break;
1972 }
1973 case Intrinsic::experimental_cttz_elts: {
1974 if (!ST->hasVInstructions())
1975 break;
1976 InstructionCost Cost = 0;
1977 Type *ArgTy = ICA.getArgTypes()[0];
1978 auto LT = getTypeLegalizationCost(Ty: ArgTy);
1979 if (!LT.second.isVector())
1980 break;
1981
1982 // If the element type is not i1, do a comparison with all-zeros.
1983 if (LT.second.getVectorElementType() != MVT::i1)
1984 Cost += getRISCVInstructionCost(OpCodes: RISCV::VMSNE_VI, VT: LT.second, CostKind);
1985
1986 Cost += getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind);
1987
1988 // If zero_is_poison is false, then we will generate additional
1989 // cmp + select instructions to convert -1 to EVL.
1990 Type *BoolTy = Type::getInt1Ty(C&: RetTy->getContext());
1991 if (ICA.getArgs().size() > 1 &&
1992 cast<ConstantInt>(Val: ICA.getArgs()[1])->isZero())
1993 Cost += getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: BoolTy, CondTy: RetTy,
1994 VecPred: CmpInst::ICMP_SLT, CostKind) +
1995 getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: RetTy, CondTy: BoolTy,
1996 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
1997
1998 return LT.first * Cost;
1999 }
2000 case Intrinsic::experimental_vp_splice: {
2001 // To support type-based query from vectorizer, set the index to 0.
2002 // Note that index only change the cost from vslide.vx to vslide.vi and in
2003 // current implementations they have same costs.
2004 return getShuffleCost(Kind: TTI::SK_Splice, DstTy: cast<VectorType>(Val: ICA.getReturnType()),
2005 SrcTy: cast<VectorType>(Val: ICA.getArgTypes()[0]), CostKind, Mask: {},
2006 Index: 0, SubTp: cast<VectorType>(Val: ICA.getReturnType()));
2007 }
2008 case Intrinsic::vp_merge: {
2009 // If an operand is a binary op and the type is legal, RISCVVectorPeephole
2010 // will likely fold the resulting vmerge.vvm away.
2011 if (ICA.getVectorInstrContext() == VectorInstrContext::BinaryOp &&
2012 getTypeLegalizationCost(Ty: RetTy).first == 1)
2013 return TTI::TCC_Free;
2014 break;
2015 }
2016 case Intrinsic::fptoui_sat:
2017 case Intrinsic::fptosi_sat: {
2018 InstructionCost Cost = 0;
2019 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
2020 Type *SrcTy = ICA.getArgTypes()[0];
2021
2022 auto SrcLT = getTypeLegalizationCost(Ty: SrcTy);
2023 auto DstLT = getTypeLegalizationCost(Ty: RetTy);
2024 if (!SrcTy->isVectorTy())
2025 break;
2026
2027 if (!SrcLT.first.isValid() || !DstLT.first.isValid())
2028 return InstructionCost::getInvalid();
2029
2030 Cost +=
2031 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
2032 Dst: RetTy, Src: SrcTy, CCH: TTI::CastContextHint::None, CostKind);
2033
2034 // Handle NaN.
2035 // vmfne v0, v8, v8 # If v8[i] is NaN set v0[i] to 1.
2036 // vmerge.vim v8, v8, 0, v0 # Convert NaN to 0.
2037 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2038 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: SrcTy, CondTy,
2039 VecPred: CmpInst::FCMP_UNO, CostKind);
2040 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
2041 VecPred: CmpInst::FCMP_UNO, CostKind);
2042 return Cost;
2043 }
2044 case Intrinsic::experimental_vector_extract_last_active: {
2045 auto *ValTy = cast<VectorType>(Val: ICA.getArgTypes()[0]);
2046 auto *MaskTy = cast<VectorType>(Val: ICA.getArgTypes()[1]);
2047
2048 auto ValLT = getTypeLegalizationCost(Ty: ValTy);
2049 auto MaskLT = getTypeLegalizationCost(Ty: MaskTy);
2050
2051 // TODO: Return cheaper cost when the entire lane is inactive.
2052 // The expected asm sequence is:
2053 // vcpop.m a0, v0
2054 // beqz a0, exit # Return passthru when the entire lane is inactive.
2055 // vid v10, v0.t
2056 // vredmaxu.vs v10, v10, v10
2057 // vmv.x.s a0, v10
2058 // zext.b a0, a0
2059 // vslidedown.vx v8, v8, a0
2060 // vmv.x.s a0, v8
2061 // exit:
2062 // ...
2063
2064 // Find a suitable type for a stepvector.
2065 ConstantRange VScaleRange(APInt(64, 1), APInt::getZero(numBits: 64));
2066 unsigned EltWidth = getTLI()->getBitWidthForCttzElements(
2067 RetVT: TLI->getVectorIdxTy(DL: getDataLayout()), EC: MaskTy->getElementCount(),
2068 /*ZeroIsPoison=*/true, VScaleRange: &VScaleRange);
2069 EltWidth = std::max(a: EltWidth, b: MaskTy->getScalarSizeInBits());
2070 Type *StepTy = Type::getIntNTy(C&: MaskTy->getContext(), N: EltWidth);
2071 auto *StepVecTy = VectorType::get(ElementType: StepTy, EC: ValTy->getElementCount());
2072 auto StepLT = getTypeLegalizationCost(Ty: StepVecTy);
2073
2074 // Currently expandVectorFindLastActive cannot handle step vector split.
2075 // So return invalid when the type needs split.
2076 // FIXME: Remove this if expandVectorFindLastActive supports split vector.
2077 if (StepLT.first > 1)
2078 return InstructionCost::getInvalid();
2079
2080 InstructionCost Cost = 0;
2081 unsigned Opcodes[] = {RISCV::VID_V, RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
2082
2083 Cost += MaskLT.first *
2084 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: MaskLT.second, CostKind);
2085 Cost += getCFInstrCost(Opcode: Instruction::CondBr, CostKind, I: nullptr);
2086 Cost += StepLT.first *
2087 getRISCVInstructionCost(OpCodes: Opcodes, VT: StepLT.second, CostKind);
2088 Cost += getCastInstrCost(Opcode: Instruction::ZExt,
2089 Dst: Type::getInt64Ty(C&: ValTy->getContext()), Src: StepTy,
2090 CCH: TTI::CastContextHint::None, CostKind, I: nullptr);
2091 Cost += ValLT.first *
2092 getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VI, RISCV::VMV_X_S},
2093 VT: ValLT.second, CostKind);
2094 return Cost;
2095 }
2096 case Intrinsic::vector_interleave2:
2097 case Intrinsic::vector_deinterleave2: {
2098 if (!ST->hasStdExtZvzip())
2099 break;
2100
2101 bool IsInterleave = ICA.getID() == Intrinsic::vector_interleave2;
2102 Type *InterleavedTy = IsInterleave ? RetTy : ICA.getArgTypes().front();
2103 // ISel does not select vzip.vv if either interleave2 input is undef.
2104 if (IsInterleave && !ICA.isTypeBasedOnly() &&
2105 any_of(Range: ICA.getArgs(),
2106 P: [](const Value *Arg) { return isa<UndefValue>(Val: Arg); }))
2107 break;
2108 if (InterleavedTy->getScalarSizeInBits() == 1)
2109 break;
2110
2111 if (auto *FVT = dyn_cast<FixedVectorType>(Val: InterleavedTy)) {
2112 auto *HalfFVT = FixedVectorType::getHalfElementsVectorType(VTy: FVT);
2113 unsigned HalfVF = HalfFVT->getNumElements();
2114 if (IsInterleave)
2115 return getShuffleCost(Kind: TTI::SK_PermuteTwoSrc, DstTy: FVT, SrcTy: HalfFVT, CostKind,
2116 Mask: createInterleaveMask(VF: HalfVF, NumVecs: 2), Index: 0, SubTp: nullptr);
2117 InstructionCost Cost = 0;
2118 for (unsigned Start = 0; Start != 2; ++Start)
2119 Cost += getShuffleCost(Kind: TTI::SK_PermuteSingleSrc, DstTy: HalfFVT, SrcTy: FVT, CostKind,
2120 Mask: createStrideMask(Start, Stride: 2, VF: HalfVF), Index: 0, SubTp: nullptr);
2121 return Cost;
2122 }
2123
2124 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: InterleavedTy);
2125 if (!LT.second.isScalableVector())
2126 break;
2127 if (IsInterleave) {
2128 if (MVT CostVT = getZvzipVZIPCostVT(InterleavedVT: LT.second); CostVT.isValid())
2129 return LT.first *
2130 getRISCVInstructionCost(OpCodes: RISCV::VZIP_VV, VT: CostVT, CostKind);
2131 } else if (MVT CostVT = getZvzipVUNZIPCostVT(InterleavedVT: LT.second); CostVT.isValid()) {
2132 return LT.first *
2133 getRISCVInstructionCost(OpCodes: {RISCV::VUNZIPE_V, RISCV::VUNZIPO_V},
2134 VT: CostVT, CostKind);
2135 }
2136 break;
2137 }
2138 }
2139
2140 if (ST->hasVInstructions() && RetTy->isVectorTy()) {
2141 if (auto LT = getTypeLegalizationCost(Ty: RetTy);
2142 LT.second.isVector()) {
2143 MVT EltTy = LT.second.getVectorElementType();
2144 if (const auto *Entry = CostTableLookup(Table: VectorIntrinsicCostTable,
2145 ISD: ICA.getID(), Ty: EltTy))
2146 return LT.first * Entry->Cost;
2147 }
2148 }
2149
2150 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
2151}
2152
2153InstructionCost
2154RISCVTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
2155 const SCEV *Ptr,
2156 TTI::TargetCostKind CostKind) const {
2157 // Address computations for vector indexed load/store likely require an offset
2158 // and/or scaling.
2159 if (ST->hasVInstructions() && PtrTy->isVectorTy())
2160 return getArithmeticInstrCost(Opcode: Instruction::Add, Ty: PtrTy, CostKind);
2161
2162 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
2163}
2164
2165InstructionCost RISCVTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
2166 Type *Src,
2167 TTI::CastContextHint CCH,
2168 TTI::TargetCostKind CostKind,
2169 const Instruction *I) const {
2170 bool IsVectorType = isa<VectorType>(Val: Dst) && isa<VectorType>(Val: Src);
2171 if (!IsVectorType)
2172 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2173
2174 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
2175 // For now, skip all fixed vector cost analysis when P extension is available
2176 // to avoid crashes in getMinRVVVectorSizeInBits()
2177 if (ST->hasStdExtP() &&
2178 (isa<FixedVectorType>(Val: Dst) || isa<FixedVectorType>(Val: Src))) {
2179 return 1; // Treat as single instruction cost for now
2180 }
2181
2182 // FIXME: Need to compute legalizing cost for illegal types. The current
2183 // code handles only legal types and those which can be trivially
2184 // promoted to legal.
2185 if (!ST->hasVInstructions() || Src->getScalarSizeInBits() > ST->getELen() ||
2186 Dst->getScalarSizeInBits() > ST->getELen())
2187 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2188
2189 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2190 assert(ISD && "Invalid opcode");
2191 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: Src);
2192 std::pair<InstructionCost, MVT> DstLT = getTypeLegalizationCost(Ty: Dst);
2193
2194 // Handle i1 source and dest cases *before* calling logic in BasicTTI.
2195 // The shared implementation doesn't model vector widening during legalization
2196 // and instead assumes scalarization. In order to scalarize an <N x i1>
2197 // vector, we need to extend/trunc to/from i8. If we don't special case
2198 // this, we can get an infinite recursion cycle.
2199 switch (ISD) {
2200 default:
2201 break;
2202 case ISD::SIGN_EXTEND:
2203 case ISD::ZERO_EXTEND:
2204 if (Src->getScalarSizeInBits() == 1) {
2205 // We do not use vsext/vzext to extend from mask vector.
2206 // Instead we use the following instructions to extend from mask vector:
2207 // vmv.v.i v8, 0
2208 // vmerge.vim v8, v8, -1, v0 (repeated per split)
2209 return getRISCVInstructionCost(OpCodes: RISCV::VMV_V_I, VT: DstLT.second, CostKind) +
2210 DstLT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VIM,
2211 VT: DstLT.second, CostKind) +
2212 DstLT.first - 1;
2213 }
2214 break;
2215 case ISD::TRUNCATE:
2216 if (Dst->getScalarSizeInBits() == 1) {
2217 // We do not use several vncvt to truncate to mask vector. So we could
2218 // not use PowDiff to calculate it.
2219 // Instead we use the following instructions to truncate to mask vector:
2220 // vand.vi v8, v8, 1
2221 // vmsne.vi v0, v8, 0
2222 return SrcLT.first *
2223 getRISCVInstructionCost(OpCodes: {RISCV::VAND_VI, RISCV::VMSNE_VI},
2224 VT: SrcLT.second, CostKind) +
2225 SrcLT.first - 1;
2226 }
2227 break;
2228 };
2229
2230 // Our actual lowering for the case where a wider legal type is available
2231 // uses promotion to the wider type. This is reflected in the result of
2232 // getTypeLegalizationCost, but BasicTTI assumes the widened cases are
2233 // scalarized if the legalized Src and Dst are not equal sized.
2234 const DataLayout &DL = this->getDataLayout();
2235 if (!SrcLT.second.isVector() || !DstLT.second.isVector() ||
2236 !SrcLT.first.isValid() || !DstLT.first.isValid() ||
2237 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Src),
2238 RHS: SrcLT.second.getSizeInBits()) ||
2239 !TypeSize::isKnownLE(LHS: DL.getTypeSizeInBits(Ty: Dst),
2240 RHS: DstLT.second.getSizeInBits()) ||
2241 SrcLT.first > 1 || DstLT.first > 1)
2242 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2243
2244 // The split cost is handled by the base getCastInstrCost
2245 assert((SrcLT.first == 1) && (DstLT.first == 1) && "Illegal type");
2246
2247 int PowDiff = (int)Log2_32(Value: DstLT.second.getScalarSizeInBits()) -
2248 (int)Log2_32(Value: SrcLT.second.getScalarSizeInBits());
2249 switch (ISD) {
2250 case ISD::SIGN_EXTEND:
2251 case ISD::ZERO_EXTEND: {
2252 if ((PowDiff < 1) || (PowDiff > 3))
2253 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2254 unsigned SExtOp[] = {RISCV::VSEXT_VF2, RISCV::VSEXT_VF4, RISCV::VSEXT_VF8};
2255 unsigned ZExtOp[] = {RISCV::VZEXT_VF2, RISCV::VZEXT_VF4, RISCV::VZEXT_VF8};
2256 unsigned Op =
2257 (ISD == ISD::SIGN_EXTEND) ? SExtOp[PowDiff - 1] : ZExtOp[PowDiff - 1];
2258 return getRISCVInstructionCost(OpCodes: Op, VT: DstLT.second, CostKind);
2259 }
2260 case ISD::TRUNCATE:
2261 case ISD::FP_EXTEND:
2262 case ISD::FP_ROUND: {
2263 // Counts of narrow/widen instructions.
2264 unsigned SrcEltSize = SrcLT.second.getScalarSizeInBits();
2265 unsigned DstEltSize = DstLT.second.getScalarSizeInBits();
2266
2267 unsigned Op = (ISD == ISD::TRUNCATE) ? RISCV::VNSRL_WI
2268 : (ISD == ISD::FP_EXTEND) ? RISCV::VFWCVT_F_F_V
2269 : RISCV::VFNCVT_F_F_W;
2270 InstructionCost Cost = 0;
2271 for (; SrcEltSize != DstEltSize;) {
2272 MVT ElementMVT = (ISD == ISD::TRUNCATE)
2273 ? MVT::getIntegerVT(BitWidth: DstEltSize)
2274 : MVT::getFloatingPointVT(BitWidth: DstEltSize);
2275 MVT DstMVT = DstLT.second.changeVectorElementType(EltVT: ElementMVT);
2276 DstEltSize =
2277 (DstEltSize > SrcEltSize) ? DstEltSize >> 1 : DstEltSize << 1;
2278 Cost += getRISCVInstructionCost(OpCodes: Op, VT: DstMVT, CostKind);
2279 }
2280 return Cost;
2281 }
2282 case ISD::FP_TO_SINT:
2283 case ISD::FP_TO_UINT: {
2284 unsigned IsSigned = ISD == ISD::FP_TO_SINT;
2285 unsigned FCVT = IsSigned ? RISCV::VFCVT_RTZ_X_F_V : RISCV::VFCVT_RTZ_XU_F_V;
2286 unsigned FWCVT =
2287 IsSigned ? RISCV::VFWCVT_RTZ_X_F_V : RISCV::VFWCVT_RTZ_XU_F_V;
2288 unsigned FNCVT =
2289 IsSigned ? RISCV::VFNCVT_RTZ_X_F_W : RISCV::VFNCVT_RTZ_XU_F_W;
2290 unsigned SrcEltSize = Src->getScalarSizeInBits();
2291 unsigned DstEltSize = Dst->getScalarSizeInBits();
2292 InstructionCost Cost = 0;
2293 if ((SrcEltSize == 16) &&
2294 (!ST->hasVInstructionsF16() || ((DstEltSize / 2) > SrcEltSize))) {
2295 // If the target only supports zvfhmin or it is fp16-to-i64 conversion
2296 // pre-widening to f32 and then convert f32 to integer
2297 VectorType *VecF32Ty =
2298 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2299 EC: cast<VectorType>(Val: Dst)->getElementCount());
2300 std::pair<InstructionCost, MVT> VecF32LT =
2301 getTypeLegalizationCost(Ty: VecF32Ty);
2302 Cost +=
2303 VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFWCVT_F_F_V,
2304 VT: VecF32LT.second, CostKind);
2305 Cost += getCastInstrCost(Opcode, Dst, Src: VecF32Ty, CCH, CostKind, I);
2306 return Cost;
2307 }
2308 if (DstEltSize == SrcEltSize)
2309 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2310 else if (DstEltSize > SrcEltSize)
2311 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2312 else { // (SrcEltSize > DstEltSize)
2313 // First do a narrowing conversion to an integer half the size, then
2314 // truncate if needed.
2315 MVT ElementVT = MVT::getIntegerVT(BitWidth: SrcEltSize / 2);
2316 MVT VecVT = DstLT.second.changeVectorElementType(EltVT: ElementVT);
2317 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: VecVT, CostKind);
2318 if ((SrcEltSize / 2) > DstEltSize) {
2319 Type *VecTy = EVT(VecVT).getTypeForEVT(Context&: Dst->getContext());
2320 Cost +=
2321 getCastInstrCost(Opcode: Instruction::Trunc, Dst, Src: VecTy, CCH, CostKind, I);
2322 }
2323 }
2324 return Cost;
2325 }
2326 case ISD::SINT_TO_FP:
2327 case ISD::UINT_TO_FP: {
2328 unsigned IsSigned = ISD == ISD::SINT_TO_FP;
2329 unsigned FCVT = IsSigned ? RISCV::VFCVT_F_X_V : RISCV::VFCVT_F_XU_V;
2330 unsigned FWCVT = IsSigned ? RISCV::VFWCVT_F_X_V : RISCV::VFWCVT_F_XU_V;
2331 unsigned FNCVT = IsSigned ? RISCV::VFNCVT_F_X_W : RISCV::VFNCVT_F_XU_W;
2332 unsigned SrcEltSize = Src->getScalarSizeInBits();
2333 unsigned DstEltSize = Dst->getScalarSizeInBits();
2334
2335 InstructionCost Cost = 0;
2336 if ((DstEltSize == 16) &&
2337 (!ST->hasVInstructionsF16() || ((SrcEltSize / 2) > DstEltSize))) {
2338 // If the target only supports zvfhmin or it is i64-to-fp16 conversion
2339 // it is converted to f32 and then converted to f16
2340 VectorType *VecF32Ty =
2341 VectorType::get(ElementType: Type::getFloatTy(C&: Dst->getContext()),
2342 EC: cast<VectorType>(Val: Dst)->getElementCount());
2343 std::pair<InstructionCost, MVT> VecF32LT =
2344 getTypeLegalizationCost(Ty: VecF32Ty);
2345 Cost += getCastInstrCost(Opcode, Dst: VecF32Ty, Src, CCH, CostKind, I);
2346 Cost += VecF32LT.first * getRISCVInstructionCost(OpCodes: RISCV::VFNCVT_F_F_W,
2347 VT: DstLT.second, CostKind);
2348 return Cost;
2349 }
2350
2351 if (DstEltSize == SrcEltSize)
2352 Cost += getRISCVInstructionCost(OpCodes: FCVT, VT: DstLT.second, CostKind);
2353 else if (DstEltSize > SrcEltSize) {
2354 if ((DstEltSize / 2) > SrcEltSize) {
2355 VectorType *VecTy =
2356 VectorType::get(ElementType: IntegerType::get(C&: Dst->getContext(), NumBits: DstEltSize / 2),
2357 EC: cast<VectorType>(Val: Dst)->getElementCount());
2358 unsigned Op = IsSigned ? Instruction::SExt : Instruction::ZExt;
2359 Cost += getCastInstrCost(Opcode: Op, Dst: VecTy, Src, CCH, CostKind, I);
2360 }
2361 Cost += getRISCVInstructionCost(OpCodes: FWCVT, VT: DstLT.second, CostKind);
2362 } else
2363 Cost += getRISCVInstructionCost(OpCodes: FNCVT, VT: DstLT.second, CostKind);
2364 return Cost;
2365 }
2366 }
2367 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
2368}
2369
2370unsigned RISCVTTIImpl::getEstimatedVLFor(VectorType *Ty) const {
2371 if (isa<ScalableVectorType>(Val: Ty)) {
2372 const unsigned EltSize = DL.getTypeSizeInBits(Ty: Ty->getElementType());
2373 const unsigned MinSize = DL.getTypeSizeInBits(Ty).getKnownMinValue();
2374 const unsigned VectorBits = *getVScaleForTuning() * RISCV::RVVBitsPerBlock;
2375 return RISCVTargetLowering::computeVLMAX(VectorBits, EltSize, MinSize);
2376 }
2377 return cast<FixedVectorType>(Val: Ty)->getNumElements();
2378}
2379
2380InstructionCost
2381RISCVTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
2382 FastMathFlags FMF,
2383 TTI::TargetCostKind CostKind) const {
2384 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2385 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2386
2387 // Skip if scalar size of Ty is bigger than ELEN.
2388 if (Ty->getScalarSizeInBits() > ST->getELen())
2389 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2390
2391 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2392 if (Ty->getElementType()->isIntegerTy(BitWidth: 1)) {
2393 // SelectionDAGBuilder does following transforms:
2394 // vector_reduce_{smin,umax}(<n x i1>) --> vector_reduce_or(<n x i1>)
2395 // vector_reduce_{smax,umin}(<n x i1>) --> vector_reduce_and(<n x i1>)
2396 if (IID == Intrinsic::umax || IID == Intrinsic::smin)
2397 return getArithmeticReductionCost(Opcode: Instruction::Or, Ty, FMF, CostKind);
2398 else
2399 return getArithmeticReductionCost(Opcode: Instruction::And, Ty, FMF, CostKind);
2400 }
2401
2402 if (IID == Intrinsic::maximum || IID == Intrinsic::minimum) {
2403 SmallVector<unsigned, 3> Opcodes;
2404 InstructionCost ExtraCost = 0;
2405 switch (IID) {
2406 case Intrinsic::maximum:
2407 if (FMF.noNaNs()) {
2408 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2409 } else {
2410 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMAX_VS,
2411 RISCV::VFMV_F_S};
2412 // Cost of Canonical Nan + branch
2413 // lui a0, 523264
2414 // fmv.w.x fa0, a0
2415 Type *DstTy = Ty->getScalarType();
2416 const unsigned EltTyBits = DstTy->getScalarSizeInBits();
2417 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2418 ExtraCost = 1 +
2419 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2420 CCH: TTI::CastContextHint::None, CostKind) +
2421 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2422 }
2423 break;
2424
2425 case Intrinsic::minimum:
2426 if (FMF.noNaNs()) {
2427 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2428 } else {
2429 Opcodes = {RISCV::VMFNE_VV, RISCV::VCPOP_M, RISCV::VFREDMIN_VS,
2430 RISCV::VFMV_F_S};
2431 // Cost of Canonical Nan + branch
2432 // lui a0, 523264
2433 // fmv.w.x fa0, a0
2434 Type *DstTy = Ty->getScalarType();
2435 const unsigned EltTyBits = DL.getTypeSizeInBits(Ty: DstTy);
2436 Type *SrcTy = IntegerType::getIntNTy(C&: DstTy->getContext(), N: EltTyBits);
2437 ExtraCost = 1 +
2438 getCastInstrCost(Opcode: Instruction::UIToFP, Dst: DstTy, Src: SrcTy,
2439 CCH: TTI::CastContextHint::None, CostKind) +
2440 getCFInstrCost(Opcode: Instruction::CondBr, CostKind);
2441 }
2442 break;
2443 }
2444 return ExtraCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2445 }
2446
2447 // IR Reduction is composed by one rvv reduction instruction and vmv
2448 unsigned SplitOp;
2449 SmallVector<unsigned, 3> Opcodes;
2450 switch (IID) {
2451 default:
2452 llvm_unreachable("Unsupported intrinsic");
2453 case Intrinsic::smax:
2454 SplitOp = RISCV::VMAX_VV;
2455 Opcodes = {RISCV::VREDMAX_VS, RISCV::VMV_X_S};
2456 break;
2457 case Intrinsic::smin:
2458 SplitOp = RISCV::VMIN_VV;
2459 Opcodes = {RISCV::VREDMIN_VS, RISCV::VMV_X_S};
2460 break;
2461 case Intrinsic::umax:
2462 SplitOp = RISCV::VMAXU_VV;
2463 Opcodes = {RISCV::VREDMAXU_VS, RISCV::VMV_X_S};
2464 break;
2465 case Intrinsic::umin:
2466 SplitOp = RISCV::VMINU_VV;
2467 Opcodes = {RISCV::VREDMINU_VS, RISCV::VMV_X_S};
2468 break;
2469 case Intrinsic::maxnum:
2470 SplitOp = RISCV::VFMAX_VV;
2471 Opcodes = {RISCV::VFREDMAX_VS, RISCV::VFMV_F_S};
2472 break;
2473 case Intrinsic::minnum:
2474 SplitOp = RISCV::VFMIN_VV;
2475 Opcodes = {RISCV::VFREDMIN_VS, RISCV::VFMV_F_S};
2476 break;
2477 }
2478 // Add a cost for data larger than LMUL8
2479 InstructionCost SplitCost =
2480 (LT.first > 1) ? (LT.first - 1) *
2481 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2482 : 0;
2483 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2484}
2485
2486InstructionCost
2487RISCVTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *Ty,
2488 std::optional<FastMathFlags> FMF,
2489 TTI::TargetCostKind CostKind) const {
2490 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
2491 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2492
2493 // Skip if scalar size of Ty is bigger than ELEN.
2494 if (Ty->getScalarSizeInBits() > ST->getELen())
2495 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2496
2497 int ISD = TLI->InstructionOpcodeToISD(Opcode);
2498 assert(ISD && "Invalid opcode");
2499
2500 if (ISD != ISD::ADD && ISD != ISD::OR && ISD != ISD::XOR && ISD != ISD::AND &&
2501 ISD != ISD::FADD)
2502 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2503
2504 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2505 Type *ElementTy = Ty->getElementType();
2506 if (ElementTy->isIntegerTy(BitWidth: 1)) {
2507 // Example sequences:
2508 // vfirst.m a0, v0
2509 // seqz a0, a0
2510 if (LT.second == MVT::v1i1)
2511 return getRISCVInstructionCost(OpCodes: RISCV::VFIRST_M, VT: LT.second, CostKind) +
2512 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2513 VecPred: CmpInst::ICMP_EQ, CostKind);
2514
2515 if (ISD == ISD::AND) {
2516 // Example sequences:
2517 // vmand.mm v8, v9, v8 ; needed every time type is split
2518 // vmnot.m v8, v0 ; alias for vmnand
2519 // vcpop.m a0, v8
2520 // seqz a0, a0
2521
2522 // See the discussion: https://github.com/llvm/llvm-project/pull/119160
2523 // For LMUL <= 8, there is no splitting,
2524 // the sequences are vmnot, vcpop and seqz.
2525 // When LMUL > 8 and split = 1,
2526 // the sequences are vmnand, vcpop and seqz.
2527 // When LMUL > 8 and split > 1,
2528 // the sequences are (LT.first-2) * vmand, vmnand, vcpop and seqz.
2529 return ((LT.first > 2) ? (LT.first - 2) : 0) *
2530 getRISCVInstructionCost(OpCodes: RISCV::VMAND_MM, VT: LT.second, CostKind) +
2531 getRISCVInstructionCost(OpCodes: RISCV::VMNAND_MM, VT: LT.second, CostKind) +
2532 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2533 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2534 VecPred: CmpInst::ICMP_EQ, CostKind);
2535 } else if (ISD == ISD::XOR || ISD == ISD::ADD) {
2536 // Example sequences:
2537 // vsetvli a0, zero, e8, mf8, ta, ma
2538 // vmxor.mm v8, v0, v8 ; needed every time type is split
2539 // vcpop.m a0, v8
2540 // andi a0, a0, 1
2541 return (LT.first - 1) *
2542 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind) +
2543 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) + 1;
2544 } else {
2545 assert(ISD == ISD::OR);
2546 // Example sequences:
2547 // vsetvli a0, zero, e8, mf8, ta, ma
2548 // vmor.mm v8, v9, v8 ; needed every time type is split
2549 // vcpop.m a0, v0
2550 // snez a0, a0
2551 return (LT.first - 1) *
2552 getRISCVInstructionCost(OpCodes: RISCV::VMOR_MM, VT: LT.second, CostKind) +
2553 getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind) +
2554 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: ElementTy, CondTy: ElementTy,
2555 VecPred: CmpInst::ICMP_NE, CostKind);
2556 }
2557 }
2558
2559 // IR Reduction of or/and is composed by one vmv and one rvv reduction
2560 // instruction, and others is composed by two vmv and one rvv reduction
2561 // instruction
2562 unsigned SplitOp;
2563 SmallVector<unsigned, 3> Opcodes;
2564 switch (ISD) {
2565 case ISD::ADD:
2566 SplitOp = RISCV::VADD_VV;
2567 Opcodes = {RISCV::VMV_S_X, RISCV::VREDSUM_VS, RISCV::VMV_X_S};
2568 break;
2569 case ISD::OR:
2570 SplitOp = RISCV::VOR_VV;
2571 Opcodes = {RISCV::VREDOR_VS, RISCV::VMV_X_S};
2572 break;
2573 case ISD::XOR:
2574 SplitOp = RISCV::VXOR_VV;
2575 Opcodes = {RISCV::VMV_S_X, RISCV::VREDXOR_VS, RISCV::VMV_X_S};
2576 break;
2577 case ISD::AND:
2578 SplitOp = RISCV::VAND_VV;
2579 Opcodes = {RISCV::VREDAND_VS, RISCV::VMV_X_S};
2580 break;
2581 case ISD::FADD:
2582 // We can't promote f16/bf16 fadd reductions.
2583 if ((LT.second.getScalarType() == MVT::f16 && !ST->hasVInstructionsF16()) ||
2584 LT.second.getScalarType() == MVT::bf16)
2585 return BaseT::getArithmeticReductionCost(Opcode, Ty, FMF, CostKind);
2586 if (TTI::requiresOrderedReduction(FMF)) {
2587 Opcodes.push_back(Elt: RISCV::VFMV_S_F);
2588 for (unsigned i = 0; i < LT.first.getValue(); i++)
2589 Opcodes.push_back(Elt: RISCV::VFREDOSUM_VS);
2590 Opcodes.push_back(Elt: RISCV::VFMV_F_S);
2591 return getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2592 }
2593 SplitOp = RISCV::VFADD_VV;
2594 Opcodes = {RISCV::VFMV_S_F, RISCV::VFREDUSUM_VS, RISCV::VFMV_F_S};
2595 break;
2596 }
2597 // Add a cost for data larger than LMUL8
2598 InstructionCost SplitCost =
2599 (LT.first > 1) ? (LT.first - 1) *
2600 getRISCVInstructionCost(OpCodes: SplitOp, VT: LT.second, CostKind)
2601 : 0;
2602 return SplitCost + getRISCVInstructionCost(OpCodes: Opcodes, VT: LT.second, CostKind);
2603}
2604
2605InstructionCost RISCVTTIImpl::getExtendedReductionCost(
2606 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
2607 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
2608 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2609 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2610 FMF, CostKind);
2611
2612 // Skip if scalar size of ResTy is bigger than ELEN.
2613 if (ResTy->getScalarSizeInBits() > ST->getELen())
2614 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2615 FMF, CostKind);
2616
2617 if (Opcode != Instruction::Add && Opcode != Instruction::FAdd)
2618 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2619 FMF, CostKind);
2620
2621 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2622
2623 if (IsUnsigned && Opcode == Instruction::Add && LT.first == 1 &&
2624 LT.second.isVectorOf(EltVT: MVT::i1)) {
2625 // Represent vector_reduce_add(ZExt(<i1 vector>)) as
2626 // ZExtOrTrunc(ctpop(bitcast <i1 vector> to in)).
2627 return getRISCVInstructionCost(OpCodes: RISCV::VCPOP_M, VT: LT.second, CostKind);
2628 }
2629
2630 if (ResTy->getScalarSizeInBits() != 2 * LT.second.getScalarSizeInBits())
2631 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy,
2632 FMF, CostKind);
2633
2634 return (LT.first - 1) +
2635 getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
2636}
2637
2638InstructionCost
2639RISCVTTIImpl::getStoreImmCost(Type *Ty, TTI::OperandValueInfo OpInfo,
2640 TTI::TargetCostKind CostKind) const {
2641 assert(OpInfo.isConstant() && "non constant operand?");
2642 if (!isa<VectorType>(Val: Ty))
2643 // FIXME: We need to account for immediate materialization here, but doing
2644 // a decent job requires more knowledge about the immediate than we
2645 // currently have here.
2646 return 0;
2647
2648 if (OpInfo.isUniform())
2649 // vmv.v.i, vmv.v.x, or vfmv.v.f
2650 // We ignore the cost of the scalar constant materialization to be consistent
2651 // with how we treat scalar constants themselves just above.
2652 return 1;
2653
2654 return getConstantPoolLoadCost(Ty, CostKind);
2655}
2656
2657InstructionCost RISCVTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
2658 Align Alignment,
2659 unsigned AddressSpace,
2660 TTI::TargetCostKind CostKind,
2661 TTI::OperandValueInfo OpInfo,
2662 const Instruction *I) const {
2663 EVT VT = TLI->getValueType(DL, Ty: Src, AllowUnknown: true);
2664 // Type legalization can't handle structs, and load latency isn't handled here
2665 if (VT == MVT::Other ||
2666 (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency))
2667 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2668 CostKind, OpInfo, I);
2669
2670 InstructionCost Cost = 0;
2671 if (Opcode == Instruction::Store && OpInfo.isConstant())
2672 Cost += getStoreImmCost(Ty: Src, OpInfo, CostKind);
2673
2674 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
2675
2676 InstructionCost BaseCost = [&]() {
2677 InstructionCost Cost = LT.first;
2678 if (CostKind != TTI::TCK_RecipThroughput)
2679 return Cost;
2680
2681 // Our actual lowering for the case where a wider legal type is available
2682 // uses the a VL predicated load on the wider type. This is reflected in
2683 // the result of getTypeLegalizationCost, but BasicTTI assumes the
2684 // widened cases are scalarized.
2685 const DataLayout &DL = this->getDataLayout();
2686 if (Src->isVectorTy() && LT.second.isVector() &&
2687 TypeSize::isKnownLT(LHS: DL.getTypeStoreSizeInBits(Ty: Src),
2688 RHS: LT.second.getSizeInBits()))
2689 return Cost;
2690
2691 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
2692 CostKind, OpInfo, I);
2693 }();
2694
2695 // Assume memory ops cost scale with the number of vector registers
2696 // possible accessed by the instruction. Note that BasicTTI already
2697 // handles the LT.first term for us.
2698 if (ST->hasVInstructions() && LT.second.isVector() &&
2699 CostKind != TTI::TCK_CodeSize)
2700 BaseCost *= TLI->getLMULCost(VT: LT.second);
2701 return Cost + BaseCost;
2702}
2703
2704InstructionCost RISCVTTIImpl::getCmpSelInstrCost(
2705 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
2706 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
2707 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
2708 if (CostKind != TTI::TCK_RecipThroughput)
2709 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2710 Op1Info, Op2Info, I);
2711
2712 if (isa<FixedVectorType>(Val: ValTy) && !ST->useRVVForFixedLengthVectors())
2713 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2714 Op1Info, Op2Info, I);
2715
2716 // Skip if scalar size of ValTy is bigger than ELEN.
2717 if (ValTy->isVectorTy() && ValTy->getScalarSizeInBits() > ST->getELen())
2718 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2719 Op1Info, Op2Info, I);
2720
2721 auto GetConstantMatCost =
2722 [&](TTI::OperandValueInfo OpInfo) -> InstructionCost {
2723 if (OpInfo.isUniform())
2724 // We return 0 we currently ignore the cost of materializing scalar
2725 // constants in GPRs.
2726 return 0;
2727
2728 return getConstantPoolLoadCost(Ty: ValTy, CostKind);
2729 };
2730
2731 InstructionCost ConstantMatCost;
2732 if (Op1Info.isConstant())
2733 ConstantMatCost += GetConstantMatCost(Op1Info);
2734 if (Op2Info.isConstant())
2735 ConstantMatCost += GetConstantMatCost(Op2Info);
2736
2737 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
2738 if (Opcode == Instruction::Select && LT.second.isVector()) {
2739 if (CondTy->isVectorTy()) {
2740 if (ValTy->getScalarSizeInBits() == 1) {
2741 // vmandn.mm v8, v8, v9
2742 // vmand.mm v9, v0, v9
2743 // vmor.mm v0, v9, v8
2744 return ConstantMatCost +
2745 LT.first *
2746 getRISCVInstructionCost(
2747 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2748 VT: LT.second, CostKind);
2749 }
2750 // vselect and max/min are supported natively.
2751 return ConstantMatCost +
2752 LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMERGE_VVM, VT: LT.second,
2753 CostKind);
2754 }
2755
2756 if (ValTy->getScalarSizeInBits() == 1) {
2757 // vmv.v.x v9, a0
2758 // vmsne.vi v9, v9, 0
2759 // vmandn.mm v8, v8, v9
2760 // vmand.mm v9, v0, v9
2761 // vmor.mm v0, v9, v8
2762 MVT InterimVT = LT.second.changeVectorElementType(EltVT: MVT::i8);
2763 return ConstantMatCost +
2764 LT.first *
2765 getRISCVInstructionCost(OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI},
2766 VT: InterimVT, CostKind) +
2767 LT.first * getRISCVInstructionCost(
2768 OpCodes: {RISCV::VMANDN_MM, RISCV::VMAND_MM, RISCV::VMOR_MM},
2769 VT: LT.second, CostKind);
2770 }
2771
2772 // vmv.v.x v10, a0
2773 // vmsne.vi v0, v10, 0
2774 // vmerge.vvm v8, v9, v8, v0
2775 return ConstantMatCost +
2776 LT.first * getRISCVInstructionCost(
2777 OpCodes: {RISCV::VMV_V_X, RISCV::VMSNE_VI, RISCV::VMERGE_VVM},
2778 VT: LT.second, CostKind);
2779 }
2780
2781 if ((Opcode == Instruction::ICmp) && ValTy->isVectorTy() &&
2782 CmpInst::isIntPredicate(P: VecPred)) {
2783 // Use VMSLT_VV to represent VMSEQ, VMSNE, VMSLTU, VMSLEU, VMSLT, VMSLE
2784 // provided they incur the same cost across all implementations
2785 return ConstantMatCost + LT.first * getRISCVInstructionCost(OpCodes: RISCV::VMSLT_VV,
2786 VT: LT.second,
2787 CostKind);
2788 }
2789
2790 if ((Opcode == Instruction::FCmp) && ValTy->isVectorTy() &&
2791 CmpInst::isFPPredicate(P: VecPred)) {
2792
2793 // Use VMXOR_MM and VMXNOR_MM to generate all true/false mask
2794 if ((VecPred == CmpInst::FCMP_FALSE) || (VecPred == CmpInst::FCMP_TRUE))
2795 return ConstantMatCost +
2796 getRISCVInstructionCost(OpCodes: RISCV::VMXOR_MM, VT: LT.second, CostKind);
2797
2798 // If we do not support the input floating point vector type, use the base
2799 // one which will calculate as:
2800 // ScalarizeCost + Num * Cost for fixed vector,
2801 // InvalidCost for scalable vector.
2802 if ((ValTy->getScalarSizeInBits() == 16 && !ST->hasVInstructionsF16()) ||
2803 (ValTy->getScalarSizeInBits() == 32 && !ST->hasVInstructionsF32()) ||
2804 (ValTy->getScalarSizeInBits() == 64 && !ST->hasVInstructionsF64()))
2805 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2806 Op1Info, Op2Info, I);
2807
2808 // Assuming vector fp compare and mask instructions are all the same cost
2809 // until a need arises to differentiate them.
2810 switch (VecPred) {
2811 case CmpInst::FCMP_ONE: // vmflt.vv + vmflt.vv + vmor.mm
2812 case CmpInst::FCMP_ORD: // vmfeq.vv + vmfeq.vv + vmand.mm
2813 case CmpInst::FCMP_UNO: // vmfne.vv + vmfne.vv + vmor.mm
2814 case CmpInst::FCMP_UEQ: // vmflt.vv + vmflt.vv + vmnor.mm
2815 return ConstantMatCost +
2816 LT.first * getRISCVInstructionCost(
2817 OpCodes: {RISCV::VMFLT_VV, RISCV::VMFLT_VV, RISCV::VMOR_MM},
2818 VT: LT.second, CostKind);
2819
2820 case CmpInst::FCMP_UGT: // vmfle.vv + vmnot.m
2821 case CmpInst::FCMP_UGE: // vmflt.vv + vmnot.m
2822 case CmpInst::FCMP_ULT: // vmfle.vv + vmnot.m
2823 case CmpInst::FCMP_ULE: // vmflt.vv + vmnot.m
2824 return ConstantMatCost +
2825 LT.first *
2826 getRISCVInstructionCost(OpCodes: {RISCV::VMFLT_VV, RISCV::VMNAND_MM},
2827 VT: LT.second, CostKind);
2828
2829 case CmpInst::FCMP_OEQ: // vmfeq.vv
2830 case CmpInst::FCMP_OGT: // vmflt.vv
2831 case CmpInst::FCMP_OGE: // vmfle.vv
2832 case CmpInst::FCMP_OLT: // vmflt.vv
2833 case CmpInst::FCMP_OLE: // vmfle.vv
2834 case CmpInst::FCMP_UNE: // vmfne.vv
2835 return ConstantMatCost +
2836 LT.first *
2837 getRISCVInstructionCost(OpCodes: RISCV::VMFLT_VV, VT: LT.second, CostKind);
2838 default:
2839 break;
2840 }
2841 }
2842
2843 // With ShortForwardBranchOpt or ConditionalMoveFusion, scalar icmp + select
2844 // instructions will lower to SELECT_CC and lower to PseudoCCMOVGPR which will
2845 // generate a conditional branch + mv. The cost of scalar (icmp + select) will
2846 // be (0 + select instr cost).
2847 if (ST->hasConditionalMoveFusion() && I && isa<ICmpInst>(Val: I) &&
2848 ValTy->isIntegerTy() && !I->user_empty()) {
2849 if (all_of(Range: I->users(), P: [&](const User *U) {
2850 return match(V: U, P: m_Select(C: m_Specific(V: I), L: m_Value(), R: m_Value())) &&
2851 U->getType()->isIntegerTy() &&
2852 !isa<ConstantData>(Val: U->getOperand(i: 1)) &&
2853 !isa<ConstantData>(Val: U->getOperand(i: 2));
2854 }))
2855 return 0;
2856 }
2857
2858 // TODO: Add cost for scalar type.
2859
2860 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
2861 Op1Info, Op2Info, I);
2862}
2863
2864InstructionCost RISCVTTIImpl::getCFInstrCost(unsigned Opcode,
2865 TTI::TargetCostKind CostKind,
2866 const Instruction *I) const {
2867 if (CostKind != TTI::TCK_RecipThroughput)
2868 return Opcode == Instruction::PHI ? 0 : 1;
2869 // Branches are assumed to be predicted.
2870 return 0;
2871}
2872
2873InstructionCost RISCVTTIImpl::getVectorInstrCost(
2874 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
2875 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
2876 assert(Val->isVectorTy() && "This must be a vector type");
2877
2878 // TODO: Add proper cost model for P extension fixed vectors (e.g., v4i16)
2879 // For now, skip all fixed vector cost analysis when P extension is available
2880 // to avoid crashes in getMinRVVVectorSizeInBits()
2881 if (ST->hasStdExtP() && isa<FixedVectorType>(Val)) {
2882 return 1; // Treat as single instruction cost for now
2883 }
2884
2885 if (Opcode != Instruction::ExtractElement &&
2886 Opcode != Instruction::InsertElement)
2887 return BaseT::getVectorInstrCost(Opcode, Val, CostKind, Index, Op0, Op1,
2888 VIC);
2889
2890 // Scalar splat operand can be folded for vector ops that support splatting
2891 // the scalar operand, so the explicit insertelement is free in this context.
2892 if (Opcode == Instruction::InsertElement &&
2893 VIC == TTI::VectorInstrContext::SplatOpFolded &&
2894 ST->sinkSplatOperands() && Index == 0)
2895 return TTI::TCC_Free;
2896
2897 // Legalize the type.
2898 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Val);
2899
2900 // This type is legalized to a scalar type.
2901 if (!LT.second.isVector()) {
2902 auto *FixedVecTy = cast<FixedVectorType>(Val);
2903 // If Index is a known constant, cost is zero.
2904 if (Index != -1U)
2905 return 0;
2906 // Extract/InsertElement with non-constant index is very costly when
2907 // scalarized; estimate cost of loads/stores sequence via the stack:
2908 // ExtractElement cost: store vector to stack, load scalar;
2909 // InsertElement cost: store vector to stack, store scalar, load vector.
2910 Type *ElemTy = FixedVecTy->getElementType();
2911 auto NumElems = FixedVecTy->getNumElements();
2912 auto Align = DL.getPrefTypeAlign(Ty: ElemTy);
2913 InstructionCost LoadCost =
2914 getMemoryOpCost(Opcode: Instruction::Load, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2915 InstructionCost StoreCost =
2916 getMemoryOpCost(Opcode: Instruction::Store, Src: ElemTy, Alignment: Align, AddressSpace: 0, CostKind);
2917 return Opcode == Instruction::ExtractElement
2918 ? StoreCost * NumElems + LoadCost
2919 : (StoreCost + LoadCost) * NumElems + StoreCost;
2920 }
2921
2922 // For unsupported scalable vector.
2923 if (LT.second.isScalableVector() && !LT.first.isValid())
2924 return LT.first;
2925
2926 // Mask vector extract/insert is expanded via e8.
2927 if (Val->getScalarSizeInBits() == 1) {
2928 VectorType *WideTy =
2929 VectorType::get(ElementType: IntegerType::get(C&: Val->getContext(), NumBits: 8),
2930 EC: cast<VectorType>(Val)->getElementCount());
2931 if (Opcode == Instruction::ExtractElement) {
2932 InstructionCost ExtendCost
2933 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2934 CCH: TTI::CastContextHint::None, CostKind);
2935 InstructionCost ExtractCost
2936 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2937 return ExtendCost + ExtractCost;
2938 }
2939 InstructionCost ExtendCost
2940 = getCastInstrCost(Opcode: Instruction::ZExt, Dst: WideTy, Src: Val,
2941 CCH: TTI::CastContextHint::None, CostKind);
2942 InstructionCost InsertCost
2943 = getVectorInstrCost(Opcode, Val: WideTy, CostKind, Index, Op0: nullptr, Op1: nullptr);
2944 InstructionCost TruncCost
2945 = getCastInstrCost(Opcode: Instruction::Trunc, Dst: Val, Src: WideTy,
2946 CCH: TTI::CastContextHint::None, CostKind);
2947 return ExtendCost + InsertCost + TruncCost;
2948 }
2949
2950
2951 // In RVV, we could use vslidedown + vmv.x.s to extract element from vector
2952 // and vslideup + vmv.s.x to insert element to vector.
2953 unsigned MoveOpc;
2954 if (LT.second.isFloatingPoint())
2955 MoveOpc = Opcode == Instruction::InsertElement ? RISCV::VFMV_S_F
2956 : RISCV::VFMV_F_S;
2957 else
2958 MoveOpc =
2959 Opcode == Instruction::InsertElement ? RISCV::VMV_S_X : RISCV::VMV_X_S;
2960 InstructionCost BaseCost =
2961 getRISCVInstructionCost(OpCodes: MoveOpc, VT: LT.second, CostKind);
2962 // When insertelement we should add the index with 1 as the input of vslideup.
2963 InstructionCost SlideCost = Opcode == Instruction::InsertElement ? 2 : 1;
2964
2965 if (Index != -1U) {
2966 // The type may be split. For fixed-width vectors we can normalize the
2967 // index to the new type.
2968 if (LT.second.isFixedLengthVector()) {
2969 unsigned Width = LT.second.getVectorNumElements();
2970 Index = Index % Width;
2971 }
2972
2973 // If exact VLEN is known, we will insert/extract into the appropriate
2974 // subvector with no additional subvector insert/extract cost.
2975 if (auto VLEN = ST->getRealVLen()) {
2976 unsigned EltSize = LT.second.getScalarSizeInBits();
2977 unsigned M1Max = *VLEN / EltSize;
2978 Index = Index % M1Max;
2979 }
2980
2981 if (Index == 0)
2982 // We can extract/insert the first element without vslidedown/vslideup.
2983 SlideCost = 0;
2984 else if (Opcode == Instruction::InsertElement)
2985 SlideCost = 1; // With a constant index, we do not need to use addi.
2986 }
2987
2988 // When the vector needs to split into multiple register groups and the index
2989 // exceeds single vector register group, we need to insert/extract the element
2990 // via stack.
2991 if (LT.first > 1 &&
2992 ((Index == -1U) || (Index >= LT.second.getVectorMinNumElements() &&
2993 LT.second.isScalableVector()))) {
2994 Type *ScalarType = Val->getScalarType();
2995 Align VecAlign = DL.getPrefTypeAlign(Ty: Val);
2996 Align SclAlign = DL.getPrefTypeAlign(Ty: ScalarType);
2997 // Extra addi for unknown index.
2998 InstructionCost IdxCost = Index == -1U ? 1 : 0;
2999
3000 // Store all split vectors into stack and load the target element.
3001 if (Opcode == Instruction::ExtractElement)
3002 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
3003 getMemoryOpCost(Opcode: Instruction::Load, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
3004 CostKind) +
3005 IdxCost;
3006
3007 // Store all split vectors into stack and store the target element and load
3008 // vectors back.
3009 return getMemoryOpCost(Opcode: Instruction::Store, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
3010 getMemoryOpCost(Opcode: Instruction::Load, Src: Val, Alignment: VecAlign, AddressSpace: 0, CostKind) +
3011 getMemoryOpCost(Opcode: Instruction::Store, Src: ScalarType, Alignment: SclAlign, AddressSpace: 0,
3012 CostKind) +
3013 IdxCost;
3014 }
3015
3016 // Extract i64 in the target that has XLEN=32 need more instruction.
3017 if (Val->getScalarType()->isIntegerTy() &&
3018 ST->getXLen() < Val->getScalarSizeInBits()) {
3019 // For extractelement, we need the following instructions:
3020 // vsetivli zero, 1, e64, m1, ta, mu (not count)
3021 // vslidedown.vx v8, v8, a0
3022 // vmv.x.s a0, v8
3023 // li a1, 32
3024 // vsrl.vx v8, v8, a1
3025 // vmv.x.s a1, v8
3026
3027 // For insertelement, we need the following instructions:
3028 // vsetivli zero, 2, e32, m4, ta, ma (don't count)
3029 // vslide1down.vx v12, v8, a0
3030 // vslide1down.vx v12, v12, a1
3031 // addi a0, a2, 1
3032 // vsetvli zero, a0, e64, m4, tu, ma (don't count)
3033 // vslideup.vx v8, v12, a2
3034
3035 // TODO: should we count these special vsetvlis?
3036 BaseCost =
3037 Opcode == Instruction::InsertElement
3038 ? getRISCVInstructionCost(OpCodes: {RISCV::VSLIDE1DOWN_VX,
3039 RISCV::VSLIDE1DOWN_VX,
3040 RISCV::VSLIDEUP_VX},
3041 VT: LT.second, CostKind)
3042 : getRISCVInstructionCost(OpCodes: {RISCV::VSLIDEDOWN_VX, RISCV::VMV_X_S,
3043 RISCV::VSRL_VX, RISCV::VMV_X_S},
3044 VT: LT.second, CostKind);
3045 }
3046 return BaseCost + SlideCost;
3047}
3048
3049InstructionCost
3050RISCVTTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
3051 TTI::TargetCostKind CostKind,
3052 unsigned Index) const {
3053 if (isa<FixedVectorType>(Val))
3054 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
3055 Index);
3056
3057 // TODO: This code replicates what LoopVectorize.cpp used to do when asking
3058 // for the cost of extracting the last lane of a scalable vector. It probably
3059 // needs a more accurate cost.
3060 ElementCount EC = cast<VectorType>(Val)->getElementCount();
3061 assert(Index < EC.getKnownMinValue() && "Unexpected reverse index");
3062 return getVectorInstrCost(Opcode, Val, CostKind,
3063 Index: EC.getKnownMinValue() - 1 - Index, Op0: nullptr,
3064 Op1: nullptr);
3065}
3066
3067/// Check to see if this instruction is expected to be combined to a simpler
3068/// operation during/before lowering. If so return the cost of the combined
3069/// operation rather than provided one. For instance, `udiv i16 %X, 2` is likely
3070/// to be combined to `lshr i16 %X, 1`, so return the cost of a `lshr` rather
3071/// than the cost of a `udiv`
3072std::optional<InstructionCost>
3073RISCVTTIImpl::getCombinedArithmeticInstructionCost(
3074 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
3075 TTI::OperandValueInfo Opd1Info, TTI::OperandValueInfo Opd2Info,
3076 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
3077 // Vector unsigned division/remainder will be simplified to shifts/masks.
3078 if ((Opcode == Instruction::UDiv || Opcode == Instruction::URem) &&
3079 Opd2Info.isConstant() && Opd2Info.isPowerOf2()) {
3080 if (Opcode == Instruction::UDiv)
3081 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind, Op1Info: Opd1Info,
3082 Op2Info: Opd2Info.getNoProps());
3083 // UREM
3084 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind, Op1Info: Opd1Info,
3085 Op2Info: Opd2Info.getNoProps());
3086 }
3087 return std::nullopt;
3088}
3089
3090InstructionCost RISCVTTIImpl::getArithmeticInstrCost(
3091 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
3092 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
3093 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
3094
3095 // TODO: Handle more cost kinds.
3096 if (CostKind != TTI::TCK_RecipThroughput)
3097 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3098 Args, CtxI);
3099
3100 if (isa<FixedVectorType>(Val: Ty) && !ST->useRVVForFixedLengthVectors())
3101 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3102 Args, CtxI);
3103
3104 // Skip if scalar size of Ty is bigger than ELEN.
3105 if (isa<VectorType>(Val: Ty) && Ty->getScalarSizeInBits() > ST->getELen())
3106 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3107 Args, CtxI);
3108
3109 if (std::optional<InstructionCost> CombinedCost =
3110 getCombinedArithmeticInstructionCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
3111 Opd2Info: Op2Info, Args, CtxI))
3112 return *CombinedCost;
3113
3114 // Legalize the type.
3115 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
3116 unsigned ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
3117
3118 // TODO: Handle scalar type.
3119 if (!LT.second.isVector()) {
3120 if (ISDOpcode == ISD::MUL && !ST->hasStdExtZmmul())
3121 return TTI::TCC_Expensive * LT.first;
3122
3123 static const CostTblEntry DivTbl[]{
3124 {.ISD: ISD::UDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
3125 {.ISD: ISD::UDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
3126 {.ISD: ISD::SDIV, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
3127 {.ISD: ISD::SDIV, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
3128 {.ISD: ISD::UREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
3129 {.ISD: ISD::UREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive},
3130 {.ISD: ISD::SREM, .Type: MVT::i32, .Cost: TTI::TCC_Expensive},
3131 {.ISD: ISD::SREM, .Type: MVT::i64, .Cost: TTI::TCC_Expensive}};
3132 if (TLI->isOperationLegalOrPromote(Op: ISDOpcode, VT: LT.second))
3133 if (const auto *Entry = CostTableLookup(Table: DivTbl, ISD: ISDOpcode, Ty: LT.second))
3134 return Entry->Cost * LT.first;
3135
3136 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3137 Args, CtxI);
3138 }
3139
3140 // f16 with zvfhmin and bf16 will be promoted to f32.
3141 // FIXME: nxv32[b]f16 will be custom lowered and split.
3142 InstructionCost CastCost = 0;
3143 if ((LT.second.getVectorElementType() == MVT::f16 ||
3144 LT.second.getVectorElementType() == MVT::bf16) &&
3145 TLI->getOperationAction(Op: ISDOpcode, VT: LT.second) ==
3146 TargetLoweringBase::LegalizeAction::Promote) {
3147 MVT PromotedVT = TLI->getTypeToPromoteTo(Op: ISDOpcode, VT: LT.second);
3148 Type *PromotedTy = EVT(PromotedVT).getTypeForEVT(Context&: Ty->getContext());
3149 Type *LegalTy = EVT(LT.second).getTypeForEVT(Context&: Ty->getContext());
3150 // Add cost of extending arguments
3151 CastCost += LT.first * Args.size() *
3152 getCastInstrCost(Opcode: Instruction::FPExt, Dst: PromotedTy, Src: LegalTy,
3153 CCH: TTI::CastContextHint::None, CostKind);
3154 // Add cost of truncating result
3155 CastCost +=
3156 LT.first * getCastInstrCost(Opcode: Instruction::FPTrunc, Dst: LegalTy, Src: PromotedTy,
3157 CCH: TTI::CastContextHint::None, CostKind);
3158 // Compute cost of op in promoted type
3159 LT.second = PromotedVT;
3160 }
3161
3162 auto getConstantMatCost =
3163 [&](unsigned Operand, TTI::OperandValueInfo OpInfo) -> InstructionCost {
3164 if (OpInfo.isUniform() &&
3165 canSplatOperand(Opcode, Operand, IntrinsicID: Intrinsic::not_intrinsic))
3166 // Two sub-cases:
3167 // * Has a 5 bit immediate operand which can be splatted.
3168 // * Has a larger immediate which must be materialized in scalar register
3169 // We return 0 for both as we currently ignore the cost of materializing
3170 // scalar constants in GPRs.
3171 return 0;
3172
3173 return getConstantPoolLoadCost(Ty, CostKind);
3174 };
3175
3176 // Add the cost of materializing any constant vectors required.
3177 InstructionCost ConstantMatCost = 0;
3178 if (Op1Info.isConstant())
3179 ConstantMatCost += getConstantMatCost(0, Op1Info);
3180 if (Op2Info.isConstant())
3181 ConstantMatCost += getConstantMatCost(1, Op2Info);
3182
3183 unsigned Op;
3184 switch (ISDOpcode) {
3185 case ISD::ADD:
3186 case ISD::SUB:
3187 Op = RISCV::VADD_VV;
3188 break;
3189 case ISD::SHL:
3190 case ISD::SRL:
3191 case ISD::SRA:
3192 Op = RISCV::VSLL_VV;
3193 break;
3194 case ISD::AND:
3195 case ISD::OR:
3196 case ISD::XOR:
3197 Op = (Ty->getScalarSizeInBits() == 1) ? RISCV::VMAND_MM : RISCV::VAND_VV;
3198 break;
3199 case ISD::MUL:
3200 case ISD::MULHS:
3201 case ISD::MULHU:
3202 Op = RISCV::VMUL_VV;
3203 break;
3204 case ISD::SDIV:
3205 case ISD::UDIV:
3206 Op = RISCV::VDIV_VV;
3207 break;
3208 case ISD::SREM:
3209 case ISD::UREM:
3210 Op = RISCV::VREM_VV;
3211 break;
3212 case ISD::FADD:
3213 case ISD::FSUB:
3214 Op = RISCV::VFADD_VV;
3215 break;
3216 case ISD::FMUL:
3217 Op = RISCV::VFMUL_VV;
3218 break;
3219 case ISD::FDIV:
3220 Op = RISCV::VFDIV_VV;
3221 break;
3222 case ISD::FNEG:
3223 Op = RISCV::VFSGNJN_VV;
3224 break;
3225 default:
3226 // Assuming all other instructions have the same cost until a need arises to
3227 // differentiate them.
3228 return CastCost + ConstantMatCost +
3229 BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info,
3230 Args, CtxI);
3231 }
3232
3233 InstructionCost InstrCost = getRISCVInstructionCost(OpCodes: Op, VT: LT.second, CostKind);
3234 // We use BasicTTIImpl to calculate scalar costs, which assumes floating point
3235 // ops are twice as expensive as integer ops. Do the same for vectors so
3236 // scalar floating point ops aren't cheaper than their vector equivalents.
3237 if (Ty->isFPOrFPVectorTy())
3238 InstrCost *= 2;
3239 return CastCost + ConstantMatCost + LT.first * InstrCost;
3240}
3241
3242// TODO: Deduplicate from TargetTransformInfoImplCRTPBase.
3243InstructionCost RISCVTTIImpl::getPointersChainCost(
3244 ArrayRef<const Value *> Ptrs, const Value *Base,
3245 const TTI::PointersChainInfo &Info, Type *AccessTy,
3246 const TTI::TargetCostKind CostKind) const {
3247 InstructionCost Cost = TTI::TCC_Free;
3248 // In the basic model we take into account GEP instructions only
3249 // (although here can come alloca instruction, a value, constants and/or
3250 // constant expressions, PHIs, bitcasts ... whatever allowed to be used as a
3251 // pointer). Typically, if Base is a not a GEP-instruction and all the
3252 // pointers are relative to the same base address, all the rest are
3253 // either GEP instructions, PHIs, bitcasts or constants. When we have same
3254 // base, we just calculate cost of each non-Base GEP as an ADD operation if
3255 // any their index is a non-const.
3256 // If no known dependencies between the pointers cost is calculated as a sum
3257 // of costs of GEP instructions.
3258 for (auto [I, V] : enumerate(First&: Ptrs)) {
3259 const auto *GEP = dyn_cast<GetElementPtrInst>(Val: V);
3260 if (!GEP)
3261 continue;
3262 if (Info.isSameBase() && V != Base) {
3263 if (GEP->hasAllConstantIndices())
3264 continue;
3265 // If the chain is unit-stride and BaseReg + stride*i is a legal
3266 // addressing mode, then presume the base GEP is sitting around in a
3267 // register somewhere and check if we can fold the offset relative to
3268 // it.
3269 unsigned Stride = DL.getTypeStoreSize(Ty: AccessTy);
3270 if (Info.isUnitStride() &&
3271 isLegalAddressingMode(Ty: AccessTy,
3272 /* BaseGV */ nullptr,
3273 /* BaseOffset */ Stride * I,
3274 /* HasBaseReg */ true,
3275 /* Scale */ 0,
3276 AddrSpace: GEP->getType()->getPointerAddressSpace()))
3277 continue;
3278 Cost += getArithmeticInstrCost(Opcode: Instruction::Add, Ty: GEP->getType(), CostKind,
3279 Op1Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None},
3280 Op2Info: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, Args: {});
3281 } else {
3282 SmallVector<const Value *> Indices(GEP->indices());
3283 Cost += getGEPCost(PointeeType: GEP->getSourceElementType(), Ptr: GEP->getPointerOperand(),
3284 Operands: Indices, CostKind, AccessType: AccessTy);
3285 }
3286 }
3287 return Cost;
3288}
3289
3290void RISCVTTIImpl::getUnrollingPreferences(
3291 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
3292 OptimizationRemarkEmitter *ORE) const {
3293 // TODO: More tuning on benchmarks and metrics with changes as needed
3294 // would apply to all settings below to enable performance.
3295
3296
3297 if (ST->enableDefaultUnroll())
3298 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
3299
3300 // Enable Upper bound unrolling universally, not dependent upon the conditions
3301 // below.
3302 UP.UpperBound = true;
3303
3304 // Disable loop unrolling for Oz and Os.
3305 UP.OptSizeThreshold = 0;
3306 UP.PartialOptSizeThreshold = 0;
3307 if (L->getHeader()->getParent()->hasOptSize())
3308 return;
3309
3310 SmallVector<BasicBlock *, 4> ExitingBlocks;
3311 L->getExitingBlocks(ExitingBlocks);
3312 LLVM_DEBUG(dbgs() << "Loop has:\n"
3313 << "Blocks: " << L->getNumBlocks() << "\n"
3314 << "Exit blocks: " << ExitingBlocks.size() << "\n");
3315
3316 // Only allow another exit other than the latch. This acts as an early exit
3317 // as it mirrors the profitability calculation of the runtime unroller.
3318 if (ExitingBlocks.size() > 2)
3319 return;
3320
3321 // Limit the CFG of the loop body for targets with a branch predictor.
3322 // Allowing 4 blocks permits if-then-else diamonds in the body.
3323 if (L->getNumBlocks() > 4)
3324 return;
3325
3326 // Scan the loop: don't unroll loops with calls as this could prevent
3327 // inlining. Don't unroll auto-vectorized loops either, though do allow
3328 // unrolling of the scalar remainder.
3329 bool IsVectorized = getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized");
3330 InstructionCost Cost = 0;
3331 for (auto *BB : L->getBlocks()) {
3332 for (auto &I : *BB) {
3333 // Both auto-vectorized loops and the scalar remainder have the
3334 // isvectorized attribute, so differentiate between them by the presence
3335 // of vector instructions.
3336 if (IsVectorized && (I.getType()->isVectorTy() ||
3337 llvm::any_of(Range: I.operand_values(), P: [](Value *V) {
3338 return V->getType()->isVectorTy();
3339 })))
3340 return;
3341
3342 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
3343 const Function *F = cast<CallBase>(Val&: I).getCalledFunction();
3344 if (!F || isLoweredToCall(F))
3345 return;
3346 }
3347
3348 SmallVector<const Value *> Operands(I.operand_values());
3349 Cost += getInstructionCost(U: &I, Operands,
3350 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
3351 }
3352 }
3353
3354 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
3355
3356 UP.Partial = true;
3357 UP.Runtime = true;
3358 UP.UnrollRemainder = true;
3359 UP.UnrollAndJam = true;
3360
3361 // Force unrolling small loops can be very useful because of the branch
3362 // taken cost of the backedge.
3363 if (Cost < 12)
3364 UP.Force = true;
3365}
3366
3367void RISCVTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
3368 TTI::PeelingPreferences &PP) const {
3369 BaseT::getPeelingPreferences(L, SE, PP);
3370}
3371
3372bool RISCVTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
3373 MemIntrinsicInfo &Info) const {
3374 const DataLayout &DL = getDataLayout();
3375 Intrinsic::ID IID = Inst->getIntrinsicID();
3376 LLVMContext &C = Inst->getContext();
3377 bool HasMask = false;
3378
3379 auto getSegNum = [](const IntrinsicInst *II, unsigned PtrOperandNo,
3380 bool IsWrite) -> int64_t {
3381 if (auto *TarExtTy =
3382 dyn_cast<TargetExtType>(Val: II->getArgOperand(i: 0)->getType()))
3383 return TarExtTy->getIntParameter(i: 0);
3384
3385 return 1;
3386 };
3387
3388 switch (IID) {
3389 case Intrinsic::riscv_vle_mask:
3390 case Intrinsic::riscv_vse_mask:
3391 case Intrinsic::riscv_vlseg2_mask:
3392 case Intrinsic::riscv_vlseg3_mask:
3393 case Intrinsic::riscv_vlseg4_mask:
3394 case Intrinsic::riscv_vlseg5_mask:
3395 case Intrinsic::riscv_vlseg6_mask:
3396 case Intrinsic::riscv_vlseg7_mask:
3397 case Intrinsic::riscv_vlseg8_mask:
3398 case Intrinsic::riscv_vsseg2_mask:
3399 case Intrinsic::riscv_vsseg3_mask:
3400 case Intrinsic::riscv_vsseg4_mask:
3401 case Intrinsic::riscv_vsseg5_mask:
3402 case Intrinsic::riscv_vsseg6_mask:
3403 case Intrinsic::riscv_vsseg7_mask:
3404 case Intrinsic::riscv_vsseg8_mask:
3405 HasMask = true;
3406 [[fallthrough]];
3407 case Intrinsic::riscv_vle:
3408 case Intrinsic::riscv_vse:
3409 case Intrinsic::riscv_vlseg2:
3410 case Intrinsic::riscv_vlseg3:
3411 case Intrinsic::riscv_vlseg4:
3412 case Intrinsic::riscv_vlseg5:
3413 case Intrinsic::riscv_vlseg6:
3414 case Intrinsic::riscv_vlseg7:
3415 case Intrinsic::riscv_vlseg8:
3416 case Intrinsic::riscv_vsseg2:
3417 case Intrinsic::riscv_vsseg3:
3418 case Intrinsic::riscv_vsseg4:
3419 case Intrinsic::riscv_vsseg5:
3420 case Intrinsic::riscv_vsseg6:
3421 case Intrinsic::riscv_vsseg7:
3422 case Intrinsic::riscv_vsseg8: {
3423 // Intrinsic interface:
3424 // riscv_vle(merge, ptr, vl)
3425 // riscv_vle_mask(merge, ptr, mask, vl, policy)
3426 // riscv_vse(val, ptr, vl)
3427 // riscv_vse_mask(val, ptr, mask, vl, policy)
3428 // riscv_vlseg#(merge, ptr, vl, sew)
3429 // riscv_vlseg#_mask(merge, ptr, mask, vl, policy, sew)
3430 // riscv_vsseg#(val, ptr, vl, sew)
3431 // riscv_vsseg#_mask(val, ptr, mask, vl, sew)
3432 bool IsWrite = Inst->getType()->isVoidTy();
3433 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3434 // The results of segment loads are TargetExtType.
3435 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3436 unsigned SEW =
3437 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3438 ->getZExtValue();
3439 Ty = TarExtTy->getTypeParameter(i: 0U);
3440 Ty = ScalableVectorType::get(
3441 ElementType: IntegerType::get(C, NumBits: SEW),
3442 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3443 }
3444 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3445 unsigned VLIndex = RVVIInfo->VLOperand;
3446 unsigned PtrOperandNo = VLIndex - 1 - HasMask;
3447 MaybeAlign Alignment =
3448 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3449 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3450 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3451 if (HasMask)
3452 Mask = Inst->getArgOperand(i: VLIndex - 1);
3453 Value *EVL = Inst->getArgOperand(i: VLIndex);
3454 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3455 // RVV uses contiguous elements as a segment.
3456 if (SegNum > 1) {
3457 unsigned ElemSize = Ty->getScalarSizeInBits();
3458 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3459 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3460 }
3461 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3462 Args&: Alignment, Args&: Mask, Args&: EVL);
3463 return true;
3464 }
3465 case Intrinsic::riscv_vlse_mask:
3466 case Intrinsic::riscv_vsse_mask:
3467 case Intrinsic::riscv_vlsseg2_mask:
3468 case Intrinsic::riscv_vlsseg3_mask:
3469 case Intrinsic::riscv_vlsseg4_mask:
3470 case Intrinsic::riscv_vlsseg5_mask:
3471 case Intrinsic::riscv_vlsseg6_mask:
3472 case Intrinsic::riscv_vlsseg7_mask:
3473 case Intrinsic::riscv_vlsseg8_mask:
3474 case Intrinsic::riscv_vssseg2_mask:
3475 case Intrinsic::riscv_vssseg3_mask:
3476 case Intrinsic::riscv_vssseg4_mask:
3477 case Intrinsic::riscv_vssseg5_mask:
3478 case Intrinsic::riscv_vssseg6_mask:
3479 case Intrinsic::riscv_vssseg7_mask:
3480 case Intrinsic::riscv_vssseg8_mask:
3481 HasMask = true;
3482 [[fallthrough]];
3483 case Intrinsic::riscv_vlse:
3484 case Intrinsic::riscv_vsse:
3485 case Intrinsic::riscv_vlsseg2:
3486 case Intrinsic::riscv_vlsseg3:
3487 case Intrinsic::riscv_vlsseg4:
3488 case Intrinsic::riscv_vlsseg5:
3489 case Intrinsic::riscv_vlsseg6:
3490 case Intrinsic::riscv_vlsseg7:
3491 case Intrinsic::riscv_vlsseg8:
3492 case Intrinsic::riscv_vssseg2:
3493 case Intrinsic::riscv_vssseg3:
3494 case Intrinsic::riscv_vssseg4:
3495 case Intrinsic::riscv_vssseg5:
3496 case Intrinsic::riscv_vssseg6:
3497 case Intrinsic::riscv_vssseg7:
3498 case Intrinsic::riscv_vssseg8: {
3499 // Intrinsic interface:
3500 // riscv_vlse(merge, ptr, stride, vl)
3501 // riscv_vlse_mask(merge, ptr, stride, mask, vl, policy)
3502 // riscv_vsse(val, ptr, stride, vl)
3503 // riscv_vsse_mask(val, ptr, stride, mask, vl, policy)
3504 // riscv_vlsseg#(merge, ptr, offset, vl, sew)
3505 // riscv_vlsseg#_mask(merge, ptr, offset, mask, vl, policy, sew)
3506 // riscv_vssseg#(val, ptr, offset, vl, sew)
3507 // riscv_vssseg#_mask(val, ptr, offset, mask, vl, sew)
3508 bool IsWrite = Inst->getType()->isVoidTy();
3509 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3510 // The results of segment loads are TargetExtType.
3511 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3512 unsigned SEW =
3513 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3514 ->getZExtValue();
3515 Ty = TarExtTy->getTypeParameter(i: 0U);
3516 Ty = ScalableVectorType::get(
3517 ElementType: IntegerType::get(C, NumBits: SEW),
3518 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3519 }
3520 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3521 unsigned VLIndex = RVVIInfo->VLOperand;
3522 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3523 MaybeAlign Alignment =
3524 Inst->getArgOperand(i: PtrOperandNo)->getPointerAlignment(DL);
3525
3526 Value *Stride = Inst->getArgOperand(i: PtrOperandNo + 1);
3527 // Use the pointer alignment as the element alignment if the stride is a
3528 // multiple of the pointer alignment. Otherwise, the element alignment
3529 // should be the greatest common divisor of pointer alignment and stride.
3530 // For simplicity, just consider unalignment for elements.
3531 unsigned PointerAlign = Alignment.valueOrOne().value();
3532 if (!isa<ConstantInt>(Val: Stride) ||
3533 cast<ConstantInt>(Val: Stride)->getZExtValue() % PointerAlign != 0)
3534 Alignment = Align(1);
3535
3536 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3537 Value *Mask = ConstantInt::getTrue(Ty: MaskType);
3538 if (HasMask)
3539 Mask = Inst->getArgOperand(i: VLIndex - 1);
3540 Value *EVL = Inst->getArgOperand(i: VLIndex);
3541 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3542 // RVV uses contiguous elements as a segment.
3543 if (SegNum > 1) {
3544 unsigned ElemSize = Ty->getScalarSizeInBits();
3545 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3546 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3547 }
3548 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3549 Args&: Alignment, Args&: Mask, Args&: EVL, Args&: Stride);
3550 return true;
3551 }
3552 case Intrinsic::riscv_vloxei_mask:
3553 case Intrinsic::riscv_vluxei_mask:
3554 case Intrinsic::riscv_vsoxei_mask:
3555 case Intrinsic::riscv_vsuxei_mask:
3556 case Intrinsic::riscv_vloxseg2_mask:
3557 case Intrinsic::riscv_vloxseg3_mask:
3558 case Intrinsic::riscv_vloxseg4_mask:
3559 case Intrinsic::riscv_vloxseg5_mask:
3560 case Intrinsic::riscv_vloxseg6_mask:
3561 case Intrinsic::riscv_vloxseg7_mask:
3562 case Intrinsic::riscv_vloxseg8_mask:
3563 case Intrinsic::riscv_vluxseg2_mask:
3564 case Intrinsic::riscv_vluxseg3_mask:
3565 case Intrinsic::riscv_vluxseg4_mask:
3566 case Intrinsic::riscv_vluxseg5_mask:
3567 case Intrinsic::riscv_vluxseg6_mask:
3568 case Intrinsic::riscv_vluxseg7_mask:
3569 case Intrinsic::riscv_vluxseg8_mask:
3570 case Intrinsic::riscv_vsoxseg2_mask:
3571 case Intrinsic::riscv_vsoxseg3_mask:
3572 case Intrinsic::riscv_vsoxseg4_mask:
3573 case Intrinsic::riscv_vsoxseg5_mask:
3574 case Intrinsic::riscv_vsoxseg6_mask:
3575 case Intrinsic::riscv_vsoxseg7_mask:
3576 case Intrinsic::riscv_vsoxseg8_mask:
3577 case Intrinsic::riscv_vsuxseg2_mask:
3578 case Intrinsic::riscv_vsuxseg3_mask:
3579 case Intrinsic::riscv_vsuxseg4_mask:
3580 case Intrinsic::riscv_vsuxseg5_mask:
3581 case Intrinsic::riscv_vsuxseg6_mask:
3582 case Intrinsic::riscv_vsuxseg7_mask:
3583 case Intrinsic::riscv_vsuxseg8_mask:
3584 HasMask = true;
3585 [[fallthrough]];
3586 case Intrinsic::riscv_vloxei:
3587 case Intrinsic::riscv_vluxei:
3588 case Intrinsic::riscv_vsoxei:
3589 case Intrinsic::riscv_vsuxei:
3590 case Intrinsic::riscv_vloxseg2:
3591 case Intrinsic::riscv_vloxseg3:
3592 case Intrinsic::riscv_vloxseg4:
3593 case Intrinsic::riscv_vloxseg5:
3594 case Intrinsic::riscv_vloxseg6:
3595 case Intrinsic::riscv_vloxseg7:
3596 case Intrinsic::riscv_vloxseg8:
3597 case Intrinsic::riscv_vluxseg2:
3598 case Intrinsic::riscv_vluxseg3:
3599 case Intrinsic::riscv_vluxseg4:
3600 case Intrinsic::riscv_vluxseg5:
3601 case Intrinsic::riscv_vluxseg6:
3602 case Intrinsic::riscv_vluxseg7:
3603 case Intrinsic::riscv_vluxseg8:
3604 case Intrinsic::riscv_vsoxseg2:
3605 case Intrinsic::riscv_vsoxseg3:
3606 case Intrinsic::riscv_vsoxseg4:
3607 case Intrinsic::riscv_vsoxseg5:
3608 case Intrinsic::riscv_vsoxseg6:
3609 case Intrinsic::riscv_vsoxseg7:
3610 case Intrinsic::riscv_vsoxseg8:
3611 case Intrinsic::riscv_vsuxseg2:
3612 case Intrinsic::riscv_vsuxseg3:
3613 case Intrinsic::riscv_vsuxseg4:
3614 case Intrinsic::riscv_vsuxseg5:
3615 case Intrinsic::riscv_vsuxseg6:
3616 case Intrinsic::riscv_vsuxseg7:
3617 case Intrinsic::riscv_vsuxseg8: {
3618 // Intrinsic interface (only listed ordered version):
3619 // riscv_vloxei(merge, ptr, index, vl)
3620 // riscv_vloxei_mask(merge, ptr, index, mask, vl, policy)
3621 // riscv_vsoxei(val, ptr, index, vl)
3622 // riscv_vsoxei_mask(val, ptr, index, mask, vl, policy)
3623 // riscv_vloxseg#(merge, ptr, index, vl, sew)
3624 // riscv_vloxseg#_mask(merge, ptr, index, mask, vl, policy, sew)
3625 // riscv_vsoxseg#(val, ptr, index, vl, sew)
3626 // riscv_vsoxseg#_mask(val, ptr, index, mask, vl, sew)
3627 bool IsWrite = Inst->getType()->isVoidTy();
3628 Type *Ty = IsWrite ? Inst->getArgOperand(i: 0)->getType() : Inst->getType();
3629 // The results of segment loads are TargetExtType.
3630 if (auto *TarExtTy = dyn_cast<TargetExtType>(Val: Ty)) {
3631 unsigned SEW =
3632 1 << cast<ConstantInt>(Val: Inst->getArgOperand(i: Inst->arg_size() - 1))
3633 ->getZExtValue();
3634 Ty = TarExtTy->getTypeParameter(i: 0U);
3635 Ty = ScalableVectorType::get(
3636 ElementType: IntegerType::get(C, NumBits: SEW),
3637 MinNumElts: cast<ScalableVectorType>(Val: Ty)->getMinNumElements() * 8 / SEW);
3638 }
3639 const auto *RVVIInfo = RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IID);
3640 unsigned VLIndex = RVVIInfo->VLOperand;
3641 unsigned PtrOperandNo = VLIndex - 2 - HasMask;
3642 Value *Mask;
3643 if (HasMask) {
3644 Mask = Inst->getArgOperand(i: VLIndex - 1);
3645 } else {
3646 // Mask cannot be nullptr here: vector GEP produces <vscale x N x ptr>,
3647 // and casting that to scalar i64 triggers a vector/scalar mismatch
3648 // assertion in CreatePointerCast. Use an all-true mask so ASan lowers it
3649 // via extractelement instead.
3650 Type *MaskType = Ty->getWithNewType(EltTy: Type::getInt1Ty(C));
3651 Mask = ConstantInt::getTrue(Ty: MaskType);
3652 }
3653 Value *EVL = Inst->getArgOperand(i: VLIndex);
3654 unsigned SegNum = getSegNum(Inst, PtrOperandNo, IsWrite);
3655 // RVV uses contiguous elements as a segment.
3656 if (SegNum > 1) {
3657 unsigned ElemSize = Ty->getScalarSizeInBits();
3658 auto *SegTy = IntegerType::get(C, NumBits: ElemSize * SegNum);
3659 Ty = VectorType::get(ElementType: SegTy, Other: cast<VectorType>(Val: Ty));
3660 }
3661 Value *OffsetOp = Inst->getArgOperand(i: PtrOperandNo + 1);
3662 Info.InterestingOperands.emplace_back(Args&: Inst, Args&: PtrOperandNo, Args&: IsWrite, Args&: Ty,
3663 Args: Align(1), Args&: Mask, Args&: EVL,
3664 /* Stride */ Args: nullptr, Args&: OffsetOp);
3665 return true;
3666 }
3667 }
3668 return false;
3669}
3670
3671unsigned RISCVTTIImpl::getRegUsageForType(Type *Ty) const {
3672 if (Ty->isVectorTy()) {
3673 // f16 with only zvfhmin and bf16 will be promoted to f32
3674 Type *EltTy = cast<VectorType>(Val: Ty)->getElementType();
3675 if ((EltTy->isHalfTy() && !ST->hasVInstructionsF16()) ||
3676 EltTy->isBFloatTy())
3677 Ty = VectorType::get(ElementType: Type::getFloatTy(C&: Ty->getContext()),
3678 Other: cast<VectorType>(Val: Ty));
3679
3680 TypeSize Size = DL.getTypeSizeInBits(Ty);
3681 if (Size.isScalable() && ST->hasVInstructions())
3682 return divideCeil(Numerator: Size.getKnownMinValue(), Denominator: RISCV::RVVBitsPerBlock);
3683
3684 if (ST->useRVVForFixedLengthVectors())
3685 return divideCeil(Numerator: Size, Denominator: ST->getRealMinVLen());
3686 }
3687
3688 return BaseT::getRegUsageForType(Ty);
3689}
3690
3691unsigned RISCVTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
3692 if (std::optional<unsigned> MaxVF = ST->getCLOpts().v_slp_max_vf)
3693 return *MaxVF;
3694
3695 // Return how many elements can fit in getRegisterBitwidth. This is the
3696 // same routine as used in LoopVectorizer. We should probably be
3697 // accounting for whether we actually have instructions with the right
3698 // lane type, but we don't have enough information to do that without
3699 // some additional plumbing which hasn't been justified yet.
3700 TypeSize RegWidth =
3701 getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector);
3702 // If no vector registers, or absurd element widths, disable
3703 // vectorization by returning 1.
3704 return std::max<unsigned>(a: 1U, b: RegWidth.getFixedValue() / ElemWidth);
3705}
3706
3707unsigned RISCVTTIImpl::getMinTripCountTailFoldingThreshold() const {
3708 return ST->getCLOpts().v_min_trip_count;
3709}
3710
3711bool RISCVTTIImpl::preferAlternateOpcodeVectorization() const {
3712 return ST->enableUnalignedVectorMem();
3713}
3714
3715TTI::AddressingModeKind
3716RISCVTTIImpl::getPreferredAddressingMode(const Loop *L,
3717 ScalarEvolution *SE) const {
3718 if (ST->hasVendorXCVmem() && !ST->is64Bit())
3719 return TTI::AMK_PostIndexed;
3720
3721 return BasicTTIImplBase::getPreferredAddressingMode(L, SE);
3722}
3723
3724bool RISCVTTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1,
3725 const TargetTransformInfo::LSRCost &C2) const {
3726 // RISC-V specific here are "instruction number 1st priority".
3727 // If we need to emit adds inside the loop to add up base registers, then
3728 // we need at least one extra temporary register.
3729 unsigned C1NumRegs = C1.NumRegs + (C1.NumBaseAdds != 0);
3730 unsigned C2NumRegs = C2.NumRegs + (C2.NumBaseAdds != 0);
3731 return std::tie(args: C1.Insns, args&: C1NumRegs, args: C1.AddRecCost,
3732 args: C1.NumIVMuls, args: C1.NumBaseAdds,
3733 args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
3734 std::tie(args: C2.Insns, args&: C2NumRegs, args: C2.AddRecCost,
3735 args: C2.NumIVMuls, args: C2.NumBaseAdds,
3736 args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
3737}
3738
3739bool RISCVTTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
3740 Align Alignment) const {
3741 auto *VTy = dyn_cast<VectorType>(Val: DataTy);
3742 if (!VTy)
3743 return false;
3744
3745 if (!isLegalMaskedLoadStore(DataType: DataTy, Alignment))
3746 return false;
3747
3748 // FIXME: If it is an i8 vector and the element count exceeds 256, we should
3749 // scalarize these types with LMUL >= maximum fixed-length LMUL.
3750 if (VTy->getElementType()->isIntegerTy(BitWidth: 8)) {
3751 uint64_t MaxEltCount = VTy->getElementCount().getKnownMinValue();
3752 if (VTy->isScalableTy())
3753 MaxEltCount *= ST->getRealMaxVLen() / RISCV::RVVBitsPerBlock;
3754 // We can't yet split any widened indices type.
3755 if (MaxEltCount > 256)
3756 return getTypeLegalizationCost(
3757 Ty: VTy->getWithNewType(EltTy: Type::getInt16Ty(C&: VTy->getContext())))
3758 .first == 1;
3759 }
3760 return true;
3761}
3762
3763bool RISCVTTIImpl::isLegalMaskedCompressStore(Type *DataTy,
3764 Align Alignment) const {
3765 return isLegalMaskedLoadStore(DataType: DataTy, Alignment);
3766}
3767
3768bool RISCVTTIImpl::isLegalBroadcastLoad(Type *ElementTy,
3769 ElementCount NumElements) const {
3770 // Optimized zero-stride loads can be treated as broadcasts.
3771 if (!ST->hasVInstructions() || !ST->hasOptimizedZeroStrideLoad())
3772 return false;
3773
3774 return TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: ElementTy));
3775}
3776
3777/// See if \p I should be considered for address type promotion. We check if \p
3778/// I is a sext with right type and used in memory accesses. If it used in a
3779/// "complex" getelementptr, we allow it to be promoted without finding other
3780/// sext instructions that sign extended the same initial value. A getelementptr
3781/// is considered as "complex" if it has more than 2 operands.
3782bool RISCVTTIImpl::shouldConsiderAddressTypePromotion(
3783 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
3784 bool Considerable = false;
3785 AllowPromotionWithoutCommonHeader = false;
3786 if (!isa<SExtInst>(Val: &I))
3787 return false;
3788 Type *ConsideredSExtType =
3789 Type::getInt64Ty(C&: I.getParent()->getParent()->getContext());
3790 if (I.getType() != ConsideredSExtType)
3791 return false;
3792 // See if the sext is the one with the right type and used in at least one
3793 // GetElementPtrInst.
3794 for (const User *U : I.users()) {
3795 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(Val: U)) {
3796 Considerable = true;
3797 // A getelementptr is considered as "complex" if it has more than 2
3798 // operands. We will promote a SExt used in such complex GEP as we
3799 // expect some computation to be merged if they are done on 64 bits.
3800 if (GEPInst->getNumOperands() > 2) {
3801 AllowPromotionWithoutCommonHeader = true;
3802 break;
3803 }
3804 }
3805 }
3806 return Considerable;
3807}
3808
3809bool RISCVTTIImpl::canSplatOperand(unsigned Opcode, int Operand,
3810 Intrinsic::ID IntrinsicID) const {
3811 switch (Opcode) {
3812 case Instruction::Add:
3813 case Instruction::Sub:
3814 case Instruction::Mul:
3815 case Instruction::And:
3816 case Instruction::Or:
3817 case Instruction::Xor:
3818 case Instruction::FAdd:
3819 case Instruction::FSub:
3820 case Instruction::FMul:
3821 case Instruction::FDiv:
3822 case Instruction::ICmp:
3823 case Instruction::FCmp:
3824 return true;
3825 case Instruction::Shl:
3826 case Instruction::LShr:
3827 case Instruction::AShr:
3828 case Instruction::UDiv:
3829 case Instruction::SDiv:
3830 case Instruction::URem:
3831 case Instruction::SRem:
3832 case Instruction::Select:
3833 return Operand == 1;
3834 case Instruction::Call:
3835 switch (IntrinsicID) {
3836 case Intrinsic::fma:
3837 case Intrinsic::fmuladd:
3838 return Operand == 0 || Operand == 1;
3839 case Intrinsic::vp_udiv:
3840 case Intrinsic::vp_sdiv:
3841 case Intrinsic::vp_urem:
3842 case Intrinsic::vp_srem:
3843 case Intrinsic::ssub_sat:
3844 case Intrinsic::usub_sat:
3845 return Operand == 1;
3846 // These intrinsics are commutative.
3847 case Intrinsic::smin:
3848 case Intrinsic::umin:
3849 case Intrinsic::smax:
3850 case Intrinsic::umax:
3851 case Intrinsic::sadd_sat:
3852 case Intrinsic::uadd_sat:
3853 return Operand == 0 || Operand == 1;
3854 default:
3855 return false;
3856 }
3857 default:
3858 return false;
3859 }
3860}
3861
3862bool RISCVTTIImpl::canSplatOperand(Instruction *I, int Operand) const {
3863 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3864 return false;
3865
3866 auto *II = dyn_cast<IntrinsicInst>(Val: I);
3867 Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic;
3868 if (II)
3869 IntrinsicID = II->getIntrinsicID();
3870
3871 return canSplatOperand(Opcode: I->getOpcode(), Operand, IntrinsicID);
3872}
3873
3874TargetTransformInfo::VectorInstrContext RISCVTTIImpl::getBuildVectorContextHint(
3875 ArrayRef<int> Mask, ArrayRef<Value *> Scalars,
3876 function_ref<bool(SmallVectorImpl<TargetTransformInfo::BuildVectorUseOp> &)>
3877 GatherUseOps) const {
3878 if (Scalars.empty() || !ST->hasVInstructions() || !ST->sinkSplatOperands() ||
3879 !ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts: Mask.size()))
3880 return VectorInstrContext::None;
3881
3882 const auto *SplatIt = find_if_not(Range&: Scalars, P: IsaPred<UndefValue>);
3883 if (SplatIt == Scalars.end() || (*SplatIt)->getType()->isIntegerTy(BitWidth: 1) ||
3884 isa<VectorType>(Val: (*SplatIt)->getType()) ||
3885 isa<ExtractElementInst>(Val: *SplatIt))
3886 return VectorInstrContext::None;
3887
3888 SmallVector<TargetTransformInfo::BuildVectorUseOp, 4> UserOps;
3889 if (!GatherUseOps(UserOps) || UserOps.empty())
3890 return VectorInstrContext::None;
3891
3892 if (all_of(Range&: UserOps,
3893 P: [this](const TargetTransformInfo::BuildVectorUseOp &UserOp) {
3894 return canSplatOperand(Opcode: UserOp.Opcode, Operand: UserOp.OperandIndex,
3895 IntrinsicID: UserOp.IntrinsicID);
3896 }))
3897 return VectorInstrContext::SplatOpFolded;
3898
3899 return VectorInstrContext::None;
3900}
3901
3902/// Check if sinking \p I's operands to I's basic block is profitable, because
3903/// the operands can be folded into a target instruction, e.g.
3904/// splats of scalars can fold into vector instructions.
3905bool RISCVTTIImpl::isProfitableToSinkOperands(
3906 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
3907 using namespace llvm::PatternMatch;
3908
3909 if (I->isBitwiseLogicOp()) {
3910 if (!I->getType()->isVectorTy()) {
3911 if (ST->hasStdExtZbb() || ST->hasStdExtZbkb()) {
3912 for (auto &Op : I->operands()) {
3913 // (and/or/xor X, (not Y)) -> (andn/orn/xnor X, Y)
3914 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3915 Ops.push_back(Elt: &Op);
3916 return true;
3917 }
3918 }
3919 }
3920 } else if (I->getOpcode() == Instruction::And && ST->hasStdExtZvkb()) {
3921 for (auto &Op : I->operands()) {
3922 // (and X, (not Y)) -> (vandn.vv X, Y)
3923 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
3924 Ops.push_back(Elt: &Op);
3925 return true;
3926 }
3927 // (and X, (splat (not Y))) -> (vandn.vx X, Y)
3928 if (match(V: Op.get(), P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()),
3929 Idx: m_ZeroInt()),
3930 v2: m_Value(), mask: m_ZeroMask()))) {
3931 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
3932 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
3933 Ops.push_back(Elt: &Not);
3934 Ops.push_back(Elt: &InsertElt);
3935 Ops.push_back(Elt: &Op);
3936 return true;
3937 }
3938 }
3939 }
3940 }
3941
3942 if (!I->getType()->isVectorTy() || !ST->hasVInstructions())
3943 return false;
3944
3945 // Don't sink splat operands if the target prefers it. Some targets requires
3946 // S2V transfer buffers and we can run out of them copying the same value
3947 // repeatedly.
3948 // FIXME: It could still be worth doing if it would improve vector register
3949 // pressure and prevent a vector spill.
3950 if (!ST->sinkSplatOperands())
3951 return false;
3952
3953 for (auto OpIdx : enumerate(First: I->operands())) {
3954 if (!canSplatOperand(I, Operand: OpIdx.index()))
3955 continue;
3956
3957 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
3958 // Make sure we are not already sinking this operand
3959 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
3960 continue;
3961
3962 // We are looking for a splat that can be sunk.
3963 if (!match(V: Op, P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Value(), Idx: m_ZeroInt()),
3964 v2: m_Value(), mask: m_ZeroMask())))
3965 continue;
3966
3967 // Don't sink i1 splats.
3968 if (cast<VectorType>(Val: Op->getType())->getElementType()->isIntegerTy(BitWidth: 1))
3969 continue;
3970
3971 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
3972 // and vector registers
3973 for (Use &U : Op->uses()) {
3974 Instruction *Insn = cast<Instruction>(Val: U.getUser());
3975 if (!canSplatOperand(I: Insn, Operand: U.getOperandNo()))
3976 return false;
3977 }
3978
3979 // Sink any fpexts since they might be used in a widening fp pattern.
3980 Use *InsertEltUse = &Op->getOperandUse(i: 0);
3981 auto *InsertElt = cast<InsertElementInst>(Val: InsertEltUse);
3982 if (isa<FPExtInst>(Val: InsertElt->getOperand(i_nocapture: 1)))
3983 Ops.push_back(Elt: &InsertElt->getOperandUse(i: 1));
3984 Ops.push_back(Elt: InsertEltUse);
3985 Ops.push_back(Elt: &OpIdx.value());
3986 }
3987 return true;
3988}
3989
3990RISCVTTIImpl::TTI::MemCmpExpansionOptions
3991RISCVTTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
3992 TTI::MemCmpExpansionOptions Options;
3993
3994 if (!ST->hasStdExtZbb() && !ST->hasStdExtZbkb() && !IsZeroCmp)
3995 return Options;
3996
3997 Options.AllowOverlappingLoads = true;
3998 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
3999 Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
4000 if (ST->is64Bit()) {
4001 Options.LoadSizes = {8, 4, 2, 1};
4002 Options.AllowedTailExpansions = {3, 5, 6};
4003 } else {
4004 Options.LoadSizes = {4, 2, 1};
4005 Options.AllowedTailExpansions = {3};
4006 }
4007
4008 if (IsZeroCmp && ST->hasVInstructions()) {
4009 unsigned VLenB = ST->getRealMinVLen() / 8;
4010 // The minimum size should be `XLen / 8 + 1`, and the maxinum size should be
4011 // `VLenB * MaxLMUL` so that it fits in a single register group.
4012 unsigned MinSize = ST->getXLen() / 8 + 1;
4013 unsigned MaxSize = VLenB * 8;
4014 for (unsigned Size = MinSize; Size <= MaxSize; Size++)
4015 Options.LoadSizes.insert(I: Options.LoadSizes.begin(), Elt: Size);
4016 }
4017 return Options;
4018}
4019
4020bool RISCVTTIImpl::shouldTreatInstructionLikeSelect(
4021 const Instruction *I) const {
4022 if (ST->getCLOpts().or_like_select) {
4023 // For the binary operators (e.g. or) we need to be more careful than
4024 // selects, here we only transform them if they are already at a natural
4025 // break point in the code - the end of a block with an unconditional
4026 // terminator.
4027 if (I->getOpcode() == Instruction::Or &&
4028 isa<UncondBrInst>(Val: I->getNextNode()))
4029 return true;
4030
4031 if (I->getOpcode() == Instruction::Add ||
4032 I->getOpcode() == Instruction::Sub)
4033 return true;
4034 }
4035 return BaseT::shouldTreatInstructionLikeSelect(I);
4036}
4037
4038bool RISCVTTIImpl::shouldCopyAttributeWhenOutliningFrom(
4039 const Function *Caller, const Attribute &Attr) const {
4040 // "interrupt" controls the prolog/epilog of interrupt handlers (and includes
4041 // restrictions on their signatures). We can outline from the bodies of these
4042 // handlers, but when we do we need to make sure we don't mark the outlined
4043 // function as an interrupt handler too.
4044 if (Attr.isStringAttribute() && Attr.getKindAsString() == "interrupt")
4045 return false;
4046
4047 return BaseT::shouldCopyAttributeWhenOutliningFrom(Caller, Attr);
4048}
4049
4050std::optional<Instruction *>
4051RISCVTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
4052 // Attach a range return attribute describing the result of vsetvli/vsetvlimax
4053 // so generic value analyses can reason about it. The verifier guarantees an
4054 // XLen result and constant VSEW/VLMUL encoding a valid vtype, so no defensive
4055 // validation is needed here.
4056 if (is_contained(Set: {Intrinsic::riscv_vsetvli, Intrinsic::riscv_vsetvlimax},
4057 Element: II.getIntrinsicID())) {
4058 // These intrinsics require the V extension; without it the VLEN queries
4059 // below would assert. Such IR would fail isel anyway, so just bail out.
4060 if (!ST->hasVInstructions())
4061 return {};
4062
4063 bool HasAVL = II.getIntrinsicID() == Intrinsic::riscv_vsetvli;
4064 unsigned Offset = HasAVL ? 1 : 0;
4065 unsigned BitWidth = II.getType()->getIntegerBitWidth();
4066 ConstantRange VLenRange(APInt(BitWidth, ST->getRealMinVLen()),
4067 APInt(BitWidth, ST->getRealMaxVLen()) + 1);
4068
4069 uint64_t VSEW = cast<ConstantInt>(Val: II.getArgOperand(i: Offset))->getZExtValue();
4070 auto VLMUL = static_cast<RISCVVType::VLMUL>(
4071 cast<ConstantInt>(Val: II.getArgOperand(i: Offset + 1))->getZExtValue());
4072 unsigned SEW = RISCVVType::decodeVSEW(VSEW);
4073 unsigned Ratio = RISCVVType::getSEWLMULRatio(SEW, VLMul: VLMUL);
4074
4075 // VLMAX = VLEN / (SEW / LMUL), clamped to >= 1 for any usable vtype.
4076 ConstantRange VLMAXRange =
4077 VLenRange.udiv(Other: ConstantRange(APInt(BitWidth, Ratio)))
4078 .umax(Other: ConstantRange(APInt(BitWidth, 1)));
4079
4080 // vsetvlimax returns exactly VLMAX; vsetvli returns vl with
4081 // 0 <= vl <= min(AVL, VLMAX). vl == AVL only when AVL <= the smallest
4082 // possible VLMAX; otherwise vl can shrink below VLMAX (to 0 at runtime), so
4083 // only the VLMAX upper bound is sound.
4084 ConstantRange VLRange = VLMAXRange;
4085 if (HasAVL) {
4086 // vl ≤ VLMAX
4087 VLRange =
4088 ConstantRange::makeAllowedICmpRegion(Pred: CmpInst::ICMP_ULE, Other: VLMAXRange);
4089
4090 Value *AVL = II.getArgOperand(i: 0);
4091 ConstantRange AVLRange = computeConstantRangeIncludingKnownBits(
4092 V: AVL, /*ForSigned=*/false,
4093 SQ: IC.getSimplifyQuery().getWithInstruction(I: &II));
4094
4095 // vl = AVL if AVL ≤ VLMAX
4096 if (AVLRange.icmp(Pred: CmpInst::ICMP_ULE, Other: VLMAXRange))
4097 return IC.replaceInstUsesWith(I&: II, V: AVL);
4098
4099 // vl ≤ AVL
4100 VLRange = VLRange.umin(Other: AVLRange.getUnsignedMax());
4101
4102 // vl > 0 if AVL > 0
4103 if (AVLRange.icmp(Pred: CmpInst::ICMP_UGT, Other: APInt::getZero(numBits: BitWidth)))
4104 VLRange = VLRange.umax(Other: APInt(BitWidth, 1));
4105
4106 // vl = VLMAX if AVL ≥ (2 * VLMAX)
4107 ConstantRange TwoVLMAX = VLMAXRange.multiply(Other: APInt(BitWidth, 2));
4108 if (AVLRange.icmp(Pred: CmpInst::ICMP_UGE, Other: TwoVLMAX))
4109 VLRange = VLRange.intersectWith(CR: VLMAXRange);
4110
4111 // ceil(AVL / 2) ≤ vl ≤ VLMAX if AVL < (2 * VLMAX)
4112 if (AVLRange.icmp(Pred: CmpInst::ICMP_ULT, Other: TwoVLMAX))
4113 VLRange = VLRange.umax(Other: APIntOps::RoundingUDiv(A: AVLRange.getUnsignedMin(),
4114 B: APInt(BitWidth, 2),
4115 RM: APInt::Rounding::UP));
4116 }
4117
4118 ConstantRange OldRange =
4119 II.getRange().value_or(u: ConstantRange::getFull(BitWidth));
4120 ConstantRange NewRange = VLRange.intersectWith(CR: OldRange);
4121 if (NewRange != OldRange) {
4122 II.addRangeRetAttr(CR: NewRange);
4123 return &II;
4124 }
4125 return {};
4126 }
4127
4128 // If all operands of a vmv.v.x are constant, fold a bitcast(vmv.v.x) to scale
4129 // the vmv.v.x, enabling removal of the bitcast. The transform helps avoid
4130 // creating redundant masks.
4131 const DataLayout &DL = IC.getDataLayout();
4132 if (II.user_empty())
4133 return {};
4134 auto *TargetVecTy = dyn_cast<ScalableVectorType>(Val: II.user_back()->getType());
4135 if (!TargetVecTy)
4136 return {};
4137 const APInt *Scalar;
4138 uint64_t VL;
4139 if (!match(V: &II, P: m_Intrinsic<Intrinsic::riscv_vmv_v_x>(
4140 Ops: m_Poison(), Ops: m_APInt(Res&: Scalar), Ops: m_ConstantInt(V&: VL))) ||
4141 !all_of(Range: II.users(), P: [TargetVecTy](User *U) {
4142 return U->getType() == TargetVecTy && match(V: U, P: m_BitCast(Op: m_Value()));
4143 }))
4144 return {};
4145 auto *SourceVecTy = cast<ScalableVectorType>(Val: II.getType());
4146 unsigned TargetEltBW = DL.getTypeSizeInBits(Ty: TargetVecTy->getElementType());
4147 unsigned SourceEltBW = DL.getTypeSizeInBits(Ty: SourceVecTy->getElementType());
4148 if (TargetEltBW % SourceEltBW)
4149 return {};
4150 unsigned TargetScale = TargetEltBW / SourceEltBW;
4151 if (VL % TargetScale || TargetScale == 1)
4152 return {};
4153 Type *VLTy = II.getOperand(i_nocapture: 2)->getType();
4154 ElementCount SourceEC = SourceVecTy->getElementCount();
4155 unsigned NewEltBW = SourceEltBW * TargetScale;
4156 if (!SourceEC.isKnownMultipleOf(RHS: TargetScale) ||
4157 !DL.fitsInLegalInteger(Width: NewEltBW))
4158 return {};
4159 auto *NewEltTy = IntegerType::get(C&: II.getContext(), NumBits: NewEltBW);
4160 if (!TLI->isLegalElementTypeForRVV(ScalarTy: TLI->getValueType(DL, Ty: NewEltTy)))
4161 return {};
4162 ElementCount NewEC = SourceEC.divideCoefficientBy(RHS: TargetScale);
4163 Type *RetTy = VectorType::get(ElementType: NewEltTy, EC: NewEC);
4164 assert(SourceVecTy->canLosslesslyBitCastTo(RetTy) &&
4165 "Lossless bitcast between types expected");
4166 APInt NewScalar = APInt::getSplat(NewLen: NewEltBW, V: *Scalar);
4167 return IC.replaceInstUsesWith(
4168 I&: II,
4169 V: IC.Builder.CreateBitCast(
4170 V: IC.Builder.CreateIntrinsic(
4171 RetTy, ID: Intrinsic::riscv_vmv_v_x,
4172 Args: {PoisonValue::get(T: RetTy), ConstantInt::get(Ty: NewEltTy, V: NewScalar),
4173 ConstantInt::get(Ty: VLTy, V: VL / TargetScale)}),
4174 DestTy: SourceVecTy));
4175}
4176