1//===- ARMTargetTransformInfo.cpp - ARM specific TTI ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ARMTargetTransformInfo.h"
10#include "ARMSubtarget.h"
11#include "MCTargetDesc/ARMAddressingModes.h"
12#include "llvm/ADT/APInt.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Analysis/LoopInfo.h"
15#include "llvm/CodeGen/CostTable.h"
16#include "llvm/CodeGen/ISDOpcodes.h"
17#include "llvm/CodeGen/ValueTypes.h"
18#include "llvm/CodeGenTypes/MachineValueType.h"
19#include "llvm/IR/BasicBlock.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/IR/DerivedTypes.h"
22#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/IntrinsicsARM.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Target/TargetMachine.h"
32#include "llvm/TargetParser/SubtargetFeature.h"
33#include "llvm/Transforms/InstCombine/InstCombiner.h"
34#include "llvm/Transforms/Utils/Local.h"
35#include "llvm/Transforms/Utils/LoopUtils.h"
36#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
37#include <algorithm>
38#include <cassert>
39#include <cstdint>
40#include <optional>
41#include <utility>
42
43using namespace llvm;
44
45#define DEBUG_TYPE "armtti"
46
47static cl::opt<bool> EnableMaskedLoadStores(
48 "enable-arm-maskedldst", cl::Hidden, cl::init(Val: true),
49 cl::desc("Enable the generation of masked loads and stores"));
50
51static cl::opt<bool> DisableLowOverheadLoops(
52 "disable-arm-loloops", cl::Hidden, cl::init(Val: false),
53 cl::desc("Disable the generation of low-overhead loops"));
54
55static cl::opt<bool>
56 AllowWLSLoops("allow-arm-wlsloops", cl::Hidden, cl::init(Val: true),
57 cl::desc("Enable the generation of WLS loops"));
58
59static cl::opt<bool> UseWidenGlobalArrays(
60 "widen-global-strings", cl::Hidden, cl::init(Val: true),
61 cl::desc("Enable the widening of global strings to alignment boundaries"));
62
63extern cl::opt<TailPredication::Mode> EnableTailPredication;
64
65extern cl::opt<bool> EnableMaskedGatherScatters;
66
67extern cl::opt<unsigned> MVEMaxSupportedInterleaveFactor;
68
69static cl::opt<int> ArmForceUnrollThreshold(
70 "arm-force-unroll-threshold", cl::init(Val: 12), cl::Hidden,
71 cl::desc(
72 "Threshold for forced unrolling of small loops in Arm architecture"));
73
74/// Convert a vector load intrinsic into a simple llvm load instruction.
75/// This is beneficial when the underlying object being addressed comes
76/// from a constant, since we get constant-folding for free.
77static Value *simplifyNeonVld1(const IntrinsicInst &II, unsigned MemAlign,
78 InstCombiner::BuilderTy &Builder) {
79 auto *IntrAlign = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
80
81 if (!IntrAlign)
82 return nullptr;
83
84 unsigned Alignment = IntrAlign->getLimitedValue() < MemAlign
85 ? MemAlign
86 : IntrAlign->getLimitedValue();
87
88 if (!isPowerOf2_32(Value: Alignment))
89 return nullptr;
90
91 return Builder.CreateAlignedLoad(Ty: II.getType(), Ptr: II.getArgOperand(i: 0),
92 Align: Align(Alignment));
93}
94
95TTI::AddressingModeKind
96ARMTTIImpl::getPreferredAddressingMode(const Loop *L,
97 ScalarEvolution *SE) const {
98 if (ST->hasMVEIntegerOps())
99 return TTI::AMK_PostIndexed;
100
101 if (L->getHeader()->getParent()->hasOptSize())
102 return TTI::AMK_None;
103
104 if (ST->isMClass() && ST->isThumb2() &&
105 L->getNumBlocks() == 1)
106 return TTI::AMK_PreIndexed;
107
108 return TTI::AMK_None;
109}
110
111std::optional<Instruction *>
112ARMTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
113 using namespace PatternMatch;
114 Intrinsic::ID IID = II.getIntrinsicID();
115 switch (IID) {
116 default:
117 break;
118 case Intrinsic::arm_neon_vld1: {
119 Align MemAlign =
120 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CxtI: &II,
121 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
122 if (Value *V = simplifyNeonVld1(II, MemAlign: MemAlign.value(), Builder&: IC.Builder)) {
123 return IC.replaceInstUsesWith(I&: II, V);
124 }
125 break;
126 }
127
128 case Intrinsic::arm_neon_vld2:
129 case Intrinsic::arm_neon_vld3:
130 case Intrinsic::arm_neon_vld4:
131 case Intrinsic::arm_neon_vld2lane:
132 case Intrinsic::arm_neon_vld3lane:
133 case Intrinsic::arm_neon_vld4lane:
134 case Intrinsic::arm_neon_vst1:
135 case Intrinsic::arm_neon_vst2:
136 case Intrinsic::arm_neon_vst3:
137 case Intrinsic::arm_neon_vst4:
138 case Intrinsic::arm_neon_vst2lane:
139 case Intrinsic::arm_neon_vst3lane:
140 case Intrinsic::arm_neon_vst4lane: {
141 Align MemAlign =
142 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CxtI: &II,
143 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
144 unsigned AlignArg = II.arg_size() - 1;
145 Value *AlignArgOp = II.getArgOperand(i: AlignArg);
146 MaybeAlign Align = cast<ConstantInt>(Val: AlignArgOp)->getMaybeAlignValue();
147 if (Align && *Align < MemAlign) {
148 return IC.replaceOperand(
149 I&: II, OpNum: AlignArg,
150 V: ConstantInt::get(Ty: Type::getInt32Ty(C&: II.getContext()), V: MemAlign.value(),
151 IsSigned: false));
152 }
153 break;
154 }
155
156 case Intrinsic::arm_neon_vld1x2:
157 case Intrinsic::arm_neon_vld1x3:
158 case Intrinsic::arm_neon_vld1x4:
159 case Intrinsic::arm_neon_vst1x2:
160 case Intrinsic::arm_neon_vst1x3:
161 case Intrinsic::arm_neon_vst1x4: {
162 Align NewAlign =
163 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CxtI: &II,
164 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
165 Align OldAlign = II.getParamAlign(ArgNo: 0).valueOrOne();
166 if (NewAlign > OldAlign)
167 II.addParamAttr(ArgNo: 0,
168 Attr: Attribute::getWithAlignment(Context&: II.getContext(), Alignment: NewAlign));
169 break;
170 }
171
172 case Intrinsic::arm_mve_pred_i2v: {
173 Value *Arg = II.getArgOperand(i: 0);
174 Value *ArgArg;
175 if (match(V: Arg, P: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
176 Ops: PatternMatch::m_Value(V&: ArgArg))) &&
177 II.getType() == ArgArg->getType()) {
178 return IC.replaceInstUsesWith(I&: II, V: ArgArg);
179 }
180 Constant *XorMask;
181 if (match(V: Arg, P: m_Xor(L: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
182 Ops: PatternMatch::m_Value(V&: ArgArg)),
183 R: PatternMatch::m_Constant(C&: XorMask))) &&
184 II.getType() == ArgArg->getType()) {
185 if (auto *CI = dyn_cast<ConstantInt>(Val: XorMask)) {
186 if (CI->getValue().trunc(width: 16).isAllOnes()) {
187 auto TrueVector = IC.Builder.CreateVectorSplat(
188 NumElts: cast<FixedVectorType>(Val: II.getType())->getNumElements(),
189 V: IC.Builder.getTrue());
190 return BinaryOperator::Create(Op: Instruction::Xor, S1: ArgArg, S2: TrueVector);
191 }
192 }
193 }
194 KnownBits ScalarKnown(32);
195 if (IC.SimplifyDemandedBits(I: &II, OpNo: 0, DemandedMask: APInt::getLowBitsSet(numBits: 32, loBitsSet: 16),
196 Known&: ScalarKnown)) {
197 return &II;
198 }
199 break;
200 }
201 case Intrinsic::arm_mve_pred_v2i: {
202 Value *Arg = II.getArgOperand(i: 0);
203 Value *ArgArg;
204 if (match(V: Arg, P: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_i2v>(
205 Ops: PatternMatch::m_Value(V&: ArgArg)))) {
206 return IC.replaceInstUsesWith(I&: II, V: ArgArg);
207 }
208
209 if (II.getMetadata(KindID: LLVMContext::MD_range))
210 break;
211
212 ConstantRange Range(APInt(32, 0), APInt(32, 0x10000));
213
214 if (auto CurrentRange = II.getRange()) {
215 Range = Range.intersectWith(CR: *CurrentRange);
216 if (Range == CurrentRange)
217 break;
218 }
219
220 II.addRangeRetAttr(CR: Range);
221 II.addRetAttr(Kind: Attribute::NoUndef);
222 return &II;
223 }
224 case Intrinsic::arm_mve_vadc:
225 case Intrinsic::arm_mve_vadc_predicated: {
226 unsigned CarryOp =
227 (II.getIntrinsicID() == Intrinsic::arm_mve_vadc_predicated) ? 3 : 2;
228 assert(II.getArgOperand(CarryOp)->getType()->getScalarSizeInBits() == 32 &&
229 "Bad type for intrinsic!");
230
231 KnownBits CarryKnown(32);
232 if (IC.SimplifyDemandedBits(I: &II, OpNo: CarryOp, DemandedMask: APInt::getOneBitSet(numBits: 32, BitNo: 29),
233 Known&: CarryKnown)) {
234 return &II;
235 }
236 break;
237 }
238 case Intrinsic::arm_mve_vmldava: {
239 Instruction *I = cast<Instruction>(Val: &II);
240 if (I->hasOneUse()) {
241 auto *User = cast<Instruction>(Val: *I->user_begin());
242 Value *OpZ;
243 if (match(V: User, P: m_c_Add(L: m_Specific(V: I), R: m_Value(V&: OpZ))) &&
244 match(V: I->getOperand(i: 3), P: m_Zero())) {
245 Value *OpX = I->getOperand(i: 4);
246 Value *OpY = I->getOperand(i: 5);
247 Type *OpTy = OpX->getType();
248
249 IC.Builder.SetInsertPoint(User);
250 Value *V =
251 IC.Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vmldava, OverloadTypes: {OpTy},
252 Args: {I->getOperand(i: 0), I->getOperand(i: 1),
253 I->getOperand(i: 2), OpZ, OpX, OpY});
254
255 IC.replaceInstUsesWith(I&: *User, V);
256 return IC.eraseInstFromFunction(I&: *User);
257 }
258 }
259 return std::nullopt;
260 }
261 }
262 return std::nullopt;
263}
264
265std::optional<Value *> ARMTTIImpl::simplifyDemandedVectorEltsIntrinsic(
266 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,
267 APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
268 std::function<void(Instruction *, unsigned, APInt, APInt &)>
269 SimplifyAndSetOp) const {
270
271 // Compute the demanded bits for a narrowing MVE intrinsic. The TopOpc is the
272 // opcode specifying a Top/Bottom instruction, which can change between
273 // instructions.
274 auto SimplifyNarrowInstrTopBottom =[&](unsigned TopOpc) {
275 unsigned NumElts = cast<FixedVectorType>(Val: II.getType())->getNumElements();
276 unsigned IsTop = cast<ConstantInt>(Val: II.getOperand(i_nocapture: TopOpc))->getZExtValue();
277
278 // The only odd/even lanes of operand 0 will only be demanded depending
279 // on whether this is a top/bottom instruction.
280 APInt DemandedElts =
281 APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
282 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
283 SimplifyAndSetOp(&II, 0, OrigDemandedElts & DemandedElts, UndefElts);
284 // The other lanes will be defined from the inserted elements.
285 UndefElts &= APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
286 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
287 return std::nullopt;
288 };
289
290 switch (II.getIntrinsicID()) {
291 default:
292 break;
293 case Intrinsic::arm_mve_vcvt_narrow:
294 SimplifyNarrowInstrTopBottom(2);
295 break;
296 case Intrinsic::arm_mve_vqmovn:
297 SimplifyNarrowInstrTopBottom(4);
298 break;
299 case Intrinsic::arm_mve_vshrn:
300 SimplifyNarrowInstrTopBottom(7);
301 break;
302 }
303
304 return std::nullopt;
305}
306
307InstructionCost ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
308 TTI::TargetCostKind CostKind) const {
309 assert(Ty->isIntegerTy());
310
311 unsigned Bits = Ty->getPrimitiveSizeInBits();
312 if (Bits == 0 || Imm.getActiveBits() >= 64)
313 return 4;
314
315 int64_t SImmVal = Imm.getSExtValue();
316 uint64_t ZImmVal = Imm.getZExtValue();
317 if (!ST->isThumb()) {
318 if ((SImmVal >= 0 && SImmVal < 65536) ||
319 (ARM_AM::getSOImmVal(Arg: ZImmVal) != -1) ||
320 (ARM_AM::getSOImmVal(Arg: ~ZImmVal) != -1))
321 return 1;
322 return ST->hasV6T2Ops() ? 2 : 3;
323 }
324 if (ST->isThumb2()) {
325 if ((SImmVal >= 0 && SImmVal < 65536) ||
326 (ARM_AM::getT2SOImmVal(Arg: ZImmVal) != -1) ||
327 (ARM_AM::getT2SOImmVal(Arg: ~ZImmVal) != -1))
328 return 1;
329 return ST->hasV6T2Ops() ? 2 : 3;
330 }
331 // Thumb1, any i8 imm cost 1.
332 if (Bits == 8 || (SImmVal >= 0 && SImmVal < 256))
333 return 1;
334 if ((~SImmVal < 256) || ARM_AM::isThumbImmShiftedVal(V: ZImmVal))
335 return 2;
336 // Load from constantpool.
337 return 3;
338}
339
340// Constants smaller than 256 fit in the immediate field of
341// Thumb1 instructions so we return a zero cost and 1 otherwise.
342InstructionCost ARMTTIImpl::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
343 const APInt &Imm,
344 Type *Ty) const {
345 if (Imm.isNonNegative() && Imm.getLimitedValue() < 256)
346 return 0;
347
348 return 1;
349}
350
351// Checks whether Inst is part of a min(max()) or max(min()) pattern
352// that will match to an SSAT instruction. Returns the instruction being
353// saturated, or null if no saturation pattern was found.
354static Value *isSSATMinMaxPattern(Instruction *Inst, const APInt &Imm) {
355 Value *LHS, *RHS;
356 ConstantInt *C;
357 SelectPatternFlavor InstSPF = matchSelectPattern(V: Inst, LHS, RHS).Flavor;
358
359 if (InstSPF == SPF_SMAX &&
360 PatternMatch::match(V: RHS, P: PatternMatch::m_ConstantInt(CI&: C)) &&
361 C->getValue() == Imm && Imm.isNegative() && Imm.isNegatedPowerOf2()) {
362
363 auto isSSatMin = [&](Value *MinInst) {
364 if (isa<SelectInst>(Val: MinInst)) {
365 Value *MinLHS, *MinRHS;
366 ConstantInt *MinC;
367 SelectPatternFlavor MinSPF =
368 matchSelectPattern(V: MinInst, LHS&: MinLHS, RHS&: MinRHS).Flavor;
369 if (MinSPF == SPF_SMIN &&
370 PatternMatch::match(V: MinRHS, P: PatternMatch::m_ConstantInt(CI&: MinC)) &&
371 MinC->getValue() == ((-Imm) - 1))
372 return true;
373 }
374 return false;
375 };
376
377 if (isSSatMin(Inst->getOperand(i: 1)))
378 return cast<Instruction>(Val: Inst->getOperand(i: 1))->getOperand(i: 1);
379 if (Inst->hasNUses(N: 2) &&
380 (isSSatMin(*Inst->user_begin()) || isSSatMin(*(++Inst->user_begin()))))
381 return Inst->getOperand(i: 1);
382 }
383 return nullptr;
384}
385
386// Look for a FP Saturation pattern, where the instruction can be simplified to
387// a fptosi.sat. max(min(fptosi)). The constant in this case is always free.
388static bool isFPSatMinMaxPattern(Instruction *Inst, const APInt &Imm) {
389 if (Imm.getBitWidth() != 64 ||
390 Imm != APInt::getHighBitsSet(numBits: 64, hiBitsSet: 33)) // -2147483648
391 return false;
392 Value *FP = isSSATMinMaxPattern(Inst, Imm);
393 if (!FP && isa<ICmpInst>(Val: Inst) && Inst->hasOneUse())
394 FP = isSSATMinMaxPattern(Inst: cast<Instruction>(Val: *Inst->user_begin()), Imm);
395 if (!FP)
396 return false;
397 return isa<FPToSIInst>(Val: FP);
398}
399
400InstructionCost ARMTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
401 const APInt &Imm, Type *Ty,
402 TTI::TargetCostKind CostKind,
403 Instruction *Inst) const {
404 // Division by a constant can be turned into multiplication, but only if we
405 // know it's constant. So it's not so much that the immediate is cheap (it's
406 // not), but that the alternative is worse.
407 // FIXME: this is probably unneeded with GlobalISel.
408 if ((Opcode == Instruction::SDiv || Opcode == Instruction::UDiv ||
409 Opcode == Instruction::SRem || Opcode == Instruction::URem) &&
410 Idx == 1)
411 return 0;
412
413 // Leave any gep offsets for the CodeGenPrepare, which will do a better job at
414 // splitting any large offsets.
415 if (Opcode == Instruction::GetElementPtr && Idx != 0)
416 return 0;
417
418 if (Opcode == Instruction::And) {
419 // UXTB/UXTH
420 if (Imm == 255 || Imm == 65535)
421 return 0;
422 // Conversion to BIC is free, and means we can use ~Imm instead.
423 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
424 b: getIntImmCost(Imm: ~Imm, Ty, CostKind));
425 }
426
427 if (Opcode == Instruction::Add)
428 // Conversion to SUB is free, and means we can use -Imm instead.
429 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
430 b: getIntImmCost(Imm: -Imm, Ty, CostKind));
431
432 if (Opcode == Instruction::ICmp && Imm.isNegative() &&
433 Ty->getIntegerBitWidth() == 32) {
434 int64_t NegImm = -Imm.getSExtValue();
435 if (ST->isThumb2() && NegImm < 1<<12)
436 // icmp X, #-C -> cmn X, #C
437 return 0;
438 if (ST->isThumb() && NegImm < 1<<8)
439 // icmp X, #-C -> adds X, #C
440 return 0;
441 }
442
443 // xor a, -1 can always be folded to MVN
444 if (Opcode == Instruction::Xor && Imm.isAllOnes())
445 return 0;
446
447 // Ensures negative constant of min(max()) or max(min()) patterns that
448 // match to SSAT instructions don't get hoisted
449 if (Inst && ((ST->hasV6Ops() && !ST->isThumb()) || ST->isThumb2()) &&
450 Ty->getIntegerBitWidth() <= 32) {
451 if (isSSATMinMaxPattern(Inst, Imm) ||
452 (isa<ICmpInst>(Val: Inst) && Inst->hasOneUse() &&
453 isSSATMinMaxPattern(Inst: cast<Instruction>(Val: *Inst->user_begin()), Imm)))
454 return 0;
455 }
456
457 if (Inst && ST->hasVFP2Base() && isFPSatMinMaxPattern(Inst, Imm))
458 return 0;
459
460 // We can convert <= -1 to < 0, which is generally quite cheap.
461 if (Inst && Opcode == Instruction::ICmp && Idx == 1 && Imm.isAllOnes()) {
462 ICmpInst::Predicate Pred = cast<ICmpInst>(Val: Inst)->getPredicate();
463 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE)
464 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
465 b: getIntImmCost(Imm: Imm + 1, Ty, CostKind));
466 }
467
468 return getIntImmCost(Imm, Ty, CostKind);
469}
470
471InstructionCost ARMTTIImpl::getCFInstrCost(unsigned Opcode,
472 TTI::TargetCostKind CostKind,
473 const Instruction *I) const {
474 if (CostKind == TTI::TCK_RecipThroughput &&
475 (ST->hasNEON() || ST->hasMVEIntegerOps())) {
476 // FIXME: The vectorizer is highly sensitive to the cost of these
477 // instructions, which suggests that it may be using the costs incorrectly.
478 // But, for now, just make them free to avoid performance regressions for
479 // vector targets.
480 return 0;
481 }
482 return BaseT::getCFInstrCost(Opcode, CostKind, I);
483}
484
485InstructionCost ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
486 Type *Src,
487 TTI::CastContextHint CCH,
488 TTI::TargetCostKind CostKind,
489 const Instruction *I) const {
490 int ISD = TLI->InstructionOpcodeToISD(Opcode);
491 assert(ISD && "Invalid opcode");
492
493 // TODO: Allow non-throughput costs that aren't binary.
494 auto AdjustCost = [&CostKind](InstructionCost Cost) -> InstructionCost {
495 if (CostKind != TTI::TCK_RecipThroughput)
496 return Cost == 0 ? 0 : 1;
497 return Cost;
498 };
499 auto IsLegalFPType = [this](EVT VT) {
500 EVT EltVT = VT.getScalarType();
501 return (EltVT == MVT::f32 && ST->hasVFP2Base()) ||
502 (EltVT == MVT::f64 && ST->hasFP64()) ||
503 (EltVT == MVT::f16 && ST->hasFullFP16());
504 };
505
506 EVT SrcTy = TLI->getValueType(DL, Ty: Src);
507 EVT DstTy = TLI->getValueType(DL, Ty: Dst);
508
509 if (!SrcTy.isSimple() || !DstTy.isSimple())
510 return AdjustCost(
511 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
512
513 // Extending masked load/Truncating masked stores is expensive because we
514 // currently don't split them. This means that we'll likely end up
515 // loading/storing each element individually (hence the high cost).
516 if ((ST->hasMVEIntegerOps() &&
517 (Opcode == Instruction::Trunc || Opcode == Instruction::ZExt ||
518 Opcode == Instruction::SExt)) ||
519 (ST->hasMVEFloatOps() &&
520 (Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc) &&
521 IsLegalFPType(SrcTy) && IsLegalFPType(DstTy)))
522 if (CCH == TTI::CastContextHint::Masked && DstTy.getSizeInBits() > 128)
523 return 2 * DstTy.getVectorNumElements() *
524 ST->getMVEVectorCostFactor(CostKind);
525
526 // The extend of other kinds of load is free
527 if (CCH == TTI::CastContextHint::Normal ||
528 CCH == TTI::CastContextHint::Masked) {
529 static const TypeConversionCostTblEntry LoadConversionTbl[] = {
530 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i32, .Src: MVT::i16, .Cost: 0},
531 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i32, .Src: MVT::i16, .Cost: 0},
532 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i32, .Src: MVT::i8, .Cost: 0},
533 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i32, .Src: MVT::i8, .Cost: 0},
534 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i16, .Src: MVT::i8, .Cost: 0},
535 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i16, .Src: MVT::i8, .Cost: 0},
536 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i32, .Cost: 1},
537 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i32, .Cost: 1},
538 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 1},
539 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 1},
540 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i8, .Cost: 1},
541 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i8, .Cost: 1},
542 };
543 if (const auto *Entry = ConvertCostTableLookup(
544 Table: LoadConversionTbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
545 return AdjustCost(Entry->Cost);
546
547 static const TypeConversionCostTblEntry MVELoadConversionTbl[] = {
548 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
549 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
550 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
551 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
552 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
553 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
554 // The following extend from a legal type to an illegal type, so need to
555 // split the load. This introduced an extra load operation, but the
556 // extend is still "free".
557 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
558 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
559 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
560 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
561 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
562 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
563 };
564 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
565 if (const auto *Entry =
566 ConvertCostTableLookup(Table: MVELoadConversionTbl, ISD,
567 Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
568 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
569 }
570
571 static const TypeConversionCostTblEntry MVEFLoadConversionTbl[] = {
572 // FPExtends are similar but also require the VCVT instructions.
573 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1},
574 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 3},
575 };
576 if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
577 if (const auto *Entry =
578 ConvertCostTableLookup(Table: MVEFLoadConversionTbl, ISD,
579 Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
580 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
581 }
582
583 // The truncate of a store is free. This is the mirror of extends above.
584 static const TypeConversionCostTblEntry MVEStoreConversionTbl[] = {
585 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
586 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
587 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
588 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
589 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 1},
590 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
591 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
592 };
593 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
594 if (const auto *Entry =
595 ConvertCostTableLookup(Table: MVEStoreConversionTbl, ISD,
596 Dst: SrcTy.getSimpleVT(), Src: DstTy.getSimpleVT()))
597 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
598 }
599
600 static const TypeConversionCostTblEntry MVEFStoreConversionTbl[] = {
601 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1},
602 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 3},
603 };
604 if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
605 if (const auto *Entry =
606 ConvertCostTableLookup(Table: MVEFStoreConversionTbl, ISD,
607 Dst: SrcTy.getSimpleVT(), Src: DstTy.getSimpleVT()))
608 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
609 }
610 }
611
612 // NEON vector operations that can extend their inputs.
613 if ((ISD == ISD::SIGN_EXTEND || ISD == ISD::ZERO_EXTEND) &&
614 I && I->hasOneUse() && ST->hasNEON() && SrcTy.isVector()) {
615 static const TypeConversionCostTblEntry NEONDoubleWidthTbl[] = {
616 // vaddl
617 { .ISD: ISD::ADD, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
618 { .ISD: ISD::ADD, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
619 // vsubl
620 { .ISD: ISD::SUB, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
621 { .ISD: ISD::SUB, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
622 // vmull
623 { .ISD: ISD::MUL, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
624 { .ISD: ISD::MUL, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
625 // vshll
626 { .ISD: ISD::SHL, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
627 { .ISD: ISD::SHL, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
628 };
629
630 auto *User = cast<Instruction>(Val: *I->user_begin());
631 int UserISD = TLI->InstructionOpcodeToISD(Opcode: User->getOpcode());
632 if (auto *Entry = ConvertCostTableLookup(Table: NEONDoubleWidthTbl, ISD: UserISD,
633 Dst: DstTy.getSimpleVT(),
634 Src: SrcTy.getSimpleVT())) {
635 return AdjustCost(Entry->Cost);
636 }
637 }
638
639 // Single to/from double precision conversions.
640 if (Src->isVectorTy() && ST->hasNEON() &&
641 ((ISD == ISD::FP_ROUND && SrcTy.getScalarType() == MVT::f64 &&
642 DstTy.getScalarType() == MVT::f32) ||
643 (ISD == ISD::FP_EXTEND && SrcTy.getScalarType() == MVT::f32 &&
644 DstTy.getScalarType() == MVT::f64))) {
645 static const CostTblEntry NEONFltDblTbl[] = {
646 // Vector fptrunc/fpext conversions.
647 {.ISD: ISD::FP_ROUND, .Type: MVT::v2f64, .Cost: 2},
648 {.ISD: ISD::FP_EXTEND, .Type: MVT::v2f32, .Cost: 2},
649 {.ISD: ISD::FP_EXTEND, .Type: MVT::v4f32, .Cost: 4}};
650
651 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
652 if (const auto *Entry = CostTableLookup(Table: NEONFltDblTbl, ISD, Ty: LT.second))
653 return AdjustCost(LT.first * Entry->Cost);
654 }
655
656 // Some arithmetic, load and store operations have specific instructions
657 // to cast up/down their types automatically at no extra cost.
658 // TODO: Get these tables to know at least what the related operations are.
659 static const TypeConversionCostTblEntry NEONVectorConversionTbl[] = {
660 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
661 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
662 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1 },
663 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1 },
664 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: 0 },
665 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i32, .Cost: 1 },
666
667 // The number of vmovl instructions for the extension.
668 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
669 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
670 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
671 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
672 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 3 },
673 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 3 },
674 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
675 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
676 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3 },
677 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3 },
678 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3 },
679 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3 },
680 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7 },
681 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7 },
682 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6 },
683 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6 },
684 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6 },
685 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6 },
686
687 // Operations that we legalize using splitting.
688 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: 6 },
689 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: 3 },
690
691 // Vector float <-> i32 conversions.
692 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1 },
693 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1 },
694
695 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3 },
696 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3 },
697 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2 },
698 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2 },
699 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1 },
700 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1 },
701 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: 3 },
702 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: 3 },
703 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3 },
704 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3 },
705 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2 },
706 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2 },
707 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4 },
708 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4 },
709 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: 2 },
710 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: 2 },
711 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: 8 },
712 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: 8 },
713 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: 4 },
714 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: 4 },
715
716 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1 },
717 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1 },
718 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 3 },
719 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 3 },
720 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2 },
721 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2 },
722
723 // Vector double <-> i32 conversions.
724 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
725 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
726
727 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4 },
728 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4 },
729 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 3 },
730 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 3 },
731 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
732 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
733
734 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2 },
735 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2 },
736 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 4 },
737 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 4 },
738 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: 8 },
739 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: 8 }
740 };
741
742 if (SrcTy.isVector() && ST->hasNEON()) {
743 if (const auto *Entry = ConvertCostTableLookup(Table: NEONVectorConversionTbl, ISD,
744 Dst: DstTy.getSimpleVT(),
745 Src: SrcTy.getSimpleVT()))
746 return AdjustCost(Entry->Cost);
747 }
748
749 // Scalar float to integer conversions.
750 static const TypeConversionCostTblEntry NEONFloatConversionTbl[] = {
751 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i1, .Src: MVT::f32, .Cost: 2 },
752 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i1, .Src: MVT::f32, .Cost: 2 },
753 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i1, .Src: MVT::f64, .Cost: 2 },
754 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i1, .Src: MVT::f64, .Cost: 2 },
755 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i8, .Src: MVT::f32, .Cost: 2 },
756 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i8, .Src: MVT::f32, .Cost: 2 },
757 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i8, .Src: MVT::f64, .Cost: 2 },
758 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i8, .Src: MVT::f64, .Cost: 2 },
759 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i16, .Src: MVT::f32, .Cost: 2 },
760 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i16, .Src: MVT::f32, .Cost: 2 },
761 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i16, .Src: MVT::f64, .Cost: 2 },
762 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i16, .Src: MVT::f64, .Cost: 2 },
763 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: 2 },
764 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: 2 },
765 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: 2 },
766 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: 2 },
767 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: 10 },
768 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: 10 },
769 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: 10 },
770 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: 10 }
771 };
772 if (SrcTy.isFloatingPoint() && ST->hasNEON()) {
773 if (const auto *Entry = ConvertCostTableLookup(Table: NEONFloatConversionTbl, ISD,
774 Dst: DstTy.getSimpleVT(),
775 Src: SrcTy.getSimpleVT()))
776 return AdjustCost(Entry->Cost);
777 }
778
779 // Scalar integer to float conversions.
780 static const TypeConversionCostTblEntry NEONIntegerConversionTbl[] = {
781 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i1, .Cost: 2 },
782 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i1, .Cost: 2 },
783 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i1, .Cost: 2 },
784 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i1, .Cost: 2 },
785 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i8, .Cost: 2 },
786 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i8, .Cost: 2 },
787 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i8, .Cost: 2 },
788 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i8, .Cost: 2 },
789 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i16, .Cost: 2 },
790 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i16, .Cost: 2 },
791 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i16, .Cost: 2 },
792 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i16, .Cost: 2 },
793 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: 2 },
794 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: 2 },
795 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: 2 },
796 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: 2 },
797 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: 10 },
798 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: 10 },
799 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: 10 },
800 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: 10 }
801 };
802
803 if (SrcTy.isInteger() && ST->hasNEON()) {
804 if (const auto *Entry = ConvertCostTableLookup(Table: NEONIntegerConversionTbl,
805 ISD, Dst: DstTy.getSimpleVT(),
806 Src: SrcTy.getSimpleVT()))
807 return AdjustCost(Entry->Cost);
808 }
809
810 // MVE extend costs, taken from codegen tests. i8->i16 or i16->i32 is one
811 // instruction, i8->i32 is two. i64 zexts are an VAND with a constant, sext
812 // are linearised so take more.
813 static const TypeConversionCostTblEntry MVEVectorConversionTbl[] = {
814 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
815 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
816 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
817 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
818 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 10 },
819 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 2 },
820 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
821 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
822 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 10 },
823 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
824 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 8 },
825 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 2 },
826 };
827
828 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
829 if (const auto *Entry = ConvertCostTableLookup(Table: MVEVectorConversionTbl,
830 ISD, Dst: DstTy.getSimpleVT(),
831 Src: SrcTy.getSimpleVT()))
832 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
833 }
834
835 if (ISD == ISD::FP_ROUND || ISD == ISD::FP_EXTEND) {
836 // As general rule, fp converts that were not matched above are scalarized
837 // and cost 1 vcvt for each lane, so long as the instruction is available.
838 // If not it will become a series of function calls.
839 const InstructionCost CallCost =
840 getCallInstrCost(F: nullptr, RetTy: Dst, Tys: {Src}, CostKind);
841 int Lanes = 1;
842 if (SrcTy.isFixedLengthVector())
843 Lanes = SrcTy.getVectorNumElements();
844
845 if (IsLegalFPType(SrcTy) && IsLegalFPType(DstTy))
846 return Lanes;
847 else
848 return Lanes * CallCost;
849 }
850
851 if (ISD == ISD::TRUNCATE && ST->hasMVEIntegerOps() &&
852 SrcTy.isFixedLengthVector()) {
853 // Treat a truncate with larger than legal source (128bits for MVE) as
854 // expensive, 2 instructions per lane.
855 if ((SrcTy.getScalarType() == MVT::i8 ||
856 SrcTy.getScalarType() == MVT::i16 ||
857 SrcTy.getScalarType() == MVT::i32) &&
858 SrcTy.getSizeInBits() > 128 &&
859 SrcTy.getSizeInBits() > DstTy.getSizeInBits())
860 return SrcTy.getVectorNumElements() * 2;
861 }
862
863 // Scalar integer conversion costs.
864 static const TypeConversionCostTblEntry ARMIntegerConversionTbl[] = {
865 // i16 -> i64 requires two dependent operations.
866 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 2 },
867
868 // Truncates on i64 are assumed to be free.
869 { .ISD: ISD::TRUNCATE, .Dst: MVT::i32, .Src: MVT::i64, .Cost: 0 },
870 { .ISD: ISD::TRUNCATE, .Dst: MVT::i16, .Src: MVT::i64, .Cost: 0 },
871 { .ISD: ISD::TRUNCATE, .Dst: MVT::i8, .Src: MVT::i64, .Cost: 0 },
872 { .ISD: ISD::TRUNCATE, .Dst: MVT::i1, .Src: MVT::i64, .Cost: 0 }
873 };
874
875 if (SrcTy.isInteger()) {
876 if (const auto *Entry = ConvertCostTableLookup(Table: ARMIntegerConversionTbl, ISD,
877 Dst: DstTy.getSimpleVT(),
878 Src: SrcTy.getSimpleVT()))
879 return AdjustCost(Entry->Cost);
880 }
881
882 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
883 ? ST->getMVEVectorCostFactor(CostKind)
884 : 1;
885 return AdjustCost(
886 BaseCost * BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
887}
888
889InstructionCost ARMTTIImpl::getVectorInstrCost(
890 unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
891 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
892 // Penalize inserting into an D-subregister. We end up with a three times
893 // lower estimated throughput on swift.
894 if (ST->hasSlowLoadDSubregister() && Opcode == Instruction::InsertElement &&
895 ValTy->isVectorTy() && ValTy->getScalarSizeInBits() <= 32)
896 return 3;
897
898 if (ST->hasNEON() && (Opcode == Instruction::InsertElement ||
899 Opcode == Instruction::ExtractElement)) {
900 // Cross-class copies are expensive on many microarchitectures,
901 // so assume they are expensive by default.
902 if (cast<VectorType>(Val: ValTy)->getElementType()->isIntegerTy())
903 return 3;
904
905 // Even if it's not a cross class copy, this likely leads to mixing
906 // of NEON and VFP code and should be therefore penalized.
907 if (ValTy->isVectorTy() &&
908 ValTy->getScalarSizeInBits() <= 32)
909 return std::max<InstructionCost>(
910 a: BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
911 VIC),
912 b: 2U);
913 }
914
915 if (ST->hasMVEIntegerOps() && (Opcode == Instruction::InsertElement ||
916 Opcode == Instruction::ExtractElement)) {
917 // Integer cross-lane moves are more expensive than float, which can
918 // sometimes just be vmovs. Integer involve being passes to GPR registers,
919 // causing more of a delay.
920 std::pair<InstructionCost, MVT> LT =
921 getTypeLegalizationCost(Ty: ValTy->getScalarType());
922 return LT.first * (ValTy->getScalarType()->isIntegerTy() ? 4 : 1);
923 }
924
925 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
926 VIC);
927}
928
929InstructionCost ARMTTIImpl::getCmpSelInstrCost(
930 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
931 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
932 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
933 int ISD = TLI->InstructionOpcodeToISD(Opcode);
934
935 // Thumb scalar code size cost for select.
936 if (CostKind == TTI::TCK_CodeSize && ISD == ISD::SELECT &&
937 ST->isThumb() && !ValTy->isVectorTy()) {
938 // Assume expensive structs.
939 if (TLI->getValueType(DL, Ty: ValTy, AllowUnknown: true) == MVT::Other)
940 return TTI::TCC_Expensive;
941
942 // Select costs can vary because they:
943 // - may require one or more conditional mov (including an IT),
944 // - can't operate directly on immediates,
945 // - require live flags, which we can't copy around easily.
946 InstructionCost Cost = getTypeLegalizationCost(Ty: ValTy).first;
947
948 // Possible IT instruction for Thumb2, or more for Thumb1.
949 ++Cost;
950
951 // i1 values may need rematerialising by using mov immediates and/or
952 // flag setting instructions.
953 if (ValTy->isIntegerTy(BitWidth: 1))
954 ++Cost;
955
956 return Cost;
957 }
958
959 // If this is a vector min/max/abs, use the cost of that intrinsic directly
960 // instead. Hopefully when min/max intrinsics are more prevalent this code
961 // will not be needed.
962 const Instruction *Sel = I;
963 if ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && Sel &&
964 Sel->hasOneUse())
965 Sel = cast<Instruction>(Val: Sel->user_back());
966 if (Sel && ValTy->isVectorTy() &&
967 (ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy())) {
968 const Value *LHS, *RHS;
969 SelectPatternFlavor SPF = matchSelectPattern(V: Sel, LHS, RHS).Flavor;
970 unsigned IID = 0;
971 switch (SPF) {
972 case SPF_ABS:
973 IID = Intrinsic::abs;
974 break;
975 case SPF_SMIN:
976 IID = Intrinsic::smin;
977 break;
978 case SPF_SMAX:
979 IID = Intrinsic::smax;
980 break;
981 case SPF_UMIN:
982 IID = Intrinsic::umin;
983 break;
984 case SPF_UMAX:
985 IID = Intrinsic::umax;
986 break;
987 case SPF_FMINNUM:
988 IID = Intrinsic::minnum;
989 break;
990 case SPF_FMAXNUM:
991 IID = Intrinsic::maxnum;
992 break;
993 default:
994 break;
995 }
996 if (IID) {
997 // The ICmp is free, the select gets the cost of the min/max/etc
998 if (Sel != I)
999 return 0;
1000 IntrinsicCostAttributes CostAttrs(IID, ValTy, {ValTy, ValTy});
1001 return getIntrinsicInstrCost(ICA: CostAttrs, CostKind);
1002 }
1003 }
1004
1005 // On NEON a vector select gets lowered to vbsl.
1006 if (ST->hasNEON() && ValTy->isVectorTy() && ISD == ISD::SELECT && CondTy) {
1007 // Lowering of some vector selects is currently far from perfect.
1008 static const TypeConversionCostTblEntry NEONVectorSelectTbl[] = {
1009 { .ISD: ISD::SELECT, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: 4*4 + 1*2 + 1 },
1010 { .ISD: ISD::SELECT, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: 50 },
1011 { .ISD: ISD::SELECT, .Dst: MVT::v16i1, .Src: MVT::v16i64, .Cost: 100 }
1012 };
1013
1014 EVT SelCondTy = TLI->getValueType(DL, Ty: CondTy);
1015 EVT SelValTy = TLI->getValueType(DL, Ty: ValTy);
1016 if (SelCondTy.isSimple() && SelValTy.isSimple()) {
1017 if (const auto *Entry = ConvertCostTableLookup(Table: NEONVectorSelectTbl, ISD,
1018 Dst: SelCondTy.getSimpleVT(),
1019 Src: SelValTy.getSimpleVT()))
1020 return Entry->Cost;
1021 }
1022
1023 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1024 return LT.first;
1025 }
1026
1027 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy() &&
1028 (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
1029 cast<FixedVectorType>(Val: ValTy)->getNumElements() > 1) {
1030 FixedVectorType *VecValTy = cast<FixedVectorType>(Val: ValTy);
1031 FixedVectorType *VecCondTy = dyn_cast_or_null<FixedVectorType>(Val: CondTy);
1032 if (!VecCondTy)
1033 VecCondTy = cast<FixedVectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecValTy));
1034
1035 // If we don't have mve.fp any fp operations will need to be scalarized.
1036 if (Opcode == Instruction::FCmp && !ST->hasMVEFloatOps()) {
1037 // One scalaization insert, one scalarization extract and the cost of the
1038 // fcmps.
1039 return BaseT::getScalarizationOverhead(InTy: VecValTy, /*Insert*/ false,
1040 /*Extract*/ true, CostKind) +
1041 BaseT::getScalarizationOverhead(InTy: VecCondTy, /*Insert*/ true,
1042 /*Extract*/ false, CostKind) +
1043 VecValTy->getNumElements() *
1044 getCmpSelInstrCost(Opcode, ValTy: ValTy->getScalarType(),
1045 CondTy: VecCondTy->getScalarType(), VecPred,
1046 CostKind, Op1Info, Op2Info, I);
1047 }
1048
1049 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1050 int BaseCost = ST->getMVEVectorCostFactor(CostKind);
1051 // There are two types - the input that specifies the type of the compare
1052 // and the output vXi1 type. Because we don't know how the output will be
1053 // split, we may need an expensive shuffle to get two in sync. This has the
1054 // effect of making larger than legal compares (v8i32 for example)
1055 // expensive.
1056 if (LT.second.isVector() && LT.second.getVectorNumElements() > 2) {
1057 if (LT.first > 1)
1058 return LT.first * BaseCost +
1059 BaseT::getScalarizationOverhead(InTy: VecCondTy, /*Insert*/ true,
1060 /*Extract*/ false, CostKind);
1061 return BaseCost;
1062 }
1063 }
1064
1065 // Default to cheap (throughput/size of 1 instruction) but adjust throughput
1066 // for "multiple beats" potentially needed by MVE instructions.
1067 int BaseCost = 1;
1068 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy())
1069 BaseCost = ST->getMVEVectorCostFactor(CostKind);
1070
1071 return BaseCost * BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred,
1072 CostKind, Op1Info, Op2Info, I);
1073}
1074
1075InstructionCost
1076ARMTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
1077 const SCEV *Ptr,
1078 TTI::TargetCostKind CostKind) const {
1079 // Address computations in vectorized code with non-consecutive addresses will
1080 // likely result in more instructions compared to scalar code where the
1081 // computation can more often be merged into the index mode. The resulting
1082 // extra micro-ops can significantly decrease throughput.
1083 unsigned NumVectorInstToHideOverhead = 10;
1084 int MaxMergeDistance = 64;
1085
1086 if (ST->hasNEON()) {
1087 if (PtrTy->isVectorTy() && SE &&
1088 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MergeDistance: MaxMergeDistance + 1))
1089 return NumVectorInstToHideOverhead;
1090
1091 // In many cases the address computation is not merged into the instruction
1092 // addressing mode.
1093 return 1;
1094 }
1095 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
1096}
1097
1098bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) const {
1099 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1100 // If a VCTP is part of a chain, it's already profitable and shouldn't be
1101 // optimized, else LSR may block tail-predication.
1102 switch (II->getIntrinsicID()) {
1103 case Intrinsic::arm_mve_vctp8:
1104 case Intrinsic::arm_mve_vctp16:
1105 case Intrinsic::arm_mve_vctp32:
1106 case Intrinsic::arm_mve_vctp64:
1107 return true;
1108 default:
1109 break;
1110 }
1111 }
1112 return false;
1113}
1114
1115bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
1116 unsigned /*AddressSpace*/,
1117 TTI::MaskKind /*MaskKind*/) const {
1118 if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
1119 return false;
1120
1121 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: DataTy)) {
1122 // Don't support v2i1 yet.
1123 if (VecTy->getNumElements() == 2)
1124 return false;
1125
1126 // We don't support extending fp types.
1127 unsigned VecWidth = DataTy->getPrimitiveSizeInBits();
1128 if (VecWidth != 128 && VecTy->getElementType()->isFloatingPointTy())
1129 return false;
1130 }
1131
1132 unsigned EltWidth = DataTy->getScalarSizeInBits();
1133 return (EltWidth == 32 && Alignment >= 4) ||
1134 (EltWidth == 16 && Alignment >= 2) || (EltWidth == 8);
1135}
1136
1137bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) const {
1138 if (!EnableMaskedGatherScatters || !ST->hasMVEIntegerOps())
1139 return false;
1140
1141 unsigned EltWidth = Ty->getScalarSizeInBits();
1142 return ((EltWidth == 32 && Alignment >= 4) ||
1143 (EltWidth == 16 && Alignment >= 2) || EltWidth == 8);
1144}
1145
1146/// Given a memcpy/memset/memmove instruction, return the number of memory
1147/// operations performed, via querying findOptimalMemOpLowering. Returns -1 if a
1148/// call is used.
1149int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const {
1150 MemOp MOp;
1151 unsigned DstAddrSpace = ~0u;
1152 unsigned SrcAddrSpace = ~0u;
1153 const Function *F = I->getParent()->getParent();
1154
1155 if (const auto *MC = dyn_cast<MemTransferInst>(Val: I)) {
1156 ConstantInt *C = dyn_cast<ConstantInt>(Val: MC->getLength());
1157 // If 'size' is not a constant, a library call will be generated.
1158 if (!C)
1159 return -1;
1160
1161 const unsigned Size = C->getValue().getZExtValue();
1162 const Align DstAlign = MC->getDestAlign().valueOrOne();
1163 const Align SrcAlign = MC->getSourceAlign().valueOrOne();
1164
1165 // Use the most restrictive of memset, memcpy, memmove.
1166 MOp = MemOp::Move(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
1167 /*IsVolatile*/ false);
1168 DstAddrSpace = MC->getDestAddressSpace();
1169 SrcAddrSpace = MC->getSourceAddressSpace();
1170 }
1171 else if (const auto *MS = dyn_cast<MemSetInst>(Val: I)) {
1172 ConstantInt *C = dyn_cast<ConstantInt>(Val: MS->getLength());
1173 // If 'size' is not a constant, a library call will be generated.
1174 if (!C)
1175 return -1;
1176
1177 const unsigned Size = C->getValue().getZExtValue();
1178 const Align DstAlign = MS->getDestAlign().valueOrOne();
1179
1180 MOp = MemOp::Set(Size, /*DstAlignCanChange*/ false, DstAlign,
1181 /*IsZeroMemset*/ false, /*IsVolatile*/ false);
1182 DstAddrSpace = MS->getDestAddressSpace();
1183 }
1184 else
1185 llvm_unreachable("Expected a memcpy/move or memset!");
1186
1187 unsigned Limit, Factor = 2;
1188 switch(I->getIntrinsicID()) {
1189 case Intrinsic::memcpy:
1190 Limit = TLI->getMaxStoresPerMemcpy(OptSize: F->hasMinSize());
1191 break;
1192 case Intrinsic::memmove:
1193 Limit = TLI->getMaxStoresPerMemmove(OptSize: F->hasMinSize());
1194 break;
1195 case Intrinsic::memset:
1196 Limit = TLI->getMaxStoresPerMemset(OptSize: F->hasMinSize());
1197 Factor = 1;
1198 break;
1199 default:
1200 llvm_unreachable("Expected a memcpy/move or memset!");
1201 }
1202
1203 // MemOps will be poplulated with a list of data types that needs to be
1204 // loaded and stored. That's why we multiply the number of elements by 2 to
1205 // get the cost for this memcpy.
1206 std::vector<EVT> MemOps;
1207 LLVMContext &C = F->getContext();
1208 if (getTLI()->findOptimalMemOpLowering(Context&: C, MemOps, Limit, Op: MOp, DstAS: DstAddrSpace,
1209 SrcAS: SrcAddrSpace, FuncAttributes: F->getAttributes(),
1210 LargestVT: nullptr))
1211 return MemOps.size() * Factor;
1212
1213 // If we can't find an optimal memop lowering, return the default cost
1214 return -1;
1215}
1216
1217InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const {
1218 int NumOps = getNumMemOps(I: cast<IntrinsicInst>(Val: I));
1219
1220 // To model the cost of a library call, we assume 1 for the call, and
1221 // 3 for the argument setup.
1222 if (NumOps == -1)
1223 return 4;
1224 return NumOps;
1225}
1226
1227InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
1228 VectorType *DstTy, VectorType *SrcTy,
1229 ArrayRef<int> Mask,
1230 TTI::TargetCostKind CostKind,
1231 int Index, VectorType *SubTp,
1232 ArrayRef<const Value *> Args,
1233 const Instruction *CxtI) const {
1234 assert((Mask.empty() || DstTy->isScalableTy() ||
1235 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
1236 "Expected the Mask to match the return size if given");
1237 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
1238 "Expected the same scalar types");
1239
1240 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1241 // Treat extractsubvector as single op permutation.
1242 bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector;
1243 if (IsExtractSubvector)
1244 Kind = TTI::SK_PermuteSingleSrc;
1245 if (ST->hasNEON()) {
1246 if (Kind == TTI::SK_Broadcast) {
1247 static const CostTblEntry NEONDupTbl[] = {
1248 // VDUP handles these cases.
1249 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1250 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1251 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1252 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1253 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 1},
1254 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i8, .Cost: 1},
1255
1256 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 1},
1257 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 1},
1258 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 1},
1259 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 1}};
1260
1261 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1262 if (const auto *Entry =
1263 CostTableLookup(Table: NEONDupTbl, ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1264 return LT.first * Entry->Cost;
1265 }
1266 if (Kind == TTI::SK_Reverse) {
1267 static const CostTblEntry NEONShuffleTbl[] = {
1268 // Reverse shuffle cost one instruction if we are shuffling within a
1269 // double word (vrev) or two if we shuffle a quad word (vrev, vext).
1270 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1271 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1272 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1273 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1274 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 1},
1275 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i8, .Cost: 1},
1276
1277 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 2},
1278 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 2},
1279 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 2},
1280 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 2}};
1281
1282 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1283 if (const auto *Entry =
1284 CostTableLookup(Table: NEONShuffleTbl, ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1285 return LT.first * Entry->Cost;
1286 }
1287 if (Kind == TTI::SK_Select) {
1288 static const CostTblEntry NEONSelShuffleTbl[] = {
1289 // Select shuffle cost table for ARM. Cost is the number of
1290 // instructions
1291 // required to create the shuffled vector.
1292
1293 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1294 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1295 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1296 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1297
1298 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 2},
1299 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 2},
1300 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 2},
1301
1302 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 16},
1303
1304 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 32}};
1305
1306 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1307 if (const auto *Entry = CostTableLookup(Table: NEONSelShuffleTbl,
1308 ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1309 return LT.first * Entry->Cost;
1310 }
1311 }
1312 if (ST->hasMVEIntegerOps()) {
1313 if (Kind == TTI::SK_Broadcast) {
1314 static const CostTblEntry MVEDupTbl[] = {
1315 // VDUP handles these cases.
1316 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 1},
1317 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 1},
1318 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 1},
1319 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 1},
1320 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8f16, .Cost: 1}};
1321
1322 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1323 if (const auto *Entry = CostTableLookup(Table: MVEDupTbl, ISD: ISD::VECTOR_SHUFFLE,
1324 Ty: LT.second))
1325 return LT.first * Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
1326 }
1327
1328 if (!Mask.empty()) {
1329 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1330 // Check for LD2/LD4 instructions, which are represented in llvm IR as
1331 // deinterleaving-shuffle(load). The shuffle cost could potentially be
1332 // free, but we model it with a cost of LT.first so that LD2/LD4 have a
1333 // higher cost than just the load.
1334 if (Args.size() >= 1 && isa<LoadInst>(Val: Args[0]) &&
1335 (LT.second.getScalarSizeInBits() == 8 ||
1336 LT.second.getScalarSizeInBits() == 16 ||
1337 LT.second.getScalarSizeInBits() == 32) &&
1338 LT.second.getSizeInBits() == 128 &&
1339 ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
1340 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2)) ||
1341 (TLI->getMaxSupportedInterleaveFactor() == 4 &&
1342 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 4))))
1343 return ST->getMVEVectorCostFactor(CostKind) *
1344 std::max<InstructionCost>(a: 1, b: LT.first / 4);
1345
1346 // Check for ST2/ST4 instructions, which are represented in llvm IR as
1347 // store(interleaving-shuffle). The shuffle cost could potentially be
1348 // free, but we model it with a cost of LT.first so that ST2/ST4 have a
1349 // higher cost than just the store.
1350 if (CxtI && CxtI->hasOneUse() && isa<StoreInst>(Val: *CxtI->user_begin()) &&
1351 (LT.second.getScalarSizeInBits() == 8 ||
1352 LT.second.getScalarSizeInBits() == 16 ||
1353 LT.second.getScalarSizeInBits() == 32) &&
1354 LT.second.getSizeInBits() == 128 &&
1355 ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
1356 ShuffleVectorInst::isInterleaveMask(
1357 Mask, Factor: 2, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2)) ||
1358 (TLI->getMaxSupportedInterleaveFactor() == 4 &&
1359 ShuffleVectorInst::isInterleaveMask(
1360 Mask, Factor: 4, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2))))
1361 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1362
1363 if (LT.second.isVector() &&
1364 Mask.size() <= LT.second.getVectorNumElements() &&
1365 (isVREVMask(M: Mask, VT: LT.second, BlockSize: 16) || isVREVMask(M: Mask, VT: LT.second, BlockSize: 32) ||
1366 isVREVMask(M: Mask, VT: LT.second, BlockSize: 64)))
1367 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1368 }
1369 }
1370
1371 // Restore optimal kind.
1372 if (IsExtractSubvector)
1373 Kind = TTI::SK_ExtractSubvector;
1374 int BaseCost = ST->hasMVEIntegerOps() && SrcTy->isVectorTy()
1375 ? ST->getMVEVectorCostFactor(CostKind)
1376 : 1;
1377 return BaseCost * BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind,
1378 Index, SubTp);
1379}
1380
1381InstructionCost ARMTTIImpl::getArithmeticInstrCost(
1382 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
1383 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
1384 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
1385 int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
1386 if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(BitWidth: 1)) {
1387 // Make operations on i1 relatively expensive as this often involves
1388 // combining predicates. AND and XOR should be easier to handle with IT
1389 // blocks.
1390 switch (ISDOpcode) {
1391 default:
1392 break;
1393 case ISD::AND:
1394 case ISD::XOR:
1395 return 2;
1396 case ISD::OR:
1397 return 3;
1398 }
1399 }
1400
1401 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1402
1403 if (ST->hasNEON()) {
1404 const unsigned FunctionCallDivCost = 20;
1405 const unsigned ReciprocalDivCost = 10;
1406 static const CostTblEntry CostTbl[] = {
1407 // Division.
1408 // These costs are somewhat random. Choose a cost of 20 to indicate that
1409 // vectorizing devision (added function call) is going to be very expensive.
1410 // Double registers types.
1411 { .ISD: ISD::SDIV, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1412 { .ISD: ISD::UDIV, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1413 { .ISD: ISD::SREM, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1414 { .ISD: ISD::UREM, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1415 { .ISD: ISD::SDIV, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1416 { .ISD: ISD::UDIV, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1417 { .ISD: ISD::SREM, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1418 { .ISD: ISD::UREM, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1419 { .ISD: ISD::SDIV, .Type: MVT::v4i16, .Cost: ReciprocalDivCost},
1420 { .ISD: ISD::UDIV, .Type: MVT::v4i16, .Cost: ReciprocalDivCost},
1421 { .ISD: ISD::SREM, .Type: MVT::v4i16, .Cost: 4 * FunctionCallDivCost},
1422 { .ISD: ISD::UREM, .Type: MVT::v4i16, .Cost: 4 * FunctionCallDivCost},
1423 { .ISD: ISD::SDIV, .Type: MVT::v8i8, .Cost: ReciprocalDivCost},
1424 { .ISD: ISD::UDIV, .Type: MVT::v8i8, .Cost: ReciprocalDivCost},
1425 { .ISD: ISD::SREM, .Type: MVT::v8i8, .Cost: 8 * FunctionCallDivCost},
1426 { .ISD: ISD::UREM, .Type: MVT::v8i8, .Cost: 8 * FunctionCallDivCost},
1427 // Quad register types.
1428 { .ISD: ISD::SDIV, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1429 { .ISD: ISD::UDIV, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1430 { .ISD: ISD::SREM, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1431 { .ISD: ISD::UREM, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1432 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1433 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1434 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1435 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1436 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1437 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1438 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1439 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1440 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1441 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1442 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1443 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1444 // Multiplication.
1445 };
1446
1447 if (const auto *Entry = CostTableLookup(Table: CostTbl, ISD: ISDOpcode, Ty: LT.second))
1448 return LT.first * Entry->Cost;
1449
1450 InstructionCost Cost = BaseT::getArithmeticInstrCost(
1451 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
1452
1453 // This is somewhat of a hack. The problem that we are facing is that SROA
1454 // creates a sequence of shift, and, or instructions to construct values.
1455 // These sequences are recognized by the ISel and have zero-cost. Not so for
1456 // the vectorized code. Because we have support for v2i64 but not i64 those
1457 // sequences look particularly beneficial to vectorize.
1458 // To work around this we increase the cost of v2i64 operations to make them
1459 // seem less beneficial.
1460 if (LT.second == MVT::v2i64 && Op2Info.isUniform() && Op2Info.isConstant())
1461 Cost += 4;
1462
1463 return Cost;
1464 }
1465
1466 // If this operation is a shift on arm/thumb2, it might well be folded into
1467 // the following instruction, hence having a cost of 0.
1468 auto LooksLikeAFreeShift = [&]() {
1469 if (ST->isThumb1Only() || Ty->isVectorTy())
1470 return false;
1471
1472 if (!CxtI || !CxtI->hasOneUse() || !CxtI->isShift())
1473 return false;
1474 if (!Op2Info.isUniform() || !Op2Info.isConstant())
1475 return false;
1476
1477 // Folded into a ADC/ADD/AND/BIC/CMP/EOR/MVN/ORR/ORN/RSB/SBC/SUB
1478 switch (cast<Instruction>(Val: CxtI->user_back())->getOpcode()) {
1479 case Instruction::Add:
1480 case Instruction::Sub:
1481 case Instruction::And:
1482 case Instruction::Xor:
1483 case Instruction::Or:
1484 case Instruction::ICmp:
1485 return true;
1486 default:
1487 return false;
1488 }
1489 };
1490 if (LooksLikeAFreeShift())
1491 return 0;
1492
1493 // When targets have both DSP and MVE we find that the
1494 // the compiler will attempt to vectorize as well as using
1495 // scalar (S/U)MLAL operations. This is in cases where we have
1496 // the pattern ext(mul(ext(i16), ext(i16))) we find
1497 // that codegen performs better when only using (S/U)MLAL scalar
1498 // ops instead of trying to mix vector ops with (S/U)MLAL ops. We therefore
1499 // check if a mul instruction is used in a (U/S)MLAL pattern.
1500 auto MulInDSPMLALPattern = [&](const Instruction *I, unsigned Opcode,
1501 Type *Ty) -> bool {
1502 if (!ST->hasDSP())
1503 return false;
1504
1505 if (!I)
1506 return false;
1507
1508 if (Opcode != Instruction::Mul)
1509 return false;
1510
1511 if (Ty->isVectorTy())
1512 return false;
1513
1514 auto ValueOpcodesEqual = [](const Value *LHS, const Value *RHS) -> bool {
1515 return cast<Instruction>(Val: LHS)->getOpcode() ==
1516 cast<Instruction>(Val: RHS)->getOpcode();
1517 };
1518 auto IsExtInst = [](const Value *V) -> bool {
1519 return isa<ZExtInst>(Val: V) || isa<SExtInst>(Val: V);
1520 };
1521 auto IsExtensionFromHalf = [](const Value *V) -> bool {
1522 return cast<Instruction>(Val: V)->getOperand(i: 0)->getType()->isIntegerTy(BitWidth: 16);
1523 };
1524
1525 // We check the arguments of the instruction to see if they're extends
1526 auto *BinOp = dyn_cast<BinaryOperator>(Val: I);
1527 if (!BinOp)
1528 return false;
1529 Value *Op0 = BinOp->getOperand(i_nocapture: 0);
1530 Value *Op1 = BinOp->getOperand(i_nocapture: 1);
1531 if (IsExtInst(Op0) && IsExtInst(Op1) && ValueOpcodesEqual(Op0, Op1)) {
1532 // We're interested in an ext of an i16
1533 if (!I->getType()->isIntegerTy(BitWidth: 32) || !IsExtensionFromHalf(Op0) ||
1534 !IsExtensionFromHalf(Op1))
1535 return false;
1536 // We need to check if this result will be further extended to i64
1537 // and that all these uses are SExt
1538 for (auto *U : I->users())
1539 if (!IsExtInst(U))
1540 return false;
1541 return true;
1542 }
1543
1544 return false;
1545 };
1546
1547 if (MulInDSPMLALPattern(CxtI, Opcode, Ty))
1548 return 0;
1549
1550 // Default to cheap (throughput/size of 1 instruction) but adjust throughput
1551 // for "multiple beats" potentially needed by MVE instructions.
1552 int BaseCost = 1;
1553 if (ST->hasMVEIntegerOps() && Ty->isVectorTy())
1554 BaseCost = ST->getMVEVectorCostFactor(CostKind);
1555
1556 // The rest of this mostly follows what is done in
1557 // BaseT::getArithmeticInstrCost, without treating floats as more expensive
1558 // that scalars or increasing the costs for custom operations. The results is
1559 // also multiplied by the MVEVectorCostFactor where appropriate.
1560 if (TLI->isOperationLegalOrCustomOrPromote(Op: ISDOpcode, VT: LT.second))
1561 return LT.first * BaseCost;
1562
1563 // Else this is expand, assume that we need to scalarize this op.
1564 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
1565 unsigned Num = VTy->getNumElements();
1566 InstructionCost Cost =
1567 getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind);
1568 // Return the cost of multiple scalar invocation plus the cost of
1569 // inserting and extracting the values.
1570 SmallVector<Type *> Tys(Args.size(), Ty);
1571 return BaseT::getScalarizationOverhead(RetTy: VTy, Args, Tys, CostKind) +
1572 Num * Cost;
1573 }
1574
1575 return BaseCost;
1576}
1577
1578InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
1579 Align Alignment,
1580 unsigned AddressSpace,
1581 TTI::TargetCostKind CostKind,
1582 TTI::OperandValueInfo OpInfo,
1583 const Instruction *I) const {
1584 // FIXME: Load latency isn't handled here
1585 if (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)
1586 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1587 CostKind, OpInfo, I);
1588
1589 // TODO: Handle other cost kinds.
1590 if (CostKind != TTI::TCK_RecipThroughput)
1591 return 1;
1592
1593 // Type legalization can't handle structs
1594 if (TLI->getValueType(DL, Ty: Src, AllowUnknown: true) == MVT::Other)
1595 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1596 CostKind);
1597
1598 if (ST->hasNEON() && Src->isVectorTy() && Alignment != Align(16) &&
1599 cast<VectorType>(Val: Src)->getElementType()->isDoubleTy()) {
1600 // Unaligned loads/stores are extremely inefficient.
1601 // We need 4 uops for vst.1/vld.1 vs 1uop for vldr/vstr.
1602 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
1603 return LT.first * 4;
1604 }
1605
1606 // MVE can optimize a fpext(load(4xhalf)) using an extending integer load.
1607 // Same for stores.
1608 if (ST->hasMVEFloatOps() && isa<FixedVectorType>(Val: Src) && I &&
1609 ((Opcode == Instruction::Load && I->hasOneUse() &&
1610 isa<FPExtInst>(Val: *I->user_begin())) ||
1611 (Opcode == Instruction::Store && isa<FPTruncInst>(Val: I->getOperand(i: 0))))) {
1612 FixedVectorType *SrcVTy = cast<FixedVectorType>(Val: Src);
1613 Type *DstTy =
1614 Opcode == Instruction::Load
1615 ? (*I->user_begin())->getType()
1616 : cast<Instruction>(Val: I->getOperand(i: 0))->getOperand(i: 0)->getType();
1617 if (SrcVTy->getNumElements() == 4 && SrcVTy->getScalarType()->isHalfTy() &&
1618 DstTy->getScalarType()->isFloatTy())
1619 return ST->getMVEVectorCostFactor(CostKind);
1620 }
1621
1622 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
1623 ? ST->getMVEVectorCostFactor(CostKind)
1624 : 1;
1625 return BaseCost * BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1626 CostKind, OpInfo, I);
1627}
1628
1629InstructionCost
1630ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1631 TTI::TargetCostKind CostKind) const {
1632 switch (MICA.getID()) {
1633 case Intrinsic::masked_scatter:
1634 case Intrinsic::masked_gather:
1635 return getGatherScatterOpCost(MICA, CostKind);
1636 case Intrinsic::masked_load:
1637 case Intrinsic::masked_store:
1638 return getMaskedMemoryOpCost(MICA, CostKind);
1639 }
1640 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1641}
1642
1643InstructionCost
1644ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1645 TTI::TargetCostKind CostKind) const {
1646 unsigned IID = MICA.getID();
1647 Type *Src = MICA.getDataType();
1648 Align Alignment = MICA.getAlignment();
1649 unsigned AddressSpace = MICA.getAddressSpace();
1650 if (ST->hasMVEIntegerOps()) {
1651 if (IID == Intrinsic::masked_load &&
1652 isLegalMaskedLoad(DataTy: Src, Alignment, AddressSpace))
1653 return ST->getMVEVectorCostFactor(CostKind);
1654 if (IID == Intrinsic::masked_store &&
1655 isLegalMaskedStore(DataTy: Src, Alignment, AddressSpace))
1656 return ST->getMVEVectorCostFactor(CostKind);
1657 }
1658 if (!isa<FixedVectorType>(Val: Src))
1659 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1660 // Scalar cost, which is currently very high due to the efficiency of the
1661 // generated code.
1662 return cast<FixedVectorType>(Val: Src)->getNumElements() * 8;
1663}
1664
1665InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost(
1666 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1667 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1668 bool UseMaskForCond, bool UseMaskForGaps) const {
1669 assert(Factor >= 2 && "Invalid interleave factor");
1670 assert(isa<VectorType>(VecTy) && "Expect a vector type");
1671
1672 // vldN/vstN doesn't support vector types of i64/f64 element.
1673 bool EltIs64Bits = DL.getTypeSizeInBits(Ty: VecTy->getScalarType()) == 64;
1674
1675 if (Factor <= TLI->getMaxSupportedInterleaveFactor() && !EltIs64Bits &&
1676 !UseMaskForCond && !UseMaskForGaps) {
1677 unsigned NumElts = cast<FixedVectorType>(Val: VecTy)->getNumElements();
1678 auto *SubVecTy =
1679 FixedVectorType::get(ElementType: VecTy->getScalarType(), NumElts: NumElts / Factor);
1680
1681 // vldN/vstN only support legal vector types of size 64 or 128 in bits.
1682 // Accesses having vector types that are a multiple of 128 bits can be
1683 // matched to more than one vldN/vstN instruction.
1684 int BaseCost =
1685 ST->hasMVEIntegerOps() ? ST->getMVEVectorCostFactor(CostKind) : 1;
1686 if (NumElts % Factor == 0 &&
1687 TLI->isLegalInterleavedAccessType(Factor, VecTy: SubVecTy, Alignment, DL))
1688 return Factor * BaseCost * TLI->getNumInterleavedAccesses(VecTy: SubVecTy, DL);
1689
1690 // Some smaller than legal interleaved patterns are cheap as we can make
1691 // use of the vmovn or vrev patterns to interleave a standard load. This is
1692 // true for v4i8, v8i8 and v4i16 at least (but not for v4f16 as it is
1693 // promoted differently). The cost of 2 here is then a load and vrev or
1694 // vmovn.
1695 if (ST->hasMVEIntegerOps() && Factor == 2 && NumElts / Factor > 2 &&
1696 VecTy->isIntOrIntVectorTy() &&
1697 DL.getTypeSizeInBits(Ty: SubVecTy).getFixedValue() <= 64)
1698 return 2 * BaseCost;
1699 }
1700
1701 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1702 Alignment, AddressSpace, CostKind,
1703 UseMaskForCond, UseMaskForGaps);
1704}
1705
1706InstructionCost
1707ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1708 TTI::TargetCostKind CostKind) const {
1709
1710 Type *DataTy = MICA.getDataType();
1711 const Value *Ptr = MICA.getPointer();
1712 bool VariableMask = MICA.getVariableMask();
1713 Align Alignment = MICA.getAlignment();
1714 const Instruction *I = MICA.getInst();
1715
1716 using namespace PatternMatch;
1717 if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
1718 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1719
1720 assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!");
1721 auto *VTy = cast<FixedVectorType>(Val: DataTy);
1722
1723 // TODO: Splitting, once we do that.
1724
1725 unsigned NumElems = VTy->getNumElements();
1726 unsigned EltSize = VTy->getScalarSizeInBits();
1727 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: DataTy);
1728
1729 // For now, it is assumed that for the MVE gather instructions the loads are
1730 // all effectively serialised. This means the cost is the scalar cost
1731 // multiplied by the number of elements being loaded. This is possibly very
1732 // conservative, but even so we still end up vectorising loops because the
1733 // cost per iteration for many loops is lower than for scalar loops.
1734 InstructionCost VectorCost =
1735 NumElems * LT.first * ST->getMVEVectorCostFactor(CostKind);
1736 // The scalarization cost should be a lot higher. We use the number of vector
1737 // elements plus the scalarization overhead. If masking is required then a lot
1738 // of little blocks will be needed and potentially a scalarized p0 mask,
1739 // greatly increasing the cost.
1740 InstructionCost ScalarCost =
1741 NumElems * LT.first + (VariableMask ? NumElems * 5 : 0) +
1742 BaseT::getScalarizationOverhead(InTy: VTy, /*Insert*/ true, /*Extract*/ false,
1743 CostKind) +
1744 BaseT::getScalarizationOverhead(InTy: VTy, /*Insert*/ false, /*Extract*/ true,
1745 CostKind);
1746
1747 if (EltSize < 8 || Alignment < EltSize / 8)
1748 return ScalarCost;
1749
1750 unsigned ExtSize = EltSize;
1751 // Check whether there's a single user that asks for an extended type
1752 if (I != nullptr) {
1753 // Dependent of the caller of this function, a gather instruction will
1754 // either have opcode Instruction::Load or be a call to the masked_gather
1755 // intrinsic
1756 if ((I->getOpcode() == Instruction::Load ||
1757 match(V: I, P: m_Intrinsic<Intrinsic::masked_gather>())) &&
1758 I->hasOneUse()) {
1759 const User *Us = *I->users().begin();
1760 if (isa<ZExtInst>(Val: Us) || isa<SExtInst>(Val: Us)) {
1761 // only allow valid type combinations
1762 unsigned TypeSize =
1763 cast<Instruction>(Val: Us)->getType()->getScalarSizeInBits();
1764 if (((TypeSize == 32 && (EltSize == 8 || EltSize == 16)) ||
1765 (TypeSize == 16 && EltSize == 8)) &&
1766 TypeSize * NumElems == 128) {
1767 ExtSize = TypeSize;
1768 }
1769 }
1770 }
1771 // Check whether the input data needs to be truncated
1772 TruncInst *T;
1773 if ((I->getOpcode() == Instruction::Store ||
1774 match(V: I, P: m_Intrinsic<Intrinsic::masked_scatter>())) &&
1775 (T = dyn_cast<TruncInst>(Val: I->getOperand(i: 0)))) {
1776 // Only allow valid type combinations
1777 unsigned TypeSize = T->getOperand(i_nocapture: 0)->getType()->getScalarSizeInBits();
1778 if (((EltSize == 16 && TypeSize == 32) ||
1779 (EltSize == 8 && (TypeSize == 32 || TypeSize == 16))) &&
1780 TypeSize * NumElems == 128)
1781 ExtSize = TypeSize;
1782 }
1783 }
1784
1785 if (ExtSize * NumElems != 128 || NumElems < 4)
1786 return ScalarCost;
1787
1788 // Any (aligned) i32 gather will not need to be scalarised.
1789 if (ExtSize == 32)
1790 return VectorCost;
1791 // For smaller types, we need to ensure that the gep's inputs are correctly
1792 // extended from a small enough value. Other sizes (including i64) are
1793 // scalarized for now.
1794 if (ExtSize != 8 && ExtSize != 16)
1795 return ScalarCost;
1796
1797 if (const auto *BC = dyn_cast<BitCastInst>(Val: Ptr))
1798 Ptr = BC->getOperand(i_nocapture: 0);
1799 if (const auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr)) {
1800 if (GEP->getNumOperands() != 2)
1801 return ScalarCost;
1802 unsigned Scale = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
1803 // Scale needs to be correct (which is only relevant for i16s).
1804 if (Scale != 1 && Scale * 8 != ExtSize)
1805 return ScalarCost;
1806 // And we need to zext (not sext) the indexes from a small enough type.
1807 if (const auto *ZExt = dyn_cast<ZExtInst>(Val: GEP->getOperand(i_nocapture: 1))) {
1808 if (ZExt->getOperand(i_nocapture: 0)->getType()->getScalarSizeInBits() <= ExtSize)
1809 return VectorCost;
1810 }
1811 return ScalarCost;
1812 }
1813 return ScalarCost;
1814}
1815
1816InstructionCost
1817ARMTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
1818 std::optional<FastMathFlags> FMF,
1819 TTI::TargetCostKind CostKind) const {
1820
1821 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1822 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1823 unsigned EltSize = ValVT.getScalarSizeInBits();
1824
1825 // In general floating point reductions are a series of elementwise
1826 // operations, with free extracts on each step. These are either in-order or
1827 // treewise depending on whether that is allowed by the fast math flags.
1828 if ((ISD == ISD::FADD || ISD == ISD::FMUL) &&
1829 ((EltSize == 32 && ST->hasVFP2Base()) ||
1830 (EltSize == 64 && ST->hasFP64()) ||
1831 (EltSize == 16 && ST->hasFullFP16()))) {
1832 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
1833 unsigned VecLimit = ST->hasMVEFloatOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1834 InstructionCost VecCost = 0;
1835 while (!TTI::requiresOrderedReduction(FMF) && isPowerOf2_32(Value: NumElts) &&
1836 NumElts * EltSize > VecLimit) {
1837 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts: NumElts / 2);
1838 VecCost += getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1839 NumElts /= 2;
1840 }
1841
1842 // For fp16 we need to extract the upper lane elements. MVE can add a
1843 // VREV+FMIN/MAX to perform another vector step instead.
1844 InstructionCost ExtractCost = 0;
1845 if (!TTI::requiresOrderedReduction(FMF) && ST->hasMVEFloatOps() &&
1846 ValVT.getVectorElementType() == MVT::f16 && NumElts == 8) {
1847 VecCost += ST->getMVEVectorCostFactor(CostKind) * 2;
1848 NumElts /= 2;
1849 } else if (ValVT.getVectorElementType() == MVT::f16)
1850 ExtractCost = NumElts / 2;
1851
1852 return VecCost + ExtractCost +
1853 NumElts *
1854 getArithmeticInstrCost(Opcode, Ty: ValTy->getElementType(), CostKind);
1855 }
1856
1857 if ((ISD == ISD::AND || ISD == ISD::OR || ISD == ISD::XOR) &&
1858 (EltSize == 64 || EltSize == 32 || EltSize == 16 || EltSize == 8)) {
1859 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
1860 unsigned VecLimit =
1861 ST->hasMVEIntegerOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1862 InstructionCost VecCost = 0;
1863 while (isPowerOf2_32(Value: NumElts) && NumElts * EltSize > VecLimit) {
1864 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts: NumElts / 2);
1865 VecCost += getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1866 NumElts /= 2;
1867 }
1868 // For i16/i8, MVE will perform a VREV + VORR/VAND/VEOR for the 64bit vector
1869 // step.
1870 if (ST->hasMVEIntegerOps() && ValVT.getScalarSizeInBits() <= 16 &&
1871 NumElts * EltSize == 64) {
1872 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts);
1873 VecCost += ST->getMVEVectorCostFactor(CostKind) +
1874 getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1875 NumElts /= 2;
1876 }
1877
1878 // From here we extract the elements and perform the and/or/xor.
1879 InstructionCost ExtractCost = NumElts;
1880 return VecCost + ExtractCost +
1881 (NumElts - 1) * getArithmeticInstrCost(
1882 Opcode, Ty: ValTy->getElementType(), CostKind);
1883 }
1884
1885 if (!ST->hasMVEIntegerOps() || !ValVT.isSimple() || ISD != ISD::ADD ||
1886 TTI::requiresOrderedReduction(FMF))
1887 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
1888
1889 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1890
1891 static const CostTblEntry CostTblAdd[]{
1892 {.ISD: ISD::ADD, .Type: MVT::v16i8, .Cost: 1},
1893 {.ISD: ISD::ADD, .Type: MVT::v8i16, .Cost: 1},
1894 {.ISD: ISD::ADD, .Type: MVT::v4i32, .Cost: 1},
1895 };
1896 if (const auto *Entry = CostTableLookup(Table: CostTblAdd, ISD, Ty: LT.second))
1897 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first;
1898
1899 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
1900}
1901
1902InstructionCost ARMTTIImpl::getExtendedReductionCost(
1903 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
1904 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
1905 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1906 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
1907
1908 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1909
1910 switch (ISD) {
1911 case ISD::ADD:
1912 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) {
1913 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1914
1915 // The legal cases are:
1916 // VADDV u/s 8/16/32
1917 // VADDLV u/s 32
1918 // Codegen currently cannot always handle larger than legal vectors very
1919 // well, especially for predicated reductions where the mask needs to be
1920 // split, so restrict to 128bit or smaller input types.
1921 unsigned RevVTSize = ResVT.getSizeInBits();
1922 if (ValVT.getSizeInBits() <= 128 &&
1923 ((LT.second == MVT::v16i8 && RevVTSize <= 32) ||
1924 (LT.second == MVT::v8i16 && RevVTSize <= 32) ||
1925 (LT.second == MVT::v4i32 && RevVTSize <= 64)))
1926 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1927 }
1928 break;
1929 default:
1930 break;
1931 }
1932 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy, FMF,
1933 CostKind);
1934}
1935
1936InstructionCost
1937ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
1938 Type *ResTy, VectorType *ValTy,
1939 TTI::TargetCostKind CostKind) const {
1940 if (RedOpcode != Instruction::Add)
1941 return InstructionCost::getInvalid(Val: CostKind);
1942 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1943 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
1944
1945 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) {
1946 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1947
1948 // The legal cases are:
1949 // VMLAV u/s 8/16/32
1950 // VMLALV u/s 16/32
1951 // Codegen currently cannot always handle larger than legal vectors very
1952 // well, especially for predicated reductions where the mask needs to be
1953 // split, so restrict to 128bit or smaller input types.
1954 unsigned RevVTSize = ResVT.getSizeInBits();
1955 if (ValVT.getSizeInBits() <= 128 &&
1956 ((LT.second == MVT::v16i8 && RevVTSize <= 32) ||
1957 (LT.second == MVT::v8i16 && RevVTSize <= 64) ||
1958 (LT.second == MVT::v4i32 && RevVTSize <= 64)))
1959 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1960 }
1961
1962 return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty: ValTy,
1963 CostKind);
1964}
1965
1966InstructionCost
1967ARMTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
1968 FastMathFlags FMF,
1969 TTI::TargetCostKind CostKind) const {
1970 EVT ValVT = TLI->getValueType(DL, Ty);
1971
1972 // In general floating point reductions are a series of elementwise
1973 // operations, with free extracts on each step. These are either in-order or
1974 // treewise depending on whether that is allowed by the fast math flags.
1975 if ((IID == Intrinsic::minnum || IID == Intrinsic::maxnum) &&
1976 ((ValVT.getVectorElementType() == MVT::f32 && ST->hasVFP2Base()) ||
1977 (ValVT.getVectorElementType() == MVT::f64 && ST->hasFP64()) ||
1978 (ValVT.getVectorElementType() == MVT::f16 && ST->hasFullFP16()))) {
1979 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
1980 unsigned EltSize = ValVT.getScalarSizeInBits();
1981 unsigned VecLimit = ST->hasMVEFloatOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1982 InstructionCost VecCost;
1983 while (isPowerOf2_32(Value: NumElts) && NumElts * EltSize > VecLimit) {
1984 Type *VecTy = FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElts/2);
1985 IntrinsicCostAttributes ICA(IID, VecTy, {VecTy, VecTy}, FMF);
1986 VecCost += getIntrinsicInstrCost(ICA, CostKind);
1987 NumElts /= 2;
1988 }
1989
1990 // For fp16 we need to extract the upper lane elements. MVE can add a
1991 // VREV+FMIN/MAX to perform another vector step instead.
1992 InstructionCost ExtractCost = 0;
1993 if (ST->hasMVEFloatOps() && ValVT.getVectorElementType() == MVT::f16 &&
1994 NumElts == 8) {
1995 VecCost += ST->getMVEVectorCostFactor(CostKind) * 2;
1996 NumElts /= 2;
1997 } else if (ValVT.getVectorElementType() == MVT::f16)
1998 ExtractCost = cast<FixedVectorType>(Val: Ty)->getNumElements() / 2;
1999
2000 IntrinsicCostAttributes ICA(IID, Ty->getElementType(),
2001 {Ty->getElementType(), Ty->getElementType()},
2002 FMF);
2003 return VecCost + ExtractCost +
2004 (NumElts - 1) * getIntrinsicInstrCost(ICA, CostKind);
2005 }
2006
2007 if (IID == Intrinsic::smin || IID == Intrinsic::smax ||
2008 IID == Intrinsic::umin || IID == Intrinsic::umax) {
2009 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2010
2011 // All costs are the same for u/s min/max. These lower to vminv, which are
2012 // given a slightly higher cost as they tend to take multiple cycles for
2013 // smaller type sizes.
2014 static const CostTblEntry CostTblAdd[]{
2015 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: 4},
2016 {.ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: 3},
2017 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: 2},
2018 };
2019 if (const auto *Entry = CostTableLookup(Table: CostTblAdd, ISD: ISD::SMIN, Ty: LT.second))
2020 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first;
2021 }
2022
2023 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2024}
2025
2026InstructionCost
2027ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
2028 TTI::TargetCostKind CostKind) const {
2029 unsigned Opc = ICA.getID();
2030 switch (Opc) {
2031 case Intrinsic::get_active_lane_mask:
2032 // Currently we make a somewhat optimistic assumption that
2033 // active_lane_mask's are always free. In reality it may be freely folded
2034 // into a tail predicated loop, expanded into a VCPT or expanded into a lot
2035 // of add/icmp code. We may need to improve this in the future, but being
2036 // able to detect if it is free or not involves looking at a lot of other
2037 // code. We currently assume that the vectorizer inserted these, and knew
2038 // what it was doing in adding one.
2039 if (ST->hasMVEIntegerOps())
2040 return 0;
2041 break;
2042 case Intrinsic::sadd_sat:
2043 case Intrinsic::ssub_sat:
2044 case Intrinsic::uadd_sat:
2045 case Intrinsic::usub_sat: {
2046 bool IsAdd = (Opc == Intrinsic::sadd_sat || Opc == Intrinsic::ssub_sat);
2047 bool IsSigned = (Opc == Intrinsic::sadd_sat || Opc == Intrinsic::ssub_sat);
2048 Type *RetTy = ICA.getReturnType();
2049
2050 if (auto *ITy = dyn_cast<IntegerType>(Val: RetTy)) {
2051 if (IsSigned && ST->hasDSP() && ITy->getBitWidth() == 32)
2052 return 1; // qadd / qsub
2053 if (ST->hasDSP() && (ITy->getBitWidth() == 8 || ITy->getBitWidth() == 16))
2054 return 2; // uqadd16 / qadd16 / uqsub16 / qsub16 + possible extend.
2055 // Otherwise return the cost of expanding the node. Generally an add +
2056 // icmp + sel.
2057 CmpInst::Predicate Pred = CmpInst::ICMP_SGT;
2058 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2059 return getArithmeticInstrCost(Opcode: IsAdd ? Instruction::Add : Instruction::Sub,
2060 Ty: RetTy, CostKind) +
2061 2 * getCmpSelInstrCost(Opcode: BinaryOperator::ICmp, ValTy: RetTy, CondTy, VecPred: Pred,
2062 CostKind) +
2063 2 * getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy, VecPred: Pred,
2064 CostKind);
2065 }
2066
2067 if (!ST->hasMVEIntegerOps())
2068 break;
2069
2070 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
2071 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 ||
2072 LT.second == MVT::v16i8) {
2073 // This is a base cost of 1 for the vqadd, plus 3 extract shifts if we
2074 // need to extend the type, as it uses shr(qadd(shl, shl)).
2075 unsigned Instrs =
2076 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1
2077 : 4;
2078 return LT.first * ST->getMVEVectorCostFactor(CostKind) * Instrs;
2079 }
2080 break;
2081 }
2082 case Intrinsic::abs:
2083 case Intrinsic::smin:
2084 case Intrinsic::smax:
2085 case Intrinsic::umin:
2086 case Intrinsic::umax: {
2087 if (!ST->hasMVEIntegerOps())
2088 break;
2089 Type *VT = ICA.getReturnType();
2090
2091 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VT);
2092 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 ||
2093 LT.second == MVT::v16i8)
2094 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2095 break;
2096 }
2097 case Intrinsic::minnum:
2098 case Intrinsic::maxnum: {
2099 if (!ST->hasMVEFloatOps())
2100 break;
2101 Type *VT = ICA.getReturnType();
2102 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VT);
2103 if (LT.second == MVT::v4f32 || LT.second == MVT::v8f16)
2104 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2105 break;
2106 }
2107 case Intrinsic::fptosi_sat:
2108 case Intrinsic::fptoui_sat: {
2109 if (ICA.getArgTypes().empty())
2110 break;
2111 bool IsSigned = Opc == Intrinsic::fptosi_sat;
2112 auto LT = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
2113 EVT MTy = TLI->getValueType(DL, Ty: ICA.getReturnType());
2114 // Check for the legal types, with the correct subtarget features.
2115 if ((ST->hasVFP2Base() && LT.second == MVT::f32 && MTy == MVT::i32) ||
2116 (ST->hasFP64() && LT.second == MVT::f64 && MTy == MVT::i32) ||
2117 (ST->hasFullFP16() && LT.second == MVT::f16 && MTy == MVT::i32))
2118 return LT.first;
2119
2120 // Equally for MVE vector types
2121 if (ST->hasMVEFloatOps() &&
2122 (LT.second == MVT::v4f32 || LT.second == MVT::v8f16) &&
2123 LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits())
2124 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2125
2126 // If we can we use a legal convert followed by a min+max
2127 if (((ST->hasVFP2Base() && LT.second == MVT::f32) ||
2128 (ST->hasFP64() && LT.second == MVT::f64) ||
2129 (ST->hasFullFP16() && LT.second == MVT::f16) ||
2130 (ST->hasMVEFloatOps() &&
2131 (LT.second == MVT::v4f32 || LT.second == MVT::v8f16))) &&
2132 LT.second.getScalarSizeInBits() >= MTy.getScalarSizeInBits()) {
2133 Type *LegalTy = Type::getIntNTy(C&: ICA.getReturnType()->getContext(),
2134 N: LT.second.getScalarSizeInBits());
2135 InstructionCost Cost =
2136 LT.second.isVector() ? ST->getMVEVectorCostFactor(CostKind) : 1;
2137 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin
2138 : Intrinsic::umin,
2139 LegalTy, {LegalTy, LegalTy});
2140 Cost += getIntrinsicInstrCost(ICA: Attrs1, CostKind);
2141 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax
2142 : Intrinsic::umax,
2143 LegalTy, {LegalTy, LegalTy});
2144 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
2145 return LT.first * Cost;
2146 }
2147 // Otherwise we need to follow the default expansion that clamps the value
2148 // using a float min/max with a fcmp+sel for nan handling when signed.
2149 Type *FPTy = ICA.getArgTypes()[0];
2150 Type *RetTy = ICA.getReturnType();
2151 IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FPTy, {FPTy, FPTy});
2152 InstructionCost Cost = getIntrinsicInstrCost(ICA: Attrs1, CostKind);
2153 IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FPTy, {FPTy, FPTy});
2154 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
2155 Cost +=
2156 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
2157 Dst: RetTy, Src: FPTy, CCH: TTI::CastContextHint::None, CostKind);
2158 if (IsSigned) {
2159 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2160 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: FPTy, CondTy,
2161 VecPred: CmpInst::FCMP_UNO, CostKind);
2162 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
2163 VecPred: CmpInst::FCMP_UNO, CostKind);
2164 }
2165 return Cost;
2166 }
2167 }
2168
2169 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
2170}
2171
2172bool ARMTTIImpl::isLoweredToCall(const Function *F) const {
2173 if (!F->isIntrinsic())
2174 return BaseT::isLoweredToCall(F);
2175
2176 // Assume all Arm-specific intrinsics map to an instruction.
2177 if (F->getName().starts_with(Prefix: "llvm.arm"))
2178 return false;
2179
2180 switch (F->getIntrinsicID()) {
2181 default: break;
2182 case Intrinsic::powi:
2183 case Intrinsic::sin:
2184 case Intrinsic::cos:
2185 case Intrinsic::sincos:
2186 case Intrinsic::pow:
2187 case Intrinsic::log:
2188 case Intrinsic::log10:
2189 case Intrinsic::log2:
2190 case Intrinsic::exp:
2191 case Intrinsic::exp2:
2192 return true;
2193 case Intrinsic::sqrt:
2194 case Intrinsic::fabs:
2195 case Intrinsic::copysign:
2196 case Intrinsic::floor:
2197 case Intrinsic::ceil:
2198 case Intrinsic::trunc:
2199 case Intrinsic::rint:
2200 case Intrinsic::nearbyint:
2201 case Intrinsic::round:
2202 case Intrinsic::canonicalize:
2203 case Intrinsic::lround:
2204 case Intrinsic::llround:
2205 case Intrinsic::lrint:
2206 case Intrinsic::llrint:
2207 if (F->getReturnType()->isDoubleTy() && !ST->hasFP64())
2208 return true;
2209 if (F->getReturnType()->isHalfTy() && !ST->hasFullFP16())
2210 return true;
2211 // Some operations can be handled by vector instructions and assume
2212 // unsupported vectors will be expanded into supported scalar ones.
2213 // TODO Handle scalar operations properly.
2214 return !ST->hasFPARMv8Base() && !ST->hasVFP2Base();
2215 case Intrinsic::masked_store:
2216 case Intrinsic::masked_load:
2217 case Intrinsic::masked_gather:
2218 case Intrinsic::masked_scatter:
2219 return !ST->hasMVEIntegerOps();
2220 case Intrinsic::sadd_with_overflow:
2221 case Intrinsic::uadd_with_overflow:
2222 case Intrinsic::ssub_with_overflow:
2223 case Intrinsic::usub_with_overflow:
2224 case Intrinsic::sadd_sat:
2225 case Intrinsic::uadd_sat:
2226 case Intrinsic::ssub_sat:
2227 case Intrinsic::usub_sat:
2228 return false;
2229 }
2230
2231 return BaseT::isLoweredToCall(F);
2232}
2233
2234bool ARMTTIImpl::maybeLoweredToCall(Instruction &I) const {
2235 unsigned ISD = TLI->InstructionOpcodeToISD(Opcode: I.getOpcode());
2236 EVT VT = TLI->getValueType(DL, Ty: I.getType(), AllowUnknown: true);
2237 if (TLI->getOperationAction(Op: ISD, VT) == TargetLowering::LibCall)
2238 return true;
2239
2240 // Check if an intrinsic will be lowered to a call and assume that any
2241 // other CallInst will generate a bl.
2242 if (auto *Call = dyn_cast<CallInst>(Val: &I)) {
2243 if (auto *II = dyn_cast<IntrinsicInst>(Val: Call)) {
2244 switch(II->getIntrinsicID()) {
2245 case Intrinsic::memcpy:
2246 case Intrinsic::memset:
2247 case Intrinsic::memmove:
2248 return getNumMemOps(I: II) == -1;
2249 default:
2250 if (const Function *F = Call->getCalledFunction())
2251 return isLoweredToCall(F);
2252 }
2253 }
2254 return true;
2255 }
2256
2257 // FPv5 provides conversions between integer, double-precision,
2258 // single-precision, and half-precision formats.
2259 switch (I.getOpcode()) {
2260 default:
2261 break;
2262 case Instruction::FPToSI:
2263 case Instruction::FPToUI:
2264 case Instruction::SIToFP:
2265 case Instruction::UIToFP:
2266 case Instruction::FPTrunc:
2267 case Instruction::FPExt:
2268 return !ST->hasFPARMv8Base();
2269 }
2270
2271 // FIXME: Unfortunately the approach of checking the Operation Action does
2272 // not catch all cases of Legalization that use library calls. Our
2273 // Legalization step categorizes some transformations into library calls as
2274 // Custom, Expand or even Legal when doing type legalization. So for now
2275 // we have to special case for instance the SDIV of 64bit integers and the
2276 // use of floating point emulation.
2277 if (VT.isInteger() && VT.getSizeInBits() >= 64) {
2278 switch (ISD) {
2279 default:
2280 break;
2281 case ISD::SDIV:
2282 case ISD::UDIV:
2283 case ISD::SREM:
2284 case ISD::UREM:
2285 case ISD::SDIVREM:
2286 case ISD::UDIVREM:
2287 return true;
2288 }
2289 }
2290
2291 // Assume all other non-float operations are supported.
2292 if (!VT.isFloatingPoint())
2293 return false;
2294
2295 // We'll need a library call to handle most floats when using soft.
2296 if (TLI->useSoftFloat()) {
2297 switch (I.getOpcode()) {
2298 default:
2299 return true;
2300 case Instruction::Alloca:
2301 case Instruction::Load:
2302 case Instruction::Store:
2303 case Instruction::Select:
2304 case Instruction::PHI:
2305 return false;
2306 }
2307 }
2308
2309 // We'll need a libcall to perform double precision operations on a single
2310 // precision only FPU.
2311 if (I.getType()->isDoubleTy() && !ST->hasFP64())
2312 return true;
2313
2314 // Likewise for half precision arithmetic.
2315 if (I.getType()->isHalfTy() && !ST->hasFullFP16())
2316 return true;
2317
2318 return false;
2319}
2320
2321bool ARMTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
2322 AssumptionCache &AC,
2323 TargetLibraryInfo *LibInfo,
2324 HardwareLoopInfo &HWLoopInfo) const {
2325 // Low-overhead branches are only supported in the 'low-overhead branch'
2326 // extension of v8.1-m.
2327 if (!ST->hasLOB() || DisableLowOverheadLoops) {
2328 LLVM_DEBUG(dbgs() << "ARMHWLoops: Disabled\n");
2329 return false;
2330 }
2331
2332 if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
2333 LLVM_DEBUG(dbgs() << "ARMHWLoops: No BETC\n");
2334 return false;
2335 }
2336
2337 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2338 if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount)) {
2339 LLVM_DEBUG(dbgs() << "ARMHWLoops: Uncomputable BETC\n");
2340 return false;
2341 }
2342
2343 const SCEV *TripCountSCEV =
2344 SE.getAddExpr(LHS: BackedgeTakenCount,
2345 RHS: SE.getOne(Ty: BackedgeTakenCount->getType()));
2346
2347 // We need to store the trip count in LR, a 32-bit register.
2348 if (SE.getUnsignedRangeMax(S: TripCountSCEV).getBitWidth() > 32) {
2349 LLVM_DEBUG(dbgs() << "ARMHWLoops: Trip count does not fit into 32bits\n");
2350 return false;
2351 }
2352
2353 // Making a call will trash LR and clear LO_BRANCH_INFO, so there's little
2354 // point in generating a hardware loop if that's going to happen.
2355
2356 auto IsHardwareLoopIntrinsic = [](Instruction &I) {
2357 if (auto *Call = dyn_cast<IntrinsicInst>(Val: &I)) {
2358 switch (Call->getIntrinsicID()) {
2359 default:
2360 break;
2361 case Intrinsic::start_loop_iterations:
2362 case Intrinsic::test_start_loop_iterations:
2363 case Intrinsic::loop_decrement:
2364 case Intrinsic::loop_decrement_reg:
2365 return true;
2366 }
2367 }
2368 return false;
2369 };
2370
2371 // Scan the instructions to see if there's any that we know will turn into a
2372 // call or if this loop is already a low-overhead loop or will become a tail
2373 // predicated loop.
2374 bool IsTailPredLoop = false;
2375 auto ScanLoop = [&](Loop *L) {
2376 for (auto *BB : L->getBlocks()) {
2377 for (auto &I : *BB) {
2378 if (maybeLoweredToCall(I) || IsHardwareLoopIntrinsic(I) ||
2379 isa<InlineAsm>(Val: I)) {
2380 LLVM_DEBUG(dbgs() << "ARMHWLoops: Bad instruction: " << I << "\n");
2381 return false;
2382 }
2383 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
2384 IsTailPredLoop |=
2385 II->getIntrinsicID() == Intrinsic::get_active_lane_mask ||
2386 II->getIntrinsicID() == Intrinsic::arm_mve_vctp8 ||
2387 II->getIntrinsicID() == Intrinsic::arm_mve_vctp16 ||
2388 II->getIntrinsicID() == Intrinsic::arm_mve_vctp32 ||
2389 II->getIntrinsicID() == Intrinsic::arm_mve_vctp64;
2390 }
2391 }
2392 return true;
2393 };
2394
2395 // Visit inner loops.
2396 for (auto *Inner : *L)
2397 if (!ScanLoop(Inner))
2398 return false;
2399
2400 if (!ScanLoop(L))
2401 return false;
2402
2403 // TODO: Check whether the trip count calculation is expensive. If L is the
2404 // inner loop but we know it has a low trip count, calculating that trip
2405 // count (in the parent loop) may be detrimental.
2406
2407 LLVMContext &C = L->getHeader()->getContext();
2408 HWLoopInfo.CounterInReg = true;
2409 HWLoopInfo.IsNestingLegal = false;
2410 HWLoopInfo.PerformEntryTest = AllowWLSLoops && !IsTailPredLoop;
2411 HWLoopInfo.CountType = Type::getInt32Ty(C);
2412 HWLoopInfo.LoopDecrement = ConstantInt::get(Ty: HWLoopInfo.CountType, V: 1);
2413 return true;
2414}
2415
2416static bool canTailPredicateInstruction(Instruction &I, int &ICmpCount) {
2417 // We don't allow icmp's, and because we only look at single block loops,
2418 // we simply count the icmps, i.e. there should only be 1 for the backedge.
2419 if (isa<ICmpInst>(Val: &I) && ++ICmpCount > 1)
2420 return false;
2421 // FIXME: This is a workaround for poor cost modelling. Min/Max intrinsics are
2422 // not currently canonical, but soon will be. Code without them uses icmp, and
2423 // so is not tail predicated as per the condition above. In order to get the
2424 // same performance we treat min and max the same as an icmp for tailpred
2425 // purposes for the moment (we often rely on non-tailpred and higher VF's to
2426 // pick more optimal instructions like VQDMULH. They need to be recognized
2427 // directly by the vectorizer).
2428 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
2429 if ((II->getIntrinsicID() == Intrinsic::smin ||
2430 II->getIntrinsicID() == Intrinsic::smax ||
2431 II->getIntrinsicID() == Intrinsic::umin ||
2432 II->getIntrinsicID() == Intrinsic::umax) &&
2433 ++ICmpCount > 1)
2434 return false;
2435
2436 if (isa<FCmpInst>(Val: &I))
2437 return false;
2438
2439 // We could allow extending/narrowing FP loads/stores, but codegen is
2440 // too inefficient so reject this for now.
2441 if (isa<FPExtInst>(Val: &I) || isa<FPTruncInst>(Val: &I))
2442 return false;
2443
2444 // Extends have to be extending-loads
2445 if (isa<SExtInst>(Val: &I) || isa<ZExtInst>(Val: &I) )
2446 if (!I.getOperand(i: 0)->hasOneUse() || !isa<LoadInst>(Val: I.getOperand(i: 0)))
2447 return false;
2448
2449 // Truncs have to be narrowing-stores
2450 if (isa<TruncInst>(Val: &I) )
2451 if (!I.hasOneUse() || !isa<StoreInst>(Val: *I.user_begin()))
2452 return false;
2453
2454 return true;
2455}
2456
2457// To set up a tail-predicated loop, we need to know the total number of
2458// elements processed by that loop. Thus, we need to determine the element
2459// size and:
2460// 1) it should be uniform for all operations in the vector loop, so we
2461// e.g. don't want any widening/narrowing operations.
2462// 2) it should be smaller than i64s because we don't have vector operations
2463// that work on i64s.
2464// 3) we don't want elements to be reversed or shuffled, to make sure the
2465// tail-predication masks/predicates the right lanes.
2466//
2467static bool canTailPredicateLoop(Loop *L, LoopInfo *LI, ScalarEvolution &SE,
2468 const DataLayout &DL,
2469 const LoopAccessInfo *LAI,
2470 const DominatorTree &DT) {
2471 LLVM_DEBUG(dbgs() << "Tail-predication: checking allowed instructions\n");
2472
2473 // If there are live-out values, it is probably a reduction. We can predicate
2474 // most reduction operations freely under MVE using a combination of
2475 // prefer-predicated-reduction-select and inloop reductions. We limit this to
2476 // floating point and integer reductions, but don't check for operators
2477 // specifically here. If the value ends up not being a reduction (and so the
2478 // vectorizer cannot tailfold the loop), we should fall back to standard
2479 // vectorization automatically.
2480 SmallVector< Instruction *, 8 > LiveOuts;
2481 LiveOuts = llvm::findDefsUsedOutsideOfLoop(L);
2482 bool ReductionsDisabled =
2483 EnableTailPredication == TailPredication::EnabledNoReductions ||
2484 EnableTailPredication == TailPredication::ForceEnabledNoReductions;
2485
2486 for (auto *I : LiveOuts) {
2487 if (!I->getType()->isIntegerTy() && !I->getType()->isFloatTy() &&
2488 !I->getType()->isHalfTy()) {
2489 LLVM_DEBUG(dbgs() << "Don't tail-predicate loop with non-integer/float "
2490 "live-out value\n");
2491 return false;
2492 }
2493 if (ReductionsDisabled) {
2494 LLVM_DEBUG(dbgs() << "Reductions not enabled\n");
2495 return false;
2496 }
2497 }
2498
2499 // Next, check that all instructions can be tail-predicated.
2500 PredicatedScalarEvolution PSE = LAI->getPSE();
2501 int ICmpCount = 0;
2502
2503 for (BasicBlock *BB : L->blocks()) {
2504 for (Instruction &I : *BB) {
2505 if (isa<PHINode>(Val: I) || isa<PseudoProbeInst>(Val: I))
2506 continue;
2507 if (!canTailPredicateInstruction(I, ICmpCount)) {
2508 LLVM_DEBUG(dbgs() << "Instruction not allowed: "; I.dump());
2509 return false;
2510 }
2511
2512 Type *T = I.getType();
2513 if (T->getScalarSizeInBits() > 32) {
2514 LLVM_DEBUG(dbgs() << "Unsupported Type: "; T->dump());
2515 return false;
2516 }
2517 if (isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I)) {
2518 Value *Ptr = getLoadStorePointerOperand(V: &I);
2519 Type *AccessTy = getLoadStoreType(I: &I);
2520 int64_t NextStride =
2521 getPtrStride(PSE, AccessTy, Ptr, Lp: L, DT).value_or(u: 0);
2522 if (NextStride == 1) {
2523 // TODO: for now only allow consecutive strides of 1. We could support
2524 // other strides as long as it is uniform, but let's keep it simple
2525 // for now.
2526 continue;
2527 } else if (NextStride == -1 ||
2528 (NextStride == 2 && MVEMaxSupportedInterleaveFactor >= 2) ||
2529 (NextStride == 4 && MVEMaxSupportedInterleaveFactor >= 4)) {
2530 LLVM_DEBUG(dbgs()
2531 << "Consecutive strides of 2 found, vld2/vstr2 can't "
2532 "be tail-predicated\n.");
2533 return false;
2534 // TODO: don't tail predicate if there is a reversed load?
2535 } else if (EnableMaskedGatherScatters) {
2536 // Gather/scatters do allow loading from arbitrary strides, at
2537 // least if they are loop invariant.
2538 // TODO: Loop variant strides should in theory work, too, but
2539 // this requires further testing.
2540 const SCEV *PtrScev = PSE.getSE()->getSCEV(V: Ptr);
2541 if (auto AR = dyn_cast<SCEVAddRecExpr>(Val: PtrScev)) {
2542 const SCEV *Step = AR->getStepRecurrence(SE&: *PSE.getSE());
2543 if (PSE.getSE()->isLoopInvariant(S: Step, L))
2544 continue;
2545 }
2546 }
2547 LLVM_DEBUG(dbgs() << "Bad stride found, can't "
2548 "tail-predicate\n.");
2549 return false;
2550 }
2551 }
2552 }
2553
2554 LLVM_DEBUG(dbgs() << "tail-predication: all instructions allowed!\n");
2555 return true;
2556}
2557
2558bool ARMTTIImpl::preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
2559 if (!EnableTailPredication) {
2560 LLVM_DEBUG(dbgs() << "Tail-folding not enabled.\n");
2561 return false;
2562 }
2563
2564 // Creating a tail-folded vector loop is the first step for generating a
2565 // tail-folded hardware loop, for which we need the MVE masked
2566 // load/stores instructions:
2567 if (!ST->hasMVEIntegerOps())
2568 return false;
2569
2570 LoopVectorizationLegality *LVL = TFI->LVL;
2571 Loop *L = LVL->getLoop();
2572
2573 // For now, restrict this to single block loops.
2574 if (L->getNumBlocks() > 1) {
2575 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: not a single block "
2576 "loop.\n");
2577 return false;
2578 }
2579
2580 assert(L->isInnermost() &&
2581 "preferTailFoldingOverEpilogue: inner-loop expected");
2582
2583 LoopInfo *LI = LVL->getLoopInfo();
2584 HardwareLoopInfo HWLoopInfo(L);
2585 if (!HWLoopInfo.canAnalyze(LI&: *LI)) {
2586 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2587 "analyzable.\n");
2588 return false;
2589 }
2590
2591 AssumptionCache *AC = LVL->getAssumptionCache();
2592 ScalarEvolution *SE = LVL->getScalarEvolution();
2593
2594 // This checks if we have the low-overhead branch architecture
2595 // extension, and if we will create a hardware-loop:
2596 if (!isHardwareLoopProfitable(L, SE&: *SE, AC&: *AC, LibInfo: TFI->TLI, HWLoopInfo)) {
2597 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2598 "profitable.\n");
2599 return false;
2600 }
2601
2602 DominatorTree *DT = LVL->getDominatorTree();
2603 if (!HWLoopInfo.isHardwareLoopCandidate(SE&: *SE, LI&: *LI, DT&: *DT)) {
2604 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2605 "a candidate.\n");
2606 return false;
2607 }
2608
2609 return canTailPredicateLoop(L, LI, SE&: *SE, DL, LAI: LVL->getLAI(),
2610 DT: *LVL->getDominatorTree());
2611}
2612
2613TailFoldingStyle ARMTTIImpl::getPreferredTailFoldingStyle() const {
2614 if (!ST->hasMVEIntegerOps() || !EnableTailPredication)
2615 return TailFoldingStyle::DataWithoutLaneMask;
2616
2617 // Intrinsic @llvm.get.active.lane.mask is supported.
2618 // It is used in the MVETailPredication pass, which requires the number of
2619 // elements processed by this vector loop to setup the tail-predicated
2620 // loop.
2621 return TailFoldingStyle::Data;
2622}
2623void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
2624 TTI::UnrollingPreferences &UP,
2625 OptimizationRemarkEmitter *ORE) const {
2626 // Enable Upper bound unrolling universally, providing that we do not see an
2627 // active lane mask, which will be better kept as a loop to become tail
2628 // predicated than to be conditionally unrolled.
2629 UP.UpperBound =
2630 !ST->hasMVEIntegerOps() || !any_of(Range&: *L->getHeader(), P: [](Instruction &I) {
2631 return isa<IntrinsicInst>(Val: I) &&
2632 cast<IntrinsicInst>(Val&: I).getIntrinsicID() ==
2633 Intrinsic::get_active_lane_mask;
2634 });
2635
2636 // Only currently enable these preferences for M-Class cores.
2637 if (!ST->isMClass())
2638 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
2639
2640 // Disable loop unrolling for Oz and Os.
2641 UP.OptSizeThreshold = 0;
2642 UP.PartialOptSizeThreshold = 0;
2643 if (L->getHeader()->getParent()->hasOptSize())
2644 return;
2645
2646 SmallVector<BasicBlock*, 4> ExitingBlocks;
2647 L->getExitingBlocks(ExitingBlocks);
2648 LLVM_DEBUG(dbgs() << "Loop has:\n"
2649 << "Blocks: " << L->getNumBlocks() << "\n"
2650 << "Exit blocks: " << ExitingBlocks.size() << "\n");
2651
2652 // Only allow another exit other than the latch. This acts as an early exit
2653 // as it mirrors the profitability calculation of the runtime unroller.
2654 if (ExitingBlocks.size() > 2)
2655 return;
2656
2657 // Limit the CFG of the loop body for targets with a branch predictor.
2658 // Allowing 4 blocks permits if-then-else diamonds in the body.
2659 if (ST->hasBranchPredictor() && L->getNumBlocks() > 4)
2660 return;
2661
2662 // Don't unroll vectorized loops, including the remainder loop
2663 if (getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized"))
2664 return;
2665
2666 // Scan the loop: don't unroll loops with calls as this could prevent
2667 // inlining.
2668 InstructionCost Cost = 0;
2669 for (auto *BB : L->getBlocks()) {
2670 for (auto &I : *BB) {
2671 // Don't unroll vectorised loop. MVE does not benefit from it as much as
2672 // scalar code.
2673 if (I.getType()->isVectorTy())
2674 return;
2675
2676 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
2677 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction()) {
2678 if (!isLoweredToCall(F))
2679 continue;
2680 }
2681 return;
2682 }
2683
2684 SmallVector<const Value*, 4> Operands(I.operand_values());
2685 Cost += getInstructionCost(U: &I, Operands,
2686 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
2687 }
2688 }
2689
2690 // On v6m cores, there are very few registers available. We can easily end up
2691 // spilling and reloading more registers in an unrolled loop. Look at the
2692 // number of LCSSA phis as a rough measure of how many registers will need to
2693 // be live out of the loop, reducing the default unroll count if more than 1
2694 // value is needed. In the long run, all of this should be being learnt by a
2695 // machine.
2696 unsigned UnrollCount = 4;
2697 if (ST->isThumb1Only()) {
2698 unsigned ExitingValues = 0;
2699 SmallVector<BasicBlock *, 4> ExitBlocks;
2700 L->getExitBlocks(ExitBlocks);
2701 for (auto *Exit : ExitBlocks) {
2702 // Count the number of LCSSA phis. Exclude values coming from GEP's as
2703 // only the last is expected to be needed for address operands.
2704 unsigned LiveOuts = count_if(Range: Exit->phis(), P: [](auto &PH) {
2705 return PH.getNumOperands() != 1 ||
2706 !isa<GetElementPtrInst>(PH.getOperand(0));
2707 });
2708 ExitingValues = ExitingValues < LiveOuts ? LiveOuts : ExitingValues;
2709 }
2710 if (ExitingValues)
2711 UnrollCount /= ExitingValues;
2712 if (UnrollCount <= 1)
2713 return;
2714 }
2715
2716 // For processors with low overhead branching (LOB), runtime unrolling the
2717 // innermost loop is often detrimental to performance. In these cases the loop
2718 // remainder gets unrolled into a series of compare-and-jump blocks, which in
2719 // deeply nested loops get executed multiple times, negating the benefits of
2720 // LOB. This is particularly noticeable when the loop trip count of the
2721 // innermost loop varies within the outer loop, such as in the case of
2722 // triangular matrix decompositions. In these cases we will prefer to not
2723 // unroll the innermost loop, with the intention for it to be executed as a
2724 // low overhead loop.
2725 bool Runtime = true;
2726 if (ST->hasLOB()) {
2727 if (SE.hasLoopInvariantBackedgeTakenCount(L)) {
2728 const SCEV *BETC = SE.getBackedgeTakenCount(L);
2729 auto *Outer = L->getOutermostLoop();
2730 if ((L != Outer && Outer != L->getParentLoop()) ||
2731 (L != Outer && BETC && !SE.isLoopInvariant(S: BETC, L: Outer))) {
2732 Runtime = false;
2733 }
2734 }
2735 }
2736
2737 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
2738 LLVM_DEBUG(dbgs() << "Default Runtime Unroll Count: " << UnrollCount << "\n");
2739
2740 UP.Partial = true;
2741 UP.Runtime = Runtime;
2742 UP.UnrollRemainder = true;
2743 UP.DefaultUnrollRuntimeCount = UnrollCount;
2744 UP.UnrollAndJam = true;
2745 UP.UnrollAndJamInnerLoopThreshold = 60;
2746
2747 // Force unrolling small loops can be very useful because of the branch
2748 // taken cost of the backedge.
2749 if (Cost < ArmForceUnrollThreshold)
2750 UP.Force = true;
2751}
2752
2753void ARMTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
2754 TTI::PeelingPreferences &PP) const {
2755 BaseT::getPeelingPreferences(L, SE, PP);
2756}
2757
2758bool ARMTTIImpl::preferInLoopReduction(RecurKind Kind, Type *Ty) const {
2759 if (!ST->hasMVEIntegerOps())
2760 return false;
2761
2762 unsigned ScalarBits = Ty->getScalarSizeInBits();
2763 switch (Kind) {
2764 case RecurKind::Add:
2765 return ScalarBits <= 64;
2766 default:
2767 return false;
2768 }
2769}
2770
2771bool ARMTTIImpl::preferPredicatedReductionSelect() const {
2772 if (!ST->hasMVEIntegerOps())
2773 return false;
2774 return true;
2775}
2776
2777InstructionCost ARMTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
2778 StackOffset BaseOffset,
2779 bool HasBaseReg, int64_t Scale,
2780 unsigned AddrSpace) const {
2781 TargetLoweringBase::AddrMode AM;
2782 AM.BaseGV = BaseGV;
2783 AM.BaseOffs = BaseOffset.getFixed();
2784 AM.HasBaseReg = HasBaseReg;
2785 AM.Scale = Scale;
2786 AM.ScalableOffset = BaseOffset.getScalable();
2787 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AS: AddrSpace)) {
2788 if (ST->hasFPAO())
2789 return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster
2790 return 0;
2791 }
2792 return InstructionCost::getInvalid();
2793}
2794
2795bool ARMTTIImpl::shouldConsiderVectorizationRegPressure() const {
2796 // MVE only has 8 vector registers, so we should consider register pressure to
2797 // avoid vectorizing when the cost of spills exceeds the gains from
2798 // vectorization.
2799 return ST->hasMVEIntegerOps();
2800}
2801
2802bool ARMTTIImpl::hasArmWideBranch(bool Thumb) const {
2803 if (Thumb) {
2804 // B.W is available in any Thumb2-supporting target, and also in every
2805 // version of Armv8-M, even Baseline which does not include the rest of
2806 // Thumb2.
2807 return ST->isThumb2() || ST->hasV8MBaselineOps();
2808 } else {
2809 // B is available in all versions of the Arm ISA, so the only question is
2810 // whether that ISA is available at all.
2811 return ST->hasARMOps();
2812 }
2813}
2814
2815/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
2816/// of the vector elements.
2817static bool areExtractExts(Value *Ext1, Value *Ext2) {
2818 using namespace PatternMatch;
2819
2820 auto areExtDoubled = [](Instruction *Ext) {
2821 return Ext->getType()->getScalarSizeInBits() ==
2822 2 * Ext->getOperand(i: 0)->getType()->getScalarSizeInBits();
2823 };
2824
2825 if (!match(V: Ext1, P: m_ZExtOrSExt(Op: m_Value())) ||
2826 !match(V: Ext2, P: m_ZExtOrSExt(Op: m_Value())) ||
2827 !areExtDoubled(cast<Instruction>(Val: Ext1)) ||
2828 !areExtDoubled(cast<Instruction>(Val: Ext2)))
2829 return false;
2830
2831 return true;
2832}
2833
2834/// Check if sinking \p I's operands to I's basic block is profitable, because
2835/// the operands can be folded into a target instruction, e.g.
2836/// sext/zext can be folded into vsubl.
2837bool ARMTTIImpl::isProfitableToSinkOperands(Instruction *I,
2838 SmallVectorImpl<Use *> &Ops) const {
2839 using namespace PatternMatch;
2840
2841 if (!I->getType()->isVectorTy())
2842 return false;
2843
2844 if (ST->hasNEON()) {
2845 switch (I->getOpcode()) {
2846 case Instruction::Sub:
2847 case Instruction::Add: {
2848 if (!areExtractExts(Ext1: I->getOperand(i: 0), Ext2: I->getOperand(i: 1)))
2849 return false;
2850 Ops.push_back(Elt: &I->getOperandUse(i: 0));
2851 Ops.push_back(Elt: &I->getOperandUse(i: 1));
2852 return true;
2853 }
2854 default:
2855 return false;
2856 }
2857 }
2858
2859 if (!ST->hasMVEIntegerOps())
2860 return false;
2861
2862 auto IsFMSMul = [&](Instruction *I) {
2863 if (!I->hasOneUse())
2864 return false;
2865 auto *Sub = cast<Instruction>(Val: *I->users().begin());
2866 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(i: 1) == I;
2867 };
2868 auto IsFMS = [&](Instruction *I) {
2869 if (match(V: I->getOperand(i: 0), P: m_FNeg(X: m_Value())) ||
2870 match(V: I->getOperand(i: 1), P: m_FNeg(X: m_Value())))
2871 return true;
2872 return false;
2873 };
2874
2875 auto IsSinker = [&](Instruction *I, int Operand) {
2876 switch (I->getOpcode()) {
2877 case Instruction::Add:
2878 case Instruction::Mul:
2879 case Instruction::FAdd:
2880 case Instruction::ICmp:
2881 case Instruction::FCmp:
2882 return true;
2883 case Instruction::FMul:
2884 return !IsFMSMul(I);
2885 case Instruction::Sub:
2886 case Instruction::FSub:
2887 case Instruction::Shl:
2888 case Instruction::LShr:
2889 case Instruction::AShr:
2890 return Operand == 1;
2891 case Instruction::Call:
2892 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2893 switch (II->getIntrinsicID()) {
2894 case Intrinsic::fma:
2895 return !IsFMS(I);
2896 case Intrinsic::sadd_sat:
2897 case Intrinsic::uadd_sat:
2898 case Intrinsic::arm_mve_add_predicated:
2899 case Intrinsic::arm_mve_mul_predicated:
2900 case Intrinsic::arm_mve_qadd_predicated:
2901 case Intrinsic::arm_mve_vhadd:
2902 case Intrinsic::arm_mve_hadd_predicated:
2903 case Intrinsic::arm_mve_vqdmull:
2904 case Intrinsic::arm_mve_vqdmull_predicated:
2905 case Intrinsic::arm_mve_vqdmulh:
2906 case Intrinsic::arm_mve_qdmulh_predicated:
2907 case Intrinsic::arm_mve_vqrdmulh:
2908 case Intrinsic::arm_mve_qrdmulh_predicated:
2909 case Intrinsic::arm_mve_fma_predicated:
2910 return true;
2911 case Intrinsic::ssub_sat:
2912 case Intrinsic::usub_sat:
2913 case Intrinsic::arm_mve_sub_predicated:
2914 case Intrinsic::arm_mve_qsub_predicated:
2915 case Intrinsic::arm_mve_hsub_predicated:
2916 case Intrinsic::arm_mve_vhsub:
2917 return Operand == 1;
2918 default:
2919 return false;
2920 }
2921 }
2922 return false;
2923 default:
2924 return false;
2925 }
2926 };
2927
2928 for (auto OpIdx : enumerate(First: I->operands())) {
2929 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
2930 // Make sure we are not already sinking this operand
2931 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
2932 continue;
2933
2934 Instruction *Shuffle = Op;
2935 if (Shuffle->getOpcode() == Instruction::BitCast)
2936 Shuffle = dyn_cast<Instruction>(Val: Shuffle->getOperand(i: 0));
2937 // We are looking for a splat that can be sunk.
2938 if (!Shuffle || !match(V: Shuffle, P: m_Shuffle(v1: m_InsertElt(Val: m_Undef(), Elt: m_Value(),
2939 Idx: m_ZeroInt()),
2940 v2: m_Undef(), mask: m_ZeroMask())))
2941 continue;
2942 if (!IsSinker(I, OpIdx.index()))
2943 continue;
2944
2945 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
2946 // and vector registers
2947 for (Use &U : Op->uses()) {
2948 Instruction *Insn = cast<Instruction>(Val: U.getUser());
2949 if (!IsSinker(Insn, U.getOperandNo()))
2950 return false;
2951 }
2952
2953 Ops.push_back(Elt: &Shuffle->getOperandUse(i: 0));
2954 if (Shuffle != Op)
2955 Ops.push_back(Elt: &Op->getOperandUse(i: 0));
2956 Ops.push_back(Elt: &OpIdx.value());
2957 }
2958 return true;
2959}
2960
2961unsigned ARMTTIImpl::getNumBytesToPadGlobalArray(unsigned Size,
2962 Type *ArrayType) const {
2963 if (!UseWidenGlobalArrays) {
2964 LLVM_DEBUG(dbgs() << "Padding global arrays disabled\n");
2965 return false;
2966 }
2967
2968 // Don't modify none integer array types
2969 if (!ArrayType || !ArrayType->isArrayTy() ||
2970 !ArrayType->getArrayElementType()->isIntegerTy())
2971 return 0;
2972
2973 // We pad to 4 byte boundaries
2974 if (Size % 4 == 0)
2975 return 0;
2976
2977 unsigned NumBytesToPad = 4 - (Size % 4);
2978 unsigned NewSize = Size + NumBytesToPad;
2979
2980 // Max number of bytes that memcpy allows for lowering to load/stores before
2981 // it uses library function (__aeabi_memcpy).
2982 unsigned MaxMemIntrinsicSize = getMaxMemIntrinsicInlineSizeThreshold();
2983
2984 if (NewSize > MaxMemIntrinsicSize)
2985 return 0;
2986
2987 return NumBytesToPad;
2988}
2989