1//===- ARMTargetTransformInfo.cpp - ARM specific TTI ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ARMTargetTransformInfo.h"
10#include "ARMSubtarget.h"
11#include "MCTargetDesc/ARMAddressingModes.h"
12#include "llvm/ADT/APInt.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Analysis/LoopInfo.h"
15#include "llvm/CodeGen/CostTable.h"
16#include "llvm/CodeGen/ISDOpcodes.h"
17#include "llvm/CodeGen/ValueTypes.h"
18#include "llvm/CodeGenTypes/MachineValueType.h"
19#include "llvm/IR/BasicBlock.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/IR/DerivedTypes.h"
22#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/IntrinsicsARM.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Target/TargetMachine.h"
32#include "llvm/TargetParser/SubtargetFeature.h"
33#include "llvm/Transforms/InstCombine/InstCombiner.h"
34#include "llvm/Transforms/Utils/Local.h"
35#include "llvm/Transforms/Utils/LoopUtils.h"
36#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
37#include <algorithm>
38#include <cassert>
39#include <cstdint>
40#include <optional>
41#include <utility>
42
43using namespace llvm;
44
45#define DEBUG_TYPE "armtti"
46
47static cl::opt<bool> EnableMaskedLoadStores(
48 "enable-arm-maskedldst", cl::Hidden, cl::init(Val: true),
49 cl::desc("Enable the generation of masked loads and stores"));
50
51static cl::opt<bool> DisableLowOverheadLoops(
52 "disable-arm-loloops", cl::Hidden, cl::init(Val: false),
53 cl::desc("Disable the generation of low-overhead loops"));
54
55static cl::opt<bool>
56 AllowWLSLoops("allow-arm-wlsloops", cl::Hidden, cl::init(Val: true),
57 cl::desc("Enable the generation of WLS loops"));
58
59static cl::opt<bool> UseWidenGlobalArrays(
60 "widen-global-strings", cl::Hidden, cl::init(Val: true),
61 cl::desc("Enable the widening of global strings to alignment boundaries"));
62
63extern cl::opt<TailPredication::Mode> EnableTailPredication;
64
65extern cl::opt<bool> EnableMaskedGatherScatters;
66
67extern cl::opt<unsigned> MVEMaxSupportedInterleaveFactor;
68
69static cl::opt<int> ArmForceUnrollThreshold(
70 "arm-force-unroll-threshold", cl::init(Val: 12), cl::Hidden,
71 cl::desc(
72 "Threshold for forced unrolling of small loops in Arm architecture"));
73
74/// Convert a vector load intrinsic into a simple llvm load instruction.
75/// This is beneficial when the underlying object being addressed comes
76/// from a constant, since we get constant-folding for free.
77static Value *simplifyNeonVld1(const IntrinsicInst &II, unsigned MemAlign,
78 InstCombiner::BuilderTy &Builder) {
79 auto *IntrAlign = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
80
81 if (!IntrAlign)
82 return nullptr;
83
84 unsigned Alignment = IntrAlign->getLimitedValue() < MemAlign
85 ? MemAlign
86 : IntrAlign->getLimitedValue();
87
88 if (!isPowerOf2_32(Value: Alignment))
89 return nullptr;
90
91 return Builder.CreateAlignedLoad(Ty: II.getType(), Ptr: II.getArgOperand(i: 0),
92 Align: Align(Alignment));
93}
94
95TTI::AddressingModeKind
96ARMTTIImpl::getPreferredAddressingMode(const Loop *L,
97 ScalarEvolution *SE) const {
98 if (ST->hasMVEIntegerOps())
99 return TTI::AMK_PostIndexed;
100
101 if (L->getHeader()->getParent()->hasOptSize())
102 return TTI::AMK_None;
103
104 if (ST->isMClass() && ST->isThumb2() &&
105 L->getNumBlocks() == 1)
106 return TTI::AMK_PreIndexed;
107
108 return TTI::AMK_None;
109}
110
111std::optional<Instruction *>
112ARMTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
113 using namespace PatternMatch;
114 Intrinsic::ID IID = II.getIntrinsicID();
115 switch (IID) {
116 default:
117 break;
118 case Intrinsic::arm_neon_vld1: {
119 Align MemAlign =
120 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CtxI: &II,
121 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
122 if (Value *V = simplifyNeonVld1(II, MemAlign: MemAlign.value(), Builder&: IC.Builder)) {
123 return IC.replaceInstUsesWith(I&: II, V);
124 }
125 break;
126 }
127
128 case Intrinsic::arm_neon_vld2:
129 case Intrinsic::arm_neon_vld3:
130 case Intrinsic::arm_neon_vld4:
131 case Intrinsic::arm_neon_vld2lane:
132 case Intrinsic::arm_neon_vld3lane:
133 case Intrinsic::arm_neon_vld4lane:
134 case Intrinsic::arm_neon_vst1:
135 case Intrinsic::arm_neon_vst2:
136 case Intrinsic::arm_neon_vst3:
137 case Intrinsic::arm_neon_vst4:
138 case Intrinsic::arm_neon_vst2lane:
139 case Intrinsic::arm_neon_vst3lane:
140 case Intrinsic::arm_neon_vst4lane: {
141 Align MemAlign =
142 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CtxI: &II,
143 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
144 unsigned AlignArg = II.arg_size() - 1;
145 Value *AlignArgOp = II.getArgOperand(i: AlignArg);
146 MaybeAlign Align = cast<ConstantInt>(Val: AlignArgOp)->getMaybeAlignValue();
147 if (Align && *Align < MemAlign) {
148 return IC.replaceOperand(
149 I&: II, OpNum: AlignArg,
150 V: ConstantInt::get(Ty: Type::getInt32Ty(C&: II.getContext()), V: MemAlign.value(),
151 IsSigned: false));
152 }
153 break;
154 }
155
156 case Intrinsic::arm_neon_vld1x2:
157 case Intrinsic::arm_neon_vld1x3:
158 case Intrinsic::arm_neon_vld1x4:
159 case Intrinsic::arm_neon_vst1x2:
160 case Intrinsic::arm_neon_vst1x3:
161 case Intrinsic::arm_neon_vst1x4: {
162 Align NewAlign =
163 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CtxI: &II,
164 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
165 Align OldAlign = II.getParamAlign(ArgNo: 0).valueOrOne();
166 if (NewAlign > OldAlign)
167 II.addParamAttr(ArgNo: 0,
168 Attr: Attribute::getWithAlignment(Context&: II.getContext(), Alignment: NewAlign));
169 break;
170 }
171
172 case Intrinsic::arm_mve_pred_i2v: {
173 Value *Arg = II.getArgOperand(i: 0);
174 Value *ArgArg;
175 if (match(V: Arg, P: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
176 Ops: PatternMatch::m_Value(V&: ArgArg))) &&
177 II.getType() == ArgArg->getType()) {
178 return IC.replaceInstUsesWith(I&: II, V: ArgArg);
179 }
180 Constant *XorMask;
181 if (match(V: Arg, P: m_Xor(L: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
182 Ops: PatternMatch::m_Value(V&: ArgArg)),
183 R: PatternMatch::m_Constant(C&: XorMask))) &&
184 II.getType() == ArgArg->getType()) {
185 if (auto *CI = dyn_cast<ConstantInt>(Val: XorMask)) {
186 if (CI->getValue().trunc(width: 16).isAllOnes()) {
187 auto TrueVector = IC.Builder.CreateVectorSplat(
188 NumElts: cast<FixedVectorType>(Val: II.getType())->getNumElements(),
189 V: IC.Builder.getTrue());
190 return BinaryOperator::Create(Op: Instruction::Xor, S1: ArgArg, S2: TrueVector);
191 }
192 }
193 }
194 KnownBits ScalarKnown(32);
195 if (IC.SimplifyDemandedBits(I: &II, OpNo: 0, DemandedMask: APInt::getLowBitsSet(numBits: 32, loBitsSet: 16),
196 Known&: ScalarKnown)) {
197 return &II;
198 }
199 break;
200 }
201 case Intrinsic::arm_mve_pred_v2i: {
202 Value *Arg = II.getArgOperand(i: 0);
203 Value *ArgArg;
204 if (match(V: Arg, P: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_i2v>(
205 Ops: PatternMatch::m_Value(V&: ArgArg)))) {
206 return IC.replaceInstUsesWith(I&: II, V: ArgArg);
207 }
208
209 if (II.getMetadata(KindID: LLVMContext::MD_range))
210 break;
211
212 ConstantRange Range(APInt(32, 0), APInt(32, 0x10000));
213
214 if (auto CurrentRange = II.getRange()) {
215 Range = Range.intersectWith(CR: *CurrentRange);
216 if (Range == CurrentRange)
217 break;
218 }
219
220 II.addRangeRetAttr(CR: Range);
221 II.addRetAttr(Kind: Attribute::NoUndef);
222 return &II;
223 }
224 case Intrinsic::arm_mve_vadc:
225 case Intrinsic::arm_mve_vadc_predicated: {
226 unsigned CarryOp =
227 (II.getIntrinsicID() == Intrinsic::arm_mve_vadc_predicated) ? 3 : 2;
228 assert(II.getArgOperand(CarryOp)->getType()->getScalarSizeInBits() == 32 &&
229 "Bad type for intrinsic!");
230
231 KnownBits CarryKnown(32);
232 if (IC.SimplifyDemandedBits(I: &II, OpNo: CarryOp, DemandedMask: APInt::getOneBitSet(numBits: 32, BitNo: 29),
233 Known&: CarryKnown)) {
234 return &II;
235 }
236 break;
237 }
238 case Intrinsic::arm_mve_vmldava: {
239 Instruction *I = cast<Instruction>(Val: &II);
240 if (I->hasOneUse()) {
241 auto *User = cast<Instruction>(Val: *I->user_begin());
242 Value *OpZ;
243 if (match(V: User, P: m_c_Add(L: m_Specific(V: I), R: m_Value(V&: OpZ))) &&
244 match(V: I->getOperand(i: 3), P: m_Zero())) {
245 Value *OpX = I->getOperand(i: 4);
246 Value *OpY = I->getOperand(i: 5);
247 Type *OpTy = OpX->getType();
248
249 IC.Builder.SetInsertPoint(User);
250 Value *V =
251 IC.Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vmldava, OverloadTypes: {OpTy},
252 Args: {I->getOperand(i: 0), I->getOperand(i: 1),
253 I->getOperand(i: 2), OpZ, OpX, OpY});
254
255 IC.replaceInstUsesWith(I&: *User, V);
256 return IC.eraseInstFromFunction(I&: *User);
257 }
258 }
259 return std::nullopt;
260 }
261 }
262 return std::nullopt;
263}
264
265std::optional<Value *> ARMTTIImpl::simplifyDemandedVectorEltsIntrinsic(
266 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,
267 APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
268 std::function<void(Instruction *, unsigned, APInt, APInt &)>
269 SimplifyAndSetOp) const {
270
271 // Compute the demanded bits for a narrowing MVE intrinsic. The TopOpc is the
272 // opcode specifying a Top/Bottom instruction, which can change between
273 // instructions.
274 auto SimplifyNarrowInstrTopBottom =[&](unsigned TopOpc) {
275 unsigned NumElts = cast<FixedVectorType>(Val: II.getType())->getNumElements();
276 unsigned IsTop = cast<ConstantInt>(Val: II.getOperand(i_nocapture: TopOpc))->getZExtValue();
277
278 // The only odd/even lanes of operand 0 will only be demanded depending
279 // on whether this is a top/bottom instruction.
280 APInt DemandedElts =
281 APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
282 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
283 SimplifyAndSetOp(&II, 0, OrigDemandedElts & DemandedElts, UndefElts);
284 // The other lanes will be defined from the inserted elements.
285 UndefElts &= APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
286 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
287 return std::nullopt;
288 };
289
290 switch (II.getIntrinsicID()) {
291 default:
292 break;
293 case Intrinsic::arm_mve_vcvt_narrow:
294 SimplifyNarrowInstrTopBottom(2);
295 break;
296 case Intrinsic::arm_mve_vqmovn:
297 SimplifyNarrowInstrTopBottom(4);
298 break;
299 case Intrinsic::arm_mve_vshrn:
300 SimplifyNarrowInstrTopBottom(7);
301 break;
302 }
303
304 return std::nullopt;
305}
306
307InstructionCost ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
308 TTI::TargetCostKind CostKind) const {
309 assert(Ty->isIntegerTy());
310
311 unsigned Bits = Ty->getPrimitiveSizeInBits();
312 if (Bits == 0 || Imm.getActiveBits() >= 64)
313 return 4;
314
315 int64_t SImmVal = Imm.getSExtValue();
316 uint64_t ZImmVal = Imm.getZExtValue();
317 if (!ST->isThumb()) {
318 if ((SImmVal >= 0 && SImmVal < 65536) ||
319 (ARM_AM::getSOImmVal(Arg: ZImmVal) != -1) ||
320 (ARM_AM::getSOImmVal(Arg: ~ZImmVal) != -1))
321 return 1;
322 return ST->hasV6T2Ops() ? 2 : 3;
323 }
324 if (ST->isThumb2()) {
325 if ((SImmVal >= 0 && SImmVal < 65536) ||
326 (ARM_AM::getT2SOImmVal(Arg: ZImmVal) != -1) ||
327 (ARM_AM::getT2SOImmVal(Arg: ~ZImmVal) != -1))
328 return 1;
329 return ST->hasV6T2Ops() ? 2 : 3;
330 }
331 // Thumb1, any i8 imm cost 1.
332 if (Bits == 8 || (SImmVal >= 0 && SImmVal < 256))
333 return 1;
334 if ((~SImmVal < 256) || ARM_AM::isThumbImmShiftedVal(V: ZImmVal))
335 return 2;
336 // Load from constantpool.
337 return 3;
338}
339
340// Constants smaller than 256 fit in the immediate field of
341// Thumb1 instructions so we return a zero cost and 1 otherwise.
342InstructionCost ARMTTIImpl::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
343 const APInt &Imm,
344 Type *Ty) const {
345 if (Imm.isNonNegative() && Imm.getLimitedValue() < 256)
346 return 0;
347
348 return 1;
349}
350
351// Checks whether Inst is part of a min(max()) or max(min()) pattern
352// that will match to an SSAT instruction. Returns the instruction being
353// saturated, or null if no saturation pattern was found.
354static Value *isSSATMinMaxPattern(Instruction *Inst, const APInt &Imm) {
355 Value *LHS, *RHS;
356 ConstantInt *C;
357 SelectPatternFlavor InstSPF = matchSelectPattern(V: Inst, LHS, RHS).Flavor;
358
359 if (InstSPF == SPF_SMAX &&
360 PatternMatch::match(V: RHS, P: PatternMatch::m_ConstantInt(CI&: C)) &&
361 C->getValue() == Imm && Imm.isNegative() && Imm.isNegatedPowerOf2()) {
362
363 auto isSSatMin = [&](Value *MinInst) {
364 if (isa<SelectInst>(Val: MinInst)) {
365 Value *MinLHS, *MinRHS;
366 ConstantInt *MinC;
367 SelectPatternFlavor MinSPF =
368 matchSelectPattern(V: MinInst, LHS&: MinLHS, RHS&: MinRHS).Flavor;
369 if (MinSPF == SPF_SMIN &&
370 PatternMatch::match(V: MinRHS, P: PatternMatch::m_ConstantInt(CI&: MinC)) &&
371 MinC->getValue() == ((-Imm) - 1))
372 return true;
373 }
374 return false;
375 };
376
377 if (isSSatMin(Inst->getOperand(i: 1)))
378 return cast<Instruction>(Val: Inst->getOperand(i: 1))->getOperand(i: 1);
379 if (Inst->hasNUses(N: 2) &&
380 (isSSatMin(*Inst->user_begin()) || isSSatMin(*(++Inst->user_begin()))))
381 return Inst->getOperand(i: 1);
382 }
383 return nullptr;
384}
385
386// Look for a FP Saturation pattern, where the instruction can be simplified to
387// a fptosi.sat. max(min(fptosi)). The constant in this case is always free.
388static bool isFPSatMinMaxPattern(Instruction *Inst, const APInt &Imm) {
389 if (Imm.getBitWidth() != 64 ||
390 Imm != APInt::getHighBitsSet(numBits: 64, hiBitsSet: 33)) // -2147483648
391 return false;
392 Value *FP = isSSATMinMaxPattern(Inst, Imm);
393 if (!FP && isa<ICmpInst>(Val: Inst) && Inst->hasOneUse())
394 FP = isSSATMinMaxPattern(Inst: cast<Instruction>(Val: *Inst->user_begin()), Imm);
395 if (!FP)
396 return false;
397 return isa<FPToSIInst>(Val: FP);
398}
399
400InstructionCost ARMTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
401 const APInt &Imm, Type *Ty,
402 TTI::TargetCostKind CostKind,
403 Instruction *Inst) const {
404 // Division by a constant can be turned into multiplication, but only if we
405 // know it's constant. So it's not so much that the immediate is cheap (it's
406 // not), but that the alternative is worse.
407 // FIXME: this is probably unneeded with GlobalISel.
408 if ((Opcode == Instruction::SDiv || Opcode == Instruction::UDiv ||
409 Opcode == Instruction::SRem || Opcode == Instruction::URem) &&
410 Idx == 1)
411 return 0;
412
413 // Leave any gep offsets for the CodeGenPrepare, which will do a better job at
414 // splitting any large offsets.
415 if (Opcode == Instruction::GetElementPtr && Idx != 0)
416 return 0;
417
418 if (Opcode == Instruction::And) {
419 // UXTB/UXTH
420 if (Imm == 255 || Imm == 65535)
421 return 0;
422 // Conversion to BIC is free, and means we can use ~Imm instead.
423 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
424 b: getIntImmCost(Imm: ~Imm, Ty, CostKind));
425 }
426
427 if (Opcode == Instruction::Add)
428 // Conversion to SUB is free, and means we can use -Imm instead.
429 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
430 b: getIntImmCost(Imm: -Imm, Ty, CostKind));
431
432 if (Opcode == Instruction::ICmp && Imm.isNegative() &&
433 Ty->getIntegerBitWidth() == 32) {
434 int64_t NegImm = -Imm.getSExtValue();
435 if (ST->isThumb2() && NegImm < 1<<12)
436 // icmp X, #-C -> cmn X, #C
437 return 0;
438 if (ST->isThumb() && NegImm < 1<<8)
439 // icmp X, #-C -> adds X, #C
440 return 0;
441 }
442
443 // xor a, -1 can always be folded to MVN
444 if (Opcode == Instruction::Xor && Imm.isAllOnes())
445 return 0;
446
447 // Ensures negative constant of min(max()) or max(min()) patterns that
448 // match to SSAT instructions don't get hoisted
449 if (Inst && ((ST->hasV6Ops() && !ST->isThumb()) || ST->isThumb2()) &&
450 Ty->getIntegerBitWidth() <= 32) {
451 if (isSSATMinMaxPattern(Inst, Imm) ||
452 (isa<ICmpInst>(Val: Inst) && Inst->hasOneUse() &&
453 isSSATMinMaxPattern(Inst: cast<Instruction>(Val: *Inst->user_begin()), Imm)))
454 return 0;
455 }
456
457 if (Inst && ST->hasVFP2Base() && isFPSatMinMaxPattern(Inst, Imm))
458 return 0;
459
460 // We can convert <= -1 to < 0, which is generally quite cheap.
461 if (Inst && Opcode == Instruction::ICmp && Idx == 1 && Imm.isAllOnes()) {
462 ICmpInst::Predicate Pred = cast<ICmpInst>(Val: Inst)->getPredicate();
463 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE)
464 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
465 b: getIntImmCost(Imm: Imm + 1, Ty, CostKind));
466 }
467
468 return getIntImmCost(Imm, Ty, CostKind);
469}
470
471InstructionCost ARMTTIImpl::getCFInstrCost(unsigned Opcode,
472 TTI::TargetCostKind CostKind,
473 const Instruction *I) const {
474 if (CostKind == TTI::TCK_RecipThroughput &&
475 (ST->hasNEON() || ST->hasMVEIntegerOps())) {
476 // FIXME: The vectorizer is highly sensitive to the cost of these
477 // instructions, which suggests that it may be using the costs incorrectly.
478 // But, for now, just make them free to avoid performance regressions for
479 // vector targets.
480 return 0;
481 }
482 return BaseT::getCFInstrCost(Opcode, CostKind, I);
483}
484
485InstructionCost ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
486 Type *Src,
487 TTI::CastContextHint CCH,
488 TTI::TargetCostKind CostKind,
489 const Instruction *I) const {
490 int ISD = TLI->InstructionOpcodeToISD(Opcode);
491 assert(ISD && "Invalid opcode");
492
493 // TODO: Allow non-throughput costs that aren't binary.
494 auto AdjustCost = [&CostKind](InstructionCost Cost) -> InstructionCost {
495 if (CostKind != TTI::TCK_RecipThroughput)
496 return Cost == 0 ? 0 : 1;
497 return Cost;
498 };
499 auto IsLegalFPType = [this](EVT VT) {
500 EVT EltVT = VT.getScalarType();
501 return (EltVT == MVT::f32 && ST->hasVFP2Base()) ||
502 (EltVT == MVT::f64 && ST->hasFP64()) ||
503 (EltVT == MVT::f16 && ST->hasFullFP16());
504 };
505
506 EVT SrcTy = TLI->getValueType(DL, Ty: Src);
507 EVT DstTy = TLI->getValueType(DL, Ty: Dst);
508
509 if (!SrcTy.isSimple() || !DstTy.isSimple())
510 return AdjustCost(
511 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
512
513 // Extending masked load/Truncating masked stores is expensive because we
514 // currently don't split them. This means that we'll likely end up
515 // loading/storing each element individually (hence the high cost).
516 if ((ST->hasMVEIntegerOps() &&
517 (Opcode == Instruction::Trunc || Opcode == Instruction::ZExt ||
518 Opcode == Instruction::SExt)) ||
519 (ST->hasMVEFloatOps() &&
520 (Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc) &&
521 IsLegalFPType(SrcTy) && IsLegalFPType(DstTy)))
522 if (CCH == TTI::CastContextHint::Masked && DstTy.getSizeInBits() > 128)
523 return 2 * DstTy.getVectorNumElements() *
524 ST->getMVEVectorCostFactor(CostKind);
525
526 // The extend of other kinds of load is free
527 if (CCH == TTI::CastContextHint::Normal ||
528 CCH == TTI::CastContextHint::Masked) {
529 static const TypeConversionCostTblEntry LoadConversionTbl[] = {
530 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i32, .Src: MVT::i16, .Cost: 0},
531 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i32, .Src: MVT::i16, .Cost: 0},
532 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i32, .Src: MVT::i8, .Cost: 0},
533 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i32, .Src: MVT::i8, .Cost: 0},
534 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i16, .Src: MVT::i8, .Cost: 0},
535 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i16, .Src: MVT::i8, .Cost: 0},
536 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i32, .Cost: 1},
537 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i32, .Cost: 1},
538 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 1},
539 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 1},
540 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i8, .Cost: 1},
541 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i8, .Cost: 1},
542 };
543 if (const auto *Entry = ConvertCostTableLookup(
544 Table: LoadConversionTbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
545 return AdjustCost(Entry->Cost);
546
547 static const TypeConversionCostTblEntry MVELoadConversionTbl[] = {
548 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
549 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
550 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
551 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
552 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
553 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
554 // The following extend from a legal type to an illegal type, so need to
555 // split the load. This introduced an extra load operation, but the
556 // extend is still "free".
557 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
558 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
559 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
560 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
561 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
562 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
563 };
564 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
565 if (const auto *Entry =
566 ConvertCostTableLookup(Table: MVELoadConversionTbl, ISD,
567 Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
568 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
569 }
570
571 static const TypeConversionCostTblEntry MVEFLoadConversionTbl[] = {
572 // FPExtends are similar but also require the VCVT instructions.
573 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1},
574 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 3},
575 };
576 if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
577 if (const auto *Entry =
578 ConvertCostTableLookup(Table: MVEFLoadConversionTbl, ISD,
579 Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
580 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
581 }
582
583 // The truncate of a store is free. This is the mirror of extends above.
584 static const TypeConversionCostTblEntry MVEStoreConversionTbl[] = {
585 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
586 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
587 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
588 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
589 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 1},
590 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
591 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
592 };
593 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
594 if (const auto *Entry =
595 ConvertCostTableLookup(Table: MVEStoreConversionTbl, ISD,
596 Dst: SrcTy.getSimpleVT(), Src: DstTy.getSimpleVT()))
597 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
598 }
599
600 static const TypeConversionCostTblEntry MVEFStoreConversionTbl[] = {
601 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1},
602 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 3},
603 };
604 if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
605 if (const auto *Entry =
606 ConvertCostTableLookup(Table: MVEFStoreConversionTbl, ISD,
607 Dst: SrcTy.getSimpleVT(), Src: DstTy.getSimpleVT()))
608 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
609 }
610 }
611
612 // NEON vector operations that can extend their inputs.
613 if ((ISD == ISD::SIGN_EXTEND || ISD == ISD::ZERO_EXTEND) &&
614 I && I->hasOneUse() && ST->hasNEON() && SrcTy.isVector()) {
615 static const TypeConversionCostTblEntry NEONDoubleWidthTbl[] = {
616 // vaddl
617 { .ISD: ISD::ADD, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
618 { .ISD: ISD::ADD, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
619 // vsubl
620 { .ISD: ISD::SUB, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
621 { .ISD: ISD::SUB, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
622 // vmull
623 { .ISD: ISD::MUL, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
624 { .ISD: ISD::MUL, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
625 // vshll
626 { .ISD: ISD::SHL, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
627 { .ISD: ISD::SHL, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
628 };
629
630 auto *User = cast<Instruction>(Val: *I->user_begin());
631 int UserISD = TLI->InstructionOpcodeToISD(Opcode: User->getOpcode());
632 if (auto *Entry = ConvertCostTableLookup(Table: NEONDoubleWidthTbl, ISD: UserISD,
633 Dst: DstTy.getSimpleVT(),
634 Src: SrcTy.getSimpleVT())) {
635 return AdjustCost(Entry->Cost);
636 }
637 }
638
639 // Single to/from double precision conversions.
640 if (Src->isVectorTy() && ST->hasNEON() &&
641 ((ISD == ISD::FP_ROUND && SrcTy.getScalarType() == MVT::f64 &&
642 DstTy.getScalarType() == MVT::f32) ||
643 (ISD == ISD::FP_EXTEND && SrcTy.getScalarType() == MVT::f32 &&
644 DstTy.getScalarType() == MVT::f64))) {
645 static const CostTblEntry NEONFltDblTbl[] = {
646 // Vector fptrunc/fpext conversions.
647 {.ISD: ISD::FP_ROUND, .Type: MVT::v2f64, .Cost: 2},
648 {.ISD: ISD::FP_EXTEND, .Type: MVT::v2f32, .Cost: 2},
649 {.ISD: ISD::FP_EXTEND, .Type: MVT::v4f32, .Cost: 4}};
650
651 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
652 if (const auto *Entry = CostTableLookup(Table: NEONFltDblTbl, ISD, Ty: LT.second))
653 return AdjustCost(LT.first * Entry->Cost);
654 }
655
656 // Some arithmetic, load and store operations have specific instructions
657 // to cast up/down their types automatically at no extra cost.
658 // TODO: Get these tables to know at least what the related operations are.
659 static const TypeConversionCostTblEntry NEONVectorConversionTbl[] = {
660 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
661 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
662 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1 },
663 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1 },
664 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: 0 },
665 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i32, .Cost: 1 },
666
667 // The number of vmovl instructions for the extension.
668 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
669 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
670 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
671 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
672 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 3 },
673 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 3 },
674 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
675 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
676 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3 },
677 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3 },
678 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3 },
679 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3 },
680 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7 },
681 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7 },
682 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6 },
683 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6 },
684 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6 },
685 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6 },
686
687 // Operations that we legalize using splitting.
688 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: 6 },
689 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: 3 },
690
691 // Vector float <-> i32 conversions.
692 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1 },
693 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1 },
694
695 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3 },
696 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3 },
697 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2 },
698 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2 },
699 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1 },
700 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1 },
701 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: 3 },
702 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: 3 },
703 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3 },
704 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3 },
705 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2 },
706 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2 },
707 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4 },
708 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4 },
709 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: 2 },
710 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: 2 },
711 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: 8 },
712 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: 8 },
713 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: 4 },
714 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: 4 },
715
716 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1 },
717 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1 },
718 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 3 },
719 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 3 },
720 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2 },
721 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2 },
722
723 // Vector double <-> i32 conversions.
724 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
725 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
726
727 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4 },
728 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4 },
729 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 3 },
730 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 3 },
731 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
732 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
733
734 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2 },
735 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2 },
736 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 4 },
737 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 4 },
738 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: 8 },
739 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: 8 }
740 };
741
742 if (SrcTy.isVector() && ST->hasNEON()) {
743 if (const auto *Entry = ConvertCostTableLookup(Table: NEONVectorConversionTbl, ISD,
744 Dst: DstTy.getSimpleVT(),
745 Src: SrcTy.getSimpleVT()))
746 return AdjustCost(Entry->Cost);
747 }
748
749 // Scalar float to integer conversions.
750 static const TypeConversionCostTblEntry NEONFloatConversionTbl[] = {
751 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i1, .Src: MVT::f32, .Cost: 2 },
752 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i1, .Src: MVT::f32, .Cost: 2 },
753 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i1, .Src: MVT::f64, .Cost: 2 },
754 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i1, .Src: MVT::f64, .Cost: 2 },
755 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i8, .Src: MVT::f32, .Cost: 2 },
756 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i8, .Src: MVT::f32, .Cost: 2 },
757 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i8, .Src: MVT::f64, .Cost: 2 },
758 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i8, .Src: MVT::f64, .Cost: 2 },
759 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i16, .Src: MVT::f32, .Cost: 2 },
760 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i16, .Src: MVT::f32, .Cost: 2 },
761 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i16, .Src: MVT::f64, .Cost: 2 },
762 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i16, .Src: MVT::f64, .Cost: 2 },
763 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: 2 },
764 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: 2 },
765 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: 2 },
766 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: 2 },
767 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: 10 },
768 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: 10 },
769 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: 10 },
770 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: 10 }
771 };
772 if (SrcTy.isFloatingPoint() && ST->hasNEON()) {
773 if (const auto *Entry = ConvertCostTableLookup(Table: NEONFloatConversionTbl, ISD,
774 Dst: DstTy.getSimpleVT(),
775 Src: SrcTy.getSimpleVT()))
776 return AdjustCost(Entry->Cost);
777 }
778
779 // Scalar integer to float conversions.
780 static const TypeConversionCostTblEntry NEONIntegerConversionTbl[] = {
781 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i1, .Cost: 2 },
782 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i1, .Cost: 2 },
783 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i1, .Cost: 2 },
784 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i1, .Cost: 2 },
785 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i8, .Cost: 2 },
786 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i8, .Cost: 2 },
787 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i8, .Cost: 2 },
788 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i8, .Cost: 2 },
789 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i16, .Cost: 2 },
790 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i16, .Cost: 2 },
791 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i16, .Cost: 2 },
792 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i16, .Cost: 2 },
793 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: 2 },
794 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: 2 },
795 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: 2 },
796 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: 2 },
797 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: 10 },
798 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: 10 },
799 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: 10 },
800 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: 10 }
801 };
802
803 if (SrcTy.isInteger() && ST->hasNEON()) {
804 if (const auto *Entry = ConvertCostTableLookup(Table: NEONIntegerConversionTbl,
805 ISD, Dst: DstTy.getSimpleVT(),
806 Src: SrcTy.getSimpleVT()))
807 return AdjustCost(Entry->Cost);
808 }
809
810 // MVE extend costs, taken from codegen tests. i8->i16 or i16->i32 is one
811 // instruction, i8->i32 is two. i64 zexts are an VAND with a constant, sext
812 // are linearised so take more.
813 static const TypeConversionCostTblEntry MVEVectorConversionTbl[] = {
814 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
815 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
816 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
817 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
818 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 10 },
819 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 2 },
820 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
821 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
822 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 10 },
823 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
824 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 8 },
825 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 2 },
826 };
827
828 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
829 if (const auto *Entry = ConvertCostTableLookup(Table: MVEVectorConversionTbl,
830 ISD, Dst: DstTy.getSimpleVT(),
831 Src: SrcTy.getSimpleVT()))
832 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
833 }
834
835 if (ISD == ISD::FP_ROUND || ISD == ISD::FP_EXTEND) {
836 // As general rule, fp converts that were not matched above are scalarized
837 // and cost 1 vcvt for each lane, so long as the instruction is available.
838 // If not it will become a series of function calls.
839 const InstructionCost CallCost =
840 getCallInstrCost(F: nullptr, RetTy: Dst, Tys: {Src}, CostKind);
841 int Lanes = 1;
842 if (SrcTy.isFixedLengthVector())
843 Lanes = SrcTy.getVectorNumElements();
844
845 if (IsLegalFPType(SrcTy) && IsLegalFPType(DstTy))
846 return Lanes;
847 else
848 return Lanes * CallCost;
849 }
850
851 if (ISD == ISD::TRUNCATE && ST->hasMVEIntegerOps() &&
852 SrcTy.isFixedLengthVector()) {
853 // Treat a truncate with larger than legal source (128bits for MVE) as
854 // expensive, 2 instructions per lane.
855 if ((SrcTy.getScalarType() == MVT::i8 ||
856 SrcTy.getScalarType() == MVT::i16 ||
857 SrcTy.getScalarType() == MVT::i32) &&
858 SrcTy.getSizeInBits() > 128 &&
859 SrcTy.getSizeInBits() > DstTy.getSizeInBits())
860 return SrcTy.getVectorNumElements() * 2;
861 }
862
863 // Scalar integer conversion costs.
864 static const TypeConversionCostTblEntry ARMIntegerConversionTbl[] = {
865 // i16 -> i64 requires two dependent operations.
866 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 2 },
867
868 // Truncates on i64 are assumed to be free.
869 { .ISD: ISD::TRUNCATE, .Dst: MVT::i32, .Src: MVT::i64, .Cost: 0 },
870 { .ISD: ISD::TRUNCATE, .Dst: MVT::i16, .Src: MVT::i64, .Cost: 0 },
871 { .ISD: ISD::TRUNCATE, .Dst: MVT::i8, .Src: MVT::i64, .Cost: 0 },
872 { .ISD: ISD::TRUNCATE, .Dst: MVT::i1, .Src: MVT::i64, .Cost: 0 }
873 };
874
875 if (SrcTy.isInteger()) {
876 if (const auto *Entry = ConvertCostTableLookup(Table: ARMIntegerConversionTbl, ISD,
877 Dst: DstTy.getSimpleVT(),
878 Src: SrcTy.getSimpleVT()))
879 return AdjustCost(Entry->Cost);
880 }
881
882 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
883 ? ST->getMVEVectorCostFactor(CostKind)
884 : 1;
885 return AdjustCost(
886 BaseCost * BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
887}
888
889InstructionCost ARMTTIImpl::getVectorInstrCost(
890 unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
891 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
892 // Penalize inserting into an D-subregister. We end up with a three times
893 // lower estimated throughput on swift.
894 if (ST->hasSlowLoadDSubregister() && Opcode == Instruction::InsertElement &&
895 ValTy->isVectorTy() && ValTy->getScalarSizeInBits() <= 32)
896 return 3;
897
898 if (ST->hasNEON() && (Opcode == Instruction::InsertElement ||
899 Opcode == Instruction::ExtractElement)) {
900 // Cross-class copies are expensive on many microarchitectures,
901 // so assume they are expensive by default.
902 if (cast<VectorType>(Val: ValTy)->getElementType()->isIntegerTy())
903 return 3;
904
905 // Even if it's not a cross class copy, this likely leads to mixing
906 // of NEON and VFP code and should be therefore penalized.
907 if (ValTy->isVectorTy() &&
908 ValTy->getScalarSizeInBits() <= 32)
909 return std::max<InstructionCost>(
910 a: BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
911 VIC),
912 b: 2U);
913 }
914
915 if (ST->hasMVEIntegerOps() && (Opcode == Instruction::InsertElement ||
916 Opcode == Instruction::ExtractElement)) {
917 // Integer cross-lane moves are more expensive than float, which can
918 // sometimes just be vmovs. Integer involve being passes to GPR registers,
919 // causing more of a delay.
920 std::pair<InstructionCost, MVT> LT =
921 getTypeLegalizationCost(Ty: ValTy->getScalarType());
922 return LT.first * (ValTy->getScalarType()->isIntegerTy() ? 4 : 1);
923 }
924
925 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
926 VIC);
927}
928
929InstructionCost ARMTTIImpl::getCmpSelInstrCost(
930 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
931 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
932 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
933 int ISD = TLI->InstructionOpcodeToISD(Opcode);
934
935 // Thumb scalar code size cost for select.
936 if (CostKind == TTI::TCK_CodeSize && ISD == ISD::SELECT &&
937 ST->isThumb() && !ValTy->isVectorTy()) {
938 // Assume expensive structs.
939 if (TLI->getValueType(DL, Ty: ValTy, AllowUnknown: true) == MVT::Other)
940 return TTI::TCC_Expensive;
941
942 // Select costs can vary because they:
943 // - may require one or more conditional mov (including an IT),
944 // - can't operate directly on immediates,
945 // - require live flags, which we can't copy around easily.
946 InstructionCost Cost = getTypeLegalizationCost(Ty: ValTy).first;
947
948 // Possible IT instruction for Thumb2, or more for Thumb1.
949 ++Cost;
950
951 // i1 values may need rematerialising by using mov immediates and/or
952 // flag setting instructions.
953 if (ValTy->isIntegerTy(BitWidth: 1))
954 ++Cost;
955
956 return Cost;
957 }
958
959 // If this is a vector min/max/abs, use the cost of that intrinsic directly
960 // instead. Hopefully when min/max intrinsics are more prevalent this code
961 // will not be needed.
962 const Instruction *Sel = I;
963 if ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && Sel &&
964 Sel->hasOneUse())
965 Sel = cast<Instruction>(Val: Sel->user_back());
966 if (Sel && ValTy->isVectorTy() &&
967 (ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy())) {
968 const Value *LHS, *RHS;
969 SelectPatternFlavor SPF = matchSelectPattern(V: Sel, LHS, RHS).Flavor;
970 unsigned IID = 0;
971 switch (SPF) {
972 case SPF_ABS:
973 IID = Intrinsic::abs;
974 break;
975 case SPF_SMIN:
976 IID = Intrinsic::smin;
977 break;
978 case SPF_SMAX:
979 IID = Intrinsic::smax;
980 break;
981 case SPF_UMIN:
982 IID = Intrinsic::umin;
983 break;
984 case SPF_UMAX:
985 IID = Intrinsic::umax;
986 break;
987 case SPF_FMINNUM:
988 IID = Intrinsic::minnum;
989 break;
990 case SPF_FMAXNUM:
991 IID = Intrinsic::maxnum;
992 break;
993 default:
994 break;
995 }
996 if (IID) {
997 // The ICmp is free, the select gets the cost of the min/max/etc
998 if (Sel != I)
999 return 0;
1000 IntrinsicCostAttributes CostAttrs(IID, ValTy, {ValTy, ValTy});
1001 return getIntrinsicInstrCost(ICA: CostAttrs, CostKind);
1002 }
1003 }
1004
1005 // On NEON a vector select gets lowered to vbsl.
1006 if (ST->hasNEON() && ValTy->isVectorTy() && ISD == ISD::SELECT && CondTy) {
1007 // Lowering of some vector selects is currently far from perfect.
1008 static const TypeConversionCostTblEntry NEONVectorSelectTbl[] = {
1009 { .ISD: ISD::SELECT, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: 4*4 + 1*2 + 1 },
1010 { .ISD: ISD::SELECT, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: 50 },
1011 { .ISD: ISD::SELECT, .Dst: MVT::v16i1, .Src: MVT::v16i64, .Cost: 100 }
1012 };
1013
1014 EVT SelCondTy = TLI->getValueType(DL, Ty: CondTy);
1015 EVT SelValTy = TLI->getValueType(DL, Ty: ValTy);
1016 if (SelCondTy.isSimple() && SelValTy.isSimple()) {
1017 if (const auto *Entry = ConvertCostTableLookup(Table: NEONVectorSelectTbl, ISD,
1018 Dst: SelCondTy.getSimpleVT(),
1019 Src: SelValTy.getSimpleVT()))
1020 return Entry->Cost;
1021 }
1022
1023 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1024 return LT.first;
1025 }
1026
1027 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy() &&
1028 (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
1029 cast<FixedVectorType>(Val: ValTy)->getNumElements() > 1) {
1030 FixedVectorType *VecValTy = cast<FixedVectorType>(Val: ValTy);
1031 FixedVectorType *VecCondTy = dyn_cast_or_null<FixedVectorType>(Val: CondTy);
1032 if (!VecCondTy)
1033 VecCondTy = cast<FixedVectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecValTy));
1034
1035 // If we don't have mve.fp any fp operations will need to be scalarized.
1036 if (Opcode == Instruction::FCmp && !ST->hasMVEFloatOps()) {
1037 // One scalaization insert, one scalarization extract and the cost of the
1038 // fcmps.
1039 return BaseT::getScalarizationOverhead(InTy: VecValTy, /*Insert*/ false,
1040 /*Extract*/ true, CostKind) +
1041 BaseT::getScalarizationOverhead(InTy: VecCondTy, /*Insert*/ true,
1042 /*Extract*/ false, CostKind) +
1043 VecValTy->getNumElements() *
1044 getCmpSelInstrCost(Opcode, ValTy: ValTy->getScalarType(),
1045 CondTy: VecCondTy->getScalarType(), VecPred,
1046 CostKind, Op1Info, Op2Info, I);
1047 }
1048
1049 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1050 int BaseCost = ST->getMVEVectorCostFactor(CostKind);
1051 // There are two types - the input that specifies the type of the compare
1052 // and the output vXi1 type. Because we don't know how the output will be
1053 // split, we may need an expensive shuffle to get two in sync. This has the
1054 // effect of making larger than legal compares (v8i32 for example)
1055 // expensive.
1056 if (LT.second.isVector() && LT.second.getVectorNumElements() > 2) {
1057 if (LT.first > 1)
1058 return LT.first * BaseCost +
1059 BaseT::getScalarizationOverhead(InTy: VecCondTy, /*Insert*/ true,
1060 /*Extract*/ false, CostKind);
1061 return BaseCost;
1062 }
1063 }
1064
1065 // Default to cheap (throughput/size of 1 instruction) but adjust throughput
1066 // for "multiple beats" potentially needed by MVE instructions.
1067 int BaseCost = 1;
1068 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy())
1069 BaseCost = ST->getMVEVectorCostFactor(CostKind);
1070
1071 return BaseCost * BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred,
1072 CostKind, Op1Info, Op2Info, I);
1073}
1074
1075InstructionCost
1076ARMTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
1077 const SCEV *Ptr,
1078 TTI::TargetCostKind CostKind) const {
1079 // Address computations in vectorized code with non-consecutive addresses will
1080 // likely result in more instructions compared to scalar code where the
1081 // computation can more often be merged into the index mode. The resulting
1082 // extra micro-ops can significantly decrease throughput.
1083 unsigned NumVectorInstToHideOverhead = 10;
1084 int MaxMergeDistance = 64;
1085
1086 if (ST->hasNEON()) {
1087 if (PtrTy->isVectorTy() && SE &&
1088 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MergeDistance: MaxMergeDistance + 1))
1089 return NumVectorInstToHideOverhead;
1090
1091 // In many cases the address computation is not merged into the instruction
1092 // addressing mode.
1093 return 1;
1094 }
1095 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
1096}
1097
1098bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) const {
1099 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1100 // If a VCTP is part of a chain, it's already profitable and shouldn't be
1101 // optimized, else LSR may block tail-predication.
1102 switch (II->getIntrinsicID()) {
1103 case Intrinsic::arm_mve_vctp8:
1104 case Intrinsic::arm_mve_vctp16:
1105 case Intrinsic::arm_mve_vctp32:
1106 case Intrinsic::arm_mve_vctp64:
1107 return true;
1108 default:
1109 break;
1110 }
1111 }
1112 return false;
1113}
1114
1115bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
1116 unsigned /*AddressSpace*/,
1117 TTI::MaskKind /*MaskKind*/) const {
1118 if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
1119 return false;
1120
1121 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: DataTy)) {
1122 // Don't support v2i1 yet.
1123 if (VecTy->getNumElements() == 2)
1124 return false;
1125
1126 // We don't support extending fp types.
1127 unsigned VecWidth = DataTy->getPrimitiveSizeInBits();
1128 if (VecWidth != 128 && VecTy->getElementType()->isFloatingPointTy())
1129 return false;
1130 }
1131
1132 unsigned EltWidth = DataTy->getScalarSizeInBits();
1133 return (EltWidth == 32 && Alignment >= 4) ||
1134 (EltWidth == 16 && Alignment >= 2) || (EltWidth == 8);
1135}
1136
1137bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) const {
1138 if (!EnableMaskedGatherScatters || !ST->hasMVEIntegerOps())
1139 return false;
1140
1141 unsigned EltWidth = Ty->getScalarSizeInBits();
1142 return ((EltWidth == 32 && Alignment >= 4) ||
1143 (EltWidth == 16 && Alignment >= 2) || EltWidth == 8);
1144}
1145
1146/// Given a memcpy/memset/memmove instruction, return the number of memory
1147/// operations performed, via querying findOptimalMemOpLowering. Returns -1 if a
1148/// call is used.
1149int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const {
1150 MemOp MOp;
1151 unsigned DstAddrSpace = ~0u;
1152 unsigned SrcAddrSpace = ~0u;
1153 const Function *F = I->getParent()->getParent();
1154
1155 if (const auto *MC = dyn_cast<MemTransferInst>(Val: I)) {
1156 ConstantInt *C = dyn_cast<ConstantInt>(Val: MC->getLength());
1157 // If 'size' is not a constant, a library call will be generated.
1158 if (!C)
1159 return -1;
1160
1161 const unsigned Size = C->getValue().getZExtValue();
1162 const Align DstAlign = MC->getDestAlign().valueOrOne();
1163 const Align SrcAlign = MC->getSourceAlign().valueOrOne();
1164
1165 // Use the most restrictive of memset, memcpy, memmove.
1166 MOp = MemOp::Move(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
1167 /*IsVolatile*/ false);
1168 DstAddrSpace = MC->getDestAddressSpace();
1169 SrcAddrSpace = MC->getSourceAddressSpace();
1170 }
1171 else if (const auto *MS = dyn_cast<MemSetInst>(Val: I)) {
1172 ConstantInt *C = dyn_cast<ConstantInt>(Val: MS->getLength());
1173 // If 'size' is not a constant, a library call will be generated.
1174 if (!C)
1175 return -1;
1176
1177 const unsigned Size = C->getValue().getZExtValue();
1178 const Align DstAlign = MS->getDestAlign().valueOrOne();
1179
1180 MOp = MemOp::Set(Size, /*DstAlignCanChange*/ false, DstAlign,
1181 /*IsZeroMemset*/ false, /*IsVolatile*/ false);
1182 DstAddrSpace = MS->getDestAddressSpace();
1183 }
1184 else
1185 llvm_unreachable("Expected a memcpy/move or memset!");
1186
1187 unsigned Limit, Factor = 2;
1188 switch(I->getIntrinsicID()) {
1189 case Intrinsic::memcpy:
1190 Limit = TLI->getMaxStoresPerMemcpy(OptSize: F->hasMinSize());
1191 break;
1192 case Intrinsic::memmove:
1193 Limit = TLI->getMaxStoresPerMemmove(OptSize: F->hasMinSize());
1194 break;
1195 case Intrinsic::memset:
1196 Limit = TLI->getMaxStoresPerMemset(OptSize: F->hasMinSize());
1197 Factor = 1;
1198 break;
1199 default:
1200 llvm_unreachable("Expected a memcpy/move or memset!");
1201 }
1202
1203 // MemOps will be poplulated with a list of data types that needs to be
1204 // loaded and stored. That's why we multiply the number of elements by 2 to
1205 // get the cost for this memcpy.
1206 std::vector<EVT> MemOps;
1207 LLVMContext &C = F->getContext();
1208 if (getTLI()->findOptimalMemOpLowering(Context&: C, MemOps, Limit, Op: MOp, DstAS: DstAddrSpace,
1209 SrcAS: SrcAddrSpace, FuncAttributes: F->getAttributes(),
1210 LargestVT: nullptr))
1211 return MemOps.size() * Factor;
1212
1213 // If we can't find an optimal memop lowering, return the default cost
1214 return -1;
1215}
1216
1217InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const {
1218 int NumOps = getNumMemOps(I: cast<IntrinsicInst>(Val: I));
1219
1220 // To model the cost of a library call, we assume 1 for the call, and
1221 // 3 for the argument setup.
1222 if (NumOps == -1)
1223 return 4;
1224 return NumOps;
1225}
1226
1227InstructionCost ARMTTIImpl::getShuffleCost(
1228 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
1229 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
1230 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
1231 TTI::VectorInstrContext VIC) const {
1232 assert((Mask.empty() || DstTy->isScalableTy() ||
1233 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
1234 "Expected the Mask to match the return size if given");
1235 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
1236 "Expected the same scalar types");
1237
1238 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1239 // Treat extractsubvector as single op permutation.
1240 bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector;
1241 if (IsExtractSubvector)
1242 Kind = TTI::SK_PermuteSingleSrc;
1243 if (ST->hasNEON()) {
1244 if (Kind == TTI::SK_Broadcast) {
1245 static const CostTblEntry NEONDupTbl[] = {
1246 // VDUP handles these cases.
1247 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1248 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1249 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1250 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1251 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 1},
1252 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i8, .Cost: 1},
1253
1254 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 1},
1255 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 1},
1256 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 1},
1257 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 1}};
1258
1259 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1260 if (const auto *Entry =
1261 CostTableLookup(Table: NEONDupTbl, ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1262 return LT.first * Entry->Cost;
1263 }
1264 if (Kind == TTI::SK_Reverse) {
1265 static const CostTblEntry NEONShuffleTbl[] = {
1266 // Reverse shuffle cost one instruction if we are shuffling within a
1267 // double word (vrev) or two if we shuffle a quad word (vrev, vext).
1268 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1269 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1270 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1271 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1272 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 1},
1273 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i8, .Cost: 1},
1274
1275 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 2},
1276 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 2},
1277 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 2},
1278 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 2}};
1279
1280 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1281 if (const auto *Entry =
1282 CostTableLookup(Table: NEONShuffleTbl, ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1283 return LT.first * Entry->Cost;
1284 }
1285 if (Kind == TTI::SK_Select) {
1286 static const CostTblEntry NEONSelShuffleTbl[] = {
1287 // Select shuffle cost table for ARM. Cost is the number of
1288 // instructions
1289 // required to create the shuffled vector.
1290
1291 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1292 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1293 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1294 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1295
1296 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 2},
1297 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 2},
1298 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 2},
1299
1300 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 16},
1301
1302 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 32}};
1303
1304 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1305 if (const auto *Entry = CostTableLookup(Table: NEONSelShuffleTbl,
1306 ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1307 return LT.first * Entry->Cost;
1308 }
1309
1310 // Check for other shuffles that are not SK_ kinds but we have native
1311 // instructions for, for example REV.
1312 if (!Mask.empty()) {
1313 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1314 unsigned Unused;
1315 if (LT.second.isVector() &&
1316 Mask.size() <= LT.second.getVectorNumElements() &&
1317 (isVREVMask(M: Mask, VT: LT.second, BlockSize: 16) || isVREVMask(M: Mask, VT: LT.second, BlockSize: 32) ||
1318 isVREVMask(M: Mask, VT: LT.second, BlockSize: 64) ||
1319 isVTRNMask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1320 isVTRN_v_undef_Mask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1321 isVZIPMask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1322 isVZIP_v_undef_Mask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1323 isVUZPMask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1324 isVUZP_v_undef_Mask(M: Mask, VT: LT.second, WhichResult&: Unused)))
1325 return LT.first;
1326 }
1327 }
1328 if (ST->hasMVEIntegerOps()) {
1329 if (Kind == TTI::SK_Broadcast) {
1330 static const CostTblEntry MVEDupTbl[] = {
1331 // VDUP handles these cases.
1332 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 1},
1333 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 1},
1334 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 1},
1335 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 1},
1336 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8f16, .Cost: 1}};
1337
1338 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1339 if (const auto *Entry = CostTableLookup(Table: MVEDupTbl, ISD: ISD::VECTOR_SHUFFLE,
1340 Ty: LT.second))
1341 return LT.first * Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
1342 }
1343
1344 if (!Mask.empty()) {
1345 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1346 // Check for LD2/LD4 instructions, which are represented in llvm IR as
1347 // deinterleaving-shuffle(load). The shuffle cost could potentially be
1348 // free, but we model it with a cost of LT.first so that LD2/LD4 have a
1349 // higher cost than just the load.
1350 if (Args.size() >= 1 && isa<LoadInst>(Val: Args[0]) &&
1351 (LT.second.getScalarSizeInBits() == 8 ||
1352 LT.second.getScalarSizeInBits() == 16 ||
1353 LT.second.getScalarSizeInBits() == 32) &&
1354 LT.second.getSizeInBits() == 128 &&
1355 ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
1356 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2)) ||
1357 (TLI->getMaxSupportedInterleaveFactor() == 4 &&
1358 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 4))))
1359 return ST->getMVEVectorCostFactor(CostKind) *
1360 std::max<InstructionCost>(a: 1, b: LT.first / 4);
1361
1362 // Check for ST2/ST4 instructions, which are represented in llvm IR as
1363 // store(interleaving-shuffle). The shuffle cost could potentially be
1364 // free, but we model it with a cost of LT.first so that ST2/ST4 have a
1365 // higher cost than just the store.
1366 if (CtxI && CtxI->hasOneUse() && isa<StoreInst>(Val: *CtxI->user_begin()) &&
1367 (LT.second.getScalarSizeInBits() == 8 ||
1368 LT.second.getScalarSizeInBits() == 16 ||
1369 LT.second.getScalarSizeInBits() == 32) &&
1370 LT.second.getSizeInBits() == 128 &&
1371 ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
1372 ShuffleVectorInst::isInterleaveMask(
1373 Mask, Factor: 2, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2)) ||
1374 (TLI->getMaxSupportedInterleaveFactor() == 4 &&
1375 ShuffleVectorInst::isInterleaveMask(
1376 Mask, Factor: 4, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2))))
1377 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1378
1379 if (LT.second.isVector() &&
1380 Mask.size() <= LT.second.getVectorNumElements() &&
1381 (isVREVMask(M: Mask, VT: LT.second, BlockSize: 16) || isVREVMask(M: Mask, VT: LT.second, BlockSize: 32) ||
1382 isVREVMask(M: Mask, VT: LT.second, BlockSize: 64)))
1383 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1384 }
1385 }
1386
1387 // Restore optimal kind.
1388 if (IsExtractSubvector)
1389 Kind = TTI::SK_ExtractSubvector;
1390 int BaseCost = ST->hasMVEIntegerOps() && SrcTy->isVectorTy()
1391 ? ST->getMVEVectorCostFactor(CostKind)
1392 : 1;
1393 return BaseCost * BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask,
1394 Index, SubTp);
1395}
1396
1397InstructionCost ARMTTIImpl::getArithmeticInstrCost(
1398 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
1399 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
1400 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
1401 int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
1402 if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(BitWidth: 1)) {
1403 // Make operations on i1 relatively expensive as this often involves
1404 // combining predicates. AND and XOR should be easier to handle with IT
1405 // blocks.
1406 switch (ISDOpcode) {
1407 default:
1408 break;
1409 case ISD::AND:
1410 case ISD::XOR:
1411 return 2;
1412 case ISD::OR:
1413 return 3;
1414 }
1415 }
1416
1417 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1418
1419 if (ST->hasNEON()) {
1420 const unsigned FunctionCallDivCost = 20;
1421 const unsigned ReciprocalDivCost = 10;
1422 static const CostTblEntry CostTbl[] = {
1423 // Division.
1424 // These costs are somewhat random. Choose a cost of 20 to indicate that
1425 // vectorizing devision (added function call) is going to be very expensive.
1426 // Double registers types.
1427 { .ISD: ISD::SDIV, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1428 { .ISD: ISD::UDIV, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1429 { .ISD: ISD::SREM, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1430 { .ISD: ISD::UREM, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1431 { .ISD: ISD::SDIV, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1432 { .ISD: ISD::UDIV, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1433 { .ISD: ISD::SREM, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1434 { .ISD: ISD::UREM, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1435 { .ISD: ISD::SDIV, .Type: MVT::v4i16, .Cost: ReciprocalDivCost},
1436 { .ISD: ISD::UDIV, .Type: MVT::v4i16, .Cost: ReciprocalDivCost},
1437 { .ISD: ISD::SREM, .Type: MVT::v4i16, .Cost: 4 * FunctionCallDivCost},
1438 { .ISD: ISD::UREM, .Type: MVT::v4i16, .Cost: 4 * FunctionCallDivCost},
1439 { .ISD: ISD::SDIV, .Type: MVT::v8i8, .Cost: ReciprocalDivCost},
1440 { .ISD: ISD::UDIV, .Type: MVT::v8i8, .Cost: ReciprocalDivCost},
1441 { .ISD: ISD::SREM, .Type: MVT::v8i8, .Cost: 8 * FunctionCallDivCost},
1442 { .ISD: ISD::UREM, .Type: MVT::v8i8, .Cost: 8 * FunctionCallDivCost},
1443 // Quad register types.
1444 { .ISD: ISD::SDIV, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1445 { .ISD: ISD::UDIV, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1446 { .ISD: ISD::SREM, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1447 { .ISD: ISD::UREM, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1448 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1449 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1450 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1451 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1452 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1453 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1454 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1455 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1456 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1457 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1458 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1459 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1460 // Multiplication.
1461 };
1462
1463 if (const auto *Entry = CostTableLookup(Table: CostTbl, ISD: ISDOpcode, Ty: LT.second))
1464 return LT.first * Entry->Cost;
1465
1466 InstructionCost Cost = BaseT::getArithmeticInstrCost(
1467 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
1468
1469 // This is somewhat of a hack. The problem that we are facing is that SROA
1470 // creates a sequence of shift, and, or instructions to construct values.
1471 // These sequences are recognized by the ISel and have zero-cost. Not so for
1472 // the vectorized code. Because we have support for v2i64 but not i64 those
1473 // sequences look particularly beneficial to vectorize.
1474 // To work around this we increase the cost of v2i64 operations to make them
1475 // seem less beneficial.
1476 if (LT.second == MVT::v2i64 && Op2Info.isUniform() && Op2Info.isConstant())
1477 Cost += 4;
1478
1479 return Cost;
1480 }
1481
1482 // If this operation is a shift on arm/thumb2, it might well be folded into
1483 // the following instruction, hence having a cost of 0.
1484 auto LooksLikeAFreeShift = [&]() {
1485 if (ST->isThumb1Only() || Ty->isVectorTy())
1486 return false;
1487
1488 if (!CtxI || !CtxI->hasOneUse() || !CtxI->isShift())
1489 return false;
1490 if (!Op2Info.isUniform() || !Op2Info.isConstant())
1491 return false;
1492
1493 // Folded into a ADC/ADD/AND/BIC/CMP/EOR/MVN/ORR/ORN/RSB/SBC/SUB
1494 switch (cast<Instruction>(Val: CtxI->user_back())->getOpcode()) {
1495 case Instruction::Add:
1496 case Instruction::Sub:
1497 case Instruction::And:
1498 case Instruction::Xor:
1499 case Instruction::Or:
1500 case Instruction::ICmp:
1501 return true;
1502 default:
1503 return false;
1504 }
1505 };
1506 if (LooksLikeAFreeShift())
1507 return 0;
1508
1509 // When targets have both DSP and MVE we find that the
1510 // the compiler will attempt to vectorize as well as using
1511 // scalar (S/U)MLAL operations. This is in cases where we have
1512 // the pattern ext(mul(ext(i16), ext(i16))) we find
1513 // that codegen performs better when only using (S/U)MLAL scalar
1514 // ops instead of trying to mix vector ops with (S/U)MLAL ops. We therefore
1515 // check if a mul instruction is used in a (U/S)MLAL pattern.
1516 auto MulInDSPMLALPattern = [&](const Instruction *I, unsigned Opcode,
1517 Type *Ty) -> bool {
1518 if (!ST->hasDSP())
1519 return false;
1520
1521 if (!I)
1522 return false;
1523
1524 if (Opcode != Instruction::Mul)
1525 return false;
1526
1527 if (Ty->isVectorTy())
1528 return false;
1529
1530 auto ValueOpcodesEqual = [](const Value *LHS, const Value *RHS) -> bool {
1531 return cast<Instruction>(Val: LHS)->getOpcode() ==
1532 cast<Instruction>(Val: RHS)->getOpcode();
1533 };
1534 auto IsExtInst = [](const Value *V) -> bool {
1535 return isa<ZExtInst>(Val: V) || isa<SExtInst>(Val: V);
1536 };
1537 auto IsExtensionFromHalf = [](const Value *V) -> bool {
1538 return cast<Instruction>(Val: V)->getOperand(i: 0)->getType()->isIntegerTy(BitWidth: 16);
1539 };
1540
1541 // We check the arguments of the instruction to see if they're extends
1542 auto *BinOp = dyn_cast<BinaryOperator>(Val: I);
1543 if (!BinOp)
1544 return false;
1545 Value *Op0 = BinOp->getOperand(i_nocapture: 0);
1546 Value *Op1 = BinOp->getOperand(i_nocapture: 1);
1547 if (IsExtInst(Op0) && IsExtInst(Op1) && ValueOpcodesEqual(Op0, Op1)) {
1548 // We're interested in an ext of an i16
1549 if (!I->getType()->isIntegerTy(BitWidth: 32) || !IsExtensionFromHalf(Op0) ||
1550 !IsExtensionFromHalf(Op1))
1551 return false;
1552 // We need to check if this result will be further extended to i64
1553 // and that all these uses are SExt
1554 for (auto *U : I->users())
1555 if (!IsExtInst(U))
1556 return false;
1557 return true;
1558 }
1559
1560 return false;
1561 };
1562
1563 if (MulInDSPMLALPattern(CtxI, Opcode, Ty))
1564 return 0;
1565
1566 // Default to cheap (throughput/size of 1 instruction) but adjust throughput
1567 // for "multiple beats" potentially needed by MVE instructions.
1568 int BaseCost = 1;
1569 if (ST->hasMVEIntegerOps() && Ty->isVectorTy())
1570 BaseCost = ST->getMVEVectorCostFactor(CostKind);
1571
1572 // The rest of this mostly follows what is done in
1573 // BaseT::getArithmeticInstrCost, without treating floats as more expensive
1574 // that scalars or increasing the costs for custom operations. The results is
1575 // also multiplied by the MVEVectorCostFactor where appropriate.
1576 if (TLI->isOperationLegalOrCustomOrPromote(Op: ISDOpcode, VT: LT.second))
1577 return LT.first * BaseCost;
1578
1579 // Else this is expand, assume that we need to scalarize this op.
1580 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
1581 unsigned Num = VTy->getNumElements();
1582 InstructionCost Cost =
1583 getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind);
1584 // Return the cost of multiple scalar invocation plus the cost of
1585 // inserting and extracting the values.
1586 SmallVector<Type *> Tys(Args.size(), Ty);
1587 return BaseT::getScalarizationOverhead(RetTy: VTy, Args, Tys, CostKind) +
1588 Num * Cost;
1589 }
1590
1591 return BaseCost;
1592}
1593
1594InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
1595 Align Alignment,
1596 unsigned AddressSpace,
1597 TTI::TargetCostKind CostKind,
1598 TTI::OperandValueInfo OpInfo,
1599 const Instruction *I) const {
1600 // FIXME: Load latency isn't handled here
1601 if (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)
1602 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1603 CostKind, OpInfo, I);
1604
1605 // TODO: Handle other cost kinds.
1606 if (CostKind != TTI::TCK_RecipThroughput)
1607 return 1;
1608
1609 // Type legalization can't handle structs
1610 if (TLI->getValueType(DL, Ty: Src, AllowUnknown: true) == MVT::Other)
1611 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1612 CostKind);
1613
1614 if (ST->hasNEON() && Src->isVectorTy() && Alignment != Align(16) &&
1615 cast<VectorType>(Val: Src)->getElementType()->isDoubleTy()) {
1616 // Unaligned loads/stores are extremely inefficient.
1617 // We need 4 uops for vst.1/vld.1 vs 1uop for vldr/vstr.
1618 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
1619 return LT.first * 4;
1620 }
1621
1622 // MVE can optimize a fpext(load(4xhalf)) using an extending integer load.
1623 // Same for stores.
1624 if (ST->hasMVEFloatOps() && isa<FixedVectorType>(Val: Src) && I &&
1625 ((Opcode == Instruction::Load && I->hasOneUse() &&
1626 isa<FPExtInst>(Val: *I->user_begin())) ||
1627 (Opcode == Instruction::Store && isa<FPTruncInst>(Val: I->getOperand(i: 0))))) {
1628 FixedVectorType *SrcVTy = cast<FixedVectorType>(Val: Src);
1629 Type *DstTy =
1630 Opcode == Instruction::Load
1631 ? (*I->user_begin())->getType()
1632 : cast<Instruction>(Val: I->getOperand(i: 0))->getOperand(i: 0)->getType();
1633 if (SrcVTy->getNumElements() == 4 && SrcVTy->getScalarType()->isHalfTy() &&
1634 DstTy->getScalarType()->isFloatTy())
1635 return ST->getMVEVectorCostFactor(CostKind);
1636 }
1637
1638 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
1639 ? ST->getMVEVectorCostFactor(CostKind)
1640 : 1;
1641 return BaseCost * BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1642 CostKind, OpInfo, I);
1643}
1644
1645InstructionCost
1646ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1647 TTI::TargetCostKind CostKind) const {
1648 switch (MICA.getID()) {
1649 case Intrinsic::masked_scatter:
1650 case Intrinsic::masked_gather:
1651 return getGatherScatterOpCost(MICA, CostKind);
1652 case Intrinsic::masked_load:
1653 case Intrinsic::masked_store:
1654 return getMaskedMemoryOpCost(MICA, CostKind);
1655 }
1656 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1657}
1658
1659InstructionCost
1660ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1661 TTI::TargetCostKind CostKind) const {
1662 unsigned IID = MICA.getID();
1663 Type *Src = MICA.getDataType();
1664 Align Alignment = MICA.getAlignment();
1665 unsigned AddressSpace = MICA.getAddressSpace();
1666 if (ST->hasMVEIntegerOps()) {
1667 if (IID == Intrinsic::masked_load &&
1668 isLegalMaskedLoad(DataTy: Src, Alignment, AddressSpace))
1669 return ST->getMVEVectorCostFactor(CostKind);
1670 if (IID == Intrinsic::masked_store &&
1671 isLegalMaskedStore(DataTy: Src, Alignment, AddressSpace))
1672 return ST->getMVEVectorCostFactor(CostKind);
1673 }
1674 if (!isa<FixedVectorType>(Val: Src))
1675 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1676 // Scalar cost, which is currently very high due to the efficiency of the
1677 // generated code.
1678 return cast<FixedVectorType>(Val: Src)->getNumElements() * 8;
1679}
1680
1681InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost(
1682 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1683 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1684 bool UseMaskForCond, bool UseMaskForGaps) const {
1685 assert(Factor >= 2 && "Invalid interleave factor");
1686 assert(isa<VectorType>(VecTy) && "Expect a vector type");
1687
1688 // vldN/vstN doesn't support vector types of i64/f64 element.
1689 bool EltIs64Bits = DL.getTypeSizeInBits(Ty: VecTy->getScalarType()) == 64;
1690
1691 if (Factor <= TLI->getMaxSupportedInterleaveFactor() && !EltIs64Bits &&
1692 !UseMaskForCond && !UseMaskForGaps) {
1693 unsigned NumElts = cast<FixedVectorType>(Val: VecTy)->getNumElements();
1694 auto *SubVecTy =
1695 FixedVectorType::get(ElementType: VecTy->getScalarType(), NumElts: NumElts / Factor);
1696
1697 // vldN/vstN only support legal vector types of size 64 or 128 in bits.
1698 // Accesses having vector types that are a multiple of 128 bits can be
1699 // matched to more than one vldN/vstN instruction.
1700 int BaseCost =
1701 ST->hasMVEIntegerOps() ? ST->getMVEVectorCostFactor(CostKind) : 1;
1702 if (NumElts % Factor == 0 &&
1703 TLI->isLegalInterleavedAccessType(Factor, VecTy: SubVecTy, Alignment, DL))
1704 return Factor * BaseCost * TLI->getNumInterleavedAccesses(VecTy: SubVecTy, DL);
1705
1706 // Some smaller than legal interleaved patterns are cheap as we can make
1707 // use of the vmovn or vrev patterns to interleave a standard load. This is
1708 // true for v4i8, v8i8 and v4i16 at least (but not for v4f16 as it is
1709 // promoted differently). The cost of 2 here is then a load and vrev or
1710 // vmovn.
1711 if (ST->hasMVEIntegerOps() && Factor == 2 && NumElts / Factor > 2 &&
1712 VecTy->isIntOrIntVectorTy() &&
1713 DL.getTypeSizeInBits(Ty: SubVecTy).getFixedValue() <= 64)
1714 return 2 * BaseCost;
1715 }
1716
1717 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1718 Alignment, AddressSpace, CostKind,
1719 UseMaskForCond, UseMaskForGaps);
1720}
1721
1722InstructionCost
1723ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1724 TTI::TargetCostKind CostKind) const {
1725
1726 Type *DataTy = MICA.getDataType();
1727 const Value *Ptr = MICA.getPointer();
1728 bool VariableMask = MICA.getVariableMask();
1729 Align Alignment = MICA.getAlignment();
1730 const Instruction *I = MICA.getInst();
1731
1732 using namespace PatternMatch;
1733 if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
1734 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1735
1736 assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!");
1737 auto *VTy = cast<FixedVectorType>(Val: DataTy);
1738
1739 // TODO: Splitting, once we do that.
1740
1741 unsigned NumElems = VTy->getNumElements();
1742 unsigned EltSize = VTy->getScalarSizeInBits();
1743 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: DataTy);
1744
1745 // For now, it is assumed that for the MVE gather instructions the loads are
1746 // all effectively serialised. This means the cost is the scalar cost
1747 // multiplied by the number of elements being loaded. This is possibly very
1748 // conservative, but even so we still end up vectorising loops because the
1749 // cost per iteration for many loops is lower than for scalar loops.
1750 InstructionCost VectorCost =
1751 NumElems * LT.first * ST->getMVEVectorCostFactor(CostKind);
1752 // The scalarization cost should be a lot higher. We use the number of vector
1753 // elements plus the scalarization overhead. If masking is required then a lot
1754 // of little blocks will be needed and potentially a scalarized p0 mask,
1755 // greatly increasing the cost.
1756 InstructionCost ScalarCost =
1757 NumElems * LT.first + (VariableMask ? NumElems * 5 : 0) +
1758 BaseT::getScalarizationOverhead(InTy: VTy, /*Insert*/ true, /*Extract*/ false,
1759 CostKind) +
1760 BaseT::getScalarizationOverhead(InTy: VTy, /*Insert*/ false, /*Extract*/ true,
1761 CostKind);
1762
1763 if (EltSize < 8 || Alignment < EltSize / 8)
1764 return ScalarCost;
1765
1766 unsigned ExtSize = EltSize;
1767 // Check whether there's a single user that asks for an extended type
1768 if (I != nullptr) {
1769 // Dependent of the caller of this function, a gather instruction will
1770 // either have opcode Instruction::Load or be a call to the masked_gather
1771 // intrinsic
1772 if ((I->getOpcode() == Instruction::Load ||
1773 match(V: I, P: m_Intrinsic<Intrinsic::masked_gather>())) &&
1774 I->hasOneUse()) {
1775 const User *Us = *I->users().begin();
1776 if (isa<ZExtInst>(Val: Us) || isa<SExtInst>(Val: Us)) {
1777 // only allow valid type combinations
1778 unsigned TypeSize =
1779 cast<Instruction>(Val: Us)->getType()->getScalarSizeInBits();
1780 if (((TypeSize == 32 && (EltSize == 8 || EltSize == 16)) ||
1781 (TypeSize == 16 && EltSize == 8)) &&
1782 TypeSize * NumElems == 128) {
1783 ExtSize = TypeSize;
1784 }
1785 }
1786 }
1787 // Check whether the input data needs to be truncated
1788 TruncInst *T;
1789 if ((I->getOpcode() == Instruction::Store ||
1790 match(V: I, P: m_Intrinsic<Intrinsic::masked_scatter>())) &&
1791 (T = dyn_cast<TruncInst>(Val: I->getOperand(i: 0)))) {
1792 // Only allow valid type combinations
1793 unsigned TypeSize = T->getOperand(i_nocapture: 0)->getType()->getScalarSizeInBits();
1794 if (((EltSize == 16 && TypeSize == 32) ||
1795 (EltSize == 8 && (TypeSize == 32 || TypeSize == 16))) &&
1796 TypeSize * NumElems == 128)
1797 ExtSize = TypeSize;
1798 }
1799 }
1800
1801 if (ExtSize * NumElems != 128 || NumElems < 4)
1802 return ScalarCost;
1803
1804 // Any (aligned) i32 gather will not need to be scalarised.
1805 if (ExtSize == 32)
1806 return VectorCost;
1807 // For smaller types, we need to ensure that the gep's inputs are correctly
1808 // extended from a small enough value. Other sizes (including i64) are
1809 // scalarized for now.
1810 if (ExtSize != 8 && ExtSize != 16)
1811 return ScalarCost;
1812
1813 if (const auto *BC = dyn_cast<BitCastInst>(Val: Ptr))
1814 Ptr = BC->getOperand(i_nocapture: 0);
1815 if (const auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr)) {
1816 if (GEP->getNumOperands() != 2)
1817 return ScalarCost;
1818 unsigned Scale = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
1819 // Scale needs to be correct (which is only relevant for i16s).
1820 if (Scale != 1 && Scale * 8 != ExtSize)
1821 return ScalarCost;
1822 // And we need to zext (not sext) the indexes from a small enough type.
1823 if (const auto *ZExt = dyn_cast<ZExtInst>(Val: GEP->getOperand(i_nocapture: 1))) {
1824 if (ZExt->getOperand(i_nocapture: 0)->getType()->getScalarSizeInBits() <= ExtSize)
1825 return VectorCost;
1826 }
1827 return ScalarCost;
1828 }
1829 return ScalarCost;
1830}
1831
1832InstructionCost
1833ARMTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
1834 std::optional<FastMathFlags> FMF,
1835 TTI::TargetCostKind CostKind) const {
1836
1837 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1838 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1839 unsigned EltSize = ValVT.getScalarSizeInBits();
1840
1841 // In general floating point reductions are a series of elementwise
1842 // operations, with free extracts on each step. These are either in-order or
1843 // treewise depending on whether that is allowed by the fast math flags.
1844 if ((ISD == ISD::FADD || ISD == ISD::FMUL) &&
1845 ((EltSize == 32 && ST->hasVFP2Base()) ||
1846 (EltSize == 64 && ST->hasFP64()) ||
1847 (EltSize == 16 && ST->hasFullFP16()))) {
1848 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
1849 unsigned VecLimit = ST->hasMVEFloatOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1850 InstructionCost VecCost = 0;
1851 while (!TTI::requiresOrderedReduction(FMF) && isPowerOf2_32(Value: NumElts) &&
1852 NumElts * EltSize > VecLimit) {
1853 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts: NumElts / 2);
1854 VecCost += getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1855 NumElts /= 2;
1856 }
1857
1858 // For fp16 we need to extract the upper lane elements. MVE can add a
1859 // VREV+FMIN/MAX to perform another vector step instead.
1860 InstructionCost ExtractCost = 0;
1861 if (!TTI::requiresOrderedReduction(FMF) && ST->hasMVEFloatOps() &&
1862 ValVT.getVectorElementType() == MVT::f16 && NumElts == 8) {
1863 VecCost += ST->getMVEVectorCostFactor(CostKind) * 2;
1864 NumElts /= 2;
1865 } else if (ValVT.getVectorElementType() == MVT::f16)
1866 ExtractCost = NumElts / 2;
1867
1868 return VecCost + ExtractCost +
1869 NumElts *
1870 getArithmeticInstrCost(Opcode, Ty: ValTy->getElementType(), CostKind);
1871 }
1872
1873 if ((ISD == ISD::AND || ISD == ISD::OR || ISD == ISD::XOR) &&
1874 (EltSize == 64 || EltSize == 32 || EltSize == 16 || EltSize == 8)) {
1875 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
1876 unsigned VecLimit =
1877 ST->hasMVEIntegerOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1878 InstructionCost VecCost = 0;
1879 while (isPowerOf2_32(Value: NumElts) && NumElts * EltSize > VecLimit) {
1880 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts: NumElts / 2);
1881 VecCost += getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1882 NumElts /= 2;
1883 }
1884 // For i16/i8, MVE will perform a VREV + VORR/VAND/VEOR for the 64bit vector
1885 // step.
1886 if (ST->hasMVEIntegerOps() && ValVT.getScalarSizeInBits() <= 16 &&
1887 NumElts * EltSize == 64) {
1888 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts);
1889 VecCost += ST->getMVEVectorCostFactor(CostKind) +
1890 getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1891 NumElts /= 2;
1892 }
1893
1894 // From here we extract the elements and perform the and/or/xor.
1895 InstructionCost ExtractCost = NumElts;
1896 return VecCost + ExtractCost +
1897 (NumElts - 1) * getArithmeticInstrCost(
1898 Opcode, Ty: ValTy->getElementType(), CostKind);
1899 }
1900
1901 if (!ST->hasMVEIntegerOps() || !ValVT.isSimple() || ISD != ISD::ADD ||
1902 TTI::requiresOrderedReduction(FMF))
1903 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
1904
1905 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1906
1907 static const CostTblEntry CostTblAdd[]{
1908 {.ISD: ISD::ADD, .Type: MVT::v16i8, .Cost: 1},
1909 {.ISD: ISD::ADD, .Type: MVT::v8i16, .Cost: 1},
1910 {.ISD: ISD::ADD, .Type: MVT::v4i32, .Cost: 1},
1911 };
1912 if (const auto *Entry = CostTableLookup(Table: CostTblAdd, ISD, Ty: LT.second))
1913 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first;
1914
1915 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
1916}
1917
1918InstructionCost ARMTTIImpl::getExtendedReductionCost(
1919 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
1920 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
1921 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1922 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
1923
1924 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1925
1926 switch (ISD) {
1927 case ISD::ADD:
1928 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) {
1929 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1930
1931 // The legal cases are:
1932 // VADDV u/s 8/16/32
1933 // VADDLV u/s 32
1934 // Codegen currently cannot always handle larger than legal vectors very
1935 // well, especially for predicated reductions where the mask needs to be
1936 // split, so restrict to 128bit or smaller input types.
1937 unsigned RevVTSize = ResVT.getSizeInBits();
1938 if (ValVT.getSizeInBits() <= 128 &&
1939 ((LT.second == MVT::v16i8 && RevVTSize <= 32) ||
1940 (LT.second == MVT::v8i16 && RevVTSize <= 32) ||
1941 (LT.second == MVT::v4i32 && RevVTSize <= 64)))
1942 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1943 }
1944 break;
1945 default:
1946 break;
1947 }
1948 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy, FMF,
1949 CostKind);
1950}
1951
1952InstructionCost
1953ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
1954 Type *ResTy, VectorType *ValTy,
1955 TTI::TargetCostKind CostKind) const {
1956 if (RedOpcode != Instruction::Add)
1957 return InstructionCost::getInvalid(Val: CostKind);
1958 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1959 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
1960
1961 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) {
1962 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1963
1964 // The legal cases are:
1965 // VMLAV u/s 8/16/32
1966 // VMLALV u/s 16/32
1967 // Codegen currently cannot always handle larger than legal vectors very
1968 // well, especially for predicated reductions where the mask needs to be
1969 // split, so restrict to 128bit or smaller input types.
1970 unsigned RevVTSize = ResVT.getSizeInBits();
1971 if (ValVT.getSizeInBits() <= 128 &&
1972 ((LT.second == MVT::v16i8 && RevVTSize <= 32) ||
1973 (LT.second == MVT::v8i16 && RevVTSize <= 64) ||
1974 (LT.second == MVT::v4i32 && RevVTSize <= 64)))
1975 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1976 }
1977
1978 return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty: ValTy,
1979 CostKind);
1980}
1981
1982InstructionCost
1983ARMTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
1984 FastMathFlags FMF,
1985 TTI::TargetCostKind CostKind) const {
1986 EVT ValVT = TLI->getValueType(DL, Ty);
1987
1988 // In general floating point reductions are a series of elementwise
1989 // operations, with free extracts on each step. These are either in-order or
1990 // treewise depending on whether that is allowed by the fast math flags.
1991 if ((IID == Intrinsic::minnum || IID == Intrinsic::maxnum) &&
1992 ((ValVT.getVectorElementType() == MVT::f32 && ST->hasVFP2Base()) ||
1993 (ValVT.getVectorElementType() == MVT::f64 && ST->hasFP64()) ||
1994 (ValVT.getVectorElementType() == MVT::f16 && ST->hasFullFP16()))) {
1995 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
1996 unsigned EltSize = ValVT.getScalarSizeInBits();
1997 unsigned VecLimit = ST->hasMVEFloatOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1998 InstructionCost VecCost;
1999 while (isPowerOf2_32(Value: NumElts) && NumElts * EltSize > VecLimit) {
2000 Type *VecTy = FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElts/2);
2001 IntrinsicCostAttributes ICA(IID, VecTy, {VecTy, VecTy}, FMF);
2002 VecCost += getIntrinsicInstrCost(ICA, CostKind);
2003 NumElts /= 2;
2004 }
2005
2006 // For fp16 we need to extract the upper lane elements. MVE can add a
2007 // VREV+FMIN/MAX to perform another vector step instead.
2008 InstructionCost ExtractCost = 0;
2009 if (ST->hasMVEFloatOps() && ValVT.getVectorElementType() == MVT::f16 &&
2010 NumElts == 8) {
2011 VecCost += ST->getMVEVectorCostFactor(CostKind) * 2;
2012 NumElts /= 2;
2013 } else if (ValVT.getVectorElementType() == MVT::f16)
2014 ExtractCost = cast<FixedVectorType>(Val: Ty)->getNumElements() / 2;
2015
2016 IntrinsicCostAttributes ICA(IID, Ty->getElementType(),
2017 {Ty->getElementType(), Ty->getElementType()},
2018 FMF);
2019 return VecCost + ExtractCost +
2020 (NumElts - 1) * getIntrinsicInstrCost(ICA, CostKind);
2021 }
2022
2023 if (IID == Intrinsic::smin || IID == Intrinsic::smax ||
2024 IID == Intrinsic::umin || IID == Intrinsic::umax) {
2025 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2026
2027 // All costs are the same for u/s min/max. These lower to vminv, which are
2028 // given a slightly higher cost as they tend to take multiple cycles for
2029 // smaller type sizes.
2030 static const CostTblEntry CostTblAdd[]{
2031 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: 4},
2032 {.ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: 3},
2033 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: 2},
2034 };
2035 if (const auto *Entry = CostTableLookup(Table: CostTblAdd, ISD: ISD::SMIN, Ty: LT.second))
2036 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first;
2037 }
2038
2039 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2040}
2041
2042InstructionCost
2043ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
2044 TTI::TargetCostKind CostKind) const {
2045 unsigned Opc = ICA.getID();
2046 switch (Opc) {
2047 case Intrinsic::get_active_lane_mask:
2048 // Currently we make a somewhat optimistic assumption that
2049 // active_lane_mask's are always free. In reality it may be freely folded
2050 // into a tail predicated loop, expanded into a VCPT or expanded into a lot
2051 // of add/icmp code. We may need to improve this in the future, but being
2052 // able to detect if it is free or not involves looking at a lot of other
2053 // code. We currently assume that the vectorizer inserted these, and knew
2054 // what it was doing in adding one.
2055 if (ST->hasMVEIntegerOps())
2056 return 0;
2057 break;
2058 case Intrinsic::sadd_sat:
2059 case Intrinsic::ssub_sat:
2060 case Intrinsic::uadd_sat:
2061 case Intrinsic::usub_sat: {
2062 bool IsAdd = (Opc == Intrinsic::sadd_sat || Opc == Intrinsic::ssub_sat);
2063 bool IsSigned = (Opc == Intrinsic::sadd_sat || Opc == Intrinsic::ssub_sat);
2064 Type *RetTy = ICA.getReturnType();
2065
2066 if (auto *ITy = dyn_cast<IntegerType>(Val: RetTy)) {
2067 if (IsSigned && ST->hasDSP() && ITy->getBitWidth() == 32)
2068 return 1; // qadd / qsub
2069 if (ST->hasDSP() && (ITy->getBitWidth() == 8 || ITy->getBitWidth() == 16))
2070 return 2; // uqadd16 / qadd16 / uqsub16 / qsub16 + possible extend.
2071 // Otherwise return the cost of expanding the node. Generally an add +
2072 // icmp + sel.
2073 CmpInst::Predicate Pred = CmpInst::ICMP_SGT;
2074 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2075 return getArithmeticInstrCost(Opcode: IsAdd ? Instruction::Add : Instruction::Sub,
2076 Ty: RetTy, CostKind) +
2077 2 * getCmpSelInstrCost(Opcode: BinaryOperator::ICmp, ValTy: RetTy, CondTy, VecPred: Pred,
2078 CostKind) +
2079 2 * getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy, VecPred: Pred,
2080 CostKind);
2081 }
2082
2083 if (!ST->hasMVEIntegerOps())
2084 break;
2085
2086 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
2087 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 ||
2088 LT.second == MVT::v16i8) {
2089 // This is a base cost of 1 for the vqadd, plus 3 extract shifts if we
2090 // need to extend the type, as it uses shr(qadd(shl, shl)).
2091 unsigned Instrs =
2092 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1
2093 : 4;
2094 return LT.first * ST->getMVEVectorCostFactor(CostKind) * Instrs;
2095 }
2096 break;
2097 }
2098 case Intrinsic::abs:
2099 case Intrinsic::smin:
2100 case Intrinsic::smax:
2101 case Intrinsic::umin:
2102 case Intrinsic::umax: {
2103 if (!ST->hasMVEIntegerOps())
2104 break;
2105 Type *VT = ICA.getReturnType();
2106
2107 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VT);
2108 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 ||
2109 LT.second == MVT::v16i8)
2110 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2111 break;
2112 }
2113 case Intrinsic::minnum:
2114 case Intrinsic::maxnum: {
2115 if (!ST->hasMVEFloatOps())
2116 break;
2117 Type *VT = ICA.getReturnType();
2118 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VT);
2119 if (LT.second == MVT::v4f32 || LT.second == MVT::v8f16)
2120 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2121 break;
2122 }
2123 case Intrinsic::fptosi_sat:
2124 case Intrinsic::fptoui_sat: {
2125 if (ICA.getArgTypes().empty())
2126 break;
2127 bool IsSigned = Opc == Intrinsic::fptosi_sat;
2128 auto LT = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
2129 EVT MTy = TLI->getValueType(DL, Ty: ICA.getReturnType());
2130 // Check for the legal types, with the correct subtarget features.
2131 if ((ST->hasVFP2Base() && LT.second == MVT::f32 && MTy == MVT::i32) ||
2132 (ST->hasFP64() && LT.second == MVT::f64 && MTy == MVT::i32) ||
2133 (ST->hasFullFP16() && LT.second == MVT::f16 && MTy == MVT::i32))
2134 return LT.first;
2135
2136 // Equally for MVE vector types
2137 if (ST->hasMVEFloatOps() &&
2138 (LT.second == MVT::v4f32 || LT.second == MVT::v8f16) &&
2139 LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits())
2140 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2141
2142 // If we can we use a legal convert followed by a min+max
2143 if (((ST->hasVFP2Base() && LT.second == MVT::f32) ||
2144 (ST->hasFP64() && LT.second == MVT::f64) ||
2145 (ST->hasFullFP16() && LT.second == MVT::f16) ||
2146 (ST->hasMVEFloatOps() &&
2147 (LT.second == MVT::v4f32 || LT.second == MVT::v8f16))) &&
2148 LT.second.getScalarSizeInBits() >= MTy.getScalarSizeInBits()) {
2149 Type *LegalTy = Type::getIntNTy(C&: ICA.getReturnType()->getContext(),
2150 N: LT.second.getScalarSizeInBits());
2151 InstructionCost Cost =
2152 LT.second.isVector() ? ST->getMVEVectorCostFactor(CostKind) : 1;
2153 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin
2154 : Intrinsic::umin,
2155 LegalTy, {LegalTy, LegalTy});
2156 Cost += getIntrinsicInstrCost(ICA: Attrs1, CostKind);
2157 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax
2158 : Intrinsic::umax,
2159 LegalTy, {LegalTy, LegalTy});
2160 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
2161 return LT.first * Cost;
2162 }
2163 // Otherwise we need to follow the default expansion that clamps the value
2164 // using a float min/max with a fcmp+sel for nan handling when signed.
2165 Type *FPTy = ICA.getArgTypes()[0];
2166 Type *RetTy = ICA.getReturnType();
2167 IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FPTy, {FPTy, FPTy});
2168 InstructionCost Cost = getIntrinsicInstrCost(ICA: Attrs1, CostKind);
2169 IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FPTy, {FPTy, FPTy});
2170 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
2171 Cost +=
2172 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
2173 Dst: RetTy, Src: FPTy, CCH: TTI::CastContextHint::None, CostKind);
2174 if (IsSigned) {
2175 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2176 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: FPTy, CondTy,
2177 VecPred: CmpInst::FCMP_UNO, CostKind);
2178 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
2179 VecPred: CmpInst::FCMP_UNO, CostKind);
2180 }
2181 return Cost;
2182 }
2183 }
2184
2185 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
2186}
2187
2188bool ARMTTIImpl::isLoweredToCall(const Function *F) const {
2189 if (!F->isIntrinsic())
2190 return BaseT::isLoweredToCall(F);
2191
2192 // Assume all Arm-specific intrinsics map to an instruction.
2193 if (F->getName().starts_with(Prefix: "llvm.arm"))
2194 return false;
2195
2196 switch (F->getIntrinsicID()) {
2197 default: break;
2198 case Intrinsic::powi:
2199 case Intrinsic::sin:
2200 case Intrinsic::cos:
2201 case Intrinsic::sincos:
2202 case Intrinsic::pow:
2203 case Intrinsic::log:
2204 case Intrinsic::log10:
2205 case Intrinsic::log2:
2206 case Intrinsic::exp:
2207 case Intrinsic::exp2:
2208 return true;
2209 case Intrinsic::sqrt:
2210 case Intrinsic::fabs:
2211 case Intrinsic::copysign:
2212 case Intrinsic::floor:
2213 case Intrinsic::ceil:
2214 case Intrinsic::trunc:
2215 case Intrinsic::rint:
2216 case Intrinsic::nearbyint:
2217 case Intrinsic::round:
2218 case Intrinsic::canonicalize:
2219 case Intrinsic::lround:
2220 case Intrinsic::llround:
2221 case Intrinsic::lrint:
2222 case Intrinsic::llrint:
2223 if (F->getReturnType()->isDoubleTy() && !ST->hasFP64())
2224 return true;
2225 if (F->getReturnType()->isHalfTy() && !ST->hasFullFP16())
2226 return true;
2227 // Some operations can be handled by vector instructions and assume
2228 // unsupported vectors will be expanded into supported scalar ones.
2229 // TODO Handle scalar operations properly.
2230 return !ST->hasFPARMv8Base() && !ST->hasVFP2Base();
2231 case Intrinsic::masked_store:
2232 case Intrinsic::masked_load:
2233 case Intrinsic::masked_gather:
2234 case Intrinsic::masked_scatter:
2235 return !ST->hasMVEIntegerOps();
2236 case Intrinsic::sadd_with_overflow:
2237 case Intrinsic::uadd_with_overflow:
2238 case Intrinsic::ssub_with_overflow:
2239 case Intrinsic::usub_with_overflow:
2240 case Intrinsic::sadd_sat:
2241 case Intrinsic::uadd_sat:
2242 case Intrinsic::ssub_sat:
2243 case Intrinsic::usub_sat:
2244 return false;
2245 }
2246
2247 return BaseT::isLoweredToCall(F);
2248}
2249
2250bool ARMTTIImpl::maybeLoweredToCall(Instruction &I) const {
2251 unsigned ISD = TLI->InstructionOpcodeToISD(Opcode: I.getOpcode());
2252 EVT VT = TLI->getValueType(DL, Ty: I.getType(), AllowUnknown: true);
2253 if (TLI->getOperationAction(Op: ISD, VT) == TargetLowering::LibCall)
2254 return true;
2255
2256 // Check if an intrinsic will be lowered to a call and assume that any
2257 // other CallInst will generate a bl.
2258 if (auto *Call = dyn_cast<CallInst>(Val: &I)) {
2259 if (auto *II = dyn_cast<IntrinsicInst>(Val: Call)) {
2260 switch(II->getIntrinsicID()) {
2261 case Intrinsic::memcpy:
2262 case Intrinsic::memset:
2263 case Intrinsic::memmove:
2264 return getNumMemOps(I: II) == -1;
2265 default:
2266 if (const Function *F = Call->getCalledFunction())
2267 return isLoweredToCall(F);
2268 }
2269 }
2270 return true;
2271 }
2272
2273 // FPv5 provides conversions between integer, double-precision,
2274 // single-precision, and half-precision formats.
2275 switch (I.getOpcode()) {
2276 default:
2277 break;
2278 case Instruction::FPToSI:
2279 case Instruction::FPToUI:
2280 case Instruction::SIToFP:
2281 case Instruction::UIToFP:
2282 case Instruction::FPTrunc:
2283 case Instruction::FPExt:
2284 return !ST->hasFPARMv8Base();
2285 }
2286
2287 // FIXME: Unfortunately the approach of checking the Operation Action does
2288 // not catch all cases of Legalization that use library calls. Our
2289 // Legalization step categorizes some transformations into library calls as
2290 // Custom, Expand or even Legal when doing type legalization. So for now
2291 // we have to special case for instance the SDIV of 64bit integers and the
2292 // use of floating point emulation.
2293 if (VT.isInteger() && VT.getSizeInBits() >= 64) {
2294 switch (ISD) {
2295 default:
2296 break;
2297 case ISD::SDIV:
2298 case ISD::UDIV:
2299 case ISD::SREM:
2300 case ISD::UREM:
2301 case ISD::SDIVREM:
2302 case ISD::UDIVREM:
2303 return true;
2304 }
2305 }
2306
2307 // Assume all other non-float operations are supported.
2308 if (!VT.isFloatingPoint())
2309 return false;
2310
2311 // We'll need a library call to handle most floats when using soft.
2312 if (TLI->useSoftFloat()) {
2313 switch (I.getOpcode()) {
2314 default:
2315 return true;
2316 case Instruction::Alloca:
2317 case Instruction::Load:
2318 case Instruction::Store:
2319 case Instruction::Select:
2320 case Instruction::PHI:
2321 return false;
2322 }
2323 }
2324
2325 // We'll need a libcall to perform double precision operations on a single
2326 // precision only FPU.
2327 if (I.getType()->isDoubleTy() && !ST->hasFP64())
2328 return true;
2329
2330 // Likewise for half precision arithmetic.
2331 if (I.getType()->isHalfTy() && !ST->hasFullFP16())
2332 return true;
2333
2334 return false;
2335}
2336
2337bool ARMTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
2338 AssumptionCache &AC,
2339 TargetLibraryInfo *LibInfo,
2340 HardwareLoopInfo &HWLoopInfo) const {
2341 // Low-overhead branches are only supported in the 'low-overhead branch'
2342 // extension of v8.1-m.
2343 if (!ST->hasLOB() || DisableLowOverheadLoops) {
2344 LLVM_DEBUG(dbgs() << "ARMHWLoops: Disabled\n");
2345 return false;
2346 }
2347
2348 if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
2349 LLVM_DEBUG(dbgs() << "ARMHWLoops: No BETC\n");
2350 return false;
2351 }
2352
2353 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2354 if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount)) {
2355 LLVM_DEBUG(dbgs() << "ARMHWLoops: Uncomputable BETC\n");
2356 return false;
2357 }
2358
2359 const SCEV *TripCountSCEV =
2360 SE.getAddExpr(LHS: BackedgeTakenCount,
2361 RHS: SE.getOne(Ty: BackedgeTakenCount->getType()));
2362
2363 // We need to store the trip count in LR, a 32-bit register.
2364 if (SE.getUnsignedRangeMax(S: TripCountSCEV).getBitWidth() > 32) {
2365 LLVM_DEBUG(dbgs() << "ARMHWLoops: Trip count does not fit into 32bits\n");
2366 return false;
2367 }
2368
2369 // Making a call will trash LR and clear LO_BRANCH_INFO, so there's little
2370 // point in generating a hardware loop if that's going to happen.
2371
2372 auto IsHardwareLoopIntrinsic = [](Instruction &I) {
2373 if (auto *Call = dyn_cast<IntrinsicInst>(Val: &I)) {
2374 switch (Call->getIntrinsicID()) {
2375 default:
2376 break;
2377 case Intrinsic::start_loop_iterations:
2378 case Intrinsic::test_start_loop_iterations:
2379 case Intrinsic::loop_decrement:
2380 case Intrinsic::loop_decrement_reg:
2381 return true;
2382 }
2383 }
2384 return false;
2385 };
2386
2387 // Scan the instructions to see if there's any that we know will turn into a
2388 // call or if this loop is already a low-overhead loop or will become a tail
2389 // predicated loop.
2390 bool IsTailPredLoop = false;
2391 auto ScanLoop = [&](Loop *L) {
2392 for (auto *BB : L->getBlocks()) {
2393 for (auto &I : *BB) {
2394 if (maybeLoweredToCall(I) || IsHardwareLoopIntrinsic(I) ||
2395 isa<InlineAsm>(Val: I)) {
2396 LLVM_DEBUG(dbgs() << "ARMHWLoops: Bad instruction: " << I << "\n");
2397 return false;
2398 }
2399 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
2400 IsTailPredLoop |=
2401 II->getIntrinsicID() == Intrinsic::get_active_lane_mask ||
2402 II->getIntrinsicID() == Intrinsic::arm_mve_vctp8 ||
2403 II->getIntrinsicID() == Intrinsic::arm_mve_vctp16 ||
2404 II->getIntrinsicID() == Intrinsic::arm_mve_vctp32 ||
2405 II->getIntrinsicID() == Intrinsic::arm_mve_vctp64;
2406 }
2407 }
2408 return true;
2409 };
2410
2411 // Visit inner loops.
2412 for (auto *Inner : *L)
2413 if (!ScanLoop(Inner))
2414 return false;
2415
2416 if (!ScanLoop(L))
2417 return false;
2418
2419 // TODO: Check whether the trip count calculation is expensive. If L is the
2420 // inner loop but we know it has a low trip count, calculating that trip
2421 // count (in the parent loop) may be detrimental.
2422
2423 LLVMContext &C = L->getHeader()->getContext();
2424 HWLoopInfo.CounterInReg = true;
2425 HWLoopInfo.IsNestingLegal = false;
2426 HWLoopInfo.PerformEntryTest = AllowWLSLoops && !IsTailPredLoop;
2427 HWLoopInfo.CountType = Type::getInt32Ty(C);
2428 HWLoopInfo.LoopDecrement = ConstantInt::get(Ty: HWLoopInfo.CountType, V: 1);
2429 return true;
2430}
2431
2432static bool canTailPredicateInstruction(Instruction &I, int &ICmpCount) {
2433 // We don't allow icmp's, and because we only look at single block loops,
2434 // we simply count the icmps, i.e. there should only be 1 for the backedge.
2435 if (isa<ICmpInst>(Val: &I) && ++ICmpCount > 1)
2436 return false;
2437 // FIXME: This is a workaround for poor cost modelling. Min/Max intrinsics are
2438 // not currently canonical, but soon will be. Code without them uses icmp, and
2439 // so is not tail predicated as per the condition above. In order to get the
2440 // same performance we treat min and max the same as an icmp for tailpred
2441 // purposes for the moment (we often rely on non-tailpred and higher VF's to
2442 // pick more optimal instructions like VQDMULH. They need to be recognized
2443 // directly by the vectorizer).
2444 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
2445 if ((II->getIntrinsicID() == Intrinsic::smin ||
2446 II->getIntrinsicID() == Intrinsic::smax ||
2447 II->getIntrinsicID() == Intrinsic::umin ||
2448 II->getIntrinsicID() == Intrinsic::umax) &&
2449 ++ICmpCount > 1)
2450 return false;
2451
2452 if (isa<FCmpInst>(Val: &I))
2453 return false;
2454
2455 // We could allow extending/narrowing FP loads/stores, but codegen is
2456 // too inefficient so reject this for now.
2457 if (isa<FPExtInst>(Val: &I) || isa<FPTruncInst>(Val: &I))
2458 return false;
2459
2460 // Extends have to be extending-loads
2461 if (isa<SExtInst>(Val: &I) || isa<ZExtInst>(Val: &I) )
2462 if (!I.getOperand(i: 0)->hasOneUse() || !isa<LoadInst>(Val: I.getOperand(i: 0)))
2463 return false;
2464
2465 // Truncs have to be narrowing-stores
2466 if (isa<TruncInst>(Val: &I) )
2467 if (!I.hasOneUse() || !isa<StoreInst>(Val: *I.user_begin()))
2468 return false;
2469
2470 return true;
2471}
2472
2473// To set up a tail-predicated loop, we need to know the total number of
2474// elements processed by that loop. Thus, we need to determine the element
2475// size and:
2476// 1) it should be uniform for all operations in the vector loop, so we
2477// e.g. don't want any widening/narrowing operations.
2478// 2) it should be smaller than i64s because we don't have vector operations
2479// that work on i64s.
2480// 3) we don't want elements to be reversed or shuffled, to make sure the
2481// tail-predication masks/predicates the right lanes.
2482//
2483static bool canTailPredicateLoop(Loop *L, LoopInfo *LI, ScalarEvolution &SE,
2484 const DataLayout &DL,
2485 const LoopAccessInfo *LAI,
2486 const DominatorTree &DT) {
2487 LLVM_DEBUG(dbgs() << "Tail-predication: checking allowed instructions\n");
2488
2489 // If there are live-out values, it is probably a reduction. We can predicate
2490 // most reduction operations freely under MVE using a combination of
2491 // prefer-predicated-reduction-select and inloop reductions. We limit this to
2492 // floating point and integer reductions, but don't check for operators
2493 // specifically here. If the value ends up not being a reduction (and so the
2494 // vectorizer cannot tailfold the loop), we should fall back to standard
2495 // vectorization automatically.
2496 SmallVector< Instruction *, 8 > LiveOuts;
2497 LiveOuts = llvm::findDefsUsedOutsideOfLoop(L);
2498 bool ReductionsDisabled =
2499 EnableTailPredication == TailPredication::EnabledNoReductions ||
2500 EnableTailPredication == TailPredication::ForceEnabledNoReductions;
2501
2502 for (auto *I : LiveOuts) {
2503 if (!I->getType()->isIntegerTy() && !I->getType()->isFloatTy() &&
2504 !I->getType()->isHalfTy()) {
2505 LLVM_DEBUG(dbgs() << "Don't tail-predicate loop with non-integer/float "
2506 "live-out value\n");
2507 return false;
2508 }
2509 if (ReductionsDisabled) {
2510 LLVM_DEBUG(dbgs() << "Reductions not enabled\n");
2511 return false;
2512 }
2513 }
2514
2515 // Next, check that all instructions can be tail-predicated.
2516 PredicatedScalarEvolution PSE = LAI->getPSE();
2517 int ICmpCount = 0;
2518
2519 for (BasicBlock *BB : L->blocks()) {
2520 for (Instruction &I : *BB) {
2521 if (isa<PHINode>(Val: I) || isa<PseudoProbeInst>(Val: I))
2522 continue;
2523 if (!canTailPredicateInstruction(I, ICmpCount)) {
2524 LLVM_DEBUG(dbgs() << "Instruction not allowed: "; I.dump());
2525 return false;
2526 }
2527
2528 Type *T = I.getType();
2529 if (T->getScalarSizeInBits() > 32) {
2530 LLVM_DEBUG(dbgs() << "Unsupported Type: "; T->dump());
2531 return false;
2532 }
2533 if (isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I)) {
2534 Value *Ptr = getLoadStorePointerOperand(V: &I);
2535 Type *AccessTy = getLoadStoreType(I: &I);
2536 int64_t NextStride =
2537 getPtrStride(PSE, AccessTy, Ptr, Lp: L, DT).value_or(u: 0);
2538 if (NextStride == 1) {
2539 // TODO: for now only allow consecutive strides of 1. We could support
2540 // other strides as long as it is uniform, but let's keep it simple
2541 // for now.
2542 continue;
2543 } else if (NextStride == -1 ||
2544 (NextStride == 2 && MVEMaxSupportedInterleaveFactor >= 2) ||
2545 (NextStride == 4 && MVEMaxSupportedInterleaveFactor >= 4)) {
2546 LLVM_DEBUG(dbgs()
2547 << "Consecutive strides of 2 found, vld2/vstr2 can't "
2548 "be tail-predicated\n.");
2549 return false;
2550 // TODO: don't tail predicate if there is a reversed load?
2551 } else if (EnableMaskedGatherScatters) {
2552 // Gather/scatters do allow loading from arbitrary strides, at
2553 // least if they are loop invariant.
2554 // TODO: Loop variant strides should in theory work, too, but
2555 // this requires further testing.
2556 const SCEV *PtrScev = PSE.getSE()->getSCEV(V: Ptr);
2557 if (auto AR = dyn_cast<SCEVAddRecExpr>(Val: PtrScev)) {
2558 const SCEV *Step = AR->getStepRecurrence(SE&: *PSE.getSE());
2559 if (PSE.getSE()->isLoopInvariant(S: Step, L))
2560 continue;
2561 }
2562 }
2563 LLVM_DEBUG(dbgs() << "Bad stride found, can't "
2564 "tail-predicate\n.");
2565 return false;
2566 }
2567 }
2568 }
2569
2570 LLVM_DEBUG(dbgs() << "tail-predication: all instructions allowed!\n");
2571 return true;
2572}
2573
2574bool ARMTTIImpl::preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
2575 if (!EnableTailPredication) {
2576 LLVM_DEBUG(dbgs() << "Tail-folding not enabled.\n");
2577 return false;
2578 }
2579
2580 // Creating a tail-folded vector loop is the first step for generating a
2581 // tail-folded hardware loop, for which we need the MVE masked
2582 // load/stores instructions:
2583 if (!ST->hasMVEIntegerOps())
2584 return false;
2585
2586 LoopVectorizationLegality *LVL = TFI->LVL;
2587 Loop *L = LVL->getLoop();
2588
2589 // For now, restrict this to single block loops.
2590 if (L->getNumBlocks() > 1) {
2591 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: not a single block "
2592 "loop.\n");
2593 return false;
2594 }
2595
2596 assert(L->isInnermost() &&
2597 "preferTailFoldingOverEpilogue: inner-loop expected");
2598
2599 LoopInfo *LI = LVL->getLoopInfo();
2600 HardwareLoopInfo HWLoopInfo(L);
2601 if (!HWLoopInfo.canAnalyze(LI&: *LI)) {
2602 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2603 "analyzable.\n");
2604 return false;
2605 }
2606
2607 AssumptionCache *AC = LVL->getAssumptionCache();
2608 ScalarEvolution *SE = LVL->getScalarEvolution();
2609
2610 // This checks if we have the low-overhead branch architecture
2611 // extension, and if we will create a hardware-loop:
2612 if (!isHardwareLoopProfitable(L, SE&: *SE, AC&: *AC, LibInfo: TFI->TLI, HWLoopInfo)) {
2613 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2614 "profitable.\n");
2615 return false;
2616 }
2617
2618 DominatorTree *DT = LVL->getDominatorTree();
2619 if (!HWLoopInfo.isHardwareLoopCandidate(SE&: *SE, LI&: *LI, DT&: *DT)) {
2620 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2621 "a candidate.\n");
2622 return false;
2623 }
2624
2625 return canTailPredicateLoop(L, LI, SE&: *SE, DL, LAI: LVL->getLAI(),
2626 DT: *LVL->getDominatorTree());
2627}
2628
2629TailFoldingStyle ARMTTIImpl::getPreferredTailFoldingStyle() const {
2630 if (!ST->hasMVEIntegerOps() || !EnableTailPredication)
2631 return TailFoldingStyle::DataWithoutLaneMask;
2632
2633 // Intrinsic @llvm.get.active.lane.mask is supported.
2634 // It is used in the MVETailPredication pass, which requires the number of
2635 // elements processed by this vector loop to setup the tail-predicated
2636 // loop.
2637 return TailFoldingStyle::Data;
2638}
2639void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
2640 TTI::UnrollingPreferences &UP,
2641 OptimizationRemarkEmitter *ORE) const {
2642 // Enable Upper bound unrolling universally, providing that we do not see an
2643 // active lane mask, which will be better kept as a loop to become tail
2644 // predicated than to be conditionally unrolled.
2645 UP.UpperBound =
2646 !ST->hasMVEIntegerOps() || !any_of(Range&: *L->getHeader(), P: [](Instruction &I) {
2647 return isa<IntrinsicInst>(Val: I) &&
2648 cast<IntrinsicInst>(Val&: I).getIntrinsicID() ==
2649 Intrinsic::get_active_lane_mask;
2650 });
2651
2652 // Only currently enable these preferences for M-Class cores.
2653 if (!ST->isMClass())
2654 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
2655
2656 // Disable loop unrolling for Oz and Os.
2657 UP.OptSizeThreshold = 0;
2658 UP.PartialOptSizeThreshold = 0;
2659 if (L->getHeader()->getParent()->hasOptSize())
2660 return;
2661
2662 SmallVector<BasicBlock*, 4> ExitingBlocks;
2663 L->getExitingBlocks(ExitingBlocks);
2664 LLVM_DEBUG(dbgs() << "Loop has:\n"
2665 << "Blocks: " << L->getNumBlocks() << "\n"
2666 << "Exit blocks: " << ExitingBlocks.size() << "\n");
2667
2668 // Only allow another exit other than the latch. This acts as an early exit
2669 // as it mirrors the profitability calculation of the runtime unroller.
2670 if (ExitingBlocks.size() > 2)
2671 return;
2672
2673 // Limit the CFG of the loop body for targets with a branch predictor.
2674 // Allowing 4 blocks permits if-then-else diamonds in the body.
2675 if (ST->hasBranchPredictor() && L->getNumBlocks() > 4)
2676 return;
2677
2678 // Don't unroll vectorized loops, including the remainder loop
2679 if (getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized"))
2680 return;
2681
2682 // Scan the loop: don't unroll loops with calls as this could prevent
2683 // inlining.
2684 InstructionCost Cost = 0;
2685 for (auto *BB : L->getBlocks()) {
2686 for (auto &I : *BB) {
2687 // Don't unroll vectorised loop. MVE does not benefit from it as much as
2688 // scalar code.
2689 if (I.getType()->isVectorTy())
2690 return;
2691
2692 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
2693 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction()) {
2694 if (!isLoweredToCall(F))
2695 continue;
2696 }
2697 return;
2698 }
2699
2700 SmallVector<const Value*, 4> Operands(I.operand_values());
2701 Cost += getInstructionCost(U: &I, Operands,
2702 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
2703 }
2704 }
2705
2706 // On v6m cores, there are very few registers available. We can easily end up
2707 // spilling and reloading more registers in an unrolled loop. Look at the
2708 // number of LCSSA phis as a rough measure of how many registers will need to
2709 // be live out of the loop, reducing the default unroll count if more than 1
2710 // value is needed. In the long run, all of this should be being learnt by a
2711 // machine.
2712 unsigned UnrollCount = 4;
2713 if (ST->isThumb1Only()) {
2714 unsigned ExitingValues = 0;
2715 SmallVector<BasicBlock *, 4> ExitBlocks;
2716 L->getExitBlocks(ExitBlocks);
2717 for (auto *Exit : ExitBlocks) {
2718 // Count the number of LCSSA phis. Exclude values coming from GEP's as
2719 // only the last is expected to be needed for address operands.
2720 unsigned LiveOuts = count_if(Range: Exit->phis(), P: [](auto &PH) {
2721 return PH.getNumOperands() != 1 ||
2722 !isa<GetElementPtrInst>(PH.getOperand(0));
2723 });
2724 ExitingValues = ExitingValues < LiveOuts ? LiveOuts : ExitingValues;
2725 }
2726 if (ExitingValues)
2727 UnrollCount /= ExitingValues;
2728 if (UnrollCount <= 1)
2729 return;
2730 }
2731
2732 // For processors with low overhead branching (LOB), runtime unrolling the
2733 // innermost loop is often detrimental to performance. In these cases the loop
2734 // remainder gets unrolled into a series of compare-and-jump blocks, which in
2735 // deeply nested loops get executed multiple times, negating the benefits of
2736 // LOB. This is particularly noticeable when the loop trip count of the
2737 // innermost loop varies within the outer loop, such as in the case of
2738 // triangular matrix decompositions. In these cases we will prefer to not
2739 // unroll the innermost loop, with the intention for it to be executed as a
2740 // low overhead loop.
2741 bool Runtime = true;
2742 if (ST->hasLOB()) {
2743 if (SE.hasLoopInvariantBackedgeTakenCount(L)) {
2744 const SCEV *BETC = SE.getBackedgeTakenCount(L);
2745 auto *Outer = L->getOutermostLoop();
2746 if ((L != Outer && Outer != L->getParentLoop()) ||
2747 (L != Outer && BETC && !SE.isLoopInvariant(S: BETC, L: Outer))) {
2748 Runtime = false;
2749 }
2750 }
2751 }
2752
2753 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
2754 LLVM_DEBUG(dbgs() << "Default Runtime Unroll Count: " << UnrollCount << "\n");
2755
2756 UP.Partial = true;
2757 UP.Runtime = Runtime;
2758 UP.UnrollRemainder = true;
2759 UP.DefaultUnrollRuntimeCount = UnrollCount;
2760 UP.UnrollAndJam = true;
2761 UP.UnrollAndJamInnerLoopThreshold = 60;
2762
2763 // Force unrolling small loops can be very useful because of the branch
2764 // taken cost of the backedge.
2765 if (Cost < ArmForceUnrollThreshold)
2766 UP.Force = true;
2767}
2768
2769void ARMTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
2770 TTI::PeelingPreferences &PP) const {
2771 BaseT::getPeelingPreferences(L, SE, PP);
2772}
2773
2774bool ARMTTIImpl::preferInLoopReduction(RecurKind Kind, Type *Ty) const {
2775 if (!ST->hasMVEIntegerOps())
2776 return false;
2777
2778 unsigned ScalarBits = Ty->getScalarSizeInBits();
2779 switch (Kind) {
2780 case RecurKind::Add:
2781 return ScalarBits <= 64;
2782 default:
2783 return false;
2784 }
2785}
2786
2787bool ARMTTIImpl::preferPredicatedReductionSelect() const {
2788 if (!ST->hasMVEIntegerOps())
2789 return false;
2790 return true;
2791}
2792
2793InstructionCost ARMTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
2794 StackOffset BaseOffset,
2795 bool HasBaseReg, int64_t Scale,
2796 unsigned AddrSpace) const {
2797 TargetLoweringBase::AddrMode AM;
2798 AM.BaseGV = BaseGV;
2799 AM.BaseOffs = BaseOffset.getFixed();
2800 AM.HasBaseReg = HasBaseReg;
2801 AM.Scale = Scale;
2802 AM.ScalableOffset = BaseOffset.getScalable();
2803 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AS: AddrSpace)) {
2804 if (ST->hasFPAO())
2805 return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster
2806 return 0;
2807 }
2808 return InstructionCost::getInvalid();
2809}
2810
2811bool ARMTTIImpl::shouldConsiderVectorizationRegPressure() const {
2812 // MVE only has 8 vector registers, so we should consider register pressure to
2813 // avoid vectorizing when the cost of spills exceeds the gains from
2814 // vectorization.
2815 return ST->hasMVEIntegerOps();
2816}
2817
2818bool ARMTTIImpl::hasArmWideBranch(bool Thumb) const {
2819 if (Thumb) {
2820 // B.W is available in any Thumb2-supporting target, and also in every
2821 // version of Armv8-M, even Baseline which does not include the rest of
2822 // Thumb2.
2823 return ST->isThumb2() || ST->hasV8MBaselineOps();
2824 } else {
2825 // B is available in all versions of the Arm ISA, so the only question is
2826 // whether that ISA is available at all.
2827 return ST->hasARMOps();
2828 }
2829}
2830
2831/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
2832/// of the vector elements.
2833static bool areExtractExts(Value *Ext1, Value *Ext2) {
2834 using namespace PatternMatch;
2835
2836 auto areExtDoubled = [](Instruction *Ext) {
2837 return Ext->getType()->getScalarSizeInBits() ==
2838 2 * Ext->getOperand(i: 0)->getType()->getScalarSizeInBits();
2839 };
2840
2841 if (!match(V: Ext1, P: m_ZExtOrSExt(Op: m_Value())) ||
2842 !match(V: Ext2, P: m_ZExtOrSExt(Op: m_Value())) ||
2843 !areExtDoubled(cast<Instruction>(Val: Ext1)) ||
2844 !areExtDoubled(cast<Instruction>(Val: Ext2)))
2845 return false;
2846
2847 return true;
2848}
2849
2850/// Check if sinking \p I's operands to I's basic block is profitable, because
2851/// the operands can be folded into a target instruction, e.g.
2852/// sext/zext can be folded into vsubl.
2853bool ARMTTIImpl::isProfitableToSinkOperands(Instruction *I,
2854 SmallVectorImpl<Use *> &Ops) const {
2855 using namespace PatternMatch;
2856
2857 if (!I->getType()->isVectorTy())
2858 return false;
2859
2860 if (ST->hasNEON()) {
2861 switch (I->getOpcode()) {
2862 case Instruction::Sub:
2863 case Instruction::Add: {
2864 if (!areExtractExts(Ext1: I->getOperand(i: 0), Ext2: I->getOperand(i: 1)))
2865 return false;
2866 Ops.push_back(Elt: &I->getOperandUse(i: 0));
2867 Ops.push_back(Elt: &I->getOperandUse(i: 1));
2868 return true;
2869 }
2870 default:
2871 return false;
2872 }
2873 }
2874
2875 if (!ST->hasMVEIntegerOps())
2876 return false;
2877
2878 auto IsFMSMul = [&](Instruction *I) {
2879 if (!I->hasOneUse())
2880 return false;
2881 auto *Sub = cast<Instruction>(Val: *I->users().begin());
2882 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(i: 1) == I;
2883 };
2884 auto IsFMS = [&](Instruction *I) {
2885 if (match(V: I->getOperand(i: 0), P: m_FNeg(X: m_Value())) ||
2886 match(V: I->getOperand(i: 1), P: m_FNeg(X: m_Value())))
2887 return true;
2888 return false;
2889 };
2890
2891 auto IsSinker = [&](Instruction *I, int Operand) {
2892 switch (I->getOpcode()) {
2893 case Instruction::Add:
2894 case Instruction::Mul:
2895 case Instruction::FAdd:
2896 case Instruction::ICmp:
2897 case Instruction::FCmp:
2898 return true;
2899 case Instruction::FMul:
2900 return !IsFMSMul(I);
2901 case Instruction::Sub:
2902 case Instruction::FSub:
2903 case Instruction::Shl:
2904 case Instruction::LShr:
2905 case Instruction::AShr:
2906 return Operand == 1;
2907 case Instruction::Call:
2908 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2909 switch (II->getIntrinsicID()) {
2910 case Intrinsic::fma:
2911 return !IsFMS(I);
2912 case Intrinsic::sadd_sat:
2913 case Intrinsic::uadd_sat:
2914 case Intrinsic::arm_mve_add_predicated:
2915 case Intrinsic::arm_mve_mul_predicated:
2916 case Intrinsic::arm_mve_qadd_predicated:
2917 case Intrinsic::arm_mve_vhadd:
2918 case Intrinsic::arm_mve_hadd_predicated:
2919 case Intrinsic::arm_mve_vqdmull:
2920 case Intrinsic::arm_mve_vqdmull_predicated:
2921 case Intrinsic::arm_mve_vqdmulh:
2922 case Intrinsic::arm_mve_qdmulh_predicated:
2923 case Intrinsic::arm_mve_vqrdmulh:
2924 case Intrinsic::arm_mve_qrdmulh_predicated:
2925 case Intrinsic::arm_mve_fma_predicated:
2926 return true;
2927 case Intrinsic::ssub_sat:
2928 case Intrinsic::usub_sat:
2929 case Intrinsic::arm_mve_sub_predicated:
2930 case Intrinsic::arm_mve_qsub_predicated:
2931 case Intrinsic::arm_mve_hsub_predicated:
2932 case Intrinsic::arm_mve_vhsub:
2933 return Operand == 1;
2934 default:
2935 return false;
2936 }
2937 }
2938 return false;
2939 default:
2940 return false;
2941 }
2942 };
2943
2944 for (auto OpIdx : enumerate(First: I->operands())) {
2945 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
2946 // Make sure we are not already sinking this operand
2947 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
2948 continue;
2949
2950 Instruction *Shuffle = Op;
2951 if (Shuffle->getOpcode() == Instruction::BitCast)
2952 Shuffle = dyn_cast<Instruction>(Val: Shuffle->getOperand(i: 0));
2953 // We are looking for a splat that can be sunk.
2954 if (!Shuffle || !match(V: Shuffle, P: m_Shuffle(v1: m_InsertElt(Val: m_Undef(), Elt: m_Value(),
2955 Idx: m_ZeroInt()),
2956 v2: m_Undef(), mask: m_ZeroMask())))
2957 continue;
2958 if (!IsSinker(I, OpIdx.index()))
2959 continue;
2960
2961 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
2962 // and vector registers
2963 for (Use &U : Op->uses()) {
2964 Instruction *Insn = cast<Instruction>(Val: U.getUser());
2965 if (!IsSinker(Insn, U.getOperandNo()))
2966 return false;
2967 }
2968
2969 Ops.push_back(Elt: &Shuffle->getOperandUse(i: 0));
2970 if (Shuffle != Op)
2971 Ops.push_back(Elt: &Op->getOperandUse(i: 0));
2972 Ops.push_back(Elt: &OpIdx.value());
2973 }
2974 return true;
2975}
2976
2977unsigned ARMTTIImpl::getNumBytesToPadGlobalArray(unsigned Size,
2978 Type *ArrayType) const {
2979 if (!UseWidenGlobalArrays) {
2980 LLVM_DEBUG(dbgs() << "Padding global arrays disabled\n");
2981 return false;
2982 }
2983
2984 // Don't modify none integer array types
2985 if (!ArrayType || !ArrayType->isArrayTy() ||
2986 !ArrayType->getArrayElementType()->isIntegerTy())
2987 return 0;
2988
2989 // We pad to 4 byte boundaries
2990 if (Size % 4 == 0)
2991 return 0;
2992
2993 unsigned NumBytesToPad = 4 - (Size % 4);
2994 unsigned NewSize = Size + NumBytesToPad;
2995
2996 // Max number of bytes that memcpy allows for lowering to load/stores before
2997 // it uses library function (__aeabi_memcpy).
2998 unsigned MaxMemIntrinsicSize = getMaxMemIntrinsicInlineSizeThreshold();
2999
3000 if (NewSize > MaxMemIntrinsicSize)
3001 return 0;
3002
3003 return NumBytesToPad;
3004}
3005