1//===- ARMTargetTransformInfo.cpp - ARM specific TTI ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ARMTargetTransformInfo.h"
10#include "ARMSubtarget.h"
11#include "MCTargetDesc/ARMAddressingModes.h"
12#include "llvm/ADT/APInt.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Analysis/LoopInfo.h"
15#include "llvm/CodeGen/CostTable.h"
16#include "llvm/CodeGen/ISDOpcodes.h"
17#include "llvm/CodeGen/ValueTypes.h"
18#include "llvm/CodeGenTypes/MachineValueType.h"
19#include "llvm/IR/BasicBlock.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/IR/DerivedTypes.h"
22#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/IntrinsicsARM.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Target/TargetMachine.h"
32#include "llvm/TargetParser/SubtargetFeature.h"
33#include "llvm/Transforms/InstCombine/InstCombiner.h"
34#include "llvm/Transforms/Utils/Local.h"
35#include "llvm/Transforms/Utils/LoopUtils.h"
36#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
37#include <algorithm>
38#include <cassert>
39#include <cstdint>
40#include <optional>
41#include <utility>
42
43using namespace llvm;
44
45#define DEBUG_TYPE "armtti"
46
47static cl::opt<bool> EnableMaskedLoadStores(
48 "enable-arm-maskedldst", cl::Hidden, cl::init(Val: true),
49 cl::desc("Enable the generation of masked loads and stores"));
50
51static cl::opt<bool> DisableLowOverheadLoops(
52 "disable-arm-loloops", cl::Hidden, cl::init(Val: false),
53 cl::desc("Disable the generation of low-overhead loops"));
54
55static cl::opt<bool>
56 AllowWLSLoops("allow-arm-wlsloops", cl::Hidden, cl::init(Val: true),
57 cl::desc("Enable the generation of WLS loops"));
58
59static cl::opt<bool> UseWidenGlobalArrays(
60 "widen-global-strings", cl::Hidden, cl::init(Val: true),
61 cl::desc("Enable the widening of global strings to alignment boundaries"));
62
63extern cl::opt<TailPredication::Mode> EnableTailPredication;
64
65extern cl::opt<bool> EnableMaskedGatherScatters;
66
67extern cl::opt<unsigned> MVEMaxSupportedInterleaveFactor;
68
69static cl::opt<int> ArmForceUnrollThreshold(
70 "arm-force-unroll-threshold", cl::init(Val: 12), cl::Hidden,
71 cl::desc(
72 "Threshold for forced unrolling of small loops in Arm architecture"));
73
74/// Convert a vector load intrinsic into a simple llvm load instruction.
75/// This is beneficial when the underlying object being addressed comes
76/// from a constant, since we get constant-folding for free.
77static Value *simplifyNeonVld1(const IntrinsicInst &II, unsigned MemAlign,
78 InstCombiner::BuilderTy &Builder) {
79 auto *IntrAlign = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
80
81 if (!IntrAlign)
82 return nullptr;
83
84 unsigned Alignment = IntrAlign->getLimitedValue() < MemAlign
85 ? MemAlign
86 : IntrAlign->getLimitedValue();
87
88 if (!isPowerOf2_32(Value: Alignment))
89 return nullptr;
90
91 return Builder.CreateAlignedLoad(Ty: II.getType(), Ptr: II.getArgOperand(i: 0),
92 Align: Align(Alignment));
93}
94
95TTI::AddressingModeKind
96ARMTTIImpl::getPreferredAddressingMode(const Loop *L,
97 ScalarEvolution *SE) const {
98 if (ST->hasMVEIntegerOps())
99 return TTI::AMK_PostIndexed;
100
101 if (L->getHeader()->getParent()->hasOptSize())
102 return TTI::AMK_None;
103
104 if (ST->isMClass() && ST->isThumb2() &&
105 L->getNumBlocks() == 1)
106 return TTI::AMK_PreIndexed;
107
108 return TTI::AMK_None;
109}
110
111std::optional<Instruction *>
112ARMTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
113 using namespace PatternMatch;
114 Intrinsic::ID IID = II.getIntrinsicID();
115 switch (IID) {
116 default:
117 break;
118 case Intrinsic::arm_neon_vld1: {
119 Align MemAlign =
120 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CxtI: &II,
121 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
122 if (Value *V = simplifyNeonVld1(II, MemAlign: MemAlign.value(), Builder&: IC.Builder)) {
123 return IC.replaceInstUsesWith(I&: II, V);
124 }
125 break;
126 }
127
128 case Intrinsic::arm_neon_vld2:
129 case Intrinsic::arm_neon_vld3:
130 case Intrinsic::arm_neon_vld4:
131 case Intrinsic::arm_neon_vld2lane:
132 case Intrinsic::arm_neon_vld3lane:
133 case Intrinsic::arm_neon_vld4lane:
134 case Intrinsic::arm_neon_vst1:
135 case Intrinsic::arm_neon_vst2:
136 case Intrinsic::arm_neon_vst3:
137 case Intrinsic::arm_neon_vst4:
138 case Intrinsic::arm_neon_vst2lane:
139 case Intrinsic::arm_neon_vst3lane:
140 case Intrinsic::arm_neon_vst4lane: {
141 Align MemAlign =
142 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CxtI: &II,
143 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
144 unsigned AlignArg = II.arg_size() - 1;
145 Value *AlignArgOp = II.getArgOperand(i: AlignArg);
146 MaybeAlign Align = cast<ConstantInt>(Val: AlignArgOp)->getMaybeAlignValue();
147 if (Align && *Align < MemAlign) {
148 return IC.replaceOperand(
149 I&: II, OpNum: AlignArg,
150 V: ConstantInt::get(Ty: Type::getInt32Ty(C&: II.getContext()), V: MemAlign.value(),
151 IsSigned: false));
152 }
153 break;
154 }
155
156 case Intrinsic::arm_neon_vld1x2:
157 case Intrinsic::arm_neon_vld1x3:
158 case Intrinsic::arm_neon_vld1x4:
159 case Intrinsic::arm_neon_vst1x2:
160 case Intrinsic::arm_neon_vst1x3:
161 case Intrinsic::arm_neon_vst1x4: {
162 Align NewAlign =
163 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CxtI: &II,
164 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
165 Align OldAlign = II.getParamAlign(ArgNo: 0).valueOrOne();
166 if (NewAlign > OldAlign)
167 II.addParamAttr(ArgNo: 0,
168 Attr: Attribute::getWithAlignment(Context&: II.getContext(), Alignment: NewAlign));
169 break;
170 }
171
172 case Intrinsic::arm_mve_pred_i2v: {
173 Value *Arg = II.getArgOperand(i: 0);
174 Value *ArgArg;
175 if (match(V: Arg, P: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
176 Ops: PatternMatch::m_Value(V&: ArgArg))) &&
177 II.getType() == ArgArg->getType()) {
178 return IC.replaceInstUsesWith(I&: II, V: ArgArg);
179 }
180 Constant *XorMask;
181 if (match(V: Arg, P: m_Xor(L: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
182 Ops: PatternMatch::m_Value(V&: ArgArg)),
183 R: PatternMatch::m_Constant(C&: XorMask))) &&
184 II.getType() == ArgArg->getType()) {
185 if (auto *CI = dyn_cast<ConstantInt>(Val: XorMask)) {
186 if (CI->getValue().trunc(width: 16).isAllOnes()) {
187 auto TrueVector = IC.Builder.CreateVectorSplat(
188 NumElts: cast<FixedVectorType>(Val: II.getType())->getNumElements(),
189 V: IC.Builder.getTrue());
190 return BinaryOperator::Create(Op: Instruction::Xor, S1: ArgArg, S2: TrueVector);
191 }
192 }
193 }
194 KnownBits ScalarKnown(32);
195 if (IC.SimplifyDemandedBits(I: &II, OpNo: 0, DemandedMask: APInt::getLowBitsSet(numBits: 32, loBitsSet: 16),
196 Known&: ScalarKnown)) {
197 return &II;
198 }
199 break;
200 }
201 case Intrinsic::arm_mve_pred_v2i: {
202 Value *Arg = II.getArgOperand(i: 0);
203 Value *ArgArg;
204 if (match(V: Arg, P: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_i2v>(
205 Ops: PatternMatch::m_Value(V&: ArgArg)))) {
206 return IC.replaceInstUsesWith(I&: II, V: ArgArg);
207 }
208
209 if (II.getMetadata(KindID: LLVMContext::MD_range))
210 break;
211
212 ConstantRange Range(APInt(32, 0), APInt(32, 0x10000));
213
214 if (auto CurrentRange = II.getRange()) {
215 Range = Range.intersectWith(CR: *CurrentRange);
216 if (Range == CurrentRange)
217 break;
218 }
219
220 II.addRangeRetAttr(CR: Range);
221 II.addRetAttr(Kind: Attribute::NoUndef);
222 return &II;
223 }
224 case Intrinsic::arm_mve_vadc:
225 case Intrinsic::arm_mve_vadc_predicated: {
226 unsigned CarryOp =
227 (II.getIntrinsicID() == Intrinsic::arm_mve_vadc_predicated) ? 3 : 2;
228 assert(II.getArgOperand(CarryOp)->getType()->getScalarSizeInBits() == 32 &&
229 "Bad type for intrinsic!");
230
231 KnownBits CarryKnown(32);
232 if (IC.SimplifyDemandedBits(I: &II, OpNo: CarryOp, DemandedMask: APInt::getOneBitSet(numBits: 32, BitNo: 29),
233 Known&: CarryKnown)) {
234 return &II;
235 }
236 break;
237 }
238 case Intrinsic::arm_mve_vmldava: {
239 Instruction *I = cast<Instruction>(Val: &II);
240 if (I->hasOneUse()) {
241 auto *User = cast<Instruction>(Val: *I->user_begin());
242 Value *OpZ;
243 if (match(V: User, P: m_c_Add(L: m_Specific(V: I), R: m_Value(V&: OpZ))) &&
244 match(V: I->getOperand(i: 3), P: m_Zero())) {
245 Value *OpX = I->getOperand(i: 4);
246 Value *OpY = I->getOperand(i: 5);
247 Type *OpTy = OpX->getType();
248
249 IC.Builder.SetInsertPoint(User);
250 Value *V =
251 IC.Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vmldava, OverloadTypes: {OpTy},
252 Args: {I->getOperand(i: 0), I->getOperand(i: 1),
253 I->getOperand(i: 2), OpZ, OpX, OpY});
254
255 IC.replaceInstUsesWith(I&: *User, V);
256 return IC.eraseInstFromFunction(I&: *User);
257 }
258 }
259 return std::nullopt;
260 }
261 }
262 return std::nullopt;
263}
264
265std::optional<Value *> ARMTTIImpl::simplifyDemandedVectorEltsIntrinsic(
266 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,
267 APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
268 std::function<void(Instruction *, unsigned, APInt, APInt &)>
269 SimplifyAndSetOp) const {
270
271 // Compute the demanded bits for a narrowing MVE intrinsic. The TopOpc is the
272 // opcode specifying a Top/Bottom instruction, which can change between
273 // instructions.
274 auto SimplifyNarrowInstrTopBottom =[&](unsigned TopOpc) {
275 unsigned NumElts = cast<FixedVectorType>(Val: II.getType())->getNumElements();
276 unsigned IsTop = cast<ConstantInt>(Val: II.getOperand(i_nocapture: TopOpc))->getZExtValue();
277
278 // The only odd/even lanes of operand 0 will only be demanded depending
279 // on whether this is a top/bottom instruction.
280 APInt DemandedElts =
281 APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
282 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
283 SimplifyAndSetOp(&II, 0, OrigDemandedElts & DemandedElts, UndefElts);
284 // The other lanes will be defined from the inserted elements.
285 UndefElts &= APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
286 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
287 return std::nullopt;
288 };
289
290 switch (II.getIntrinsicID()) {
291 default:
292 break;
293 case Intrinsic::arm_mve_vcvt_narrow:
294 SimplifyNarrowInstrTopBottom(2);
295 break;
296 case Intrinsic::arm_mve_vqmovn:
297 SimplifyNarrowInstrTopBottom(4);
298 break;
299 case Intrinsic::arm_mve_vshrn:
300 SimplifyNarrowInstrTopBottom(7);
301 break;
302 }
303
304 return std::nullopt;
305}
306
307InstructionCost ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
308 TTI::TargetCostKind CostKind) const {
309 assert(Ty->isIntegerTy());
310
311 unsigned Bits = Ty->getPrimitiveSizeInBits();
312 if (Bits == 0 || Imm.getActiveBits() >= 64)
313 return 4;
314
315 int64_t SImmVal = Imm.getSExtValue();
316 uint64_t ZImmVal = Imm.getZExtValue();
317 if (!ST->isThumb()) {
318 if ((SImmVal >= 0 && SImmVal < 65536) ||
319 (ARM_AM::getSOImmVal(Arg: ZImmVal) != -1) ||
320 (ARM_AM::getSOImmVal(Arg: ~ZImmVal) != -1))
321 return 1;
322 return ST->hasV6T2Ops() ? 2 : 3;
323 }
324 if (ST->isThumb2()) {
325 if ((SImmVal >= 0 && SImmVal < 65536) ||
326 (ARM_AM::getT2SOImmVal(Arg: ZImmVal) != -1) ||
327 (ARM_AM::getT2SOImmVal(Arg: ~ZImmVal) != -1))
328 return 1;
329 return ST->hasV6T2Ops() ? 2 : 3;
330 }
331 // Thumb1, any i8 imm cost 1.
332 if (Bits == 8 || (SImmVal >= 0 && SImmVal < 256))
333 return 1;
334 if ((~SImmVal < 256) || ARM_AM::isThumbImmShiftedVal(V: ZImmVal))
335 return 2;
336 // Load from constantpool.
337 return 3;
338}
339
340// Constants smaller than 256 fit in the immediate field of
341// Thumb1 instructions so we return a zero cost and 1 otherwise.
342InstructionCost ARMTTIImpl::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
343 const APInt &Imm,
344 Type *Ty) const {
345 if (Imm.isNonNegative() && Imm.getLimitedValue() < 256)
346 return 0;
347
348 return 1;
349}
350
351// Checks whether Inst is part of a min(max()) or max(min()) pattern
352// that will match to an SSAT instruction. Returns the instruction being
353// saturated, or null if no saturation pattern was found.
354static Value *isSSATMinMaxPattern(Instruction *Inst, const APInt &Imm) {
355 Value *LHS, *RHS;
356 ConstantInt *C;
357 SelectPatternFlavor InstSPF = matchSelectPattern(V: Inst, LHS, RHS).Flavor;
358
359 if (InstSPF == SPF_SMAX &&
360 PatternMatch::match(V: RHS, P: PatternMatch::m_ConstantInt(CI&: C)) &&
361 C->getValue() == Imm && Imm.isNegative() && Imm.isNegatedPowerOf2()) {
362
363 auto isSSatMin = [&](Value *MinInst) {
364 if (isa<SelectInst>(Val: MinInst)) {
365 Value *MinLHS, *MinRHS;
366 ConstantInt *MinC;
367 SelectPatternFlavor MinSPF =
368 matchSelectPattern(V: MinInst, LHS&: MinLHS, RHS&: MinRHS).Flavor;
369 if (MinSPF == SPF_SMIN &&
370 PatternMatch::match(V: MinRHS, P: PatternMatch::m_ConstantInt(CI&: MinC)) &&
371 MinC->getValue() == ((-Imm) - 1))
372 return true;
373 }
374 return false;
375 };
376
377 if (isSSatMin(Inst->getOperand(i: 1)))
378 return cast<Instruction>(Val: Inst->getOperand(i: 1))->getOperand(i: 1);
379 if (Inst->hasNUses(N: 2) &&
380 (isSSatMin(*Inst->user_begin()) || isSSatMin(*(++Inst->user_begin()))))
381 return Inst->getOperand(i: 1);
382 }
383 return nullptr;
384}
385
386// Look for a FP Saturation pattern, where the instruction can be simplified to
387// a fptosi.sat. max(min(fptosi)). The constant in this case is always free.
388static bool isFPSatMinMaxPattern(Instruction *Inst, const APInt &Imm) {
389 if (Imm.getBitWidth() != 64 ||
390 Imm != APInt::getHighBitsSet(numBits: 64, hiBitsSet: 33)) // -2147483648
391 return false;
392 Value *FP = isSSATMinMaxPattern(Inst, Imm);
393 if (!FP && isa<ICmpInst>(Val: Inst) && Inst->hasOneUse())
394 FP = isSSATMinMaxPattern(Inst: cast<Instruction>(Val: *Inst->user_begin()), Imm);
395 if (!FP)
396 return false;
397 return isa<FPToSIInst>(Val: FP);
398}
399
400InstructionCost ARMTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
401 const APInt &Imm, Type *Ty,
402 TTI::TargetCostKind CostKind,
403 Instruction *Inst) const {
404 // Division by a constant can be turned into multiplication, but only if we
405 // know it's constant. So it's not so much that the immediate is cheap (it's
406 // not), but that the alternative is worse.
407 // FIXME: this is probably unneeded with GlobalISel.
408 if ((Opcode == Instruction::SDiv || Opcode == Instruction::UDiv ||
409 Opcode == Instruction::SRem || Opcode == Instruction::URem) &&
410 Idx == 1)
411 return 0;
412
413 // Leave any gep offsets for the CodeGenPrepare, which will do a better job at
414 // splitting any large offsets.
415 if (Opcode == Instruction::GetElementPtr && Idx != 0)
416 return 0;
417
418 if (Opcode == Instruction::And) {
419 // UXTB/UXTH
420 if (Imm == 255 || Imm == 65535)
421 return 0;
422 // Conversion to BIC is free, and means we can use ~Imm instead.
423 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
424 b: getIntImmCost(Imm: ~Imm, Ty, CostKind));
425 }
426
427 if (Opcode == Instruction::Add)
428 // Conversion to SUB is free, and means we can use -Imm instead.
429 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
430 b: getIntImmCost(Imm: -Imm, Ty, CostKind));
431
432 if (Opcode == Instruction::ICmp && Imm.isNegative() &&
433 Ty->getIntegerBitWidth() == 32) {
434 int64_t NegImm = -Imm.getSExtValue();
435 if (ST->isThumb2() && NegImm < 1<<12)
436 // icmp X, #-C -> cmn X, #C
437 return 0;
438 if (ST->isThumb() && NegImm < 1<<8)
439 // icmp X, #-C -> adds X, #C
440 return 0;
441 }
442
443 // xor a, -1 can always be folded to MVN
444 if (Opcode == Instruction::Xor && Imm.isAllOnes())
445 return 0;
446
447 // Ensures negative constant of min(max()) or max(min()) patterns that
448 // match to SSAT instructions don't get hoisted
449 if (Inst && ((ST->hasV6Ops() && !ST->isThumb()) || ST->isThumb2()) &&
450 Ty->getIntegerBitWidth() <= 32) {
451 if (isSSATMinMaxPattern(Inst, Imm) ||
452 (isa<ICmpInst>(Val: Inst) && Inst->hasOneUse() &&
453 isSSATMinMaxPattern(Inst: cast<Instruction>(Val: *Inst->user_begin()), Imm)))
454 return 0;
455 }
456
457 if (Inst && ST->hasVFP2Base() && isFPSatMinMaxPattern(Inst, Imm))
458 return 0;
459
460 // We can convert <= -1 to < 0, which is generally quite cheap.
461 if (Inst && Opcode == Instruction::ICmp && Idx == 1 && Imm.isAllOnes()) {
462 ICmpInst::Predicate Pred = cast<ICmpInst>(Val: Inst)->getPredicate();
463 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE)
464 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
465 b: getIntImmCost(Imm: Imm + 1, Ty, CostKind));
466 }
467
468 return getIntImmCost(Imm, Ty, CostKind);
469}
470
471InstructionCost ARMTTIImpl::getCFInstrCost(unsigned Opcode,
472 TTI::TargetCostKind CostKind,
473 const Instruction *I) const {
474 if (CostKind == TTI::TCK_RecipThroughput &&
475 (ST->hasNEON() || ST->hasMVEIntegerOps())) {
476 // FIXME: The vectorizer is highly sensitive to the cost of these
477 // instructions, which suggests that it may be using the costs incorrectly.
478 // But, for now, just make them free to avoid performance regressions for
479 // vector targets.
480 return 0;
481 }
482 return BaseT::getCFInstrCost(Opcode, CostKind, I);
483}
484
485InstructionCost ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
486 Type *Src,
487 TTI::CastContextHint CCH,
488 TTI::TargetCostKind CostKind,
489 const Instruction *I) const {
490 int ISD = TLI->InstructionOpcodeToISD(Opcode);
491 assert(ISD && "Invalid opcode");
492
493 // TODO: Allow non-throughput costs that aren't binary.
494 auto AdjustCost = [&CostKind](InstructionCost Cost) -> InstructionCost {
495 if (CostKind != TTI::TCK_RecipThroughput)
496 return Cost == 0 ? 0 : 1;
497 return Cost;
498 };
499 auto IsLegalFPType = [this](EVT VT) {
500 EVT EltVT = VT.getScalarType();
501 return (EltVT == MVT::f32 && ST->hasVFP2Base()) ||
502 (EltVT == MVT::f64 && ST->hasFP64()) ||
503 (EltVT == MVT::f16 && ST->hasFullFP16());
504 };
505
506 EVT SrcTy = TLI->getValueType(DL, Ty: Src);
507 EVT DstTy = TLI->getValueType(DL, Ty: Dst);
508
509 if (!SrcTy.isSimple() || !DstTy.isSimple())
510 return AdjustCost(
511 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
512
513 // Extending masked load/Truncating masked stores is expensive because we
514 // currently don't split them. This means that we'll likely end up
515 // loading/storing each element individually (hence the high cost).
516 if ((ST->hasMVEIntegerOps() &&
517 (Opcode == Instruction::Trunc || Opcode == Instruction::ZExt ||
518 Opcode == Instruction::SExt)) ||
519 (ST->hasMVEFloatOps() &&
520 (Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc) &&
521 IsLegalFPType(SrcTy) && IsLegalFPType(DstTy)))
522 if (CCH == TTI::CastContextHint::Masked && DstTy.getSizeInBits() > 128)
523 return 2 * DstTy.getVectorNumElements() *
524 ST->getMVEVectorCostFactor(CostKind);
525
526 // The extend of other kinds of load is free
527 if (CCH == TTI::CastContextHint::Normal ||
528 CCH == TTI::CastContextHint::Masked) {
529 static const TypeConversionCostTblEntry LoadConversionTbl[] = {
530 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i32, .Src: MVT::i16, .Cost: 0},
531 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i32, .Src: MVT::i16, .Cost: 0},
532 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i32, .Src: MVT::i8, .Cost: 0},
533 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i32, .Src: MVT::i8, .Cost: 0},
534 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i16, .Src: MVT::i8, .Cost: 0},
535 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i16, .Src: MVT::i8, .Cost: 0},
536 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i32, .Cost: 1},
537 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i32, .Cost: 1},
538 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 1},
539 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 1},
540 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i8, .Cost: 1},
541 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i8, .Cost: 1},
542 };
543 if (const auto *Entry = ConvertCostTableLookup(
544 Table: LoadConversionTbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
545 return AdjustCost(Entry->Cost);
546
547 static const TypeConversionCostTblEntry MVELoadConversionTbl[] = {
548 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
549 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
550 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
551 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
552 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
553 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
554 // The following extend from a legal type to an illegal type, so need to
555 // split the load. This introduced an extra load operation, but the
556 // extend is still "free".
557 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
558 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
559 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
560 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
561 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
562 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
563 };
564 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
565 if (const auto *Entry =
566 ConvertCostTableLookup(Table: MVELoadConversionTbl, ISD,
567 Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
568 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
569 }
570
571 static const TypeConversionCostTblEntry MVEFLoadConversionTbl[] = {
572 // FPExtends are similar but also require the VCVT instructions.
573 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1},
574 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 3},
575 };
576 if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
577 if (const auto *Entry =
578 ConvertCostTableLookup(Table: MVEFLoadConversionTbl, ISD,
579 Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
580 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
581 }
582
583 // The truncate of a store is free. This is the mirror of extends above.
584 static const TypeConversionCostTblEntry MVEStoreConversionTbl[] = {
585 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
586 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
587 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
588 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
589 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 1},
590 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
591 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
592 };
593 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
594 if (const auto *Entry =
595 ConvertCostTableLookup(Table: MVEStoreConversionTbl, ISD,
596 Dst: SrcTy.getSimpleVT(), Src: DstTy.getSimpleVT()))
597 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
598 }
599
600 static const TypeConversionCostTblEntry MVEFStoreConversionTbl[] = {
601 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1},
602 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 3},
603 };
604 if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
605 if (const auto *Entry =
606 ConvertCostTableLookup(Table: MVEFStoreConversionTbl, ISD,
607 Dst: SrcTy.getSimpleVT(), Src: DstTy.getSimpleVT()))
608 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
609 }
610 }
611
612 // NEON vector operations that can extend their inputs.
613 if ((ISD == ISD::SIGN_EXTEND || ISD == ISD::ZERO_EXTEND) &&
614 I && I->hasOneUse() && ST->hasNEON() && SrcTy.isVector()) {
615 static const TypeConversionCostTblEntry NEONDoubleWidthTbl[] = {
616 // vaddl
617 { .ISD: ISD::ADD, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
618 { .ISD: ISD::ADD, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
619 // vsubl
620 { .ISD: ISD::SUB, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
621 { .ISD: ISD::SUB, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
622 // vmull
623 { .ISD: ISD::MUL, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
624 { .ISD: ISD::MUL, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
625 // vshll
626 { .ISD: ISD::SHL, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
627 { .ISD: ISD::SHL, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
628 };
629
630 auto *User = cast<Instruction>(Val: *I->user_begin());
631 int UserISD = TLI->InstructionOpcodeToISD(Opcode: User->getOpcode());
632 if (auto *Entry = ConvertCostTableLookup(Table: NEONDoubleWidthTbl, ISD: UserISD,
633 Dst: DstTy.getSimpleVT(),
634 Src: SrcTy.getSimpleVT())) {
635 return AdjustCost(Entry->Cost);
636 }
637 }
638
639 // Single to/from double precision conversions.
640 if (Src->isVectorTy() && ST->hasNEON() &&
641 ((ISD == ISD::FP_ROUND && SrcTy.getScalarType() == MVT::f64 &&
642 DstTy.getScalarType() == MVT::f32) ||
643 (ISD == ISD::FP_EXTEND && SrcTy.getScalarType() == MVT::f32 &&
644 DstTy.getScalarType() == MVT::f64))) {
645 static const CostTblEntry NEONFltDblTbl[] = {
646 // Vector fptrunc/fpext conversions.
647 {.ISD: ISD::FP_ROUND, .Type: MVT::v2f64, .Cost: 2},
648 {.ISD: ISD::FP_EXTEND, .Type: MVT::v2f32, .Cost: 2},
649 {.ISD: ISD::FP_EXTEND, .Type: MVT::v4f32, .Cost: 4}};
650
651 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
652 if (const auto *Entry = CostTableLookup(Table: NEONFltDblTbl, ISD, Ty: LT.second))
653 return AdjustCost(LT.first * Entry->Cost);
654 }
655
656 // Some arithmetic, load and store operations have specific instructions
657 // to cast up/down their types automatically at no extra cost.
658 // TODO: Get these tables to know at least what the related operations are.
659 static const TypeConversionCostTblEntry NEONVectorConversionTbl[] = {
660 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
661 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
662 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1 },
663 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1 },
664 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: 0 },
665 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i32, .Cost: 1 },
666
667 // The number of vmovl instructions for the extension.
668 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
669 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
670 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
671 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
672 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 3 },
673 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 3 },
674 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
675 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
676 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3 },
677 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3 },
678 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3 },
679 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3 },
680 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7 },
681 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7 },
682 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6 },
683 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6 },
684 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6 },
685 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6 },
686
687 // Operations that we legalize using splitting.
688 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: 6 },
689 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: 3 },
690
691 // Vector float <-> i32 conversions.
692 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1 },
693 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1 },
694
695 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3 },
696 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3 },
697 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2 },
698 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2 },
699 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1 },
700 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1 },
701 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: 3 },
702 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: 3 },
703 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3 },
704 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3 },
705 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2 },
706 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2 },
707 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4 },
708 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4 },
709 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: 2 },
710 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: 2 },
711 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: 8 },
712 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: 8 },
713 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: 4 },
714 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: 4 },
715
716 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1 },
717 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1 },
718 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 3 },
719 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 3 },
720 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2 },
721 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2 },
722
723 // Vector double <-> i32 conversions.
724 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
725 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
726
727 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4 },
728 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4 },
729 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 3 },
730 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 3 },
731 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
732 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
733
734 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2 },
735 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2 },
736 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 4 },
737 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 4 },
738 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: 8 },
739 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: 8 }
740 };
741
742 if (SrcTy.isVector() && ST->hasNEON()) {
743 if (const auto *Entry = ConvertCostTableLookup(Table: NEONVectorConversionTbl, ISD,
744 Dst: DstTy.getSimpleVT(),
745 Src: SrcTy.getSimpleVT()))
746 return AdjustCost(Entry->Cost);
747 }
748
749 // Scalar float to integer conversions.
750 static const TypeConversionCostTblEntry NEONFloatConversionTbl[] = {
751 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i1, .Src: MVT::f32, .Cost: 2 },
752 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i1, .Src: MVT::f32, .Cost: 2 },
753 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i1, .Src: MVT::f64, .Cost: 2 },
754 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i1, .Src: MVT::f64, .Cost: 2 },
755 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i8, .Src: MVT::f32, .Cost: 2 },
756 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i8, .Src: MVT::f32, .Cost: 2 },
757 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i8, .Src: MVT::f64, .Cost: 2 },
758 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i8, .Src: MVT::f64, .Cost: 2 },
759 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i16, .Src: MVT::f32, .Cost: 2 },
760 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i16, .Src: MVT::f32, .Cost: 2 },
761 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i16, .Src: MVT::f64, .Cost: 2 },
762 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i16, .Src: MVT::f64, .Cost: 2 },
763 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: 2 },
764 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: 2 },
765 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: 2 },
766 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: 2 },
767 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: 10 },
768 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: 10 },
769 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: 10 },
770 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: 10 }
771 };
772 if (SrcTy.isFloatingPoint() && ST->hasNEON()) {
773 if (const auto *Entry = ConvertCostTableLookup(Table: NEONFloatConversionTbl, ISD,
774 Dst: DstTy.getSimpleVT(),
775 Src: SrcTy.getSimpleVT()))
776 return AdjustCost(Entry->Cost);
777 }
778
779 // Scalar integer to float conversions.
780 static const TypeConversionCostTblEntry NEONIntegerConversionTbl[] = {
781 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i1, .Cost: 2 },
782 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i1, .Cost: 2 },
783 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i1, .Cost: 2 },
784 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i1, .Cost: 2 },
785 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i8, .Cost: 2 },
786 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i8, .Cost: 2 },
787 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i8, .Cost: 2 },
788 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i8, .Cost: 2 },
789 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i16, .Cost: 2 },
790 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i16, .Cost: 2 },
791 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i16, .Cost: 2 },
792 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i16, .Cost: 2 },
793 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: 2 },
794 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: 2 },
795 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: 2 },
796 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: 2 },
797 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: 10 },
798 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: 10 },
799 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: 10 },
800 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: 10 }
801 };
802
803 if (SrcTy.isInteger() && ST->hasNEON()) {
804 if (const auto *Entry = ConvertCostTableLookup(Table: NEONIntegerConversionTbl,
805 ISD, Dst: DstTy.getSimpleVT(),
806 Src: SrcTy.getSimpleVT()))
807 return AdjustCost(Entry->Cost);
808 }
809
810 // MVE extend costs, taken from codegen tests. i8->i16 or i16->i32 is one
811 // instruction, i8->i32 is two. i64 zexts are an VAND with a constant, sext
812 // are linearised so take more.
813 static const TypeConversionCostTblEntry MVEVectorConversionTbl[] = {
814 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
815 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
816 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
817 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
818 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 10 },
819 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 2 },
820 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
821 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
822 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 10 },
823 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
824 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 8 },
825 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 2 },
826 };
827
828 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
829 if (const auto *Entry = ConvertCostTableLookup(Table: MVEVectorConversionTbl,
830 ISD, Dst: DstTy.getSimpleVT(),
831 Src: SrcTy.getSimpleVT()))
832 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
833 }
834
835 if (ISD == ISD::FP_ROUND || ISD == ISD::FP_EXTEND) {
836 // As general rule, fp converts that were not matched above are scalarized
837 // and cost 1 vcvt for each lane, so long as the instruction is available.
838 // If not it will become a series of function calls.
839 const InstructionCost CallCost =
840 getCallInstrCost(F: nullptr, RetTy: Dst, Tys: {Src}, CostKind);
841 int Lanes = 1;
842 if (SrcTy.isFixedLengthVector())
843 Lanes = SrcTy.getVectorNumElements();
844
845 if (IsLegalFPType(SrcTy) && IsLegalFPType(DstTy))
846 return Lanes;
847 else
848 return Lanes * CallCost;
849 }
850
851 if (ISD == ISD::TRUNCATE && ST->hasMVEIntegerOps() &&
852 SrcTy.isFixedLengthVector()) {
853 // Treat a truncate with larger than legal source (128bits for MVE) as
854 // expensive, 2 instructions per lane.
855 if ((SrcTy.getScalarType() == MVT::i8 ||
856 SrcTy.getScalarType() == MVT::i16 ||
857 SrcTy.getScalarType() == MVT::i32) &&
858 SrcTy.getSizeInBits() > 128 &&
859 SrcTy.getSizeInBits() > DstTy.getSizeInBits())
860 return SrcTy.getVectorNumElements() * 2;
861 }
862
863 // Scalar integer conversion costs.
864 static const TypeConversionCostTblEntry ARMIntegerConversionTbl[] = {
865 // i16 -> i64 requires two dependent operations.
866 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 2 },
867
868 // Truncates on i64 are assumed to be free.
869 { .ISD: ISD::TRUNCATE, .Dst: MVT::i32, .Src: MVT::i64, .Cost: 0 },
870 { .ISD: ISD::TRUNCATE, .Dst: MVT::i16, .Src: MVT::i64, .Cost: 0 },
871 { .ISD: ISD::TRUNCATE, .Dst: MVT::i8, .Src: MVT::i64, .Cost: 0 },
872 { .ISD: ISD::TRUNCATE, .Dst: MVT::i1, .Src: MVT::i64, .Cost: 0 }
873 };
874
875 if (SrcTy.isInteger()) {
876 if (const auto *Entry = ConvertCostTableLookup(Table: ARMIntegerConversionTbl, ISD,
877 Dst: DstTy.getSimpleVT(),
878 Src: SrcTy.getSimpleVT()))
879 return AdjustCost(Entry->Cost);
880 }
881
882 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
883 ? ST->getMVEVectorCostFactor(CostKind)
884 : 1;
885 return AdjustCost(
886 BaseCost * BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
887}
888
889InstructionCost ARMTTIImpl::getVectorInstrCost(
890 unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
891 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
892 // Penalize inserting into an D-subregister. We end up with a three times
893 // lower estimated throughput on swift.
894 if (ST->hasSlowLoadDSubregister() && Opcode == Instruction::InsertElement &&
895 ValTy->isVectorTy() && ValTy->getScalarSizeInBits() <= 32)
896 return 3;
897
898 if (ST->hasNEON() && (Opcode == Instruction::InsertElement ||
899 Opcode == Instruction::ExtractElement)) {
900 // Cross-class copies are expensive on many microarchitectures,
901 // so assume they are expensive by default.
902 if (cast<VectorType>(Val: ValTy)->getElementType()->isIntegerTy())
903 return 3;
904
905 // Even if it's not a cross class copy, this likely leads to mixing
906 // of NEON and VFP code and should be therefore penalized.
907 if (ValTy->isVectorTy() &&
908 ValTy->getScalarSizeInBits() <= 32)
909 return std::max<InstructionCost>(
910 a: BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
911 VIC),
912 b: 2U);
913 }
914
915 if (ST->hasMVEIntegerOps() && (Opcode == Instruction::InsertElement ||
916 Opcode == Instruction::ExtractElement)) {
917 // Integer cross-lane moves are more expensive than float, which can
918 // sometimes just be vmovs. Integer involve being passes to GPR registers,
919 // causing more of a delay.
920 std::pair<InstructionCost, MVT> LT =
921 getTypeLegalizationCost(Ty: ValTy->getScalarType());
922 return LT.first * (ValTy->getScalarType()->isIntegerTy() ? 4 : 1);
923 }
924
925 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
926 VIC);
927}
928
929InstructionCost ARMTTIImpl::getCmpSelInstrCost(
930 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
931 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
932 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
933 int ISD = TLI->InstructionOpcodeToISD(Opcode);
934
935 // Thumb scalar code size cost for select.
936 if (CostKind == TTI::TCK_CodeSize && ISD == ISD::SELECT &&
937 ST->isThumb() && !ValTy->isVectorTy()) {
938 // Assume expensive structs.
939 if (TLI->getValueType(DL, Ty: ValTy, AllowUnknown: true) == MVT::Other)
940 return TTI::TCC_Expensive;
941
942 // Select costs can vary because they:
943 // - may require one or more conditional mov (including an IT),
944 // - can't operate directly on immediates,
945 // - require live flags, which we can't copy around easily.
946 InstructionCost Cost = getTypeLegalizationCost(Ty: ValTy).first;
947
948 // Possible IT instruction for Thumb2, or more for Thumb1.
949 ++Cost;
950
951 // i1 values may need rematerialising by using mov immediates and/or
952 // flag setting instructions.
953 if (ValTy->isIntegerTy(BitWidth: 1))
954 ++Cost;
955
956 return Cost;
957 }
958
959 // If this is a vector min/max/abs, use the cost of that intrinsic directly
960 // instead. Hopefully when min/max intrinsics are more prevalent this code
961 // will not be needed.
962 const Instruction *Sel = I;
963 if ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && Sel &&
964 Sel->hasOneUse())
965 Sel = cast<Instruction>(Val: Sel->user_back());
966 if (Sel && ValTy->isVectorTy() &&
967 (ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy())) {
968 const Value *LHS, *RHS;
969 SelectPatternFlavor SPF = matchSelectPattern(V: Sel, LHS, RHS).Flavor;
970 unsigned IID = 0;
971 switch (SPF) {
972 case SPF_ABS:
973 IID = Intrinsic::abs;
974 break;
975 case SPF_SMIN:
976 IID = Intrinsic::smin;
977 break;
978 case SPF_SMAX:
979 IID = Intrinsic::smax;
980 break;
981 case SPF_UMIN:
982 IID = Intrinsic::umin;
983 break;
984 case SPF_UMAX:
985 IID = Intrinsic::umax;
986 break;
987 case SPF_FMINNUM:
988 IID = Intrinsic::minnum;
989 break;
990 case SPF_FMAXNUM:
991 IID = Intrinsic::maxnum;
992 break;
993 default:
994 break;
995 }
996 if (IID) {
997 // The ICmp is free, the select gets the cost of the min/max/etc
998 if (Sel != I)
999 return 0;
1000 IntrinsicCostAttributes CostAttrs(IID, ValTy, {ValTy, ValTy});
1001 return getIntrinsicInstrCost(ICA: CostAttrs, CostKind);
1002 }
1003 }
1004
1005 // On NEON a vector select gets lowered to vbsl.
1006 if (ST->hasNEON() && ValTy->isVectorTy() && ISD == ISD::SELECT && CondTy) {
1007 // Lowering of some vector selects is currently far from perfect.
1008 static const TypeConversionCostTblEntry NEONVectorSelectTbl[] = {
1009 { .ISD: ISD::SELECT, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: 4*4 + 1*2 + 1 },
1010 { .ISD: ISD::SELECT, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: 50 },
1011 { .ISD: ISD::SELECT, .Dst: MVT::v16i1, .Src: MVT::v16i64, .Cost: 100 }
1012 };
1013
1014 EVT SelCondTy = TLI->getValueType(DL, Ty: CondTy);
1015 EVT SelValTy = TLI->getValueType(DL, Ty: ValTy);
1016 if (SelCondTy.isSimple() && SelValTy.isSimple()) {
1017 if (const auto *Entry = ConvertCostTableLookup(Table: NEONVectorSelectTbl, ISD,
1018 Dst: SelCondTy.getSimpleVT(),
1019 Src: SelValTy.getSimpleVT()))
1020 return Entry->Cost;
1021 }
1022
1023 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1024 return LT.first;
1025 }
1026
1027 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy() &&
1028 (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
1029 cast<FixedVectorType>(Val: ValTy)->getNumElements() > 1) {
1030 FixedVectorType *VecValTy = cast<FixedVectorType>(Val: ValTy);
1031 FixedVectorType *VecCondTy = dyn_cast_or_null<FixedVectorType>(Val: CondTy);
1032 if (!VecCondTy)
1033 VecCondTy = cast<FixedVectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecValTy));
1034
1035 // If we don't have mve.fp any fp operations will need to be scalarized.
1036 if (Opcode == Instruction::FCmp && !ST->hasMVEFloatOps()) {
1037 // One scalaization insert, one scalarization extract and the cost of the
1038 // fcmps.
1039 return BaseT::getScalarizationOverhead(InTy: VecValTy, /*Insert*/ false,
1040 /*Extract*/ true, CostKind) +
1041 BaseT::getScalarizationOverhead(InTy: VecCondTy, /*Insert*/ true,
1042 /*Extract*/ false, CostKind) +
1043 VecValTy->getNumElements() *
1044 getCmpSelInstrCost(Opcode, ValTy: ValTy->getScalarType(),
1045 CondTy: VecCondTy->getScalarType(), VecPred,
1046 CostKind, Op1Info, Op2Info, I);
1047 }
1048
1049 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1050 int BaseCost = ST->getMVEVectorCostFactor(CostKind);
1051 // There are two types - the input that specifies the type of the compare
1052 // and the output vXi1 type. Because we don't know how the output will be
1053 // split, we may need an expensive shuffle to get two in sync. This has the
1054 // effect of making larger than legal compares (v8i32 for example)
1055 // expensive.
1056 if (LT.second.isVector() && LT.second.getVectorNumElements() > 2) {
1057 if (LT.first > 1)
1058 return LT.first * BaseCost +
1059 BaseT::getScalarizationOverhead(InTy: VecCondTy, /*Insert*/ true,
1060 /*Extract*/ false, CostKind);
1061 return BaseCost;
1062 }
1063 }
1064
1065 // Default to cheap (throughput/size of 1 instruction) but adjust throughput
1066 // for "multiple beats" potentially needed by MVE instructions.
1067 int BaseCost = 1;
1068 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy())
1069 BaseCost = ST->getMVEVectorCostFactor(CostKind);
1070
1071 return BaseCost * BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred,
1072 CostKind, Op1Info, Op2Info, I);
1073}
1074
1075InstructionCost
1076ARMTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
1077 const SCEV *Ptr,
1078 TTI::TargetCostKind CostKind) const {
1079 // Address computations in vectorized code with non-consecutive addresses will
1080 // likely result in more instructions compared to scalar code where the
1081 // computation can more often be merged into the index mode. The resulting
1082 // extra micro-ops can significantly decrease throughput.
1083 unsigned NumVectorInstToHideOverhead = 10;
1084 int MaxMergeDistance = 64;
1085
1086 if (ST->hasNEON()) {
1087 if (PtrTy->isVectorTy() && SE &&
1088 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MergeDistance: MaxMergeDistance + 1))
1089 return NumVectorInstToHideOverhead;
1090
1091 // In many cases the address computation is not merged into the instruction
1092 // addressing mode.
1093 return 1;
1094 }
1095 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
1096}
1097
1098bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) const {
1099 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1100 // If a VCTP is part of a chain, it's already profitable and shouldn't be
1101 // optimized, else LSR may block tail-predication.
1102 switch (II->getIntrinsicID()) {
1103 case Intrinsic::arm_mve_vctp8:
1104 case Intrinsic::arm_mve_vctp16:
1105 case Intrinsic::arm_mve_vctp32:
1106 case Intrinsic::arm_mve_vctp64:
1107 return true;
1108 default:
1109 break;
1110 }
1111 }
1112 return false;
1113}
1114
1115bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
1116 unsigned /*AddressSpace*/,
1117 TTI::MaskKind /*MaskKind*/) const {
1118 if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
1119 return false;
1120
1121 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: DataTy)) {
1122 // Don't support v2i1 yet.
1123 if (VecTy->getNumElements() == 2)
1124 return false;
1125
1126 // We don't support extending fp types.
1127 unsigned VecWidth = DataTy->getPrimitiveSizeInBits();
1128 if (VecWidth != 128 && VecTy->getElementType()->isFloatingPointTy())
1129 return false;
1130 }
1131
1132 unsigned EltWidth = DataTy->getScalarSizeInBits();
1133 return (EltWidth == 32 && Alignment >= 4) ||
1134 (EltWidth == 16 && Alignment >= 2) || (EltWidth == 8);
1135}
1136
1137bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) const {
1138 if (!EnableMaskedGatherScatters || !ST->hasMVEIntegerOps())
1139 return false;
1140
1141 unsigned EltWidth = Ty->getScalarSizeInBits();
1142 return ((EltWidth == 32 && Alignment >= 4) ||
1143 (EltWidth == 16 && Alignment >= 2) || EltWidth == 8);
1144}
1145
1146/// Given a memcpy/memset/memmove instruction, return the number of memory
1147/// operations performed, via querying findOptimalMemOpLowering. Returns -1 if a
1148/// call is used.
1149int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const {
1150 MemOp MOp;
1151 unsigned DstAddrSpace = ~0u;
1152 unsigned SrcAddrSpace = ~0u;
1153 const Function *F = I->getParent()->getParent();
1154
1155 if (const auto *MC = dyn_cast<MemTransferInst>(Val: I)) {
1156 ConstantInt *C = dyn_cast<ConstantInt>(Val: MC->getLength());
1157 // If 'size' is not a constant, a library call will be generated.
1158 if (!C)
1159 return -1;
1160
1161 const unsigned Size = C->getValue().getZExtValue();
1162 const Align DstAlign = MC->getDestAlign().valueOrOne();
1163 const Align SrcAlign = MC->getSourceAlign().valueOrOne();
1164
1165 // Use the most restrictive of memset, memcpy, memmove.
1166 MOp = MemOp::Move(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
1167 /*IsVolatile*/ false);
1168 DstAddrSpace = MC->getDestAddressSpace();
1169 SrcAddrSpace = MC->getSourceAddressSpace();
1170 }
1171 else if (const auto *MS = dyn_cast<MemSetInst>(Val: I)) {
1172 ConstantInt *C = dyn_cast<ConstantInt>(Val: MS->getLength());
1173 // If 'size' is not a constant, a library call will be generated.
1174 if (!C)
1175 return -1;
1176
1177 const unsigned Size = C->getValue().getZExtValue();
1178 const Align DstAlign = MS->getDestAlign().valueOrOne();
1179
1180 MOp = MemOp::Set(Size, /*DstAlignCanChange*/ false, DstAlign,
1181 /*IsZeroMemset*/ false, /*IsVolatile*/ false);
1182 DstAddrSpace = MS->getDestAddressSpace();
1183 }
1184 else
1185 llvm_unreachable("Expected a memcpy/move or memset!");
1186
1187 unsigned Limit, Factor = 2;
1188 switch(I->getIntrinsicID()) {
1189 case Intrinsic::memcpy:
1190 Limit = TLI->getMaxStoresPerMemcpy(OptSize: F->hasMinSize());
1191 break;
1192 case Intrinsic::memmove:
1193 Limit = TLI->getMaxStoresPerMemmove(OptSize: F->hasMinSize());
1194 break;
1195 case Intrinsic::memset:
1196 Limit = TLI->getMaxStoresPerMemset(OptSize: F->hasMinSize());
1197 Factor = 1;
1198 break;
1199 default:
1200 llvm_unreachable("Expected a memcpy/move or memset!");
1201 }
1202
1203 // MemOps will be poplulated with a list of data types that needs to be
1204 // loaded and stored. That's why we multiply the number of elements by 2 to
1205 // get the cost for this memcpy.
1206 std::vector<EVT> MemOps;
1207 LLVMContext &C = F->getContext();
1208 if (getTLI()->findOptimalMemOpLowering(Context&: C, MemOps, Limit, Op: MOp, DstAS: DstAddrSpace,
1209 SrcAS: SrcAddrSpace, FuncAttributes: F->getAttributes(),
1210 LargestVT: nullptr))
1211 return MemOps.size() * Factor;
1212
1213 // If we can't find an optimal memop lowering, return the default cost
1214 return -1;
1215}
1216
1217InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const {
1218 int NumOps = getNumMemOps(I: cast<IntrinsicInst>(Val: I));
1219
1220 // To model the cost of a library call, we assume 1 for the call, and
1221 // 3 for the argument setup.
1222 if (NumOps == -1)
1223 return 4;
1224 return NumOps;
1225}
1226
1227InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
1228 VectorType *DstTy, VectorType *SrcTy,
1229 TTI::TargetCostKind CostKind,
1230 ArrayRef<int> Mask, int Index,
1231 VectorType *SubTp,
1232 ArrayRef<const Value *> Args,
1233 const Instruction *CxtI) const {
1234 assert((Mask.empty() || DstTy->isScalableTy() ||
1235 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
1236 "Expected the Mask to match the return size if given");
1237 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
1238 "Expected the same scalar types");
1239
1240 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1241 // Treat extractsubvector as single op permutation.
1242 bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector;
1243 if (IsExtractSubvector)
1244 Kind = TTI::SK_PermuteSingleSrc;
1245 if (ST->hasNEON()) {
1246 if (Kind == TTI::SK_Broadcast) {
1247 static const CostTblEntry NEONDupTbl[] = {
1248 // VDUP handles these cases.
1249 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1250 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1251 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1252 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1253 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 1},
1254 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i8, .Cost: 1},
1255
1256 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 1},
1257 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 1},
1258 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 1},
1259 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 1}};
1260
1261 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1262 if (const auto *Entry =
1263 CostTableLookup(Table: NEONDupTbl, ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1264 return LT.first * Entry->Cost;
1265 }
1266 if (Kind == TTI::SK_Reverse) {
1267 static const CostTblEntry NEONShuffleTbl[] = {
1268 // Reverse shuffle cost one instruction if we are shuffling within a
1269 // double word (vrev) or two if we shuffle a quad word (vrev, vext).
1270 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1271 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1272 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1273 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1274 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 1},
1275 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i8, .Cost: 1},
1276
1277 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 2},
1278 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 2},
1279 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 2},
1280 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 2}};
1281
1282 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1283 if (const auto *Entry =
1284 CostTableLookup(Table: NEONShuffleTbl, ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1285 return LT.first * Entry->Cost;
1286 }
1287 if (Kind == TTI::SK_Select) {
1288 static const CostTblEntry NEONSelShuffleTbl[] = {
1289 // Select shuffle cost table for ARM. Cost is the number of
1290 // instructions
1291 // required to create the shuffled vector.
1292
1293 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1294 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1295 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1296 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1297
1298 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 2},
1299 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 2},
1300 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 2},
1301
1302 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 16},
1303
1304 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 32}};
1305
1306 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1307 if (const auto *Entry = CostTableLookup(Table: NEONSelShuffleTbl,
1308 ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1309 return LT.first * Entry->Cost;
1310 }
1311
1312 // Check for other shuffles that are not SK_ kinds but we have native
1313 // instructions for, for example REV.
1314 if (!Mask.empty()) {
1315 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1316 if (LT.second.isVector() &&
1317 Mask.size() <= LT.second.getVectorNumElements() &&
1318 (isVREVMask(M: Mask, VT: LT.second, BlockSize: 16) || isVREVMask(M: Mask, VT: LT.second, BlockSize: 32) ||
1319 isVREVMask(M: Mask, VT: LT.second, BlockSize: 64)))
1320 return LT.first;
1321 }
1322 }
1323 if (ST->hasMVEIntegerOps()) {
1324 if (Kind == TTI::SK_Broadcast) {
1325 static const CostTblEntry MVEDupTbl[] = {
1326 // VDUP handles these cases.
1327 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 1},
1328 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 1},
1329 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 1},
1330 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 1},
1331 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8f16, .Cost: 1}};
1332
1333 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1334 if (const auto *Entry = CostTableLookup(Table: MVEDupTbl, ISD: ISD::VECTOR_SHUFFLE,
1335 Ty: LT.second))
1336 return LT.first * Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
1337 }
1338
1339 if (!Mask.empty()) {
1340 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1341 // Check for LD2/LD4 instructions, which are represented in llvm IR as
1342 // deinterleaving-shuffle(load). The shuffle cost could potentially be
1343 // free, but we model it with a cost of LT.first so that LD2/LD4 have a
1344 // higher cost than just the load.
1345 if (Args.size() >= 1 && isa<LoadInst>(Val: Args[0]) &&
1346 (LT.second.getScalarSizeInBits() == 8 ||
1347 LT.second.getScalarSizeInBits() == 16 ||
1348 LT.second.getScalarSizeInBits() == 32) &&
1349 LT.second.getSizeInBits() == 128 &&
1350 ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
1351 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2)) ||
1352 (TLI->getMaxSupportedInterleaveFactor() == 4 &&
1353 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 4))))
1354 return ST->getMVEVectorCostFactor(CostKind) *
1355 std::max<InstructionCost>(a: 1, b: LT.first / 4);
1356
1357 // Check for ST2/ST4 instructions, which are represented in llvm IR as
1358 // store(interleaving-shuffle). The shuffle cost could potentially be
1359 // free, but we model it with a cost of LT.first so that ST2/ST4 have a
1360 // higher cost than just the store.
1361 if (CxtI && CxtI->hasOneUse() && isa<StoreInst>(Val: *CxtI->user_begin()) &&
1362 (LT.second.getScalarSizeInBits() == 8 ||
1363 LT.second.getScalarSizeInBits() == 16 ||
1364 LT.second.getScalarSizeInBits() == 32) &&
1365 LT.second.getSizeInBits() == 128 &&
1366 ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
1367 ShuffleVectorInst::isInterleaveMask(
1368 Mask, Factor: 2, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2)) ||
1369 (TLI->getMaxSupportedInterleaveFactor() == 4 &&
1370 ShuffleVectorInst::isInterleaveMask(
1371 Mask, Factor: 4, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2))))
1372 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1373
1374 if (LT.second.isVector() &&
1375 Mask.size() <= LT.second.getVectorNumElements() &&
1376 (isVREVMask(M: Mask, VT: LT.second, BlockSize: 16) || isVREVMask(M: Mask, VT: LT.second, BlockSize: 32) ||
1377 isVREVMask(M: Mask, VT: LT.second, BlockSize: 64)))
1378 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1379 }
1380 }
1381
1382 // Restore optimal kind.
1383 if (IsExtractSubvector)
1384 Kind = TTI::SK_ExtractSubvector;
1385 int BaseCost = ST->hasMVEIntegerOps() && SrcTy->isVectorTy()
1386 ? ST->getMVEVectorCostFactor(CostKind)
1387 : 1;
1388 return BaseCost * BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask,
1389 Index, SubTp);
1390}
1391
1392InstructionCost ARMTTIImpl::getArithmeticInstrCost(
1393 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
1394 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
1395 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
1396 int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
1397 if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(BitWidth: 1)) {
1398 // Make operations on i1 relatively expensive as this often involves
1399 // combining predicates. AND and XOR should be easier to handle with IT
1400 // blocks.
1401 switch (ISDOpcode) {
1402 default:
1403 break;
1404 case ISD::AND:
1405 case ISD::XOR:
1406 return 2;
1407 case ISD::OR:
1408 return 3;
1409 }
1410 }
1411
1412 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1413
1414 if (ST->hasNEON()) {
1415 const unsigned FunctionCallDivCost = 20;
1416 const unsigned ReciprocalDivCost = 10;
1417 static const CostTblEntry CostTbl[] = {
1418 // Division.
1419 // These costs are somewhat random. Choose a cost of 20 to indicate that
1420 // vectorizing devision (added function call) is going to be very expensive.
1421 // Double registers types.
1422 { .ISD: ISD::SDIV, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1423 { .ISD: ISD::UDIV, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1424 { .ISD: ISD::SREM, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1425 { .ISD: ISD::UREM, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1426 { .ISD: ISD::SDIV, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1427 { .ISD: ISD::UDIV, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1428 { .ISD: ISD::SREM, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1429 { .ISD: ISD::UREM, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1430 { .ISD: ISD::SDIV, .Type: MVT::v4i16, .Cost: ReciprocalDivCost},
1431 { .ISD: ISD::UDIV, .Type: MVT::v4i16, .Cost: ReciprocalDivCost},
1432 { .ISD: ISD::SREM, .Type: MVT::v4i16, .Cost: 4 * FunctionCallDivCost},
1433 { .ISD: ISD::UREM, .Type: MVT::v4i16, .Cost: 4 * FunctionCallDivCost},
1434 { .ISD: ISD::SDIV, .Type: MVT::v8i8, .Cost: ReciprocalDivCost},
1435 { .ISD: ISD::UDIV, .Type: MVT::v8i8, .Cost: ReciprocalDivCost},
1436 { .ISD: ISD::SREM, .Type: MVT::v8i8, .Cost: 8 * FunctionCallDivCost},
1437 { .ISD: ISD::UREM, .Type: MVT::v8i8, .Cost: 8 * FunctionCallDivCost},
1438 // Quad register types.
1439 { .ISD: ISD::SDIV, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1440 { .ISD: ISD::UDIV, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1441 { .ISD: ISD::SREM, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1442 { .ISD: ISD::UREM, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1443 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1444 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1445 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1446 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1447 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1448 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1449 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1450 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1451 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1452 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1453 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1454 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1455 // Multiplication.
1456 };
1457
1458 if (const auto *Entry = CostTableLookup(Table: CostTbl, ISD: ISDOpcode, Ty: LT.second))
1459 return LT.first * Entry->Cost;
1460
1461 InstructionCost Cost = BaseT::getArithmeticInstrCost(
1462 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
1463
1464 // This is somewhat of a hack. The problem that we are facing is that SROA
1465 // creates a sequence of shift, and, or instructions to construct values.
1466 // These sequences are recognized by the ISel and have zero-cost. Not so for
1467 // the vectorized code. Because we have support for v2i64 but not i64 those
1468 // sequences look particularly beneficial to vectorize.
1469 // To work around this we increase the cost of v2i64 operations to make them
1470 // seem less beneficial.
1471 if (LT.second == MVT::v2i64 && Op2Info.isUniform() && Op2Info.isConstant())
1472 Cost += 4;
1473
1474 return Cost;
1475 }
1476
1477 // If this operation is a shift on arm/thumb2, it might well be folded into
1478 // the following instruction, hence having a cost of 0.
1479 auto LooksLikeAFreeShift = [&]() {
1480 if (ST->isThumb1Only() || Ty->isVectorTy())
1481 return false;
1482
1483 if (!CxtI || !CxtI->hasOneUse() || !CxtI->isShift())
1484 return false;
1485 if (!Op2Info.isUniform() || !Op2Info.isConstant())
1486 return false;
1487
1488 // Folded into a ADC/ADD/AND/BIC/CMP/EOR/MVN/ORR/ORN/RSB/SBC/SUB
1489 switch (cast<Instruction>(Val: CxtI->user_back())->getOpcode()) {
1490 case Instruction::Add:
1491 case Instruction::Sub:
1492 case Instruction::And:
1493 case Instruction::Xor:
1494 case Instruction::Or:
1495 case Instruction::ICmp:
1496 return true;
1497 default:
1498 return false;
1499 }
1500 };
1501 if (LooksLikeAFreeShift())
1502 return 0;
1503
1504 // When targets have both DSP and MVE we find that the
1505 // the compiler will attempt to vectorize as well as using
1506 // scalar (S/U)MLAL operations. This is in cases where we have
1507 // the pattern ext(mul(ext(i16), ext(i16))) we find
1508 // that codegen performs better when only using (S/U)MLAL scalar
1509 // ops instead of trying to mix vector ops with (S/U)MLAL ops. We therefore
1510 // check if a mul instruction is used in a (U/S)MLAL pattern.
1511 auto MulInDSPMLALPattern = [&](const Instruction *I, unsigned Opcode,
1512 Type *Ty) -> bool {
1513 if (!ST->hasDSP())
1514 return false;
1515
1516 if (!I)
1517 return false;
1518
1519 if (Opcode != Instruction::Mul)
1520 return false;
1521
1522 if (Ty->isVectorTy())
1523 return false;
1524
1525 auto ValueOpcodesEqual = [](const Value *LHS, const Value *RHS) -> bool {
1526 return cast<Instruction>(Val: LHS)->getOpcode() ==
1527 cast<Instruction>(Val: RHS)->getOpcode();
1528 };
1529 auto IsExtInst = [](const Value *V) -> bool {
1530 return isa<ZExtInst>(Val: V) || isa<SExtInst>(Val: V);
1531 };
1532 auto IsExtensionFromHalf = [](const Value *V) -> bool {
1533 return cast<Instruction>(Val: V)->getOperand(i: 0)->getType()->isIntegerTy(BitWidth: 16);
1534 };
1535
1536 // We check the arguments of the instruction to see if they're extends
1537 auto *BinOp = dyn_cast<BinaryOperator>(Val: I);
1538 if (!BinOp)
1539 return false;
1540 Value *Op0 = BinOp->getOperand(i_nocapture: 0);
1541 Value *Op1 = BinOp->getOperand(i_nocapture: 1);
1542 if (IsExtInst(Op0) && IsExtInst(Op1) && ValueOpcodesEqual(Op0, Op1)) {
1543 // We're interested in an ext of an i16
1544 if (!I->getType()->isIntegerTy(BitWidth: 32) || !IsExtensionFromHalf(Op0) ||
1545 !IsExtensionFromHalf(Op1))
1546 return false;
1547 // We need to check if this result will be further extended to i64
1548 // and that all these uses are SExt
1549 for (auto *U : I->users())
1550 if (!IsExtInst(U))
1551 return false;
1552 return true;
1553 }
1554
1555 return false;
1556 };
1557
1558 if (MulInDSPMLALPattern(CxtI, Opcode, Ty))
1559 return 0;
1560
1561 // Default to cheap (throughput/size of 1 instruction) but adjust throughput
1562 // for "multiple beats" potentially needed by MVE instructions.
1563 int BaseCost = 1;
1564 if (ST->hasMVEIntegerOps() && Ty->isVectorTy())
1565 BaseCost = ST->getMVEVectorCostFactor(CostKind);
1566
1567 // The rest of this mostly follows what is done in
1568 // BaseT::getArithmeticInstrCost, without treating floats as more expensive
1569 // that scalars or increasing the costs for custom operations. The results is
1570 // also multiplied by the MVEVectorCostFactor where appropriate.
1571 if (TLI->isOperationLegalOrCustomOrPromote(Op: ISDOpcode, VT: LT.second))
1572 return LT.first * BaseCost;
1573
1574 // Else this is expand, assume that we need to scalarize this op.
1575 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
1576 unsigned Num = VTy->getNumElements();
1577 InstructionCost Cost =
1578 getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind);
1579 // Return the cost of multiple scalar invocation plus the cost of
1580 // inserting and extracting the values.
1581 SmallVector<Type *> Tys(Args.size(), Ty);
1582 return BaseT::getScalarizationOverhead(RetTy: VTy, Args, Tys, CostKind) +
1583 Num * Cost;
1584 }
1585
1586 return BaseCost;
1587}
1588
1589InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
1590 Align Alignment,
1591 unsigned AddressSpace,
1592 TTI::TargetCostKind CostKind,
1593 TTI::OperandValueInfo OpInfo,
1594 const Instruction *I) const {
1595 // FIXME: Load latency isn't handled here
1596 if (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)
1597 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1598 CostKind, OpInfo, I);
1599
1600 // TODO: Handle other cost kinds.
1601 if (CostKind != TTI::TCK_RecipThroughput)
1602 return 1;
1603
1604 // Type legalization can't handle structs
1605 if (TLI->getValueType(DL, Ty: Src, AllowUnknown: true) == MVT::Other)
1606 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1607 CostKind);
1608
1609 if (ST->hasNEON() && Src->isVectorTy() && Alignment != Align(16) &&
1610 cast<VectorType>(Val: Src)->getElementType()->isDoubleTy()) {
1611 // Unaligned loads/stores are extremely inefficient.
1612 // We need 4 uops for vst.1/vld.1 vs 1uop for vldr/vstr.
1613 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
1614 return LT.first * 4;
1615 }
1616
1617 // MVE can optimize a fpext(load(4xhalf)) using an extending integer load.
1618 // Same for stores.
1619 if (ST->hasMVEFloatOps() && isa<FixedVectorType>(Val: Src) && I &&
1620 ((Opcode == Instruction::Load && I->hasOneUse() &&
1621 isa<FPExtInst>(Val: *I->user_begin())) ||
1622 (Opcode == Instruction::Store && isa<FPTruncInst>(Val: I->getOperand(i: 0))))) {
1623 FixedVectorType *SrcVTy = cast<FixedVectorType>(Val: Src);
1624 Type *DstTy =
1625 Opcode == Instruction::Load
1626 ? (*I->user_begin())->getType()
1627 : cast<Instruction>(Val: I->getOperand(i: 0))->getOperand(i: 0)->getType();
1628 if (SrcVTy->getNumElements() == 4 && SrcVTy->getScalarType()->isHalfTy() &&
1629 DstTy->getScalarType()->isFloatTy())
1630 return ST->getMVEVectorCostFactor(CostKind);
1631 }
1632
1633 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
1634 ? ST->getMVEVectorCostFactor(CostKind)
1635 : 1;
1636 return BaseCost * BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1637 CostKind, OpInfo, I);
1638}
1639
1640InstructionCost
1641ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1642 TTI::TargetCostKind CostKind) const {
1643 switch (MICA.getID()) {
1644 case Intrinsic::masked_scatter:
1645 case Intrinsic::masked_gather:
1646 return getGatherScatterOpCost(MICA, CostKind);
1647 case Intrinsic::masked_load:
1648 case Intrinsic::masked_store:
1649 return getMaskedMemoryOpCost(MICA, CostKind);
1650 }
1651 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1652}
1653
1654InstructionCost
1655ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1656 TTI::TargetCostKind CostKind) const {
1657 unsigned IID = MICA.getID();
1658 Type *Src = MICA.getDataType();
1659 Align Alignment = MICA.getAlignment();
1660 unsigned AddressSpace = MICA.getAddressSpace();
1661 if (ST->hasMVEIntegerOps()) {
1662 if (IID == Intrinsic::masked_load &&
1663 isLegalMaskedLoad(DataTy: Src, Alignment, AddressSpace))
1664 return ST->getMVEVectorCostFactor(CostKind);
1665 if (IID == Intrinsic::masked_store &&
1666 isLegalMaskedStore(DataTy: Src, Alignment, AddressSpace))
1667 return ST->getMVEVectorCostFactor(CostKind);
1668 }
1669 if (!isa<FixedVectorType>(Val: Src))
1670 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1671 // Scalar cost, which is currently very high due to the efficiency of the
1672 // generated code.
1673 return cast<FixedVectorType>(Val: Src)->getNumElements() * 8;
1674}
1675
1676InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost(
1677 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1678 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1679 bool UseMaskForCond, bool UseMaskForGaps) const {
1680 assert(Factor >= 2 && "Invalid interleave factor");
1681 assert(isa<VectorType>(VecTy) && "Expect a vector type");
1682
1683 // vldN/vstN doesn't support vector types of i64/f64 element.
1684 bool EltIs64Bits = DL.getTypeSizeInBits(Ty: VecTy->getScalarType()) == 64;
1685
1686 if (Factor <= TLI->getMaxSupportedInterleaveFactor() && !EltIs64Bits &&
1687 !UseMaskForCond && !UseMaskForGaps) {
1688 unsigned NumElts = cast<FixedVectorType>(Val: VecTy)->getNumElements();
1689 auto *SubVecTy =
1690 FixedVectorType::get(ElementType: VecTy->getScalarType(), NumElts: NumElts / Factor);
1691
1692 // vldN/vstN only support legal vector types of size 64 or 128 in bits.
1693 // Accesses having vector types that are a multiple of 128 bits can be
1694 // matched to more than one vldN/vstN instruction.
1695 int BaseCost =
1696 ST->hasMVEIntegerOps() ? ST->getMVEVectorCostFactor(CostKind) : 1;
1697 if (NumElts % Factor == 0 &&
1698 TLI->isLegalInterleavedAccessType(Factor, VecTy: SubVecTy, Alignment, DL))
1699 return Factor * BaseCost * TLI->getNumInterleavedAccesses(VecTy: SubVecTy, DL);
1700
1701 // Some smaller than legal interleaved patterns are cheap as we can make
1702 // use of the vmovn or vrev patterns to interleave a standard load. This is
1703 // true for v4i8, v8i8 and v4i16 at least (but not for v4f16 as it is
1704 // promoted differently). The cost of 2 here is then a load and vrev or
1705 // vmovn.
1706 if (ST->hasMVEIntegerOps() && Factor == 2 && NumElts / Factor > 2 &&
1707 VecTy->isIntOrIntVectorTy() &&
1708 DL.getTypeSizeInBits(Ty: SubVecTy).getFixedValue() <= 64)
1709 return 2 * BaseCost;
1710 }
1711
1712 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1713 Alignment, AddressSpace, CostKind,
1714 UseMaskForCond, UseMaskForGaps);
1715}
1716
1717InstructionCost
1718ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1719 TTI::TargetCostKind CostKind) const {
1720
1721 Type *DataTy = MICA.getDataType();
1722 const Value *Ptr = MICA.getPointer();
1723 bool VariableMask = MICA.getVariableMask();
1724 Align Alignment = MICA.getAlignment();
1725 const Instruction *I = MICA.getInst();
1726
1727 using namespace PatternMatch;
1728 if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
1729 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1730
1731 assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!");
1732 auto *VTy = cast<FixedVectorType>(Val: DataTy);
1733
1734 // TODO: Splitting, once we do that.
1735
1736 unsigned NumElems = VTy->getNumElements();
1737 unsigned EltSize = VTy->getScalarSizeInBits();
1738 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: DataTy);
1739
1740 // For now, it is assumed that for the MVE gather instructions the loads are
1741 // all effectively serialised. This means the cost is the scalar cost
1742 // multiplied by the number of elements being loaded. This is possibly very
1743 // conservative, but even so we still end up vectorising loops because the
1744 // cost per iteration for many loops is lower than for scalar loops.
1745 InstructionCost VectorCost =
1746 NumElems * LT.first * ST->getMVEVectorCostFactor(CostKind);
1747 // The scalarization cost should be a lot higher. We use the number of vector
1748 // elements plus the scalarization overhead. If masking is required then a lot
1749 // of little blocks will be needed and potentially a scalarized p0 mask,
1750 // greatly increasing the cost.
1751 InstructionCost ScalarCost =
1752 NumElems * LT.first + (VariableMask ? NumElems * 5 : 0) +
1753 BaseT::getScalarizationOverhead(InTy: VTy, /*Insert*/ true, /*Extract*/ false,
1754 CostKind) +
1755 BaseT::getScalarizationOverhead(InTy: VTy, /*Insert*/ false, /*Extract*/ true,
1756 CostKind);
1757
1758 if (EltSize < 8 || Alignment < EltSize / 8)
1759 return ScalarCost;
1760
1761 unsigned ExtSize = EltSize;
1762 // Check whether there's a single user that asks for an extended type
1763 if (I != nullptr) {
1764 // Dependent of the caller of this function, a gather instruction will
1765 // either have opcode Instruction::Load or be a call to the masked_gather
1766 // intrinsic
1767 if ((I->getOpcode() == Instruction::Load ||
1768 match(V: I, P: m_Intrinsic<Intrinsic::masked_gather>())) &&
1769 I->hasOneUse()) {
1770 const User *Us = *I->users().begin();
1771 if (isa<ZExtInst>(Val: Us) || isa<SExtInst>(Val: Us)) {
1772 // only allow valid type combinations
1773 unsigned TypeSize =
1774 cast<Instruction>(Val: Us)->getType()->getScalarSizeInBits();
1775 if (((TypeSize == 32 && (EltSize == 8 || EltSize == 16)) ||
1776 (TypeSize == 16 && EltSize == 8)) &&
1777 TypeSize * NumElems == 128) {
1778 ExtSize = TypeSize;
1779 }
1780 }
1781 }
1782 // Check whether the input data needs to be truncated
1783 TruncInst *T;
1784 if ((I->getOpcode() == Instruction::Store ||
1785 match(V: I, P: m_Intrinsic<Intrinsic::masked_scatter>())) &&
1786 (T = dyn_cast<TruncInst>(Val: I->getOperand(i: 0)))) {
1787 // Only allow valid type combinations
1788 unsigned TypeSize = T->getOperand(i_nocapture: 0)->getType()->getScalarSizeInBits();
1789 if (((EltSize == 16 && TypeSize == 32) ||
1790 (EltSize == 8 && (TypeSize == 32 || TypeSize == 16))) &&
1791 TypeSize * NumElems == 128)
1792 ExtSize = TypeSize;
1793 }
1794 }
1795
1796 if (ExtSize * NumElems != 128 || NumElems < 4)
1797 return ScalarCost;
1798
1799 // Any (aligned) i32 gather will not need to be scalarised.
1800 if (ExtSize == 32)
1801 return VectorCost;
1802 // For smaller types, we need to ensure that the gep's inputs are correctly
1803 // extended from a small enough value. Other sizes (including i64) are
1804 // scalarized for now.
1805 if (ExtSize != 8 && ExtSize != 16)
1806 return ScalarCost;
1807
1808 if (const auto *BC = dyn_cast<BitCastInst>(Val: Ptr))
1809 Ptr = BC->getOperand(i_nocapture: 0);
1810 if (const auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr)) {
1811 if (GEP->getNumOperands() != 2)
1812 return ScalarCost;
1813 unsigned Scale = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
1814 // Scale needs to be correct (which is only relevant for i16s).
1815 if (Scale != 1 && Scale * 8 != ExtSize)
1816 return ScalarCost;
1817 // And we need to zext (not sext) the indexes from a small enough type.
1818 if (const auto *ZExt = dyn_cast<ZExtInst>(Val: GEP->getOperand(i_nocapture: 1))) {
1819 if (ZExt->getOperand(i_nocapture: 0)->getType()->getScalarSizeInBits() <= ExtSize)
1820 return VectorCost;
1821 }
1822 return ScalarCost;
1823 }
1824 return ScalarCost;
1825}
1826
1827InstructionCost
1828ARMTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
1829 std::optional<FastMathFlags> FMF,
1830 TTI::TargetCostKind CostKind) const {
1831
1832 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1833 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1834 unsigned EltSize = ValVT.getScalarSizeInBits();
1835
1836 // In general floating point reductions are a series of elementwise
1837 // operations, with free extracts on each step. These are either in-order or
1838 // treewise depending on whether that is allowed by the fast math flags.
1839 if ((ISD == ISD::FADD || ISD == ISD::FMUL) &&
1840 ((EltSize == 32 && ST->hasVFP2Base()) ||
1841 (EltSize == 64 && ST->hasFP64()) ||
1842 (EltSize == 16 && ST->hasFullFP16()))) {
1843 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
1844 unsigned VecLimit = ST->hasMVEFloatOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1845 InstructionCost VecCost = 0;
1846 while (!TTI::requiresOrderedReduction(FMF) && isPowerOf2_32(Value: NumElts) &&
1847 NumElts * EltSize > VecLimit) {
1848 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts: NumElts / 2);
1849 VecCost += getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1850 NumElts /= 2;
1851 }
1852
1853 // For fp16 we need to extract the upper lane elements. MVE can add a
1854 // VREV+FMIN/MAX to perform another vector step instead.
1855 InstructionCost ExtractCost = 0;
1856 if (!TTI::requiresOrderedReduction(FMF) && ST->hasMVEFloatOps() &&
1857 ValVT.getVectorElementType() == MVT::f16 && NumElts == 8) {
1858 VecCost += ST->getMVEVectorCostFactor(CostKind) * 2;
1859 NumElts /= 2;
1860 } else if (ValVT.getVectorElementType() == MVT::f16)
1861 ExtractCost = NumElts / 2;
1862
1863 return VecCost + ExtractCost +
1864 NumElts *
1865 getArithmeticInstrCost(Opcode, Ty: ValTy->getElementType(), CostKind);
1866 }
1867
1868 if ((ISD == ISD::AND || ISD == ISD::OR || ISD == ISD::XOR) &&
1869 (EltSize == 64 || EltSize == 32 || EltSize == 16 || EltSize == 8)) {
1870 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
1871 unsigned VecLimit =
1872 ST->hasMVEIntegerOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1873 InstructionCost VecCost = 0;
1874 while (isPowerOf2_32(Value: NumElts) && NumElts * EltSize > VecLimit) {
1875 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts: NumElts / 2);
1876 VecCost += getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1877 NumElts /= 2;
1878 }
1879 // For i16/i8, MVE will perform a VREV + VORR/VAND/VEOR for the 64bit vector
1880 // step.
1881 if (ST->hasMVEIntegerOps() && ValVT.getScalarSizeInBits() <= 16 &&
1882 NumElts * EltSize == 64) {
1883 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts);
1884 VecCost += ST->getMVEVectorCostFactor(CostKind) +
1885 getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1886 NumElts /= 2;
1887 }
1888
1889 // From here we extract the elements and perform the and/or/xor.
1890 InstructionCost ExtractCost = NumElts;
1891 return VecCost + ExtractCost +
1892 (NumElts - 1) * getArithmeticInstrCost(
1893 Opcode, Ty: ValTy->getElementType(), CostKind);
1894 }
1895
1896 if (!ST->hasMVEIntegerOps() || !ValVT.isSimple() || ISD != ISD::ADD ||
1897 TTI::requiresOrderedReduction(FMF))
1898 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
1899
1900 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1901
1902 static const CostTblEntry CostTblAdd[]{
1903 {.ISD: ISD::ADD, .Type: MVT::v16i8, .Cost: 1},
1904 {.ISD: ISD::ADD, .Type: MVT::v8i16, .Cost: 1},
1905 {.ISD: ISD::ADD, .Type: MVT::v4i32, .Cost: 1},
1906 };
1907 if (const auto *Entry = CostTableLookup(Table: CostTblAdd, ISD, Ty: LT.second))
1908 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first;
1909
1910 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
1911}
1912
1913InstructionCost ARMTTIImpl::getExtendedReductionCost(
1914 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
1915 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
1916 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1917 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
1918
1919 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1920
1921 switch (ISD) {
1922 case ISD::ADD:
1923 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) {
1924 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1925
1926 // The legal cases are:
1927 // VADDV u/s 8/16/32
1928 // VADDLV u/s 32
1929 // Codegen currently cannot always handle larger than legal vectors very
1930 // well, especially for predicated reductions where the mask needs to be
1931 // split, so restrict to 128bit or smaller input types.
1932 unsigned RevVTSize = ResVT.getSizeInBits();
1933 if (ValVT.getSizeInBits() <= 128 &&
1934 ((LT.second == MVT::v16i8 && RevVTSize <= 32) ||
1935 (LT.second == MVT::v8i16 && RevVTSize <= 32) ||
1936 (LT.second == MVT::v4i32 && RevVTSize <= 64)))
1937 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1938 }
1939 break;
1940 default:
1941 break;
1942 }
1943 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy, FMF,
1944 CostKind);
1945}
1946
1947InstructionCost
1948ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
1949 Type *ResTy, VectorType *ValTy,
1950 TTI::TargetCostKind CostKind) const {
1951 if (RedOpcode != Instruction::Add)
1952 return InstructionCost::getInvalid(Val: CostKind);
1953 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1954 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
1955
1956 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) {
1957 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1958
1959 // The legal cases are:
1960 // VMLAV u/s 8/16/32
1961 // VMLALV u/s 16/32
1962 // Codegen currently cannot always handle larger than legal vectors very
1963 // well, especially for predicated reductions where the mask needs to be
1964 // split, so restrict to 128bit or smaller input types.
1965 unsigned RevVTSize = ResVT.getSizeInBits();
1966 if (ValVT.getSizeInBits() <= 128 &&
1967 ((LT.second == MVT::v16i8 && RevVTSize <= 32) ||
1968 (LT.second == MVT::v8i16 && RevVTSize <= 64) ||
1969 (LT.second == MVT::v4i32 && RevVTSize <= 64)))
1970 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1971 }
1972
1973 return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty: ValTy,
1974 CostKind);
1975}
1976
1977InstructionCost
1978ARMTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
1979 FastMathFlags FMF,
1980 TTI::TargetCostKind CostKind) const {
1981 EVT ValVT = TLI->getValueType(DL, Ty);
1982
1983 // In general floating point reductions are a series of elementwise
1984 // operations, with free extracts on each step. These are either in-order or
1985 // treewise depending on whether that is allowed by the fast math flags.
1986 if ((IID == Intrinsic::minnum || IID == Intrinsic::maxnum) &&
1987 ((ValVT.getVectorElementType() == MVT::f32 && ST->hasVFP2Base()) ||
1988 (ValVT.getVectorElementType() == MVT::f64 && ST->hasFP64()) ||
1989 (ValVT.getVectorElementType() == MVT::f16 && ST->hasFullFP16()))) {
1990 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
1991 unsigned EltSize = ValVT.getScalarSizeInBits();
1992 unsigned VecLimit = ST->hasMVEFloatOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1993 InstructionCost VecCost;
1994 while (isPowerOf2_32(Value: NumElts) && NumElts * EltSize > VecLimit) {
1995 Type *VecTy = FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElts/2);
1996 IntrinsicCostAttributes ICA(IID, VecTy, {VecTy, VecTy}, FMF);
1997 VecCost += getIntrinsicInstrCost(ICA, CostKind);
1998 NumElts /= 2;
1999 }
2000
2001 // For fp16 we need to extract the upper lane elements. MVE can add a
2002 // VREV+FMIN/MAX to perform another vector step instead.
2003 InstructionCost ExtractCost = 0;
2004 if (ST->hasMVEFloatOps() && ValVT.getVectorElementType() == MVT::f16 &&
2005 NumElts == 8) {
2006 VecCost += ST->getMVEVectorCostFactor(CostKind) * 2;
2007 NumElts /= 2;
2008 } else if (ValVT.getVectorElementType() == MVT::f16)
2009 ExtractCost = cast<FixedVectorType>(Val: Ty)->getNumElements() / 2;
2010
2011 IntrinsicCostAttributes ICA(IID, Ty->getElementType(),
2012 {Ty->getElementType(), Ty->getElementType()},
2013 FMF);
2014 return VecCost + ExtractCost +
2015 (NumElts - 1) * getIntrinsicInstrCost(ICA, CostKind);
2016 }
2017
2018 if (IID == Intrinsic::smin || IID == Intrinsic::smax ||
2019 IID == Intrinsic::umin || IID == Intrinsic::umax) {
2020 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2021
2022 // All costs are the same for u/s min/max. These lower to vminv, which are
2023 // given a slightly higher cost as they tend to take multiple cycles for
2024 // smaller type sizes.
2025 static const CostTblEntry CostTblAdd[]{
2026 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: 4},
2027 {.ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: 3},
2028 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: 2},
2029 };
2030 if (const auto *Entry = CostTableLookup(Table: CostTblAdd, ISD: ISD::SMIN, Ty: LT.second))
2031 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first;
2032 }
2033
2034 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2035}
2036
2037InstructionCost
2038ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
2039 TTI::TargetCostKind CostKind) const {
2040 unsigned Opc = ICA.getID();
2041 switch (Opc) {
2042 case Intrinsic::get_active_lane_mask:
2043 // Currently we make a somewhat optimistic assumption that
2044 // active_lane_mask's are always free. In reality it may be freely folded
2045 // into a tail predicated loop, expanded into a VCPT or expanded into a lot
2046 // of add/icmp code. We may need to improve this in the future, but being
2047 // able to detect if it is free or not involves looking at a lot of other
2048 // code. We currently assume that the vectorizer inserted these, and knew
2049 // what it was doing in adding one.
2050 if (ST->hasMVEIntegerOps())
2051 return 0;
2052 break;
2053 case Intrinsic::sadd_sat:
2054 case Intrinsic::ssub_sat:
2055 case Intrinsic::uadd_sat:
2056 case Intrinsic::usub_sat: {
2057 bool IsAdd = (Opc == Intrinsic::sadd_sat || Opc == Intrinsic::ssub_sat);
2058 bool IsSigned = (Opc == Intrinsic::sadd_sat || Opc == Intrinsic::ssub_sat);
2059 Type *RetTy = ICA.getReturnType();
2060
2061 if (auto *ITy = dyn_cast<IntegerType>(Val: RetTy)) {
2062 if (IsSigned && ST->hasDSP() && ITy->getBitWidth() == 32)
2063 return 1; // qadd / qsub
2064 if (ST->hasDSP() && (ITy->getBitWidth() == 8 || ITy->getBitWidth() == 16))
2065 return 2; // uqadd16 / qadd16 / uqsub16 / qsub16 + possible extend.
2066 // Otherwise return the cost of expanding the node. Generally an add +
2067 // icmp + sel.
2068 CmpInst::Predicate Pred = CmpInst::ICMP_SGT;
2069 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2070 return getArithmeticInstrCost(Opcode: IsAdd ? Instruction::Add : Instruction::Sub,
2071 Ty: RetTy, CostKind) +
2072 2 * getCmpSelInstrCost(Opcode: BinaryOperator::ICmp, ValTy: RetTy, CondTy, VecPred: Pred,
2073 CostKind) +
2074 2 * getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy, VecPred: Pred,
2075 CostKind);
2076 }
2077
2078 if (!ST->hasMVEIntegerOps())
2079 break;
2080
2081 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
2082 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 ||
2083 LT.second == MVT::v16i8) {
2084 // This is a base cost of 1 for the vqadd, plus 3 extract shifts if we
2085 // need to extend the type, as it uses shr(qadd(shl, shl)).
2086 unsigned Instrs =
2087 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1
2088 : 4;
2089 return LT.first * ST->getMVEVectorCostFactor(CostKind) * Instrs;
2090 }
2091 break;
2092 }
2093 case Intrinsic::abs:
2094 case Intrinsic::smin:
2095 case Intrinsic::smax:
2096 case Intrinsic::umin:
2097 case Intrinsic::umax: {
2098 if (!ST->hasMVEIntegerOps())
2099 break;
2100 Type *VT = ICA.getReturnType();
2101
2102 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VT);
2103 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 ||
2104 LT.second == MVT::v16i8)
2105 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2106 break;
2107 }
2108 case Intrinsic::minnum:
2109 case Intrinsic::maxnum: {
2110 if (!ST->hasMVEFloatOps())
2111 break;
2112 Type *VT = ICA.getReturnType();
2113 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VT);
2114 if (LT.second == MVT::v4f32 || LT.second == MVT::v8f16)
2115 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2116 break;
2117 }
2118 case Intrinsic::fptosi_sat:
2119 case Intrinsic::fptoui_sat: {
2120 if (ICA.getArgTypes().empty())
2121 break;
2122 bool IsSigned = Opc == Intrinsic::fptosi_sat;
2123 auto LT = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
2124 EVT MTy = TLI->getValueType(DL, Ty: ICA.getReturnType());
2125 // Check for the legal types, with the correct subtarget features.
2126 if ((ST->hasVFP2Base() && LT.second == MVT::f32 && MTy == MVT::i32) ||
2127 (ST->hasFP64() && LT.second == MVT::f64 && MTy == MVT::i32) ||
2128 (ST->hasFullFP16() && LT.second == MVT::f16 && MTy == MVT::i32))
2129 return LT.first;
2130
2131 // Equally for MVE vector types
2132 if (ST->hasMVEFloatOps() &&
2133 (LT.second == MVT::v4f32 || LT.second == MVT::v8f16) &&
2134 LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits())
2135 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2136
2137 // If we can we use a legal convert followed by a min+max
2138 if (((ST->hasVFP2Base() && LT.second == MVT::f32) ||
2139 (ST->hasFP64() && LT.second == MVT::f64) ||
2140 (ST->hasFullFP16() && LT.second == MVT::f16) ||
2141 (ST->hasMVEFloatOps() &&
2142 (LT.second == MVT::v4f32 || LT.second == MVT::v8f16))) &&
2143 LT.second.getScalarSizeInBits() >= MTy.getScalarSizeInBits()) {
2144 Type *LegalTy = Type::getIntNTy(C&: ICA.getReturnType()->getContext(),
2145 N: LT.second.getScalarSizeInBits());
2146 InstructionCost Cost =
2147 LT.second.isVector() ? ST->getMVEVectorCostFactor(CostKind) : 1;
2148 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin
2149 : Intrinsic::umin,
2150 LegalTy, {LegalTy, LegalTy});
2151 Cost += getIntrinsicInstrCost(ICA: Attrs1, CostKind);
2152 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax
2153 : Intrinsic::umax,
2154 LegalTy, {LegalTy, LegalTy});
2155 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
2156 return LT.first * Cost;
2157 }
2158 // Otherwise we need to follow the default expansion that clamps the value
2159 // using a float min/max with a fcmp+sel for nan handling when signed.
2160 Type *FPTy = ICA.getArgTypes()[0];
2161 Type *RetTy = ICA.getReturnType();
2162 IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FPTy, {FPTy, FPTy});
2163 InstructionCost Cost = getIntrinsicInstrCost(ICA: Attrs1, CostKind);
2164 IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FPTy, {FPTy, FPTy});
2165 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
2166 Cost +=
2167 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
2168 Dst: RetTy, Src: FPTy, CCH: TTI::CastContextHint::None, CostKind);
2169 if (IsSigned) {
2170 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2171 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: FPTy, CondTy,
2172 VecPred: CmpInst::FCMP_UNO, CostKind);
2173 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
2174 VecPred: CmpInst::FCMP_UNO, CostKind);
2175 }
2176 return Cost;
2177 }
2178 }
2179
2180 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
2181}
2182
2183bool ARMTTIImpl::isLoweredToCall(const Function *F) const {
2184 if (!F->isIntrinsic())
2185 return BaseT::isLoweredToCall(F);
2186
2187 // Assume all Arm-specific intrinsics map to an instruction.
2188 if (F->getName().starts_with(Prefix: "llvm.arm"))
2189 return false;
2190
2191 switch (F->getIntrinsicID()) {
2192 default: break;
2193 case Intrinsic::powi:
2194 case Intrinsic::sin:
2195 case Intrinsic::cos:
2196 case Intrinsic::sincos:
2197 case Intrinsic::pow:
2198 case Intrinsic::log:
2199 case Intrinsic::log10:
2200 case Intrinsic::log2:
2201 case Intrinsic::exp:
2202 case Intrinsic::exp2:
2203 return true;
2204 case Intrinsic::sqrt:
2205 case Intrinsic::fabs:
2206 case Intrinsic::copysign:
2207 case Intrinsic::floor:
2208 case Intrinsic::ceil:
2209 case Intrinsic::trunc:
2210 case Intrinsic::rint:
2211 case Intrinsic::nearbyint:
2212 case Intrinsic::round:
2213 case Intrinsic::canonicalize:
2214 case Intrinsic::lround:
2215 case Intrinsic::llround:
2216 case Intrinsic::lrint:
2217 case Intrinsic::llrint:
2218 if (F->getReturnType()->isDoubleTy() && !ST->hasFP64())
2219 return true;
2220 if (F->getReturnType()->isHalfTy() && !ST->hasFullFP16())
2221 return true;
2222 // Some operations can be handled by vector instructions and assume
2223 // unsupported vectors will be expanded into supported scalar ones.
2224 // TODO Handle scalar operations properly.
2225 return !ST->hasFPARMv8Base() && !ST->hasVFP2Base();
2226 case Intrinsic::masked_store:
2227 case Intrinsic::masked_load:
2228 case Intrinsic::masked_gather:
2229 case Intrinsic::masked_scatter:
2230 return !ST->hasMVEIntegerOps();
2231 case Intrinsic::sadd_with_overflow:
2232 case Intrinsic::uadd_with_overflow:
2233 case Intrinsic::ssub_with_overflow:
2234 case Intrinsic::usub_with_overflow:
2235 case Intrinsic::sadd_sat:
2236 case Intrinsic::uadd_sat:
2237 case Intrinsic::ssub_sat:
2238 case Intrinsic::usub_sat:
2239 return false;
2240 }
2241
2242 return BaseT::isLoweredToCall(F);
2243}
2244
2245bool ARMTTIImpl::maybeLoweredToCall(Instruction &I) const {
2246 unsigned ISD = TLI->InstructionOpcodeToISD(Opcode: I.getOpcode());
2247 EVT VT = TLI->getValueType(DL, Ty: I.getType(), AllowUnknown: true);
2248 if (TLI->getOperationAction(Op: ISD, VT) == TargetLowering::LibCall)
2249 return true;
2250
2251 // Check if an intrinsic will be lowered to a call and assume that any
2252 // other CallInst will generate a bl.
2253 if (auto *Call = dyn_cast<CallInst>(Val: &I)) {
2254 if (auto *II = dyn_cast<IntrinsicInst>(Val: Call)) {
2255 switch(II->getIntrinsicID()) {
2256 case Intrinsic::memcpy:
2257 case Intrinsic::memset:
2258 case Intrinsic::memmove:
2259 return getNumMemOps(I: II) == -1;
2260 default:
2261 if (const Function *F = Call->getCalledFunction())
2262 return isLoweredToCall(F);
2263 }
2264 }
2265 return true;
2266 }
2267
2268 // FPv5 provides conversions between integer, double-precision,
2269 // single-precision, and half-precision formats.
2270 switch (I.getOpcode()) {
2271 default:
2272 break;
2273 case Instruction::FPToSI:
2274 case Instruction::FPToUI:
2275 case Instruction::SIToFP:
2276 case Instruction::UIToFP:
2277 case Instruction::FPTrunc:
2278 case Instruction::FPExt:
2279 return !ST->hasFPARMv8Base();
2280 }
2281
2282 // FIXME: Unfortunately the approach of checking the Operation Action does
2283 // not catch all cases of Legalization that use library calls. Our
2284 // Legalization step categorizes some transformations into library calls as
2285 // Custom, Expand or even Legal when doing type legalization. So for now
2286 // we have to special case for instance the SDIV of 64bit integers and the
2287 // use of floating point emulation.
2288 if (VT.isInteger() && VT.getSizeInBits() >= 64) {
2289 switch (ISD) {
2290 default:
2291 break;
2292 case ISD::SDIV:
2293 case ISD::UDIV:
2294 case ISD::SREM:
2295 case ISD::UREM:
2296 case ISD::SDIVREM:
2297 case ISD::UDIVREM:
2298 return true;
2299 }
2300 }
2301
2302 // Assume all other non-float operations are supported.
2303 if (!VT.isFloatingPoint())
2304 return false;
2305
2306 // We'll need a library call to handle most floats when using soft.
2307 if (TLI->useSoftFloat()) {
2308 switch (I.getOpcode()) {
2309 default:
2310 return true;
2311 case Instruction::Alloca:
2312 case Instruction::Load:
2313 case Instruction::Store:
2314 case Instruction::Select:
2315 case Instruction::PHI:
2316 return false;
2317 }
2318 }
2319
2320 // We'll need a libcall to perform double precision operations on a single
2321 // precision only FPU.
2322 if (I.getType()->isDoubleTy() && !ST->hasFP64())
2323 return true;
2324
2325 // Likewise for half precision arithmetic.
2326 if (I.getType()->isHalfTy() && !ST->hasFullFP16())
2327 return true;
2328
2329 return false;
2330}
2331
2332bool ARMTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
2333 AssumptionCache &AC,
2334 TargetLibraryInfo *LibInfo,
2335 HardwareLoopInfo &HWLoopInfo) const {
2336 // Low-overhead branches are only supported in the 'low-overhead branch'
2337 // extension of v8.1-m.
2338 if (!ST->hasLOB() || DisableLowOverheadLoops) {
2339 LLVM_DEBUG(dbgs() << "ARMHWLoops: Disabled\n");
2340 return false;
2341 }
2342
2343 if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
2344 LLVM_DEBUG(dbgs() << "ARMHWLoops: No BETC\n");
2345 return false;
2346 }
2347
2348 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2349 if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount)) {
2350 LLVM_DEBUG(dbgs() << "ARMHWLoops: Uncomputable BETC\n");
2351 return false;
2352 }
2353
2354 const SCEV *TripCountSCEV =
2355 SE.getAddExpr(LHS: BackedgeTakenCount,
2356 RHS: SE.getOne(Ty: BackedgeTakenCount->getType()));
2357
2358 // We need to store the trip count in LR, a 32-bit register.
2359 if (SE.getUnsignedRangeMax(S: TripCountSCEV).getBitWidth() > 32) {
2360 LLVM_DEBUG(dbgs() << "ARMHWLoops: Trip count does not fit into 32bits\n");
2361 return false;
2362 }
2363
2364 // Making a call will trash LR and clear LO_BRANCH_INFO, so there's little
2365 // point in generating a hardware loop if that's going to happen.
2366
2367 auto IsHardwareLoopIntrinsic = [](Instruction &I) {
2368 if (auto *Call = dyn_cast<IntrinsicInst>(Val: &I)) {
2369 switch (Call->getIntrinsicID()) {
2370 default:
2371 break;
2372 case Intrinsic::start_loop_iterations:
2373 case Intrinsic::test_start_loop_iterations:
2374 case Intrinsic::loop_decrement:
2375 case Intrinsic::loop_decrement_reg:
2376 return true;
2377 }
2378 }
2379 return false;
2380 };
2381
2382 // Scan the instructions to see if there's any that we know will turn into a
2383 // call or if this loop is already a low-overhead loop or will become a tail
2384 // predicated loop.
2385 bool IsTailPredLoop = false;
2386 auto ScanLoop = [&](Loop *L) {
2387 for (auto *BB : L->getBlocks()) {
2388 for (auto &I : *BB) {
2389 if (maybeLoweredToCall(I) || IsHardwareLoopIntrinsic(I) ||
2390 isa<InlineAsm>(Val: I)) {
2391 LLVM_DEBUG(dbgs() << "ARMHWLoops: Bad instruction: " << I << "\n");
2392 return false;
2393 }
2394 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
2395 IsTailPredLoop |=
2396 II->getIntrinsicID() == Intrinsic::get_active_lane_mask ||
2397 II->getIntrinsicID() == Intrinsic::arm_mve_vctp8 ||
2398 II->getIntrinsicID() == Intrinsic::arm_mve_vctp16 ||
2399 II->getIntrinsicID() == Intrinsic::arm_mve_vctp32 ||
2400 II->getIntrinsicID() == Intrinsic::arm_mve_vctp64;
2401 }
2402 }
2403 return true;
2404 };
2405
2406 // Visit inner loops.
2407 for (auto *Inner : *L)
2408 if (!ScanLoop(Inner))
2409 return false;
2410
2411 if (!ScanLoop(L))
2412 return false;
2413
2414 // TODO: Check whether the trip count calculation is expensive. If L is the
2415 // inner loop but we know it has a low trip count, calculating that trip
2416 // count (in the parent loop) may be detrimental.
2417
2418 LLVMContext &C = L->getHeader()->getContext();
2419 HWLoopInfo.CounterInReg = true;
2420 HWLoopInfo.IsNestingLegal = false;
2421 HWLoopInfo.PerformEntryTest = AllowWLSLoops && !IsTailPredLoop;
2422 HWLoopInfo.CountType = Type::getInt32Ty(C);
2423 HWLoopInfo.LoopDecrement = ConstantInt::get(Ty: HWLoopInfo.CountType, V: 1);
2424 return true;
2425}
2426
2427static bool canTailPredicateInstruction(Instruction &I, int &ICmpCount) {
2428 // We don't allow icmp's, and because we only look at single block loops,
2429 // we simply count the icmps, i.e. there should only be 1 for the backedge.
2430 if (isa<ICmpInst>(Val: &I) && ++ICmpCount > 1)
2431 return false;
2432 // FIXME: This is a workaround for poor cost modelling. Min/Max intrinsics are
2433 // not currently canonical, but soon will be. Code without them uses icmp, and
2434 // so is not tail predicated as per the condition above. In order to get the
2435 // same performance we treat min and max the same as an icmp for tailpred
2436 // purposes for the moment (we often rely on non-tailpred and higher VF's to
2437 // pick more optimal instructions like VQDMULH. They need to be recognized
2438 // directly by the vectorizer).
2439 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
2440 if ((II->getIntrinsicID() == Intrinsic::smin ||
2441 II->getIntrinsicID() == Intrinsic::smax ||
2442 II->getIntrinsicID() == Intrinsic::umin ||
2443 II->getIntrinsicID() == Intrinsic::umax) &&
2444 ++ICmpCount > 1)
2445 return false;
2446
2447 if (isa<FCmpInst>(Val: &I))
2448 return false;
2449
2450 // We could allow extending/narrowing FP loads/stores, but codegen is
2451 // too inefficient so reject this for now.
2452 if (isa<FPExtInst>(Val: &I) || isa<FPTruncInst>(Val: &I))
2453 return false;
2454
2455 // Extends have to be extending-loads
2456 if (isa<SExtInst>(Val: &I) || isa<ZExtInst>(Val: &I) )
2457 if (!I.getOperand(i: 0)->hasOneUse() || !isa<LoadInst>(Val: I.getOperand(i: 0)))
2458 return false;
2459
2460 // Truncs have to be narrowing-stores
2461 if (isa<TruncInst>(Val: &I) )
2462 if (!I.hasOneUse() || !isa<StoreInst>(Val: *I.user_begin()))
2463 return false;
2464
2465 return true;
2466}
2467
2468// To set up a tail-predicated loop, we need to know the total number of
2469// elements processed by that loop. Thus, we need to determine the element
2470// size and:
2471// 1) it should be uniform for all operations in the vector loop, so we
2472// e.g. don't want any widening/narrowing operations.
2473// 2) it should be smaller than i64s because we don't have vector operations
2474// that work on i64s.
2475// 3) we don't want elements to be reversed or shuffled, to make sure the
2476// tail-predication masks/predicates the right lanes.
2477//
2478static bool canTailPredicateLoop(Loop *L, LoopInfo *LI, ScalarEvolution &SE,
2479 const DataLayout &DL,
2480 const LoopAccessInfo *LAI,
2481 const DominatorTree &DT) {
2482 LLVM_DEBUG(dbgs() << "Tail-predication: checking allowed instructions\n");
2483
2484 // If there are live-out values, it is probably a reduction. We can predicate
2485 // most reduction operations freely under MVE using a combination of
2486 // prefer-predicated-reduction-select and inloop reductions. We limit this to
2487 // floating point and integer reductions, but don't check for operators
2488 // specifically here. If the value ends up not being a reduction (and so the
2489 // vectorizer cannot tailfold the loop), we should fall back to standard
2490 // vectorization automatically.
2491 SmallVector< Instruction *, 8 > LiveOuts;
2492 LiveOuts = llvm::findDefsUsedOutsideOfLoop(L);
2493 bool ReductionsDisabled =
2494 EnableTailPredication == TailPredication::EnabledNoReductions ||
2495 EnableTailPredication == TailPredication::ForceEnabledNoReductions;
2496
2497 for (auto *I : LiveOuts) {
2498 if (!I->getType()->isIntegerTy() && !I->getType()->isFloatTy() &&
2499 !I->getType()->isHalfTy()) {
2500 LLVM_DEBUG(dbgs() << "Don't tail-predicate loop with non-integer/float "
2501 "live-out value\n");
2502 return false;
2503 }
2504 if (ReductionsDisabled) {
2505 LLVM_DEBUG(dbgs() << "Reductions not enabled\n");
2506 return false;
2507 }
2508 }
2509
2510 // Next, check that all instructions can be tail-predicated.
2511 PredicatedScalarEvolution PSE = LAI->getPSE();
2512 int ICmpCount = 0;
2513
2514 for (BasicBlock *BB : L->blocks()) {
2515 for (Instruction &I : *BB) {
2516 if (isa<PHINode>(Val: I) || isa<PseudoProbeInst>(Val: I))
2517 continue;
2518 if (!canTailPredicateInstruction(I, ICmpCount)) {
2519 LLVM_DEBUG(dbgs() << "Instruction not allowed: "; I.dump());
2520 return false;
2521 }
2522
2523 Type *T = I.getType();
2524 if (T->getScalarSizeInBits() > 32) {
2525 LLVM_DEBUG(dbgs() << "Unsupported Type: "; T->dump());
2526 return false;
2527 }
2528 if (isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I)) {
2529 Value *Ptr = getLoadStorePointerOperand(V: &I);
2530 Type *AccessTy = getLoadStoreType(I: &I);
2531 int64_t NextStride =
2532 getPtrStride(PSE, AccessTy, Ptr, Lp: L, DT).value_or(u: 0);
2533 if (NextStride == 1) {
2534 // TODO: for now only allow consecutive strides of 1. We could support
2535 // other strides as long as it is uniform, but let's keep it simple
2536 // for now.
2537 continue;
2538 } else if (NextStride == -1 ||
2539 (NextStride == 2 && MVEMaxSupportedInterleaveFactor >= 2) ||
2540 (NextStride == 4 && MVEMaxSupportedInterleaveFactor >= 4)) {
2541 LLVM_DEBUG(dbgs()
2542 << "Consecutive strides of 2 found, vld2/vstr2 can't "
2543 "be tail-predicated\n.");
2544 return false;
2545 // TODO: don't tail predicate if there is a reversed load?
2546 } else if (EnableMaskedGatherScatters) {
2547 // Gather/scatters do allow loading from arbitrary strides, at
2548 // least if they are loop invariant.
2549 // TODO: Loop variant strides should in theory work, too, but
2550 // this requires further testing.
2551 const SCEV *PtrScev = PSE.getSE()->getSCEV(V: Ptr);
2552 if (auto AR = dyn_cast<SCEVAddRecExpr>(Val: PtrScev)) {
2553 const SCEV *Step = AR->getStepRecurrence(SE&: *PSE.getSE());
2554 if (PSE.getSE()->isLoopInvariant(S: Step, L))
2555 continue;
2556 }
2557 }
2558 LLVM_DEBUG(dbgs() << "Bad stride found, can't "
2559 "tail-predicate\n.");
2560 return false;
2561 }
2562 }
2563 }
2564
2565 LLVM_DEBUG(dbgs() << "tail-predication: all instructions allowed!\n");
2566 return true;
2567}
2568
2569bool ARMTTIImpl::preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
2570 if (!EnableTailPredication) {
2571 LLVM_DEBUG(dbgs() << "Tail-folding not enabled.\n");
2572 return false;
2573 }
2574
2575 // Creating a tail-folded vector loop is the first step for generating a
2576 // tail-folded hardware loop, for which we need the MVE masked
2577 // load/stores instructions:
2578 if (!ST->hasMVEIntegerOps())
2579 return false;
2580
2581 LoopVectorizationLegality *LVL = TFI->LVL;
2582 Loop *L = LVL->getLoop();
2583
2584 // For now, restrict this to single block loops.
2585 if (L->getNumBlocks() > 1) {
2586 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: not a single block "
2587 "loop.\n");
2588 return false;
2589 }
2590
2591 assert(L->isInnermost() &&
2592 "preferTailFoldingOverEpilogue: inner-loop expected");
2593
2594 LoopInfo *LI = LVL->getLoopInfo();
2595 HardwareLoopInfo HWLoopInfo(L);
2596 if (!HWLoopInfo.canAnalyze(LI&: *LI)) {
2597 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2598 "analyzable.\n");
2599 return false;
2600 }
2601
2602 AssumptionCache *AC = LVL->getAssumptionCache();
2603 ScalarEvolution *SE = LVL->getScalarEvolution();
2604
2605 // This checks if we have the low-overhead branch architecture
2606 // extension, and if we will create a hardware-loop:
2607 if (!isHardwareLoopProfitable(L, SE&: *SE, AC&: *AC, LibInfo: TFI->TLI, HWLoopInfo)) {
2608 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2609 "profitable.\n");
2610 return false;
2611 }
2612
2613 DominatorTree *DT = LVL->getDominatorTree();
2614 if (!HWLoopInfo.isHardwareLoopCandidate(SE&: *SE, LI&: *LI, DT&: *DT)) {
2615 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2616 "a candidate.\n");
2617 return false;
2618 }
2619
2620 return canTailPredicateLoop(L, LI, SE&: *SE, DL, LAI: LVL->getLAI(),
2621 DT: *LVL->getDominatorTree());
2622}
2623
2624TailFoldingStyle ARMTTIImpl::getPreferredTailFoldingStyle() const {
2625 if (!ST->hasMVEIntegerOps() || !EnableTailPredication)
2626 return TailFoldingStyle::DataWithoutLaneMask;
2627
2628 // Intrinsic @llvm.get.active.lane.mask is supported.
2629 // It is used in the MVETailPredication pass, which requires the number of
2630 // elements processed by this vector loop to setup the tail-predicated
2631 // loop.
2632 return TailFoldingStyle::Data;
2633}
2634void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
2635 TTI::UnrollingPreferences &UP,
2636 OptimizationRemarkEmitter *ORE) const {
2637 // Enable Upper bound unrolling universally, providing that we do not see an
2638 // active lane mask, which will be better kept as a loop to become tail
2639 // predicated than to be conditionally unrolled.
2640 UP.UpperBound =
2641 !ST->hasMVEIntegerOps() || !any_of(Range&: *L->getHeader(), P: [](Instruction &I) {
2642 return isa<IntrinsicInst>(Val: I) &&
2643 cast<IntrinsicInst>(Val&: I).getIntrinsicID() ==
2644 Intrinsic::get_active_lane_mask;
2645 });
2646
2647 // Only currently enable these preferences for M-Class cores.
2648 if (!ST->isMClass())
2649 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
2650
2651 // Disable loop unrolling for Oz and Os.
2652 UP.OptSizeThreshold = 0;
2653 UP.PartialOptSizeThreshold = 0;
2654 if (L->getHeader()->getParent()->hasOptSize())
2655 return;
2656
2657 SmallVector<BasicBlock*, 4> ExitingBlocks;
2658 L->getExitingBlocks(ExitingBlocks);
2659 LLVM_DEBUG(dbgs() << "Loop has:\n"
2660 << "Blocks: " << L->getNumBlocks() << "\n"
2661 << "Exit blocks: " << ExitingBlocks.size() << "\n");
2662
2663 // Only allow another exit other than the latch. This acts as an early exit
2664 // as it mirrors the profitability calculation of the runtime unroller.
2665 if (ExitingBlocks.size() > 2)
2666 return;
2667
2668 // Limit the CFG of the loop body for targets with a branch predictor.
2669 // Allowing 4 blocks permits if-then-else diamonds in the body.
2670 if (ST->hasBranchPredictor() && L->getNumBlocks() > 4)
2671 return;
2672
2673 // Don't unroll vectorized loops, including the remainder loop
2674 if (getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized"))
2675 return;
2676
2677 // Scan the loop: don't unroll loops with calls as this could prevent
2678 // inlining.
2679 InstructionCost Cost = 0;
2680 for (auto *BB : L->getBlocks()) {
2681 for (auto &I : *BB) {
2682 // Don't unroll vectorised loop. MVE does not benefit from it as much as
2683 // scalar code.
2684 if (I.getType()->isVectorTy())
2685 return;
2686
2687 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
2688 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction()) {
2689 if (!isLoweredToCall(F))
2690 continue;
2691 }
2692 return;
2693 }
2694
2695 SmallVector<const Value*, 4> Operands(I.operand_values());
2696 Cost += getInstructionCost(U: &I, Operands,
2697 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
2698 }
2699 }
2700
2701 // On v6m cores, there are very few registers available. We can easily end up
2702 // spilling and reloading more registers in an unrolled loop. Look at the
2703 // number of LCSSA phis as a rough measure of how many registers will need to
2704 // be live out of the loop, reducing the default unroll count if more than 1
2705 // value is needed. In the long run, all of this should be being learnt by a
2706 // machine.
2707 unsigned UnrollCount = 4;
2708 if (ST->isThumb1Only()) {
2709 unsigned ExitingValues = 0;
2710 SmallVector<BasicBlock *, 4> ExitBlocks;
2711 L->getExitBlocks(ExitBlocks);
2712 for (auto *Exit : ExitBlocks) {
2713 // Count the number of LCSSA phis. Exclude values coming from GEP's as
2714 // only the last is expected to be needed for address operands.
2715 unsigned LiveOuts = count_if(Range: Exit->phis(), P: [](auto &PH) {
2716 return PH.getNumOperands() != 1 ||
2717 !isa<GetElementPtrInst>(PH.getOperand(0));
2718 });
2719 ExitingValues = ExitingValues < LiveOuts ? LiveOuts : ExitingValues;
2720 }
2721 if (ExitingValues)
2722 UnrollCount /= ExitingValues;
2723 if (UnrollCount <= 1)
2724 return;
2725 }
2726
2727 // For processors with low overhead branching (LOB), runtime unrolling the
2728 // innermost loop is often detrimental to performance. In these cases the loop
2729 // remainder gets unrolled into a series of compare-and-jump blocks, which in
2730 // deeply nested loops get executed multiple times, negating the benefits of
2731 // LOB. This is particularly noticeable when the loop trip count of the
2732 // innermost loop varies within the outer loop, such as in the case of
2733 // triangular matrix decompositions. In these cases we will prefer to not
2734 // unroll the innermost loop, with the intention for it to be executed as a
2735 // low overhead loop.
2736 bool Runtime = true;
2737 if (ST->hasLOB()) {
2738 if (SE.hasLoopInvariantBackedgeTakenCount(L)) {
2739 const SCEV *BETC = SE.getBackedgeTakenCount(L);
2740 auto *Outer = L->getOutermostLoop();
2741 if ((L != Outer && Outer != L->getParentLoop()) ||
2742 (L != Outer && BETC && !SE.isLoopInvariant(S: BETC, L: Outer))) {
2743 Runtime = false;
2744 }
2745 }
2746 }
2747
2748 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
2749 LLVM_DEBUG(dbgs() << "Default Runtime Unroll Count: " << UnrollCount << "\n");
2750
2751 UP.Partial = true;
2752 UP.Runtime = Runtime;
2753 UP.UnrollRemainder = true;
2754 UP.DefaultUnrollRuntimeCount = UnrollCount;
2755 UP.UnrollAndJam = true;
2756 UP.UnrollAndJamInnerLoopThreshold = 60;
2757
2758 // Force unrolling small loops can be very useful because of the branch
2759 // taken cost of the backedge.
2760 if (Cost < ArmForceUnrollThreshold)
2761 UP.Force = true;
2762}
2763
2764void ARMTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
2765 TTI::PeelingPreferences &PP) const {
2766 BaseT::getPeelingPreferences(L, SE, PP);
2767}
2768
2769bool ARMTTIImpl::preferInLoopReduction(RecurKind Kind, Type *Ty) const {
2770 if (!ST->hasMVEIntegerOps())
2771 return false;
2772
2773 unsigned ScalarBits = Ty->getScalarSizeInBits();
2774 switch (Kind) {
2775 case RecurKind::Add:
2776 return ScalarBits <= 64;
2777 default:
2778 return false;
2779 }
2780}
2781
2782bool ARMTTIImpl::preferPredicatedReductionSelect() const {
2783 if (!ST->hasMVEIntegerOps())
2784 return false;
2785 return true;
2786}
2787
2788InstructionCost ARMTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
2789 StackOffset BaseOffset,
2790 bool HasBaseReg, int64_t Scale,
2791 unsigned AddrSpace) const {
2792 TargetLoweringBase::AddrMode AM;
2793 AM.BaseGV = BaseGV;
2794 AM.BaseOffs = BaseOffset.getFixed();
2795 AM.HasBaseReg = HasBaseReg;
2796 AM.Scale = Scale;
2797 AM.ScalableOffset = BaseOffset.getScalable();
2798 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AS: AddrSpace)) {
2799 if (ST->hasFPAO())
2800 return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster
2801 return 0;
2802 }
2803 return InstructionCost::getInvalid();
2804}
2805
2806bool ARMTTIImpl::shouldConsiderVectorizationRegPressure() const {
2807 // MVE only has 8 vector registers, so we should consider register pressure to
2808 // avoid vectorizing when the cost of spills exceeds the gains from
2809 // vectorization.
2810 return ST->hasMVEIntegerOps();
2811}
2812
2813bool ARMTTIImpl::hasArmWideBranch(bool Thumb) const {
2814 if (Thumb) {
2815 // B.W is available in any Thumb2-supporting target, and also in every
2816 // version of Armv8-M, even Baseline which does not include the rest of
2817 // Thumb2.
2818 return ST->isThumb2() || ST->hasV8MBaselineOps();
2819 } else {
2820 // B is available in all versions of the Arm ISA, so the only question is
2821 // whether that ISA is available at all.
2822 return ST->hasARMOps();
2823 }
2824}
2825
2826/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
2827/// of the vector elements.
2828static bool areExtractExts(Value *Ext1, Value *Ext2) {
2829 using namespace PatternMatch;
2830
2831 auto areExtDoubled = [](Instruction *Ext) {
2832 return Ext->getType()->getScalarSizeInBits() ==
2833 2 * Ext->getOperand(i: 0)->getType()->getScalarSizeInBits();
2834 };
2835
2836 if (!match(V: Ext1, P: m_ZExtOrSExt(Op: m_Value())) ||
2837 !match(V: Ext2, P: m_ZExtOrSExt(Op: m_Value())) ||
2838 !areExtDoubled(cast<Instruction>(Val: Ext1)) ||
2839 !areExtDoubled(cast<Instruction>(Val: Ext2)))
2840 return false;
2841
2842 return true;
2843}
2844
2845/// Check if sinking \p I's operands to I's basic block is profitable, because
2846/// the operands can be folded into a target instruction, e.g.
2847/// sext/zext can be folded into vsubl.
2848bool ARMTTIImpl::isProfitableToSinkOperands(Instruction *I,
2849 SmallVectorImpl<Use *> &Ops) const {
2850 using namespace PatternMatch;
2851
2852 if (!I->getType()->isVectorTy())
2853 return false;
2854
2855 if (ST->hasNEON()) {
2856 switch (I->getOpcode()) {
2857 case Instruction::Sub:
2858 case Instruction::Add: {
2859 if (!areExtractExts(Ext1: I->getOperand(i: 0), Ext2: I->getOperand(i: 1)))
2860 return false;
2861 Ops.push_back(Elt: &I->getOperandUse(i: 0));
2862 Ops.push_back(Elt: &I->getOperandUse(i: 1));
2863 return true;
2864 }
2865 default:
2866 return false;
2867 }
2868 }
2869
2870 if (!ST->hasMVEIntegerOps())
2871 return false;
2872
2873 auto IsFMSMul = [&](Instruction *I) {
2874 if (!I->hasOneUse())
2875 return false;
2876 auto *Sub = cast<Instruction>(Val: *I->users().begin());
2877 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(i: 1) == I;
2878 };
2879 auto IsFMS = [&](Instruction *I) {
2880 if (match(V: I->getOperand(i: 0), P: m_FNeg(X: m_Value())) ||
2881 match(V: I->getOperand(i: 1), P: m_FNeg(X: m_Value())))
2882 return true;
2883 return false;
2884 };
2885
2886 auto IsSinker = [&](Instruction *I, int Operand) {
2887 switch (I->getOpcode()) {
2888 case Instruction::Add:
2889 case Instruction::Mul:
2890 case Instruction::FAdd:
2891 case Instruction::ICmp:
2892 case Instruction::FCmp:
2893 return true;
2894 case Instruction::FMul:
2895 return !IsFMSMul(I);
2896 case Instruction::Sub:
2897 case Instruction::FSub:
2898 case Instruction::Shl:
2899 case Instruction::LShr:
2900 case Instruction::AShr:
2901 return Operand == 1;
2902 case Instruction::Call:
2903 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2904 switch (II->getIntrinsicID()) {
2905 case Intrinsic::fma:
2906 return !IsFMS(I);
2907 case Intrinsic::sadd_sat:
2908 case Intrinsic::uadd_sat:
2909 case Intrinsic::arm_mve_add_predicated:
2910 case Intrinsic::arm_mve_mul_predicated:
2911 case Intrinsic::arm_mve_qadd_predicated:
2912 case Intrinsic::arm_mve_vhadd:
2913 case Intrinsic::arm_mve_hadd_predicated:
2914 case Intrinsic::arm_mve_vqdmull:
2915 case Intrinsic::arm_mve_vqdmull_predicated:
2916 case Intrinsic::arm_mve_vqdmulh:
2917 case Intrinsic::arm_mve_qdmulh_predicated:
2918 case Intrinsic::arm_mve_vqrdmulh:
2919 case Intrinsic::arm_mve_qrdmulh_predicated:
2920 case Intrinsic::arm_mve_fma_predicated:
2921 return true;
2922 case Intrinsic::ssub_sat:
2923 case Intrinsic::usub_sat:
2924 case Intrinsic::arm_mve_sub_predicated:
2925 case Intrinsic::arm_mve_qsub_predicated:
2926 case Intrinsic::arm_mve_hsub_predicated:
2927 case Intrinsic::arm_mve_vhsub:
2928 return Operand == 1;
2929 default:
2930 return false;
2931 }
2932 }
2933 return false;
2934 default:
2935 return false;
2936 }
2937 };
2938
2939 for (auto OpIdx : enumerate(First: I->operands())) {
2940 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
2941 // Make sure we are not already sinking this operand
2942 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
2943 continue;
2944
2945 Instruction *Shuffle = Op;
2946 if (Shuffle->getOpcode() == Instruction::BitCast)
2947 Shuffle = dyn_cast<Instruction>(Val: Shuffle->getOperand(i: 0));
2948 // We are looking for a splat that can be sunk.
2949 if (!Shuffle || !match(V: Shuffle, P: m_Shuffle(v1: m_InsertElt(Val: m_Undef(), Elt: m_Value(),
2950 Idx: m_ZeroInt()),
2951 v2: m_Undef(), mask: m_ZeroMask())))
2952 continue;
2953 if (!IsSinker(I, OpIdx.index()))
2954 continue;
2955
2956 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
2957 // and vector registers
2958 for (Use &U : Op->uses()) {
2959 Instruction *Insn = cast<Instruction>(Val: U.getUser());
2960 if (!IsSinker(Insn, U.getOperandNo()))
2961 return false;
2962 }
2963
2964 Ops.push_back(Elt: &Shuffle->getOperandUse(i: 0));
2965 if (Shuffle != Op)
2966 Ops.push_back(Elt: &Op->getOperandUse(i: 0));
2967 Ops.push_back(Elt: &OpIdx.value());
2968 }
2969 return true;
2970}
2971
2972unsigned ARMTTIImpl::getNumBytesToPadGlobalArray(unsigned Size,
2973 Type *ArrayType) const {
2974 if (!UseWidenGlobalArrays) {
2975 LLVM_DEBUG(dbgs() << "Padding global arrays disabled\n");
2976 return false;
2977 }
2978
2979 // Don't modify none integer array types
2980 if (!ArrayType || !ArrayType->isArrayTy() ||
2981 !ArrayType->getArrayElementType()->isIntegerTy())
2982 return 0;
2983
2984 // We pad to 4 byte boundaries
2985 if (Size % 4 == 0)
2986 return 0;
2987
2988 unsigned NumBytesToPad = 4 - (Size % 4);
2989 unsigned NewSize = Size + NumBytesToPad;
2990
2991 // Max number of bytes that memcpy allows for lowering to load/stores before
2992 // it uses library function (__aeabi_memcpy).
2993 unsigned MaxMemIntrinsicSize = getMaxMemIntrinsicInlineSizeThreshold();
2994
2995 if (NewSize > MaxMemIntrinsicSize)
2996 return 0;
2997
2998 return NumBytesToPad;
2999}
3000