1//===- ARMTargetTransformInfo.cpp - ARM specific TTI ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "ARMTargetTransformInfo.h"
10#include "ARMSubtarget.h"
11#include "MCTargetDesc/ARMAddressingModes.h"
12#include "llvm/ADT/APInt.h"
13#include "llvm/ADT/SmallVector.h"
14#include "llvm/Analysis/LoopInfo.h"
15#include "llvm/CodeGen/CostTable.h"
16#include "llvm/CodeGen/ISDOpcodes.h"
17#include "llvm/CodeGen/ValueTypes.h"
18#include "llvm/CodeGenTypes/MachineValueType.h"
19#include "llvm/IR/BasicBlock.h"
20#include "llvm/IR/DataLayout.h"
21#include "llvm/IR/DerivedTypes.h"
22#include "llvm/IR/Instruction.h"
23#include "llvm/IR/Instructions.h"
24#include "llvm/IR/IntrinsicInst.h"
25#include "llvm/IR/Intrinsics.h"
26#include "llvm/IR/IntrinsicsARM.h"
27#include "llvm/IR/PatternMatch.h"
28#include "llvm/IR/Type.h"
29#include "llvm/Support/Casting.h"
30#include "llvm/Support/KnownBits.h"
31#include "llvm/Target/TargetMachine.h"
32#include "llvm/TargetParser/SubtargetFeature.h"
33#include "llvm/Transforms/InstCombine/InstCombiner.h"
34#include "llvm/Transforms/Utils/Local.h"
35#include "llvm/Transforms/Utils/LoopUtils.h"
36#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
37#include <algorithm>
38#include <cassert>
39#include <cstdint>
40#include <optional>
41#include <utility>
42
43using namespace llvm;
44
45#define DEBUG_TYPE "armtti"
46
47static cl::opt<bool> EnableMaskedLoadStores(
48 "enable-arm-maskedldst", cl::Hidden, cl::init(Val: true),
49 cl::desc("Enable the generation of masked loads and stores"));
50
51static cl::opt<bool> DisableLowOverheadLoops(
52 "disable-arm-loloops", cl::Hidden, cl::init(Val: false),
53 cl::desc("Disable the generation of low-overhead loops"));
54
55static cl::opt<bool>
56 AllowWLSLoops("allow-arm-wlsloops", cl::Hidden, cl::init(Val: true),
57 cl::desc("Enable the generation of WLS loops"));
58
59extern cl::opt<TailPredication::Mode> EnableTailPredication;
60
61extern cl::opt<bool> EnableMaskedGatherScatters;
62
63extern cl::opt<unsigned> MVEMaxSupportedInterleaveFactor;
64
65static cl::opt<int> ArmForceUnrollThreshold(
66 "arm-force-unroll-threshold", cl::init(Val: 12), cl::Hidden,
67 cl::desc(
68 "Threshold for forced unrolling of small loops in Arm architecture"));
69
70/// Convert a vector load intrinsic into a simple llvm load instruction.
71/// This is beneficial when the underlying object being addressed comes
72/// from a constant, since we get constant-folding for free.
73static Value *simplifyNeonVld1(const IntrinsicInst &II, unsigned MemAlign,
74 InstCombiner::BuilderTy &Builder) {
75 auto *IntrAlign = dyn_cast<ConstantInt>(Val: II.getArgOperand(i: 1));
76
77 if (!IntrAlign)
78 return nullptr;
79
80 unsigned Alignment = IntrAlign->getLimitedValue() < MemAlign
81 ? MemAlign
82 : IntrAlign->getLimitedValue();
83
84 if (!isPowerOf2_32(Value: Alignment))
85 return nullptr;
86
87 return Builder.CreateAlignedLoad(Ty: II.getType(), Ptr: II.getArgOperand(i: 0),
88 Align: Align(Alignment));
89}
90
91TTI::AddressingModeKind
92ARMTTIImpl::getPreferredAddressingMode(const Loop *L,
93 ScalarEvolution *SE) const {
94 if (ST->hasMVEIntegerOps())
95 return TTI::AMK_PostIndexed;
96
97 if (L->getHeader()->getParent()->hasOptSize())
98 return TTI::AMK_None;
99
100 if (ST->isMClass() && ST->isThumb2() &&
101 L->getNumBlocks() == 1)
102 return TTI::AMK_PreIndexed;
103
104 return TTI::AMK_None;
105}
106
107std::optional<Instruction *>
108ARMTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
109 using namespace PatternMatch;
110 Intrinsic::ID IID = II.getIntrinsicID();
111 switch (IID) {
112 default:
113 break;
114 case Intrinsic::arm_neon_vld1: {
115 Align MemAlign =
116 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CtxI: &II,
117 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
118 if (Value *V = simplifyNeonVld1(II, MemAlign: MemAlign.value(), Builder&: IC.Builder)) {
119 return IC.replaceInstUsesWith(I&: II, V);
120 }
121 break;
122 }
123
124 case Intrinsic::arm_neon_vld2:
125 case Intrinsic::arm_neon_vld3:
126 case Intrinsic::arm_neon_vld4:
127 case Intrinsic::arm_neon_vld2lane:
128 case Intrinsic::arm_neon_vld3lane:
129 case Intrinsic::arm_neon_vld4lane:
130 case Intrinsic::arm_neon_vst1:
131 case Intrinsic::arm_neon_vst2:
132 case Intrinsic::arm_neon_vst3:
133 case Intrinsic::arm_neon_vst4:
134 case Intrinsic::arm_neon_vst2lane:
135 case Intrinsic::arm_neon_vst3lane:
136 case Intrinsic::arm_neon_vst4lane: {
137 Align MemAlign =
138 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CtxI: &II,
139 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
140 unsigned AlignArg = II.arg_size() - 1;
141 Value *AlignArgOp = II.getArgOperand(i: AlignArg);
142 MaybeAlign Align = cast<ConstantInt>(Val: AlignArgOp)->getMaybeAlignValue();
143 if (Align && *Align < MemAlign) {
144 return IC.replaceOperand(
145 I&: II, OpNum: AlignArg,
146 V: ConstantInt::get(Ty: Type::getInt32Ty(C&: II.getContext()), V: MemAlign.value(),
147 IsSigned: false));
148 }
149 break;
150 }
151
152 case Intrinsic::arm_neon_vld1x2:
153 case Intrinsic::arm_neon_vld1x3:
154 case Intrinsic::arm_neon_vld1x4:
155 case Intrinsic::arm_neon_vst1x2:
156 case Intrinsic::arm_neon_vst1x3:
157 case Intrinsic::arm_neon_vst1x4: {
158 Align NewAlign =
159 getKnownAlignment(V: II.getArgOperand(i: 0), DL: IC.getDataLayout(), CtxI: &II,
160 AC: &IC.getAssumptionCache(), DT: &IC.getDominatorTree());
161 Align OldAlign = II.getParamAlign(ArgNo: 0).valueOrOne();
162 if (NewAlign > OldAlign)
163 II.addParamAttr(ArgNo: 0,
164 Attr: Attribute::getWithAlignment(Context&: II.getContext(), Alignment: NewAlign));
165 break;
166 }
167
168 case Intrinsic::arm_mve_pred_i2v: {
169 Value *Arg = II.getArgOperand(i: 0);
170 Value *ArgArg;
171 if (match(V: Arg, P: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
172 Ops: PatternMatch::m_Value(V&: ArgArg))) &&
173 II.getType() == ArgArg->getType()) {
174 return IC.replaceInstUsesWith(I&: II, V: ArgArg);
175 }
176 Constant *XorMask;
177 if (match(V: Arg, P: m_Xor(L: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_v2i>(
178 Ops: PatternMatch::m_Value(V&: ArgArg)),
179 R: PatternMatch::m_Constant(C&: XorMask))) &&
180 II.getType() == ArgArg->getType()) {
181 if (auto *CI = dyn_cast<ConstantInt>(Val: XorMask)) {
182 if (CI->getValue().trunc(width: 16).isAllOnes()) {
183 auto TrueVector = IC.Builder.CreateVectorSplat(
184 NumElts: cast<FixedVectorType>(Val: II.getType())->getNumElements(),
185 V: IC.Builder.getTrue());
186 return BinaryOperator::Create(Op: Instruction::Xor, S1: ArgArg, S2: TrueVector);
187 }
188 }
189 }
190 KnownBits ScalarKnown(32);
191 if (IC.SimplifyDemandedBits(I: &II, OpNo: 0, DemandedMask: APInt::getLowBitsSet(numBits: 32, loBitsSet: 16),
192 Known&: ScalarKnown)) {
193 return &II;
194 }
195 break;
196 }
197 case Intrinsic::arm_mve_pred_v2i: {
198 Value *Arg = II.getArgOperand(i: 0);
199 Value *ArgArg;
200 if (match(V: Arg, P: PatternMatch::m_Intrinsic<Intrinsic::arm_mve_pred_i2v>(
201 Ops: PatternMatch::m_Value(V&: ArgArg)))) {
202 return IC.replaceInstUsesWith(I&: II, V: ArgArg);
203 }
204
205 if (II.getMetadata(KindID: LLVMContext::MD_range))
206 break;
207
208 ConstantRange Range(APInt(32, 0), APInt(32, 0x10000));
209
210 if (auto CurrentRange = II.getRange()) {
211 Range = Range.intersectWith(CR: *CurrentRange);
212 if (Range == CurrentRange)
213 break;
214 }
215
216 II.addRangeRetAttr(CR: Range);
217 II.addRetAttr(Kind: Attribute::NoUndef);
218 return &II;
219 }
220 case Intrinsic::arm_mve_vadc:
221 case Intrinsic::arm_mve_vadc_predicated: {
222 unsigned CarryOp =
223 (II.getIntrinsicID() == Intrinsic::arm_mve_vadc_predicated) ? 3 : 2;
224 assert(II.getArgOperand(CarryOp)->getType()->getScalarSizeInBits() == 32 &&
225 "Bad type for intrinsic!");
226
227 KnownBits CarryKnown(32);
228 if (IC.SimplifyDemandedBits(I: &II, OpNo: CarryOp, DemandedMask: APInt::getOneBitSet(numBits: 32, BitNo: 29),
229 Known&: CarryKnown)) {
230 return &II;
231 }
232 break;
233 }
234 case Intrinsic::arm_mve_vmldava: {
235 Instruction *I = cast<Instruction>(Val: &II);
236 if (I->hasOneUse()) {
237 auto *User = cast<Instruction>(Val: *I->user_begin());
238 Value *OpZ;
239 if (match(V: User, P: m_c_Add(L: m_Specific(V: I), R: m_Value(V&: OpZ))) &&
240 match(V: I->getOperand(i: 3), P: m_Zero())) {
241 Value *OpX = I->getOperand(i: 4);
242 Value *OpY = I->getOperand(i: 5);
243 Type *OpTy = OpX->getType();
244
245 IC.Builder.SetInsertPoint(User);
246 Value *V =
247 IC.Builder.CreateIntrinsic(ID: Intrinsic::arm_mve_vmldava, OverloadTypes: {OpTy},
248 Args: {I->getOperand(i: 0), I->getOperand(i: 1),
249 I->getOperand(i: 2), OpZ, OpX, OpY});
250
251 IC.replaceInstUsesWith(I&: *User, V);
252 return IC.eraseInstFromFunction(I&: *User);
253 }
254 }
255 return std::nullopt;
256 }
257 }
258 return std::nullopt;
259}
260
261std::optional<Value *> ARMTTIImpl::simplifyDemandedVectorEltsIntrinsic(
262 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,
263 APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
264 std::function<void(Instruction *, unsigned, APInt, APInt &)>
265 SimplifyAndSetOp) const {
266
267 // Compute the demanded bits for a narrowing MVE intrinsic. The TopOpc is the
268 // opcode specifying a Top/Bottom instruction, which can change between
269 // instructions.
270 auto SimplifyNarrowInstrTopBottom =[&](unsigned TopOpc) {
271 unsigned NumElts = cast<FixedVectorType>(Val: II.getType())->getNumElements();
272 unsigned IsTop = cast<ConstantInt>(Val: II.getOperand(i_nocapture: TopOpc))->getZExtValue();
273
274 // The only odd/even lanes of operand 0 will only be demanded depending
275 // on whether this is a top/bottom instruction.
276 APInt DemandedElts =
277 APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
278 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
279 SimplifyAndSetOp(&II, 0, OrigDemandedElts & DemandedElts, UndefElts);
280 // The other lanes will be defined from the inserted elements.
281 UndefElts &= APInt::getSplat(NewLen: NumElts, V: IsTop ? APInt::getLowBitsSet(numBits: 2, loBitsSet: 1)
282 : APInt::getHighBitsSet(numBits: 2, hiBitsSet: 1));
283 return std::nullopt;
284 };
285
286 switch (II.getIntrinsicID()) {
287 default:
288 break;
289 case Intrinsic::arm_mve_vcvt_narrow:
290 SimplifyNarrowInstrTopBottom(2);
291 break;
292 case Intrinsic::arm_mve_vqmovn:
293 SimplifyNarrowInstrTopBottom(4);
294 break;
295 case Intrinsic::arm_mve_vshrn:
296 SimplifyNarrowInstrTopBottom(7);
297 break;
298 }
299
300 return std::nullopt;
301}
302
303InstructionCost ARMTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
304 TTI::TargetCostKind CostKind) const {
305 assert(Ty->isIntegerTy());
306
307 unsigned Bits = Ty->getPrimitiveSizeInBits();
308 if (Bits == 0 || Imm.getActiveBits() >= 64)
309 return 4;
310
311 int64_t SImmVal = Imm.getSExtValue();
312 uint64_t ZImmVal = Imm.getZExtValue();
313 if (!ST->isThumb()) {
314 if ((SImmVal >= 0 && SImmVal < 65536) ||
315 (ARM_AM::getSOImmVal(Arg: ZImmVal) != -1) ||
316 (ARM_AM::getSOImmVal(Arg: ~ZImmVal) != -1))
317 return 1;
318 return ST->hasV6T2Ops() ? 2 : 3;
319 }
320 if (ST->isThumb2()) {
321 if ((SImmVal >= 0 && SImmVal < 65536) ||
322 (ARM_AM::getT2SOImmVal(Arg: ZImmVal) != -1) ||
323 (ARM_AM::getT2SOImmVal(Arg: ~ZImmVal) != -1))
324 return 1;
325 return ST->hasV6T2Ops() ? 2 : 3;
326 }
327 // Thumb1, any i8 imm cost 1.
328 if (Bits == 8 || (SImmVal >= 0 && SImmVal < 256))
329 return 1;
330 if ((~SImmVal < 256) || ARM_AM::isThumbImmShiftedVal(V: ZImmVal))
331 return 2;
332 // Load from constantpool.
333 return 3;
334}
335
336// Constants smaller than 256 fit in the immediate field of
337// Thumb1 instructions so we return a zero cost and 1 otherwise.
338InstructionCost ARMTTIImpl::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx,
339 const APInt &Imm,
340 Type *Ty) const {
341 if (Imm.isNonNegative() && Imm.getLimitedValue() < 256)
342 return 0;
343
344 return 1;
345}
346
347// Checks whether Inst is part of a min(max()) or max(min()) pattern
348// that will match to an SSAT instruction. Returns the instruction being
349// saturated, or null if no saturation pattern was found.
350static Value *isSSATMinMaxPattern(Instruction *Inst, const APInt &Imm) {
351 Value *LHS, *RHS;
352 ConstantInt *C;
353 SelectPatternFlavor InstSPF = matchSelectPattern(V: Inst, LHS, RHS).Flavor;
354
355 if (InstSPF == SPF_SMAX &&
356 PatternMatch::match(V: RHS, P: PatternMatch::m_ConstantInt(CI&: C)) &&
357 C->getValue() == Imm && Imm.isNegative() && Imm.isNegatedPowerOf2()) {
358
359 auto isSSatMin = [&](Value *MinInst) {
360 if (isa<SelectInst>(Val: MinInst)) {
361 Value *MinLHS, *MinRHS;
362 ConstantInt *MinC;
363 SelectPatternFlavor MinSPF =
364 matchSelectPattern(V: MinInst, LHS&: MinLHS, RHS&: MinRHS).Flavor;
365 if (MinSPF == SPF_SMIN &&
366 PatternMatch::match(V: MinRHS, P: PatternMatch::m_ConstantInt(CI&: MinC)) &&
367 MinC->getValue() == ((-Imm) - 1))
368 return true;
369 }
370 return false;
371 };
372
373 if (isSSatMin(Inst->getOperand(i: 1)))
374 return cast<Instruction>(Val: Inst->getOperand(i: 1))->getOperand(i: 1);
375 if (Inst->hasNUses(N: 2) &&
376 (isSSatMin(*Inst->user_begin()) || isSSatMin(*(++Inst->user_begin()))))
377 return Inst->getOperand(i: 1);
378 }
379 return nullptr;
380}
381
382// Look for a FP Saturation pattern, where the instruction can be simplified to
383// a fptosi.sat. max(min(fptosi)). The constant in this case is always free.
384static bool isFPSatMinMaxPattern(Instruction *Inst, const APInt &Imm) {
385 if (Imm.getBitWidth() != 64 ||
386 Imm != APInt::getHighBitsSet(numBits: 64, hiBitsSet: 33)) // -2147483648
387 return false;
388 Value *FP = isSSATMinMaxPattern(Inst, Imm);
389 if (!FP && isa<ICmpInst>(Val: Inst) && Inst->hasOneUse())
390 FP = isSSATMinMaxPattern(Inst: cast<Instruction>(Val: *Inst->user_begin()), Imm);
391 if (!FP)
392 return false;
393 return isa<FPToSIInst>(Val: FP);
394}
395
396InstructionCost ARMTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
397 const APInt &Imm, Type *Ty,
398 TTI::TargetCostKind CostKind,
399 Instruction *Inst) const {
400 // Division by a constant can be turned into multiplication, but only if we
401 // know it's constant. So it's not so much that the immediate is cheap (it's
402 // not), but that the alternative is worse.
403 // FIXME: this is probably unneeded with GlobalISel.
404 if ((Opcode == Instruction::SDiv || Opcode == Instruction::UDiv ||
405 Opcode == Instruction::SRem || Opcode == Instruction::URem) &&
406 Idx == 1)
407 return 0;
408
409 // Leave any gep offsets for the CodeGenPrepare, which will do a better job at
410 // splitting any large offsets.
411 if (Opcode == Instruction::GetElementPtr && Idx != 0)
412 return 0;
413
414 if (Opcode == Instruction::And) {
415 // UXTB/UXTH
416 if (Imm == 255 || Imm == 65535)
417 return 0;
418 // Conversion to BIC is free, and means we can use ~Imm instead.
419 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
420 b: getIntImmCost(Imm: ~Imm, Ty, CostKind));
421 }
422
423 if (Opcode == Instruction::Add)
424 // Conversion to SUB is free, and means we can use -Imm instead.
425 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
426 b: getIntImmCost(Imm: -Imm, Ty, CostKind));
427
428 if (Opcode == Instruction::ICmp && Imm.isNegative() &&
429 Ty->getIntegerBitWidth() == 32) {
430 int64_t NegImm = -Imm.getSExtValue();
431 if (ST->isThumb2() && NegImm < 1<<12)
432 // icmp X, #-C -> cmn X, #C
433 return 0;
434 if (ST->isThumb() && NegImm < 1<<8)
435 // icmp X, #-C -> adds X, #C
436 return 0;
437 }
438
439 // xor a, -1 can always be folded to MVN
440 if (Opcode == Instruction::Xor && Imm.isAllOnes())
441 return 0;
442
443 // Ensures negative constant of min(max()) or max(min()) patterns that
444 // match to SSAT instructions don't get hoisted
445 if (Inst && ((ST->hasV6Ops() && !ST->isThumb()) || ST->isThumb2()) &&
446 Ty->getIntegerBitWidth() <= 32) {
447 if (isSSATMinMaxPattern(Inst, Imm) ||
448 (isa<ICmpInst>(Val: Inst) && Inst->hasOneUse() &&
449 isSSATMinMaxPattern(Inst: cast<Instruction>(Val: *Inst->user_begin()), Imm)))
450 return 0;
451 }
452
453 if (Inst && ST->hasVFP2Base() && isFPSatMinMaxPattern(Inst, Imm))
454 return 0;
455
456 // We can convert <= -1 to < 0, which is generally quite cheap.
457 if (Inst && Opcode == Instruction::ICmp && Idx == 1 && Imm.isAllOnes()) {
458 ICmpInst::Predicate Pred = cast<ICmpInst>(Val: Inst)->getPredicate();
459 if (Pred == ICmpInst::ICMP_SGT || Pred == ICmpInst::ICMP_SLE)
460 return std::min(a: getIntImmCost(Imm, Ty, CostKind),
461 b: getIntImmCost(Imm: Imm + 1, Ty, CostKind));
462 }
463
464 return getIntImmCost(Imm, Ty, CostKind);
465}
466
467InstructionCost ARMTTIImpl::getCFInstrCost(unsigned Opcode,
468 TTI::TargetCostKind CostKind,
469 const Instruction *I) const {
470 if (CostKind == TTI::TCK_RecipThroughput &&
471 (ST->hasNEON() || ST->hasMVEIntegerOps())) {
472 // FIXME: The vectorizer is highly sensitive to the cost of these
473 // instructions, which suggests that it may be using the costs incorrectly.
474 // But, for now, just make them free to avoid performance regressions for
475 // vector targets.
476 return 0;
477 }
478 return BaseT::getCFInstrCost(Opcode, CostKind, I);
479}
480
481InstructionCost ARMTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
482 Type *Src,
483 TTI::CastContextHint CCH,
484 TTI::TargetCostKind CostKind,
485 const Instruction *I) const {
486 int ISD = TLI->InstructionOpcodeToISD(Opcode);
487 assert(ISD && "Invalid opcode");
488
489 // TODO: Allow non-throughput costs that aren't binary.
490 auto AdjustCost = [&CostKind](InstructionCost Cost) -> InstructionCost {
491 if (CostKind != TTI::TCK_RecipThroughput)
492 return Cost == 0 ? 0 : 1;
493 return Cost;
494 };
495 auto IsLegalFPType = [this](EVT VT) {
496 EVT EltVT = VT.getScalarType();
497 return (EltVT == MVT::f32 && ST->hasVFP2Base()) ||
498 (EltVT == MVT::f64 && ST->hasFP64()) ||
499 (EltVT == MVT::f16 && ST->hasFullFP16());
500 };
501
502 EVT SrcTy = TLI->getValueType(DL, Ty: Src);
503 EVT DstTy = TLI->getValueType(DL, Ty: Dst);
504
505 if (!SrcTy.isSimple() || !DstTy.isSimple())
506 return AdjustCost(
507 BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
508
509 // Extending masked load/Truncating masked stores is expensive because we
510 // currently don't split them. This means that we'll likely end up
511 // loading/storing each element individually (hence the high cost).
512 if ((ST->hasMVEIntegerOps() &&
513 (Opcode == Instruction::Trunc || Opcode == Instruction::ZExt ||
514 Opcode == Instruction::SExt)) ||
515 (ST->hasMVEFloatOps() &&
516 (Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc) &&
517 IsLegalFPType(SrcTy) && IsLegalFPType(DstTy)))
518 if (CCH == TTI::CastContextHint::Masked && DstTy.getSizeInBits() > 128)
519 return 2 * DstTy.getVectorNumElements() *
520 ST->getMVEVectorCostFactor(CostKind);
521
522 // The extend of other kinds of load is free
523 if (CCH == TTI::CastContextHint::Normal ||
524 CCH == TTI::CastContextHint::Masked) {
525 static const TypeConversionCostTblEntry LoadConversionTbl[] = {
526 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i32, .Src: MVT::i16, .Cost: 0},
527 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i32, .Src: MVT::i16, .Cost: 0},
528 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i32, .Src: MVT::i8, .Cost: 0},
529 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i32, .Src: MVT::i8, .Cost: 0},
530 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i16, .Src: MVT::i8, .Cost: 0},
531 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i16, .Src: MVT::i8, .Cost: 0},
532 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i32, .Cost: 1},
533 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i32, .Cost: 1},
534 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 1},
535 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 1},
536 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i8, .Cost: 1},
537 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::i64, .Src: MVT::i8, .Cost: 1},
538 };
539 if (const auto *Entry = ConvertCostTableLookup(
540 Table: LoadConversionTbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
541 return AdjustCost(Entry->Cost);
542
543 static const TypeConversionCostTblEntry MVELoadConversionTbl[] = {
544 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
545 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
546 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
547 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
548 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
549 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
550 // The following extend from a legal type to an illegal type, so need to
551 // split the load. This introduced an extra load operation, but the
552 // extend is still "free".
553 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
554 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
555 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
556 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
557 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
558 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
559 };
560 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
561 if (const auto *Entry =
562 ConvertCostTableLookup(Table: MVELoadConversionTbl, ISD,
563 Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
564 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
565 }
566
567 static const TypeConversionCostTblEntry MVEFLoadConversionTbl[] = {
568 // FPExtends are similar but also require the VCVT instructions.
569 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1},
570 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 3},
571 };
572 if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
573 if (const auto *Entry =
574 ConvertCostTableLookup(Table: MVEFLoadConversionTbl, ISD,
575 Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
576 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
577 }
578
579 // The truncate of a store is free. This is the mirror of extends above.
580 static const TypeConversionCostTblEntry MVEStoreConversionTbl[] = {
581 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0},
582 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 0},
583 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0},
584 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 1},
585 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 1},
586 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 3},
587 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 1},
588 };
589 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
590 if (const auto *Entry =
591 ConvertCostTableLookup(Table: MVEStoreConversionTbl, ISD,
592 Dst: SrcTy.getSimpleVT(), Src: DstTy.getSimpleVT()))
593 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
594 }
595
596 static const TypeConversionCostTblEntry MVEFStoreConversionTbl[] = {
597 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1},
598 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 3},
599 };
600 if (SrcTy.isVector() && ST->hasMVEFloatOps()) {
601 if (const auto *Entry =
602 ConvertCostTableLookup(Table: MVEFStoreConversionTbl, ISD,
603 Dst: SrcTy.getSimpleVT(), Src: DstTy.getSimpleVT()))
604 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
605 }
606 }
607
608 // NEON vector operations that can extend their inputs.
609 if ((ISD == ISD::SIGN_EXTEND || ISD == ISD::ZERO_EXTEND) &&
610 I && I->hasOneUse() && ST->hasNEON() && SrcTy.isVector()) {
611 static const TypeConversionCostTblEntry NEONDoubleWidthTbl[] = {
612 // vaddl
613 { .ISD: ISD::ADD, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
614 { .ISD: ISD::ADD, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
615 // vsubl
616 { .ISD: ISD::SUB, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
617 { .ISD: ISD::SUB, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
618 // vmull
619 { .ISD: ISD::MUL, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
620 { .ISD: ISD::MUL, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
621 // vshll
622 { .ISD: ISD::SHL, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 0 },
623 { .ISD: ISD::SHL, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 0 },
624 };
625
626 auto *User = cast<Instruction>(Val: *I->user_begin());
627 int UserISD = TLI->InstructionOpcodeToISD(Opcode: User->getOpcode());
628 if (auto *Entry = ConvertCostTableLookup(Table: NEONDoubleWidthTbl, ISD: UserISD,
629 Dst: DstTy.getSimpleVT(),
630 Src: SrcTy.getSimpleVT())) {
631 return AdjustCost(Entry->Cost);
632 }
633 }
634
635 // Single to/from double precision conversions.
636 if (Src->isVectorTy() && ST->hasNEON() &&
637 ((ISD == ISD::FP_ROUND && SrcTy.getScalarType() == MVT::f64 &&
638 DstTy.getScalarType() == MVT::f32) ||
639 (ISD == ISD::FP_EXTEND && SrcTy.getScalarType() == MVT::f32 &&
640 DstTy.getScalarType() == MVT::f64))) {
641 static const CostTblEntry NEONFltDblTbl[] = {
642 // Vector fptrunc/fpext conversions.
643 {.ISD: ISD::FP_ROUND, .Type: MVT::v2f64, .Cost: 2},
644 {.ISD: ISD::FP_EXTEND, .Type: MVT::v2f32, .Cost: 2},
645 {.ISD: ISD::FP_EXTEND, .Type: MVT::v4f32, .Cost: 4}};
646
647 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
648 if (const auto *Entry = CostTableLookup(Table: NEONFltDblTbl, ISD, Ty: LT.second))
649 return AdjustCost(LT.first * Entry->Cost);
650 }
651
652 // Some arithmetic, load and store operations have specific instructions
653 // to cast up/down their types automatically at no extra cost.
654 // TODO: Get these tables to know at least what the related operations are.
655 static const TypeConversionCostTblEntry NEONVectorConversionTbl[] = {
656 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
657 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
658 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1 },
659 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 1 },
660 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: 0 },
661 { .ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i32, .Cost: 1 },
662
663 // The number of vmovl instructions for the extension.
664 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
665 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
666 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
667 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
668 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 3 },
669 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 3 },
670 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
671 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
672 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3 },
673 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3 },
674 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3 },
675 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3 },
676 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7 },
677 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7 },
678 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6 },
679 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6 },
680 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6 },
681 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6 },
682
683 // Operations that we legalize using splitting.
684 { .ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: 6 },
685 { .ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: 3 },
686
687 // Vector float <-> i32 conversions.
688 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1 },
689 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1 },
690
691 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3 },
692 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3 },
693 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2 },
694 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 2 },
695 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1 },
696 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1 },
697 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: 3 },
698 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i1, .Cost: 3 },
699 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3 },
700 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3 },
701 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2 },
702 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2 },
703 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4 },
704 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4 },
705 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: 2 },
706 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i32, .Cost: 2 },
707 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: 8 },
708 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i16, .Cost: 8 },
709 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: 4 },
710 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i32, .Cost: 4 },
711
712 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1 },
713 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1 },
714 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 3 },
715 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 3 },
716 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2 },
717 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2 },
718
719 // Vector double <-> i32 conversions.
720 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
721 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
722
723 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4 },
724 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4 },
725 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 3 },
726 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 3 },
727 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
728 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2 },
729
730 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2 },
731 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2 },
732 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 4 },
733 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f32, .Cost: 4 },
734 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: 8 },
735 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v16f32, .Cost: 8 }
736 };
737
738 if (SrcTy.isVector() && ST->hasNEON()) {
739 if (const auto *Entry = ConvertCostTableLookup(Table: NEONVectorConversionTbl, ISD,
740 Dst: DstTy.getSimpleVT(),
741 Src: SrcTy.getSimpleVT()))
742 return AdjustCost(Entry->Cost);
743 }
744
745 // Scalar float to integer conversions.
746 static const TypeConversionCostTblEntry NEONFloatConversionTbl[] = {
747 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i1, .Src: MVT::f32, .Cost: 2 },
748 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i1, .Src: MVT::f32, .Cost: 2 },
749 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i1, .Src: MVT::f64, .Cost: 2 },
750 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i1, .Src: MVT::f64, .Cost: 2 },
751 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i8, .Src: MVT::f32, .Cost: 2 },
752 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i8, .Src: MVT::f32, .Cost: 2 },
753 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i8, .Src: MVT::f64, .Cost: 2 },
754 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i8, .Src: MVT::f64, .Cost: 2 },
755 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i16, .Src: MVT::f32, .Cost: 2 },
756 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i16, .Src: MVT::f32, .Cost: 2 },
757 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i16, .Src: MVT::f64, .Cost: 2 },
758 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i16, .Src: MVT::f64, .Cost: 2 },
759 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: 2 },
760 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f32, .Cost: 2 },
761 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: 2 },
762 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i32, .Src: MVT::f64, .Cost: 2 },
763 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: 10 },
764 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f32, .Cost: 10 },
765 { .ISD: ISD::FP_TO_SINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: 10 },
766 { .ISD: ISD::FP_TO_UINT, .Dst: MVT::i64, .Src: MVT::f64, .Cost: 10 }
767 };
768 if (SrcTy.isFloatingPoint() && ST->hasNEON()) {
769 if (const auto *Entry = ConvertCostTableLookup(Table: NEONFloatConversionTbl, ISD,
770 Dst: DstTy.getSimpleVT(),
771 Src: SrcTy.getSimpleVT()))
772 return AdjustCost(Entry->Cost);
773 }
774
775 // Scalar integer to float conversions.
776 static const TypeConversionCostTblEntry NEONIntegerConversionTbl[] = {
777 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i1, .Cost: 2 },
778 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i1, .Cost: 2 },
779 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i1, .Cost: 2 },
780 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i1, .Cost: 2 },
781 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i8, .Cost: 2 },
782 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i8, .Cost: 2 },
783 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i8, .Cost: 2 },
784 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i8, .Cost: 2 },
785 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i16, .Cost: 2 },
786 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i16, .Cost: 2 },
787 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i16, .Cost: 2 },
788 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i16, .Cost: 2 },
789 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: 2 },
790 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i32, .Cost: 2 },
791 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: 2 },
792 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i32, .Cost: 2 },
793 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: 10 },
794 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f32, .Src: MVT::i64, .Cost: 10 },
795 { .ISD: ISD::SINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: 10 },
796 { .ISD: ISD::UINT_TO_FP, .Dst: MVT::f64, .Src: MVT::i64, .Cost: 10 }
797 };
798
799 if (SrcTy.isInteger() && ST->hasNEON()) {
800 if (const auto *Entry = ConvertCostTableLookup(Table: NEONIntegerConversionTbl,
801 ISD, Dst: DstTy.getSimpleVT(),
802 Src: SrcTy.getSimpleVT()))
803 return AdjustCost(Entry->Cost);
804 }
805
806 // MVE extend costs, taken from codegen tests. i8->i16 or i16->i32 is one
807 // instruction, i8->i32 is two. i64 zexts are an VAND with a constant, sext
808 // are linearised so take more.
809 static const TypeConversionCostTblEntry MVEVectorConversionTbl[] = {
810 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
811 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i16, .Src: MVT::v8i8, .Cost: 1 },
812 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
813 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i8, .Cost: 2 },
814 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 10 },
815 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i8, .Cost: 2 },
816 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
817 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i32, .Src: MVT::v4i16, .Cost: 1 },
818 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 10 },
819 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i16, .Cost: 2 },
820 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 8 },
821 { .ISD: ISD::ZERO_EXTEND, .Dst: MVT::v2i64, .Src: MVT::v2i32, .Cost: 2 },
822 };
823
824 if (SrcTy.isVector() && ST->hasMVEIntegerOps()) {
825 if (const auto *Entry = ConvertCostTableLookup(Table: MVEVectorConversionTbl,
826 ISD, Dst: DstTy.getSimpleVT(),
827 Src: SrcTy.getSimpleVT()))
828 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
829 }
830
831 if (ISD == ISD::FP_ROUND || ISD == ISD::FP_EXTEND) {
832 // As general rule, fp converts that were not matched above are scalarized
833 // and cost 1 vcvt for each lane, so long as the instruction is available.
834 // If not it will become a series of function calls.
835 const InstructionCost CallCost =
836 getCallInstrCost(F: nullptr, RetTy: Dst, Tys: {Src}, CostKind);
837 int Lanes = 1;
838 if (SrcTy.isFixedLengthVector())
839 Lanes = SrcTy.getVectorNumElements();
840
841 if (IsLegalFPType(SrcTy) && IsLegalFPType(DstTy))
842 return Lanes;
843 else
844 return Lanes * CallCost;
845 }
846
847 if (ISD == ISD::TRUNCATE && ST->hasMVEIntegerOps() &&
848 SrcTy.isFixedLengthVector()) {
849 // Treat a truncate with larger than legal source (128bits for MVE) as
850 // expensive, 2 instructions per lane.
851 if ((SrcTy.getScalarType() == MVT::i8 ||
852 SrcTy.getScalarType() == MVT::i16 ||
853 SrcTy.getScalarType() == MVT::i32) &&
854 SrcTy.getSizeInBits() > 128 &&
855 SrcTy.getSizeInBits() > DstTy.getSizeInBits())
856 return SrcTy.getVectorNumElements() * 2;
857 }
858
859 // Scalar integer conversion costs.
860 static const TypeConversionCostTblEntry ARMIntegerConversionTbl[] = {
861 // i16 -> i64 requires two dependent operations.
862 { .ISD: ISD::SIGN_EXTEND, .Dst: MVT::i64, .Src: MVT::i16, .Cost: 2 },
863
864 // Truncates on i64 are assumed to be free.
865 { .ISD: ISD::TRUNCATE, .Dst: MVT::i32, .Src: MVT::i64, .Cost: 0 },
866 { .ISD: ISD::TRUNCATE, .Dst: MVT::i16, .Src: MVT::i64, .Cost: 0 },
867 { .ISD: ISD::TRUNCATE, .Dst: MVT::i8, .Src: MVT::i64, .Cost: 0 },
868 { .ISD: ISD::TRUNCATE, .Dst: MVT::i1, .Src: MVT::i64, .Cost: 0 }
869 };
870
871 if (SrcTy.isInteger()) {
872 if (const auto *Entry = ConvertCostTableLookup(Table: ARMIntegerConversionTbl, ISD,
873 Dst: DstTy.getSimpleVT(),
874 Src: SrcTy.getSimpleVT()))
875 return AdjustCost(Entry->Cost);
876 }
877
878 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
879 ? ST->getMVEVectorCostFactor(CostKind)
880 : 1;
881 return AdjustCost(
882 BaseCost * BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
883}
884
885InstructionCost ARMTTIImpl::getVectorInstrCost(
886 unsigned Opcode, Type *ValTy, TTI::TargetCostKind CostKind, unsigned Index,
887 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
888 // Penalize inserting into an D-subregister. We end up with a three times
889 // lower estimated throughput on swift.
890 if (ST->hasSlowLoadDSubregister() && Opcode == Instruction::InsertElement &&
891 ValTy->isVectorTy() && ValTy->getScalarSizeInBits() <= 32)
892 return 3;
893
894 if (ST->hasNEON() && (Opcode == Instruction::InsertElement ||
895 Opcode == Instruction::ExtractElement)) {
896 // Cross-class copies are expensive on many microarchitectures,
897 // so assume they are expensive by default.
898 if (cast<VectorType>(Val: ValTy)->getElementType()->isIntegerTy())
899 return 3;
900
901 // Even if it's not a cross class copy, this likely leads to mixing
902 // of NEON and VFP code and should be therefore penalized.
903 if (ValTy->isVectorTy() &&
904 ValTy->getScalarSizeInBits() <= 32)
905 return std::max<InstructionCost>(
906 a: BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
907 VIC),
908 b: 2U);
909 }
910
911 if (ST->hasMVEIntegerOps() && (Opcode == Instruction::InsertElement ||
912 Opcode == Instruction::ExtractElement)) {
913 // Integer cross-lane moves are more expensive than float, which can
914 // sometimes just be vmovs. Integer involve being passes to GPR registers,
915 // causing more of a delay.
916 std::pair<InstructionCost, MVT> LT =
917 getTypeLegalizationCost(Ty: ValTy->getScalarType());
918 return LT.first * (ValTy->getScalarType()->isIntegerTy() ? 4 : 1);
919 }
920
921 return BaseT::getVectorInstrCost(Opcode, Val: ValTy, CostKind, Index, Op0, Op1,
922 VIC);
923}
924
925InstructionCost ARMTTIImpl::getCmpSelInstrCost(
926 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
927 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
928 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
929 int ISD = TLI->InstructionOpcodeToISD(Opcode);
930
931 // Thumb scalar code size cost for select.
932 if (CostKind == TTI::TCK_CodeSize && ISD == ISD::SELECT &&
933 ST->isThumb() && !ValTy->isVectorTy()) {
934 // Assume expensive structs.
935 if (TLI->getValueType(DL, Ty: ValTy, AllowUnknown: true) == MVT::Other)
936 return TTI::TCC_Expensive;
937
938 // Select costs can vary because they:
939 // - may require one or more conditional mov (including an IT),
940 // - can't operate directly on immediates,
941 // - require live flags, which we can't copy around easily.
942 InstructionCost Cost = getTypeLegalizationCost(Ty: ValTy).first;
943
944 // Possible IT instruction for Thumb2, or more for Thumb1.
945 ++Cost;
946
947 // i1 values may need rematerialising by using mov immediates and/or
948 // flag setting instructions.
949 if (ValTy->isIntegerTy(BitWidth: 1))
950 ++Cost;
951
952 return Cost;
953 }
954
955 // If this is a vector min/max/abs, use the cost of that intrinsic directly
956 // instead. Hopefully when min/max intrinsics are more prevalent this code
957 // will not be needed.
958 const Instruction *Sel = I;
959 if ((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) && Sel &&
960 Sel->hasOneUse())
961 Sel = cast<Instruction>(Val: Sel->user_back());
962 if (Sel && ValTy->isVectorTy() &&
963 (ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy())) {
964 const Value *LHS, *RHS;
965 SelectPatternFlavor SPF = matchSelectPattern(V: Sel, LHS, RHS).Flavor;
966 unsigned IID = 0;
967 switch (SPF) {
968 case SPF_ABS:
969 IID = Intrinsic::abs;
970 break;
971 case SPF_SMIN:
972 IID = Intrinsic::smin;
973 break;
974 case SPF_SMAX:
975 IID = Intrinsic::smax;
976 break;
977 case SPF_UMIN:
978 IID = Intrinsic::umin;
979 break;
980 case SPF_UMAX:
981 IID = Intrinsic::umax;
982 break;
983 case SPF_FMINNUM:
984 IID = Intrinsic::minnum;
985 break;
986 case SPF_FMAXNUM:
987 IID = Intrinsic::maxnum;
988 break;
989 default:
990 break;
991 }
992 if (IID) {
993 // The ICmp is free, the select gets the cost of the min/max/etc
994 if (Sel != I)
995 return 0;
996 IntrinsicCostAttributes CostAttrs(IID, ValTy, {ValTy, ValTy});
997 return getIntrinsicInstrCost(ICA: CostAttrs, CostKind);
998 }
999 }
1000
1001 // On NEON a vector select gets lowered to vbsl.
1002 if (ST->hasNEON() && ValTy->isVectorTy() && ISD == ISD::SELECT && CondTy) {
1003 // Lowering of some vector selects is currently far from perfect.
1004 static const TypeConversionCostTblEntry NEONVectorSelectTbl[] = {
1005 { .ISD: ISD::SELECT, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: 4*4 + 1*2 + 1 },
1006 { .ISD: ISD::SELECT, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: 50 },
1007 { .ISD: ISD::SELECT, .Dst: MVT::v16i1, .Src: MVT::v16i64, .Cost: 100 }
1008 };
1009
1010 EVT SelCondTy = TLI->getValueType(DL, Ty: CondTy);
1011 EVT SelValTy = TLI->getValueType(DL, Ty: ValTy);
1012 if (SelCondTy.isSimple() && SelValTy.isSimple()) {
1013 if (const auto *Entry = ConvertCostTableLookup(Table: NEONVectorSelectTbl, ISD,
1014 Dst: SelCondTy.getSimpleVT(),
1015 Src: SelValTy.getSimpleVT()))
1016 return Entry->Cost;
1017 }
1018
1019 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1020 return LT.first;
1021 }
1022
1023 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy() &&
1024 (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
1025 cast<FixedVectorType>(Val: ValTy)->getNumElements() > 1) {
1026 FixedVectorType *VecValTy = cast<FixedVectorType>(Val: ValTy);
1027 FixedVectorType *VecCondTy = dyn_cast_or_null<FixedVectorType>(Val: CondTy);
1028 if (!VecCondTy)
1029 VecCondTy = cast<FixedVectorType>(Val: CmpInst::makeCmpResultType(opnd_type: VecValTy));
1030
1031 // If we don't have mve.fp any fp operations will need to be scalarized.
1032 if (Opcode == Instruction::FCmp && !ST->hasMVEFloatOps()) {
1033 // One scalaization insert, one scalarization extract and the cost of the
1034 // fcmps.
1035 return BaseT::getScalarizationOverhead(InTy: VecValTy, /*Insert*/ false,
1036 /*Extract*/ true, CostKind) +
1037 BaseT::getScalarizationOverhead(InTy: VecCondTy, /*Insert*/ true,
1038 /*Extract*/ false, CostKind) +
1039 VecValTy->getNumElements() *
1040 getCmpSelInstrCost(Opcode, ValTy: ValTy->getScalarType(),
1041 CondTy: VecCondTy->getScalarType(), VecPred,
1042 CostKind, Op1Info, Op2Info, I);
1043 }
1044
1045 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1046 int BaseCost = ST->getMVEVectorCostFactor(CostKind);
1047 // There are two types - the input that specifies the type of the compare
1048 // and the output vXi1 type. Because we don't know how the output will be
1049 // split, we may need an expensive shuffle to get two in sync. This has the
1050 // effect of making larger than legal compares (v8i32 for example)
1051 // expensive.
1052 if (LT.second.isVector() && LT.second.getVectorNumElements() > 2) {
1053 if (LT.first > 1)
1054 return LT.first * BaseCost +
1055 BaseT::getScalarizationOverhead(InTy: VecCondTy, /*Insert*/ true,
1056 /*Extract*/ false, CostKind);
1057 return BaseCost;
1058 }
1059 }
1060
1061 // Default to cheap (throughput/size of 1 instruction) but adjust throughput
1062 // for "multiple beats" potentially needed by MVE instructions.
1063 int BaseCost = 1;
1064 if (ST->hasMVEIntegerOps() && ValTy->isVectorTy())
1065 BaseCost = ST->getMVEVectorCostFactor(CostKind);
1066
1067 return BaseCost * BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred,
1068 CostKind, Op1Info, Op2Info, I);
1069}
1070
1071InstructionCost
1072ARMTTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
1073 const SCEV *Ptr,
1074 TTI::TargetCostKind CostKind) const {
1075 // Address computations in vectorized code with non-consecutive addresses will
1076 // likely result in more instructions compared to scalar code where the
1077 // computation can more often be merged into the index mode. The resulting
1078 // extra micro-ops can significantly decrease throughput.
1079 unsigned NumVectorInstToHideOverhead = 10;
1080 int MaxMergeDistance = 64;
1081
1082 if (ST->hasNEON()) {
1083 if (PtrTy->isVectorTy() && SE &&
1084 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MergeDistance: MaxMergeDistance + 1))
1085 return NumVectorInstToHideOverhead;
1086
1087 // In many cases the address computation is not merged into the instruction
1088 // addressing mode.
1089 return 1;
1090 }
1091 return BaseT::getAddressComputationCost(PtrTy, SE, Ptr, CostKind);
1092}
1093
1094bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) const {
1095 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
1096 // If a VCTP is part of a chain, it's already profitable and shouldn't be
1097 // optimized, else LSR may block tail-predication.
1098 switch (II->getIntrinsicID()) {
1099 case Intrinsic::arm_mve_vctp8:
1100 case Intrinsic::arm_mve_vctp16:
1101 case Intrinsic::arm_mve_vctp32:
1102 case Intrinsic::arm_mve_vctp64:
1103 return true;
1104 default:
1105 break;
1106 }
1107 }
1108 return false;
1109}
1110
1111bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment,
1112 unsigned /*AddressSpace*/,
1113 TTI::MaskKind /*MaskKind*/) const {
1114 if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps())
1115 return false;
1116
1117 if (auto *VecTy = dyn_cast<FixedVectorType>(Val: DataTy)) {
1118 // Don't support v2i1 yet.
1119 if (VecTy->getNumElements() == 2)
1120 return false;
1121
1122 // We don't support extending fp types.
1123 unsigned VecWidth = DataTy->getPrimitiveSizeInBits();
1124 if (VecWidth != 128 && VecTy->getElementType()->isFloatingPointTy())
1125 return false;
1126 }
1127
1128 unsigned EltWidth = DataTy->getScalarSizeInBits();
1129 return (EltWidth == 32 && Alignment >= 4) ||
1130 (EltWidth == 16 && Alignment >= 2) || (EltWidth == 8);
1131}
1132
1133bool ARMTTIImpl::isLegalMaskedGather(Type *Ty, Align Alignment) const {
1134 if (!EnableMaskedGatherScatters || !ST->hasMVEIntegerOps())
1135 return false;
1136
1137 unsigned EltWidth = Ty->getScalarSizeInBits();
1138 return ((EltWidth == 32 && Alignment >= 4) ||
1139 (EltWidth == 16 && Alignment >= 2) || EltWidth == 8);
1140}
1141
1142/// Given a memcpy/memset/memmove instruction, return the number of memory
1143/// operations performed, via querying findOptimalMemOpLowering. Returns -1 if a
1144/// call is used.
1145int ARMTTIImpl::getNumMemOps(const IntrinsicInst *I) const {
1146 MemOp MOp;
1147 unsigned DstAddrSpace = ~0u;
1148 unsigned SrcAddrSpace = ~0u;
1149 const Function *F = I->getParent()->getParent();
1150
1151 if (const auto *MC = dyn_cast<MemTransferInst>(Val: I)) {
1152 ConstantInt *C = dyn_cast<ConstantInt>(Val: MC->getLength());
1153 // If 'size' is not a constant, a library call will be generated.
1154 if (!C)
1155 return -1;
1156
1157 const unsigned Size = C->getValue().getZExtValue();
1158 const Align DstAlign = MC->getDestAlign().valueOrOne();
1159 const Align SrcAlign = MC->getSourceAlign().valueOrOne();
1160
1161 // Use the most restrictive of memset, memcpy, memmove.
1162 MOp = MemOp::Move(Size, /*DstAlignCanChange*/ false, DstAlign, SrcAlign,
1163 /*IsVolatile*/ false);
1164 DstAddrSpace = MC->getDestAddressSpace();
1165 SrcAddrSpace = MC->getSourceAddressSpace();
1166 }
1167 else if (const auto *MS = dyn_cast<MemSetInst>(Val: I)) {
1168 ConstantInt *C = dyn_cast<ConstantInt>(Val: MS->getLength());
1169 // If 'size' is not a constant, a library call will be generated.
1170 if (!C)
1171 return -1;
1172
1173 const unsigned Size = C->getValue().getZExtValue();
1174 const Align DstAlign = MS->getDestAlign().valueOrOne();
1175
1176 MOp = MemOp::Set(Size, /*DstAlignCanChange*/ false, DstAlign,
1177 /*IsZeroMemset*/ false, /*IsVolatile*/ false);
1178 DstAddrSpace = MS->getDestAddressSpace();
1179 }
1180 else
1181 llvm_unreachable("Expected a memcpy/move or memset!");
1182
1183 unsigned Limit, Factor = 2;
1184 switch(I->getIntrinsicID()) {
1185 case Intrinsic::memcpy:
1186 Limit = TLI->getMaxStoresPerMemcpy(OptSize: F->hasMinSize());
1187 break;
1188 case Intrinsic::memmove:
1189 Limit = TLI->getMaxStoresPerMemmove(OptSize: F->hasMinSize());
1190 break;
1191 case Intrinsic::memset:
1192 Limit = TLI->getMaxStoresPerMemset(OptSize: F->hasMinSize());
1193 Factor = 1;
1194 break;
1195 default:
1196 llvm_unreachable("Expected a memcpy/move or memset!");
1197 }
1198
1199 // MemOps will be poplulated with a list of data types that needs to be
1200 // loaded and stored. That's why we multiply the number of elements by 2 to
1201 // get the cost for this memcpy.
1202 std::vector<EVT> MemOps;
1203 LLVMContext &C = F->getContext();
1204 if (getTLI()->findOptimalMemOpLowering(Context&: C, MemOps, Limit, Op: MOp, DstAS: DstAddrSpace,
1205 SrcAS: SrcAddrSpace, FuncAttributes: F->getAttributes(),
1206 LargestVT: nullptr))
1207 return MemOps.size() * Factor;
1208
1209 // If we can't find an optimal memop lowering, return the default cost
1210 return -1;
1211}
1212
1213InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) const {
1214 int NumOps = getNumMemOps(I: cast<IntrinsicInst>(Val: I));
1215
1216 // To model the cost of a library call, we assume 1 for the call, and
1217 // 3 for the argument setup.
1218 if (NumOps == -1)
1219 return 4;
1220 return NumOps;
1221}
1222
1223InstructionCost ARMTTIImpl::getShuffleCost(
1224 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
1225 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
1226 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
1227 TTI::VectorInstrContext VIC) const {
1228 assert((Mask.empty() || DstTy->isScalableTy() ||
1229 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
1230 "Expected the Mask to match the return size if given");
1231 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
1232 "Expected the same scalar types");
1233
1234 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
1235 // Treat extractsubvector as single op permutation.
1236 bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector;
1237 if (IsExtractSubvector)
1238 Kind = TTI::SK_PermuteSingleSrc;
1239 if (ST->hasNEON()) {
1240 if (Kind == TTI::SK_Broadcast) {
1241 static const CostTblEntry NEONDupTbl[] = {
1242 // VDUP handles these cases.
1243 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1244 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1245 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1246 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1247 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 1},
1248 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i8, .Cost: 1},
1249
1250 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 1},
1251 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 1},
1252 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 1},
1253 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 1}};
1254
1255 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1256 if (const auto *Entry =
1257 CostTableLookup(Table: NEONDupTbl, ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1258 return LT.first * Entry->Cost;
1259 }
1260 if (Kind == TTI::SK_Reverse) {
1261 static const CostTblEntry NEONShuffleTbl[] = {
1262 // Reverse shuffle cost one instruction if we are shuffling within a
1263 // double word (vrev) or two if we shuffle a quad word (vrev, vext).
1264 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1265 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1266 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1267 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1268 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 1},
1269 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i8, .Cost: 1},
1270
1271 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 2},
1272 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 2},
1273 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 2},
1274 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 2}};
1275
1276 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1277 if (const auto *Entry =
1278 CostTableLookup(Table: NEONShuffleTbl, ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1279 return LT.first * Entry->Cost;
1280 }
1281 if (Kind == TTI::SK_Select) {
1282 static const CostTblEntry NEONSelShuffleTbl[] = {
1283 // Select shuffle cost table for ARM. Cost is the number of
1284 // instructions
1285 // required to create the shuffled vector.
1286
1287 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f32, .Cost: 1},
1288 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i64, .Cost: 1},
1289 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2f64, .Cost: 1},
1290 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v2i32, .Cost: 1},
1291
1292 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 2},
1293 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 2},
1294 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i16, .Cost: 2},
1295
1296 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 16},
1297
1298 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 32}};
1299
1300 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1301 if (const auto *Entry = CostTableLookup(Table: NEONSelShuffleTbl,
1302 ISD: ISD::VECTOR_SHUFFLE, Ty: LT.second))
1303 return LT.first * Entry->Cost;
1304 }
1305
1306 // Check for other shuffles that are not SK_ kinds but we have native
1307 // instructions for, for example REV.
1308 if (!Mask.empty()) {
1309 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1310 unsigned Unused;
1311 if (LT.second.isVector() &&
1312 Mask.size() <= LT.second.getVectorNumElements() &&
1313 (isVREVMask(M: Mask, VT: LT.second, BlockSize: 16) || isVREVMask(M: Mask, VT: LT.second, BlockSize: 32) ||
1314 isVREVMask(M: Mask, VT: LT.second, BlockSize: 64) ||
1315 isVTRNMask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1316 isVTRN_v_undef_Mask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1317 isVZIPMask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1318 isVZIP_v_undef_Mask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1319 isVUZPMask(M: Mask, VT: LT.second, WhichResult&: Unused) ||
1320 isVUZP_v_undef_Mask(M: Mask, VT: LT.second, WhichResult&: Unused)))
1321 return LT.first;
1322 }
1323 }
1324 if (ST->hasMVEIntegerOps()) {
1325 if (Kind == TTI::SK_Broadcast) {
1326 static const CostTblEntry MVEDupTbl[] = {
1327 // VDUP handles these cases.
1328 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4i32, .Cost: 1},
1329 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8i16, .Cost: 1},
1330 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v16i8, .Cost: 1},
1331 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v4f32, .Cost: 1},
1332 {.ISD: ISD::VECTOR_SHUFFLE, .Type: MVT::v8f16, .Cost: 1}};
1333
1334 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1335 if (const auto *Entry = CostTableLookup(Table: MVEDupTbl, ISD: ISD::VECTOR_SHUFFLE,
1336 Ty: LT.second))
1337 return LT.first * Entry->Cost * ST->getMVEVectorCostFactor(CostKind);
1338 }
1339
1340 if (!Mask.empty()) {
1341 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
1342 // Check for LD2/LD4 instructions, which are represented in llvm IR as
1343 // deinterleaving-shuffle(load). The shuffle cost could potentially be
1344 // free, but we model it with a cost of LT.first so that LD2/LD4 have a
1345 // higher cost than just the load.
1346 if (Args.size() >= 1 && isa<LoadInst>(Val: Args[0]) &&
1347 (LT.second.getScalarSizeInBits() == 8 ||
1348 LT.second.getScalarSizeInBits() == 16 ||
1349 LT.second.getScalarSizeInBits() == 32) &&
1350 LT.second.getSizeInBits() == 128 &&
1351 ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
1352 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2)) ||
1353 (TLI->getMaxSupportedInterleaveFactor() == 4 &&
1354 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 4))))
1355 return ST->getMVEVectorCostFactor(CostKind) *
1356 std::max<InstructionCost>(a: 1, b: LT.first / 4);
1357
1358 // Check for ST2/ST4 instructions, which are represented in llvm IR as
1359 // store(interleaving-shuffle). The shuffle cost could potentially be
1360 // free, but we model it with a cost of LT.first so that ST2/ST4 have a
1361 // higher cost than just the store.
1362 if (CtxI && CtxI->hasOneUse() && isa<StoreInst>(Val: *CtxI->user_begin()) &&
1363 (LT.second.getScalarSizeInBits() == 8 ||
1364 LT.second.getScalarSizeInBits() == 16 ||
1365 LT.second.getScalarSizeInBits() == 32) &&
1366 LT.second.getSizeInBits() == 128 &&
1367 ((TLI->getMaxSupportedInterleaveFactor() >= 2 &&
1368 ShuffleVectorInst::isInterleaveMask(
1369 Mask, Factor: 2, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2)) ||
1370 (TLI->getMaxSupportedInterleaveFactor() == 4 &&
1371 ShuffleVectorInst::isInterleaveMask(
1372 Mask, Factor: 4, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2))))
1373 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1374
1375 if (LT.second.isVector() &&
1376 Mask.size() <= LT.second.getVectorNumElements() &&
1377 (isVREVMask(M: Mask, VT: LT.second, BlockSize: 16) || isVREVMask(M: Mask, VT: LT.second, BlockSize: 32) ||
1378 isVREVMask(M: Mask, VT: LT.second, BlockSize: 64)))
1379 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1380 }
1381 }
1382
1383 // Restore optimal kind.
1384 if (IsExtractSubvector)
1385 Kind = TTI::SK_ExtractSubvector;
1386 int BaseCost = ST->hasMVEIntegerOps() && SrcTy->isVectorTy()
1387 ? ST->getMVEVectorCostFactor(CostKind)
1388 : 1;
1389 return BaseCost * BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask,
1390 Index, SubTp);
1391}
1392
1393InstructionCost ARMTTIImpl::getArithmeticInstrCost(
1394 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
1395 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
1396 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
1397 int ISDOpcode = TLI->InstructionOpcodeToISD(Opcode);
1398 if (ST->isThumb() && CostKind == TTI::TCK_CodeSize && Ty->isIntegerTy(BitWidth: 1)) {
1399 // Make operations on i1 relatively expensive as this often involves
1400 // combining predicates. AND and XOR should be easier to handle with IT
1401 // blocks.
1402 switch (ISDOpcode) {
1403 default:
1404 break;
1405 case ISD::AND:
1406 case ISD::XOR:
1407 return 2;
1408 case ISD::OR:
1409 return 3;
1410 }
1411 }
1412
1413 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
1414
1415 if (ST->hasNEON()) {
1416 const unsigned FunctionCallDivCost = 20;
1417 const unsigned ReciprocalDivCost = 10;
1418 static const CostTblEntry CostTbl[] = {
1419 // Division.
1420 // These costs are somewhat random. Choose a cost of 20 to indicate that
1421 // vectorizing devision (added function call) is going to be very expensive.
1422 // Double registers types.
1423 { .ISD: ISD::SDIV, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1424 { .ISD: ISD::UDIV, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1425 { .ISD: ISD::SREM, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1426 { .ISD: ISD::UREM, .Type: MVT::v1i64, .Cost: 1 * FunctionCallDivCost},
1427 { .ISD: ISD::SDIV, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1428 { .ISD: ISD::UDIV, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1429 { .ISD: ISD::SREM, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1430 { .ISD: ISD::UREM, .Type: MVT::v2i32, .Cost: 2 * FunctionCallDivCost},
1431 { .ISD: ISD::SDIV, .Type: MVT::v4i16, .Cost: ReciprocalDivCost},
1432 { .ISD: ISD::UDIV, .Type: MVT::v4i16, .Cost: ReciprocalDivCost},
1433 { .ISD: ISD::SREM, .Type: MVT::v4i16, .Cost: 4 * FunctionCallDivCost},
1434 { .ISD: ISD::UREM, .Type: MVT::v4i16, .Cost: 4 * FunctionCallDivCost},
1435 { .ISD: ISD::SDIV, .Type: MVT::v8i8, .Cost: ReciprocalDivCost},
1436 { .ISD: ISD::UDIV, .Type: MVT::v8i8, .Cost: ReciprocalDivCost},
1437 { .ISD: ISD::SREM, .Type: MVT::v8i8, .Cost: 8 * FunctionCallDivCost},
1438 { .ISD: ISD::UREM, .Type: MVT::v8i8, .Cost: 8 * FunctionCallDivCost},
1439 // Quad register types.
1440 { .ISD: ISD::SDIV, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1441 { .ISD: ISD::UDIV, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1442 { .ISD: ISD::SREM, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1443 { .ISD: ISD::UREM, .Type: MVT::v2i64, .Cost: 2 * FunctionCallDivCost},
1444 { .ISD: ISD::SDIV, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1445 { .ISD: ISD::UDIV, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1446 { .ISD: ISD::SREM, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1447 { .ISD: ISD::UREM, .Type: MVT::v4i32, .Cost: 4 * FunctionCallDivCost},
1448 { .ISD: ISD::SDIV, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1449 { .ISD: ISD::UDIV, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1450 { .ISD: ISD::SREM, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1451 { .ISD: ISD::UREM, .Type: MVT::v8i16, .Cost: 8 * FunctionCallDivCost},
1452 { .ISD: ISD::SDIV, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1453 { .ISD: ISD::UDIV, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1454 { .ISD: ISD::SREM, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1455 { .ISD: ISD::UREM, .Type: MVT::v16i8, .Cost: 16 * FunctionCallDivCost},
1456 // Multiplication.
1457 };
1458
1459 if (const auto *Entry = CostTableLookup(Table: CostTbl, ISD: ISDOpcode, Ty: LT.second))
1460 return LT.first * Entry->Cost;
1461
1462 InstructionCost Cost = BaseT::getArithmeticInstrCost(
1463 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
1464
1465 // This is somewhat of a hack. The problem that we are facing is that SROA
1466 // creates a sequence of shift, and, or instructions to construct values.
1467 // These sequences are recognized by the ISel and have zero-cost. Not so for
1468 // the vectorized code. Because we have support for v2i64 but not i64 those
1469 // sequences look particularly beneficial to vectorize.
1470 // To work around this we increase the cost of v2i64 operations to make them
1471 // seem less beneficial.
1472 if (LT.second == MVT::v2i64 && Op2Info.isUniform() && Op2Info.isConstant())
1473 Cost += 4;
1474
1475 return Cost;
1476 }
1477
1478 // If this operation is a shift on arm/thumb2, it might well be folded into
1479 // the following instruction, hence having a cost of 0.
1480 auto LooksLikeAFreeShift = [&]() {
1481 if (ST->isThumb1Only() || Ty->isVectorTy())
1482 return false;
1483
1484 if (!CtxI || !CtxI->hasOneUse() || !CtxI->isShift())
1485 return false;
1486 if (!Op2Info.isUniform() || !Op2Info.isConstant())
1487 return false;
1488
1489 // Folded into a ADC/ADD/AND/BIC/CMP/EOR/MVN/ORR/ORN/RSB/SBC/SUB
1490 switch (cast<Instruction>(Val: CtxI->user_back())->getOpcode()) {
1491 case Instruction::Add:
1492 case Instruction::Sub:
1493 case Instruction::And:
1494 case Instruction::Xor:
1495 case Instruction::Or:
1496 case Instruction::ICmp:
1497 return true;
1498 default:
1499 return false;
1500 }
1501 };
1502 if (LooksLikeAFreeShift())
1503 return 0;
1504
1505 // When targets have both DSP and MVE we find that the
1506 // the compiler will attempt to vectorize as well as using
1507 // scalar (S/U)MLAL operations. This is in cases where we have
1508 // the pattern ext(mul(ext(i16), ext(i16))) we find
1509 // that codegen performs better when only using (S/U)MLAL scalar
1510 // ops instead of trying to mix vector ops with (S/U)MLAL ops. We therefore
1511 // check if a mul instruction is used in a (U/S)MLAL pattern.
1512 auto MulInDSPMLALPattern = [&](const Instruction *I, unsigned Opcode,
1513 Type *Ty) -> bool {
1514 if (!ST->hasDSP())
1515 return false;
1516
1517 if (!I)
1518 return false;
1519
1520 if (Opcode != Instruction::Mul)
1521 return false;
1522
1523 if (Ty->isVectorTy())
1524 return false;
1525
1526 auto ValueOpcodesEqual = [](const Value *LHS, const Value *RHS) -> bool {
1527 return cast<Instruction>(Val: LHS)->getOpcode() ==
1528 cast<Instruction>(Val: RHS)->getOpcode();
1529 };
1530 auto IsExtInst = [](const Value *V) -> bool {
1531 return isa<ZExtInst>(Val: V) || isa<SExtInst>(Val: V);
1532 };
1533 auto IsExtensionFromHalf = [](const Value *V) -> bool {
1534 return cast<Instruction>(Val: V)->getOperand(i: 0)->getType()->isIntegerTy(BitWidth: 16);
1535 };
1536
1537 // We check the arguments of the instruction to see if they're extends
1538 auto *BinOp = dyn_cast<BinaryOperator>(Val: I);
1539 if (!BinOp)
1540 return false;
1541 Value *Op0 = BinOp->getOperand(i_nocapture: 0);
1542 Value *Op1 = BinOp->getOperand(i_nocapture: 1);
1543 if (IsExtInst(Op0) && IsExtInst(Op1) && ValueOpcodesEqual(Op0, Op1)) {
1544 // We're interested in an ext of an i16
1545 if (!I->getType()->isIntegerTy(BitWidth: 32) || !IsExtensionFromHalf(Op0) ||
1546 !IsExtensionFromHalf(Op1))
1547 return false;
1548 // We need to check if this result will be further extended to i64
1549 // and that all these uses are SExt
1550 for (auto *U : I->users())
1551 if (!IsExtInst(U))
1552 return false;
1553 return true;
1554 }
1555
1556 return false;
1557 };
1558
1559 if (MulInDSPMLALPattern(CtxI, Opcode, Ty))
1560 return 0;
1561
1562 // Default to cheap (throughput/size of 1 instruction) but adjust throughput
1563 // for "multiple beats" potentially needed by MVE instructions.
1564 int BaseCost = 1;
1565 if (ST->hasMVEIntegerOps() && Ty->isVectorTy())
1566 BaseCost = ST->getMVEVectorCostFactor(CostKind);
1567
1568 // The rest of this mostly follows what is done in
1569 // BaseT::getArithmeticInstrCost, without treating floats as more expensive
1570 // that scalars or increasing the costs for custom operations. The results is
1571 // also multiplied by the MVEVectorCostFactor where appropriate.
1572 if (TLI->isOperationLegalOrCustomOrPromote(Op: ISDOpcode, VT: LT.second))
1573 return LT.first * BaseCost;
1574
1575 // Else this is expand, assume that we need to scalarize this op.
1576 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
1577 unsigned Num = VTy->getNumElements();
1578 InstructionCost Cost =
1579 getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind);
1580 // Return the cost of multiple scalar invocation plus the cost of
1581 // inserting and extracting the values.
1582 SmallVector<Type *> Tys(Args.size(), Ty);
1583 return BaseT::getScalarizationOverhead(RetTy: VTy, Args, Tys, CostKind) +
1584 Num * Cost;
1585 }
1586
1587 return BaseCost;
1588}
1589
1590InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
1591 Align Alignment,
1592 unsigned AddressSpace,
1593 TTI::TargetCostKind CostKind,
1594 TTI::OperandValueInfo OpInfo,
1595 const Instruction *I) const {
1596 // FIXME: Load latency isn't handled here
1597 if (Opcode == Instruction::Load && CostKind == TTI::TCK_Latency)
1598 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1599 CostKind, OpInfo, I);
1600
1601 // TODO: Handle other cost kinds.
1602 if (CostKind != TTI::TCK_RecipThroughput)
1603 return 1;
1604
1605 // Type legalization can't handle structs
1606 if (TLI->getValueType(DL, Ty: Src, AllowUnknown: true) == MVT::Other)
1607 return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1608 CostKind);
1609
1610 if (ST->hasNEON() && Src->isVectorTy() && Alignment != Align(16) &&
1611 cast<VectorType>(Val: Src)->getElementType()->isDoubleTy()) {
1612 // Unaligned loads/stores are extremely inefficient.
1613 // We need 4 uops for vst.1/vld.1 vs 1uop for vldr/vstr.
1614 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Src);
1615 return LT.first * 4;
1616 }
1617
1618 // MVE can optimize a fpext(load(4xhalf)) using an extending integer load.
1619 // Same for stores.
1620 if (ST->hasMVEFloatOps() && isa<FixedVectorType>(Val: Src) && I &&
1621 ((Opcode == Instruction::Load && I->hasOneUse() &&
1622 isa<FPExtInst>(Val: *I->user_begin())) ||
1623 (Opcode == Instruction::Store && isa<FPTruncInst>(Val: I->getOperand(i: 0))))) {
1624 FixedVectorType *SrcVTy = cast<FixedVectorType>(Val: Src);
1625 Type *DstTy =
1626 Opcode == Instruction::Load
1627 ? (*I->user_begin())->getType()
1628 : cast<Instruction>(Val: I->getOperand(i: 0))->getOperand(i: 0)->getType();
1629 if (SrcVTy->getNumElements() == 4 && SrcVTy->getScalarType()->isHalfTy() &&
1630 DstTy->getScalarType()->isFloatTy())
1631 return ST->getMVEVectorCostFactor(CostKind);
1632 }
1633
1634 int BaseCost = ST->hasMVEIntegerOps() && Src->isVectorTy()
1635 ? ST->getMVEVectorCostFactor(CostKind)
1636 : 1;
1637 return BaseCost * BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
1638 CostKind, OpInfo, I);
1639}
1640
1641InstructionCost
1642ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
1643 TTI::TargetCostKind CostKind) const {
1644 switch (MICA.getID()) {
1645 case Intrinsic::masked_scatter:
1646 case Intrinsic::masked_gather:
1647 return getGatherScatterOpCost(MICA, CostKind);
1648 case Intrinsic::masked_load:
1649 case Intrinsic::masked_store:
1650 return getMaskedMemoryOpCost(MICA, CostKind);
1651 }
1652 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1653}
1654
1655InstructionCost
1656ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
1657 TTI::TargetCostKind CostKind) const {
1658 unsigned IID = MICA.getID();
1659 Type *Src = MICA.getDataType();
1660 Align Alignment = MICA.getAlignment();
1661 unsigned AddressSpace = MICA.getAddressSpace();
1662 if (ST->hasMVEIntegerOps()) {
1663 if (IID == Intrinsic::masked_load &&
1664 isLegalMaskedLoad(DataTy: Src, Alignment, AddressSpace))
1665 return ST->getMVEVectorCostFactor(CostKind);
1666 if (IID == Intrinsic::masked_store &&
1667 isLegalMaskedStore(DataTy: Src, Alignment, AddressSpace))
1668 return ST->getMVEVectorCostFactor(CostKind);
1669 }
1670 if (!isa<FixedVectorType>(Val: Src))
1671 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1672 // Scalar cost, which is currently very high due to the efficiency of the
1673 // generated code.
1674 return cast<FixedVectorType>(Val: Src)->getNumElements() * 8;
1675}
1676
1677InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost(
1678 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1679 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1680 bool UseMaskForCond, bool UseMaskForGaps) const {
1681 assert(Factor >= 2 && "Invalid interleave factor");
1682 assert(isa<VectorType>(VecTy) && "Expect a vector type");
1683
1684 // vldN/vstN doesn't support vector types of i64/f64 element.
1685 bool EltIs64Bits = DL.getTypeSizeInBits(Ty: VecTy->getScalarType()) == 64;
1686
1687 if (Factor <= TLI->getMaxSupportedInterleaveFactor() && !EltIs64Bits &&
1688 !UseMaskForCond && !UseMaskForGaps) {
1689 unsigned NumElts = cast<FixedVectorType>(Val: VecTy)->getNumElements();
1690 auto *SubVecTy =
1691 FixedVectorType::get(ElementType: VecTy->getScalarType(), NumElts: NumElts / Factor);
1692
1693 // vldN/vstN only support legal vector types of size 64 or 128 in bits.
1694 // Accesses having vector types that are a multiple of 128 bits can be
1695 // matched to more than one vldN/vstN instruction.
1696 int BaseCost =
1697 ST->hasMVEIntegerOps() ? ST->getMVEVectorCostFactor(CostKind) : 1;
1698 if (NumElts % Factor == 0 &&
1699 TLI->isLegalInterleavedAccessType(Factor, VecTy: SubVecTy, Alignment, DL))
1700 return Factor * BaseCost * TLI->getNumInterleavedAccesses(VecTy: SubVecTy, DL);
1701
1702 // Some smaller than legal interleaved patterns are cheap as we can make
1703 // use of the vmovn or vrev patterns to interleave a standard load. This is
1704 // true for v4i8, v8i8 and v4i16 at least (but not for v4f16 as it is
1705 // promoted differently). The cost of 2 here is then a load and vrev or
1706 // vmovn.
1707 if (ST->hasMVEIntegerOps() && Factor == 2 && NumElts / Factor > 2 &&
1708 VecTy->isIntOrIntVectorTy() &&
1709 DL.getTypeSizeInBits(Ty: SubVecTy).getFixedValue() <= 64)
1710 return 2 * BaseCost;
1711 }
1712
1713 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1714 Alignment, AddressSpace, CostKind,
1715 UseMaskForCond, UseMaskForGaps);
1716}
1717
1718InstructionCost
1719ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
1720 TTI::TargetCostKind CostKind) const {
1721
1722 Type *DataTy = MICA.getDataType();
1723 const Value *Ptr = MICA.getPointer();
1724 bool VariableMask = MICA.getVariableMask();
1725 Align Alignment = MICA.getAlignment();
1726 const Instruction *I = MICA.getInst();
1727
1728 using namespace PatternMatch;
1729 if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters)
1730 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
1731
1732 assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!");
1733 auto *VTy = cast<FixedVectorType>(Val: DataTy);
1734
1735 // TODO: Splitting, once we do that.
1736
1737 unsigned NumElems = VTy->getNumElements();
1738 unsigned EltSize = VTy->getScalarSizeInBits();
1739 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: DataTy);
1740
1741 // For now, it is assumed that for the MVE gather instructions the loads are
1742 // all effectively serialised. This means the cost is the scalar cost
1743 // multiplied by the number of elements being loaded. This is possibly very
1744 // conservative, but even so we still end up vectorising loops because the
1745 // cost per iteration for many loops is lower than for scalar loops.
1746 InstructionCost VectorCost =
1747 NumElems * LT.first * ST->getMVEVectorCostFactor(CostKind);
1748 // The scalarization cost should be a lot higher. We use the number of vector
1749 // elements plus the scalarization overhead. If masking is required then a lot
1750 // of little blocks will be needed and potentially a scalarized p0 mask,
1751 // greatly increasing the cost.
1752 InstructionCost ScalarCost =
1753 NumElems * LT.first + (VariableMask ? NumElems * 5 : 0) +
1754 BaseT::getScalarizationOverhead(InTy: VTy, /*Insert*/ true, /*Extract*/ false,
1755 CostKind) +
1756 BaseT::getScalarizationOverhead(InTy: VTy, /*Insert*/ false, /*Extract*/ true,
1757 CostKind);
1758
1759 if (EltSize < 8 || Alignment < EltSize / 8)
1760 return ScalarCost;
1761
1762 unsigned ExtSize = EltSize;
1763 // Check whether there's a single user that asks for an extended type
1764 if (I != nullptr) {
1765 // Dependent of the caller of this function, a gather instruction will
1766 // either have opcode Instruction::Load or be a call to the masked_gather
1767 // intrinsic
1768 if ((I->getOpcode() == Instruction::Load ||
1769 match(V: I, P: m_Intrinsic<Intrinsic::masked_gather>())) &&
1770 I->hasOneUse()) {
1771 const User *Us = *I->users().begin();
1772 if (isa<ZExtInst>(Val: Us) || isa<SExtInst>(Val: Us)) {
1773 // only allow valid type combinations
1774 unsigned TypeSize =
1775 cast<Instruction>(Val: Us)->getType()->getScalarSizeInBits();
1776 if (((TypeSize == 32 && (EltSize == 8 || EltSize == 16)) ||
1777 (TypeSize == 16 && EltSize == 8)) &&
1778 TypeSize * NumElems == 128) {
1779 ExtSize = TypeSize;
1780 }
1781 }
1782 }
1783 // Check whether the input data needs to be truncated
1784 TruncInst *T;
1785 if ((I->getOpcode() == Instruction::Store ||
1786 match(V: I, P: m_Intrinsic<Intrinsic::masked_scatter>())) &&
1787 (T = dyn_cast<TruncInst>(Val: I->getOperand(i: 0)))) {
1788 // Only allow valid type combinations
1789 unsigned TypeSize = T->getOperand(i_nocapture: 0)->getType()->getScalarSizeInBits();
1790 if (((EltSize == 16 && TypeSize == 32) ||
1791 (EltSize == 8 && (TypeSize == 32 || TypeSize == 16))) &&
1792 TypeSize * NumElems == 128)
1793 ExtSize = TypeSize;
1794 }
1795 }
1796
1797 if (ExtSize * NumElems != 128 || NumElems < 4)
1798 return ScalarCost;
1799
1800 // Any (aligned) i32 gather will not need to be scalarised.
1801 if (ExtSize == 32)
1802 return VectorCost;
1803 // For smaller types, we need to ensure that the gep's inputs are correctly
1804 // extended from a small enough value. Other sizes (including i64) are
1805 // scalarized for now.
1806 if (ExtSize != 8 && ExtSize != 16)
1807 return ScalarCost;
1808
1809 if (const auto *BC = dyn_cast<BitCastInst>(Val: Ptr))
1810 Ptr = BC->getOperand(i_nocapture: 0);
1811 if (const auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptr)) {
1812 if (GEP->getNumOperands() != 2)
1813 return ScalarCost;
1814 unsigned Scale = DL.getTypeAllocSize(Ty: GEP->getResultElementType());
1815 // Scale needs to be correct (which is only relevant for i16s).
1816 if (Scale != 1 && Scale * 8 != ExtSize)
1817 return ScalarCost;
1818 // And we need to zext (not sext) the indexes from a small enough type.
1819 if (const auto *ZExt = dyn_cast<ZExtInst>(Val: GEP->getOperand(i_nocapture: 1))) {
1820 if (ZExt->getOperand(i_nocapture: 0)->getType()->getScalarSizeInBits() <= ExtSize)
1821 return VectorCost;
1822 }
1823 return ScalarCost;
1824 }
1825 return ScalarCost;
1826}
1827
1828InstructionCost
1829ARMTTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
1830 std::optional<FastMathFlags> FMF,
1831 TTI::TargetCostKind CostKind) const {
1832
1833 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1834 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1835 unsigned EltSize = ValVT.getScalarSizeInBits();
1836
1837 // In general floating point reductions are a series of elementwise
1838 // operations, with free extracts on each step. These are either in-order or
1839 // treewise depending on whether that is allowed by the fast math flags.
1840 if ((ISD == ISD::FADD || ISD == ISD::FMUL) &&
1841 ((EltSize == 32 && ST->hasVFP2Base()) ||
1842 (EltSize == 64 && ST->hasFP64()) ||
1843 (EltSize == 16 && ST->hasFullFP16()))) {
1844 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
1845 unsigned VecLimit = ST->hasMVEFloatOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1846 InstructionCost VecCost = 0;
1847 while (!TTI::requiresOrderedReduction(FMF) && isPowerOf2_32(Value: NumElts) &&
1848 NumElts * EltSize > VecLimit) {
1849 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts: NumElts / 2);
1850 VecCost += getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1851 NumElts /= 2;
1852 }
1853
1854 // For fp16 we need to extract the upper lane elements. MVE can add a
1855 // VREV+FMIN/MAX to perform another vector step instead.
1856 InstructionCost ExtractCost = 0;
1857 if (!TTI::requiresOrderedReduction(FMF) && ST->hasMVEFloatOps() &&
1858 ValVT.getVectorElementType() == MVT::f16 && NumElts == 8) {
1859 VecCost += ST->getMVEVectorCostFactor(CostKind) * 2;
1860 NumElts /= 2;
1861 } else if (ValVT.getVectorElementType() == MVT::f16)
1862 ExtractCost = NumElts / 2;
1863
1864 return VecCost + ExtractCost +
1865 NumElts *
1866 getArithmeticInstrCost(Opcode, Ty: ValTy->getElementType(), CostKind);
1867 }
1868
1869 if ((ISD == ISD::AND || ISD == ISD::OR || ISD == ISD::XOR) &&
1870 (EltSize == 64 || EltSize == 32 || EltSize == 16 || EltSize == 8)) {
1871 unsigned NumElts = cast<FixedVectorType>(Val: ValTy)->getNumElements();
1872 unsigned VecLimit =
1873 ST->hasMVEIntegerOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1874 InstructionCost VecCost = 0;
1875 while (isPowerOf2_32(Value: NumElts) && NumElts * EltSize > VecLimit) {
1876 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts: NumElts / 2);
1877 VecCost += getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1878 NumElts /= 2;
1879 }
1880 // For i16/i8, MVE will perform a VREV + VORR/VAND/VEOR for the 64bit vector
1881 // step.
1882 if (ST->hasMVEIntegerOps() && ValVT.getScalarSizeInBits() <= 16 &&
1883 NumElts * EltSize == 64) {
1884 Type *VecTy = FixedVectorType::get(ElementType: ValTy->getElementType(), NumElts);
1885 VecCost += ST->getMVEVectorCostFactor(CostKind) +
1886 getArithmeticInstrCost(Opcode, Ty: VecTy, CostKind);
1887 NumElts /= 2;
1888 }
1889
1890 // From here we extract the elements and perform the and/or/xor.
1891 InstructionCost ExtractCost = NumElts;
1892 return VecCost + ExtractCost +
1893 (NumElts - 1) * getArithmeticInstrCost(
1894 Opcode, Ty: ValTy->getElementType(), CostKind);
1895 }
1896
1897 if (!ST->hasMVEIntegerOps() || !ValVT.isSimple() || ISD != ISD::ADD ||
1898 TTI::requiresOrderedReduction(FMF))
1899 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
1900
1901 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1902
1903 static const CostTblEntry CostTblAdd[]{
1904 {.ISD: ISD::ADD, .Type: MVT::v16i8, .Cost: 1},
1905 {.ISD: ISD::ADD, .Type: MVT::v8i16, .Cost: 1},
1906 {.ISD: ISD::ADD, .Type: MVT::v4i32, .Cost: 1},
1907 };
1908 if (const auto *Entry = CostTableLookup(Table: CostTblAdd, ISD, Ty: LT.second))
1909 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first;
1910
1911 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
1912}
1913
1914InstructionCost ARMTTIImpl::getExtendedReductionCost(
1915 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *ValTy,
1916 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
1917 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1918 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
1919
1920 int ISD = TLI->InstructionOpcodeToISD(Opcode);
1921
1922 switch (ISD) {
1923 case ISD::ADD:
1924 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) {
1925 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1926
1927 // The legal cases are:
1928 // VADDV u/s 8/16/32
1929 // VADDLV u/s 32
1930 // Codegen currently cannot always handle larger than legal vectors very
1931 // well, especially for predicated reductions where the mask needs to be
1932 // split, so restrict to 128bit or smaller input types.
1933 unsigned RevVTSize = ResVT.getSizeInBits();
1934 if (ValVT.getSizeInBits() <= 128 &&
1935 ((LT.second == MVT::v16i8 && RevVTSize <= 32) ||
1936 (LT.second == MVT::v8i16 && RevVTSize <= 32) ||
1937 (LT.second == MVT::v4i32 && RevVTSize <= 64)))
1938 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1939 }
1940 break;
1941 default:
1942 break;
1943 }
1944 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: ValTy, FMF,
1945 CostKind);
1946}
1947
1948InstructionCost
1949ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
1950 Type *ResTy, VectorType *ValTy,
1951 TTI::TargetCostKind CostKind) const {
1952 if (RedOpcode != Instruction::Add)
1953 return InstructionCost::getInvalid(Val: CostKind);
1954 EVT ValVT = TLI->getValueType(DL, Ty: ValTy);
1955 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
1956
1957 if (ST->hasMVEIntegerOps() && ValVT.isSimple() && ResVT.isSimple()) {
1958 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
1959
1960 // The legal cases are:
1961 // VMLAV u/s 8/16/32
1962 // VMLALV u/s 16/32
1963 // Codegen currently cannot always handle larger than legal vectors very
1964 // well, especially for predicated reductions where the mask needs to be
1965 // split, so restrict to 128bit or smaller input types.
1966 unsigned RevVTSize = ResVT.getSizeInBits();
1967 if (ValVT.getSizeInBits() <= 128 &&
1968 ((LT.second == MVT::v16i8 && RevVTSize <= 32) ||
1969 (LT.second == MVT::v8i16 && RevVTSize <= 64) ||
1970 (LT.second == MVT::v4i32 && RevVTSize <= 64)))
1971 return ST->getMVEVectorCostFactor(CostKind) * LT.first;
1972 }
1973
1974 return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty: ValTy,
1975 CostKind);
1976}
1977
1978InstructionCost
1979ARMTTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
1980 FastMathFlags FMF,
1981 TTI::TargetCostKind CostKind) const {
1982 EVT ValVT = TLI->getValueType(DL, Ty);
1983
1984 // In general floating point reductions are a series of elementwise
1985 // operations, with free extracts on each step. These are either in-order or
1986 // treewise depending on whether that is allowed by the fast math flags.
1987 if ((IID == Intrinsic::minnum || IID == Intrinsic::maxnum) &&
1988 ((ValVT.getVectorElementType() == MVT::f32 && ST->hasVFP2Base()) ||
1989 (ValVT.getVectorElementType() == MVT::f64 && ST->hasFP64()) ||
1990 (ValVT.getVectorElementType() == MVT::f16 && ST->hasFullFP16()))) {
1991 unsigned NumElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
1992 unsigned EltSize = ValVT.getScalarSizeInBits();
1993 unsigned VecLimit = ST->hasMVEFloatOps() ? 128 : (ST->hasNEON() ? 64 : -1);
1994 InstructionCost VecCost;
1995 while (isPowerOf2_32(Value: NumElts) && NumElts * EltSize > VecLimit) {
1996 Type *VecTy = FixedVectorType::get(ElementType: Ty->getElementType(), NumElts: NumElts/2);
1997 IntrinsicCostAttributes ICA(IID, VecTy, {VecTy, VecTy}, FMF);
1998 VecCost += getIntrinsicInstrCost(ICA, CostKind);
1999 NumElts /= 2;
2000 }
2001
2002 // For fp16 we need to extract the upper lane elements. MVE can add a
2003 // VREV+FMIN/MAX to perform another vector step instead.
2004 InstructionCost ExtractCost = 0;
2005 if (ST->hasMVEFloatOps() && ValVT.getVectorElementType() == MVT::f16 &&
2006 NumElts == 8) {
2007 VecCost += ST->getMVEVectorCostFactor(CostKind) * 2;
2008 NumElts /= 2;
2009 } else if (ValVT.getVectorElementType() == MVT::f16)
2010 ExtractCost = cast<FixedVectorType>(Val: Ty)->getNumElements() / 2;
2011
2012 IntrinsicCostAttributes ICA(IID, Ty->getElementType(),
2013 {Ty->getElementType(), Ty->getElementType()},
2014 FMF);
2015 return VecCost + ExtractCost +
2016 (NumElts - 1) * getIntrinsicInstrCost(ICA, CostKind);
2017 }
2018
2019 if (IID == Intrinsic::smin || IID == Intrinsic::smax ||
2020 IID == Intrinsic::umin || IID == Intrinsic::umax) {
2021 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
2022
2023 // All costs are the same for u/s min/max. These lower to vminv, which are
2024 // given a slightly higher cost as they tend to take multiple cycles for
2025 // smaller type sizes.
2026 static const CostTblEntry CostTblAdd[]{
2027 {.ISD: ISD::SMIN, .Type: MVT::v16i8, .Cost: 4},
2028 {.ISD: ISD::SMIN, .Type: MVT::v8i16, .Cost: 3},
2029 {.ISD: ISD::SMIN, .Type: MVT::v4i32, .Cost: 2},
2030 };
2031 if (const auto *Entry = CostTableLookup(Table: CostTblAdd, ISD: ISD::SMIN, Ty: LT.second))
2032 return Entry->Cost * ST->getMVEVectorCostFactor(CostKind) * LT.first;
2033 }
2034
2035 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
2036}
2037
2038InstructionCost
2039ARMTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
2040 TTI::TargetCostKind CostKind) const {
2041 unsigned Opc = ICA.getID();
2042 switch (Opc) {
2043 case Intrinsic::get_active_lane_mask:
2044 // Currently we make a somewhat optimistic assumption that
2045 // active_lane_mask's are always free. In reality it may be freely folded
2046 // into a tail predicated loop, expanded into a VCPT or expanded into a lot
2047 // of add/icmp code. We may need to improve this in the future, but being
2048 // able to detect if it is free or not involves looking at a lot of other
2049 // code. We currently assume that the vectorizer inserted these, and knew
2050 // what it was doing in adding one.
2051 if (ST->hasMVEIntegerOps())
2052 return 0;
2053 break;
2054 case Intrinsic::sadd_sat:
2055 case Intrinsic::ssub_sat:
2056 case Intrinsic::uadd_sat:
2057 case Intrinsic::usub_sat: {
2058 bool IsAdd = (Opc == Intrinsic::sadd_sat || Opc == Intrinsic::ssub_sat);
2059 bool IsSigned = (Opc == Intrinsic::sadd_sat || Opc == Intrinsic::ssub_sat);
2060 Type *RetTy = ICA.getReturnType();
2061
2062 if (auto *ITy = dyn_cast<IntegerType>(Val: RetTy)) {
2063 if (IsSigned && ST->hasDSP() && ITy->getBitWidth() == 32)
2064 return 1; // qadd / qsub
2065 if (ST->hasDSP() && (ITy->getBitWidth() == 8 || ITy->getBitWidth() == 16))
2066 return 2; // uqadd16 / qadd16 / uqsub16 / qsub16 + possible extend.
2067 // Otherwise return the cost of expanding the node. Generally an add +
2068 // icmp + sel.
2069 CmpInst::Predicate Pred = CmpInst::ICMP_SGT;
2070 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2071 return getArithmeticInstrCost(Opcode: IsAdd ? Instruction::Add : Instruction::Sub,
2072 Ty: RetTy, CostKind) +
2073 2 * getCmpSelInstrCost(Opcode: BinaryOperator::ICmp, ValTy: RetTy, CondTy, VecPred: Pred,
2074 CostKind) +
2075 2 * getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy, VecPred: Pred,
2076 CostKind);
2077 }
2078
2079 if (!ST->hasMVEIntegerOps())
2080 break;
2081
2082 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: RetTy);
2083 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 ||
2084 LT.second == MVT::v16i8) {
2085 // This is a base cost of 1 for the vqadd, plus 3 extract shifts if we
2086 // need to extend the type, as it uses shr(qadd(shl, shl)).
2087 unsigned Instrs =
2088 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1
2089 : 4;
2090 return LT.first * ST->getMVEVectorCostFactor(CostKind) * Instrs;
2091 }
2092 break;
2093 }
2094 case Intrinsic::abs:
2095 case Intrinsic::smin:
2096 case Intrinsic::smax:
2097 case Intrinsic::umin:
2098 case Intrinsic::umax: {
2099 if (!ST->hasMVEIntegerOps())
2100 break;
2101 Type *VT = ICA.getReturnType();
2102
2103 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VT);
2104 if (LT.second == MVT::v4i32 || LT.second == MVT::v8i16 ||
2105 LT.second == MVT::v16i8)
2106 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2107 break;
2108 }
2109 case Intrinsic::minnum:
2110 case Intrinsic::maxnum: {
2111 if (!ST->hasMVEFloatOps())
2112 break;
2113 Type *VT = ICA.getReturnType();
2114 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VT);
2115 if (LT.second == MVT::v4f32 || LT.second == MVT::v8f16)
2116 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2117 break;
2118 }
2119 case Intrinsic::fptosi_sat:
2120 case Intrinsic::fptoui_sat: {
2121 if (ICA.getArgTypes().empty())
2122 break;
2123 bool IsSigned = Opc == Intrinsic::fptosi_sat;
2124 auto LT = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
2125 EVT MTy = TLI->getValueType(DL, Ty: ICA.getReturnType());
2126 // Check for the legal types, with the correct subtarget features.
2127 if ((ST->hasVFP2Base() && LT.second == MVT::f32 && MTy == MVT::i32) ||
2128 (ST->hasFP64() && LT.second == MVT::f64 && MTy == MVT::i32) ||
2129 (ST->hasFullFP16() && LT.second == MVT::f16 && MTy == MVT::i32))
2130 return LT.first;
2131
2132 // Equally for MVE vector types
2133 if (ST->hasMVEFloatOps() &&
2134 (LT.second == MVT::v4f32 || LT.second == MVT::v8f16) &&
2135 LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits())
2136 return LT.first * ST->getMVEVectorCostFactor(CostKind);
2137
2138 // If we can we use a legal convert followed by a min+max
2139 if (((ST->hasVFP2Base() && LT.second == MVT::f32) ||
2140 (ST->hasFP64() && LT.second == MVT::f64) ||
2141 (ST->hasFullFP16() && LT.second == MVT::f16) ||
2142 (ST->hasMVEFloatOps() &&
2143 (LT.second == MVT::v4f32 || LT.second == MVT::v8f16))) &&
2144 LT.second.getScalarSizeInBits() >= MTy.getScalarSizeInBits()) {
2145 Type *LegalTy = Type::getIntNTy(C&: ICA.getReturnType()->getContext(),
2146 N: LT.second.getScalarSizeInBits());
2147 InstructionCost Cost =
2148 LT.second.isVector() ? ST->getMVEVectorCostFactor(CostKind) : 1;
2149 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin
2150 : Intrinsic::umin,
2151 LegalTy, {LegalTy, LegalTy});
2152 Cost += getIntrinsicInstrCost(ICA: Attrs1, CostKind);
2153 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax
2154 : Intrinsic::umax,
2155 LegalTy, {LegalTy, LegalTy});
2156 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
2157 return LT.first * Cost;
2158 }
2159 // Otherwise we need to follow the default expansion that clamps the value
2160 // using a float min/max with a fcmp+sel for nan handling when signed.
2161 Type *FPTy = ICA.getArgTypes()[0];
2162 Type *RetTy = ICA.getReturnType();
2163 IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FPTy, {FPTy, FPTy});
2164 InstructionCost Cost = getIntrinsicInstrCost(ICA: Attrs1, CostKind);
2165 IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FPTy, {FPTy, FPTy});
2166 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
2167 Cost +=
2168 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
2169 Dst: RetTy, Src: FPTy, CCH: TTI::CastContextHint::None, CostKind);
2170 if (IsSigned) {
2171 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
2172 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: FPTy, CondTy,
2173 VecPred: CmpInst::FCMP_UNO, CostKind);
2174 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
2175 VecPred: CmpInst::FCMP_UNO, CostKind);
2176 }
2177 return Cost;
2178 }
2179 }
2180
2181 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
2182}
2183
2184bool ARMTTIImpl::isLoweredToCall(const Function *F) const {
2185 if (!F->isIntrinsic())
2186 return BaseT::isLoweredToCall(F);
2187
2188 // Assume all Arm-specific intrinsics map to an instruction.
2189 if (F->getName().starts_with(Prefix: "llvm.arm"))
2190 return false;
2191
2192 switch (F->getIntrinsicID()) {
2193 default: break;
2194 case Intrinsic::powi:
2195 case Intrinsic::sin:
2196 case Intrinsic::cos:
2197 case Intrinsic::sincos:
2198 case Intrinsic::pow:
2199 case Intrinsic::log:
2200 case Intrinsic::log10:
2201 case Intrinsic::log2:
2202 case Intrinsic::exp:
2203 case Intrinsic::exp2:
2204 return true;
2205 case Intrinsic::sqrt:
2206 case Intrinsic::fabs:
2207 case Intrinsic::copysign:
2208 case Intrinsic::floor:
2209 case Intrinsic::ceil:
2210 case Intrinsic::trunc:
2211 case Intrinsic::rint:
2212 case Intrinsic::nearbyint:
2213 case Intrinsic::round:
2214 case Intrinsic::canonicalize:
2215 case Intrinsic::lround:
2216 case Intrinsic::llround:
2217 case Intrinsic::lrint:
2218 case Intrinsic::llrint:
2219 if (F->getReturnType()->isDoubleTy() && !ST->hasFP64())
2220 return true;
2221 if (F->getReturnType()->isHalfTy() && !ST->hasFullFP16())
2222 return true;
2223 // Some operations can be handled by vector instructions and assume
2224 // unsupported vectors will be expanded into supported scalar ones.
2225 // TODO Handle scalar operations properly.
2226 return !ST->hasFPARMv8Base() && !ST->hasVFP2Base();
2227 case Intrinsic::masked_store:
2228 case Intrinsic::masked_load:
2229 case Intrinsic::masked_gather:
2230 case Intrinsic::masked_scatter:
2231 return !ST->hasMVEIntegerOps();
2232 case Intrinsic::sadd_with_overflow:
2233 case Intrinsic::uadd_with_overflow:
2234 case Intrinsic::ssub_with_overflow:
2235 case Intrinsic::usub_with_overflow:
2236 case Intrinsic::sadd_sat:
2237 case Intrinsic::uadd_sat:
2238 case Intrinsic::ssub_sat:
2239 case Intrinsic::usub_sat:
2240 return false;
2241 }
2242
2243 return BaseT::isLoweredToCall(F);
2244}
2245
2246bool ARMTTIImpl::maybeLoweredToCall(Instruction &I) const {
2247 unsigned ISD = TLI->InstructionOpcodeToISD(Opcode: I.getOpcode());
2248 EVT VT = TLI->getValueType(DL, Ty: I.getType(), AllowUnknown: true);
2249 if (TLI->getOperationAction(Op: ISD, VT) == TargetLowering::LibCall)
2250 return true;
2251
2252 // Check if an intrinsic will be lowered to a call and assume that any
2253 // other CallInst will generate a bl.
2254 if (auto *Call = dyn_cast<CallInst>(Val: &I)) {
2255 if (auto *II = dyn_cast<IntrinsicInst>(Val: Call)) {
2256 switch(II->getIntrinsicID()) {
2257 case Intrinsic::memcpy:
2258 case Intrinsic::memset:
2259 case Intrinsic::memmove:
2260 return getNumMemOps(I: II) == -1;
2261 default:
2262 if (const Function *F = Call->getCalledFunction())
2263 return isLoweredToCall(F);
2264 }
2265 }
2266 return true;
2267 }
2268
2269 // FPv5 provides conversions between integer, double-precision,
2270 // single-precision, and half-precision formats.
2271 switch (I.getOpcode()) {
2272 default:
2273 break;
2274 case Instruction::FPToSI:
2275 case Instruction::FPToUI:
2276 case Instruction::SIToFP:
2277 case Instruction::UIToFP:
2278 case Instruction::FPTrunc:
2279 case Instruction::FPExt:
2280 return !ST->hasFPARMv8Base();
2281 }
2282
2283 // FIXME: Unfortunately the approach of checking the Operation Action does
2284 // not catch all cases of Legalization that use library calls. Our
2285 // Legalization step categorizes some transformations into library calls as
2286 // Custom, Expand or even Legal when doing type legalization. So for now
2287 // we have to special case for instance the SDIV of 64bit integers and the
2288 // use of floating point emulation.
2289 if (VT.isInteger() && VT.getSizeInBits() >= 64) {
2290 switch (ISD) {
2291 default:
2292 break;
2293 case ISD::SDIV:
2294 case ISD::UDIV:
2295 case ISD::SREM:
2296 case ISD::UREM:
2297 case ISD::SDIVREM:
2298 case ISD::UDIVREM:
2299 return true;
2300 }
2301 }
2302
2303 // Assume all other non-float operations are supported.
2304 if (!VT.isFloatingPoint())
2305 return false;
2306
2307 // We'll need a library call to handle most floats when using soft.
2308 if (TLI->useSoftFloat()) {
2309 switch (I.getOpcode()) {
2310 default:
2311 return true;
2312 case Instruction::Alloca:
2313 case Instruction::Load:
2314 case Instruction::Store:
2315 case Instruction::Select:
2316 case Instruction::PHI:
2317 return false;
2318 }
2319 }
2320
2321 // We'll need a libcall to perform double precision operations on a single
2322 // precision only FPU.
2323 if (I.getType()->isDoubleTy() && !ST->hasFP64())
2324 return true;
2325
2326 // Likewise for half precision arithmetic.
2327 if (I.getType()->isHalfTy() && !ST->hasFullFP16())
2328 return true;
2329
2330 return false;
2331}
2332
2333bool ARMTTIImpl::isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE,
2334 AssumptionCache &AC,
2335 TargetLibraryInfo *LibInfo,
2336 HardwareLoopInfo &HWLoopInfo) const {
2337 // Low-overhead branches are only supported in the 'low-overhead branch'
2338 // extension of v8.1-m.
2339 if (!ST->hasLOB() || DisableLowOverheadLoops) {
2340 LLVM_DEBUG(dbgs() << "ARMHWLoops: Disabled\n");
2341 return false;
2342 }
2343
2344 if (!SE.hasLoopInvariantBackedgeTakenCount(L)) {
2345 LLVM_DEBUG(dbgs() << "ARMHWLoops: No BETC\n");
2346 return false;
2347 }
2348
2349 const SCEV *BackedgeTakenCount = SE.getBackedgeTakenCount(L);
2350 if (isa<SCEVCouldNotCompute>(Val: BackedgeTakenCount)) {
2351 LLVM_DEBUG(dbgs() << "ARMHWLoops: Uncomputable BETC\n");
2352 return false;
2353 }
2354
2355 const SCEV *TripCountSCEV =
2356 SE.getAddExpr(LHS: BackedgeTakenCount,
2357 RHS: SE.getOne(Ty: BackedgeTakenCount->getType()));
2358
2359 // We need to store the trip count in LR, a 32-bit register.
2360 if (SE.getUnsignedRangeMax(S: TripCountSCEV).getBitWidth() > 32) {
2361 LLVM_DEBUG(dbgs() << "ARMHWLoops: Trip count does not fit into 32bits\n");
2362 return false;
2363 }
2364
2365 // Making a call will trash LR and clear LO_BRANCH_INFO, so there's little
2366 // point in generating a hardware loop if that's going to happen.
2367
2368 auto IsHardwareLoopIntrinsic = [](Instruction &I) {
2369 if (auto *Call = dyn_cast<IntrinsicInst>(Val: &I)) {
2370 switch (Call->getIntrinsicID()) {
2371 default:
2372 break;
2373 case Intrinsic::start_loop_iterations:
2374 case Intrinsic::test_start_loop_iterations:
2375 case Intrinsic::loop_decrement:
2376 case Intrinsic::loop_decrement_reg:
2377 return true;
2378 }
2379 }
2380 return false;
2381 };
2382
2383 // Scan the instructions to see if there's any that we know will turn into a
2384 // call or if this loop is already a low-overhead loop or will become a tail
2385 // predicated loop.
2386 bool IsTailPredLoop = false;
2387 auto ScanLoop = [&](Loop *L) {
2388 for (auto *BB : L->getBlocks()) {
2389 for (auto &I : *BB) {
2390 if (maybeLoweredToCall(I) || IsHardwareLoopIntrinsic(I) ||
2391 isa<InlineAsm>(Val: I)) {
2392 LLVM_DEBUG(dbgs() << "ARMHWLoops: Bad instruction: " << I << "\n");
2393 return false;
2394 }
2395 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
2396 IsTailPredLoop |=
2397 II->getIntrinsicID() == Intrinsic::get_active_lane_mask ||
2398 II->getIntrinsicID() == Intrinsic::arm_mve_vctp8 ||
2399 II->getIntrinsicID() == Intrinsic::arm_mve_vctp16 ||
2400 II->getIntrinsicID() == Intrinsic::arm_mve_vctp32 ||
2401 II->getIntrinsicID() == Intrinsic::arm_mve_vctp64;
2402 }
2403 }
2404 return true;
2405 };
2406
2407 // Visit inner loops.
2408 for (auto *Inner : *L)
2409 if (!ScanLoop(Inner))
2410 return false;
2411
2412 if (!ScanLoop(L))
2413 return false;
2414
2415 // TODO: Check whether the trip count calculation is expensive. If L is the
2416 // inner loop but we know it has a low trip count, calculating that trip
2417 // count (in the parent loop) may be detrimental.
2418
2419 LLVMContext &C = L->getHeader()->getContext();
2420 HWLoopInfo.CounterInReg = true;
2421 HWLoopInfo.IsNestingLegal = false;
2422 HWLoopInfo.PerformEntryTest = AllowWLSLoops && !IsTailPredLoop;
2423 HWLoopInfo.CountType = Type::getInt32Ty(C);
2424 HWLoopInfo.LoopDecrement = ConstantInt::get(Ty: HWLoopInfo.CountType, V: 1);
2425 return true;
2426}
2427
2428static bool canTailPredicateInstruction(Instruction &I, int &ICmpCount) {
2429 // We don't allow icmp's, and because we only look at single block loops,
2430 // we simply count the icmps, i.e. there should only be 1 for the backedge.
2431 if (isa<ICmpInst>(Val: &I) && ++ICmpCount > 1)
2432 return false;
2433 // FIXME: This is a workaround for poor cost modelling. Min/Max intrinsics are
2434 // not currently canonical, but soon will be. Code without them uses icmp, and
2435 // so is not tail predicated as per the condition above. In order to get the
2436 // same performance we treat min and max the same as an icmp for tailpred
2437 // purposes for the moment (we often rely on non-tailpred and higher VF's to
2438 // pick more optimal instructions like VQDMULH. They need to be recognized
2439 // directly by the vectorizer).
2440 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I))
2441 if ((II->getIntrinsicID() == Intrinsic::smin ||
2442 II->getIntrinsicID() == Intrinsic::smax ||
2443 II->getIntrinsicID() == Intrinsic::umin ||
2444 II->getIntrinsicID() == Intrinsic::umax) &&
2445 ++ICmpCount > 1)
2446 return false;
2447
2448 if (isa<FCmpInst>(Val: &I))
2449 return false;
2450
2451 // We could allow extending/narrowing FP loads/stores, but codegen is
2452 // too inefficient so reject this for now.
2453 if (isa<FPExtInst>(Val: &I) || isa<FPTruncInst>(Val: &I))
2454 return false;
2455
2456 // Extends have to be extending-loads
2457 if (isa<SExtInst>(Val: &I) || isa<ZExtInst>(Val: &I) )
2458 if (!I.getOperand(i: 0)->hasOneUse() || !isa<LoadInst>(Val: I.getOperand(i: 0)))
2459 return false;
2460
2461 // Truncs have to be narrowing-stores
2462 if (isa<TruncInst>(Val: &I) )
2463 if (!I.hasOneUse() || !isa<StoreInst>(Val: *I.user_begin()))
2464 return false;
2465
2466 return true;
2467}
2468
2469// To set up a tail-predicated loop, we need to know the total number of
2470// elements processed by that loop. Thus, we need to determine the element
2471// size and:
2472// 1) it should be uniform for all operations in the vector loop, so we
2473// e.g. don't want any widening/narrowing operations.
2474// 2) it should be smaller than i64s because we don't have vector operations
2475// that work on i64s.
2476// 3) we don't want elements to be reversed or shuffled, to make sure the
2477// tail-predication masks/predicates the right lanes.
2478//
2479static bool canTailPredicateLoop(Loop *L, LoopInfo *LI, ScalarEvolution &SE,
2480 const DataLayout &DL,
2481 const LoopAccessInfo *LAI,
2482 const DominatorTree &DT) {
2483 LLVM_DEBUG(dbgs() << "Tail-predication: checking allowed instructions\n");
2484
2485 // If there are live-out values, it is probably a reduction. We can predicate
2486 // most reduction operations freely under MVE using a combination of
2487 // prefer-predicated-reduction-select and inloop reductions. We limit this to
2488 // floating point and integer reductions, but don't check for operators
2489 // specifically here. If the value ends up not being a reduction (and so the
2490 // vectorizer cannot tailfold the loop), we should fall back to standard
2491 // vectorization automatically.
2492 SmallVector< Instruction *, 8 > LiveOuts;
2493 LiveOuts = llvm::findDefsUsedOutsideOfLoop(L);
2494 bool ReductionsDisabled =
2495 EnableTailPredication == TailPredication::EnabledNoReductions ||
2496 EnableTailPredication == TailPredication::ForceEnabledNoReductions;
2497
2498 for (auto *I : LiveOuts) {
2499 if (!I->getType()->isIntegerTy() && !I->getType()->isFloatTy() &&
2500 !I->getType()->isHalfTy()) {
2501 LLVM_DEBUG(dbgs() << "Don't tail-predicate loop with non-integer/float "
2502 "live-out value\n");
2503 return false;
2504 }
2505 if (ReductionsDisabled) {
2506 LLVM_DEBUG(dbgs() << "Reductions not enabled\n");
2507 return false;
2508 }
2509 }
2510
2511 // Next, check that all instructions can be tail-predicated.
2512 PredicatedScalarEvolution PSE = LAI->getPSE();
2513 int ICmpCount = 0;
2514
2515 for (BasicBlock *BB : L->blocks()) {
2516 for (Instruction &I : *BB) {
2517 if (isa<PHINode>(Val: I) || isa<PseudoProbeInst>(Val: I))
2518 continue;
2519 if (!canTailPredicateInstruction(I, ICmpCount)) {
2520 LLVM_DEBUG(dbgs() << "Instruction not allowed: "; I.dump());
2521 return false;
2522 }
2523
2524 Type *T = I.getType();
2525 if (T->getScalarSizeInBits() > 32) {
2526 LLVM_DEBUG(dbgs() << "Unsupported Type: "; T->dump());
2527 return false;
2528 }
2529 if (isa<StoreInst>(Val: I) || isa<LoadInst>(Val: I)) {
2530 Value *Ptr = getLoadStorePointerOperand(V: &I);
2531 Type *AccessTy = getLoadStoreType(I: &I);
2532 int64_t NextStride =
2533 getPtrStride(PSE, AccessTy, Ptr, Lp: L, DT).value_or(u: 0);
2534 if (NextStride == 1) {
2535 // TODO: for now only allow consecutive strides of 1. We could support
2536 // other strides as long as it is uniform, but let's keep it simple
2537 // for now.
2538 continue;
2539 } else if (NextStride == -1 ||
2540 (NextStride == 2 && MVEMaxSupportedInterleaveFactor >= 2) ||
2541 (NextStride == 4 && MVEMaxSupportedInterleaveFactor >= 4)) {
2542 LLVM_DEBUG(dbgs()
2543 << "Consecutive strides of 2 found, vld2/vstr2 can't "
2544 "be tail-predicated\n.");
2545 return false;
2546 // TODO: don't tail predicate if there is a reversed load?
2547 } else if (EnableMaskedGatherScatters) {
2548 // Gather/scatters do allow loading from arbitrary strides, at
2549 // least if they are loop invariant.
2550 // TODO: Loop variant strides should in theory work, too, but
2551 // this requires further testing.
2552 const SCEV *PtrScev = PSE.getSE()->getSCEV(V: Ptr);
2553 if (auto AR = dyn_cast<SCEVAddRecExpr>(Val: PtrScev)) {
2554 const SCEV *Step = AR->getStepRecurrence(SE&: *PSE.getSE());
2555 if (PSE.getSE()->isLoopInvariant(S: Step, L))
2556 continue;
2557 }
2558 }
2559 LLVM_DEBUG(dbgs() << "Bad stride found, can't "
2560 "tail-predicate\n.");
2561 return false;
2562 }
2563 }
2564 }
2565
2566 LLVM_DEBUG(dbgs() << "tail-predication: all instructions allowed!\n");
2567 return true;
2568}
2569
2570bool ARMTTIImpl::preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
2571 if (!EnableTailPredication) {
2572 LLVM_DEBUG(dbgs() << "Tail-folding not enabled.\n");
2573 return false;
2574 }
2575
2576 // Creating a tail-folded vector loop is the first step for generating a
2577 // tail-folded hardware loop, for which we need the MVE masked
2578 // load/stores instructions:
2579 if (!ST->hasMVEIntegerOps())
2580 return false;
2581
2582 LoopVectorizationLegality *LVL = TFI->LVL;
2583 Loop *L = LVL->getLoop();
2584
2585 // For now, restrict this to single block loops.
2586 if (L->getNumBlocks() > 1) {
2587 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: not a single block "
2588 "loop.\n");
2589 return false;
2590 }
2591
2592 assert(L->isInnermost() &&
2593 "preferTailFoldingOverEpilogue: inner-loop expected");
2594
2595 LoopInfo *LI = LVL->getLoopInfo();
2596 HardwareLoopInfo HWLoopInfo(L);
2597 if (!HWLoopInfo.canAnalyze(LI&: *LI)) {
2598 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2599 "analyzable.\n");
2600 return false;
2601 }
2602
2603 AssumptionCache *AC = LVL->getAssumptionCache();
2604 ScalarEvolution *SE = LVL->getScalarEvolution();
2605
2606 // This checks if we have the low-overhead branch architecture
2607 // extension, and if we will create a hardware-loop:
2608 if (!isHardwareLoopProfitable(L, SE&: *SE, AC&: *AC, LibInfo: TFI->TLI, HWLoopInfo)) {
2609 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2610 "profitable.\n");
2611 return false;
2612 }
2613
2614 DominatorTree *DT = LVL->getDominatorTree();
2615 if (!HWLoopInfo.isHardwareLoopCandidate(SE&: *SE, LI&: *LI, DT&: *DT)) {
2616 LLVM_DEBUG(dbgs() << "preferTailFoldingOverEpilogue: hardware-loop is not "
2617 "a candidate.\n");
2618 return false;
2619 }
2620
2621 return canTailPredicateLoop(L, LI, SE&: *SE, DL, LAI: LVL->getLAI(),
2622 DT: *LVL->getDominatorTree());
2623}
2624
2625TailFoldingStyle ARMTTIImpl::getPreferredTailFoldingStyle() const {
2626 if (!ST->hasMVEIntegerOps() || !EnableTailPredication)
2627 return TailFoldingStyle::DataWithoutLaneMask;
2628
2629 // Intrinsic @llvm.get.active.lane.mask is supported.
2630 // It is used in the MVETailPredication pass, which requires the number of
2631 // elements processed by this vector loop to setup the tail-predicated
2632 // loop.
2633 return TailFoldingStyle::Data;
2634}
2635void ARMTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
2636 TTI::UnrollingPreferences &UP,
2637 OptimizationRemarkEmitter *ORE) const {
2638 // Enable Upper bound unrolling universally, providing that we do not see an
2639 // active lane mask, which will be better kept as a loop to become tail
2640 // predicated than to be conditionally unrolled.
2641 UP.UpperBound =
2642 !ST->hasMVEIntegerOps() || !any_of(Range&: *L->getHeader(), P: [](Instruction &I) {
2643 return isa<IntrinsicInst>(Val: I) &&
2644 cast<IntrinsicInst>(Val&: I).getIntrinsicID() ==
2645 Intrinsic::get_active_lane_mask;
2646 });
2647
2648 // Only currently enable these preferences for M-Class cores.
2649 if (!ST->isMClass())
2650 return BasicTTIImplBase::getUnrollingPreferences(L, SE, UP, ORE);
2651
2652 // Disable loop unrolling for Oz and Os.
2653 UP.OptSizeThreshold = 0;
2654 UP.PartialOptSizeThreshold = 0;
2655 if (L->getHeader()->getParent()->hasOptSize())
2656 return;
2657
2658 SmallVector<BasicBlock*, 4> ExitingBlocks;
2659 L->getExitingBlocks(ExitingBlocks);
2660 LLVM_DEBUG(dbgs() << "Loop has:\n"
2661 << "Blocks: " << L->getNumBlocks() << "\n"
2662 << "Exit blocks: " << ExitingBlocks.size() << "\n");
2663
2664 // Only allow another exit other than the latch. This acts as an early exit
2665 // as it mirrors the profitability calculation of the runtime unroller.
2666 if (ExitingBlocks.size() > 2)
2667 return;
2668
2669 // Limit the CFG of the loop body for targets with a branch predictor.
2670 // Allowing 4 blocks permits if-then-else diamonds in the body.
2671 if (ST->hasBranchPredictor() && L->getNumBlocks() > 4)
2672 return;
2673
2674 // Don't unroll vectorized loops, including the remainder loop
2675 if (getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized"))
2676 return;
2677
2678 // Scan the loop: don't unroll loops with calls as this could prevent
2679 // inlining.
2680 InstructionCost Cost = 0;
2681 for (auto *BB : L->getBlocks()) {
2682 for (auto &I : *BB) {
2683 // Don't unroll vectorised loop. MVE does not benefit from it as much as
2684 // scalar code.
2685 if (I.getType()->isVectorTy())
2686 return;
2687
2688 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I)) {
2689 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction()) {
2690 if (!isLoweredToCall(F))
2691 continue;
2692 }
2693 return;
2694 }
2695
2696 SmallVector<const Value*, 4> Operands(I.operand_values());
2697 Cost += getInstructionCost(U: &I, Operands,
2698 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
2699 }
2700 }
2701
2702 // On v6m cores, there are very few registers available. We can easily end up
2703 // spilling and reloading more registers in an unrolled loop. Look at the
2704 // number of LCSSA phis as a rough measure of how many registers will need to
2705 // be live out of the loop, reducing the default unroll count if more than 1
2706 // value is needed. In the long run, all of this should be being learnt by a
2707 // machine.
2708 unsigned UnrollCount = 4;
2709 if (ST->isThumb1Only()) {
2710 unsigned ExitingValues = 0;
2711 SmallVector<BasicBlock *, 4> ExitBlocks;
2712 L->getExitBlocks(ExitBlocks);
2713 for (auto *Exit : ExitBlocks) {
2714 // Count the number of LCSSA phis. Exclude values coming from GEP's as
2715 // only the last is expected to be needed for address operands.
2716 unsigned LiveOuts = count_if(Range: Exit->phis(), P: [](auto &PH) {
2717 return PH.getNumOperands() != 1 ||
2718 !isa<GetElementPtrInst>(PH.getOperand(0));
2719 });
2720 ExitingValues = ExitingValues < LiveOuts ? LiveOuts : ExitingValues;
2721 }
2722 if (ExitingValues)
2723 UnrollCount /= ExitingValues;
2724 if (UnrollCount <= 1)
2725 return;
2726 }
2727
2728 // For processors with low overhead branching (LOB), runtime unrolling the
2729 // innermost loop is often detrimental to performance. In these cases the loop
2730 // remainder gets unrolled into a series of compare-and-jump blocks, which in
2731 // deeply nested loops get executed multiple times, negating the benefits of
2732 // LOB. This is particularly noticeable when the loop trip count of the
2733 // innermost loop varies within the outer loop, such as in the case of
2734 // triangular matrix decompositions. In these cases we will prefer to not
2735 // unroll the innermost loop, with the intention for it to be executed as a
2736 // low overhead loop.
2737 bool Runtime = true;
2738 if (ST->hasLOB()) {
2739 if (SE.hasLoopInvariantBackedgeTakenCount(L)) {
2740 const SCEV *BETC = SE.getBackedgeTakenCount(L);
2741 auto *Outer = L->getOutermostLoop();
2742 if ((L != Outer && Outer != L->getParentLoop()) ||
2743 (L != Outer && BETC && !SE.isLoopInvariant(S: BETC, L: Outer))) {
2744 Runtime = false;
2745 }
2746 }
2747 }
2748
2749 LLVM_DEBUG(dbgs() << "Cost of loop: " << Cost << "\n");
2750 LLVM_DEBUG(dbgs() << "Default Runtime Unroll Count: " << UnrollCount << "\n");
2751
2752 UP.Partial = true;
2753 UP.Runtime = Runtime;
2754 UP.UnrollRemainder = true;
2755 UP.DefaultUnrollRuntimeCount = UnrollCount;
2756 UP.UnrollAndJam = true;
2757 UP.UnrollAndJamInnerLoopThreshold = 60;
2758
2759 // Force unrolling small loops can be very useful because of the branch
2760 // taken cost of the backedge.
2761 if (Cost < ArmForceUnrollThreshold)
2762 UP.Force = true;
2763}
2764
2765void ARMTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
2766 TTI::PeelingPreferences &PP) const {
2767 BaseT::getPeelingPreferences(L, SE, PP);
2768}
2769
2770bool ARMTTIImpl::preferInLoopReduction(RecurKind Kind, Type *Ty) const {
2771 if (!ST->hasMVEIntegerOps())
2772 return false;
2773
2774 unsigned ScalarBits = Ty->getScalarSizeInBits();
2775 switch (Kind) {
2776 case RecurKind::Add:
2777 return ScalarBits <= 64;
2778 default:
2779 return false;
2780 }
2781}
2782
2783bool ARMTTIImpl::preferPredicatedReductionSelect() const {
2784 if (!ST->hasMVEIntegerOps())
2785 return false;
2786 return true;
2787}
2788
2789InstructionCost ARMTTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
2790 StackOffset BaseOffset,
2791 bool HasBaseReg, int64_t Scale,
2792 unsigned AddrSpace) const {
2793 TargetLoweringBase::AddrMode AM;
2794 AM.BaseGV = BaseGV;
2795 AM.BaseOffs = BaseOffset.getFixed();
2796 AM.HasBaseReg = HasBaseReg;
2797 AM.Scale = Scale;
2798 AM.ScalableOffset = BaseOffset.getScalable();
2799 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AS: AddrSpace)) {
2800 if (ST->hasFPAO())
2801 return AM.Scale < 0 ? 1 : 0; // positive offsets execute faster
2802 return 0;
2803 }
2804 return InstructionCost::getInvalid();
2805}
2806
2807bool ARMTTIImpl::shouldConsiderVectorizationRegPressure() const {
2808 // MVE only has 8 vector registers, so we should consider register pressure to
2809 // avoid vectorizing when the cost of spills exceeds the gains from
2810 // vectorization.
2811 return ST->hasMVEIntegerOps();
2812}
2813
2814bool ARMTTIImpl::hasArmWideBranch(bool Thumb) const {
2815 if (Thumb) {
2816 // B.W is available in any Thumb2-supporting target, and also in every
2817 // version of Armv8-M, even Baseline which does not include the rest of
2818 // Thumb2.
2819 return ST->isThumb2() || ST->hasV8MBaselineOps();
2820 } else {
2821 // B is available in all versions of the Arm ISA, so the only question is
2822 // whether that ISA is available at all.
2823 return ST->hasARMOps();
2824 }
2825}
2826
2827/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
2828/// of the vector elements.
2829static bool areExtractExts(Value *Ext1, Value *Ext2) {
2830 using namespace PatternMatch;
2831
2832 auto areExtDoubled = [](Instruction *Ext) {
2833 return Ext->getType()->getScalarSizeInBits() ==
2834 2 * Ext->getOperand(i: 0)->getType()->getScalarSizeInBits();
2835 };
2836
2837 if (!match(V: Ext1, P: m_ZExtOrSExt(Op: m_Value())) ||
2838 !match(V: Ext2, P: m_ZExtOrSExt(Op: m_Value())) ||
2839 !areExtDoubled(cast<Instruction>(Val: Ext1)) ||
2840 !areExtDoubled(cast<Instruction>(Val: Ext2)))
2841 return false;
2842
2843 return true;
2844}
2845
2846/// Check if sinking \p I's operands to I's basic block is profitable, because
2847/// the operands can be folded into a target instruction, e.g.
2848/// sext/zext can be folded into vsubl.
2849bool ARMTTIImpl::isProfitableToSinkOperands(Instruction *I,
2850 SmallVectorImpl<Use *> &Ops) const {
2851 using namespace PatternMatch;
2852
2853 if (!I->getType()->isVectorTy())
2854 return false;
2855
2856 if (ST->hasNEON()) {
2857 switch (I->getOpcode()) {
2858 case Instruction::Sub:
2859 case Instruction::Add: {
2860 if (!areExtractExts(Ext1: I->getOperand(i: 0), Ext2: I->getOperand(i: 1)))
2861 return false;
2862 Ops.push_back(Elt: &I->getOperandUse(i: 0));
2863 Ops.push_back(Elt: &I->getOperandUse(i: 1));
2864 return true;
2865 }
2866 default:
2867 return false;
2868 }
2869 }
2870
2871 if (!ST->hasMVEIntegerOps())
2872 return false;
2873
2874 auto IsFMSMul = [&](Instruction *I) {
2875 if (!I->hasOneUse())
2876 return false;
2877 auto *Sub = cast<Instruction>(Val: *I->users().begin());
2878 return Sub->getOpcode() == Instruction::FSub && Sub->getOperand(i: 1) == I;
2879 };
2880 auto IsFMS = [&](Instruction *I) {
2881 if (match(V: I->getOperand(i: 0), P: m_FNeg(X: m_Value())) ||
2882 match(V: I->getOperand(i: 1), P: m_FNeg(X: m_Value())))
2883 return true;
2884 return false;
2885 };
2886
2887 auto IsSinker = [&](Instruction *I, int Operand) {
2888 switch (I->getOpcode()) {
2889 case Instruction::Add:
2890 case Instruction::Mul:
2891 case Instruction::FAdd:
2892 case Instruction::ICmp:
2893 case Instruction::FCmp:
2894 return true;
2895 case Instruction::FMul:
2896 return !IsFMSMul(I);
2897 case Instruction::Sub:
2898 case Instruction::FSub:
2899 case Instruction::Shl:
2900 case Instruction::LShr:
2901 case Instruction::AShr:
2902 return Operand == 1;
2903 case Instruction::Call:
2904 if (auto *II = dyn_cast<IntrinsicInst>(Val: I)) {
2905 switch (II->getIntrinsicID()) {
2906 case Intrinsic::fma:
2907 return !IsFMS(I);
2908 case Intrinsic::sadd_sat:
2909 case Intrinsic::uadd_sat:
2910 case Intrinsic::arm_mve_add_predicated:
2911 case Intrinsic::arm_mve_mul_predicated:
2912 case Intrinsic::arm_mve_qadd_predicated:
2913 case Intrinsic::arm_mve_vhadd:
2914 case Intrinsic::arm_mve_hadd_predicated:
2915 case Intrinsic::arm_mve_vqdmull:
2916 case Intrinsic::arm_mve_vqdmull_predicated:
2917 case Intrinsic::arm_mve_vqdmulh:
2918 case Intrinsic::arm_mve_qdmulh_predicated:
2919 case Intrinsic::arm_mve_vqrdmulh:
2920 case Intrinsic::arm_mve_qrdmulh_predicated:
2921 case Intrinsic::arm_mve_fma_predicated:
2922 return true;
2923 case Intrinsic::ssub_sat:
2924 case Intrinsic::usub_sat:
2925 case Intrinsic::arm_mve_sub_predicated:
2926 case Intrinsic::arm_mve_qsub_predicated:
2927 case Intrinsic::arm_mve_hsub_predicated:
2928 case Intrinsic::arm_mve_vhsub:
2929 return Operand == 1;
2930 default:
2931 return false;
2932 }
2933 }
2934 return false;
2935 default:
2936 return false;
2937 }
2938 };
2939
2940 for (auto OpIdx : enumerate(First: I->operands())) {
2941 Instruction *Op = dyn_cast<Instruction>(Val: OpIdx.value().get());
2942 // Make sure we are not already sinking this operand
2943 if (!Op || any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
2944 continue;
2945
2946 Instruction *Shuffle = Op;
2947 if (Shuffle->getOpcode() == Instruction::BitCast)
2948 Shuffle = dyn_cast<Instruction>(Val: Shuffle->getOperand(i: 0));
2949 // We are looking for a splat that can be sunk.
2950 if (!Shuffle || !match(V: Shuffle, P: m_Shuffle(v1: m_InsertElt(Val: m_Undef(), Elt: m_Value(),
2951 Idx: m_ZeroInt()),
2952 v2: m_Undef(), mask: m_ZeroMask())))
2953 continue;
2954 if (!IsSinker(I, OpIdx.index()))
2955 continue;
2956
2957 // All uses of the shuffle should be sunk to avoid duplicating it across gpr
2958 // and vector registers
2959 for (Use &U : Op->uses()) {
2960 Instruction *Insn = cast<Instruction>(Val: U.getUser());
2961 if (!IsSinker(Insn, U.getOperandNo()))
2962 return false;
2963 }
2964
2965 Ops.push_back(Elt: &Shuffle->getOperandUse(i: 0));
2966 if (Shuffle != Op)
2967 Ops.push_back(Elt: &Op->getOperandUse(i: 0));
2968 Ops.push_back(Elt: &OpIdx.value());
2969 }
2970 return true;
2971}
2972