1//===-- AArch64TargetTransformInfo.cpp - AArch64 specific TTI -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64TargetTransformInfo.h"
10#include "AArch64ExpandImm.h"
11#include "AArch64PerfectShuffle.h"
12#include "AArch64SMEAttributes.h"
13#include "MCTargetDesc/AArch64AddressingModes.h"
14#include "llvm/ADT/DenseMap.h"
15#include "llvm/Analysis/LoopInfo.h"
16#include "llvm/Analysis/TargetTransformInfo.h"
17#include "llvm/CodeGen/BasicTTIImpl.h"
18#include "llvm/CodeGen/CostTable.h"
19#include "llvm/CodeGen/TargetLowering.h"
20#include "llvm/IR/DerivedTypes.h"
21#include "llvm/IR/IntrinsicInst.h"
22#include "llvm/IR/Intrinsics.h"
23#include "llvm/IR/IntrinsicsAArch64.h"
24#include "llvm/IR/PatternMatch.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/TargetParser/AArch64TargetParser.h"
27#include "llvm/Transforms/InstCombine/InstCombiner.h"
28#include "llvm/Transforms/Utils/UnrollLoop.h"
29#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
30#include <algorithm>
31#include <optional>
32using namespace llvm;
33using namespace llvm::PatternMatch;
34
35#define DEBUG_TYPE "aarch64tti"
36
37static cl::opt<bool> EnableFalkorHWPFUnrollFix("enable-falkor-hwpf-unroll-fix",
38 cl::init(Val: true), cl::Hidden);
39
40static cl::opt<bool> SVEPreferFixedOverScalableIfEqualCost(
41 "sve-prefer-fixed-over-scalable-if-equal", cl::Hidden);
42
43static cl::opt<unsigned> SVEGatherOverhead("sve-gather-overhead", cl::init(Val: 10),
44 cl::Hidden);
45
46static cl::opt<unsigned> SVEScatterOverhead("sve-scatter-overhead",
47 cl::init(Val: 10), cl::Hidden);
48
49static cl::opt<unsigned> SVETailFoldInsnThreshold("sve-tail-folding-insn-threshold",
50 cl::init(Val: 15), cl::Hidden);
51
52static cl::opt<unsigned>
53 NeonNonConstStrideOverhead("neon-nonconst-stride-overhead", cl::init(Val: 10),
54 cl::Hidden);
55
56static cl::opt<unsigned> CallPenaltyChangeSM(
57 "call-penalty-sm-change", cl::init(Val: 5), cl::Hidden,
58 cl::desc(
59 "Penalty of calling a function that requires a change to PSTATE.SM"));
60
61static cl::opt<unsigned> InlineCallPenaltyChangeSM(
62 "inline-call-penalty-sm-change", cl::init(Val: 10), cl::Hidden,
63 cl::desc("Penalty of inlining a call that requires a change to PSTATE.SM"));
64
65static cl::opt<bool> EnableOrLikeSelectOpt("enable-aarch64-or-like-select",
66 cl::init(Val: true), cl::Hidden);
67
68static cl::opt<bool> EnableLSRCostOpt("enable-aarch64-lsr-cost-opt",
69 cl::init(Val: true), cl::Hidden);
70
71// A complete guess as to a reasonable cost.
72static cl::opt<unsigned>
73 BaseHistCntCost("aarch64-base-histcnt-cost", cl::init(Val: 8), cl::Hidden,
74 cl::desc("The cost of a histcnt instruction"));
75
76static cl::opt<unsigned> DMBLookaheadThreshold(
77 "dmb-lookahead-threshold", cl::init(Val: 10), cl::Hidden,
78 cl::desc("The number of instructions to search for a redundant dmb"));
79
80static cl::opt<int> Aarch64ForceUnrollThreshold(
81 "aarch64-force-unroll-threshold", cl::init(Val: 0), cl::Hidden,
82 cl::desc("Threshold for forced unrolling of small loops in AArch64"));
83
84namespace {
85class TailFoldingOption {
86 // These bitfields will only ever be set to something non-zero in operator=,
87 // when setting the -sve-tail-folding option. This option should always be of
88 // the form (default|simple|all|disable)[+(Flag1|Flag2|etc)], where here
89 // InitialBits is one of (disabled|all|simple). EnableBits represents
90 // additional flags we're enabling, and DisableBits for those flags we're
91 // disabling. The default flag is tracked in the variable NeedsDefault, since
92 // at the time of setting the option we may not know what the default value
93 // for the CPU is.
94 TailFoldingOpts InitialBits = TailFoldingOpts::Disabled;
95 TailFoldingOpts EnableBits = TailFoldingOpts::Disabled;
96 TailFoldingOpts DisableBits = TailFoldingOpts::Disabled;
97
98 // This value needs to be initialised to true in case the user does not
99 // explicitly set the -sve-tail-folding option.
100 bool NeedsDefault = true;
101
102 void setInitialBits(TailFoldingOpts Bits) { InitialBits = Bits; }
103
104 void setNeedsDefault(bool V) { NeedsDefault = V; }
105
106 void setEnableBit(TailFoldingOpts Bit) {
107 EnableBits |= Bit;
108 DisableBits &= ~Bit;
109 }
110
111 void setDisableBit(TailFoldingOpts Bit) {
112 EnableBits &= ~Bit;
113 DisableBits |= Bit;
114 }
115
116 TailFoldingOpts getBits(TailFoldingOpts DefaultBits) const {
117 TailFoldingOpts Bits = TailFoldingOpts::Disabled;
118
119 assert((InitialBits == TailFoldingOpts::Disabled || !NeedsDefault) &&
120 "Initial bits should only include one of "
121 "(disabled|all|simple|default)");
122 Bits = NeedsDefault ? DefaultBits : InitialBits;
123 Bits |= EnableBits;
124 Bits &= ~DisableBits;
125
126 return Bits;
127 }
128
129 void reportError(std::string Opt) {
130 errs() << "invalid argument '" << Opt
131 << "' to -sve-tail-folding=; the option should be of the form\n"
132 " (disabled|all|default|simple)[+(reductions|recurrences"
133 "|reverse|noreductions|norecurrences|noreverse)]\n";
134 report_fatal_error(reason: "Unrecognised tail-folding option");
135 }
136
137public:
138
139 void operator=(const std::string &Val) {
140 // If the user explicitly sets -sve-tail-folding= then treat as an error.
141 if (Val.empty()) {
142 reportError(Opt: "");
143 return;
144 }
145
146 // Since the user is explicitly setting the option we don't automatically
147 // need the default unless they require it.
148 setNeedsDefault(false);
149
150 SmallVector<StringRef, 4> TailFoldTypes;
151 StringRef(Val).split(A&: TailFoldTypes, Separator: '+', MaxSplit: -1, KeepEmpty: false);
152
153 unsigned StartIdx = 1;
154 if (TailFoldTypes[0] == "disabled")
155 setInitialBits(TailFoldingOpts::Disabled);
156 else if (TailFoldTypes[0] == "all")
157 setInitialBits(TailFoldingOpts::All);
158 else if (TailFoldTypes[0] == "default")
159 setNeedsDefault(true);
160 else if (TailFoldTypes[0] == "simple")
161 setInitialBits(TailFoldingOpts::Simple);
162 else {
163 StartIdx = 0;
164 setInitialBits(TailFoldingOpts::Disabled);
165 }
166
167 for (unsigned I = StartIdx; I < TailFoldTypes.size(); I++) {
168 if (TailFoldTypes[I] == "reductions")
169 setEnableBit(TailFoldingOpts::Reductions);
170 else if (TailFoldTypes[I] == "recurrences")
171 setEnableBit(TailFoldingOpts::Recurrences);
172 else if (TailFoldTypes[I] == "reverse")
173 setEnableBit(TailFoldingOpts::Reverse);
174 else if (TailFoldTypes[I] == "noreductions")
175 setDisableBit(TailFoldingOpts::Reductions);
176 else if (TailFoldTypes[I] == "norecurrences")
177 setDisableBit(TailFoldingOpts::Recurrences);
178 else if (TailFoldTypes[I] == "noreverse")
179 setDisableBit(TailFoldingOpts::Reverse);
180 else
181 reportError(Opt: Val);
182 }
183 }
184
185 bool satisfies(TailFoldingOpts DefaultBits, TailFoldingOpts Required) const {
186 return (getBits(DefaultBits) & Required) == Required;
187 }
188};
189} // namespace
190
191TailFoldingOption TailFoldingOptionLoc;
192
193static cl::opt<TailFoldingOption, true, cl::parser<std::string>> SVETailFolding(
194 "sve-tail-folding",
195 cl::desc(
196 "Control the use of vectorisation using tail-folding for SVE where the"
197 " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:"
198 "\ndisabled (Initial) No loop types will vectorize using "
199 "tail-folding"
200 "\ndefault (Initial) Uses the default tail-folding settings for "
201 "the target CPU"
202 "\nall (Initial) All legal loop types will vectorize using "
203 "tail-folding"
204 "\nsimple (Initial) Use tail-folding for simple loops (not "
205 "reductions or recurrences)"
206 "\nreductions Use tail-folding for loops containing reductions"
207 "\nnoreductions Inverse of above"
208 "\nrecurrences Use tail-folding for loops containing fixed order "
209 "recurrences"
210 "\nnorecurrences Inverse of above"
211 "\nreverse Use tail-folding for loops requiring reversed "
212 "predicates"
213 "\nnoreverse Inverse of above"),
214 cl::location(L&: TailFoldingOptionLoc));
215
216// Experimental option that will only be fully functional when the
217// code-generator is changed to use SVE instead of NEON for all fixed-width
218// operations.
219static cl::opt<bool> EnableFixedwidthAutovecInStreamingMode(
220 "enable-fixedwidth-autovec-in-streaming-mode", cl::init(Val: false), cl::Hidden);
221
222// Experimental option that will only be fully functional when the cost-model
223// and code-generator have been changed to avoid using scalable vector
224// instructions that are not legal in streaming SVE mode.
225static cl::opt<bool> EnableScalableAutovecInStreamingMode(
226 "enable-scalable-autovec-in-streaming-mode", cl::init(Val: false), cl::Hidden);
227
228static bool isSMEABIRoutineCall(const CallInst &CI,
229 const AArch64TargetLowering &TLI) {
230 const auto *F = CI.getCalledFunction();
231 return F &&
232 SMEAttrs(F->getName(), TLI.getRuntimeLibcallsInfo()).isSMEABIRoutine();
233}
234
235/// Returns true if the function has explicit operations that can only be
236/// lowered using incompatible instructions for the selected mode. This also
237/// returns true if the function F may use or modify ZA state.
238static bool hasPossibleIncompatibleOps(const Function *F,
239 const AArch64TargetLowering &TLI) {
240 for (const BasicBlock &BB : *F) {
241 for (const Instruction &I : BB) {
242 // Be conservative for now and assume that any call to inline asm or to
243 // intrinsics could could result in non-streaming ops (e.g. calls to
244 // @llvm.aarch64.* or @llvm.gather/scatter intrinsics). We can assume that
245 // all native LLVM instructions can be lowered to compatible instructions.
246 if (isa<CallInst>(Val: I) && !I.isDebugOrPseudoInst() &&
247 (cast<CallInst>(Val: I).isInlineAsm() || isa<IntrinsicInst>(Val: I) ||
248 isSMEABIRoutineCall(CI: cast<CallInst>(Val: I), TLI)))
249 return true;
250 }
251 }
252 return false;
253}
254
255static void extractAttrFeatures(const Function &F, const AArch64TTIImpl *TTI,
256 SmallVectorImpl<StringRef> &Features) {
257 StringRef AttributeStr =
258 TTI->isMultiversionedFunction(F) ? "fmv-features" : "target-features";
259 StringRef FeatureStr = F.getFnAttribute(Kind: AttributeStr).getValueAsString();
260 FeatureStr.split(A&: Features, Separator: ",");
261}
262
263APInt AArch64TTIImpl::getFeatureMask(const Function &F) const {
264 SmallVector<StringRef, 8> Features;
265 extractAttrFeatures(F, TTI: this, Features);
266 return AArch64::getCpuSupportsMask(Features);
267}
268
269APInt AArch64TTIImpl::getPriorityMask(const Function &F) const {
270 SmallVector<StringRef, 8> Features;
271 extractAttrFeatures(F, TTI: this, Features);
272 return AArch64::getFMVPriority(Features);
273}
274
275bool AArch64TTIImpl::isMultiversionedFunction(const Function &F) const {
276 return F.hasFnAttribute(Kind: "fmv-features");
277}
278
279bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
280 const Function *Callee) const {
281 SMECallAttrs CallAttrs(*Caller, *Callee);
282
283 // Never inline a function explicitly marked as being streaming,
284 // into a non-streaming function. Assume it was marked as streaming
285 // for a reason.
286 if (CallAttrs.caller().hasNonStreamingInterfaceAndBody() &&
287 CallAttrs.callee().hasStreamingInterfaceOrBody())
288 return false;
289
290 // When inlining, we should consider the body of the function, not the
291 // interface.
292 if (CallAttrs.callee().hasStreamingBody()) {
293 CallAttrs.callee().set(M: SMEAttrs::SM_Compatible, Enable: false);
294 CallAttrs.callee().set(M: SMEAttrs::SM_Enabled, Enable: true);
295 }
296
297 if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0())
298 return false;
299
300 if (CallAttrs.requiresLazySave() || CallAttrs.requiresSMChange() ||
301 CallAttrs.requiresPreservingZT0() ||
302 CallAttrs.requiresPreservingAllZAState()) {
303 if (hasPossibleIncompatibleOps(F: Callee, TLI: *getTLI()))
304 return false;
305 }
306
307 return BaseT::areInlineCompatible(Caller, Callee);
308}
309
310bool AArch64TTIImpl::areTypesABICompatible(const Function *Caller,
311 const Function *Callee,
312 ArrayRef<Type *> Types) const {
313 if (!BaseT::areTypesABICompatible(Caller, Callee, Types))
314 return false;
315
316 // We need to ensure that argument promotion does not attempt to promote
317 // pointers to fixed-length vector types larger than 128 bits like
318 // <8 x float> (and pointers to aggregate types which have such fixed-length
319 // vector type members) into the values of the pointees. Such vector types
320 // are used for SVE VLS but there is no ABI for SVE VLS arguments and the
321 // backend cannot lower such value arguments. The 128-bit fixed-length SVE
322 // types can be safely treated as 128-bit NEON types and they cannot be
323 // distinguished in IR.
324 if (ST->useSVEForFixedLengthVectors() && llvm::any_of(Range&: Types, P: [](Type *Ty) {
325 auto FVTy = dyn_cast<FixedVectorType>(Val: Ty);
326 return FVTy &&
327 FVTy->getScalarSizeInBits() * FVTy->getNumElements() > 128;
328 }))
329 return false;
330
331 return true;
332}
333
334unsigned
335AArch64TTIImpl::getInlineCallPenalty(const Function *F, const CallBase &Call,
336 unsigned DefaultCallPenalty) const {
337 // This function calculates a penalty for executing Call in F.
338 //
339 // There are two ways this function can be called:
340 // (1) F:
341 // call from F -> G (the call here is Call)
342 //
343 // For (1), Call.getCaller() == F, so it will always return a high cost if
344 // a streaming-mode change is required (thus promoting the need to inline the
345 // function)
346 //
347 // (2) F:
348 // call from F -> G (the call here is not Call)
349 // G:
350 // call from G -> H (the call here is Call)
351 //
352 // For (2), if after inlining the body of G into F the call to H requires a
353 // streaming-mode change, and the call to G from F would also require a
354 // streaming-mode change, then there is benefit to do the streaming-mode
355 // change only once and avoid inlining of G into F.
356
357 SMEAttrs FAttrs(*F);
358 SMECallAttrs CallAttrs(Call, &getTLI()->getRuntimeLibcallsInfo());
359
360 if (SMECallAttrs(FAttrs, CallAttrs.callee()).requiresSMChange()) {
361 if (F == Call.getCaller()) // (1)
362 return CallPenaltyChangeSM * DefaultCallPenalty;
363 if (SMECallAttrs(FAttrs, CallAttrs.caller()).requiresSMChange()) // (2)
364 return InlineCallPenaltyChangeSM * DefaultCallPenalty;
365 }
366
367 return DefaultCallPenalty;
368}
369
370bool AArch64TTIImpl::shouldMaximizeVectorBandwidth(
371 TargetTransformInfo::RegisterKind K) const {
372 assert(K != TargetTransformInfo::RGK_Scalar);
373
374 if (K == TargetTransformInfo::RGK_FixedWidthVector && ST->isNeonAvailable())
375 return true;
376
377 return K == TargetTransformInfo::RGK_ScalableVector &&
378 ST->isSVEorStreamingSVEAvailable() &&
379 !ST->disableMaximizeScalableBandwidth();
380}
381
382/// Calculate the cost of materializing a 64-bit value. This helper
383/// method might only calculate a fraction of a larger immediate. Therefore it
384/// is valid to return a cost of ZERO.
385InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) const {
386 // Check if the immediate can be encoded within an instruction.
387 if (Val == 0 || AArch64_AM::isLogicalImmediate(imm: Val, regSize: 64))
388 return 0;
389
390 if (Val < 0)
391 Val = ~Val;
392
393 // Calculate how many moves we will need to materialize this constant.
394 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
395 AArch64_IMM::expandMOVImm(Imm: Val, BitSize: 64, Insn);
396 return Insn.size();
397}
398
399/// Calculate the cost of materializing the given constant.
400InstructionCost
401AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
402 TTI::TargetCostKind CostKind) const {
403 assert(Ty->isIntegerTy());
404
405 unsigned BitSize = Ty->getPrimitiveSizeInBits();
406 if (BitSize == 0)
407 return ~0U;
408
409 // Sign-extend all constants to a multiple of 64-bit.
410 APInt ImmVal = Imm;
411 if (BitSize & 0x3f)
412 ImmVal = Imm.sext(width: (BitSize + 63) & ~0x3fU);
413
414 // Split the constant into 64-bit chunks and calculate the cost for each
415 // chunk.
416 InstructionCost Cost = 0;
417 for (unsigned ShiftVal = 0; ShiftVal < BitSize; ShiftVal += 64) {
418 APInt Tmp = ImmVal.ashr(ShiftAmt: ShiftVal).sextOrTrunc(width: 64);
419 int64_t Val = Tmp.getSExtValue();
420 Cost += getIntImmCost(Val);
421 }
422 // We need at least one instruction to materialze the constant.
423 return std::max<InstructionCost>(a: 1, b: Cost);
424}
425
426InstructionCost AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
427 const APInt &Imm, Type *Ty,
428 TTI::TargetCostKind CostKind,
429 Instruction *Inst) const {
430 assert(Ty->isIntegerTy());
431
432 unsigned BitSize = Ty->getPrimitiveSizeInBits();
433 // There is no cost model for constants with a bit size of 0. Return TCC_Free
434 // here, so that constant hoisting will ignore this constant.
435 if (BitSize == 0)
436 return TTI::TCC_Free;
437
438 unsigned ImmIdx = ~0U;
439 switch (Opcode) {
440 default:
441 return TTI::TCC_Free;
442 case Instruction::GetElementPtr:
443 // Always hoist the base address of a GetElementPtr.
444 if (Idx == 0)
445 return 2 * TTI::TCC_Basic;
446 return TTI::TCC_Free;
447 case Instruction::Store:
448 ImmIdx = 0;
449 break;
450 case Instruction::Add:
451 case Instruction::Sub:
452 case Instruction::Mul:
453 case Instruction::UDiv:
454 case Instruction::SDiv:
455 case Instruction::URem:
456 case Instruction::SRem:
457 case Instruction::And:
458 case Instruction::Or:
459 case Instruction::Xor:
460 case Instruction::ICmp:
461 ImmIdx = 1;
462 break;
463 // Always return TCC_Free for the shift value of a shift instruction.
464 case Instruction::Shl:
465 case Instruction::LShr:
466 case Instruction::AShr:
467 if (Idx == 1)
468 return TTI::TCC_Free;
469 break;
470 case Instruction::Trunc:
471 case Instruction::ZExt:
472 case Instruction::SExt:
473 case Instruction::IntToPtr:
474 case Instruction::PtrToInt:
475 case Instruction::BitCast:
476 case Instruction::PHI:
477 case Instruction::Call:
478 case Instruction::Select:
479 case Instruction::Ret:
480 case Instruction::Load:
481 break;
482 }
483
484 if (Idx == ImmIdx) {
485 int NumConstants = (BitSize + 63) / 64;
486 InstructionCost Cost = AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
487 return (Cost <= NumConstants * TTI::TCC_Basic)
488 ? static_cast<int>(TTI::TCC_Free)
489 : Cost;
490 }
491 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
492}
493
494InstructionCost
495AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
496 const APInt &Imm, Type *Ty,
497 TTI::TargetCostKind CostKind) const {
498 assert(Ty->isIntegerTy());
499
500 unsigned BitSize = Ty->getPrimitiveSizeInBits();
501 // There is no cost model for constants with a bit size of 0. Return TCC_Free
502 // here, so that constant hoisting will ignore this constant.
503 if (BitSize == 0)
504 return TTI::TCC_Free;
505
506 // Most (all?) AArch64 intrinsics do not support folding immediates into the
507 // selected instruction, so we compute the materialization cost for the
508 // immediate directly.
509 if (IID >= Intrinsic::aarch64_addg && IID <= Intrinsic::aarch64_udiv)
510 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
511
512 switch (IID) {
513 default:
514 return TTI::TCC_Free;
515 case Intrinsic::sadd_with_overflow:
516 case Intrinsic::uadd_with_overflow:
517 case Intrinsic::ssub_with_overflow:
518 case Intrinsic::usub_with_overflow:
519 case Intrinsic::smul_with_overflow:
520 case Intrinsic::umul_with_overflow:
521 if (Idx == 1) {
522 int NumConstants = (BitSize + 63) / 64;
523 InstructionCost Cost = AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
524 return (Cost <= NumConstants * TTI::TCC_Basic)
525 ? static_cast<int>(TTI::TCC_Free)
526 : Cost;
527 }
528 break;
529 case Intrinsic::experimental_stackmap:
530 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(x: Imm.getSExtValue())))
531 return TTI::TCC_Free;
532 break;
533 case Intrinsic::experimental_patchpoint_void:
534 case Intrinsic::experimental_patchpoint:
535 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(x: Imm.getSExtValue())))
536 return TTI::TCC_Free;
537 break;
538 case Intrinsic::experimental_gc_statepoint:
539 if ((Idx < 5) || (Imm.getBitWidth() <= 64 && isInt<64>(x: Imm.getSExtValue())))
540 return TTI::TCC_Free;
541 break;
542 }
543 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
544}
545
546TargetTransformInfo::PopcntSupportKind
547AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) const {
548 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
549 if (TyWidth == 32 || TyWidth == 64)
550 return TTI::PSK_FastHardware;
551 // TODO: AArch64TargetLowering::LowerCTPOP() supports 128bit popcount.
552 return TTI::PSK_Software;
553}
554
555InstructionCost AArch64TTIImpl::getBranchMispredictPenalty() const {
556 // MispredictPenalty is defined per-CPU in AArch64Sched*.td (e.g.,
557 // AArch64SchedNeoverseV2.td).
558 return ST->getMispredictionPenalty();
559}
560
561static bool isUnpackedVectorVT(EVT VecVT) {
562 return VecVT.isScalableVector() &&
563 VecVT.getSizeInBits().getKnownMinValue() < AArch64::SVEBitsPerBlock;
564}
565
566static InstructionCost getHistogramCost(const AArch64Subtarget *ST,
567 const IntrinsicCostAttributes &ICA) {
568 // We need to know at least the number of elements in the vector of buckets
569 // and the size of each element to update.
570 if (ICA.getArgTypes().size() < 2)
571 return InstructionCost::getInvalid();
572
573 // Only interested in costing for the hardware instruction from SVE2.
574 if (!ST->hasSVE2())
575 return InstructionCost::getInvalid();
576
577 Type *BucketPtrsTy = ICA.getArgTypes()[0]; // Type of vector of pointers
578 Type *EltTy = ICA.getArgTypes()[1]; // Type of bucket elements
579 unsigned TotalHistCnts = 1;
580
581 unsigned EltSize = EltTy->getScalarSizeInBits();
582 // Only allow (up to 64b) integers or pointers
583 if ((!EltTy->isIntegerTy() && !EltTy->isPointerTy()) || EltSize > 64)
584 return InstructionCost::getInvalid();
585
586 // FIXME: We should be able to generate histcnt for fixed-length vectors
587 // using ptrue with a specific VL.
588 if (VectorType *VTy = dyn_cast<VectorType>(Val: BucketPtrsTy)) {
589 unsigned EC = VTy->getElementCount().getKnownMinValue();
590 if (!isPowerOf2_64(Value: EC) || !VTy->isScalableTy())
591 return InstructionCost::getInvalid();
592
593 // HistCnt only supports 32b and 64b element types
594 unsigned LegalEltSize = EltSize <= 32 ? 32 : 64;
595
596 if (EC == 2 || (LegalEltSize == 32 && EC == 4))
597 return InstructionCost(BaseHistCntCost);
598
599 unsigned NaturalVectorWidth = AArch64::SVEBitsPerBlock / LegalEltSize;
600 TotalHistCnts = EC / NaturalVectorWidth;
601
602 return InstructionCost(BaseHistCntCost * TotalHistCnts);
603 }
604
605 return InstructionCost::getInvalid();
606}
607
608InstructionCost
609AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
610 TTI::TargetCostKind CostKind) const {
611 // The code-generator is currently not able to handle scalable vectors
612 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
613 // it. This change will be removed when code-generation for these types is
614 // sufficiently reliable.
615 auto *RetTy = ICA.getReturnType();
616 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: RetTy))
617 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
618 return InstructionCost::getInvalid();
619
620 switch (ICA.getID()) {
621 case Intrinsic::experimental_vector_histogram_add: {
622 InstructionCost HistCost = getHistogramCost(ST, ICA);
623 // If the cost isn't valid, we may still be able to scalarize
624 if (HistCost.isValid())
625 return HistCost;
626 break;
627 }
628 case Intrinsic::clmul: {
629 auto LT = getTypeLegalizationCost(Ty: RetTy);
630
631 // PMUL v8i8/v16i8 is always available on AArch64
632 if (ST->hasNEON()) {
633 if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
634 return LT.first;
635
636 // Scalar i8 lowers through scalar/vector moves around PMUL.
637 if (TLI->getValueType(DL, Ty: RetTy, AllowUnknown: true) == MVT::i8) {
638 auto *VecTy =
639 FixedVectorType::get(ElementType: Type::getInt8Ty(C&: RetTy->getContext()), NumElts: 8);
640 return 1 +
641 getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: VecTy, CostKind,
642 Index: -1, Op0: nullptr, Op1: nullptr) *
643 2 +
644 getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy, CostKind,
645 Index: -1, Op0: nullptr, Op1: nullptr);
646 }
647 }
648
649 if (LT.second.SimpleTy == MVT::nxv2i64)
650 if (ST->hasSVEAES() && (ST->isSVEAvailable() || ST->hasSSVE_AES()))
651 return LT.first * 3;
652
653 if (ST->hasSVE2() || ST->hasSME()) {
654 switch (LT.second.SimpleTy) {
655 case MVT::nxv16i8:
656 return LT.first;
657 case MVT::nxv8i16:
658 return LT.first * 6;
659 case MVT::nxv4i32:
660 return LT.first * 3;
661 case MVT::nxv2i64:
662 return LT.first * 8;
663 default:
664 break;
665 }
666 }
667
668 // Avoid +sve giving this cost 2 due to custom lowering: It's very slow
669 if (LT.second.SimpleTy == MVT::nxv2i64)
670 return 192;
671
672 if (ST->hasAES()) {
673 switch (LT.second.SimpleTy) {
674 case MVT::i16:
675 case MVT::i32:
676 case MVT::i64:
677 case MVT::i128: {
678 auto *VecTy =
679 FixedVectorType::get(ElementType: Type::getInt64Ty(C&: RetTy->getContext()), NumElts: 1);
680 return LT.first *
681 (1 +
682 getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: VecTy, CostKind,
683 Index: -1, Op0: nullptr, Op1: nullptr) *
684 2 +
685 getVectorInstrCost(Opcode: Instruction::InsertElement, Val: VecTy, CostKind,
686 Index: -1, Op0: nullptr, Op1: nullptr));
687 }
688 case MVT::v1i64:
689 return LT.first;
690 case MVT::v2i64:
691 return LT.first * 3;
692 case MVT::v2i32:
693 return LT.first * 6;
694 case MVT::v4i32:
695 return LT.first * 11;
696 case MVT::v4i16:
697 return LT.first * 14;
698 default:
699 break;
700 }
701 }
702 break;
703 }
704 case Intrinsic::umin:
705 case Intrinsic::umax:
706 case Intrinsic::smin:
707 case Intrinsic::smax: {
708 static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
709 MVT::v8i16, MVT::v2i32, MVT::v4i32,
710 MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32,
711 MVT::nxv2i64};
712 auto LT = getTypeLegalizationCost(Ty: RetTy);
713 // v2i64 types get converted to cmp+bif hence the cost of 2
714 if (LT.second == MVT::v2i64)
715 return LT.first * 2;
716 if (any_of(Range: ValidMinMaxTys, P: equal_to(Arg&: LT.second)))
717 return LT.first;
718 break;
719 }
720 case Intrinsic::scmp:
721 case Intrinsic::ucmp: {
722 static const CostTblEntry BitreverseTbl[] = {
723 {.ISD: Intrinsic::scmp, .Type: MVT::i32, .Cost: 3}, // cmp+cset+csinv
724 {.ISD: Intrinsic::scmp, .Type: MVT::i64, .Cost: 3}, // cmp+cset+csinv
725 {.ISD: Intrinsic::scmp, .Type: MVT::v8i8, .Cost: 3}, // cmgt+cmgt+sub
726 {.ISD: Intrinsic::scmp, .Type: MVT::v16i8, .Cost: 3}, // cmgt+cmgt+sub
727 {.ISD: Intrinsic::scmp, .Type: MVT::v4i16, .Cost: 3}, // cmgt+cmgt+sub
728 {.ISD: Intrinsic::scmp, .Type: MVT::v8i16, .Cost: 3}, // cmgt+cmgt+sub
729 {.ISD: Intrinsic::scmp, .Type: MVT::v2i32, .Cost: 3}, // cmgt+cmgt+sub
730 {.ISD: Intrinsic::scmp, .Type: MVT::v4i32, .Cost: 3}, // cmgt+cmgt+sub
731 {.ISD: Intrinsic::scmp, .Type: MVT::v1i64, .Cost: 3}, // cmgt+cmgt+sub
732 {.ISD: Intrinsic::scmp, .Type: MVT::v2i64, .Cost: 3}, // cmgt+cmgt+sub
733 };
734 const auto LT = getTypeLegalizationCost(Ty: RetTy);
735 const auto *Entry =
736 CostTableLookup(Table: BitreverseTbl, ISD: Intrinsic::scmp, Ty: LT.second);
737 if (Entry)
738 return Entry->Cost * LT.first;
739 break;
740 }
741 case Intrinsic::sadd_sat:
742 case Intrinsic::ssub_sat:
743 case Intrinsic::uadd_sat:
744 case Intrinsic::usub_sat: {
745 static const auto ValidSatTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
746 MVT::v8i16, MVT::v2i32, MVT::v4i32,
747 MVT::v2i64};
748 auto LT = getTypeLegalizationCost(Ty: RetTy);
749 // This is a base cost of 1 for the vadd, plus 3 extract shifts if we
750 // need to extend the type, as it uses shr(qadd(shl, shl)).
751 unsigned Instrs =
752 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1 : 4;
753 if (any_of(Range: ValidSatTys, P: equal_to(Arg&: LT.second)))
754 return LT.first * Instrs;
755
756 TypeSize TS = getDataLayout().getTypeSizeInBits(Ty: RetTy);
757 uint64_t VectorSize = TS.getKnownMinValue();
758
759 if (ST->isSVEAvailable() && VectorSize >= 128 && isPowerOf2_64(Value: VectorSize))
760 return LT.first * Instrs;
761
762 break;
763 }
764 case Intrinsic::abs: {
765 static const auto ValidAbsTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
766 MVT::v8i16, MVT::v2i32, MVT::v4i32,
767 MVT::v2i64, MVT::nxv16i8, MVT::nxv8i16,
768 MVT::nxv4i32, MVT::nxv2i64};
769 auto LT = getTypeLegalizationCost(Ty: RetTy);
770 if (any_of(Range: ValidAbsTys, P: equal_to(Arg&: LT.second)))
771 return LT.first;
772 break;
773 }
774 case Intrinsic::bswap: {
775 static const auto ValidAbsTys = {MVT::v4i16, MVT::v8i16, MVT::v2i32,
776 MVT::v4i32, MVT::v2i64};
777 auto LT = getTypeLegalizationCost(Ty: RetTy);
778 if (any_of(Range: ValidAbsTys, P: equal_to(Arg&: LT.second)) &&
779 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits())
780 return LT.first;
781 break;
782 }
783 case Intrinsic::fma:
784 case Intrinsic::fmuladd: {
785 // Given a fma or fmuladd, cost it the same as a fmul instruction which are
786 // usually the same for costs. TODO: Add fp16 and bf16 expansion costs.
787 Type *EltTy = RetTy->getScalarType();
788 if (EltTy->isFloatTy() || EltTy->isDoubleTy() ||
789 (EltTy->isHalfTy() && ST->hasFullFP16()))
790 return getArithmeticInstrCost(Opcode: Instruction::FMul, Ty: RetTy, CostKind);
791 break;
792 }
793 case Intrinsic::stepvector: {
794 InstructionCost Cost = 1; // Cost of the `index' instruction
795 auto LT = getTypeLegalizationCost(Ty: RetTy);
796 // Legalisation of illegal vectors involves an `index' instruction plus
797 // (LT.first - 1) vector adds.
798 if (LT.first > 1) {
799 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Context&: RetTy->getContext());
800 InstructionCost AddCost =
801 getArithmeticInstrCost(Opcode: Instruction::Add, Ty: LegalVTy, CostKind);
802 Cost += AddCost * (LT.first - 1);
803 }
804 return Cost;
805 }
806 case Intrinsic::vector_extract:
807 case Intrinsic::vector_insert: {
808 // If both the vector and subvector types are legal types and the index
809 // is 0, then this should be a no-op or simple operation; return a
810 // relatively low cost.
811
812 // If arguments aren't actually supplied, then we cannot determine the
813 // value of the index. We also want to skip predicate types.
814 if (ICA.getArgs().size() != ICA.getArgTypes().size() ||
815 ICA.getReturnType()->getScalarType()->isIntegerTy(BitWidth: 1))
816 break;
817
818 LLVMContext &C = RetTy->getContext();
819 EVT VecVT = getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[0]);
820 bool IsExtract = ICA.getID() == Intrinsic::vector_extract;
821 EVT SubVecVT = IsExtract ? getTLI()->getValueType(DL, Ty: RetTy)
822 : getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[1]);
823 // Skip this if either the vector or subvector types are unpacked
824 // SVE types; they may get lowered to stack stores and loads.
825 if (isUnpackedVectorVT(VecVT) || isUnpackedVectorVT(VecVT: SubVecVT))
826 break;
827
828 TargetLoweringBase::LegalizeKind SubVecLK =
829 getTLI()->getTypeConversion(Context&: C, VT: SubVecVT);
830 TargetLoweringBase::LegalizeKind VecLK =
831 getTLI()->getTypeConversion(Context&: C, VT: VecVT);
832 const Value *Idx = IsExtract ? ICA.getArgs()[1] : ICA.getArgs()[2];
833 const ConstantInt *CIdx = cast<ConstantInt>(Val: Idx);
834 if (SubVecLK.first == TargetLoweringBase::TypeLegal &&
835 VecLK.first == TargetLoweringBase::TypeLegal && CIdx->isZero())
836 return TTI::TCC_Free;
837 break;
838 }
839 case Intrinsic::bitreverse: {
840 static const CostTblEntry BitreverseTbl[] = {
841 {.ISD: Intrinsic::bitreverse, .Type: MVT::i32, .Cost: 1},
842 {.ISD: Intrinsic::bitreverse, .Type: MVT::i64, .Cost: 1},
843 {.ISD: Intrinsic::bitreverse, .Type: MVT::v8i8, .Cost: 1},
844 {.ISD: Intrinsic::bitreverse, .Type: MVT::v16i8, .Cost: 1},
845 {.ISD: Intrinsic::bitreverse, .Type: MVT::v4i16, .Cost: 2},
846 {.ISD: Intrinsic::bitreverse, .Type: MVT::v8i16, .Cost: 2},
847 {.ISD: Intrinsic::bitreverse, .Type: MVT::v2i32, .Cost: 2},
848 {.ISD: Intrinsic::bitreverse, .Type: MVT::v4i32, .Cost: 2},
849 {.ISD: Intrinsic::bitreverse, .Type: MVT::v1i64, .Cost: 2},
850 {.ISD: Intrinsic::bitreverse, .Type: MVT::v2i64, .Cost: 2},
851 };
852 const auto LegalisationCost = getTypeLegalizationCost(Ty: RetTy);
853 const auto *Entry =
854 CostTableLookup(Table: BitreverseTbl, ISD: ICA.getID(), Ty: LegalisationCost.second);
855 if (Entry) {
856 // Cost Model is using the legal type(i32) that i8 and i16 will be
857 // converted to +1 so that we match the actual lowering cost
858 if (TLI->getValueType(DL, Ty: RetTy, AllowUnknown: true) == MVT::i8 ||
859 TLI->getValueType(DL, Ty: RetTy, AllowUnknown: true) == MVT::i16)
860 return LegalisationCost.first * Entry->Cost + 1;
861
862 return LegalisationCost.first * Entry->Cost;
863 }
864 break;
865 }
866 case Intrinsic::ctpop: {
867 if (!ST->hasNEON()) {
868 // 32-bit or 64-bit ctpop without NEON is 12 instructions.
869 return getTypeLegalizationCost(Ty: RetTy).first * 12;
870 }
871 static const CostTblEntry CtpopCostTbl[] = {
872 {.ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: 4},
873 {.ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: 3},
874 {.ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: 2},
875 {.ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: 1},
876 {.ISD: ISD::CTPOP, .Type: MVT::i64, .Cost: 4},
877 {.ISD: ISD::CTPOP, .Type: MVT::v2i32, .Cost: 3},
878 {.ISD: ISD::CTPOP, .Type: MVT::v4i16, .Cost: 2},
879 {.ISD: ISD::CTPOP, .Type: MVT::v8i8, .Cost: 1},
880 {.ISD: ISD::CTPOP, .Type: MVT::i32, .Cost: 5},
881 // SVE types (For targets that override NEON for fixed length vectors)
882 {.ISD: ISD::CTPOP, .Type: MVT::nxv2i64, .Cost: 1},
883 {.ISD: ISD::CTPOP, .Type: MVT::nxv4i32, .Cost: 1},
884 {.ISD: ISD::CTPOP, .Type: MVT::nxv8i16, .Cost: 1},
885 {.ISD: ISD::CTPOP, .Type: MVT::nxv16i8, .Cost: 1},
886 };
887 auto LT = getTypeLegalizationCost(Ty: RetTy);
888 MVT MTy = LT.second;
889
890 // When SVE is available CNT will be used for fixed and scalable vectors.
891 if (ST->isSVEorStreamingSVEAvailable() && MTy.isFixedLengthVector())
892 MTy = MVT::getScalableVectorVT(VT: MTy.getVectorElementType(),
893 NumElements: 128 / MTy.getScalarSizeInBits());
894
895 if (const auto *Entry = CostTableLookup(Table: CtpopCostTbl, ISD: ISD::CTPOP, Ty: MTy)) {
896 // Extra cost of +1 when illegal vector types are legalized by promoting
897 // the integer type.
898 int ExtraCost = MTy.isVector() && MTy.getScalarSizeInBits() !=
899 RetTy->getScalarSizeInBits()
900 ? 1
901 : 0;
902 return LT.first * Entry->Cost + ExtraCost;
903 }
904 break;
905 }
906 case Intrinsic::sadd_with_overflow:
907 case Intrinsic::uadd_with_overflow:
908 case Intrinsic::ssub_with_overflow:
909 case Intrinsic::usub_with_overflow:
910 case Intrinsic::smul_with_overflow:
911 case Intrinsic::umul_with_overflow: {
912 static const CostTblEntry WithOverflowCostTbl[] = {
913 {.ISD: Intrinsic::sadd_with_overflow, .Type: MVT::i8, .Cost: 3},
914 {.ISD: Intrinsic::uadd_with_overflow, .Type: MVT::i8, .Cost: 3},
915 {.ISD: Intrinsic::sadd_with_overflow, .Type: MVT::i16, .Cost: 3},
916 {.ISD: Intrinsic::uadd_with_overflow, .Type: MVT::i16, .Cost: 3},
917 {.ISD: Intrinsic::sadd_with_overflow, .Type: MVT::i32, .Cost: 1},
918 {.ISD: Intrinsic::uadd_with_overflow, .Type: MVT::i32, .Cost: 1},
919 {.ISD: Intrinsic::sadd_with_overflow, .Type: MVT::i64, .Cost: 1},
920 {.ISD: Intrinsic::uadd_with_overflow, .Type: MVT::i64, .Cost: 1},
921 {.ISD: Intrinsic::ssub_with_overflow, .Type: MVT::i8, .Cost: 3},
922 {.ISD: Intrinsic::usub_with_overflow, .Type: MVT::i8, .Cost: 3},
923 {.ISD: Intrinsic::ssub_with_overflow, .Type: MVT::i16, .Cost: 3},
924 {.ISD: Intrinsic::usub_with_overflow, .Type: MVT::i16, .Cost: 3},
925 {.ISD: Intrinsic::ssub_with_overflow, .Type: MVT::i32, .Cost: 1},
926 {.ISD: Intrinsic::usub_with_overflow, .Type: MVT::i32, .Cost: 1},
927 {.ISD: Intrinsic::ssub_with_overflow, .Type: MVT::i64, .Cost: 1},
928 {.ISD: Intrinsic::usub_with_overflow, .Type: MVT::i64, .Cost: 1},
929 {.ISD: Intrinsic::smul_with_overflow, .Type: MVT::i8, .Cost: 5},
930 {.ISD: Intrinsic::umul_with_overflow, .Type: MVT::i8, .Cost: 4},
931 {.ISD: Intrinsic::smul_with_overflow, .Type: MVT::i16, .Cost: 5},
932 {.ISD: Intrinsic::umul_with_overflow, .Type: MVT::i16, .Cost: 4},
933 {.ISD: Intrinsic::smul_with_overflow, .Type: MVT::i32, .Cost: 2}, // eg umull;tst
934 {.ISD: Intrinsic::umul_with_overflow, .Type: MVT::i32, .Cost: 2}, // eg umull;cmp sxtw
935 {.ISD: Intrinsic::smul_with_overflow, .Type: MVT::i64, .Cost: 3}, // eg mul;smulh;cmp
936 {.ISD: Intrinsic::umul_with_overflow, .Type: MVT::i64, .Cost: 3}, // eg mul;umulh;cmp asr
937 };
938 EVT MTy = TLI->getValueType(DL, Ty: RetTy->getContainedType(i: 0), AllowUnknown: true);
939 if (MTy.isSimple())
940 if (const auto *Entry = CostTableLookup(Table: WithOverflowCostTbl, ISD: ICA.getID(),
941 Ty: MTy.getSimpleVT()))
942 return Entry->Cost;
943 break;
944 }
945 case Intrinsic::fptosi_sat:
946 case Intrinsic::fptoui_sat: {
947 if (ICA.getArgTypes().empty())
948 break;
949 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
950 auto LT = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
951 EVT MTy = TLI->getValueType(DL, Ty: RetTy);
952 // Check for the legal types, which are where the size of the input and the
953 // output are the same, or we are using cvt f64->i32 or f32->i64.
954 if ((LT.second == MVT::f32 || LT.second == MVT::f64 ||
955 LT.second == MVT::v2f32 || LT.second == MVT::v4f32 ||
956 LT.second == MVT::v2f64)) {
957 if ((LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits() ||
958 (LT.second == MVT::f64 && MTy == MVT::i32) ||
959 (LT.second == MVT::f32 && MTy == MVT::i64)))
960 return LT.first;
961 // Extending vector types v2f32->v2i64, fcvtl*2 + fcvt*2
962 if (LT.second.getScalarType() == MVT::f32 && MTy.isFixedLengthVector() &&
963 MTy.getScalarSizeInBits() == 64)
964 return LT.first * (MTy.getVectorNumElements() > 2 ? 4 : 2);
965 }
966 // Similarly for fp16 sizes. Without FullFP16 we generally need to fcvt to
967 // f32.
968 if (LT.second.getScalarType() == MVT::f16 && !ST->hasFullFP16())
969 return LT.first + getIntrinsicInstrCost(
970 ICA: {ICA.getID(),
971 RetTy,
972 {ICA.getArgTypes()[0]->getWithNewType(
973 EltTy: Type::getFloatTy(C&: RetTy->getContext()))}},
974 CostKind);
975 if ((LT.second == MVT::f16 && MTy == MVT::i32) ||
976 (LT.second == MVT::f16 && MTy == MVT::i64) ||
977 ((LT.second == MVT::v4f16 || LT.second == MVT::v8f16) &&
978 (LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits())))
979 return LT.first;
980 // Extending vector types v8f16->v8i32, fcvtl*2 + fcvt*2
981 if (LT.second.getScalarType() == MVT::f16 && MTy.isFixedLengthVector() &&
982 MTy.getScalarSizeInBits() == 32)
983 return LT.first * (MTy.getVectorNumElements() > 4 ? 4 : 2);
984 // Extending vector types v8f16->v8i32. These current scalarize but the
985 // codegen could be better.
986 if (LT.second.getScalarType() == MVT::f16 && MTy.isFixedLengthVector() &&
987 MTy.getScalarSizeInBits() == 64)
988 return MTy.getVectorNumElements() * 3;
989
990 // If we can we use a legal convert followed by a min+max
991 if ((LT.second.getScalarType() == MVT::f32 ||
992 LT.second.getScalarType() == MVT::f64 ||
993 LT.second.getScalarType() == MVT::f16) &&
994 LT.second.getScalarSizeInBits() >= MTy.getScalarSizeInBits()) {
995 Type *LegalTy =
996 Type::getIntNTy(C&: RetTy->getContext(), N: LT.second.getScalarSizeInBits());
997 if (LT.second.isVector())
998 LegalTy = VectorType::get(ElementType: LegalTy, EC: LT.second.getVectorElementCount());
999 InstructionCost Cost = 1;
1000 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin
1001 : Intrinsic::umin,
1002 LegalTy, {LegalTy, LegalTy});
1003 Cost += getIntrinsicInstrCost(ICA: Attrs1, CostKind);
1004 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax
1005 : Intrinsic::umax,
1006 LegalTy, {LegalTy, LegalTy});
1007 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
1008 return LT.first * Cost +
1009 ((LT.second.getScalarType() != MVT::f16 || ST->hasFullFP16()) ? 0
1010 : 1);
1011 }
1012 // Otherwise we need to follow the default expansion that clamps the value
1013 // using a float min/max with a fcmp+sel for nan handling when signed.
1014 Type *FPTy = ICA.getArgTypes()[0]->getScalarType();
1015 RetTy = RetTy->getScalarType();
1016 if (LT.second.isVector()) {
1017 FPTy = VectorType::get(ElementType: FPTy, EC: LT.second.getVectorElementCount());
1018 RetTy = VectorType::get(ElementType: RetTy, EC: LT.second.getVectorElementCount());
1019 }
1020 IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FPTy, {FPTy, FPTy});
1021 InstructionCost Cost = getIntrinsicInstrCost(ICA: Attrs1, CostKind);
1022 IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FPTy, {FPTy, FPTy});
1023 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
1024 Cost +=
1025 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
1026 Dst: RetTy, Src: FPTy, CCH: TTI::CastContextHint::None, CostKind);
1027 if (IsSigned) {
1028 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
1029 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: FPTy, CondTy,
1030 VecPred: CmpInst::FCMP_UNO, CostKind);
1031 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
1032 VecPred: CmpInst::FCMP_UNO, CostKind);
1033 }
1034 return LT.first * Cost;
1035 }
1036 case Intrinsic::fshl:
1037 case Intrinsic::fshr: {
1038 if (ICA.getArgs().empty())
1039 break;
1040
1041 const TTI::OperandValueInfo OpInfoZ = TTI::getOperandInfo(V: ICA.getArgs()[2]);
1042
1043 // ROTR / ROTL is a funnel shift with equal first and second operand. For
1044 // ROTR on integer registers (i32/i64) this can be done in a single ror
1045 // instruction. A fshl with a non-constant shift uses a neg + ror.
1046 if (RetTy->isIntegerTy() && ICA.getArgs()[0] == ICA.getArgs()[1] &&
1047 (RetTy->getPrimitiveSizeInBits() == 32 ||
1048 RetTy->getPrimitiveSizeInBits() == 64)) {
1049 InstructionCost NegCost =
1050 (ICA.getID() == Intrinsic::fshl && !OpInfoZ.isConstant()) ? 1 : 0;
1051 return 1 + NegCost;
1052 }
1053
1054 // TODO: Add handling for fshl where third argument is not a constant.
1055 if (!OpInfoZ.isConstant())
1056 break;
1057
1058 const auto LegalisationCost = getTypeLegalizationCost(Ty: RetTy);
1059 if (OpInfoZ.isUniform()) {
1060 static const CostTblEntry FshlTbl[] = {
1061 {.ISD: Intrinsic::fshl, .Type: MVT::v4i32, .Cost: 2}, // shl + usra
1062 {.ISD: Intrinsic::fshl, .Type: MVT::v2i64, .Cost: 2}, {.ISD: Intrinsic::fshl, .Type: MVT::v16i8, .Cost: 2},
1063 {.ISD: Intrinsic::fshl, .Type: MVT::v8i16, .Cost: 2}, {.ISD: Intrinsic::fshl, .Type: MVT::v2i32, .Cost: 2},
1064 {.ISD: Intrinsic::fshl, .Type: MVT::v8i8, .Cost: 2}, {.ISD: Intrinsic::fshl, .Type: MVT::v4i16, .Cost: 2}};
1065 // Costs for both fshl & fshr are the same, so just pass Intrinsic::fshl
1066 // to avoid having to duplicate the costs.
1067 const auto *Entry =
1068 CostTableLookup(Table: FshlTbl, ISD: Intrinsic::fshl, Ty: LegalisationCost.second);
1069 if (Entry)
1070 return LegalisationCost.first * Entry->Cost;
1071 }
1072
1073 auto TyL = getTypeLegalizationCost(Ty: RetTy);
1074 if (!RetTy->isIntegerTy())
1075 break;
1076
1077 // Estimate cost manually, as types like i8 and i16 will get promoted to
1078 // i32 and CostTableLookup will ignore the extra conversion cost.
1079 bool HigherCost = (RetTy->getScalarSizeInBits() != 32 &&
1080 RetTy->getScalarSizeInBits() < 64) ||
1081 (RetTy->getScalarSizeInBits() % 64 != 0);
1082 unsigned ExtraCost = HigherCost ? 1 : 0;
1083 if (RetTy->getScalarSizeInBits() == 32 ||
1084 RetTy->getScalarSizeInBits() == 64)
1085 ExtraCost = 0; // fhsl/fshr for i32 and i64 can be lowered to a single
1086 // extr instruction.
1087 else if (HigherCost)
1088 ExtraCost = 1;
1089 else
1090 break;
1091 return TyL.first + ExtraCost;
1092 }
1093 case Intrinsic::get_active_lane_mask: {
1094 auto RetTy = cast<VectorType>(Val: ICA.getReturnType());
1095 EVT RetVT = getTLI()->getValueType(DL, Ty: RetTy);
1096 EVT OpVT = getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[0]);
1097 if (getTLI()->shouldExpandGetActiveLaneMask(VT: RetVT, OpVT))
1098 break;
1099
1100 if (RetTy->isScalableTy()) {
1101 if (TLI->getTypeAction(Context&: RetTy->getContext(), VT: RetVT) !=
1102 TargetLowering::TypeSplitVector)
1103 break;
1104
1105 auto LT = getTypeLegalizationCost(Ty: RetTy);
1106 InstructionCost Cost = LT.first;
1107 // When SVE2p1 or SME2 is available, we can halve getTypeLegalizationCost
1108 // as get_active_lane_mask may lower to the sve_whilelo_x2 intrinsic, e.g.
1109 // nxv32i1 = get_active_lane_mask(base, idx) ->
1110 // {nxv16i1, nxv16i1} = sve_whilelo_x2(base, idx)
1111 if (ST->hasSVE2p1() || ST->hasSME2()) {
1112 Cost /= 2;
1113 if (Cost == 1)
1114 return Cost;
1115 }
1116
1117 // If more than one whilelo intrinsic is required, include the extra cost
1118 // required by the saturating add & select required to increment the
1119 // start value after the first intrinsic call.
1120 Type *OpTy = ICA.getArgTypes()[0];
1121 IntrinsicCostAttributes AddAttrs(Intrinsic::uadd_sat, OpTy, {OpTy, OpTy});
1122 InstructionCost SplitCost = getIntrinsicInstrCost(ICA: AddAttrs, CostKind);
1123 Type *CondTy = OpTy->getWithNewBitWidth(NewBitWidth: 1);
1124 SplitCost += getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: OpTy, CondTy,
1125 VecPred: CmpInst::ICMP_UGT, CostKind);
1126 return Cost + (SplitCost * (Cost - 1));
1127 } else if (!getTLI()->isTypeLegal(VT: RetVT)) {
1128 // We don't have enough context at this point to determine if the mask
1129 // is going to be kept live after the block, which will force the vXi1
1130 // type to be expanded to legal vectors of integers, e.g. v4i1->v4i32.
1131 // For now, we just assume the vectorizer created this intrinsic and
1132 // the result will be the input for a PHI. In this case the cost will
1133 // be extremely high for fixed-width vectors.
1134 // NOTE: getScalarizationOverhead returns a cost that's far too
1135 // pessimistic for the actual generated codegen. In reality there are
1136 // two instructions generated per lane.
1137 return cast<FixedVectorType>(Val: RetTy)->getNumElements() * 2;
1138 }
1139 break;
1140 }
1141 case Intrinsic::experimental_vector_match: {
1142 auto *NeedleTy = cast<FixedVectorType>(Val: ICA.getArgTypes()[1]);
1143 EVT SearchVT = getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[0]);
1144 unsigned SearchSize = NeedleTy->getNumElements();
1145 if (!getTLI()->shouldExpandVectorMatch(VT: SearchVT, SearchSize)) {
1146 // Base cost for MATCH instructions. At least on the Neoverse V2 and
1147 // Neoverse V3, these are cheap operations with the same latency as a
1148 // vector ADD. In most cases, however, we also need to do an extra DUP.
1149 // For fixed-length vectors we currently need an extra five--six
1150 // instructions besides the MATCH.
1151 InstructionCost Cost = 4;
1152 if (isa<FixedVectorType>(Val: RetTy))
1153 Cost += 10;
1154 return Cost;
1155 }
1156 break;
1157 }
1158 case Intrinsic::cttz: {
1159 auto LT = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
1160 if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
1161 return LT.first * 2;
1162 if (LT.second == MVT::v4i16 || LT.second == MVT::v8i16 ||
1163 LT.second == MVT::v2i32 || LT.second == MVT::v4i32)
1164 return LT.first * 3;
1165 break;
1166 }
1167 case Intrinsic::experimental_cttz_elts: {
1168 EVT ArgVT = getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[0]);
1169 if (!getTLI()->shouldExpandCttzElements(VT: ArgVT)) {
1170 // This will consist of a SVE brkb and a cntp instruction. These
1171 // typically have the same latency and half the throughput as a vector
1172 // add instruction.
1173 return 4;
1174 }
1175 break;
1176 }
1177 case Intrinsic::loop_dependence_raw_mask:
1178 case Intrinsic::loop_dependence_war_mask: {
1179 // The whilewr/rw instructions require SVE2 or SME.
1180 if (ST->hasSVE2() || ST->hasSME()) {
1181 EVT VecVT = getTLI()->getValueType(DL, Ty: RetTy);
1182 unsigned EltSizeInBytes =
1183 cast<ConstantInt>(Val: ICA.getArgs()[2])->getZExtValue();
1184 if (!is_contained(Set: {1u, 2u, 4u, 8u}, Element: EltSizeInBytes) ||
1185 VecVT.getVectorMinNumElements() != (16 / EltSizeInBytes))
1186 break;
1187 // For fixed-vector types we need to AND the mask with a ptrue vl<N>.
1188 return isa<FixedVectorType>(Val: RetTy) ? 2 : 1;
1189 }
1190 break;
1191 }
1192 case Intrinsic::experimental_vector_extract_last_active:
1193 if (ST->isSVEorStreamingSVEAvailable()) {
1194 auto [LegalCost, _] = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
1195 // This should turn into chained clastb instructions.
1196 return LegalCost;
1197 }
1198 break;
1199 case Intrinsic::pow: {
1200 // For scalar calls we know the target has the libcall, and for fixed-width
1201 // vectors we know for the worst case it can be scalarised.
1202 EVT VT = getTLI()->getValueType(DL, Ty: RetTy);
1203 RTLIB::Libcall LC = RTLIB::getPOW(RetVT: VT);
1204 bool HasLibcall = getTLI()->getLibcallImpl(Call: LC) != RTLIB::Unsupported;
1205 bool CanLowerWithLibcalls = !isa<ScalableVectorType>(Val: RetTy) || HasLibcall;
1206
1207 // If we know that the call can be lowered with libcalls then it's safe to
1208 // reduce the costs in some cases. This is important for scalable vectors,
1209 // since we cannot scalarize the call in the absence of a vector math
1210 // library.
1211 if (CanLowerWithLibcalls && ICA.getInst() && !ICA.getArgs().empty()) {
1212 // If we know the fast math flags and the exponent is a constant then the
1213 // cost may be less for some exponents like 0.25 and 0.75.
1214 const Constant *ExpC = dyn_cast<Constant>(Val: ICA.getArgs()[1]);
1215 if (ExpC && isa<VectorType>(Val: ExpC->getType()))
1216 ExpC = ExpC->getSplatValue();
1217 if (auto *ExpF = dyn_cast_or_null<ConstantFP>(Val: ExpC)) {
1218 // The argument must be a FP constant.
1219 bool Is025 = ExpF->getValueAPF().isExactlyValue(V: 0.25);
1220 bool Is075 = ExpF->getValueAPF().isExactlyValue(V: 0.75);
1221 FastMathFlags FMF = ICA.getInst()->getFastMathFlags();
1222 if ((Is025 || Is075) && FMF.noInfs() && FMF.approxFunc() &&
1223 (!Is025 || FMF.noSignedZeros())) {
1224 IntrinsicCostAttributes Attrs(Intrinsic::sqrt, RetTy, {RetTy}, FMF);
1225 InstructionCost Sqrt = getIntrinsicInstrCost(ICA: Attrs, CostKind);
1226 if (Is025)
1227 return 2 * Sqrt;
1228 InstructionCost FMul =
1229 getArithmeticInstrCost(Opcode: Instruction::FMul, Ty: RetTy, CostKind);
1230 return (Sqrt * 2) + FMul;
1231 }
1232 // TODO: For 1/3 exponents we expect the cbrt call to be slightly
1233 // cheaper than pow.
1234 }
1235 }
1236
1237 if (HasLibcall)
1238 return getCallInstrCost(F: nullptr, RetTy, Tys: ICA.getArgTypes(), CostKind);
1239 break;
1240 }
1241 case Intrinsic::sqrt:
1242 case Intrinsic::fabs:
1243 case Intrinsic::ceil:
1244 case Intrinsic::floor:
1245 case Intrinsic::nearbyint:
1246 case Intrinsic::round:
1247 case Intrinsic::rint:
1248 case Intrinsic::roundeven:
1249 case Intrinsic::trunc:
1250 case Intrinsic::minnum:
1251 case Intrinsic::maxnum:
1252 case Intrinsic::minimum:
1253 case Intrinsic::maximum: {
1254 if (isa<ScalableVectorType>(Val: RetTy) && ST->isSVEorStreamingSVEAvailable()) {
1255 auto LT = getTypeLegalizationCost(Ty: RetTy);
1256 return LT.first;
1257 }
1258 break;
1259 }
1260 default:
1261 break;
1262 }
1263 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1264}
1265
1266/// The function will remove redundant reinterprets casting in the presence
1267/// of the control flow
1268static std::optional<Instruction *> processPhiNode(InstCombiner &IC,
1269 IntrinsicInst &II) {
1270 SmallVector<Instruction *, 32> Worklist;
1271 auto RequiredType = II.getType();
1272
1273 auto *PN = dyn_cast<PHINode>(Val: II.getArgOperand(i: 0));
1274 assert(PN && "Expected Phi Node!");
1275
1276 // Don't create a new Phi unless we can remove the old one.
1277 if (!PN->hasOneUse())
1278 return std::nullopt;
1279
1280 for (Value *IncValPhi : PN->incoming_values()) {
1281 auto *Reinterpret = dyn_cast<IntrinsicInst>(Val: IncValPhi);
1282 if (!Reinterpret ||
1283 Reinterpret->getIntrinsicID() !=
1284 Intrinsic::aarch64_sve_convert_to_svbool ||
1285 RequiredType != Reinterpret->getArgOperand(i: 0)->getType())
1286 return std::nullopt;
1287 }
1288
1289 // Create the new Phi
1290 IC.Builder.SetInsertPoint(PN);
1291 PHINode *NPN = IC.Builder.CreatePHI(Ty: RequiredType, NumReservedValues: PN->getNumIncomingValues());
1292 Worklist.push_back(Elt: PN);
1293
1294 for (unsigned I = 0; I < PN->getNumIncomingValues(); I++) {
1295 auto *Reinterpret = cast<Instruction>(Val: PN->getIncomingValue(i: I));
1296 NPN->addIncoming(V: Reinterpret->getOperand(i: 0), BB: PN->getIncomingBlock(i: I));
1297 Worklist.push_back(Elt: Reinterpret);
1298 }
1299
1300 // Cleanup Phi Node and reinterprets
1301 return IC.replaceInstUsesWith(I&: II, V: NPN);
1302}
1303
1304// A collection of properties common to SVE intrinsics that allow for combines
1305// to be written without needing to know the specific intrinsic.
1306struct SVEIntrinsicInfo {
1307 //
1308 // Helper routines for common intrinsic definitions.
1309 //
1310
1311 // e.g. llvm.aarch64.sve.add pg, op1, op2
1312 // with IID ==> llvm.aarch64.sve.add_u
1313 static SVEIntrinsicInfo
1314 defaultMergingOp(Intrinsic::ID IID = Intrinsic::not_intrinsic) {
1315 return SVEIntrinsicInfo()
1316 .setGoverningPredicateOperandIdx(0)
1317 .setOperandIdxInactiveLanesTakenFrom(1)
1318 .setMatchingUndefIntrinsic(IID);
1319 }
1320
1321 // e.g. llvm.aarch64.sve.neg inactive, pg, op
1322 static SVEIntrinsicInfo defaultMergingUnaryOp() {
1323 return SVEIntrinsicInfo()
1324 .setGoverningPredicateOperandIdx(1)
1325 .setOperandIdxInactiveLanesTakenFrom(0)
1326 .setOperandIdxWithNoActiveLanes(0);
1327 }
1328
1329 // e.g. llvm.aarch64.sve.fcvtnt inactive, pg, op
1330 static SVEIntrinsicInfo defaultMergingUnaryNarrowingTopOp() {
1331 return SVEIntrinsicInfo()
1332 .setGoverningPredicateOperandIdx(1)
1333 .setOperandIdxInactiveLanesTakenFrom(0);
1334 }
1335
1336 // e.g. llvm.aarch64.sve.add_u pg, op1, op2
1337 static SVEIntrinsicInfo defaultUndefOp() {
1338 return SVEIntrinsicInfo()
1339 .setGoverningPredicateOperandIdx(0)
1340 .setInactiveLanesAreNotDefined();
1341 }
1342
1343 // e.g. llvm.aarch64.sve.prf pg, ptr (GPIndex = 0)
1344 // llvm.aarch64.sve.st1 data, pg, ptr (GPIndex = 1)
1345 static SVEIntrinsicInfo defaultVoidOp(unsigned GPIndex) {
1346 return SVEIntrinsicInfo()
1347 .setGoverningPredicateOperandIdx(GPIndex)
1348 .setInactiveLanesAreUnused();
1349 }
1350
1351 // e.g. llvm.aarch64.sve.cmpeq pg, op1, op2
1352 // llvm.aarch64.sve.ld1 pg, ptr
1353 static SVEIntrinsicInfo defaultZeroingOp() {
1354 return SVEIntrinsicInfo()
1355 .setGoverningPredicateOperandIdx(0)
1356 .setInactiveLanesAreUnused()
1357 .setResultIsZeroInitialized();
1358 }
1359
1360 // All properties relate to predication and thus having a general predicate
1361 // is the minimum requirement to say there is intrinsic info to act on.
1362 explicit operator bool() const { return hasGoverningPredicate(); }
1363
1364 //
1365 // Properties relating to the governing predicate.
1366 //
1367
1368 bool hasGoverningPredicate() const {
1369 return GoverningPredicateIdx != std::numeric_limits<unsigned>::max();
1370 }
1371
1372 unsigned getGoverningPredicateOperandIdx() const {
1373 assert(hasGoverningPredicate() && "Propery not set!");
1374 return GoverningPredicateIdx;
1375 }
1376
1377 SVEIntrinsicInfo &setGoverningPredicateOperandIdx(unsigned Index) {
1378 assert(!hasGoverningPredicate() && "Cannot set property twice!");
1379 GoverningPredicateIdx = Index;
1380 return *this;
1381 }
1382
1383 //
1384 // Properties relating to operations the intrinsic could be transformed into.
1385 // NOTE: This does not mean such a transformation is always possible, but the
1386 // knowledge makes it possible to reuse existing optimisations without needing
1387 // to embed specific handling for each intrinsic. For example, instruction
1388 // simplification can be used to optimise an intrinsic's active lanes.
1389 //
1390
1391 bool hasMatchingUndefIntrinsic() const {
1392 return UndefIntrinsic != Intrinsic::not_intrinsic;
1393 }
1394
1395 Intrinsic::ID getMatchingUndefIntrinsic() const {
1396 assert(hasMatchingUndefIntrinsic() && "Propery not set!");
1397 return UndefIntrinsic;
1398 }
1399
1400 SVEIntrinsicInfo &setMatchingUndefIntrinsic(Intrinsic::ID IID) {
1401 assert(!hasMatchingUndefIntrinsic() && "Cannot set property twice!");
1402 UndefIntrinsic = IID;
1403 return *this;
1404 }
1405
1406 bool hasMatchingIROpode() const { return IROpcode != 0; }
1407
1408 unsigned getMatchingIROpode() const {
1409 assert(hasMatchingIROpode() && "Propery not set!");
1410 return IROpcode;
1411 }
1412
1413 SVEIntrinsicInfo &setMatchingIROpcode(unsigned Opcode) {
1414 assert(!hasMatchingIROpode() && "Cannot set property twice!");
1415 IROpcode = Opcode;
1416 return *this;
1417 }
1418
1419 //
1420 // Properties relating to the result of inactive lanes.
1421 //
1422
1423 bool inactiveLanesTakenFromOperand() const {
1424 return ResultLanes == InactiveLanesTakenFromOperand;
1425 }
1426
1427 unsigned getOperandIdxInactiveLanesTakenFrom() const {
1428 assert(inactiveLanesTakenFromOperand() && "Propery not set!");
1429 return OperandIdxForInactiveLanes;
1430 }
1431
1432 SVEIntrinsicInfo &setOperandIdxInactiveLanesTakenFrom(unsigned Index) {
1433 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1434 ResultLanes = InactiveLanesTakenFromOperand;
1435 OperandIdxForInactiveLanes = Index;
1436 return *this;
1437 }
1438
1439 bool inactiveLanesAreNotDefined() const {
1440 return ResultLanes == InactiveLanesAreNotDefined;
1441 }
1442
1443 SVEIntrinsicInfo &setInactiveLanesAreNotDefined() {
1444 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1445 ResultLanes = InactiveLanesAreNotDefined;
1446 return *this;
1447 }
1448
1449 bool inactiveLanesAreUnused() const {
1450 return ResultLanes == InactiveLanesAreUnused;
1451 }
1452
1453 SVEIntrinsicInfo &setInactiveLanesAreUnused() {
1454 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1455 ResultLanes = InactiveLanesAreUnused;
1456 return *this;
1457 }
1458
1459 // NOTE: Whilst not limited to only inactive lanes, the common use case is:
1460 // inactiveLanesAreZeroed =
1461 // resultIsZeroInitialized() && inactiveLanesAreUnused()
1462 bool resultIsZeroInitialized() const { return ResultIsZeroInitialized; }
1463
1464 SVEIntrinsicInfo &setResultIsZeroInitialized() {
1465 ResultIsZeroInitialized = true;
1466 return *this;
1467 }
1468
1469 //
1470 // The first operand of unary merging operations is typically only used to
1471 // set the result for inactive lanes. Knowing this allows us to deadcode the
1472 // operand when we can prove there are no inactive lanes.
1473 //
1474
1475 bool hasOperandWithNoActiveLanes() const {
1476 return OperandIdxWithNoActiveLanes != std::numeric_limits<unsigned>::max();
1477 }
1478
1479 unsigned getOperandIdxWithNoActiveLanes() const {
1480 assert(hasOperandWithNoActiveLanes() && "Propery not set!");
1481 return OperandIdxWithNoActiveLanes;
1482 }
1483
1484 SVEIntrinsicInfo &setOperandIdxWithNoActiveLanes(unsigned Index) {
1485 assert(!hasOperandWithNoActiveLanes() && "Cannot set property twice!");
1486 OperandIdxWithNoActiveLanes = Index;
1487 return *this;
1488 }
1489
1490private:
1491 unsigned GoverningPredicateIdx = std::numeric_limits<unsigned>::max();
1492
1493 Intrinsic::ID UndefIntrinsic = Intrinsic::not_intrinsic;
1494 unsigned IROpcode = 0;
1495
1496 enum PredicationStyle {
1497 Uninitialized,
1498 InactiveLanesTakenFromOperand,
1499 InactiveLanesAreNotDefined,
1500 InactiveLanesAreUnused
1501 } ResultLanes = Uninitialized;
1502
1503 bool ResultIsZeroInitialized = false;
1504 unsigned OperandIdxForInactiveLanes = std::numeric_limits<unsigned>::max();
1505 unsigned OperandIdxWithNoActiveLanes = std::numeric_limits<unsigned>::max();
1506};
1507
1508static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II) {
1509 // Some SVE intrinsics do not use scalable vector types, but since they are
1510 // not relevant from an SVEIntrinsicInfo perspective, they are also ignored.
1511 if (!isa<ScalableVectorType>(Val: II.getType()) &&
1512 all_of(Range: II.args(), P: [&](const Value *V) {
1513 return !isa<ScalableVectorType>(Val: V->getType());
1514 }))
1515 return SVEIntrinsicInfo();
1516
1517 Intrinsic::ID IID = II.getIntrinsicID();
1518 switch (IID) {
1519 default:
1520 break;
1521 case Intrinsic::aarch64_sve_fcvt_bf16f32_v2:
1522 case Intrinsic::aarch64_sve_fcvt_f16f32:
1523 case Intrinsic::aarch64_sve_fcvt_f16f64:
1524 case Intrinsic::aarch64_sve_fcvt_f32f16:
1525 case Intrinsic::aarch64_sve_fcvt_f32f64:
1526 case Intrinsic::aarch64_sve_fcvt_f64f16:
1527 case Intrinsic::aarch64_sve_fcvt_f64f32:
1528 case Intrinsic::aarch64_sve_fcvtlt_f32f16:
1529 case Intrinsic::aarch64_sve_fcvtlt_f64f32:
1530 case Intrinsic::aarch64_sve_fcvtx_f32f64:
1531 case Intrinsic::aarch64_sve_fcvtzs:
1532 case Intrinsic::aarch64_sve_fcvtzs_i32f16:
1533 case Intrinsic::aarch64_sve_fcvtzs_i32f64:
1534 case Intrinsic::aarch64_sve_fcvtzs_i64f16:
1535 case Intrinsic::aarch64_sve_fcvtzs_i64f32:
1536 case Intrinsic::aarch64_sve_fcvtzu:
1537 case Intrinsic::aarch64_sve_fcvtzu_i32f16:
1538 case Intrinsic::aarch64_sve_fcvtzu_i32f64:
1539 case Intrinsic::aarch64_sve_fcvtzu_i64f16:
1540 case Intrinsic::aarch64_sve_fcvtzu_i64f32:
1541 case Intrinsic::aarch64_sve_revb:
1542 case Intrinsic::aarch64_sve_revh:
1543 case Intrinsic::aarch64_sve_revw:
1544 case Intrinsic::aarch64_sve_revd:
1545 case Intrinsic::aarch64_sve_scvtf:
1546 case Intrinsic::aarch64_sve_scvtf_f16i32:
1547 case Intrinsic::aarch64_sve_scvtf_f16i64:
1548 case Intrinsic::aarch64_sve_scvtf_f32i64:
1549 case Intrinsic::aarch64_sve_scvtf_f64i32:
1550 case Intrinsic::aarch64_sve_ucvtf:
1551 case Intrinsic::aarch64_sve_ucvtf_f16i32:
1552 case Intrinsic::aarch64_sve_ucvtf_f16i64:
1553 case Intrinsic::aarch64_sve_ucvtf_f32i64:
1554 case Intrinsic::aarch64_sve_ucvtf_f64i32:
1555 return SVEIntrinsicInfo::defaultMergingUnaryOp();
1556
1557 case Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2:
1558 case Intrinsic::aarch64_sve_fcvtnt_f16f32:
1559 case Intrinsic::aarch64_sve_fcvtnt_f32f64:
1560 case Intrinsic::aarch64_sve_fcvtxnt_f32f64:
1561 return SVEIntrinsicInfo::defaultMergingUnaryNarrowingTopOp();
1562
1563 case Intrinsic::aarch64_sve_fabd:
1564 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fabd_u);
1565 case Intrinsic::aarch64_sve_fadd:
1566 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fadd_u)
1567 .setMatchingIROpcode(Instruction::FAdd);
1568 case Intrinsic::aarch64_sve_fdiv:
1569 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fdiv_u)
1570 .setMatchingIROpcode(Instruction::FDiv);
1571 case Intrinsic::aarch64_sve_fmax:
1572 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmax_u);
1573 case Intrinsic::aarch64_sve_fmaxnm:
1574 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmaxnm_u);
1575 case Intrinsic::aarch64_sve_fmin:
1576 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmin_u);
1577 case Intrinsic::aarch64_sve_fminnm:
1578 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fminnm_u);
1579 case Intrinsic::aarch64_sve_fmla:
1580 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmla_u);
1581 case Intrinsic::aarch64_sve_fmls:
1582 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmls_u);
1583 case Intrinsic::aarch64_sve_fmul:
1584 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmul_u)
1585 .setMatchingIROpcode(Instruction::FMul);
1586 case Intrinsic::aarch64_sve_fmulx:
1587 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmulx_u);
1588 case Intrinsic::aarch64_sve_fnmla:
1589 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fnmla_u);
1590 case Intrinsic::aarch64_sve_fnmls:
1591 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fnmls_u);
1592 case Intrinsic::aarch64_sve_fsub:
1593 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fsub_u)
1594 .setMatchingIROpcode(Instruction::FSub);
1595 case Intrinsic::aarch64_sve_add:
1596 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_add_u)
1597 .setMatchingIROpcode(Instruction::Add);
1598 case Intrinsic::aarch64_sve_mla:
1599 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_mla_u);
1600 case Intrinsic::aarch64_sve_mls:
1601 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_mls_u);
1602 case Intrinsic::aarch64_sve_mul:
1603 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_mul_u)
1604 .setMatchingIROpcode(Instruction::Mul);
1605 case Intrinsic::aarch64_sve_sabd:
1606 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sabd_u);
1607 case Intrinsic::aarch64_sve_sdiv:
1608 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sdiv_u)
1609 .setMatchingIROpcode(Instruction::SDiv);
1610 case Intrinsic::aarch64_sve_smax:
1611 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_smax_u);
1612 case Intrinsic::aarch64_sve_smin:
1613 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_smin_u);
1614 case Intrinsic::aarch64_sve_smulh:
1615 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_smulh_u);
1616 case Intrinsic::aarch64_sve_sub:
1617 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sub_u)
1618 .setMatchingIROpcode(Instruction::Sub);
1619 case Intrinsic::aarch64_sve_uabd:
1620 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uabd_u);
1621 case Intrinsic::aarch64_sve_udiv:
1622 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_udiv_u)
1623 .setMatchingIROpcode(Instruction::UDiv);
1624 case Intrinsic::aarch64_sve_umax:
1625 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_umax_u);
1626 case Intrinsic::aarch64_sve_umin:
1627 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_umin_u);
1628 case Intrinsic::aarch64_sve_umulh:
1629 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_umulh_u);
1630 case Intrinsic::aarch64_sve_asr:
1631 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_asr_u)
1632 .setMatchingIROpcode(Instruction::AShr);
1633 case Intrinsic::aarch64_sve_lsl:
1634 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_lsl_u)
1635 .setMatchingIROpcode(Instruction::Shl);
1636 case Intrinsic::aarch64_sve_lsr:
1637 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_lsr_u)
1638 .setMatchingIROpcode(Instruction::LShr);
1639 case Intrinsic::aarch64_sve_and:
1640 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_and_u)
1641 .setMatchingIROpcode(Instruction::And);
1642 case Intrinsic::aarch64_sve_bic:
1643 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_bic_u);
1644 case Intrinsic::aarch64_sve_eor:
1645 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_eor_u)
1646 .setMatchingIROpcode(Instruction::Xor);
1647 case Intrinsic::aarch64_sve_orr:
1648 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_orr_u)
1649 .setMatchingIROpcode(Instruction::Or);
1650 case Intrinsic::aarch64_sve_shsub:
1651 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_shsub_u);
1652 case Intrinsic::aarch64_sve_shsubr:
1653 return SVEIntrinsicInfo::defaultMergingOp();
1654 case Intrinsic::aarch64_sve_sqrshl:
1655 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sqrshl_u);
1656 case Intrinsic::aarch64_sve_sqshl:
1657 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sqshl_u);
1658 case Intrinsic::aarch64_sve_sqsub:
1659 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sqsub_u);
1660 case Intrinsic::aarch64_sve_srshl:
1661 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_srshl_u);
1662 case Intrinsic::aarch64_sve_uhsub:
1663 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uhsub_u);
1664 case Intrinsic::aarch64_sve_uhsubr:
1665 return SVEIntrinsicInfo::defaultMergingOp();
1666 case Intrinsic::aarch64_sve_uqrshl:
1667 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uqrshl_u);
1668 case Intrinsic::aarch64_sve_uqshl:
1669 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uqshl_u);
1670 case Intrinsic::aarch64_sve_uqsub:
1671 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uqsub_u);
1672 case Intrinsic::aarch64_sve_urshl:
1673 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_urshl_u);
1674
1675 case Intrinsic::aarch64_sve_add_u:
1676 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1677 Instruction::Add);
1678 case Intrinsic::aarch64_sve_and_u:
1679 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1680 Instruction::And);
1681 case Intrinsic::aarch64_sve_asr_u:
1682 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1683 Instruction::AShr);
1684 case Intrinsic::aarch64_sve_eor_u:
1685 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1686 Instruction::Xor);
1687 case Intrinsic::aarch64_sve_fadd_u:
1688 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1689 Instruction::FAdd);
1690 case Intrinsic::aarch64_sve_fdiv_u:
1691 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1692 Instruction::FDiv);
1693 case Intrinsic::aarch64_sve_fmul_u:
1694 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1695 Instruction::FMul);
1696 case Intrinsic::aarch64_sve_fsub_u:
1697 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1698 Instruction::FSub);
1699 case Intrinsic::aarch64_sve_lsl_u:
1700 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1701 Instruction::Shl);
1702 case Intrinsic::aarch64_sve_lsr_u:
1703 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1704 Instruction::LShr);
1705 case Intrinsic::aarch64_sve_mul_u:
1706 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1707 Instruction::Mul);
1708 case Intrinsic::aarch64_sve_orr_u:
1709 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1710 Instruction::Or);
1711 case Intrinsic::aarch64_sve_sdiv_u:
1712 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1713 Instruction::SDiv);
1714 case Intrinsic::aarch64_sve_sub_u:
1715 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1716 Instruction::Sub);
1717 case Intrinsic::aarch64_sve_udiv_u:
1718 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1719 Instruction::UDiv);
1720
1721 case Intrinsic::aarch64_sve_addqv:
1722 case Intrinsic::aarch64_sve_and_z:
1723 case Intrinsic::aarch64_sve_bic_z:
1724 case Intrinsic::aarch64_sve_brka_z:
1725 case Intrinsic::aarch64_sve_brkb_z:
1726 case Intrinsic::aarch64_sve_brkn_z:
1727 case Intrinsic::aarch64_sve_brkpa_z:
1728 case Intrinsic::aarch64_sve_brkpb_z:
1729 case Intrinsic::aarch64_sve_cntp:
1730 case Intrinsic::aarch64_sve_compact:
1731 case Intrinsic::aarch64_sve_eor_z:
1732 case Intrinsic::aarch64_sve_eorv:
1733 case Intrinsic::aarch64_sve_eorqv:
1734 case Intrinsic::aarch64_sve_nand_z:
1735 case Intrinsic::aarch64_sve_nor_z:
1736 case Intrinsic::aarch64_sve_orn_z:
1737 case Intrinsic::aarch64_sve_orr_z:
1738 case Intrinsic::aarch64_sve_orv:
1739 case Intrinsic::aarch64_sve_orqv:
1740 case Intrinsic::aarch64_sve_pnext:
1741 case Intrinsic::aarch64_sve_rdffr_z:
1742 case Intrinsic::aarch64_sve_saddv:
1743 case Intrinsic::aarch64_sve_uaddv:
1744 case Intrinsic::aarch64_sve_umaxv:
1745 case Intrinsic::aarch64_sve_umaxqv:
1746 case Intrinsic::aarch64_sve_cmpeq:
1747 case Intrinsic::aarch64_sve_cmpeq_wide:
1748 case Intrinsic::aarch64_sve_cmpge:
1749 case Intrinsic::aarch64_sve_cmpge_wide:
1750 case Intrinsic::aarch64_sve_cmpgt:
1751 case Intrinsic::aarch64_sve_cmpgt_wide:
1752 case Intrinsic::aarch64_sve_cmphi:
1753 case Intrinsic::aarch64_sve_cmphi_wide:
1754 case Intrinsic::aarch64_sve_cmphs:
1755 case Intrinsic::aarch64_sve_cmphs_wide:
1756 case Intrinsic::aarch64_sve_cmple_wide:
1757 case Intrinsic::aarch64_sve_cmplo_wide:
1758 case Intrinsic::aarch64_sve_cmpls_wide:
1759 case Intrinsic::aarch64_sve_cmplt_wide:
1760 case Intrinsic::aarch64_sve_cmpne:
1761 case Intrinsic::aarch64_sve_cmpne_wide:
1762 case Intrinsic::aarch64_sve_facge:
1763 case Intrinsic::aarch64_sve_facgt:
1764 case Intrinsic::aarch64_sve_fcmpeq:
1765 case Intrinsic::aarch64_sve_fcmpge:
1766 case Intrinsic::aarch64_sve_fcmpgt:
1767 case Intrinsic::aarch64_sve_fcmpne:
1768 case Intrinsic::aarch64_sve_fcmpuo:
1769 case Intrinsic::aarch64_sve_ld1:
1770 case Intrinsic::aarch64_sve_ld1_gather:
1771 case Intrinsic::aarch64_sve_ld1_gather_index:
1772 case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
1773 case Intrinsic::aarch64_sve_ld1_gather_sxtw:
1774 case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
1775 case Intrinsic::aarch64_sve_ld1_gather_uxtw:
1776 case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
1777 case Intrinsic::aarch64_sve_ld1q_gather_index:
1778 case Intrinsic::aarch64_sve_ld1q_gather_scalar_offset:
1779 case Intrinsic::aarch64_sve_ld1q_gather_vector_offset:
1780 case Intrinsic::aarch64_sve_ld1ro:
1781 case Intrinsic::aarch64_sve_ld1rq:
1782 case Intrinsic::aarch64_sve_ld1udq:
1783 case Intrinsic::aarch64_sve_ld1uwq:
1784 case Intrinsic::aarch64_sve_ld2_sret:
1785 case Intrinsic::aarch64_sve_ld2q_sret:
1786 case Intrinsic::aarch64_sve_ld3_sret:
1787 case Intrinsic::aarch64_sve_ld3q_sret:
1788 case Intrinsic::aarch64_sve_ld4_sret:
1789 case Intrinsic::aarch64_sve_ld4q_sret:
1790 case Intrinsic::aarch64_sve_ldff1:
1791 case Intrinsic::aarch64_sve_ldff1_gather:
1792 case Intrinsic::aarch64_sve_ldff1_gather_index:
1793 case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
1794 case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
1795 case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
1796 case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
1797 case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
1798 case Intrinsic::aarch64_sve_ldnf1:
1799 case Intrinsic::aarch64_sve_ldnt1:
1800 case Intrinsic::aarch64_sve_ldnt1_gather:
1801 case Intrinsic::aarch64_sve_ldnt1_gather_index:
1802 case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
1803 case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
1804 return SVEIntrinsicInfo::defaultZeroingOp();
1805
1806 case Intrinsic::aarch64_sve_prf:
1807 case Intrinsic::aarch64_sve_prfb_gather_index:
1808 case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
1809 case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
1810 case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
1811 case Intrinsic::aarch64_sve_prfd_gather_index:
1812 case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
1813 case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
1814 case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
1815 case Intrinsic::aarch64_sve_prfh_gather_index:
1816 case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
1817 case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
1818 case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
1819 case Intrinsic::aarch64_sve_prfw_gather_index:
1820 case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
1821 case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
1822 case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
1823 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 0);
1824
1825 case Intrinsic::aarch64_sve_st1_scatter:
1826 case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
1827 case Intrinsic::aarch64_sve_st1_scatter_sxtw:
1828 case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
1829 case Intrinsic::aarch64_sve_st1_scatter_uxtw:
1830 case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
1831 case Intrinsic::aarch64_sve_st1dq:
1832 case Intrinsic::aarch64_sve_st1q_scatter_index:
1833 case Intrinsic::aarch64_sve_st1q_scatter_scalar_offset:
1834 case Intrinsic::aarch64_sve_st1q_scatter_vector_offset:
1835 case Intrinsic::aarch64_sve_st1wq:
1836 case Intrinsic::aarch64_sve_stnt1:
1837 case Intrinsic::aarch64_sve_stnt1_scatter:
1838 case Intrinsic::aarch64_sve_stnt1_scatter_index:
1839 case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
1840 case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
1841 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 1);
1842 case Intrinsic::aarch64_sve_st2:
1843 case Intrinsic::aarch64_sve_st2q:
1844 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 2);
1845 case Intrinsic::aarch64_sve_st3:
1846 case Intrinsic::aarch64_sve_st3q:
1847 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 3);
1848 case Intrinsic::aarch64_sve_st4:
1849 case Intrinsic::aarch64_sve_st4q:
1850 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 4);
1851 }
1852
1853 return SVEIntrinsicInfo();
1854}
1855
1856static bool isAllActivePredicate(Value *Pred) {
1857 Value *UncastedPred;
1858
1859 // Look through predicate casts that only remove lanes.
1860 if (match(V: Pred, P: m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>(
1861 Ops: m_Value(V&: UncastedPred)))) {
1862 auto *OrigPredTy = cast<ScalableVectorType>(Val: Pred->getType());
1863 Pred = UncastedPred;
1864
1865 if (match(V: Pred, P: m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(
1866 Ops: m_Value(V&: UncastedPred))))
1867 // If the predicate has the same or less lanes than the uncasted predicate
1868 // then we know the casting has no effect.
1869 if (OrigPredTy->getMinNumElements() <=
1870 cast<ScalableVectorType>(Val: UncastedPred->getType())
1871 ->getMinNumElements())
1872 Pred = UncastedPred;
1873 }
1874
1875 auto *C = dyn_cast<Constant>(Val: Pred);
1876 return C && C->isAllOnesValue();
1877}
1878
1879// Simplify `V` by only considering the operations that affect active lanes.
1880// This function should only return existing Values or newly created Constants.
1881static Value *stripInactiveLanes(Value *V, const Value *Pg) {
1882 auto *Dup = dyn_cast<IntrinsicInst>(Val: V);
1883 if (Dup && Dup->getIntrinsicID() == Intrinsic::aarch64_sve_dup &&
1884 Dup->getOperand(i_nocapture: 1) == Pg && isa<Constant>(Val: Dup->getOperand(i_nocapture: 2)))
1885 return ConstantVector::getSplat(
1886 EC: cast<VectorType>(Val: V->getType())->getElementCount(),
1887 Elt: cast<Constant>(Val: Dup->getOperand(i_nocapture: 2)));
1888
1889 return V;
1890}
1891
1892static std::optional<Instruction *>
1893simplifySVEIntrinsicBinOp(InstCombiner &IC, IntrinsicInst &II,
1894 const SVEIntrinsicInfo &IInfo) {
1895 const unsigned Opc = IInfo.getMatchingIROpode();
1896 assert(Instruction::isBinaryOp(Opc) && "Expected a binary operation!");
1897
1898 Value *Pg = II.getOperand(i_nocapture: 0);
1899 Value *Op1 = II.getOperand(i_nocapture: 1);
1900 Value *Op2 = II.getOperand(i_nocapture: 2);
1901 const DataLayout &DL = II.getDataLayout();
1902
1903 // Canonicalise constants to the RHS.
1904 if (Instruction::isCommutative(Opcode: Opc) && IInfo.inactiveLanesAreNotDefined() &&
1905 isa<Constant>(Val: Op1) && !isa<Constant>(Val: Op2)) {
1906 IC.replaceOperand(I&: II, OpNum: 1, V: Op2);
1907 IC.replaceOperand(I&: II, OpNum: 2, V: Op1);
1908 return &II;
1909 }
1910
1911 // Only active lanes matter when simplifying the operation.
1912 Op1 = stripInactiveLanes(V: Op1, Pg);
1913 Op2 = stripInactiveLanes(V: Op2, Pg);
1914
1915 Value *SimpleII;
1916 if (auto FII = dyn_cast<FPMathOperator>(Val: &II))
1917 SimpleII = simplifyBinOp(Opcode: Opc, LHS: Op1, RHS: Op2, FMF: FII->getFastMathFlags(), Q: DL);
1918 else
1919 SimpleII = simplifyBinOp(Opcode: Opc, LHS: Op1, RHS: Op2, Q: DL);
1920
1921 // An SVE intrinsic's result is always defined. However, this is not the case
1922 // for its equivalent IR instruction (e.g. when shifting by an amount more
1923 // than the data's bitwidth). Simplifications to an undefined result must be
1924 // ignored to preserve the intrinsic's expected behaviour.
1925 if (!SimpleII || isa<UndefValue>(Val: SimpleII))
1926 return std::nullopt;
1927
1928 if (IInfo.inactiveLanesAreNotDefined())
1929 return IC.replaceInstUsesWith(I&: II, V: SimpleII);
1930
1931 Value *Inactive = II.getOperand(i_nocapture: IInfo.getOperandIdxInactiveLanesTakenFrom());
1932
1933 // The intrinsic does nothing (e.g. sve.mul(pg, A, 1.0)).
1934 if (SimpleII == Inactive)
1935 return IC.replaceInstUsesWith(I&: II, V: SimpleII);
1936
1937 // Inactive lanes must be preserved.
1938 SimpleII = IC.Builder.CreateSelect(C: Pg, True: SimpleII, False: Inactive);
1939 return IC.replaceInstUsesWith(I&: II, V: SimpleII);
1940}
1941
1942// Use SVE intrinsic info to eliminate redundant operands and/or canonicalise
1943// to operations with less strict inactive lane requirements.
1944static std::optional<Instruction *>
1945simplifySVEIntrinsic(InstCombiner &IC, IntrinsicInst &II,
1946 const SVEIntrinsicInfo &IInfo) {
1947 if (!IInfo.hasGoverningPredicate())
1948 return std::nullopt;
1949
1950 auto *OpPredicate = II.getOperand(i_nocapture: IInfo.getGoverningPredicateOperandIdx());
1951
1952 // If there are no active lanes.
1953 if (match(V: OpPredicate, P: m_ZeroInt())) {
1954 if (IInfo.inactiveLanesTakenFromOperand())
1955 return IC.replaceInstUsesWith(
1956 I&: II, V: II.getOperand(i_nocapture: IInfo.getOperandIdxInactiveLanesTakenFrom()));
1957
1958 if (IInfo.inactiveLanesAreUnused()) {
1959 if (IInfo.resultIsZeroInitialized())
1960 IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty: II.getType()));
1961
1962 return IC.eraseInstFromFunction(I&: II);
1963 }
1964 }
1965
1966 // If there are no inactive lanes.
1967 if (isAllActivePredicate(Pred: OpPredicate)) {
1968 if (IInfo.hasOperandWithNoActiveLanes()) {
1969 unsigned OpIdx = IInfo.getOperandIdxWithNoActiveLanes();
1970 if (!isa<UndefValue>(Val: II.getOperand(i_nocapture: OpIdx)))
1971 return IC.replaceOperand(I&: II, OpNum: OpIdx, V: UndefValue::get(T: II.getType()));
1972 }
1973
1974 if (IInfo.hasMatchingUndefIntrinsic()) {
1975 auto *NewDecl = Intrinsic::getOrInsertDeclaration(
1976 M: II.getModule(), id: IInfo.getMatchingUndefIntrinsic(), OverloadTys: {II.getType()});
1977 II.setCalledFunction(NewDecl);
1978 return &II;
1979 }
1980 }
1981
1982 // Operation specific simplifications.
1983 if (IInfo.hasMatchingIROpode() &&
1984 Instruction::isBinaryOp(Opcode: IInfo.getMatchingIROpode()))
1985 return simplifySVEIntrinsicBinOp(IC, II, IInfo);
1986
1987 return std::nullopt;
1988}
1989
1990// (from_svbool (binop (to_svbool pred) (svbool_t _) (svbool_t _))))
1991// => (binop (pred) (from_svbool _) (from_svbool _))
1992//
1993// The above transformation eliminates a `to_svbool` in the predicate
1994// operand of bitwise operation `binop` by narrowing the vector width of
1995// the operation. For example, it would convert a `<vscale x 16 x i1>
1996// and` into a `<vscale x 4 x i1> and`. This is profitable because
1997// to_svbool must zero the new lanes during widening, whereas
1998// from_svbool is free.
1999static std::optional<Instruction *>
2000tryCombineFromSVBoolBinOp(InstCombiner &IC, IntrinsicInst &II) {
2001 auto BinOp = dyn_cast<IntrinsicInst>(Val: II.getOperand(i_nocapture: 0));
2002 if (!BinOp)
2003 return std::nullopt;
2004
2005 auto IntrinsicID = BinOp->getIntrinsicID();
2006 switch (IntrinsicID) {
2007 case Intrinsic::aarch64_sve_and_z:
2008 case Intrinsic::aarch64_sve_bic_z:
2009 case Intrinsic::aarch64_sve_eor_z:
2010 case Intrinsic::aarch64_sve_nand_z:
2011 case Intrinsic::aarch64_sve_nor_z:
2012 case Intrinsic::aarch64_sve_orn_z:
2013 case Intrinsic::aarch64_sve_orr_z:
2014 break;
2015 default:
2016 return std::nullopt;
2017 }
2018
2019 auto BinOpPred = BinOp->getOperand(i_nocapture: 0);
2020 auto BinOpOp1 = BinOp->getOperand(i_nocapture: 1);
2021 auto BinOpOp2 = BinOp->getOperand(i_nocapture: 2);
2022
2023 auto PredIntr = dyn_cast<IntrinsicInst>(Val: BinOpPred);
2024 if (!PredIntr ||
2025 PredIntr->getIntrinsicID() != Intrinsic::aarch64_sve_convert_to_svbool)
2026 return std::nullopt;
2027
2028 auto PredOp = PredIntr->getOperand(i_nocapture: 0);
2029 auto PredOpTy = cast<VectorType>(Val: PredOp->getType());
2030 if (PredOpTy != II.getType())
2031 return std::nullopt;
2032
2033 SmallVector<Value *> NarrowedBinOpArgs = {PredOp};
2034 auto NarrowBinOpOp1 = IC.Builder.CreateIntrinsic(
2035 ID: Intrinsic::aarch64_sve_convert_from_svbool, OverloadTypes: {PredOpTy}, Args: {BinOpOp1});
2036 NarrowedBinOpArgs.push_back(Elt: NarrowBinOpOp1);
2037 if (BinOpOp1 == BinOpOp2)
2038 NarrowedBinOpArgs.push_back(Elt: NarrowBinOpOp1);
2039 else
2040 NarrowedBinOpArgs.push_back(Elt: IC.Builder.CreateIntrinsic(
2041 ID: Intrinsic::aarch64_sve_convert_from_svbool, OverloadTypes: {PredOpTy}, Args: {BinOpOp2}));
2042
2043 auto NarrowedBinOp =
2044 IC.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {PredOpTy}, Args: NarrowedBinOpArgs);
2045 return IC.replaceInstUsesWith(I&: II, V: NarrowedBinOp);
2046}
2047
2048static std::optional<Instruction *>
2049instCombineConvertFromSVBool(InstCombiner &IC, IntrinsicInst &II) {
2050 // If the reinterpret instruction operand is a PHI Node
2051 if (isa<PHINode>(Val: II.getArgOperand(i: 0)))
2052 return processPhiNode(IC, II);
2053
2054 if (auto BinOpCombine = tryCombineFromSVBoolBinOp(IC, II))
2055 return BinOpCombine;
2056
2057 // Ignore converts to/from svcount_t.
2058 if (isa<TargetExtType>(Val: II.getArgOperand(i: 0)->getType()) ||
2059 isa<TargetExtType>(Val: II.getType()))
2060 return std::nullopt;
2061
2062 SmallVector<Instruction *, 32> CandidatesForRemoval;
2063 Value *Cursor = II.getOperand(i_nocapture: 0), *EarliestReplacement = nullptr;
2064
2065 const auto *IVTy = cast<VectorType>(Val: II.getType());
2066
2067 // Walk the chain of conversions.
2068 while (Cursor) {
2069 // If the type of the cursor has fewer lanes than the final result, zeroing
2070 // must take place, which breaks the equivalence chain.
2071 const auto *CursorVTy = cast<VectorType>(Val: Cursor->getType());
2072 if (CursorVTy->getElementCount().getKnownMinValue() <
2073 IVTy->getElementCount().getKnownMinValue())
2074 break;
2075
2076 // If the cursor has the same type as I, it is a viable replacement.
2077 if (Cursor->getType() == IVTy)
2078 EarliestReplacement = Cursor;
2079
2080 auto *IntrinsicCursor = dyn_cast<IntrinsicInst>(Val: Cursor);
2081
2082 // If this is not an SVE conversion intrinsic, this is the end of the chain.
2083 if (!IntrinsicCursor || !(IntrinsicCursor->getIntrinsicID() ==
2084 Intrinsic::aarch64_sve_convert_to_svbool ||
2085 IntrinsicCursor->getIntrinsicID() ==
2086 Intrinsic::aarch64_sve_convert_from_svbool))
2087 break;
2088
2089 CandidatesForRemoval.insert(I: CandidatesForRemoval.begin(), Elt: IntrinsicCursor);
2090 Cursor = IntrinsicCursor->getOperand(i_nocapture: 0);
2091 }
2092
2093 // If no viable replacement in the conversion chain was found, there is
2094 // nothing to do.
2095 if (!EarliestReplacement)
2096 return std::nullopt;
2097
2098 return IC.replaceInstUsesWith(I&: II, V: EarliestReplacement);
2099}
2100
2101static std::optional<Instruction *> instCombineSVESel(InstCombiner &IC,
2102 IntrinsicInst &II) {
2103 // svsel(ptrue, x, y) => x
2104 auto *OpPredicate = II.getOperand(i_nocapture: 0);
2105 if (isAllActivePredicate(Pred: OpPredicate))
2106 return IC.replaceInstUsesWith(I&: II, V: II.getOperand(i_nocapture: 1));
2107
2108 auto Select =
2109 IC.Builder.CreateSelect(C: OpPredicate, True: II.getOperand(i_nocapture: 1), False: II.getOperand(i_nocapture: 2));
2110 return IC.replaceInstUsesWith(I&: II, V: Select);
2111}
2112
2113static std::optional<Instruction *> instCombineSVEDup(InstCombiner &IC,
2114 IntrinsicInst &II) {
2115 Value *Pg = II.getOperand(i_nocapture: 1);
2116
2117 // sve.dup(V, all_active, X) ==> splat(X)
2118 if (isAllActivePredicate(Pred: Pg)) {
2119 auto *RetTy = cast<ScalableVectorType>(Val: II.getType());
2120 Value *Splat = IC.Builder.CreateVectorSplat(EC: RetTy->getElementCount(),
2121 V: II.getArgOperand(i: 2));
2122 return IC.replaceInstUsesWith(I&: II, V: Splat);
2123 }
2124
2125 if (!match(V: Pg, P: m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
2126 Ops: m_SpecificInt(V: AArch64SVEPredPattern::vl1))))
2127 return std::nullopt;
2128
2129 // sve.dup(V, sve.ptrue(vl1), X) ==> insertelement V, X, 0
2130 Value *Insert = IC.Builder.CreateInsertElement(
2131 Vec: II.getArgOperand(i: 0), NewElt: II.getArgOperand(i: 2), Idx: uint64_t(0));
2132 return IC.replaceInstUsesWith(I&: II, V: Insert);
2133}
2134
2135static std::optional<Instruction *> instCombineSVEDupX(InstCombiner &IC,
2136 IntrinsicInst &II) {
2137 // Replace DupX with a regular IR splat.
2138 auto *RetTy = cast<ScalableVectorType>(Val: II.getType());
2139 Value *Splat = IC.Builder.CreateVectorSplat(EC: RetTy->getElementCount(),
2140 V: II.getArgOperand(i: 0));
2141 Splat->takeName(V: &II);
2142 return IC.replaceInstUsesWith(I&: II, V: Splat);
2143}
2144
2145// xor(cmpne(%pg, %lhs, %rhs), %pg)
2146// -> cmpeq(%pg, %lhs, %rhs)
2147static std::optional<Instruction *> instCombineXorSVECmpCC(InstCombiner &IC,
2148 IntrinsicInst &II) {
2149 if (!II.hasOneUse())
2150 return std::nullopt;
2151 auto *User = cast<Instruction>(Val: *II.user_begin());
2152 if (!match(V: User, P: m_c_Xor(L: m_Specific(V: &II), R: m_Specific(V: II.getOperand(i_nocapture: 0)))))
2153 return std::nullopt;
2154
2155 Intrinsic::ID IID;
2156 switch (II.getIntrinsicID()) {
2157 case Intrinsic::aarch64_sve_cmpne:
2158 IID = Intrinsic::aarch64_sve_cmpeq;
2159 break;
2160 case Intrinsic::aarch64_sve_cmpne_wide:
2161 IID = Intrinsic::aarch64_sve_cmpeq_wide;
2162 break;
2163 case Intrinsic::aarch64_sve_cmpeq:
2164 IID = Intrinsic::aarch64_sve_cmpne;
2165 break;
2166 case Intrinsic::aarch64_sve_cmpeq_wide:
2167 IID = Intrinsic::aarch64_sve_cmpne_wide;
2168 break;
2169 default:
2170 return std::nullopt;
2171 }
2172
2173 IC.Builder.SetInsertPoint(User);
2174 Value *CMPCC = IC.Builder.CreateIntrinsic(
2175 ID: IID, OverloadTypes: II.getOperand(i_nocapture: 1)->getType(),
2176 Args: {II.getOperand(i_nocapture: 0), II.getOperand(i_nocapture: 1), II.getOperand(i_nocapture: 2)});
2177 IC.replaceInstUsesWith(I&: *User, V: CMPCC);
2178 IC.eraseInstFromFunction(I&: *User);
2179 return &II;
2180}
2181
2182// zext(cmpne(ptrue, %v, 0))
2183// -> umin(%pg, %v, 1)
2184static std::optional<Instruction *> instCombineZExtSVECmpNE(InstCombiner &IC,
2185 IntrinsicInst &II) {
2186 if (!isAllActivePredicate(Pred: II.getOperand(i_nocapture: 0)) ||
2187 !match(V: II.getOperand(i_nocapture: 2), P: m_Zero()))
2188 return std::nullopt;
2189
2190 for (auto *U : II.users()) {
2191 if (match(V: U, P: m_ZExt(Op: m_Specific(V: &II)))) {
2192 auto *User = cast<Instruction>(Val: U);
2193 Type *Ty = II.getOperand(i_nocapture: 1)->getType();
2194 if (User->getType() != Ty)
2195 continue;
2196 IC.Builder.SetInsertPoint(User);
2197 Value *UMin = IC.Builder.CreateIntrinsic(
2198 ID: Intrinsic::aarch64_sve_umin, OverloadTypes: Ty,
2199 Args: {II.getOperand(i_nocapture: 0), II.getOperand(i_nocapture: 1), ConstantInt::get(Ty, V: 1)});
2200 IC.replaceInstUsesWith(I&: *User, V: UMin);
2201 IC.eraseInstFromFunction(I&: *User);
2202 return &II;
2203 }
2204 }
2205 return std::nullopt;
2206}
2207
2208static std::optional<Instruction *> instCombineSVECmpNE(InstCombiner &IC,
2209 IntrinsicInst &II) {
2210 LLVMContext &Ctx = II.getContext();
2211
2212 if (auto Res = instCombineXorSVECmpCC(IC, II))
2213 return Res;
2214
2215 if (auto Res = instCombineZExtSVECmpNE(IC, II))
2216 return Res;
2217
2218 if (!isAllActivePredicate(Pred: II.getArgOperand(i: 0)))
2219 return std::nullopt;
2220
2221 // Check that we have a compare of zero..
2222 auto *SplatValue =
2223 dyn_cast_or_null<ConstantInt>(Val: getSplatValue(V: II.getArgOperand(i: 2)));
2224 if (!SplatValue || !SplatValue->isZero())
2225 return std::nullopt;
2226
2227 // ..against a dupq
2228 auto *DupQLane = dyn_cast<IntrinsicInst>(Val: II.getArgOperand(i: 1));
2229 if (!DupQLane ||
2230 DupQLane->getIntrinsicID() != Intrinsic::aarch64_sve_dupq_lane)
2231 return std::nullopt;
2232
2233 // Where the dupq is a lane 0 replicate of a vector insert
2234 auto *DupQLaneIdx = dyn_cast<ConstantInt>(Val: DupQLane->getArgOperand(i: 1));
2235 if (!DupQLaneIdx || !DupQLaneIdx->isZero())
2236 return std::nullopt;
2237
2238 auto *VecIns = dyn_cast<IntrinsicInst>(Val: DupQLane->getArgOperand(i: 0));
2239 if (!VecIns || VecIns->getIntrinsicID() != Intrinsic::vector_insert)
2240 return std::nullopt;
2241
2242 // Where the vector insert is a fixed constant vector insert into undef at
2243 // index zero
2244 if (!isa<UndefValue>(Val: VecIns->getArgOperand(i: 0)))
2245 return std::nullopt;
2246
2247 if (!cast<ConstantInt>(Val: VecIns->getArgOperand(i: 2))->isZero())
2248 return std::nullopt;
2249
2250 auto *ConstVec = dyn_cast<Constant>(Val: VecIns->getArgOperand(i: 1));
2251 if (!ConstVec)
2252 return std::nullopt;
2253
2254 auto *VecTy = dyn_cast<FixedVectorType>(Val: ConstVec->getType());
2255 auto *OutTy = dyn_cast<ScalableVectorType>(Val: II.getType());
2256 if (!VecTy || !OutTy || VecTy->getNumElements() != OutTy->getMinNumElements())
2257 return std::nullopt;
2258
2259 unsigned NumElts = VecTy->getNumElements();
2260 unsigned PredicateBits = 0;
2261
2262 // Expand intrinsic operands to a 16-bit byte level predicate
2263 for (unsigned I = 0; I < NumElts; ++I) {
2264 auto *Arg = dyn_cast<ConstantInt>(Val: ConstVec->getAggregateElement(Elt: I));
2265 if (!Arg)
2266 return std::nullopt;
2267 if (!Arg->isZero())
2268 PredicateBits |= 1 << (I * (16 / NumElts));
2269 }
2270
2271 // If all bits are zero bail early with an empty predicate
2272 if (PredicateBits == 0) {
2273 auto *PFalse = Constant::getNullValue(Ty: II.getType());
2274 PFalse->takeName(V: &II);
2275 return IC.replaceInstUsesWith(I&: II, V: PFalse);
2276 }
2277
2278 // Calculate largest predicate type used (where byte predicate is largest)
2279 unsigned Mask = 8;
2280 for (unsigned I = 0; I < 16; ++I)
2281 if ((PredicateBits & (1 << I)) != 0)
2282 Mask |= (I % 8);
2283
2284 unsigned PredSize = Mask & -Mask;
2285 auto *PredType = ScalableVectorType::get(
2286 ElementType: Type::getInt1Ty(C&: Ctx), MinNumElts: AArch64::SVEBitsPerBlock / (PredSize * 8));
2287
2288 // Ensure all relevant bits are set
2289 for (unsigned I = 0; I < 16; I += PredSize)
2290 if ((PredicateBits & (1 << I)) == 0)
2291 return std::nullopt;
2292
2293 auto *ConvertToSVBool =
2294 IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_to_svbool,
2295 OverloadTypes: PredType, Args: ConstantInt::getTrue(Ty: PredType));
2296 auto *ConvertFromSVBool =
2297 IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_from_svbool,
2298 OverloadTypes: II.getType(), Args: ConvertToSVBool);
2299
2300 ConvertFromSVBool->takeName(V: &II);
2301 return IC.replaceInstUsesWith(I&: II, V: ConvertFromSVBool);
2302}
2303
2304static std::optional<Instruction *> instCombineSVELast(InstCombiner &IC,
2305 IntrinsicInst &II) {
2306 Value *Pg = II.getArgOperand(i: 0);
2307 Value *Vec = II.getArgOperand(i: 1);
2308 auto IntrinsicID = II.getIntrinsicID();
2309 bool IsAfter = IntrinsicID == Intrinsic::aarch64_sve_lasta;
2310
2311 // lastX(splat(X)) --> X
2312 if (auto *SplatVal = getSplatValue(V: Vec))
2313 return IC.replaceInstUsesWith(I&: II, V: SplatVal);
2314
2315 // If x and/or y is a splat value then:
2316 // lastX (binop (x, y)) --> binop(lastX(x), lastX(y))
2317 Value *LHS, *RHS;
2318 if (match(V: Vec, P: m_OneUse(SubPattern: m_BinOp(L: m_Value(V&: LHS), R: m_Value(V&: RHS))))) {
2319 if (isSplatValue(V: LHS) || isSplatValue(V: RHS)) {
2320 auto *OldBinOp = cast<BinaryOperator>(Val: Vec);
2321 auto OpC = OldBinOp->getOpcode();
2322 auto *NewLHS =
2323 IC.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Vec->getType()}, Args: {Pg, LHS});
2324 auto *NewRHS =
2325 IC.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Vec->getType()}, Args: {Pg, RHS});
2326 auto *NewBinOp = BinaryOperator::CreateWithCopiedFlags(
2327 Opc: OpC, V1: NewLHS, V2: NewRHS, CopyO: OldBinOp, Name: OldBinOp->getName(), InsertBefore: II.getIterator());
2328 return IC.replaceInstUsesWith(I&: II, V: NewBinOp);
2329 }
2330 }
2331
2332 auto *C = dyn_cast<Constant>(Val: Pg);
2333 if (IsAfter && C && C->isNullValue()) {
2334 // The intrinsic is extracting lane 0 so use an extract instead.
2335 auto *IdxTy = Type::getInt64Ty(C&: II.getContext());
2336 auto *Extract = ExtractElementInst::Create(Vec, Idx: ConstantInt::get(Ty: IdxTy, V: 0));
2337 Extract->insertBefore(InsertPos: II.getIterator());
2338 Extract->takeName(V: &II);
2339 return IC.replaceInstUsesWith(I&: II, V: Extract);
2340 }
2341
2342 auto *IntrPG = dyn_cast<IntrinsicInst>(Val: Pg);
2343 if (!IntrPG)
2344 return std::nullopt;
2345
2346 if (IntrPG->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)
2347 return std::nullopt;
2348
2349 const auto PTruePattern =
2350 cast<ConstantInt>(Val: IntrPG->getOperand(i_nocapture: 0))->getZExtValue();
2351
2352 // Can the intrinsic's predicate be converted to a known constant index?
2353 unsigned MinNumElts = getNumElementsFromSVEPredPattern(Pattern: PTruePattern);
2354 if (!MinNumElts)
2355 return std::nullopt;
2356
2357 unsigned Idx = MinNumElts - 1;
2358 // Increment the index if extracting the element after the last active
2359 // predicate element.
2360 if (IsAfter)
2361 ++Idx;
2362
2363 // Ignore extracts whose index is larger than the known minimum vector
2364 // length. NOTE: This is an artificial constraint where we prefer to
2365 // maintain what the user asked for until an alternative is proven faster.
2366 auto *PgVTy = cast<ScalableVectorType>(Val: Pg->getType());
2367 if (Idx >= PgVTy->getMinNumElements())
2368 return std::nullopt;
2369
2370 // The intrinsic is extracting a fixed lane so use an extract instead.
2371 auto *IdxTy = Type::getInt64Ty(C&: II.getContext());
2372 auto *Extract = ExtractElementInst::Create(Vec, Idx: ConstantInt::get(Ty: IdxTy, V: Idx));
2373 Extract->insertBefore(InsertPos: II.getIterator());
2374 Extract->takeName(V: &II);
2375 return IC.replaceInstUsesWith(I&: II, V: Extract);
2376}
2377
2378static std::optional<Instruction *> instCombineSVECondLast(InstCombiner &IC,
2379 IntrinsicInst &II) {
2380 // The SIMD&FP variant of CLAST[AB] is significantly faster than the scalar
2381 // integer variant across a variety of micro-architectures. Replace scalar
2382 // integer CLAST[AB] intrinsic with optimal SIMD&FP variant. A simple
2383 // bitcast-to-fp + clast[ab] + bitcast-to-int will cost a cycle or two more
2384 // depending on the micro-architecture, but has been observed as generally
2385 // being faster, particularly when the CLAST[AB] op is a loop-carried
2386 // dependency.
2387 Value *Pg = II.getArgOperand(i: 0);
2388 Value *Fallback = II.getArgOperand(i: 1);
2389 Value *Vec = II.getArgOperand(i: 2);
2390 Type *Ty = II.getType();
2391
2392 if (!Ty->isIntegerTy())
2393 return std::nullopt;
2394
2395 Type *FPTy;
2396 switch (cast<IntegerType>(Val: Ty)->getBitWidth()) {
2397 default:
2398 return std::nullopt;
2399 case 16:
2400 FPTy = IC.Builder.getHalfTy();
2401 break;
2402 case 32:
2403 FPTy = IC.Builder.getFloatTy();
2404 break;
2405 case 64:
2406 FPTy = IC.Builder.getDoubleTy();
2407 break;
2408 }
2409
2410 Value *FPFallBack = IC.Builder.CreateBitCast(V: Fallback, DestTy: FPTy);
2411 auto *FPVTy = VectorType::get(
2412 ElementType: FPTy, EC: cast<VectorType>(Val: Vec->getType())->getElementCount());
2413 Value *FPVec = IC.Builder.CreateBitCast(V: Vec, DestTy: FPVTy);
2414 auto *FPII = IC.Builder.CreateIntrinsic(
2415 ID: II.getIntrinsicID(), OverloadTypes: {FPVec->getType()}, Args: {Pg, FPFallBack, FPVec});
2416 Value *FPIItoInt = IC.Builder.CreateBitCast(V: FPII, DestTy: II.getType());
2417 return IC.replaceInstUsesWith(I&: II, V: FPIItoInt);
2418}
2419
2420static std::optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
2421 IntrinsicInst &II) {
2422 // Replace rdffr with predicated rdffr.z intrinsic, so that optimizePTestInstr
2423 // can work with RDFFR_PP for ptest elimination.
2424 auto *RDFFR = IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_rdffr_z,
2425 Args: ConstantInt::getTrue(Ty: II.getType()));
2426 RDFFR->takeName(V: &II);
2427 return IC.replaceInstUsesWith(I&: II, V: RDFFR);
2428}
2429
2430static std::optional<Instruction *>
2431instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) {
2432 const auto Pattern = cast<ConstantInt>(Val: II.getArgOperand(i: 0))->getZExtValue();
2433
2434 if (Pattern == AArch64SVEPredPattern::all) {
2435 Value *Cnt = IC.Builder.CreateElementCount(
2436 Ty: II.getType(), EC: ElementCount::getScalable(MinVal: NumElts));
2437 Cnt->takeName(V: &II);
2438 return IC.replaceInstUsesWith(I&: II, V: Cnt);
2439 }
2440
2441 unsigned MinNumElts = getNumElementsFromSVEPredPattern(Pattern);
2442
2443 return MinNumElts && NumElts >= MinNumElts
2444 ? std::optional<Instruction *>(IC.replaceInstUsesWith(
2445 I&: II, V: ConstantInt::get(Ty: II.getType(), V: MinNumElts)))
2446 : std::nullopt;
2447}
2448
2449static std::optional<Instruction *>
2450instCombineSMECntsd(InstCombiner &IC, IntrinsicInst &II,
2451 const AArch64Subtarget *ST) {
2452 if (!ST->isStreaming())
2453 return std::nullopt;
2454
2455 // In streaming-mode, aarch64_sme_cntds is equivalent to aarch64_sve_cntd
2456 // with SVEPredPattern::all
2457 Value *Cnt =
2458 IC.Builder.CreateElementCount(Ty: II.getType(), EC: ElementCount::getScalable(MinVal: 2));
2459 Cnt->takeName(V: &II);
2460 return IC.replaceInstUsesWith(I&: II, V: Cnt);
2461}
2462
2463static std::optional<Instruction *> instCombineSVEPTest(InstCombiner &IC,
2464 IntrinsicInst &II) {
2465 Value *PgVal = II.getArgOperand(i: 0);
2466 Value *OpVal = II.getArgOperand(i: 1);
2467
2468 // PTEST_<FIRST|LAST>(X, X) is equivalent to PTEST_ANY(X, X).
2469 // Later optimizations prefer this form.
2470 if (PgVal == OpVal &&
2471 (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_first ||
2472 II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_last)) {
2473 Value *Ops[] = {PgVal, OpVal};
2474 Type *Tys[] = {PgVal->getType()};
2475
2476 auto *PTest =
2477 IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_ptest_any, OverloadTypes: Tys, Args: Ops);
2478 PTest->takeName(V: &II);
2479
2480 return IC.replaceInstUsesWith(I&: II, V: PTest);
2481 }
2482
2483 IntrinsicInst *Pg = dyn_cast<IntrinsicInst>(Val: PgVal);
2484 IntrinsicInst *Op = dyn_cast<IntrinsicInst>(Val: OpVal);
2485
2486 if (!Pg || !Op)
2487 return std::nullopt;
2488
2489 Intrinsic::ID OpIID = Op->getIntrinsicID();
2490
2491 if (Pg->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool &&
2492 OpIID == Intrinsic::aarch64_sve_convert_to_svbool &&
2493 Pg->getArgOperand(i: 0)->getType() == Op->getArgOperand(i: 0)->getType()) {
2494 Value *Ops[] = {Pg->getArgOperand(i: 0), Op->getArgOperand(i: 0)};
2495 Type *Tys[] = {Pg->getArgOperand(i: 0)->getType()};
2496
2497 auto *PTest = IC.Builder.CreateIntrinsic(ID: II.getIntrinsicID(), OverloadTypes: Tys, Args: Ops);
2498
2499 PTest->takeName(V: &II);
2500 return IC.replaceInstUsesWith(I&: II, V: PTest);
2501 }
2502
2503 // Transform PTEST_ANY(X=OP(PG,...), X) -> PTEST_ANY(PG, X)).
2504 // Later optimizations may rewrite sequence to use the flag-setting variant
2505 // of instruction X to remove PTEST.
2506 if ((Pg == Op) && (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_any) &&
2507 ((OpIID == Intrinsic::aarch64_sve_brka_z) ||
2508 (OpIID == Intrinsic::aarch64_sve_brkb_z) ||
2509 (OpIID == Intrinsic::aarch64_sve_brkpa_z) ||
2510 (OpIID == Intrinsic::aarch64_sve_brkpb_z) ||
2511 (OpIID == Intrinsic::aarch64_sve_rdffr_z) ||
2512 (OpIID == Intrinsic::aarch64_sve_and_z) ||
2513 (OpIID == Intrinsic::aarch64_sve_bic_z) ||
2514 (OpIID == Intrinsic::aarch64_sve_eor_z) ||
2515 (OpIID == Intrinsic::aarch64_sve_nand_z) ||
2516 (OpIID == Intrinsic::aarch64_sve_nor_z) ||
2517 (OpIID == Intrinsic::aarch64_sve_orn_z) ||
2518 (OpIID == Intrinsic::aarch64_sve_orr_z))) {
2519 Value *Ops[] = {Pg->getArgOperand(i: 0), Pg};
2520 Type *Tys[] = {Pg->getType()};
2521
2522 auto *PTest = IC.Builder.CreateIntrinsic(ID: II.getIntrinsicID(), OverloadTypes: Tys, Args: Ops);
2523 PTest->takeName(V: &II);
2524
2525 return IC.replaceInstUsesWith(I&: II, V: PTest);
2526 }
2527
2528 return std::nullopt;
2529}
2530
2531template <Intrinsic::ID MulOpc, Intrinsic::ID FuseOpc>
2532static std::optional<Instruction *>
2533instCombineSVEVectorFuseMulAddSub(InstCombiner &IC, IntrinsicInst &II,
2534 bool MergeIntoAddendOp) {
2535 Value *P = II.getOperand(i_nocapture: 0);
2536 Value *MulOp0, *MulOp1, *AddendOp, *Mul;
2537 if (MergeIntoAddendOp) {
2538 AddendOp = II.getOperand(i_nocapture: 1);
2539 Mul = II.getOperand(i_nocapture: 2);
2540 } else {
2541 AddendOp = II.getOperand(i_nocapture: 2);
2542 Mul = II.getOperand(i_nocapture: 1);
2543 }
2544
2545 if (!match(Mul, m_Intrinsic<MulOpc>(m_Specific(V: P), m_Value(V&: MulOp0),
2546 m_Value(V&: MulOp1))))
2547 return std::nullopt;
2548
2549 if (!Mul->hasOneUse())
2550 return std::nullopt;
2551
2552 Instruction *FMFSource = nullptr;
2553 if (II.getType()->isFPOrFPVectorTy()) {
2554 llvm::FastMathFlags FAddFlags = II.getFastMathFlags();
2555 // Stop the combine when the flags on the inputs differ in case dropping
2556 // flags would lead to us missing out on more beneficial optimizations.
2557 if (FAddFlags != cast<CallInst>(Val: Mul)->getFastMathFlags())
2558 return std::nullopt;
2559 if (!FAddFlags.allowContract())
2560 return std::nullopt;
2561 FMFSource = &II;
2562 }
2563
2564 Value *Res;
2565 if (MergeIntoAddendOp)
2566 Res = IC.Builder.CreateIntrinsic(ID: FuseOpc, OverloadTypes: {II.getType()},
2567 Args: {P, AddendOp, MulOp0, MulOp1}, FMFSource);
2568 else
2569 Res = IC.Builder.CreateIntrinsic(ID: FuseOpc, OverloadTypes: {II.getType()},
2570 Args: {P, MulOp0, MulOp1, AddendOp}, FMFSource);
2571
2572 return IC.replaceInstUsesWith(I&: II, V: Res);
2573}
2574
2575static std::optional<Instruction *>
2576instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
2577 Value *Pred = II.getOperand(i_nocapture: 0);
2578 Value *PtrOp = II.getOperand(i_nocapture: 1);
2579 Type *VecTy = II.getType();
2580
2581 if (isAllActivePredicate(Pred)) {
2582 LoadInst *Load = IC.Builder.CreateLoad(Ty: VecTy, Ptr: PtrOp);
2583 Load->copyMetadata(SrcInst: II);
2584 return IC.replaceInstUsesWith(I&: II, V: Load);
2585 }
2586
2587 CallInst *MaskedLoad =
2588 IC.Builder.CreateMaskedLoad(Ty: VecTy, Ptr: PtrOp, Alignment: PtrOp->getPointerAlignment(DL),
2589 Mask: Pred, PassThru: ConstantAggregateZero::get(Ty: VecTy));
2590 MaskedLoad->copyMetadata(SrcInst: II);
2591 return IC.replaceInstUsesWith(I&: II, V: MaskedLoad);
2592}
2593
2594static std::optional<Instruction *>
2595instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
2596 Value *VecOp = II.getOperand(i_nocapture: 0);
2597 Value *Pred = II.getOperand(i_nocapture: 1);
2598 Value *PtrOp = II.getOperand(i_nocapture: 2);
2599
2600 if (isAllActivePredicate(Pred)) {
2601 StoreInst *Store = IC.Builder.CreateStore(Val: VecOp, Ptr: PtrOp);
2602 Store->copyMetadata(SrcInst: II);
2603 return IC.eraseInstFromFunction(I&: II);
2604 }
2605
2606 CallInst *MaskedStore = IC.Builder.CreateMaskedStore(
2607 Val: VecOp, Ptr: PtrOp, Alignment: PtrOp->getPointerAlignment(DL), Mask: Pred);
2608 MaskedStore->copyMetadata(SrcInst: II);
2609 return IC.eraseInstFromFunction(I&: II);
2610}
2611
2612static Instruction::BinaryOps intrinsicIDToBinOpCode(unsigned Intrinsic) {
2613 switch (Intrinsic) {
2614 case Intrinsic::aarch64_sve_fmul_u:
2615 return Instruction::BinaryOps::FMul;
2616 case Intrinsic::aarch64_sve_fadd_u:
2617 return Instruction::BinaryOps::FAdd;
2618 case Intrinsic::aarch64_sve_fsub_u:
2619 return Instruction::BinaryOps::FSub;
2620 default:
2621 return Instruction::BinaryOpsEnd;
2622 }
2623}
2624
2625static std::optional<Instruction *>
2626instCombineSVEVectorBinOp(InstCombiner &IC, IntrinsicInst &II) {
2627 // Bail due to missing support for ISD::STRICT_ scalable vector operations.
2628 if (II.isStrictFP())
2629 return std::nullopt;
2630
2631 auto *OpPredicate = II.getOperand(i_nocapture: 0);
2632 auto BinOpCode = intrinsicIDToBinOpCode(Intrinsic: II.getIntrinsicID());
2633 if (BinOpCode == Instruction::BinaryOpsEnd ||
2634 !isAllActivePredicate(Pred: OpPredicate))
2635 return std::nullopt;
2636 auto BinOp = IC.Builder.CreateBinOpFMF(
2637 Opc: BinOpCode, LHS: II.getOperand(i_nocapture: 1), RHS: II.getOperand(i_nocapture: 2), FMFSource: II.getFastMathFlags());
2638 return IC.replaceInstUsesWith(I&: II, V: BinOp);
2639}
2640
2641static std::optional<Instruction *>
2642instCombineSVEVectorMlaU(InstCombiner &IC, IntrinsicInst &II) {
2643 assert(II.getIntrinsicID() == Intrinsic::aarch64_sve_mla_u &&
2644 "Expected MLA_U intrinsic");
2645 Value *Acc = II.getArgOperand(i: 1);
2646 Value *MulOp0 = II.getArgOperand(i: 2);
2647 Value *MulOp1 = II.getArgOperand(i: 3);
2648
2649 // For mla_u, inactive lanes are undefined, so it is valid to drop the
2650 // predicate when replacing mla_u(acc, x, 1) with add(acc, x) or
2651 // mla_u(acc, x, -1) with sub(acc, x).
2652 if (match(V: MulOp0, P: m_One()))
2653 return IC.replaceInstUsesWith(I&: II, V: IC.Builder.CreateAdd(LHS: Acc, RHS: MulOp1));
2654 if (match(V: MulOp1, P: m_One()))
2655 return IC.replaceInstUsesWith(I&: II, V: IC.Builder.CreateAdd(LHS: Acc, RHS: MulOp0));
2656 if (match(V: MulOp0, P: m_AllOnes()))
2657 return IC.replaceInstUsesWith(I&: II, V: IC.Builder.CreateSub(LHS: Acc, RHS: MulOp1));
2658 if (match(V: MulOp1, P: m_AllOnes()))
2659 return IC.replaceInstUsesWith(I&: II, V: IC.Builder.CreateSub(LHS: Acc, RHS: MulOp0));
2660
2661 if (isa<Constant>(Val: MulOp0) && !isa<Constant>(Val: MulOp1)) {
2662 II.setArgOperand(i: 2, v: MulOp1);
2663 II.setArgOperand(i: 3, v: MulOp0);
2664 return &II;
2665 }
2666
2667 return std::nullopt;
2668}
2669
2670static std::optional<Instruction *>
2671instCombineSVEPairwiseAddLong(InstCombiner &IC, IntrinsicInst &II) {
2672 assert((II.getIntrinsicID() == Intrinsic::aarch64_sve_sadalp ||
2673 II.getIntrinsicID() == Intrinsic::aarch64_sve_uadalp) &&
2674 "Expected SADALP or UADALP intrinsic");
2675
2676 // Simplify add(adalp(pg, zeroinitializer, in), wide_acc)
2677 // -> adalp(pg, wide_acc, in)
2678 auto *User = dyn_cast_or_null<Instruction>(Val: II.getUniqueUndroppableUser());
2679 if (!User || !match(V: II.getArgOperand(i: 1), P: m_Zero()))
2680 return std::nullopt;
2681
2682 Value *Acc;
2683 if (!match(V: User, P: m_c_Add(L: m_Specific(V: &II), R: m_Value(V&: Acc))))
2684 return std::nullopt;
2685
2686 IC.Builder.SetInsertPoint(User);
2687 Value *PairwiseAddLong = IC.Builder.CreateIntrinsic(
2688 ID: II.getIntrinsicID(), OverloadTypes: {II.getType()},
2689 Args: {II.getArgOperand(i: 0), Acc, II.getArgOperand(i: 2)});
2690
2691 IC.replaceInstUsesWith(I&: *User, V: PairwiseAddLong);
2692 IC.eraseInstFromFunction(I&: *User);
2693 return &II; // II is now trivially dead and will get erased.
2694}
2695
2696static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
2697 IntrinsicInst &II) {
2698 if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
2699 Intrinsic::aarch64_sve_mla>(
2700 IC, II, MergeIntoAddendOp: true))
2701 return MLA;
2702 if (auto MAD = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
2703 Intrinsic::aarch64_sve_mad>(
2704 IC, II, MergeIntoAddendOp: false))
2705 return MAD;
2706 return std::nullopt;
2707}
2708
2709static std::optional<Instruction *>
2710instCombineSVEVectorFAdd(InstCombiner &IC, IntrinsicInst &II) {
2711 if (auto FMLA =
2712 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
2713 Intrinsic::aarch64_sve_fmla>(IC, II,
2714 MergeIntoAddendOp: true))
2715 return FMLA;
2716 if (auto FMAD =
2717 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
2718 Intrinsic::aarch64_sve_fmad>(IC, II,
2719 MergeIntoAddendOp: false))
2720 return FMAD;
2721 if (auto FMLA =
2722 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
2723 Intrinsic::aarch64_sve_fmla>(IC, II,
2724 MergeIntoAddendOp: true))
2725 return FMLA;
2726 return std::nullopt;
2727}
2728
2729static std::optional<Instruction *>
2730instCombineSVEVectorFAddU(InstCombiner &IC, IntrinsicInst &II) {
2731 if (auto FMLA =
2732 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
2733 Intrinsic::aarch64_sve_fmla>(IC, II,
2734 MergeIntoAddendOp: true))
2735 return FMLA;
2736 if (auto FMAD =
2737 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
2738 Intrinsic::aarch64_sve_fmad>(IC, II,
2739 MergeIntoAddendOp: false))
2740 return FMAD;
2741 if (auto FMLA_U =
2742 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
2743 Intrinsic::aarch64_sve_fmla_u>(
2744 IC, II, MergeIntoAddendOp: true))
2745 return FMLA_U;
2746 return instCombineSVEVectorBinOp(IC, II);
2747}
2748
2749static std::optional<Instruction *>
2750instCombineSVEVectorFSub(InstCombiner &IC, IntrinsicInst &II) {
2751 if (auto FMLS =
2752 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
2753 Intrinsic::aarch64_sve_fmls>(IC, II,
2754 MergeIntoAddendOp: true))
2755 return FMLS;
2756 if (auto FMSB =
2757 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
2758 Intrinsic::aarch64_sve_fnmsb>(
2759 IC, II, MergeIntoAddendOp: false))
2760 return FMSB;
2761 if (auto FMLS =
2762 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
2763 Intrinsic::aarch64_sve_fmls>(IC, II,
2764 MergeIntoAddendOp: true))
2765 return FMLS;
2766 return std::nullopt;
2767}
2768
2769static std::optional<Instruction *>
2770instCombineSVEVectorFSubU(InstCombiner &IC, IntrinsicInst &II) {
2771 if (auto FMLS =
2772 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
2773 Intrinsic::aarch64_sve_fmls>(IC, II,
2774 MergeIntoAddendOp: true))
2775 return FMLS;
2776 if (auto FMSB =
2777 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
2778 Intrinsic::aarch64_sve_fnmsb>(
2779 IC, II, MergeIntoAddendOp: false))
2780 return FMSB;
2781 if (auto FMLS_U =
2782 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
2783 Intrinsic::aarch64_sve_fmls_u>(
2784 IC, II, MergeIntoAddendOp: true))
2785 return FMLS_U;
2786 return instCombineSVEVectorBinOp(IC, II);
2787}
2788
2789static std::optional<Instruction *> instCombineSVEVectorSub(InstCombiner &IC,
2790 IntrinsicInst &II) {
2791 if (auto MLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
2792 Intrinsic::aarch64_sve_mls>(
2793 IC, II, MergeIntoAddendOp: true))
2794 return MLS;
2795 return std::nullopt;
2796}
2797
2798static std::optional<Instruction *> instCombineSVEUnpack(InstCombiner &IC,
2799 IntrinsicInst &II) {
2800 Value *UnpackArg = II.getArgOperand(i: 0);
2801 auto *RetTy = cast<ScalableVectorType>(Val: II.getType());
2802 bool IsSigned = II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpkhi ||
2803 II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpklo;
2804
2805 // Hi = uunpkhi(splat(X)) --> Hi = splat(extend(X))
2806 // Lo = uunpklo(splat(X)) --> Lo = splat(extend(X))
2807 if (auto *ScalarArg = getSplatValue(V: UnpackArg)) {
2808 ScalarArg =
2809 IC.Builder.CreateIntCast(V: ScalarArg, DestTy: RetTy->getScalarType(), isSigned: IsSigned);
2810 Value *NewVal =
2811 IC.Builder.CreateVectorSplat(EC: RetTy->getElementCount(), V: ScalarArg);
2812 NewVal->takeName(V: &II);
2813 return IC.replaceInstUsesWith(I&: II, V: NewVal);
2814 }
2815
2816 return std::nullopt;
2817}
2818static std::optional<Instruction *> instCombineSVETBL(InstCombiner &IC,
2819 IntrinsicInst &II) {
2820 auto *OpVal = II.getOperand(i_nocapture: 0);
2821 auto *OpIndices = II.getOperand(i_nocapture: 1);
2822 VectorType *VTy = cast<VectorType>(Val: II.getType());
2823
2824 // Check whether OpIndices is a constant splat value < minimal element count
2825 // of result.
2826 auto *SplatValue = dyn_cast_or_null<ConstantInt>(Val: getSplatValue(V: OpIndices));
2827 if (!SplatValue ||
2828 SplatValue->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue()))
2829 return std::nullopt;
2830
2831 // Convert sve_tbl(OpVal sve_dup_x(SplatValue)) to
2832 // splat_vector(extractelement(OpVal, SplatValue)) for further optimization.
2833 auto *Extract = IC.Builder.CreateExtractElement(Vec: OpVal, Idx: SplatValue);
2834 auto *VectorSplat =
2835 IC.Builder.CreateVectorSplat(EC: VTy->getElementCount(), V: Extract);
2836
2837 VectorSplat->takeName(V: &II);
2838 return IC.replaceInstUsesWith(I&: II, V: VectorSplat);
2839}
2840
2841static std::optional<Instruction *> instCombineSVEUzp1(InstCombiner &IC,
2842 IntrinsicInst &II) {
2843 Value *A, *B;
2844 Type *RetTy = II.getType();
2845 constexpr Intrinsic::ID FromSVB = Intrinsic::aarch64_sve_convert_from_svbool;
2846 constexpr Intrinsic::ID ToSVB = Intrinsic::aarch64_sve_convert_to_svbool;
2847
2848 // uzp1(to_svbool(A), to_svbool(B)) --> <A, B>
2849 // uzp1(from_svbool(to_svbool(A)), from_svbool(to_svbool(B))) --> <A, B>
2850 if ((match(V: II.getArgOperand(i: 0),
2851 P: m_Intrinsic<FromSVB>(Ops: m_Intrinsic<ToSVB>(Ops: m_Value(V&: A)))) &&
2852 match(V: II.getArgOperand(i: 1),
2853 P: m_Intrinsic<FromSVB>(Ops: m_Intrinsic<ToSVB>(Ops: m_Value(V&: B))))) ||
2854 (match(V: II.getArgOperand(i: 0), P: m_Intrinsic<ToSVB>(Ops: m_Value(V&: A))) &&
2855 match(V: II.getArgOperand(i: 1), P: m_Intrinsic<ToSVB>(Ops: m_Value(V&: B))))) {
2856 auto *TyA = cast<ScalableVectorType>(Val: A->getType());
2857 if (TyA == B->getType() &&
2858 RetTy == ScalableVectorType::getDoubleElementsVectorType(VTy: TyA)) {
2859 auto *SubVec = IC.Builder.CreateInsertVector(
2860 DstType: RetTy, SrcVec: PoisonValue::get(T: RetTy), SubVec: A, Idx: uint64_t(0));
2861 auto *ConcatVec = IC.Builder.CreateInsertVector(DstType: RetTy, SrcVec: SubVec, SubVec: B,
2862 Idx: TyA->getMinNumElements());
2863 ConcatVec->takeName(V: &II);
2864 return IC.replaceInstUsesWith(I&: II, V: ConcatVec);
2865 }
2866 }
2867
2868 return std::nullopt;
2869}
2870
2871static std::optional<Instruction *> instCombineSVEZip(InstCombiner &IC,
2872 IntrinsicInst &II) {
2873 // zip1(uzp1(A, B), uzp2(A, B)) --> A
2874 // zip2(uzp1(A, B), uzp2(A, B)) --> B
2875 Value *A, *B;
2876 if (match(V: II.getArgOperand(i: 0),
2877 P: m_Intrinsic<Intrinsic::aarch64_sve_uzp1>(Ops: m_Value(V&: A), Ops: m_Value(V&: B))) &&
2878 match(V: II.getArgOperand(i: 1), P: m_Intrinsic<Intrinsic::aarch64_sve_uzp2>(
2879 Ops: m_Specific(V: A), Ops: m_Specific(V: B))))
2880 return IC.replaceInstUsesWith(
2881 I&: II, V: (II.getIntrinsicID() == Intrinsic::aarch64_sve_zip1 ? A : B));
2882
2883 return std::nullopt;
2884}
2885
2886static std::optional<Instruction *>
2887instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) {
2888 Value *Mask = II.getOperand(i_nocapture: 0);
2889 Value *BasePtr = II.getOperand(i_nocapture: 1);
2890 Value *Index = II.getOperand(i_nocapture: 2);
2891 Type *Ty = II.getType();
2892 Value *PassThru = ConstantAggregateZero::get(Ty);
2893
2894 // Contiguous gather => masked load.
2895 // (sve.ld1.gather.index Mask BasePtr (sve.index IndexBase 1))
2896 // => (masked.load (gep BasePtr IndexBase) Align Mask zeroinitializer)
2897 Value *IndexBase;
2898 if (match(V: Index, P: m_Intrinsic<Intrinsic::aarch64_sve_index>(Ops: m_Value(V&: IndexBase),
2899 Ops: m_One()))) {
2900 Align Alignment =
2901 BasePtr->getPointerAlignment(DL: II.getDataLayout());
2902
2903 Value *Ptr = IC.Builder.CreateGEP(Ty: cast<VectorType>(Val: Ty)->getElementType(),
2904 Ptr: BasePtr, IdxList: IndexBase);
2905 CallInst *MaskedLoad =
2906 IC.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, Mask, PassThru);
2907 MaskedLoad->takeName(V: &II);
2908 return IC.replaceInstUsesWith(I&: II, V: MaskedLoad);
2909 }
2910
2911 return std::nullopt;
2912}
2913
2914static std::optional<Instruction *>
2915instCombineST1ScatterIndex(InstCombiner &IC, IntrinsicInst &II) {
2916 Value *Val = II.getOperand(i_nocapture: 0);
2917 Value *Mask = II.getOperand(i_nocapture: 1);
2918 Value *BasePtr = II.getOperand(i_nocapture: 2);
2919 Value *Index = II.getOperand(i_nocapture: 3);
2920 Type *Ty = Val->getType();
2921
2922 // Contiguous scatter => masked store.
2923 // (sve.st1.scatter.index Value Mask BasePtr (sve.index IndexBase 1))
2924 // => (masked.store Value (gep BasePtr IndexBase) Align Mask)
2925 Value *IndexBase;
2926 if (match(V: Index, P: m_Intrinsic<Intrinsic::aarch64_sve_index>(Ops: m_Value(V&: IndexBase),
2927 Ops: m_One()))) {
2928 Align Alignment =
2929 BasePtr->getPointerAlignment(DL: II.getDataLayout());
2930
2931 Value *Ptr = IC.Builder.CreateGEP(Ty: cast<VectorType>(Val: Ty)->getElementType(),
2932 Ptr: BasePtr, IdxList: IndexBase);
2933 (void)IC.Builder.CreateMaskedStore(Val, Ptr, Alignment, Mask);
2934
2935 return IC.eraseInstFromFunction(I&: II);
2936 }
2937
2938 return std::nullopt;
2939}
2940
2941static std::optional<Instruction *> instCombineSVESDIV(InstCombiner &IC,
2942 IntrinsicInst &II) {
2943 Type *Int32Ty = IC.Builder.getInt32Ty();
2944 Value *Pred = II.getOperand(i_nocapture: 0);
2945 Value *Vec = II.getOperand(i_nocapture: 1);
2946 Value *DivVec = II.getOperand(i_nocapture: 2);
2947
2948 Value *SplatValue = getSplatValue(V: DivVec);
2949 ConstantInt *SplatConstantInt = dyn_cast_or_null<ConstantInt>(Val: SplatValue);
2950 if (!SplatConstantInt)
2951 return std::nullopt;
2952
2953 APInt Divisor = SplatConstantInt->getValue();
2954 const int64_t DivisorValue = Divisor.getSExtValue();
2955 if (DivisorValue == -1)
2956 return std::nullopt;
2957 if (DivisorValue == 1)
2958 IC.replaceInstUsesWith(I&: II, V: Vec);
2959
2960 if (Divisor.isPowerOf2()) {
2961 Constant *DivisorLog2 = ConstantInt::get(Ty: Int32Ty, V: Divisor.logBase2());
2962 auto ASRD = IC.Builder.CreateIntrinsic(
2963 ID: Intrinsic::aarch64_sve_asrd, OverloadTypes: {II.getType()}, Args: {Pred, Vec, DivisorLog2});
2964 return IC.replaceInstUsesWith(I&: II, V: ASRD);
2965 }
2966 if (Divisor.isNegatedPowerOf2()) {
2967 Divisor.negate();
2968 Constant *DivisorLog2 = ConstantInt::get(Ty: Int32Ty, V: Divisor.logBase2());
2969 auto ASRD = IC.Builder.CreateIntrinsic(
2970 ID: Intrinsic::aarch64_sve_asrd, OverloadTypes: {II.getType()}, Args: {Pred, Vec, DivisorLog2});
2971 auto NEG = IC.Builder.CreateIntrinsic(
2972 ID: Intrinsic::aarch64_sve_neg, OverloadTypes: {ASRD->getType()}, Args: {ASRD, Pred, ASRD});
2973 return IC.replaceInstUsesWith(I&: II, V: NEG);
2974 }
2975
2976 return std::nullopt;
2977}
2978
2979bool SimplifyValuePattern(SmallVector<Value *> &Vec, bool AllowPoison) {
2980 size_t VecSize = Vec.size();
2981 if (VecSize == 1)
2982 return true;
2983 if (!isPowerOf2_64(Value: VecSize))
2984 return false;
2985 size_t HalfVecSize = VecSize / 2;
2986
2987 for (auto LHS = Vec.begin(), RHS = Vec.begin() + HalfVecSize;
2988 RHS != Vec.end(); LHS++, RHS++) {
2989 if (*LHS != nullptr && *RHS != nullptr) {
2990 if (*LHS == *RHS)
2991 continue;
2992 else
2993 return false;
2994 }
2995 if (!AllowPoison)
2996 return false;
2997 if (*LHS == nullptr && *RHS != nullptr)
2998 *LHS = *RHS;
2999 }
3000
3001 Vec.resize(N: HalfVecSize);
3002 SimplifyValuePattern(Vec, AllowPoison);
3003 return true;
3004}
3005
3006// Try to simplify dupqlane patterns like dupqlane(f32 A, f32 B, f32 A, f32 B)
3007// to dupqlane(f64(C)) where C is A concatenated with B
3008static std::optional<Instruction *> instCombineSVEDupqLane(InstCombiner &IC,
3009 IntrinsicInst &II) {
3010 Value *CurrentInsertElt = nullptr, *Default = nullptr;
3011 if (!match(V: II.getOperand(i_nocapture: 0),
3012 P: m_Intrinsic<Intrinsic::vector_insert>(
3013 Ops: m_Value(V&: Default), Ops: m_Value(V&: CurrentInsertElt), Ops: m_Value())) ||
3014 !isa<FixedVectorType>(Val: CurrentInsertElt->getType()))
3015 return std::nullopt;
3016 auto IIScalableTy = cast<ScalableVectorType>(Val: II.getType());
3017
3018 // Insert the scalars into a container ordered by InsertElement index
3019 SmallVector<Value *> Elts(IIScalableTy->getMinNumElements(), nullptr);
3020 while (auto InsertElt = dyn_cast<InsertElementInst>(Val: CurrentInsertElt)) {
3021 auto Idx = cast<ConstantInt>(Val: InsertElt->getOperand(i_nocapture: 2));
3022 Elts[Idx->getValue().getZExtValue()] = InsertElt->getOperand(i_nocapture: 1);
3023 CurrentInsertElt = InsertElt->getOperand(i_nocapture: 0);
3024 }
3025
3026 bool AllowPoison =
3027 isa<PoisonValue>(Val: CurrentInsertElt) && isa<PoisonValue>(Val: Default);
3028 if (!SimplifyValuePattern(Vec&: Elts, AllowPoison))
3029 return std::nullopt;
3030
3031 // Rebuild the simplified chain of InsertElements. e.g. (a, b, a, b) as (a, b)
3032 Value *InsertEltChain = PoisonValue::get(T: CurrentInsertElt->getType());
3033 for (size_t I = 0; I < Elts.size(); I++) {
3034 if (Elts[I] == nullptr)
3035 continue;
3036 InsertEltChain = IC.Builder.CreateInsertElement(Vec: InsertEltChain, NewElt: Elts[I],
3037 Idx: IC.Builder.getInt64(C: I));
3038 }
3039 if (InsertEltChain == nullptr)
3040 return std::nullopt;
3041
3042 // Splat the simplified sequence, e.g. (f16 a, f16 b, f16 c, f16 d) as one i64
3043 // value or (f16 a, f16 b) as one i32 value. This requires an InsertSubvector
3044 // be bitcast to a type wide enough to fit the sequence, be splatted, and then
3045 // be narrowed back to the original type.
3046 unsigned PatternWidth = IIScalableTy->getScalarSizeInBits() * Elts.size();
3047 unsigned PatternElementCount = IIScalableTy->getScalarSizeInBits() *
3048 IIScalableTy->getMinNumElements() /
3049 PatternWidth;
3050
3051 IntegerType *WideTy = IC.Builder.getIntNTy(N: PatternWidth);
3052 auto *WideScalableTy = ScalableVectorType::get(ElementType: WideTy, MinNumElts: PatternElementCount);
3053 auto *WideShuffleMaskTy =
3054 ScalableVectorType::get(ElementType: IC.Builder.getInt32Ty(), MinNumElts: PatternElementCount);
3055
3056 auto InsertSubvector = IC.Builder.CreateInsertVector(
3057 DstType: II.getType(), SrcVec: PoisonValue::get(T: II.getType()), SubVec: InsertEltChain,
3058 Idx: uint64_t(0));
3059 auto WideBitcast =
3060 IC.Builder.CreateBitOrPointerCast(V: InsertSubvector, DestTy: WideScalableTy);
3061 auto WideShuffleMask = ConstantAggregateZero::get(Ty: WideShuffleMaskTy);
3062 auto WideShuffle = IC.Builder.CreateShuffleVector(
3063 V1: WideBitcast, V2: PoisonValue::get(T: WideScalableTy), Mask: WideShuffleMask);
3064 auto NarrowBitcast =
3065 IC.Builder.CreateBitOrPointerCast(V: WideShuffle, DestTy: II.getType());
3066
3067 return IC.replaceInstUsesWith(I&: II, V: NarrowBitcast);
3068}
3069
3070static std::optional<Instruction *> instCombineMaxMinNM(InstCombiner &IC,
3071 IntrinsicInst &II) {
3072 Value *A = II.getArgOperand(i: 0);
3073 Value *B = II.getArgOperand(i: 1);
3074 if (A == B)
3075 return IC.replaceInstUsesWith(I&: II, V: A);
3076
3077 return std::nullopt;
3078}
3079
3080static std::optional<Instruction *> instCombineSVESrshl(InstCombiner &IC,
3081 IntrinsicInst &II) {
3082 Value *Pred = II.getOperand(i_nocapture: 0);
3083 Value *Vec = II.getOperand(i_nocapture: 1);
3084 Value *Shift = II.getOperand(i_nocapture: 2);
3085
3086 // Convert SRSHL into the simpler LSL intrinsic when fed by an ABS intrinsic.
3087 Value *AbsPred, *MergedValue;
3088 if (!match(V: Vec, P: m_Intrinsic<Intrinsic::aarch64_sve_sqabs>(
3089 Ops: m_Value(V&: MergedValue), Ops: m_Value(V&: AbsPred), Ops: m_Value())) &&
3090 !match(V: Vec, P: m_Intrinsic<Intrinsic::aarch64_sve_abs>(
3091 Ops: m_Value(V&: MergedValue), Ops: m_Value(V&: AbsPred), Ops: m_Value())))
3092
3093 return std::nullopt;
3094
3095 // Transform is valid if any of the following are true:
3096 // * The ABS merge value is an undef or non-negative
3097 // * The ABS predicate is all active
3098 // * The ABS predicate and the SRSHL predicates are the same
3099 if (!isa<UndefValue>(Val: MergedValue) && !match(V: MergedValue, P: m_NonNegative()) &&
3100 AbsPred != Pred && !isAllActivePredicate(Pred: AbsPred))
3101 return std::nullopt;
3102
3103 // Only valid when the shift amount is non-negative, otherwise the rounding
3104 // behaviour of SRSHL cannot be ignored.
3105 if (!match(V: Shift, P: m_NonNegative()))
3106 return std::nullopt;
3107
3108 auto LSL = IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_lsl,
3109 OverloadTypes: {II.getType()}, Args: {Pred, Vec, Shift});
3110
3111 return IC.replaceInstUsesWith(I&: II, V: LSL);
3112}
3113
3114static std::optional<Instruction *> instCombineSVEInsr(InstCombiner &IC,
3115 IntrinsicInst &II) {
3116 Value *Vec = II.getOperand(i_nocapture: 0);
3117
3118 if (getSplatValue(V: Vec) == II.getOperand(i_nocapture: 1))
3119 return IC.replaceInstUsesWith(I&: II, V: Vec);
3120
3121 return std::nullopt;
3122}
3123
3124static std::optional<Instruction *> instCombineDMB(InstCombiner &IC,
3125 IntrinsicInst &II) {
3126 // If this barrier is post-dominated by identical one we can remove it
3127 auto *NI = II.getNextNode();
3128 unsigned LookaheadThreshold = DMBLookaheadThreshold;
3129 auto CanSkipOver = [](Instruction *I) {
3130 return !I->mayReadOrWriteMemory() && !I->mayHaveSideEffects();
3131 };
3132 while (LookaheadThreshold-- && CanSkipOver(NI)) {
3133 auto *NIBB = NI->getParent();
3134 NI = NI->getNextNode();
3135 if (!NI) {
3136 if (auto *SuccBB = NIBB->getUniqueSuccessor())
3137 NI = &*SuccBB->getFirstNonPHIOrDbgOrLifetime();
3138 else
3139 break;
3140 }
3141 }
3142 auto *NextII = dyn_cast_or_null<IntrinsicInst>(Val: NI);
3143 if (NextII && II.isIdenticalTo(I: NextII))
3144 return IC.eraseInstFromFunction(I&: II);
3145
3146 return std::nullopt;
3147}
3148
3149static std::optional<Instruction *> instCombineWhilelo(InstCombiner &IC,
3150 IntrinsicInst &II) {
3151 return IC.replaceInstUsesWith(
3152 I&: II,
3153 V: IC.Builder.CreateIntrinsic(ID: Intrinsic::get_active_lane_mask,
3154 OverloadTypes: {II.getType(), II.getOperand(i_nocapture: 0)->getType()},
3155 Args: {II.getOperand(i_nocapture: 0), II.getOperand(i_nocapture: 1)}));
3156}
3157
3158static std::optional<Instruction *> instCombinePTrue(InstCombiner &IC,
3159 IntrinsicInst &II) {
3160 unsigned PredPattern = cast<ConstantInt>(Val: II.getOperand(i_nocapture: 0))->getZExtValue();
3161 // SVE vector length is a power-of-two, thus pow2 is synonymous with all.
3162 if (PredPattern == AArch64SVEPredPattern::all ||
3163 PredPattern == AArch64SVEPredPattern::pow2)
3164 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getTrue(Ty: II.getType()));
3165 return std::nullopt;
3166}
3167
3168static std::optional<Instruction *> instCombineSVEUxt(InstCombiner &IC,
3169 IntrinsicInst &II,
3170 unsigned NumBits) {
3171 Value *Passthru = II.getOperand(i_nocapture: 0);
3172 Value *Pg = II.getOperand(i_nocapture: 1);
3173 Value *Op = II.getOperand(i_nocapture: 2);
3174
3175 // Convert UXT[BHW] to AND.
3176 if (isa<UndefValue>(Val: Passthru) || isAllActivePredicate(Pred: Pg)) {
3177 auto *Ty = cast<VectorType>(Val: II.getType());
3178 auto MaskValue = APInt::getLowBitsSet(numBits: Ty->getScalarSizeInBits(), loBitsSet: NumBits);
3179 auto *Mask = ConstantInt::get(Ty, V: MaskValue);
3180 auto *And = IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_and_u, OverloadTypes: {Ty},
3181 Args: {Pg, Op, Mask});
3182 return IC.replaceInstUsesWith(I&: II, V: And);
3183 }
3184
3185 return std::nullopt;
3186}
3187
3188static std::optional<Instruction *>
3189instCombineInStreamingMode(InstCombiner &IC, IntrinsicInst &II) {
3190 SMEAttrs FnSMEAttrs(*II.getFunction());
3191 bool IsStreaming = FnSMEAttrs.hasStreamingInterfaceOrBody();
3192 if (IsStreaming || !FnSMEAttrs.hasStreamingCompatibleInterface())
3193 return IC.replaceInstUsesWith(
3194 I&: II, V: ConstantInt::getBool(Ty: II.getType(), V: IsStreaming));
3195 return std::nullopt;
3196}
3197
3198std::optional<Instruction *>
3199AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
3200 IntrinsicInst &II) const {
3201 const SVEIntrinsicInfo &IInfo = constructSVEIntrinsicInfo(II);
3202 if (std::optional<Instruction *> I = simplifySVEIntrinsic(IC, II, IInfo))
3203 return I;
3204
3205 Intrinsic::ID IID = II.getIntrinsicID();
3206 switch (IID) {
3207 default:
3208 break;
3209 case Intrinsic::aarch64_dmb:
3210 return instCombineDMB(IC, II);
3211 case Intrinsic::aarch64_neon_fmaxnm:
3212 case Intrinsic::aarch64_neon_fminnm:
3213 return instCombineMaxMinNM(IC, II);
3214 case Intrinsic::aarch64_sve_convert_from_svbool:
3215 return instCombineConvertFromSVBool(IC, II);
3216 case Intrinsic::aarch64_sve_dup:
3217 return instCombineSVEDup(IC, II);
3218 case Intrinsic::aarch64_sve_dup_x:
3219 return instCombineSVEDupX(IC, II);
3220 case Intrinsic::aarch64_sve_cmpeq:
3221 case Intrinsic::aarch64_sve_cmpeq_wide:
3222 return instCombineXorSVECmpCC(IC, II);
3223 case Intrinsic::aarch64_sve_cmpne:
3224 case Intrinsic::aarch64_sve_cmpne_wide:
3225 return instCombineSVECmpNE(IC, II);
3226 case Intrinsic::aarch64_sve_rdffr:
3227 return instCombineRDFFR(IC, II);
3228 case Intrinsic::aarch64_sve_lasta:
3229 case Intrinsic::aarch64_sve_lastb:
3230 return instCombineSVELast(IC, II);
3231 case Intrinsic::aarch64_sve_clasta_n:
3232 case Intrinsic::aarch64_sve_clastb_n:
3233 return instCombineSVECondLast(IC, II);
3234 case Intrinsic::aarch64_sve_cntd:
3235 return instCombineSVECntElts(IC, II, NumElts: 2);
3236 case Intrinsic::aarch64_sve_cntw:
3237 return instCombineSVECntElts(IC, II, NumElts: 4);
3238 case Intrinsic::aarch64_sve_cnth:
3239 return instCombineSVECntElts(IC, II, NumElts: 8);
3240 case Intrinsic::aarch64_sve_cntb:
3241 return instCombineSVECntElts(IC, II, NumElts: 16);
3242 case Intrinsic::aarch64_sme_cntsd:
3243 return instCombineSMECntsd(IC, II, ST);
3244 case Intrinsic::aarch64_sve_ptest_any:
3245 case Intrinsic::aarch64_sve_ptest_first:
3246 case Intrinsic::aarch64_sve_ptest_last:
3247 return instCombineSVEPTest(IC, II);
3248 case Intrinsic::aarch64_sve_fadd:
3249 return instCombineSVEVectorFAdd(IC, II);
3250 case Intrinsic::aarch64_sve_fadd_u:
3251 return instCombineSVEVectorFAddU(IC, II);
3252 case Intrinsic::aarch64_sve_fmul_u:
3253 return instCombineSVEVectorBinOp(IC, II);
3254 case Intrinsic::aarch64_sve_fsub:
3255 return instCombineSVEVectorFSub(IC, II);
3256 case Intrinsic::aarch64_sve_fsub_u:
3257 return instCombineSVEVectorFSubU(IC, II);
3258 case Intrinsic::aarch64_sve_add:
3259 return instCombineSVEVectorAdd(IC, II);
3260 case Intrinsic::aarch64_sve_add_u:
3261 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u,
3262 Intrinsic::aarch64_sve_mla_u>(
3263 IC, II, MergeIntoAddendOp: true);
3264 case Intrinsic::aarch64_sve_mla_u:
3265 return instCombineSVEVectorMlaU(IC, II);
3266 case Intrinsic::aarch64_sve_sadalp:
3267 case Intrinsic::aarch64_sve_uadalp:
3268 return instCombineSVEPairwiseAddLong(IC, II);
3269 case Intrinsic::aarch64_sve_sub:
3270 return instCombineSVEVectorSub(IC, II);
3271 case Intrinsic::aarch64_sve_sub_u:
3272 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u,
3273 Intrinsic::aarch64_sve_mls_u>(
3274 IC, II, MergeIntoAddendOp: true);
3275 case Intrinsic::aarch64_sve_tbl:
3276 return instCombineSVETBL(IC, II);
3277 case Intrinsic::aarch64_sve_uunpkhi:
3278 case Intrinsic::aarch64_sve_uunpklo:
3279 case Intrinsic::aarch64_sve_sunpkhi:
3280 case Intrinsic::aarch64_sve_sunpklo:
3281 return instCombineSVEUnpack(IC, II);
3282 case Intrinsic::aarch64_sve_uzp1:
3283 return instCombineSVEUzp1(IC, II);
3284 case Intrinsic::aarch64_sve_zip1:
3285 case Intrinsic::aarch64_sve_zip2:
3286 return instCombineSVEZip(IC, II);
3287 case Intrinsic::aarch64_sve_ld1_gather_index:
3288 return instCombineLD1GatherIndex(IC, II);
3289 case Intrinsic::aarch64_sve_st1_scatter_index:
3290 return instCombineST1ScatterIndex(IC, II);
3291 case Intrinsic::aarch64_sve_ld1:
3292 return instCombineSVELD1(IC, II, DL);
3293 case Intrinsic::aarch64_sve_st1:
3294 return instCombineSVEST1(IC, II, DL);
3295 case Intrinsic::aarch64_sve_sdiv:
3296 return instCombineSVESDIV(IC, II);
3297 case Intrinsic::aarch64_sve_sel:
3298 return instCombineSVESel(IC, II);
3299 case Intrinsic::aarch64_sve_srshl:
3300 return instCombineSVESrshl(IC, II);
3301 case Intrinsic::aarch64_sve_dupq_lane:
3302 return instCombineSVEDupqLane(IC, II);
3303 case Intrinsic::aarch64_sve_insr:
3304 return instCombineSVEInsr(IC, II);
3305 case Intrinsic::aarch64_sve_whilelo:
3306 return instCombineWhilelo(IC, II);
3307 case Intrinsic::aarch64_sve_ptrue:
3308 return instCombinePTrue(IC, II);
3309 case Intrinsic::aarch64_sve_uxtb:
3310 return instCombineSVEUxt(IC, II, NumBits: 8);
3311 case Intrinsic::aarch64_sve_uxth:
3312 return instCombineSVEUxt(IC, II, NumBits: 16);
3313 case Intrinsic::aarch64_sve_uxtw:
3314 return instCombineSVEUxt(IC, II, NumBits: 32);
3315 case Intrinsic::aarch64_sme_in_streaming_mode:
3316 return instCombineInStreamingMode(IC, II);
3317 }
3318
3319 return std::nullopt;
3320}
3321
3322std::optional<Value *> AArch64TTIImpl::simplifyDemandedVectorEltsIntrinsic(
3323 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,
3324 APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
3325 std::function<void(Instruction *, unsigned, APInt, APInt &)>
3326 SimplifyAndSetOp) const {
3327 switch (II.getIntrinsicID()) {
3328 default:
3329 break;
3330 case Intrinsic::aarch64_neon_fcvtxn:
3331 case Intrinsic::aarch64_neon_rshrn:
3332 case Intrinsic::aarch64_neon_sqrshrn:
3333 case Intrinsic::aarch64_neon_sqrshrun:
3334 case Intrinsic::aarch64_neon_sqshrn:
3335 case Intrinsic::aarch64_neon_sqshrun:
3336 case Intrinsic::aarch64_neon_sqxtn:
3337 case Intrinsic::aarch64_neon_sqxtun:
3338 case Intrinsic::aarch64_neon_uqrshrn:
3339 case Intrinsic::aarch64_neon_uqshrn:
3340 case Intrinsic::aarch64_neon_uqxtn:
3341 SimplifyAndSetOp(&II, 0, OrigDemandedElts, UndefElts);
3342 break;
3343 }
3344
3345 return std::nullopt;
3346}
3347
3348bool AArch64TTIImpl::enableScalableVectorization() const {
3349 return ST->isSVEAvailable() || (ST->isSVEorStreamingSVEAvailable() &&
3350 EnableScalableAutovecInStreamingMode);
3351}
3352
3353TypeSize
3354AArch64TTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
3355 switch (K) {
3356 case TargetTransformInfo::RGK_Scalar:
3357 return TypeSize::getFixed(ExactSize: 64);
3358 case TargetTransformInfo::RGK_FixedWidthVector:
3359 if (ST->useSVEForFixedLengthVectors() &&
3360 (ST->isSVEAvailable() || EnableFixedwidthAutovecInStreamingMode))
3361 return TypeSize::getFixed(
3362 ExactSize: std::max(a: ST->getMinSVEVectorSizeInBits(), b: 128u));
3363 else if (ST->isNeonAvailable())
3364 return TypeSize::getFixed(ExactSize: 128);
3365 else
3366 return TypeSize::getFixed(ExactSize: 0);
3367 case TargetTransformInfo::RGK_ScalableVector:
3368 if (ST->isSVEAvailable() || (ST->isSVEorStreamingSVEAvailable() &&
3369 EnableScalableAutovecInStreamingMode))
3370 return TypeSize::getScalable(MinimumSize: 128);
3371 else
3372 return TypeSize::getScalable(MinimumSize: 0);
3373 }
3374 llvm_unreachable("Unsupported register kind");
3375}
3376
3377bool AArch64TTIImpl::isSingleExtWideningInstruction(
3378 unsigned Opcode, Type *DstTy, ArrayRef<const Value *> Args,
3379 Type *SrcOverrideTy) const {
3380 // A helper that returns a vector type from the given type. The number of
3381 // elements in type Ty determines the vector width.
3382 auto toVectorTy = [&](Type *ArgTy) {
3383 return VectorType::get(ElementType: ArgTy->getScalarType(),
3384 EC: cast<VectorType>(Val: DstTy)->getElementCount());
3385 };
3386
3387 // Exit early if DstTy is not a vector type whose elements are one of [i16,
3388 // i32, i64]. SVE doesn't generally have the same set of instructions to
3389 // perform an extend with the add/sub/mul. There are SMULLB style
3390 // instructions, but they operate on top/bottom, requiring some sort of lane
3391 // interleaving to be used with zext/sext.
3392 unsigned DstEltSize = DstTy->getScalarSizeInBits();
3393 if (!useNeonVector(Ty: DstTy) || Args.size() != 2 ||
3394 (DstEltSize != 16 && DstEltSize != 32 && DstEltSize != 64))
3395 return false;
3396
3397 Type *SrcTy = SrcOverrideTy;
3398 switch (Opcode) {
3399 case Instruction::Add: // UADDW(2), SADDW(2).
3400 case Instruction::Sub: { // USUBW(2), SSUBW(2).
3401 // The second operand needs to be an extend
3402 if (isa<SExtInst>(Val: Args[1]) || isa<ZExtInst>(Val: Args[1])) {
3403 if (!SrcTy)
3404 SrcTy =
3405 toVectorTy(cast<Instruction>(Val: Args[1])->getOperand(i: 0)->getType());
3406 break;
3407 }
3408
3409 if (Opcode == Instruction::Sub)
3410 return false;
3411
3412 // UADDW(2), SADDW(2) can be commutted.
3413 if (isa<SExtInst>(Val: Args[0]) || isa<ZExtInst>(Val: Args[0])) {
3414 if (!SrcTy)
3415 SrcTy =
3416 toVectorTy(cast<Instruction>(Val: Args[0])->getOperand(i: 0)->getType());
3417 break;
3418 }
3419 return false;
3420 }
3421 default:
3422 return false;
3423 }
3424
3425 // Legalize the destination type and ensure it can be used in a widening
3426 // operation.
3427 auto DstTyL = getTypeLegalizationCost(Ty: DstTy);
3428 if (!DstTyL.second.isVector() || DstEltSize != DstTy->getScalarSizeInBits())
3429 return false;
3430
3431 // Legalize the source type and ensure it can be used in a widening
3432 // operation.
3433 assert(SrcTy && "Expected some SrcTy");
3434 auto SrcTyL = getTypeLegalizationCost(Ty: SrcTy);
3435 unsigned SrcElTySize = SrcTyL.second.getScalarSizeInBits();
3436 if (!SrcTyL.second.isVector() || SrcElTySize != SrcTy->getScalarSizeInBits())
3437 return false;
3438
3439 // Get the total number of vector elements in the legalized types.
3440 InstructionCost NumDstEls =
3441 DstTyL.first * DstTyL.second.getVectorMinNumElements();
3442 InstructionCost NumSrcEls =
3443 SrcTyL.first * SrcTyL.second.getVectorMinNumElements();
3444
3445 // Return true if the legalized types have the same number of vector elements
3446 // and the destination element type size is twice that of the source type.
3447 return NumDstEls == NumSrcEls && 2 * SrcElTySize == DstEltSize;
3448}
3449
3450Type *AArch64TTIImpl::isBinExtWideningInstruction(unsigned Opcode, Type *DstTy,
3451 ArrayRef<const Value *> Args,
3452 Type *SrcOverrideTy) const {
3453 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
3454 Opcode != Instruction::Mul)
3455 return nullptr;
3456
3457 // Exit early if DstTy is not a vector type whose elements are one of [i16,
3458 // i32, i64]. SVE doesn't generally have the same set of instructions to
3459 // perform an extend with the add/sub/mul. There are SMULLB style
3460 // instructions, but they operate on top/bottom, requiring some sort of lane
3461 // interleaving to be used with zext/sext.
3462 unsigned DstEltSize = DstTy->getScalarSizeInBits();
3463 if (!useNeonVector(Ty: DstTy) || Args.size() != 2 ||
3464 (DstEltSize != 16 && DstEltSize != 32 && DstEltSize != 64))
3465 return nullptr;
3466
3467 auto getScalarSizeWithOverride = [&](const Value *V) {
3468 if (SrcOverrideTy)
3469 return SrcOverrideTy->getScalarSizeInBits();
3470 return cast<Instruction>(Val: V)
3471 ->getOperand(i: 0)
3472 ->getType()
3473 ->getScalarSizeInBits();
3474 };
3475
3476 unsigned MaxEltSize = 0;
3477 if ((isa<SExtInst>(Val: Args[0]) && isa<SExtInst>(Val: Args[1])) ||
3478 (isa<ZExtInst>(Val: Args[0]) && isa<ZExtInst>(Val: Args[1]))) {
3479 unsigned EltSize0 = getScalarSizeWithOverride(Args[0]);
3480 unsigned EltSize1 = getScalarSizeWithOverride(Args[1]);
3481 MaxEltSize = std::max(a: EltSize0, b: EltSize1);
3482 } else if (isa<SExtInst, ZExtInst>(Val: Args[0]) &&
3483 isa<SExtInst, ZExtInst>(Val: Args[1])) {
3484 unsigned EltSize0 = getScalarSizeWithOverride(Args[0]);
3485 unsigned EltSize1 = getScalarSizeWithOverride(Args[1]);
3486 // mul(sext, zext) will become smull(sext, zext) if the extends are large
3487 // enough.
3488 if (EltSize0 >= DstEltSize / 2 || EltSize1 >= DstEltSize / 2)
3489 return nullptr;
3490 MaxEltSize = DstEltSize / 2;
3491 } else if (Opcode == Instruction::Mul &&
3492 (isa<ZExtInst>(Val: Args[0]) || isa<ZExtInst>(Val: Args[1]))) {
3493 // If one of the operands is a Zext and the other has enough zero bits
3494 // to be treated as unsigned, we can still generate a umull, meaning the
3495 // zext is free.
3496 KnownBits Known =
3497 computeKnownBits(V: isa<ZExtInst>(Val: Args[0]) ? Args[1] : Args[0], DL);
3498 if (Args[0]->getType()->getScalarSizeInBits() -
3499 Known.Zero.countLeadingOnes() >
3500 DstTy->getScalarSizeInBits() / 2)
3501 return nullptr;
3502
3503 MaxEltSize =
3504 getScalarSizeWithOverride(isa<ZExtInst>(Val: Args[0]) ? Args[0] : Args[1]);
3505 } else
3506 return nullptr;
3507
3508 if (MaxEltSize * 2 > DstEltSize)
3509 return nullptr;
3510
3511 Type *ExtTy = DstTy->getWithNewBitWidth(NewBitWidth: MaxEltSize * 2);
3512 if (ExtTy->getPrimitiveSizeInBits() <= 64)
3513 return nullptr;
3514 return ExtTy;
3515}
3516
3517// s/urhadd instructions implement the following pattern, making the
3518// extends free:
3519// %x = add ((zext i8 -> i16), 1)
3520// %y = (zext i8 -> i16)
3521// trunc i16 (lshr (add %x, %y), 1) -> i8
3522//
3523bool AArch64TTIImpl::isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
3524 Type *Src) const {
3525 // The source should be a legal vector type.
3526 if (!Src->isVectorTy() || !TLI->isTypeLegal(VT: TLI->getValueType(DL, Ty: Src)) ||
3527 (Src->isScalableTy() && !ST->hasSVE2()))
3528 return false;
3529
3530 if (ExtUser->getOpcode() != Instruction::Add || !ExtUser->hasOneUse())
3531 return false;
3532
3533 // Look for trunc/shl/add before trying to match the pattern.
3534 const Instruction *Add = ExtUser;
3535 auto *AddUser =
3536 dyn_cast_or_null<Instruction>(Val: Add->getUniqueUndroppableUser());
3537 if (AddUser && AddUser->getOpcode() == Instruction::Add)
3538 Add = AddUser;
3539
3540 auto *Shr = dyn_cast_or_null<Instruction>(Val: Add->getUniqueUndroppableUser());
3541 if (!Shr || Shr->getOpcode() != Instruction::LShr)
3542 return false;
3543
3544 auto *Trunc = dyn_cast_or_null<Instruction>(Val: Shr->getUniqueUndroppableUser());
3545 if (!Trunc || Trunc->getOpcode() != Instruction::Trunc ||
3546 Src->getScalarSizeInBits() !=
3547 cast<CastInst>(Val: Trunc)->getDestTy()->getScalarSizeInBits())
3548 return false;
3549
3550 // Try to match the whole pattern. Ext could be either the first or second
3551 // m_ZExtOrSExt matched.
3552 Instruction *Ex1, *Ex2;
3553 if (!(match(V: Add, P: m_c_Add(L: m_Instruction(I&: Ex1),
3554 R: m_c_Add(L: m_Instruction(I&: Ex2), R: m_One())))))
3555 return false;
3556
3557 // Ensure both extends are of the same type
3558 if (match(V: Ex1, P: m_ZExtOrSExt(Op: m_Value())) &&
3559 Ex1->getOpcode() == Ex2->getOpcode())
3560 return true;
3561
3562 return false;
3563}
3564
3565InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
3566 Type *Src,
3567 TTI::CastContextHint CCH,
3568 TTI::TargetCostKind CostKind,
3569 const Instruction *I) const {
3570 int ISD = TLI->InstructionOpcodeToISD(Opcode);
3571 assert(ISD && "Invalid opcode");
3572 // If the cast is observable, and it is used by a widening instruction (e.g.,
3573 // uaddl, saddw, etc.), it may be free.
3574 if (I && I->hasOneUser()) {
3575 auto *SingleUser = cast<Instruction>(Val: *I->user_begin());
3576 SmallVector<const Value *, 4> Operands(SingleUser->operand_values());
3577 if (Type *ExtTy = isBinExtWideningInstruction(
3578 Opcode: SingleUser->getOpcode(), DstTy: Dst, Args: Operands,
3579 SrcOverrideTy: Src != I->getOperand(i: 0)->getType() ? Src : nullptr)) {
3580 // The cost from Src->Src*2 needs to be added if required, the cost from
3581 // Src*2->ExtTy is free.
3582 if (ExtTy->getScalarSizeInBits() > Src->getScalarSizeInBits() * 2) {
3583 Type *DoubleSrcTy =
3584 Src->getWithNewBitWidth(NewBitWidth: Src->getScalarSizeInBits() * 2);
3585 return getCastInstrCost(Opcode, Dst: DoubleSrcTy, Src,
3586 CCH: TTI::CastContextHint::None, CostKind);
3587 }
3588
3589 return 0;
3590 }
3591
3592 if (isSingleExtWideningInstruction(
3593 Opcode: SingleUser->getOpcode(), DstTy: Dst, Args: Operands,
3594 SrcOverrideTy: Src != I->getOperand(i: 0)->getType() ? Src : nullptr)) {
3595 // For adds only count the second operand as free if both operands are
3596 // extends but not the same operation. (i.e both operands are not free in
3597 // add(sext, zext)).
3598 if (SingleUser->getOpcode() == Instruction::Add) {
3599 if (I == SingleUser->getOperand(i: 1) ||
3600 (isa<CastInst>(Val: SingleUser->getOperand(i: 1)) &&
3601 cast<CastInst>(Val: SingleUser->getOperand(i: 1))->getOpcode() == Opcode))
3602 return 0;
3603 } else {
3604 // Others are free so long as isSingleExtWideningInstruction
3605 // returned true.
3606 return 0;
3607 }
3608 }
3609
3610 // The cast will be free for the s/urhadd instructions
3611 if ((isa<ZExtInst>(Val: I) || isa<SExtInst>(Val: I)) &&
3612 isExtPartOfAvgExpr(ExtUser: SingleUser, Dst, Src))
3613 return 0;
3614 }
3615
3616 EVT SrcTy = TLI->getValueType(DL, Ty: Src);
3617 EVT DstTy = TLI->getValueType(DL, Ty: Dst);
3618
3619 if (!SrcTy.isSimple() || !DstTy.isSimple())
3620 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
3621
3622 // For the moment we do not have lowering for SVE1-only fptrunc f64->bf16 as
3623 // we use fcvtx under SVE2. Give them invalid costs.
3624 if (!ST->hasSVE2() && !ST->isStreamingSVEAvailable() &&
3625 ISD == ISD::FP_ROUND && SrcTy.isScalableVector() &&
3626 DstTy.getScalarType() == MVT::bf16 && SrcTy.getScalarType() == MVT::f64)
3627 return InstructionCost::getInvalid();
3628
3629 static const TypeConversionCostTblEntry BF16Tbl[] = {
3630 {.ISD: ISD::FP_ROUND, .Dst: MVT::bf16, .Src: MVT::f32, .Cost: 1}, // bfcvt
3631 {.ISD: ISD::FP_ROUND, .Dst: MVT::bf16, .Src: MVT::f64, .Cost: 1}, // bfcvt
3632 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4bf16, .Src: MVT::v4f32, .Cost: 1}, // bfcvtn
3633 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8bf16, .Src: MVT::v8f32, .Cost: 2}, // bfcvtn+bfcvtn2
3634 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2bf16, .Src: MVT::v2f64, .Cost: 2}, // bfcvtn+fcvtn
3635 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4bf16, .Src: MVT::v4f64, .Cost: 3}, // fcvtn+fcvtl2+bfcvtn
3636 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8bf16, .Src: MVT::v8f64, .Cost: 6}, // 2 * fcvtn+fcvtn2+bfcvtn
3637 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2bf16, .Src: MVT::nxv2f32, .Cost: 1}, // bfcvt
3638 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4bf16, .Src: MVT::nxv4f32, .Cost: 1}, // bfcvt
3639 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8bf16, .Src: MVT::nxv8f32, .Cost: 3}, // bfcvt+bfcvt+uzp1
3640 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2bf16, .Src: MVT::nxv2f64, .Cost: 2}, // fcvtx+bfcvt
3641 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4bf16, .Src: MVT::nxv4f64, .Cost: 5}, // 2*fcvtx+2*bfcvt+uzp1
3642 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8bf16, .Src: MVT::nxv8f64, .Cost: 11}, // 4*fcvt+4*bfcvt+3*uzp
3643 };
3644
3645 if (ST->hasBF16())
3646 if (const auto *Entry = ConvertCostTableLookup(
3647 Table: BF16Tbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
3648 return Entry->Cost;
3649
3650 // We have to estimate a cost of fixed length operation upon
3651 // SVE registers(operations) with the number of registers required
3652 // for a fixed type to be represented upon SVE registers.
3653 EVT WiderTy = SrcTy.bitsGT(VT: DstTy) ? SrcTy : DstTy;
3654 if (SrcTy.isFixedLengthVector() && DstTy.isFixedLengthVector() &&
3655 SrcTy.getVectorNumElements() == DstTy.getVectorNumElements() &&
3656 ST->useSVEForFixedLengthVectors(VT: WiderTy)) {
3657 std::pair<InstructionCost, MVT> LT =
3658 getTypeLegalizationCost(Ty: WiderTy.getTypeForEVT(Context&: Dst->getContext()));
3659 unsigned NumElements =
3660 AArch64::SVEBitsPerBlock / LT.second.getScalarSizeInBits();
3661 return LT.first *
3662 getCastInstrCost(
3663 Opcode,
3664 Dst: ScalableVectorType::get(ElementType: Dst->getScalarType(), MinNumElts: NumElements),
3665 Src: ScalableVectorType::get(ElementType: Src->getScalarType(), MinNumElts: NumElements), CCH,
3666 CostKind, I);
3667 }
3668
3669 // Symbolic constants for the SVE sitofp/uitofp entries in the table below
3670 // The cost of unpacking twice is artificially increased for now in order
3671 // to avoid regressions against NEON, which will use tbl instructions directly
3672 // instead of multiple layers of [s|u]unpk[lo|hi].
3673 // We use the unpacks in cases where the destination type is illegal and
3674 // requires splitting of the input, even if the input type itself is legal.
3675 const unsigned int SVE_EXT_COST = 1;
3676 const unsigned int SVE_FCVT_COST = 1;
3677 const unsigned int SVE_UNPACK_ONCE = 4;
3678 const unsigned int SVE_UNPACK_TWICE = 16;
3679
3680 static const TypeConversionCostTblEntry ConversionTbl[] = {
3681 {.ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i64, .Cost: 1}, // xtn
3682 {.ISD: ISD::TRUNCATE, .Dst: MVT::v2i16, .Src: MVT::v2i64, .Cost: 1}, // xtn
3683 {.ISD: ISD::TRUNCATE, .Dst: MVT::v2i32, .Src: MVT::v2i64, .Cost: 1}, // xtn
3684 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i32, .Cost: 1}, // xtn
3685 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i64, .Cost: 3}, // 2 xtn + 1 uzp1
3686 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i32, .Cost: 1}, // xtn
3687 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i64, .Cost: 2}, // 1 uzp1 + 1 xtn
3688 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: 1}, // 1 uzp1
3689 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i16, .Cost: 1}, // 1 xtn
3690 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: 2}, // 1 uzp1 + 1 xtn
3691 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i64, .Cost: 4}, // 3 x uzp1 + xtn
3692 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: 1}, // 1 uzp1
3693 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i64, .Cost: 3}, // 3 x uzp1
3694 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i64, .Cost: 2}, // 2 x uzp1
3695 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: 1}, // uzp1
3696 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: 3}, // (2 + 1) x uzp1
3697 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i64, .Cost: 7}, // (4 + 2 + 1) x uzp1
3698 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: 2}, // 2 x uzp1
3699 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i64, .Cost: 6}, // (4 + 2) x uzp1
3700 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i32, .Src: MVT::v16i64, .Cost: 4}, // 4 x uzp1
3701
3702 // Truncations on nxvmiN
3703 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i1, .Src: MVT::nxv2i8, .Cost: 2},
3704 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i1, .Src: MVT::nxv2i16, .Cost: 2},
3705 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i1, .Src: MVT::nxv2i32, .Cost: 2},
3706 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i1, .Src: MVT::nxv2i64, .Cost: 2},
3707 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i1, .Src: MVT::nxv4i8, .Cost: 2},
3708 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i1, .Src: MVT::nxv4i16, .Cost: 2},
3709 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i1, .Src: MVT::nxv4i32, .Cost: 2},
3710 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i1, .Src: MVT::nxv4i64, .Cost: 5},
3711 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i1, .Src: MVT::nxv8i8, .Cost: 2},
3712 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i1, .Src: MVT::nxv8i16, .Cost: 2},
3713 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i1, .Src: MVT::nxv8i32, .Cost: 5},
3714 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i1, .Src: MVT::nxv8i64, .Cost: 11},
3715 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv16i1, .Src: MVT::nxv16i8, .Cost: 2},
3716 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i8, .Src: MVT::nxv2i16, .Cost: 0},
3717 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i8, .Src: MVT::nxv2i32, .Cost: 0},
3718 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i8, .Src: MVT::nxv2i64, .Cost: 0},
3719 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i16, .Src: MVT::nxv2i32, .Cost: 0},
3720 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i16, .Src: MVT::nxv2i64, .Cost: 0},
3721 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i32, .Src: MVT::nxv2i64, .Cost: 0},
3722 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i8, .Src: MVT::nxv4i16, .Cost: 0},
3723 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i8, .Src: MVT::nxv4i32, .Cost: 0},
3724 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i8, .Src: MVT::nxv4i64, .Cost: 1},
3725 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i16, .Src: MVT::nxv4i32, .Cost: 0},
3726 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i16, .Src: MVT::nxv4i64, .Cost: 1},
3727 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i32, .Src: MVT::nxv4i64, .Cost: 1},
3728 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i8, .Src: MVT::nxv8i16, .Cost: 0},
3729 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i8, .Src: MVT::nxv8i32, .Cost: 1},
3730 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i8, .Src: MVT::nxv8i64, .Cost: 3},
3731 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i16, .Src: MVT::nxv8i32, .Cost: 1},
3732 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i16, .Src: MVT::nxv8i64, .Cost: 3},
3733 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv16i8, .Src: MVT::nxv16i16, .Cost: 1},
3734 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv16i8, .Src: MVT::nxv16i32, .Cost: 3},
3735 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv16i8, .Src: MVT::nxv16i64, .Cost: 7},
3736
3737 // The number of shll instructions for the extension.
3738 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3},
3739 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3},
3740 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: 2},
3741 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: 2},
3742 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3},
3743 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3},
3744 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 2},
3745 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 2},
3746 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7},
3747 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7},
3748 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6},
3749 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6},
3750 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 2},
3751 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 2},
3752 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6},
3753 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6},
3754
3755 // FP Ext and trunc
3756 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f64, .Src: MVT::f32, .Cost: 1}, // fcvt
3757 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v2f64, .Src: MVT::v2f32, .Cost: 1}, // fcvtl
3758 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4f32, .Cost: 2}, // fcvtl+fcvtl2
3759 // FP16
3760 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f32, .Src: MVT::f16, .Cost: 1}, // fcvt
3761 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f64, .Src: MVT::f16, .Cost: 1}, // fcvt
3762 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1}, // fcvtl
3763 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 2}, // fcvtl+fcvtl2
3764 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v2f64, .Src: MVT::v2f16, .Cost: 2}, // fcvtl+fcvtl
3765 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4f16, .Cost: 3}, // fcvtl+fcvtl2+fcvtl
3766 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8f16, .Cost: 6}, // 2 * fcvtl+fcvtl2+fcvtl
3767 // BF16 (uses shift)
3768 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f32, .Src: MVT::bf16, .Cost: 1}, // shl
3769 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f64, .Src: MVT::bf16, .Cost: 2}, // shl+fcvt
3770 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4bf16, .Cost: 1}, // shll
3771 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8bf16, .Cost: 2}, // shll+shll2
3772 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v2f64, .Src: MVT::v2bf16, .Cost: 2}, // shll+fcvtl
3773 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4bf16, .Cost: 3}, // shll+fcvtl+fcvtl2
3774 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8bf16, .Cost: 6}, // 2 * shll+fcvtl+fcvtl2
3775 // FP Ext and trunc
3776 {.ISD: ISD::FP_ROUND, .Dst: MVT::f32, .Src: MVT::f64, .Cost: 1}, // fcvt
3777 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2f32, .Src: MVT::v2f64, .Cost: 1}, // fcvtn
3778 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f32, .Src: MVT::v4f64, .Cost: 2}, // fcvtn+fcvtn2
3779 // FP16
3780 {.ISD: ISD::FP_ROUND, .Dst: MVT::f16, .Src: MVT::f32, .Cost: 1}, // fcvt
3781 {.ISD: ISD::FP_ROUND, .Dst: MVT::f16, .Src: MVT::f64, .Cost: 1}, // fcvt
3782 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f16, .Src: MVT::v4f32, .Cost: 1}, // fcvtn
3783 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8f16, .Src: MVT::v8f32, .Cost: 2}, // fcvtn+fcvtn2
3784 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2f16, .Src: MVT::v2f64, .Cost: 2}, // fcvtn+fcvtn
3785 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f16, .Src: MVT::v4f64, .Cost: 3}, // fcvtn+fcvtn2+fcvtn
3786 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8f16, .Src: MVT::v8f64, .Cost: 6}, // 2 * fcvtn+fcvtn2+fcvtn
3787 // BF16 (more complex, with +bf16 is handled above)
3788 {.ISD: ISD::FP_ROUND, .Dst: MVT::bf16, .Src: MVT::f32, .Cost: 8}, // Expansion is ~8 insns
3789 {.ISD: ISD::FP_ROUND, .Dst: MVT::bf16, .Src: MVT::f64, .Cost: 9}, // fcvtn + above
3790 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2bf16, .Src: MVT::v2f32, .Cost: 8},
3791 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4bf16, .Src: MVT::v4f32, .Cost: 8},
3792 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8bf16, .Src: MVT::v8f32, .Cost: 15},
3793 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2bf16, .Src: MVT::v2f64, .Cost: 9},
3794 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4bf16, .Src: MVT::v4f64, .Cost: 10},
3795 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8bf16, .Src: MVT::v8f64, .Cost: 19},
3796
3797 // LowerVectorINT_TO_FP:
3798 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1},
3799 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1},
3800 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: 1},
3801 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1},
3802 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1},
3803 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: 1},
3804
3805 // SVE: to nxv2f16
3806 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i8,
3807 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3808 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
3809 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
3810 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
3811 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i8,
3812 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3813 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
3814 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
3815 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
3816
3817 // SVE: to nxv4f16
3818 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i8,
3819 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3820 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i16, .Cost: SVE_FCVT_COST},
3821 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i32, .Cost: SVE_FCVT_COST},
3822 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i8,
3823 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3824 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i16, .Cost: SVE_FCVT_COST},
3825 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i32, .Cost: SVE_FCVT_COST},
3826
3827 // SVE: to nxv8f16
3828 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f16, .Src: MVT::nxv8i8,
3829 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3830 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f16, .Src: MVT::nxv8i16, .Cost: SVE_FCVT_COST},
3831 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f16, .Src: MVT::nxv8i8,
3832 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3833 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f16, .Src: MVT::nxv8i16, .Cost: SVE_FCVT_COST},
3834
3835 // SVE: to nxv16f16
3836 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv16f16, .Src: MVT::nxv16i8,
3837 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3838 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv16f16, .Src: MVT::nxv16i8,
3839 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3840
3841 // Complex: to v2f32
3842 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3},
3843 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 3},
3844 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3},
3845 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 3},
3846
3847 // SVE: to nxv2f32
3848 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i8,
3849 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3850 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
3851 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
3852 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
3853 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i8,
3854 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3855 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
3856 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
3857 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
3858
3859 // Complex: to v4f32
3860 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 4},
3861 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2},
3862 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3},
3863 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2},
3864
3865 // SVE: to nxv4f32
3866 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i8,
3867 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3868 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i16, .Cost: SVE_FCVT_COST},
3869 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i32, .Cost: SVE_FCVT_COST},
3870 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i8,
3871 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3872 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i16, .Cost: SVE_FCVT_COST},
3873 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i32, .Cost: SVE_FCVT_COST},
3874
3875 // Complex: to v8f32
3876 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i8, .Cost: 10},
3877 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4},
3878 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i8, .Cost: 10},
3879 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4},
3880
3881 // SVE: to nxv8f32
3882 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f32, .Src: MVT::nxv8i8,
3883 .Cost: SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3884 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f32, .Src: MVT::nxv8i16,
3885 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3886 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f32, .Src: MVT::nxv8i8,
3887 .Cost: SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3888 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f32, .Src: MVT::nxv8i16,
3889 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3890
3891 // SVE: to nxv16f32
3892 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv16f32, .Src: MVT::nxv16i8,
3893 .Cost: SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
3894 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv16f32, .Src: MVT::nxv16i8,
3895 .Cost: SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
3896
3897 // Complex: to v16f32
3898 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i8, .Cost: 21},
3899 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i8, .Cost: 21},
3900
3901 // Complex: to v2f64
3902 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4},
3903 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 4},
3904 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2},
3905 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4},
3906 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 4},
3907 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2},
3908
3909 // SVE: to nxv2f64
3910 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i8,
3911 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3912 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
3913 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
3914 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
3915 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i8,
3916 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
3917 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
3918 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
3919 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
3920
3921 // Complex: to v4f64
3922 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: 4},
3923 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: 4},
3924
3925 // SVE: to nxv4f64
3926 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i8,
3927 .Cost: SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3928 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i16,
3929 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3930 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i32,
3931 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3932 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i8,
3933 .Cost: SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3934 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i16,
3935 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3936 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i32,
3937 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
3938
3939 // SVE: to nxv8f64
3940 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f64, .Src: MVT::nxv8i8,
3941 .Cost: SVE_EXT_COST + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
3942 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f64, .Src: MVT::nxv8i16,
3943 .Cost: SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
3944 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f64, .Src: MVT::nxv8i8,
3945 .Cost: SVE_EXT_COST + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
3946 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f64, .Src: MVT::nxv8i16,
3947 .Cost: SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
3948
3949 // LowerVectorFP_TO_INT
3950 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i32, .Src: MVT::v2f32, .Cost: 1},
3951 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1},
3952 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i64, .Src: MVT::v2f64, .Cost: 1},
3953 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i32, .Src: MVT::v2f32, .Cost: 1},
3954 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1},
3955 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i64, .Src: MVT::v2f64, .Cost: 1},
3956
3957 // Complex, from v2f32: legal type is v2i32 (no cost) or v2i64 (1 ext).
3958 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i64, .Src: MVT::v2f32, .Cost: 2},
3959 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i16, .Src: MVT::v2f32, .Cost: 1},
3960 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i8, .Src: MVT::v2f32, .Cost: 1},
3961 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i64, .Src: MVT::v2f32, .Cost: 2},
3962 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i16, .Src: MVT::v2f32, .Cost: 1},
3963 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i8, .Src: MVT::v2f32, .Cost: 1},
3964
3965 // Complex, from v4f32: legal type is v4i16, 1 narrowing => ~2
3966 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2},
3967 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 2},
3968 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2},
3969 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 2},
3970
3971 // Complex, from v2f64: legal type is v2i32, 1 narrowing => ~2.
3972 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2},
3973 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i16, .Src: MVT::v2f64, .Cost: 2},
3974 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i8, .Src: MVT::v2f64, .Cost: 2},
3975 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2},
3976 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i16, .Src: MVT::v2f64, .Cost: 2},
3977 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i8, .Src: MVT::v2f64, .Cost: 2},
3978
3979 // Complex, from nxv2f32.
3980 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f32, .Cost: 1},
3981 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f32, .Cost: 1},
3982 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f32, .Cost: 1},
3983 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f32, .Cost: 1},
3984 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f32, .Cost: 1},
3985 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f32, .Cost: 1},
3986 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f32, .Cost: 1},
3987 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f32, .Cost: 1},
3988
3989 // Complex, from nxv2f64.
3990 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f64, .Cost: 1},
3991 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f64, .Cost: 1},
3992 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f64, .Cost: 1},
3993 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f64, .Cost: 1},
3994 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i1, .Src: MVT::nxv2f64, .Cost: 1},
3995 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f64, .Cost: 1},
3996 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f64, .Cost: 1},
3997 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f64, .Cost: 1},
3998 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f64, .Cost: 1},
3999 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i1, .Src: MVT::nxv2f64, .Cost: 1},
4000
4001 // Complex, from nxv4f32.
4002 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i64, .Src: MVT::nxv4f32, .Cost: 4},
4003 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f32, .Cost: 1},
4004 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f32, .Cost: 1},
4005 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f32, .Cost: 1},
4006 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i1, .Src: MVT::nxv4f32, .Cost: 1},
4007 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i64, .Src: MVT::nxv4f32, .Cost: 4},
4008 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f32, .Cost: 1},
4009 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f32, .Cost: 1},
4010 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f32, .Cost: 1},
4011 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i1, .Src: MVT::nxv4f32, .Cost: 1},
4012
4013 // Complex, from nxv8f64. Illegal -> illegal conversions not required.
4014 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f64, .Cost: 7},
4015 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f64, .Cost: 7},
4016 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f64, .Cost: 7},
4017 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f64, .Cost: 7},
4018
4019 // Complex, from nxv4f64. Illegal -> illegal conversions not required.
4020 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f64, .Cost: 3},
4021 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f64, .Cost: 3},
4022 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f64, .Cost: 3},
4023 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f64, .Cost: 3},
4024 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f64, .Cost: 3},
4025 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f64, .Cost: 3},
4026
4027 // Complex, from nxv8f32. Illegal -> illegal conversions not required.
4028 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f32, .Cost: 3},
4029 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f32, .Cost: 3},
4030 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f32, .Cost: 3},
4031 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f32, .Cost: 3},
4032
4033 // Complex, from nxv8f16.
4034 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i64, .Src: MVT::nxv8f16, .Cost: 10},
4035 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i32, .Src: MVT::nxv8f16, .Cost: 4},
4036 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f16, .Cost: 1},
4037 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f16, .Cost: 1},
4038 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i1, .Src: MVT::nxv8f16, .Cost: 1},
4039 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i64, .Src: MVT::nxv8f16, .Cost: 10},
4040 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i32, .Src: MVT::nxv8f16, .Cost: 4},
4041 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f16, .Cost: 1},
4042 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f16, .Cost: 1},
4043 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i1, .Src: MVT::nxv8f16, .Cost: 1},
4044
4045 // Complex, from nxv4f16.
4046 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i64, .Src: MVT::nxv4f16, .Cost: 4},
4047 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f16, .Cost: 1},
4048 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f16, .Cost: 1},
4049 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f16, .Cost: 1},
4050 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i64, .Src: MVT::nxv4f16, .Cost: 4},
4051 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f16, .Cost: 1},
4052 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f16, .Cost: 1},
4053 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f16, .Cost: 1},
4054
4055 // Complex, from nxv2f16.
4056 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f16, .Cost: 1},
4057 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f16, .Cost: 1},
4058 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f16, .Cost: 1},
4059 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f16, .Cost: 1},
4060 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f16, .Cost: 1},
4061 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f16, .Cost: 1},
4062 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f16, .Cost: 1},
4063 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f16, .Cost: 1},
4064
4065 // Truncate from nxvmf32 to nxvmf16.
4066 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2f16, .Src: MVT::nxv2f32, .Cost: 1},
4067 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4f16, .Src: MVT::nxv4f32, .Cost: 1},
4068 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8f16, .Src: MVT::nxv8f32, .Cost: 3},
4069
4070 // Truncate from nxvmf32 to nxvmbf16.
4071 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2bf16, .Src: MVT::nxv2f32, .Cost: 8},
4072 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4bf16, .Src: MVT::nxv4f32, .Cost: 8},
4073 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8bf16, .Src: MVT::nxv8f32, .Cost: 17},
4074
4075 // Truncate from nxvmf64 to nxvmf16.
4076 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2f16, .Src: MVT::nxv2f64, .Cost: 1},
4077 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4f16, .Src: MVT::nxv4f64, .Cost: 3},
4078 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8f16, .Src: MVT::nxv8f64, .Cost: 7},
4079
4080 // Truncate from nxvmf64 to nxvmbf16.
4081 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2bf16, .Src: MVT::nxv2f64, .Cost: 9},
4082 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4bf16, .Src: MVT::nxv4f64, .Cost: 19},
4083 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8bf16, .Src: MVT::nxv8f64, .Cost: 39},
4084
4085 // Truncate from nxvmf64 to nxvmf32.
4086 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2f32, .Src: MVT::nxv2f64, .Cost: 1},
4087 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4f32, .Src: MVT::nxv4f64, .Cost: 3},
4088 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8f32, .Src: MVT::nxv8f64, .Cost: 6},
4089
4090 // Extend from nxvmf16 to nxvmf32.
4091 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f32, .Src: MVT::nxv2f16, .Cost: 1},
4092 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f32, .Src: MVT::nxv4f16, .Cost: 1},
4093 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f32, .Src: MVT::nxv8f16, .Cost: 2},
4094
4095 // Extend from nxvmbf16 to nxvmf32.
4096 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f32, .Src: MVT::nxv2bf16, .Cost: 1}, // lsl
4097 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f32, .Src: MVT::nxv4bf16, .Cost: 1}, // lsl
4098 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f32, .Src: MVT::nxv8bf16, .Cost: 4}, // unpck+unpck+lsl+lsl
4099
4100 // Extend from nxvmf16 to nxvmf64.
4101 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f64, .Src: MVT::nxv2f16, .Cost: 1},
4102 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f64, .Src: MVT::nxv4f16, .Cost: 2},
4103 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f64, .Src: MVT::nxv8f16, .Cost: 4},
4104
4105 // Extend from nxvmbf16 to nxvmf64.
4106 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f64, .Src: MVT::nxv2bf16, .Cost: 2}, // lsl+fcvt
4107 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f64, .Src: MVT::nxv4bf16, .Cost: 6}, // 2*unpck+2*lsl+2*fcvt
4108 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f64, .Src: MVT::nxv8bf16, .Cost: 14}, // 6*unpck+4*lsl+4*fcvt
4109
4110 // Extend from nxvmf32 to nxvmf64.
4111 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f64, .Src: MVT::nxv2f32, .Cost: 1},
4112 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f64, .Src: MVT::nxv4f32, .Cost: 2},
4113 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f64, .Src: MVT::nxv8f32, .Cost: 6},
4114
4115 // Bitcasts from float to integer
4116 {.ISD: ISD::BITCAST, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i16, .Cost: 0},
4117 {.ISD: ISD::BITCAST, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i16, .Cost: 0},
4118 {.ISD: ISD::BITCAST, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i32, .Cost: 0},
4119
4120 // Bitcasts from integer to float
4121 {.ISD: ISD::BITCAST, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f16, .Cost: 0},
4122 {.ISD: ISD::BITCAST, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f16, .Cost: 0},
4123 {.ISD: ISD::BITCAST, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f32, .Cost: 0},
4124
4125 // Add cost for extending to illegal -too wide- scalable vectors.
4126 // zero/sign extend are implemented by multiple unpack operations,
4127 // where each operation has a cost of 1.
4128 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv16i16, .Src: MVT::nxv16i8, .Cost: 2},
4129 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv16i32, .Src: MVT::nxv16i8, .Cost: 6},
4130 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv16i64, .Src: MVT::nxv16i8, .Cost: 14},
4131 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv8i32, .Src: MVT::nxv8i16, .Cost: 2},
4132 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv8i64, .Src: MVT::nxv8i16, .Cost: 6},
4133 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv4i64, .Src: MVT::nxv4i32, .Cost: 2},
4134
4135 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv16i16, .Src: MVT::nxv16i8, .Cost: 2},
4136 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv16i32, .Src: MVT::nxv16i8, .Cost: 6},
4137 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv16i64, .Src: MVT::nxv16i8, .Cost: 14},
4138 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv8i32, .Src: MVT::nxv8i16, .Cost: 2},
4139 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv8i64, .Src: MVT::nxv8i16, .Cost: 6},
4140 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv4i64, .Src: MVT::nxv4i32, .Cost: 2},
4141 };
4142
4143 if (const auto *Entry = ConvertCostTableLookup(
4144 Table: ConversionTbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
4145 return Entry->Cost;
4146
4147 static const TypeConversionCostTblEntry FP16Tbl[] = {
4148 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f16, .Cost: 1}, // fcvtzs
4149 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f16, .Cost: 1},
4150 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f16, .Cost: 1}, // fcvtzs
4151 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f16, .Cost: 1},
4152 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f16, .Cost: 2}, // fcvtl+fcvtzs
4153 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f16, .Cost: 2},
4154 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i8, .Src: MVT::v8f16, .Cost: 2}, // fcvtzs+xtn
4155 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i8, .Src: MVT::v8f16, .Cost: 2},
4156 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f16, .Cost: 1}, // fcvtzs
4157 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f16, .Cost: 1},
4158 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f16, .Cost: 4}, // 2*fcvtl+2*fcvtzs
4159 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f16, .Cost: 4},
4160 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v16f16, .Cost: 3}, // 2*fcvtzs+xtn
4161 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v16f16, .Cost: 3},
4162 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v16f16, .Cost: 2}, // 2*fcvtzs
4163 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v16f16, .Cost: 2},
4164 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i32, .Src: MVT::v16f16, .Cost: 8}, // 4*fcvtl+4*fcvtzs
4165 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i32, .Src: MVT::v16f16, .Cost: 8},
4166 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f16, .Src: MVT::v8i8, .Cost: 2}, // ushll + ucvtf
4167 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f16, .Src: MVT::v8i8, .Cost: 2}, // sshll + scvtf
4168 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f16, .Src: MVT::v16i8, .Cost: 4}, // 2 * ushl(2) + 2 * ucvtf
4169 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f16, .Src: MVT::v16i8, .Cost: 4}, // 2 * sshl(2) + 2 * scvtf
4170 };
4171
4172 if (ST->hasFullFP16())
4173 if (const auto *Entry = ConvertCostTableLookup(
4174 Table: FP16Tbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
4175 return Entry->Cost;
4176
4177 // INT_TO_FP of i64->f32 will scalarize, which is required to avoid
4178 // double-rounding issues.
4179 if ((ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP) &&
4180 DstTy.getScalarType() == MVT::f32 && SrcTy.getScalarSizeInBits() > 32 &&
4181 isa<FixedVectorType>(Val: Dst) && isa<FixedVectorType>(Val: Src))
4182 return cast<FixedVectorType>(Val: Dst)->getNumElements() *
4183 getCastInstrCost(Opcode, Dst: Dst->getScalarType(),
4184 Src: Src->getScalarType(), CCH, CostKind) +
4185 BaseT::getScalarizationOverhead(InTy: cast<FixedVectorType>(Val: Src), Insert: false,
4186 Extract: true, CostKind) +
4187 BaseT::getScalarizationOverhead(InTy: cast<FixedVectorType>(Val: Dst), Insert: true,
4188 Extract: false, CostKind);
4189
4190 if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
4191 CCH == TTI::CastContextHint::Masked &&
4192 ST->isSVEorStreamingSVEAvailable() &&
4193 TLI->getTypeAction(Context&: Src->getContext(), VT: SrcTy) ==
4194 TargetLowering::TypePromoteInteger &&
4195 TLI->getTypeAction(Context&: Dst->getContext(), VT: DstTy) ==
4196 TargetLowering::TypeSplitVector) {
4197 // The standard behaviour in the backend for these cases is to split the
4198 // extend up into two parts:
4199 // 1. Perform an extending load or masked load up to the legal type.
4200 // 2. Extend the loaded data to the final type.
4201 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: Src);
4202 Type *LegalTy = EVT(SrcLT.second).getTypeForEVT(Context&: Src->getContext());
4203 InstructionCost Part1 = AArch64TTIImpl::getCastInstrCost(
4204 Opcode, Dst: LegalTy, Src, CCH, CostKind, I);
4205 InstructionCost Part2 = AArch64TTIImpl::getCastInstrCost(
4206 Opcode, Dst, Src: LegalTy, CCH: TTI::CastContextHint::None, CostKind, I);
4207 return Part1 + Part2;
4208 }
4209
4210 // The BasicTTIImpl version only deals with CCH==TTI::CastContextHint::Normal,
4211 // but we also want to include the TTI::CastContextHint::Masked case too.
4212 if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
4213 CCH == TTI::CastContextHint::Masked &&
4214 ST->isSVEorStreamingSVEAvailable() && TLI->isTypeLegal(VT: DstTy))
4215 CCH = TTI::CastContextHint::Normal;
4216
4217 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
4218}
4219
4220InstructionCost
4221AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
4222 VectorType *VecTy, unsigned Index,
4223 TTI::TargetCostKind CostKind) const {
4224
4225 // Make sure we were given a valid extend opcode.
4226 assert((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
4227 "Invalid opcode");
4228
4229 // We are extending an element we extract from a vector, so the source type
4230 // of the extend is the element type of the vector.
4231 auto *Src = VecTy->getElementType();
4232
4233 // Sign- and zero-extends are for integer types only.
4234 assert(isa<IntegerType>(Dst) && isa<IntegerType>(Src) && "Invalid type");
4235
4236 // Get the cost for the extract. We compute the cost (if any) for the extend
4237 // below.
4238 InstructionCost Cost = getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: VecTy,
4239 CostKind, Index, Op0: nullptr, Op1: nullptr);
4240
4241 // Legalize the types.
4242 auto VecLT = getTypeLegalizationCost(Ty: VecTy);
4243 auto DstVT = TLI->getValueType(DL, Ty: Dst);
4244 auto SrcVT = TLI->getValueType(DL, Ty: Src);
4245
4246 // If the resulting type is still a vector and the destination type is legal,
4247 // we may get the extension for free. If not, get the default cost for the
4248 // extend.
4249 if (!VecLT.second.isVector() || !TLI->isTypeLegal(VT: DstVT))
4250 return Cost + getCastInstrCost(Opcode, Dst, Src, CCH: TTI::CastContextHint::None,
4251 CostKind);
4252
4253 // The destination type should be larger than the element type. If not, get
4254 // the default cost for the extend.
4255 if (DstVT.getFixedSizeInBits() < SrcVT.getFixedSizeInBits())
4256 return Cost + getCastInstrCost(Opcode, Dst, Src, CCH: TTI::CastContextHint::None,
4257 CostKind);
4258
4259 switch (Opcode) {
4260 default:
4261 llvm_unreachable("Opcode should be either SExt or ZExt");
4262
4263 // For sign-extends, we only need a smov, which performs the extension
4264 // automatically.
4265 case Instruction::SExt:
4266 return Cost;
4267
4268 // For zero-extends, the extend is performed automatically by a umov unless
4269 // the destination type is i64 and the element type is i8 or i16.
4270 case Instruction::ZExt:
4271 if (DstVT.getSizeInBits() != 64u || SrcVT.getSizeInBits() == 32u)
4272 return Cost;
4273 }
4274
4275 // If we are unable to perform the extend for free, get the default cost.
4276 return Cost + getCastInstrCost(Opcode, Dst, Src, CCH: TTI::CastContextHint::None,
4277 CostKind);
4278}
4279
4280InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
4281 TTI::TargetCostKind CostKind,
4282 const Instruction *I) const {
4283 if (CostKind != TTI::TCK_RecipThroughput)
4284 return Opcode == Instruction::PHI ? 0 : 1;
4285 assert(CostKind == TTI::TCK_RecipThroughput && "unexpected CostKind");
4286 // Branches are assumed to be predicted.
4287 return 0;
4288}
4289
4290InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
4291 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
4292 const Instruction *I, Value *Scalar,
4293 ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
4294 TTI::VectorInstrContext VIC) const {
4295 assert(Val->isVectorTy() && "This must be a vector type");
4296
4297 if (Index != -1U) {
4298 // Legalize the type.
4299 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Val);
4300
4301 // This type is legalized to a scalar type.
4302 if (!LT.second.isVector())
4303 return 0;
4304
4305 // The type may be split. For fixed-width vectors we can normalize the
4306 // index to the new type.
4307 if (LT.second.isFixedLengthVector()) {
4308 unsigned Width = LT.second.getVectorNumElements();
4309 Index = Index % Width;
4310 }
4311
4312 // The element at index zero is already inside the vector.
4313 // - For a insert-element or extract-element
4314 // instruction that extracts integers, an explicit FPR -> GPR move is
4315 // needed. So it has non-zero cost.
4316 if (Index == 0 && !Val->getScalarType()->isIntegerTy())
4317 return 0;
4318
4319 // This is recognising a LD1 single-element structure to one lane of one
4320 // register instruction. I.e., if this is an `insertelement` instruction,
4321 // and its second operand is a load, then we will generate a LD1, which
4322 // are expensive instructions on some uArchs.
4323 if (VIC == TTI::VectorInstrContext::Load) {
4324 if (ST->hasFastLD1Single())
4325 return 0;
4326 return CostKind == TTI::TCK_CodeSize
4327 ? 0
4328 : ST->getVectorInsertExtractBaseCost() + 1;
4329 }
4330
4331 // i1 inserts and extract will include an extra cset or cmp of the vector
4332 // value. Increase the cost by 1 to account.
4333 if (Val->getScalarSizeInBits() == 1)
4334 return CostKind == TTI::TCK_CodeSize
4335 ? 2
4336 : ST->getVectorInsertExtractBaseCost() + 1;
4337
4338 // FIXME:
4339 // If the extract-element and insert-element instructions could be
4340 // simplified away (e.g., could be combined into users by looking at use-def
4341 // context), they have no cost. This is not done in the first place for
4342 // compile-time considerations.
4343 }
4344
4345 // In case of Neon, if there exists extractelement from lane != 0 such that
4346 // 1. extractelement does not necessitate a move from vector_reg -> GPR.
4347 // 2. extractelement result feeds into fmul.
4348 // 3. Other operand of fmul is an extractelement from lane 0 or lane
4349 // equivalent to 0.
4350 // then the extractelement can be merged with fmul in the backend and it
4351 // incurs no cost.
4352 // e.g.
4353 // define double @foo(<2 x double> %a) {
4354 // %1 = extractelement <2 x double> %a, i32 0
4355 // %2 = extractelement <2 x double> %a, i32 1
4356 // %res = fmul double %1, %2
4357 // ret double %res
4358 // }
4359 // %2 and %res can be merged in the backend to generate fmul d0, d0, v1.d[1]
4360 auto ExtractCanFuseWithFmul = [&]() {
4361 // We bail out if the extract is from lane 0.
4362 if (Index == 0)
4363 return false;
4364
4365 // Check if the scalar element type of the vector operand of ExtractElement
4366 // instruction is one of the allowed types.
4367 auto IsAllowedScalarTy = [&](const Type *T) {
4368 return T->isFloatTy() || T->isDoubleTy() ||
4369 (T->isHalfTy() && ST->hasFullFP16());
4370 };
4371
4372 // Check if the extractelement user is scalar fmul.
4373 auto IsUserFMulScalarTy = [](const Value *EEUser) {
4374 // Check if the user is scalar fmul.
4375 const auto *BO = dyn_cast<BinaryOperator>(Val: EEUser);
4376 return BO && BO->getOpcode() == BinaryOperator::FMul &&
4377 !BO->getType()->isVectorTy();
4378 };
4379
4380 // Check if the extract index is from lane 0 or lane equivalent to 0 for a
4381 // certain scalar type and a certain vector register width.
4382 auto IsExtractLaneEquivalentToZero = [&](unsigned Idx, unsigned EltSz) {
4383 auto RegWidth =
4384 getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector)
4385 .getFixedValue();
4386 return Idx == 0 || (RegWidth != 0 && (Idx * EltSz) % RegWidth == 0);
4387 };
4388
4389 // Check if the type constraints on input vector type and result scalar type
4390 // of extractelement instruction are satisfied.
4391 if (!isa<FixedVectorType>(Val) || !IsAllowedScalarTy(Val->getScalarType()))
4392 return false;
4393
4394 if (Scalar) {
4395 DenseMap<User *, unsigned> UserToExtractIdx;
4396 for (auto *U : Scalar->users()) {
4397 if (!IsUserFMulScalarTy(U))
4398 return false;
4399 // Recording entry for the user is important. Index value is not
4400 // important.
4401 UserToExtractIdx[U];
4402 }
4403 if (UserToExtractIdx.empty())
4404 return false;
4405 for (auto &[S, U, L] : ScalarUserAndIdx) {
4406 for (auto *U : S->users()) {
4407 if (UserToExtractIdx.contains(Val: U)) {
4408 auto *FMul = cast<BinaryOperator>(Val: U);
4409 auto *Op0 = FMul->getOperand(i_nocapture: 0);
4410 auto *Op1 = FMul->getOperand(i_nocapture: 1);
4411 if ((Op0 == S && Op1 == S) || Op0 != S || Op1 != S) {
4412 UserToExtractIdx[U] = L;
4413 break;
4414 }
4415 }
4416 }
4417 }
4418 for (auto &[U, L] : UserToExtractIdx) {
4419 if (!IsExtractLaneEquivalentToZero(Index, Val->getScalarSizeInBits()) &&
4420 !IsExtractLaneEquivalentToZero(L, Val->getScalarSizeInBits()))
4421 return false;
4422 }
4423 } else {
4424 const auto *EE = cast<ExtractElementInst>(Val: I);
4425
4426 const auto *IdxOp = dyn_cast<ConstantInt>(Val: EE->getIndexOperand());
4427 if (!IdxOp)
4428 return false;
4429
4430 return !EE->users().empty() && all_of(Range: EE->users(), P: [&](const User *U) {
4431 if (!IsUserFMulScalarTy(U))
4432 return false;
4433
4434 // Check if the other operand of extractelement is also extractelement
4435 // from lane equivalent to 0.
4436 const auto *BO = cast<BinaryOperator>(Val: U);
4437 const auto *OtherEE = dyn_cast<ExtractElementInst>(
4438 Val: BO->getOperand(i_nocapture: 0) == EE ? BO->getOperand(i_nocapture: 1) : BO->getOperand(i_nocapture: 0));
4439 if (OtherEE) {
4440 const auto *IdxOp = dyn_cast<ConstantInt>(Val: OtherEE->getIndexOperand());
4441 if (!IdxOp)
4442 return false;
4443 return IsExtractLaneEquivalentToZero(
4444 cast<ConstantInt>(Val: OtherEE->getIndexOperand())
4445 ->getValue()
4446 .getZExtValue(),
4447 OtherEE->getType()->getScalarSizeInBits());
4448 }
4449 return true;
4450 });
4451 }
4452 return true;
4453 };
4454
4455 if (Opcode == Instruction::ExtractElement && (I || Scalar) &&
4456 ExtractCanFuseWithFmul())
4457 return 0;
4458
4459 // All other insert/extracts cost this much.
4460 return CostKind == TTI::TCK_CodeSize ? 1
4461 : ST->getVectorInsertExtractBaseCost();
4462}
4463
4464InstructionCost AArch64TTIImpl::getVectorInstrCost(
4465 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
4466 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
4467 // Treat insert at lane 0 into a poison vector as having zero cost. This
4468 // ensures vector broadcasts via an insert + shuffle (and will be lowered to a
4469 // single dup) are treated as cheap.
4470 if (Opcode == Instruction::InsertElement && Index == 0 && Op0 &&
4471 isa<PoisonValue>(Val: Op0))
4472 return 0;
4473 return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, I: nullptr,
4474 Scalar: nullptr, ScalarUserAndIdx: {}, VIC);
4475}
4476
4477InstructionCost AArch64TTIImpl::getVectorInstrCost(
4478 unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index,
4479 Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
4480 TTI::VectorInstrContext VIC) const {
4481 return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, I: nullptr, Scalar,
4482 ScalarUserAndIdx, VIC);
4483}
4484
4485InstructionCost
4486AArch64TTIImpl::getVectorInstrCost(const Instruction &I, Type *Val,
4487 TTI::TargetCostKind CostKind, unsigned Index,
4488 TTI::VectorInstrContext VIC) const {
4489 return getVectorInstrCostHelper(Opcode: I.getOpcode(), Val, CostKind, Index, I: &I,
4490 Scalar: nullptr, ScalarUserAndIdx: {}, VIC);
4491}
4492
4493InstructionCost
4494AArch64TTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Val,
4495 TTI::TargetCostKind CostKind,
4496 unsigned Index) const {
4497 if (isa<FixedVectorType>(Val))
4498 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val, CostKind,
4499 Index);
4500
4501 // This typically requires both while and lastb instructions in order
4502 // to extract the last element. If this is in a loop the while
4503 // instruction can at least be hoisted out, although it will consume a
4504 // predicate register. The cost should be more expensive than the base
4505 // extract cost, which is 2 for most CPUs.
4506 return CostKind == TTI::TCK_CodeSize
4507 ? 2
4508 : ST->getVectorInsertExtractBaseCost() + 1;
4509}
4510
4511InstructionCost AArch64TTIImpl::getScalarizationOverhead(
4512 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
4513 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
4514 TTI::VectorInstrContext VIC) const {
4515 if (isa<ScalableVectorType>(Val: Ty))
4516 return InstructionCost::getInvalid();
4517 if (Ty->getElementType()->isFloatingPointTy())
4518 return BaseT::getScalarizationOverhead(InTy: Ty, DemandedElts, Insert, Extract,
4519 CostKind);
4520 unsigned VecInstCost =
4521 CostKind == TTI::TCK_CodeSize ? 1 : ST->getVectorInsertExtractBaseCost();
4522 return DemandedElts.popcount() * (Insert + Extract) * VecInstCost;
4523}
4524
4525std::optional<InstructionCost> AArch64TTIImpl::getFP16BF16PromoteCost(
4526 Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
4527 TTI::OperandValueInfo Op2Info, bool IncludeTrunc, bool CanUseSVE,
4528 std::function<InstructionCost(Type *)> InstCost) const {
4529 if (!Ty->getScalarType()->isHalfTy() && !Ty->getScalarType()->isBFloatTy())
4530 return std::nullopt;
4531 if (Ty->getScalarType()->isHalfTy() && ST->hasFullFP16())
4532 return std::nullopt;
4533 // If we have +sve-b16b16 the operation can be promoted to SVE.
4534 if (CanUseSVE && ST->hasSVEB16B16() && ST->isNonStreamingSVEorSME2Available())
4535 return std::nullopt;
4536
4537 Type *PromotedTy = Ty->getWithNewType(EltTy: Type::getFloatTy(C&: Ty->getContext()));
4538 InstructionCost Cost = getCastInstrCost(Opcode: Instruction::FPExt, Dst: PromotedTy, Src: Ty,
4539 CCH: TTI::CastContextHint::None, CostKind);
4540 if (!Op1Info.isConstant() && !Op2Info.isConstant())
4541 Cost *= 2;
4542 Cost += InstCost(PromotedTy);
4543 if (IncludeTrunc)
4544 Cost += getCastInstrCost(Opcode: Instruction::FPTrunc, Dst: Ty, Src: PromotedTy,
4545 CCH: TTI::CastContextHint::None, CostKind);
4546 return Cost;
4547}
4548
4549InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
4550 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
4551 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
4552 ArrayRef<const Value *> Args, const Instruction *CxtI) const {
4553
4554 // The code-generator is currently not able to handle scalable vectors
4555 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
4556 // it. This change will be removed when code-generation for these types is
4557 // sufficiently reliable.
4558 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: Ty))
4559 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
4560 return InstructionCost::getInvalid();
4561
4562 // TODO: Handle more cost kinds.
4563 if (CostKind != TTI::TCK_RecipThroughput)
4564 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
4565 Opd2Info: Op2Info, Args, CxtI);
4566
4567 // Legalize the type.
4568 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
4569 int ISD = TLI->InstructionOpcodeToISD(Opcode);
4570
4571 // Increase the cost for half and bfloat types if not architecturally
4572 // supported.
4573 if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
4574 ISD == ISD::FDIV || ISD == ISD::FREM) {
4575 if (auto PromotedCost = getFP16BF16PromoteCost(
4576 Ty, CostKind, Op1Info, Op2Info, /*IncludeTrunc=*/true,
4577 // There is not native support for fdiv/frem even with +sve-b16b16.
4578 /*CanUseSVE=*/ISD != ISD::FDIV && ISD != ISD::FREM,
4579 InstCost: [&](Type *PromotedTy) {
4580 return getArithmeticInstrCost(Opcode, Ty: PromotedTy, CostKind,
4581 Op1Info, Op2Info);
4582 }))
4583 return *PromotedCost;
4584
4585 // fp128 all go via libcalls
4586 if (Ty->getScalarType()->isFP128Ty())
4587 return (CostKind == TTI::TCK_CodeSize ? 1 : 10) * LT.first;
4588 }
4589
4590 // If the operation is a widening instruction (smull or umull) and both
4591 // operands are extends the cost can be cheaper by considering that the
4592 // operation will operate on the narrowest type size possible (double the
4593 // largest input size) and a further extend.
4594 if (Type *ExtTy = isBinExtWideningInstruction(Opcode, DstTy: Ty, Args)) {
4595 if (ExtTy != Ty)
4596 return getArithmeticInstrCost(Opcode, Ty: ExtTy, CostKind) +
4597 getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: ExtTy,
4598 CCH: TTI::CastContextHint::None, CostKind);
4599 return LT.first;
4600 }
4601
4602 switch (ISD) {
4603 default:
4604 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
4605 Opd2Info: Op2Info);
4606 case ISD::ADD:
4607 case ISD::SUB:
4608 return LT.first; // Also works for i128
4609 case ISD::MUL: {
4610 // i128 multiply is umulh + 2*madd + mul and grows ~O(Bitwidth^2). For
4611 // scalable vectors the cost of LT.first will be invalid, leading to an
4612 // invalid cost overall.
4613 unsigned Mul64CostFactor = (CostKind == TTI::TCK_RecipThroughput &&
4614 ST->hasLimited64bitVectorMulBandwidth())
4615 ? 4
4616 : 1;
4617 if (Ty->getScalarSizeInBits() > 64) {
4618 unsigned NumLanes = isa<FixedVectorType>(Val: Ty)
4619 ? cast<FixedVectorType>(Val: Ty)->getNumElements()
4620 : 1;
4621 InstructionCost CostPerLane = LT.first / NumLanes;
4622 return CostPerLane * CostPerLane * NumLanes * Mul64CostFactor;
4623 }
4624
4625 if (LT.second == MVT::v2i64) {
4626 // When SVE is available, then we can lower the v2i64 operation using
4627 // the SVE mul instruction, which has a lower cost.
4628 if (ST->hasSVE())
4629 return LT.first * Mul64CostFactor;
4630
4631 // When SVE is not available, there is no MUL.2d instruction,
4632 // which means mul <2 x i64> is expensive as elements are extracted
4633 // from the vectors and the muls scalarized.
4634 // As getScalarizationOverhead is a bit too pessimistic, we
4635 // estimate the cost for a i64 vector directly here, which is:
4636 // - four 2-cost i64 extracts,
4637 // - two 2-cost i64 inserts, and
4638 // - two 1-cost muls.
4639 // So, for a v2i64 with LT.First = 1 the cost is 14, and for a v4i64 with
4640 // LT.first = 2 the cost is 28.
4641 return cast<VectorType>(Val: Ty)->getElementCount().getKnownMinValue() *
4642 (getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind) +
4643 getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: Ty, CostKind, Index: -1,
4644 Op0: nullptr, Op1: nullptr) *
4645 2 +
4646 getVectorInstrCost(Opcode: Instruction::InsertElement, Val: Ty, CostKind, Index: -1,
4647 Op0: nullptr, Op1: nullptr));
4648 }
4649
4650 if (LT.second == MVT::nxv2i64)
4651 return LT.first * Mul64CostFactor;
4652
4653 return LT.first;
4654 }
4655 case ISD::SREM:
4656 case ISD::SDIV:
4657 /*
4658 Notes for sdiv/srem specific costs:
4659 1. This only considers the cases where the divisor is constant, uniform and
4660 (pow-of-2/non-pow-of-2). Other cases are not important since they either
4661 result in some form of (ldr + adrp), corresponding to constant vectors, or
4662 scalarization of the division operation.
4663 2. Constant divisors, either negative in whole or partially, don't result in
4664 significantly different codegen as compared to positive constant divisors.
4665 So, we don't consider negative divisors separately.
4666 3. If the codegen is significantly different with SVE, it has been indicated
4667 using comments at appropriate places.
4668
4669 sdiv specific cases:
4670 -----------------------------------------------------------------------
4671 codegen | pow-of-2 | Type
4672 -----------------------------------------------------------------------
4673 add + cmp + csel + asr | Y | i64
4674 add + cmp + csel + asr | Y | i32
4675 -----------------------------------------------------------------------
4676
4677 srem specific cases:
4678 -----------------------------------------------------------------------
4679 codegen | pow-of-2 | Type
4680 -----------------------------------------------------------------------
4681 negs + and + and + csneg | Y | i64
4682 negs + and + and + csneg | Y | i32
4683 -----------------------------------------------------------------------
4684
4685 other sdiv/srem cases:
4686 -------------------------------------------------------------------------
4687 common codegen | + srem | + sdiv | pow-of-2 | Type
4688 -------------------------------------------------------------------------
4689 smulh + asr + add + add | - | - | N | i64
4690 smull + lsr + add + add | - | - | N | i32
4691 usra | and + sub | sshr | Y | <2 x i64>
4692 2 * (scalar code) | - | - | N | <2 x i64>
4693 usra | bic + sub | sshr + neg | Y | <4 x i32>
4694 smull2 + smull + uzp2 | mls | - | N | <4 x i32>
4695 + sshr + usra | | | |
4696 -------------------------------------------------------------------------
4697 */
4698 if (Op2Info.isConstant() && Op2Info.isUniform()) {
4699 InstructionCost AddCost =
4700 getArithmeticInstrCost(Opcode: Instruction::Add, Ty, CostKind,
4701 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
4702 InstructionCost AsrCost =
4703 getArithmeticInstrCost(Opcode: Instruction::AShr, Ty, CostKind,
4704 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
4705 InstructionCost MulCost =
4706 getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind,
4707 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
4708 // add/cmp/csel/csneg should have similar cost while asr/negs/and should
4709 // have similar cost.
4710 auto VT = TLI->getValueType(DL, Ty);
4711 if (VT.isScalarInteger() && VT.getSizeInBits() <= 64) {
4712 if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) {
4713 // Neg can be folded into the asr instruction.
4714 return ISD == ISD::SDIV ? (3 * AddCost + AsrCost)
4715 : (3 * AsrCost + AddCost);
4716 } else {
4717 return MulCost + AsrCost + 2 * AddCost;
4718 }
4719 } else if (VT.isVector()) {
4720 InstructionCost UsraCost = 2 * AsrCost;
4721 if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) {
4722 // Division with scalable types corresponds to native 'asrd'
4723 // instruction when SVE is available.
4724 // e.g. %1 = sdiv <vscale x 4 x i32> %a, splat (i32 8)
4725
4726 // One more for the negation in SDIV
4727 InstructionCost Cost =
4728 (Op2Info.isNegatedPowerOf2() && ISD == ISD::SDIV) ? AsrCost : 0;
4729 if (Ty->isScalableTy() && ST->hasSVE())
4730 Cost += 2 * AsrCost;
4731 else {
4732 Cost +=
4733 UsraCost +
4734 (ISD == ISD::SDIV
4735 ? (LT.second.getScalarType() == MVT::i64 ? 1 : 2) * AsrCost
4736 : 2 * AddCost);
4737 }
4738 return Cost;
4739 } else if (LT.second == MVT::v2i64) {
4740 return VT.getVectorNumElements() *
4741 getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind,
4742 Op1Info: Op1Info.getNoProps(),
4743 Op2Info: Op2Info.getNoProps());
4744 } else {
4745 // When SVE is available, we get:
4746 // smulh + lsr + add/sub + asr + add/sub.
4747 if (Ty->isScalableTy() && ST->hasSVE())
4748 return MulCost /*smulh cost*/ + 2 * AddCost + 2 * AsrCost;
4749 return 2 * MulCost + AddCost /*uzp2 cost*/ + AsrCost + UsraCost;
4750 }
4751 }
4752 }
4753 if (Op2Info.isConstant() && !Op2Info.isUniform() &&
4754 LT.second.isFixedLengthVector()) {
4755 // FIXME: When the constant vector is non-uniform, this may result in
4756 // loading the vector from constant pool or in some cases, may also result
4757 // in scalarization. For now, we are approximating this with the
4758 // scalarization cost.
4759 auto ExtractCost = 2 * getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: Ty,
4760 CostKind, Index: -1, Op0: nullptr, Op1: nullptr);
4761 auto InsertCost = getVectorInstrCost(Opcode: Instruction::InsertElement, Val: Ty,
4762 CostKind, Index: -1, Op0: nullptr, Op1: nullptr);
4763 unsigned NElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
4764 return ExtractCost + InsertCost +
4765 NElts * getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(),
4766 CostKind, Op1Info: Op1Info.getNoProps(),
4767 Op2Info: Op2Info.getNoProps());
4768 }
4769 [[fallthrough]];
4770 case ISD::UDIV:
4771 case ISD::UREM: {
4772 auto VT = TLI->getValueType(DL, Ty);
4773 if (Op2Info.isConstant()) {
4774 // If the operand is a power of 2 we can use the shift or and cost.
4775 if (ISD == ISD::UDIV && Op2Info.isPowerOf2())
4776 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind,
4777 Op1Info: Op1Info.getNoProps(),
4778 Op2Info: Op2Info.getNoProps());
4779 if (ISD == ISD::UREM && Op2Info.isPowerOf2())
4780 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind,
4781 Op1Info: Op1Info.getNoProps(),
4782 Op2Info: Op2Info.getNoProps());
4783
4784 if (ISD == ISD::UDIV || ISD == ISD::UREM) {
4785 // Divides by a constant are expanded to MULHU + SUB + SRL + ADD + SRL.
4786 // The MULHU will be expanded to UMULL for the types not listed below,
4787 // and will become a pair of UMULL+MULL2 for 128bit vectors.
4788 bool HasMULH = VT == MVT::i64 || LT.second == MVT::nxv2i64 ||
4789 LT.second == MVT::nxv4i32 || LT.second == MVT::nxv8i16 ||
4790 LT.second == MVT::nxv16i8;
4791 bool Is128bit = LT.second.is128BitVector();
4792
4793 InstructionCost MulCost =
4794 getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind,
4795 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
4796 InstructionCost AddCost =
4797 getArithmeticInstrCost(Opcode: Instruction::Add, Ty, CostKind,
4798 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
4799 InstructionCost ShrCost =
4800 getArithmeticInstrCost(Opcode: Instruction::AShr, Ty, CostKind,
4801 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
4802 InstructionCost DivCost = MulCost * (Is128bit ? 2 : 1) + // UMULL/UMULH
4803 (HasMULH ? 0 : ShrCost) + // UMULL shift
4804 AddCost * 2 + ShrCost;
4805 return DivCost + (ISD == ISD::UREM ? MulCost + AddCost : 0);
4806 }
4807 }
4808
4809 // div i128's are lowered as libcalls. Pass nullptr as (u)divti3 calls are
4810 // emitted by the backend even when those functions are not declared in the
4811 // module.
4812 if (!VT.isVector() && VT.getSizeInBits() > 64)
4813 return getCallInstrCost(/*Function*/ F: nullptr, RetTy: Ty, Tys: {Ty, Ty}, CostKind);
4814
4815 InstructionCost Cost = BaseT::getArithmeticInstrCost(
4816 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
4817 if (Ty->isVectorTy() && (ISD == ISD::SDIV || ISD == ISD::UDIV)) {
4818 if (TLI->isOperationLegalOrCustom(Op: ISD, VT: LT.second) && ST->hasSVE()) {
4819 // SDIV/UDIV operations are lowered using SVE, then we can have less
4820 // costs.
4821 if (VT.isSimple() && isa<FixedVectorType>(Val: Ty) &&
4822 Ty->getPrimitiveSizeInBits().getFixedValue() < 128) {
4823 static const CostTblEntry DivTbl[]{
4824 {.ISD: ISD::SDIV, .Type: MVT::v2i8, .Cost: 5}, {.ISD: ISD::SDIV, .Type: MVT::v4i8, .Cost: 8},
4825 {.ISD: ISD::SDIV, .Type: MVT::v8i8, .Cost: 8}, {.ISD: ISD::SDIV, .Type: MVT::v2i16, .Cost: 5},
4826 {.ISD: ISD::SDIV, .Type: MVT::v4i16, .Cost: 5}, {.ISD: ISD::SDIV, .Type: MVT::v2i32, .Cost: 1},
4827 {.ISD: ISD::UDIV, .Type: MVT::v2i8, .Cost: 5}, {.ISD: ISD::UDIV, .Type: MVT::v4i8, .Cost: 8},
4828 {.ISD: ISD::UDIV, .Type: MVT::v8i8, .Cost: 8}, {.ISD: ISD::UDIV, .Type: MVT::v2i16, .Cost: 5},
4829 {.ISD: ISD::UDIV, .Type: MVT::v4i16, .Cost: 5}, {.ISD: ISD::UDIV, .Type: MVT::v2i32, .Cost: 1}};
4830
4831 const auto *Entry = CostTableLookup(Table: DivTbl, ISD, Ty: VT.getSimpleVT());
4832 if (nullptr != Entry)
4833 return Entry->Cost;
4834 }
4835 // For 8/16-bit elements, the cost is higher because the type
4836 // requires promotion and possibly splitting:
4837 if (LT.second.getScalarType() == MVT::i8)
4838 Cost *= 8;
4839 else if (LT.second.getScalarType() == MVT::i16)
4840 Cost *= 4;
4841 return Cost;
4842 } else {
4843 // If one of the operands is a uniform constant then the cost for each
4844 // element is Cost for insertion, extraction and division.
4845 // Insertion cost = 2, Extraction Cost = 2, Division = cost for the
4846 // operation with scalar type
4847 if ((Op1Info.isConstant() && Op1Info.isUniform()) ||
4848 (Op2Info.isConstant() && Op2Info.isUniform())) {
4849 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
4850 InstructionCost DivCost = BaseT::getArithmeticInstrCost(
4851 Opcode, Ty: Ty->getScalarType(), CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
4852 return (4 + DivCost) * VTy->getNumElements();
4853 }
4854 }
4855 // On AArch64, without SVE, vector divisions are expanded
4856 // into scalar divisions of each pair of elements.
4857 Cost += getVectorInstrCost(Opcode: Instruction::ExtractElement, Val: Ty, CostKind,
4858 Index: -1, Op0: nullptr, Op1: nullptr);
4859 Cost += getVectorInstrCost(Opcode: Instruction::InsertElement, Val: Ty, CostKind, Index: -1,
4860 Op0: nullptr, Op1: nullptr);
4861 }
4862
4863 // TODO: if one of the arguments is scalar, then it's not necessary to
4864 // double the cost of handling the vector elements.
4865 Cost += Cost;
4866 }
4867 return Cost;
4868 }
4869 case ISD::XOR:
4870 case ISD::OR:
4871 case ISD::AND:
4872 case ISD::SRL:
4873 case ISD::SRA:
4874 case ISD::SHL:
4875 // These nodes are marked as 'custom' for combining purposes only.
4876 // We know that they are legal. See LowerAdd in ISelLowering.
4877 return LT.first;
4878
4879 case ISD::FNEG:
4880 // Scalar fmul(fneg) or fneg(fmul) can be converted to fnmul
4881 if ((Ty->isFloatTy() || Ty->isDoubleTy() ||
4882 (Ty->isHalfTy() && ST->hasFullFP16())) &&
4883 CxtI &&
4884 ((CxtI->hasOneUse() &&
4885 match(V: *CxtI->user_begin(), P: m_FMul(L: m_Value(), R: m_Value()))) ||
4886 match(V: CxtI->getOperand(i: 0), P: m_FMul(L: m_Value(), R: m_Value()))))
4887 return 0;
4888 [[fallthrough]];
4889 case ISD::FADD:
4890 case ISD::FSUB:
4891 if (!Ty->getScalarType()->isFP128Ty())
4892 return LT.first;
4893 [[fallthrough]];
4894 case ISD::FMUL:
4895 case ISD::FDIV:
4896 // These nodes are marked as 'custom' just to lower them to SVE.
4897 // We know said lowering will incur no additional cost.
4898 if (!Ty->getScalarType()->isFP128Ty())
4899 return 2 * LT.first;
4900
4901 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
4902 Opd2Info: Op2Info);
4903 case ISD::FREM:
4904 // Pass nullptr as fmod/fmodf calls are emitted by the backend even when
4905 // those functions are not declared in the module.
4906 if (!Ty->isVectorTy())
4907 return getCallInstrCost(/*Function*/ F: nullptr, RetTy: Ty, Tys: {Ty, Ty}, CostKind);
4908 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
4909 Opd2Info: Op2Info);
4910 }
4911}
4912
4913InstructionCost
4914AArch64TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
4915 const SCEV *Ptr,
4916 TTI::TargetCostKind CostKind) const {
4917 // Address computations in vectorized code with non-consecutive addresses will
4918 // likely result in more instructions compared to scalar code where the
4919 // computation can more often be merged into the index mode. The resulting
4920 // extra micro-ops can significantly decrease throughput.
4921 unsigned NumVectorInstToHideOverhead = NeonNonConstStrideOverhead;
4922 int MaxMergeDistance = 64;
4923
4924 if (PtrTy->isVectorTy() && SE &&
4925 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MergeDistance: MaxMergeDistance + 1))
4926 return NumVectorInstToHideOverhead;
4927
4928 // In many cases the address computation is not merged into the instruction
4929 // addressing mode.
4930 return 1;
4931}
4932
4933/// Check whether Opcode1 has less throughput according to the scheduling
4934/// model than Opcode2.
4935bool AArch64TTIImpl::hasKnownLowerThroughputFromSchedulingModel(
4936 unsigned Opcode1, unsigned Opcode2) const {
4937 const MCSchedModel &Sched = ST->getSchedModel();
4938 const TargetInstrInfo *TII = ST->getInstrInfo();
4939 if (!Sched.hasInstrSchedModel())
4940 return false;
4941
4942 const MCSchedClassDesc *SCD1 =
4943 Sched.getSchedClassDesc(SchedClassIdx: TII->get(Opcode: Opcode1).getSchedClass());
4944 const MCSchedClassDesc *SCD2 =
4945 Sched.getSchedClassDesc(SchedClassIdx: TII->get(Opcode: Opcode2).getSchedClass());
4946 // We cannot handle variant scheduling classes without an MI. If we need to
4947 // support them for any of the instructions we query the information of we
4948 // might need to add a way to resolve them without a MI or not use the
4949 // scheduling info.
4950 assert(!SCD1->isVariant() && !SCD2->isVariant() &&
4951 "Cannot handle variant scheduling classes without an MI");
4952 if (!SCD1->isValid() || !SCD2->isValid())
4953 return false;
4954
4955 return MCSchedModel::getReciprocalThroughput(STI: *ST, SCDesc: *SCD1) >
4956 MCSchedModel::getReciprocalThroughput(STI: *ST, SCDesc: *SCD2);
4957}
4958
4959InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
4960 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
4961 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
4962 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
4963 // We don't lower some vector selects well that are wider than the register
4964 // width. TODO: Improve this with different cost kinds.
4965 if (isa<FixedVectorType>(Val: ValTy) && Opcode == Instruction::Select) {
4966 // We would need this many instructions to hide the scalarization happening.
4967 const int AmortizationCost = 20;
4968
4969 // If VecPred is not set, check if we can get a predicate from the context
4970 // instruction, if its type matches the requested ValTy.
4971 if (VecPred == CmpInst::BAD_ICMP_PREDICATE && I && I->getType() == ValTy) {
4972 CmpPredicate CurrentPred;
4973 if (match(V: I, P: m_Select(C: m_Cmp(Pred&: CurrentPred, L: m_Value(), R: m_Value()), L: m_Value(),
4974 R: m_Value())))
4975 VecPred = CurrentPred;
4976 }
4977 // Check if we have a compare/select chain that can be lowered using
4978 // a (F)CMxx & BFI pair.
4979 if (CmpInst::isIntPredicate(P: VecPred) || VecPred == CmpInst::FCMP_OLE ||
4980 VecPred == CmpInst::FCMP_OLT || VecPred == CmpInst::FCMP_OGT ||
4981 VecPred == CmpInst::FCMP_OGE || VecPred == CmpInst::FCMP_OEQ ||
4982 VecPred == CmpInst::FCMP_UNE) {
4983 static const auto ValidMinMaxTys = {
4984 MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
4985 MVT::v4i32, MVT::v2i64, MVT::v2f32, MVT::v4f32, MVT::v2f64};
4986 static const auto ValidFP16MinMaxTys = {MVT::v4f16, MVT::v8f16};
4987
4988 auto LT = getTypeLegalizationCost(Ty: ValTy);
4989 if (any_of(Range: ValidMinMaxTys, P: equal_to(Arg&: LT.second)) ||
4990 (ST->hasFullFP16() &&
4991 any_of(Range: ValidFP16MinMaxTys, P: equal_to(Arg&: LT.second))))
4992 return LT.first;
4993 }
4994
4995 static const TypeConversionCostTblEntry VectorSelectTbl[] = {
4996 {.ISD: Instruction::Select, .Dst: MVT::v2i1, .Src: MVT::v2f32, .Cost: 2},
4997 {.ISD: Instruction::Select, .Dst: MVT::v2i1, .Src: MVT::v2f64, .Cost: 2},
4998 {.ISD: Instruction::Select, .Dst: MVT::v4i1, .Src: MVT::v4f32, .Cost: 2},
4999 {.ISD: Instruction::Select, .Dst: MVT::v4i1, .Src: MVT::v4f16, .Cost: 2},
5000 {.ISD: Instruction::Select, .Dst: MVT::v8i1, .Src: MVT::v8f16, .Cost: 2},
5001 {.ISD: Instruction::Select, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: 16},
5002 {.ISD: Instruction::Select, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: 8},
5003 {.ISD: Instruction::Select, .Dst: MVT::v16i1, .Src: MVT::v16i32, .Cost: 16},
5004 {.ISD: Instruction::Select, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: 4 * AmortizationCost},
5005 {.ISD: Instruction::Select, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: 8 * AmortizationCost},
5006 {.ISD: Instruction::Select, .Dst: MVT::v16i1, .Src: MVT::v16i64, .Cost: 16 * AmortizationCost}};
5007
5008 EVT SelCondTy = TLI->getValueType(DL, Ty: CondTy);
5009 EVT SelValTy = TLI->getValueType(DL, Ty: ValTy);
5010 if (SelCondTy.isSimple() && SelValTy.isSimple()) {
5011 if (const auto *Entry = ConvertCostTableLookup(Table: VectorSelectTbl, ISD: Opcode,
5012 Dst: SelCondTy.getSimpleVT(),
5013 Src: SelValTy.getSimpleVT()))
5014 return Entry->Cost;
5015 }
5016 }
5017
5018 if (Opcode == Instruction::FCmp) {
5019 if (auto PromotedCost = getFP16BF16PromoteCost(
5020 Ty: ValTy, CostKind, Op1Info, Op2Info, /*IncludeTrunc=*/false,
5021 // TODO: Consider costing SVE FCMPs.
5022 /*CanUseSVE=*/false, InstCost: [&](Type *PromotedTy) {
5023 InstructionCost Cost =
5024 getCmpSelInstrCost(Opcode, ValTy: PromotedTy, CondTy, VecPred,
5025 CostKind, Op1Info, Op2Info);
5026 if (isa<VectorType>(Val: PromotedTy))
5027 Cost += getCastInstrCost(
5028 Opcode: Instruction::Trunc,
5029 Dst: VectorType::getInteger(VTy: cast<VectorType>(Val: ValTy)),
5030 Src: VectorType::getInteger(VTy: cast<VectorType>(Val: PromotedTy)),
5031 CCH: TTI::CastContextHint::None, CostKind);
5032 return Cost;
5033 }))
5034 return *PromotedCost;
5035
5036 auto LT = getTypeLegalizationCost(Ty: ValTy);
5037 // Model unknown fp compares as a libcall.
5038 if (LT.second.getScalarType() != MVT::f64 &&
5039 LT.second.getScalarType() != MVT::f32 &&
5040 LT.second.getScalarType() != MVT::f16)
5041 return LT.first * getCallInstrCost(/*Function*/ F: nullptr, RetTy: ValTy,
5042 Tys: {ValTy, ValTy}, CostKind);
5043
5044 // Some comparison operators require expanding to multiple compares + or.
5045 unsigned Factor = 1;
5046 if (!CondTy->isVectorTy() &&
5047 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ))
5048 Factor = 2; // fcmp with 2 selects
5049 else if (isa<FixedVectorType>(Val: ValTy) &&
5050 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ ||
5051 VecPred == FCmpInst::FCMP_ORD || VecPred == FCmpInst::FCMP_UNO))
5052 Factor = 3; // fcmxx+fcmyy+or
5053 else if (isa<ScalableVectorType>(Val: ValTy) &&
5054 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ))
5055 Factor = 3; // fcmxx+fcmyy+or
5056
5057 if (isa<ScalableVectorType>(Val: ValTy) &&
5058 CostKind == TTI::TCK_RecipThroughput &&
5059 hasKnownLowerThroughputFromSchedulingModel(Opcode1: AArch64::FCMEQ_PPzZZ_S,
5060 Opcode2: AArch64::FCMEQv4f32))
5061 Factor *= 2;
5062
5063 return Factor * (CostKind == TTI::TCK_Latency ? 2 : LT.first);
5064 }
5065
5066 // Treat the icmp in icmp(and, 0) or icmp(and, -1/1) when it can be folded to
5067 // icmp(and, 0) as free, as we can make use of ands, but only if the
5068 // comparison is not unsigned. FIXME: Enable for non-throughput cost kinds
5069 // providing it will not cause performance regressions.
5070 if (CostKind == TTI::TCK_RecipThroughput && ValTy->isIntegerTy() &&
5071 Opcode == Instruction::ICmp && I && !CmpInst::isUnsigned(Pred: VecPred) &&
5072 TLI->isTypeLegal(VT: TLI->getValueType(DL, Ty: ValTy)) &&
5073 match(V: I->getOperand(i: 0), P: m_And(L: m_Value(), R: m_Value()))) {
5074 if (match(V: I->getOperand(i: 1), P: m_Zero()))
5075 return 0;
5076
5077 // x >= 1 / x < 1 -> x > 0 / x <= 0
5078 if (match(V: I->getOperand(i: 1), P: m_One()) &&
5079 (VecPred == CmpInst::ICMP_SLT || VecPred == CmpInst::ICMP_SGE))
5080 return 0;
5081
5082 // x <= -1 / x > -1 -> x > 0 / x <= 0
5083 if (match(V: I->getOperand(i: 1), P: m_AllOnes()) &&
5084 (VecPred == CmpInst::ICMP_SLE || VecPred == CmpInst::ICMP_SGT))
5085 return 0;
5086 }
5087
5088 // The base case handles scalable vectors fine for now, since it treats the
5089 // cost as 1 * legalization cost.
5090 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
5091 Op1Info, Op2Info, I);
5092}
5093
5094AArch64TTIImpl::TTI::MemCmpExpansionOptions
5095AArch64TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
5096 TTI::MemCmpExpansionOptions Options;
5097 if (ST->requiresStrictAlign()) {
5098 // TODO: Add cost modeling for strict align. Misaligned loads expand to
5099 // a bunch of instructions when strict align is enabled.
5100 return Options;
5101 }
5102 Options.AllowOverlappingLoads = true;
5103 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
5104 Options.NumLoadsPerBlock = Options.MaxNumLoads;
5105 // TODO: Though vector loads usually perform well on AArch64, in some targets
5106 // they may wake up the FP unit, which raises the power consumption. Perhaps
5107 // they could be used with no holds barred (-O3).
5108 Options.LoadSizes = {8, 4, 2, 1};
5109 Options.AllowedTailExpansions = {3, 5, 6};
5110 return Options;
5111}
5112
5113bool AArch64TTIImpl::prefersVectorizedAddressing() const {
5114 return ST->hasSVE();
5115}
5116
5117InstructionCost
5118AArch64TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
5119 TTI::TargetCostKind CostKind) const {
5120 switch (MICA.getID()) {
5121 case Intrinsic::masked_scatter:
5122 case Intrinsic::masked_gather:
5123 return getGatherScatterOpCost(MICA, CostKind);
5124 case Intrinsic::masked_load:
5125 case Intrinsic::masked_expandload:
5126 case Intrinsic::masked_store:
5127 return getMaskedMemoryOpCost(MICA, CostKind);
5128 }
5129 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5130}
5131
5132InstructionCost
5133AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
5134 TTI::TargetCostKind CostKind) const {
5135 Type *Src = MICA.getDataType();
5136
5137 if (useNeonVector(Ty: Src))
5138 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5139 auto LT = getTypeLegalizationCost(Ty: Src);
5140 if (!LT.first.isValid())
5141 return InstructionCost::getInvalid();
5142
5143 // Return an invalid cost for element types that we are unable to lower.
5144 auto *VT = cast<VectorType>(Val: Src);
5145 if (VT->getElementType()->isIntegerTy(BitWidth: 1))
5146 return InstructionCost::getInvalid();
5147
5148 // The code-generator is currently not able to handle scalable vectors
5149 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
5150 // it. This change will be removed when code-generation for these types is
5151 // sufficiently reliable.
5152 if (VT->getElementCount() == ElementCount::getScalable(MinVal: 1))
5153 return InstructionCost::getInvalid();
5154
5155 InstructionCost MemOpCost = LT.first;
5156 if (MICA.getID() == Intrinsic::masked_expandload) {
5157 if (!isLegalMaskedExpandLoad(DataTy: Src, Alignment: MICA.getAlignment()))
5158 return InstructionCost::getInvalid();
5159
5160 // Operation will be split into expand of masked.load
5161 MemOpCost *= 2;
5162 }
5163
5164 // If we need to split the memory operation, we will also need to split the
5165 // mask. This will likely lead to overestimating the cost in some cases if
5166 // multiple memory operations use the same mask, but we often don't have
5167 // enough context to figure that out here.
5168 //
5169 // If the elements being loaded are bytes then the mask will already be split,
5170 // since the number of bits in a P register matches the number of bytes in a
5171 // Z register.
5172 if (LT.first > 1 && LT.second.getScalarSizeInBits() > 8)
5173 return MemOpCost * 2;
5174
5175 return MemOpCost;
5176}
5177
5178// This function returns gather/scatter overhead either from
5179// user-provided value or specialized values per-target from \p ST.
5180static unsigned getSVEGatherScatterOverhead(unsigned Opcode,
5181 const AArch64Subtarget *ST) {
5182 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
5183 "Should be called on only load or stores.");
5184 switch (Opcode) {
5185 case Instruction::Load:
5186 if (SVEGatherOverhead.getNumOccurrences() > 0)
5187 return SVEGatherOverhead;
5188 return ST->getGatherOverhead();
5189 break;
5190 case Instruction::Store:
5191 if (SVEScatterOverhead.getNumOccurrences() > 0)
5192 return SVEScatterOverhead;
5193 return ST->getScatterOverhead();
5194 break;
5195 default:
5196 llvm_unreachable("Shouldn't have reached here");
5197 }
5198}
5199
5200InstructionCost
5201AArch64TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
5202 TTI::TargetCostKind CostKind) const {
5203
5204 unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
5205 MICA.getID() == Intrinsic::vp_gather)
5206 ? Instruction::Load
5207 : Instruction::Store;
5208
5209 Type *DataTy = MICA.getDataType();
5210 Align Alignment = MICA.getAlignment();
5211 const Instruction *I = MICA.getInst();
5212
5213 if (useNeonVector(Ty: DataTy) || !isLegalMaskedGatherScatter(DataType: DataTy))
5214 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5215 auto *VT = cast<VectorType>(Val: DataTy);
5216 auto LT = getTypeLegalizationCost(Ty: DataTy);
5217 if (!LT.first.isValid())
5218 return InstructionCost::getInvalid();
5219
5220 // Return an invalid cost for element types that we are unable to lower.
5221 if (!LT.second.isVector() ||
5222 !isElementTypeLegalForScalableVector(Ty: VT->getElementType()) ||
5223 VT->getElementType()->isIntegerTy(BitWidth: 1))
5224 return InstructionCost::getInvalid();
5225
5226 // The code-generator is currently not able to handle scalable vectors
5227 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
5228 // it. This change will be removed when code-generation for these types is
5229 // sufficiently reliable.
5230 if (VT->getElementCount() == ElementCount::getScalable(MinVal: 1))
5231 return InstructionCost::getInvalid();
5232
5233 ElementCount LegalVF = LT.second.getVectorElementCount();
5234 InstructionCost MemOpCost =
5235 getMemoryOpCost(Opcode, Src: VT->getElementType(), Alignment, AddressSpace: 0, CostKind,
5236 OpInfo: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
5237 // Add on an overhead cost for using gathers/scatters.
5238 MemOpCost *= getSVEGatherScatterOverhead(Opcode, ST);
5239 return LT.first * MemOpCost * getMaxNumElements(VF: LegalVF);
5240}
5241
5242bool AArch64TTIImpl::useNeonVector(const Type *Ty) const {
5243 return isa<FixedVectorType>(Val: Ty) && !ST->useSVEForFixedLengthVectors();
5244}
5245
5246InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
5247 Align Alignment,
5248 unsigned AddressSpace,
5249 TTI::TargetCostKind CostKind,
5250 TTI::OperandValueInfo OpInfo,
5251 const Instruction *I) const {
5252 EVT VT = TLI->getValueType(DL, Ty, AllowUnknown: true);
5253 // Type legalization can't handle structs
5254 if (VT == MVT::Other)
5255 return BaseT::getMemoryOpCost(Opcode, Src: Ty, Alignment, AddressSpace,
5256 CostKind);
5257
5258 auto LT = getTypeLegalizationCost(Ty);
5259 if (!LT.first.isValid())
5260 return InstructionCost::getInvalid();
5261
5262 // The code-generator is currently not able to handle scalable vectors
5263 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
5264 // it. This change will be removed when code-generation for these types is
5265 // sufficiently reliable.
5266 // We also only support full register predicate loads and stores.
5267 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: Ty))
5268 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1) ||
5269 (VTy->getElementType()->isIntegerTy(BitWidth: 1) &&
5270 !VTy->getElementCount().isKnownMultipleOf(
5271 RHS: ElementCount::getScalable(MinVal: 16))))
5272 return InstructionCost::getInvalid();
5273
5274 // TODO: consider latency as well for TCK_SizeAndLatency.
5275 if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
5276 return LT.first;
5277
5278 if (CostKind == TTI::TCK_Latency) {
5279 // Latency doesn't make much sense for stores, so just return 1
5280 if (Opcode == Instruction::Store)
5281 return 1;
5282 // If the subtarget has overridden the load latency then use that instead of
5283 // querying the SchedModel.
5284 if (ST->getFixedLoadLatency())
5285 return (LT.first - 1) + ST->getFixedLoadLatency();
5286 // We expect the load to become LT.first loads of type LT.second. The
5287 // latency will be the latency of the last load plus the time it gets to get
5288 // there, which will be the amount of other loads before that (i.e. total
5289 // loads - 1) multiplied by how long it takes to get through them (the
5290 // reciprocal of the throughput). We get the latency and reciprocal
5291 // throughput from the SchedModel, and assume that the loads become the
5292 // variant with unsigned integer offset.
5293 unsigned Inst = 0;
5294 if (LT.second.isScalableVector() ||
5295 ST->useSVEForFixedLengthVectors(VT: LT.second)) {
5296 Inst = AArch64::LDR_ZXI;
5297 } else if (LT.second.isVector() || LT.second.isFloatingPoint()) {
5298 switch (LT.second.getSizeInBits()) {
5299 case 8:
5300 Inst = AArch64::LDRBui;
5301 break;
5302 case 16:
5303 Inst = AArch64::LDRHui;
5304 break;
5305 case 32:
5306 Inst = AArch64::LDRSui;
5307 break;
5308 case 64:
5309 Inst = AArch64::LDRDui;
5310 break;
5311 case 128:
5312 Inst = AArch64::LDRQui;
5313 break;
5314 default:
5315 llvm_unreachable("Unexpected float or vector type");
5316 }
5317 } else {
5318 switch (LT.second.getSizeInBits()) {
5319 case 8:
5320 Inst = AArch64::LDRBBui;
5321 break;
5322 case 16:
5323 Inst = AArch64::LDRHHui;
5324 break;
5325 case 32:
5326 Inst = AArch64::LDRWui;
5327 break;
5328 case 64:
5329 Inst = AArch64::LDRXui;
5330 break;
5331 default:
5332 llvm_unreachable("Unexpected integer type");
5333 }
5334 }
5335 const MCSchedModel &Sched = ST->getSchedModel();
5336 const TargetInstrInfo *TII = ST->getInstrInfo();
5337 unsigned SchedClass = TII->get(Opcode: Inst).getSchedClass();
5338 const MCSchedClassDesc *SCD = Sched.getSchedClassDesc(SchedClassIdx: SchedClass);
5339 // We need to convert the number of loads before the last to a float here,
5340 // as the reciprocal throughput may be fractional.
5341 float NumLoads = (LT.first - 1).getValue();
5342 return NumLoads * Sched.getReciprocalThroughput(STI: *ST, SCDesc: *SCD) +
5343 Sched.computeInstrLatency(STI: *ST, SCDesc: *SCD);
5344 }
5345
5346 if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store &&
5347 LT.second.is128BitVector() && Alignment < Align(16)) {
5348 // Unaligned stores are extremely inefficient. We don't split all
5349 // unaligned 128-bit stores because the negative impact that has shown in
5350 // practice on inlined block copy code.
5351 // We make such stores expensive so that we will only vectorize if there
5352 // are 6 other instructions getting vectorized.
5353 const int AmortizationCost = 6;
5354
5355 return LT.first * 2 * AmortizationCost;
5356 }
5357
5358 // Opaque ptr or ptr vector types are i64s and can be lowered to STP/LDPs.
5359 if (Ty->isPtrOrPtrVectorTy())
5360 return LT.first;
5361
5362 if (useNeonVector(Ty)) {
5363 // Check truncating stores and extending loads.
5364 if (Ty->getScalarSizeInBits() != LT.second.getScalarSizeInBits()) {
5365 // v4i8 types are lowered to scalar a load/store and sshll/xtn.
5366 if (VT == MVT::v4i8)
5367 return 2;
5368 // Otherwise we need to scalarize.
5369 return cast<FixedVectorType>(Val: Ty)->getNumElements() * 2;
5370 }
5371 EVT EltVT = VT.getVectorElementType();
5372 unsigned EltSize = EltVT.getScalarSizeInBits();
5373 if (!isPowerOf2_32(Value: EltSize) || EltSize < 8 || EltSize > 64 ||
5374 VT.getVectorNumElements() >= (128 / EltSize) || Alignment != Align(1))
5375 return LT.first;
5376 // FIXME: v3i8 lowering currently is very inefficient, due to automatic
5377 // widening to v4i8, which produces suboptimal results.
5378 if (VT.getVectorNumElements() == 3 && EltVT == MVT::i8)
5379 return LT.first;
5380
5381 // Check non-power-of-2 loads/stores for legal vector element types with
5382 // NEON. Non-power-of-2 memory ops will get broken down to a set of
5383 // operations on smaller power-of-2 ops, including ld1/st1.
5384 LLVMContext &C = Ty->getContext();
5385 InstructionCost Cost(0);
5386 SmallVector<EVT> TypeWorklist;
5387 TypeWorklist.push_back(Elt: VT);
5388 while (!TypeWorklist.empty()) {
5389 EVT CurrVT = TypeWorklist.pop_back_val();
5390 unsigned CurrNumElements = CurrVT.getVectorNumElements();
5391 if (isPowerOf2_32(Value: CurrNumElements)) {
5392 Cost += 1;
5393 continue;
5394 }
5395
5396 unsigned PrevPow2 = NextPowerOf2(A: CurrNumElements) / 2;
5397 TypeWorklist.push_back(Elt: EVT::getVectorVT(Context&: C, VT: EltVT, NumElements: PrevPow2));
5398 TypeWorklist.push_back(
5399 Elt: EVT::getVectorVT(Context&: C, VT: EltVT, NumElements: CurrNumElements - PrevPow2));
5400 }
5401 return Cost;
5402 }
5403
5404 return LT.first;
5405}
5406
5407InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
5408 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
5409 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
5410 bool UseMaskForCond, bool UseMaskForGaps) const {
5411 assert(Factor >= 2 && "Invalid interleave factor");
5412 auto *VecVTy = cast<VectorType>(Val: VecTy);
5413
5414 if (VecTy->isScalableTy() && !ST->hasSVE())
5415 return InstructionCost::getInvalid();
5416
5417 // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
5418 // only have lowering for power-of-2 factors.
5419 // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
5420 // InterleavedAccessPass for ld3/st3
5421 if (VecTy->isScalableTy() && !isPowerOf2_32(Value: Factor))
5422 return InstructionCost::getInvalid();
5423
5424 // Vectorization for masked interleaved accesses is only enabled for scalable
5425 // VF.
5426 if (!VecTy->isScalableTy() && (UseMaskForCond || UseMaskForGaps))
5427 return InstructionCost::getInvalid();
5428
5429 if (!UseMaskForGaps && Factor <= TLI->getMaxSupportedInterleaveFactor()) {
5430 ElementCount EC = VecVTy->getElementCount();
5431 auto *SubVecTy = VectorType::get(ElementType: VecVTy->getElementType(),
5432 EC: EC.divideCoefficientBy(RHS: Factor));
5433
5434 // ldN/stN only support legal vector types of size 64 or 128 in bits.
5435 // Accesses having vector types that are a multiple of 128 bits can be
5436 // matched to more than one ldN/stN instruction.
5437 bool UseScalable;
5438 if (EC.isKnownMultipleOf(RHS: Factor) &&
5439 TLI->isLegalInterleavedAccessType(VecTy: SubVecTy, DL, UseScalable))
5440 return Factor * TLI->getNumInterleavedAccesses(VecTy: SubVecTy, DL, UseScalable);
5441
5442 // Cost the alternative approach for scalable vectors where the interleave
5443 // factor is larger than the VF: use a contiguous load/store of the full
5444 // wide vector followed by deinterleave/interleave shuffles.
5445 if (VecTy->isScalableTy() && EC.isKnownMultipleOf(RHS: Factor)) {
5446 if (SubVecTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
5447 return InstructionCost::getInvalid();
5448
5449 // Cost of the contiguous memory operation on the wide vector.
5450 InstructionCost MemCost;
5451 if (UseMaskForCond) {
5452 unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
5453 : Intrinsic::masked_store;
5454 MemCost = getMemIntrinsicInstrCost(
5455 MICA: MemIntrinsicCostAttributes(IID, VecTy, Alignment, AddressSpace),
5456 CostKind);
5457 } else {
5458 MemCost =
5459 getMemoryOpCost(Opcode, Ty: VecTy, Alignment, AddressSpace, CostKind);
5460 }
5461
5462 // llvm.vector.deinterleaveN is lowered as a binary tree of deinterleave2
5463 // operations. The tree has Log2(Factor) levels, with Factor UZP/ZIP
5464 // operations at each level, giving a total shuffle cost of
5465 // Factor * Log2(Factor).
5466 auto SubVecCost = getTypeLegalizationCost(Ty: SubVecTy);
5467 auto ResultCost = getTypeLegalizationCost(Ty: VecTy);
5468 llvm::InstructionCost LegalizationCost = SubVecCost.first;
5469
5470 // FIXME: A temporary increase to the cost in cases where the input
5471 // element type is 4x the output type. Otherwise it produces an SVE tail
5472 // loop which is significantly larger than the NEON equivalent.
5473 if (Opcode == Instruction::Store && Factor == 4 &&
5474 SubVecCost.second.getScalarSizeInBits() ==
5475 (4 * ResultCost.second.getScalarSizeInBits()))
5476 LegalizationCost *= 4;
5477
5478 return MemCost + (Factor * LegalizationCost) + (Factor * Log2_64(Value: Factor));
5479 }
5480 }
5481
5482 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
5483 Alignment, AddressSpace, CostKind,
5484 UseMaskForCond, UseMaskForGaps);
5485}
5486
5487InstructionCost
5488AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
5489 InstructionCost Cost = 0;
5490 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
5491 for (auto *I : Tys) {
5492 if (!I->isVectorTy())
5493 continue;
5494 if (I->getScalarSizeInBits() * cast<FixedVectorType>(Val: I)->getNumElements() ==
5495 128)
5496 Cost += getMemoryOpCost(Opcode: Instruction::Store, Ty: I, Alignment: Align(128), AddressSpace: 0, CostKind) +
5497 getMemoryOpCost(Opcode: Instruction::Load, Ty: I, Alignment: Align(128), AddressSpace: 0, CostKind);
5498 }
5499 return Cost;
5500}
5501
5502bool AArch64TTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
5503 Align Alignment) const {
5504 // Neon types should be scalarised when we are not choosing to use SVE.
5505 if (useNeonVector(Ty: DataTy))
5506 return false;
5507
5508 // Return true only if we are able to lower using the SVE2p2/SME2p2
5509 // expand instruction.
5510 return (ST->isSVEAvailable() && ST->hasSVE2p2()) ||
5511 (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2());
5512}
5513
5514unsigned
5515AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF,
5516 bool HasUnorderedReductions) const {
5517 if (VF.isScalar() || (HasUnorderedReductions && VF.getKnownMinValue() <= 4))
5518 return 4;
5519 return ST->getMaxInterleaveFactor();
5520}
5521
5522// For Falkor, we want to avoid having too many strided loads in a loop since
5523// that can exhaust the HW prefetcher resources. We adjust the unroller
5524// MaxCount preference below to attempt to ensure unrolling doesn't create too
5525// many strided loads.
5526static void
5527getFalkorUnrollingPreferences(Loop *L, ScalarEvolution &SE,
5528 TargetTransformInfo::UnrollingPreferences &UP) {
5529 enum { MaxStridedLoads = 7 };
5530 auto countStridedLoads = [](Loop *L, ScalarEvolution &SE) {
5531 int StridedLoads = 0;
5532 // FIXME? We could make this more precise by looking at the CFG and
5533 // e.g. not counting loads in each side of an if-then-else diamond.
5534 for (const auto BB : L->blocks()) {
5535 for (auto &I : *BB) {
5536 LoadInst *LMemI = dyn_cast<LoadInst>(Val: &I);
5537 if (!LMemI)
5538 continue;
5539
5540 Value *PtrValue = LMemI->getPointerOperand();
5541 if (L->isLoopInvariant(V: PtrValue))
5542 continue;
5543
5544 const SCEV *LSCEV = SE.getSCEV(V: PtrValue);
5545 const SCEVAddRecExpr *LSCEVAddRec = dyn_cast<SCEVAddRecExpr>(Val: LSCEV);
5546 if (!LSCEVAddRec || !LSCEVAddRec->isAffine())
5547 continue;
5548
5549 // FIXME? We could take pairing of unrolled load copies into account
5550 // by looking at the AddRec, but we would probably have to limit this
5551 // to loops with no stores or other memory optimization barriers.
5552 ++StridedLoads;
5553 // We've seen enough strided loads that seeing more won't make a
5554 // difference.
5555 if (StridedLoads > MaxStridedLoads / 2)
5556 return StridedLoads;
5557 }
5558 }
5559 return StridedLoads;
5560 };
5561
5562 int StridedLoads = countStridedLoads(L, SE);
5563 LLVM_DEBUG(dbgs() << "falkor-hwpf: detected " << StridedLoads
5564 << " strided loads\n");
5565 // Pick the largest power of 2 unroll count that won't result in too many
5566 // strided loads.
5567 if (StridedLoads) {
5568 UP.MaxCount = 1 << Log2_32(Value: MaxStridedLoads / StridedLoads);
5569 LLVM_DEBUG(dbgs() << "falkor-hwpf: setting unroll MaxCount to "
5570 << UP.MaxCount << '\n');
5571 }
5572}
5573
5574// This function returns true if the loop:
5575// 1. Has a valid cost, and
5576// 2. Has a cost within the supplied budget.
5577// Otherwise it returns false.
5578static bool isLoopSizeWithinBudget(Loop *L, const AArch64TTIImpl &TTI,
5579 InstructionCost Budget,
5580 unsigned *FinalSize) {
5581 // Estimate the size of the loop.
5582 InstructionCost LoopCost = 0;
5583
5584 for (auto *BB : L->getBlocks()) {
5585 for (auto &I : *BB) {
5586 SmallVector<const Value *, 4> Operands(I.operand_values());
5587 InstructionCost Cost =
5588 TTI.getInstructionCost(U: &I, Operands, CostKind: TTI::TCK_CodeSize);
5589 // This can happen with intrinsics that don't currently have a cost model
5590 // or for some operations that require SVE.
5591 if (!Cost.isValid())
5592 return false;
5593
5594 LoopCost += Cost;
5595 if (LoopCost > Budget)
5596 return false;
5597 }
5598 }
5599
5600 if (FinalSize)
5601 *FinalSize = LoopCost.getValue();
5602 return true;
5603}
5604
5605static bool shouldUnrollMultiExitLoop(Loop *L, ScalarEvolution &SE,
5606 const AArch64TTIImpl &TTI) {
5607 // Only consider loops with unknown trip counts for which we can determine
5608 // a symbolic expression. Multi-exit loops with small known trip counts will
5609 // likely be unrolled anyway.
5610 const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L);
5611 if (isa<SCEVConstant>(Val: BTC) || isa<SCEVCouldNotCompute>(Val: BTC))
5612 return false;
5613
5614 // It might not be worth unrolling loops with low max trip counts. Restrict
5615 // this to max trip counts > 32 for now.
5616 unsigned MaxTC = SE.getSmallConstantMaxTripCount(L);
5617 if (MaxTC > 0 && MaxTC <= 32)
5618 return false;
5619
5620 // Make sure the loop size is <= 5.
5621 if (!isLoopSizeWithinBudget(L, TTI, Budget: 5, FinalSize: nullptr))
5622 return false;
5623
5624 // Small search loops with multiple exits can be highly beneficial to unroll.
5625 // We only care about loops with exactly two exiting blocks, although each
5626 // block could jump to the same exit block.
5627 ArrayRef<BasicBlock *> Blocks = L->getBlocks();
5628 if (Blocks.size() != 2)
5629 return false;
5630
5631 if (any_of(Range&: Blocks, P: [](BasicBlock *BB) {
5632 return !isa<UncondBrInst, CondBrInst>(Val: BB->getTerminator());
5633 }))
5634 return false;
5635
5636 return true;
5637}
5638
5639/// For Apple CPUs, we want to runtime-unroll loops to make better use if the
5640/// OOO engine's wide instruction window and various predictors.
5641static void
5642getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
5643 TargetTransformInfo::UnrollingPreferences &UP,
5644 const AArch64TTIImpl &TTI) {
5645 // Limit loops with structure that is highly likely to benefit from runtime
5646 // unrolling; that is we exclude outer loops and loops with many blocks (i.e.
5647 // likely with complex control flow). Note that the heuristics here may be
5648 // overly conservative and we err on the side of avoiding runtime unrolling
5649 // rather than unroll excessively. They are all subject to further refinement.
5650 if (!L->isInnermost() || L->getNumBlocks() > 8)
5651 return;
5652
5653 // Loops with multiple exits are handled by common code.
5654 if (!L->getExitBlock())
5655 return;
5656
5657 // Check if the loop contains any reductions that could be parallelized when
5658 // unrolling. If so, enable partial unrolling, if the trip count is know to be
5659 // a multiple of 2.
5660 bool HasParellelizableReductions =
5661 L->getNumBlocks() == 1 &&
5662 any_of(Range: L->getHeader()->phis(),
5663 P: [&SE, L](PHINode &Phi) {
5664 return canParallelizeReductionWhenUnrolling(Phi, L, SE: &SE);
5665 }) &&
5666 isLoopSizeWithinBudget(L, TTI, Budget: 12, FinalSize: nullptr);
5667 if (HasParellelizableReductions &&
5668 SE.getSmallConstantTripMultiple(L, ExitingBlock: L->getExitingBlock()) % 2 == 0) {
5669 UP.Partial = true;
5670 UP.MaxCount = 4;
5671 UP.AddAdditionalAccumulators = true;
5672 }
5673
5674 const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L);
5675 if (isa<SCEVConstant>(Val: BTC) || isa<SCEVCouldNotCompute>(Val: BTC) ||
5676 (SE.getSmallConstantMaxTripCount(L) > 0 &&
5677 SE.getSmallConstantMaxTripCount(L) <= 32))
5678 return;
5679
5680 if (findStringMetadataForLoop(TheLoop: L, Name: "llvm.loop.isvectorized"))
5681 return;
5682
5683 if (SE.getSymbolicMaxBackedgeTakenCount(L) != SE.getBackedgeTakenCount(L))
5684 return;
5685
5686 // Limit to loops with trip counts that are cheap to expand.
5687 UP.SCEVExpansionBudget = 1;
5688
5689 if (HasParellelizableReductions) {
5690 UP.Runtime = true;
5691 UP.DefaultUnrollRuntimeCount = 4;
5692 UP.AddAdditionalAccumulators = true;
5693 }
5694
5695 // Try to unroll small loops, of few-blocks with low budget, if they have
5696 // load/store dependencies, to expose more parallel memory access streams,
5697 // or if they do little work inside a block (i.e. load -> X -> store pattern).
5698 BasicBlock *Header = L->getHeader();
5699 BasicBlock *Latch = L->getLoopLatch();
5700 if (Header == Latch) {
5701 // Estimate the size of the loop.
5702 unsigned Size;
5703 unsigned Width = 10;
5704 if (!isLoopSizeWithinBudget(L, TTI, Budget: Width, FinalSize: &Size))
5705 return;
5706
5707 // Try to find an unroll count that maximizes the use of the instruction
5708 // window, i.e. trying to fetch as many instructions per cycle as possible.
5709 unsigned MaxInstsPerLine = 16;
5710 unsigned UC = 1;
5711 unsigned BestUC = 1;
5712 unsigned SizeWithBestUC = BestUC * Size;
5713 while (UC <= 8) {
5714 unsigned SizeWithUC = UC * Size;
5715 if (SizeWithUC > 48)
5716 break;
5717 if ((SizeWithUC % MaxInstsPerLine) == 0 ||
5718 (SizeWithBestUC % MaxInstsPerLine) < (SizeWithUC % MaxInstsPerLine)) {
5719 BestUC = UC;
5720 SizeWithBestUC = BestUC * Size;
5721 }
5722 UC++;
5723 }
5724
5725 if (BestUC == 1)
5726 return;
5727
5728 SmallPtrSet<Value *, 8> LoadedValuesPlus;
5729 SmallVector<StoreInst *> Stores;
5730 for (auto *BB : L->blocks()) {
5731 for (auto &I : *BB) {
5732 Value *Ptr = getLoadStorePointerOperand(V: &I);
5733 if (!Ptr)
5734 continue;
5735 const SCEV *PtrSCEV = SE.getSCEV(V: Ptr);
5736 if (SE.isLoopInvariant(S: PtrSCEV, L))
5737 continue;
5738 if (isa<LoadInst>(Val: &I)) {
5739 LoadedValuesPlus.insert(Ptr: &I);
5740 // Include in-loop 1st users of loaded values.
5741 for (auto *U : I.users())
5742 if (L->contains(Inst: cast<Instruction>(Val: U)))
5743 LoadedValuesPlus.insert(Ptr: U);
5744 } else
5745 Stores.push_back(Elt: cast<StoreInst>(Val: &I));
5746 }
5747 }
5748
5749 if (none_of(Range&: Stores, P: [&LoadedValuesPlus](StoreInst *SI) {
5750 return LoadedValuesPlus.contains(Ptr: SI->getOperand(i_nocapture: 0));
5751 }))
5752 return;
5753
5754 UP.Runtime = true;
5755 UP.DefaultUnrollRuntimeCount = BestUC;
5756 return;
5757 }
5758
5759 // Try to runtime-unroll loops with early-continues depending on loop-varying
5760 // loads; this helps with branch-prediction for the early-continues.
5761 auto *Term = dyn_cast<CondBrInst>(Val: Header->getTerminator());
5762 SmallVector<BasicBlock *> Preds(predecessors(BB: Latch));
5763 if (!Term || Preds.size() == 1 || !llvm::is_contained(Range&: Preds, Element: Header) ||
5764 none_of(Range&: Preds, P: [L](BasicBlock *Pred) { return L->contains(BB: Pred); }))
5765 return;
5766
5767 std::function<bool(Instruction *, unsigned)> DependsOnLoopLoad =
5768 [&](Instruction *I, unsigned Depth) -> bool {
5769 if (isa<PHINode>(Val: I) || L->isLoopInvariant(V: I) || Depth > 8)
5770 return false;
5771
5772 if (isa<LoadInst>(Val: I))
5773 return true;
5774
5775 return any_of(Range: I->operands(), P: [&](Value *V) {
5776 auto *I = dyn_cast<Instruction>(Val: V);
5777 return I && DependsOnLoopLoad(I, Depth + 1);
5778 });
5779 };
5780 CmpPredicate Pred;
5781 Instruction *I;
5782 if (match(V: Term, P: m_Br(C: m_ICmp(Pred, L: m_Instruction(I), R: m_Value()), T: m_Value(),
5783 F: m_Value())) &&
5784 DependsOnLoopLoad(I, 0)) {
5785 UP.Runtime = true;
5786 }
5787}
5788
5789void AArch64TTIImpl::getUnrollingPreferences(
5790 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
5791 OptimizationRemarkEmitter *ORE) const {
5792 // Enable partial unrolling and runtime unrolling.
5793 BaseT::getUnrollingPreferences(L, SE, UP, ORE);
5794
5795 UP.UpperBound = true;
5796
5797 // For inner loop, it is more likely to be a hot one, and the runtime check
5798 // can be promoted out from LICM pass, so the overhead is less, let's try
5799 // a larger threshold to unroll more loops.
5800 if (L->getLoopDepth() > 1)
5801 UP.PartialThreshold *= 2;
5802
5803 // Disable partial & runtime unrolling on -Os.
5804 UP.PartialOptSizeThreshold = 0;
5805
5806 // Scan the loop: don't unroll loops with calls as this could prevent
5807 // inlining. Don't unroll auto-vectorized loops either, though do allow
5808 // unrolling of the scalar remainder.
5809 bool IsVectorized = getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized");
5810 InstructionCost Cost = 0;
5811 for (auto *BB : L->getBlocks()) {
5812 for (auto &I : *BB) {
5813 // Both auto-vectorized loops and the scalar remainder have the
5814 // isvectorized attribute, so differentiate between them by the presence
5815 // of vector instructions.
5816 if (IsVectorized && I.getType()->isVectorTy())
5817 return;
5818 if (isa<CallBase>(Val: I)) {
5819 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I))
5820 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction())
5821 if (!isLoweredToCall(F))
5822 continue;
5823 return;
5824 }
5825
5826 SmallVector<const Value *, 4> Operands(I.operand_values());
5827 Cost += getInstructionCost(U: &I, Operands,
5828 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
5829 }
5830 }
5831
5832 // Apply subtarget-specific unrolling preferences.
5833 if (ST->isAppleMLike())
5834 getAppleRuntimeUnrollPreferences(L, SE, UP, TTI: *this);
5835 else if (ST->getProcFamily() == AArch64Subtarget::Falkor &&
5836 EnableFalkorHWPFUnrollFix)
5837 getFalkorUnrollingPreferences(L, SE, UP);
5838
5839 // If this is a small, multi-exit loop similar to something like std::find,
5840 // then there is typically a performance improvement achieved by unrolling.
5841 if (!L->getExitBlock() && shouldUnrollMultiExitLoop(L, SE, TTI: *this)) {
5842 UP.RuntimeUnrollMultiExit = true;
5843 UP.Runtime = true;
5844 // Limit unroll count.
5845 UP.DefaultUnrollRuntimeCount = 4;
5846 // Allow slightly more costly trip-count expansion to catch search loops
5847 // with pointer inductions.
5848 UP.SCEVExpansionBudget = 5;
5849 return;
5850 }
5851
5852 // Enable runtime unrolling for in-order models
5853 // If mcpu is omitted, getProcFamily() returns AArch64Subtarget::Others, so by
5854 // checking for that case, we can ensure that the default behaviour is
5855 // unchanged
5856 if (ST->getProcFamily() != AArch64Subtarget::Generic &&
5857 !ST->getSchedModel().isOutOfOrder()) {
5858 UP.Runtime = true;
5859 UP.Partial = true;
5860 UP.UnrollRemainder = true;
5861 UP.DefaultUnrollRuntimeCount = 4;
5862
5863 UP.UnrollAndJam = true;
5864 UP.UnrollAndJamInnerLoopThreshold = 60;
5865 }
5866
5867 // Force unrolling small loops can be very useful because of the branch
5868 // taken cost of the backedge.
5869 if (Cost < Aarch64ForceUnrollThreshold)
5870 UP.Force = true;
5871}
5872
5873void AArch64TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
5874 TTI::PeelingPreferences &PP) const {
5875 BaseT::getPeelingPreferences(L, SE, PP);
5876}
5877
5878Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
5879 Type *ExpectedType,
5880 bool CanCreate) const {
5881 switch (Inst->getIntrinsicID()) {
5882 default:
5883 return nullptr;
5884 case Intrinsic::aarch64_neon_st1x2:
5885 case Intrinsic::aarch64_neon_st1x3:
5886 case Intrinsic::aarch64_neon_st1x4:
5887 case Intrinsic::aarch64_neon_st2:
5888 case Intrinsic::aarch64_neon_st3:
5889 case Intrinsic::aarch64_neon_st4: {
5890 // Create a struct type
5891 StructType *ST = dyn_cast<StructType>(Val: ExpectedType);
5892 if (!CanCreate || !ST)
5893 return nullptr;
5894 unsigned NumElts = Inst->arg_size() - 1;
5895 if (ST->getNumElements() != NumElts)
5896 return nullptr;
5897 for (unsigned i = 0, e = NumElts; i != e; ++i) {
5898 if (Inst->getArgOperand(i)->getType() != ST->getElementType(N: i))
5899 return nullptr;
5900 }
5901 Value *Res = PoisonValue::get(T: ExpectedType);
5902 IRBuilder<> Builder(Inst);
5903 for (unsigned i = 0, e = NumElts; i != e; ++i) {
5904 Value *L = Inst->getArgOperand(i);
5905 Res = Builder.CreateInsertValue(Agg: Res, Val: L, Idxs: i);
5906 }
5907 return Res;
5908 }
5909 case Intrinsic::aarch64_neon_ld1x2:
5910 case Intrinsic::aarch64_neon_ld1x3:
5911 case Intrinsic::aarch64_neon_ld1x4:
5912 case Intrinsic::aarch64_neon_ld2:
5913 case Intrinsic::aarch64_neon_ld3:
5914 case Intrinsic::aarch64_neon_ld4:
5915 if (Inst->getType() == ExpectedType)
5916 return Inst;
5917 return nullptr;
5918 }
5919}
5920
5921bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
5922 MemIntrinsicInfo &Info) const {
5923 switch (Inst->getIntrinsicID()) {
5924 default:
5925 break;
5926 case Intrinsic::aarch64_neon_ld1x2:
5927 case Intrinsic::aarch64_neon_ld1x3:
5928 case Intrinsic::aarch64_neon_ld1x4:
5929 case Intrinsic::aarch64_neon_ld2:
5930 case Intrinsic::aarch64_neon_ld3:
5931 case Intrinsic::aarch64_neon_ld4:
5932 Info.ReadMem = true;
5933 Info.WriteMem = false;
5934 Info.PtrVal = Inst->getArgOperand(i: 0);
5935 break;
5936 case Intrinsic::aarch64_neon_st1x2:
5937 case Intrinsic::aarch64_neon_st1x3:
5938 case Intrinsic::aarch64_neon_st1x4:
5939 case Intrinsic::aarch64_neon_st2:
5940 case Intrinsic::aarch64_neon_st3:
5941 case Intrinsic::aarch64_neon_st4:
5942 Info.ReadMem = false;
5943 Info.WriteMem = true;
5944 Info.PtrVal = Inst->getArgOperand(i: Inst->arg_size() - 1);
5945 break;
5946 }
5947
5948 // Use the ID of neon load as the "matching id".
5949 switch (Inst->getIntrinsicID()) {
5950 default:
5951 return false;
5952 case Intrinsic::aarch64_neon_ld1x2:
5953 case Intrinsic::aarch64_neon_st1x2:
5954 Info.MatchingId = Intrinsic::aarch64_neon_ld1x2;
5955 break;
5956 case Intrinsic::aarch64_neon_ld1x3:
5957 case Intrinsic::aarch64_neon_st1x3:
5958 Info.MatchingId = Intrinsic::aarch64_neon_ld1x3;
5959 break;
5960 case Intrinsic::aarch64_neon_ld1x4:
5961 case Intrinsic::aarch64_neon_st1x4:
5962 Info.MatchingId = Intrinsic::aarch64_neon_ld1x4;
5963 break;
5964 case Intrinsic::aarch64_neon_ld2:
5965 case Intrinsic::aarch64_neon_st2:
5966 Info.MatchingId = Intrinsic::aarch64_neon_ld2;
5967 break;
5968 case Intrinsic::aarch64_neon_ld3:
5969 case Intrinsic::aarch64_neon_st3:
5970 Info.MatchingId = Intrinsic::aarch64_neon_ld3;
5971 break;
5972 case Intrinsic::aarch64_neon_ld4:
5973 case Intrinsic::aarch64_neon_st4:
5974 Info.MatchingId = Intrinsic::aarch64_neon_ld4;
5975 break;
5976 }
5977 return true;
5978}
5979
5980/// See if \p I should be considered for address type promotion. We check if \p
5981/// I is a sext with right type and used in memory accesses. If it used in a
5982/// "complex" getelementptr, we allow it to be promoted without finding other
5983/// sext instructions that sign extended the same initial value. A getelementptr
5984/// is considered as "complex" if it has more than 2 operands.
5985bool AArch64TTIImpl::shouldConsiderAddressTypePromotion(
5986 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
5987 bool Considerable = false;
5988 AllowPromotionWithoutCommonHeader = false;
5989 if (!isa<SExtInst>(Val: &I))
5990 return false;
5991 Type *ConsideredSExtType =
5992 Type::getInt64Ty(C&: I.getParent()->getParent()->getContext());
5993 if (I.getType() != ConsideredSExtType)
5994 return false;
5995 // See if the sext is the one with the right type and used in at least one
5996 // GetElementPtrInst.
5997 for (const User *U : I.users()) {
5998 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(Val: U)) {
5999 Considerable = true;
6000 // A getelementptr is considered as "complex" if it has more than 2
6001 // operands. We will promote a SExt used in such complex GEP as we
6002 // expect some computation to be merged if they are done on 64 bits.
6003 if (GEPInst->getNumOperands() > 2) {
6004 AllowPromotionWithoutCommonHeader = true;
6005 break;
6006 }
6007 }
6008 }
6009 return Considerable;
6010}
6011
6012bool AArch64TTIImpl::isLegalToVectorizeReduction(
6013 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {
6014 if (!VF.isScalable())
6015 return true;
6016
6017 Type *Ty = RdxDesc.getRecurrenceType();
6018 if (Ty->isBFloatTy() || !isElementTypeLegalForScalableVector(Ty))
6019 return false;
6020
6021 switch (RdxDesc.getRecurrenceKind()) {
6022 case RecurKind::Sub:
6023 case RecurKind::FSub:
6024 case RecurKind::AddChainWithSubs:
6025 case RecurKind::FAddChainWithSubs:
6026 case RecurKind::Add:
6027 case RecurKind::FAdd:
6028 case RecurKind::And:
6029 case RecurKind::Or:
6030 case RecurKind::Xor:
6031 case RecurKind::SMin:
6032 case RecurKind::SMax:
6033 case RecurKind::UMin:
6034 case RecurKind::UMax:
6035 case RecurKind::FMin:
6036 case RecurKind::FMax:
6037 case RecurKind::FMulAdd:
6038 case RecurKind::AnyOf:
6039 case RecurKind::FindLast:
6040 return true;
6041 default:
6042 return false;
6043 }
6044}
6045
6046InstructionCost
6047AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
6048 FastMathFlags FMF,
6049 TTI::TargetCostKind CostKind) const {
6050 // The code-generator is currently not able to handle scalable vectors
6051 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
6052 // it. This change will be removed when code-generation for these types is
6053 // sufficiently reliable.
6054 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: Ty))
6055 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
6056 return InstructionCost::getInvalid();
6057
6058 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
6059
6060 if (LT.second.getScalarType() == MVT::f16 && !ST->hasFullFP16())
6061 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
6062
6063 InstructionCost LegalizationCost = 0;
6064 if (LT.first > 1) {
6065 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Context&: Ty->getContext());
6066 IntrinsicCostAttributes Attrs(IID, LegalVTy, {LegalVTy, LegalVTy}, FMF);
6067 LegalizationCost = getIntrinsicInstrCost(ICA: Attrs, CostKind) * (LT.first - 1);
6068 }
6069
6070 return LegalizationCost + /*Cost of horizontal reduction*/ 2;
6071}
6072
6073InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE(
6074 unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) const {
6075 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
6076 InstructionCost LegalizationCost = 0;
6077 if (LT.first > 1) {
6078 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Context&: ValTy->getContext());
6079 LegalizationCost = getArithmeticInstrCost(Opcode, Ty: LegalVTy, CostKind);
6080 LegalizationCost *= LT.first - 1;
6081 }
6082
6083 int ISD = TLI->InstructionOpcodeToISD(Opcode);
6084 assert(ISD && "Invalid opcode");
6085 // Add the final reduction cost for the legal horizontal reduction
6086 switch (ISD) {
6087 case ISD::ADD:
6088 case ISD::AND:
6089 case ISD::OR:
6090 case ISD::XOR:
6091 case ISD::FADD:
6092 return LegalizationCost + 2;
6093 default:
6094 return InstructionCost::getInvalid();
6095 }
6096}
6097
6098InstructionCost
6099AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
6100 std::optional<FastMathFlags> FMF,
6101 TTI::TargetCostKind CostKind) const {
6102 // The code-generator is currently not able to handle scalable vectors
6103 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
6104 // it. This change will be removed when code-generation for these types is
6105 // sufficiently reliable.
6106 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: ValTy))
6107 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
6108 return InstructionCost::getInvalid();
6109
6110 if (TTI::requiresOrderedReduction(FMF)) {
6111 if (auto *FixedVTy = dyn_cast<FixedVectorType>(Val: ValTy)) {
6112 InstructionCost BaseCost =
6113 BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
6114 // Add on extra cost to reflect the extra overhead on some CPUs. We still
6115 // end up vectorizing for more computationally intensive loops.
6116 return BaseCost + FixedVTy->getNumElements();
6117 }
6118
6119 if (Opcode != Instruction::FAdd || ValTy->getElementType()->isBFloatTy())
6120 return InstructionCost::getInvalid();
6121
6122 auto *VTy = cast<ScalableVectorType>(Val: ValTy);
6123 InstructionCost Cost =
6124 getArithmeticInstrCost(Opcode, Ty: VTy->getScalarType(), CostKind);
6125 Cost *= getMaxNumElements(VF: VTy->getElementCount());
6126 return Cost;
6127 }
6128
6129 if (isa<ScalableVectorType>(Val: ValTy))
6130 return getArithmeticReductionCostSVE(Opcode, ValTy, CostKind);
6131
6132 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
6133 MVT MTy = LT.second;
6134 int ISD = TLI->InstructionOpcodeToISD(Opcode);
6135 assert(ISD && "Invalid opcode");
6136
6137 // Horizontal adds can use the 'addv' instruction. We model the cost of these
6138 // instructions as twice a normal vector add, plus 1 for each legalization
6139 // step (LT.first). This is the only arithmetic vector reduction operation for
6140 // which we have an instruction.
6141 // OR, XOR and AND costs should match the codegen from:
6142 // OR: llvm/test/CodeGen/AArch64/reduce-or.ll
6143 // XOR: llvm/test/CodeGen/AArch64/reduce-xor.ll
6144 // AND: llvm/test/CodeGen/AArch64/reduce-and.ll
6145 static const CostTblEntry CostTblNoPairwise[]{
6146 {.ISD: ISD::ADD, .Type: MVT::v8i8, .Cost: 2},
6147 {.ISD: ISD::ADD, .Type: MVT::v16i8, .Cost: 2},
6148 {.ISD: ISD::ADD, .Type: MVT::v4i16, .Cost: 2},
6149 {.ISD: ISD::ADD, .Type: MVT::v8i16, .Cost: 2},
6150 {.ISD: ISD::ADD, .Type: MVT::v2i32, .Cost: 2},
6151 {.ISD: ISD::ADD, .Type: MVT::v4i32, .Cost: 2},
6152 {.ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: 2},
6153 {.ISD: ISD::OR, .Type: MVT::v8i8, .Cost: 5}, // fmov + orr_lsr + orr_lsr + lsr + orr
6154 {.ISD: ISD::OR, .Type: MVT::v16i8, .Cost: 7}, // ext + orr + same as v8i8
6155 {.ISD: ISD::OR, .Type: MVT::v4i16, .Cost: 4}, // fmov + orr_lsr + lsr + orr
6156 {.ISD: ISD::OR, .Type: MVT::v8i16, .Cost: 6}, // ext + orr + same as v4i16
6157 {.ISD: ISD::OR, .Type: MVT::v2i32, .Cost: 3}, // fmov + lsr + orr
6158 {.ISD: ISD::OR, .Type: MVT::v4i32, .Cost: 5}, // ext + orr + same as v2i32
6159 {.ISD: ISD::OR, .Type: MVT::v2i64, .Cost: 3}, // ext + orr + fmov
6160 {.ISD: ISD::XOR, .Type: MVT::v8i8, .Cost: 5}, // Same as above for or...
6161 {.ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: 7},
6162 {.ISD: ISD::XOR, .Type: MVT::v4i16, .Cost: 4},
6163 {.ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: 6},
6164 {.ISD: ISD::XOR, .Type: MVT::v2i32, .Cost: 3},
6165 {.ISD: ISD::XOR, .Type: MVT::v4i32, .Cost: 5},
6166 {.ISD: ISD::XOR, .Type: MVT::v2i64, .Cost: 3},
6167 {.ISD: ISD::AND, .Type: MVT::v8i8, .Cost: 5}, // Same as above for or...
6168 {.ISD: ISD::AND, .Type: MVT::v16i8, .Cost: 7},
6169 {.ISD: ISD::AND, .Type: MVT::v4i16, .Cost: 4},
6170 {.ISD: ISD::AND, .Type: MVT::v8i16, .Cost: 6},
6171 {.ISD: ISD::AND, .Type: MVT::v2i32, .Cost: 3},
6172 {.ISD: ISD::AND, .Type: MVT::v4i32, .Cost: 5},
6173 {.ISD: ISD::AND, .Type: MVT::v2i64, .Cost: 3},
6174 };
6175 switch (ISD) {
6176 default:
6177 break;
6178 case ISD::FADD:
6179 if (Type *EltTy = ValTy->getScalarType();
6180 // FIXME: For half types without fullfp16 support, this could extend and
6181 // use a fp32 faddp reduction but current codegen unrolls.
6182 MTy.isVector() && (EltTy->isFloatTy() || EltTy->isDoubleTy() ||
6183 (EltTy->isHalfTy() && ST->hasFullFP16()))) {
6184 const unsigned NElts = MTy.getVectorNumElements();
6185 if (ValTy->getElementCount().getFixedValue() >= 2 && NElts >= 2 &&
6186 isPowerOf2_32(Value: NElts))
6187 // Reduction corresponding to series of fadd instructions is lowered to
6188 // series of faddp instructions. faddp has latency/throughput that
6189 // matches fadd instruction and hence, every faddp instruction can be
6190 // considered to have a relative cost = 1 with
6191 // CostKind = TCK_RecipThroughput.
6192 // An faddp will pairwise add vector elements, so the size of input
6193 // vector reduces by half every time, requiring
6194 // #(faddp instructions) = log2_32(NElts).
6195 return (LT.first - 1) + /*No of faddp instructions*/ Log2_32(Value: NElts);
6196 }
6197 break;
6198 case ISD::ADD:
6199 if (const auto *Entry = CostTableLookup(Table: CostTblNoPairwise, ISD, Ty: MTy))
6200 return (LT.first - 1) + Entry->Cost;
6201 break;
6202 case ISD::XOR:
6203 case ISD::AND:
6204 case ISD::OR:
6205 const auto *Entry = CostTableLookup(Table: CostTblNoPairwise, ISD, Ty: MTy);
6206 if (!Entry)
6207 break;
6208 auto *ValVTy = cast<FixedVectorType>(Val: ValTy);
6209 if (MTy.getVectorNumElements() <= ValVTy->getNumElements() &&
6210 isPowerOf2_32(Value: ValVTy->getNumElements())) {
6211 InstructionCost ExtraCost = 0;
6212 if (LT.first != 1) {
6213 // Type needs to be split, so there is an extra cost of LT.first - 1
6214 // arithmetic ops.
6215 auto *Ty = FixedVectorType::get(ElementType: ValTy->getElementType(),
6216 NumElts: MTy.getVectorNumElements());
6217 ExtraCost = getArithmeticInstrCost(Opcode, Ty, CostKind);
6218 ExtraCost *= LT.first - 1;
6219 }
6220 // All and/or/xor of i1 will be lowered with maxv/minv/addv + fmov
6221 auto Cost = ValVTy->getElementType()->isIntegerTy(BitWidth: 1) ? 2 : Entry->Cost;
6222 return Cost + ExtraCost;
6223 }
6224 break;
6225 }
6226 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
6227}
6228
6229InstructionCost AArch64TTIImpl::getExtendedReductionCost(
6230 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *VecTy,
6231 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
6232 EVT VecVT = TLI->getValueType(DL, Ty: VecTy);
6233 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
6234
6235 if (Opcode == Instruction::Add && VecVT.isSimple() && ResVT.isSimple() &&
6236 VecVT.getSizeInBits() >= 64) {
6237 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VecTy);
6238
6239 // The legal cases are:
6240 // UADDLV 8/16/32->32
6241 // UADDLP 32->64
6242 unsigned RevVTSize = ResVT.getSizeInBits();
6243 if (((LT.second == MVT::v8i8 || LT.second == MVT::v16i8) &&
6244 RevVTSize <= 32) ||
6245 ((LT.second == MVT::v4i16 || LT.second == MVT::v8i16) &&
6246 RevVTSize <= 32) ||
6247 ((LT.second == MVT::v2i32 || LT.second == MVT::v4i32) &&
6248 RevVTSize <= 64))
6249 return (LT.first - 1) * 2 + 2;
6250 }
6251
6252 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: VecTy, FMF,
6253 CostKind);
6254}
6255
6256InstructionCost
6257AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
6258 Type *ResTy, VectorType *VecTy,
6259 TTI::TargetCostKind CostKind) const {
6260 EVT VecVT = TLI->getValueType(DL, Ty: VecTy);
6261 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
6262
6263 if (ST->hasDotProd() && VecVT.isSimple() && ResVT.isSimple() &&
6264 RedOpcode == Instruction::Add) {
6265 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VecTy);
6266
6267 // The legal cases with dotprod are
6268 // UDOT 8->32
6269 // Which requires an additional uaddv to sum the i32 values.
6270 if ((LT.second == MVT::v8i8 || LT.second == MVT::v16i8) &&
6271 ResVT == MVT::i32)
6272 return LT.first + 2;
6273 }
6274
6275 return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty: VecTy,
6276 CostKind);
6277}
6278
6279InstructionCost
6280AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index,
6281 TTI::TargetCostKind CostKind) const {
6282 static const CostTblEntry ShuffleTbl[] = {
6283 { .ISD: TTI::SK_Splice, .Type: MVT::nxv16i8, .Cost: 1 },
6284 { .ISD: TTI::SK_Splice, .Type: MVT::nxv8i16, .Cost: 1 },
6285 { .ISD: TTI::SK_Splice, .Type: MVT::nxv4i32, .Cost: 1 },
6286 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2i64, .Cost: 1 },
6287 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2f16, .Cost: 1 },
6288 { .ISD: TTI::SK_Splice, .Type: MVT::nxv4f16, .Cost: 1 },
6289 { .ISD: TTI::SK_Splice, .Type: MVT::nxv8f16, .Cost: 1 },
6290 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2bf16, .Cost: 1 },
6291 { .ISD: TTI::SK_Splice, .Type: MVT::nxv4bf16, .Cost: 1 },
6292 { .ISD: TTI::SK_Splice, .Type: MVT::nxv8bf16, .Cost: 1 },
6293 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2f32, .Cost: 1 },
6294 { .ISD: TTI::SK_Splice, .Type: MVT::nxv4f32, .Cost: 1 },
6295 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2f64, .Cost: 1 },
6296 };
6297
6298 // The code-generator is currently not able to handle scalable vectors
6299 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
6300 // it. This change will be removed when code-generation for these types is
6301 // sufficiently reliable.
6302 if (Tp->getElementCount() == ElementCount::getScalable(MinVal: 1))
6303 return InstructionCost::getInvalid();
6304
6305 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Tp);
6306 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Context&: Tp->getContext());
6307 EVT PromotedVT = LT.second.getScalarType() == MVT::i1
6308 ? TLI->getPromotedVTForPredicate(VT: EVT(LT.second))
6309 : LT.second;
6310 Type *PromotedVTy = EVT(PromotedVT).getTypeForEVT(Context&: Tp->getContext());
6311 InstructionCost LegalizationCost = 0;
6312 if (Index < 0) {
6313 LegalizationCost =
6314 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: PromotedVTy, CondTy: PromotedVTy,
6315 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind) +
6316 getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: PromotedVTy, CondTy: LegalVTy,
6317 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
6318 }
6319
6320 // Predicated splice are promoted when lowering. See AArch64ISelLowering.cpp
6321 // Cost performed on a promoted type.
6322 if (LT.second.getScalarType() == MVT::i1) {
6323 LegalizationCost +=
6324 getCastInstrCost(Opcode: Instruction::ZExt, Dst: PromotedVTy, Src: LegalVTy,
6325 CCH: TTI::CastContextHint::None, CostKind) +
6326 getCastInstrCost(Opcode: Instruction::Trunc, Dst: LegalVTy, Src: PromotedVTy,
6327 CCH: TTI::CastContextHint::None, CostKind);
6328 }
6329 const auto *Entry =
6330 CostTableLookup(Table: ShuffleTbl, ISD: TTI::SK_Splice, Ty: PromotedVT.getSimpleVT());
6331 assert(Entry && "Illegal Type for Splice");
6332 LegalizationCost += Entry->Cost;
6333 return LegalizationCost * LT.first;
6334}
6335
6336InstructionCost AArch64TTIImpl::getPartialReductionCost(
6337 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
6338 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
6339 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
6340 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
6341 InstructionCost Invalid = InstructionCost::getInvalid();
6342
6343 if (CostKind != TTI::TCK_RecipThroughput)
6344 return Invalid;
6345
6346 if ((Opcode != Instruction::Add && Opcode != Instruction::Sub &&
6347 Opcode != Instruction::FAdd && Opcode != Instruction::FSub) ||
6348 OpAExtend == TTI::PR_None)
6349 return Invalid;
6350
6351 // Floating-point partial reductions are invalid if `reassoc` and `contract`
6352 // are not allowed.
6353 if (AccumType->isFloatingPointTy()) {
6354 assert(FMF && "Missing FastMathFlags for floating-point partial reduction");
6355 if (!FMF->allowReassoc() || !FMF->allowContract())
6356 return Invalid;
6357 } else {
6358 assert(!FMF &&
6359 "FastMathFlags only apply to floating-point partial reductions");
6360 }
6361
6362 assert((BinOp || (OpBExtend == TTI::PR_None && !InputTypeB)) &&
6363 (!BinOp || (OpBExtend != TTI::PR_None && InputTypeB)) &&
6364 "Unexpected values for OpBExtend or InputTypeB");
6365
6366 // We only support multiply binary operations for now, and for muls we
6367 // require the types being extended to be the same.
6368 if (BinOp && ((*BinOp != Instruction::Mul && *BinOp != Instruction::FMul) ||
6369 InputTypeA != InputTypeB))
6370 return Invalid;
6371
6372 bool IsUSDot = OpBExtend != TTI::PR_None && OpAExtend != OpBExtend;
6373 // USDot is natively supported with +i8mm. With plain +dotprod, SUMLA is
6374 // lowered to two udots plus an eor and a sub.
6375 if (IsUSDot && !ST->hasMatMulInt8() && !ST->hasDotProd())
6376 // FIXME: Remove this early bailout in favour of expand cost.
6377 return Invalid;
6378
6379 unsigned Ratio =
6380 AccumType->getScalarSizeInBits() / InputTypeA->getScalarSizeInBits();
6381 if (VF.getKnownMinValue() <= Ratio)
6382 return Invalid;
6383
6384 VectorType *InputVectorType = VectorType::get(ElementType: InputTypeA, EC: VF);
6385 VectorType *AccumVectorType =
6386 VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: Ratio));
6387 // We don't yet support all kinds of legalization.
6388 auto TC = TLI->getTypeConversion(Context&: AccumVectorType->getContext(),
6389 VT: EVT::getEVT(Ty: AccumVectorType));
6390 switch (TC.first) {
6391 default:
6392 return Invalid;
6393 case TargetLowering::TypeLegal:
6394 case TargetLowering::TypePromoteInteger:
6395 case TargetLowering::TypeSplitVector:
6396 // The legalised type (e.g. after splitting) must be legal too.
6397 if (TLI->getTypeAction(Context&: AccumVectorType->getContext(), VT: TC.second) !=
6398 TargetLowering::TypeLegal)
6399 return Invalid;
6400 break;
6401 }
6402
6403 std::pair<InstructionCost, MVT> AccumLT =
6404 getTypeLegalizationCost(Ty: AccumVectorType);
6405 std::pair<InstructionCost, MVT> InputLT =
6406 getTypeLegalizationCost(Ty: InputVectorType);
6407
6408 // Returns true if the subtarget supports the operation for a given type.
6409 auto IsSupported = [&](bool SVEPred, bool NEONPred) -> bool {
6410 return (ST->isSVEorStreamingSVEAvailable() && SVEPred) ||
6411 (AccumLT.second.isFixedLengthVector() &&
6412 AccumLT.second.getSizeInBits() <= 128 && ST->isNeonAvailable() &&
6413 NEONPred);
6414 };
6415
6416 bool IsSub = Opcode == Instruction::Sub || Opcode == Instruction::FSub;
6417 InstructionCost Cost = InputLT.first * TTI::TCC_Basic;
6418 // Integer partial sub-reductions that don't map to a specific instruction,
6419 // carry an extra cost for implementing a double negation:
6420 // partial_reduce_umls acc, lhs, rhs
6421 // <=> -partial_reduce_umla -acc, lhs, rhs
6422 InstructionCost INegCost = IsSub ? 2 * InputLT.first * TTI::TCC_Basic : 0;
6423
6424 if (AccumLT.second.getScalarType() == MVT::i32 &&
6425 InputLT.second.getScalarType() == MVT::i8) {
6426 // i8 -> i32 is natively supported with udot/sdot for both NEON and SVE.
6427 if (!IsUSDot && IsSupported(true, ST->hasDotProd()))
6428 return Cost + INegCost;
6429 // i8 -> i32 usdot requires +i8mm
6430 if (IsUSDot && IsSupported(ST->hasMatMulInt8(), ST->hasMatMulInt8()))
6431 return Cost + INegCost;
6432 // Without +i8mm, lower SUMLA via two udots plus an eor and a sub on plain
6433 // +dotprod targets. Note that this is only implemented for NEON, as all
6434 // modern CPUs with SVE also have +i8mm. Charge an extra factor for the
6435 // expansion.
6436 if (IsUSDot && IsSupported(false, ST->hasDotProd()))
6437 return Cost * 3 + INegCost;
6438 }
6439
6440 if (ST->isSVEorStreamingSVEAvailable() && !IsUSDot) {
6441 // i16 -> i64 is natively supported for udot/sdot
6442 if (AccumLT.second.getScalarType() == MVT::i64 &&
6443 InputLT.second.getScalarType() == MVT::i16)
6444 return Cost + INegCost;
6445 // i16 -> i32 is natively supported with SVE2p1 udot/sdot.
6446 // For sub-reductions, we prefer using the *mlslb/t instructions.
6447 if (AccumLT.second.getScalarType() == MVT::i32 &&
6448 InputLT.second.getScalarType() == MVT::i16 &&
6449 (ST->hasSVE2p1() || ST->hasSME2()) && !IsSub)
6450 return Cost;
6451 // i8 -> i64 is supported with an extra level of extends
6452 if (AccumLT.second.getScalarType() == MVT::i64 &&
6453 InputLT.second.getScalarType() == MVT::i8)
6454 // FIXME: This cost should probably be a little higher, e.g. Cost + 2
6455 // because it requires two extra extends on the inputs. But if we'd change
6456 // that now, a regular reduction would be cheaper because the costs of
6457 // the extends in the IR are still counted. This can be fixed
6458 // after https://github.com/llvm/llvm-project/pull/147302 has landed.
6459 return Cost + INegCost;
6460 // i8 -> i16 is natively supported with SVE2p3 udot/sdot
6461 // For sub-reductions, we prefer using the *mlslb/t instructions.
6462 if (AccumLT.second.getScalarType() == MVT::i16 &&
6463 InputLT.second.getScalarType() == MVT::i8 &&
6464 (ST->hasSVE2p3() || ST->hasSME2p3()) && !IsSub)
6465 return Cost;
6466 }
6467
6468 // f16 -> f32 is natively supported for fdot using either
6469 // SVE or NEON instruction.
6470 if (Opcode == Instruction::FAdd && !IsSub &&
6471 IsSupported(ST->hasSME2() || ST->hasSVE2p1(), ST->hasF16F32DOT()) &&
6472 AccumLT.second.getScalarType() == MVT::f32 &&
6473 InputLT.second.getScalarType() == MVT::f16)
6474 return Cost;
6475
6476 // For a ratio of 2, we can use *mlal and *mlsl top/bottom instructions.
6477 if (Ratio == 2 && !IsUSDot) {
6478 MVT InVT = InputLT.second.getScalarType();
6479
6480 // SVE2 [us]ml[as]lb/t and NEON [us]ml[as]l(2)
6481 if (IsSupported(ST->hasSVE2() || ST->hasSME(), true) &&
6482 llvm::is_contained(Set: {MVT::i8, MVT::i16, MVT::i32}, Element: InVT.SimpleTy))
6483 return Cost * 2;
6484
6485 // SVE2 fml[as]lb/t and NEON fml[as]l(2)
6486 if (IsSupported(ST->hasSVE2(), ST->hasFP16FML()) && InVT == MVT::f16)
6487 return Cost * 2;
6488
6489 // SME2/SVE2p1 bfmlslb/t
6490 if (IsSupported(ST->hasSVE2p1() || ST->hasSME2(), false) &&
6491 InVT == MVT::bf16 && IsSub)
6492 return Cost * 2;
6493
6494 // FP partial sub-reductions that don't map to a specific instruction,
6495 // carry an extra cost for implementing an extra negation:
6496 // partial_reduce_fmls acc, lhs, rhs
6497 // <=> partial_reduce_fmla acc, lhs, -rhs
6498 InstructionCost FNegCost = IsSub ? InputLT.first * TTI::TCC_Basic : 0;
6499
6500 // SVE and NEON bfmlalb/t
6501 if (IsSupported(ST->hasBF16(), ST->hasBF16()) && InVT == MVT::bf16)
6502 return Cost * 2 + FNegCost;
6503 }
6504
6505 return BaseT::getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
6506 AccumType, VF, OpAExtend, OpBExtend,
6507 BinOp, CostKind, FMF);
6508}
6509
6510InstructionCost
6511AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy,
6512 VectorType *SrcTy, ArrayRef<int> Mask,
6513 TTI::TargetCostKind CostKind, int Index,
6514 VectorType *SubTp, ArrayRef<const Value *> Args,
6515 const Instruction *CxtI) const {
6516 assert((Mask.empty() || DstTy->isScalableTy() ||
6517 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
6518 "Expected the Mask to match the return size if given");
6519 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
6520 "Expected the same scalar types");
6521 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
6522
6523 // If we have a Mask, and the LT is being legalized somehow, split the Mask
6524 // into smaller vectors and sum the cost of each shuffle.
6525 if (!Mask.empty() && isa<FixedVectorType>(Val: SrcTy) && LT.second.isVector() &&
6526 LT.second.getScalarSizeInBits() * Mask.size() > 128 &&
6527 SrcTy->getScalarSizeInBits() == LT.second.getScalarSizeInBits() &&
6528 Mask.size() > LT.second.getVectorNumElements() && !Index && !SubTp) {
6529 // Check for LD3/LD4 instructions, which are represented in llvm IR as
6530 // deinterleaving-shuffle(load). The shuffle cost could potentially be free,
6531 // but we model it with a cost of LT.first so that LD3/LD4 have a higher
6532 // cost than just the load.
6533 if (Args.size() >= 1 && isa<LoadInst>(Val: Args[0]) &&
6534 (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 3) ||
6535 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 4)))
6536 return std::max<InstructionCost>(a: 1, b: LT.first / 4);
6537
6538 // Check for ST3/ST4 instructions, which are represented in llvm IR as
6539 // store(interleaving-shuffle). The shuffle cost could potentially be free,
6540 // but we model it with a cost of LT.first so that ST3/ST4 have a higher
6541 // cost than just the store.
6542 if (CxtI && CxtI->hasOneUse() && isa<StoreInst>(Val: *CxtI->user_begin()) &&
6543 (ShuffleVectorInst::isInterleaveMask(
6544 Mask, Factor: 4, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2) ||
6545 ShuffleVectorInst::isInterleaveMask(
6546 Mask, Factor: 3, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2)))
6547 return LT.first;
6548
6549 unsigned TpNumElts = Mask.size();
6550 unsigned LTNumElts = LT.second.getVectorNumElements();
6551 unsigned NumVecs = (TpNumElts + LTNumElts - 1) / LTNumElts;
6552 VectorType *NTp = VectorType::get(ElementType: SrcTy->getScalarType(),
6553 EC: LT.second.getVectorElementCount());
6554 InstructionCost Cost;
6555 std::map<std::tuple<unsigned, unsigned, SmallVector<int>>, InstructionCost>
6556 PreviousCosts;
6557 for (unsigned N = 0; N < NumVecs; N++) {
6558 SmallVector<int> NMask;
6559 // Split the existing mask into chunks of size LTNumElts. Track the source
6560 // sub-vectors to ensure the result has at most 2 inputs.
6561 unsigned Source1 = -1U, Source2 = -1U;
6562 unsigned NumSources = 0;
6563 for (unsigned E = 0; E < LTNumElts; E++) {
6564 int MaskElt = (N * LTNumElts + E < TpNumElts) ? Mask[N * LTNumElts + E]
6565 : PoisonMaskElem;
6566 if (MaskElt < 0) {
6567 NMask.push_back(Elt: PoisonMaskElem);
6568 continue;
6569 }
6570
6571 // Calculate which source from the input this comes from and whether it
6572 // is new to us.
6573 unsigned Source = MaskElt / LTNumElts;
6574 if (NumSources == 0) {
6575 Source1 = Source;
6576 NumSources = 1;
6577 } else if (NumSources == 1 && Source != Source1) {
6578 Source2 = Source;
6579 NumSources = 2;
6580 } else if (NumSources >= 2 && Source != Source1 && Source != Source2) {
6581 NumSources++;
6582 }
6583
6584 // Add to the new mask. For the NumSources>2 case these are not correct,
6585 // but are only used for the modular lane number.
6586 if (Source == Source1)
6587 NMask.push_back(Elt: MaskElt % LTNumElts);
6588 else if (Source == Source2)
6589 NMask.push_back(Elt: MaskElt % LTNumElts + LTNumElts);
6590 else
6591 NMask.push_back(Elt: MaskElt % LTNumElts);
6592 }
6593 // Check if we have already generated this sub-shuffle, which means we
6594 // will have already generated the output. For example a <16 x i32> splat
6595 // will be the same sub-splat 4 times, which only needs to be generated
6596 // once and reused.
6597 auto Result =
6598 PreviousCosts.insert(x: {std::make_tuple(args&: Source1, args&: Source2, args&: NMask), 0});
6599 // Check if it was already in the map (already costed).
6600 if (!Result.second)
6601 continue;
6602 // If the sub-mask has at most 2 input sub-vectors then re-cost it using
6603 // getShuffleCost. If not then cost it using the worst case as the number
6604 // of element moves into a new vector.
6605 InstructionCost NCost =
6606 NumSources <= 2
6607 ? getShuffleCost(Kind: NumSources <= 1 ? TTI::SK_PermuteSingleSrc
6608 : TTI::SK_PermuteTwoSrc,
6609 DstTy: NTp, SrcTy: NTp, Mask: NMask, CostKind, Index: 0, SubTp: nullptr, Args,
6610 CxtI)
6611 : LTNumElts;
6612 Result.first->second = NCost;
6613 Cost += NCost;
6614 }
6615 return Cost;
6616 }
6617
6618 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
6619 bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector;
6620 // A subvector extract can be implemented with a NEON/SVE ext (or trivial
6621 // extract, if from lane 0) for 128-bit NEON vectors or legal SVE vectors.
6622 // This currently only handles low or high extracts to prevent SLP vectorizer
6623 // regressions.
6624 // Note that SVE's ext instruction is destructive, but it can be fused with
6625 // a movprfx to act like a constructive instruction.
6626 if (IsExtractSubvector && LT.second.isFixedLengthVector()) {
6627 if (LT.second.getFixedSizeInBits() >= 128 &&
6628 cast<FixedVectorType>(Val: SubTp)->getNumElements() ==
6629 LT.second.getVectorNumElements() / 2) {
6630 if (Index == 0)
6631 return 0;
6632 if (Index == (int)LT.second.getVectorNumElements() / 2)
6633 return 1;
6634 }
6635 Kind = TTI::SK_PermuteSingleSrc;
6636 }
6637 // FIXME: This was added to keep the costs equal when adding DstTys. Update
6638 // the code to handle length-changing shuffles.
6639 if (Kind == TTI::SK_InsertSubvector) {
6640 LT = getTypeLegalizationCost(Ty: DstTy);
6641 SrcTy = DstTy;
6642 }
6643
6644 // Check for identity masks, which we can treat as free for both fixed and
6645 // scalable vector paths.
6646 if (!Mask.empty() && LT.second.isFixedLengthVector() &&
6647 (Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc) &&
6648 all_of(Range: enumerate(First&: Mask), P: [](const auto &M) {
6649 return M.value() < 0 || M.value() == (int)M.index();
6650 }))
6651 return 0;
6652
6653 // Segmented shuffle matching.
6654 if (Kind == TTI::SK_PermuteSingleSrc && isa<FixedVectorType>(Val: SrcTy) &&
6655 !Mask.empty() && SrcTy->getPrimitiveSizeInBits().isNonZero() &&
6656 SrcTy->getPrimitiveSizeInBits().isKnownMultipleOf(
6657 RHS: AArch64::SVEBitsPerBlock)) {
6658
6659 FixedVectorType *VTy = cast<FixedVectorType>(Val: SrcTy);
6660 unsigned Segments =
6661 VTy->getPrimitiveSizeInBits() / AArch64::SVEBitsPerBlock;
6662 unsigned SegmentElts = VTy->getNumElements() / Segments;
6663
6664 // dupq zd.t, zn.t[idx]
6665 if ((ST->hasSVE2p1() || ST->hasSME2p1()) &&
6666 ST->isSVEorStreamingSVEAvailable() &&
6667 isDUPQMask(Mask, Segments, SegmentSize: SegmentElts))
6668 return LT.first;
6669
6670 // mov zd.q, vn
6671 if (ST->isSVEorStreamingSVEAvailable() &&
6672 isDUPFirstSegmentMask(Mask, Segments, SegmentSize: SegmentElts))
6673 return LT.first;
6674 }
6675
6676 // Check for broadcast loads, which are supported by the LD1R instruction.
6677 // In terms of code-size, the shuffle vector is free when a load + dup get
6678 // folded into a LD1R. That's what we check and return here. For performance
6679 // and reciprocal throughput, a LD1R is not completely free. In this case, we
6680 // return the cost for the broadcast below (i.e. 1 for most/all types), so
6681 // that we model the load + dup sequence slightly higher because LD1R is a
6682 // high latency instruction.
6683 if (CostKind == TTI::TCK_CodeSize && Kind == TTI::SK_Broadcast) {
6684 bool IsLoad = !Args.empty() && isa<LoadInst>(Val: Args[0]);
6685 if (IsLoad && LT.second.isVector() &&
6686 isLegalBroadcastLoad(ElementTy: SrcTy->getElementType(),
6687 NumElements: LT.second.getVectorElementCount()))
6688 return 0;
6689 }
6690
6691 // If we have 4 elements for the shuffle and a Mask, get the cost straight
6692 // from the perfect shuffle tables.
6693 if (Mask.size() == 4 &&
6694 SrcTy->getElementCount() == ElementCount::getFixed(MinVal: 4) &&
6695 (SrcTy->getScalarSizeInBits() == 16 ||
6696 SrcTy->getScalarSizeInBits() == 32) &&
6697 all_of(Range&: Mask, P: [](int E) { return E < 8; }))
6698 return getPerfectShuffleCost(M: Mask);
6699
6700 // Check for other shuffles that are not SK_ kinds but we have native
6701 // instructions for, for example ZIP and UZP.
6702 unsigned Unused;
6703 if (LT.second.isFixedLengthVector() &&
6704 LT.second.getVectorNumElements() == Mask.size() &&
6705 (Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc ||
6706 // Discrepancies between isTRNMask and ShuffleVectorInst::isTransposeMask
6707 // mean that we can end up with shuffles that satisfy isTRNMask, but end
6708 // up labelled as TTI::SK_InsertSubvector. (e.g. {2, 0}).
6709 Kind == TTI::SK_InsertSubvector) &&
6710 (isZIPMask(M: Mask, NumElts: LT.second.getVectorNumElements(), WhichResultOut&: Unused, OperandOrderOut&: Unused) ||
6711 isTRNMask(M: Mask, NumElts: LT.second.getVectorNumElements(), WhichResultOut&: Unused, OperandOrderOut&: Unused) ||
6712 isUZPMask(M: Mask, NumElts: LT.second.getVectorNumElements(), WhichResultOut&: Unused) ||
6713 isREVMask(M: Mask, EltSize: LT.second.getScalarSizeInBits(),
6714 NumElts: LT.second.getVectorNumElements(), BlockSize: 16) ||
6715 isREVMask(M: Mask, EltSize: LT.second.getScalarSizeInBits(),
6716 NumElts: LT.second.getVectorNumElements(), BlockSize: 32) ||
6717 isREVMask(M: Mask, EltSize: LT.second.getScalarSizeInBits(),
6718 NumElts: LT.second.getVectorNumElements(), BlockSize: 64) ||
6719 // Check for non-zero lane splats
6720 all_of(Range: drop_begin(RangeOrContainer&: Mask),
6721 P: [&Mask](int M) { return M < 0 || M == Mask[0]; })))
6722 return 1;
6723
6724 if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose ||
6725 Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc ||
6726 Kind == TTI::SK_Reverse || Kind == TTI::SK_Splice) {
6727 static const CostTblEntry ShuffleTbl[] = {
6728 // Broadcast shuffle kinds can be performed with 'dup'.
6729 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i8, .Cost: 1},
6730 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i8, .Cost: 1},
6731 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i16, .Cost: 1},
6732 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i16, .Cost: 1},
6733 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i32, .Cost: 1},
6734 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i32, .Cost: 1},
6735 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i64, .Cost: 1},
6736 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4f16, .Cost: 1},
6737 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f16, .Cost: 1},
6738 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4bf16, .Cost: 1},
6739 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8bf16, .Cost: 1},
6740 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2f32, .Cost: 1},
6741 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4f32, .Cost: 1},
6742 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2f64, .Cost: 1},
6743 // Transpose shuffle kinds can be performed with 'trn1/trn2' and
6744 // 'zip1/zip2' instructions.
6745 {.ISD: TTI::SK_Transpose, .Type: MVT::v8i8, .Cost: 1},
6746 {.ISD: TTI::SK_Transpose, .Type: MVT::v16i8, .Cost: 1},
6747 {.ISD: TTI::SK_Transpose, .Type: MVT::v4i16, .Cost: 1},
6748 {.ISD: TTI::SK_Transpose, .Type: MVT::v8i16, .Cost: 1},
6749 {.ISD: TTI::SK_Transpose, .Type: MVT::v2i32, .Cost: 1},
6750 {.ISD: TTI::SK_Transpose, .Type: MVT::v4i32, .Cost: 1},
6751 {.ISD: TTI::SK_Transpose, .Type: MVT::v2i64, .Cost: 1},
6752 {.ISD: TTI::SK_Transpose, .Type: MVT::v4f16, .Cost: 1},
6753 {.ISD: TTI::SK_Transpose, .Type: MVT::v8f16, .Cost: 1},
6754 {.ISD: TTI::SK_Transpose, .Type: MVT::v4bf16, .Cost: 1},
6755 {.ISD: TTI::SK_Transpose, .Type: MVT::v8bf16, .Cost: 1},
6756 {.ISD: TTI::SK_Transpose, .Type: MVT::v2f32, .Cost: 1},
6757 {.ISD: TTI::SK_Transpose, .Type: MVT::v4f32, .Cost: 1},
6758 {.ISD: TTI::SK_Transpose, .Type: MVT::v2f64, .Cost: 1},
6759 // Select shuffle kinds.
6760 // TODO: handle vXi8/vXi16.
6761 {.ISD: TTI::SK_Select, .Type: MVT::v2i32, .Cost: 1}, // mov.
6762 {.ISD: TTI::SK_Select, .Type: MVT::v4i32, .Cost: 2}, // rev+trn (or similar).
6763 {.ISD: TTI::SK_Select, .Type: MVT::v2i64, .Cost: 1}, // mov.
6764 {.ISD: TTI::SK_Select, .Type: MVT::v2f32, .Cost: 1}, // mov.
6765 {.ISD: TTI::SK_Select, .Type: MVT::v4f32, .Cost: 2}, // rev+trn (or similar).
6766 {.ISD: TTI::SK_Select, .Type: MVT::v2f64, .Cost: 1}, // mov.
6767 // PermuteSingleSrc shuffle kinds.
6768 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i32, .Cost: 1}, // mov.
6769 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i32, .Cost: 3}, // perfectshuffle worst case.
6770 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i64, .Cost: 1}, // mov.
6771 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2f32, .Cost: 1}, // mov.
6772 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f32, .Cost: 3}, // perfectshuffle worst case.
6773 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2f64, .Cost: 1}, // mov.
6774 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i16, .Cost: 3}, // perfectshuffle worst case.
6775 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f16, .Cost: 3}, // perfectshuffle worst case.
6776 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4bf16, .Cost: 3}, // same
6777 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i16, .Cost: 8}, // constpool + load + tbl
6778 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f16, .Cost: 8}, // constpool + load + tbl
6779 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8bf16, .Cost: 8}, // constpool + load + tbl
6780 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i8, .Cost: 8}, // constpool + load + tbl
6781 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i8, .Cost: 8}, // constpool + load + tbl
6782 // Reverse can be lowered with `rev`.
6783 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i32, .Cost: 1}, // REV64
6784 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i32, .Cost: 2}, // REV64; EXT
6785 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i64, .Cost: 1}, // EXT
6786 {.ISD: TTI::SK_Reverse, .Type: MVT::v2f32, .Cost: 1}, // REV64
6787 {.ISD: TTI::SK_Reverse, .Type: MVT::v4f32, .Cost: 2}, // REV64; EXT
6788 {.ISD: TTI::SK_Reverse, .Type: MVT::v2f64, .Cost: 1}, // EXT
6789 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f16, .Cost: 2}, // REV64; EXT
6790 {.ISD: TTI::SK_Reverse, .Type: MVT::v8bf16, .Cost: 2}, // REV64; EXT
6791 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i16, .Cost: 2}, // REV64; EXT
6792 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i8, .Cost: 2}, // REV64; EXT
6793 {.ISD: TTI::SK_Reverse, .Type: MVT::v4f16, .Cost: 1}, // REV64
6794 {.ISD: TTI::SK_Reverse, .Type: MVT::v4bf16, .Cost: 1}, // REV64
6795 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i16, .Cost: 1}, // REV64
6796 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i8, .Cost: 1}, // REV64
6797 // Splice can all be lowered as `ext`.
6798 {.ISD: TTI::SK_Splice, .Type: MVT::v2i32, .Cost: 1},
6799 {.ISD: TTI::SK_Splice, .Type: MVT::v4i32, .Cost: 1},
6800 {.ISD: TTI::SK_Splice, .Type: MVT::v2i64, .Cost: 1},
6801 {.ISD: TTI::SK_Splice, .Type: MVT::v2f32, .Cost: 1},
6802 {.ISD: TTI::SK_Splice, .Type: MVT::v4f32, .Cost: 1},
6803 {.ISD: TTI::SK_Splice, .Type: MVT::v2f64, .Cost: 1},
6804 {.ISD: TTI::SK_Splice, .Type: MVT::v8f16, .Cost: 1},
6805 {.ISD: TTI::SK_Splice, .Type: MVT::v8bf16, .Cost: 1},
6806 {.ISD: TTI::SK_Splice, .Type: MVT::v8i16, .Cost: 1},
6807 {.ISD: TTI::SK_Splice, .Type: MVT::v16i8, .Cost: 1},
6808 {.ISD: TTI::SK_Splice, .Type: MVT::v4f16, .Cost: 1},
6809 {.ISD: TTI::SK_Splice, .Type: MVT::v4bf16, .Cost: 1},
6810 {.ISD: TTI::SK_Splice, .Type: MVT::v4i16, .Cost: 1},
6811 {.ISD: TTI::SK_Splice, .Type: MVT::v8i8, .Cost: 1},
6812 // Broadcast shuffle kinds for scalable vectors
6813 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv16i8, .Cost: 1},
6814 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv8i16, .Cost: 1},
6815 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4i32, .Cost: 1},
6816 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2i64, .Cost: 1},
6817 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2f16, .Cost: 1},
6818 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4f16, .Cost: 1},
6819 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv8f16, .Cost: 1},
6820 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2bf16, .Cost: 1},
6821 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4bf16, .Cost: 1},
6822 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv8bf16, .Cost: 1},
6823 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2f32, .Cost: 1},
6824 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4f32, .Cost: 1},
6825 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2f64, .Cost: 1},
6826 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv16i1, .Cost: 1},
6827 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv8i1, .Cost: 1},
6828 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4i1, .Cost: 1},
6829 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2i1, .Cost: 1},
6830 // Handle the cases for vector.reverse with scalable vectors
6831 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv16i8, .Cost: 1},
6832 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv8i16, .Cost: 1},
6833 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4i32, .Cost: 1},
6834 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2i64, .Cost: 1},
6835 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2f16, .Cost: 1},
6836 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4f16, .Cost: 1},
6837 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv8f16, .Cost: 1},
6838 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2bf16, .Cost: 1},
6839 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4bf16, .Cost: 1},
6840 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv8bf16, .Cost: 1},
6841 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2f32, .Cost: 1},
6842 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4f32, .Cost: 1},
6843 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2f64, .Cost: 1},
6844 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv16i1, .Cost: 1},
6845 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv8i1, .Cost: 1},
6846 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4i1, .Cost: 1},
6847 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2i1, .Cost: 1},
6848 };
6849 if (const auto *Entry = CostTableLookup(Table: ShuffleTbl, ISD: Kind, Ty: LT.second))
6850 return LT.first * Entry->Cost;
6851 }
6852
6853 if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(Val: SrcTy))
6854 return getSpliceCost(Tp: SrcTy, Index, CostKind);
6855
6856 // Inserting a subvector can often be done with either a D, S or H register
6857 // move, so long as the inserted vector is "aligned".
6858 if (Kind == TTI::SK_InsertSubvector && LT.second.isFixedLengthVector() &&
6859 LT.second.getSizeInBits() <= 128 && SubTp) {
6860 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
6861 if (SubLT.second.isVector()) {
6862 int NumElts = LT.second.getVectorNumElements();
6863 int NumSubElts = SubLT.second.getVectorNumElements();
6864 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
6865 return SubLT.first;
6866 }
6867 }
6868
6869 // Restore optimal kind.
6870 if (IsExtractSubvector)
6871 Kind = TTI::SK_ExtractSubvector;
6872 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp,
6873 Args, CxtI);
6874}
6875
6876static bool containsDecreasingPointers(Loop *TheLoop,
6877 PredicatedScalarEvolution *PSE,
6878 const DominatorTree &DT) {
6879 const auto &Strides = DenseMap<Value *, const SCEV *>();
6880 for (BasicBlock *BB : TheLoop->blocks()) {
6881 // Scan the instructions in the block and look for addresses that are
6882 // consecutive and decreasing.
6883 for (Instruction &I : *BB) {
6884 if (isa<LoadInst>(Val: &I) || isa<StoreInst>(Val: &I)) {
6885 Value *Ptr = getLoadStorePointerOperand(V: &I);
6886 Type *AccessTy = getLoadStoreType(I: &I);
6887 if (getPtrStride(PSE&: *PSE, AccessTy, Ptr, Lp: TheLoop, DT, StridesMap: Strides,
6888 /*Assume=*/true, /*ShouldCheckWrap=*/false)
6889 .value_or(u: 0) < 0)
6890 return true;
6891 }
6892 }
6893 }
6894 return false;
6895}
6896
6897bool AArch64TTIImpl::preferFixedOverScalableIfEqualCost(bool IsEpilogue) const {
6898 if (SVEPreferFixedOverScalableIfEqualCost.getNumOccurrences())
6899 return SVEPreferFixedOverScalableIfEqualCost;
6900 // For cases like post-LTO vectorization, when we eventually know the trip
6901 // count, epilogue with fixed-width vectorization can be deleted if the trip
6902 // count is less than the epilogue iterations. That's why we prefer
6903 // fixed-width vectorization in epilogue in case of equal costs.
6904 if (IsEpilogue)
6905 return true;
6906 return ST->useFixedOverScalableIfEqualCost();
6907}
6908
6909unsigned AArch64TTIImpl::getEpilogueVectorizationMinVF() const {
6910 return ST->getEpilogueVectorizationMinVF();
6911}
6912
6913bool AArch64TTIImpl::preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
6914 if (!ST->hasSVE())
6915 return false;
6916
6917 // We don't currently support vectorisation with interleaving for SVE - with
6918 // such loops we're better off not using tail-folding. This gives us a chance
6919 // to fall back on fixed-width vectorisation using NEON's ld2/st2/etc.
6920 if (TFI->IAI->hasGroups())
6921 return false;
6922
6923 TailFoldingOpts Required = TailFoldingOpts::Disabled;
6924 if (TFI->LVL->getReductionVars().size())
6925 Required |= TailFoldingOpts::Reductions;
6926 if (TFI->LVL->getFixedOrderRecurrences().size())
6927 Required |= TailFoldingOpts::Recurrences;
6928
6929 // We call this to discover whether any load/store pointers in the loop have
6930 // negative strides. This will require extra work to reverse the loop
6931 // predicate, which may be expensive.
6932 if (containsDecreasingPointers(TheLoop: TFI->LVL->getLoop(),
6933 PSE: TFI->LVL->getPredicatedScalarEvolution(),
6934 DT: *TFI->LVL->getDominatorTree()))
6935 Required |= TailFoldingOpts::Reverse;
6936 if (Required == TailFoldingOpts::Disabled)
6937 Required |= TailFoldingOpts::Simple;
6938
6939 if (!TailFoldingOptionLoc.satisfies(DefaultBits: ST->getSVETailFoldingDefaultOpts(),
6940 Required))
6941 return false;
6942
6943 // Don't tail-fold for tight loops where we would be better off interleaving
6944 // with an unpredicated loop.
6945 unsigned NumInsns = 0;
6946 for (BasicBlock *BB : TFI->LVL->getLoop()->blocks()) {
6947 NumInsns += BB->size();
6948 }
6949
6950 // We expect 4 of these to be a IV PHI, IV add, IV compare and branch.
6951 return NumInsns >= SVETailFoldInsnThreshold;
6952}
6953
6954InstructionCost
6955AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
6956 StackOffset BaseOffset, bool HasBaseReg,
6957 int64_t Scale, unsigned AddrSpace) const {
6958 // Scaling factors are not free at all.
6959 // Operands | Rt Latency
6960 // -------------------------------------------
6961 // Rt, [Xn, Xm] | 4
6962 // -------------------------------------------
6963 // Rt, [Xn, Xm, lsl #imm] | Rn: 4 Rm: 5
6964 // Rt, [Xn, Wm, <extend> #imm] |
6965 TargetLoweringBase::AddrMode AM;
6966 AM.BaseGV = BaseGV;
6967 AM.BaseOffs = BaseOffset.getFixed();
6968 AM.HasBaseReg = HasBaseReg;
6969 AM.Scale = Scale;
6970 AM.ScalableOffset = BaseOffset.getScalable();
6971 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AS: AddrSpace))
6972 // Scale represents reg2 * scale, thus account for 1 if
6973 // it is not equal to 0 or 1.
6974 return AM.Scale != 0 && AM.Scale != 1;
6975 return InstructionCost::getInvalid();
6976}
6977
6978bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(
6979 const Instruction *I) const {
6980 if (EnableOrLikeSelectOpt) {
6981 // For the binary operators (e.g. or) we need to be more careful than
6982 // selects, here we only transform them if they are already at a natural
6983 // break point in the code - the end of a block with an unconditional
6984 // terminator.
6985 if (I->getOpcode() == Instruction::Or &&
6986 isa<UncondBrInst>(Val: I->getNextNode()))
6987 return true;
6988
6989 if (I->getOpcode() == Instruction::Add ||
6990 I->getOpcode() == Instruction::Sub)
6991 return true;
6992 }
6993 return BaseT::shouldTreatInstructionLikeSelect(I);
6994}
6995
6996bool AArch64TTIImpl::isLSRCostLess(
6997 const TargetTransformInfo::LSRCost &C1,
6998 const TargetTransformInfo::LSRCost &C2) const {
6999 // AArch64 specific here is adding the number of instructions to the
7000 // comparison (though not as the first consideration, as some targets do)
7001 // along with changing the priority of the base additions.
7002 // TODO: Maybe a more nuanced tradeoff between instruction count
7003 // and number of registers? To be investigated at a later date.
7004 if (EnableLSRCostOpt)
7005 return std::tie(args: C1.NumRegs, args: C1.Insns, args: C1.NumBaseAdds, args: C1.AddRecCost,
7006 args: C1.NumIVMuls, args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
7007 std::tie(args: C2.NumRegs, args: C2.Insns, args: C2.NumBaseAdds, args: C2.AddRecCost,
7008 args: C2.NumIVMuls, args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
7009
7010 return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
7011}
7012
7013static bool isSplatShuffle(Value *V) {
7014 if (auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: V))
7015 return all_equal(Range: Shuf->getShuffleMask());
7016 return false;
7017}
7018
7019/// Check if both Op1 and Op2 are shufflevector extracts of either the lower
7020/// or upper half of the vector elements.
7021static bool areExtractShuffleVectors(Value *Op1, Value *Op2,
7022 bool AllowSplat = false) {
7023 // Scalable types can't be extract shuffle vectors.
7024 if (Op1->getType()->isScalableTy() || Op2->getType()->isScalableTy())
7025 return false;
7026
7027 auto areTypesHalfed = [](Value *FullV, Value *HalfV) {
7028 auto *FullTy = FullV->getType();
7029 auto *HalfTy = HalfV->getType();
7030 return FullTy->getPrimitiveSizeInBits().getFixedValue() ==
7031 2 * HalfTy->getPrimitiveSizeInBits().getFixedValue();
7032 };
7033
7034 auto extractHalf = [](Value *FullV, Value *HalfV) {
7035 auto *FullVT = cast<FixedVectorType>(Val: FullV->getType());
7036 auto *HalfVT = cast<FixedVectorType>(Val: HalfV->getType());
7037 return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
7038 };
7039
7040 ArrayRef<int> M1, M2;
7041 Value *S1Op1 = nullptr, *S2Op1 = nullptr;
7042 if (!match(V: Op1, P: m_Shuffle(v1: m_Value(V&: S1Op1), v2: m_Undef(), mask: m_Mask(M1))) ||
7043 !match(V: Op2, P: m_Shuffle(v1: m_Value(V&: S2Op1), v2: m_Undef(), mask: m_Mask(M2))))
7044 return false;
7045
7046 // If we allow splats, set S1Op1/S2Op1 to nullptr for the relevant arg so that
7047 // it is not checked as an extract below.
7048 if (AllowSplat && isSplatShuffle(V: Op1))
7049 S1Op1 = nullptr;
7050 if (AllowSplat && isSplatShuffle(V: Op2))
7051 S2Op1 = nullptr;
7052
7053 // Check that the operands are half as wide as the result and we extract
7054 // half of the elements of the input vectors.
7055 if ((S1Op1 && (!areTypesHalfed(S1Op1, Op1) || !extractHalf(S1Op1, Op1))) ||
7056 (S2Op1 && (!areTypesHalfed(S2Op1, Op2) || !extractHalf(S2Op1, Op2))))
7057 return false;
7058
7059 // Check the mask extracts either the lower or upper half of vector
7060 // elements.
7061 int M1Start = 0;
7062 int M2Start = 0;
7063 int NumElements = cast<FixedVectorType>(Val: Op1->getType())->getNumElements() * 2;
7064 if ((S1Op1 &&
7065 !ShuffleVectorInst::isExtractSubvectorMask(Mask: M1, NumSrcElts: NumElements, Index&: M1Start)) ||
7066 (S2Op1 &&
7067 !ShuffleVectorInst::isExtractSubvectorMask(Mask: M2, NumSrcElts: NumElements, Index&: M2Start)))
7068 return false;
7069
7070 if ((M1Start != 0 && M1Start != (NumElements / 2)) ||
7071 (M2Start != 0 && M2Start != (NumElements / 2)))
7072 return false;
7073 if (S1Op1 && S2Op1 && M1Start != M2Start)
7074 return false;
7075
7076 return true;
7077}
7078
7079/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
7080/// of the vector elements.
7081static bool areExtractExts(Value *Ext1, Value *Ext2) {
7082 auto areExtDoubled = [](Instruction *Ext) {
7083 return Ext->getType()->getScalarSizeInBits() ==
7084 2 * Ext->getOperand(i: 0)->getType()->getScalarSizeInBits();
7085 };
7086
7087 if (!match(V: Ext1, P: m_ZExtOrSExt(Op: m_Value())) ||
7088 !match(V: Ext2, P: m_ZExtOrSExt(Op: m_Value())) ||
7089 !areExtDoubled(cast<Instruction>(Val: Ext1)) ||
7090 !areExtDoubled(cast<Instruction>(Val: Ext2)))
7091 return false;
7092
7093 return true;
7094}
7095
7096/// Check if Op could be used with vmull_high_p64 intrinsic.
7097static bool isOperandOfVmullHighP64(Value *Op) {
7098 Value *VectorOperand = nullptr;
7099 ConstantInt *ElementIndex = nullptr;
7100 return match(V: Op, P: m_ExtractElt(Val: m_Value(V&: VectorOperand),
7101 Idx: m_ConstantInt(CI&: ElementIndex))) &&
7102 ElementIndex->getValue() == 1 &&
7103 isa<FixedVectorType>(Val: VectorOperand->getType()) &&
7104 cast<FixedVectorType>(Val: VectorOperand->getType())->getNumElements() == 2;
7105}
7106
7107/// Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
7108static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2) {
7109 return isOperandOfVmullHighP64(Op: Op1) && isOperandOfVmullHighP64(Op: Op2);
7110}
7111
7112static bool shouldSinkVectorOfPtrs(Value *Ptrs, SmallVectorImpl<Use *> &Ops) {
7113 // Restrict ourselves to the form CodeGenPrepare typically constructs.
7114 auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptrs);
7115 if (!GEP || GEP->getNumOperands() != 2)
7116 return false;
7117
7118 Value *Base = GEP->getOperand(i_nocapture: 0);
7119 Value *Offsets = GEP->getOperand(i_nocapture: 1);
7120
7121 // We only care about scalar_base+vector_offsets.
7122 if (Base->getType()->isVectorTy() || !Offsets->getType()->isVectorTy())
7123 return false;
7124
7125 // Sink extends that would allow us to use 32-bit offset vectors.
7126 if (isa<SExtInst>(Val: Offsets) || isa<ZExtInst>(Val: Offsets)) {
7127 auto *OffsetsInst = cast<Instruction>(Val: Offsets);
7128 if (OffsetsInst->getType()->getScalarSizeInBits() > 32 &&
7129 OffsetsInst->getOperand(i: 0)->getType()->getScalarSizeInBits() <= 32)
7130 Ops.push_back(Elt: &GEP->getOperandUse(i: 1));
7131 }
7132
7133 // Sink the GEP.
7134 return true;
7135}
7136
7137/// We want to sink following cases:
7138/// (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale;
7139/// (add|sub|gep) A, ((mul|shl) zext(vscale), imm);
7140static bool shouldSinkVScale(Value *Op, SmallVectorImpl<Use *> &Ops) {
7141 if (match(V: Op, P: m_VScale()))
7142 return true;
7143 if (match(V: Op, P: m_Shl(L: m_VScale(), R: m_ConstantInt())) ||
7144 match(V: Op, P: m_Mul(L: m_VScale(), R: m_ConstantInt()))) {
7145 Ops.push_back(Elt: &cast<Instruction>(Val: Op)->getOperandUse(i: 0));
7146 return true;
7147 }
7148 if (match(V: Op, P: m_Shl(L: m_ZExt(Op: m_VScale()), R: m_ConstantInt())) ||
7149 match(V: Op, P: m_Mul(L: m_ZExt(Op: m_VScale()), R: m_ConstantInt()))) {
7150 Value *ZExtOp = cast<Instruction>(Val: Op)->getOperand(i: 0);
7151 Ops.push_back(Elt: &cast<Instruction>(Val: ZExtOp)->getOperandUse(i: 0));
7152 Ops.push_back(Elt: &cast<Instruction>(Val: Op)->getOperandUse(i: 0));
7153 return true;
7154 }
7155 return false;
7156}
7157
7158static bool isFNeg(Value *Op) { return match(V: Op, P: m_FNeg(X: m_Value())); }
7159
7160/// Check if sinking \p I's operands to I's basic block is profitable, because
7161/// the operands can be folded into a target instruction, e.g.
7162/// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2).
7163bool AArch64TTIImpl::isProfitableToSinkOperands(
7164 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
7165 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
7166 switch (II->getIntrinsicID()) {
7167 case Intrinsic::aarch64_neon_smull:
7168 case Intrinsic::aarch64_neon_umull:
7169 if (areExtractShuffleVectors(Op1: II->getOperand(i_nocapture: 0), Op2: II->getOperand(i_nocapture: 1),
7170 /*AllowSplat=*/true)) {
7171 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7172 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7173 return true;
7174 }
7175 [[fallthrough]];
7176
7177 case Intrinsic::fma:
7178 case Intrinsic::fmuladd:
7179 if (isa<VectorType>(Val: I->getType()) &&
7180 cast<VectorType>(Val: I->getType())->getElementType()->isHalfTy() &&
7181 !ST->hasFullFP16())
7182 return false;
7183
7184 if (isFNeg(Op: II->getOperand(i_nocapture: 0)))
7185 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7186 if (isFNeg(Op: II->getOperand(i_nocapture: 1)))
7187 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7188
7189 [[fallthrough]];
7190 case Intrinsic::aarch64_neon_sqdmull:
7191 case Intrinsic::aarch64_neon_sqdmulh:
7192 case Intrinsic::aarch64_neon_sqrdmulh:
7193 // Sink splats for index lane variants
7194 if (isSplatShuffle(V: II->getOperand(i_nocapture: 0)))
7195 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7196 if (isSplatShuffle(V: II->getOperand(i_nocapture: 1)))
7197 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7198 return !Ops.empty();
7199 case Intrinsic::aarch64_neon_fmlal:
7200 case Intrinsic::aarch64_neon_fmlal2:
7201 case Intrinsic::aarch64_neon_fmlsl:
7202 case Intrinsic::aarch64_neon_fmlsl2:
7203 // Sink splats for index lane variants
7204 if (isSplatShuffle(V: II->getOperand(i_nocapture: 1)))
7205 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7206 if (isSplatShuffle(V: II->getOperand(i_nocapture: 2)))
7207 Ops.push_back(Elt: &II->getOperandUse(i: 2));
7208 return !Ops.empty();
7209 case Intrinsic::aarch64_sve_ptest_first:
7210 case Intrinsic::aarch64_sve_ptest_last:
7211 if (auto *IIOp = dyn_cast<IntrinsicInst>(Val: II->getOperand(i_nocapture: 0)))
7212 if (IIOp->getIntrinsicID() == Intrinsic::aarch64_sve_ptrue)
7213 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7214 return !Ops.empty();
7215 case Intrinsic::aarch64_sme_write_horiz:
7216 case Intrinsic::aarch64_sme_write_vert:
7217 case Intrinsic::aarch64_sme_writeq_horiz:
7218 case Intrinsic::aarch64_sme_writeq_vert: {
7219 auto *Idx = dyn_cast<Instruction>(Val: II->getOperand(i_nocapture: 1));
7220 if (!Idx || Idx->getOpcode() != Instruction::Add)
7221 return false;
7222 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7223 return true;
7224 }
7225 case Intrinsic::aarch64_sme_read_horiz:
7226 case Intrinsic::aarch64_sme_read_vert:
7227 case Intrinsic::aarch64_sme_readq_horiz:
7228 case Intrinsic::aarch64_sme_readq_vert:
7229 case Intrinsic::aarch64_sme_ld1b_vert:
7230 case Intrinsic::aarch64_sme_ld1h_vert:
7231 case Intrinsic::aarch64_sme_ld1w_vert:
7232 case Intrinsic::aarch64_sme_ld1d_vert:
7233 case Intrinsic::aarch64_sme_ld1q_vert:
7234 case Intrinsic::aarch64_sme_st1b_vert:
7235 case Intrinsic::aarch64_sme_st1h_vert:
7236 case Intrinsic::aarch64_sme_st1w_vert:
7237 case Intrinsic::aarch64_sme_st1d_vert:
7238 case Intrinsic::aarch64_sme_st1q_vert:
7239 case Intrinsic::aarch64_sme_ld1b_horiz:
7240 case Intrinsic::aarch64_sme_ld1h_horiz:
7241 case Intrinsic::aarch64_sme_ld1w_horiz:
7242 case Intrinsic::aarch64_sme_ld1d_horiz:
7243 case Intrinsic::aarch64_sme_ld1q_horiz:
7244 case Intrinsic::aarch64_sme_st1b_horiz:
7245 case Intrinsic::aarch64_sme_st1h_horiz:
7246 case Intrinsic::aarch64_sme_st1w_horiz:
7247 case Intrinsic::aarch64_sme_st1d_horiz:
7248 case Intrinsic::aarch64_sme_st1q_horiz: {
7249 auto *Idx = dyn_cast<Instruction>(Val: II->getOperand(i_nocapture: 3));
7250 if (!Idx || Idx->getOpcode() != Instruction::Add)
7251 return false;
7252 Ops.push_back(Elt: &II->getOperandUse(i: 3));
7253 return true;
7254 }
7255 case Intrinsic::aarch64_neon_pmull:
7256 if (!areExtractShuffleVectors(Op1: II->getOperand(i_nocapture: 0), Op2: II->getOperand(i_nocapture: 1)))
7257 return false;
7258 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7259 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7260 return true;
7261 case Intrinsic::aarch64_neon_pmull64:
7262 if (!areOperandsOfVmullHighP64(Op1: II->getArgOperand(i: 0),
7263 Op2: II->getArgOperand(i: 1)))
7264 return false;
7265 Ops.push_back(Elt: &II->getArgOperandUse(i: 0));
7266 Ops.push_back(Elt: &II->getArgOperandUse(i: 1));
7267 return true;
7268 case Intrinsic::masked_gather:
7269 if (!shouldSinkVectorOfPtrs(Ptrs: II->getArgOperand(i: 0), Ops))
7270 return false;
7271 Ops.push_back(Elt: &II->getArgOperandUse(i: 0));
7272 return true;
7273 case Intrinsic::masked_scatter:
7274 if (!shouldSinkVectorOfPtrs(Ptrs: II->getArgOperand(i: 1), Ops))
7275 return false;
7276 Ops.push_back(Elt: &II->getArgOperandUse(i: 1));
7277 return true;
7278 default:
7279 return false;
7280 }
7281 }
7282
7283 auto ShouldSinkCondition = [](Value *Cond,
7284 SmallVectorImpl<Use *> &Ops) -> bool {
7285 if (!isa<IntrinsicInst>(Val: Cond))
7286 return false;
7287 auto *II = dyn_cast<IntrinsicInst>(Val: Cond);
7288 if (II->getIntrinsicID() != Intrinsic::vector_reduce_or ||
7289 !isa<ScalableVectorType>(Val: II->getOperand(i_nocapture: 0)->getType()))
7290 return false;
7291 if (isa<CmpInst>(Val: II->getOperand(i_nocapture: 0)))
7292 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7293 return true;
7294 };
7295
7296 switch (I->getOpcode()) {
7297 case Instruction::GetElementPtr:
7298 case Instruction::Add:
7299 case Instruction::Sub:
7300 // Sink vscales closer to uses for better isel
7301 for (unsigned Op = 0; Op < I->getNumOperands(); ++Op) {
7302 if (shouldSinkVScale(Op: I->getOperand(i: Op), Ops)) {
7303 Ops.push_back(Elt: &I->getOperandUse(i: Op));
7304 return true;
7305 }
7306 }
7307 break;
7308 case Instruction::Select: {
7309 if (!ShouldSinkCondition(I->getOperand(i: 0), Ops))
7310 return false;
7311
7312 Ops.push_back(Elt: &I->getOperandUse(i: 0));
7313 return true;
7314 }
7315 case Instruction::UncondBr:
7316 return false;
7317 case Instruction::CondBr: {
7318 if (!ShouldSinkCondition(cast<CondBrInst>(Val: I)->getCondition(), Ops))
7319 return false;
7320
7321 Ops.push_back(Elt: &I->getOperandUse(i: 0));
7322 return true;
7323 }
7324 case Instruction::FMul:
7325 // fmul with contract flag can be combined with fadd into fma.
7326 // Sinking fneg into this block enables fmls pattern.
7327 if (cast<FPMathOperator>(Val: I)->hasAllowContract()) {
7328 if (isFNeg(Op: I->getOperand(i: 0)))
7329 Ops.push_back(Elt: &I->getOperandUse(i: 0));
7330 if (isFNeg(Op: I->getOperand(i: 1)))
7331 Ops.push_back(Elt: &I->getOperandUse(i: 1));
7332 }
7333 break;
7334
7335 // Type | BIC | ORN | EON
7336 // ----------------+-----------+-----------+-----------
7337 // scalar | Base | Base | Base
7338 // scalar w/shift | - | - | -
7339 // fixed vector | NEON/Base | NEON/Base | BSL2N/Base
7340 // scalable vector | SVE | - | BSL2N
7341 case Instruction::Xor:
7342 // EON only for scalars (possibly expanded fixed vectors)
7343 // and vectors using the SVE2/SME BSL2N instruction.
7344 if (I->getType()->isVectorTy() && ST->isNeonAvailable()) {
7345 bool HasBSL2N =
7346 ST->isSVEorStreamingSVEAvailable() && (ST->hasSVE2() || ST->hasSME());
7347 if (!HasBSL2N)
7348 break;
7349 }
7350 [[fallthrough]];
7351 case Instruction::And:
7352 case Instruction::Or:
7353 // Even though we could use the SVE2/SME BSL2N instruction,
7354 // it might pessimize with an extra MOV depending on register allocation.
7355 if (I->getOpcode() == Instruction::Or &&
7356 isa<ScalableVectorType>(Val: I->getType()))
7357 break;
7358 // Shift can be fold into scalar AND/ORR/EOR,
7359 // but not the non-negated operand of BIC/ORN/EON.
7360 if (!(I->getType()->isVectorTy() && ST->hasNEON()) &&
7361 match(V: I, P: m_c_BinOp(L: m_Shift(L: m_Value(), R: m_ConstantInt()), R: m_Value())))
7362 break;
7363 for (auto &Op : I->operands()) {
7364 // (and/or/xor X, (not Y)) -> (bic/orn/eon X, Y)
7365 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
7366 Ops.push_back(Elt: &Op);
7367 return true;
7368 }
7369 // (and/or/xor X, (splat (not Y))) -> (bic/orn/eon X, (splat Y))
7370 if (match(V: Op.get(),
7371 P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()), Idx: m_ZeroInt()),
7372 v2: m_Value(), mask: m_ZeroMask()))) {
7373 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
7374 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
7375 Ops.push_back(Elt: &Not);
7376 Ops.push_back(Elt: &InsertElt);
7377 Ops.push_back(Elt: &Op);
7378 return true;
7379 }
7380 }
7381 break;
7382 default:
7383 break;
7384 }
7385
7386 if (!I->getType()->isVectorTy())
7387 return !Ops.empty();
7388
7389 switch (I->getOpcode()) {
7390 case Instruction::Sub:
7391 case Instruction::Add: {
7392 if (!areExtractExts(Ext1: I->getOperand(i: 0), Ext2: I->getOperand(i: 1)))
7393 return false;
7394
7395 // If the exts' operands extract either the lower or upper elements, we
7396 // can sink them too.
7397 auto Ext1 = cast<Instruction>(Val: I->getOperand(i: 0));
7398 auto Ext2 = cast<Instruction>(Val: I->getOperand(i: 1));
7399 if (areExtractShuffleVectors(Op1: Ext1->getOperand(i: 0), Op2: Ext2->getOperand(i: 0))) {
7400 Ops.push_back(Elt: &Ext1->getOperandUse(i: 0));
7401 Ops.push_back(Elt: &Ext2->getOperandUse(i: 0));
7402 }
7403
7404 Ops.push_back(Elt: &I->getOperandUse(i: 0));
7405 Ops.push_back(Elt: &I->getOperandUse(i: 1));
7406
7407 return true;
7408 }
7409 case Instruction::Or: {
7410 // Pattern: Or(And(MaskValue, A), And(Not(MaskValue), B)) ->
7411 // bitselect(MaskValue, A, B) where Not(MaskValue) = Xor(MaskValue, -1)
7412 if (ST->hasNEON()) {
7413 Instruction *OtherAnd, *IA, *IB;
7414 Value *MaskValue;
7415 // MainAnd refers to And instruction that has 'Not' as one of its operands
7416 if (match(V: I, P: m_c_Or(L: m_OneUse(SubPattern: m_Instruction(I&: OtherAnd)),
7417 R: m_OneUse(SubPattern: m_c_And(L: m_OneUse(SubPattern: m_Not(V: m_Value(V&: MaskValue))),
7418 R: m_Instruction(I&: IA)))))) {
7419 if (match(V: OtherAnd,
7420 P: m_c_And(L: m_Specific(V: MaskValue), R: m_Instruction(I&: IB)))) {
7421 Instruction *MainAnd = I->getOperand(i: 0) == OtherAnd
7422 ? cast<Instruction>(Val: I->getOperand(i: 1))
7423 : cast<Instruction>(Val: I->getOperand(i: 0));
7424
7425 // Both Ands should be in same basic block as Or
7426 if (I->getParent() != MainAnd->getParent() ||
7427 I->getParent() != OtherAnd->getParent())
7428 return false;
7429
7430 // Non-mask operands of both Ands should also be in same basic block
7431 if (I->getParent() != IA->getParent() ||
7432 I->getParent() != IB->getParent())
7433 return false;
7434
7435 Ops.push_back(
7436 Elt: &MainAnd->getOperandUse(i: MainAnd->getOperand(i: 0) == IA ? 1 : 0));
7437 Ops.push_back(Elt: &I->getOperandUse(i: 0));
7438 Ops.push_back(Elt: &I->getOperandUse(i: 1));
7439
7440 return true;
7441 }
7442 }
7443 }
7444
7445 return false;
7446 }
7447 case Instruction::Mul: {
7448 auto ShouldSinkSplatForIndexedVariant = [](Value *V) {
7449 auto *Ty = cast<VectorType>(Val: V->getType());
7450 // For SVE the lane-indexing is within 128-bits, so we can't fold splats.
7451 if (Ty->isScalableTy())
7452 return false;
7453
7454 // Indexed variants of Mul exist for i16 and i32 element types only.
7455 return Ty->getScalarSizeInBits() == 16 || Ty->getScalarSizeInBits() == 32;
7456 };
7457
7458 int NumZExts = 0, NumSExts = 0;
7459 for (auto &Op : I->operands()) {
7460 // Make sure we are not already sinking this operand
7461 if (any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
7462 continue;
7463
7464 if (match(V: &Op, P: m_ZExtOrSExt(Op: m_Value()))) {
7465 auto *Ext = cast<Instruction>(Val&: Op);
7466 auto *ExtOp = Ext->getOperand(i: 0);
7467 if (isSplatShuffle(V: ExtOp) && ShouldSinkSplatForIndexedVariant(ExtOp))
7468 Ops.push_back(Elt: &Ext->getOperandUse(i: 0));
7469 Ops.push_back(Elt: &Op);
7470
7471 if (isa<SExtInst>(Val: Ext)) {
7472 NumSExts++;
7473 } else {
7474 NumZExts++;
7475 // A zext(a) is also a sext(zext(a)), if we take more than 2 steps.
7476 if (Ext->getOperand(i: 0)->getType()->getScalarSizeInBits() * 2 <
7477 I->getType()->getScalarSizeInBits())
7478 NumSExts++;
7479 }
7480
7481 continue;
7482 }
7483
7484 ShuffleVectorInst *Shuffle = dyn_cast<ShuffleVectorInst>(Val&: Op);
7485 if (!Shuffle)
7486 continue;
7487
7488 // If the Shuffle is a splat and the operand is a zext/sext, sinking the
7489 // operand and the s/zext can help create indexed s/umull. This is
7490 // especially useful to prevent i64 mul being scalarized.
7491 if (isSplatShuffle(V: Shuffle) &&
7492 match(V: Shuffle->getOperand(i_nocapture: 0), P: m_ZExtOrSExt(Op: m_Value()))) {
7493 Ops.push_back(Elt: &Shuffle->getOperandUse(i: 0));
7494 Ops.push_back(Elt: &Op);
7495 if (match(V: Shuffle->getOperand(i_nocapture: 0), P: m_SExt(Op: m_Value())))
7496 NumSExts++;
7497 else
7498 NumZExts++;
7499 continue;
7500 }
7501
7502 Value *ShuffleOperand = Shuffle->getOperand(i_nocapture: 0);
7503 InsertElementInst *Insert = dyn_cast<InsertElementInst>(Val: ShuffleOperand);
7504 if (!Insert)
7505 continue;
7506
7507 Instruction *OperandInstr = dyn_cast<Instruction>(Val: Insert->getOperand(i_nocapture: 1));
7508 if (!OperandInstr)
7509 continue;
7510
7511 ConstantInt *ElementConstant =
7512 dyn_cast<ConstantInt>(Val: Insert->getOperand(i_nocapture: 2));
7513 // Check that the insertelement is inserting into element 0
7514 if (!ElementConstant || !ElementConstant->isZero())
7515 continue;
7516
7517 unsigned Opcode = OperandInstr->getOpcode();
7518 if (Opcode == Instruction::SExt)
7519 NumSExts++;
7520 else if (Opcode == Instruction::ZExt)
7521 NumZExts++;
7522 else {
7523 // If we find that the top bits are known 0, then we can sink and allow
7524 // the backend to generate a umull.
7525 unsigned Bitwidth = I->getType()->getScalarSizeInBits();
7526 APInt UpperMask = APInt::getHighBitsSet(numBits: Bitwidth, hiBitsSet: Bitwidth / 2);
7527 if (!MaskedValueIsZero(V: OperandInstr, Mask: UpperMask, SQ: DL))
7528 continue;
7529 NumZExts++;
7530 }
7531
7532 // And(Load) is excluded to prevent CGP getting stuck in a loop of sinking
7533 // the And, just to hoist it again back to the load.
7534 if (!match(V: OperandInstr, P: m_And(L: m_Load(Op: m_Value()), R: m_Value())))
7535 Ops.push_back(Elt: &Insert->getOperandUse(i: 1));
7536 Ops.push_back(Elt: &Shuffle->getOperandUse(i: 0));
7537 Ops.push_back(Elt: &Op);
7538 }
7539
7540 // It is profitable to sink if we found two of the same type of extends.
7541 if (!Ops.empty() && (NumSExts == 2 || NumZExts == 2))
7542 return true;
7543
7544 // Otherwise, see if we should sink splats for indexed variants.
7545 if (!ShouldSinkSplatForIndexedVariant(I))
7546 return false;
7547
7548 Ops.clear();
7549 if (isSplatShuffle(V: I->getOperand(i: 0)))
7550 Ops.push_back(Elt: &I->getOperandUse(i: 0));
7551 if (isSplatShuffle(V: I->getOperand(i: 1)))
7552 Ops.push_back(Elt: &I->getOperandUse(i: 1));
7553
7554 return !Ops.empty();
7555 }
7556 case Instruction::FMul: {
7557 // For SVE the lane-indexing is within 128-bits, so we can't fold splats.
7558 if (I->getType()->isScalableTy())
7559 return !Ops.empty();
7560
7561 if (cast<VectorType>(Val: I->getType())->getElementType()->isHalfTy() &&
7562 !ST->hasFullFP16())
7563 return !Ops.empty();
7564
7565 // Sink splats for index lane variants
7566 if (isSplatShuffle(V: I->getOperand(i: 0)))
7567 Ops.push_back(Elt: &I->getOperandUse(i: 0));
7568 if (isSplatShuffle(V: I->getOperand(i: 1)))
7569 Ops.push_back(Elt: &I->getOperandUse(i: 1));
7570 return !Ops.empty();
7571 }
7572 default:
7573 return false;
7574 }
7575 return false;
7576}
7577