1//===-- AArch64TargetTransformInfo.cpp - AArch64 specific TTI -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AArch64TargetTransformInfo.h"
10#include "AArch64ExpandImm.h"
11#include "AArch64PerfectShuffle.h"
12#include "AArch64SMEAttributes.h"
13#include "MCTargetDesc/AArch64AddressingModes.h"
14#include "llvm/ADT/DenseMap.h"
15#include "llvm/ADT/bit.h"
16#include "llvm/Analysis/LoopInfo.h"
17#include "llvm/Analysis/TargetTransformInfo.h"
18#include "llvm/CodeGen/BasicTTIImpl.h"
19#include "llvm/CodeGen/CostTable.h"
20#include "llvm/CodeGen/TargetLowering.h"
21#include "llvm/IR/DerivedTypes.h"
22#include "llvm/IR/IntrinsicInst.h"
23#include "llvm/IR/Intrinsics.h"
24#include "llvm/IR/IntrinsicsAArch64.h"
25#include "llvm/IR/PatternMatch.h"
26#include "llvm/Support/Debug.h"
27#include "llvm/TargetParser/AArch64TargetParser.h"
28#include "llvm/Transforms/InstCombine/InstCombiner.h"
29#include "llvm/Transforms/Utils/UnrollLoop.h"
30#include "llvm/Transforms/Vectorize/LoopVectorizationLegality.h"
31#include <algorithm>
32#include <optional>
33using namespace llvm;
34using namespace llvm::PatternMatch;
35
36#define DEBUG_TYPE "aarch64tti"
37
38namespace {
39class TailFoldingOption {
40 // These bitfields will only ever be set to something non-zero in operator=,
41 // when setting the -sve-tail-folding option. This option should always be of
42 // the form (default|simple|all|disable)[+(Flag1|Flag2|etc)], where here
43 // InitialBits is one of (disabled|all|simple). EnableBits represents
44 // additional flags we're enabling, and DisableBits for those flags we're
45 // disabling. The default flag is tracked in the variable NeedsDefault, since
46 // at the time of setting the option we may not know what the default value
47 // for the CPU is.
48 TailFoldingOpts InitialBits = TailFoldingOpts::Disabled;
49 TailFoldingOpts EnableBits = TailFoldingOpts::Disabled;
50 TailFoldingOpts DisableBits = TailFoldingOpts::Disabled;
51
52 // This value needs to be initialised to true in case the user does not
53 // explicitly set the -sve-tail-folding option.
54 bool NeedsDefault = true;
55
56 void setInitialBits(TailFoldingOpts Bits) { InitialBits = Bits; }
57
58 void setNeedsDefault(bool V) { NeedsDefault = V; }
59
60 void setEnableBit(TailFoldingOpts Bit) {
61 EnableBits |= Bit;
62 DisableBits &= ~Bit;
63 }
64
65 void setDisableBit(TailFoldingOpts Bit) {
66 EnableBits &= ~Bit;
67 DisableBits |= Bit;
68 }
69
70 TailFoldingOpts getBits(TailFoldingOpts DefaultBits) const {
71 TailFoldingOpts Bits = TailFoldingOpts::Disabled;
72
73 assert((InitialBits == TailFoldingOpts::Disabled || !NeedsDefault) &&
74 "Initial bits should only include one of "
75 "(disabled|all|simple|default)");
76 Bits = NeedsDefault ? DefaultBits : InitialBits;
77 Bits |= EnableBits;
78 Bits &= ~DisableBits;
79
80 return Bits;
81 }
82
83 void reportError(std::string Opt) {
84 errs() << "invalid argument '" << Opt
85 << "' to -sve-tail-folding=; the option should be of the form\n"
86 " (disabled|all|default|simple)[+(reductions|recurrences"
87 "|reverse|noreductions|norecurrences|noreverse)]\n";
88 report_fatal_error(reason: "Unrecognised tail-folding option");
89 }
90
91public:
92
93 void operator=(const std::string &Val) {
94 // If the user explicitly sets -sve-tail-folding= then treat as an error.
95 if (Val.empty()) {
96 reportError(Opt: "");
97 return;
98 }
99
100 // Since the user is explicitly setting the option we don't automatically
101 // need the default unless they require it.
102 setNeedsDefault(false);
103
104 SmallVector<StringRef, 4> TailFoldTypes;
105 StringRef(Val).split(A&: TailFoldTypes, Separator: '+', MaxSplit: -1, KeepEmpty: false);
106
107 unsigned StartIdx = 1;
108 if (TailFoldTypes[0] == "disabled")
109 setInitialBits(TailFoldingOpts::Disabled);
110 else if (TailFoldTypes[0] == "all")
111 setInitialBits(TailFoldingOpts::All);
112 else if (TailFoldTypes[0] == "default")
113 setNeedsDefault(true);
114 else if (TailFoldTypes[0] == "simple")
115 setInitialBits(TailFoldingOpts::Simple);
116 else {
117 StartIdx = 0;
118 setInitialBits(TailFoldingOpts::Disabled);
119 }
120
121 for (unsigned I = StartIdx; I < TailFoldTypes.size(); I++) {
122 if (TailFoldTypes[I] == "reductions")
123 setEnableBit(TailFoldingOpts::Reductions);
124 else if (TailFoldTypes[I] == "recurrences")
125 setEnableBit(TailFoldingOpts::Recurrences);
126 else if (TailFoldTypes[I] == "reverse")
127 setEnableBit(TailFoldingOpts::Reverse);
128 else if (TailFoldTypes[I] == "noreductions")
129 setDisableBit(TailFoldingOpts::Reductions);
130 else if (TailFoldTypes[I] == "norecurrences")
131 setDisableBit(TailFoldingOpts::Recurrences);
132 else if (TailFoldTypes[I] == "noreverse")
133 setDisableBit(TailFoldingOpts::Reverse);
134 else
135 reportError(Opt: Val);
136 }
137 }
138
139 bool isDisabled(TailFoldingOpts DefaultBits) const {
140 return getBits(DefaultBits) == TailFoldingOpts::Disabled;
141 }
142
143 bool satisfies(TailFoldingOpts DefaultBits, TailFoldingOpts Required) const {
144 return (getBits(DefaultBits) & Required) == Required;
145 }
146};
147} // namespace
148
149TailFoldingOption TailFoldingOptionLoc;
150
151static cl::opt<TailFoldingOption, true, cl::parser<std::string>> SVETailFolding(
152 "sve-tail-folding",
153 cl::desc(
154 "Control the use of vectorisation using tail-folding for SVE where the"
155 " option is specified in the form (Initial)[+(Flag1|Flag2|...)]:"
156 "\ndisabled (Initial) No loop types will vectorize using "
157 "tail-folding"
158 "\ndefault (Initial) Uses the default tail-folding settings for "
159 "the target CPU"
160 "\nall (Initial) All legal loop types will vectorize using "
161 "tail-folding"
162 "\nsimple (Initial) Use tail-folding for simple loops (not "
163 "reductions or recurrences)"
164 "\nreductions Use tail-folding for loops containing reductions"
165 "\nnoreductions Inverse of above"
166 "\nrecurrences Use tail-folding for loops containing fixed order "
167 "recurrences"
168 "\nnorecurrences Inverse of above"
169 "\nreverse Use tail-folding for loops requiring reversed "
170 "predicates"
171 "\nnoreverse Inverse of above"),
172 cl::location(L&: TailFoldingOptionLoc));
173
174/// Returns true if the function has explicit operations that can only be
175/// lowered using incompatible instructions for the selected mode. This also
176/// returns true if the function F may use or modify ZA state.
177bool AArch64TTIImpl::hasPossibleIncompatibleOps(const Function *F,
178 bool ConsiderZA,
179 bool ConsiderZT,
180 bool ConsiderSM) const {
181 if (!ConsiderZA && !ConsiderZT && !ConsiderSM)
182 return false;
183
184 bool IsAlwaysInline = F->hasFnAttribute(Kind: Attribute::AlwaysInline);
185 // If the interface has vscale-dependent arguments/return value, then the ACLE
186 // describes that the function can only have defined behaviour if vscale is
187 // the same for both streaming/non-streaming mode, meaning that we can assume
188 // vscale is equivalent.
189 // FIXME: Handle the case where vscale_range is known and equivalent for
190 // caller and callee.
191 bool AssumeVScaleIsEquivalent =
192 F->getReturnType()->isScalableTy() ||
193 any_of(Range: F->getFunctionType()->params(),
194 P: [](const Type *T) { return T->isScalableTy(); });
195
196 for (const BasicBlock &BB : *F) {
197 for (const Instruction &I : BB) {
198 if (auto *CB = dyn_cast<CallBase>(Val: &I)) {
199 SMECallAttrs CallAttrs(*CB, &TLI->getRuntimeLibcallsInfo());
200
201 // Inline asm must be rejected as it could use SME state.
202 if (CB->isInlineAsm())
203 return true;
204
205 if (CallAttrs.callee().isSMEABIRoutine())
206 return true;
207
208 // If the callee has calls to streaming compatible functions, then those
209 // may have statements which depend on SME state (e.g. vscale or
210 // explicit reads of PSTATE.SM). If we were to inline those calls,
211 // behaviour may change after inlining because the streaming-compatible
212 // function would be executed in a different streaming mode.
213 // Conservatively avoid inlining for now.
214 if (ConsiderSM && CallAttrs.callee().hasStreamingCompatibleInterface())
215 return true;
216 }
217
218 if (!ConsiderSM)
219 continue;
220
221 // Inlining operations on fixed-length vectors when the streaming
222 // mode does not match is rejected because performance may be impacted.
223 // This decision should eventually be moved to the cost-model.
224 if (!IsAlwaysInline && (isa<FixedVectorType>(Val: I.getType()) ||
225 any_of(Range: I.operand_values(), P: [](const Value *V) {
226 return isa<FixedVectorType>(Val: V->getType());
227 })))
228 return true;
229
230 if (auto *II = dyn_cast<IntrinsicInst>(Val: &I)) {
231 unsigned IID = II->getIntrinsicID();
232 switch (IID) {
233 default:
234 if (!Intrinsic::isTargetIntrinsic(IID))
235 break;
236 // Conservatively assume target intrinsics are vscale dependent
237 // or invalid in the current mode.
238 return true;
239 case Intrinsic::masked_gather:
240 case Intrinsic::masked_scatter:
241 case Intrinsic::masked_compressstore:
242 case Intrinsic::experimental_vector_histogram_add:
243 // Instructions that are not available in streaming mode.
244 // Fixed-length operations can still be code-generated by scalarizing.
245 // FIXME: Consider FEAT_FA64.
246 if (I.getOperand(i: 0)->getType()->isScalableTy())
247 return true;
248 break;
249 case Intrinsic::vscale:
250 if (!AssumeVScaleIsEquivalent)
251 return true;
252 break;
253 }
254 }
255
256 // Determine if the operation is vscale dependent by looking at it's
257 // operand/result types.
258 if (!AssumeVScaleIsEquivalent &&
259 (I.getType()->isScalableTy() ||
260 any_of(
261 Range: I.operand_values(),
262 P: [](const Value *V) { return V->getType()->isScalableTy(); }) ||
263 (isa<GetElementPtrInst>(Val: I) && cast<GetElementPtrInst>(Val: I)
264 .getSourceElementType()
265 ->isScalableTy()) ||
266 (isa<AllocaInst>(Val: I) && cast<AllocaInst>(Val: I).isScalable())))
267 return true;
268 }
269 }
270
271 return false;
272}
273
274static void extractAttrFeatures(const Function &F, const AArch64TTIImpl *TTI,
275 SmallVectorImpl<StringRef> &Features) {
276 StringRef AttributeStr =
277 TTI->isMultiversionedFunction(F) ? "fmv-features" : "target-features";
278 StringRef FeatureStr = F.getFnAttribute(Kind: AttributeStr).getValueAsString();
279 FeatureStr.split(A&: Features, Separator: ",");
280}
281
282APInt AArch64TTIImpl::getFeatureMask(const Function &F) const {
283 SmallVector<StringRef, 8> Features;
284 extractAttrFeatures(F, TTI: this, Features);
285 return AArch64::getCpuSupportsMask(Features);
286}
287
288APInt AArch64TTIImpl::getPriorityMask(const Function &F) const {
289 SmallVector<StringRef, 8> Features;
290 extractAttrFeatures(F, TTI: this, Features);
291 return AArch64::getFMVPriority(Features);
292}
293
294bool AArch64TTIImpl::isMultiversionedFunction(const Function &F) const {
295 return F.hasFnAttribute(Kind: "fmv-features");
296}
297
298/// The compiler must not inline when that may alter the behavior of the
299/// program. This is especially relevant around SME which implements different
300/// runtime modes and maintains external state through attributes.
301///
302/// The compiler must not inline when:
303//
304/// * The module is compiled with 'strict-fp' and the called function
305/// implements a different FP environment than the caller.
306///
307/// * The called function has operations that are incompatible in the mode
308/// of the caller, e.g.:
309/// * inlining non-streaming-only instructions into a streaming function.
310/// * inlining streaming-only instructions into a non-streaming function.
311///
312/// Inline asm blocks must be entered and exited in the [streaming] mode of
313/// the parent function. There is no language-level mechanism to inform the
314/// compiler that a particular inline asm block is streaming compatible, so
315/// the compiler must reject this as a candidate for inlining.
316///
317/// * The called function contains vscale-dependent operations but otherwise
318/// does not take/return VL-dependent arguments (see definition in the
319/// ACLE (Arm C/C++ Language Extensions)).
320///
321/// * The called function sets up new ZA/ZT state into a function that already
322/// has ZA or ZT state, as that is not valid as per the ACLE.
323///
324/// The compiler should not inline when:
325//
326/// * The called function has fixed-length vectors and the caller is in
327/// streaming mode, as this may cause performance regressions. This should
328/// never result in an error to be reported.
329///
330/// * The called function sets up new ZA/ZT state into a function that has no
331/// ZA and no ZT state, as the compiler currently cannot transfer the
332/// attribute to the caller. It may also impact performance, but that should
333/// be covered by AArch64TTIImpl::getInlineCallPenalty().
334///
335/// If a callee is incompatible with its caller, then the inliner should prevent
336/// inlining. LLVM's `alwaysinline` attribute has no impact on whether a callee
337/// is inline compatible. This is especially relevant with streaming-mode
338/// attributes, where `alwaysinline`ing a function into a function with a
339/// different streaming mode may otherwise impact runtime behaviour.
340bool AArch64TTIImpl::areInlineCompatible(const Function *Caller,
341 const Function *Callee) const {
342 SMECallAttrs CallAttrs(*Caller, *Callee);
343
344 // Never inline a function explicitly marked as being streaming,
345 // into a non-streaming function. Assume it was marked as streaming
346 // for a reason.
347 if (CallAttrs.caller().hasNonStreamingInterfaceAndBody() &&
348 CallAttrs.callee().hasStreamingInterfaceOrBody())
349 return false;
350
351 if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0())
352 return false;
353
354 // When inlining, we should consider the body of the function, not the
355 // interface.
356 if (CallAttrs.callee().hasStreamingBody()) {
357 CallAttrs.callee().set(M: SMEAttrs::SM_Compatible, Enable: false);
358 CallAttrs.callee().set(M: SMEAttrs::SM_Enabled, Enable: true);
359 }
360
361 bool ConsiderZA = CallAttrs.requiresZASave();
362 bool ConsiderZT = CallAttrs.requiresPreservingZT0() ||
363 CallAttrs.requiresPreservingAllZAState();
364 bool ConsiderSM = CallAttrs.requiresSMChange();
365
366 // FP environment is interpreted differently between streaming mode and
367 // non-streaming mode, so are not inline-compatible.
368 // FIXME: Analyze whether the callee actually has any FP operations.
369 if (ConsiderSM && (Caller->isStrictFP() || Callee->isStrictFP()))
370 return false;
371
372 if (hasPossibleIncompatibleOps(F: Callee, ConsiderZA, ConsiderZT, ConsiderSM))
373 return false;
374
375 return BaseT::areInlineCompatible(Caller, Callee);
376}
377
378bool AArch64TTIImpl::areTypesABICompatible(const Function *Caller,
379 const Function *Callee,
380 ArrayRef<Type *> Types) const {
381 if (!BaseT::areTypesABICompatible(Caller, Callee, Types))
382 return false;
383
384 // We need to ensure that argument promotion does not attempt to promote
385 // pointers to fixed-length vector types larger than 128 bits like
386 // <8 x float> (and pointers to aggregate types which have such fixed-length
387 // vector type members) into the values of the pointees. Such vector types
388 // are used for SVE VLS but there is no ABI for SVE VLS arguments and the
389 // backend cannot lower such value arguments. The 128-bit fixed-length SVE
390 // types can be safely treated as 128-bit NEON types and they cannot be
391 // distinguished in IR.
392 if (ST->useSVEForFixedLengthVectors() && llvm::any_of(Range&: Types, P: [](Type *Ty) {
393 auto FVTy = dyn_cast<FixedVectorType>(Val: Ty);
394 return FVTy &&
395 FVTy->getScalarSizeInBits() * FVTy->getNumElements() > 128;
396 }))
397 return false;
398
399 return true;
400}
401
402unsigned
403AArch64TTIImpl::getInlineCallPenalty(const Function *F, const CallBase &Call,
404 unsigned DefaultCallPenalty) const {
405 // This function calculates a penalty for executing Call in F.
406 //
407 // There are two ways this function can be called:
408 // (1) F:
409 // call from F -> G (the call here is Call)
410 //
411 // For (1), Call.getCaller() == F, so it will always return a high cost if
412 // a streaming-mode change is required (thus promoting the need to inline the
413 // function)
414 //
415 // (2) F:
416 // call from F -> G (the call here is not Call)
417 // G:
418 // call from G -> H (the call here is Call)
419 //
420 // For (2), if after inlining the body of G into F the call to H requires a
421 // streaming-mode change, and the call to G from F would also require a
422 // streaming-mode change, then there is benefit to do the streaming-mode
423 // change only once and avoid inlining of G into F.
424
425 SMEAttrs FAttrs(*F);
426 SMECallAttrs CallAttrs(Call, &getTLI()->getRuntimeLibcallsInfo());
427
428 if (SMECallAttrs(FAttrs, CallAttrs.callee()).requiresSMChange()) {
429 if (F == Call.getCaller()) // (1)
430 return ST->getCLOpts().call_penalty_sm_change * DefaultCallPenalty;
431 if (SMECallAttrs(FAttrs, CallAttrs.caller()).requiresSMChange()) // (2)
432 return ST->getCLOpts().inline_call_penalty_sm_change * DefaultCallPenalty;
433 }
434
435 return DefaultCallPenalty;
436}
437
438bool AArch64TTIImpl::shouldMaximizeVectorBandwidth(
439 TargetTransformInfo::RegisterKind K) const {
440 assert(K != TargetTransformInfo::RGK_Scalar);
441
442 if (K == TargetTransformInfo::RGK_FixedWidthVector && ST->isNeonAvailable())
443 return true;
444
445 return K == TargetTransformInfo::RGK_ScalableVector &&
446 ST->isSVEorStreamingSVEAvailable() &&
447 !ST->disableMaximizeScalableBandwidth();
448}
449
450/// Calculate the cost of materializing a 64-bit value. This helper
451/// method might only calculate a fraction of a larger immediate. Therefore it
452/// is valid to return a cost of ZERO.
453InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) const {
454 // Check if the immediate can be encoded within an instruction.
455 if (Val == 0 || AArch64_AM::isLogicalImmediate(imm: Val, regSize: 64))
456 return 0;
457
458 if (Val < 0)
459 Val = ~Val;
460
461 // Calculate how many moves we will need to materialize this constant.
462 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
463 AArch64_IMM::expandMOVImm(Imm: Val, BitSize: 64, Insn);
464 return Insn.size();
465}
466
467/// Calculate the cost of materializing the given constant.
468InstructionCost
469AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
470 TTI::TargetCostKind CostKind) const {
471 assert(Ty->isIntegerTy());
472
473 unsigned BitSize = Ty->getPrimitiveSizeInBits();
474 if (BitSize == 0)
475 return ~0U;
476
477 // Sign-extend all constants to a multiple of 64-bit.
478 APInt ImmVal = Imm;
479 if (BitSize & 0x3f)
480 ImmVal = Imm.sext(width: (BitSize + 63) & ~0x3fU);
481
482 // Split the constant into 64-bit chunks and calculate the cost for each
483 // chunk.
484 InstructionCost Cost = 0;
485 for (unsigned ShiftVal = 0; ShiftVal < BitSize; ShiftVal += 64) {
486 APInt Tmp = ImmVal.ashr(ShiftAmt: ShiftVal).sextOrTrunc(width: 64);
487 int64_t Val = Tmp.getSExtValue();
488 Cost += getIntImmCost(Val);
489 }
490 // We need at least one instruction to materialze the constant.
491 return std::max<InstructionCost>(a: 1, b: Cost);
492}
493
494InstructionCost AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
495 const APInt &Imm, Type *Ty,
496 TTI::TargetCostKind CostKind,
497 Instruction *Inst) const {
498 assert(Ty->isIntegerTy());
499
500 unsigned BitSize = Ty->getPrimitiveSizeInBits();
501 // There is no cost model for constants with a bit size of 0. Return TCC_Free
502 // here, so that constant hoisting will ignore this constant.
503 if (BitSize == 0)
504 return TTI::TCC_Free;
505
506 unsigned ImmIdx = ~0U;
507 switch (Opcode) {
508 default:
509 return TTI::TCC_Free;
510 case Instruction::GetElementPtr:
511 // Always hoist the base address of a GetElementPtr.
512 if (Idx == 0)
513 return 2 * TTI::TCC_Basic;
514 return TTI::TCC_Free;
515 case Instruction::Store:
516 ImmIdx = 0;
517 break;
518 case Instruction::Add:
519 case Instruction::Sub:
520 case Instruction::Mul:
521 case Instruction::UDiv:
522 case Instruction::SDiv:
523 case Instruction::URem:
524 case Instruction::SRem:
525 case Instruction::And:
526 case Instruction::Or:
527 case Instruction::Xor:
528 case Instruction::ICmp:
529 ImmIdx = 1;
530 break;
531 // Always return TCC_Free for the shift value of a shift instruction.
532 case Instruction::Shl:
533 case Instruction::LShr:
534 case Instruction::AShr:
535 if (Idx == 1)
536 return TTI::TCC_Free;
537 break;
538 case Instruction::Trunc:
539 case Instruction::ZExt:
540 case Instruction::SExt:
541 case Instruction::IntToPtr:
542 case Instruction::PtrToInt:
543 case Instruction::BitCast:
544 case Instruction::PHI:
545 case Instruction::Call:
546 case Instruction::Select:
547 case Instruction::Ret:
548 case Instruction::Load:
549 break;
550 }
551
552 if (Idx == ImmIdx) {
553 int NumConstants = (BitSize + 63) / 64;
554 InstructionCost Cost = AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
555 return (Cost <= NumConstants * TTI::TCC_Basic)
556 ? static_cast<int>(TTI::TCC_Free)
557 : Cost;
558 }
559 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
560}
561
562InstructionCost
563AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
564 const APInt &Imm, Type *Ty,
565 TTI::TargetCostKind CostKind) const {
566 assert(Ty->isIntegerTy());
567
568 unsigned BitSize = Ty->getPrimitiveSizeInBits();
569 // There is no cost model for constants with a bit size of 0. Return TCC_Free
570 // here, so that constant hoisting will ignore this constant.
571 if (BitSize == 0)
572 return TTI::TCC_Free;
573
574 // Most (all?) AArch64 intrinsics do not support folding immediates into the
575 // selected instruction, so we compute the materialization cost for the
576 // immediate directly.
577 if (IID >= Intrinsic::aarch64_addg && IID <= Intrinsic::aarch64_udiv)
578 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
579
580 switch (IID) {
581 default:
582 return TTI::TCC_Free;
583 case Intrinsic::sadd_with_overflow:
584 case Intrinsic::uadd_with_overflow:
585 case Intrinsic::ssub_with_overflow:
586 case Intrinsic::usub_with_overflow:
587 case Intrinsic::smul_with_overflow:
588 case Intrinsic::umul_with_overflow:
589 if (Idx == 1) {
590 int NumConstants = (BitSize + 63) / 64;
591 InstructionCost Cost = AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
592 return (Cost <= NumConstants * TTI::TCC_Basic)
593 ? static_cast<int>(TTI::TCC_Free)
594 : Cost;
595 }
596 break;
597 case Intrinsic::experimental_stackmap:
598 if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(x: Imm.getSExtValue())))
599 return TTI::TCC_Free;
600 break;
601 case Intrinsic::experimental_patchpoint_void:
602 case Intrinsic::experimental_patchpoint:
603 if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(x: Imm.getSExtValue())))
604 return TTI::TCC_Free;
605 break;
606 case Intrinsic::experimental_gc_statepoint:
607 if ((Idx < 5) || (Imm.getBitWidth() <= 64 && isInt<64>(x: Imm.getSExtValue())))
608 return TTI::TCC_Free;
609 break;
610 }
611 return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
612}
613
614TargetTransformInfo::PopcntSupportKind
615AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) const {
616 assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2");
617 if (TyWidth == 32 || TyWidth == 64)
618 return TTI::PSK_FastHardware;
619 // TODO: AArch64TargetLowering::LowerCTPOP() supports 128bit popcount.
620 return TTI::PSK_Software;
621}
622
623InstructionCost AArch64TTIImpl::getBranchMispredictPenalty() const {
624 // MispredictPenalty is defined per-CPU in AArch64Sched*.td (e.g.,
625 // AArch64SchedNeoverseV2.td).
626 return ST->getMispredictionPenalty();
627}
628
629static bool isUnpackedVectorVT(EVT VecVT) {
630 return VecVT.isScalableVector() &&
631 VecVT.getSizeInBits().getKnownMinValue() < AArch64::SVEBitsPerBlock;
632}
633
634static InstructionCost getHistogramCost(const AArch64Subtarget *ST,
635 const IntrinsicCostAttributes &ICA) {
636 // We need to know at least the number of elements in the vector of buckets
637 // and the size of each element to update.
638 if (ICA.getArgTypes().size() < 2)
639 return InstructionCost::getInvalid();
640
641 // Only interested in costing for the hardware instruction from SVE2.
642 if (!ST->hasSVE2())
643 return InstructionCost::getInvalid();
644
645 Type *BucketPtrsTy = ICA.getArgTypes()[0]; // Type of vector of pointers
646 Type *EltTy = ICA.getArgTypes()[1]; // Type of bucket elements
647 unsigned TotalHistCnts = 1;
648
649 unsigned EltSize = EltTy->getScalarSizeInBits();
650 // Only allow (up to 64b) integers or pointers
651 if ((!EltTy->isIntegerTy() && !EltTy->isPointerTy()) || EltSize > 64)
652 return InstructionCost::getInvalid();
653
654 // FIXME: We should be able to generate histcnt for fixed-length vectors
655 // using ptrue with a specific VL.
656 if (VectorType *VTy = dyn_cast<VectorType>(Val: BucketPtrsTy)) {
657 unsigned EC = VTy->getElementCount().getKnownMinValue();
658 if (!isPowerOf2_64(Value: EC) || !VTy->isScalableTy() || EC == 1)
659 return InstructionCost::getInvalid();
660
661 // HistCnt only supports 32b and 64b element types
662 unsigned LegalEltSize = EltSize <= 32 ? 32 : 64;
663
664 if (EC == 2 || (LegalEltSize == 32 && EC == 4))
665 return InstructionCost(ST->getCLOpts().base_histcnt_cost);
666
667 unsigned NaturalVectorWidth = AArch64::SVEBitsPerBlock / LegalEltSize;
668 TotalHistCnts = EC / NaturalVectorWidth;
669
670 return InstructionCost(ST->getCLOpts().base_histcnt_cost * TotalHistCnts);
671 }
672
673 return InstructionCost::getInvalid();
674}
675
676InstructionCost
677AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
678 TTI::TargetCostKind CostKind) const {
679 // The code-generator is currently not able to handle scalable vectors
680 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
681 // it. This change will be removed when code-generation for these types is
682 // sufficiently reliable.
683 // Only allow masked ld/st to pass through to getMemIntrinsicInstrCost().
684 auto *RetTy = ICA.getReturnType();
685 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: RetTy))
686 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1) &&
687 !is_contained(Set: {Intrinsic::masked_load, Intrinsic::masked_store},
688 Element: ICA.getID()))
689 return InstructionCost::getInvalid();
690
691 switch (ICA.getID()) {
692 case Intrinsic::experimental_vector_histogram_add: {
693 InstructionCost HistCost = getHistogramCost(ST, ICA);
694 // If the cost isn't valid, we may still be able to scalarize
695 if (HistCost.isValid())
696 return HistCost;
697 break;
698 }
699 case Intrinsic::experimental_vector_histogram_uadd_sat:
700 case Intrinsic::experimental_vector_histogram_umax:
701 case Intrinsic::experimental_vector_histogram_umin:
702 // No hardware support for these histogram variants on AArch64.
703 return InstructionCost::getInvalid();
704 case Intrinsic::clmul: {
705 auto LT = getTypeLegalizationCost(Ty: RetTy);
706
707 // PMUL v8i8/v16i8 is always available on AArch64
708 if (ST->hasNEON()) {
709 if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
710 return LT.first;
711
712 // Scalar i8 lowers through scalar/vector moves around PMUL.
713 if (TLI->getValueType(DL, Ty: RetTy, AllowUnknown: true) == MVT::i8) {
714 auto *VecTy =
715 FixedVectorType::get(ElementType: Type::getInt8Ty(C&: RetTy->getContext()), NumElts: 8);
716 return 1 +
717 getVectorInstrCost(Opcode: Instruction::ExtractElement, Ty: VecTy, CostKind,
718 Index: -1, Op0: nullptr, Op1: nullptr) *
719 2 +
720 getVectorInstrCost(Opcode: Instruction::InsertElement, Ty: VecTy, CostKind,
721 Index: -1, Op0: nullptr, Op1: nullptr);
722 }
723 }
724
725 if (LT.second.SimpleTy == MVT::nxv2i64)
726 if (ST->hasSVEAES() && (ST->isSVEAvailable() || ST->hasSSVE_AES()))
727 return LT.first * 3;
728
729 if (ST->hasSVE2() || ST->hasSME()) {
730 switch (LT.second.SimpleTy) {
731 case MVT::nxv16i8:
732 return LT.first;
733 case MVT::nxv8i16:
734 return LT.first * 6;
735 case MVT::nxv4i32:
736 return LT.first * 3;
737 case MVT::nxv2i64:
738 return LT.first * 8;
739 default:
740 break;
741 }
742 }
743
744 // Avoid +sve giving this cost 2 due to custom lowering: It's very slow
745 if (LT.second.SimpleTy == MVT::nxv2i64)
746 return 192;
747
748 if (ST->hasAES()) {
749 switch (LT.second.SimpleTy) {
750 case MVT::i16:
751 case MVT::i32:
752 case MVT::i64:
753 case MVT::i128: {
754 auto *VecTy =
755 FixedVectorType::get(ElementType: Type::getInt64Ty(C&: RetTy->getContext()), NumElts: 1);
756 return LT.first *
757 (1 +
758 getVectorInstrCost(Opcode: Instruction::ExtractElement, Ty: VecTy, CostKind,
759 Index: -1, Op0: nullptr, Op1: nullptr) *
760 2 +
761 getVectorInstrCost(Opcode: Instruction::InsertElement, Ty: VecTy, CostKind,
762 Index: -1, Op0: nullptr, Op1: nullptr));
763 }
764 case MVT::v1i64:
765 return LT.first;
766 case MVT::v2i64:
767 return LT.first * 3;
768 case MVT::v2i32:
769 return LT.first * 6;
770 case MVT::v4i32:
771 return LT.first * 11;
772 case MVT::v4i16:
773 return LT.first * 14;
774 default:
775 break;
776 }
777 }
778 break;
779 }
780 case Intrinsic::smulh:
781 case Intrinsic::umulh: {
782 InstructionCost MulCost =
783 getArithmeticInstrCost(Opcode: Instruction::Mul, Ty: RetTy, CostKind);
784
785 // Wide types like i128 need every partial product of a wide multiply
786 // and the carry chain between them. This is a crude cost approximation.
787 if (RetTy->getScalarSizeInBits() > 64)
788 return 2 * MulCost;
789
790 auto LT = getTypeLegalizationCost(Ty: RetTy);
791 MVT MTy = LT.second;
792
793 // Per-register cost to extend the operands.
794 InstructionCost ExtraCost =
795 LT.first *
796 (MTy.getScalarSizeInBits() != RetTy->getScalarSizeInBits() ? 2 : 0);
797
798 // MULH and MUL lower similarly for 64-bit products. For vectors, they
799 // also share the reduced vector multiply bandwidth seen on some cores.
800 if (MTy.getScalarSizeInBits() == 64)
801 return MulCost + ExtraCost;
802
803 // Scalable vectors (and fixed-length vectors, when possible) use SVE.
804 if (MTy.isScalableVector() ||
805 (MTy.isFixedLengthVector() && ST->isSVEorStreamingSVEAvailable()))
806 return LT.first + ExtraCost;
807
808 static const CostTblEntry MulHighCostTbl[] = {
809 {.ISD: ISD::MULHU, .Type: MVT::i32, .Cost: 2}, // [su]mull + lsr
810
811 {.ISD: ISD::MULHU, .Type: MVT::v8i8, .Cost: 2}, // [su]mull + shrn
812 {.ISD: ISD::MULHU, .Type: MVT::v4i16, .Cost: 2}, // "
813 {.ISD: ISD::MULHU, .Type: MVT::v2i32, .Cost: 2}, // "
814
815 {.ISD: ISD::MULHU, .Type: MVT::v16i8, .Cost: 3}, // [su]mull + [su]mull2 + uzp2
816 {.ISD: ISD::MULHU, .Type: MVT::v8i16, .Cost: 3}, // "
817 {.ISD: ISD::MULHU, .Type: MVT::v4i32, .Cost: 3}, // "
818 };
819
820 // Currently we assume SMULH has the same cost as UMULH.
821 if (const auto *Entry = CostTableLookup(Table: MulHighCostTbl, ISD: ISD::MULHU, Ty: MTy))
822 return LT.first * Entry->Cost + ExtraCost;
823
824 break;
825 }
826 case Intrinsic::umin:
827 case Intrinsic::umax:
828 case Intrinsic::smin:
829 case Intrinsic::smax: {
830 static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
831 MVT::v8i16, MVT::v2i32, MVT::v4i32,
832 MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32,
833 MVT::nxv2i64};
834 auto LT = getTypeLegalizationCost(Ty: RetTy);
835 // Type promotion for v2i8 and v2i16 types have a heavy cost when
836 // vectorising. Account for this cost to avoid vectorising unprofitable
837 // examples when vectorising loops with low trip counts.
838 bool IsSigned =
839 ICA.getID() == Intrinsic::smin || ICA.getID() == Intrinsic::smax;
840 EVT VT = TLI->getValueType(DL, Ty: RetTy, /*AllowUnknown=*/true);
841 if (VT == MVT::v2i8 || VT == MVT::v2i16 || VT == MVT::v4i8)
842 return LT.first * (IsSigned ? 5 : 3);
843 // v2i64 types get converted to cmp+bif hence the cost of 2
844 if (LT.second == MVT::v2i64)
845 return LT.first * 2;
846 if (any_of(Range: ValidMinMaxTys, P: equal_to(Arg&: LT.second)))
847 return LT.first;
848 break;
849 }
850 case Intrinsic::scmp:
851 case Intrinsic::ucmp: {
852 static const CostTblEntry BitreverseTbl[] = {
853 {.ISD: Intrinsic::scmp, .Type: MVT::i32, .Cost: 3}, // cmp+cset+csinv
854 {.ISD: Intrinsic::scmp, .Type: MVT::i64, .Cost: 3}, // cmp+cset+csinv
855 {.ISD: Intrinsic::scmp, .Type: MVT::v8i8, .Cost: 3}, // cmgt+cmgt+sub
856 {.ISD: Intrinsic::scmp, .Type: MVT::v16i8, .Cost: 3}, // cmgt+cmgt+sub
857 {.ISD: Intrinsic::scmp, .Type: MVT::v4i16, .Cost: 3}, // cmgt+cmgt+sub
858 {.ISD: Intrinsic::scmp, .Type: MVT::v8i16, .Cost: 3}, // cmgt+cmgt+sub
859 {.ISD: Intrinsic::scmp, .Type: MVT::v2i32, .Cost: 3}, // cmgt+cmgt+sub
860 {.ISD: Intrinsic::scmp, .Type: MVT::v4i32, .Cost: 3}, // cmgt+cmgt+sub
861 {.ISD: Intrinsic::scmp, .Type: MVT::v1i64, .Cost: 3}, // cmgt+cmgt+sub
862 {.ISD: Intrinsic::scmp, .Type: MVT::v2i64, .Cost: 3}, // cmgt+cmgt+sub
863 };
864 const auto LT = getTypeLegalizationCost(Ty: RetTy);
865 const auto *Entry =
866 CostTableLookup(Table: BitreverseTbl, ISD: Intrinsic::scmp, Ty: LT.second);
867 if (Entry)
868 return Entry->Cost * LT.first;
869 break;
870 }
871 case Intrinsic::sadd_sat:
872 case Intrinsic::ssub_sat:
873 case Intrinsic::uadd_sat:
874 case Intrinsic::usub_sat: {
875 static const auto ValidSatTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
876 MVT::v8i16, MVT::v2i32, MVT::v4i32,
877 MVT::v2i64};
878 auto LT = getTypeLegalizationCost(Ty: RetTy);
879 // This is a base cost of 1 for the vadd, plus 3 extract shifts if we
880 // need to extend the type, as it uses shr(qadd(shl, shl)).
881 unsigned Instrs =
882 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits() ? 1 : 4;
883 if (any_of(Range: ValidSatTys, P: equal_to(Arg&: LT.second)))
884 return LT.first * Instrs;
885
886 TypeSize TS = getDataLayout().getTypeSizeInBits(Ty: RetTy);
887 uint64_t VectorSize = TS.getKnownMinValue();
888
889 if (ST->isSVEAvailable() && VectorSize >= 128 && isPowerOf2_64(Value: VectorSize))
890 return LT.first * Instrs;
891
892 break;
893 }
894 case Intrinsic::abs: {
895 static const auto ValidAbsTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
896 MVT::v8i16, MVT::v2i32, MVT::v4i32,
897 MVT::v2i64, MVT::nxv16i8, MVT::nxv8i16,
898 MVT::nxv4i32, MVT::nxv2i64};
899 auto LT = getTypeLegalizationCost(Ty: RetTy);
900 if (any_of(Range: ValidAbsTys, P: equal_to(Arg&: LT.second)))
901 return LT.first;
902 break;
903 }
904 case Intrinsic::bswap: {
905 static const auto ValidAbsTys = {MVT::v4i16, MVT::v8i16, MVT::v2i32,
906 MVT::v4i32, MVT::v2i64};
907 auto LT = getTypeLegalizationCost(Ty: RetTy);
908 if (any_of(Range: ValidAbsTys, P: equal_to(Arg&: LT.second)) &&
909 LT.second.getScalarSizeInBits() == RetTy->getScalarSizeInBits())
910 return LT.first;
911 break;
912 }
913 case Intrinsic::fma:
914 case Intrinsic::fmuladd: {
915 // Given a fma or fmuladd, cost it the same as a fmul instruction which are
916 // usually the same for costs. TODO: Add fp16 and bf16 expansion costs.
917 Type *EltTy = RetTy->getScalarType();
918 if (EltTy->isFloatTy() || EltTy->isDoubleTy() ||
919 (EltTy->isHalfTy() && ST->hasFullFP16()))
920 return getArithmeticInstrCost(Opcode: Instruction::FMul, Ty: RetTy, CostKind);
921 break;
922 }
923 case Intrinsic::stepvector: {
924 InstructionCost Cost = 1; // Cost of the `index' instruction
925 auto LT = getTypeLegalizationCost(Ty: RetTy);
926 // Legalisation of illegal vectors involves an `index' instruction plus
927 // (LT.first - 1) vector adds.
928 if (LT.first > 1) {
929 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Context&: RetTy->getContext());
930 InstructionCost AddCost =
931 getArithmeticInstrCost(Opcode: Instruction::Add, Ty: LegalVTy, CostKind);
932 Cost += AddCost * (LT.first - 1);
933 }
934 return Cost;
935 }
936 case Intrinsic::vector_extract:
937 case Intrinsic::vector_insert: {
938 // If both the vector and subvector types are legal types and the index
939 // is 0, then this should be a no-op or simple operation; return a
940 // relatively low cost.
941
942 // If arguments aren't actually supplied, then we cannot determine the
943 // value of the index. We also want to skip predicate types.
944 if (ICA.getArgs().size() != ICA.getArgTypes().size() ||
945 ICA.getReturnType()->getScalarType()->isIntegerTy(BitWidth: 1))
946 break;
947
948 LLVMContext &C = RetTy->getContext();
949 EVT VecVT = getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[0]);
950 bool IsExtract = ICA.getID() == Intrinsic::vector_extract;
951 EVT SubVecVT = IsExtract ? getTLI()->getValueType(DL, Ty: RetTy)
952 : getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[1]);
953 // Skip this if either the vector or subvector types are unpacked
954 // SVE types; they may get lowered to stack stores and loads.
955 if (isUnpackedVectorVT(VecVT) || isUnpackedVectorVT(VecVT: SubVecVT))
956 break;
957
958 TargetLoweringBase::LegalizeKind SubVecLK =
959 getTLI()->getTypeConversion(Context&: C, VT: SubVecVT);
960 TargetLoweringBase::LegalizeKind VecLK =
961 getTLI()->getTypeConversion(Context&: C, VT: VecVT);
962 const Value *Idx = IsExtract ? ICA.getArgs()[1] : ICA.getArgs()[2];
963 const ConstantInt *CIdx = cast<ConstantInt>(Val: Idx);
964 if (SubVecLK.first == TargetLoweringBase::TypeLegal &&
965 VecLK.first == TargetLoweringBase::TypeLegal && CIdx->isZero())
966 return TTI::TCC_Free;
967 break;
968 }
969 case Intrinsic::bitreverse: {
970 static const CostTblEntry BitreverseTbl[] = {
971 {.ISD: Intrinsic::bitreverse, .Type: MVT::i32, .Cost: 1},
972 {.ISD: Intrinsic::bitreverse, .Type: MVT::i64, .Cost: 1},
973 {.ISD: Intrinsic::bitreverse, .Type: MVT::v8i8, .Cost: 1},
974 {.ISD: Intrinsic::bitreverse, .Type: MVT::v16i8, .Cost: 1},
975 {.ISD: Intrinsic::bitreverse, .Type: MVT::v4i16, .Cost: 2},
976 {.ISD: Intrinsic::bitreverse, .Type: MVT::v8i16, .Cost: 2},
977 {.ISD: Intrinsic::bitreverse, .Type: MVT::v2i32, .Cost: 2},
978 {.ISD: Intrinsic::bitreverse, .Type: MVT::v4i32, .Cost: 2},
979 {.ISD: Intrinsic::bitreverse, .Type: MVT::v1i64, .Cost: 2},
980 {.ISD: Intrinsic::bitreverse, .Type: MVT::v2i64, .Cost: 2},
981 };
982 const auto LegalisationCost = getTypeLegalizationCost(Ty: RetTy);
983 const auto *Entry =
984 CostTableLookup(Table: BitreverseTbl, ISD: ICA.getID(), Ty: LegalisationCost.second);
985 if (Entry) {
986 // Cost Model is using the legal type(i32) that i8 and i16 will be
987 // converted to +1 so that we match the actual lowering cost
988 if (TLI->getValueType(DL, Ty: RetTy, AllowUnknown: true) == MVT::i8 ||
989 TLI->getValueType(DL, Ty: RetTy, AllowUnknown: true) == MVT::i16)
990 return LegalisationCost.first * Entry->Cost + 1;
991
992 return LegalisationCost.first * Entry->Cost;
993 }
994 break;
995 }
996 case Intrinsic::ctpop: {
997 auto LT = getTypeLegalizationCost(Ty: RetTy);
998 MVT MTy = LT.second;
999
1000 if (ST->hasCSSC() && !RetTy->isVectorTy()) {
1001 int ExtraCost =
1002 MTy.getScalarSizeInBits() != RetTy->getScalarSizeInBits() ? 1 : 0;
1003 return LT.first + ExtraCost;
1004 }
1005 if (!ST->hasNEON()) {
1006 // 32-bit or 64-bit ctpop without NEON is 12 instructions.
1007 return getTypeLegalizationCost(Ty: RetTy).first * 12;
1008 }
1009 static const CostTblEntry CtpopCostTbl[] = {
1010 {.ISD: ISD::CTPOP, .Type: MVT::v2i64, .Cost: 4},
1011 {.ISD: ISD::CTPOP, .Type: MVT::v4i32, .Cost: 3},
1012 {.ISD: ISD::CTPOP, .Type: MVT::v8i16, .Cost: 2},
1013 {.ISD: ISD::CTPOP, .Type: MVT::v16i8, .Cost: 1},
1014 {.ISD: ISD::CTPOP, .Type: MVT::i64, .Cost: 4},
1015 {.ISD: ISD::CTPOP, .Type: MVT::v2i32, .Cost: 3},
1016 {.ISD: ISD::CTPOP, .Type: MVT::v4i16, .Cost: 2},
1017 {.ISD: ISD::CTPOP, .Type: MVT::v8i8, .Cost: 1},
1018 {.ISD: ISD::CTPOP, .Type: MVT::i32, .Cost: 5},
1019 // SVE types (For targets that override NEON for fixed length vectors)
1020 {.ISD: ISD::CTPOP, .Type: MVT::nxv2i64, .Cost: 1},
1021 {.ISD: ISD::CTPOP, .Type: MVT::nxv4i32, .Cost: 1},
1022 {.ISD: ISD::CTPOP, .Type: MVT::nxv8i16, .Cost: 1},
1023 {.ISD: ISD::CTPOP, .Type: MVT::nxv16i8, .Cost: 1},
1024 };
1025
1026 // When SVE is available CNT will be used for fixed and scalable vectors.
1027 if (ST->isSVEorStreamingSVEAvailable() && MTy.isFixedLengthVector())
1028 MTy = MVT::getScalableVectorVT(VT: MTy.getVectorElementType(),
1029 NumElements: 128 / MTy.getScalarSizeInBits());
1030
1031 if (const auto *Entry = CostTableLookup(Table: CtpopCostTbl, ISD: ISD::CTPOP, Ty: MTy)) {
1032 // Extra cost of +1 when illegal vector types are legalized by promoting
1033 // the integer type.
1034 int ExtraCost = MTy.isVector() && MTy.getScalarSizeInBits() !=
1035 RetTy->getScalarSizeInBits()
1036 ? 1
1037 : 0;
1038 return LT.first * Entry->Cost + ExtraCost;
1039 }
1040 break;
1041 }
1042 case Intrinsic::sadd_with_overflow:
1043 case Intrinsic::uadd_with_overflow:
1044 case Intrinsic::ssub_with_overflow:
1045 case Intrinsic::usub_with_overflow:
1046 case Intrinsic::smul_with_overflow:
1047 case Intrinsic::umul_with_overflow: {
1048 static const CostTblEntry WithOverflowCostTbl[] = {
1049 {.ISD: Intrinsic::sadd_with_overflow, .Type: MVT::i8, .Cost: 3},
1050 {.ISD: Intrinsic::uadd_with_overflow, .Type: MVT::i8, .Cost: 3},
1051 {.ISD: Intrinsic::sadd_with_overflow, .Type: MVT::i16, .Cost: 3},
1052 {.ISD: Intrinsic::uadd_with_overflow, .Type: MVT::i16, .Cost: 3},
1053 {.ISD: Intrinsic::sadd_with_overflow, .Type: MVT::i32, .Cost: 1},
1054 {.ISD: Intrinsic::uadd_with_overflow, .Type: MVT::i32, .Cost: 1},
1055 {.ISD: Intrinsic::sadd_with_overflow, .Type: MVT::i64, .Cost: 1},
1056 {.ISD: Intrinsic::uadd_with_overflow, .Type: MVT::i64, .Cost: 1},
1057 {.ISD: Intrinsic::ssub_with_overflow, .Type: MVT::i8, .Cost: 3},
1058 {.ISD: Intrinsic::usub_with_overflow, .Type: MVT::i8, .Cost: 3},
1059 {.ISD: Intrinsic::ssub_with_overflow, .Type: MVT::i16, .Cost: 3},
1060 {.ISD: Intrinsic::usub_with_overflow, .Type: MVT::i16, .Cost: 3},
1061 {.ISD: Intrinsic::ssub_with_overflow, .Type: MVT::i32, .Cost: 1},
1062 {.ISD: Intrinsic::usub_with_overflow, .Type: MVT::i32, .Cost: 1},
1063 {.ISD: Intrinsic::ssub_with_overflow, .Type: MVT::i64, .Cost: 1},
1064 {.ISD: Intrinsic::usub_with_overflow, .Type: MVT::i64, .Cost: 1},
1065 {.ISD: Intrinsic::smul_with_overflow, .Type: MVT::i8, .Cost: 5},
1066 {.ISD: Intrinsic::umul_with_overflow, .Type: MVT::i8, .Cost: 4},
1067 {.ISD: Intrinsic::smul_with_overflow, .Type: MVT::i16, .Cost: 5},
1068 {.ISD: Intrinsic::umul_with_overflow, .Type: MVT::i16, .Cost: 4},
1069 {.ISD: Intrinsic::smul_with_overflow, .Type: MVT::i32, .Cost: 2}, // eg umull;tst
1070 {.ISD: Intrinsic::umul_with_overflow, .Type: MVT::i32, .Cost: 2}, // eg umull;cmp sxtw
1071 {.ISD: Intrinsic::smul_with_overflow, .Type: MVT::i64, .Cost: 3}, // eg mul;smulh;cmp
1072 {.ISD: Intrinsic::umul_with_overflow, .Type: MVT::i64, .Cost: 3}, // eg mul;umulh;cmp asr
1073 };
1074 EVT MTy = TLI->getValueType(DL, Ty: RetTy->getContainedType(i: 0), AllowUnknown: true);
1075 if (MTy.isSimple())
1076 if (const auto *Entry = CostTableLookup(Table: WithOverflowCostTbl, ISD: ICA.getID(),
1077 Ty: MTy.getSimpleVT()))
1078 return Entry->Cost;
1079 break;
1080 }
1081 case Intrinsic::fptosi_sat:
1082 case Intrinsic::fptoui_sat: {
1083 if (ICA.getArgTypes().empty())
1084 break;
1085 bool IsSigned = ICA.getID() == Intrinsic::fptosi_sat;
1086 auto LT = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
1087 EVT MTy = TLI->getValueType(DL, Ty: RetTy);
1088 // Check for the legal types, which are where the size of the input and the
1089 // output are the same, or we are using cvt f64->i32 or f32->i64.
1090 if ((LT.second == MVT::f32 || LT.second == MVT::f64 ||
1091 LT.second == MVT::v2f32 || LT.second == MVT::v4f32 ||
1092 LT.second == MVT::v2f64)) {
1093 if ((LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits() ||
1094 (LT.second == MVT::f64 && MTy == MVT::i32) ||
1095 (LT.second == MVT::f32 && MTy == MVT::i64)))
1096 return LT.first;
1097 // Extending vector types v2f32->v2i64, fcvtl*2 + fcvt*2
1098 if (LT.second.getScalarType() == MVT::f32 && MTy.isFixedLengthVector() &&
1099 MTy.getScalarSizeInBits() == 64)
1100 return LT.first * (MTy.getVectorNumElements() > 2 ? 4 : 2);
1101 }
1102 // Similarly for fp16 sizes. Without FullFP16 we generally need to fcvt to
1103 // f32.
1104 if (LT.second.getScalarType() == MVT::f16 && !ST->hasFullFP16())
1105 return LT.first + getIntrinsicInstrCost(
1106 ICA: {ICA.getID(),
1107 RetTy,
1108 {ICA.getArgTypes()[0]->getWithNewType(
1109 EltTy: Type::getFloatTy(C&: RetTy->getContext()))}},
1110 CostKind);
1111 if ((LT.second == MVT::f16 && MTy == MVT::i32) ||
1112 (LT.second == MVT::f16 && MTy == MVT::i64) ||
1113 ((LT.second == MVT::v4f16 || LT.second == MVT::v8f16) &&
1114 (LT.second.getScalarSizeInBits() == MTy.getScalarSizeInBits())))
1115 return LT.first;
1116 // Extending vector types v8f16->v8i32, fcvtl*2 + fcvt*2
1117 if (LT.second.getScalarType() == MVT::f16 && MTy.isFixedLengthVector() &&
1118 MTy.getScalarSizeInBits() == 32)
1119 return LT.first * (MTy.getVectorNumElements() > 4 ? 4 : 2);
1120 // Extending vector types v8f16->v8i32. These current scalarize but the
1121 // codegen could be better.
1122 if (LT.second.getScalarType() == MVT::f16 && MTy.isFixedLengthVector() &&
1123 MTy.getScalarSizeInBits() == 64)
1124 return MTy.getVectorNumElements() * 3;
1125
1126 // If we can we use a legal convert followed by a min+max
1127 if ((LT.second.getScalarType() == MVT::f32 ||
1128 LT.second.getScalarType() == MVT::f64 ||
1129 LT.second.getScalarType() == MVT::f16) &&
1130 LT.second.getScalarSizeInBits() >= MTy.getScalarSizeInBits()) {
1131 Type *LegalTy =
1132 Type::getIntNTy(C&: RetTy->getContext(), N: LT.second.getScalarSizeInBits());
1133 if (LT.second.isVector())
1134 LegalTy = VectorType::get(ElementType: LegalTy, EC: LT.second.getVectorElementCount());
1135 InstructionCost Cost = 1;
1136 IntrinsicCostAttributes Attrs1(IsSigned ? Intrinsic::smin
1137 : Intrinsic::umin,
1138 LegalTy, {LegalTy, LegalTy});
1139 Cost += getIntrinsicInstrCost(ICA: Attrs1, CostKind);
1140 IntrinsicCostAttributes Attrs2(IsSigned ? Intrinsic::smax
1141 : Intrinsic::umax,
1142 LegalTy, {LegalTy, LegalTy});
1143 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
1144 return LT.first * Cost +
1145 ((LT.second.getScalarType() != MVT::f16 || ST->hasFullFP16()) ? 0
1146 : 1);
1147 }
1148 // Otherwise we need to follow the default expansion that clamps the value
1149 // using a float min/max with a fcmp+sel for nan handling when signed.
1150 Type *FPTy = ICA.getArgTypes()[0]->getScalarType();
1151 RetTy = RetTy->getScalarType();
1152 if (LT.second.isVector()) {
1153 FPTy = VectorType::get(ElementType: FPTy, EC: LT.second.getVectorElementCount());
1154 RetTy = VectorType::get(ElementType: RetTy, EC: LT.second.getVectorElementCount());
1155 }
1156 IntrinsicCostAttributes Attrs1(Intrinsic::minnum, FPTy, {FPTy, FPTy});
1157 InstructionCost Cost = getIntrinsicInstrCost(ICA: Attrs1, CostKind);
1158 IntrinsicCostAttributes Attrs2(Intrinsic::maxnum, FPTy, {FPTy, FPTy});
1159 Cost += getIntrinsicInstrCost(ICA: Attrs2, CostKind);
1160 Cost +=
1161 getCastInstrCost(Opcode: IsSigned ? Instruction::FPToSI : Instruction::FPToUI,
1162 Dst: RetTy, Src: FPTy, CCH: TTI::CastContextHint::None, CostKind);
1163 if (IsSigned) {
1164 Type *CondTy = RetTy->getWithNewBitWidth(NewBitWidth: 1);
1165 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::FCmp, ValTy: FPTy, CondTy,
1166 VecPred: CmpInst::FCMP_UNO, CostKind);
1167 Cost += getCmpSelInstrCost(Opcode: BinaryOperator::Select, ValTy: RetTy, CondTy,
1168 VecPred: CmpInst::FCMP_UNO, CostKind);
1169 }
1170 return LT.first * Cost;
1171 }
1172 case Intrinsic::fshl:
1173 case Intrinsic::fshr: {
1174 if (ICA.getArgs().empty())
1175 break;
1176
1177 const TTI::OperandValueInfo OpInfoZ = TTI::getOperandInfo(V: ICA.getArgs()[2]);
1178
1179 // ROTR / ROTL is a funnel shift with equal first and second operand. For
1180 // ROTR on integer registers (i32/i64) this can be done in a single ror
1181 // instruction. A fshl with a non-constant shift uses a neg + ror.
1182 if (RetTy->isIntegerTy() && ICA.getArgs()[0] == ICA.getArgs()[1] &&
1183 (RetTy->getPrimitiveSizeInBits() == 32 ||
1184 RetTy->getPrimitiveSizeInBits() == 64)) {
1185 InstructionCost NegCost =
1186 (ICA.getID() == Intrinsic::fshl && !OpInfoZ.isConstant()) ? 1 : 0;
1187 return 1 + NegCost;
1188 }
1189
1190 // TODO: Add handling for fshl where third argument is not a constant.
1191 if (!OpInfoZ.isConstant())
1192 break;
1193
1194 const auto LegalisationCost = getTypeLegalizationCost(Ty: RetTy);
1195 if (OpInfoZ.isUniform()) {
1196 static const CostTblEntry FshlTbl[] = {
1197 {.ISD: Intrinsic::fshl, .Type: MVT::v4i32, .Cost: 2}, // shl + usra
1198 {.ISD: Intrinsic::fshl, .Type: MVT::v2i64, .Cost: 2}, {.ISD: Intrinsic::fshl, .Type: MVT::v16i8, .Cost: 2},
1199 {.ISD: Intrinsic::fshl, .Type: MVT::v8i16, .Cost: 2}, {.ISD: Intrinsic::fshl, .Type: MVT::v2i32, .Cost: 2},
1200 {.ISD: Intrinsic::fshl, .Type: MVT::v8i8, .Cost: 2}, {.ISD: Intrinsic::fshl, .Type: MVT::v4i16, .Cost: 2}};
1201 // Costs for both fshl & fshr are the same, so just pass Intrinsic::fshl
1202 // to avoid having to duplicate the costs.
1203 const auto *Entry =
1204 CostTableLookup(Table: FshlTbl, ISD: Intrinsic::fshl, Ty: LegalisationCost.second);
1205 if (Entry)
1206 return LegalisationCost.first * Entry->Cost;
1207 }
1208
1209 auto TyL = getTypeLegalizationCost(Ty: RetTy);
1210 if (!RetTy->isIntegerTy())
1211 break;
1212
1213 // Estimate cost manually, as types like i8 and i16 will get promoted to
1214 // i32 and CostTableLookup will ignore the extra conversion cost.
1215 bool HigherCost = (RetTy->getScalarSizeInBits() != 32 &&
1216 RetTy->getScalarSizeInBits() < 64) ||
1217 (RetTy->getScalarSizeInBits() % 64 != 0);
1218 unsigned ExtraCost = HigherCost ? 1 : 0;
1219 if (RetTy->getScalarSizeInBits() == 32 ||
1220 RetTy->getScalarSizeInBits() == 64)
1221 ExtraCost = 0; // fhsl/fshr for i32 and i64 can be lowered to a single
1222 // extr instruction.
1223 else if (HigherCost)
1224 ExtraCost = 1;
1225 else
1226 break;
1227 return TyL.first + ExtraCost;
1228 }
1229 case Intrinsic::get_active_lane_mask: {
1230 auto RetTy = cast<VectorType>(Val: ICA.getReturnType());
1231 EVT RetVT = getTLI()->getValueType(DL, Ty: RetTy);
1232 EVT OpVT = getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[0]);
1233 if (getTLI()->shouldExpandGetActiveLaneMask(VT: RetVT, OpVT))
1234 break;
1235
1236 if (RetTy->isScalableTy()) {
1237 if (TLI->getTypeAction(Context&: RetTy->getContext(), VT: RetVT) !=
1238 TargetLowering::TypeSplitVector)
1239 break;
1240
1241 auto LT = getTypeLegalizationCost(Ty: RetTy);
1242 InstructionCost Cost = LT.first;
1243 // When SVE2p1 or SME2 is available, we can halve getTypeLegalizationCost
1244 // as get_active_lane_mask may lower to the sve_whilelo_x2 intrinsic, e.g.
1245 // nxv32i1 = get_active_lane_mask(base, idx) ->
1246 // {nxv16i1, nxv16i1} = sve_whilelo_x2(base, idx)
1247 if (ST->hasSVE2p1() || ST->hasSME2()) {
1248 Cost /= 2;
1249 if (Cost == 1)
1250 return Cost;
1251 }
1252
1253 // If more than one whilelo intrinsic is required, include the extra cost
1254 // required by the saturating add & select required to increment the
1255 // start value after the first intrinsic call.
1256 Type *OpTy = ICA.getArgTypes()[0];
1257 IntrinsicCostAttributes AddAttrs(Intrinsic::uadd_sat, OpTy, {OpTy, OpTy});
1258 InstructionCost SplitCost = getIntrinsicInstrCost(ICA: AddAttrs, CostKind);
1259 Type *CondTy = OpTy->getWithNewBitWidth(NewBitWidth: 1);
1260 SplitCost += getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: OpTy, CondTy,
1261 VecPred: CmpInst::ICMP_UGT, CostKind);
1262 return Cost + (SplitCost * (Cost - 1));
1263 } else if (!getTLI()->isTypeLegal(VT: RetVT)) {
1264 // We don't have enough context at this point to determine if the mask
1265 // is going to be kept live after the block, which will force the vXi1
1266 // type to be expanded to legal vectors of integers, e.g. v4i1->v4i32.
1267 // For now, we just assume the vectorizer created this intrinsic and
1268 // the result will be the input for a PHI. In this case the cost will
1269 // be extremely high for fixed-width vectors.
1270 // NOTE: getScalarizationOverhead returns a cost that's far too
1271 // pessimistic for the actual generated codegen. In reality there are
1272 // two instructions generated per lane.
1273 return cast<FixedVectorType>(Val: RetTy)->getNumElements() * 2;
1274 }
1275 break;
1276 }
1277 case Intrinsic::experimental_vector_match: {
1278 if (!ST->hasSVE2() || !ST->isSVEAvailable())
1279 break;
1280
1281 auto *NeedleTy = cast<FixedVectorType>(Val: ICA.getArgTypes()[1]);
1282
1283 // We expand vector.matches with <= 2 elements to a chain of compares.
1284 unsigned SearchSize = NeedleTy->getNumElements();
1285 if (SearchSize <= 2)
1286 break;
1287
1288 auto [LegalParts, SearchVT] = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
1289 if (!is_contained(
1290 Set: {MVT::nxv8i16, MVT::nxv16i8, MVT::v8i16, MVT::v16i8, MVT::v8i8},
1291 Element: SearchVT.SimpleTy))
1292 break;
1293
1294 unsigned ElementSizeInBits = SearchVT.getScalarSizeInBits();
1295
1296 // Number of needle elements we can compare per `match` instruction.
1297 unsigned NeedleEltsPerMatch = AArch64::SVEBitsPerBlock / ElementSizeInBits;
1298
1299 // How many `match` instructions we need to match `SearchSize` elements.
1300 unsigned MatchesRequiredForNeedle =
1301 llvm::divideCeil(Numerator: SearchSize, Denominator: NeedleEltsPerMatch);
1302
1303 // Base cost for MATCH instructions. At least on the Neoverse V2 and
1304 // Neoverse V3, these are cheap operations with the same latency as a
1305 // vector ADD. In most cases, however, we also need to do an extra DUP.
1306 // For fixed-length vectors we currently need an extra five--six
1307 // instructions besides the MATCH.
1308 InstructionCost Cost = 4;
1309 if (isa<FixedVectorType>(Val: RetTy))
1310 Cost += 10;
1311
1312 return Cost * LegalParts * MatchesRequiredForNeedle;
1313 }
1314 case Intrinsic::cttz: {
1315 auto LT = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
1316 if (LT.second == MVT::v8i8 || LT.second == MVT::v16i8)
1317 return LT.first * 2;
1318 if (LT.second == MVT::v4i16 || LT.second == MVT::v8i16 ||
1319 LT.second == MVT::v2i32 || LT.second == MVT::v4i32)
1320 return LT.first * 3;
1321 break;
1322 }
1323 case Intrinsic::experimental_cttz_elts: {
1324 EVT ArgVT = getTLI()->getValueType(DL, Ty: ICA.getArgTypes()[0]);
1325 if (getTLI()->isOperationCustom(Op: ISD::CTTZ_ELTS, VT: ArgVT)) {
1326 // This will consist of a SVE brkb and a cntp instruction. These
1327 // typically have the same latency and half the throughput as a vector
1328 // add instruction.
1329 return 4;
1330 }
1331 break;
1332 }
1333 case Intrinsic::loop_dependence_raw_mask:
1334 case Intrinsic::loop_dependence_war_mask: {
1335 // The whilewr/rw instructions require SVE2 or SME.
1336 if (ST->hasSVE2() || ST->hasSME()) {
1337 EVT VecVT = getTLI()->getValueType(DL, Ty: RetTy);
1338 unsigned EltSizeInBytes =
1339 cast<ConstantInt>(Val: ICA.getArgs()[2])->getZExtValue();
1340 if (!is_contained(Set: {1u, 2u, 4u, 8u}, Element: EltSizeInBytes) ||
1341 VecVT.getVectorMinNumElements() != (16 / EltSizeInBytes))
1342 break;
1343 // For fixed-vector types we need to AND the mask with a ptrue vl<N>.
1344 return isa<FixedVectorType>(Val: RetTy) ? 2 : 1;
1345 }
1346 break;
1347 }
1348 case Intrinsic::experimental_vector_extract_last_active:
1349 if (ST->isSVEorStreamingSVEAvailable()) {
1350 auto [LegalCost, _] = getTypeLegalizationCost(Ty: ICA.getArgTypes()[0]);
1351 // This should turn into chained clastb instructions.
1352 return LegalCost;
1353 }
1354 break;
1355 case Intrinsic::pow: {
1356 // For scalar calls we know the target has the libcall, and for fixed-width
1357 // vectors we know for the worst case it can be scalarised.
1358 EVT VT = getTLI()->getValueType(DL, Ty: RetTy);
1359 RTLIB::Libcall LC = RTLIB::getPOW(VT);
1360 bool HasLibcall =
1361 getTLI()->getLibcallLoweringInfo().getVectorLibcallImplOrMasked(Call: LC) !=
1362 RTLIB::Unsupported;
1363 bool CanLowerWithLibcalls = !isa<ScalableVectorType>(Val: RetTy) || HasLibcall;
1364
1365 // If we know that the call can be lowered with libcalls then it's safe to
1366 // reduce the costs in some cases. This is important for scalable vectors,
1367 // since we cannot scalarize the call in the absence of a vector math
1368 // library.
1369 if (CanLowerWithLibcalls && ICA.getInst() && !ICA.getArgs().empty()) {
1370 // If we know the fast math flags and the exponent is a constant then the
1371 // cost may be less for some exponents like 0.25 and 0.75.
1372 const Constant *ExpC = dyn_cast<Constant>(Val: ICA.getArgs()[1]);
1373 if (ExpC && isa<VectorType>(Val: ExpC->getType()))
1374 ExpC = ExpC->getSplatValue();
1375 if (auto *ExpF = dyn_cast_or_null<ConstantFP>(Val: ExpC)) {
1376 // The argument must be a FP constant.
1377 bool Is025 = ExpF->getValueAPF().isExactlyValue(V: 0.25);
1378 bool Is075 = ExpF->getValueAPF().isExactlyValue(V: 0.75);
1379 FastMathFlags FMF = ICA.getInst()->getFastMathFlags();
1380 if ((Is025 || Is075) && FMF.noInfs() && FMF.approxFunc() &&
1381 (!Is025 || FMF.noSignedZeros())) {
1382 IntrinsicCostAttributes Attrs(Intrinsic::sqrt, RetTy, {RetTy}, FMF);
1383 InstructionCost Sqrt = getIntrinsicInstrCost(ICA: Attrs, CostKind);
1384 if (Is025)
1385 return 2 * Sqrt;
1386 InstructionCost FMul =
1387 getArithmeticInstrCost(Opcode: Instruction::FMul, Ty: RetTy, CostKind);
1388 return (Sqrt * 2) + FMul;
1389 }
1390 // TODO: For 1/3 exponents we expect the cbrt call to be slightly
1391 // cheaper than pow.
1392 }
1393 }
1394
1395 if (HasLibcall)
1396 return getCallInstrCost(F: nullptr, RetTy, Tys: ICA.getArgTypes(), CostKind);
1397 break;
1398 }
1399 case Intrinsic::sqrt:
1400 case Intrinsic::fabs:
1401 case Intrinsic::ceil:
1402 case Intrinsic::floor:
1403 case Intrinsic::nearbyint:
1404 case Intrinsic::round:
1405 case Intrinsic::rint:
1406 case Intrinsic::roundeven:
1407 case Intrinsic::trunc:
1408 case Intrinsic::minnum:
1409 case Intrinsic::maxnum:
1410 case Intrinsic::minimum:
1411 case Intrinsic::maximum: {
1412 if (isa<ScalableVectorType>(Val: RetTy) && ST->isSVEorStreamingSVEAvailable()) {
1413 auto LT = getTypeLegalizationCost(Ty: RetTy);
1414 return LT.first;
1415 }
1416 break;
1417 }
1418 default:
1419 break;
1420 }
1421 return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1422}
1423
1424/// The function will remove redundant reinterprets casting in the presence
1425/// of the control flow
1426static std::optional<Instruction *> processPhiNode(InstCombiner &IC,
1427 IntrinsicInst &II) {
1428 SmallVector<Instruction *, 32> Worklist;
1429 auto RequiredType = II.getType();
1430
1431 auto *PN = dyn_cast<PHINode>(Val: II.getArgOperand(i: 0));
1432 assert(PN && "Expected Phi Node!");
1433
1434 // Don't create a new Phi unless we can remove the old one.
1435 if (!PN->hasOneUse())
1436 return std::nullopt;
1437
1438 for (Value *IncValPhi : PN->incoming_values()) {
1439 auto *Reinterpret = dyn_cast<IntrinsicInst>(Val: IncValPhi);
1440 if (!Reinterpret ||
1441 Reinterpret->getIntrinsicID() !=
1442 Intrinsic::aarch64_sve_convert_to_svbool ||
1443 RequiredType != Reinterpret->getArgOperand(i: 0)->getType())
1444 return std::nullopt;
1445 }
1446
1447 // Create the new Phi
1448 IC.Builder.SetInsertPoint(PN);
1449 PHINode *NPN = IC.Builder.CreatePHI(Ty: RequiredType, NumReservedValues: PN->getNumIncomingValues());
1450 Worklist.push_back(Elt: PN);
1451
1452 for (unsigned I = 0; I < PN->getNumIncomingValues(); I++) {
1453 auto *Reinterpret = cast<Instruction>(Val: PN->getIncomingValue(i: I));
1454 NPN->addIncoming(V: Reinterpret->getOperand(i: 0), BB: PN->getIncomingBlock(i: I));
1455 Worklist.push_back(Elt: Reinterpret);
1456 }
1457
1458 // Cleanup Phi Node and reinterprets
1459 return IC.replaceInstUsesWith(I&: II, V: NPN);
1460}
1461
1462// A collection of properties common to SVE intrinsics that allow for combines
1463// to be written without needing to know the specific intrinsic.
1464struct SVEIntrinsicInfo {
1465 //
1466 // Helper routines for common intrinsic definitions.
1467 //
1468
1469 // e.g. llvm.aarch64.sve.add pg, op1, op2
1470 // with IID ==> llvm.aarch64.sve.add_u
1471 static SVEIntrinsicInfo
1472 defaultMergingOp(Intrinsic::ID IID = Intrinsic::not_intrinsic) {
1473 return SVEIntrinsicInfo()
1474 .setGoverningPredicateOperandIdx(0)
1475 .setOperandIdxInactiveLanesTakenFrom(1)
1476 .setMatchingUndefIntrinsic(IID);
1477 }
1478
1479 // e.g. llvm.aarch64.sve.neg inactive, pg, op
1480 static SVEIntrinsicInfo defaultMergingUnaryOp() {
1481 return SVEIntrinsicInfo()
1482 .setGoverningPredicateOperandIdx(1)
1483 .setOperandIdxInactiveLanesTakenFrom(0)
1484 .setOperandIdxWithNoActiveLanes(0);
1485 }
1486
1487 // e.g. llvm.aarch64.sve.fcvtnt inactive, pg, op
1488 static SVEIntrinsicInfo defaultMergingUnaryNarrowingTopOp() {
1489 return SVEIntrinsicInfo()
1490 .setGoverningPredicateOperandIdx(1)
1491 .setOperandIdxInactiveLanesTakenFrom(0);
1492 }
1493
1494 // e.g. llvm.aarch64.sve.add_u pg, op1, op2
1495 static SVEIntrinsicInfo defaultUndefOp() {
1496 return SVEIntrinsicInfo()
1497 .setGoverningPredicateOperandIdx(0)
1498 .setInactiveLanesAreNotDefined();
1499 }
1500
1501 // e.g. llvm.aarch64.sve.prf pg, ptr (GPIndex = 0)
1502 // llvm.aarch64.sve.st1 data, pg, ptr (GPIndex = 1)
1503 static SVEIntrinsicInfo defaultVoidOp(unsigned GPIndex) {
1504 return SVEIntrinsicInfo()
1505 .setGoverningPredicateOperandIdx(GPIndex)
1506 .setInactiveLanesAreUnused();
1507 }
1508
1509 // e.g. llvm.aarch64.sve.cmpeq pg, op1, op2
1510 // llvm.aarch64.sve.ld1 pg, ptr
1511 static SVEIntrinsicInfo defaultZeroingOp() {
1512 return SVEIntrinsicInfo()
1513 .setGoverningPredicateOperandIdx(0)
1514 .setInactiveLanesAreUnused()
1515 .setResultIsZeroInitialized();
1516 }
1517
1518 // All properties relate to predication and thus having a general predicate
1519 // is the minimum requirement to say there is intrinsic info to act on.
1520 explicit operator bool() const { return hasGoverningPredicate(); }
1521
1522 //
1523 // Properties relating to the governing predicate.
1524 //
1525
1526 bool hasGoverningPredicate() const {
1527 return GoverningPredicateIdx != std::numeric_limits<unsigned>::max();
1528 }
1529
1530 unsigned getGoverningPredicateOperandIdx() const {
1531 assert(hasGoverningPredicate() && "Property not set!");
1532 return GoverningPredicateIdx;
1533 }
1534
1535 SVEIntrinsicInfo &setGoverningPredicateOperandIdx(unsigned Index) {
1536 assert(!hasGoverningPredicate() && "Cannot set property twice!");
1537 GoverningPredicateIdx = Index;
1538 return *this;
1539 }
1540
1541 //
1542 // Properties relating to operations the intrinsic could be transformed into.
1543 // NOTE: This does not mean such a transformation is always possible, but the
1544 // knowledge makes it possible to reuse existing optimisations without needing
1545 // to embed specific handling for each intrinsic. For example, instruction
1546 // simplification can be used to optimise an intrinsic's active lanes.
1547 //
1548
1549 //
1550 // Intrinsic that produces the same result for active lanes.
1551 //
1552
1553 bool hasMatchingUndefIntrinsic() const {
1554 return UndefIntrinsic != Intrinsic::not_intrinsic;
1555 }
1556
1557 Intrinsic::ID getMatchingUndefIntrinsic() const {
1558 assert(hasMatchingUndefIntrinsic() && "Property not set!");
1559 return UndefIntrinsic;
1560 }
1561
1562 SVEIntrinsicInfo &setMatchingUndefIntrinsic(Intrinsic::ID IID) {
1563 assert(!hasMatchingUndefIntrinsic() && "Cannot set property twice!");
1564 UndefIntrinsic = IID;
1565 return *this;
1566 }
1567
1568 //
1569 // Instruction where active lanes produce the same result.
1570 //
1571
1572 bool hasMatchingIROpode() const { return IROpcode != 0; }
1573
1574 unsigned getMatchingIROpode() const {
1575 assert(hasMatchingIROpode() && "Property not set!");
1576 return IROpcode;
1577 }
1578
1579 SVEIntrinsicInfo &setMatchingIROpcode(unsigned Opcode) {
1580 assert(!hasMatchingIROpode() && "Cannot set property twice!");
1581 IROpcode = Opcode;
1582 return *this;
1583 }
1584
1585 bool hasCmpPredicate() const {
1586 return CmpPredicate != CmpInst::BAD_ICMP_PREDICATE;
1587 }
1588
1589 CmpInst::Predicate getCmpPredicate() const {
1590 assert(hasCmpPredicate() && "Property not set!");
1591 return CmpPredicate;
1592 }
1593
1594 SVEIntrinsicInfo &setCmpPredicate(CmpInst::Predicate Pred) {
1595 assert(!hasCmpPredicate() && "Cannot set property twice!");
1596 CmpPredicate = Pred;
1597
1598 if (CmpInst::isFPPredicate(P: Pred))
1599 return setMatchingIROpcode(Instruction::FCmp);
1600
1601 if (CmpInst::isIntPredicate(P: Pred))
1602 return setMatchingIROpcode(Instruction::ICmp);
1603
1604 llvm_unreachable("Unsupported compare predicate!");
1605 }
1606
1607 //
1608 // Properties relating to the result of inactive lanes.
1609 //
1610
1611 bool inactiveLanesTakenFromOperand() const {
1612 return ResultLanes == InactiveLanesTakenFromOperand;
1613 }
1614
1615 unsigned getOperandIdxInactiveLanesTakenFrom() const {
1616 assert(inactiveLanesTakenFromOperand() && "Property not set!");
1617 return OperandIdxForInactiveLanes;
1618 }
1619
1620 SVEIntrinsicInfo &setOperandIdxInactiveLanesTakenFrom(unsigned Index) {
1621 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1622 ResultLanes = InactiveLanesTakenFromOperand;
1623 OperandIdxForInactiveLanes = Index;
1624 return *this;
1625 }
1626
1627 bool inactiveLanesAreNotDefined() const {
1628 return ResultLanes == InactiveLanesAreNotDefined;
1629 }
1630
1631 SVEIntrinsicInfo &setInactiveLanesAreNotDefined() {
1632 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1633 ResultLanes = InactiveLanesAreNotDefined;
1634 return *this;
1635 }
1636
1637 bool inactiveLanesAreUnused() const {
1638 return ResultLanes == InactiveLanesAreUnused;
1639 }
1640
1641 SVEIntrinsicInfo &setInactiveLanesAreUnused() {
1642 assert(ResultLanes == Uninitialized && "Cannot set property twice!");
1643 ResultLanes = InactiveLanesAreUnused;
1644 return *this;
1645 }
1646
1647 // NOTE: Whilst not limited to only inactive lanes, the common use case is:
1648 // inactiveLanesAreZeroed =
1649 // resultIsZeroInitialized() && inactiveLanesAreUnused()
1650 bool resultIsZeroInitialized() const { return ResultIsZeroInitialized; }
1651
1652 SVEIntrinsicInfo &setResultIsZeroInitialized() {
1653 ResultIsZeroInitialized = true;
1654 return *this;
1655 }
1656
1657 //
1658 // The first operand of unary merging operations is typically only used to
1659 // set the result for inactive lanes. Knowing this allows us to deadcode the
1660 // operand when we can prove there are no inactive lanes.
1661 //
1662
1663 bool hasOperandWithNoActiveLanes() const {
1664 return OperandIdxWithNoActiveLanes != std::numeric_limits<unsigned>::max();
1665 }
1666
1667 unsigned getOperandIdxWithNoActiveLanes() const {
1668 assert(hasOperandWithNoActiveLanes() && "Property not set!");
1669 return OperandIdxWithNoActiveLanes;
1670 }
1671
1672 SVEIntrinsicInfo &setOperandIdxWithNoActiveLanes(unsigned Index) {
1673 assert(!hasOperandWithNoActiveLanes() && "Cannot set property twice!");
1674 OperandIdxWithNoActiveLanes = Index;
1675 return *this;
1676 }
1677
1678private:
1679 unsigned GoverningPredicateIdx = std::numeric_limits<unsigned>::max();
1680
1681 Intrinsic::ID UndefIntrinsic = Intrinsic::not_intrinsic;
1682 unsigned IROpcode = 0;
1683 CmpInst::Predicate CmpPredicate = CmpInst::BAD_ICMP_PREDICATE;
1684
1685 enum PredicationStyle {
1686 Uninitialized,
1687 InactiveLanesTakenFromOperand,
1688 InactiveLanesAreNotDefined,
1689 InactiveLanesAreUnused
1690 } ResultLanes = Uninitialized;
1691
1692 bool ResultIsZeroInitialized = false;
1693 unsigned OperandIdxForInactiveLanes = std::numeric_limits<unsigned>::max();
1694 unsigned OperandIdxWithNoActiveLanes = std::numeric_limits<unsigned>::max();
1695};
1696
1697static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II) {
1698 // Some SVE intrinsics do not use scalable vector types, but since they are
1699 // not relevant from an SVEIntrinsicInfo perspective, they are also ignored.
1700 if (!isa<ScalableVectorType>(Val: II.getType()) &&
1701 all_of(Range: II.args(), P: [&](const Value *V) {
1702 return !isa<ScalableVectorType>(Val: V->getType());
1703 }))
1704 return SVEIntrinsicInfo();
1705
1706 Intrinsic::ID IID = II.getIntrinsicID();
1707 switch (IID) {
1708 default:
1709 break;
1710 case Intrinsic::aarch64_sve_fcvt_bf16f32_v2:
1711 case Intrinsic::aarch64_sve_fcvt_f16f32:
1712 case Intrinsic::aarch64_sve_fcvt_f16f64:
1713 case Intrinsic::aarch64_sve_fcvt_f32f16:
1714 case Intrinsic::aarch64_sve_fcvt_f32f64:
1715 case Intrinsic::aarch64_sve_fcvt_f64f16:
1716 case Intrinsic::aarch64_sve_fcvt_f64f32:
1717 case Intrinsic::aarch64_sve_fcvtlt_f32f16:
1718 case Intrinsic::aarch64_sve_fcvtlt_f64f32:
1719 case Intrinsic::aarch64_sve_fcvtx_f32f64:
1720 case Intrinsic::aarch64_sve_fcvtzs:
1721 case Intrinsic::aarch64_sve_fcvtzs_i32f16:
1722 case Intrinsic::aarch64_sve_fcvtzs_i32f64:
1723 case Intrinsic::aarch64_sve_fcvtzs_i64f16:
1724 case Intrinsic::aarch64_sve_fcvtzs_i64f32:
1725 case Intrinsic::aarch64_sve_fcvtzu:
1726 case Intrinsic::aarch64_sve_fcvtzu_i32f16:
1727 case Intrinsic::aarch64_sve_fcvtzu_i32f64:
1728 case Intrinsic::aarch64_sve_fcvtzu_i64f16:
1729 case Intrinsic::aarch64_sve_fcvtzu_i64f32:
1730 case Intrinsic::aarch64_sve_revb:
1731 case Intrinsic::aarch64_sve_revh:
1732 case Intrinsic::aarch64_sve_revw:
1733 case Intrinsic::aarch64_sve_revd:
1734 case Intrinsic::aarch64_sve_scvtf:
1735 case Intrinsic::aarch64_sve_scvtf_f16i32:
1736 case Intrinsic::aarch64_sve_scvtf_f16i64:
1737 case Intrinsic::aarch64_sve_scvtf_f32i64:
1738 case Intrinsic::aarch64_sve_scvtf_f64i32:
1739 case Intrinsic::aarch64_sve_ucvtf:
1740 case Intrinsic::aarch64_sve_ucvtf_f16i32:
1741 case Intrinsic::aarch64_sve_ucvtf_f16i64:
1742 case Intrinsic::aarch64_sve_ucvtf_f32i64:
1743 case Intrinsic::aarch64_sve_ucvtf_f64i32:
1744 return SVEIntrinsicInfo::defaultMergingUnaryOp();
1745
1746 case Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2:
1747 case Intrinsic::aarch64_sve_fcvtnt_f16f32:
1748 case Intrinsic::aarch64_sve_fcvtnt_f32f64:
1749 case Intrinsic::aarch64_sve_fcvtxnt_f32f64:
1750 return SVEIntrinsicInfo::defaultMergingUnaryNarrowingTopOp();
1751
1752 case Intrinsic::aarch64_sve_fabd:
1753 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fabd_u);
1754 case Intrinsic::aarch64_sve_fadd:
1755 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fadd_u)
1756 .setMatchingIROpcode(Instruction::FAdd);
1757 case Intrinsic::aarch64_sve_fdiv:
1758 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fdiv_u)
1759 .setMatchingIROpcode(Instruction::FDiv);
1760 case Intrinsic::aarch64_sve_fmax:
1761 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmax_u);
1762 case Intrinsic::aarch64_sve_fmaxnm:
1763 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmaxnm_u);
1764 case Intrinsic::aarch64_sve_fmin:
1765 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmin_u);
1766 case Intrinsic::aarch64_sve_fminnm:
1767 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fminnm_u);
1768 case Intrinsic::aarch64_sve_fmla:
1769 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmla_u);
1770 case Intrinsic::aarch64_sve_fmls:
1771 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmls_u);
1772 case Intrinsic::aarch64_sve_fmul:
1773 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmul_u)
1774 .setMatchingIROpcode(Instruction::FMul);
1775 case Intrinsic::aarch64_sve_fmulx:
1776 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fmulx_u);
1777 case Intrinsic::aarch64_sve_fnmla:
1778 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fnmla_u);
1779 case Intrinsic::aarch64_sve_fnmls:
1780 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fnmls_u);
1781 case Intrinsic::aarch64_sve_fsub:
1782 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_fsub_u)
1783 .setMatchingIROpcode(Instruction::FSub);
1784 case Intrinsic::aarch64_sve_add:
1785 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_add_u)
1786 .setMatchingIROpcode(Instruction::Add);
1787 case Intrinsic::aarch64_sve_mla:
1788 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_mla_u);
1789 case Intrinsic::aarch64_sve_mls:
1790 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_mls_u);
1791 case Intrinsic::aarch64_sve_mul:
1792 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_mul_u)
1793 .setMatchingIROpcode(Instruction::Mul);
1794 case Intrinsic::aarch64_sve_sabd:
1795 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sabd_u);
1796 case Intrinsic::aarch64_sve_sdiv:
1797 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sdiv_u)
1798 .setMatchingIROpcode(Instruction::SDiv);
1799 case Intrinsic::aarch64_sve_smax:
1800 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_smax_u);
1801 case Intrinsic::aarch64_sve_smin:
1802 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_smin_u);
1803 case Intrinsic::aarch64_sve_smulh:
1804 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_smulh_u);
1805 case Intrinsic::aarch64_sve_sub:
1806 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sub_u)
1807 .setMatchingIROpcode(Instruction::Sub);
1808 case Intrinsic::aarch64_sve_uabd:
1809 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uabd_u);
1810 case Intrinsic::aarch64_sve_udiv:
1811 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_udiv_u)
1812 .setMatchingIROpcode(Instruction::UDiv);
1813 case Intrinsic::aarch64_sve_umax:
1814 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_umax_u);
1815 case Intrinsic::aarch64_sve_umin:
1816 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_umin_u);
1817 case Intrinsic::aarch64_sve_umulh:
1818 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_umulh_u);
1819 case Intrinsic::aarch64_sve_asr:
1820 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_asr_u)
1821 .setMatchingIROpcode(Instruction::AShr);
1822 case Intrinsic::aarch64_sve_lsl:
1823 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_lsl_u)
1824 .setMatchingIROpcode(Instruction::Shl);
1825 case Intrinsic::aarch64_sve_lsr:
1826 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_lsr_u)
1827 .setMatchingIROpcode(Instruction::LShr);
1828 case Intrinsic::aarch64_sve_and:
1829 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_and_u)
1830 .setMatchingIROpcode(Instruction::And);
1831 case Intrinsic::aarch64_sve_bic:
1832 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_bic_u);
1833 case Intrinsic::aarch64_sve_eor:
1834 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_eor_u)
1835 .setMatchingIROpcode(Instruction::Xor);
1836 case Intrinsic::aarch64_sve_orr:
1837 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_orr_u)
1838 .setMatchingIROpcode(Instruction::Or);
1839 case Intrinsic::aarch64_sve_shsub:
1840 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_shsub_u);
1841 case Intrinsic::aarch64_sve_shsubr:
1842 return SVEIntrinsicInfo::defaultMergingOp();
1843 case Intrinsic::aarch64_sve_sqrshl:
1844 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sqrshl_u);
1845 case Intrinsic::aarch64_sve_sqshl:
1846 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sqshl_u);
1847 case Intrinsic::aarch64_sve_sqsub:
1848 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_sqsub_u);
1849 case Intrinsic::aarch64_sve_srshl:
1850 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_srshl_u);
1851 case Intrinsic::aarch64_sve_uhsub:
1852 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uhsub_u);
1853 case Intrinsic::aarch64_sve_uhsubr:
1854 return SVEIntrinsicInfo::defaultMergingOp();
1855 case Intrinsic::aarch64_sve_uqrshl:
1856 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uqrshl_u);
1857 case Intrinsic::aarch64_sve_uqshl:
1858 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uqshl_u);
1859 case Intrinsic::aarch64_sve_uqsub:
1860 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_uqsub_u);
1861 case Intrinsic::aarch64_sve_urshl:
1862 return SVEIntrinsicInfo::defaultMergingOp(IID: Intrinsic::aarch64_sve_urshl_u);
1863
1864 case Intrinsic::aarch64_sve_add_u:
1865 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1866 Instruction::Add);
1867 case Intrinsic::aarch64_sve_and_u:
1868 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1869 Instruction::And);
1870 case Intrinsic::aarch64_sve_asr_u:
1871 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1872 Instruction::AShr);
1873 case Intrinsic::aarch64_sve_eor_u:
1874 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1875 Instruction::Xor);
1876 case Intrinsic::aarch64_sve_fadd_u:
1877 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1878 Instruction::FAdd);
1879 case Intrinsic::aarch64_sve_fdiv_u:
1880 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1881 Instruction::FDiv);
1882 case Intrinsic::aarch64_sve_fmul_u:
1883 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1884 Instruction::FMul);
1885 case Intrinsic::aarch64_sve_fsub_u:
1886 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1887 Instruction::FSub);
1888 case Intrinsic::aarch64_sve_lsl_u:
1889 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1890 Instruction::Shl);
1891 case Intrinsic::aarch64_sve_lsr_u:
1892 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1893 Instruction::LShr);
1894 case Intrinsic::aarch64_sve_mul_u:
1895 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1896 Instruction::Mul);
1897 case Intrinsic::aarch64_sve_orr_u:
1898 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1899 Instruction::Or);
1900 case Intrinsic::aarch64_sve_sdiv_u:
1901 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1902 Instruction::SDiv);
1903 case Intrinsic::aarch64_sve_sub_u:
1904 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1905 Instruction::Sub);
1906 case Intrinsic::aarch64_sve_udiv_u:
1907 return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode(
1908 Instruction::UDiv);
1909
1910 case Intrinsic::aarch64_sve_addqv:
1911 case Intrinsic::aarch64_sve_bic_z:
1912 case Intrinsic::aarch64_sve_brka_z:
1913 case Intrinsic::aarch64_sve_brkb_z:
1914 case Intrinsic::aarch64_sve_brkn_z:
1915 case Intrinsic::aarch64_sve_brkpa_z:
1916 case Intrinsic::aarch64_sve_brkpb_z:
1917 case Intrinsic::aarch64_sve_cntp:
1918 case Intrinsic::aarch64_sve_compact:
1919 case Intrinsic::aarch64_sve_eorv:
1920 case Intrinsic::aarch64_sve_eorqv:
1921 case Intrinsic::aarch64_sve_nand_z:
1922 case Intrinsic::aarch64_sve_nor_z:
1923 case Intrinsic::aarch64_sve_orn_z:
1924 case Intrinsic::aarch64_sve_orv:
1925 case Intrinsic::aarch64_sve_orqv:
1926 case Intrinsic::aarch64_sve_pnext:
1927 case Intrinsic::aarch64_sve_rdffr_z:
1928 case Intrinsic::aarch64_sve_saddv:
1929 case Intrinsic::aarch64_sve_uaddv:
1930 case Intrinsic::aarch64_sve_umaxv:
1931 case Intrinsic::aarch64_sve_umaxqv:
1932 case Intrinsic::aarch64_sve_facge:
1933 case Intrinsic::aarch64_sve_facgt:
1934 case Intrinsic::aarch64_sve_ld1:
1935 case Intrinsic::aarch64_sve_ld1_gather:
1936 case Intrinsic::aarch64_sve_ld1_gather_index:
1937 case Intrinsic::aarch64_sve_ld1_gather_scalar_offset:
1938 case Intrinsic::aarch64_sve_ld1_gather_sxtw:
1939 case Intrinsic::aarch64_sve_ld1_gather_sxtw_index:
1940 case Intrinsic::aarch64_sve_ld1_gather_uxtw:
1941 case Intrinsic::aarch64_sve_ld1_gather_uxtw_index:
1942 case Intrinsic::aarch64_sve_ld1q_gather_index:
1943 case Intrinsic::aarch64_sve_ld1q_gather_scalar_offset:
1944 case Intrinsic::aarch64_sve_ld1q_gather_vector_offset:
1945 case Intrinsic::aarch64_sve_ld1ro:
1946 case Intrinsic::aarch64_sve_ld1rq:
1947 case Intrinsic::aarch64_sve_ld1udq:
1948 case Intrinsic::aarch64_sve_ld1uwq:
1949 case Intrinsic::aarch64_sve_ld2_sret:
1950 case Intrinsic::aarch64_sve_ld2q_sret:
1951 case Intrinsic::aarch64_sve_ld3_sret:
1952 case Intrinsic::aarch64_sve_ld3q_sret:
1953 case Intrinsic::aarch64_sve_ld4_sret:
1954 case Intrinsic::aarch64_sve_ld4q_sret:
1955 case Intrinsic::aarch64_sve_ldff1:
1956 case Intrinsic::aarch64_sve_ldff1_gather:
1957 case Intrinsic::aarch64_sve_ldff1_gather_index:
1958 case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset:
1959 case Intrinsic::aarch64_sve_ldff1_gather_sxtw:
1960 case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index:
1961 case Intrinsic::aarch64_sve_ldff1_gather_uxtw:
1962 case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index:
1963 case Intrinsic::aarch64_sve_ldnf1:
1964 case Intrinsic::aarch64_sve_ldnt1:
1965 case Intrinsic::aarch64_sve_ldnt1_gather:
1966 case Intrinsic::aarch64_sve_ldnt1_gather_index:
1967 case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset:
1968 case Intrinsic::aarch64_sve_ldnt1_gather_uxtw:
1969 return SVEIntrinsicInfo::defaultZeroingOp();
1970
1971 case Intrinsic::aarch64_sve_and_z:
1972 return SVEIntrinsicInfo::defaultZeroingOp().setMatchingIROpcode(
1973 Instruction::And);
1974 case Intrinsic::aarch64_sve_orr_z:
1975 return SVEIntrinsicInfo::defaultZeroingOp().setMatchingIROpcode(
1976 Instruction::Or);
1977 case Intrinsic::aarch64_sve_eor_z:
1978 return SVEIntrinsicInfo::defaultZeroingOp().setMatchingIROpcode(
1979 Instruction::Xor);
1980
1981 case Intrinsic::aarch64_sve_cmpeq:
1982 case Intrinsic::aarch64_sve_cmpeq_wide:
1983 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
1984 CmpInst::ICMP_EQ);
1985 case Intrinsic::aarch64_sve_cmpge:
1986 case Intrinsic::aarch64_sve_cmpge_wide:
1987 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
1988 CmpInst::ICMP_SGE);
1989 case Intrinsic::aarch64_sve_cmpgt:
1990 case Intrinsic::aarch64_sve_cmpgt_wide:
1991 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
1992 CmpInst::ICMP_SGT);
1993 case Intrinsic::aarch64_sve_cmphi:
1994 case Intrinsic::aarch64_sve_cmphi_wide:
1995 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
1996 CmpInst::ICMP_UGT);
1997 case Intrinsic::aarch64_sve_cmphs:
1998 case Intrinsic::aarch64_sve_cmphs_wide:
1999 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2000 CmpInst::ICMP_UGE);
2001 case Intrinsic::aarch64_sve_cmple_wide:
2002 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2003 CmpInst::ICMP_SLE);
2004 case Intrinsic::aarch64_sve_cmplo_wide:
2005 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2006 CmpInst::ICMP_ULT);
2007 case Intrinsic::aarch64_sve_cmpls_wide:
2008 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2009 CmpInst::ICMP_ULE);
2010 case Intrinsic::aarch64_sve_cmplt_wide:
2011 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2012 CmpInst::ICMP_SLT);
2013 case Intrinsic::aarch64_sve_cmpne:
2014 case Intrinsic::aarch64_sve_cmpne_wide:
2015 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2016 CmpInst::ICMP_NE);
2017 case Intrinsic::aarch64_sve_fcmpeq:
2018 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2019 CmpInst::FCMP_OEQ);
2020 case Intrinsic::aarch64_sve_fcmpge:
2021 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2022 CmpInst::FCMP_OGE);
2023 case Intrinsic::aarch64_sve_fcmpgt:
2024 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2025 CmpInst::FCMP_OGT);
2026 case Intrinsic::aarch64_sve_fcmpne:
2027 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2028 CmpInst::FCMP_UNE);
2029 case Intrinsic::aarch64_sve_fcmpuo:
2030 return SVEIntrinsicInfo::defaultZeroingOp().setCmpPredicate(
2031 CmpInst::FCMP_UNO);
2032
2033 case Intrinsic::aarch64_sve_prf:
2034 case Intrinsic::aarch64_sve_prfb_gather_index:
2035 case Intrinsic::aarch64_sve_prfb_gather_scalar_offset:
2036 case Intrinsic::aarch64_sve_prfb_gather_sxtw_index:
2037 case Intrinsic::aarch64_sve_prfb_gather_uxtw_index:
2038 case Intrinsic::aarch64_sve_prfd_gather_index:
2039 case Intrinsic::aarch64_sve_prfd_gather_scalar_offset:
2040 case Intrinsic::aarch64_sve_prfd_gather_sxtw_index:
2041 case Intrinsic::aarch64_sve_prfd_gather_uxtw_index:
2042 case Intrinsic::aarch64_sve_prfh_gather_index:
2043 case Intrinsic::aarch64_sve_prfh_gather_scalar_offset:
2044 case Intrinsic::aarch64_sve_prfh_gather_sxtw_index:
2045 case Intrinsic::aarch64_sve_prfh_gather_uxtw_index:
2046 case Intrinsic::aarch64_sve_prfw_gather_index:
2047 case Intrinsic::aarch64_sve_prfw_gather_scalar_offset:
2048 case Intrinsic::aarch64_sve_prfw_gather_sxtw_index:
2049 case Intrinsic::aarch64_sve_prfw_gather_uxtw_index:
2050 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 0);
2051
2052 case Intrinsic::aarch64_sve_st1_scatter:
2053 case Intrinsic::aarch64_sve_st1_scatter_scalar_offset:
2054 case Intrinsic::aarch64_sve_st1_scatter_sxtw:
2055 case Intrinsic::aarch64_sve_st1_scatter_sxtw_index:
2056 case Intrinsic::aarch64_sve_st1_scatter_uxtw:
2057 case Intrinsic::aarch64_sve_st1_scatter_uxtw_index:
2058 case Intrinsic::aarch64_sve_st1dq:
2059 case Intrinsic::aarch64_sve_st1q_scatter_index:
2060 case Intrinsic::aarch64_sve_st1q_scatter_scalar_offset:
2061 case Intrinsic::aarch64_sve_st1q_scatter_vector_offset:
2062 case Intrinsic::aarch64_sve_st1wq:
2063 case Intrinsic::aarch64_sve_stnt1:
2064 case Intrinsic::aarch64_sve_stnt1_scatter:
2065 case Intrinsic::aarch64_sve_stnt1_scatter_index:
2066 case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset:
2067 case Intrinsic::aarch64_sve_stnt1_scatter_uxtw:
2068 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 1);
2069 case Intrinsic::aarch64_sve_st2:
2070 case Intrinsic::aarch64_sve_st2q:
2071 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 2);
2072 case Intrinsic::aarch64_sve_st3:
2073 case Intrinsic::aarch64_sve_st3q:
2074 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 3);
2075 case Intrinsic::aarch64_sve_st4:
2076 case Intrinsic::aarch64_sve_st4q:
2077 return SVEIntrinsicInfo::defaultVoidOp(GPIndex: 4);
2078 }
2079
2080 return SVEIntrinsicInfo();
2081}
2082
2083static bool isAllActivePredicate(Value *Pred) {
2084 Value *UncastedPred;
2085
2086 // Look through predicate casts that only remove lanes.
2087 if (match(V: Pred, P: m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>(
2088 Ops: m_Value(V&: UncastedPred)))) {
2089 auto *OrigPredTy = cast<ScalableVectorType>(Val: Pred->getType());
2090 Pred = UncastedPred;
2091
2092 if (match(V: Pred, P: m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(
2093 Ops: m_Value(V&: UncastedPred))))
2094 // If the predicate has the same or less lanes than the uncasted predicate
2095 // then we know the casting has no effect.
2096 if (OrigPredTy->getMinNumElements() <=
2097 cast<ScalableVectorType>(Val: UncastedPred->getType())
2098 ->getMinNumElements())
2099 Pred = UncastedPred;
2100 }
2101
2102 auto *C = dyn_cast<Constant>(Val: Pred);
2103 return C && C->isAllOnesValue();
2104}
2105
2106// Simplify `V` by only considering the operations that affect active lanes.
2107// This function should only return existing Values or newly created Constants.
2108static Value *stripInactiveLanes(Value *V, const Value *Pg) {
2109 auto *Dup = dyn_cast<IntrinsicInst>(Val: V);
2110 if (Dup && Dup->getIntrinsicID() == Intrinsic::aarch64_sve_dup &&
2111 Dup->getOperand(i_nocapture: 1) == Pg && isa<Constant>(Val: Dup->getOperand(i_nocapture: 2)))
2112 return ConstantVector::getSplat(
2113 EC: cast<VectorType>(Val: V->getType())->getElementCount(),
2114 Elt: cast<Constant>(Val: Dup->getOperand(i_nocapture: 2)));
2115
2116 return V;
2117}
2118
2119static std::optional<Instruction *>
2120simplifySVEIntrinsicBinOp(InstCombiner &IC, IntrinsicInst &II,
2121 const SVEIntrinsicInfo &IInfo) {
2122 const unsigned Opc = IInfo.getMatchingIROpode();
2123 assert(Instruction::isBinaryOp(Opc) && "Expected a binary operation!");
2124
2125 Value *Pg = II.getOperand(i_nocapture: 0);
2126 Value *Op1 = II.getOperand(i_nocapture: 1);
2127 Value *Op2 = II.getOperand(i_nocapture: 2);
2128 const DataLayout &DL = II.getDataLayout();
2129
2130 // Canonicalise constants to the RHS.
2131 if (Instruction::isCommutative(Opcode: Opc) && IInfo.inactiveLanesAreNotDefined() &&
2132 isa<Constant>(Val: Op1) && !isa<Constant>(Val: Op2)) {
2133 IC.replaceOperand(I&: II, OpNum: 1, V: Op2);
2134 IC.replaceOperand(I&: II, OpNum: 2, V: Op1);
2135 return &II;
2136 }
2137
2138 // Only active lanes matter when simplifying the operation.
2139 Op1 = stripInactiveLanes(V: Op1, Pg);
2140 Op2 = stripInactiveLanes(V: Op2, Pg);
2141
2142 Value *SimpleII;
2143 if (auto FII = dyn_cast<FPMathOperator>(Val: &II))
2144 SimpleII = simplifyBinOp(Opcode: Opc, LHS: Op1, RHS: Op2, FMF: FII->getFastMathFlags(), Q: DL);
2145 else
2146 SimpleII = simplifyBinOp(Opcode: Opc, LHS: Op1, RHS: Op2, Q: DL);
2147
2148 // If both operands are convert.to.svbool from the same narrower predicate
2149 // type, try to simplify the operation at that narrower type. This is valid
2150 // because the conversions zero the lanes not represented by the narrower
2151 // type, so those lanes of the result are zero either way.
2152 Value *NarrowOp1, *NarrowOp2;
2153 if (!SimpleII &&
2154 match(V: Op1, P: m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(
2155 Ops: m_Value(V&: NarrowOp1))) &&
2156 match(V: Op2, P: m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(
2157 Ops: m_Value(V&: NarrowOp2))) &&
2158 NarrowOp1->getType() == NarrowOp2->getType()) {
2159 Value *SimpleNarrow = simplifyBinOp(Opcode: Opc, LHS: NarrowOp1, RHS: NarrowOp2, Q: DL);
2160 if (SimpleNarrow && !isa<UndefValue>(Val: SimpleNarrow)) {
2161 if (match(V: SimpleNarrow, P: m_ZeroInt()))
2162 SimpleII = Constant::getNullValue(Ty: II.getType());
2163 else if (SimpleNarrow == NarrowOp1)
2164 SimpleII = Op1;
2165 else if (SimpleNarrow == NarrowOp2)
2166 SimpleII = Op2;
2167 else
2168 SimpleII =
2169 IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_to_svbool,
2170 OverloadTypes: SimpleNarrow->getType(), Args: SimpleNarrow);
2171 }
2172 }
2173
2174 // An SVE intrinsic's result is always defined. However, this is not the case
2175 // for its equivalent IR instruction (e.g. when shifting by an amount more
2176 // than the data's bitwidth). Simplifications to an undefined result must be
2177 // ignored to preserve the intrinsic's expected behaviour.
2178 if (!SimpleII || isa<UndefValue>(Val: SimpleII))
2179 return std::nullopt;
2180
2181 if (IInfo.inactiveLanesAreNotDefined())
2182 return IC.replaceInstUsesWith(I&: II, V: SimpleII);
2183
2184 Value *Inactive =
2185 IInfo.resultIsZeroInitialized()
2186 ? Constant::getNullValue(Ty: II.getType())
2187 : II.getOperand(i_nocapture: IInfo.getOperandIdxInactiveLanesTakenFrom());
2188
2189 // The intrinsic does nothing (e.g. sve.mul(pg, A, 1.0)).
2190 if (SimpleII == Inactive)
2191 return IC.replaceInstUsesWith(I&: II, V: SimpleII);
2192
2193 // Inactive lanes must be preserved.
2194 SimpleII = IC.Builder.CreateSelect(C: Pg, True: SimpleII, False: Inactive);
2195 return IC.replaceInstUsesWith(I&: II, V: SimpleII);
2196}
2197
2198static std::optional<Instruction *>
2199simplifySVEIntrinsicCompare(InstCombiner &IC, IntrinsicInst &II,
2200 const SVEIntrinsicInfo &IInfo) {
2201 const unsigned Opc = IInfo.getMatchingIROpode();
2202 assert((Opc == Instruction::ICmp || Opc == Instruction::FCmp) &&
2203 "Expected a compare operation!");
2204
2205 Value *Pg = II.getOperand(i_nocapture: 0);
2206 Value *LHS = II.getOperand(i_nocapture: 1);
2207 Value *RHS = II.getOperand(i_nocapture: 2);
2208 CmpInst::Predicate CmpPred = IInfo.getCmpPredicate();
2209 bool IsWideICmp =
2210 Opc == Instruction::ICmp && LHS->getType() != RHS->getType();
2211 assert((IsWideICmp || LHS->getType() == RHS->getType()) &&
2212 "Unexpected wide compare!");
2213
2214 // Canonicalise constants to the RHS.
2215 if ((ICmpInst::isCommutative(P: CmpPred) || FCmpInst::isCommutative(Pred: CmpPred)) &&
2216 isa<Constant>(Val: LHS) && !isa<Constant>(Val: RHS) && !IsWideICmp) {
2217 IC.replaceOperand(I&: II, OpNum: 1, V: RHS);
2218 IC.replaceOperand(I&: II, OpNum: 2, V: LHS);
2219 return &II;
2220 }
2221
2222 // Only active lanes matter when simplifying the operation.
2223 LHS = stripInactiveLanes(V: LHS, Pg);
2224 RHS = stripInactiveLanes(V: RHS, Pg);
2225
2226 if (IsWideICmp) {
2227 // We can do more for wide compares, but not using simplifyCmpInst.
2228 const APInt *LHSVal, *RHSVal;
2229 if (!match(V: LHS, P: m_APInt(Res&: LHSVal)) || !match(V: RHS, P: m_APInt(Res&: RHSVal)))
2230 return std::nullopt;
2231
2232 // Consider cmpge.wide(..., <vscale x 4 x i32> LHS, <vscale x 2 x i64> RHS),
2233 // we must reconstruct the constants because LHS has the wrong element type,
2234 // and RHS the wrong element count.
2235 Type *WideVT = VectorType::get(ElementType: RHS->getType()->getScalarType(),
2236 Other: cast<VectorType>(Val: LHS->getType()));
2237 // NOTE: Wide equality comparisons are signed.
2238 if (ICmpInst::isUnsigned(Pred: CmpPred)) {
2239 LHS = ConstantInt::get(Ty: WideVT, V: LHSVal->getZExtValue());
2240 RHS = ConstantInt::get(Ty: WideVT, V: RHSVal->getZExtValue());
2241 } else {
2242 LHS = ConstantInt::get(Ty: WideVT, V: LHSVal->getSExtValue());
2243 RHS = ConstantInt::get(Ty: WideVT, V: RHSVal->getSExtValue());
2244 }
2245 }
2246
2247 // TODO: Allow fast-math flags for calls to compare intrinsics.
2248 const DataLayout &DL = II.getDataLayout();
2249 Value *SimpleII = simplifyCmpInst(Predicate: CmpPred, LHS, RHS, Q: DL);
2250
2251 // No simplification happened.
2252 if (!SimpleII)
2253 return std::nullopt;
2254
2255 assert(IInfo.resultIsZeroInitialized() && "Expected a zeroing operation!");
2256
2257 if (match(V: SimpleII, P: m_ZeroInt()))
2258 return IC.replaceInstUsesWith(I&: II, V: SimpleII);
2259
2260 // Inactive lanes must be zeroed.
2261 SimpleII = IC.Builder.CreateLogicalAnd(Cond1: Pg, Cond2: SimpleII);
2262 return IC.replaceInstUsesWith(I&: II, V: SimpleII);
2263}
2264
2265// Use SVE intrinsic info to eliminate redundant operands and/or canonicalise
2266// to operations with less strict inactive lane requirements.
2267static std::optional<Instruction *>
2268simplifySVEIntrinsic(InstCombiner &IC, IntrinsicInst &II,
2269 const SVEIntrinsicInfo &IInfo) {
2270 if (!IInfo.hasGoverningPredicate())
2271 return std::nullopt;
2272
2273 auto *OpPredicate = II.getOperand(i_nocapture: IInfo.getGoverningPredicateOperandIdx());
2274
2275 // If there are no active lanes.
2276 if (match(V: OpPredicate, P: m_ZeroInt())) {
2277 if (IInfo.inactiveLanesTakenFromOperand())
2278 return IC.replaceInstUsesWith(
2279 I&: II, V: II.getOperand(i_nocapture: IInfo.getOperandIdxInactiveLanesTakenFrom()));
2280
2281 if (IInfo.inactiveLanesAreUnused()) {
2282 if (IInfo.resultIsZeroInitialized())
2283 IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty: II.getType()));
2284
2285 return IC.eraseInstFromFunction(I&: II);
2286 }
2287 }
2288
2289 // If there are no inactive lanes.
2290 if (isAllActivePredicate(Pred: OpPredicate)) {
2291 if (IInfo.hasOperandWithNoActiveLanes()) {
2292 unsigned OpIdx = IInfo.getOperandIdxWithNoActiveLanes();
2293 if (!isa<UndefValue>(Val: II.getOperand(i_nocapture: OpIdx)))
2294 return IC.replaceOperand(I&: II, OpNum: OpIdx, V: UndefValue::get(T: II.getType()));
2295 }
2296
2297 if (IInfo.hasMatchingUndefIntrinsic()) {
2298 auto *NewDecl = Intrinsic::getOrInsertDeclaration(
2299 M: II.getModule(), id: IInfo.getMatchingUndefIntrinsic(), OverloadTys: {II.getType()});
2300 II.setCalledFunction(NewDecl);
2301 return &II;
2302 }
2303 }
2304
2305 if (!IInfo.hasMatchingIROpode())
2306 return std::nullopt;
2307
2308 //
2309 // Operation specific simplifications.
2310 //
2311
2312 unsigned Opc = IInfo.getMatchingIROpode();
2313
2314 if (Instruction::isBinaryOp(Opcode: Opc))
2315 return simplifySVEIntrinsicBinOp(IC, II, IInfo);
2316
2317 if (Opc == Instruction::FCmp || Opc == Instruction::ICmp)
2318 return simplifySVEIntrinsicCompare(IC, II, IInfo);
2319
2320 return std::nullopt;
2321}
2322
2323// (from_svbool (binop (to_svbool pred) (svbool_t _) (svbool_t _))))
2324// => (binop (pred) (from_svbool _) (from_svbool _))
2325//
2326// The above transformation eliminates a `to_svbool` in the predicate
2327// operand of bitwise operation `binop` by narrowing the vector width of
2328// the operation. For example, it would convert a `<vscale x 16 x i1>
2329// and` into a `<vscale x 4 x i1> and`. This is profitable because
2330// to_svbool must zero the new lanes during widening, whereas
2331// from_svbool is free.
2332static std::optional<Instruction *>
2333tryCombineFromSVBoolBinOp(InstCombiner &IC, IntrinsicInst &II) {
2334 auto m_ConvertToSVBool = [](auto P) {
2335 return m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>(P);
2336 };
2337 constexpr Intrinsic::ID ConvertFromSVBool =
2338 Intrinsic::aarch64_sve_convert_from_svbool;
2339
2340 Type *Ty = II.getType();
2341 Value *LHS, *RHS, *NarrowLHS, *NarrowRHS;
2342
2343 if (match(V: II.getOperand(i_nocapture: 0),
2344 P: m_LogicalAnd(L: m_Value(V&: LHS),
2345 R: m_ConvertToSVBool(m_SpecificType(RefTy: Ty, V&: NarrowRHS))))) {
2346 NarrowLHS = IC.Builder.CreateIntrinsic(ID: ConvertFromSVBool, OverloadTypes: Ty, Args: LHS);
2347 Value *NarrowAnd = IC.Builder.CreateLogicalAnd(Cond1: NarrowLHS, Cond2: NarrowRHS);
2348 return IC.replaceInstUsesWith(I&: II, V: NarrowAnd);
2349 }
2350
2351 if (match(V: II.getOperand(i_nocapture: 0),
2352 P: m_LogicalAnd(L: m_ConvertToSVBool(m_SpecificType(RefTy: Ty, V&: NarrowLHS)),
2353 R: m_Value(V&: RHS)))) {
2354 NarrowRHS = IC.Builder.CreateIntrinsic(ID: ConvertFromSVBool, OverloadTypes: Ty, Args: RHS);
2355 Value *NarrowAnd = IC.Builder.CreateLogicalAnd(Cond1: NarrowLHS, Cond2: NarrowRHS);
2356 return IC.replaceInstUsesWith(I&: II, V: NarrowAnd);
2357 }
2358
2359 auto BinOp = dyn_cast<IntrinsicInst>(Val: II.getOperand(i_nocapture: 0));
2360 if (!BinOp)
2361 return std::nullopt;
2362
2363 Intrinsic::ID BinOpIID = BinOp->getIntrinsicID();
2364 switch (BinOpIID) {
2365 case Intrinsic::aarch64_sve_and_z:
2366 case Intrinsic::aarch64_sve_bic_z:
2367 case Intrinsic::aarch64_sve_eor_z:
2368 case Intrinsic::aarch64_sve_nand_z:
2369 case Intrinsic::aarch64_sve_nor_z:
2370 case Intrinsic::aarch64_sve_orn_z:
2371 case Intrinsic::aarch64_sve_orr_z:
2372 break;
2373 default:
2374 return std::nullopt;
2375 }
2376
2377 Value *BinOpPred = BinOp->getOperand(i_nocapture: 0);
2378 Value *BinOpOp1 = BinOp->getOperand(i_nocapture: 1);
2379 Value *BinOpOp2 = BinOp->getOperand(i_nocapture: 2);
2380
2381 Value *NarrowBinOpPred;
2382 if (!match(V: BinOpPred, P: m_ConvertToSVBool(m_SpecificType(RefTy: Ty, V&: NarrowBinOpPred))))
2383 return std::nullopt;
2384
2385 Value *NarrowBinOpOp1 =
2386 IC.Builder.CreateIntrinsic(ID: ConvertFromSVBool, OverloadTypes: Ty, Args: BinOpOp1);
2387 Value *NarrowBinOpOp2 = NarrowBinOpOp1;
2388 if (BinOpOp1 != BinOpOp2)
2389 NarrowBinOpOp2 =
2390 IC.Builder.CreateIntrinsic(ID: ConvertFromSVBool, OverloadTypes: Ty, Args: BinOpOp2);
2391 Value *NarrowedBinOp = IC.Builder.CreateIntrinsic(
2392 ID: BinOpIID, OverloadTypes: Ty, Args: {NarrowBinOpPred, NarrowBinOpOp1, NarrowBinOpOp2});
2393 return IC.replaceInstUsesWith(I&: II, V: NarrowedBinOp);
2394}
2395
2396static std::optional<Instruction *>
2397instCombineConvertFromSVBool(InstCombiner &IC, IntrinsicInst &II) {
2398 // If the reinterpret instruction operand is a PHI Node
2399 if (isa<PHINode>(Val: II.getArgOperand(i: 0)))
2400 return processPhiNode(IC, II);
2401
2402 if (auto BinOpCombine = tryCombineFromSVBoolBinOp(IC, II))
2403 return BinOpCombine;
2404
2405 // Ignore converts to/from svcount_t.
2406 if (isa<TargetExtType>(Val: II.getArgOperand(i: 0)->getType()) ||
2407 isa<TargetExtType>(Val: II.getType()))
2408 return std::nullopt;
2409
2410 SmallVector<Instruction *, 32> CandidatesForRemoval;
2411 Value *Cursor = II.getOperand(i_nocapture: 0), *EarliestReplacement = nullptr;
2412
2413 const auto *IVTy = cast<VectorType>(Val: II.getType());
2414
2415 // Walk the chain of conversions.
2416 while (Cursor) {
2417 // If the type of the cursor has fewer lanes than the final result, zeroing
2418 // must take place, which breaks the equivalence chain.
2419 const auto *CursorVTy = cast<VectorType>(Val: Cursor->getType());
2420 if (CursorVTy->getElementCount().getKnownMinValue() <
2421 IVTy->getElementCount().getKnownMinValue())
2422 break;
2423
2424 // If the cursor has the same type as I, it is a viable replacement.
2425 if (Cursor->getType() == IVTy)
2426 EarliestReplacement = Cursor;
2427
2428 auto *IntrinsicCursor = dyn_cast<IntrinsicInst>(Val: Cursor);
2429
2430 // If this is not an SVE conversion intrinsic, this is the end of the chain.
2431 if (!IntrinsicCursor || !(IntrinsicCursor->getIntrinsicID() ==
2432 Intrinsic::aarch64_sve_convert_to_svbool ||
2433 IntrinsicCursor->getIntrinsicID() ==
2434 Intrinsic::aarch64_sve_convert_from_svbool))
2435 break;
2436
2437 CandidatesForRemoval.insert(I: CandidatesForRemoval.begin(), Elt: IntrinsicCursor);
2438 Cursor = IntrinsicCursor->getOperand(i_nocapture: 0);
2439 }
2440
2441 // If no viable replacement in the conversion chain was found, there is
2442 // nothing to do.
2443 if (!EarliestReplacement)
2444 return std::nullopt;
2445
2446 return IC.replaceInstUsesWith(I&: II, V: EarliestReplacement);
2447}
2448
2449static std::optional<Instruction *> instCombineSVESel(InstCombiner &IC,
2450 IntrinsicInst &II) {
2451 // svsel(ptrue, x, y) => x
2452 auto *OpPredicate = II.getOperand(i_nocapture: 0);
2453 if (isAllActivePredicate(Pred: OpPredicate))
2454 return IC.replaceInstUsesWith(I&: II, V: II.getOperand(i_nocapture: 1));
2455
2456 auto Select =
2457 IC.Builder.CreateSelect(C: OpPredicate, True: II.getOperand(i_nocapture: 1), False: II.getOperand(i_nocapture: 2));
2458 return IC.replaceInstUsesWith(I&: II, V: Select);
2459}
2460
2461static std::optional<Instruction *> instCombineSVEDup(InstCombiner &IC,
2462 IntrinsicInst &II) {
2463 Value *Pg = II.getOperand(i_nocapture: 1);
2464
2465 // sve.dup(V, all_active, X) ==> splat(X)
2466 if (isAllActivePredicate(Pred: Pg)) {
2467 auto *RetTy = cast<ScalableVectorType>(Val: II.getType());
2468 Value *Splat = IC.Builder.CreateVectorSplat(EC: RetTy->getElementCount(),
2469 V: II.getArgOperand(i: 2));
2470 return IC.replaceInstUsesWith(I&: II, V: Splat);
2471 }
2472
2473 if (!match(V: Pg, P: m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
2474 Ops: m_SpecificInt(V: AArch64SVEPredPattern::vl1))))
2475 return std::nullopt;
2476
2477 // sve.dup(V, sve.ptrue(vl1), X) ==> insertelement V, X, 0
2478 Value *Insert = IC.Builder.CreateInsertElement(
2479 Vec: II.getArgOperand(i: 0), NewElt: II.getArgOperand(i: 2), Idx: uint64_t(0));
2480 return IC.replaceInstUsesWith(I&: II, V: Insert);
2481}
2482
2483static std::optional<Instruction *> instCombineSVEDupX(InstCombiner &IC,
2484 IntrinsicInst &II) {
2485 // Replace DupX with a regular IR splat.
2486 auto *RetTy = cast<ScalableVectorType>(Val: II.getType());
2487 Value *Splat = IC.Builder.CreateVectorSplat(EC: RetTy->getElementCount(),
2488 V: II.getArgOperand(i: 0));
2489 Splat->takeName(V: &II);
2490 return IC.replaceInstUsesWith(I&: II, V: Splat);
2491}
2492
2493// xor(cmpne(%pg, %lhs, %rhs), %pg)
2494// -> cmpeq(%pg, %lhs, %rhs)
2495static std::optional<Instruction *> instCombineXorSVECmpCC(InstCombiner &IC,
2496 IntrinsicInst &II) {
2497 if (!II.hasOneUse())
2498 return std::nullopt;
2499 auto *User = cast<Instruction>(Val: *II.user_begin());
2500 if (!match(V: User, P: m_c_Xor(L: m_Specific(V: &II), R: m_Specific(V: II.getOperand(i_nocapture: 0)))))
2501 return std::nullopt;
2502
2503 Intrinsic::ID IID;
2504 switch (II.getIntrinsicID()) {
2505 case Intrinsic::aarch64_sve_cmpne:
2506 IID = Intrinsic::aarch64_sve_cmpeq;
2507 break;
2508 case Intrinsic::aarch64_sve_cmpne_wide:
2509 IID = Intrinsic::aarch64_sve_cmpeq_wide;
2510 break;
2511 case Intrinsic::aarch64_sve_cmpeq:
2512 IID = Intrinsic::aarch64_sve_cmpne;
2513 break;
2514 case Intrinsic::aarch64_sve_cmpeq_wide:
2515 IID = Intrinsic::aarch64_sve_cmpne_wide;
2516 break;
2517 default:
2518 return std::nullopt;
2519 }
2520
2521 IC.Builder.SetInsertPoint(User);
2522 Value *CMPCC = IC.Builder.CreateIntrinsic(
2523 ID: IID, OverloadTypes: II.getOperand(i_nocapture: 1)->getType(),
2524 Args: {II.getOperand(i_nocapture: 0), II.getOperand(i_nocapture: 1), II.getOperand(i_nocapture: 2)});
2525 IC.replaceInstUsesWith(I&: *User, V: CMPCC);
2526 IC.eraseInstFromFunction(I&: *User);
2527 return &II;
2528}
2529
2530// zext(cmpne(ptrue, %v, 0))
2531// -> umin(%pg, %v, 1)
2532static std::optional<Instruction *> instCombineZExtSVECmpNE(InstCombiner &IC,
2533 IntrinsicInst &II) {
2534 if (!isAllActivePredicate(Pred: II.getOperand(i_nocapture: 0)) ||
2535 !match(V: II.getOperand(i_nocapture: 2), P: m_Zero()))
2536 return std::nullopt;
2537
2538 for (auto *U : II.users()) {
2539 if (match(V: U, P: m_ZExt(Op: m_Specific(V: &II)))) {
2540 auto *User = cast<Instruction>(Val: U);
2541 Type *Ty = II.getOperand(i_nocapture: 1)->getType();
2542 if (User->getType() != Ty)
2543 continue;
2544 IC.Builder.SetInsertPoint(User);
2545 Value *UMin = IC.Builder.CreateIntrinsic(
2546 ID: Intrinsic::aarch64_sve_umin, OverloadTypes: Ty,
2547 Args: {II.getOperand(i_nocapture: 0), II.getOperand(i_nocapture: 1), ConstantInt::get(Ty, V: 1)});
2548 IC.replaceInstUsesWith(I&: *User, V: UMin);
2549 IC.eraseInstFromFunction(I&: *User);
2550 return &II;
2551 }
2552 }
2553 return std::nullopt;
2554}
2555
2556static std::optional<Instruction *> instCombineSVECmpNE(InstCombiner &IC,
2557 IntrinsicInst &II) {
2558 LLVMContext &Ctx = II.getContext();
2559
2560 if (auto Res = instCombineXorSVECmpCC(IC, II))
2561 return Res;
2562
2563 if (auto Res = instCombineZExtSVECmpNE(IC, II))
2564 return Res;
2565
2566 if (!isAllActivePredicate(Pred: II.getArgOperand(i: 0)))
2567 return std::nullopt;
2568
2569 // Check that we have a compare of zero..
2570 auto *SplatValue =
2571 dyn_cast_or_null<ConstantInt>(Val: getSplatValue(V: II.getArgOperand(i: 2)));
2572 if (!SplatValue || !SplatValue->isZero())
2573 return std::nullopt;
2574
2575 // ..against a dupq
2576 auto *DupQLane = dyn_cast<IntrinsicInst>(Val: II.getArgOperand(i: 1));
2577 if (!DupQLane ||
2578 DupQLane->getIntrinsicID() != Intrinsic::aarch64_sve_dupq_lane)
2579 return std::nullopt;
2580
2581 // Where the dupq is a lane 0 replicate of a vector insert
2582 auto *DupQLaneIdx = dyn_cast<ConstantInt>(Val: DupQLane->getArgOperand(i: 1));
2583 if (!DupQLaneIdx || !DupQLaneIdx->isZero())
2584 return std::nullopt;
2585
2586 auto *VecIns = dyn_cast<IntrinsicInst>(Val: DupQLane->getArgOperand(i: 0));
2587 if (!VecIns || VecIns->getIntrinsicID() != Intrinsic::vector_insert)
2588 return std::nullopt;
2589
2590 // Where the vector insert is a fixed constant vector insert into undef at
2591 // index zero
2592 if (!isa<UndefValue>(Val: VecIns->getArgOperand(i: 0)))
2593 return std::nullopt;
2594
2595 if (!cast<ConstantInt>(Val: VecIns->getArgOperand(i: 2))->isZero())
2596 return std::nullopt;
2597
2598 auto *ConstVec = dyn_cast<Constant>(Val: VecIns->getArgOperand(i: 1));
2599 if (!ConstVec)
2600 return std::nullopt;
2601
2602 auto *VecTy = dyn_cast<FixedVectorType>(Val: ConstVec->getType());
2603 auto *OutTy = dyn_cast<ScalableVectorType>(Val: II.getType());
2604 if (!VecTy || !OutTy || VecTy->getNumElements() != OutTy->getMinNumElements())
2605 return std::nullopt;
2606
2607 unsigned NumElts = VecTy->getNumElements();
2608 unsigned PredicateBits = 0;
2609
2610 // Expand intrinsic operands to a 16-bit byte level predicate
2611 for (unsigned I = 0; I < NumElts; ++I) {
2612 auto *Arg = dyn_cast<ConstantInt>(Val: ConstVec->getAggregateElement(Elt: I));
2613 if (!Arg)
2614 return std::nullopt;
2615 if (!Arg->isZero())
2616 PredicateBits |= 1 << (I * (16 / NumElts));
2617 }
2618
2619 // If all bits are zero bail early with an empty predicate
2620 if (PredicateBits == 0) {
2621 auto *PFalse = Constant::getNullValue(Ty: II.getType());
2622 PFalse->takeName(V: &II);
2623 return IC.replaceInstUsesWith(I&: II, V: PFalse);
2624 }
2625
2626 // Calculate largest predicate type used (where byte predicate is largest)
2627 unsigned Mask = 8;
2628 for (unsigned I = 0; I < 16; ++I)
2629 if ((PredicateBits & (1 << I)) != 0)
2630 Mask |= (I % 8);
2631
2632 unsigned PredSize = Mask & -Mask;
2633 auto *PredType = ScalableVectorType::get(
2634 ElementType: Type::getInt1Ty(C&: Ctx), MinNumElts: AArch64::SVEBitsPerBlock / (PredSize * 8));
2635
2636 // Ensure all relevant bits are set
2637 for (unsigned I = 0; I < 16; I += PredSize)
2638 if ((PredicateBits & (1 << I)) == 0)
2639 return std::nullopt;
2640
2641 auto *ConvertToSVBool =
2642 IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_to_svbool,
2643 OverloadTypes: PredType, Args: ConstantInt::getTrue(Ty: PredType));
2644 auto *ConvertFromSVBool =
2645 IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_convert_from_svbool,
2646 OverloadTypes: II.getType(), Args: ConvertToSVBool);
2647
2648 ConvertFromSVBool->takeName(V: &II);
2649 return IC.replaceInstUsesWith(I&: II, V: ConvertFromSVBool);
2650}
2651
2652static std::optional<Instruction *> instCombineSVELast(InstCombiner &IC,
2653 IntrinsicInst &II) {
2654 Value *Pg = II.getArgOperand(i: 0);
2655 Value *Vec = II.getArgOperand(i: 1);
2656 auto IntrinsicID = II.getIntrinsicID();
2657 bool IsAfter = IntrinsicID == Intrinsic::aarch64_sve_lasta;
2658
2659 // lastX(splat(X)) --> X
2660 if (auto *SplatVal = getSplatValue(V: Vec))
2661 return IC.replaceInstUsesWith(I&: II, V: SplatVal);
2662
2663 // If x and/or y is a splat value then:
2664 // lastX (binop (x, y)) --> binop(lastX(x), lastX(y))
2665 Value *LHS, *RHS;
2666 if (match(V: Vec, P: m_OneUse(SubPattern: m_BinOp(L: m_Value(V&: LHS), R: m_Value(V&: RHS))))) {
2667 if (isSplatValue(V: LHS) || isSplatValue(V: RHS)) {
2668 auto *OldBinOp = cast<BinaryOperator>(Val: Vec);
2669 auto OpC = OldBinOp->getOpcode();
2670 auto *NewLHS =
2671 IC.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Vec->getType()}, Args: {Pg, LHS});
2672 auto *NewRHS =
2673 IC.Builder.CreateIntrinsic(ID: IntrinsicID, OverloadTypes: {Vec->getType()}, Args: {Pg, RHS});
2674 auto *NewBinOp = BinaryOperator::CreateWithCopiedFlags(
2675 Opc: OpC, V1: NewLHS, V2: NewRHS, CopyO: OldBinOp, Name: OldBinOp->getName(), InsertBefore: II.getIterator());
2676 return IC.replaceInstUsesWith(I&: II, V: NewBinOp);
2677 }
2678 }
2679
2680 auto *C = dyn_cast<Constant>(Val: Pg);
2681 if (IsAfter && C && C->isNullValue()) {
2682 // The intrinsic is extracting lane 0 so use an extract instead.
2683 auto *IdxTy = Type::getInt64Ty(C&: II.getContext());
2684 auto *Extract = ExtractElementInst::Create(Vec, Idx: ConstantInt::get(Ty: IdxTy, V: 0));
2685 Extract->insertBefore(InsertPos: II.getIterator());
2686 Extract->takeName(V: &II);
2687 return IC.replaceInstUsesWith(I&: II, V: Extract);
2688 }
2689
2690 auto *IntrPG = dyn_cast<IntrinsicInst>(Val: Pg);
2691 if (!IntrPG)
2692 return std::nullopt;
2693
2694 if (IntrPG->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue)
2695 return std::nullopt;
2696
2697 const auto PTruePattern =
2698 cast<ConstantInt>(Val: IntrPG->getOperand(i_nocapture: 0))->getZExtValue();
2699
2700 // Can the intrinsic's predicate be converted to a known constant index?
2701 unsigned MinNumElts = getNumElementsFromSVEPredPattern(Pattern: PTruePattern);
2702 if (!MinNumElts)
2703 return std::nullopt;
2704
2705 unsigned Idx = MinNumElts - 1;
2706 // Increment the index if extracting the element after the last active
2707 // predicate element.
2708 if (IsAfter)
2709 ++Idx;
2710
2711 // Ignore extracts whose index is larger than the known minimum vector
2712 // length. NOTE: This is an artificial constraint where we prefer to
2713 // maintain what the user asked for until an alternative is proven faster.
2714 auto *PgVTy = cast<ScalableVectorType>(Val: Pg->getType());
2715 if (Idx >= PgVTy->getMinNumElements())
2716 return std::nullopt;
2717
2718 // The intrinsic is extracting a fixed lane so use an extract instead.
2719 auto *IdxTy = Type::getInt64Ty(C&: II.getContext());
2720 auto *Extract = ExtractElementInst::Create(Vec, Idx: ConstantInt::get(Ty: IdxTy, V: Idx));
2721 Extract->insertBefore(InsertPos: II.getIterator());
2722 Extract->takeName(V: &II);
2723 return IC.replaceInstUsesWith(I&: II, V: Extract);
2724}
2725
2726static std::optional<Instruction *> instCombineSVECondLast(InstCombiner &IC,
2727 IntrinsicInst &II) {
2728 // The SIMD&FP variant of CLAST[AB] is significantly faster than the scalar
2729 // integer variant across a variety of micro-architectures. Replace scalar
2730 // integer CLAST[AB] intrinsic with optimal SIMD&FP variant. A simple
2731 // bitcast-to-fp + clast[ab] + bitcast-to-int will cost a cycle or two more
2732 // depending on the micro-architecture, but has been observed as generally
2733 // being faster, particularly when the CLAST[AB] op is a loop-carried
2734 // dependency.
2735 Value *Pg = II.getArgOperand(i: 0);
2736 Value *Fallback = II.getArgOperand(i: 1);
2737 Value *Vec = II.getArgOperand(i: 2);
2738 Type *Ty = II.getType();
2739
2740 if (!Ty->isIntegerTy())
2741 return std::nullopt;
2742
2743 Type *FPTy;
2744 switch (cast<IntegerType>(Val: Ty)->getBitWidth()) {
2745 default:
2746 return std::nullopt;
2747 case 16:
2748 FPTy = IC.Builder.getHalfTy();
2749 break;
2750 case 32:
2751 FPTy = IC.Builder.getFloatTy();
2752 break;
2753 case 64:
2754 FPTy = IC.Builder.getDoubleTy();
2755 break;
2756 }
2757
2758 Value *FPFallBack = IC.Builder.CreateBitCast(V: Fallback, DestTy: FPTy);
2759 auto *FPVTy = VectorType::get(
2760 ElementType: FPTy, EC: cast<VectorType>(Val: Vec->getType())->getElementCount());
2761 Value *FPVec = IC.Builder.CreateBitCast(V: Vec, DestTy: FPVTy);
2762 auto *FPII = IC.Builder.CreateIntrinsic(
2763 ID: II.getIntrinsicID(), OverloadTypes: {FPVec->getType()}, Args: {Pg, FPFallBack, FPVec});
2764 Value *FPIItoInt = IC.Builder.CreateBitCast(V: FPII, DestTy: II.getType());
2765 return IC.replaceInstUsesWith(I&: II, V: FPIItoInt);
2766}
2767
2768static std::optional<Instruction *> instCombineRDFFR(InstCombiner &IC,
2769 IntrinsicInst &II) {
2770 // Replace rdffr with predicated rdffr.z intrinsic, so that optimizePTestInstr
2771 // can work with RDFFR_PP for ptest elimination.
2772 auto *RDFFR = IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_rdffr_z,
2773 Args: ConstantInt::getTrue(Ty: II.getType()));
2774 RDFFR->takeName(V: &II);
2775 return IC.replaceInstUsesWith(I&: II, V: RDFFR);
2776}
2777
2778static std::optional<Instruction *>
2779instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) {
2780 const auto Pattern = cast<ConstantInt>(Val: II.getArgOperand(i: 0))->getZExtValue();
2781
2782 if (Pattern == AArch64SVEPredPattern::all) {
2783 Value *Cnt = IC.Builder.CreateElementCount(
2784 Ty: II.getType(), EC: ElementCount::getScalable(MinVal: NumElts));
2785 Cnt->takeName(V: &II);
2786 return IC.replaceInstUsesWith(I&: II, V: Cnt);
2787 }
2788
2789 unsigned MinNumElts = getNumElementsFromSVEPredPattern(Pattern);
2790
2791 return MinNumElts && NumElts >= MinNumElts
2792 ? std::optional<Instruction *>(IC.replaceInstUsesWith(
2793 I&: II, V: ConstantInt::get(Ty: II.getType(), V: MinNumElts)))
2794 : std::nullopt;
2795}
2796
2797static std::optional<Instruction *>
2798instCombineSMECntsd(InstCombiner &IC, IntrinsicInst &II,
2799 const AArch64Subtarget *ST) {
2800 if (!ST->isStreaming())
2801 return std::nullopt;
2802
2803 // In streaming-mode, aarch64_sme_cntds is equivalent to aarch64_sve_cntd
2804 // with SVEPredPattern::all
2805 Value *Cnt =
2806 IC.Builder.CreateElementCount(Ty: II.getType(), EC: ElementCount::getScalable(MinVal: 2));
2807 Cnt->takeName(V: &II);
2808 return IC.replaceInstUsesWith(I&: II, V: Cnt);
2809}
2810
2811static std::optional<Instruction *> instCombineSVEPTest(InstCombiner &IC,
2812 IntrinsicInst &II) {
2813 Value *PgVal = II.getArgOperand(i: 0);
2814 Value *OpVal = II.getArgOperand(i: 1);
2815
2816 // PTEST_<FIRST|LAST>(X, X) is equivalent to PTEST_ANY(X, X).
2817 // Later optimizations prefer this form.
2818 if (PgVal == OpVal &&
2819 (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_first ||
2820 II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_last)) {
2821 Value *Ops[] = {PgVal, OpVal};
2822 Type *Tys[] = {PgVal->getType()};
2823
2824 auto *PTest =
2825 IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_ptest_any, OverloadTypes: Tys, Args: Ops);
2826 PTest->takeName(V: &II);
2827
2828 return IC.replaceInstUsesWith(I&: II, V: PTest);
2829 }
2830
2831 IntrinsicInst *Pg = dyn_cast<IntrinsicInst>(Val: PgVal);
2832 IntrinsicInst *Op = dyn_cast<IntrinsicInst>(Val: OpVal);
2833
2834 if (!Pg || !Op)
2835 return std::nullopt;
2836
2837 Intrinsic::ID OpIID = Op->getIntrinsicID();
2838
2839 if (Pg->getIntrinsicID() == Intrinsic::aarch64_sve_convert_to_svbool &&
2840 OpIID == Intrinsic::aarch64_sve_convert_to_svbool &&
2841 Pg->getArgOperand(i: 0)->getType() == Op->getArgOperand(i: 0)->getType()) {
2842 Value *Ops[] = {Pg->getArgOperand(i: 0), Op->getArgOperand(i: 0)};
2843 Type *Tys[] = {Pg->getArgOperand(i: 0)->getType()};
2844
2845 auto *PTest = IC.Builder.CreateIntrinsic(ID: II.getIntrinsicID(), OverloadTypes: Tys, Args: Ops);
2846
2847 PTest->takeName(V: &II);
2848 return IC.replaceInstUsesWith(I&: II, V: PTest);
2849 }
2850
2851 // Transform PTEST_ANY(X=OP(PG,...), X) -> PTEST_ANY(PG, X)).
2852 // Later optimizations may rewrite sequence to use the flag-setting variant
2853 // of instruction X to remove PTEST.
2854 if ((Pg == Op) && (II.getIntrinsicID() == Intrinsic::aarch64_sve_ptest_any) &&
2855 ((OpIID == Intrinsic::aarch64_sve_brka_z) ||
2856 (OpIID == Intrinsic::aarch64_sve_brkb_z) ||
2857 (OpIID == Intrinsic::aarch64_sve_brkpa_z) ||
2858 (OpIID == Intrinsic::aarch64_sve_brkpb_z) ||
2859 (OpIID == Intrinsic::aarch64_sve_rdffr_z) ||
2860 (OpIID == Intrinsic::aarch64_sve_and_z) ||
2861 (OpIID == Intrinsic::aarch64_sve_bic_z) ||
2862 (OpIID == Intrinsic::aarch64_sve_eor_z) ||
2863 (OpIID == Intrinsic::aarch64_sve_nand_z) ||
2864 (OpIID == Intrinsic::aarch64_sve_nor_z) ||
2865 (OpIID == Intrinsic::aarch64_sve_orn_z) ||
2866 (OpIID == Intrinsic::aarch64_sve_orr_z))) {
2867 Value *Ops[] = {Pg->getArgOperand(i: 0), Pg};
2868 Type *Tys[] = {Pg->getType()};
2869
2870 auto *PTest = IC.Builder.CreateIntrinsic(ID: II.getIntrinsicID(), OverloadTypes: Tys, Args: Ops);
2871 PTest->takeName(V: &II);
2872
2873 return IC.replaceInstUsesWith(I&: II, V: PTest);
2874 }
2875
2876 return std::nullopt;
2877}
2878
2879template <Intrinsic::ID MulOpc, Intrinsic::ID FuseOpc>
2880static std::optional<Instruction *>
2881instCombineSVEVectorFuseMulAddSub(InstCombiner &IC, IntrinsicInst &II,
2882 bool MergeIntoAddendOp) {
2883 Value *P = II.getOperand(i_nocapture: 0);
2884 Value *MulOp0, *MulOp1, *AddendOp, *Mul;
2885 if (MergeIntoAddendOp) {
2886 AddendOp = II.getOperand(i_nocapture: 1);
2887 Mul = II.getOperand(i_nocapture: 2);
2888 } else {
2889 AddendOp = II.getOperand(i_nocapture: 2);
2890 Mul = II.getOperand(i_nocapture: 1);
2891 }
2892
2893 if (!match(Mul, m_Intrinsic<MulOpc>(m_Specific(V: P), m_Value(V&: MulOp0),
2894 m_Value(V&: MulOp1))))
2895 return std::nullopt;
2896
2897 if (!Mul->hasOneUse())
2898 return std::nullopt;
2899
2900 Instruction *FMFSource = nullptr;
2901 if (II.getType()->isFPOrFPVectorTy()) {
2902 llvm::FastMathFlags FAddFlags = II.getFastMathFlags();
2903 // Stop the combine when the flags on the inputs differ in case dropping
2904 // flags would lead to us missing out on more beneficial optimizations.
2905 if (FAddFlags != cast<CallInst>(Val: Mul)->getFastMathFlags())
2906 return std::nullopt;
2907 if (!FAddFlags.allowContract())
2908 return std::nullopt;
2909 FMFSource = &II;
2910 }
2911
2912 Value *Res;
2913 if (MergeIntoAddendOp)
2914 Res = IC.Builder.CreateIntrinsic(ID: FuseOpc, OverloadTypes: {II.getType()},
2915 Args: {P, AddendOp, MulOp0, MulOp1}, FMFSource);
2916 else
2917 Res = IC.Builder.CreateIntrinsic(ID: FuseOpc, OverloadTypes: {II.getType()},
2918 Args: {P, MulOp0, MulOp1, AddendOp}, FMFSource);
2919
2920 return IC.replaceInstUsesWith(I&: II, V: Res);
2921}
2922
2923static std::optional<Instruction *>
2924instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
2925 Value *Pred = II.getOperand(i_nocapture: 0);
2926 Value *PtrOp = II.getOperand(i_nocapture: 1);
2927 Type *VecTy = II.getType();
2928
2929 if (isAllActivePredicate(Pred)) {
2930 LoadInst *Load = IC.Builder.CreateLoad(Ty: VecTy, Ptr: PtrOp);
2931 Load->copyMetadata(SrcInst: II);
2932 return IC.replaceInstUsesWith(I&: II, V: Load);
2933 }
2934
2935 CallInst *MaskedLoad =
2936 IC.Builder.CreateMaskedLoad(Ty: VecTy, Ptr: PtrOp, Alignment: PtrOp->getPointerAlignment(DL),
2937 Mask: Pred, PassThru: ConstantAggregateZero::get(Ty: VecTy));
2938 MaskedLoad->copyMetadata(SrcInst: II);
2939 return IC.replaceInstUsesWith(I&: II, V: MaskedLoad);
2940}
2941
2942static std::optional<Instruction *>
2943instCombineSVEST1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) {
2944 Value *VecOp = II.getOperand(i_nocapture: 0);
2945 Value *Pred = II.getOperand(i_nocapture: 1);
2946 Value *PtrOp = II.getOperand(i_nocapture: 2);
2947
2948 if (isAllActivePredicate(Pred)) {
2949 StoreInst *Store = IC.Builder.CreateStore(Val: VecOp, Ptr: PtrOp);
2950 Store->copyMetadata(SrcInst: II);
2951 return IC.eraseInstFromFunction(I&: II);
2952 }
2953
2954 CallInst *MaskedStore = IC.Builder.CreateMaskedStore(
2955 Val: VecOp, Ptr: PtrOp, Alignment: PtrOp->getPointerAlignment(DL), Mask: Pred);
2956 MaskedStore->copyMetadata(SrcInst: II);
2957 return IC.eraseInstFromFunction(I&: II);
2958}
2959
2960static Instruction::BinaryOps intrinsicIDToBinOpCode(unsigned Intrinsic) {
2961 switch (Intrinsic) {
2962 case Intrinsic::aarch64_sve_fmul_u:
2963 return Instruction::BinaryOps::FMul;
2964 case Intrinsic::aarch64_sve_fadd_u:
2965 return Instruction::BinaryOps::FAdd;
2966 case Intrinsic::aarch64_sve_fsub_u:
2967 return Instruction::BinaryOps::FSub;
2968 default:
2969 return Instruction::BinaryOpsEnd;
2970 }
2971}
2972
2973static std::optional<Instruction *>
2974instCombineSVEVectorBinOp(InstCombiner &IC, IntrinsicInst &II) {
2975 // Bail due to missing support for ISD::STRICT_ scalable vector operations.
2976 if (II.isStrictFP())
2977 return std::nullopt;
2978
2979 auto *OpPredicate = II.getOperand(i_nocapture: 0);
2980 auto BinOpCode = intrinsicIDToBinOpCode(Intrinsic: II.getIntrinsicID());
2981 if (BinOpCode == Instruction::BinaryOpsEnd ||
2982 !isAllActivePredicate(Pred: OpPredicate))
2983 return std::nullopt;
2984 auto BinOp = IC.Builder.CreateBinOpFMF(
2985 Opc: BinOpCode, LHS: II.getOperand(i_nocapture: 1), RHS: II.getOperand(i_nocapture: 2), FMFSource: II.getFastMathFlags());
2986 return IC.replaceInstUsesWith(I&: II, V: BinOp);
2987}
2988
2989static std::optional<Instruction *>
2990instCombineSVEVectorMlaU(InstCombiner &IC, IntrinsicInst &II) {
2991 assert(II.getIntrinsicID() == Intrinsic::aarch64_sve_mla_u &&
2992 "Expected MLA_U intrinsic");
2993 Value *Acc = II.getArgOperand(i: 1);
2994 Value *MulOp0 = II.getArgOperand(i: 2);
2995 Value *MulOp1 = II.getArgOperand(i: 3);
2996
2997 // For mla_u, inactive lanes are undefined, so it is valid to drop the
2998 // predicate when replacing mla_u(acc, x, 1) with add(acc, x) or
2999 // mla_u(acc, x, -1) with sub(acc, x).
3000 if (match(V: MulOp0, P: m_One()))
3001 return IC.replaceInstUsesWith(I&: II, V: IC.Builder.CreateAdd(LHS: Acc, RHS: MulOp1));
3002 if (match(V: MulOp1, P: m_One()))
3003 return IC.replaceInstUsesWith(I&: II, V: IC.Builder.CreateAdd(LHS: Acc, RHS: MulOp0));
3004 if (match(V: MulOp0, P: m_AllOnes()))
3005 return IC.replaceInstUsesWith(I&: II, V: IC.Builder.CreateSub(LHS: Acc, RHS: MulOp1));
3006 if (match(V: MulOp1, P: m_AllOnes()))
3007 return IC.replaceInstUsesWith(I&: II, V: IC.Builder.CreateSub(LHS: Acc, RHS: MulOp0));
3008
3009 if (isa<Constant>(Val: MulOp0) && !isa<Constant>(Val: MulOp1)) {
3010 II.setArgOperand(i: 2, v: MulOp1);
3011 II.setArgOperand(i: 3, v: MulOp0);
3012 return &II;
3013 }
3014
3015 return std::nullopt;
3016}
3017
3018static std::optional<Instruction *>
3019instCombineSVEPairwiseAddLong(InstCombiner &IC, IntrinsicInst &II) {
3020 assert((II.getIntrinsicID() == Intrinsic::aarch64_sve_sadalp ||
3021 II.getIntrinsicID() == Intrinsic::aarch64_sve_uadalp) &&
3022 "Expected SADALP or UADALP intrinsic");
3023
3024 // Simplify add(adalp(pg, zeroinitializer, in), wide_acc)
3025 // -> adalp(pg, wide_acc, in)
3026 auto *User = dyn_cast_or_null<Instruction>(Val: II.getUniqueUndroppableUser());
3027 if (!User || !match(V: II.getArgOperand(i: 1), P: m_Zero()))
3028 return std::nullopt;
3029
3030 Value *Acc;
3031 if (!match(V: User, P: m_c_Add(L: m_Specific(V: &II), R: m_Value(V&: Acc))))
3032 return std::nullopt;
3033
3034 IC.Builder.SetInsertPoint(User);
3035 Value *PairwiseAddLong = IC.Builder.CreateIntrinsic(
3036 ID: II.getIntrinsicID(), OverloadTypes: {II.getType()},
3037 Args: {II.getArgOperand(i: 0), Acc, II.getArgOperand(i: 2)});
3038
3039 IC.replaceInstUsesWith(I&: *User, V: PairwiseAddLong);
3040 IC.eraseInstFromFunction(I&: *User);
3041 return &II; // II is now trivially dead and will get erased.
3042}
3043
3044static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC,
3045 IntrinsicInst &II) {
3046 if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
3047 Intrinsic::aarch64_sve_mla>(
3048 IC, II, MergeIntoAddendOp: true))
3049 return MLA;
3050 if (auto MAD = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
3051 Intrinsic::aarch64_sve_mad>(
3052 IC, II, MergeIntoAddendOp: false))
3053 return MAD;
3054 return std::nullopt;
3055}
3056
3057static std::optional<Instruction *>
3058instCombineSVEVectorFAdd(InstCombiner &IC, IntrinsicInst &II) {
3059 if (auto FMLA =
3060 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3061 Intrinsic::aarch64_sve_fmla>(IC, II,
3062 MergeIntoAddendOp: true))
3063 return FMLA;
3064 if (auto FMAD =
3065 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3066 Intrinsic::aarch64_sve_fmad>(IC, II,
3067 MergeIntoAddendOp: false))
3068 return FMAD;
3069 if (auto FMLA =
3070 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
3071 Intrinsic::aarch64_sve_fmla>(IC, II,
3072 MergeIntoAddendOp: true))
3073 return FMLA;
3074 return std::nullopt;
3075}
3076
3077static std::optional<Instruction *>
3078instCombineSVEVectorFAddU(InstCombiner &IC, IntrinsicInst &II) {
3079 if (auto FMLA =
3080 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3081 Intrinsic::aarch64_sve_fmla>(IC, II,
3082 MergeIntoAddendOp: true))
3083 return FMLA;
3084 if (auto FMAD =
3085 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3086 Intrinsic::aarch64_sve_fmad>(IC, II,
3087 MergeIntoAddendOp: false))
3088 return FMAD;
3089 if (auto FMLA_U =
3090 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
3091 Intrinsic::aarch64_sve_fmla_u>(
3092 IC, II, MergeIntoAddendOp: true))
3093 return FMLA_U;
3094 return instCombineSVEVectorBinOp(IC, II);
3095}
3096
3097static std::optional<Instruction *>
3098instCombineSVEVectorFSub(InstCombiner &IC, IntrinsicInst &II) {
3099 if (auto FMLS =
3100 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3101 Intrinsic::aarch64_sve_fmls>(IC, II,
3102 MergeIntoAddendOp: true))
3103 return FMLS;
3104 if (auto FMSB =
3105 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3106 Intrinsic::aarch64_sve_fnmsb>(
3107 IC, II, MergeIntoAddendOp: false))
3108 return FMSB;
3109 if (auto FMLS =
3110 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
3111 Intrinsic::aarch64_sve_fmls>(IC, II,
3112 MergeIntoAddendOp: true))
3113 return FMLS;
3114 return std::nullopt;
3115}
3116
3117static std::optional<Instruction *>
3118instCombineSVEVectorFSubU(InstCombiner &IC, IntrinsicInst &II) {
3119 if (auto FMLS =
3120 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3121 Intrinsic::aarch64_sve_fmls>(IC, II,
3122 MergeIntoAddendOp: true))
3123 return FMLS;
3124 if (auto FMSB =
3125 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul,
3126 Intrinsic::aarch64_sve_fnmsb>(
3127 IC, II, MergeIntoAddendOp: false))
3128 return FMSB;
3129 if (auto FMLS_U =
3130 instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul_u,
3131 Intrinsic::aarch64_sve_fmls_u>(
3132 IC, II, MergeIntoAddendOp: true))
3133 return FMLS_U;
3134 return instCombineSVEVectorBinOp(IC, II);
3135}
3136
3137static std::optional<Instruction *> instCombineSVEVectorSub(InstCombiner &IC,
3138 IntrinsicInst &II) {
3139 if (auto MLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul,
3140 Intrinsic::aarch64_sve_mls>(
3141 IC, II, MergeIntoAddendOp: true))
3142 return MLS;
3143 return std::nullopt;
3144}
3145
3146static std::optional<Instruction *> instCombineSVEUnpack(InstCombiner &IC,
3147 IntrinsicInst &II) {
3148 Value *UnpackArg = II.getArgOperand(i: 0);
3149 auto *RetTy = cast<ScalableVectorType>(Val: II.getType());
3150 bool IsSigned = II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpkhi ||
3151 II.getIntrinsicID() == Intrinsic::aarch64_sve_sunpklo;
3152
3153 // Hi = uunpkhi(splat(X)) --> Hi = splat(extend(X))
3154 // Lo = uunpklo(splat(X)) --> Lo = splat(extend(X))
3155 if (auto *ScalarArg = getSplatValue(V: UnpackArg)) {
3156 ScalarArg =
3157 IC.Builder.CreateIntCast(V: ScalarArg, DestTy: RetTy->getScalarType(), isSigned: IsSigned);
3158 Value *NewVal =
3159 IC.Builder.CreateVectorSplat(EC: RetTy->getElementCount(), V: ScalarArg);
3160 NewVal->takeName(V: &II);
3161 return IC.replaceInstUsesWith(I&: II, V: NewVal);
3162 }
3163
3164 return std::nullopt;
3165}
3166static std::optional<Instruction *> instCombineSVETBL(InstCombiner &IC,
3167 IntrinsicInst &II) {
3168 auto *OpVal = II.getOperand(i_nocapture: 0);
3169 auto *OpIndices = II.getOperand(i_nocapture: 1);
3170 VectorType *VTy = cast<VectorType>(Val: II.getType());
3171
3172 // Check whether OpIndices is a constant splat value < minimal element count
3173 // of result.
3174 auto *SplatValue = dyn_cast_or_null<ConstantInt>(Val: getSplatValue(V: OpIndices));
3175 if (!SplatValue ||
3176 SplatValue->getValue().uge(RHS: VTy->getElementCount().getKnownMinValue()))
3177 return std::nullopt;
3178
3179 // Convert sve_tbl(OpVal sve_dup_x(SplatValue)) to
3180 // splat_vector(extractelement(OpVal, SplatValue)) for further optimization.
3181 auto *Extract = IC.Builder.CreateExtractElement(Vec: OpVal, Idx: SplatValue);
3182 auto *VectorSplat =
3183 IC.Builder.CreateVectorSplat(EC: VTy->getElementCount(), V: Extract);
3184
3185 VectorSplat->takeName(V: &II);
3186 return IC.replaceInstUsesWith(I&: II, V: VectorSplat);
3187}
3188
3189static std::optional<Instruction *> instCombineSVEUzp1(InstCombiner &IC,
3190 IntrinsicInst &II) {
3191 Value *A, *B;
3192 Type *RetTy = II.getType();
3193 constexpr Intrinsic::ID FromSVB = Intrinsic::aarch64_sve_convert_from_svbool;
3194 constexpr Intrinsic::ID ToSVB = Intrinsic::aarch64_sve_convert_to_svbool;
3195
3196 // uzp1(to_svbool(A), to_svbool(B)) --> <A, B>
3197 // uzp1(from_svbool(to_svbool(A)), from_svbool(to_svbool(B))) --> <A, B>
3198 if ((match(V: II.getArgOperand(i: 0),
3199 P: m_Intrinsic<FromSVB>(Ops: m_Intrinsic<ToSVB>(Ops: m_Value(V&: A)))) &&
3200 match(V: II.getArgOperand(i: 1),
3201 P: m_Intrinsic<FromSVB>(Ops: m_Intrinsic<ToSVB>(Ops: m_Value(V&: B))))) ||
3202 (match(V: II.getArgOperand(i: 0), P: m_Intrinsic<ToSVB>(Ops: m_Value(V&: A))) &&
3203 match(V: II.getArgOperand(i: 1), P: m_Intrinsic<ToSVB>(Ops: m_Value(V&: B))))) {
3204 auto *TyA = cast<ScalableVectorType>(Val: A->getType());
3205 if (TyA == B->getType() &&
3206 RetTy == ScalableVectorType::getDoubleElementsVectorType(VTy: TyA)) {
3207 auto *SubVec = IC.Builder.CreateInsertVector(
3208 DstType: RetTy, SrcVec: PoisonValue::get(T: RetTy), SubVec: A, Idx: uint64_t(0));
3209 auto *ConcatVec = IC.Builder.CreateInsertVector(DstType: RetTy, SrcVec: SubVec, SubVec: B,
3210 Idx: TyA->getMinNumElements());
3211 ConcatVec->takeName(V: &II);
3212 return IC.replaceInstUsesWith(I&: II, V: ConcatVec);
3213 }
3214 }
3215
3216 return std::nullopt;
3217}
3218
3219static std::optional<Instruction *> instCombineSVEZip(InstCombiner &IC,
3220 IntrinsicInst &II) {
3221 // zip1(uzp1(A, B), uzp2(A, B)) --> A
3222 // zip2(uzp1(A, B), uzp2(A, B)) --> B
3223 Value *A, *B;
3224 if (match(V: II.getArgOperand(i: 0),
3225 P: m_Intrinsic<Intrinsic::aarch64_sve_uzp1>(Ops: m_Value(V&: A), Ops: m_Value(V&: B))) &&
3226 match(V: II.getArgOperand(i: 1), P: m_Intrinsic<Intrinsic::aarch64_sve_uzp2>(
3227 Ops: m_Specific(V: A), Ops: m_Specific(V: B))))
3228 return IC.replaceInstUsesWith(
3229 I&: II, V: (II.getIntrinsicID() == Intrinsic::aarch64_sve_zip1 ? A : B));
3230
3231 return std::nullopt;
3232}
3233
3234static std::optional<Instruction *>
3235instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) {
3236 Value *Mask = II.getOperand(i_nocapture: 0);
3237 Value *BasePtr = II.getOperand(i_nocapture: 1);
3238 Value *Index = II.getOperand(i_nocapture: 2);
3239 Type *Ty = II.getType();
3240 Value *PassThru = ConstantAggregateZero::get(Ty);
3241
3242 // Contiguous gather => masked load.
3243 // (sve.ld1.gather.index Mask BasePtr (sve.index IndexBase 1))
3244 // => (masked.load (gep BasePtr IndexBase) Align Mask zeroinitializer)
3245 Value *IndexBase;
3246 if (match(V: Index, P: m_Intrinsic<Intrinsic::aarch64_sve_index>(Ops: m_Value(V&: IndexBase),
3247 Ops: m_One()))) {
3248 Align Alignment =
3249 BasePtr->getPointerAlignment(DL: II.getDataLayout());
3250
3251 Value *Ptr = IC.Builder.CreateGEP(Ty: cast<VectorType>(Val: Ty)->getElementType(),
3252 Ptr: BasePtr, IdxList: IndexBase);
3253 CallInst *MaskedLoad =
3254 IC.Builder.CreateMaskedLoad(Ty, Ptr, Alignment, Mask, PassThru);
3255 MaskedLoad->takeName(V: &II);
3256 return IC.replaceInstUsesWith(I&: II, V: MaskedLoad);
3257 }
3258
3259 return std::nullopt;
3260}
3261
3262static std::optional<Instruction *>
3263instCombineST1ScatterIndex(InstCombiner &IC, IntrinsicInst &II) {
3264 Value *Val = II.getOperand(i_nocapture: 0);
3265 Value *Mask = II.getOperand(i_nocapture: 1);
3266 Value *BasePtr = II.getOperand(i_nocapture: 2);
3267 Value *Index = II.getOperand(i_nocapture: 3);
3268 Type *Ty = Val->getType();
3269
3270 // Contiguous scatter => masked store.
3271 // (sve.st1.scatter.index Value Mask BasePtr (sve.index IndexBase 1))
3272 // => (masked.store Value (gep BasePtr IndexBase) Align Mask)
3273 Value *IndexBase;
3274 if (match(V: Index, P: m_Intrinsic<Intrinsic::aarch64_sve_index>(Ops: m_Value(V&: IndexBase),
3275 Ops: m_One()))) {
3276 Align Alignment =
3277 BasePtr->getPointerAlignment(DL: II.getDataLayout());
3278
3279 Value *Ptr = IC.Builder.CreateGEP(Ty: cast<VectorType>(Val: Ty)->getElementType(),
3280 Ptr: BasePtr, IdxList: IndexBase);
3281 (void)IC.Builder.CreateMaskedStore(Val, Ptr, Alignment, Mask);
3282
3283 return IC.eraseInstFromFunction(I&: II);
3284 }
3285
3286 return std::nullopt;
3287}
3288
3289static std::optional<Instruction *> instCombineSVESDIV(InstCombiner &IC,
3290 IntrinsicInst &II) {
3291 Type *Int32Ty = IC.Builder.getInt32Ty();
3292 Value *Pred = II.getOperand(i_nocapture: 0);
3293 Value *Vec = II.getOperand(i_nocapture: 1);
3294 Value *DivVec = II.getOperand(i_nocapture: 2);
3295
3296 Value *SplatValue = getSplatValue(V: DivVec);
3297 ConstantInt *SplatConstantInt = dyn_cast_or_null<ConstantInt>(Val: SplatValue);
3298 if (!SplatConstantInt)
3299 return std::nullopt;
3300
3301 APInt Divisor = SplatConstantInt->getValue();
3302 const int64_t DivisorValue = Divisor.getSExtValue();
3303 if (DivisorValue == -1)
3304 return std::nullopt;
3305 if (DivisorValue == 1)
3306 IC.replaceInstUsesWith(I&: II, V: Vec);
3307
3308 if (Divisor.isPowerOf2()) {
3309 Constant *DivisorLog2 = ConstantInt::get(Ty: Int32Ty, V: Divisor.logBase2());
3310 auto ASRD = IC.Builder.CreateIntrinsic(
3311 ID: Intrinsic::aarch64_sve_asrd, OverloadTypes: {II.getType()}, Args: {Pred, Vec, DivisorLog2});
3312 return IC.replaceInstUsesWith(I&: II, V: ASRD);
3313 }
3314 if (Divisor.isNegatedPowerOf2()) {
3315 Divisor.negate();
3316 Constant *DivisorLog2 = ConstantInt::get(Ty: Int32Ty, V: Divisor.logBase2());
3317 auto ASRD = IC.Builder.CreateIntrinsic(
3318 ID: Intrinsic::aarch64_sve_asrd, OverloadTypes: {II.getType()}, Args: {Pred, Vec, DivisorLog2});
3319 auto NEG = IC.Builder.CreateIntrinsic(
3320 ID: Intrinsic::aarch64_sve_neg, OverloadTypes: {ASRD->getType()}, Args: {ASRD, Pred, ASRD});
3321 return IC.replaceInstUsesWith(I&: II, V: NEG);
3322 }
3323
3324 return std::nullopt;
3325}
3326
3327/// Match Or(And(MASK, A), And(Not(MASK), B)), which ISel selects as
3328/// BSL/BIT/BIF. Both Ands must have one use and be in the Or's block. MainAnd,
3329///
3330/// OtherAnd, A and B are out parameters with the corresponding instructions.
3331/// MainAnd refers to And that has Not as one of its operands.
3332static bool matchBSL(const Instruction *I, Instruction *&MainAnd,
3333 Instruction *&OtherAnd, Value *&A, Value *&B) {
3334 Value *MaskValue;
3335 if (!match(V: I,
3336 P: m_c_Or(L: m_OneUse(SubPattern: m_Instruction(I&: OtherAnd)),
3337 R: m_OneUse(SubPattern: m_c_And(L: m_Not(V: m_Value(V&: MaskValue)), R: m_Value(V&: A))))))
3338 return false;
3339
3340 if (!match(V: OtherAnd, P: m_c_And(L: m_Specific(V: MaskValue), R: m_Value(V&: B))))
3341 return false;
3342
3343 MainAnd = I->getOperand(i: 0) == OtherAnd ? cast<Instruction>(Val: I->getOperand(i: 1))
3344 : cast<Instruction>(Val: I->getOperand(i: 0));
3345
3346 // Both Ands should be in same basic block as Or
3347 if (I->getParent() != MainAnd->getParent() ||
3348 I->getParent() != OtherAnd->getParent())
3349 return false;
3350
3351 return true;
3352}
3353
3354bool SimplifyValuePattern(SmallVector<Value *> &Vec, bool AllowPoison) {
3355 size_t VecSize = Vec.size();
3356 if (VecSize == 1)
3357 return true;
3358 if (!isPowerOf2_64(Value: VecSize))
3359 return false;
3360 size_t HalfVecSize = VecSize / 2;
3361
3362 for (auto LHS = Vec.begin(), RHS = Vec.begin() + HalfVecSize;
3363 RHS != Vec.end(); LHS++, RHS++) {
3364 if (*LHS != nullptr && *RHS != nullptr) {
3365 if (*LHS == *RHS)
3366 continue;
3367 else
3368 return false;
3369 }
3370 if (!AllowPoison)
3371 return false;
3372 if (*LHS == nullptr && *RHS != nullptr)
3373 *LHS = *RHS;
3374 }
3375
3376 Vec.resize(N: HalfVecSize);
3377 SimplifyValuePattern(Vec, AllowPoison);
3378 return true;
3379}
3380
3381// Try to simplify dupqlane patterns like dupqlane(f32 A, f32 B, f32 A, f32 B)
3382// to dupqlane(f64(C)) where C is A concatenated with B
3383static std::optional<Instruction *> instCombineSVEDupqLane(InstCombiner &IC,
3384 IntrinsicInst &II) {
3385 Value *CurrentInsertElt = nullptr, *Default = nullptr;
3386 if (!match(V: II.getOperand(i_nocapture: 0),
3387 P: m_Intrinsic<Intrinsic::vector_insert>(
3388 Ops: m_Value(V&: Default), Ops: m_Value(V&: CurrentInsertElt), Ops: m_Value())) ||
3389 !isa<FixedVectorType>(Val: CurrentInsertElt->getType()))
3390 return std::nullopt;
3391 auto IIScalableTy = cast<ScalableVectorType>(Val: II.getType());
3392
3393 // Insert the scalars into a container ordered by InsertElement index
3394 SmallVector<Value *> Elts(IIScalableTy->getMinNumElements(), nullptr);
3395 while (auto InsertElt = dyn_cast<InsertElementInst>(Val: CurrentInsertElt)) {
3396 auto Idx = cast<ConstantInt>(Val: InsertElt->getOperand(i_nocapture: 2));
3397 Elts[Idx->getValue().getZExtValue()] = InsertElt->getOperand(i_nocapture: 1);
3398 CurrentInsertElt = InsertElt->getOperand(i_nocapture: 0);
3399 }
3400
3401 bool AllowPoison =
3402 isa<PoisonValue>(Val: CurrentInsertElt) && isa<PoisonValue>(Val: Default);
3403 if (!SimplifyValuePattern(Vec&: Elts, AllowPoison))
3404 return std::nullopt;
3405
3406 // Rebuild the simplified chain of InsertElements. e.g. (a, b, a, b) as (a, b)
3407 Value *InsertEltChain = PoisonValue::get(T: CurrentInsertElt->getType());
3408 for (size_t I = 0; I < Elts.size(); I++) {
3409 if (Elts[I] == nullptr)
3410 continue;
3411 InsertEltChain = IC.Builder.CreateInsertElement(Vec: InsertEltChain, NewElt: Elts[I],
3412 Idx: IC.Builder.getInt64(C: I));
3413 }
3414 if (InsertEltChain == nullptr)
3415 return std::nullopt;
3416
3417 // Splat the simplified sequence, e.g. (f16 a, f16 b, f16 c, f16 d) as one i64
3418 // value or (f16 a, f16 b) as one i32 value. This requires an InsertSubvector
3419 // be bitcast to a type wide enough to fit the sequence, be splatted, and then
3420 // be narrowed back to the original type.
3421 unsigned PatternWidth = IIScalableTy->getScalarSizeInBits() * Elts.size();
3422 unsigned PatternElementCount = IIScalableTy->getScalarSizeInBits() *
3423 IIScalableTy->getMinNumElements() /
3424 PatternWidth;
3425
3426 IntegerType *WideTy = IC.Builder.getIntNTy(N: PatternWidth);
3427 auto *WideScalableTy = ScalableVectorType::get(ElementType: WideTy, MinNumElts: PatternElementCount);
3428 auto *WideShuffleMaskTy =
3429 ScalableVectorType::get(ElementType: IC.Builder.getInt32Ty(), MinNumElts: PatternElementCount);
3430
3431 auto InsertSubvector = IC.Builder.CreateInsertVector(
3432 DstType: II.getType(), SrcVec: PoisonValue::get(T: II.getType()), SubVec: InsertEltChain,
3433 Idx: uint64_t(0));
3434 auto WideBitcast =
3435 IC.Builder.CreateBitOrPointerCast(V: InsertSubvector, DestTy: WideScalableTy);
3436 auto WideShuffleMask = ConstantAggregateZero::get(Ty: WideShuffleMaskTy);
3437 auto WideShuffle = IC.Builder.CreateShuffleVector(
3438 V1: WideBitcast, V2: PoisonValue::get(T: WideScalableTy), Mask: WideShuffleMask);
3439 auto NarrowBitcast =
3440 IC.Builder.CreateBitOrPointerCast(V: WideShuffle, DestTy: II.getType());
3441
3442 return IC.replaceInstUsesWith(I&: II, V: NarrowBitcast);
3443}
3444
3445static std::optional<Instruction *> instCombineMaxMinNM(InstCombiner &IC,
3446 IntrinsicInst &II) {
3447 Value *A = II.getArgOperand(i: 0);
3448 Value *B = II.getArgOperand(i: 1);
3449 if (A == B)
3450 return IC.replaceInstUsesWith(I&: II, V: A);
3451
3452 return std::nullopt;
3453}
3454
3455static std::optional<Instruction *> instCombineSVESrshl(InstCombiner &IC,
3456 IntrinsicInst &II) {
3457 Value *Pred = II.getOperand(i_nocapture: 0);
3458 Value *Vec = II.getOperand(i_nocapture: 1);
3459 Value *Shift = II.getOperand(i_nocapture: 2);
3460
3461 // Convert SRSHL into the simpler LSL intrinsic when fed by an ABS intrinsic.
3462 Value *AbsPred, *MergedValue;
3463 if (!match(V: Vec, P: m_Intrinsic<Intrinsic::aarch64_sve_sqabs>(
3464 Ops: m_Value(V&: MergedValue), Ops: m_Value(V&: AbsPred), Ops: m_Value())) &&
3465 !match(V: Vec, P: m_Intrinsic<Intrinsic::aarch64_sve_abs>(
3466 Ops: m_Value(V&: MergedValue), Ops: m_Value(V&: AbsPred), Ops: m_Value())))
3467
3468 return std::nullopt;
3469
3470 // Transform is valid if any of the following are true:
3471 // * The ABS merge value is an undef or non-negative
3472 // * The ABS predicate is all active
3473 // * The ABS predicate and the SRSHL predicates are the same
3474 if (!isa<UndefValue>(Val: MergedValue) && !match(V: MergedValue, P: m_NonNegative()) &&
3475 AbsPred != Pred && !isAllActivePredicate(Pred: AbsPred))
3476 return std::nullopt;
3477
3478 // Only valid when the shift amount is non-negative, otherwise the rounding
3479 // behaviour of SRSHL cannot be ignored.
3480 if (!match(V: Shift, P: m_NonNegative()))
3481 return std::nullopt;
3482
3483 auto LSL = IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_lsl,
3484 OverloadTypes: {II.getType()}, Args: {Pred, Vec, Shift});
3485
3486 return IC.replaceInstUsesWith(I&: II, V: LSL);
3487}
3488
3489static std::optional<Instruction *> instCombineSVEInsr(InstCombiner &IC,
3490 IntrinsicInst &II) {
3491 Value *Vec = II.getOperand(i_nocapture: 0);
3492
3493 if (getSplatValue(V: Vec) == II.getOperand(i_nocapture: 1))
3494 return IC.replaceInstUsesWith(I&: II, V: Vec);
3495
3496 return std::nullopt;
3497}
3498
3499static std::optional<Instruction *>
3500instCombineDMB(InstCombiner &IC, IntrinsicInst &II,
3501 unsigned LookaheadThreshold) {
3502 // If this barrier is post-dominated by identical one we can remove it
3503 auto *NI = II.getNextNode();
3504 auto CanSkipOver = [](Instruction *I) {
3505 return !I->mayReadOrWriteMemory() && !I->mayHaveSideEffects();
3506 };
3507 while (LookaheadThreshold-- && CanSkipOver(NI)) {
3508 auto *NIBB = NI->getParent();
3509 NI = NI->getNextNode();
3510 if (!NI) {
3511 if (auto *SuccBB = NIBB->getUniqueSuccessor())
3512 NI = &*SuccBB->getFirstNonPHIOrLifetime();
3513 else
3514 break;
3515 }
3516 }
3517 auto *NextII = dyn_cast_or_null<IntrinsicInst>(Val: NI);
3518 if (NextII && II.isIdenticalTo(I: NextII))
3519 return IC.eraseInstFromFunction(I&: II);
3520
3521 return std::nullopt;
3522}
3523
3524static std::optional<Instruction *> instCombineWhilelo(InstCombiner &IC,
3525 IntrinsicInst &II) {
3526 return IC.replaceInstUsesWith(
3527 I&: II,
3528 V: IC.Builder.CreateIntrinsic(ID: Intrinsic::get_active_lane_mask,
3529 OverloadTypes: {II.getType(), II.getOperand(i_nocapture: 0)->getType()},
3530 Args: {II.getOperand(i_nocapture: 0), II.getOperand(i_nocapture: 1)}));
3531}
3532
3533static std::optional<Instruction *> instCombinePTrue(InstCombiner &IC,
3534 IntrinsicInst &II) {
3535 unsigned PredPattern = cast<ConstantInt>(Val: II.getOperand(i_nocapture: 0))->getZExtValue();
3536 // SVE vector length is a power-of-two, thus pow2 is synonymous with all.
3537 if (PredPattern == AArch64SVEPredPattern::all ||
3538 PredPattern == AArch64SVEPredPattern::pow2)
3539 return IC.replaceInstUsesWith(I&: II, V: ConstantInt::getTrue(Ty: II.getType()));
3540 return std::nullopt;
3541}
3542
3543static std::optional<Instruction *> instCombineSVEUxt(InstCombiner &IC,
3544 IntrinsicInst &II,
3545 unsigned NumBits) {
3546 Value *Passthru = II.getOperand(i_nocapture: 0);
3547 Value *Pg = II.getOperand(i_nocapture: 1);
3548 Value *Op = II.getOperand(i_nocapture: 2);
3549
3550 // Convert UXT[BHW] to AND.
3551 if (isa<UndefValue>(Val: Passthru) || isAllActivePredicate(Pred: Pg)) {
3552 auto *Ty = cast<VectorType>(Val: II.getType());
3553 auto MaskValue = APInt::getLowBitsSet(numBits: Ty->getScalarSizeInBits(), loBitsSet: NumBits);
3554 auto *Mask = ConstantInt::get(Ty, V: MaskValue);
3555 auto *And = IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_and_u, OverloadTypes: {Ty},
3556 Args: {Pg, Op, Mask});
3557 return IC.replaceInstUsesWith(I&: II, V: And);
3558 }
3559
3560 return std::nullopt;
3561}
3562
3563static std::optional<Instruction *>
3564instCombineInStreamingMode(InstCombiner &IC, IntrinsicInst &II) {
3565 SMEAttrs FnSMEAttrs(*II.getFunction());
3566 bool IsStreaming = FnSMEAttrs.hasStreamingInterfaceOrBody();
3567 if (IsStreaming || !FnSMEAttrs.hasStreamingCompatibleInterface())
3568 return IC.replaceInstUsesWith(
3569 I&: II, V: ConstantInt::getBool(Ty: II.getType(), V: IsStreaming));
3570 return std::nullopt;
3571}
3572
3573static std::optional<Instruction *> instCombineSVEUMin(InstCombiner &IC,
3574 IntrinsicInst &II) {
3575 // umin(umin(A, 1), umin(B, 1)) -> umin(umin(A,B), 1)
3576 constexpr Intrinsic::ID UMinID = Intrinsic::aarch64_sve_umin_u;
3577 Value *A, *B;
3578 Value *Pg = II.getOperand(i_nocapture: 0);
3579 if (match(V: II.getOperand(i_nocapture: 1), P: m_OneUse(SubPattern: m_Intrinsic<UMinID>(
3580 Ops: m_Specific(V: Pg), Ops: m_Value(V&: A), Ops: m_One()))) &&
3581 match(V: II.getOperand(i_nocapture: 2), P: m_OneUse(SubPattern: m_Intrinsic<UMinID>(
3582 Ops: m_Specific(V: Pg), Ops: m_Value(V&: B), Ops: m_One())))) {
3583 Value *NewUMin =
3584 IC.Builder.CreateIntrinsic(ID: UMinID, OverloadTypes: II.getType(), Args: {Pg, A, B});
3585 Value *NewLogicalUMin = IC.Builder.CreateIntrinsic(
3586 ID: UMinID, OverloadTypes: II.getType(), Args: {Pg, NewUMin, ConstantInt::get(Ty: II.getType(), V: 1)});
3587 return IC.replaceInstUsesWith(I&: II, V: NewLogicalUMin);
3588 }
3589
3590 // umin(umin(A, 1), 1) -> umin(A, 1)
3591 if (match(V: II.getOperand(i_nocapture: 1),
3592 P: m_Intrinsic<UMinID>(Ops: m_Specific(V: Pg), Ops: m_Value(), Ops: m_One())) &&
3593 match(V: II.getOperand(i_nocapture: 2), P: m_One()))
3594 return IC.replaceInstUsesWith(I&: II, V: II.getOperand(i_nocapture: 1));
3595
3596 return std::nullopt;
3597}
3598
3599static std::optional<Instruction *> instCombineSVEOrr(InstCombiner &IC,
3600 IntrinsicInst &II) {
3601 // orr(umin(A, 1), umin(B, 1)) -> umin(orr(A, B), 1)
3602 constexpr Intrinsic::ID UMinID = Intrinsic::aarch64_sve_umin_u;
3603 Value *Pg = II.getOperand(i_nocapture: 0);
3604
3605 Value *A, *B;
3606 if (!match(V: II.getOperand(i_nocapture: 1), P: m_OneUse(SubPattern: m_Intrinsic<UMinID>(
3607 Ops: m_Specific(V: Pg), Ops: m_Value(V&: A), Ops: m_One()))) ||
3608 !match(V: II.getOperand(i_nocapture: 2), P: m_OneUse(SubPattern: m_Intrinsic<UMinID>(
3609 Ops: m_Specific(V: Pg), Ops: m_Value(V&: B), Ops: m_One()))))
3610 return std::nullopt;
3611
3612 Value *NewOrr = IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_orr_u,
3613 OverloadTypes: II.getType(), Args: {Pg, A, B});
3614 Value *NewUMin = IC.Builder.CreateIntrinsic(
3615 ID: UMinID, OverloadTypes: II.getType(), Args: {Pg, NewOrr, ConstantInt::get(Ty: II.getType(), V: 1)});
3616 return IC.replaceInstUsesWith(I&: II, V: NewUMin);
3617}
3618
3619static std::optional<Instruction *> instCombineSVEAnd(InstCombiner &IC,
3620 IntrinsicInst &II) {
3621 // and(cmphs(pg, ConstA, A), cmphs(pg, A, ConstB))
3622 // ->
3623 // cmphs(pg, ConstA - ConstB, sub(pg, A, ConstB))
3624 constexpr Intrinsic::ID CmphsID = Intrinsic::aarch64_sve_cmphs;
3625 Value *Pg = II.getOperand(i_nocapture: 0);
3626 Value *LHS = II.getOperand(i_nocapture: 1);
3627 Value *RHS = II.getOperand(i_nocapture: 2);
3628
3629 Value *A, *PgLHS, *PgRHS;
3630 uint64_t ConstA, ConstB;
3631 if (!match(V: LHS, P: m_Intrinsic<CmphsID>(Ops: m_Value(V&: PgLHS), Ops: m_ConstantInt(V&: ConstA),
3632 Ops: m_Value(V&: A))) ||
3633 !match(V: RHS, P: m_Intrinsic<CmphsID>(Ops: m_Value(V&: PgRHS), Ops: m_Specific(V: A),
3634 Ops: m_ConstantInt(V&: ConstB))) ||
3635 !LHS->hasOneUser() || !RHS->hasOneUser())
3636 return std::nullopt;
3637
3638 // Always false regardless of predication
3639 if (ConstB > ConstA)
3640 return IC.replaceInstUsesWith(I&: II, V: Constant::getNullValue(Ty: II.getType()));
3641
3642 // The predicate for both CMPHSs must match.
3643 // The predicate for the AND can either be equal to the CMPHS predicates, or
3644 // either of the CMPHS values.
3645 if (PgLHS != PgRHS || (Pg != LHS && Pg != RHS && Pg != PgLHS))
3646 return std::nullopt;
3647
3648 Type *VecTy = A->getType();
3649 Constant *Base = ConstantInt::get(Ty: VecTy, V: ConstB);
3650 Value *Sub = IC.Builder.CreateIntrinsic(ID: Intrinsic::aarch64_sve_sub_u, OverloadTypes: VecTy,
3651 Args: {PgLHS, A, Base});
3652 Constant *Limit = ConstantInt::get(Ty: VecTy, V: ConstA - ConstB);
3653 Value *NewCmphs =
3654 IC.Builder.CreateIntrinsic(ID: CmphsID, OverloadTypes: VecTy, Args: {PgLHS, Limit, Sub});
3655
3656 return IC.replaceInstUsesWith(I&: II, V: NewCmphs);
3657}
3658
3659std::optional<Instruction *>
3660AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC,
3661 IntrinsicInst &II) const {
3662 const SVEIntrinsicInfo &IInfo = constructSVEIntrinsicInfo(II);
3663 if (std::optional<Instruction *> I = simplifySVEIntrinsic(IC, II, IInfo))
3664 return I;
3665
3666 Intrinsic::ID IID = II.getIntrinsicID();
3667 switch (IID) {
3668 default:
3669 break;
3670 case Intrinsic::aarch64_dmb:
3671 return instCombineDMB(IC, II, LookaheadThreshold: ST->getCLOpts().dmb_lookahead_threshold);
3672 case Intrinsic::aarch64_neon_fmaxnm:
3673 case Intrinsic::aarch64_neon_fminnm:
3674 return instCombineMaxMinNM(IC, II);
3675 case Intrinsic::aarch64_sve_convert_from_svbool:
3676 return instCombineConvertFromSVBool(IC, II);
3677 case Intrinsic::aarch64_sve_dup:
3678 return instCombineSVEDup(IC, II);
3679 case Intrinsic::aarch64_sve_dup_x:
3680 return instCombineSVEDupX(IC, II);
3681 case Intrinsic::aarch64_sve_cmpeq:
3682 case Intrinsic::aarch64_sve_cmpeq_wide:
3683 return instCombineXorSVECmpCC(IC, II);
3684 case Intrinsic::aarch64_sve_cmpne:
3685 case Intrinsic::aarch64_sve_cmpne_wide:
3686 return instCombineSVECmpNE(IC, II);
3687 case Intrinsic::aarch64_sve_rdffr:
3688 return instCombineRDFFR(IC, II);
3689 case Intrinsic::aarch64_sve_lasta:
3690 case Intrinsic::aarch64_sve_lastb:
3691 return instCombineSVELast(IC, II);
3692 case Intrinsic::aarch64_sve_clasta_n:
3693 case Intrinsic::aarch64_sve_clastb_n:
3694 return instCombineSVECondLast(IC, II);
3695 case Intrinsic::aarch64_sve_cntd:
3696 return instCombineSVECntElts(IC, II, NumElts: 2);
3697 case Intrinsic::aarch64_sve_cntw:
3698 return instCombineSVECntElts(IC, II, NumElts: 4);
3699 case Intrinsic::aarch64_sve_cnth:
3700 return instCombineSVECntElts(IC, II, NumElts: 8);
3701 case Intrinsic::aarch64_sve_cntb:
3702 return instCombineSVECntElts(IC, II, NumElts: 16);
3703 case Intrinsic::aarch64_sme_cntsd:
3704 return instCombineSMECntsd(IC, II, ST);
3705 case Intrinsic::aarch64_sve_ptest_any:
3706 case Intrinsic::aarch64_sve_ptest_first:
3707 case Intrinsic::aarch64_sve_ptest_last:
3708 return instCombineSVEPTest(IC, II);
3709 case Intrinsic::aarch64_sve_fadd:
3710 return instCombineSVEVectorFAdd(IC, II);
3711 case Intrinsic::aarch64_sve_fadd_u:
3712 return instCombineSVEVectorFAddU(IC, II);
3713 case Intrinsic::aarch64_sve_fmul_u:
3714 return instCombineSVEVectorBinOp(IC, II);
3715 case Intrinsic::aarch64_sve_fsub:
3716 return instCombineSVEVectorFSub(IC, II);
3717 case Intrinsic::aarch64_sve_fsub_u:
3718 return instCombineSVEVectorFSubU(IC, II);
3719 case Intrinsic::aarch64_sve_add:
3720 return instCombineSVEVectorAdd(IC, II);
3721 case Intrinsic::aarch64_sve_add_u:
3722 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u,
3723 Intrinsic::aarch64_sve_mla_u>(
3724 IC, II, MergeIntoAddendOp: true);
3725 case Intrinsic::aarch64_sve_mla_u:
3726 return instCombineSVEVectorMlaU(IC, II);
3727 case Intrinsic::aarch64_sve_sadalp:
3728 case Intrinsic::aarch64_sve_uadalp:
3729 return instCombineSVEPairwiseAddLong(IC, II);
3730 case Intrinsic::aarch64_sve_sub:
3731 return instCombineSVEVectorSub(IC, II);
3732 case Intrinsic::aarch64_sve_sub_u:
3733 return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u,
3734 Intrinsic::aarch64_sve_mls_u>(
3735 IC, II, MergeIntoAddendOp: true);
3736 case Intrinsic::aarch64_sve_tbl:
3737 return instCombineSVETBL(IC, II);
3738 case Intrinsic::aarch64_sve_uunpkhi:
3739 case Intrinsic::aarch64_sve_uunpklo:
3740 case Intrinsic::aarch64_sve_sunpkhi:
3741 case Intrinsic::aarch64_sve_sunpklo:
3742 return instCombineSVEUnpack(IC, II);
3743 case Intrinsic::aarch64_sve_uzp1:
3744 return instCombineSVEUzp1(IC, II);
3745 case Intrinsic::aarch64_sve_zip1:
3746 case Intrinsic::aarch64_sve_zip2:
3747 return instCombineSVEZip(IC, II);
3748 case Intrinsic::aarch64_sve_ld1_gather_index:
3749 return instCombineLD1GatherIndex(IC, II);
3750 case Intrinsic::aarch64_sve_st1_scatter_index:
3751 return instCombineST1ScatterIndex(IC, II);
3752 case Intrinsic::aarch64_sve_ld1:
3753 return instCombineSVELD1(IC, II, DL);
3754 case Intrinsic::aarch64_sve_st1:
3755 return instCombineSVEST1(IC, II, DL);
3756 case Intrinsic::aarch64_sve_sdiv:
3757 return instCombineSVESDIV(IC, II);
3758 case Intrinsic::aarch64_sve_sel:
3759 return instCombineSVESel(IC, II);
3760 case Intrinsic::aarch64_sve_srshl:
3761 return instCombineSVESrshl(IC, II);
3762 case Intrinsic::aarch64_sve_dupq_lane:
3763 return instCombineSVEDupqLane(IC, II);
3764 case Intrinsic::aarch64_sve_insr:
3765 return instCombineSVEInsr(IC, II);
3766 case Intrinsic::aarch64_sve_whilelo:
3767 return instCombineWhilelo(IC, II);
3768 case Intrinsic::aarch64_sve_ptrue:
3769 return instCombinePTrue(IC, II);
3770 case Intrinsic::aarch64_sve_uxtb:
3771 return instCombineSVEUxt(IC, II, NumBits: 8);
3772 case Intrinsic::aarch64_sve_uxth:
3773 return instCombineSVEUxt(IC, II, NumBits: 16);
3774 case Intrinsic::aarch64_sve_uxtw:
3775 return instCombineSVEUxt(IC, II, NumBits: 32);
3776 case Intrinsic::aarch64_sme_in_streaming_mode:
3777 return instCombineInStreamingMode(IC, II);
3778 case Intrinsic::aarch64_sve_umin_u:
3779 return instCombineSVEUMin(IC, II);
3780 case Intrinsic::aarch64_sve_orr_u:
3781 return instCombineSVEOrr(IC, II);
3782 case Intrinsic::aarch64_sve_and_z:
3783 return instCombineSVEAnd(IC, II);
3784 }
3785
3786 return std::nullopt;
3787}
3788
3789std::optional<Value *> AArch64TTIImpl::simplifyDemandedVectorEltsIntrinsic(
3790 InstCombiner &IC, IntrinsicInst &II, APInt OrigDemandedElts,
3791 APInt &UndefElts, APInt &UndefElts2, APInt &UndefElts3,
3792 std::function<void(Instruction *, unsigned, APInt, APInt &)>
3793 SimplifyAndSetOp) const {
3794 switch (II.getIntrinsicID()) {
3795 default:
3796 break;
3797 case Intrinsic::aarch64_neon_fcvtxn:
3798 case Intrinsic::aarch64_neon_rshrn:
3799 case Intrinsic::aarch64_neon_sqrshrn:
3800 case Intrinsic::aarch64_neon_sqrshrun:
3801 case Intrinsic::aarch64_neon_sqshrn:
3802 case Intrinsic::aarch64_neon_sqshrun:
3803 case Intrinsic::aarch64_neon_sqxtn:
3804 case Intrinsic::aarch64_neon_sqxtun:
3805 case Intrinsic::aarch64_neon_uqrshrn:
3806 case Intrinsic::aarch64_neon_uqshrn:
3807 case Intrinsic::aarch64_neon_uqxtn:
3808 SimplifyAndSetOp(&II, 0, OrigDemandedElts, UndefElts);
3809 break;
3810 }
3811
3812 return std::nullopt;
3813}
3814
3815bool AArch64TTIImpl::enableScalableVectorization() const {
3816 return ST->isSVEAvailable() ||
3817 (ST->isSVEorStreamingSVEAvailable() &&
3818 ST->getCLOpts().enable_scalable_autovec_in_streaming_mode);
3819}
3820
3821TypeSize
3822AArch64TTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
3823 switch (K) {
3824 case TargetTransformInfo::RGK_Scalar:
3825 return TypeSize::getFixed(ExactSize: 64);
3826 case TargetTransformInfo::RGK_FixedWidthVector:
3827 if (ST->useSVEForFixedLengthVectors() &&
3828 (ST->isSVEAvailable() ||
3829 ST->getCLOpts().enable_fixedwidth_autovec_in_streaming_mode))
3830 return TypeSize::getFixed(
3831 ExactSize: std::max(a: ST->getMinSVEVectorSizeInBits(), b: 128u));
3832 else if (ST->isNeonAvailable())
3833 return TypeSize::getFixed(ExactSize: 128);
3834 else
3835 return TypeSize::getFixed(ExactSize: 0);
3836 case TargetTransformInfo::RGK_ScalableVector:
3837 if (ST->isSVEAvailable() ||
3838 (ST->isSVEorStreamingSVEAvailable() &&
3839 ST->getCLOpts().enable_scalable_autovec_in_streaming_mode))
3840 return TypeSize::getScalable(MinimumSize: 128);
3841 else
3842 return TypeSize::getScalable(MinimumSize: 0);
3843 }
3844 llvm_unreachable("Unsupported register kind");
3845}
3846
3847bool AArch64TTIImpl::isSingleExtWideningInstruction(
3848 unsigned Opcode, Type *DstTy, ArrayRef<const Value *> Args,
3849 Type *SrcOverrideTy) const {
3850 // A helper that returns a vector type from the given type. The number of
3851 // elements in type Ty determines the vector width.
3852 auto toVectorTy = [&](Type *ArgTy) {
3853 return VectorType::get(ElementType: ArgTy->getScalarType(),
3854 EC: cast<VectorType>(Val: DstTy)->getElementCount());
3855 };
3856
3857 // Exit early if DstTy is not a vector type whose elements are one of [i16,
3858 // i32, i64]. SVE doesn't generally have the same set of instructions to
3859 // perform an extend with the add/sub/mul. There are SMULLB style
3860 // instructions, but they operate on top/bottom, requiring some sort of lane
3861 // interleaving to be used with zext/sext.
3862 unsigned DstEltSize = DstTy->getScalarSizeInBits();
3863 if (!useNeonVector(Ty: DstTy) || Args.size() != 2 ||
3864 (DstEltSize != 16 && DstEltSize != 32 && DstEltSize != 64))
3865 return false;
3866
3867 Type *SrcTy = SrcOverrideTy;
3868 switch (Opcode) {
3869 case Instruction::Add: // UADDW(2), SADDW(2).
3870 case Instruction::Sub: { // USUBW(2), SSUBW(2).
3871 // The second operand needs to be an extend
3872 if (isa<SExtInst>(Val: Args[1]) || isa<ZExtInst>(Val: Args[1])) {
3873 if (!SrcTy)
3874 SrcTy =
3875 toVectorTy(cast<Instruction>(Val: Args[1])->getOperand(i: 0)->getType());
3876 break;
3877 }
3878
3879 if (Opcode == Instruction::Sub)
3880 return false;
3881
3882 // UADDW(2), SADDW(2) can be commutted.
3883 if (isa<SExtInst>(Val: Args[0]) || isa<ZExtInst>(Val: Args[0])) {
3884 if (!SrcTy)
3885 SrcTy =
3886 toVectorTy(cast<Instruction>(Val: Args[0])->getOperand(i: 0)->getType());
3887 break;
3888 }
3889 return false;
3890 }
3891 default:
3892 return false;
3893 }
3894
3895 // Legalize the destination type and ensure it can be used in a widening
3896 // operation.
3897 auto DstTyL = getTypeLegalizationCost(Ty: DstTy);
3898 if (!DstTyL.second.isVector() || DstEltSize != DstTy->getScalarSizeInBits())
3899 return false;
3900
3901 // Legalize the source type and ensure it can be used in a widening
3902 // operation.
3903 assert(SrcTy && "Expected some SrcTy");
3904 auto SrcTyL = getTypeLegalizationCost(Ty: SrcTy);
3905 unsigned SrcElTySize = SrcTyL.second.getScalarSizeInBits();
3906 if (!SrcTyL.second.isVector() || SrcElTySize != SrcTy->getScalarSizeInBits())
3907 return false;
3908
3909 // Get the total number of vector elements in the legalized types.
3910 InstructionCost NumDstEls =
3911 DstTyL.first * DstTyL.second.getVectorMinNumElements();
3912 InstructionCost NumSrcEls =
3913 SrcTyL.first * SrcTyL.second.getVectorMinNumElements();
3914
3915 // Return true if the legalized types have the same number of vector elements
3916 // and the destination element type size is twice that of the source type.
3917 return NumDstEls == NumSrcEls && 2 * SrcElTySize == DstEltSize;
3918}
3919
3920Type *AArch64TTIImpl::isBinExtWideningInstruction(unsigned Opcode, Type *DstTy,
3921 ArrayRef<const Value *> Args,
3922 Type *SrcOverrideTy) const {
3923 if (Opcode != Instruction::Add && Opcode != Instruction::Sub &&
3924 Opcode != Instruction::Mul)
3925 return nullptr;
3926
3927 // Exit early if DstTy is not a vector type whose elements are one of [i16,
3928 // i32, i64]. SVE doesn't generally have the same set of instructions to
3929 // perform an extend with the add/sub/mul. There are SMULLB style
3930 // instructions, but they operate on top/bottom, requiring some sort of lane
3931 // interleaving to be used with zext/sext.
3932 unsigned DstEltSize = DstTy->getScalarSizeInBits();
3933 if (!useNeonVector(Ty: DstTy) || Args.size() != 2 ||
3934 (DstEltSize != 16 && DstEltSize != 32 && DstEltSize != 64))
3935 return nullptr;
3936
3937 auto getScalarSizeWithOverride = [&](const Value *V) {
3938 if (SrcOverrideTy)
3939 return SrcOverrideTy->getScalarSizeInBits();
3940 return cast<Instruction>(Val: V)
3941 ->getOperand(i: 0)
3942 ->getType()
3943 ->getScalarSizeInBits();
3944 };
3945
3946 unsigned MaxEltSize = 0;
3947 if ((isa<SExtInst>(Val: Args[0]) && isa<SExtInst>(Val: Args[1])) ||
3948 (isa<ZExtInst>(Val: Args[0]) && isa<ZExtInst>(Val: Args[1]))) {
3949 unsigned EltSize0 = getScalarSizeWithOverride(Args[0]);
3950 unsigned EltSize1 = getScalarSizeWithOverride(Args[1]);
3951 MaxEltSize = std::max(a: EltSize0, b: EltSize1);
3952 } else if (isa<SExtInst, ZExtInst>(Val: Args[0]) &&
3953 isa<SExtInst, ZExtInst>(Val: Args[1])) {
3954 unsigned EltSize0 = getScalarSizeWithOverride(Args[0]);
3955 unsigned EltSize1 = getScalarSizeWithOverride(Args[1]);
3956 // mul(sext, zext) will become smull(sext, zext) if the extends are large
3957 // enough.
3958 if (EltSize0 >= DstEltSize / 2 || EltSize1 >= DstEltSize / 2)
3959 return nullptr;
3960 MaxEltSize = DstEltSize / 2;
3961 } else if (Opcode == Instruction::Mul &&
3962 (isa<ZExtInst>(Val: Args[0]) || isa<ZExtInst>(Val: Args[1]))) {
3963 // If one of the operands is a Zext and the other has enough zero bits
3964 // to be treated as unsigned, we can still generate a umull, meaning the
3965 // zext is free.
3966 KnownBits Known =
3967 computeKnownBits(V: isa<ZExtInst>(Val: Args[0]) ? Args[1] : Args[0], DL);
3968 if (Args[0]->getType()->getScalarSizeInBits() -
3969 Known.Zero.countLeadingOnes() >
3970 DstTy->getScalarSizeInBits() / 2)
3971 return nullptr;
3972
3973 MaxEltSize =
3974 getScalarSizeWithOverride(isa<ZExtInst>(Val: Args[0]) ? Args[0] : Args[1]);
3975 } else
3976 return nullptr;
3977
3978 if (MaxEltSize * 2 > DstEltSize)
3979 return nullptr;
3980
3981 Type *ExtTy = DstTy->getWithNewBitWidth(NewBitWidth: MaxEltSize * 2);
3982 if (ExtTy->getPrimitiveSizeInBits() <= 64)
3983 return nullptr;
3984 return ExtTy;
3985}
3986
3987// s/urhadd instructions implement the following pattern, making the
3988// extends free:
3989// %x = add ((zext i8 -> i16), 1)
3990// %y = (zext i8 -> i16)
3991// trunc i16 (lshr (add %x, %y), 1) -> i8
3992//
3993bool AArch64TTIImpl::isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst,
3994 Type *Src) const {
3995 // The source should be a legal vector type.
3996 if (!Src->isVectorTy() || !TLI->isTypeLegal(VT: TLI->getValueType(DL, Ty: Src)) ||
3997 (Src->isScalableTy() && !ST->hasSVE2()))
3998 return false;
3999
4000 if (ExtUser->getOpcode() != Instruction::Add || !ExtUser->hasOneUse())
4001 return false;
4002
4003 // Look for trunc/shl/add before trying to match the pattern.
4004 const Instruction *Add = ExtUser;
4005 auto *AddUser =
4006 dyn_cast_or_null<Instruction>(Val: Add->getUniqueUndroppableUser());
4007 if (AddUser && AddUser->getOpcode() == Instruction::Add)
4008 Add = AddUser;
4009
4010 auto *Shr = dyn_cast_or_null<Instruction>(Val: Add->getUniqueUndroppableUser());
4011 if (!Shr || Shr->getOpcode() != Instruction::LShr)
4012 return false;
4013
4014 auto *Trunc = dyn_cast_or_null<Instruction>(Val: Shr->getUniqueUndroppableUser());
4015 if (!Trunc || Trunc->getOpcode() != Instruction::Trunc ||
4016 Src->getScalarSizeInBits() !=
4017 cast<CastInst>(Val: Trunc)->getDestTy()->getScalarSizeInBits())
4018 return false;
4019
4020 // Try to match the whole pattern. Ext could be either the first or second
4021 // m_ZExtOrSExt matched.
4022 Instruction *Ex1, *Ex2;
4023 if (!(match(V: Add, P: m_c_Add(L: m_Instruction(I&: Ex1),
4024 R: m_c_Add(L: m_Instruction(I&: Ex2), R: m_One())))))
4025 return false;
4026
4027 // Ensure both extends are of the same type
4028 if (match(V: Ex1, P: m_ZExtOrSExt(Op: m_Value())) &&
4029 Ex1->getOpcode() == Ex2->getOpcode())
4030 return true;
4031
4032 return false;
4033}
4034
4035InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
4036 Type *Src,
4037 TTI::CastContextHint CCH,
4038 TTI::TargetCostKind CostKind,
4039 const Instruction *I) const {
4040 int ISD = TLI->InstructionOpcodeToISD(Opcode);
4041 assert(ISD && "Invalid opcode");
4042
4043 // Codegen is not able to select all <vscale x 1 x Ty> casts yet.
4044 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: Dst);
4045 VTy && VTy->getElementCount() == ElementCount::getScalable(MinVal: 1)) {
4046 if (!is_contained(Set: {ISD::TRUNCATE, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND,
4047 ISD::BITCAST, ISD::ADDRSPACECAST},
4048 Element: ISD))
4049 return InstructionCost::getInvalid();
4050 }
4051
4052 // If the cast is observable, and it is used by a widening instruction (e.g.,
4053 // uaddl, saddw, etc.), it may be free.
4054 if (I && !I->users().empty()) {
4055 // Determine whether Usr can absorb the cast from Src to Dst into a
4056 // widening instruction (e.g. uaddl, saddw, urhadd), making the cast free
4057 // with respect to that user.
4058 auto GetUserAbsorbedCastCost =
4059 [&](const Instruction *Usr) -> std::optional<InstructionCost> {
4060 SmallVector<const Value *, 4> Operands(Usr->operand_values());
4061
4062 if (Type *ExtTy = isBinExtWideningInstruction(
4063 Opcode: Usr->getOpcode(), DstTy: Dst, Args: Operands,
4064 SrcOverrideTy: Src != I->getOperand(i: 0)->getType() ? Src : nullptr)) {
4065 // The cost from Src->Src*2 needs to be added if required, the cost
4066 // from Src*2->ExtTy is free.
4067 if (ExtTy->getScalarSizeInBits() > Src->getScalarSizeInBits() * 2) {
4068 Type *DoubleSrcTy =
4069 Src->getWithNewBitWidth(NewBitWidth: Src->getScalarSizeInBits() * 2);
4070 return getCastInstrCost(Opcode, Dst: DoubleSrcTy, Src,
4071 CCH: TTI::CastContextHint::None, CostKind);
4072 }
4073
4074 return 0;
4075 }
4076
4077 if (isSingleExtWideningInstruction(
4078 Opcode: Usr->getOpcode(), DstTy: Dst, Args: Operands,
4079 SrcOverrideTy: Src != I->getOperand(i: 0)->getType() ? Src : nullptr)) {
4080 // For adds only count the second operand as free if both operands
4081 // are extends but not the same operation. (i.e both operands are
4082 // not free in add(sext, zext)).
4083 if (Usr->getOpcode() == Instruction::Add) {
4084 if (I == Usr->getOperand(i: 1) ||
4085 (isa<CastInst>(Val: Usr->getOperand(i: 1)) &&
4086 cast<CastInst>(Val: Usr->getOperand(i: 1))->getOpcode() == Opcode))
4087 return 0;
4088 } else {
4089 // Others are free so long as isSingleExtWideningInstruction
4090 // returned true.
4091 return 0;
4092 }
4093 }
4094
4095 if (!isa<ZExtInst, SExtInst>(Val: I))
4096 return std::nullopt;
4097
4098 // The cast will be free for the s/urhadd instructions
4099 if (isExtPartOfAvgExpr(ExtUser: Usr, Dst, Src))
4100 return 0;
4101
4102 // [US]SHLL/SHLL fold a doubling extend into a shift left by a splat
4103 // immediate of at most the source element width.
4104 const APInt *ShAmt;
4105 unsigned SrcBits = Src->getScalarSizeInBits();
4106 if (match(V: Usr, P: m_Shl(L: m_Specific(V: I), R: m_APInt(Res&: ShAmt))) &&
4107 ShAmt->ule(RHS: SrcBits) && useNeonVector(Ty: Dst) &&
4108 Dst->getScalarSizeInBits() == 2 * SrcBits &&
4109 getTypeLegalizationCost(Ty: Src).second.getScalarSizeInBits() == SrcBits)
4110 return 0;
4111
4112 return std::nullopt;
4113 };
4114
4115 InstructionCost MaxAbsorbedCost = 0;
4116 bool AllUsersAbsorbCast = true;
4117 for (const User *U : I->users()) {
4118 auto *Usr = cast<Instruction>(Val: U);
4119 std::optional<InstructionCost> UserCost = GetUserAbsorbedCastCost(Usr);
4120 if (!UserCost) {
4121 AllUsersAbsorbCast = false;
4122 break;
4123 }
4124 MaxAbsorbedCost = std::max(a: MaxAbsorbedCost, b: *UserCost);
4125 }
4126
4127 if (AllUsersAbsorbCast)
4128 return MaxAbsorbedCost;
4129 }
4130
4131 // A vector compare already produces a lane mask (CMxx/FCMxx), so sign
4132 // extending it to the compare width is free. That only holds while the
4133 // result stays a vector; scalarized lanes still need a CSETM each.
4134 const Instruction *Cmp;
4135 const Value *CmpOp;
4136 if (I && useNeonVector(Ty: Dst) &&
4137 match(V: I, P: m_SExt(Op: m_Instruction(I&: Cmp, P: m_Cmp(L: m_Value(V&: CmpOp), R: m_Value())))) &&
4138 Cmp->getParent() == I->getParent() &&
4139 CmpOp->getType()->getScalarSizeInBits() == Dst->getScalarSizeInBits() &&
4140 getTypeLegalizationCost(Ty: Dst).second.isVector())
4141 return 0;
4142
4143 EVT SrcTy = TLI->getValueType(DL, Ty: Src);
4144 EVT DstTy = TLI->getValueType(DL, Ty: Dst);
4145
4146 // SVE has no elements wider than 64 bits. The generic cost for a bitcast
4147 // between unsupported scalable types can otherwise appear to be free.
4148 if ((SrcTy.isScalableVector() && SrcTy.getScalarSizeInBits() > 64) ||
4149 (DstTy.isScalableVector() && DstTy.getScalarSizeInBits() > 64))
4150 return InstructionCost::getInvalid();
4151
4152 // From a vector to a scalarized vector will be an series of extract-element
4153 // and extends.
4154 if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
4155 DstTy.getScalarSizeInBits() > 64 && SrcTy.getScalarSizeInBits() <= 64 &&
4156 DstTy.isFixedLengthVector()) {
4157 InstructionCost LaneCost = getVectorInstrCost(
4158 Opcode: Instruction::ExtractElement, Ty: Src, CostKind, Index: -1, Op0: nullptr, Op1: nullptr);
4159 InstructionCost ExtCost = getCastInstrCost(
4160 Opcode, Dst: Dst->getScalarType(), Src: Src->getScalarType(), CCH, CostKind);
4161 return DstTy.getVectorNumElements() * (LaneCost + ExtCost);
4162 }
4163
4164 if (!SrcTy.isSimple() || !DstTy.isSimple())
4165 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
4166
4167 // For the moment we do not have lowering for SVE1-only fptrunc f64->bf16 as
4168 // we use fcvtx under SVE2. Give them invalid costs.
4169 if (!ST->hasSVE2() && !ST->isStreamingSVEAvailable() &&
4170 ISD == ISD::FP_ROUND && SrcTy.isScalableVector() &&
4171 DstTy.getScalarType() == MVT::bf16 && SrcTy.getScalarType() == MVT::f64)
4172 return InstructionCost::getInvalid();
4173
4174 static const TypeConversionCostTblEntry BF16Tbl[] = {
4175 {.ISD: ISD::FP_ROUND, .Dst: MVT::bf16, .Src: MVT::f32, .Cost: 1}, // bfcvt
4176 {.ISD: ISD::FP_ROUND, .Dst: MVT::bf16, .Src: MVT::f64, .Cost: 1}, // bfcvt
4177 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4bf16, .Src: MVT::v4f32, .Cost: 1}, // bfcvtn
4178 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8bf16, .Src: MVT::v8f32, .Cost: 2}, // bfcvtn+bfcvtn2
4179 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2bf16, .Src: MVT::v2f64, .Cost: 2}, // bfcvtn+fcvtn
4180 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4bf16, .Src: MVT::v4f64, .Cost: 3}, // fcvtn+fcvtl2+bfcvtn
4181 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8bf16, .Src: MVT::v8f64, .Cost: 6}, // 2 * fcvtn+fcvtn2+bfcvtn
4182 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2bf16, .Src: MVT::nxv2f32, .Cost: 1}, // bfcvt
4183 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4bf16, .Src: MVT::nxv4f32, .Cost: 1}, // bfcvt
4184 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8bf16, .Src: MVT::nxv8f32, .Cost: 3}, // bfcvt+bfcvt+uzp1
4185 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2bf16, .Src: MVT::nxv2f64, .Cost: 2}, // fcvtx+bfcvt
4186 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4bf16, .Src: MVT::nxv4f64, .Cost: 5}, // 2*fcvtx+2*bfcvt+uzp1
4187 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8bf16, .Src: MVT::nxv8f64, .Cost: 11}, // 4*fcvt+4*bfcvt+3*uzp
4188 };
4189
4190 if (ST->hasBF16())
4191 if (const auto *Entry = ConvertCostTableLookup(
4192 Table: BF16Tbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
4193 return Entry->Cost;
4194
4195 // We have to estimate a cost of fixed length operation upon
4196 // SVE registers(operations) with the number of registers required
4197 // for a fixed type to be represented upon SVE registers.
4198 EVT WiderTy = SrcTy.bitsGT(VT: DstTy) ? SrcTy : DstTy;
4199 if (SrcTy.isFixedLengthVector() && DstTy.isFixedLengthVector() &&
4200 SrcTy.getVectorNumElements() == DstTy.getVectorNumElements() &&
4201 ST->useSVEForFixedLengthVectors(VT: WiderTy)) {
4202 std::pair<InstructionCost, MVT> LT =
4203 getTypeLegalizationCost(Ty: WiderTy.getTypeForEVT(Context&: Dst->getContext()));
4204 unsigned NumElements =
4205 AArch64::SVEBitsPerBlock / LT.second.getScalarSizeInBits();
4206 return LT.first *
4207 getCastInstrCost(
4208 Opcode,
4209 Dst: ScalableVectorType::get(ElementType: Dst->getScalarType(), MinNumElts: NumElements),
4210 Src: ScalableVectorType::get(ElementType: Src->getScalarType(), MinNumElts: NumElements), CCH,
4211 CostKind, I);
4212 }
4213
4214 // Symbolic constants for the SVE sitofp/uitofp entries in the table below
4215 // The cost of unpacking twice is artificially increased for now in order
4216 // to avoid regressions against NEON, which will use tbl instructions directly
4217 // instead of multiple layers of [s|u]unpk[lo|hi].
4218 // We use the unpacks in cases where the destination type is illegal and
4219 // requires splitting of the input, even if the input type itself is legal.
4220 const unsigned int SVE_EXT_COST = 1;
4221 const unsigned int SVE_FCVT_COST = 1;
4222 const unsigned int SVE_UNPACK_ONCE = 4;
4223 const unsigned int SVE_UNPACK_TWICE = 16;
4224
4225 static const TypeConversionCostTblEntry ConversionTbl[] = {
4226 {.ISD: ISD::TRUNCATE, .Dst: MVT::v2i8, .Src: MVT::v2i64, .Cost: 1}, // xtn
4227 {.ISD: ISD::TRUNCATE, .Dst: MVT::v2i16, .Src: MVT::v2i64, .Cost: 1}, // xtn
4228 {.ISD: ISD::TRUNCATE, .Dst: MVT::v2i32, .Src: MVT::v2i64, .Cost: 1}, // xtn
4229 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i32, .Cost: 1}, // xtn
4230 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i8, .Src: MVT::v4i64, .Cost: 3}, // 2 xtn + 1 uzp1
4231 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i32, .Cost: 1}, // xtn
4232 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i16, .Src: MVT::v4i64, .Cost: 2}, // 1 uzp1 + 1 xtn
4233 {.ISD: ISD::TRUNCATE, .Dst: MVT::v4i32, .Src: MVT::v4i64, .Cost: 1}, // 1 uzp1
4234 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i16, .Cost: 1}, // 1 xtn
4235 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i32, .Cost: 2}, // 1 uzp1 + 1 xtn
4236 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i8, .Src: MVT::v8i64, .Cost: 4}, // 3 x uzp1 + xtn
4237 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i32, .Cost: 1}, // 1 uzp1
4238 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i16, .Src: MVT::v8i64, .Cost: 3}, // 3 x uzp1
4239 {.ISD: ISD::TRUNCATE, .Dst: MVT::v8i32, .Src: MVT::v8i64, .Cost: 2}, // 2 x uzp1
4240 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i16, .Cost: 1}, // uzp1
4241 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i32, .Cost: 3}, // (2 + 1) x uzp1
4242 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i8, .Src: MVT::v16i64, .Cost: 7}, // (4 + 2 + 1) x uzp1
4243 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i32, .Cost: 2}, // 2 x uzp1
4244 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i16, .Src: MVT::v16i64, .Cost: 6}, // (4 + 2) x uzp1
4245 {.ISD: ISD::TRUNCATE, .Dst: MVT::v16i32, .Src: MVT::v16i64, .Cost: 4}, // 4 x uzp1
4246
4247 // Truncations on nxvmiN
4248 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i1, .Src: MVT::nxv2i8, .Cost: 2},
4249 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i1, .Src: MVT::nxv2i16, .Cost: 2},
4250 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i1, .Src: MVT::nxv2i32, .Cost: 2},
4251 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i1, .Src: MVT::nxv2i64, .Cost: 2},
4252 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i1, .Src: MVT::nxv4i8, .Cost: 2},
4253 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i1, .Src: MVT::nxv4i16, .Cost: 2},
4254 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i1, .Src: MVT::nxv4i32, .Cost: 2},
4255 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i1, .Src: MVT::nxv4i64, .Cost: 5},
4256 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i1, .Src: MVT::nxv8i8, .Cost: 2},
4257 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i1, .Src: MVT::nxv8i16, .Cost: 2},
4258 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i1, .Src: MVT::nxv8i32, .Cost: 5},
4259 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i1, .Src: MVT::nxv8i64, .Cost: 11},
4260 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv16i1, .Src: MVT::nxv16i8, .Cost: 2},
4261 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i8, .Src: MVT::nxv2i16, .Cost: 0},
4262 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i8, .Src: MVT::nxv2i32, .Cost: 0},
4263 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i8, .Src: MVT::nxv2i64, .Cost: 0},
4264 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i16, .Src: MVT::nxv2i32, .Cost: 0},
4265 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i16, .Src: MVT::nxv2i64, .Cost: 0},
4266 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv2i32, .Src: MVT::nxv2i64, .Cost: 0},
4267 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i8, .Src: MVT::nxv4i16, .Cost: 0},
4268 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i8, .Src: MVT::nxv4i32, .Cost: 0},
4269 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i8, .Src: MVT::nxv4i64, .Cost: 1},
4270 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i16, .Src: MVT::nxv4i32, .Cost: 0},
4271 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i16, .Src: MVT::nxv4i64, .Cost: 1},
4272 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv4i32, .Src: MVT::nxv4i64, .Cost: 1},
4273 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i8, .Src: MVT::nxv8i16, .Cost: 0},
4274 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i8, .Src: MVT::nxv8i32, .Cost: 1},
4275 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i8, .Src: MVT::nxv8i64, .Cost: 3},
4276 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i16, .Src: MVT::nxv8i32, .Cost: 1},
4277 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv8i16, .Src: MVT::nxv8i64, .Cost: 3},
4278 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv16i8, .Src: MVT::nxv16i16, .Cost: 1},
4279 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv16i8, .Src: MVT::nxv16i32, .Cost: 3},
4280 {.ISD: ISD::TRUNCATE, .Dst: MVT::nxv16i8, .Src: MVT::nxv16i64, .Cost: 7},
4281
4282 // The number of shll instructions for the extension.
4283 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3},
4284 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i16, .Cost: 3},
4285 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: 2},
4286 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v4i64, .Src: MVT::v4i32, .Cost: 2},
4287 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3},
4288 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i8, .Cost: 3},
4289 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 2},
4290 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i32, .Src: MVT::v8i16, .Cost: 2},
4291 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7},
4292 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i8, .Cost: 7},
4293 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6},
4294 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v8i64, .Src: MVT::v8i16, .Cost: 6},
4295 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 2},
4296 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i16, .Src: MVT::v16i8, .Cost: 2},
4297 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6},
4298 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::v16i32, .Src: MVT::v16i8, .Cost: 6},
4299
4300 // FP Ext and trunc
4301 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f64, .Src: MVT::f32, .Cost: 1}, // fcvt
4302 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v2f64, .Src: MVT::v2f32, .Cost: 1}, // fcvtl
4303 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4f32, .Cost: 2}, // fcvtl+fcvtl2
4304 // FP16
4305 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f32, .Src: MVT::f16, .Cost: 1}, // fcvt
4306 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f64, .Src: MVT::f16, .Cost: 1}, // fcvt
4307 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4f16, .Cost: 1}, // fcvtl
4308 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8f16, .Cost: 2}, // fcvtl+fcvtl2
4309 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v2f64, .Src: MVT::v2f16, .Cost: 2}, // fcvtl+fcvtl
4310 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4f16, .Cost: 3}, // fcvtl+fcvtl2+fcvtl
4311 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8f16, .Cost: 6}, // 2 * fcvtl+fcvtl2+fcvtl
4312 // BF16 (uses shift)
4313 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f32, .Src: MVT::bf16, .Cost: 1}, // shl
4314 {.ISD: ISD::FP_EXTEND, .Dst: MVT::f64, .Src: MVT::bf16, .Cost: 2}, // shl+fcvt
4315 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f32, .Src: MVT::v4bf16, .Cost: 1}, // shll
4316 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f32, .Src: MVT::v8bf16, .Cost: 2}, // shll+shll2
4317 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v2f64, .Src: MVT::v2bf16, .Cost: 2}, // shll+fcvtl
4318 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v4f64, .Src: MVT::v4bf16, .Cost: 3}, // shll+fcvtl+fcvtl2
4319 {.ISD: ISD::FP_EXTEND, .Dst: MVT::v8f64, .Src: MVT::v8bf16, .Cost: 6}, // 2 * shll+fcvtl+fcvtl2
4320 // FP Ext and trunc
4321 {.ISD: ISD::FP_ROUND, .Dst: MVT::f32, .Src: MVT::f64, .Cost: 1}, // fcvt
4322 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2f32, .Src: MVT::v2f64, .Cost: 1}, // fcvtn
4323 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f32, .Src: MVT::v4f64, .Cost: 2}, // fcvtn+fcvtn2
4324 // FP16
4325 {.ISD: ISD::FP_ROUND, .Dst: MVT::f16, .Src: MVT::f32, .Cost: 1}, // fcvt
4326 {.ISD: ISD::FP_ROUND, .Dst: MVT::f16, .Src: MVT::f64, .Cost: 1}, // fcvt
4327 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f16, .Src: MVT::v4f32, .Cost: 1}, // fcvtn
4328 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8f16, .Src: MVT::v8f32, .Cost: 2}, // fcvtn+fcvtn2
4329 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2f16, .Src: MVT::v2f64, .Cost: 2}, // fcvtn+fcvtn
4330 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4f16, .Src: MVT::v4f64, .Cost: 3}, // fcvtn+fcvtn2+fcvtn
4331 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8f16, .Src: MVT::v8f64, .Cost: 6}, // 2 * fcvtn+fcvtn2+fcvtn
4332 // BF16 (more complex, with +bf16 is handled above)
4333 {.ISD: ISD::FP_ROUND, .Dst: MVT::bf16, .Src: MVT::f32, .Cost: 8}, // Expansion is ~8 insns
4334 {.ISD: ISD::FP_ROUND, .Dst: MVT::bf16, .Src: MVT::f64, .Cost: 9}, // fcvtn + above
4335 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2bf16, .Src: MVT::v2f32, .Cost: 8},
4336 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4bf16, .Src: MVT::v4f32, .Cost: 8},
4337 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8bf16, .Src: MVT::v8f32, .Cost: 15},
4338 {.ISD: ISD::FP_ROUND, .Dst: MVT::v2bf16, .Src: MVT::v2f64, .Cost: 9},
4339 {.ISD: ISD::FP_ROUND, .Dst: MVT::v4bf16, .Src: MVT::v4f64, .Cost: 10},
4340 {.ISD: ISD::FP_ROUND, .Dst: MVT::v8bf16, .Src: MVT::v8f64, .Cost: 19},
4341
4342 // LowerVectorINT_TO_FP:
4343 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1},
4344 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1},
4345 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: 1},
4346 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i32, .Cost: 1},
4347 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i32, .Cost: 1},
4348 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i64, .Cost: 1},
4349
4350 // SVE: to nxv2f16
4351 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i8,
4352 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4353 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
4354 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
4355 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
4356 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i8,
4357 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4358 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
4359 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
4360 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
4361
4362 // SVE: to nxv4f16
4363 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i8,
4364 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4365 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i16, .Cost: SVE_FCVT_COST},
4366 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i32, .Cost: SVE_FCVT_COST},
4367 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i8,
4368 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4369 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i16, .Cost: SVE_FCVT_COST},
4370 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i32, .Cost: SVE_FCVT_COST},
4371
4372 // SVE: to nxv8f16
4373 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f16, .Src: MVT::nxv8i8,
4374 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4375 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f16, .Src: MVT::nxv8i16, .Cost: SVE_FCVT_COST},
4376 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f16, .Src: MVT::nxv8i8,
4377 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4378 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f16, .Src: MVT::nxv8i16, .Cost: SVE_FCVT_COST},
4379
4380 // SVE: to nxv16f16
4381 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv16f16, .Src: MVT::nxv16i8,
4382 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4383 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv16f16, .Src: MVT::nxv16i8,
4384 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4385
4386 // Complex: to v2f32
4387 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3},
4388 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 3},
4389 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i8, .Cost: 3},
4390 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f32, .Src: MVT::v2i16, .Cost: 3},
4391
4392 // SVE: to nxv2f32
4393 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i8,
4394 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4395 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
4396 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
4397 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
4398 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i8,
4399 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4400 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
4401 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
4402 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
4403
4404 // Complex: to v4f32
4405 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 4},
4406 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2},
4407 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i8, .Cost: 3},
4408 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f32, .Src: MVT::v4i16, .Cost: 2},
4409
4410 // SVE: to nxv4f32
4411 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i8,
4412 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4413 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i16, .Cost: SVE_FCVT_COST},
4414 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i32, .Cost: SVE_FCVT_COST},
4415 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i8,
4416 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4417 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i16, .Cost: SVE_FCVT_COST},
4418 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f32, .Src: MVT::nxv4i32, .Cost: SVE_FCVT_COST},
4419
4420 // Complex: to v8f32
4421 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i8, .Cost: 10},
4422 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4},
4423 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i8, .Cost: 10},
4424 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f32, .Src: MVT::v8i16, .Cost: 4},
4425
4426 // SVE: to nxv8f32
4427 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f32, .Src: MVT::nxv8i8,
4428 .Cost: SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4429 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f32, .Src: MVT::nxv8i16,
4430 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4431 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f32, .Src: MVT::nxv8i8,
4432 .Cost: SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4433 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f32, .Src: MVT::nxv8i16,
4434 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4435
4436 // SVE: to nxv16f32
4437 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv16f32, .Src: MVT::nxv16i8,
4438 .Cost: SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4439 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv16f32, .Src: MVT::nxv16i8,
4440 .Cost: SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4441
4442 // Complex: to v16f32
4443 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i8, .Cost: 21},
4444 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f32, .Src: MVT::v16i8, .Cost: 21},
4445
4446 // Complex: to v2f64
4447 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4},
4448 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 4},
4449 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2},
4450 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i8, .Cost: 4},
4451 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i16, .Cost: 4},
4452 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v2f64, .Src: MVT::v2i32, .Cost: 2},
4453
4454 // SVE: to nxv2f64
4455 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i8,
4456 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4457 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
4458 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
4459 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
4460 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i8,
4461 .Cost: SVE_EXT_COST + SVE_FCVT_COST},
4462 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i16, .Cost: SVE_FCVT_COST},
4463 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i32, .Cost: SVE_FCVT_COST},
4464 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv2f64, .Src: MVT::nxv2i64, .Cost: SVE_FCVT_COST},
4465
4466 // Complex: to v4f64
4467 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: 4},
4468 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v4f64, .Src: MVT::v4i32, .Cost: 4},
4469
4470 // SVE: to nxv4f64
4471 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i8,
4472 .Cost: SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4473 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i16,
4474 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4475 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i32,
4476 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4477 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i8,
4478 .Cost: SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4479 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i16,
4480 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4481 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv4f64, .Src: MVT::nxv4i32,
4482 .Cost: SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST},
4483
4484 // SVE: to nxv8f64
4485 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f64, .Src: MVT::nxv8i8,
4486 .Cost: SVE_EXT_COST + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4487 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::nxv8f64, .Src: MVT::nxv8i16,
4488 .Cost: SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4489 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f64, .Src: MVT::nxv8i8,
4490 .Cost: SVE_EXT_COST + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4491 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::nxv8f64, .Src: MVT::nxv8i16,
4492 .Cost: SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST},
4493
4494 // LowerVectorFP_TO_INT
4495 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i32, .Src: MVT::v2f32, .Cost: 1},
4496 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1},
4497 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i64, .Src: MVT::v2f64, .Cost: 1},
4498 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i32, .Src: MVT::v2f32, .Cost: 1},
4499 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f32, .Cost: 1},
4500 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i64, .Src: MVT::v2f64, .Cost: 1},
4501
4502 // Complex, from v2f32: legal type is v2i32 (no cost) or v2i64 (1 ext).
4503 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i64, .Src: MVT::v2f32, .Cost: 2},
4504 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i16, .Src: MVT::v2f32, .Cost: 1},
4505 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i8, .Src: MVT::v2f32, .Cost: 1},
4506 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i64, .Src: MVT::v2f32, .Cost: 2},
4507 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i16, .Src: MVT::v2f32, .Cost: 1},
4508 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i8, .Src: MVT::v2f32, .Cost: 1},
4509
4510 // Complex, from v4f32: legal type is v4i16, 1 narrowing => ~2
4511 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2},
4512 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 2},
4513 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f32, .Cost: 2},
4514 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f32, .Cost: 2},
4515
4516 // Complex, from v2f64: legal type is v2i32, 1 narrowing => ~2.
4517 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2},
4518 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i16, .Src: MVT::v2f64, .Cost: 2},
4519 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v2i8, .Src: MVT::v2f64, .Cost: 2},
4520 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i32, .Src: MVT::v2f64, .Cost: 2},
4521 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i16, .Src: MVT::v2f64, .Cost: 2},
4522 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v2i8, .Src: MVT::v2f64, .Cost: 2},
4523
4524 // Complex, from nxv2f32.
4525 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f32, .Cost: 1},
4526 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f32, .Cost: 1},
4527 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f32, .Cost: 1},
4528 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f32, .Cost: 1},
4529 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f32, .Cost: 1},
4530 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f32, .Cost: 1},
4531 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f32, .Cost: 1},
4532 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f32, .Cost: 1},
4533
4534 // Complex, from nxv2f64.
4535 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f64, .Cost: 1},
4536 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f64, .Cost: 1},
4537 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f64, .Cost: 1},
4538 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f64, .Cost: 1},
4539 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i1, .Src: MVT::nxv2f64, .Cost: 1},
4540 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f64, .Cost: 1},
4541 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f64, .Cost: 1},
4542 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f64, .Cost: 1},
4543 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f64, .Cost: 1},
4544 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i1, .Src: MVT::nxv2f64, .Cost: 1},
4545
4546 // Complex, from nxv4f32.
4547 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i64, .Src: MVT::nxv4f32, .Cost: 4},
4548 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f32, .Cost: 1},
4549 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f32, .Cost: 1},
4550 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f32, .Cost: 1},
4551 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i1, .Src: MVT::nxv4f32, .Cost: 1},
4552 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i64, .Src: MVT::nxv4f32, .Cost: 4},
4553 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f32, .Cost: 1},
4554 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f32, .Cost: 1},
4555 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f32, .Cost: 1},
4556 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i1, .Src: MVT::nxv4f32, .Cost: 1},
4557
4558 // Complex, from nxv8f64. Illegal -> illegal conversions not required.
4559 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f64, .Cost: 7},
4560 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f64, .Cost: 7},
4561 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f64, .Cost: 7},
4562 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f64, .Cost: 7},
4563
4564 // Complex, from nxv4f64. Illegal -> illegal conversions not required.
4565 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f64, .Cost: 3},
4566 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f64, .Cost: 3},
4567 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f64, .Cost: 3},
4568 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f64, .Cost: 3},
4569 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f64, .Cost: 3},
4570 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f64, .Cost: 3},
4571
4572 // Complex, from nxv8f32. Illegal -> illegal conversions not required.
4573 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f32, .Cost: 3},
4574 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f32, .Cost: 3},
4575 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f32, .Cost: 3},
4576 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f32, .Cost: 3},
4577
4578 // Complex, from nxv8f16.
4579 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i64, .Src: MVT::nxv8f16, .Cost: 10},
4580 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i32, .Src: MVT::nxv8f16, .Cost: 4},
4581 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f16, .Cost: 1},
4582 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f16, .Cost: 1},
4583 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv8i1, .Src: MVT::nxv8f16, .Cost: 1},
4584 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i64, .Src: MVT::nxv8f16, .Cost: 10},
4585 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i32, .Src: MVT::nxv8f16, .Cost: 4},
4586 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i16, .Src: MVT::nxv8f16, .Cost: 1},
4587 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i8, .Src: MVT::nxv8f16, .Cost: 1},
4588 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv8i1, .Src: MVT::nxv8f16, .Cost: 1},
4589
4590 // Complex, from nxv4f16.
4591 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i64, .Src: MVT::nxv4f16, .Cost: 4},
4592 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f16, .Cost: 1},
4593 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f16, .Cost: 1},
4594 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f16, .Cost: 1},
4595 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i64, .Src: MVT::nxv4f16, .Cost: 4},
4596 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i32, .Src: MVT::nxv4f16, .Cost: 1},
4597 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f16, .Cost: 1},
4598 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv4i8, .Src: MVT::nxv4f16, .Cost: 1},
4599
4600 // Complex, from nxv2f16.
4601 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f16, .Cost: 1},
4602 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f16, .Cost: 1},
4603 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f16, .Cost: 1},
4604 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f16, .Cost: 1},
4605 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i64, .Src: MVT::nxv2f16, .Cost: 1},
4606 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f16, .Cost: 1},
4607 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f16, .Cost: 1},
4608 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::nxv2i8, .Src: MVT::nxv2f16, .Cost: 1},
4609
4610 // Truncate from nxvmf32 to nxvmf16.
4611 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2f16, .Src: MVT::nxv2f32, .Cost: 1},
4612 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4f16, .Src: MVT::nxv4f32, .Cost: 1},
4613 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8f16, .Src: MVT::nxv8f32, .Cost: 3},
4614
4615 // Truncate from nxvmf32 to nxvmbf16.
4616 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2bf16, .Src: MVT::nxv2f32, .Cost: 8},
4617 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4bf16, .Src: MVT::nxv4f32, .Cost: 8},
4618 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8bf16, .Src: MVT::nxv8f32, .Cost: 17},
4619
4620 // Truncate from nxvmf64 to nxvmf16.
4621 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2f16, .Src: MVT::nxv2f64, .Cost: 1},
4622 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4f16, .Src: MVT::nxv4f64, .Cost: 3},
4623 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8f16, .Src: MVT::nxv8f64, .Cost: 7},
4624
4625 // Truncate from nxvmf64 to nxvmbf16.
4626 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2bf16, .Src: MVT::nxv2f64, .Cost: 9},
4627 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4bf16, .Src: MVT::nxv4f64, .Cost: 19},
4628 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8bf16, .Src: MVT::nxv8f64, .Cost: 39},
4629
4630 // Truncate from nxvmf64 to nxvmf32.
4631 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv2f32, .Src: MVT::nxv2f64, .Cost: 1},
4632 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv4f32, .Src: MVT::nxv4f64, .Cost: 3},
4633 {.ISD: ISD::FP_ROUND, .Dst: MVT::nxv8f32, .Src: MVT::nxv8f64, .Cost: 6},
4634
4635 // Extend from nxvmf16 to nxvmf32.
4636 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f32, .Src: MVT::nxv2f16, .Cost: 1},
4637 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f32, .Src: MVT::nxv4f16, .Cost: 1},
4638 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f32, .Src: MVT::nxv8f16, .Cost: 2},
4639
4640 // Extend from nxvmbf16 to nxvmf32.
4641 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f32, .Src: MVT::nxv2bf16, .Cost: 1}, // lsl
4642 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f32, .Src: MVT::nxv4bf16, .Cost: 1}, // lsl
4643 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f32, .Src: MVT::nxv8bf16, .Cost: 4}, // unpck+unpck+lsl+lsl
4644
4645 // Extend from nxvmf16 to nxvmf64.
4646 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f64, .Src: MVT::nxv2f16, .Cost: 1},
4647 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f64, .Src: MVT::nxv4f16, .Cost: 2},
4648 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f64, .Src: MVT::nxv8f16, .Cost: 4},
4649
4650 // Extend from nxvmbf16 to nxvmf64.
4651 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f64, .Src: MVT::nxv2bf16, .Cost: 2}, // lsl+fcvt
4652 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f64, .Src: MVT::nxv4bf16, .Cost: 6}, // 2*unpck+2*lsl+2*fcvt
4653 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f64, .Src: MVT::nxv8bf16, .Cost: 14}, // 6*unpck+4*lsl+4*fcvt
4654
4655 // Extend from nxvmf32 to nxvmf64.
4656 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv2f64, .Src: MVT::nxv2f32, .Cost: 1},
4657 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv4f64, .Src: MVT::nxv4f32, .Cost: 2},
4658 {.ISD: ISD::FP_EXTEND, .Dst: MVT::nxv8f64, .Src: MVT::nxv8f32, .Cost: 6},
4659
4660 // Bitcasts from float to integer
4661 {.ISD: ISD::BITCAST, .Dst: MVT::nxv2f16, .Src: MVT::nxv2i16, .Cost: 0},
4662 {.ISD: ISD::BITCAST, .Dst: MVT::nxv4f16, .Src: MVT::nxv4i16, .Cost: 0},
4663 {.ISD: ISD::BITCAST, .Dst: MVT::nxv2f32, .Src: MVT::nxv2i32, .Cost: 0},
4664
4665 // Bitcasts from integer to float
4666 {.ISD: ISD::BITCAST, .Dst: MVT::nxv2i16, .Src: MVT::nxv2f16, .Cost: 0},
4667 {.ISD: ISD::BITCAST, .Dst: MVT::nxv4i16, .Src: MVT::nxv4f16, .Cost: 0},
4668 {.ISD: ISD::BITCAST, .Dst: MVT::nxv2i32, .Src: MVT::nxv2f32, .Cost: 0},
4669
4670 // Add cost for extending to illegal -too wide- scalable vectors.
4671 // zero/sign extend are implemented by multiple unpack operations,
4672 // where each operation has a cost of 1.
4673 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv16i16, .Src: MVT::nxv16i8, .Cost: 2},
4674 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv16i32, .Src: MVT::nxv16i8, .Cost: 6},
4675 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv16i64, .Src: MVT::nxv16i8, .Cost: 14},
4676 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv8i32, .Src: MVT::nxv8i16, .Cost: 2},
4677 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv8i64, .Src: MVT::nxv8i16, .Cost: 6},
4678 {.ISD: ISD::ZERO_EXTEND, .Dst: MVT::nxv4i64, .Src: MVT::nxv4i32, .Cost: 2},
4679
4680 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv16i16, .Src: MVT::nxv16i8, .Cost: 2},
4681 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv16i32, .Src: MVT::nxv16i8, .Cost: 6},
4682 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv16i64, .Src: MVT::nxv16i8, .Cost: 14},
4683 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv8i32, .Src: MVT::nxv8i16, .Cost: 2},
4684 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv8i64, .Src: MVT::nxv8i16, .Cost: 6},
4685 {.ISD: ISD::SIGN_EXTEND, .Dst: MVT::nxv4i64, .Src: MVT::nxv4i32, .Cost: 2},
4686 };
4687
4688 if (const auto *Entry = ConvertCostTableLookup(
4689 Table: ConversionTbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
4690 return Entry->Cost;
4691
4692 static const TypeConversionCostTblEntry FP16Tbl[] = {
4693 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i8, .Src: MVT::v4f16, .Cost: 1}, // fcvtzs
4694 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i8, .Src: MVT::v4f16, .Cost: 1},
4695 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i16, .Src: MVT::v4f16, .Cost: 1}, // fcvtzs
4696 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i16, .Src: MVT::v4f16, .Cost: 1},
4697 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v4i32, .Src: MVT::v4f16, .Cost: 2}, // fcvtl+fcvtzs
4698 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v4i32, .Src: MVT::v4f16, .Cost: 2},
4699 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i8, .Src: MVT::v8f16, .Cost: 2}, // fcvtzs+xtn
4700 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i8, .Src: MVT::v8f16, .Cost: 2},
4701 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i16, .Src: MVT::v8f16, .Cost: 1}, // fcvtzs
4702 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i16, .Src: MVT::v8f16, .Cost: 1},
4703 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v8i32, .Src: MVT::v8f16, .Cost: 4}, // 2*fcvtl+2*fcvtzs
4704 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v8i32, .Src: MVT::v8f16, .Cost: 4},
4705 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i8, .Src: MVT::v16f16, .Cost: 3}, // 2*fcvtzs+xtn
4706 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i8, .Src: MVT::v16f16, .Cost: 3},
4707 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i16, .Src: MVT::v16f16, .Cost: 2}, // 2*fcvtzs
4708 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i16, .Src: MVT::v16f16, .Cost: 2},
4709 {.ISD: ISD::FP_TO_SINT, .Dst: MVT::v16i32, .Src: MVT::v16f16, .Cost: 8}, // 4*fcvtl+4*fcvtzs
4710 {.ISD: ISD::FP_TO_UINT, .Dst: MVT::v16i32, .Src: MVT::v16f16, .Cost: 8},
4711 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v8f16, .Src: MVT::v8i8, .Cost: 2}, // ushll + ucvtf
4712 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v8f16, .Src: MVT::v8i8, .Cost: 2}, // sshll + scvtf
4713 {.ISD: ISD::UINT_TO_FP, .Dst: MVT::v16f16, .Src: MVT::v16i8, .Cost: 4}, // 2 * ushl(2) + 2 * ucvtf
4714 {.ISD: ISD::SINT_TO_FP, .Dst: MVT::v16f16, .Src: MVT::v16i8, .Cost: 4}, // 2 * sshl(2) + 2 * scvtf
4715 };
4716
4717 if (ST->hasFullFP16())
4718 if (const auto *Entry = ConvertCostTableLookup(
4719 Table: FP16Tbl, ISD, Dst: DstTy.getSimpleVT(), Src: SrcTy.getSimpleVT()))
4720 return Entry->Cost;
4721
4722 // INT_TO_FP of i64->f32 will scalarize, which is required to avoid
4723 // double-rounding issues.
4724 if ((ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP) &&
4725 DstTy.getScalarType() == MVT::f32 && SrcTy.getScalarSizeInBits() > 32 &&
4726 isa<FixedVectorType>(Val: Dst) && isa<FixedVectorType>(Val: Src))
4727 return cast<FixedVectorType>(Val: Dst)->getNumElements() *
4728 getCastInstrCost(Opcode, Dst: Dst->getScalarType(),
4729 Src: Src->getScalarType(), CCH, CostKind) +
4730 BaseT::getScalarizationOverhead(InTy: cast<FixedVectorType>(Val: Src), Insert: false,
4731 Extract: true, CostKind) +
4732 BaseT::getScalarizationOverhead(InTy: cast<FixedVectorType>(Val: Dst), Insert: true,
4733 Extract: false, CostKind);
4734
4735 if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
4736 CCH == TTI::CastContextHint::Masked &&
4737 ST->isSVEorStreamingSVEAvailable() &&
4738 TLI->getTypeAction(Context&: Src->getContext(), VT: SrcTy) ==
4739 TargetLowering::TypePromoteInteger &&
4740 TLI->getTypeAction(Context&: Dst->getContext(), VT: DstTy) ==
4741 TargetLowering::TypeSplitVector) {
4742 // The standard behaviour in the backend for these cases is to split the
4743 // extend up into two parts:
4744 // 1. Perform an extending load or masked load up to the legal type.
4745 // 2. Extend the loaded data to the final type.
4746 std::pair<InstructionCost, MVT> SrcLT = getTypeLegalizationCost(Ty: Src);
4747 Type *LegalTy = EVT(SrcLT.second).getTypeForEVT(Context&: Src->getContext());
4748 InstructionCost Part1 = AArch64TTIImpl::getCastInstrCost(
4749 Opcode, Dst: LegalTy, Src, CCH, CostKind, I);
4750 InstructionCost Part2 = AArch64TTIImpl::getCastInstrCost(
4751 Opcode, Dst, Src: LegalTy, CCH: TTI::CastContextHint::None, CostKind, I);
4752 return Part1 + Part2;
4753 }
4754
4755 // The BasicTTIImpl version only deals with CCH==TTI::CastContextHint::Normal,
4756 // but we also want to include the TTI::CastContextHint::Masked case too.
4757 if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) &&
4758 CCH == TTI::CastContextHint::Masked &&
4759 ST->isSVEorStreamingSVEAvailable() && TLI->isTypeLegal(VT: DstTy))
4760 CCH = TTI::CastContextHint::Normal;
4761
4762 return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
4763}
4764
4765InstructionCost
4766AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
4767 VectorType *VecTy, unsigned Index,
4768 TTI::TargetCostKind CostKind) const {
4769
4770 // Make sure we were given a valid extend opcode.
4771 assert((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) &&
4772 "Invalid opcode");
4773
4774 // We are extending an element we extract from a vector, so the source type
4775 // of the extend is the element type of the vector.
4776 auto *Src = VecTy->getElementType();
4777
4778 // Sign- and zero-extends are for integer types only.
4779 assert(isa<IntegerType>(Dst) && isa<IntegerType>(Src) && "Invalid type");
4780
4781 // Get the cost for the extract. We compute the cost (if any) for the extend
4782 // below.
4783 InstructionCost Cost = getVectorInstrCost(Opcode: Instruction::ExtractElement, Ty: VecTy,
4784 CostKind, Index, Op0: nullptr, Op1: nullptr);
4785
4786 // Legalize the types.
4787 auto VecLT = getTypeLegalizationCost(Ty: VecTy);
4788 auto DstVT = TLI->getValueType(DL, Ty: Dst);
4789 auto SrcVT = TLI->getValueType(DL, Ty: Src);
4790
4791 // If the resulting type is still a vector and the destination type is legal,
4792 // we may get the extension for free. If not, get the default cost for the
4793 // extend.
4794 if (!VecLT.second.isVector() || !TLI->isTypeLegal(VT: DstVT))
4795 return Cost + getCastInstrCost(Opcode, Dst, Src, CCH: TTI::CastContextHint::None,
4796 CostKind);
4797
4798 // The destination type should be larger than the element type. If not, get
4799 // the default cost for the extend.
4800 if (DstVT.getFixedSizeInBits() < SrcVT.getFixedSizeInBits())
4801 return Cost + getCastInstrCost(Opcode, Dst, Src, CCH: TTI::CastContextHint::None,
4802 CostKind);
4803
4804 switch (Opcode) {
4805 default:
4806 llvm_unreachable("Opcode should be either SExt or ZExt");
4807
4808 // For sign-extends, we only need a smov, which performs the extension
4809 // automatically.
4810 case Instruction::SExt:
4811 return Cost;
4812
4813 // For zero-extends, the extend is performed automatically by a umov unless
4814 // the destination type is i64 and the element type is i8 or i16.
4815 case Instruction::ZExt:
4816 if (DstVT.getSizeInBits() != 64u || SrcVT.getSizeInBits() == 32u)
4817 return Cost;
4818 }
4819
4820 // If we are unable to perform the extend for free, get the default cost.
4821 return Cost + getCastInstrCost(Opcode, Dst, Src, CCH: TTI::CastContextHint::None,
4822 CostKind);
4823}
4824
4825InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
4826 TTI::TargetCostKind CostKind,
4827 const Instruction *I) const {
4828 if (CostKind != TTI::TCK_RecipThroughput)
4829 return Opcode == Instruction::PHI ? 0 : 1;
4830 assert(CostKind == TTI::TCK_RecipThroughput && "unexpected CostKind");
4831 // Branches are assumed to be predicted.
4832 return 0;
4833}
4834
4835InstructionCost AArch64TTIImpl::getVectorInstrCostHelper(
4836 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, unsigned Index,
4837 const Instruction *I, Value *Scalar,
4838 ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
4839 TTI::VectorInstrContext VIC) const {
4840 assert(Ty->isVectorTy() && "This must be a vector type");
4841
4842 if (Index != -1U) {
4843 // Legalize the type.
4844 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
4845
4846 // This type is legalized to a scalar type.
4847 if (!LT.second.isVector())
4848 return 0;
4849
4850 // The type may be split. For fixed-width vectors we can normalize the
4851 // index to the new type.
4852 if (LT.second.isFixedLengthVector()) {
4853 unsigned Width = LT.second.getVectorNumElements();
4854 Index = Index % Width;
4855 }
4856
4857 // The element at index zero is already inside the vector.
4858 // - For a insert-element or extract-element
4859 // instruction that extracts integers, an explicit FPR -> GPR move is
4860 // needed. So it has non-zero cost.
4861 if (Index == 0 && !Ty->getScalarType()->isIntegerTy())
4862 return 0;
4863
4864 // SVE has no scalar move to an arbitrary lane above the low 128-bit portion
4865 // of a Z register, e.g. there is no equivalent of "mov z0.d[9], d0".
4866 // Fixed-length vectors wider than 128 bits therefore need
4867 // [splice]/index/pred/splat/cmp/pred-mov when scalarizing inserts for those
4868 // lanes, so model them as more expensive than ordinary NEON lane accesses.
4869 if (ST->useSVEForFixedLengthVectors()) {
4870 InstructionCost Cost = CostKind == TTI::TCK_CodeSize
4871 ? 1
4872 : ST->getVectorInsertExtractBaseCost();
4873 if (Index * Ty->getScalarSizeInBits() < 128)
4874 return Cost;
4875 if (Index * Ty->getScalarSizeInBits() < 512 &&
4876 Opcode == Instruction::ExtractElement)
4877 // Integer extracts (>128b, <512b) require extra mov from FPR -> GPR.
4878 return Ty->getScalarType()->isIntegerTy() ? Cost + 1 : Cost;
4879 if (Opcode == Instruction::ExtractElement)
4880 return Cost + 2; // cost of mov imm + whilels + lastb
4881 if (Opcode == Instruction::InsertElement)
4882 return Cost + 3; // cost of insert with cmp/splice
4883 llvm_unreachable("unexpected opcode");
4884 }
4885
4886 // This is recognising a LD1 single-element structure to one lane of one
4887 // register instruction. I.e., if this is an `insertelement` instruction,
4888 // and its second operand is a load, then we will generate a LD1, which
4889 // are expensive instructions on some uArchs.
4890 if (VIC == TTI::VectorInstrContext::Load) {
4891 if (ST->hasFastLD1Single())
4892 return 0;
4893 return CostKind == TTI::TCK_CodeSize
4894 ? 0
4895 : ST->getVectorInsertExtractBaseCost() + 1;
4896 }
4897
4898 // i1 inserts and extract will include an extra cset or cmp of the vector
4899 // value. Increase the cost by 1 to account.
4900 if (Ty->getScalarSizeInBits() == 1)
4901 return CostKind == TTI::TCK_CodeSize
4902 ? 2
4903 : ST->getVectorInsertExtractBaseCost() + 1;
4904
4905 // FIXME:
4906 // If the extract-element and insert-element instructions could be
4907 // simplified away (e.g., could be combined into users by looking at use-def
4908 // context), they have no cost. This is not done in the first place for
4909 // compile-time considerations.
4910 }
4911
4912 // In case of Neon, if there exists extractelement from lane != 0 such that
4913 // 1. extractelement does not necessitate a move from vector_reg -> GPR.
4914 // 2. extractelement result feeds into fmul.
4915 // 3. Other operand of fmul is an extractelement from lane 0 or lane
4916 // equivalent to 0.
4917 // then the extractelement can be merged with fmul in the backend and it
4918 // incurs no cost.
4919 // e.g.
4920 // define double @foo(<2 x double> %a) {
4921 // %1 = extractelement <2 x double> %a, i32 0
4922 // %2 = extractelement <2 x double> %a, i32 1
4923 // %res = fmul double %1, %2
4924 // ret double %res
4925 // }
4926 // %2 and %res can be merged in the backend to generate fmul d0, d0, v1.d[1]
4927 auto ExtractCanFuseWithFmul = [&]() {
4928 // We bail out if the extract is from lane 0.
4929 if (Index == 0)
4930 return false;
4931
4932 // Check if the scalar element type of the vector operand of ExtractElement
4933 // instruction is one of the allowed types.
4934 auto IsAllowedScalarTy = [&](const Type *T) {
4935 return T->isFloatTy() || T->isDoubleTy() ||
4936 (T->isHalfTy() && ST->hasFullFP16());
4937 };
4938
4939 // Check if the extractelement user is scalar fmul.
4940 auto IsUserFMulScalarTy = [](const Value *EEUser) {
4941 // Check if the user is scalar fmul.
4942 const auto *BO = dyn_cast<BinaryOperator>(Val: EEUser);
4943 return BO && BO->getOpcode() == BinaryOperator::FMul &&
4944 !BO->getType()->isVectorTy();
4945 };
4946
4947 // Check if the extract index is from lane 0 or lane equivalent to 0 for a
4948 // certain scalar type and a certain vector register width.
4949 auto IsExtractLaneEquivalentToZero = [&](unsigned Idx, unsigned EltSz) {
4950 auto RegWidth =
4951 getRegisterBitWidth(K: TargetTransformInfo::RGK_FixedWidthVector)
4952 .getFixedValue();
4953 return Idx == 0 || (RegWidth != 0 && (Idx * EltSz) % RegWidth == 0);
4954 };
4955
4956 // Check if the type constraints on input vector type and result scalar type
4957 // of extractelement instruction are satisfied.
4958 if (!isa<FixedVectorType>(Val: Ty) || !IsAllowedScalarTy(Ty->getScalarType()))
4959 return false;
4960
4961 if (Scalar) {
4962 DenseMap<User *, unsigned> UserToExtractIdx;
4963 for (auto *U : Scalar->users()) {
4964 if (!IsUserFMulScalarTy(U))
4965 return false;
4966 // Recording entry for the user is important. Index value is not
4967 // important.
4968 UserToExtractIdx[U];
4969 }
4970 if (UserToExtractIdx.empty())
4971 return false;
4972 for (auto &[S, U, L] : ScalarUserAndIdx) {
4973 for (auto *U : S->users()) {
4974 if (UserToExtractIdx.contains(Val: U)) {
4975 auto *FMul = cast<BinaryOperator>(Val: U);
4976 auto *Op0 = FMul->getOperand(i_nocapture: 0);
4977 auto *Op1 = FMul->getOperand(i_nocapture: 1);
4978 if ((Op0 == S && Op1 == S) || Op0 != S || Op1 != S) {
4979 UserToExtractIdx[U] = L;
4980 break;
4981 }
4982 }
4983 }
4984 }
4985 for (auto &[U, L] : UserToExtractIdx) {
4986 if (!IsExtractLaneEquivalentToZero(Index, Ty->getScalarSizeInBits()) &&
4987 !IsExtractLaneEquivalentToZero(L, Ty->getScalarSizeInBits()))
4988 return false;
4989 }
4990 } else {
4991 const auto *EE = cast<ExtractElementInst>(Val: I);
4992
4993 const auto *IdxOp = dyn_cast<ConstantInt>(Val: EE->getIndexOperand());
4994 if (!IdxOp)
4995 return false;
4996
4997 return !EE->users().empty() && all_of(Range: EE->users(), P: [&](const User *U) {
4998 if (!IsUserFMulScalarTy(U))
4999 return false;
5000
5001 // Check if the other operand of extractelement is also extractelement
5002 // from lane equivalent to 0.
5003 const auto *BO = cast<BinaryOperator>(Val: U);
5004 const auto *OtherEE = dyn_cast<ExtractElementInst>(
5005 Val: BO->getOperand(i_nocapture: 0) == EE ? BO->getOperand(i_nocapture: 1) : BO->getOperand(i_nocapture: 0));
5006 if (OtherEE) {
5007 const auto *IdxOp = dyn_cast<ConstantInt>(Val: OtherEE->getIndexOperand());
5008 if (!IdxOp)
5009 return false;
5010 return IsExtractLaneEquivalentToZero(
5011 cast<ConstantInt>(Val: OtherEE->getIndexOperand())
5012 ->getValue()
5013 .getZExtValue(),
5014 OtherEE->getType()->getScalarSizeInBits());
5015 }
5016 return true;
5017 });
5018 }
5019 return true;
5020 };
5021
5022 if (Opcode == Instruction::ExtractElement && (I || Scalar) &&
5023 ExtractCanFuseWithFmul())
5024 return 0;
5025
5026 // All other insert/extracts cost this much.
5027 return CostKind == TTI::TCK_CodeSize ? 1
5028 : ST->getVectorInsertExtractBaseCost();
5029}
5030
5031InstructionCost AArch64TTIImpl::getVectorInstrCost(
5032 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, unsigned Index,
5033 const Value *Op0, const Value *Op1, TTI::VectorInstrContext VIC) const {
5034 // Treat insert at lane 0 into a poison vector as having zero cost. This
5035 // ensures vector broadcasts via an insert + shuffle (and will be lowered to a
5036 // single dup) are treated as cheap.
5037 if (Opcode == Instruction::InsertElement && Index == 0 && Op0 &&
5038 isa<PoisonValue>(Val: Op0))
5039 return 0;
5040 return getVectorInstrCostHelper(Opcode, Ty, CostKind, Index, I: nullptr, Scalar: nullptr,
5041 ScalarUserAndIdx: {}, VIC);
5042}
5043
5044InstructionCost AArch64TTIImpl::getVectorInstrCost(
5045 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, unsigned Index,
5046 Value *Scalar, ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx,
5047 TTI::VectorInstrContext VIC) const {
5048 return getVectorInstrCostHelper(Opcode, Ty, CostKind, Index, I: nullptr, Scalar,
5049 ScalarUserAndIdx, VIC);
5050}
5051
5052InstructionCost
5053AArch64TTIImpl::getVectorInstrCost(const Instruction &I, Type *Ty,
5054 TTI::TargetCostKind CostKind, unsigned Index,
5055 TTI::VectorInstrContext VIC) const {
5056 return getVectorInstrCostHelper(Opcode: I.getOpcode(), Ty, CostKind, Index, I: &I,
5057 Scalar: nullptr, ScalarUserAndIdx: {}, VIC);
5058}
5059
5060InstructionCost
5061AArch64TTIImpl::getIndexedVectorInstrCostFromEnd(unsigned Opcode, Type *Ty,
5062 TTI::TargetCostKind CostKind,
5063 unsigned Index) const {
5064 if (isa<FixedVectorType>(Val: Ty))
5065 return BaseT::getIndexedVectorInstrCostFromEnd(Opcode, Val: Ty, CostKind, Index);
5066
5067 // This typically requires both while and lastb instructions in order
5068 // to extract the last element. If this is in a loop the while
5069 // instruction can at least be hoisted out, although it will consume a
5070 // predicate register. The cost should be more expensive than the base
5071 // extract cost, which is 2 for most CPUs.
5072 return CostKind == TTI::TCK_CodeSize
5073 ? 2
5074 : ST->getVectorInsertExtractBaseCost() + 1;
5075}
5076
5077InstructionCost AArch64TTIImpl::getScalarizationOverhead(
5078 VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract,
5079 TTI::TargetCostKind CostKind, bool ForPoisonSrc, ArrayRef<Value *> VL,
5080 TTI::VectorInstrContext VIC) const {
5081 if (isa<ScalableVectorType>(Val: Ty))
5082 return InstructionCost::getInvalid();
5083
5084 // There's no scalarization overhead if ld1/st1 is cheap and the
5085 // insert/extracts can be folded into the load/stores.
5086 if (ST->hasFastLD1Single()) {
5087 if ((VIC == TTI::VectorInstrContext::Store && Extract) ||
5088 (VIC == TTI::VectorInstrContext::Load && Insert))
5089 return 0;
5090 }
5091
5092 if (Ty->getElementType()->isFloatingPointTy())
5093 return BaseT::getScalarizationOverhead(InTy: Ty, DemandedElts, Insert, Extract,
5094 CostKind);
5095 unsigned VecInstCost =
5096 CostKind == TTI::TCK_CodeSize ? 1 : ST->getVectorInsertExtractBaseCost();
5097 return DemandedElts.popcount() * (Insert + Extract) * VecInstCost;
5098}
5099
5100std::optional<InstructionCost> AArch64TTIImpl::getFP16BF16PromoteCost(
5101 Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
5102 TTI::OperandValueInfo Op2Info, bool IncludeTrunc, bool CanUseSVE,
5103 std::function<InstructionCost(Type *)> InstCost) const {
5104 if (!Ty->getScalarType()->isHalfTy() && !Ty->getScalarType()->isBFloatTy())
5105 return std::nullopt;
5106 if (Ty->getScalarType()->isHalfTy() && ST->hasFullFP16())
5107 return std::nullopt;
5108 // If we have +sve-b16b16 the operation can be promoted to SVE.
5109 if (CanUseSVE && ST->hasSVEB16B16() && ST->isNonStreamingSVEorSME2Available())
5110 return std::nullopt;
5111
5112 Type *PromotedTy = Ty->getWithNewType(EltTy: Type::getFloatTy(C&: Ty->getContext()));
5113 InstructionCost Cost = getCastInstrCost(Opcode: Instruction::FPExt, Dst: PromotedTy, Src: Ty,
5114 CCH: TTI::CastContextHint::None, CostKind);
5115 if (!Op1Info.isConstant() && !Op2Info.isConstant())
5116 Cost *= 2;
5117 Cost += InstCost(PromotedTy);
5118 if (IncludeTrunc)
5119 Cost += getCastInstrCost(Opcode: Instruction::FPTrunc, Dst: Ty, Src: PromotedTy,
5120 CCH: TTI::CastContextHint::None, CostKind);
5121 return Cost;
5122}
5123
5124InstructionCost AArch64TTIImpl::getArithmeticInstrCost(
5125 unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
5126 TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info,
5127 ArrayRef<const Value *> Args, const Instruction *CtxI) const {
5128
5129 // The code-generator is currently not able to handle scalable vectors
5130 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
5131 // it until all instructions are vetted.
5132 int ISD = TLI->InstructionOpcodeToISD(Opcode);
5133 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: Ty))
5134 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
5135 if (!is_contained(Set: {ISD::ADD, ISD::SUB, ISD::MUL}, Element: ISD))
5136 return InstructionCost::getInvalid();
5137
5138 if (CtxI && CtxI->getOpcode() == Opcode && !CtxI->use_empty() &&
5139 useNeonVector(Ty)) {
5140 // The Ands of a bit select are folded into the BSL that the Or becomes.
5141 Instruction *MainAnd, *OtherAnd;
5142 Value *A, *B;
5143 if (Opcode == Instruction::And && CtxI->hasOneUse() &&
5144 matchBSL(I: CtxI->user_back(), MainAnd, OtherAnd, A, B))
5145 return 0;
5146
5147 // A Not is folded into the BIC, ORN or BSL of each And/Or using it, unless
5148 // that And/Or negates both of its operands.
5149 if (match(V: CtxI, P: m_Not(V: m_Value())) &&
5150 all_of(Range: CtxI->users(), P: [&](const User *U) {
5151 const auto *UI = cast<Instruction>(Val: U);
5152 return is_contained(Set: {Instruction::And, Instruction::Or},
5153 Element: UI->getOpcode()) &&
5154 UI->getParent() == CtxI->getParent() &&
5155 !match(V: UI, P: m_BinOp(L: m_Not(V: m_Value()), R: m_Not(V: m_Value())));
5156 }))
5157 return 0;
5158 }
5159
5160 // Legalize the type.
5161 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
5162
5163 // TODO: Handle more cost kinds for floating point operations.
5164 if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
5165 ISD == ISD::FDIV || ISD == ISD::FREM || ISD == ISD::FNEG)
5166 if (CostKind != TTI::TCK_RecipThroughput)
5167 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
5168 Opd2Info: Op2Info, Args, CtxI);
5169
5170 if (ISD == ISD::FADD || ISD == ISD::FSUB || ISD == ISD::FMUL ||
5171 ISD == ISD::FDIV || ISD == ISD::FREM) {
5172 // Increase the cost for half and bfloat types if not architecturally
5173 // supported.
5174 if (auto PromotedCost = getFP16BF16PromoteCost(
5175 Ty, CostKind, Op1Info, Op2Info, /*IncludeTrunc=*/true,
5176 // There is not native support for fdiv/frem even with +sve-b16b16.
5177 /*CanUseSVE=*/ISD != ISD::FDIV && ISD != ISD::FREM,
5178 InstCost: [&](Type *PromotedTy) {
5179 return getArithmeticInstrCost(Opcode, Ty: PromotedTy, CostKind,
5180 Op1Info, Op2Info);
5181 }))
5182 return *PromotedCost;
5183
5184 // fp128 all go via libcalls
5185 if (Ty->getScalarType()->isFP128Ty())
5186 return (CostKind == TTI::TCK_CodeSize ? 1 : 10) * LT.first;
5187 }
5188
5189 // If the operation is a widening instruction (smull or umull) and both
5190 // operands are extends the cost can be cheaper by considering that the
5191 // operation will operate on the narrowest type size possible (double the
5192 // largest input size) and a further extend.
5193 if (Type *ExtTy = isBinExtWideningInstruction(Opcode, DstTy: Ty, Args)) {
5194 if (ExtTy != Ty)
5195 return getArithmeticInstrCost(Opcode, Ty: ExtTy, CostKind) +
5196 getCastInstrCost(Opcode: Instruction::ZExt, Dst: Ty, Src: ExtTy,
5197 CCH: TTI::CastContextHint::None, CostKind);
5198 return LT.first;
5199 }
5200
5201 switch (ISD) {
5202 default:
5203 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
5204 Opd2Info: Op2Info);
5205 case ISD::ADD:
5206 case ISD::SUB:
5207 return LT.first; // Also works for i128
5208 case ISD::MUL: {
5209 // i128 multiply is umulh + 2*madd + mul and grows ~O(Bitwidth^2). For
5210 // scalable vectors the cost of LT.first will be invalid, leading to an
5211 // invalid cost overall.
5212 unsigned Mul64CostFactor = (CostKind == TTI::TCK_RecipThroughput &&
5213 ST->hasLimited64bitVectorMulBandwidth())
5214 ? 4
5215 : 1;
5216 if (Ty->getScalarSizeInBits() > 64) {
5217 unsigned NumLanes = isa<FixedVectorType>(Val: Ty)
5218 ? cast<FixedVectorType>(Val: Ty)->getNumElements()
5219 : 1;
5220 InstructionCost CostPerLane = LT.first / NumLanes;
5221 return CostPerLane * CostPerLane * NumLanes * Mul64CostFactor;
5222 }
5223
5224 if (LT.second == MVT::v2i64) {
5225 // When SVE is available, then we can lower the v2i64 operation using
5226 // the SVE mul instruction, which has a lower cost.
5227 if (ST->hasSVE())
5228 return LT.first * Mul64CostFactor;
5229
5230 // When SVE is not available, there is no MUL.2d instruction,
5231 // which means mul <2 x i64> is expensive as elements are extracted
5232 // from the vectors and the muls scalarized.
5233 // As getScalarizationOverhead is a bit too pessimistic, we
5234 // estimate the cost for a i64 vector directly here, which is:
5235 // - four 2-cost i64 extracts,
5236 // - two 2-cost i64 inserts, and
5237 // - two 1-cost muls.
5238 // So, for a v2i64 with LT.First = 1 the cost is 14, and for a v4i64 with
5239 // LT.first = 2 the cost is 28.
5240 return cast<VectorType>(Val: Ty)->getElementCount().getKnownMinValue() *
5241 (getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind) +
5242 getVectorInstrCost(Opcode: Instruction::ExtractElement, Ty, CostKind, Index: -1,
5243 Op0: nullptr, Op1: nullptr) *
5244 2 +
5245 getVectorInstrCost(Opcode: Instruction::InsertElement, Ty, CostKind, Index: -1,
5246 Op0: nullptr, Op1: nullptr));
5247 }
5248
5249 if (LT.second == MVT::nxv2i64)
5250 return LT.first * Mul64CostFactor;
5251
5252 return LT.first;
5253 }
5254 case ISD::SREM:
5255 case ISD::SDIV:
5256 /*
5257 Notes for sdiv/srem specific costs:
5258 1. This only considers the cases where the divisor is constant, uniform and
5259 (pow-of-2/non-pow-of-2). Other cases are not important since they either
5260 result in some form of (ldr + adrp), corresponding to constant vectors, or
5261 scalarization of the division operation.
5262 2. Constant divisors, either negative in whole or partially, don't result in
5263 significantly different codegen as compared to positive constant divisors.
5264 So, we don't consider negative divisors separately.
5265 3. If the codegen is significantly different with SVE, it has been indicated
5266 using comments at appropriate places.
5267
5268 sdiv specific cases:
5269 -----------------------------------------------------------------------
5270 codegen | pow-of-2 | Type
5271 -----------------------------------------------------------------------
5272 add + cmp + csel + asr | Y | i64
5273 add + cmp + csel + asr | Y | i32
5274 -----------------------------------------------------------------------
5275
5276 srem specific cases:
5277 -----------------------------------------------------------------------
5278 codegen | pow-of-2 | Type
5279 -----------------------------------------------------------------------
5280 negs + and + and + csneg | Y | i64
5281 negs + and + and + csneg | Y | i32
5282 -----------------------------------------------------------------------
5283
5284 other sdiv/srem cases:
5285 -------------------------------------------------------------------------
5286 common codegen | + srem | + sdiv | pow-of-2 | Type
5287 -------------------------------------------------------------------------
5288 smulh + asr + add + add | - | - | N | i64
5289 smull + lsr + add + add | - | - | N | i32
5290 usra | and + sub | sshr | Y | <2 x i64>
5291 2 * (scalar code) | - | - | N | <2 x i64>
5292 usra | bic + sub | sshr + neg | Y | <4 x i32>
5293 smull2 + smull + uzp2 | mls | - | N | <4 x i32>
5294 + sshr + usra | | | |
5295 -------------------------------------------------------------------------
5296 */
5297 if (Op2Info.isConstant() && Op2Info.isUniform()) {
5298 InstructionCost AddCost =
5299 getArithmeticInstrCost(Opcode: Instruction::Add, Ty, CostKind,
5300 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
5301 InstructionCost AsrCost =
5302 getArithmeticInstrCost(Opcode: Instruction::AShr, Ty, CostKind,
5303 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
5304 InstructionCost MulCost =
5305 getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind,
5306 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
5307 // add/cmp/csel/csneg should have similar cost while asr/negs/and should
5308 // have similar cost.
5309 auto VT = TLI->getValueType(DL, Ty);
5310 if (VT.isScalarInteger() && VT.getSizeInBits() <= 64) {
5311 if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) {
5312 // Neg can be folded into the asr instruction.
5313 return ISD == ISD::SDIV ? (3 * AddCost + AsrCost)
5314 : (3 * AsrCost + AddCost);
5315 } else {
5316 return MulCost + AsrCost + 2 * AddCost;
5317 }
5318 } else if (VT.isVector()) {
5319 InstructionCost UsraCost = 2 * AsrCost;
5320 if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) {
5321 // Division with scalable types corresponds to native 'asrd'
5322 // instruction when SVE is available.
5323 // e.g. %1 = sdiv <vscale x 4 x i32> %a, splat (i32 8)
5324
5325 // One more for the negation in SDIV
5326 InstructionCost Cost =
5327 (Op2Info.isNegatedPowerOf2() && ISD == ISD::SDIV) ? AsrCost : 0;
5328 if (Ty->isScalableTy() && ST->hasSVE())
5329 Cost += 2 * AsrCost;
5330 else {
5331 Cost +=
5332 UsraCost +
5333 (ISD == ISD::SDIV
5334 ? (LT.second.getScalarType() == MVT::i64 ? 1 : 2) * AsrCost
5335 : 2 * AddCost);
5336 }
5337 return Cost;
5338 } else if (LT.second == MVT::v2i64) {
5339 return VT.getVectorNumElements() *
5340 getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(), CostKind,
5341 Op1Info: Op1Info.getNoProps(),
5342 Op2Info: Op2Info.getNoProps());
5343 } else {
5344 // When SVE is available, we get:
5345 // smulh + lsr + add/sub + asr + add/sub.
5346 if (Ty->isScalableTy() && ST->hasSVE())
5347 return MulCost /*smulh cost*/ + 2 * AddCost + 2 * AsrCost;
5348 return 2 * MulCost + AddCost /*uzp2 cost*/ + AsrCost + UsraCost;
5349 }
5350 }
5351 }
5352 if (Op2Info.isConstant() && !Op2Info.isUniform() &&
5353 LT.second.isFixedLengthVector()) {
5354 // FIXME: When the constant vector is non-uniform, this may result in
5355 // loading the vector from constant pool or in some cases, may also result
5356 // in scalarization. For now, we are approximating this with the
5357 // scalarization cost.
5358 auto ExtractCost = 2 * getVectorInstrCost(Opcode: Instruction::ExtractElement, Ty,
5359 CostKind, Index: -1, Op0: nullptr, Op1: nullptr);
5360 auto InsertCost = getVectorInstrCost(Opcode: Instruction::InsertElement, Ty,
5361 CostKind, Index: -1, Op0: nullptr, Op1: nullptr);
5362 unsigned NElts = cast<FixedVectorType>(Val: Ty)->getNumElements();
5363 return ExtractCost + InsertCost +
5364 NElts * getArithmeticInstrCost(Opcode, Ty: Ty->getScalarType(),
5365 CostKind, Op1Info: Op1Info.getNoProps(),
5366 Op2Info: Op2Info.getNoProps());
5367 }
5368 [[fallthrough]];
5369 case ISD::UDIV:
5370 case ISD::UREM: {
5371 auto VT = TLI->getValueType(DL, Ty);
5372 if (Op2Info.isConstant()) {
5373 // If the operand is a power of 2 we can use the shift or and cost.
5374 if (ISD == ISD::UDIV && Op2Info.isPowerOf2())
5375 return getArithmeticInstrCost(Opcode: Instruction::LShr, Ty, CostKind,
5376 Op1Info: Op1Info.getNoProps(),
5377 Op2Info: Op2Info.getNoProps());
5378 if (ISD == ISD::UREM && Op2Info.isPowerOf2())
5379 return getArithmeticInstrCost(Opcode: Instruction::And, Ty, CostKind,
5380 Op1Info: Op1Info.getNoProps(),
5381 Op2Info: Op2Info.getNoProps());
5382
5383 if (ISD == ISD::UDIV || ISD == ISD::UREM) {
5384 // Divides by a constant are expanded to MULHU + SUB + SRL + ADD + SRL.
5385 // The MULHU will be expanded to UMULL for the types not listed below,
5386 // and will become a pair of UMULL+MULL2 for 128bit vectors.
5387 bool HasMULH = VT == MVT::i64 || LT.second == MVT::nxv2i64 ||
5388 LT.second == MVT::nxv4i32 || LT.second == MVT::nxv8i16 ||
5389 LT.second == MVT::nxv16i8;
5390 bool Is128bit = LT.second.is128BitVector();
5391
5392 InstructionCost MulCost =
5393 getArithmeticInstrCost(Opcode: Instruction::Mul, Ty, CostKind,
5394 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
5395 InstructionCost AddCost =
5396 getArithmeticInstrCost(Opcode: Instruction::Add, Ty, CostKind,
5397 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
5398 InstructionCost ShrCost =
5399 getArithmeticInstrCost(Opcode: Instruction::AShr, Ty, CostKind,
5400 Op1Info: Op1Info.getNoProps(), Op2Info: Op2Info.getNoProps());
5401 InstructionCost DivCost = MulCost * (Is128bit ? 2 : 1) + // UMULL/UMULH
5402 (HasMULH ? 0 : ShrCost) + // UMULL shift
5403 AddCost * 2 + ShrCost;
5404 return DivCost + (ISD == ISD::UREM ? MulCost + AddCost : 0);
5405 }
5406 }
5407
5408 // div i128's are lowered as libcalls. Pass nullptr as (u)divti3 calls are
5409 // emitted by the backend even when those functions are not declared in the
5410 // module.
5411 if (!VT.isVector() && VT.getSizeInBits() > 64)
5412 return getCallInstrCost(/*Function*/ F: nullptr, RetTy: Ty, Tys: {Ty, Ty}, CostKind);
5413
5414 InstructionCost Cost = BaseT::getArithmeticInstrCost(
5415 Opcode, Ty, CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
5416 if (Ty->isVectorTy() && (ISD == ISD::SDIV || ISD == ISD::UDIV)) {
5417 if (TLI->isOperationLegalOrCustom(Op: ISD, VT: LT.second) && ST->hasSVE()) {
5418 // SDIV/UDIV operations are lowered using SVE, then we can have less
5419 // costs.
5420 if (VT.isSimple() && isa<FixedVectorType>(Val: Ty) &&
5421 Ty->getPrimitiveSizeInBits().getFixedValue() < 128) {
5422 static const CostTblEntry DivTbl[]{
5423 {.ISD: ISD::SDIV, .Type: MVT::v2i8, .Cost: 5}, {.ISD: ISD::SDIV, .Type: MVT::v4i8, .Cost: 8},
5424 {.ISD: ISD::SDIV, .Type: MVT::v8i8, .Cost: 8}, {.ISD: ISD::SDIV, .Type: MVT::v2i16, .Cost: 5},
5425 {.ISD: ISD::SDIV, .Type: MVT::v4i16, .Cost: 5}, {.ISD: ISD::SDIV, .Type: MVT::v2i32, .Cost: 1},
5426 {.ISD: ISD::UDIV, .Type: MVT::v2i8, .Cost: 5}, {.ISD: ISD::UDIV, .Type: MVT::v4i8, .Cost: 8},
5427 {.ISD: ISD::UDIV, .Type: MVT::v8i8, .Cost: 8}, {.ISD: ISD::UDIV, .Type: MVT::v2i16, .Cost: 5},
5428 {.ISD: ISD::UDIV, .Type: MVT::v4i16, .Cost: 5}, {.ISD: ISD::UDIV, .Type: MVT::v2i32, .Cost: 1}};
5429
5430 const auto *Entry = CostTableLookup(Table: DivTbl, ISD, Ty: VT.getSimpleVT());
5431 if (nullptr != Entry)
5432 return Entry->Cost;
5433 }
5434 // A non-power-of-2 count can't divide as a single whole-register op
5435 // (an inactive lane's leftover value could be a zero divisor and
5436 // trap), so the legalizer emits one div per whole register plus one
5437 // per set bit of the remainder (e.g. <7 x i32> emits 3 divs, not 2).
5438 if (auto *FVTy = dyn_cast<FixedVectorType>(Val: Ty);
5439 FVTy && LT.second.isFixedLengthVector()) {
5440 unsigned NumElts = FVTy->getNumElements();
5441 unsigned RegElts = LT.second.getVectorNumElements();
5442 if (RegElts > 0)
5443 Cost = (NumElts / RegElts + popcount(Value: NumElts % RegElts)) * 2;
5444 }
5445 // For 8/16-bit elements, the cost is higher because the type
5446 // requires promotion and possibly splitting:
5447 if (LT.second.getScalarType() == MVT::i8)
5448 Cost *= 8;
5449 else if (LT.second.getScalarType() == MVT::i16)
5450 Cost *= 4;
5451 return Cost;
5452 } else {
5453 // If one of the operands is a uniform constant then the cost for each
5454 // element is Cost for insertion, extraction and division.
5455 // Insertion cost = 2, Extraction Cost = 2, Division = cost for the
5456 // operation with scalar type
5457 if ((Op1Info.isConstant() && Op1Info.isUniform()) ||
5458 (Op2Info.isConstant() && Op2Info.isUniform())) {
5459 if (auto *VTy = dyn_cast<FixedVectorType>(Val: Ty)) {
5460 InstructionCost DivCost = BaseT::getArithmeticInstrCost(
5461 Opcode, Ty: Ty->getScalarType(), CostKind, Opd1Info: Op1Info, Opd2Info: Op2Info);
5462 return (4 + DivCost) * VTy->getNumElements();
5463 }
5464 }
5465 // On AArch64, without SVE, vector divisions are expanded
5466 // into scalar divisions of each pair of elements.
5467 Cost += getVectorInstrCost(Opcode: Instruction::ExtractElement, Ty, CostKind,
5468 Index: -1, Op0: nullptr, Op1: nullptr);
5469 Cost += getVectorInstrCost(Opcode: Instruction::InsertElement, Ty, CostKind, Index: -1,
5470 Op0: nullptr, Op1: nullptr);
5471 }
5472
5473 // TODO: if one of the arguments is scalar, then it's not necessary to
5474 // double the cost of handling the vector elements.
5475 Cost += Cost;
5476 }
5477 return Cost;
5478 }
5479 case ISD::XOR:
5480 case ISD::OR:
5481 case ISD::AND:
5482 // TODO: revisit these costs as it's not accurate enough for non-uniform
5483 // constant.
5484 return LT.first;
5485 case ISD::SRL:
5486 case ISD::SRA:
5487 case ISD::SHL: {
5488 // Immediate vector shifts require uniform shift amounts. Non-uniform
5489 // constants therefore use variable shifts and require materializing the
5490 // shift vector. Account for a shift and materialization per legalized
5491 // vector, together with shared setup.
5492 // This cost is for (ldr, shl) + adrp
5493 // TODO: These costs are based on CodeSize only, consider other CostKinds.
5494 if (Op2Info.isConstant() && !Op2Info.isUniform() &&
5495 LT.second.isFixedLengthVector())
5496 return 2 * LT.first + 1;
5497
5498 // Marked 'custom' for combining purposes; a uniform shift amount still
5499 // lowers to a single legal instruction.
5500 return LT.first;
5501 }
5502
5503 case ISD::FNEG:
5504 // Scalar fmul(fneg) or fneg(fmul) can be converted to fnmul
5505 if ((Ty->isFloatTy() || Ty->isDoubleTy() ||
5506 (Ty->isHalfTy() && ST->hasFullFP16())) &&
5507 CtxI &&
5508 ((CtxI->hasOneUse() &&
5509 match(V: *CtxI->user_begin(), P: m_FMul(L: m_Value(), R: m_Value()))) ||
5510 match(V: CtxI->getOperand(i: 0), P: m_FMul(L: m_Value(), R: m_Value()))))
5511 return 0;
5512 [[fallthrough]];
5513 case ISD::FADD:
5514 case ISD::FSUB:
5515 if (!Ty->getScalarType()->isFP128Ty())
5516 return LT.first;
5517 [[fallthrough]];
5518 case ISD::FMUL:
5519 case ISD::FDIV:
5520 // These nodes are marked as 'custom' just to lower them to SVE.
5521 // We know said lowering will incur no additional cost.
5522 if (!Ty->getScalarType()->isFP128Ty())
5523 return 2 * LT.first;
5524
5525 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
5526 Opd2Info: Op2Info);
5527 case ISD::FREM:
5528 // Pass nullptr as fmod/fmodf calls are emitted by the backend even when
5529 // those functions are not declared in the module.
5530 if (!Ty->isVectorTy())
5531 return getCallInstrCost(/*Function*/ F: nullptr, RetTy: Ty, Tys: {Ty, Ty}, CostKind);
5532 return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Opd1Info: Op1Info,
5533 Opd2Info: Op2Info);
5534 }
5535}
5536
5537InstructionCost
5538AArch64TTIImpl::getAddressComputationCost(Type *PtrTy, ScalarEvolution *SE,
5539 const SCEV *Ptr,
5540 TTI::TargetCostKind CostKind) const {
5541 // Address computations in vectorized code with non-consecutive addresses will
5542 // likely result in more instructions compared to scalar code where the
5543 // computation can more often be merged into the index mode. The resulting
5544 // extra micro-ops can significantly decrease throughput.
5545 unsigned NumVectorInstToHideOverhead =
5546 ST->getCLOpts().neon_nonconst_stride_overhead;
5547 int MaxMergeDistance = 64;
5548
5549 if (PtrTy->isVectorTy() && SE &&
5550 !BaseT::isConstantStridedAccessLessThan(SE, Ptr, MergeDistance: MaxMergeDistance + 1))
5551 return NumVectorInstToHideOverhead;
5552
5553 // In many cases the address computation is not merged into the instruction
5554 // addressing mode.
5555 return 1;
5556}
5557
5558/// Check whether Opcode1 has less throughput according to the scheduling
5559/// model than Opcode2.
5560bool AArch64TTIImpl::hasKnownLowerThroughputFromSchedulingModel(
5561 unsigned Opcode1, unsigned Opcode2) const {
5562 const MCSchedModel &Sched = ST->getSchedModel();
5563 const TargetInstrInfo *TII = ST->getInstrInfo();
5564 if (!Sched.hasInstrSchedModel())
5565 return false;
5566
5567 const MCSchedClassDesc *SCD1 =
5568 Sched.getSchedClassDesc(SchedClassIdx: TII->get(Opcode: Opcode1).getSchedClass());
5569 const MCSchedClassDesc *SCD2 =
5570 Sched.getSchedClassDesc(SchedClassIdx: TII->get(Opcode: Opcode2).getSchedClass());
5571 // We cannot handle variant scheduling classes without an MI. If we need to
5572 // support them for any of the instructions we query the information of we
5573 // might need to add a way to resolve them without a MI or not use the
5574 // scheduling info.
5575 assert(!SCD1->isVariant() && !SCD2->isVariant() &&
5576 "Cannot handle variant scheduling classes without an MI");
5577 if (!SCD1->isValid() || !SCD2->isValid())
5578 return false;
5579
5580 return MCSchedModel::getReciprocalThroughput(STI: *ST, SCDesc: *SCD1) >
5581 MCSchedModel::getReciprocalThroughput(STI: *ST, SCDesc: *SCD2);
5582}
5583
5584InstructionCost AArch64TTIImpl::getCmpSelInstrCost(
5585 unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred,
5586 TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info,
5587 TTI::OperandValueInfo Op2Info, const Instruction *I) const {
5588 // We don't lower some vector selects well that are wider than the register
5589 // width. TODO: Improve this with different cost kinds.
5590 if (isa<FixedVectorType>(Val: ValTy) && Opcode == Instruction::Select) {
5591 // We would need this many instructions to hide the scalarization happening.
5592 const int AmortizationCost = 20;
5593
5594 // If VecPred is not set, check if we can get a predicate from the context
5595 // instruction, if its type matches the requested ValTy.
5596 if (VecPred == CmpInst::BAD_ICMP_PREDICATE && I && I->getType() == ValTy) {
5597 CmpPredicate CurrentPred;
5598 if (match(V: I, P: m_Select(C: m_Cmp(Pred&: CurrentPred, L: m_Value(), R: m_Value()), L: m_Value(),
5599 R: m_Value())))
5600 VecPred = CurrentPred;
5601 }
5602 // Check if we have a compare/select chain that can be lowered using
5603 // a (F)CMxx & BFI pair.
5604 if (CmpInst::isIntPredicate(P: VecPred) || VecPred == CmpInst::FCMP_OLE ||
5605 VecPred == CmpInst::FCMP_OLT || VecPred == CmpInst::FCMP_OGT ||
5606 VecPred == CmpInst::FCMP_OGE || VecPred == CmpInst::FCMP_OEQ ||
5607 VecPred == CmpInst::FCMP_UNE) {
5608 static const auto ValidMinMaxTys = {
5609 MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
5610 MVT::v4i32, MVT::v2i64, MVT::v2f32, MVT::v4f32, MVT::v2f64};
5611 static const auto ValidFP16MinMaxTys = {MVT::v4f16, MVT::v8f16};
5612
5613 auto LT = getTypeLegalizationCost(Ty: ValTy);
5614 if (any_of(Range: ValidMinMaxTys, P: equal_to(Arg&: LT.second)) ||
5615 (ST->hasFullFP16() &&
5616 any_of(Range: ValidFP16MinMaxTys, P: equal_to(Arg&: LT.second))))
5617 return LT.first;
5618 }
5619
5620 static const TypeConversionCostTblEntry VectorSelectTbl[] = {
5621 {.ISD: Instruction::Select, .Dst: MVT::v2i1, .Src: MVT::v2f32, .Cost: 2},
5622 {.ISD: Instruction::Select, .Dst: MVT::v2i1, .Src: MVT::v2f64, .Cost: 2},
5623 {.ISD: Instruction::Select, .Dst: MVT::v4i1, .Src: MVT::v4f32, .Cost: 2},
5624 {.ISD: Instruction::Select, .Dst: MVT::v4i1, .Src: MVT::v4f16, .Cost: 2},
5625 {.ISD: Instruction::Select, .Dst: MVT::v8i1, .Src: MVT::v8f16, .Cost: 2},
5626 {.ISD: Instruction::Select, .Dst: MVT::v16i1, .Src: MVT::v16i16, .Cost: 16},
5627 {.ISD: Instruction::Select, .Dst: MVT::v8i1, .Src: MVT::v8i32, .Cost: 8},
5628 {.ISD: Instruction::Select, .Dst: MVT::v16i1, .Src: MVT::v16i32, .Cost: 16},
5629 {.ISD: Instruction::Select, .Dst: MVT::v4i1, .Src: MVT::v4i64, .Cost: 4 * AmortizationCost},
5630 {.ISD: Instruction::Select, .Dst: MVT::v8i1, .Src: MVT::v8i64, .Cost: 8 * AmortizationCost},
5631 {.ISD: Instruction::Select, .Dst: MVT::v16i1, .Src: MVT::v16i64, .Cost: 16 * AmortizationCost}};
5632
5633 EVT SelCondTy = TLI->getValueType(DL, Ty: CondTy);
5634 EVT SelValTy = TLI->getValueType(DL, Ty: ValTy);
5635 if (SelCondTy.isSimple() && SelValTy.isSimple()) {
5636 if (const auto *Entry = ConvertCostTableLookup(Table: VectorSelectTbl, ISD: Opcode,
5637 Dst: SelCondTy.getSimpleVT(),
5638 Src: SelValTy.getSimpleVT()))
5639 return Entry->Cost;
5640 }
5641 }
5642
5643 if (Opcode == Instruction::FCmp) {
5644 if (auto PromotedCost = getFP16BF16PromoteCost(
5645 Ty: ValTy, CostKind, Op1Info, Op2Info, /*IncludeTrunc=*/false,
5646 // TODO: Consider costing SVE FCMPs.
5647 /*CanUseSVE=*/false, InstCost: [&](Type *PromotedTy) {
5648 InstructionCost Cost =
5649 getCmpSelInstrCost(Opcode, ValTy: PromotedTy, CondTy, VecPred,
5650 CostKind, Op1Info, Op2Info);
5651 if (isa<VectorType>(Val: PromotedTy))
5652 Cost += getCastInstrCost(
5653 Opcode: Instruction::Trunc,
5654 Dst: VectorType::getInteger(VTy: cast<VectorType>(Val: ValTy)),
5655 Src: VectorType::getInteger(VTy: cast<VectorType>(Val: PromotedTy)),
5656 CCH: TTI::CastContextHint::None, CostKind);
5657 return Cost;
5658 }))
5659 return *PromotedCost;
5660
5661 auto LT = getTypeLegalizationCost(Ty: ValTy);
5662 // Model unknown fp compares as a libcall.
5663 if (LT.second.getScalarType() != MVT::f64 &&
5664 LT.second.getScalarType() != MVT::f32 &&
5665 LT.second.getScalarType() != MVT::f16)
5666 return LT.first * getCallInstrCost(/*Function*/ F: nullptr, RetTy: ValTy,
5667 Tys: {ValTy, ValTy}, CostKind);
5668
5669 // Some comparison operators require expanding to multiple compares + or.
5670 unsigned Factor = 1;
5671 if (!CondTy->isVectorTy() &&
5672 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ))
5673 Factor = 2; // fcmp with 2 selects
5674 else if (isa<FixedVectorType>(Val: ValTy) &&
5675 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ ||
5676 VecPred == FCmpInst::FCMP_ORD || VecPred == FCmpInst::FCMP_UNO))
5677 Factor = 3; // fcmxx+fcmyy+or
5678 else if (isa<ScalableVectorType>(Val: ValTy) &&
5679 (VecPred == FCmpInst::FCMP_ONE || VecPred == FCmpInst::FCMP_UEQ))
5680 Factor = 3; // fcmxx+fcmyy+or
5681
5682 if (isa<ScalableVectorType>(Val: ValTy) &&
5683 CostKind == TTI::TCK_RecipThroughput &&
5684 hasKnownLowerThroughputFromSchedulingModel(Opcode1: AArch64::FCMEQ_PPzZZ_S,
5685 Opcode2: AArch64::FCMEQv4f32))
5686 Factor *= 2;
5687
5688 return Factor * (CostKind == TTI::TCK_Latency ? 2 : LT.first);
5689 }
5690
5691 // Treat the icmp in icmp(and, 0) or icmp(and, -1/1) when it can be folded to
5692 // icmp(and, 0) as free, as we can make use of ands, but only if the
5693 // comparison is not unsigned. FIXME: Enable for non-throughput cost kinds
5694 // providing it will not cause performance regressions.
5695 if (CostKind == TTI::TCK_RecipThroughput && ValTy->isIntegerTy() &&
5696 Opcode == Instruction::ICmp && I && !CmpInst::isUnsigned(Pred: VecPred) &&
5697 TLI->isTypeLegal(VT: TLI->getValueType(DL, Ty: ValTy)) &&
5698 match(V: I->getOperand(i: 0), P: m_And(L: m_Value(), R: m_Value()))) {
5699 if (match(V: I->getOperand(i: 1), P: m_Zero()))
5700 return 0;
5701
5702 // x >= 1 / x < 1 -> x > 0 / x <= 0
5703 if (match(V: I->getOperand(i: 1), P: m_One()) &&
5704 (VecPred == CmpInst::ICMP_SLT || VecPred == CmpInst::ICMP_SGE))
5705 return 0;
5706
5707 // x <= -1 / x > -1 -> x > 0 / x <= 0
5708 if (match(V: I->getOperand(i: 1), P: m_AllOnes()) &&
5709 (VecPred == CmpInst::ICMP_SLE || VecPred == CmpInst::ICMP_SGT))
5710 return 0;
5711 }
5712
5713 // The base case handles scalable vectors fine for now, since it treats the
5714 // cost as 1 * legalization cost.
5715 return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
5716 Op1Info, Op2Info, I);
5717}
5718
5719AArch64TTIImpl::TTI::MemCmpExpansionOptions
5720AArch64TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
5721 TTI::MemCmpExpansionOptions Options;
5722 if (ST->requiresStrictAlign()) {
5723 // TODO: Add cost modeling for strict align. Misaligned loads expand to
5724 // a bunch of instructions when strict align is enabled.
5725 return Options;
5726 }
5727 Options.AllowOverlappingLoads = true;
5728 Options.MaxNumLoads = TLI->getMaxExpandSizeMemcmp(OptSize);
5729 Options.NumLoadsPerBlock = IsZeroCmp ? Options.MaxNumLoads : 1;
5730 // TODO: Though vector loads usually perform well on AArch64, in some targets
5731 // they may wake up the FP unit, which raises the power consumption. Perhaps
5732 // they could be used with no holds barred (-O3).
5733 Options.LoadSizes = {8, 4, 2, 1};
5734 Options.AllowedTailExpansions = {3, 5, 6};
5735 return Options;
5736}
5737
5738bool AArch64TTIImpl::prefersVectorizedAddressing() const {
5739 return ST->hasSVE();
5740}
5741
5742InstructionCost
5743AArch64TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA,
5744 TTI::TargetCostKind CostKind) const {
5745 switch (MICA.getID()) {
5746 case Intrinsic::masked_scatter:
5747 case Intrinsic::masked_gather:
5748 return getGatherScatterOpCost(MICA, CostKind);
5749 case Intrinsic::masked_load:
5750 case Intrinsic::masked_store:
5751 case Intrinsic::masked_expandload:
5752 case Intrinsic::masked_compressstore:
5753 return getMaskedMemoryOpCost(MICA, CostKind);
5754 }
5755 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5756}
5757
5758InstructionCost
5759AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
5760 TTI::TargetCostKind CostKind) const {
5761 Type *Src = MICA.getDataType();
5762
5763 if (useNeonVector(Ty: Src))
5764 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5765 auto LT = getTypeLegalizationCost(Ty: Src);
5766 if (!LT.first.isValid())
5767 return InstructionCost::getInvalid();
5768
5769 // Return an invalid cost for element types that we are unable to lower.
5770 auto *VT = cast<VectorType>(Val: Src);
5771 if (VT->getElementType()->isIntegerTy(BitWidth: 1))
5772 return InstructionCost::getInvalid();
5773
5774 // <vscale x 1 x eltty> operations require mask adaptation.
5775 // Allow it for normal masked ld/st
5776 if (VT->getElementCount() == ElementCount::getScalable(MinVal: 1))
5777 return is_contained(Set: {Intrinsic::masked_load, Intrinsic::masked_store},
5778 Element: MICA.getID())
5779 ? LT.first + 1
5780 : InstructionCost::getInvalid();
5781
5782 InstructionCost MemOpCost = LT.first;
5783 if (MICA.getID() == Intrinsic::masked_expandload) {
5784 if (!isLegalMaskedExpandLoad(DataTy: Src, Alignment: MICA.getAlignment()))
5785 return InstructionCost::getInvalid();
5786
5787 // Operation will be split into expand of masked.load
5788 // Something like:
5789 // cntp x8, p0, p0.s
5790 // whilelo p1.s, xzr, x8
5791 // ld1w { z1.s }, p1/z, [x0]
5792 // expand z1.s, p0, z1.s
5793 MemOpCost *= 4;
5794 }
5795
5796 if (MICA.getID() == Intrinsic::masked_compressstore) {
5797 if (!isLegalMaskedCompressStore(DataType: Src, Alignment: MICA.getAlignment()))
5798 return InstructionCost::getInvalid();
5799
5800 // A compress store lowers to something like:
5801 // compact z0.s, p0, z0.s
5802 // cntp x8, p1, p0.s
5803 // whilelo p0.s, xzr, x8
5804 // st1w { z0.s }, p0, [x0]
5805 MemOpCost *= 4;
5806 }
5807
5808 // If we need to split the memory operation, we will also need to split the
5809 // mask. This will likely lead to overestimating the cost in some cases if
5810 // multiple memory operations use the same mask, but we often don't have
5811 // enough context to figure that out here.
5812 //
5813 // If the elements being loaded are bytes then the mask will already be split,
5814 // since the number of bits in a P register matches the number of bytes in a
5815 // Z register.
5816 if (LT.first > 1 && LT.second.getScalarSizeInBits() > 8)
5817 return MemOpCost * 2;
5818
5819 return MemOpCost;
5820}
5821
5822// This function returns gather/scatter overhead either from
5823// user-provided value or specialized values per-target from \p ST.
5824static unsigned getSVEGatherScatterOverhead(unsigned Opcode,
5825 const AArch64Subtarget *ST) {
5826 assert((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
5827 "Should be called on only load or stores.");
5828 switch (Opcode) {
5829 case Instruction::Load:
5830 return ST->getCLOpts().sve_gather_overhead.value_or(
5831 u: ST->getGatherOverhead());
5832 break;
5833 case Instruction::Store:
5834 return ST->getCLOpts().sve_scatter_overhead.value_or(
5835 u: ST->getScatterOverhead());
5836 break;
5837 default:
5838 llvm_unreachable("Shouldn't have reached here");
5839 }
5840}
5841
5842InstructionCost
5843AArch64TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA,
5844 TTI::TargetCostKind CostKind) const {
5845
5846 unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather ||
5847 MICA.getID() == Intrinsic::vp_gather)
5848 ? Instruction::Load
5849 : Instruction::Store;
5850
5851 Type *DataTy = MICA.getDataType();
5852 Align Alignment = MICA.getAlignment();
5853 const Instruction *I = MICA.getInst();
5854
5855 if (useNeonVector(Ty: DataTy) || !isLegalMaskedGatherScatter(DataType: DataTy))
5856 return BaseT::getMemIntrinsicInstrCost(MICA, CostKind);
5857 auto *VT = cast<VectorType>(Val: DataTy);
5858 auto LT = getTypeLegalizationCost(Ty: DataTy);
5859 if (!LT.first.isValid())
5860 return InstructionCost::getInvalid();
5861
5862 // Return an invalid cost for element types that we are unable to lower.
5863 if (!LT.second.isVector() ||
5864 !isElementTypeLegalForScalableVector(Ty: VT->getElementType()) ||
5865 VT->getElementType()->isIntegerTy(BitWidth: 1))
5866 return InstructionCost::getInvalid();
5867
5868 // The code-generator is currently not able to handle scalable vectors
5869 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
5870 // it. This change will be removed when code-generation for these types is
5871 // sufficiently reliable.
5872 if (VT->getElementCount() == ElementCount::getScalable(MinVal: 1))
5873 return InstructionCost::getInvalid();
5874
5875 ElementCount LegalVF = LT.second.getVectorElementCount();
5876 InstructionCost MemOpCost =
5877 getMemoryOpCost(Opcode, Src: VT->getElementType(), Alignment, AddressSpace: 0, CostKind,
5878 OpInfo: {.Kind: TTI::OK_AnyValue, .Properties: TTI::OP_None}, I);
5879 // Add on an overhead cost for using gathers/scatters.
5880 MemOpCost *= getSVEGatherScatterOverhead(Opcode, ST);
5881 return LT.first * MemOpCost * getMaxNumElements(VF: LegalVF);
5882}
5883
5884bool AArch64TTIImpl::useNeonVector(const Type *Ty) const {
5885 return isa<FixedVectorType>(Val: Ty) && !ST->useSVEForFixedLengthVectors();
5886}
5887
5888InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
5889 Align Alignment,
5890 unsigned AddressSpace,
5891 TTI::TargetCostKind CostKind,
5892 TTI::OperandValueInfo OpInfo,
5893 const Instruction *I) const {
5894 EVT VT = TLI->getValueType(DL, Ty, AllowUnknown: true);
5895 // Type legalization can't handle structs
5896 if (VT == MVT::Other)
5897 return BaseT::getMemoryOpCost(Opcode, Src: Ty, Alignment, AddressSpace,
5898 CostKind);
5899
5900 auto LT = getTypeLegalizationCost(Ty);
5901 if (!LT.first.isValid())
5902 return InstructionCost::getInvalid();
5903
5904 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: Ty)) {
5905
5906 // We only support full register predicate loads and stores.
5907 if (VTy->getElementType()->isIntegerTy(BitWidth: 1) &&
5908 !VTy->getElementCount().isKnownMultipleOf(
5909 RHS: ElementCount::getScalable(MinVal: 16)))
5910 return InstructionCost::getInvalid();
5911
5912 // <vscale x 1 x eltty> operations require crafting a new mask.
5913 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1)) {
5914 Intrinsic::ID IID = Opcode == Instruction::Load ? Intrinsic::masked_load
5915 : Intrinsic::masked_store;
5916 return getMaskedMemoryOpCost(
5917 MICA: MemIntrinsicCostAttributes(IID, Ty, Alignment, AddressSpace),
5918 CostKind) +
5919 1;
5920 }
5921 }
5922
5923 // TODO: consider latency as well for TCK_SizeAndLatency.
5924 if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency)
5925 return LT.first;
5926
5927 if (CostKind == TTI::TCK_Latency) {
5928 // Latency doesn't make much sense for stores, so just return 1
5929 if (Opcode == Instruction::Store)
5930 return 1;
5931 // If the subtarget has overridden the load latency then use that instead of
5932 // querying the SchedModel.
5933 if (ST->getFixedLoadLatency())
5934 return (LT.first - 1) + ST->getFixedLoadLatency();
5935 // We expect the load to become LT.first loads of type LT.second. The
5936 // latency will be the latency of the last load plus the time it gets to get
5937 // there, which will be the amount of other loads before that (i.e. total
5938 // loads - 1) multiplied by how long it takes to get through them (the
5939 // reciprocal of the throughput). We get the latency and reciprocal
5940 // throughput from the SchedModel, and assume that the loads become the
5941 // variant with unsigned integer offset.
5942 unsigned Inst = 0;
5943 if (LT.second.isScalableVector() ||
5944 ST->useSVEForFixedLengthVectors(VT: LT.second)) {
5945 Inst = AArch64::LDR_ZXI;
5946 } else if (LT.second.isVector() || LT.second.isFloatingPoint()) {
5947 switch (LT.second.getSizeInBits()) {
5948 case 8:
5949 Inst = AArch64::LDRBui;
5950 break;
5951 case 16:
5952 Inst = AArch64::LDRHui;
5953 break;
5954 case 32:
5955 Inst = AArch64::LDRSui;
5956 break;
5957 case 64:
5958 Inst = AArch64::LDRDui;
5959 break;
5960 case 128:
5961 Inst = AArch64::LDRQui;
5962 break;
5963 default:
5964 llvm_unreachable("Unexpected float or vector type");
5965 }
5966 } else {
5967 switch (LT.second.getSizeInBits()) {
5968 case 8:
5969 Inst = AArch64::LDRBBui;
5970 break;
5971 case 16:
5972 Inst = AArch64::LDRHHui;
5973 break;
5974 case 32:
5975 Inst = AArch64::LDRWui;
5976 break;
5977 case 64:
5978 Inst = AArch64::LDRXui;
5979 break;
5980 default:
5981 llvm_unreachable("Unexpected integer type");
5982 }
5983 }
5984 const MCSchedModel &Sched = ST->getSchedModel();
5985 const TargetInstrInfo *TII = ST->getInstrInfo();
5986 unsigned SchedClass = TII->get(Opcode: Inst).getSchedClass();
5987 const MCSchedClassDesc *SCD = Sched.hasInstrSchedModel()
5988 ? Sched.getSchedClassDesc(SchedClassIdx: SchedClass)
5989 : nullptr;
5990 // If the cpu has no scheduling model, or it doesn't describe the load, then
5991 // fall back to the default load latency. Variant scheduling classes can't
5992 // be resolved without a MachineInstr, so treat them the same way.
5993 if (!SCD || !SCD->isValid() || SCD->isVariant())
5994 return (LT.first - 1) + ST->getLoadLatency();
5995 // We need to convert the number of loads before the last to a float here,
5996 // as the reciprocal throughput may be fractional.
5997 float NumLoads = (LT.first - 1).getValue();
5998 return NumLoads * Sched.getReciprocalThroughput(STI: *ST, SCDesc: *SCD) +
5999 Sched.computeInstrLatency(STI: *ST, SCDesc: *SCD);
6000 }
6001
6002 if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store &&
6003 LT.second.is128BitVector() && Alignment < Align(16)) {
6004 // Unaligned stores are extremely inefficient. We don't split all
6005 // unaligned 128-bit stores because the negative impact that has shown in
6006 // practice on inlined block copy code.
6007 // We make such stores expensive so that we will only vectorize if there
6008 // are 6 other instructions getting vectorized.
6009 const int AmortizationCost = 6;
6010
6011 return LT.first * 2 * AmortizationCost;
6012 }
6013
6014 // Opaque ptr or ptr vector types are i64s and can be lowered to STP/LDPs.
6015 if (Ty->isPtrOrPtrVectorTy())
6016 return LT.first;
6017
6018 if (useNeonVector(Ty)) {
6019 // Check truncating stores and extending loads.
6020 if (Ty->getScalarSizeInBits() != LT.second.getScalarSizeInBits()) {
6021 // v4i8 types are lowered to scalar a load/store and sshll/xtn.
6022 if (VT == MVT::v4i8)
6023 return 2;
6024 // Otherwise we need to scalarize.
6025 return cast<FixedVectorType>(Val: Ty)->getNumElements() * 2;
6026 }
6027 EVT EltVT = VT.getVectorElementType();
6028 unsigned EltSize = EltVT.getScalarSizeInBits();
6029 if (!isPowerOf2_32(Value: EltSize) || EltSize < 8 || EltSize > 64 ||
6030 Alignment != Align(1))
6031 return LT.first;
6032 // FIXME: v3i8 lowering currently is very inefficient, due to automatic
6033 // widening to v4i8, which produces suboptimal results.
6034 if (VT.getVectorNumElements() == 3 && EltVT == MVT::i8)
6035 return LT.first;
6036
6037 // Check non-power-of-2 loads/stores for legal vector element types with
6038 // NEON. Non-power-of-2 memory ops will get broken down to a set of
6039 // operations on smaller power-of-2 ops, including ld1/st1.
6040 InstructionCost Cost = VT.getVectorNumElements() / (128 / EltSize);
6041 unsigned Remainder = VT.getVectorNumElements() % (128 / EltSize);
6042 if (Remainder != 0) {
6043 SmallVector<std::pair<unsigned, unsigned>> TypeWorklist;
6044 TypeWorklist.push_back(Elt: {Remainder, 0});
6045 while (!TypeWorklist.empty()) {
6046 auto [CurrNumElements, Offset] = TypeWorklist.pop_back_val();
6047 if (isPowerOf2_32(Value: CurrNumElements)) {
6048 // 1 per load/store + possible lane insert.
6049 Cost += 1 + (Offset == 0 ? 0 : 1);
6050 continue;
6051 }
6052
6053 unsigned PrevPow2 = NextPowerOf2(A: CurrNumElements) / 2;
6054 TypeWorklist.push_back(Elt: {PrevPow2, Offset});
6055 TypeWorklist.push_back(Elt: {CurrNumElements - PrevPow2, Offset + PrevPow2});
6056 }
6057 }
6058 return Cost;
6059 }
6060
6061 return LT.first;
6062}
6063
6064InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
6065 unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
6066 Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
6067 bool UseMaskForCond, bool UseMaskForGaps) const {
6068 assert(Factor >= 2 && "Invalid interleave factor");
6069 auto *VecVTy = cast<VectorType>(Val: VecTy);
6070
6071 if (VecTy->isScalableTy() && !ST->hasSVE())
6072 return InstructionCost::getInvalid();
6073
6074 // Scalable VFs emit vector.[de]interleave intrinsics, for which the target
6075 // supports factors up to the maximum supported interleave factor.
6076 if (VecTy->isScalableTy()) {
6077 if (Factor > TLI->getMaxSupportedInterleaveFactor())
6078 return InstructionCost::getInvalid();
6079
6080 if (Factor == 3 &&
6081 DL.getTypeSizeInBits(Ty: VecTy).getKnownMinValue() != (3 * 128))
6082 return InstructionCost::getInvalid();
6083 }
6084
6085 auto LT = getTypeLegalizationCost(Ty: VecTy);
6086 unsigned MaxNativeInterleaveFactor = TLI->getMaxSupportedInterleaveFactor();
6087 // Vectorization for masked interleaved accesses is only enabled for scalable
6088 // VF. For fixed-length SVE, avoid non-native interleave factor because
6089 // the generic fallback costs wide fixed-vector shuffles too optimistically.
6090 if (!VecTy->isScalableTy() &&
6091 (UseMaskForCond || UseMaskForGaps ||
6092 (Factor > MaxNativeInterleaveFactor &&
6093 TLI->useSVEForFixedLengthVectorVT(VT: LT.second))))
6094 return InstructionCost::getInvalid();
6095
6096 if (!UseMaskForGaps && Factor <= MaxNativeInterleaveFactor) {
6097 ElementCount EC = VecVTy->getElementCount();
6098 auto *SubVecTy = VectorType::get(ElementType: VecVTy->getElementType(),
6099 EC: EC.divideCoefficientBy(RHS: Factor));
6100
6101 // ldN/stN only support legal vector types of size 64 or 128 in bits.
6102 // Accesses having vector types that are a multiple of 128 bits can be
6103 // matched to more than one ldN/stN instruction.
6104 bool UseScalable;
6105 if (EC.isKnownMultipleOf(RHS: Factor) &&
6106 TLI->isLegalInterleavedAccessType(VecTy: SubVecTy, DL, UseScalable))
6107 return Factor * TLI->getNumInterleavedAccesses(VecTy: SubVecTy, DL, UseScalable);
6108
6109 // Cost the alternative approach for scalable vectors where the interleave
6110 // factor is larger than the VF: use a contiguous load/store of the full
6111 // wide vector followed by deinterleave/interleave shuffles.
6112 if (VecTy->isScalableTy() && EC.isKnownMultipleOf(RHS: Factor)) {
6113 if (SubVecTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
6114 return InstructionCost::getInvalid();
6115
6116 // Cost of the contiguous memory operation on the wide vector.
6117 InstructionCost MemCost;
6118 if (UseMaskForCond) {
6119 unsigned IID = Opcode == Instruction::Load ? Intrinsic::masked_load
6120 : Intrinsic::masked_store;
6121 MemCost = getMemIntrinsicInstrCost(
6122 MICA: MemIntrinsicCostAttributes(IID, VecTy, Alignment, AddressSpace),
6123 CostKind);
6124 } else {
6125 MemCost =
6126 getMemoryOpCost(Opcode, Ty: VecTy, Alignment, AddressSpace, CostKind);
6127 }
6128
6129 // llvm.vector.deinterleaveN is lowered as a binary tree of deinterleave2
6130 // operations. The tree has Log2(Factor) levels, with Factor UZP/ZIP
6131 // operations at each level, giving a total shuffle cost of
6132 // Factor * Log2(Factor).
6133 auto SubVecCost = getTypeLegalizationCost(Ty: SubVecTy);
6134 auto ResultCost = getTypeLegalizationCost(Ty: VecTy);
6135 llvm::InstructionCost LegalizationCost = SubVecCost.first;
6136
6137 // FIXME: A temporary increase to the cost in cases where the input
6138 // element type is 4x the output type. Otherwise it produces an SVE tail
6139 // loop which is significantly larger than the NEON equivalent.
6140 if (Opcode == Instruction::Store && Factor == 4 &&
6141 SubVecCost.second.getScalarSizeInBits() ==
6142 (4 * ResultCost.second.getScalarSizeInBits()))
6143 LegalizationCost *= 4;
6144
6145 return MemCost + (Factor * LegalizationCost) + (Factor * Log2_64(Value: Factor));
6146 }
6147 }
6148
6149 return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
6150 Alignment, AddressSpace, CostKind,
6151 UseMaskForCond, UseMaskForGaps);
6152}
6153
6154InstructionCost
6155AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const {
6156 InstructionCost Cost = 0;
6157 TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
6158 for (auto *I : Tys) {
6159 if (!I->isVectorTy())
6160 continue;
6161 if (I->getScalarSizeInBits() * cast<FixedVectorType>(Val: I)->getNumElements() ==
6162 128)
6163 Cost += getMemoryOpCost(Opcode: Instruction::Store, Ty: I, Alignment: Align(128), AddressSpace: 0, CostKind) +
6164 getMemoryOpCost(Opcode: Instruction::Load, Ty: I, Alignment: Align(128), AddressSpace: 0, CostKind);
6165 }
6166 return Cost;
6167}
6168
6169bool AArch64TTIImpl::isLegalMaskedExpandLoad(Type *DataTy,
6170 Align Alignment) const {
6171 // Neon types should be scalarised when we are not choosing to use SVE.
6172 if (useNeonVector(Ty: DataTy))
6173 return false;
6174
6175 // Return true only if we are able to lower using the SVE2p2/SME2p2
6176 // expand instruction.
6177 return (ST->isSVEAvailable() && ST->hasSVE2p2()) ||
6178 (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2());
6179}
6180
6181bool AArch64TTIImpl::isLegalSpeculativeLoad(Type *DataType,
6182 unsigned AddressSpace) const {
6183 // Matches AArch64TargetLowering::emitCanLoadSpeculatively: only address
6184 // space 0 and power-of-2 sizes up to the 16-byte MTE tag granule.
6185 // TODO: Support scalable vectors.
6186 if (AddressSpace != 0)
6187 return false;
6188 TypeSize Size = DL.getTypeStoreSize(Ty: DataType);
6189 return !Size.isScalable() && isPowerOf2_64(Value: Size.getFixedValue()) &&
6190 Size.getFixedValue() <= 16;
6191}
6192
6193bool AArch64TTIImpl::hasMultiVectorLoadStore(
6194 unsigned NumVectors, TTI::MaskSource Mask, VectorType *VectorTy,
6195 bool IsStore, std::optional<Instruction::CastOps> CastHint) const {
6196 if (NumVectors <= 1 || !ST->enableSubRegLiveness() || !ST->hasSVE2p1())
6197 return false;
6198
6199 // TODO: Support masked multi-vector loads/stores.
6200 if (Mask != TTI::MaskSource::None)
6201 return false;
6202
6203 // A null vector type queries whether the target supports multi-vector memory
6204 // operations in general.
6205 if (!VectorTy)
6206 return true;
6207
6208 if (!isa<ScalableVectorType>(Val: VectorTy))
6209 return false;
6210
6211 // Conservatively, avoid using multi-vector loads when it's possible we could
6212 // use extending loads instead. Note: We can ignore stores as we only use
6213 // truncating stores when the store vector-width is < a full SVE vector.
6214 if (!IsStore &&
6215 (CastHint == Instruction::ZExt || CastHint == Instruction::SExt))
6216 return false;
6217
6218 // For unpredicated loads/stores allow any pow-of-two multiple of a vector >=
6219 // to a single z-register. We can split operations wider than a single
6220 // multi-vector load/store during ISEL.
6221 return isPowerOf2_32(Value: NumVectors) &&
6222 DL.getTypeSizeInBits(Ty: VectorTy).isKnownMultipleOf(
6223 RHS: AArch64::SVEBitsPerBlock);
6224}
6225
6226unsigned
6227AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF,
6228 bool HasUnorderedReductions) const {
6229 if (VF.isScalar() || (HasUnorderedReductions && VF.getKnownMinValue() <= 4))
6230 return 4;
6231 return ST->getMaxInterleaveFactor();
6232}
6233
6234// For Falkor, we want to avoid having too many strided loads in a loop since
6235// that can exhaust the HW prefetcher resources. We adjust the unroller
6236// MaxCount preference below to attempt to ensure unrolling doesn't create too
6237// many strided loads.
6238static void
6239getFalkorUnrollingPreferences(Loop *L, ScalarEvolution &SE,
6240 TargetTransformInfo::UnrollingPreferences &UP) {
6241 enum { MaxStridedLoads = 7 };
6242 auto countStridedLoads = [](Loop *L, ScalarEvolution &SE) {
6243 int StridedLoads = 0;
6244 // FIXME? We could make this more precise by looking at the CFG and
6245 // e.g. not counting loads in each side of an if-then-else diamond.
6246 for (const auto BB : L->blocks()) {
6247 for (auto &I : *BB) {
6248 LoadInst *LMemI = dyn_cast<LoadInst>(Val: &I);
6249 if (!LMemI)
6250 continue;
6251
6252 Value *PtrValue = LMemI->getPointerOperand();
6253 if (L->isLoopInvariant(V: PtrValue))
6254 continue;
6255
6256 const SCEV *LSCEV = SE.getSCEV(V: PtrValue);
6257 const SCEVAddRecExpr *LSCEVAddRec = dyn_cast<SCEVAddRecExpr>(Val: LSCEV);
6258 if (!LSCEVAddRec || !LSCEVAddRec->isAffine())
6259 continue;
6260
6261 // FIXME? We could take pairing of unrolled load copies into account
6262 // by looking at the AddRec, but we would probably have to limit this
6263 // to loops with no stores or other memory optimization barriers.
6264 ++StridedLoads;
6265 // We've seen enough strided loads that seeing more won't make a
6266 // difference.
6267 if (StridedLoads > MaxStridedLoads / 2)
6268 return StridedLoads;
6269 }
6270 }
6271 return StridedLoads;
6272 };
6273
6274 int StridedLoads = countStridedLoads(L, SE);
6275 LLVM_DEBUG(dbgs() << "falkor-hwpf: detected " << StridedLoads
6276 << " strided loads\n");
6277 // Pick the largest power of 2 unroll count that won't result in too many
6278 // strided loads.
6279 if (StridedLoads) {
6280 UP.MaxCount = 1 << Log2_32(Value: MaxStridedLoads / StridedLoads);
6281 LLVM_DEBUG(dbgs() << "falkor-hwpf: setting unroll MaxCount to "
6282 << UP.MaxCount << '\n');
6283 }
6284}
6285
6286// This function returns true if the loop:
6287// 1. Has a valid cost, and
6288// 2. Has a cost within the supplied budget.
6289// Otherwise it returns false.
6290static bool isLoopSizeWithinBudget(Loop *L, const AArch64TTIImpl &TTI,
6291 InstructionCost Budget,
6292 unsigned *FinalSize) {
6293 // Estimate the size of the loop.
6294 InstructionCost LoopCost = 0;
6295
6296 for (auto *BB : L->getBlocks()) {
6297 for (auto &I : *BB) {
6298 SmallVector<const Value *, 4> Operands(I.operand_values());
6299 InstructionCost Cost =
6300 TTI.getInstructionCost(U: &I, Operands, CostKind: TTI::TCK_CodeSize);
6301 // This can happen with intrinsics that don't currently have a cost model
6302 // or for some operations that require SVE.
6303 if (!Cost.isValid())
6304 return false;
6305
6306 LoopCost += Cost;
6307 if (LoopCost > Budget)
6308 return false;
6309 }
6310 }
6311
6312 if (FinalSize)
6313 *FinalSize = LoopCost.getValue();
6314 return true;
6315}
6316
6317static bool shouldUnrollMultiExitLoop(Loop *L, ScalarEvolution &SE,
6318 const AArch64TTIImpl &TTI) {
6319 // Only consider loops with unknown trip counts for which we can determine
6320 // a symbolic expression. Multi-exit loops with small known trip counts will
6321 // likely be unrolled anyway.
6322 const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L);
6323 if (isa<SCEVConstant>(Val: BTC) || isa<SCEVCouldNotCompute>(Val: BTC))
6324 return false;
6325
6326 // It might not be worth unrolling loops with low max trip counts. Restrict
6327 // this to max trip counts > 32 for now.
6328 unsigned MaxTC = SE.getSmallConstantMaxTripCount(L);
6329 if (MaxTC > 0 && MaxTC <= 32)
6330 return false;
6331
6332 // Make sure the loop size is <= 5.
6333 if (!isLoopSizeWithinBudget(L, TTI, Budget: 5, FinalSize: nullptr))
6334 return false;
6335
6336 // Small search loops with multiple exits can be highly beneficial to unroll.
6337 // We only care about loops with exactly two exiting blocks, although each
6338 // block could jump to the same exit block.
6339 ArrayRef<BasicBlock *> Blocks = L->getBlocks();
6340 if (Blocks.size() != 2)
6341 return false;
6342
6343 if (any_of(Range&: Blocks, P: [](BasicBlock *BB) {
6344 return !isa<UncondBrInst, CondBrInst>(Val: BB->getTerminator());
6345 }))
6346 return false;
6347
6348 return true;
6349}
6350
6351/// For Apple CPUs, we want to runtime-unroll loops to make better use if the
6352/// OOO engine's wide instruction window and various predictors.
6353static void
6354getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
6355 TargetTransformInfo::UnrollingPreferences &UP,
6356 const AArch64TTIImpl &TTI) {
6357 // Limit loops with structure that is highly likely to benefit from runtime
6358 // unrolling; that is we exclude outer loops and loops with many blocks (i.e.
6359 // likely with complex control flow). Note that the heuristics here may be
6360 // overly conservative and we err on the side of avoiding runtime unrolling
6361 // rather than unroll excessively. They are all subject to further refinement.
6362 if (!L->isInnermost() || L->getNumBlocks() > 8)
6363 return;
6364
6365 // Loops with multiple exits are handled by common code.
6366 if (!L->getExitBlock())
6367 return;
6368
6369 // Check if the loop contains any reductions that could be parallelized when
6370 // unrolling. If so, enable partial unrolling, if the trip count is know to be
6371 // a multiple of 2.
6372 bool HasParellelizableReductions =
6373 L->getNumBlocks() == 1 &&
6374 any_of(Range: L->getHeader()->phis(),
6375 P: [&SE, L](PHINode &Phi) {
6376 return canParallelizeReductionWhenUnrolling(Phi, L, SE: &SE);
6377 }) &&
6378 isLoopSizeWithinBudget(L, TTI, Budget: 12, FinalSize: nullptr);
6379 if (HasParellelizableReductions &&
6380 SE.getSmallConstantTripMultiple(L, ExitingBlock: L->getExitingBlock()) % 2 == 0) {
6381 UP.Partial = true;
6382 UP.MaxCount = 4;
6383 UP.AddAdditionalAccumulators = true;
6384 }
6385
6386 const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L);
6387 if (isa<SCEVConstant>(Val: BTC) || isa<SCEVCouldNotCompute>(Val: BTC) ||
6388 (SE.getSmallConstantMaxTripCount(L) > 0 &&
6389 SE.getSmallConstantMaxTripCount(L) <= 32))
6390 return;
6391
6392 if (findStringMetadataForLoop(TheLoop: L, Name: "llvm.loop.isvectorized"))
6393 return;
6394
6395 if (SE.getSymbolicMaxBackedgeTakenCount(L) != SE.getBackedgeTakenCount(L))
6396 return;
6397
6398 // Limit to loops with trip counts that are cheap to expand.
6399 UP.SCEVExpansionBudget = 1;
6400
6401 if (HasParellelizableReductions) {
6402 UP.Runtime = true;
6403 UP.DefaultUnrollRuntimeCount = 4;
6404 UP.AddAdditionalAccumulators = true;
6405 }
6406
6407 // Try to unroll small, single-block loops with low budget, if they have
6408 // load/store dependencies, to expose more parallel memory access streams,
6409 // or if they do little work inside a block (i.e. load -> X -> store pattern).
6410 BasicBlock *Header = L->getHeader();
6411 BasicBlock *Latch = L->getLoopLatch();
6412 if (Header == Latch) {
6413 // Estimate the size of the loop.
6414 unsigned Size;
6415 unsigned Width = 10;
6416 if (!isLoopSizeWithinBudget(L, TTI, Budget: Width, FinalSize: &Size))
6417 return;
6418
6419 // Try to find an unroll count that maximizes the use of the instruction
6420 // window, i.e. trying to fetch as many instructions per cycle as possible.
6421 unsigned MaxInstsPerLine = 16;
6422 unsigned UC = 1;
6423 unsigned BestUC = 1;
6424 unsigned SizeWithBestUC = BestUC * Size;
6425 while (UC <= 8) {
6426 unsigned SizeWithUC = UC * Size;
6427 if (SizeWithUC > 48)
6428 break;
6429 if ((SizeWithUC % MaxInstsPerLine) == 0 ||
6430 (SizeWithBestUC % MaxInstsPerLine) < (SizeWithUC % MaxInstsPerLine)) {
6431 BestUC = UC;
6432 SizeWithBestUC = BestUC * Size;
6433 }
6434 UC++;
6435 }
6436
6437 if (BestUC == 1)
6438 return;
6439
6440 SmallPtrSet<Value *, 8> LoadedValuesPlus;
6441 SmallVector<StoreInst *> Stores;
6442 for (auto *BB : L->blocks()) {
6443 for (auto &I : *BB) {
6444 Value *Ptr = getLoadStorePointerOperand(V: &I);
6445 if (!Ptr)
6446 continue;
6447 const SCEV *PtrSCEV = SE.getSCEV(V: Ptr);
6448 if (SE.isLoopInvariant(S: PtrSCEV, L))
6449 continue;
6450 if (isa<LoadInst>(Val: &I)) {
6451 LoadedValuesPlus.insert(Ptr: &I);
6452 // Include in-loop 1st users of loaded values.
6453 for (auto *U : I.users())
6454 if (L->contains(Inst: cast<Instruction>(Val: U)))
6455 LoadedValuesPlus.insert(Ptr: U);
6456 } else
6457 Stores.push_back(Elt: cast<StoreInst>(Val: &I));
6458 }
6459 }
6460
6461 if (none_of(Range&: Stores, P: [&LoadedValuesPlus](StoreInst *SI) {
6462 return LoadedValuesPlus.contains(Ptr: SI->getOperand(i_nocapture: 0));
6463 }))
6464 return;
6465
6466 UP.Runtime = true;
6467 UP.DefaultUnrollRuntimeCount = BestUC;
6468 return;
6469 }
6470
6471 // Try to runtime-unroll loops with early-continues depending on loop-varying
6472 // loads; this helps with branch-prediction for the early-continues.
6473 auto *Term = dyn_cast<CondBrInst>(Val: Header->getTerminator());
6474 SmallVector<BasicBlock *> Preds(predecessors(BB: Latch));
6475 if (!Term || Preds.size() == 1 || !llvm::is_contained(Range&: Preds, Element: Header) ||
6476 none_of(Range&: Preds, P: [L](BasicBlock *Pred) { return L->contains(BB: Pred); }))
6477 return;
6478
6479 std::function<bool(Instruction *, unsigned)> DependsOnLoopLoad =
6480 [&](Instruction *I, unsigned Depth) -> bool {
6481 if (isa<PHINode>(Val: I) || L->isLoopInvariant(V: I) || Depth > 8)
6482 return false;
6483
6484 if (isa<LoadInst>(Val: I))
6485 return true;
6486
6487 return any_of(Range: I->operands(), P: [&](Value *V) {
6488 auto *I = dyn_cast<Instruction>(Val: V);
6489 return I && DependsOnLoopLoad(I, Depth + 1);
6490 });
6491 };
6492 CmpPredicate Pred;
6493 Instruction *I;
6494 if (match(V: Term, P: m_Br(C: m_ICmp(Pred, L: m_Instruction(I), R: m_Value()), T: m_Value(),
6495 F: m_Value())) &&
6496 DependsOnLoopLoad(I, 0)) {
6497 UP.Runtime = true;
6498 }
6499}
6500
6501void AArch64TTIImpl::getUnrollingPreferences(
6502 Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP,
6503 OptimizationRemarkEmitter *ORE) const {
6504 // Enable partial unrolling and runtime unrolling.
6505 BaseT::getUnrollingPreferences(L, SE, UP, ORE);
6506
6507 UP.UpperBound = true;
6508
6509 // A loop can have a small maximum trip count while SCEV still cannot
6510 // form an exact backedge count - typically a data-dependent exit, e.g.
6511 // shifting a value until it reaches zero. Unlike for counted loops, the
6512 // unrolled body keeps an exit test per iteration, and whether that pays
6513 // off depends on how many iterations the loop usually runs, which is
6514 // unknown at compile time; the code growth and extra branches are certain.
6515 // Be conservative and hold such loops to a lower upper bound; 5 still lets
6516 // smaller early-exit loops unroll. Also disable runtime unrolling, which
6517 // would clamp the unroll count to the known maximum trip count and produce
6518 // the same complete unroll.
6519 if (L->getExitingBlock() && !SE.isBackedgeTakenCountMaxOrZero(L) &&
6520 isa<SCEVCouldNotCompute>(Val: SE.getBackedgeTakenCount(L))) {
6521 UP.MaxUpperBound = 5;
6522 UP.Runtime = false;
6523 }
6524
6525 // For inner loop, it is more likely to be a hot one, and the runtime check
6526 // can be promoted out from LICM pass, so the overhead is less, let's try
6527 // a larger threshold to unroll more loops.
6528 if (L->getLoopDepth() > 1)
6529 UP.PartialThreshold *= 2;
6530
6531 // Disable partial & runtime unrolling on -Os.
6532 UP.PartialOptSizeThreshold = 0;
6533
6534 // Scan the loop: don't unroll loops with calls as this could prevent
6535 // inlining. Don't unroll auto-vectorized loops either, though do allow
6536 // unrolling of the scalar remainder.
6537 bool IsVectorized = getBooleanLoopAttribute(TheLoop: L, Name: "llvm.loop.isvectorized");
6538 InstructionCost Cost = 0;
6539 for (auto *BB : L->getBlocks()) {
6540 for (auto &I : *BB) {
6541 // Both auto-vectorized loops and the scalar remainder have the
6542 // isvectorized attribute, so differentiate between them by the presence
6543 // of vector instructions.
6544 if (IsVectorized && I.getType()->isVectorTy())
6545 return;
6546 if (isa<CallBase>(Val: I)) {
6547 if (isa<CallInst>(Val: I) || isa<InvokeInst>(Val: I))
6548 if (const Function *F = cast<CallBase>(Val&: I).getCalledFunction())
6549 if (!isLoweredToCall(F))
6550 continue;
6551 return;
6552 }
6553
6554 // The cost is only compared against Aarch64ForceUnrollThreshold below.
6555 if (Cost >= ST->getCLOpts().force_unroll_threshold)
6556 continue;
6557 SmallVector<const Value *, 4> Operands(I.operand_values());
6558 Cost += getInstructionCost(U: &I, Operands,
6559 CostKind: TargetTransformInfo::TCK_SizeAndLatency);
6560 }
6561 }
6562
6563 // Apply subtarget-specific unrolling preferences.
6564 if (ST->isAppleMLike())
6565 getAppleRuntimeUnrollPreferences(L, SE, UP, TTI: *this);
6566 else if (ST->getProcFamily() == AArch64Subtarget::Falkor &&
6567 ST->getCLOpts().enable_falkor_hwpf_unroll_fix)
6568 getFalkorUnrollingPreferences(L, SE, UP);
6569
6570 // If this is a small, multi-exit loop similar to something like std::find,
6571 // then there is typically a performance improvement achieved by unrolling.
6572 if (!L->getExitBlock() && shouldUnrollMultiExitLoop(L, SE, TTI: *this)) {
6573 UP.RuntimeUnrollMultiExit = true;
6574 UP.Runtime = true;
6575 // Limit unroll count.
6576 UP.DefaultUnrollRuntimeCount = 4;
6577 // Allow slightly more costly trip-count expansion to catch search loops
6578 // with pointer inductions.
6579 UP.SCEVExpansionBudget = 5;
6580 return;
6581 }
6582
6583 // Enable runtime unrolling for in-order models
6584 // If mcpu is omitted, getProcFamily() returns AArch64Subtarget::Others, so by
6585 // checking for that case, we can ensure that the default behaviour is
6586 // unchanged
6587 if (ST->getProcFamily() != AArch64Subtarget::Generic &&
6588 !ST->getSchedModel().isOutOfOrder()) {
6589 UP.Runtime = true;
6590 UP.Partial = true;
6591 UP.UnrollRemainder = true;
6592 UP.DefaultUnrollRuntimeCount = 4;
6593
6594 UP.UnrollAndJam = true;
6595 UP.UnrollAndJamInnerLoopThreshold = 60;
6596 }
6597
6598 // Force unrolling small loops can be very useful because of the branch
6599 // taken cost of the backedge.
6600 if (Cost < ST->getCLOpts().force_unroll_threshold)
6601 UP.Force = true;
6602}
6603
6604void AArch64TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
6605 TTI::PeelingPreferences &PP) const {
6606 BaseT::getPeelingPreferences(L, SE, PP);
6607}
6608
6609Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst,
6610 Type *ExpectedType,
6611 bool CanCreate) const {
6612 switch (Inst->getIntrinsicID()) {
6613 default:
6614 return nullptr;
6615 case Intrinsic::aarch64_neon_st1x2:
6616 case Intrinsic::aarch64_neon_st1x3:
6617 case Intrinsic::aarch64_neon_st1x4:
6618 case Intrinsic::aarch64_neon_st2:
6619 case Intrinsic::aarch64_neon_st3:
6620 case Intrinsic::aarch64_neon_st4: {
6621 // Create a struct type
6622 StructType *ST = dyn_cast<StructType>(Val: ExpectedType);
6623 if (!CanCreate || !ST)
6624 return nullptr;
6625 unsigned NumElts = Inst->arg_size() - 1;
6626 if (ST->getNumElements() != NumElts)
6627 return nullptr;
6628 for (unsigned i = 0, e = NumElts; i != e; ++i) {
6629 if (Inst->getArgOperand(i)->getType() != ST->getElementType(N: i))
6630 return nullptr;
6631 }
6632 Value *Res = PoisonValue::get(T: ExpectedType);
6633 IRBuilder<> Builder(Inst);
6634 for (unsigned i = 0, e = NumElts; i != e; ++i) {
6635 Value *L = Inst->getArgOperand(i);
6636 Res = Builder.CreateInsertValue(Agg: Res, Val: L, Idxs: i);
6637 }
6638 return Res;
6639 }
6640 case Intrinsic::aarch64_neon_ld1x2:
6641 case Intrinsic::aarch64_neon_ld1x3:
6642 case Intrinsic::aarch64_neon_ld1x4:
6643 case Intrinsic::aarch64_neon_ld2:
6644 case Intrinsic::aarch64_neon_ld3:
6645 case Intrinsic::aarch64_neon_ld4:
6646 if (Inst->getType() == ExpectedType)
6647 return Inst;
6648 return nullptr;
6649 }
6650}
6651
6652bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst,
6653 MemIntrinsicInfo &Info) const {
6654 switch (Inst->getIntrinsicID()) {
6655 default:
6656 break;
6657 case Intrinsic::aarch64_neon_ld1x2:
6658 case Intrinsic::aarch64_neon_ld1x3:
6659 case Intrinsic::aarch64_neon_ld1x4:
6660 case Intrinsic::aarch64_neon_ld2:
6661 case Intrinsic::aarch64_neon_ld3:
6662 case Intrinsic::aarch64_neon_ld4:
6663 Info.ReadMem = true;
6664 Info.WriteMem = false;
6665 Info.PtrVal = Inst->getArgOperand(i: 0);
6666 break;
6667 case Intrinsic::aarch64_neon_st1x2:
6668 case Intrinsic::aarch64_neon_st1x3:
6669 case Intrinsic::aarch64_neon_st1x4:
6670 case Intrinsic::aarch64_neon_st2:
6671 case Intrinsic::aarch64_neon_st3:
6672 case Intrinsic::aarch64_neon_st4:
6673 Info.ReadMem = false;
6674 Info.WriteMem = true;
6675 Info.PtrVal = Inst->getArgOperand(i: Inst->arg_size() - 1);
6676 break;
6677 }
6678
6679 // Use the ID of neon load as the "matching id".
6680 switch (Inst->getIntrinsicID()) {
6681 default:
6682 return false;
6683 case Intrinsic::aarch64_neon_ld1x2:
6684 case Intrinsic::aarch64_neon_st1x2:
6685 Info.MatchingId = Intrinsic::aarch64_neon_ld1x2;
6686 break;
6687 case Intrinsic::aarch64_neon_ld1x3:
6688 case Intrinsic::aarch64_neon_st1x3:
6689 Info.MatchingId = Intrinsic::aarch64_neon_ld1x3;
6690 break;
6691 case Intrinsic::aarch64_neon_ld1x4:
6692 case Intrinsic::aarch64_neon_st1x4:
6693 Info.MatchingId = Intrinsic::aarch64_neon_ld1x4;
6694 break;
6695 case Intrinsic::aarch64_neon_ld2:
6696 case Intrinsic::aarch64_neon_st2:
6697 Info.MatchingId = Intrinsic::aarch64_neon_ld2;
6698 break;
6699 case Intrinsic::aarch64_neon_ld3:
6700 case Intrinsic::aarch64_neon_st3:
6701 Info.MatchingId = Intrinsic::aarch64_neon_ld3;
6702 break;
6703 case Intrinsic::aarch64_neon_ld4:
6704 case Intrinsic::aarch64_neon_st4:
6705 Info.MatchingId = Intrinsic::aarch64_neon_ld4;
6706 break;
6707 }
6708 return true;
6709}
6710
6711/// See if \p I should be considered for address type promotion. We check if \p
6712/// I is a sext with right type and used in memory accesses. If it used in a
6713/// "complex" getelementptr, we allow it to be promoted without finding other
6714/// sext instructions that sign extended the same initial value. A getelementptr
6715/// is considered as "complex" if it has more than 2 operands.
6716bool AArch64TTIImpl::shouldConsiderAddressTypePromotion(
6717 const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const {
6718 bool Considerable = false;
6719 AllowPromotionWithoutCommonHeader = false;
6720 if (!isa<SExtInst>(Val: &I))
6721 return false;
6722 Type *ConsideredSExtType =
6723 Type::getInt64Ty(C&: I.getParent()->getParent()->getContext());
6724 if (I.getType() != ConsideredSExtType)
6725 return false;
6726 // See if the sext is the one with the right type and used in at least one
6727 // GetElementPtrInst.
6728 for (const User *U : I.users()) {
6729 if (const GetElementPtrInst *GEPInst = dyn_cast<GetElementPtrInst>(Val: U)) {
6730 Considerable = true;
6731 // A getelementptr is considered as "complex" if it has more than 2
6732 // operands. We will promote a SExt used in such complex GEP as we
6733 // expect some computation to be merged if they are done on 64 bits.
6734 if (GEPInst->getNumOperands() > 2) {
6735 AllowPromotionWithoutCommonHeader = true;
6736 break;
6737 }
6738 }
6739 }
6740 return Considerable;
6741}
6742
6743bool AArch64TTIImpl::isLegalToVectorizeReduction(
6744 const RecurrenceDescriptor &RdxDesc, ElementCount VF) const {
6745 if (!VF.isScalable())
6746 return true;
6747
6748 Type *Ty = RdxDesc.getRecurrenceType();
6749 if (Ty->isBFloatTy() || !isElementTypeLegalForScalableVector(Ty))
6750 return false;
6751
6752 switch (RdxDesc.getRecurrenceKind()) {
6753 case RecurKind::Sub:
6754 case RecurKind::FSub:
6755 case RecurKind::AddChainWithSubs:
6756 case RecurKind::FAddChainWithSubs:
6757 case RecurKind::Add:
6758 case RecurKind::FAdd:
6759 case RecurKind::And:
6760 case RecurKind::Or:
6761 case RecurKind::Xor:
6762 case RecurKind::SMin:
6763 case RecurKind::SMax:
6764 case RecurKind::UMin:
6765 case RecurKind::UMax:
6766 case RecurKind::FMin:
6767 case RecurKind::FMax:
6768 case RecurKind::FMulAdd:
6769 case RecurKind::AnyOf:
6770 case RecurKind::FindLast:
6771 return true;
6772 default:
6773 return false;
6774 }
6775}
6776
6777InstructionCost
6778AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty,
6779 FastMathFlags FMF,
6780 TTI::TargetCostKind CostKind) const {
6781 // The code-generator is currently not able to handle scalable vectors
6782 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
6783 // it. This change will be removed when code-generation for these types is
6784 // sufficiently reliable.
6785 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: Ty))
6786 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
6787 return InstructionCost::getInvalid();
6788
6789 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty);
6790
6791 if (LT.second.getScalarType() == MVT::f16 && !ST->hasFullFP16())
6792 return BaseT::getMinMaxReductionCost(IID, Ty, FMF, CostKind);
6793
6794 InstructionCost LegalizationCost = 0;
6795 if (LT.first > 1) {
6796 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Context&: Ty->getContext());
6797 IntrinsicCostAttributes Attrs(IID, LegalVTy, {LegalVTy, LegalVTy}, FMF);
6798 LegalizationCost = getIntrinsicInstrCost(ICA: Attrs, CostKind) * (LT.first - 1);
6799 }
6800
6801 return LegalizationCost + /*Cost of horizontal reduction*/ 2;
6802}
6803
6804InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE(
6805 unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) const {
6806 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
6807 InstructionCost LegalizationCost = 0;
6808 if (LT.first > 1) {
6809 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Context&: ValTy->getContext());
6810 LegalizationCost = getArithmeticInstrCost(Opcode, Ty: LegalVTy, CostKind);
6811 LegalizationCost *= LT.first - 1;
6812 }
6813
6814 int ISD = TLI->InstructionOpcodeToISD(Opcode);
6815 assert(ISD && "Invalid opcode");
6816 // Add the final reduction cost for the legal horizontal reduction
6817 switch (ISD) {
6818 case ISD::ADD:
6819 case ISD::AND:
6820 case ISD::OR:
6821 case ISD::XOR:
6822 case ISD::FADD:
6823 return LegalizationCost + 2;
6824 default:
6825 return InstructionCost::getInvalid();
6826 }
6827}
6828
6829InstructionCost
6830AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy,
6831 std::optional<FastMathFlags> FMF,
6832 TTI::TargetCostKind CostKind) const {
6833 // The code-generator is currently not able to handle scalable vectors
6834 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
6835 // it. This change will be removed when code-generation for these types is
6836 // sufficiently reliable.
6837 if (auto *VTy = dyn_cast<ScalableVectorType>(Val: ValTy))
6838 if (VTy->getElementCount() == ElementCount::getScalable(MinVal: 1))
6839 return InstructionCost::getInvalid();
6840
6841 if (TTI::requiresOrderedReduction(FMF)) {
6842 if (auto *FixedVTy = dyn_cast<FixedVectorType>(Val: ValTy)) {
6843 InstructionCost BaseCost =
6844 BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
6845 // Add on extra cost to reflect the extra overhead on some CPUs. We still
6846 // end up vectorizing for more computationally intensive loops.
6847 return BaseCost + FixedVTy->getNumElements();
6848 }
6849
6850 if (Opcode != Instruction::FAdd || ValTy->getElementType()->isBFloatTy())
6851 return InstructionCost::getInvalid();
6852
6853 auto *VTy = cast<ScalableVectorType>(Val: ValTy);
6854 InstructionCost Cost =
6855 getArithmeticInstrCost(Opcode, Ty: VTy->getScalarType(), CostKind);
6856 Cost *= getMaxNumElements(VF: VTy->getElementCount());
6857 return Cost;
6858 }
6859
6860 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: ValTy);
6861 MVT MTy = LT.second;
6862
6863 if (isa<ScalableVectorType>(Val: ValTy) || TLI->useSVEForFixedLengthVectorVT(VT: MTy))
6864 return getArithmeticReductionCostSVE(Opcode, ValTy, CostKind);
6865
6866 int ISD = TLI->InstructionOpcodeToISD(Opcode);
6867 assert(ISD && "Invalid opcode");
6868
6869 // Horizontal adds can use the 'addv' instruction. We model the cost of these
6870 // instructions as twice a normal vector add, plus 1 for each legalization
6871 // step (LT.first). This is the only arithmetic vector reduction operation for
6872 // which we have an instruction.
6873 // OR, XOR and AND costs should match the codegen from:
6874 // OR: llvm/test/CodeGen/AArch64/reduce-or.ll
6875 // XOR: llvm/test/CodeGen/AArch64/reduce-xor.ll
6876 // AND: llvm/test/CodeGen/AArch64/reduce-and.ll
6877 static const CostTblEntry CostTblNoPairwise[]{
6878 {.ISD: ISD::ADD, .Type: MVT::v8i8, .Cost: 2},
6879 {.ISD: ISD::ADD, .Type: MVT::v16i8, .Cost: 2},
6880 {.ISD: ISD::ADD, .Type: MVT::v4i16, .Cost: 2},
6881 {.ISD: ISD::ADD, .Type: MVT::v8i16, .Cost: 2},
6882 {.ISD: ISD::ADD, .Type: MVT::v2i32, .Cost: 2},
6883 {.ISD: ISD::ADD, .Type: MVT::v4i32, .Cost: 2},
6884 {.ISD: ISD::ADD, .Type: MVT::v2i64, .Cost: 2},
6885 {.ISD: ISD::OR, .Type: MVT::v8i8, .Cost: 5}, // fmov + orr_lsr + orr_lsr + lsr + orr
6886 {.ISD: ISD::OR, .Type: MVT::v16i8, .Cost: 7}, // ext + orr + same as v8i8
6887 {.ISD: ISD::OR, .Type: MVT::v4i16, .Cost: 4}, // fmov + orr_lsr + lsr + orr
6888 {.ISD: ISD::OR, .Type: MVT::v8i16, .Cost: 6}, // ext + orr + same as v4i16
6889 {.ISD: ISD::OR, .Type: MVT::v2i32, .Cost: 3}, // fmov + lsr + orr
6890 {.ISD: ISD::OR, .Type: MVT::v4i32, .Cost: 5}, // ext + orr + same as v2i32
6891 {.ISD: ISD::OR, .Type: MVT::v2i64, .Cost: 3}, // ext + orr + fmov
6892 {.ISD: ISD::XOR, .Type: MVT::v8i8, .Cost: 5}, // Same as above for or...
6893 {.ISD: ISD::XOR, .Type: MVT::v16i8, .Cost: 7},
6894 {.ISD: ISD::XOR, .Type: MVT::v4i16, .Cost: 4},
6895 {.ISD: ISD::XOR, .Type: MVT::v8i16, .Cost: 6},
6896 {.ISD: ISD::XOR, .Type: MVT::v2i32, .Cost: 3},
6897 {.ISD: ISD::XOR, .Type: MVT::v4i32, .Cost: 5},
6898 {.ISD: ISD::XOR, .Type: MVT::v2i64, .Cost: 3},
6899 {.ISD: ISD::AND, .Type: MVT::v8i8, .Cost: 5}, // Same as above for or...
6900 {.ISD: ISD::AND, .Type: MVT::v16i8, .Cost: 7},
6901 {.ISD: ISD::AND, .Type: MVT::v4i16, .Cost: 4},
6902 {.ISD: ISD::AND, .Type: MVT::v8i16, .Cost: 6},
6903 {.ISD: ISD::AND, .Type: MVT::v2i32, .Cost: 3},
6904 {.ISD: ISD::AND, .Type: MVT::v4i32, .Cost: 5},
6905 {.ISD: ISD::AND, .Type: MVT::v2i64, .Cost: 3},
6906 };
6907 switch (ISD) {
6908 default:
6909 break;
6910 case ISD::FADD:
6911 if (Type *EltTy = ValTy->getScalarType();
6912 // FIXME: For half types without fullfp16 support, this could extend and
6913 // use a fp32 faddp reduction but current codegen unrolls.
6914 MTy.isVector() && (EltTy->isFloatTy() || EltTy->isDoubleTy() ||
6915 (EltTy->isHalfTy() && ST->hasFullFP16()))) {
6916 const unsigned NElts = MTy.getVectorNumElements();
6917 if (ValTy->getElementCount().getFixedValue() >= 2 && NElts >= 2 &&
6918 isPowerOf2_32(Value: NElts))
6919 // Reduction corresponding to series of fadd instructions is lowered to
6920 // series of faddp instructions. faddp has latency/throughput that
6921 // matches fadd instruction and hence, every faddp instruction can be
6922 // considered to have a relative cost = 1 with
6923 // CostKind = TCK_RecipThroughput.
6924 // An faddp will pairwise add vector elements, so the size of input
6925 // vector reduces by half every time, requiring
6926 // #(faddp instructions) = log2_32(NElts).
6927 return (LT.first - 1) + /*No of faddp instructions*/ Log2_32(Value: NElts);
6928 }
6929 break;
6930 case ISD::ADD:
6931 if (const auto *Entry = CostTableLookup(Table: CostTblNoPairwise, ISD, Ty: MTy))
6932 return (LT.first - 1) + Entry->Cost;
6933 break;
6934 case ISD::XOR:
6935 case ISD::AND:
6936 case ISD::OR:
6937 const auto *Entry = CostTableLookup(Table: CostTblNoPairwise, ISD, Ty: MTy);
6938 if (!Entry)
6939 break;
6940 auto *ValVTy = cast<FixedVectorType>(Val: ValTy);
6941 if (MTy.getVectorNumElements() <= ValVTy->getNumElements() &&
6942 isPowerOf2_32(Value: ValVTy->getNumElements())) {
6943 InstructionCost ExtraCost = 0;
6944 if (LT.first != 1) {
6945 // Type needs to be split, so there is an extra cost of LT.first - 1
6946 // arithmetic ops.
6947 auto *Ty = FixedVectorType::get(ElementType: ValTy->getElementType(),
6948 NumElts: MTy.getVectorNumElements());
6949 ExtraCost = getArithmeticInstrCost(Opcode, Ty, CostKind);
6950 ExtraCost *= LT.first - 1;
6951 }
6952 // All and/or/xor of i1 will be lowered with maxv/minv/addv + fmov
6953 auto Cost = ValVTy->getElementType()->isIntegerTy(BitWidth: 1) ? 2 : Entry->Cost;
6954 return Cost + ExtraCost;
6955 }
6956 break;
6957 }
6958 return BaseT::getArithmeticReductionCost(Opcode, Ty: ValTy, FMF, CostKind);
6959}
6960
6961InstructionCost AArch64TTIImpl::getExtendedReductionCost(
6962 unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *VecTy,
6963 std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const {
6964 EVT VecVT = TLI->getValueType(DL, Ty: VecTy);
6965 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
6966
6967 if (Opcode == Instruction::Add && VecVT.isSimple() && ResVT.isSimple() &&
6968 VecVT.getSizeInBits() >= 64) {
6969 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VecTy);
6970
6971 // The legal cases are:
6972 // UADDLV 8/16/32->32
6973 // UADDLP 32->64
6974 unsigned RevVTSize = ResVT.getSizeInBits();
6975 if (((LT.second == MVT::v8i8 || LT.second == MVT::v16i8) &&
6976 RevVTSize <= 32) ||
6977 ((LT.second == MVT::v4i16 || LT.second == MVT::v8i16) &&
6978 RevVTSize <= 32) ||
6979 ((LT.second == MVT::v2i32 || LT.second == MVT::v4i32) &&
6980 RevVTSize <= 64))
6981 return (LT.first - 1) * 2 + 2;
6982 }
6983
6984 return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, Ty: VecTy, FMF,
6985 CostKind);
6986}
6987
6988InstructionCost
6989AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
6990 Type *ResTy, VectorType *VecTy,
6991 TTI::TargetCostKind CostKind) const {
6992 EVT VecVT = TLI->getValueType(DL, Ty: VecTy);
6993 EVT ResVT = TLI->getValueType(DL, Ty: ResTy);
6994
6995 if (ST->hasDotProd() && VecVT.isSimple() && ResVT.isSimple() &&
6996 RedOpcode == Instruction::Add) {
6997 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: VecTy);
6998
6999 // The legal cases with dotprod are
7000 // UDOT 8->32
7001 // Which requires an additional uaddv to sum the i32 values.
7002 if ((LT.second == MVT::v8i8 || LT.second == MVT::v16i8) &&
7003 ResVT == MVT::i32)
7004 return LT.first + 2;
7005 }
7006
7007 return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty: VecTy,
7008 CostKind);
7009}
7010
7011InstructionCost
7012AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index,
7013 TTI::TargetCostKind CostKind) const {
7014 static const CostTblEntry ShuffleTbl[] = {
7015 { .ISD: TTI::SK_Splice, .Type: MVT::nxv16i8, .Cost: 1 },
7016 { .ISD: TTI::SK_Splice, .Type: MVT::nxv8i16, .Cost: 1 },
7017 { .ISD: TTI::SK_Splice, .Type: MVT::nxv4i32, .Cost: 1 },
7018 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2i64, .Cost: 1 },
7019 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2f16, .Cost: 1 },
7020 { .ISD: TTI::SK_Splice, .Type: MVT::nxv4f16, .Cost: 1 },
7021 { .ISD: TTI::SK_Splice, .Type: MVT::nxv8f16, .Cost: 1 },
7022 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2bf16, .Cost: 1 },
7023 { .ISD: TTI::SK_Splice, .Type: MVT::nxv4bf16, .Cost: 1 },
7024 { .ISD: TTI::SK_Splice, .Type: MVT::nxv8bf16, .Cost: 1 },
7025 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2f32, .Cost: 1 },
7026 { .ISD: TTI::SK_Splice, .Type: MVT::nxv4f32, .Cost: 1 },
7027 { .ISD: TTI::SK_Splice, .Type: MVT::nxv2f64, .Cost: 1 },
7028 };
7029
7030 // The code-generator is currently not able to handle scalable vectors
7031 // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting
7032 // it. This change will be removed when code-generation for these types is
7033 // sufficiently reliable.
7034 if (Tp->getElementCount() == ElementCount::getScalable(MinVal: 1))
7035 return InstructionCost::getInvalid();
7036
7037 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: Tp);
7038 Type *LegalVTy = EVT(LT.second).getTypeForEVT(Context&: Tp->getContext());
7039 EVT PromotedVT = LT.second.getScalarType() == MVT::i1
7040 ? TLI->getPromotedVTForPredicate(VT: EVT(LT.second))
7041 : LT.second;
7042 Type *PromotedVTy = EVT(PromotedVT).getTypeForEVT(Context&: Tp->getContext());
7043 InstructionCost LegalizationCost = 0;
7044 if (Index < 0) {
7045 LegalizationCost =
7046 getCmpSelInstrCost(Opcode: Instruction::ICmp, ValTy: PromotedVTy, CondTy: PromotedVTy,
7047 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind) +
7048 getCmpSelInstrCost(Opcode: Instruction::Select, ValTy: PromotedVTy, CondTy: LegalVTy,
7049 VecPred: CmpInst::BAD_ICMP_PREDICATE, CostKind);
7050 }
7051
7052 // Predicated splice are promoted when lowering. See AArch64ISelLowering.cpp
7053 // Cost performed on a promoted type.
7054 if (LT.second.getScalarType() == MVT::i1) {
7055 LegalizationCost +=
7056 getCastInstrCost(Opcode: Instruction::ZExt, Dst: PromotedVTy, Src: LegalVTy,
7057 CCH: TTI::CastContextHint::None, CostKind) +
7058 getCastInstrCost(Opcode: Instruction::Trunc, Dst: LegalVTy, Src: PromotedVTy,
7059 CCH: TTI::CastContextHint::None, CostKind);
7060 }
7061 const auto *Entry =
7062 CostTableLookup(Table: ShuffleTbl, ISD: TTI::SK_Splice, Ty: PromotedVT.getSimpleVT());
7063 assert(Entry && "Illegal Type for Splice");
7064 LegalizationCost += Entry->Cost;
7065 return LegalizationCost * LT.first;
7066}
7067
7068InstructionCost AArch64TTIImpl::getPartialReductionCost(
7069 unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType,
7070 ElementCount VF, TTI::PartialReductionExtendKind OpAExtend,
7071 TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp,
7072 TTI::TargetCostKind CostKind, std::optional<FastMathFlags> FMF) const {
7073 InstructionCost Invalid = InstructionCost::getInvalid();
7074
7075 if (CostKind != TTI::TCK_RecipThroughput)
7076 return Invalid;
7077
7078 if ((Opcode != Instruction::Add && Opcode != Instruction::Sub &&
7079 Opcode != Instruction::FAdd && Opcode != Instruction::FSub))
7080 return Invalid;
7081
7082 // Floating-point partial reductions are invalid if `reassoc` and `contract`
7083 // are not allowed.
7084 if (AccumType->isFloatingPointTy()) {
7085 assert(FMF && "Missing FastMathFlags for floating-point partial reduction");
7086 if (!FMF->allowReassoc() || !FMF->allowContract())
7087 return Invalid;
7088 } else {
7089 assert(!FMF &&
7090 "FastMathFlags only apply to floating-point partial reductions");
7091 }
7092
7093 assert((BinOp || (OpBExtend == TTI::PR_None && !InputTypeB)) &&
7094 (!BinOp || (OpBExtend != TTI::PR_None && InputTypeB)) &&
7095 "Unexpected values for OpBExtend or InputTypeB");
7096
7097 // We only support multiply binary operations for now, and for muls we
7098 // require the types being extended to be the same.
7099 if (BinOp && ((*BinOp != Instruction::Mul && *BinOp != Instruction::FMul) ||
7100 InputTypeA != InputTypeB))
7101 return Invalid;
7102
7103 // We only support the following element sizes.
7104 if (!is_contained(Set: {8u, 16u, 32u, 64u}, Element: AccumType->getScalarSizeInBits()))
7105 return Invalid;
7106
7107 // If none of the operands are extended and there's no extra BinOp, just
7108 // cost this as the equivalent arithmetic instruction.
7109 // TODO: Depending on VF and element type, we may be able to improve on this.
7110 if (!OpAExtend) {
7111 assert(!OpBExtend && "Extended second operand without extended first.");
7112 assert(InputTypeA == AccumType && "Type mismatch with no extensions.");
7113
7114 VectorType *VTy = VectorType::get(ElementType: AccumType, EC: VF);
7115 return getArithmeticInstrCost(Opcode, Ty: VTy, CostKind);
7116 }
7117
7118 bool IsUSDot = OpBExtend != TTI::PR_None && OpAExtend != OpBExtend;
7119 // USDot is natively supported with +i8mm. With plain +dotprod, SUMLA is
7120 // lowered to two udots plus an eor and a sub.
7121 if (IsUSDot && !ST->hasMatMulInt8() && !ST->hasDotProd())
7122 // FIXME: Remove this early bailout in favour of expand cost.
7123 return Invalid;
7124
7125 unsigned Ratio =
7126 AccumType->getScalarSizeInBits() / InputTypeA->getScalarSizeInBits();
7127 if (VF.getKnownMinValue() <= Ratio)
7128 return Invalid;
7129
7130 VectorType *InputVectorType = VectorType::get(ElementType: InputTypeA, EC: VF);
7131 VectorType *AccumVectorType =
7132 VectorType::get(ElementType: AccumType, EC: VF.divideCoefficientBy(RHS: Ratio));
7133 // We don't yet support all kinds of legalization.
7134 auto TC = TLI->getTypeConversion(Context&: AccumVectorType->getContext(),
7135 VT: EVT::getEVT(Ty: AccumVectorType));
7136 switch (TC.first) {
7137 default:
7138 return Invalid;
7139 case TargetLowering::TypeLegal:
7140 case TargetLowering::TypePromoteInteger:
7141 case TargetLowering::TypeSplitVector:
7142 // The legalised type (e.g. after splitting) must be legal too.
7143 if (TLI->getTypeAction(Context&: AccumVectorType->getContext(), VT: TC.second) !=
7144 TargetLowering::TypeLegal)
7145 return Invalid;
7146 break;
7147 }
7148
7149 std::pair<InstructionCost, MVT> AccumLT =
7150 getTypeLegalizationCost(Ty: AccumVectorType);
7151 std::pair<InstructionCost, MVT> InputLT =
7152 getTypeLegalizationCost(Ty: InputVectorType);
7153
7154 // Returns true if the subtarget supports the operation for a given type.
7155 auto IsSupported = [&](bool SVEPred, bool NEONPred) -> bool {
7156 return (ST->isSVEorStreamingSVEAvailable() && SVEPred) ||
7157 (AccumLT.second.isFixedLengthVector() &&
7158 AccumLT.second.getSizeInBits() <= 128 && ST->isNeonAvailable() &&
7159 NEONPred);
7160 };
7161
7162 bool IsSub = Opcode == Instruction::Sub || Opcode == Instruction::FSub;
7163 InstructionCost Cost = InputLT.first * TTI::TCC_Basic;
7164 // Integer partial sub-reductions that don't map to a specific instruction,
7165 // carry an extra cost for implementing a double negation:
7166 // partial_reduce_umls acc, lhs, rhs
7167 // <=> -partial_reduce_umla -acc, lhs, rhs
7168 InstructionCost INegCost = IsSub ? 2 * InputLT.first * TTI::TCC_Basic : 0;
7169
7170 if (AccumLT.second.getScalarType() == MVT::i32 &&
7171 InputLT.second.getScalarType() == MVT::i8) {
7172 // i8 -> i32 is natively supported with udot/sdot for both NEON and SVE.
7173 if (!IsUSDot && IsSupported(true, ST->hasDotProd()))
7174 return Cost + INegCost;
7175 // i8 -> i32 usdot requires +i8mm
7176 if (IsUSDot && IsSupported(ST->hasMatMulInt8(), ST->hasMatMulInt8()))
7177 return Cost + INegCost;
7178 // Without +i8mm, lower SUMLA via two udots plus an eor and a sub on plain
7179 // +dotprod targets. Note that this is only implemented for NEON, as all
7180 // modern CPUs with SVE also have +i8mm. Charge an extra factor for the
7181 // expansion.
7182 if (IsUSDot && IsSupported(false, ST->hasDotProd()))
7183 return Cost * 3 + INegCost;
7184 }
7185
7186 if (ST->isSVEorStreamingSVEAvailable() && !IsUSDot) {
7187 // i16 -> i64 is natively supported for udot/sdot
7188 if (AccumLT.second.getScalarType() == MVT::i64 &&
7189 InputLT.second.getScalarType() == MVT::i16)
7190 return Cost + INegCost;
7191 // i16 -> i32 is natively supported with SVE2p1 udot/sdot.
7192 // For sub-reductions, we prefer using the *mlslb/t instructions.
7193 if (AccumLT.second.getScalarType() == MVT::i32 &&
7194 InputLT.second.getScalarType() == MVT::i16 &&
7195 (ST->hasSVE2p1() || ST->hasSME2()) && !IsSub)
7196 return Cost;
7197 // i8 -> i64 is supported with an extra level of extends
7198 if (AccumLT.second.getScalarType() == MVT::i64 &&
7199 InputLT.second.getScalarType() == MVT::i8)
7200 // FIXME: Also account for folding the i32 dot product results into the
7201 // i64 accumulator, as the NEON case below does.
7202 return Cost + INegCost;
7203 // i8 -> i16 is natively supported with SVE2p3 udot/sdot
7204 // For sub-reductions, we prefer using the *mlslb/t instructions.
7205 if (AccumLT.second.getScalarType() == MVT::i16 &&
7206 InputLT.second.getScalarType() == MVT::i8 &&
7207 (ST->hasSVE2p3() || ST->hasSME2p3()) && !IsSub)
7208 return Cost;
7209 }
7210
7211 // With +dotprod, NEON lowers an extended i8 -> i64 partial reduction to a
7212 // udot/sdot into a zeroed i32 vector followed by an [SU]ADALP into the i64
7213 // accumulator. Charge one basic cost for each of the two steps per legal
7214 // input vector. This must stay after the SVE cases, as IsSupported's NEON
7215 // predicate also holds for fixed-length vectors on SVE targets.
7216 // TODO: Handle zero-extended multiplies and sub-reductions. In loops,
7217 // CodeGenPrepare turns zero extends that do not directly feed the partial
7218 // reduction (such as the operands of a multiply, or of the negation in a
7219 // sub-reduction) into TBLs, which hides the udot from instruction selection.
7220 // Sign-extended sub-reductions have not been measured yet.
7221 // TODO: Without +dotprod, pure adds with a ratio of 4 or 8 lower to a chain
7222 // of [SU]ADDLPs and an [SU]ADALP, which is not costed yet either.
7223 bool IsSExtMul = BinOp && OpAExtend == TTI::PR_SignExtend &&
7224 OpBExtend == TTI::PR_SignExtend;
7225 if ((!BinOp || IsSExtMul) && !IsSub &&
7226 AccumLT.second.getScalarType() == MVT::i64 &&
7227 InputLT.second.getScalarType() == MVT::i8 &&
7228 IsSupported(false, ST->hasDotProd()))
7229 return Cost * 2;
7230
7231 // f16 -> f32 is natively supported for fdot using either
7232 // SVE or NEON instruction.
7233 if (Opcode == Instruction::FAdd && !IsSub &&
7234 IsSupported(ST->hasSME2() || ST->hasSVE2p1(), ST->hasF16F32DOT()) &&
7235 AccumLT.second.getScalarType() == MVT::f32 &&
7236 InputLT.second.getScalarType() == MVT::f16)
7237 return Cost;
7238
7239 // For a ratio of 2, we can use *mlal and *mlsl top/bottom instructions.
7240 if (Ratio == 2 && !IsUSDot) {
7241 MVT InVT = InputLT.second.getScalarType();
7242
7243 // SVE2 [us]ml[as]lb/t and NEON [us]ml[as]l(2). A pure widening add with a
7244 // ratio of 2 can use [SU]ADALP instead.
7245 if (IsSupported(ST->hasSVE2() || ST->hasSME(), true) &&
7246 llvm::is_contained(Set: {MVT::i8, MVT::i16, MVT::i32}, Element: InVT.SimpleTy))
7247 return (BinOp || IsSub) ? Cost * 2 : Cost;
7248
7249 // SVE2 fml[as]lb/t and NEON fml[as]l(2)
7250 if (IsSupported(ST->hasSVE2(), ST->hasFP16FML()) && InVT == MVT::f16)
7251 return Cost * 2;
7252
7253 // SME2/SVE2p1 bfmlslb/t
7254 if (IsSupported(ST->hasSVE2p1() || ST->hasSME2(), false) &&
7255 InVT == MVT::bf16 && IsSub)
7256 return Cost * 2;
7257
7258 // FP partial sub-reductions that don't map to a specific instruction,
7259 // carry an extra cost for implementing an extra negation:
7260 // partial_reduce_fmls acc, lhs, rhs
7261 // <=> partial_reduce_fmla acc, lhs, -rhs
7262 InstructionCost FNegCost = IsSub ? InputLT.first * TTI::TCC_Basic : 0;
7263
7264 // SVE and NEON bfmlalb/t
7265 if (IsSupported(ST->hasBF16(), ST->hasBF16()) && InVT == MVT::bf16)
7266 return Cost * 2 + FNegCost;
7267 }
7268
7269 return BaseT::getPartialReductionCost(Opcode, InputTypeA, InputTypeB,
7270 AccumType, VF, OpAExtend, OpBExtend,
7271 BinOp, CostKind, FMF);
7272}
7273
7274InstructionCost AArch64TTIImpl::getShuffleCost(
7275 TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy,
7276 TTI::TargetCostKind CostKind, ArrayRef<int> Mask, int Index,
7277 VectorType *SubTp, ArrayRef<const Value *> Args, const Instruction *CtxI,
7278 TTI::VectorInstrContext VIC) const {
7279 assert((Mask.empty() || DstTy->isScalableTy() ||
7280 Mask.size() == DstTy->getElementCount().getKnownMinValue()) &&
7281 "Expected the Mask to match the return size if given");
7282 assert(SrcTy->getScalarType() == DstTy->getScalarType() &&
7283 "Expected the same scalar types");
7284 std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Ty: SrcTy);
7285
7286 // If we have a Mask, and the LT is being legalized somehow, split the Mask
7287 // into smaller vectors and sum the cost of each shuffle.
7288 if (!Mask.empty() && isa<FixedVectorType>(Val: SrcTy) && LT.second.isVector() &&
7289 LT.second.getScalarSizeInBits() * Mask.size() > 128 &&
7290 SrcTy->getScalarSizeInBits() == LT.second.getScalarSizeInBits() &&
7291 Mask.size() > LT.second.getVectorNumElements() && !Index && !SubTp) {
7292 // Check for LD3/LD4 instructions, which are represented in llvm IR as
7293 // deinterleaving-shuffle(load). The shuffle cost could potentially be free,
7294 // but we model it with a cost of LT.first so that LD3/LD4 have a higher
7295 // cost than just the load.
7296 if (Args.size() >= 1 && isa<LoadInst>(Val: Args[0]) &&
7297 (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 3) ||
7298 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 4)))
7299 return std::max<InstructionCost>(a: 1, b: LT.first / 4);
7300
7301 // Check for ST3/ST4 instructions, which are represented in llvm IR as
7302 // store(interleaving-shuffle). The shuffle cost could potentially be free,
7303 // but we model it with a cost of LT.first so that ST3/ST4 have a higher
7304 // cost than just the store.
7305 if (CtxI && CtxI->hasOneUse() && isa<StoreInst>(Val: *CtxI->user_begin()) &&
7306 (ShuffleVectorInst::isInterleaveMask(
7307 Mask, Factor: 4, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2) ||
7308 ShuffleVectorInst::isInterleaveMask(
7309 Mask, Factor: 3, NumInputElts: SrcTy->getElementCount().getKnownMinValue() * 2)))
7310 return LT.first;
7311
7312 unsigned TpNumElts = Mask.size();
7313 unsigned LTNumElts = LT.second.getVectorNumElements();
7314 unsigned NumVecs = (TpNumElts + LTNumElts - 1) / LTNumElts;
7315 VectorType *NTp = VectorType::get(ElementType: SrcTy->getScalarType(),
7316 EC: LT.second.getVectorElementCount());
7317 InstructionCost Cost;
7318 std::map<std::tuple<unsigned, unsigned, SmallVector<int>>, InstructionCost>
7319 PreviousCosts;
7320 for (unsigned N = 0; N < NumVecs; N++) {
7321 SmallVector<int> NMask;
7322 // Split the existing mask into chunks of size LTNumElts. Track the source
7323 // sub-vectors to ensure the result has at most 2 inputs.
7324 unsigned Source1 = -1U, Source2 = -1U;
7325 unsigned NumSources = 0;
7326 for (unsigned E = 0; E < LTNumElts; E++) {
7327 int MaskElt = (N * LTNumElts + E < TpNumElts) ? Mask[N * LTNumElts + E]
7328 : PoisonMaskElem;
7329 if (MaskElt < 0) {
7330 NMask.push_back(Elt: PoisonMaskElem);
7331 continue;
7332 }
7333
7334 // Calculate which source from the input this comes from and whether it
7335 // is new to us.
7336 unsigned Source = MaskElt / LTNumElts;
7337 if (NumSources == 0) {
7338 Source1 = Source;
7339 NumSources = 1;
7340 } else if (NumSources == 1 && Source != Source1) {
7341 Source2 = Source;
7342 NumSources = 2;
7343 } else if (NumSources >= 2 && Source != Source1 && Source != Source2) {
7344 NumSources++;
7345 }
7346
7347 // Add to the new mask. For the NumSources>2 case these are not correct,
7348 // but are only used for the modular lane number.
7349 if (Source == Source1)
7350 NMask.push_back(Elt: MaskElt % LTNumElts);
7351 else if (Source == Source2)
7352 NMask.push_back(Elt: MaskElt % LTNumElts + LTNumElts);
7353 else
7354 NMask.push_back(Elt: MaskElt % LTNumElts);
7355 }
7356 // Check if we have already generated this sub-shuffle, which means we
7357 // will have already generated the output. For example a <16 x i32> splat
7358 // will be the same sub-splat 4 times, which only needs to be generated
7359 // once and reused.
7360 auto Result =
7361 PreviousCosts.insert(x: {std::make_tuple(args&: Source1, args&: Source2, args&: NMask), 0});
7362 // Check if it was already in the map (already costed).
7363 if (!Result.second)
7364 continue;
7365 // If the sub-mask has at most 2 input sub-vectors then re-cost it using
7366 // getShuffleCost. If not then cost it using the worst case as the number
7367 // of element moves into a new vector.
7368 InstructionCost NCost =
7369 NumSources <= 2
7370 ? getShuffleCost(Kind: NumSources <= 1 ? TTI::SK_PermuteSingleSrc
7371 : TTI::SK_PermuteTwoSrc,
7372 DstTy: NTp, SrcTy: NTp, CostKind, Mask: NMask, Index: 0, SubTp: nullptr, Args,
7373 CtxI)
7374 : LTNumElts;
7375 Result.first->second = NCost;
7376 Cost += NCost;
7377 }
7378 return Cost;
7379 }
7380
7381 Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTy&: SubTp);
7382 bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector;
7383 // A subvector extract can be implemented with a NEON/SVE ext (or trivial
7384 // extract, if from lane 0) for 128-bit NEON vectors or legal SVE vectors.
7385 // This currently only handles low or high extracts to prevent SLP vectorizer
7386 // regressions.
7387 // Note that SVE's ext instruction is destructive, but it can be fused with
7388 // a movprfx to act like a constructive instruction.
7389 if (IsExtractSubvector && LT.second.isFixedLengthVector()) {
7390 if (LT.second.getFixedSizeInBits() >= 128 &&
7391 cast<FixedVectorType>(Val: SubTp)->getNumElements() ==
7392 LT.second.getVectorNumElements() / 2) {
7393 if (Index == 0)
7394 return 0;
7395 if (Index == (int)LT.second.getVectorNumElements() / 2)
7396 return 1;
7397 }
7398 Kind = TTI::SK_PermuteSingleSrc;
7399 }
7400 // FIXME: This was added to keep the costs equal when adding DstTys. Update
7401 // the code to handle length-changing shuffles.
7402 if (Kind == TTI::SK_InsertSubvector) {
7403 LT = getTypeLegalizationCost(Ty: DstTy);
7404 SrcTy = DstTy;
7405 }
7406
7407 // Check for identity masks, which we can treat as free for both fixed and
7408 // scalable vector paths.
7409 if (!Mask.empty() && LT.second.isFixedLengthVector() &&
7410 (Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc) &&
7411 all_of(Range: enumerate(First&: Mask), P: [](const auto &M) {
7412 return M.value() < 0 || M.value() == (int)M.index();
7413 }))
7414 return 0;
7415
7416 // Segmented shuffle matching.
7417 if (Kind == TTI::SK_PermuteSingleSrc && isa<FixedVectorType>(Val: SrcTy) &&
7418 !Mask.empty() && SrcTy->getPrimitiveSizeInBits().isNonZero() &&
7419 SrcTy->getPrimitiveSizeInBits().isKnownMultipleOf(
7420 RHS: AArch64::SVEBitsPerBlock)) {
7421
7422 FixedVectorType *VTy = cast<FixedVectorType>(Val: SrcTy);
7423 unsigned Segments =
7424 VTy->getPrimitiveSizeInBits() / AArch64::SVEBitsPerBlock;
7425 unsigned SegmentElts = VTy->getNumElements() / Segments;
7426
7427 // dupq zd.t, zn.t[idx]
7428 if ((ST->hasSVE2p1() || ST->hasSME2p1()) &&
7429 ST->isSVEorStreamingSVEAvailable() &&
7430 isDUPQMask(Mask, Segments, SegmentSize: SegmentElts))
7431 return LT.first;
7432
7433 // mov zd.q, vn
7434 if (ST->isSVEorStreamingSVEAvailable() &&
7435 isDUPFirstSegmentMask(Mask, Segments, SegmentSize: SegmentElts))
7436 return LT.first;
7437 }
7438
7439 // Check for broadcast loads, which are supported by the LD1R instruction.
7440 // In terms of code-size, the shuffle vector is free when a load + dup get
7441 // folded into a LD1R. That's what we check and return here. For performance
7442 // and reciprocal throughput, a LD1R is not completely free. In this case, we
7443 // return the cost for the broadcast below (i.e. 1 for most/all types), so
7444 // that we model the load + dup sequence slightly higher because LD1R is a
7445 // high latency instruction.
7446 if (CostKind == TTI::TCK_CodeSize && Kind == TTI::SK_Broadcast) {
7447 bool IsLoad = !Args.empty() && isa<LoadInst>(Val: Args[0]);
7448 if (IsLoad && LT.second.isVector() &&
7449 isLegalBroadcastLoad(ElementTy: SrcTy->getElementType(),
7450 NumElements: LT.second.getVectorElementCount()))
7451 return 0;
7452 }
7453
7454 // If we have 4 elements for the shuffle and a Mask, get the cost straight
7455 // from the perfect shuffle tables.
7456 if (Mask.size() == 4 &&
7457 SrcTy->getElementCount() == ElementCount::getFixed(MinVal: 4) &&
7458 (SrcTy->getScalarSizeInBits() == 16 ||
7459 SrcTy->getScalarSizeInBits() == 32) &&
7460 all_of(Range&: Mask, P: [](int E) { return E < 8; }))
7461 return getPerfectShuffleCost(M: Mask);
7462
7463 // Check for other shuffles that are not SK_ kinds but we have native
7464 // instructions for, for example ZIP and UZP.
7465 unsigned Unused;
7466 if (LT.second.isFixedLengthVector() &&
7467 LT.second.getVectorNumElements() == Mask.size() &&
7468 (Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc ||
7469 // Discrepancies between isTRNMask and ShuffleVectorInst::isTransposeMask
7470 // mean that we can end up with shuffles that satisfy isTRNMask, but end
7471 // up labelled as TTI::SK_InsertSubvector. (e.g. {2, 0}).
7472 Kind == TTI::SK_InsertSubvector) &&
7473 (isZIPMask(M: Mask, NumElts: LT.second.getVectorNumElements(), WhichResultOut&: Unused, OperandOrderOut&: Unused) ||
7474 isTRNMask(M: Mask, NumElts: LT.second.getVectorNumElements(), WhichResultOut&: Unused, OperandOrderOut&: Unused) ||
7475 isUZPMask(M: Mask, NumElts: LT.second.getVectorNumElements(), WhichResultOut&: Unused) ||
7476 isREVMask(M: Mask, EltSize: LT.second.getScalarSizeInBits(),
7477 NumElts: LT.second.getVectorNumElements(), BlockSize: 16) ||
7478 isREVMask(M: Mask, EltSize: LT.second.getScalarSizeInBits(),
7479 NumElts: LT.second.getVectorNumElements(), BlockSize: 32) ||
7480 isREVMask(M: Mask, EltSize: LT.second.getScalarSizeInBits(),
7481 NumElts: LT.second.getVectorNumElements(), BlockSize: 64) ||
7482 // Check for non-zero lane splats
7483 all_of(Range: drop_begin(RangeOrContainer&: Mask),
7484 P: [&Mask](int M) { return M < 0 || M == Mask[0]; })))
7485 return 1;
7486
7487 if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose ||
7488 Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc ||
7489 Kind == TTI::SK_Reverse || Kind == TTI::SK_Splice) {
7490 static const CostTblEntry ShuffleTbl[] = {
7491 // Broadcast shuffle kinds can be performed with 'dup'.
7492 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i8, .Cost: 1},
7493 {.ISD: TTI::SK_Broadcast, .Type: MVT::v16i8, .Cost: 1},
7494 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i16, .Cost: 1},
7495 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8i16, .Cost: 1},
7496 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i32, .Cost: 1},
7497 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4i32, .Cost: 1},
7498 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2i64, .Cost: 1},
7499 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4f16, .Cost: 1},
7500 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8f16, .Cost: 1},
7501 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4bf16, .Cost: 1},
7502 {.ISD: TTI::SK_Broadcast, .Type: MVT::v8bf16, .Cost: 1},
7503 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2f32, .Cost: 1},
7504 {.ISD: TTI::SK_Broadcast, .Type: MVT::v4f32, .Cost: 1},
7505 {.ISD: TTI::SK_Broadcast, .Type: MVT::v2f64, .Cost: 1},
7506 // Transpose shuffle kinds can be performed with 'trn1/trn2' and
7507 // 'zip1/zip2' instructions.
7508 {.ISD: TTI::SK_Transpose, .Type: MVT::v8i8, .Cost: 1},
7509 {.ISD: TTI::SK_Transpose, .Type: MVT::v16i8, .Cost: 1},
7510 {.ISD: TTI::SK_Transpose, .Type: MVT::v4i16, .Cost: 1},
7511 {.ISD: TTI::SK_Transpose, .Type: MVT::v8i16, .Cost: 1},
7512 {.ISD: TTI::SK_Transpose, .Type: MVT::v2i32, .Cost: 1},
7513 {.ISD: TTI::SK_Transpose, .Type: MVT::v4i32, .Cost: 1},
7514 {.ISD: TTI::SK_Transpose, .Type: MVT::v2i64, .Cost: 1},
7515 {.ISD: TTI::SK_Transpose, .Type: MVT::v4f16, .Cost: 1},
7516 {.ISD: TTI::SK_Transpose, .Type: MVT::v8f16, .Cost: 1},
7517 {.ISD: TTI::SK_Transpose, .Type: MVT::v4bf16, .Cost: 1},
7518 {.ISD: TTI::SK_Transpose, .Type: MVT::v8bf16, .Cost: 1},
7519 {.ISD: TTI::SK_Transpose, .Type: MVT::v2f32, .Cost: 1},
7520 {.ISD: TTI::SK_Transpose, .Type: MVT::v4f32, .Cost: 1},
7521 {.ISD: TTI::SK_Transpose, .Type: MVT::v2f64, .Cost: 1},
7522 // Select shuffle kinds.
7523 // TODO: handle vXi8/vXi16.
7524 {.ISD: TTI::SK_Select, .Type: MVT::v2i32, .Cost: 1}, // mov.
7525 {.ISD: TTI::SK_Select, .Type: MVT::v4i32, .Cost: 2}, // rev+trn (or similar).
7526 {.ISD: TTI::SK_Select, .Type: MVT::v2i64, .Cost: 1}, // mov.
7527 {.ISD: TTI::SK_Select, .Type: MVT::v2f32, .Cost: 1}, // mov.
7528 {.ISD: TTI::SK_Select, .Type: MVT::v4f32, .Cost: 2}, // rev+trn (or similar).
7529 {.ISD: TTI::SK_Select, .Type: MVT::v2f64, .Cost: 1}, // mov.
7530 // PermuteSingleSrc shuffle kinds.
7531 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i32, .Cost: 1}, // mov.
7532 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i32, .Cost: 3}, // perfectshuffle worst case.
7533 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2i64, .Cost: 1}, // mov.
7534 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2f32, .Cost: 1}, // mov.
7535 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f32, .Cost: 3}, // perfectshuffle worst case.
7536 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v2f64, .Cost: 1}, // mov.
7537 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4i16, .Cost: 3}, // perfectshuffle worst case.
7538 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4f16, .Cost: 3}, // perfectshuffle worst case.
7539 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v4bf16, .Cost: 3}, // same
7540 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i16, .Cost: 8}, // constpool + load + tbl
7541 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8f16, .Cost: 8}, // constpool + load + tbl
7542 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8bf16, .Cost: 8}, // constpool + load + tbl
7543 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v8i8, .Cost: 8}, // constpool + load + tbl
7544 {.ISD: TTI::SK_PermuteSingleSrc, .Type: MVT::v16i8, .Cost: 8}, // constpool + load + tbl
7545 // Reverse can be lowered with `rev`.
7546 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i32, .Cost: 1}, // REV64
7547 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i32, .Cost: 2}, // REV64; EXT
7548 {.ISD: TTI::SK_Reverse, .Type: MVT::v2i64, .Cost: 1}, // EXT
7549 {.ISD: TTI::SK_Reverse, .Type: MVT::v2f32, .Cost: 1}, // REV64
7550 {.ISD: TTI::SK_Reverse, .Type: MVT::v4f32, .Cost: 2}, // REV64; EXT
7551 {.ISD: TTI::SK_Reverse, .Type: MVT::v2f64, .Cost: 1}, // EXT
7552 {.ISD: TTI::SK_Reverse, .Type: MVT::v8f16, .Cost: 2}, // REV64; EXT
7553 {.ISD: TTI::SK_Reverse, .Type: MVT::v8bf16, .Cost: 2}, // REV64; EXT
7554 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i16, .Cost: 2}, // REV64; EXT
7555 {.ISD: TTI::SK_Reverse, .Type: MVT::v16i8, .Cost: 2}, // REV64; EXT
7556 {.ISD: TTI::SK_Reverse, .Type: MVT::v4f16, .Cost: 1}, // REV64
7557 {.ISD: TTI::SK_Reverse, .Type: MVT::v4bf16, .Cost: 1}, // REV64
7558 {.ISD: TTI::SK_Reverse, .Type: MVT::v4i16, .Cost: 1}, // REV64
7559 {.ISD: TTI::SK_Reverse, .Type: MVT::v8i8, .Cost: 1}, // REV64
7560 // Splice can all be lowered as `ext`.
7561 {.ISD: TTI::SK_Splice, .Type: MVT::v2i32, .Cost: 1},
7562 {.ISD: TTI::SK_Splice, .Type: MVT::v4i32, .Cost: 1},
7563 {.ISD: TTI::SK_Splice, .Type: MVT::v2i64, .Cost: 1},
7564 {.ISD: TTI::SK_Splice, .Type: MVT::v2f32, .Cost: 1},
7565 {.ISD: TTI::SK_Splice, .Type: MVT::v4f32, .Cost: 1},
7566 {.ISD: TTI::SK_Splice, .Type: MVT::v2f64, .Cost: 1},
7567 {.ISD: TTI::SK_Splice, .Type: MVT::v8f16, .Cost: 1},
7568 {.ISD: TTI::SK_Splice, .Type: MVT::v8bf16, .Cost: 1},
7569 {.ISD: TTI::SK_Splice, .Type: MVT::v8i16, .Cost: 1},
7570 {.ISD: TTI::SK_Splice, .Type: MVT::v16i8, .Cost: 1},
7571 {.ISD: TTI::SK_Splice, .Type: MVT::v4f16, .Cost: 1},
7572 {.ISD: TTI::SK_Splice, .Type: MVT::v4bf16, .Cost: 1},
7573 {.ISD: TTI::SK_Splice, .Type: MVT::v4i16, .Cost: 1},
7574 {.ISD: TTI::SK_Splice, .Type: MVT::v8i8, .Cost: 1},
7575 // Broadcast shuffle kinds for scalable vectors
7576 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv16i8, .Cost: 1},
7577 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv8i16, .Cost: 1},
7578 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4i32, .Cost: 1},
7579 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2i64, .Cost: 1},
7580 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2f16, .Cost: 1},
7581 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4f16, .Cost: 1},
7582 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv8f16, .Cost: 1},
7583 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2bf16, .Cost: 1},
7584 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4bf16, .Cost: 1},
7585 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv8bf16, .Cost: 1},
7586 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2f32, .Cost: 1},
7587 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4f32, .Cost: 1},
7588 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2f64, .Cost: 1},
7589 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv16i1, .Cost: 1},
7590 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv8i1, .Cost: 1},
7591 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv4i1, .Cost: 1},
7592 {.ISD: TTI::SK_Broadcast, .Type: MVT::nxv2i1, .Cost: 1},
7593 // Handle the cases for vector.reverse with scalable vectors
7594 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv16i8, .Cost: 1},
7595 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv8i16, .Cost: 1},
7596 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4i32, .Cost: 1},
7597 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2i64, .Cost: 1},
7598 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2f16, .Cost: 1},
7599 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4f16, .Cost: 1},
7600 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv8f16, .Cost: 1},
7601 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2bf16, .Cost: 1},
7602 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4bf16, .Cost: 1},
7603 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv8bf16, .Cost: 1},
7604 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2f32, .Cost: 1},
7605 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4f32, .Cost: 1},
7606 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2f64, .Cost: 1},
7607 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv16i1, .Cost: 1},
7608 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv8i1, .Cost: 1},
7609 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv4i1, .Cost: 1},
7610 {.ISD: TTI::SK_Reverse, .Type: MVT::nxv2i1, .Cost: 1},
7611 };
7612 if (const auto *Entry = CostTableLookup(Table: ShuffleTbl, ISD: Kind, Ty: LT.second))
7613 return LT.first * Entry->Cost;
7614 }
7615
7616 if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(Val: SrcTy))
7617 return getSpliceCost(Tp: SrcTy, Index, CostKind);
7618
7619 // Inserting a subvector can often be done with either a D, S or H register
7620 // move, so long as the inserted vector is "aligned".
7621 if (Kind == TTI::SK_InsertSubvector && LT.second.isFixedLengthVector() &&
7622 LT.second.getSizeInBits() <= 128 && SubTp) {
7623 std::pair<InstructionCost, MVT> SubLT = getTypeLegalizationCost(Ty: SubTp);
7624 if (SubLT.second.isVector()) {
7625 int NumElts = LT.second.getVectorNumElements();
7626 int NumSubElts = SubLT.second.getVectorNumElements();
7627 if ((Index % NumSubElts) == 0 && (NumElts % NumSubElts) == 0)
7628 return SubLT.first;
7629 }
7630 }
7631
7632 // Restore optimal kind.
7633 if (IsExtractSubvector)
7634 Kind = TTI::SK_ExtractSubvector;
7635 return BaseT::getShuffleCost(Kind, DstTy, SrcTy, CostKind, Mask, Index, SubTp,
7636 Args, CtxI);
7637}
7638
7639static bool containsDecreasingPointers(Loop *TheLoop,
7640 PredicatedScalarEvolution *PSE,
7641 const DominatorTree &DT) {
7642 const auto &Strides = SymbolicStrideMap();
7643 for (BasicBlock *BB : TheLoop->blocks()) {
7644 // Scan the instructions in the block and look for addresses that are
7645 // consecutive and decreasing.
7646 for (Instruction &I : *BB) {
7647 if (isa<LoadInst>(Val: &I) || isa<StoreInst>(Val: &I)) {
7648 Value *Ptr = getLoadStorePointerOperand(V: &I);
7649 Type *AccessTy = getLoadStoreType(I: &I);
7650 // Analyze assuming predicates will be added, but discard them; this
7651 // query only guides tail-folding and must not add runtime checks to the
7652 // loop.
7653 SmallVector<const SCEVPredicate *> Predicates;
7654 if (getPtrStride(PSE&: *PSE, AccessTy, Ptr, Lp: TheLoop, DT, StridesMap: Strides,
7655 /*ShouldCheckWrap=*/false, Predicates: &Predicates)
7656 .value_or(u: 0) < 0)
7657 return true;
7658 }
7659 }
7660 }
7661 return false;
7662}
7663
7664bool AArch64TTIImpl::preferFixedOverScalableIfEqualCost() const {
7665 return valueOr(X: ST->getCLOpts().sve_prefer_fixed_over_scalable_if_equal,
7666 Default: ST->useFixedOverScalableIfEqualCost());
7667}
7668
7669unsigned AArch64TTIImpl::getEpilogueVectorizationMinVF() const {
7670 return ST->getEpilogueVectorizationMinVF();
7671}
7672
7673bool AArch64TTIImpl::preferTailFoldingOverEpilogue(TailFoldingInfo *TFI) const {
7674 TailFoldingOpts DefaultOpts = ST->getSVETailFoldingDefaultOpts();
7675 if (!ST->hasSVE() || TailFoldingOptionLoc.isDisabled(DefaultBits: DefaultOpts))
7676 return false;
7677
7678 // We don't currently support vectorisation with interleaving for SVE - with
7679 // such loops we're better off not using tail-folding. This gives us a chance
7680 // to fall back on fixed-width vectorisation using NEON's ld2/st2/etc.
7681 if (TFI->IAI->hasGroups())
7682 return false;
7683
7684 TailFoldingOpts Required = TailFoldingOpts::Disabled;
7685 if (TFI->LVL->getReductionVars().size())
7686 Required |= TailFoldingOpts::Reductions;
7687 if (TFI->LVL->getFixedOrderRecurrences().size())
7688 Required |= TailFoldingOpts::Recurrences;
7689
7690 // We call this to discover whether any load/store pointers in the loop have
7691 // negative strides. This will require extra work to reverse the loop
7692 // predicate, which may be expensive.
7693 if (containsDecreasingPointers(TheLoop: TFI->LVL->getLoop(),
7694 PSE: TFI->LVL->getPredicatedScalarEvolution(),
7695 DT: *TFI->LVL->getDominatorTree()))
7696 Required |= TailFoldingOpts::Reverse;
7697 if (Required == TailFoldingOpts::Disabled)
7698 Required |= TailFoldingOpts::Simple;
7699
7700 if (!TailFoldingOptionLoc.satisfies(DefaultBits: DefaultOpts, Required))
7701 return false;
7702
7703 // Don't tail-fold for tight loops where we would be better off interleaving
7704 // with an unpredicated loop.
7705 unsigned NumInsns = 0;
7706 for (BasicBlock *BB : TFI->LVL->getLoop()->blocks()) {
7707 NumInsns += BB->size();
7708 }
7709
7710 // We expect 4 of these to be a IV PHI, IV add, IV compare and branch.
7711 return NumInsns >= ST->getCLOpts().sve_tail_folding_insn_threshold;
7712}
7713
7714InstructionCost
7715AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
7716 StackOffset BaseOffset, bool HasBaseReg,
7717 int64_t Scale, unsigned AddrSpace) const {
7718 // Scaling factors are not free at all.
7719 // Operands | Rt Latency
7720 // -------------------------------------------
7721 // Rt, [Xn, Xm] | 4
7722 // -------------------------------------------
7723 // Rt, [Xn, Xm, lsl #imm] | Rn: 4 Rm: 5
7724 // Rt, [Xn, Wm, <extend> #imm] |
7725 TargetLoweringBase::AddrMode AM;
7726 AM.BaseGV = BaseGV;
7727 AM.BaseOffs = BaseOffset.getFixed();
7728 AM.HasBaseReg = HasBaseReg;
7729 AM.Scale = Scale;
7730 AM.ScalableOffset = BaseOffset.getScalable();
7731 if (getTLI()->isLegalAddressingMode(DL, AM, Ty, AS: AddrSpace))
7732 // Scale represents reg2 * scale, thus account for 1 if
7733 // it is not equal to 0 or 1.
7734 return AM.Scale != 0 && AM.Scale != 1;
7735 return InstructionCost::getInvalid();
7736}
7737
7738bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(
7739 const Instruction *I) const {
7740 if (ST->getCLOpts().enable_aarch64_or_like_select) {
7741 // For the binary operators (e.g. or) we need to be more careful than
7742 // selects, here we only transform them if they are already at a natural
7743 // break point in the code - the end of a block with an unconditional
7744 // terminator.
7745 if (I->getOpcode() == Instruction::Or &&
7746 isa<UncondBrInst>(Val: I->getNextNode()))
7747 return true;
7748
7749 if (I->getOpcode() == Instruction::Add ||
7750 I->getOpcode() == Instruction::Sub)
7751 return true;
7752 }
7753 return BaseT::shouldTreatInstructionLikeSelect(I);
7754}
7755
7756bool AArch64TTIImpl::isLSRCostLess(
7757 const TargetTransformInfo::LSRCost &C1,
7758 const TargetTransformInfo::LSRCost &C2) const {
7759 // AArch64 specific here is adding the number of instructions to the
7760 // comparison (though not as the first consideration, as some targets do)
7761 // along with changing the priority of the base additions.
7762 // TODO: Maybe a more nuanced tradeoff between instruction count
7763 // and number of registers? To be investigated at a later date.
7764 if (ST->getCLOpts().enable_aarch64_lsr_cost_opt)
7765 return std::tie(args: C1.NumRegs, args: C1.Insns, args: C1.NumBaseAdds, args: C1.AddRecCost,
7766 args: C1.NumIVMuls, args: C1.ScaleCost, args: C1.ImmCost, args: C1.SetupCost) <
7767 std::tie(args: C2.NumRegs, args: C2.Insns, args: C2.NumBaseAdds, args: C2.AddRecCost,
7768 args: C2.NumIVMuls, args: C2.ScaleCost, args: C2.ImmCost, args: C2.SetupCost);
7769
7770 return TargetTransformInfoImplBase::isLSRCostLess(C1, C2);
7771}
7772
7773static bool isSplatShuffle(Value *V) {
7774 if (auto *Shuf = dyn_cast<ShuffleVectorInst>(Val: V))
7775 return all_equal(Range: Shuf->getShuffleMask());
7776 return false;
7777}
7778
7779/// Check if both Op1 and Op2 are shufflevector extracts of either the lower
7780/// or upper half of the vector elements.
7781static bool areExtractShuffleVectors(Value *Op1, Value *Op2,
7782 bool AllowSplat = false) {
7783 // Scalable types can't be extract shuffle vectors.
7784 if (Op1->getType()->isScalableTy() || Op2->getType()->isScalableTy())
7785 return false;
7786
7787 auto areTypesHalfed = [](Value *FullV, Value *HalfV) {
7788 auto *FullTy = FullV->getType();
7789 auto *HalfTy = HalfV->getType();
7790 return FullTy->getPrimitiveSizeInBits().getFixedValue() ==
7791 2 * HalfTy->getPrimitiveSizeInBits().getFixedValue();
7792 };
7793
7794 auto extractHalf = [](Value *FullV, Value *HalfV) {
7795 auto *FullVT = cast<FixedVectorType>(Val: FullV->getType());
7796 auto *HalfVT = cast<FixedVectorType>(Val: HalfV->getType());
7797 return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
7798 };
7799
7800 ArrayRef<int> M1, M2;
7801 Value *S1Op1 = nullptr, *S2Op1 = nullptr;
7802 if (!match(V: Op1, P: m_Shuffle(v1: m_Value(V&: S1Op1), v2: m_Undef(), mask: m_Mask(M1))) ||
7803 !match(V: Op2, P: m_Shuffle(v1: m_Value(V&: S2Op1), v2: m_Undef(), mask: m_Mask(M2))))
7804 return false;
7805
7806 // If we allow splats, set S1Op1/S2Op1 to nullptr for the relevant arg so that
7807 // it is not checked as an extract below.
7808 if (AllowSplat && isSplatShuffle(V: Op1))
7809 S1Op1 = nullptr;
7810 if (AllowSplat && isSplatShuffle(V: Op2))
7811 S2Op1 = nullptr;
7812
7813 // Check that the operands are half as wide as the result and we extract
7814 // half of the elements of the input vectors.
7815 if ((S1Op1 && (!areTypesHalfed(S1Op1, Op1) || !extractHalf(S1Op1, Op1))) ||
7816 (S2Op1 && (!areTypesHalfed(S2Op1, Op2) || !extractHalf(S2Op1, Op2))))
7817 return false;
7818
7819 // Check the mask extracts either the lower or upper half of vector
7820 // elements.
7821 int M1Start = 0;
7822 int M2Start = 0;
7823 int NumElements = cast<FixedVectorType>(Val: Op1->getType())->getNumElements() * 2;
7824 if ((S1Op1 &&
7825 !ShuffleVectorInst::isExtractSubvectorMask(Mask: M1, NumSrcElts: NumElements, Index&: M1Start)) ||
7826 (S2Op1 &&
7827 !ShuffleVectorInst::isExtractSubvectorMask(Mask: M2, NumSrcElts: NumElements, Index&: M2Start)))
7828 return false;
7829
7830 if ((M1Start != 0 && M1Start != (NumElements / 2)) ||
7831 (M2Start != 0 && M2Start != (NumElements / 2)))
7832 return false;
7833 if (S1Op1 && S2Op1 && M1Start != M2Start)
7834 return false;
7835
7836 return true;
7837}
7838
7839/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
7840/// of the vector elements.
7841static bool areExtractExts(Value *Ext1, Value *Ext2) {
7842 auto areExtDoubled = [](Instruction *Ext) {
7843 return Ext->getType()->getScalarSizeInBits() ==
7844 2 * Ext->getOperand(i: 0)->getType()->getScalarSizeInBits();
7845 };
7846
7847 if (!match(V: Ext1, P: m_ZExtOrSExt(Op: m_Value())) ||
7848 !match(V: Ext2, P: m_ZExtOrSExt(Op: m_Value())) ||
7849 !areExtDoubled(cast<Instruction>(Val: Ext1)) ||
7850 !areExtDoubled(cast<Instruction>(Val: Ext2)))
7851 return false;
7852
7853 return true;
7854}
7855
7856/// Check if Op could be used with vmull_high_p64 intrinsic.
7857static bool isOperandOfVmullHighP64(Value *Op) {
7858 Value *VectorOperand = nullptr;
7859 ConstantInt *ElementIndex = nullptr;
7860 return match(V: Op, P: m_ExtractElt(Val: m_Value(V&: VectorOperand),
7861 Idx: m_ConstantInt(CI&: ElementIndex))) &&
7862 ElementIndex->getValue() == 1 &&
7863 isa<FixedVectorType>(Val: VectorOperand->getType()) &&
7864 cast<FixedVectorType>(Val: VectorOperand->getType())->getNumElements() == 2;
7865}
7866
7867/// Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
7868static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2) {
7869 return isOperandOfVmullHighP64(Op: Op1) && isOperandOfVmullHighP64(Op: Op2);
7870}
7871
7872static bool shouldSinkVectorOfPtrs(Value *Ptrs, SmallVectorImpl<Use *> &Ops) {
7873 // Restrict ourselves to the form CodeGenPrepare typically constructs.
7874 auto *GEP = dyn_cast<GetElementPtrInst>(Val: Ptrs);
7875 if (!GEP || GEP->getNumOperands() != 2)
7876 return false;
7877
7878 Value *Base = GEP->getOperand(i_nocapture: 0);
7879 Value *Offsets = GEP->getOperand(i_nocapture: 1);
7880
7881 // We only care about scalar_base+vector_offsets.
7882 if (Base->getType()->isVectorTy() || !Offsets->getType()->isVectorTy())
7883 return false;
7884
7885 // Sink extends that would allow us to use 32-bit offset vectors.
7886 if (isa<SExtInst>(Val: Offsets) || isa<ZExtInst>(Val: Offsets)) {
7887 auto *OffsetsInst = cast<Instruction>(Val: Offsets);
7888 if (OffsetsInst->getType()->getScalarSizeInBits() > 32 &&
7889 OffsetsInst->getOperand(i: 0)->getType()->getScalarSizeInBits() <= 32)
7890 Ops.push_back(Elt: &GEP->getOperandUse(i: 1));
7891 }
7892
7893 // Sink the GEP.
7894 return true;
7895}
7896
7897/// We want to sink following cases:
7898/// (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale;
7899/// (add|sub|gep) A, ((mul|shl) zext(vscale), imm);
7900static bool shouldSinkVScale(Value *Op, SmallVectorImpl<Use *> &Ops) {
7901 if (match(V: Op, P: m_VScale()))
7902 return true;
7903 if (match(V: Op, P: m_Shl(L: m_VScale(), R: m_ConstantInt())) ||
7904 match(V: Op, P: m_Mul(L: m_VScale(), R: m_ConstantInt()))) {
7905 Ops.push_back(Elt: &cast<Instruction>(Val: Op)->getOperandUse(i: 0));
7906 return true;
7907 }
7908 if (match(V: Op, P: m_Shl(L: m_ZExt(Op: m_VScale()), R: m_ConstantInt())) ||
7909 match(V: Op, P: m_Mul(L: m_ZExt(Op: m_VScale()), R: m_ConstantInt()))) {
7910 Value *ZExtOp = cast<Instruction>(Val: Op)->getOperand(i: 0);
7911 Ops.push_back(Elt: &cast<Instruction>(Val: ZExtOp)->getOperandUse(i: 0));
7912 Ops.push_back(Elt: &cast<Instruction>(Val: Op)->getOperandUse(i: 0));
7913 return true;
7914 }
7915 return false;
7916}
7917
7918static bool isFNeg(Value *Op) { return match(V: Op, P: m_FNeg(X: m_Value())); }
7919
7920/// Check if sinking \p I's operands to I's basic block is profitable, because
7921/// the operands can be folded into a target instruction, e.g.
7922/// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2).
7923bool AArch64TTIImpl::isProfitableToSinkOperands(
7924 Instruction *I, SmallVectorImpl<Use *> &Ops) const {
7925 if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: I)) {
7926 switch (II->getIntrinsicID()) {
7927 case Intrinsic::aarch64_neon_smull:
7928 case Intrinsic::aarch64_neon_umull:
7929 if (areExtractShuffleVectors(Op1: II->getOperand(i_nocapture: 0), Op2: II->getOperand(i_nocapture: 1),
7930 /*AllowSplat=*/true)) {
7931 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7932 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7933 return true;
7934 }
7935 [[fallthrough]];
7936
7937 case Intrinsic::fma:
7938 case Intrinsic::fmuladd:
7939 if (isa<VectorType>(Val: I->getType()) &&
7940 cast<VectorType>(Val: I->getType())->getElementType()->isHalfTy() &&
7941 !ST->hasFullFP16())
7942 return false;
7943
7944 if (isFNeg(Op: II->getOperand(i_nocapture: 0)))
7945 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7946 if (isFNeg(Op: II->getOperand(i_nocapture: 1)))
7947 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7948
7949 [[fallthrough]];
7950 case Intrinsic::aarch64_neon_sqdmull:
7951 case Intrinsic::aarch64_neon_sqdmulh:
7952 case Intrinsic::aarch64_neon_sqrdmulh:
7953 // Sink splats for index lane variants
7954 if (isSplatShuffle(V: II->getOperand(i_nocapture: 0)))
7955 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7956 if (isSplatShuffle(V: II->getOperand(i_nocapture: 1)))
7957 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7958 return !Ops.empty();
7959 case Intrinsic::aarch64_neon_fmlal:
7960 case Intrinsic::aarch64_neon_fmlal2:
7961 case Intrinsic::aarch64_neon_fmlsl:
7962 case Intrinsic::aarch64_neon_fmlsl2:
7963 // Sink splats for index lane variants
7964 if (isSplatShuffle(V: II->getOperand(i_nocapture: 1)))
7965 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7966 if (isSplatShuffle(V: II->getOperand(i_nocapture: 2)))
7967 Ops.push_back(Elt: &II->getOperandUse(i: 2));
7968 return !Ops.empty();
7969 case Intrinsic::aarch64_sve_ptest_first:
7970 case Intrinsic::aarch64_sve_ptest_last:
7971 if (auto *IIOp = dyn_cast<IntrinsicInst>(Val: II->getOperand(i_nocapture: 0)))
7972 if (IIOp->getIntrinsicID() == Intrinsic::aarch64_sve_ptrue)
7973 Ops.push_back(Elt: &II->getOperandUse(i: 0));
7974 return !Ops.empty();
7975 case Intrinsic::aarch64_sme_write_horiz:
7976 case Intrinsic::aarch64_sme_write_vert:
7977 case Intrinsic::aarch64_sme_writeq_horiz:
7978 case Intrinsic::aarch64_sme_writeq_vert: {
7979 auto *Idx = dyn_cast<Instruction>(Val: II->getOperand(i_nocapture: 1));
7980 if (!Idx || Idx->getOpcode() != Instruction::Add)
7981 return false;
7982 Ops.push_back(Elt: &II->getOperandUse(i: 1));
7983 return true;
7984 }
7985 case Intrinsic::aarch64_sme_read_horiz:
7986 case Intrinsic::aarch64_sme_read_vert:
7987 case Intrinsic::aarch64_sme_readq_horiz:
7988 case Intrinsic::aarch64_sme_readq_vert:
7989 case Intrinsic::aarch64_sme_ld1b_vert:
7990 case Intrinsic::aarch64_sme_ld1h_vert:
7991 case Intrinsic::aarch64_sme_ld1w_vert:
7992 case Intrinsic::aarch64_sme_ld1d_vert:
7993 case Intrinsic::aarch64_sme_ld1q_vert:
7994 case Intrinsic::aarch64_sme_st1b_vert:
7995 case Intrinsic::aarch64_sme_st1h_vert:
7996 case Intrinsic::aarch64_sme_st1w_vert:
7997 case Intrinsic::aarch64_sme_st1d_vert:
7998 case Intrinsic::aarch64_sme_st1q_vert:
7999 case Intrinsic::aarch64_sme_ld1b_horiz:
8000 case Intrinsic::aarch64_sme_ld1h_horiz:
8001 case Intrinsic::aarch64_sme_ld1w_horiz:
8002 case Intrinsic::aarch64_sme_ld1d_horiz:
8003 case Intrinsic::aarch64_sme_ld1q_horiz:
8004 case Intrinsic::aarch64_sme_st1b_horiz:
8005 case Intrinsic::aarch64_sme_st1h_horiz:
8006 case Intrinsic::aarch64_sme_st1w_horiz:
8007 case Intrinsic::aarch64_sme_st1d_horiz:
8008 case Intrinsic::aarch64_sme_st1q_horiz: {
8009 auto *Idx = dyn_cast<Instruction>(Val: II->getOperand(i_nocapture: 3));
8010 if (!Idx || Idx->getOpcode() != Instruction::Add)
8011 return false;
8012 Ops.push_back(Elt: &II->getOperandUse(i: 3));
8013 return true;
8014 }
8015 case Intrinsic::aarch64_neon_pmull:
8016 if (!areExtractShuffleVectors(Op1: II->getOperand(i_nocapture: 0), Op2: II->getOperand(i_nocapture: 1)))
8017 return false;
8018 Ops.push_back(Elt: &II->getOperandUse(i: 0));
8019 Ops.push_back(Elt: &II->getOperandUse(i: 1));
8020 return true;
8021 case Intrinsic::aarch64_neon_pmull64:
8022 if (!areOperandsOfVmullHighP64(Op1: II->getArgOperand(i: 0),
8023 Op2: II->getArgOperand(i: 1)))
8024 return false;
8025 Ops.push_back(Elt: &II->getArgOperandUse(i: 0));
8026 Ops.push_back(Elt: &II->getArgOperandUse(i: 1));
8027 return true;
8028 case Intrinsic::masked_gather:
8029 if (!shouldSinkVectorOfPtrs(Ptrs: II->getArgOperand(i: 0), Ops))
8030 return false;
8031 Ops.push_back(Elt: &II->getArgOperandUse(i: 0));
8032 return true;
8033 case Intrinsic::masked_scatter:
8034 if (!shouldSinkVectorOfPtrs(Ptrs: II->getArgOperand(i: 1), Ops))
8035 return false;
8036 Ops.push_back(Elt: &II->getArgOperandUse(i: 1));
8037 return true;
8038 default:
8039 return false;
8040 }
8041 }
8042
8043 auto ShouldSinkCondition = [](Value *Cond,
8044 SmallVectorImpl<Use *> &Ops) -> bool {
8045 if (!isa<IntrinsicInst>(Val: Cond))
8046 return false;
8047 auto *II = dyn_cast<IntrinsicInst>(Val: Cond);
8048 if (II->getIntrinsicID() != Intrinsic::vector_reduce_or ||
8049 !isa<ScalableVectorType>(Val: II->getOperand(i_nocapture: 0)->getType()))
8050 return false;
8051 if (isa<CmpInst>(Val: II->getOperand(i_nocapture: 0)))
8052 Ops.push_back(Elt: &II->getOperandUse(i: 0));
8053 return true;
8054 };
8055
8056 switch (I->getOpcode()) {
8057 case Instruction::GetElementPtr:
8058 case Instruction::Add:
8059 case Instruction::Sub:
8060 // Sink vscales closer to uses for better isel
8061 for (unsigned Op = 0; Op < I->getNumOperands(); ++Op) {
8062 if (shouldSinkVScale(Op: I->getOperand(i: Op), Ops)) {
8063 Ops.push_back(Elt: &I->getOperandUse(i: Op));
8064 return true;
8065 }
8066 }
8067 break;
8068 case Instruction::Select: {
8069 if (!ShouldSinkCondition(I->getOperand(i: 0), Ops))
8070 return false;
8071
8072 Ops.push_back(Elt: &I->getOperandUse(i: 0));
8073 return true;
8074 }
8075 case Instruction::UncondBr:
8076 return false;
8077 case Instruction::CondBr: {
8078 if (!ShouldSinkCondition(cast<CondBrInst>(Val: I)->getCondition(), Ops))
8079 return false;
8080
8081 Ops.push_back(Elt: &I->getOperandUse(i: 0));
8082 return true;
8083 }
8084 case Instruction::FMul:
8085 // fmul with contract flag can be combined with fadd into fma.
8086 // Sinking fneg into this block enables fmls pattern.
8087 if (cast<FPMathOperator>(Val: I)->hasAllowContract()) {
8088 if (isFNeg(Op: I->getOperand(i: 0)))
8089 Ops.push_back(Elt: &I->getOperandUse(i: 0));
8090 if (isFNeg(Op: I->getOperand(i: 1)))
8091 Ops.push_back(Elt: &I->getOperandUse(i: 1));
8092 }
8093 break;
8094
8095 // Type | BIC | ORN | EON
8096 // ----------------+-----------+-----------+-----------
8097 // scalar | Base | Base | Base
8098 // scalar w/shift | - | - | -
8099 // fixed vector | NEON/Base | NEON/Base | BSL2N/Base
8100 // scalable vector | SVE | - | BSL2N
8101 case Instruction::Xor:
8102 // EON only for scalars (possibly expanded fixed vectors)
8103 // and vectors using the SVE2/SME BSL2N instruction.
8104 if (I->getType()->isVectorTy() && ST->isNeonAvailable()) {
8105 bool HasBSL2N =
8106 ST->isSVEorStreamingSVEAvailable() && (ST->hasSVE2() || ST->hasSME());
8107 if (!HasBSL2N)
8108 break;
8109 }
8110 [[fallthrough]];
8111 case Instruction::And:
8112 case Instruction::Or:
8113 // Even though we could use the SVE2/SME BSL2N instruction,
8114 // it might pessimize with an extra MOV depending on register allocation.
8115 if (I->getOpcode() == Instruction::Or &&
8116 isa<ScalableVectorType>(Val: I->getType()))
8117 break;
8118 // Shift can be fold into scalar AND/ORR/EOR,
8119 // but not the non-negated operand of BIC/ORN/EON.
8120 if (!(I->getType()->isVectorTy() && ST->hasNEON()) &&
8121 match(V: I, P: m_c_BinOp(L: m_Shift(L: m_Value(), R: m_ConstantInt()), R: m_Value())))
8122 break;
8123 for (auto &Op : I->operands()) {
8124 // (and/or/xor X, (not Y)) -> (bic/orn/eon X, Y)
8125 if (match(V: Op.get(), P: m_Not(V: m_Value()))) {
8126 Ops.push_back(Elt: &Op);
8127 return true;
8128 }
8129 // (and/or/xor X, (splat (not Y))) -> (bic/orn/eon X, (splat Y))
8130 if (match(V: Op.get(),
8131 P: m_Shuffle(v1: m_InsertElt(Val: m_Value(), Elt: m_Not(V: m_Value()), Idx: m_ZeroInt()),
8132 v2: m_Value(), mask: m_ZeroMask()))) {
8133 Use &InsertElt = cast<Instruction>(Val&: Op)->getOperandUse(i: 0);
8134 Use &Not = cast<Instruction>(Val&: InsertElt)->getOperandUse(i: 1);
8135 Ops.push_back(Elt: &Not);
8136 Ops.push_back(Elt: &InsertElt);
8137 Ops.push_back(Elt: &Op);
8138 return true;
8139 }
8140 }
8141 break;
8142 default:
8143 break;
8144 }
8145
8146 if (!I->getType()->isVectorTy())
8147 return !Ops.empty();
8148
8149 switch (I->getOpcode()) {
8150 case Instruction::Sub:
8151 case Instruction::Add: {
8152 if (!areExtractExts(Ext1: I->getOperand(i: 0), Ext2: I->getOperand(i: 1)))
8153 return false;
8154
8155 // If the exts' operands extract either the lower or upper elements, we
8156 // can sink them too.
8157 auto Ext1 = cast<Instruction>(Val: I->getOperand(i: 0));
8158 auto Ext2 = cast<Instruction>(Val: I->getOperand(i: 1));
8159 if (areExtractShuffleVectors(Op1: Ext1->getOperand(i: 0), Op2: Ext2->getOperand(i: 0))) {
8160 Ops.push_back(Elt: &Ext1->getOperandUse(i: 0));
8161 Ops.push_back(Elt: &Ext2->getOperandUse(i: 0));
8162 }
8163
8164 Ops.push_back(Elt: &I->getOperandUse(i: 0));
8165 Ops.push_back(Elt: &I->getOperandUse(i: 1));
8166
8167 return true;
8168 }
8169 case Instruction::Or: {
8170 // Pattern: Or(And(MaskValue, A), And(Not(MaskValue), B)) ->
8171 // bitselect(MaskValue, A, B) where Not(MaskValue) = Xor(MaskValue, -1)
8172 if (ST->hasNEON()) {
8173 // MainAnd refers to And instruction that has 'Not' as one of its operands
8174 Instruction *MainAnd, *OtherAnd;
8175 Value *A, *B;
8176 if (matchBSL(I, MainAnd, OtherAnd, A, B)) {
8177 Ops.push_back(
8178 Elt: &MainAnd->getOperandUse(i: MainAnd->getOperand(i: 0) == A ? 1 : 0));
8179 Ops.push_back(Elt: &I->getOperandUse(i: 0));
8180 Ops.push_back(Elt: &I->getOperandUse(i: 1));
8181
8182 return true;
8183 }
8184 }
8185
8186 return false;
8187 }
8188 case Instruction::Mul: {
8189 auto ShouldSinkSplatForIndexedVariant = [](Value *V) {
8190 auto *Ty = cast<VectorType>(Val: V->getType());
8191 // For SVE the lane-indexing is within 128-bits, so we can't fold splats.
8192 if (Ty->isScalableTy())
8193 return false;
8194
8195 // Indexed variants of Mul exist for i16 and i32 element types only.
8196 return Ty->getScalarSizeInBits() == 16 || Ty->getScalarSizeInBits() == 32;
8197 };
8198
8199 int NumZExts = 0, NumSExts = 0;
8200 for (auto &Op : I->operands()) {
8201 // Make sure we are not already sinking this operand
8202 if (any_of(Range&: Ops, P: [&](Use *U) { return U->get() == Op; }))
8203 continue;
8204
8205 if (match(V: &Op, P: m_ZExtOrSExt(Op: m_Value()))) {
8206 auto *Ext = cast<Instruction>(Val&: Op);
8207 auto *ExtOp = Ext->getOperand(i: 0);
8208 if (isSplatShuffle(V: ExtOp) && ShouldSinkSplatForIndexedVariant(ExtOp))
8209 Ops.push_back(Elt: &Ext->getOperandUse(i: 0));
8210 Ops.push_back(Elt: &Op);
8211
8212 if (isa<SExtInst>(Val: Ext)) {
8213 NumSExts++;
8214 } else {
8215 NumZExts++;
8216 // A zext(a) is also a sext(zext(a)), if we take more than 2 steps.
8217 if (Ext->getOperand(i: 0)->getType()->getScalarSizeInBits() * 2 <
8218 I->getType()->getScalarSizeInBits())
8219 NumSExts++;
8220 }
8221
8222 continue;
8223 }
8224
8225 ShuffleVectorInst *Shuffle = dyn_cast<ShuffleVectorInst>(Val&: Op);
8226 if (!Shuffle)
8227 continue;
8228
8229 // If the Shuffle is a splat and the operand is a zext/sext, sinking the
8230 // operand and the s/zext can help create indexed s/umull. This is
8231 // especially useful to prevent i64 mul being scalarized.
8232 if (isSplatShuffle(V: Shuffle) &&
8233 match(V: Shuffle->getOperand(i_nocapture: 0), P: m_ZExtOrSExt(Op: m_Value()))) {
8234 Ops.push_back(Elt: &Shuffle->getOperandUse(i: 0));
8235 Ops.push_back(Elt: &Op);
8236 if (match(V: Shuffle->getOperand(i_nocapture: 0), P: m_SExt(Op: m_Value())))
8237 NumSExts++;
8238 else
8239 NumZExts++;
8240 continue;
8241 }
8242
8243 Value *ShuffleOperand = Shuffle->getOperand(i_nocapture: 0);
8244 InsertElementInst *Insert = dyn_cast<InsertElementInst>(Val: ShuffleOperand);
8245 if (!Insert)
8246 continue;
8247
8248 Instruction *OperandInstr = dyn_cast<Instruction>(Val: Insert->getOperand(i_nocapture: 1));
8249 if (!OperandInstr)
8250 continue;
8251
8252 ConstantInt *ElementConstant =
8253 dyn_cast<ConstantInt>(Val: Insert->getOperand(i_nocapture: 2));
8254 // Check that the insertelement is inserting into element 0
8255 if (!ElementConstant || !ElementConstant->isZero())
8256 continue;
8257
8258 unsigned Opcode = OperandInstr->getOpcode();
8259 if (Opcode == Instruction::SExt)
8260 NumSExts++;
8261 else if (Opcode == Instruction::ZExt)
8262 NumZExts++;
8263 else {
8264 // If we find that the top bits are known 0, then we can sink and allow
8265 // the backend to generate a umull.
8266 unsigned Bitwidth = I->getType()->getScalarSizeInBits();
8267 APInt UpperMask = APInt::getHighBitsSet(numBits: Bitwidth, hiBitsSet: Bitwidth / 2);
8268 if (!MaskedValueIsZero(V: OperandInstr, Mask: UpperMask, SQ: DL))
8269 continue;
8270 NumZExts++;
8271 }
8272
8273 // And(Load) is excluded to prevent CGP getting stuck in a loop of sinking
8274 // the And, just to hoist it again back to the load.
8275 if (!match(V: OperandInstr, P: m_And(L: m_Load(Op: m_Value()), R: m_Value())))
8276 Ops.push_back(Elt: &Insert->getOperandUse(i: 1));
8277 Ops.push_back(Elt: &Shuffle->getOperandUse(i: 0));
8278 Ops.push_back(Elt: &Op);
8279 }
8280
8281 // It is profitable to sink if we found two of the same type of extends.
8282 if (!Ops.empty() && (NumSExts == 2 || NumZExts == 2))
8283 return true;
8284
8285 // Otherwise, see if we should sink splats for indexed variants.
8286 if (!ShouldSinkSplatForIndexedVariant(I))
8287 return false;
8288
8289 Ops.clear();
8290 if (isSplatShuffle(V: I->getOperand(i: 0)))
8291 Ops.push_back(Elt: &I->getOperandUse(i: 0));
8292 if (isSplatShuffle(V: I->getOperand(i: 1)))
8293 Ops.push_back(Elt: &I->getOperandUse(i: 1));
8294
8295 return !Ops.empty();
8296 }
8297 case Instruction::FMul: {
8298 // For SVE the lane-indexing is within 128-bits, so we can't fold splats.
8299 if (I->getType()->isScalableTy())
8300 return !Ops.empty();
8301
8302 if (cast<VectorType>(Val: I->getType())->getElementType()->isHalfTy() &&
8303 !ST->hasFullFP16())
8304 return !Ops.empty();
8305
8306 // Sink splats for index lane variants
8307 if (isSplatShuffle(V: I->getOperand(i: 0)))
8308 Ops.push_back(Elt: &I->getOperandUse(i: 0));
8309 if (isSplatShuffle(V: I->getOperand(i: 1)))
8310 Ops.push_back(Elt: &I->getOperandUse(i: 1));
8311 return !Ops.empty();
8312 }
8313 default:
8314 return false;
8315 }
8316 return false;
8317}
8318
8319bool AArch64TTIImpl::isLegalMaskedCompressStore(Type *DataType,
8320 Align Alignment) const {
8321 if (!(ST->isSVEAvailable() ||
8322 (ST->isSVEorStreamingSVEAvailable() && ST->hasSME2p2())))
8323 return false;
8324
8325 if (isa<FixedVectorType>(Val: DataType) &&
8326 DataType->getPrimitiveSizeInBits().getFixedValue() < 128)
8327 return false;
8328
8329 if (!isa<VectorType>(Val: DataType))
8330 return isElementTypeLegalForScalableVector(Ty: DataType);
8331
8332 auto LT = getTypeLegalizationCost(Ty: DataType);
8333 if (!LT.first.isValid())
8334 return false;
8335
8336 // Use the i32 or i64 compact instructions for f16/bf16 unpacked types.
8337 LLVMContext &Ctx = DataType->getContext();
8338 switch (LT.second.SimpleTy) {
8339 case MVT::nxv2f16:
8340 case MVT::nxv2bf16:
8341 return isElementTypeLegalForCompressStore(Ty: Type::getInt64Ty(C&: Ctx));
8342 case MVT::nxv4f16:
8343 case MVT::nxv4bf16:
8344 return isElementTypeLegalForCompressStore(Ty: Type::getInt32Ty(C&: Ctx));
8345 default:
8346 break;
8347 }
8348
8349 return isElementTypeLegalForCompressStore(
8350 Ty: EVT(LT.second.getScalarType()).getTypeForEVT(Context&: Ctx));
8351}
8352