1//==-- AArch64ISelLowering.h - AArch64 DAG Lowering Interface ----*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that AArch64 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H
15#define LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H
16
17#include "llvm/CodeGen/CallingConvLower.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/SelectionDAG.h"
20#include "llvm/CodeGen/TargetLowering.h"
21#include "llvm/IR/CallingConv.h"
22#include "llvm/IR/Instruction.h"
23
24namespace llvm {
25
26class AArch64TargetMachine;
27
28namespace AArch64 {
29/// Possible values of current rounding mode, which is specified in bits
30/// 23:22 of FPCR.
31enum Rounding {
32 RN = 0, // Round to Nearest
33 RP = 1, // Round towards Plus infinity
34 RM = 2, // Round towards Minus infinity
35 RZ = 3, // Round towards Zero
36 rmMask = 3 // Bit mask selecting rounding mode
37};
38
39// Bit position of rounding mode bits in FPCR.
40const unsigned RoundingBitsPos = 22;
41
42// Reserved bits should be preserved when modifying FPCR.
43const uint64_t ReservedFPControlBits = 0xfffffffff80040f8;
44
45// Registers used to pass function arguments.
46ArrayRef<MCPhysReg> getGPRArgRegs();
47ArrayRef<MCPhysReg> getFPRArgRegs();
48
49/// Maximum allowed number of unprobed bytes above SP at an ABI
50/// boundary.
51const unsigned StackProbeMaxUnprobedStack = 1024;
52
53/// Maximum number of iterations to unroll for a constant size probing loop.
54const unsigned StackProbeMaxLoopUnroll = 4;
55
56} // namespace AArch64
57
58namespace ARM64AS {
59enum : unsigned { PTR32_SPTR = 270, PTR32_UPTR = 271, PTR64 = 272 };
60}
61
62class AArch64Subtarget;
63
64class AArch64TargetLowering : public TargetLowering {
65public:
66 explicit AArch64TargetLowering(const TargetMachine &TM,
67 const AArch64Subtarget &STI);
68
69 const AArch64TargetMachine &getTM() const;
70
71 /// Control the following reassociation of operands: (op (op x, c1), y) -> (op
72 /// (op x, y), c1) where N0 is (op x, c1) and N1 is y.
73 bool isReassocProfitable(SelectionDAG &DAG, SDValue N0,
74 SDValue N1) const override;
75
76 /// Selects the correct CCAssignFn for a given CallingConvention value.
77 CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const;
78
79 /// Selects the correct CCAssignFn for a given CallingConvention value.
80 CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC) const;
81
82 /// Determine which of the bits specified in Mask are known to be either zero
83 /// or one and return them in the KnownZero/KnownOne bitsets.
84 void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,
85 const APInt &DemandedElts,
86 const SelectionDAG &DAG,
87 unsigned Depth = 0) const override;
88
89 unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
90 const APInt &DemandedElts,
91 const SelectionDAG &DAG,
92 unsigned Depth) const override;
93
94 unsigned computeNumSignBitsForTargetInstr(GISelValueTracking &Analysis,
95 Register R,
96 const APInt &DemandedElts,
97 const MachineRegisterInfo &MRI,
98 unsigned Depth = 0) const override;
99
100 MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override {
101 if ((AS == ARM64AS::PTR32_SPTR) || (AS == ARM64AS::PTR32_UPTR)) {
102 // These are 32-bit pointers created using the `__ptr32` extension or
103 // similar. They are handled by marking them as being in a different
104 // address space, and will be extended to 64-bits when used as the target
105 // of a load or store operation, or cast to a 64-bit pointer type.
106 return MVT::i32;
107 } else {
108 // Returning i64 unconditionally here (i.e. even for ILP32) means that the
109 // *DAG* representation of pointers will always be 64-bits. They will be
110 // truncated and extended when transferred to memory, but the 64-bit DAG
111 // allows us to use AArch64's addressing modes much more easily.
112 return MVT::i64;
113 }
114 }
115
116 unsigned getVectorIdxWidth(const DataLayout &DL) const override {
117 // The VectorIdx type is i64, with both normal and ilp32.
118 return 64;
119 }
120
121 bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
122 const APInt &DemandedElts,
123 TargetLoweringOpt &TLO) const override;
124
125 MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override;
126
127 /// Returns true if the target allows unaligned memory accesses of the
128 /// specified type.
129 bool allowsMisalignedMemoryAccesses(
130 EVT VT, unsigned AddrSpace = 0, Align Alignment = Align(1),
131 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
132 unsigned *Fast = nullptr) const override;
133 /// LLT variant.
134 bool allowsMisalignedMemoryAccesses(LLT Ty, unsigned AddrSpace,
135 Align Alignment,
136 MachineMemOperand::Flags Flags,
137 unsigned *Fast = nullptr) const override;
138
139 /// Provide custom lowering hooks for some operations.
140 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
141
142 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
143
144 /// This method returns a target specific FastISel object, or null if the
145 /// target does not support "fast" ISel.
146 FastISel *
147 createFastISel(FunctionLoweringInfo &funcInfo,
148 const TargetLibraryInfo *libInfo,
149 const LibcallLoweringInfo *libcallLowering) const override;
150
151 bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
152
153 bool isFPImmLegalAsFMov(const APFloat &Imm, EVT VT) const;
154
155 FPImmAction getFPImmAction(const APFloat &Imm, EVT VT,
156 bool ForCodeSize) const override;
157
158 /// Return true if the given shuffle mask can be codegen'd directly, or if it
159 /// should be stack expanded.
160 bool isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
161
162 /// Similar to isShuffleMaskLegal. Return true is the given 'select with zero'
163 /// shuffle mask can be codegen'd directly.
164 bool isVectorClearMaskLegal(ArrayRef<int> M, EVT VT) const override;
165
166 /// Return the ISD::SETCC ValueType.
167 EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
168 EVT VT) const override;
169
170 SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
171
172 MachineBasicBlock *EmitF128CSEL(MachineInstr &MI,
173 MachineBasicBlock *BB) const;
174
175 MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,
176 MachineBasicBlock *BB) const;
177
178 MachineBasicBlock *EmitLoweredSetFpmr(MachineInstr &MI,
179 MachineBasicBlock *MBB) const;
180
181 MachineBasicBlock *EmitDynamicProbedAlloc(MachineInstr &MI,
182 MachineBasicBlock *MBB) const;
183
184 MachineBasicBlock *EmitCheckMatchingVL(MachineInstr &MI,
185 MachineBasicBlock *MBB) const;
186
187 MachineBasicBlock *EmitTileLoad(unsigned Opc, unsigned BaseReg,
188 MachineInstr &MI,
189 MachineBasicBlock *BB) const;
190 MachineBasicBlock *EmitFill(MachineInstr &MI, MachineBasicBlock *BB) const;
191 MachineBasicBlock *EmitZAInstr(unsigned Opc, unsigned BaseReg,
192 MachineInstr &MI, MachineBasicBlock *BB) const;
193 MachineBasicBlock *EmitZTInstr(MachineInstr &MI, MachineBasicBlock *BB,
194 unsigned Opcode, bool Op0IsDef) const;
195 MachineBasicBlock *EmitZero(MachineInstr &MI, MachineBasicBlock *BB) const;
196 MachineBasicBlock *EmitEntryPStateSM(MachineInstr &MI,
197 MachineBasicBlock *BB) const;
198
199 /// Replace (0, vreg) discriminator components with the operands of blend
200 /// or with (immediate, NoRegister) when possible.
201 void fixupPtrauthDiscriminator(MachineInstr &MI, MachineBasicBlock *BB,
202 MachineOperand &IntDiscOp,
203 MachineOperand &AddrDiscOp,
204 const TargetRegisterClass *AddrDiscRC) const;
205
206 MachineBasicBlock *
207 EmitInstrWithCustomInserter(MachineInstr &MI,
208 MachineBasicBlock *MBB) const override;
209
210 void getTgtMemIntrinsic(SmallVectorImpl<IntrinsicInfo> &Infos,
211 const CallBase &I, MachineFunction &MF,
212 unsigned Intrinsic) const override;
213
214 bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT,
215 std::optional<unsigned> ByteOffset) const override;
216
217 bool shouldRemoveRedundantExtend(SDValue Op) const override;
218
219 bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
220 bool isTruncateFree(EVT VT1, EVT VT2) const override;
221
222 bool isProfitableToHoist(Instruction *I) const override;
223
224 bool isZExtFree(Type *Ty1, Type *Ty2) const override;
225 bool isZExtFree(EVT VT1, EVT VT2) const override;
226 bool isZExtFree(SDValue Val, EVT VT2) const override;
227
228 bool optimizeExtendOrTruncateConversion(
229 Instruction *I, Loop *L, const TargetTransformInfo &TTI) const override;
230
231 bool optimizeVectorCrossLaneOperation(Instruction *I, Loop *L) const override;
232
233 bool hasPairedLoad(EVT LoadedType, Align &RequiredAlignment) const override;
234
235 unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
236
237 bool lowerInterleavedLoad(Instruction *Load, Value *Mask,
238 ArrayRef<ShuffleVectorInst *> Shuffles,
239 ArrayRef<unsigned> Indices, unsigned Factor,
240 const APInt &GapMask) const override;
241 bool lowerInterleavedStore(Instruction *Store, Value *Mask,
242 ShuffleVectorInst *SVI, unsigned Factor,
243 const APInt &GapMask) const override;
244
245 bool isLegalAddImmediate(int64_t) const override;
246 bool isLegalAddScalableImmediate(int64_t) const override;
247 bool isLegalICmpImmediate(int64_t) const override;
248
249 bool isMulAddWithConstProfitable(SDValue AddNode,
250 SDValue ConstNode) const override;
251
252 bool shouldConsiderGEPOffsetSplit() const override;
253
254 EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
255 const AttributeList &FuncAttributes) const override;
256
257 LLT getOptimalMemOpLLT(const MemOp &Op,
258 const AttributeList &FuncAttributes) const override;
259
260 bool findOptimalMemOpLowering(LLVMContext &Context, std::vector<EVT> &MemOps,
261 unsigned Limit, const MemOp &Op, unsigned DstAS,
262 unsigned SrcAS,
263 const AttributeList &FuncAttributes,
264 EVT *LargestVT = nullptr) const override;
265
266 /// Return true if the addressing mode represented by AM is legal for this
267 /// target, for a load/store of the specified type.
268 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
269 unsigned AS,
270 Instruction *I = nullptr) const override;
271
272 int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset,
273 int64_t MaxOffset) const override;
274
275 /// Return true if an FMA operation is faster than a pair of fmul and fadd
276 /// instructions. fmuladd intrinsics will be expanded to FMAs when this method
277 /// returns true, otherwise fmuladd is expanded to fmul + fadd.
278 bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
279 EVT VT) const override;
280 bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override;
281
282 bool generateFMAsInMachineCombiner(EVT VT,
283 CodeGenOptLevel OptLevel) const override;
284
285 /// Return true if the target has native support for
286 /// the specified value type and it is 'desirable' to use the type for the
287 /// given node type.
288 bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;
289
290 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
291 ArrayRef<MCPhysReg> getRoundingControlRegisters() const override;
292
293 bool isNarrowingProfitable(SDNode *N, EVT SrcVT, EVT DestVT) const override;
294
295 /// Returns false if N is a bit extraction pattern of (X >> C) & Mask.
296 bool isDesirableToCommuteWithShift(const SDNode *N,
297 CombineLevel Level) const override;
298
299 bool isDesirableToPullExtFromShl(const MachineInstr &MI) const override {
300 return false;
301 }
302
303 /// Returns false if N is a bit extraction pattern of (X >> C) & Mask.
304 bool isDesirableToCommuteXorWithShift(const SDNode *N) const override;
305
306 /// Return true if it is profitable to fold a pair of shifts into a mask.
307 bool shouldFoldConstantShiftPairToMask(const SDNode *N) const override;
308
309 /// Return true if it is profitable to fold a pair of shifts into a mask.
310 bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override {
311 EVT VT = Y.getValueType();
312
313 if (VT.isVector())
314 return false;
315
316 return VT.getScalarSizeInBits() <= 64;
317 }
318
319 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT,
320 unsigned SelectOpcode, SDValue X,
321 SDValue Y) const override;
322
323 /// Returns true if it is beneficial to convert a load of a constant
324 /// to just the constant itself.
325 bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
326 Type *Ty) const override;
327
328 /// Return the cost of EXTRACT_SUBVECTOR for this result type with this
329 /// index.
330 ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT,
331 unsigned Index) const override;
332
333 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
334 bool MathUsed) const override {
335 // Using overflow ops for overflow checks only should beneficial on
336 // AArch64.
337 return TargetLowering::shouldFormOverflowOp(Opcode, VT, MathUsed: true);
338 }
339
340 // Return true if the target wants to optimize the mul overflow intrinsic
341 // for the given \p VT.
342 bool shouldOptimizeMulOverflowWithZeroHighBits(LLVMContext &Context,
343 EVT VT) const override;
344
345 Instruction *emitLeadingFence(IRBuilderBase &Builder, Instruction *Inst,
346 AtomicOrdering Ord) const override;
347 Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,
348 AtomicOrdering Ord) const override;
349 Value *emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr,
350 AtomicOrdering Ord) const override;
351 Value *emitCanLoadSpeculatively(IRBuilderBase &Builder, Value *Ptr,
352 Value *SizeInBytes) const override;
353
354 void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override;
355
356 bool isOpSuitableForLDPSTP(const Instruction *I) const;
357 bool isOpSuitableForLSE128(const Instruction *I) const;
358 bool isOpSuitableForRCPC3(const Instruction *I) const;
359 bool isOpSuitableForLSCP(const Instruction *I) const;
360 bool shouldInsertFencesForAtomic(const Instruction *I) const override;
361 bool shouldInsertTrailingSeqCstFenceForAtomicStore(
362 const Instruction *I) const override;
363
364 TargetLoweringBase::AtomicExpansionKind
365 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
366 TargetLoweringBase::AtomicExpansionKind
367 shouldCastAtomicLoadInIR(LoadInst *LI) const override;
368
369 TargetLoweringBase::AtomicExpansionKind
370 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
371 TargetLoweringBase::AtomicExpansionKind
372 shouldCastAtomicStoreInIR(StoreInst *SI) const override;
373 TargetLoweringBase::AtomicExpansionKind
374 shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const override;
375
376 TargetLoweringBase::AtomicExpansionKind
377 shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *AI) const override;
378
379 bool shouldIssueAtomicLoadForAtomicEmulationLoop() const override {
380 return false;
381 }
382
383 bool useLoadStackGuardNode(const Module &M) const override;
384 bool useStackGuardMixFP() const override;
385 SDValue emitStackGuardMixFP(SelectionDAG &DAG, SDValue Val,
386 const SDLoc &DL) const override;
387 TargetLoweringBase::LegalizeTypeAction
388 getPreferredVectorAction(MVT VT) const override;
389
390 /// If the target has a standard location for the stack protector cookie,
391 /// returns the address of that location. Otherwise, returns nullptr.
392 Value *getIRStackGuard(IRBuilderBase &IRB,
393 const LibcallLoweringInfo &Libcalls) const override;
394
395 void
396 insertSSPDeclarations(Module &M,
397 const LibcallLoweringInfo &Libcalls) const override;
398
399 /// If the target has a standard location for the unsafe stack pointer,
400 /// returns the address of that location. Otherwise, returns nullptr.
401 Value *getSafeStackPointerLocation(
402 IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const override;
403
404 /// If a physical register, this returns the register that receives the
405 /// exception address on entry to an EH pad.
406 Register
407 getExceptionPointerRegister(ExceptionHandling EH,
408 const Constant *PersonalityFn) const override;
409
410 /// If a physical register, this returns the register that receives the
411 /// exception typeid on entry to a landing pad.
412 Register
413 getExceptionSelectorRegister(ExceptionHandling EH,
414 const Constant *PersonalityFn) const override;
415
416 bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
417
418 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
419 const MachineFunction &MF) const override;
420
421 bool isCheapToSpeculateCttz(Type *) const override {
422 return true;
423 }
424
425 bool isCheapToSpeculateCtlz(Type *) const override {
426 return true;
427 }
428
429 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
430
431 bool hasAndNotCompare(SDValue V) const override {
432 // We can use bics for any scalar.
433 return V.getValueType().isScalarInteger();
434 }
435
436 bool hasAndNot(SDValue Y) const override {
437 EVT VT = Y.getValueType();
438
439 if (!VT.isVector())
440 return hasAndNotCompare(V: Y);
441
442 if (VT.isScalableVector())
443 return true;
444
445 return VT.getFixedSizeInBits() >= 64; // vector 'bic'
446 }
447
448 bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
449 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
450 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
451 SelectionDAG &DAG) const override;
452
453 ShiftLegalizationStrategy
454 preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N,
455 unsigned ExpansionFactor) const override;
456
457 CondMergingParams
458 getJumpConditionMergingParams(Instruction::BinaryOps Opc, const Value *Lhs,
459 const Value *Rhs,
460 const Function *F) const override;
461
462 bool shouldTransformSignedTruncationCheck(EVT XVT,
463 unsigned KeptBits) const override {
464 // For vectors, we don't have a preference..
465 if (XVT.isVector())
466 return false;
467
468 auto VTIsOk = [](EVT VT) -> bool {
469 return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
470 VT == MVT::i64;
471 };
472
473 // We are ok with KeptBitsVT being byte/word/dword, what SXT supports.
474 // XVT will be larger than KeptBitsVT.
475 MVT KeptBitsVT = MVT::getIntegerVT(BitWidth: KeptBits);
476 return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
477 }
478
479 bool preferIncOfAddToSubOfNot(EVT VT) const override;
480
481 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override;
482
483 bool preferSelectsOverBooleanArithmetic(EVT VT) const override;
484
485 bool isComplexDeinterleavingSupported() const override;
486 bool isComplexDeinterleavingOperationSupported(
487 ComplexDeinterleavingOperation Operation, Type *Ty) const override;
488
489 Value *createComplexDeinterleavingIR(
490 IRBuilderBase &B, ComplexDeinterleavingOperation OperationType,
491 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
492 Value *Accumulator = nullptr) const override;
493
494 bool supportSplitCSR(MachineFunction *MF) const override {
495 return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
496 MF->getFunction().hasFnAttribute(Kind: Attribute::NoUnwind);
497 }
498 void initializeSplitCSR(MachineBasicBlock *Entry) const override;
499 void insertCopiesSplitCSR(
500 MachineBasicBlock *Entry,
501 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
502
503 bool supportSwiftError() const override {
504 return true;
505 }
506
507 bool supportPtrAuthBundles() const override { return true; }
508
509 bool supportKCFIBundles() const override { return true; }
510
511 MachineInstr *EmitKCFICheck(MachineBasicBlock &MBB,
512 MachineBasicBlock::instr_iterator &MBBI,
513 const TargetInstrInfo *TII) const override;
514
515 bool shallExtractConstSplatVectorElementToStore(
516 Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const override;
517
518 /// Enable aggressive FMA fusion on targets that want it.
519 bool enableAggressiveFMAFusion(EVT VT) const override;
520
521 bool aggressivelyPreferBuildVectorSources(EVT VecVT) const override {
522 return true;
523 }
524
525 /// Returns the size of the platform's va_list object.
526 unsigned getVaListSizeInBits(const DataLayout &DL) const override;
527
528 /// Returns true if \p VecTy is a legal interleaved access type. This
529 /// function checks the vector element type and the overall width of the
530 /// vector.
531 bool isLegalInterleavedAccessType(VectorType *VecTy, const DataLayout &DL,
532 bool &UseScalable) const;
533
534 /// Returns the number of interleaved accesses that will be generated when
535 /// lowering accesses of the given type.
536 unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL,
537 bool UseScalable) const;
538
539 MachineMemOperand::Flags getTargetMMOFlags(
540 const Instruction &I) const override;
541
542 bool functionArgumentNeedsConsecutiveRegisters(
543 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
544 const DataLayout &DL) const override;
545
546 /// Used for exception handling on Win64.
547 bool needsFixedCatchObjects() const override;
548
549 bool fallBackToDAGISel(const Instruction &Inst) const override;
550
551 /// SVE code generation for fixed length vectors does not custom lower
552 /// BUILD_VECTOR. This makes BUILD_VECTOR legalisation a source of stores to
553 /// merge. However, merging them creates a BUILD_VECTOR that is just as
554 /// illegal as the original, thus leading to an infinite legalisation loop.
555 /// NOTE: Once BUILD_VECTOR is legal or can be custom lowered for all legal
556 /// vector types this override can be removed.
557 bool mergeStoresAfterLegalization(EVT VT) const override;
558
559 // If the platform/function should have a redzone, return the size in bytes.
560 unsigned getRedZoneSize(const Function &F) const {
561 if (F.hasFnAttribute(Kind: Attribute::NoRedZone))
562 return 0;
563 return 128;
564 }
565
566 bool isAllActivePredicate(const SelectionDAG &DAG, SDValue N) const;
567 EVT getPromotedVTForPredicate(EVT VT) const;
568
569 EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty,
570 bool AllowUnknown = false) const override;
571
572 bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;
573
574 /// If a change in streaming mode is required on entry to/return from a
575 /// function call it emits and returns the corresponding SMSTART or SMSTOP
576 /// node. \p Condition should be one of the enum values from
577 /// AArch64SME::ToggleCondition.
578 SDValue changeStreamingMode(SelectionDAG &DAG, SDLoc DL, bool Enable,
579 SDValue Chain, SDValue InGlue, unsigned Condition,
580 bool InsertVectorLengthCheck = false) const;
581
582 /// Returns true if \p RdxOp should be lowered to a SVE reduction. If a SVE2
583 /// pairwise operation can be used for the reduction \p PairwiseOpIID is set
584 /// to its intrinsic ID.
585 bool
586 shouldLowerReductionToSVE(SDValue RdxOp,
587 std::optional<Intrinsic::ID> &PairwiseOpIID) const;
588
589 // Normally SVE is only used for byte size vectors that do not fit within a
590 // NEON vector. This changes when OverrideNEON is true, allowing SVE to be
591 // used for 64bit and 128bit vectors as well.
592 bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false) const;
593
594 // Follow NEON ABI rules even when using SVE for fixed length vectors.
595 MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
596 EVT VT) const override;
597 unsigned getNumRegistersForCallingConv(LLVMContext &Context,
598 CallingConv::ID CC,
599 EVT VT) const override;
600 unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context,
601 CallingConv::ID CC, EVT VT,
602 EVT &IntermediateVT,
603 unsigned &NumIntermediates,
604 MVT &RegisterVT) const override;
605
606 bool preferVectorizedNonPowerOfTwoTypeBreakdown() const override {
607 return true;
608 }
609
610 /// True if stack clash protection is enabled for this functions.
611 bool hasInlineStackProbe(const MachineFunction &MF) const override;
612
613 /// In AArch64, true if FEAT_CPA is present. Allows pointer arithmetic
614 /// semantics to be preserved for instruction selection.
615 bool shouldPreservePtrArith(const Function &F, EVT PtrVT) const override;
616
617 // Match a register name (e.g. "x5", "d5", "sp") to its register number, with
618 // no validity filtering. This is the single entry point for the generated
619 // register-name matcher, shared with getRegisterByName.
620 Register matchRegisterName(StringRef RegName) const;
621
622private:
623 /// Keep a pointer to the AArch64Subtarget around so that we can
624 /// make the right decision when generating code for different targets.
625 const AArch64Subtarget *Subtarget;
626
627 bool isExtFreeImpl(const Instruction *Ext) const override;
628
629 void addTypeForNEON(MVT VT);
630 void addTypeForFixedLengthSVE(MVT VT);
631 void addDRType(MVT VT);
632 void addQRType(MVT VT);
633
634 bool shouldExpandBuildVectorWithShuffles(EVT, unsigned) const override;
635
636 SDValue lowerEHPadEntry(SDValue Chain, SDLoc const &DL,
637 SelectionDAG &DAG) const override;
638
639 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
640 bool isVarArg,
641 const SmallVectorImpl<ISD::InputArg> &Ins,
642 const SDLoc &DL, SelectionDAG &DAG,
643 SmallVectorImpl<SDValue> &InVals) const override;
644
645 void AdjustInstrPostInstrSelection(MachineInstr &MI,
646 SDNode *Node) const override;
647
648 SDValue LowerCall(CallLoweringInfo & /*CLI*/,
649 SmallVectorImpl<SDValue> &InVals) const override;
650
651 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
652 CallingConv::ID CallConv, bool isVarArg,
653 const SmallVectorImpl<CCValAssign> &RVLocs,
654 const SDLoc &DL, SelectionDAG &DAG,
655 SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
656 SDValue ThisVal, bool RequiresSMChange) const;
657
658 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
659 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
660 SDValue LowerStore128(SDValue Op, SelectionDAG &DAG) const;
661 SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const;
662 SDValue LowerSMULFIXSAT(SDValue Op, SelectionDAG &DAG) const;
663 SDValue LowerFMUL(SDValue Op, SelectionDAG &DAG) const;
664 SDValue LowerFMA(SDValue Op, SelectionDAG &DAG) const;
665 SDValue LowerCLMUL(SDValue Op, SelectionDAG &DAG) const;
666
667 SDValue LowerMGATHER(SDValue Op, SelectionDAG &DAG) const;
668 SDValue LowerMSCATTER(SDValue Op, SelectionDAG &DAG) const;
669
670 SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) const;
671
672 SDValue LowerVECTOR_COMPRESS(SDValue Op, SelectionDAG &DAG) const;
673
674 SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
675 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
676 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
677
678 bool
679 isEligibleForTailCallOptimization(const CallLoweringInfo &CLI) const;
680
681 /// Finds the incoming stack arguments which overlap the given fixed stack
682 /// object and incorporates their load into the current chain. This prevents
683 /// an upcoming store from clobbering the stack argument before it's used.
684 SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG,
685 MachineFrameInfo &MFI, int ClobberedFI) const;
686
687 bool DoesCalleeRestoreStack(CallingConv::ID CallCC, bool TailCallOpt) const;
688
689 void saveVarArgRegisters(CCState &CCInfo, SelectionDAG &DAG, const SDLoc &DL,
690 SDValue &Chain) const;
691
692 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
693 bool isVarArg,
694 const SmallVectorImpl<ISD::OutputArg> &Outs,
695 LLVMContext &Context, const Type *RetTy) const override;
696
697 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
698 const SmallVectorImpl<ISD::OutputArg> &Outs,
699 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
700 SelectionDAG &DAG) const override;
701
702 SDValue getTargetNode(GlobalAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
703 unsigned Flag) const;
704 SDValue getTargetNode(JumpTableSDNode *N, EVT Ty, SelectionDAG &DAG,
705 unsigned Flag) const;
706 SDValue getTargetNode(ConstantPoolSDNode *N, EVT Ty, SelectionDAG &DAG,
707 unsigned Flag) const;
708 SDValue getTargetNode(BlockAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
709 unsigned Flag) const;
710 SDValue getTargetNode(ExternalSymbolSDNode *N, EVT Ty, SelectionDAG &DAG,
711 unsigned Flag) const;
712 template <class NodeTy>
713 SDValue getGOT(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
714 template <class NodeTy>
715 SDValue getAddrLarge(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
716 template <class NodeTy>
717 SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
718 template <class NodeTy>
719 SDValue getAddrTiny(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
720 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
721 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
722 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
723 SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
724 SDValue LowerELFGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
725 SDValue LowerELFTLSLocalExec(const GlobalValue *GV, SDValue ThreadBase,
726 const SDLoc &DL, SelectionDAG &DAG) const;
727 SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,
728 SelectionDAG &DAG) const;
729 SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
730 SDValue LowerPtrAuthGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
731 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
732 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
733 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
734 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
735 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
736 SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,
737 SDValue TVal, SDValue FVal,
738 iterator_range<SDNode::user_iterator> Users,
739 SDNodeFlags Flags, const SDLoc &dl,
740 SelectionDAG &DAG) const;
741 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
742 SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
743 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
744 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
745 SDValue LowerBRIND(SDValue Op, SelectionDAG &DAG) const;
746 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
747 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
748 SDValue LowerAAPCS_VASTART(SDValue Op, SelectionDAG &DAG) const;
749 SDValue LowerDarwin_VASTART(SDValue Op, SelectionDAG &DAG) const;
750 SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const;
751 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
752 SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
753 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
754 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
755 SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
756 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
757 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
758 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
759 SDValue LowerGET_FPMODE(SDValue Op, SelectionDAG &DAG) const;
760 SDValue LowerSET_FPMODE(SDValue Op, SelectionDAG &DAG) const;
761 SDValue LowerRESET_FPMODE(SDValue Op, SelectionDAG &DAG) const;
762 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
763 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
764 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
765 SDValue LowerEXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
766 SDValue LowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
767 SDValue LowerVECTOR_REPEAT(SDValue Op, SelectionDAG &DAG) const;
768 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
769 SDValue LowerVECTOR_REVERSE(SDValue Op, SelectionDAG &DAG) const;
770 SDValue LowerVECTOR_SHUFFLE_VAR(SDValue Op, SelectionDAG &DAG) const;
771 SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
772 SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;
773 SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG,
774 unsigned NewOp) const;
775 SDValue LowerToScalableOp(SDValue Op, SelectionDAG &DAG) const;
776 SDValue LowerVECTOR_SPLICE(SDValue Op, SelectionDAG &DAG) const;
777 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
778 SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
779 SDValue LowerVECTOR_DEINTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
780 SDValue LowerVECTOR_INTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
781 SDValue LowerVECTOR_HISTOGRAM(SDValue Op, SelectionDAG &DAG) const;
782 SDValue LowerPARTIAL_REDUCE_MLA(SDValue Op, SelectionDAG &DAG) const;
783 SDValue LowerGET_ACTIVE_LANE_MASK(SDValue Op, SelectionDAG &DAG) const;
784 SDValue LowerMASK_BEFOREFIRST(SDValue Op, SelectionDAG &DAG) const;
785 SDValue LowerDIV(SDValue Op, SelectionDAG &DAG) const;
786 SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
787 SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
788 SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) const;
789 SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;
790 SDValue LowerCTPOP_PARITY(SDValue Op, SelectionDAG &DAG) const;
791 SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) const;
792 SDValue LowerBitreverse(SDValue Op, SelectionDAG &DAG) const;
793 SDValue LowerMinMax(SDValue Op, SelectionDAG &DAG) const;
794 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
795 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
796 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
797 SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
798 SDValue LowerVectorFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
799 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
800 SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
801 SDValue LowerVectorXRINT(SDValue Op, SelectionDAG &DAG) const;
802 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
803 SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
804 SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
805 SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) const;
806 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
807 SDValue LowerLOOP_DEPENDENCE_MASK(SDValue Op, SelectionDAG &DAG) const;
808 SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
809 SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const;
810 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
811 SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
812 SDValue LowerVECREDUCE_MUL(SDValue Op, SelectionDAG &DAG) const;
813 SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
814 SDValue LowerInlineDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
815 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
816 SDValue LowerMSTORE(SDValue Op, SelectionDAG &DAG) const;
817 SDValue LowerFCANONICALIZE(SDValue Op, SelectionDAG &DAG) const;
818 SDValue LowerAVG(SDValue Op, SelectionDAG &DAG, unsigned NewOp) const;
819 SDValue LowerFLDEXP(SDValue Op, SelectionDAG &DAG) const;
820
821 SDValue LowerFPToIntToSVE(SDValue Op, SelectionDAG &DAG) const;
822
823 SDValue LowerFixedLengthVectorIntDivideToSVE(SDValue Op,
824 SelectionDAG &DAG) const;
825 SDValue LowerFixedLengthVectorIntExtendToSVE(SDValue Op,
826 SelectionDAG &DAG) const;
827 SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
828 SDValue LowerFixedLengthVectorMLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
829 SDValue LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp, SelectionDAG &DAG) const;
830 SDValue LowerPredReductionToSVE(SDValue ScalarOp, SelectionDAG &DAG) const;
831 SDValue LowerReductionToSVE(SDValue Op, SelectionDAG &DAG) const;
832 SDValue LowerFixedLengthVectorSelectToSVE(SDValue Op, SelectionDAG &DAG) const;
833 SDValue LowerFixedLengthVectorSetccToSVE(SDValue Op, SelectionDAG &DAG) const;
834 SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
835 SDValue LowerFixedLengthVectorMStoreToSVE(SDValue Op,
836 SelectionDAG &DAG) const;
837 SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
838 SelectionDAG &DAG) const;
839 SDValue LowerFixedLengthExtractVectorElt(SDValue Op, SelectionDAG &DAG) const;
840 SDValue LowerFixedLengthInsertVectorElt(SDValue Op, SelectionDAG &DAG) const;
841 SDValue LowerFixedLengthBitcastToSVE(SDValue Op, SelectionDAG &DAG) const;
842 SDValue LowerFixedLengthConcatVectorsToSVE(SDValue Op,
843 SelectionDAG &DAG) const;
844 SDValue LowerFixedLengthFPExtendToSVE(SDValue Op, SelectionDAG &DAG) const;
845 SDValue LowerFixedLengthFPRoundToSVE(SDValue Op, SelectionDAG &DAG) const;
846 SDValue LowerFixedLengthIntToFPToSVE(SDValue Op, SelectionDAG &DAG) const;
847 SDValue LowerFixedLengthFPToIntToSVE(SDValue Op, SelectionDAG &DAG) const;
848 SDValue LowerFixedLengthVECTOR_SHUFFLEToSVE(SDValue Op,
849 SelectionDAG &DAG) const;
850 SDValue LowerFixedLengthBuildVectorToSVE(SDValue Op, SelectionDAG &DAG) const;
851 SDValue LowerFixedLengthVectorCompressToSVE(SDValue Op,
852 SelectionDAG &DAG) const;
853
854 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
855 SmallVectorImpl<SDNode *> &Created) const override;
856 SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
857 SmallVectorImpl<SDNode *> &Created) const override;
858 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
859 int &ExtraSteps, bool &UseOneConst,
860 bool Reciprocal) const override;
861 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
862 int &ExtraSteps) const override;
863 SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG,
864 const DenormalMode &Mode,
865 SDNodeFlags Flags = {}) const override;
866 SDValue getSqrtResultForDenormInput(SDValue Operand,
867 SelectionDAG &DAG) const override;
868 unsigned combineRepeatedFPDivisors() const override;
869
870 ConstraintType getConstraintType(StringRef Constraint) const override;
871 Register getRegisterByName(const char* RegName, LLT VT,
872 const MachineFunction &MF) const override;
873
874private:
875 /// Examine constraint string and operand type and determine a weight value.
876 /// The operand object must already have been set up with the operand type.
877 ConstraintWeight
878 getSingleConstraintMatchWeight(AsmOperandInfo &info,
879 const char *constraint) const override;
880
881 std::pair<unsigned, const TargetRegisterClass *>
882 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
883 StringRef Constraint, MVT VT) const override;
884
885 const char *LowerXConstraint(EVT ConstraintVT) const override;
886
887 void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
888 std::vector<SDValue> &Ops,
889 SelectionDAG &DAG) const override;
890
891 InlineAsm::ConstraintCode
892 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
893 if (ConstraintCode == "Q")
894 return InlineAsm::ConstraintCode::Q;
895 // FIXME: clang has code for 'Ump', 'Utf', 'Usa', and 'Ush' but these are
896 // followed by llvm_unreachable so we'll leave them unimplemented in
897 // the backend for now.
898 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
899 }
900
901 /// Handle Lowering flag assembly outputs.
902 SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
903 const SDLoc &DL,
904 const AsmOperandInfo &Constraint,
905 SelectionDAG &DAG) const override;
906
907 bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const override;
908 bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override;
909 bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
910 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
911 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
912 bool getIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
913 SDValue &Offset, SelectionDAG &DAG,
914 ISD::MemIndexedMode AM) const;
915 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
916 ISD::MemIndexedMode &AM,
917 SelectionDAG &DAG) const override;
918 bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
919 SDValue &Offset, ISD::MemIndexedMode &AM,
920 SelectionDAG &DAG) const override;
921 bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset,
922 bool IsPre, MachineRegisterInfo &MRI) const override;
923
924 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
925 SelectionDAG &DAG) const override;
926 void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
927 SelectionDAG &DAG) const;
928 void ReplaceExtractSubVectorResults(SDNode *N,
929 SmallVectorImpl<SDValue> &Results,
930 SelectionDAG &DAG) const;
931 void ReplaceGetActiveLaneMaskResults(SDNode *N,
932 SmallVectorImpl<SDValue> &Results,
933 SelectionDAG &DAG) const;
934
935 bool shouldNormalizeToSelectSequence(LLVMContext &, EVT, EVT) const override;
936
937 void finalizeLowering(MachineFunction &MF) const override;
938
939 bool shouldLocalize(const MachineInstr &MI,
940 const TargetTransformInfo *TTI) const override;
941
942 bool SimplifyDemandedBitsForTargetNode(SDValue Op,
943 const APInt &OriginalDemandedBits,
944 const APInt &OriginalDemandedElts,
945 KnownBits &Known,
946 TargetLoweringOpt &TLO,
947 unsigned Depth) const override;
948
949 bool canCreateUndefOrPoisonForTargetNode(
950 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
951 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const override;
952
953 bool isTargetCanonicalConstantNode(SDValue Op) const override;
954
955 // With the exception of data-predicate transitions, no instructions are
956 // required to cast between legal scalable vector types. However:
957 // 1. Packed and unpacked types have different bit lengths, meaning BITCAST
958 // is not universally useable.
959 // 2. Most unpacked integer types are not legal and thus integer extends
960 // cannot be used to convert between unpacked and packed types.
961 // These can make "bitcasting" a multiphase process. REINTERPRET_CAST is used
962 // to transition between unpacked and packed types of the same element type,
963 // with BITCAST used otherwise.
964 // This function does not handle predicate bitcasts.
965 SDValue getSVESafeBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) const;
966
967 // Returns the runtime value for PSTATE.SM by generating a call to
968 // __arm_sme_state.
969 SDValue getRuntimePStateSM(SelectionDAG &DAG, SDValue Chain, SDLoc DL,
970 EVT VT) const;
971
972 bool preferScalarizeSplat(SDNode *N) const override;
973
974 unsigned getMinimumJumpTableEntries() const override;
975
976 bool shouldScalarizeBinop(SDValue VecOp) const override {
977 return VecOp.getOpcode() == ISD::SETCC;
978 }
979
980 bool hasMultipleConditionRegisters(EVT VT) const override {
981 return VT.isScalableVector();
982 }
983};
984
985namespace AArch64 {
986FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
987 const TargetLibraryInfo *libInfo,
988 const LibcallLoweringInfo *libcallLowering);
989
990// Determine the effective TLS model for an ELF global, applying
991// AArch64-specific restrictions and configuration.
992TLSModel::Model getELFTLSModel(const GlobalValue *GV,
993 const AArch64TargetMachine &TM,
994 bool HasELFSignedGOT);
995} // end namespace AArch64
996
997} // end namespace llvm
998
999#endif
1000