1//===- AArch64InstrInfo.h - AArch64 Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64INSTRINFO_H
14#define LLVM_LIB_TARGET_AARCH64_AARCH64INSTRINFO_H
15
16#include "AArch64.h"
17#include "AArch64RegisterInfo.h"
18#include "llvm/CodeGen/TargetInstrInfo.h"
19#include "llvm/Support/TypeSize.h"
20#include <optional>
21
22#define GET_INSTRINFO_HEADER
23#include "AArch64GenInstrInfo.inc"
24
25namespace llvm {
26
27class AArch64Subtarget;
28
29static const MachineMemOperand::Flags MOSuppressPair =
30 MachineMemOperand::MOTargetFlag1;
31static const MachineMemOperand::Flags MOStridedAccess =
32 MachineMemOperand::MOTargetFlag2;
33
34#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
35
36// AArch64 MachineCombiner patterns
37enum AArch64MachineCombinerPattern : unsigned {
38 // These are patterns used to reduce the length of dependence chain.
39 SUBADD_OP1 = MachineCombinerPattern::TARGET_PATTERN_START,
40 SUBADD_OP2,
41
42 // These are multiply-add patterns matched by the AArch64 machine combiner.
43 MULADDW_OP1,
44 MULADDW_OP2,
45 MULSUBW_OP1,
46 MULSUBW_OP2,
47 MULADDWI_OP1,
48 MULSUBWI_OP1,
49 MULADDX_OP1,
50 MULADDX_OP2,
51 MULSUBX_OP1,
52 MULSUBX_OP2,
53 MULADDXI_OP1,
54 MULSUBXI_OP1,
55 // NEON integers vectors
56 MULADDv8i8_OP1,
57 MULADDv8i8_OP2,
58 MULADDv16i8_OP1,
59 MULADDv16i8_OP2,
60 MULADDv4i16_OP1,
61 MULADDv4i16_OP2,
62 MULADDv8i16_OP1,
63 MULADDv8i16_OP2,
64 MULADDv2i32_OP1,
65 MULADDv2i32_OP2,
66 MULADDv4i32_OP1,
67 MULADDv4i32_OP2,
68
69 MULSUBv8i8_OP1,
70 MULSUBv8i8_OP2,
71 MULSUBv16i8_OP1,
72 MULSUBv16i8_OP2,
73 MULSUBv4i16_OP1,
74 MULSUBv4i16_OP2,
75 MULSUBv8i16_OP1,
76 MULSUBv8i16_OP2,
77 MULSUBv2i32_OP1,
78 MULSUBv2i32_OP2,
79 MULSUBv4i32_OP1,
80 MULSUBv4i32_OP2,
81
82 MULADDv4i16_indexed_OP1,
83 MULADDv4i16_indexed_OP2,
84 MULADDv8i16_indexed_OP1,
85 MULADDv8i16_indexed_OP2,
86 MULADDv2i32_indexed_OP1,
87 MULADDv2i32_indexed_OP2,
88 MULADDv4i32_indexed_OP1,
89 MULADDv4i32_indexed_OP2,
90
91 MULSUBv4i16_indexed_OP1,
92 MULSUBv4i16_indexed_OP2,
93 MULSUBv8i16_indexed_OP1,
94 MULSUBv8i16_indexed_OP2,
95 MULSUBv2i32_indexed_OP1,
96 MULSUBv2i32_indexed_OP2,
97 MULSUBv4i32_indexed_OP1,
98 MULSUBv4i32_indexed_OP2,
99
100 // Floating Point
101 FMULADDH_OP1,
102 FMULADDH_OP2,
103 FMULSUBH_OP1,
104 FMULSUBH_OP2,
105 FMULADDS_OP1,
106 FMULADDS_OP2,
107 FMULSUBS_OP1,
108 FMULSUBS_OP2,
109 FMULADDD_OP1,
110 FMULADDD_OP2,
111 FMULSUBD_OP1,
112 FMULSUBD_OP2,
113 FNMULSUBH_OP1,
114 FNMULSUBS_OP1,
115 FNMULSUBD_OP1,
116 FMLAv1i32_indexed_OP1,
117 FMLAv1i32_indexed_OP2,
118 FMLAv1i64_indexed_OP1,
119 FMLAv1i64_indexed_OP2,
120 FMLAv4f16_OP1,
121 FMLAv4f16_OP2,
122 FMLAv8f16_OP1,
123 FMLAv8f16_OP2,
124 FMLAv2f32_OP2,
125 FMLAv2f32_OP1,
126 FMLAv2f64_OP1,
127 FMLAv2f64_OP2,
128 FMLAv4i16_indexed_OP1,
129 FMLAv4i16_indexed_OP2,
130 FMLAv8i16_indexed_OP1,
131 FMLAv8i16_indexed_OP2,
132 FMLAv2i32_indexed_OP1,
133 FMLAv2i32_indexed_OP2,
134 FMLAv2i64_indexed_OP1,
135 FMLAv2i64_indexed_OP2,
136 FMLAv4f32_OP1,
137 FMLAv4f32_OP2,
138 FMLAv4i32_indexed_OP1,
139 FMLAv4i32_indexed_OP2,
140 FMLSv1i32_indexed_OP2,
141 FMLSv1i64_indexed_OP2,
142 FMLSv4f16_OP1,
143 FMLSv4f16_OP2,
144 FMLSv8f16_OP1,
145 FMLSv8f16_OP2,
146 FMLSv2f32_OP1,
147 FMLSv2f32_OP2,
148 FMLSv2f64_OP1,
149 FMLSv2f64_OP2,
150 FMLSv4i16_indexed_OP1,
151 FMLSv4i16_indexed_OP2,
152 FMLSv8i16_indexed_OP1,
153 FMLSv8i16_indexed_OP2,
154 FMLSv2i32_indexed_OP1,
155 FMLSv2i32_indexed_OP2,
156 FMLSv2i64_indexed_OP1,
157 FMLSv2i64_indexed_OP2,
158 FMLSv4f32_OP1,
159 FMLSv4f32_OP2,
160 FMLSv4i32_indexed_OP1,
161 FMLSv4i32_indexed_OP2,
162
163 FMULv2i32_indexed_OP1,
164 FMULv2i32_indexed_OP2,
165 FMULv2i64_indexed_OP1,
166 FMULv2i64_indexed_OP2,
167 FMULv4i16_indexed_OP1,
168 FMULv4i16_indexed_OP2,
169 FMULv4i32_indexed_OP1,
170 FMULv4i32_indexed_OP2,
171 FMULv8i16_indexed_OP1,
172 FMULv8i16_indexed_OP2,
173
174 FNMADD,
175
176 GATHER_LANE_i32,
177 GATHER_LANE_i16,
178 GATHER_LANE_i8
179};
180class AArch64InstrInfo final : public AArch64GenInstrInfo {
181 const AArch64RegisterInfo RI;
182 const AArch64Subtarget &Subtarget;
183
184public:
185 explicit AArch64InstrInfo(const AArch64Subtarget &STI);
186
187 /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info. As
188 /// such, whenever a client has an instance of instruction info, it should
189 /// always be able to get register info as well (through this method).
190 const AArch64RegisterInfo &getRegisterInfo() const { return RI; }
191
192 const TargetRegisterClass *getInlineAsmMemoryOperandRegClass(
193 InlineAsm::ConstraintCode C) const override {
194 return &AArch64::GPR64spRegClass;
195 }
196
197 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
198
199 bool isAsCheapAsAMove(const MachineInstr &MI) const override;
200
201 bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg,
202 Register &DstReg, unsigned &SubIdx) const override;
203
204 bool
205 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
206 const MachineInstr &MIb) const override;
207
208 Register isLoadFromStackSlot(const MachineInstr &MI,
209 int &FrameIndex) const override;
210 Register isStoreToStackSlot(const MachineInstr &MI,
211 int &FrameIndex) const override;
212
213 /// Check for post-frame ptr elimination stack locations as well. This uses a
214 /// heuristic so it isn't reliable for correctness.
215 Register isStoreToStackSlotPostFE(const MachineInstr &MI,
216 int &FrameIndex) const override;
217 /// Check for post-frame ptr elimination stack locations as well. This uses a
218 /// heuristic so it isn't reliable for correctness.
219 Register isLoadFromStackSlotPostFE(const MachineInstr &MI,
220 int &FrameIndex) const override;
221
222 /// Does this instruction set its full destination register to zero?
223 static bool isGPRZero(const MachineInstr &MI);
224
225 /// Does this instruction rename a GPR without modifying bits?
226 static bool isGPRCopy(const MachineInstr &MI);
227
228 /// Does this instruction rename an FPR without modifying bits?
229 static bool isFPRCopy(const MachineInstr &MI);
230
231 /// Return true if pairing the given load or store is hinted to be
232 /// unprofitable.
233 static bool isLdStPairSuppressed(const MachineInstr &MI);
234
235 /// Return true if the given load or store is a strided memory access.
236 static bool isStridedAccess(const MachineInstr &MI);
237
238 /// Return true if it has an unscaled load/store offset.
239 static bool hasUnscaledLdStOffset(unsigned Opc);
240 static bool hasUnscaledLdStOffset(MachineInstr &MI) {
241 return hasUnscaledLdStOffset(Opc: MI.getOpcode());
242 }
243
244 /// Returns the unscaled load/store for the scaled load/store opcode,
245 /// if there is a corresponding unscaled variant available.
246 static std::optional<unsigned> getUnscaledLdSt(unsigned Opc);
247
248 /// Scaling factor for (scaled or unscaled) load or store.
249 static int getMemScale(unsigned Opc);
250 static int getMemScale(const MachineInstr &MI) {
251 return getMemScale(Opc: MI.getOpcode());
252 }
253
254 /// Returns whether the instruction is a pre-indexed load.
255 static bool isPreLd(const MachineInstr &MI);
256
257 /// Returns whether the instruction is a pre-indexed store.
258 static bool isPreSt(const MachineInstr &MI);
259
260 /// Returns whether the instruction is a pre-indexed load/store.
261 static bool isPreLdSt(const MachineInstr &MI);
262
263 /// Returns whether the instruction is a zero-extending load.
264 static bool isZExtLoad(const MachineInstr &MI);
265
266 /// Returns whether the instruction is a sign-extending load.
267 static bool isSExtLoad(const MachineInstr &MI);
268
269 /// Returns whether the instruction is a paired load/store.
270 static bool isPairedLdSt(const MachineInstr &MI);
271
272 /// Returns the base register operator of a load/store.
273 static const MachineOperand &getLdStBaseOp(const MachineInstr &MI);
274
275 /// Returns the immediate offset operator of a load/store.
276 static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI);
277
278 /// Returns whether the physical register is FP or NEON.
279 static bool isFpOrNEON(Register Reg);
280
281 /// Returns the shift amount operator of a load/store.
282 static const MachineOperand &getLdStAmountOp(const MachineInstr &MI);
283
284 /// Returns whether the instruction is FP or NEON.
285 static bool isFpOrNEON(const MachineInstr &MI);
286
287 /// Returns whether the instruction is in H form (16 bit operands)
288 static bool isHForm(const MachineInstr &MI);
289
290 /// Returns whether the instruction is in Q form (128 bit operands)
291 static bool isQForm(const MachineInstr &MI);
292
293 /// Returns whether the instruction can be compatible with non-zero BTYPE.
294 static bool hasBTISemantics(const MachineInstr &MI);
295
296 /// Returns the index for the immediate for a given instruction.
297 static unsigned getLoadStoreImmIdx(unsigned Opc);
298
299 /// Return true if pairing the given load or store may be paired with another.
300 static bool isPairableLdStInst(const MachineInstr &MI);
301
302 /// Returns true if MI is one of the TCRETURN* instructions.
303 static bool isTailCallReturnInst(const MachineInstr &MI);
304
305 /// Return the opcode that set flags when possible. The caller is
306 /// responsible for ensuring the opc has a flag setting equivalent.
307 static unsigned convertToFlagSettingOpc(unsigned Opc);
308
309 /// Return true if this is a load/store that can be potentially paired/merged.
310 bool isCandidateToMergeOrPair(const MachineInstr &MI) const;
311
312 /// Hint that pairing the given load or store is unprofitable.
313 static void suppressLdStPair(MachineInstr &MI);
314
315 std::optional<ExtAddrMode>
316 getAddrModeFromMemoryOp(const MachineInstr &MemI,
317 const TargetRegisterInfo *TRI) const override;
318
319 bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
320 const MachineInstr &AddrI,
321 ExtAddrMode &AM) const override;
322
323 MachineInstr *emitLdStWithAddr(MachineInstr &MemI,
324 const ExtAddrMode &AM) const override;
325
326 bool getMemOperandsWithOffsetWidth(
327 const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
328 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
329 const TargetRegisterInfo *TRI) const override;
330
331 /// If \p OffsetIsScalable is set to 'true', the offset is scaled by `vscale`.
332 /// This is true for some SVE instructions like ldr/str that have a
333 /// 'reg + imm' addressing mode where the immediate is an index to the
334 /// scalable vector located at 'reg + imm * vscale x #bytes'.
335 bool getMemOperandWithOffsetWidth(const MachineInstr &MI,
336 const MachineOperand *&BaseOp,
337 int64_t &Offset, bool &OffsetIsScalable,
338 TypeSize &Width,
339 const TargetRegisterInfo *TRI) const;
340
341 /// Return the immediate offset of the base register in a load/store \p LdSt.
342 MachineOperand &getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const;
343
344 /// Returns true if opcode \p Opc is a memory operation. If it is, set
345 /// \p Scale, \p Width, \p MinOffset, and \p MaxOffset accordingly.
346 ///
347 /// For unscaled instructions, \p Scale is set to 1. All values are in bytes.
348 /// MinOffset/MaxOffset are the un-scaled limits of the immediate in the
349 /// instruction, the actual offset limit is [MinOffset*Scale,
350 /// MaxOffset*Scale].
351 static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width,
352 int64_t &MinOffset, int64_t &MaxOffset);
353
354 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
355 int64_t Offset1, bool OffsetIsScalable1,
356 ArrayRef<const MachineOperand *> BaseOps2,
357 int64_t Offset2, bool OffsetIsScalable2,
358 unsigned ClusterSize,
359 unsigned NumBytes) const override;
360
361 void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
362 const DebugLoc &DL, MCRegister DestReg,
363 MCRegister SrcReg, bool KillSrc,
364 llvm::ArrayRef<unsigned> Indices) const;
365 void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
366 const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
367 bool KillSrc, unsigned Opcode, unsigned ZeroReg,
368 llvm::ArrayRef<unsigned> Indices) const;
369 void copyPhysRegImpl(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
370 const DebugLoc &DL, Register DestReg, Register SrcReg,
371 bool KillSrc, bool RenamableDest = false,
372 bool RenamableSrc = false) const;
373 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
374 const DebugLoc &DL, Register DestReg, Register SrcReg,
375 bool KillSrc, bool RenamableDest = false,
376 bool RenamableSrc = false) const override;
377
378 void storeRegToStackSlot(
379 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
380 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
381 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
382
383 void loadRegFromStackSlot(
384 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
385 Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
386 Register VReg, unsigned SubReg = 0,
387 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
388
389 // This tells target independent code that it is okay to pass instructions
390 // with subreg operands to foldMemoryOperandImpl.
391 bool isSubregFoldable() const override { return true; }
392
393 using TargetInstrInfo::foldMemoryOperandImpl;
394 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
395 ArrayRef<unsigned> Ops, int FrameIndex,
396 MachineInstr *&CopyMI,
397 LiveIntervals *LIS = nullptr,
398 VirtRegMap *VRM = nullptr) const override;
399
400 /// \returns true if a branch from an instruction with opcode \p BranchOpc
401 /// bytes is capable of jumping to a position \p BrOffset bytes away.
402 bool isBranchOffsetInRange(unsigned BranchOpc,
403 int64_t BrOffset) const override;
404
405 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
406
407 void insertIndirectBranch(MachineBasicBlock &MBB,
408 MachineBasicBlock &NewDestBB,
409 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
410 int64_t BrOffset, RegScavenger *RS) const override;
411
412 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
413 MachineBasicBlock *&FBB,
414 SmallVectorImpl<MachineOperand> &Cond,
415 bool AllowModify = false) const override;
416 bool analyzeBranchPredicate(MachineBasicBlock &MBB,
417 MachineBranchPredicate &MBP,
418 bool AllowModify) const override;
419 unsigned removeBranch(MachineBasicBlock &MBB,
420 int *BytesRemoved = nullptr) const override;
421 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
422 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
423 const DebugLoc &DL,
424 int *BytesAdded = nullptr) const override;
425
426 /// Inserts the compare instruction needed to un-fuse a fused conditional
427 /// branch instruction and returns the condition code of the original fused
428 /// branch.
429 AArch64CC::CondCode insertCmpForCondBr(MachineBasicBlock &MBB,
430 MachineBasicBlock::iterator MI,
431 const DebugLoc &DL,
432 ArrayRef<MachineOperand> Cond) const;
433
434 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
435 analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
436
437 bool
438 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
439 bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
440 Register, Register, Register, int &, int &,
441 int &) const override;
442 void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
443 const DebugLoc &DL, Register DstReg,
444 ArrayRef<MachineOperand> Cond, Register TrueReg,
445 Register FalseReg) const override;
446
447 void insertNoop(MachineBasicBlock &MBB,
448 MachineBasicBlock::iterator MI) const override;
449
450 MCInst getNop() const override;
451
452 bool isSchedulingBoundary(const MachineInstr &MI,
453 const MachineBasicBlock *MBB,
454 const MachineFunction &MF) const override;
455
456 /// analyzeCompare - For a comparison instruction, return the source registers
457 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
458 /// Return true if the comparison instruction can be analyzed.
459 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
460 Register &SrcReg2, int64_t &CmpMask,
461 int64_t &CmpValue) const override;
462 /// optimizeCompareInstr - Convert the instruction supplying the argument to
463 /// the comparison into one that sets the zero bit in the flags register.
464 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
465 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
466 const MachineRegisterInfo *MRI) const override;
467 bool optimizeCondBranch(MachineInstr &MI) const override;
468
469 CombinerObjective getCombinerObjective(unsigned Pattern) const override;
470 /// Return true when a code sequence can improve throughput. It
471 /// should be called only for instructions in loops.
472 /// \param Pattern - combiner pattern
473 bool isThroughputPattern(unsigned Pattern) const override;
474 /// Return true when there is potentially a faster code sequence
475 /// for an instruction chain ending in ``Root``. All potential patterns are
476 /// listed in the ``Patterns`` array.
477 bool getMachineCombinerPatterns(MachineInstr &Root,
478 SmallVectorImpl<unsigned> &Patterns,
479 bool DoRegPressureReduce) const override;
480 /// Return true when Inst is associative and commutative so that it can be
481 /// reassociated. If Invert is true, then the inverse of Inst operation must
482 /// be checked.
483 bool isAssociativeAndCommutative(const MachineInstr &Inst,
484 bool Invert) const override;
485
486 /// Returns true if \P Opcode is an instruction which performs accumulation
487 /// into a destination register.
488 bool isAccumulationOpcode(unsigned Opcode) const override;
489
490 /// Returns an opcode which defines the accumulator used by \P Opcode.
491 unsigned getAccumulationStartOpcode(unsigned Opcode) const override;
492
493 unsigned
494 getReduceOpcodeForAccumulator(unsigned int AccumulatorOpCode) const override;
495
496 /// When getMachineCombinerPatterns() finds patterns, this function
497 /// generates the instructions that could replace the original code
498 /// sequence
499 void genAlternativeCodeSequence(
500 MachineInstr &Root, unsigned Pattern,
501 SmallVectorImpl<MachineInstr *> &InsInstrs,
502 SmallVectorImpl<MachineInstr *> &DelInstrs,
503 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const override;
504 /// AArch64 supports MachineCombiner.
505 bool useMachineCombiner() const override;
506
507 bool expandPostRAPseudo(MachineInstr &MI) const override;
508
509 std::pair<unsigned, unsigned>
510 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
511 ArrayRef<std::pair<unsigned, const char *>>
512 getSerializableDirectMachineOperandTargetFlags() const override;
513 ArrayRef<std::pair<unsigned, const char *>>
514 getSerializableBitmaskMachineOperandTargetFlags() const override;
515 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
516 getSerializableMachineMemOperandTargetFlags() const override;
517
518 bool isFunctionSafeToOutlineFrom(MachineFunction &MF,
519 bool OutlineFromLinkOnceODRs) const override;
520 std::optional<std::unique_ptr<outliner::OutlinedFunction>>
521 getOutliningCandidateInfo(
522 const MachineModuleInfo &MMI,
523 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
524 unsigned MinRepeats) const override;
525 void mergeOutliningCandidateAttributes(
526 Function &F, std::vector<outliner::Candidate> &Candidates) const override;
527 outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI,
528 MachineBasicBlock::iterator &MIT,
529 unsigned Flags) const override;
530 SmallVector<
531 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
532 getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override;
533 void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF,
534 const outliner::OutlinedFunction &OF) const override;
535 MachineBasicBlock::iterator
536 insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
537 MachineBasicBlock::iterator &It, MachineFunction &MF,
538 outliner::Candidate &C) const override;
539 bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override;
540
541 void buildClearRegister(Register Reg, MachineBasicBlock &MBB,
542 MachineBasicBlock::iterator Iter, DebugLoc &DL,
543 bool AllowSideEffects = true) const override;
544
545 /// Returns the vector element size (B, H, S or D) of an SVE opcode.
546 uint64_t getElementSizeForOpcode(unsigned Opc) const;
547 /// Returns true if the opcode is for an SVE instruction that sets the
548 /// condition codes as if it's results had been fed to a PTEST instruction
549 /// along with the same general predicate.
550 bool isPTestLikeOpcode(unsigned Opc) const;
551 /// Returns true if the opcode is for an SVE WHILE## instruction.
552 bool isWhileOpcode(unsigned Opc) const;
553 /// Returns true if the instruction has a shift by immediate that can be
554 /// executed in one cycle less.
555 static bool isFalkorShiftExtFast(const MachineInstr &MI);
556 /// Return true if the instructions is a SEH instruction used for unwinding
557 /// on Windows.
558 static bool isSEHInstruction(const MachineInstr &MI);
559
560 std::optional<RegImmPair> isAddImmediate(const MachineInstr &MI,
561 Register Reg) const override;
562
563 bool isFunctionSafeToSplit(const MachineFunction &MF) const override;
564
565 bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override;
566
567 std::optional<ParamLoadedValue>
568 describeLoadedValue(const MachineInstr &MI, Register Reg) const override;
569
570 unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override;
571
572 bool isExtendLikelyToBeFolded(MachineInstr &ExtMI,
573 MachineRegisterInfo &MRI) const override;
574
575 static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset,
576 int64_t &NumBytes,
577 int64_t &NumPredicateVectors,
578 int64_t &NumDataVectors);
579 static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset,
580 int64_t &ByteSized,
581 int64_t &VGSized);
582
583 // Return true if address of the form BaseReg + Scale * ScaledReg + Offset can
584 // be used for a load/store of NumBytes. BaseReg is always present and
585 // implicit.
586 bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
587 unsigned Scale) const;
588
589 // Decrement the SP, issuing probes along the way. `TargetReg` is the new top
590 // of the stack. `FrameSetup` is passed as true, if the allocation is a part
591 // of constructing the activation frame of a function.
592 MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI,
593 Register TargetReg,
594 bool FrameSetup) const;
595
596 static int
597 findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr);
598
599 /// Insert a `PAUTH_EPILOGUE` pseudo before the first terminator in \p MBB to
600 /// authenticate the return address. Adds an implicit def of X16 when the
601 /// branch protection uses PAuthLR but the subtarget lacks PAuthLR
602 /// instructions. If the epilogue has callee-popped argument stack to restore,
603 /// it additionally implicit defines X15 and X17 to cover clobbered registers
604 /// for the required sequence on subtargets both with and without PAuthLR
605 /// instructions.
606 void createPauthEpilogueInstr(MachineBasicBlock &MBB, DebugLoc DL) const;
607
608#define GET_INSTRINFO_HELPER_DECLS
609#include "AArch64GenInstrInfo.inc"
610
611protected:
612 /// If the specific machine instruction is an instruction that moves/copies
613 /// value from one register to another register return destination and source
614 /// registers as machine operands.
615 std::optional<DestSourcePair>
616 isCopyInstrImpl(const MachineInstr &MI) const override;
617 std::optional<DestSourcePair>
618 isCopyLikeInstrImpl(const MachineInstr &MI) const override;
619
620private:
621 /// Sets the offsets on outlined instructions in \p MBB which use SP
622 /// so that they will be valid post-outlining.
623 ///
624 /// \param MBB A \p MachineBasicBlock in an outlined function.
625 void fixupPostOutline(MachineBasicBlock &MBB) const;
626
627 void instantiateCondBranch(MachineBasicBlock &MBB, const DebugLoc &DL,
628 MachineBasicBlock *TBB,
629 ArrayRef<MachineOperand> Cond) const;
630 bool substituteCmpToZero(MachineInstr &CmpInstr, unsigned SrcReg,
631 const MachineRegisterInfo &MRI) const;
632 bool removeCmpToZeroOrOne(MachineInstr &CmpInstr, unsigned SrcReg,
633 int CmpValue, const MachineRegisterInfo &MRI) const;
634
635 /// Returns an unused general-purpose register which can be used for
636 /// constructing an outlined call if one exists. Returns 0 otherwise.
637 Register findRegisterToSaveLRTo(outliner::Candidate &C) const;
638
639 /// Remove a ptest of a predicate-generating operation that already sets, or
640 /// can be made to set, the condition codes in an identical manner
641 bool optimizePTestInstr(MachineInstr *PTest, unsigned MaskReg,
642 unsigned PredReg,
643 const MachineRegisterInfo *MRI) const;
644 std::optional<unsigned>
645 canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
646 MachineInstr *Pred, const MachineRegisterInfo *MRI) const;
647
648 /// verifyInstruction - Perform target specific instruction verification.
649 bool verifyInstruction(const MachineInstr &MI,
650 StringRef &ErrInfo) const override;
651};
652
653struct UsedNZCV {
654 bool N = false;
655 bool Z = false;
656 bool C = false;
657 bool V = false;
658
659 UsedNZCV() = default;
660
661 UsedNZCV &operator|=(const UsedNZCV &UsedFlags) {
662 this->N |= UsedFlags.N;
663 this->Z |= UsedFlags.Z;
664 this->C |= UsedFlags.C;
665 this->V |= UsedFlags.V;
666 return *this;
667 }
668};
669
670/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
671/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
672/// \returns std::nullopt otherwise.
673///
674/// Collect instructions using that flags in \p CCUseInstrs if provided.
675std::optional<UsedNZCV>
676examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
677 const TargetRegisterInfo &TRI,
678 SmallVectorImpl<MachineInstr *> *CCUseInstrs = nullptr);
679
680/// Return true if there is an instruction /after/ \p DefMI and before \p UseMI
681/// which either reads or clobbers NZCV.
682bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
683 const MachineInstr &UseMI,
684 const TargetRegisterInfo *TRI);
685
686MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg,
687 unsigned Reg, const StackOffset &Offset,
688 bool LastAdjustmentWasScalable = true);
689MCCFIInstruction
690createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg,
691 const StackOffset &OffsetFromDefCFA,
692 std::optional<int64_t> IncomingVGOffsetFromDefCFA);
693
694/// emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg
695/// plus Offset. This is intended to be used from within the prolog/epilog
696/// insertion (PEI) pass, where a virtual scratch register may be allocated
697/// if necessary, to be replaced by the scavenger at the end of PEI.
698void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
699 const DebugLoc &DL, unsigned DestReg, unsigned SrcReg,
700 StackOffset Offset, const TargetInstrInfo *TII,
701 MachineInstr::MIFlag = MachineInstr::NoFlags,
702 bool SetNZCV = false, bool NeedsWinCFI = false,
703 bool *HasWinCFI = nullptr, bool EmitCFAOffset = false,
704 StackOffset InitialOffset = {},
705 unsigned FrameReg = AArch64::SP);
706
707/// rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the
708/// FP. Return false if the offset could not be handled directly in MI, and
709/// return the left-over portion by reference.
710bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
711 unsigned FrameReg, StackOffset &Offset,
712 const AArch64InstrInfo *TII);
713
714/// Use to report the frame offset status in isAArch64FrameOffsetLegal.
715enum AArch64FrameOffsetStatus {
716 AArch64FrameOffsetCannotUpdate = 0x0, ///< Offset cannot apply.
717 AArch64FrameOffsetIsLegal = 0x1, ///< Offset is legal.
718 AArch64FrameOffsetCanUpdate = 0x2 ///< Offset can apply, at least partly.
719};
720
721/// Check if the @p Offset is a valid frame offset for @p MI.
722/// The returned value reports the validity of the frame offset for @p MI.
723/// It uses the values defined by AArch64FrameOffsetStatus for that.
724/// If result == AArch64FrameOffsetCannotUpdate, @p MI cannot be updated to
725/// use an offset.eq
726/// If result & AArch64FrameOffsetIsLegal, @p Offset can completely be
727/// rewritten in @p MI.
728/// If result & AArch64FrameOffsetCanUpdate, @p Offset contains the
729/// amount that is off the limit of the legal offset.
730/// If set, @p OutUseUnscaledOp will contain the whether @p MI should be
731/// turned into an unscaled operator, which opcode is in @p OutUnscaledOp.
732/// If set, @p EmittableOffset contains the amount that can be set in @p MI
733/// (possibly with @p OutUnscaledOp if OutUseUnscaledOp is true) and that
734/// is a legal offset.
735int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset,
736 bool *OutUseUnscaledOp = nullptr,
737 unsigned *OutUnscaledOp = nullptr,
738 int64_t *EmittableOffset = nullptr);
739
740bool optimizeTerminators(MachineBasicBlock *MBB, const TargetInstrInfo &TII);
741
742static inline bool isUncondBranchOpcode(int Opc) { return Opc == AArch64::B; }
743
744static inline bool isCondBranchOpcode(int Opc) {
745 switch (Opc) {
746 case AArch64::Bcc:
747 case AArch64::CBZW:
748 case AArch64::CBZX:
749 case AArch64::CBNZW:
750 case AArch64::CBNZX:
751 case AArch64::TBZW:
752 case AArch64::TBZX:
753 case AArch64::TBNZW:
754 case AArch64::TBNZX:
755 case AArch64::CBWPri:
756 case AArch64::CBXPri:
757 case AArch64::CBBAssertExt:
758 case AArch64::CBHAssertExt:
759 case AArch64::CBWPrr:
760 case AArch64::CBXPrr:
761 return true;
762 default:
763 return false;
764 }
765}
766
767static inline bool isIndirectBranchOpcode(int Opc) {
768 switch (Opc) {
769 case AArch64::BR:
770 case AArch64::BRAA:
771 case AArch64::BRAB:
772 case AArch64::BRAAZ:
773 case AArch64::BRABZ:
774 return true;
775 }
776 return false;
777}
778
779static inline bool isIndirectCallOpcode(unsigned Opc) {
780 switch (Opc) {
781 case AArch64::BLR:
782 case AArch64::BLRAA:
783 case AArch64::BLRAB:
784 case AArch64::BLRAAZ:
785 case AArch64::BLRABZ:
786 return true;
787 default:
788 return false;
789 }
790}
791
792static inline bool isPTrueOpcode(unsigned Opc) {
793 switch (Opc) {
794 case AArch64::PTRUE_B:
795 case AArch64::PTRUE_H:
796 case AArch64::PTRUE_S:
797 case AArch64::PTRUE_D:
798 return true;
799 default:
800 return false;
801 }
802}
803
804/// Return opcode to be used for indirect calls.
805unsigned getBLRCallOpcode(const MachineFunction &MF);
806
807/// Return XPAC opcode to be used for a ptrauth strip using the given key.
808static inline unsigned getXPACOpcodeForKey(AArch64PACKey::ID K) {
809 using namespace AArch64PACKey;
810 switch (K) {
811 case IA: case IB: return AArch64::XPACI;
812 case DA: case DB: return AArch64::XPACD;
813 }
814 llvm_unreachable("Unhandled AArch64PACKey::ID enum");
815}
816
817/// Return AUT opcode to be used for a ptrauth auth using the given key, or its
818/// AUT*Z variant that doesn't take a discriminator operand, using zero instead.
819static inline unsigned getAUTOpcodeForKey(AArch64PACKey::ID K, bool Zero) {
820 using namespace AArch64PACKey;
821 switch (K) {
822 case IA: return Zero ? AArch64::AUTIZA : AArch64::AUTIA;
823 case IB: return Zero ? AArch64::AUTIZB : AArch64::AUTIB;
824 case DA: return Zero ? AArch64::AUTDZA : AArch64::AUTDA;
825 case DB: return Zero ? AArch64::AUTDZB : AArch64::AUTDB;
826 }
827 llvm_unreachable("Unhandled AArch64PACKey::ID enum");
828}
829
830/// Return PAC opcode to be used for a ptrauth sign using the given key, or its
831/// PAC*Z variant that doesn't take a discriminator operand, using zero instead.
832static inline unsigned getPACOpcodeForKey(AArch64PACKey::ID K, bool Zero) {
833 using namespace AArch64PACKey;
834 switch (K) {
835 case IA: return Zero ? AArch64::PACIZA : AArch64::PACIA;
836 case IB: return Zero ? AArch64::PACIZB : AArch64::PACIB;
837 case DA: return Zero ? AArch64::PACDZA : AArch64::PACDA;
838 case DB: return Zero ? AArch64::PACDZB : AArch64::PACDB;
839 }
840 llvm_unreachable("Unhandled AArch64PACKey::ID enum");
841}
842
843/// Return B(L)RA opcode to be used for an authenticated branch or call using
844/// the given key, or its B(L)RA*Z variant that doesn't take a discriminator
845/// operand, using zero instead.
846static inline unsigned getBranchOpcodeForKey(bool IsCall, AArch64PACKey::ID K,
847 bool Zero) {
848 using namespace AArch64PACKey;
849 static const unsigned BranchOpcode[2][2] = {
850 {AArch64::BRAA, AArch64::BRAAZ},
851 {AArch64::BRAB, AArch64::BRABZ},
852 };
853 static const unsigned CallOpcode[2][2] = {
854 {AArch64::BLRAA, AArch64::BLRAAZ},
855 {AArch64::BLRAB, AArch64::BLRABZ},
856 };
857
858 assert((K == IA || K == IB) && "B(L)RA* instructions require IA or IB key");
859 if (IsCall)
860 return CallOpcode[K == IB][Zero];
861 return BranchOpcode[K == IB][Zero];
862}
863
864// struct TSFlags {
865#define TSFLAG_ELEMENT_SIZE_TYPE(X) (X) // 3-bits
866#define TSFLAG_DESTRUCTIVE_INST_TYPE(X) ((X) << 3) // 4-bits
867#define TSFLAG_FALSE_LANE_TYPE(X) ((X) << 7) // 2-bits
868#define TSFLAG_INSTR_FLAGS(X) ((X) << 9) // 2-bits
869#define TSFLAG_SME_MATRIX_TYPE(X) ((X) << 11) // 3-bits
870// }
871
872namespace AArch64 {
873
874// clang-format off
875enum ElementSizeType {
876 ElementSizeMask = TSFLAG_ELEMENT_SIZE_TYPE(0x7),
877 ElementSizeNone = TSFLAG_ELEMENT_SIZE_TYPE(0x0),
878 ElementSizeB = TSFLAG_ELEMENT_SIZE_TYPE(0x1),
879 ElementSizeH = TSFLAG_ELEMENT_SIZE_TYPE(0x2),
880 ElementSizeS = TSFLAG_ELEMENT_SIZE_TYPE(0x3),
881 ElementSizeD = TSFLAG_ELEMENT_SIZE_TYPE(0x4),
882};
883
884enum DestructiveInstType {
885 DestructiveInstTypeMask = TSFLAG_DESTRUCTIVE_INST_TYPE(0xf),
886 NotDestructive = TSFLAG_DESTRUCTIVE_INST_TYPE(0x0),
887 DestructiveOther = TSFLAG_DESTRUCTIVE_INST_TYPE(0x1),
888 DestructiveUnary = TSFLAG_DESTRUCTIVE_INST_TYPE(0x2),
889 DestructiveBinaryImm = TSFLAG_DESTRUCTIVE_INST_TYPE(0x3),
890 DestructiveBinaryShImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0x4),
891 DestructiveBinary = TSFLAG_DESTRUCTIVE_INST_TYPE(0x5),
892 DestructiveBinaryComm = TSFLAG_DESTRUCTIVE_INST_TYPE(0x6),
893 DestructiveBinaryCommWithRev = TSFLAG_DESTRUCTIVE_INST_TYPE(0x7),
894 DestructiveTernaryCommWithRev = TSFLAG_DESTRUCTIVE_INST_TYPE(0x8),
895 Destructive2xRegImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0x9),
896 DestructiveUnaryPassthru = TSFLAG_DESTRUCTIVE_INST_TYPE(0xa),
897 DestructivePredicate = TSFLAG_DESTRUCTIVE_INST_TYPE(0xb),
898 DestructiveBinaryImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0xc),
899};
900
901enum FalseLaneType {
902 FalseLanesMask = TSFLAG_FALSE_LANE_TYPE(0x3),
903 FalseLanesZero = TSFLAG_FALSE_LANE_TYPE(0x1),
904 FalseLanesUndef = TSFLAG_FALSE_LANE_TYPE(0x2),
905};
906
907// clang-format on
908
909// NOTE: This is a bit field.
910static const uint64_t InstrFlagIsWhile = TSFLAG_INSTR_FLAGS(0x1);
911static const uint64_t InstrFlagIsPTestLike = TSFLAG_INSTR_FLAGS(0x2);
912
913enum SMEMatrixType {
914 SMEMatrixTypeMask = TSFLAG_SME_MATRIX_TYPE(0x7),
915 SMEMatrixNone = TSFLAG_SME_MATRIX_TYPE(0x0),
916 SMEMatrixTileB = TSFLAG_SME_MATRIX_TYPE(0x1),
917 SMEMatrixTileH = TSFLAG_SME_MATRIX_TYPE(0x2),
918 SMEMatrixTileS = TSFLAG_SME_MATRIX_TYPE(0x3),
919 SMEMatrixTileD = TSFLAG_SME_MATRIX_TYPE(0x4),
920 SMEMatrixTileQ = TSFLAG_SME_MATRIX_TYPE(0x5),
921 SMEMatrixArray = TSFLAG_SME_MATRIX_TYPE(0x6),
922};
923
924#undef TSFLAG_ELEMENT_SIZE_TYPE
925#undef TSFLAG_DESTRUCTIVE_INST_TYPE
926#undef TSFLAG_FALSE_LANE_TYPE
927#undef TSFLAG_INSTR_FLAGS
928#undef TSFLAG_SME_MATRIX_TYPE
929
930int32_t getSVEPseudoMap(uint32_t Opcode);
931int32_t getSVERevInstr(uint32_t Opcode);
932int32_t getSVENonRevInstr(uint32_t Opcode);
933
934int32_t getSMEPseudoMap(uint32_t Opcode);
935}
936
937} // end namespace llvm
938
939#endif
940