1//===- AArch64InstrInfo.h - AArch64 Instruction Information -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64INSTRINFO_H
14#define LLVM_LIB_TARGET_AARCH64_AARCH64INSTRINFO_H
15
16#include "AArch64.h"
17#include "AArch64RegisterInfo.h"
18#include "llvm/CodeGen/TargetInstrInfo.h"
19#include "llvm/Support/AArch64MemoryHints.h"
20#include "llvm/Support/TypeSize.h"
21#include <optional>
22
23#define GET_INSTRINFO_HEADER
24#include "AArch64GenInstrInfo.inc"
25
26namespace llvm {
27
28class AArch64Subtarget;
29
30static const MachineMemOperand::Flags MOSuppressPair =
31 MachineMemOperand::MOTargetFlag1;
32static const MachineMemOperand::Flags MOStridedAccess =
33 MachineMemOperand::MOTargetFlag2;
34
35#define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
36
37// AArch64 MachineCombiner patterns
38enum AArch64MachineCombinerPattern : unsigned {
39 // These are patterns used to reduce the length of dependence chain.
40 SUBADD_OP1 = MachineCombinerPattern::TARGET_PATTERN_START,
41 SUBADD_OP2,
42
43 // These are multiply-add patterns matched by the AArch64 machine combiner.
44 MULADDW_OP1,
45 MULADDW_OP2,
46 MULSUBW_OP1,
47 MULSUBW_OP2,
48 MULADDWI_OP1,
49 MULSUBWI_OP1,
50 MULADDWS_OP1,
51 MULADDX_OP1,
52 MULADDX_OP2,
53 MULSUBX_OP1,
54 MULSUBX_OP2,
55 MULADDXI_OP1,
56 MULSUBXI_OP1,
57 MULADDXS_OP1,
58 // NEON integers vectors
59 MULADDv8i8_OP1,
60 MULADDv8i8_OP2,
61 MULADDv16i8_OP1,
62 MULADDv16i8_OP2,
63 MULADDv4i16_OP1,
64 MULADDv4i16_OP2,
65 MULADDv8i16_OP1,
66 MULADDv8i16_OP2,
67 MULADDv2i32_OP1,
68 MULADDv2i32_OP2,
69 MULADDv4i32_OP1,
70 MULADDv4i32_OP2,
71
72 MULSUBv8i8_OP1,
73 MULSUBv8i8_OP2,
74 MULSUBv16i8_OP1,
75 MULSUBv16i8_OP2,
76 MULSUBv4i16_OP1,
77 MULSUBv4i16_OP2,
78 MULSUBv8i16_OP1,
79 MULSUBv8i16_OP2,
80 MULSUBv2i32_OP1,
81 MULSUBv2i32_OP2,
82 MULSUBv4i32_OP1,
83 MULSUBv4i32_OP2,
84
85 MULADDv4i16_indexed_OP1,
86 MULADDv4i16_indexed_OP2,
87 MULADDv8i16_indexed_OP1,
88 MULADDv8i16_indexed_OP2,
89 MULADDv2i32_indexed_OP1,
90 MULADDv2i32_indexed_OP2,
91 MULADDv4i32_indexed_OP1,
92 MULADDv4i32_indexed_OP2,
93
94 MULSUBv4i16_indexed_OP1,
95 MULSUBv4i16_indexed_OP2,
96 MULSUBv8i16_indexed_OP1,
97 MULSUBv8i16_indexed_OP2,
98 MULSUBv2i32_indexed_OP1,
99 MULSUBv2i32_indexed_OP2,
100 MULSUBv4i32_indexed_OP1,
101 MULSUBv4i32_indexed_OP2,
102
103 // Floating Point
104 FMULADDH_OP1,
105 FMULADDH_OP2,
106 FMULSUBH_OP1,
107 FMULSUBH_OP2,
108 FMULADDS_OP1,
109 FMULADDS_OP2,
110 FMULSUBS_OP1,
111 FMULSUBS_OP2,
112 FMULADDD_OP1,
113 FMULADDD_OP2,
114 FMULSUBD_OP1,
115 FMULSUBD_OP2,
116 FNMULSUBH_OP1,
117 FNMULSUBS_OP1,
118 FNMULSUBD_OP1,
119 FMLAv1i32_indexed_OP1,
120 FMLAv1i32_indexed_OP2,
121 FMLAv1i64_indexed_OP1,
122 FMLAv1i64_indexed_OP2,
123 FMLAv4f16_OP1,
124 FMLAv4f16_OP2,
125 FMLAv8f16_OP1,
126 FMLAv8f16_OP2,
127 FMLAv2f32_OP2,
128 FMLAv2f32_OP1,
129 FMLAv2f64_OP1,
130 FMLAv2f64_OP2,
131 FMLAv4i16_indexed_OP1,
132 FMLAv4i16_indexed_OP2,
133 FMLAv8i16_indexed_OP1,
134 FMLAv8i16_indexed_OP2,
135 FMLAv2i32_indexed_OP1,
136 FMLAv2i32_indexed_OP2,
137 FMLAv2i64_indexed_OP1,
138 FMLAv2i64_indexed_OP2,
139 FMLAv4f32_OP1,
140 FMLAv4f32_OP2,
141 FMLAv4i32_indexed_OP1,
142 FMLAv4i32_indexed_OP2,
143 FMLSv1i32_indexed_OP2,
144 FMLSv1i64_indexed_OP2,
145 FMLSv4f16_OP1,
146 FMLSv4f16_OP2,
147 FMLSv8f16_OP1,
148 FMLSv8f16_OP2,
149 FMLSv2f32_OP1,
150 FMLSv2f32_OP2,
151 FMLSv2f64_OP1,
152 FMLSv2f64_OP2,
153 FMLSv4i16_indexed_OP1,
154 FMLSv4i16_indexed_OP2,
155 FMLSv8i16_indexed_OP1,
156 FMLSv8i16_indexed_OP2,
157 FMLSv2i32_indexed_OP1,
158 FMLSv2i32_indexed_OP2,
159 FMLSv2i64_indexed_OP1,
160 FMLSv2i64_indexed_OP2,
161 FMLSv4f32_OP1,
162 FMLSv4f32_OP2,
163 FMLSv4i32_indexed_OP1,
164 FMLSv4i32_indexed_OP2,
165
166 FMULv2i32_indexed_OP1,
167 FMULv2i32_indexed_OP2,
168 FMULv2i64_indexed_OP1,
169 FMULv2i64_indexed_OP2,
170 FMULv4i16_indexed_OP1,
171 FMULv4i16_indexed_OP2,
172 FMULv4i32_indexed_OP1,
173 FMULv4i32_indexed_OP2,
174 FMULv8i16_indexed_OP1,
175 FMULv8i16_indexed_OP2,
176
177 FNMADD,
178
179 GATHER_LANE_i32,
180 GATHER_LANE_i16,
181 GATHER_LANE_i8
182};
183class AArch64InstrInfo final : public AArch64GenInstrInfo {
184 const AArch64RegisterInfo RI;
185 const AArch64Subtarget &Subtarget;
186
187public:
188 explicit AArch64InstrInfo(const AArch64Subtarget &STI);
189
190 /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info. As
191 /// such, whenever a client has an instance of instruction info, it should
192 /// always be able to get register info as well (through this method).
193 const AArch64RegisterInfo &getRegisterInfo() const { return RI; }
194
195 const TargetRegisterClass *getInlineAsmMemoryOperandRegClass(
196 InlineAsm::ConstraintCode C) const override {
197 return &AArch64::GPR64spRegClass;
198 }
199
200 unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
201
202 bool isAsCheapAsAMove(const MachineInstr &MI) const override;
203
204 bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg,
205 Register &DstReg, unsigned &SubIdx) const override;
206
207 bool
208 areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
209 const MachineInstr &MIb) const override;
210
211 Register isLoadFromStackSlot(const MachineInstr &MI,
212 int &FrameIndex) const override;
213 Register isStoreToStackSlot(const MachineInstr &MI,
214 int &FrameIndex) const override;
215
216 /// Check for post-frame ptr elimination stack locations as well. This uses a
217 /// heuristic so it isn't reliable for correctness.
218 Register isStoreToStackSlotPostFE(const MachineInstr &MI,
219 int &FrameIndex) const override;
220 /// Check for post-frame ptr elimination stack locations as well. This uses a
221 /// heuristic so it isn't reliable for correctness.
222 Register isLoadFromStackSlotPostFE(const MachineInstr &MI,
223 int &FrameIndex) const override;
224
225 /// Does this instruction set its full destination register to zero?
226 static bool isGPRZero(const MachineInstr &MI);
227
228 /// Does this instruction rename a GPR without modifying bits?
229 static bool isGPRCopy(const MachineInstr &MI);
230
231 /// Does this instruction rename an FPR without modifying bits?
232 static bool isFPRCopy(const MachineInstr &MI);
233
234 /// Return true if pairing the given load or store is hinted to be
235 /// unprofitable.
236 static bool isLdStPairSuppressed(const MachineInstr &MI);
237
238 /// Return true if the given load or store is a strided memory access.
239 static bool isStridedAccess(const MachineInstr &MI);
240
241 /// Return true if it has an unscaled load/store offset.
242 static bool hasUnscaledLdStOffset(unsigned Opc);
243 static bool hasUnscaledLdStOffset(MachineInstr &MI) {
244 return hasUnscaledLdStOffset(Opc: MI.getOpcode());
245 }
246
247 /// Returns the unscaled load/store for the scaled load/store opcode,
248 /// if there is a corresponding unscaled variant available.
249 static std::optional<unsigned> getUnscaledLdSt(unsigned Opc);
250
251 /// Scaling factor for (scaled or unscaled) load or store.
252 static int getMemScale(unsigned Opc);
253 static int getMemScale(const MachineInstr &MI) {
254 return getMemScale(Opc: MI.getOpcode());
255 }
256
257 /// Returns whether the instruction is a pre-indexed load.
258 static bool isPreLd(const MachineInstr &MI);
259
260 /// Returns whether the instruction is a pre-indexed store.
261 static bool isPreSt(const MachineInstr &MI);
262
263 /// Returns whether the instruction is a pre-indexed load/store.
264 static bool isPreLdSt(const MachineInstr &MI);
265
266 /// Returns whether the instruction is a zero-extending load.
267 static bool isZExtLoad(const MachineInstr &MI);
268
269 /// Returns whether the instruction is a sign-extending load.
270 static bool isSExtLoad(const MachineInstr &MI);
271
272 /// Returns whether the instruction is a paired load/store.
273 static bool isPairedLdSt(const MachineInstr &MI);
274
275 /// Returns the base register operator of a load/store.
276 static const MachineOperand &getLdStBaseOp(const MachineInstr &MI);
277
278 /// Returns the immediate offset operator of a load/store.
279 static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI);
280
281 /// Returns whether the physical register is FP or NEON.
282 static bool isFpOrNEON(Register Reg);
283
284 /// Returns the shift amount operator of a load/store.
285 static const MachineOperand &getLdStAmountOp(const MachineInstr &MI);
286
287 /// Returns whether the instruction is FP or NEON.
288 static bool isFpOrNEON(const MachineInstr &MI);
289
290 /// Returns whether the instruction is in H form (16 bit operands)
291 static bool isHForm(const MachineInstr &MI);
292
293 /// Returns whether the instruction is in Q form (128 bit operands)
294 static bool isQForm(const MachineInstr &MI);
295
296 /// Returns whether the instruction can be compatible with non-zero BTYPE.
297 static bool hasBTISemantics(const MachineInstr &MI);
298
299 /// Returns the index for the immediate for a given instruction.
300 static unsigned getLoadStoreImmIdx(unsigned Opc);
301
302 /// Return true if pairing the given load or store may be paired with another.
303 static bool isPairableLdStInst(const MachineInstr &MI);
304
305 /// Returns true if MI is one of the TCRETURN* instructions.
306 static bool isTailCallReturnInst(const MachineInstr &MI);
307
308 /// Return the opcode that set flags when possible. The caller is
309 /// responsible for ensuring the opc has a flag setting equivalent.
310 static unsigned convertToFlagSettingOpc(unsigned Opc);
311
312 /// Return true if this is a load/store that can be potentially paired/merged.
313 bool isCandidateToMergeOrPair(const MachineInstr &MI) const;
314
315 /// Hint that pairing the given load or store is unprofitable.
316 static void suppressLdStPair(MachineInstr &MI);
317
318 std::optional<ExtAddrMode>
319 getAddrModeFromMemoryOp(const MachineInstr &MemI) const override;
320
321 bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
322 const MachineInstr &AddrI,
323 ExtAddrMode &AM) const override;
324
325 MachineInstr *emitLdStWithAddr(MachineInstr &MemI,
326 const ExtAddrMode &AM) const override;
327
328 bool getMemOperandsWithOffsetWidth(
329 const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
330 int64_t &Offset, bool &OffsetIsScalable,
331 LocationSize &Width) const override;
332
333 /// If \p OffsetIsScalable is set to 'true', the offset is scaled by `vscale`.
334 /// This is true for some SVE instructions like ldr/str that have a
335 /// 'reg + imm' addressing mode where the immediate is an index to the
336 /// scalable vector located at 'reg + imm * vscale x #bytes'.
337 bool getMemOperandWithOffsetWidth(const MachineInstr &MI,
338 const MachineOperand *&BaseOp,
339 int64_t &Offset, bool &OffsetIsScalable,
340 TypeSize &Width) const;
341
342 /// Return the immediate offset of the base register in a load/store \p LdSt.
343 MachineOperand &getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const;
344
345 /// Returns true if opcode \p Opc is a memory operation. If it is, set
346 /// \p Scale, \p Width, \p MinOffset, and \p MaxOffset accordingly.
347 ///
348 /// For unscaled instructions, \p Scale is set to 1. All values are in bytes.
349 /// MinOffset/MaxOffset are the un-scaled limits of the immediate in the
350 /// instruction, the actual offset limit is [MinOffset*Scale,
351 /// MaxOffset*Scale].
352 static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width,
353 int64_t &MinOffset, int64_t &MaxOffset);
354
355 bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
356 int64_t Offset1, bool OffsetIsScalable1,
357 ArrayRef<const MachineOperand *> BaseOps2,
358 int64_t Offset2, bool OffsetIsScalable2,
359 unsigned ClusterSize,
360 unsigned NumBytes) const override;
361
362 void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
363 const DebugLoc &DL, MCRegister DestReg,
364 MCRegister SrcReg, bool KillSrc,
365 llvm::ArrayRef<unsigned> Indices) const;
366 void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
367 const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
368 bool KillSrc, unsigned Opcode, unsigned ZeroReg,
369 llvm::ArrayRef<unsigned> Indices) const;
370 void copyPhysRegImpl(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
371 const DebugLoc &DL, Register DestReg, Register SrcReg,
372 bool KillSrc, bool RenamableDest = false,
373 bool RenamableSrc = false) const;
374 void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
375 const DebugLoc &DL, Register DestReg, Register SrcReg,
376 bool KillSrc, bool RenamableDest = false,
377 bool RenamableSrc = false) const override;
378
379 void storeRegToStackSlot(
380 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, Register SrcReg,
381 bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg,
382 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
383
384 void loadRegFromStackSlot(
385 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
386 Register DestReg, int FrameIndex, const TargetRegisterClass *RC,
387 Register VReg, unsigned SubReg = 0,
388 MachineInstr::MIFlag Flags = MachineInstr::NoFlags) const override;
389
390 // This tells target independent code that it is okay to pass instructions
391 // with subreg operands to foldMemoryOperandImpl.
392 bool isSubregFoldable() const override { return true; }
393
394 using TargetInstrInfo::foldMemoryOperandImpl;
395 MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
396 ArrayRef<unsigned> Ops, int FrameIndex,
397 MachineInstr *&CopyMI,
398 LiveIntervals *LIS = nullptr,
399 VirtRegMap *VRM = nullptr) const override;
400
401 /// \returns true if a branch from an instruction with opcode \p BranchOpc
402 /// bytes is capable of jumping to a position \p BrOffset bytes away.
403 bool isBranchOffsetInRange(unsigned BranchOpc,
404 int64_t BrOffset) const override;
405
406 MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
407
408 void insertIndirectBranch(MachineBasicBlock &MBB,
409 MachineBasicBlock &NewDestBB,
410 MachineBasicBlock &RestoreBB, const DebugLoc &DL,
411 int64_t BrOffset, RegScavenger *RS) const override;
412
413 bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
414 MachineBasicBlock *&FBB,
415 SmallVectorImpl<MachineOperand> &Cond,
416 bool AllowModify = false) const override;
417 bool analyzeBranchPredicate(MachineBasicBlock &MBB,
418 MachineBranchPredicate &MBP,
419 bool AllowModify) const override;
420 unsigned removeBranch(MachineBasicBlock &MBB,
421 int *BytesRemoved = nullptr) const override;
422 unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
423 MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
424 const DebugLoc &DL,
425 int *BytesAdded = nullptr) const override;
426
427 /// Inserts the compare instruction needed to un-fuse a fused conditional
428 /// branch instruction and returns the condition code of the original fused
429 /// branch.
430 AArch64CC::CondCode insertCmpForCondBr(MachineBasicBlock &MBB,
431 MachineBasicBlock::iterator MI,
432 const DebugLoc &DL,
433 ArrayRef<MachineOperand> Cond) const;
434
435 std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
436 analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
437
438 bool
439 reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
440 bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
441 Register, Register, Register,
442 SelectExpansion &) const override;
443 void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
444 const DebugLoc &DL, Register DstReg,
445 ArrayRef<MachineOperand> Cond, Register TrueReg,
446 Register FalseReg) const override;
447
448 void insertNoop(MachineBasicBlock &MBB,
449 MachineBasicBlock::iterator MI) const override;
450
451 MCInst getNop() const override;
452
453 bool isSchedulingBoundary(const MachineInstr &MI,
454 const MachineBasicBlock *MBB,
455 const MachineFunction &MF) const override;
456
457 /// analyzeCompare - For a comparison instruction, return the source registers
458 /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
459 /// Return true if the comparison instruction can be analyzed.
460 bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
461 Register &SrcReg2, int64_t &CmpMask,
462 int64_t &CmpValue) const override;
463 /// optimizeCompareInstr - Convert the instruction supplying the argument to
464 /// the comparison into one that sets the zero bit in the flags register.
465 bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
466 Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
467 const MachineRegisterInfo *MRI) const override;
468 bool optimizeCondBranch(MachineInstr &MI) const override;
469
470 CombinerObjective getCombinerObjective(unsigned Pattern) const override;
471 /// Return true when a code sequence can improve throughput. It
472 /// should be called only for instructions in loops.
473 /// \param Pattern - combiner pattern
474 bool isThroughputPattern(unsigned Pattern) const override;
475 /// Return true when there is potentially a faster code sequence
476 /// for an instruction chain ending in ``Root``. All potential patterns are
477 /// listed in the ``Patterns`` array.
478 bool getMachineCombinerPatterns(MachineInstr &Root,
479 SmallVectorImpl<unsigned> &Patterns,
480 bool DoRegPressureReduce) const override;
481 /// Return true when Inst is associative and commutative so that it can be
482 /// reassociated. If Invert is true, then the inverse of Inst operation must
483 /// be checked.
484 bool isAssociativeAndCommutative(const MachineInstr &Inst,
485 bool Invert) const override;
486
487 /// Returns true if \P Opcode is an instruction which performs accumulation
488 /// into a destination register.
489 bool isAccumulationOpcode(unsigned Opcode) const override;
490
491 /// Returns an opcode which defines the accumulator used by \P Opcode.
492 unsigned getAccumulationStartOpcode(unsigned Opcode) const override;
493
494 unsigned
495 getReduceOpcodeForAccumulator(unsigned int AccumulatorOpCode) const override;
496
497 /// When getMachineCombinerPatterns() finds patterns, this function
498 /// generates the instructions that could replace the original code
499 /// sequence
500 void genAlternativeCodeSequence(
501 MachineInstr &Root, unsigned Pattern,
502 SmallVectorImpl<MachineInstr *> &InsInstrs,
503 SmallVectorImpl<MachineInstr *> &DelInstrs,
504 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const override;
505 /// AArch64 supports MachineCombiner.
506 bool useMachineCombiner() const override;
507
508 bool expandPostRAPseudo(MachineInstr &MI) const override;
509
510 std::pair<unsigned, unsigned>
511 decomposeMachineOperandsTargetFlags(unsigned TF) const override;
512 ArrayRef<std::pair<unsigned, const char *>>
513 getSerializableDirectMachineOperandTargetFlags() const override;
514 ArrayRef<std::pair<unsigned, const char *>>
515 getSerializableBitmaskMachineOperandTargetFlags() const override;
516 ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
517 getSerializableMachineMemOperandTargetFlags() const override;
518
519 bool isFunctionSafeToOutlineFrom(MachineFunction &MF,
520 bool OutlineFromLinkOnceODRs) const override;
521 std::optional<std::unique_ptr<outliner::OutlinedFunction>>
522 getOutliningCandidateInfo(
523 const MachineModuleInfo &MMI,
524 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
525 unsigned MinRepeats) const override;
526 void mergeOutliningCandidateAttributes(
527 Function &F, std::vector<outliner::Candidate> &Candidates) const override;
528 outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI,
529 MachineBasicBlock::iterator &MIT,
530 unsigned Flags) const override;
531 SmallVector<
532 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
533 getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override;
534 void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF,
535 const outliner::OutlinedFunction &OF) const override;
536 MachineBasicBlock::iterator
537 insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
538 MachineBasicBlock::iterator &It, MachineFunction &MF,
539 outliner::Candidate &C) const override;
540 bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override;
541
542 void buildClearRegister(Register Reg, MachineBasicBlock &MBB,
543 MachineBasicBlock::iterator Iter, DebugLoc &DL,
544 bool AllowSideEffects = true) const override;
545
546 /// Returns the vector element size (B, H, S or D) of an SVE opcode.
547 uint64_t getElementSizeForOpcode(unsigned Opc) const;
548 /// Returns true if the opcode is for an SVE instruction that sets the
549 /// condition codes as if it's results had been fed to a PTEST instruction
550 /// along with the same general predicate.
551 bool isPTestLikeOpcode(unsigned Opc) const;
552 /// Returns true if the opcode is for an SVE WHILE## instruction.
553 bool isWhileOpcode(unsigned Opc) const;
554 /// Returns true if the instruction has a shift by immediate that can be
555 /// executed in one cycle less.
556 static bool isFalkorShiftExtFast(const MachineInstr &MI);
557 /// Return true if the instructions is a SEH instruction used for unwinding
558 /// on Windows.
559 static bool isSEHInstruction(const MachineInstr &MI);
560
561 std::optional<RegImmPair> isAddImmediate(const MachineInstr &MI,
562 Register Reg) const override;
563
564 bool isFunctionSafeToSplit(const MachineFunction &MF) const override;
565
566 bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override;
567
568 std::optional<ParamLoadedValue>
569 describeLoadedValue(const MachineInstr &MI, Register Reg) const override;
570
571 unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override;
572
573 bool isExtendLikelyToBeFolded(MachineInstr &ExtMI,
574 MachineRegisterInfo &MRI) const override;
575
576 static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset,
577 int64_t &NumBytes,
578 int64_t &NumPredicateVectors,
579 int64_t &NumDataVectors);
580 static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset,
581 int64_t &ByteSized,
582 int64_t &VGSized);
583
584 // Return true if address of the form BaseReg + Scale * ScaledReg + Offset can
585 // be used for a load/store of NumBytes. BaseReg is always present and
586 // implicit.
587 bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
588 unsigned Scale) const;
589
590 // Decrement the SP, issuing probes along the way. `TargetReg` is the new top
591 // of the stack. `FrameSetup` is passed as true, if the allocation is a part
592 // of constructing the activation frame of a function.
593 MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI,
594 Register TargetReg,
595 bool FrameSetup) const;
596
597 static int
598 findCondCodeUseOperandIdxForBranchOrSelect(const MachineInstr &Instr);
599
600 /// Insert a `PAUTH_EPILOGUE` pseudo before the first terminator in \p MBB to
601 /// authenticate the return address. Adds an implicit def of X16 when the
602 /// branch protection uses PAuthLR but the subtarget lacks PAuthLR
603 /// instructions. If the epilogue has callee-popped argument stack to restore,
604 /// it additionally implicit defines X15 and X17 to cover clobbered registers
605 /// for the required sequence on subtargets both with and without PAuthLR
606 /// instructions.
607 void createPauthEpilogueInstr(MachineBasicBlock &MBB, DebugLoc DL) const;
608
609#define GET_INSTRINFO_HELPER_DECLS
610#include "AArch64GenInstrInfo.inc"
611
612protected:
613 /// If the specific machine instruction is an instruction that moves/copies
614 /// value from one register to another register return destination and source
615 /// registers as machine operands.
616 std::optional<DestSourcePair>
617 isCopyInstrImpl(const MachineInstr &MI) const override;
618 std::optional<DestSourcePair>
619 isCopyLikeInstrImpl(const MachineInstr &MI) const override;
620
621private:
622 /// Returns the size of \p MI, excluding the LFI deferred LR guard that may
623 /// be flushed before a control-flow instruction (added by
624 /// getInstSizeInBytes).
625 unsigned getInstSizeInBytesNoGuard(const MachineInstr &MI) const;
626
627 /// Sets the offsets on outlined instructions in \p MBB which use SP
628 /// so that they will be valid post-outlining.
629 ///
630 /// \param MBB A \p MachineBasicBlock in an outlined function.
631 void fixupPostOutline(MachineBasicBlock &MBB) const;
632
633 void instantiateCondBranch(MachineBasicBlock &MBB, const DebugLoc &DL,
634 MachineBasicBlock *TBB,
635 ArrayRef<MachineOperand> Cond) const;
636 bool substituteCmpToZero(MachineInstr &CmpInstr, unsigned SrcReg,
637 const MachineRegisterInfo &MRI) const;
638 bool removeCmpToZeroOrOne(MachineInstr &CmpInstr, unsigned SrcReg,
639 int CmpValue, const MachineRegisterInfo &MRI) const;
640
641 /// Returns an unused general-purpose register which can be used for
642 /// constructing an outlined call if one exists. Returns 0 otherwise.
643 Register findRegisterToSaveLRTo(outliner::Candidate &C) const;
644
645 /// Remove a ptest of a predicate-generating operation that already sets, or
646 /// can be made to set, the condition codes in an identical manner
647 bool optimizePTestInstr(MachineInstr *PTest, unsigned MaskReg,
648 unsigned PredReg,
649 const MachineRegisterInfo *MRI) const;
650 std::optional<unsigned>
651 canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
652 MachineInstr *Pred, const MachineRegisterInfo *MRI) const;
653
654 /// verifyInstruction - Perform target specific instruction verification.
655 bool verifyInstruction(const MachineInstr &MI,
656 StringRef &ErrInfo) const override;
657};
658
659struct UsedNZCV {
660 bool N = false;
661 bool Z = false;
662 bool C = false;
663 bool V = false;
664
665 UsedNZCV() = default;
666
667 UsedNZCV &operator|=(const UsedNZCV &UsedFlags) {
668 this->N |= UsedFlags.N;
669 this->Z |= UsedFlags.Z;
670 this->C |= UsedFlags.C;
671 this->V |= UsedFlags.V;
672 return *this;
673 }
674};
675
676/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
677/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
678/// \returns std::nullopt otherwise.
679///
680/// Collect instructions using that flags in \p CCUseInstrs if provided.
681std::optional<UsedNZCV>
682examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
683 const TargetRegisterInfo &TRI,
684 SmallVectorImpl<MachineInstr *> *CCUseInstrs = nullptr);
685
686/// Return true if there is an instruction /after/ \p DefMI and before \p UseMI
687/// which either reads or clobbers NZCV.
688bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
689 const MachineInstr &UseMI,
690 const TargetRegisterInfo *TRI);
691
692MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg,
693 unsigned Reg, const StackOffset &Offset,
694 bool LastAdjustmentWasScalable = true);
695MCCFIInstruction
696createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg,
697 const StackOffset &OffsetFromDefCFA,
698 std::optional<int64_t> IncomingVGOffsetFromDefCFA);
699
700/// emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg
701/// plus Offset. This is intended to be used from within the prolog/epilog
702/// insertion (PEI) pass, where a virtual scratch register may be allocated
703/// if necessary, to be replaced by the scavenger at the end of PEI.
704void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
705 const DebugLoc &DL, unsigned DestReg, unsigned SrcReg,
706 StackOffset Offset, const TargetInstrInfo *TII,
707 MachineInstr::MIFlag = MachineInstr::NoFlags,
708 bool SetNZCV = false, bool NeedsWinCFI = false,
709 bool *HasWinCFI = nullptr, bool EmitCFAOffset = false,
710 StackOffset InitialOffset = {},
711 unsigned FrameReg = AArch64::SP);
712
713/// rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the
714/// FP. Return false if the offset could not be handled directly in MI, and
715/// return the left-over portion by reference.
716bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
717 unsigned FrameReg, StackOffset &Offset,
718 const AArch64InstrInfo *TII);
719
720/// Use to report the frame offset status in isAArch64FrameOffsetLegal.
721enum AArch64FrameOffsetStatus {
722 AArch64FrameOffsetCannotUpdate = 0x0, ///< Offset cannot apply.
723 AArch64FrameOffsetIsLegal = 0x1, ///< Offset is legal.
724 AArch64FrameOffsetCanUpdate = 0x2 ///< Offset can apply, at least partly.
725};
726
727/// Check if the @p Offset is a valid frame offset for @p MI.
728/// The returned value reports the validity of the frame offset for @p MI.
729/// It uses the values defined by AArch64FrameOffsetStatus for that.
730/// If result == AArch64FrameOffsetCannotUpdate, @p MI cannot be updated to
731/// use an offset.eq
732/// If result & AArch64FrameOffsetIsLegal, @p Offset can completely be
733/// rewritten in @p MI.
734/// If result & AArch64FrameOffsetCanUpdate, @p Offset contains the
735/// amount that is off the limit of the legal offset.
736/// If set, @p OutUseUnscaledOp will contain the whether @p MI should be
737/// turned into an unscaled operator, which opcode is in @p OutUnscaledOp.
738/// If set, @p EmittableOffset contains the amount that can be set in @p MI
739/// (possibly with @p OutUnscaledOp if OutUseUnscaledOp is true) and that
740/// is a legal offset.
741int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset,
742 bool *OutUseUnscaledOp = nullptr,
743 unsigned *OutUnscaledOp = nullptr,
744 int64_t *EmittableOffset = nullptr);
745
746bool optimizeTerminators(MachineBasicBlock *MBB, const TargetInstrInfo &TII);
747
748static inline bool isUncondBranchOpcode(int Opc) { return Opc == AArch64::B; }
749
750static inline bool isCondBranchOpcode(int Opc) {
751 switch (Opc) {
752 case AArch64::Bcc:
753 case AArch64::CBZW:
754 case AArch64::CBZX:
755 case AArch64::CBNZW:
756 case AArch64::CBNZX:
757 case AArch64::TBZW:
758 case AArch64::TBZX:
759 case AArch64::TBNZW:
760 case AArch64::TBNZX:
761 case AArch64::CBWPri:
762 case AArch64::CBXPri:
763 case AArch64::CBBAssertExt:
764 case AArch64::CBHAssertExt:
765 case AArch64::CBWPrr:
766 case AArch64::CBXPrr:
767 return true;
768 default:
769 return false;
770 }
771}
772
773static inline bool isIndirectBranchOpcode(int Opc) {
774 switch (Opc) {
775 case AArch64::BR:
776 case AArch64::BRAA:
777 case AArch64::BRAB:
778 case AArch64::BRAAZ:
779 case AArch64::BRABZ:
780 return true;
781 }
782 return false;
783}
784
785static inline bool isIndirectCallOpcode(unsigned Opc) {
786 switch (Opc) {
787 case AArch64::BLR:
788 case AArch64::BLRAA:
789 case AArch64::BLRAB:
790 case AArch64::BLRAAZ:
791 case AArch64::BLRABZ:
792 return true;
793 default:
794 return false;
795 }
796}
797
798static inline bool isPTrueOpcode(unsigned Opc) {
799 switch (Opc) {
800 case AArch64::PTRUE_B:
801 case AArch64::PTRUE_H:
802 case AArch64::PTRUE_S:
803 case AArch64::PTRUE_D:
804 return true;
805 default:
806 return false;
807 }
808}
809
810/// Return opcode to be used for indirect calls.
811unsigned getBLRCallOpcode(const MachineFunction &MF);
812
813/// Return XPAC opcode to be used for a ptrauth strip using the given key.
814static inline unsigned getXPACOpcodeForKey(AArch64PACKey::ID K) {
815 using namespace AArch64PACKey;
816 switch (K) {
817 case IA: case IB: return AArch64::XPACI;
818 case DA: case DB: return AArch64::XPACD;
819 }
820 llvm_unreachable("Unhandled AArch64PACKey::ID enum");
821}
822
823/// Return AUT opcode to be used for a ptrauth auth using the given key, or its
824/// AUT*Z variant that doesn't take a discriminator operand, using zero instead.
825static inline unsigned getAUTOpcodeForKey(AArch64PACKey::ID K, bool Zero) {
826 using namespace AArch64PACKey;
827 switch (K) {
828 case IA: return Zero ? AArch64::AUTIZA : AArch64::AUTIA;
829 case IB: return Zero ? AArch64::AUTIZB : AArch64::AUTIB;
830 case DA: return Zero ? AArch64::AUTDZA : AArch64::AUTDA;
831 case DB: return Zero ? AArch64::AUTDZB : AArch64::AUTDB;
832 }
833 llvm_unreachable("Unhandled AArch64PACKey::ID enum");
834}
835
836/// Return PAC opcode to be used for a ptrauth sign using the given key, or its
837/// PAC*Z variant that doesn't take a discriminator operand, using zero instead.
838static inline unsigned getPACOpcodeForKey(AArch64PACKey::ID K, bool Zero) {
839 using namespace AArch64PACKey;
840 switch (K) {
841 case IA: return Zero ? AArch64::PACIZA : AArch64::PACIA;
842 case IB: return Zero ? AArch64::PACIZB : AArch64::PACIB;
843 case DA: return Zero ? AArch64::PACDZA : AArch64::PACDA;
844 case DB: return Zero ? AArch64::PACDZB : AArch64::PACDB;
845 }
846 llvm_unreachable("Unhandled AArch64PACKey::ID enum");
847}
848
849/// Return B(L)RA opcode to be used for an authenticated branch or call using
850/// the given key, or its B(L)RA*Z variant that doesn't take a discriminator
851/// operand, using zero instead.
852static inline unsigned getBranchOpcodeForKey(bool IsCall, AArch64PACKey::ID K,
853 bool Zero) {
854 using namespace AArch64PACKey;
855 static const unsigned BranchOpcode[2][2] = {
856 {AArch64::BRAA, AArch64::BRAAZ},
857 {AArch64::BRAB, AArch64::BRABZ},
858 };
859 static const unsigned CallOpcode[2][2] = {
860 {AArch64::BLRAA, AArch64::BLRAAZ},
861 {AArch64::BLRAB, AArch64::BLRABZ},
862 };
863
864 assert((K == IA || K == IB) && "B(L)RA* instructions require IA or IB key");
865 if (IsCall)
866 return CallOpcode[K == IB][Zero];
867 return BranchOpcode[K == IB][Zero];
868}
869
870// struct TSFlags {
871#define TSFLAG_ELEMENT_SIZE_TYPE(X) (X) // 3-bits
872#define TSFLAG_DESTRUCTIVE_INST_TYPE(X) ((X) << 3) // 4-bits
873#define TSFLAG_FALSE_LANE_TYPE(X) ((X) << 7) // 2-bits
874#define TSFLAG_INSTR_FLAGS(X) ((X) << 9) // 2-bits
875#define TSFLAG_SME_MATRIX_TYPE(X) ((X) << 11) // 3-bits
876// }
877
878namespace AArch64 {
879
880// clang-format off
881enum ElementSizeType {
882 ElementSizeMask = TSFLAG_ELEMENT_SIZE_TYPE(0x7),
883 ElementSizeNone = TSFLAG_ELEMENT_SIZE_TYPE(0x0),
884 ElementSizeB = TSFLAG_ELEMENT_SIZE_TYPE(0x1),
885 ElementSizeH = TSFLAG_ELEMENT_SIZE_TYPE(0x2),
886 ElementSizeS = TSFLAG_ELEMENT_SIZE_TYPE(0x3),
887 ElementSizeD = TSFLAG_ELEMENT_SIZE_TYPE(0x4),
888};
889
890enum DestructiveInstType {
891 DestructiveInstTypeMask = TSFLAG_DESTRUCTIVE_INST_TYPE(0xf),
892 NotDestructive = TSFLAG_DESTRUCTIVE_INST_TYPE(0x0),
893 DestructiveOther = TSFLAG_DESTRUCTIVE_INST_TYPE(0x1),
894 DestructiveUnary = TSFLAG_DESTRUCTIVE_INST_TYPE(0x2),
895 DestructiveBinaryImm = TSFLAG_DESTRUCTIVE_INST_TYPE(0x3),
896 DestructiveBinaryShImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0x4),
897 DestructiveBinary = TSFLAG_DESTRUCTIVE_INST_TYPE(0x5),
898 DestructiveBinaryComm = TSFLAG_DESTRUCTIVE_INST_TYPE(0x6),
899 DestructiveBinaryCommWithRev = TSFLAG_DESTRUCTIVE_INST_TYPE(0x7),
900 DestructiveTernaryCommWithRev = TSFLAG_DESTRUCTIVE_INST_TYPE(0x8),
901 Destructive2xRegImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0x9),
902 DestructiveUnaryPassthru = TSFLAG_DESTRUCTIVE_INST_TYPE(0xa),
903 DestructivePredicate = TSFLAG_DESTRUCTIVE_INST_TYPE(0xb),
904 DestructiveBinaryImmUnpred = TSFLAG_DESTRUCTIVE_INST_TYPE(0xc),
905};
906
907enum FalseLaneType {
908 FalseLanesMask = TSFLAG_FALSE_LANE_TYPE(0x3),
909 FalseLanesZero = TSFLAG_FALSE_LANE_TYPE(0x1),
910 FalseLanesUndef = TSFLAG_FALSE_LANE_TYPE(0x2),
911};
912
913// clang-format on
914
915// NOTE: This is a bit field.
916static const uint64_t InstrFlagIsWhile = TSFLAG_INSTR_FLAGS(0x1);
917static const uint64_t InstrFlagIsPTestLike = TSFLAG_INSTR_FLAGS(0x2);
918
919enum SMEMatrixType {
920 SMEMatrixTypeMask = TSFLAG_SME_MATRIX_TYPE(0x7),
921 SMEMatrixNone = TSFLAG_SME_MATRIX_TYPE(0x0),
922 SMEMatrixTileB = TSFLAG_SME_MATRIX_TYPE(0x1),
923 SMEMatrixTileH = TSFLAG_SME_MATRIX_TYPE(0x2),
924 SMEMatrixTileS = TSFLAG_SME_MATRIX_TYPE(0x3),
925 SMEMatrixTileD = TSFLAG_SME_MATRIX_TYPE(0x4),
926 SMEMatrixTileQ = TSFLAG_SME_MATRIX_TYPE(0x5),
927 SMEMatrixArray = TSFLAG_SME_MATRIX_TYPE(0x6),
928};
929
930#undef TSFLAG_ELEMENT_SIZE_TYPE
931#undef TSFLAG_DESTRUCTIVE_INST_TYPE
932#undef TSFLAG_FALSE_LANE_TYPE
933#undef TSFLAG_INSTR_FLAGS
934#undef TSFLAG_SME_MATRIX_TYPE
935
936int32_t getSVEPseudoMap(uint32_t Opcode);
937int32_t getSVERevInstr(uint32_t Opcode);
938int32_t getSVENonRevInstr(uint32_t Opcode);
939
940int32_t getSMEPseudoMap(uint32_t Opcode);
941}
942
943} // end namespace llvm
944
945#endif
946