1//===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64PointerAuth.h"
17#include "AArch64Subtarget.h"
18#include "MCTargetDesc/AArch64AddressingModes.h"
19#include "MCTargetDesc/AArch64MCLFIRewriter.h"
20#include "MCTargetDesc/AArch64MCTargetDesc.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/Analysis/AliasAnalysis.h"
28#include "llvm/CodeGen/CFIInstBuilder.h"
29#include "llvm/CodeGen/LivePhysRegs.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineCombinerPattern.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstr.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/RegisterScavenging.h"
41#include "llvm/CodeGen/StackMaps.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/CodeGen/TargetSubtargetInfo.h"
44#include "llvm/IR/DebugInfoMetadata.h"
45#include "llvm/IR/DebugLoc.h"
46#include "llvm/IR/GlobalValue.h"
47#include "llvm/IR/Module.h"
48#include "llvm/MC/MCAsmInfo.h"
49#include "llvm/MC/MCInst.h"
50#include "llvm/MC/MCInstBuilder.h"
51#include "llvm/MC/MCInstrDesc.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CodeGen.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Support/LEB128.h"
57#include "llvm/Support/MathExtras.h"
58#include "llvm/Target/TargetMachine.h"
59#include "llvm/Target/TargetOptions.h"
60#include <cassert>
61#include <cstdint>
62#include <iterator>
63#include <utility>
64
65using namespace llvm;
66
67#define GET_INSTRINFO_CTOR_DTOR
68#include "AArch64GenInstrInfo.inc"
69
70#define DEBUG_TYPE "AArch64InstrInfo"
71
72STATISTIC(NumCopyInstrs, "Number of COPY instructions expanded");
73STATISTIC(NumZCRegMoveInstrsGPR, "Number of zero-cycle GPR register move "
74 "instructions expanded from canonical COPY");
75STATISTIC(NumZCRegMoveInstrsFPR, "Number of zero-cycle FPR register move "
76 "instructions expanded from canonical COPY");
77STATISTIC(NumZCZeroingInstrsGPR, "Number of zero-cycle GPR zeroing "
78 "instructions expanded from canonical COPY");
79// NumZCZeroingInstrsFPR is counted at AArch64AsmPrinter
80
81static cl::opt<unsigned>
82 CBDisplacementBits("aarch64-cb-offset-bits", cl::Hidden, cl::init(Val: 9),
83 cl::desc("Restrict range of CB instructions (DEBUG)"));
84
85static cl::opt<unsigned> TBZDisplacementBits(
86 "aarch64-tbz-offset-bits", cl::Hidden, cl::init(Val: 14),
87 cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
88
89static cl::opt<unsigned> CBZDisplacementBits(
90 "aarch64-cbz-offset-bits", cl::Hidden, cl::init(Val: 19),
91 cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
92
93static cl::opt<unsigned>
94 BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(Val: 19),
95 cl::desc("Restrict range of Bcc instructions (DEBUG)"));
96
97static cl::opt<unsigned>
98 BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(Val: 26),
99 cl::desc("Restrict range of B instructions (DEBUG)"));
100
101static cl::opt<unsigned> GatherOptSearchLimit(
102 "aarch64-search-limit", cl::Hidden, cl::init(Val: 2048),
103 cl::desc("Restrict range of instructions to search for the "
104 "machine-combiner gather pattern optimization"));
105
106AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
107 : AArch64GenInstrInfo(STI, RI, AArch64::ADJCALLSTACKDOWN,
108 AArch64::ADJCALLSTACKUP, AArch64::CATCHRET),
109 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
110
111/// Return the maximum number of bytes of code the specified instruction may be
112/// after LFI rewriting. If the instruction is not rewritten, std::nullopt is
113/// returned (use default sizing).
114///
115/// NOTE: the size estimates here must be kept in sync with the rewrites in
116/// AArch64MCLFIRewriter.cpp. Sizes may be overestimates of the rewritten
117/// instruction sequences.
118static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
119 switch (MI.getOpcode()) {
120 case AArch64::SVC:
121 // SVC expands to 4 instructions.
122 return 16;
123 case AArch64::BR:
124 case AArch64::BLR:
125 // Indirect branches/calls expand to 2 instructions (guard + br/blr).
126 return 8;
127 case AArch64::RET:
128 // RET through LR is not rewritten, but RET through another register
129 // expands to 2 instructions (guard + ret).
130 if (MI.getOperand(i: 0).getReg() != AArch64::LR)
131 return 8;
132 return 4;
133 case AArch64::RETAA:
134 case AArch64::RETAB:
135 // Authenticated returns expand to 3 instructions (authenticate + guard +
136 // ret).
137 return 12;
138 case AArch64::BRAA:
139 case AArch64::BRAAZ:
140 case AArch64::BRAB:
141 case AArch64::BRABZ:
142 case AArch64::BLRAA:
143 case AArch64::BLRAAZ:
144 case AArch64::BLRAB:
145 case AArch64::BLRABZ:
146 // Authenticated branches/calls expand to 3 instructions (authenticate +
147 // guard + branch).
148 return 12;
149 case AArch64::AUTIASP:
150 case AArch64::AUTIBSP:
151 case AArch64::AUTIAZ:
152 case AArch64::AUTIBZ:
153 case AArch64::XPACLRI:
154 // Authenticating LR expands to the instruction plus a deferred LR guard.
155 return 8;
156 case AArch64::SYSxt:
157 // VA-based DC/IC ops (op1=3, Cn=7, op2=1) expand to 2 instructions.
158 if (MI.getOperand(i: 0).getImm() == 3 && MI.getOperand(i: 1).getImm() == 7 &&
159 MI.getOperand(i: 3).getImm() == 1)
160 return 8;
161 return std::nullopt;
162 default:
163 break;
164 }
165
166 // Detect instructions that explicitly define SP or LR.
167 bool ModifiesLR = false;
168 bool ModifiesSP = false;
169 for (const MachineOperand &MO : MI.defs()) {
170 if (!MO.isReg())
171 continue;
172 if (MO.getReg() == AArch64::LR)
173 ModifiesLR = true;
174 else if (MO.getReg() == AArch64::SP)
175 ModifiesSP = true;
176 }
177
178 // Memory accesses expand to a base-register guard plus the rewritten access
179 // (8 bytes), with an extra base-register update for pre/post-index forms (12
180 // bytes total). If the access also defines LR, an LR mask is appended (+4
181 // bytes). Depending on additional optimizations that the rewriter performs,
182 // this may be an overestimate.
183 if (MI.mayLoadOrStore()) {
184 unsigned Size = isLFIPrePostMemAccess(Opcode: MI.getOpcode()) ? 12 : 8;
185 if (ModifiesLR)
186 Size += 4;
187 return Size;
188 }
189
190 // Non memory operations that modify LR or SP expand to 2 instructions.
191 if (ModifiesSP || ModifiesLR)
192 return 8;
193
194 // Default case: instructions that don't cause expansion.
195 // - TP accesses in LFI are a single load/store, so no expansion.
196 // - All remaining instructions are not rewritten.
197 return std::nullopt;
198}
199
200/// GetInstSize - Return the number of bytes of code the specified
201/// instruction may be. This returns the maximum number of bytes.
202unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
203 const MachineBasicBlock &MBB = *MI.getParent();
204 const MachineFunction *MF = MBB.getParent();
205 const Function &F = MF->getFunction();
206 const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
207
208 {
209 auto Op = MI.getOpcode();
210 if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
211 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(), MAI);
212 }
213
214 // Meta-instructions emit no code.
215 if (MI.isMetaInstruction())
216 return 0;
217
218 // FIXME: We currently only handle pseudoinstructions that don't get expanded
219 // before the assembly printer.
220 unsigned NumBytes = 0;
221 const MCInstrDesc &Desc = MI.getDesc();
222
223 // LFI rewriter expansions that supersede normal sizing.
224 const auto &STI = MF->getSubtarget<AArch64Subtarget>();
225 if (STI.isLFI())
226 if (auto Size = getLFIInstSizeInBytes(MI))
227 return *Size;
228
229 if (!MI.isBundle() && isTailCallReturnInst(MI)) {
230 NumBytes = Desc.getSize() ? Desc.getSize() : 4;
231
232 const auto *MFI = MF->getInfo<AArch64FunctionInfo>();
233 if (!MFI->shouldSignReturnAddress(MF: *MF))
234 return NumBytes;
235
236 auto Method = STI.getAuthenticatedLRCheckMethod(MF: *MF);
237 NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
238 return NumBytes;
239 }
240
241 // Size should be preferably set in
242 // llvm/lib/Target/AArch64/AArch64InstrInfo.td (default case).
243 // Specific cases handle instructions of variable sizes
244 switch (Desc.getOpcode()) {
245 default:
246 if (Desc.getSize())
247 return Desc.getSize();
248
249 // Anything not explicitly designated otherwise (i.e. pseudo-instructions
250 // with fixed constant size but not specified in .td file) is a normal
251 // 4-byte insn.
252 NumBytes = 4;
253 break;
254 case TargetOpcode::STACKMAP:
255 // The upper bound for a stackmap intrinsic is the full length of its shadow
256 NumBytes = StackMapOpers(&MI).getNumPatchBytes();
257 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
258 break;
259 case TargetOpcode::PATCHPOINT:
260 // The size of the patchpoint intrinsic is the number of bytes requested
261 NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
262 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
263 break;
264 case TargetOpcode::STATEPOINT:
265 NumBytes = StatepointOpers(&MI).getNumPatchBytes();
266 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
267 // No patch bytes means a normal call inst is emitted
268 if (NumBytes == 0)
269 NumBytes = 4;
270 break;
271 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
272 // If `patchable-function-entry` is set, PATCHABLE_FUNCTION_ENTER
273 // instructions are expanded to the specified number of NOPs. Otherwise,
274 // they are expanded to 36-byte XRay sleds.
275 NumBytes =
276 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry", Default: 9) * 4;
277 break;
278 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
279 case TargetOpcode::PATCHABLE_TAIL_CALL:
280 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
281 // An XRay sled can be 4 bytes of alignment plus a 32-byte block.
282 NumBytes = 36;
283 break;
284 case TargetOpcode::PATCHABLE_EVENT_CALL:
285 // EVENT_CALL XRay sleds are exactly 6 instructions long (no alignment).
286 NumBytes = 24;
287 break;
288
289 case AArch64::SPACE:
290 NumBytes = MI.getOperand(i: 1).getImm();
291 break;
292 case AArch64::MOVaddr:
293 case AArch64::MOVaddrJT:
294 case AArch64::MOVaddrCP:
295 case AArch64::MOVaddrBA:
296 case AArch64::MOVaddrTLS:
297 case AArch64::MOVaddrEXT: {
298 // Use the same logic as the pseudo expansion to count instructions.
299 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
300 AArch64_IMM::expandMOVAddr(Opcode: Desc.getOpcode(),
301 TargetFlags: MI.getOperand(i: 1).getTargetFlags(),
302 IsTargetMachO: Subtarget.isTargetMachO(), Insn);
303 NumBytes = Insn.size() * 4;
304 break;
305 }
306
307 case AArch64::MOVi32imm:
308 case AArch64::MOVi64imm: {
309 // Use the same logic as the pseudo expansion to count instructions.
310 unsigned BitSize = Desc.getOpcode() == AArch64::MOVi32imm ? 32 : 64;
311 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
312 AArch64_IMM::expandMOVImm(Imm: MI.getOperand(i: 1).getImm(), BitSize, Insn);
313 NumBytes = Insn.size() * 4;
314 break;
315 }
316
317 case TargetOpcode::BUNDLE:
318 NumBytes = getInstBundleSize(MI);
319 break;
320 }
321
322 return NumBytes;
323}
324
325static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
326 SmallVectorImpl<MachineOperand> &Cond) {
327 // Block ends with fall-through condbranch.
328 switch (LastInst->getOpcode()) {
329 default:
330 llvm_unreachable("Unknown branch instruction?");
331 case AArch64::Bcc:
332 Target = LastInst->getOperand(i: 1).getMBB();
333 Cond.push_back(Elt: LastInst->getOperand(i: 0));
334 break;
335 case AArch64::CBZW:
336 case AArch64::CBZX:
337 case AArch64::CBNZW:
338 case AArch64::CBNZX:
339 Target = LastInst->getOperand(i: 1).getMBB();
340 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
341 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
342 Cond.push_back(Elt: LastInst->getOperand(i: 0));
343 break;
344 case AArch64::TBZW:
345 case AArch64::TBZX:
346 case AArch64::TBNZW:
347 case AArch64::TBNZX:
348 Target = LastInst->getOperand(i: 2).getMBB();
349 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
350 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
351 Cond.push_back(Elt: LastInst->getOperand(i: 0));
352 Cond.push_back(Elt: LastInst->getOperand(i: 1));
353 break;
354 case AArch64::CBWPri:
355 case AArch64::CBXPri:
356 case AArch64::CBWPrr:
357 case AArch64::CBXPrr:
358 Target = LastInst->getOperand(i: 3).getMBB();
359 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
360 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
361 Cond.push_back(Elt: LastInst->getOperand(i: 0));
362 Cond.push_back(Elt: LastInst->getOperand(i: 1));
363 Cond.push_back(Elt: LastInst->getOperand(i: 2));
364 break;
365 case AArch64::CBBAssertExt:
366 case AArch64::CBHAssertExt:
367 Target = LastInst->getOperand(i: 3).getMBB();
368 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1)); // -1
369 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode())); // Opc
370 Cond.push_back(Elt: LastInst->getOperand(i: 0)); // Cond
371 Cond.push_back(Elt: LastInst->getOperand(i: 1)); // Op0
372 Cond.push_back(Elt: LastInst->getOperand(i: 2)); // Op1
373 Cond.push_back(Elt: LastInst->getOperand(i: 4)); // Ext0
374 Cond.push_back(Elt: LastInst->getOperand(i: 5)); // Ext1
375 break;
376 }
377}
378
379static unsigned getBranchDisplacementBits(unsigned Opc) {
380 switch (Opc) {
381 default:
382 llvm_unreachable("unexpected opcode!");
383 case AArch64::B:
384 return BDisplacementBits;
385 case AArch64::TBNZW:
386 case AArch64::TBZW:
387 case AArch64::TBNZX:
388 case AArch64::TBZX:
389 return TBZDisplacementBits;
390 case AArch64::CBNZW:
391 case AArch64::CBZW:
392 case AArch64::CBNZX:
393 case AArch64::CBZX:
394 return CBZDisplacementBits;
395 case AArch64::Bcc:
396 return BCCDisplacementBits;
397 case AArch64::CBWPri:
398 case AArch64::CBXPri:
399 case AArch64::CBBAssertExt:
400 case AArch64::CBHAssertExt:
401 case AArch64::CBWPrr:
402 case AArch64::CBXPrr:
403 return CBDisplacementBits;
404 }
405}
406
407bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
408 int64_t BrOffset) const {
409 unsigned Bits = getBranchDisplacementBits(Opc: BranchOp);
410 assert(Bits >= 3 && "max branch displacement must be enough to jump"
411 "over conditional branch expansion");
412 return isIntN(N: Bits, x: BrOffset / 4);
413}
414
415MachineBasicBlock *
416AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
417 switch (MI.getOpcode()) {
418 default:
419 llvm_unreachable("unexpected opcode!");
420 case AArch64::B:
421 return MI.getOperand(i: 0).getMBB();
422 case AArch64::TBZW:
423 case AArch64::TBNZW:
424 case AArch64::TBZX:
425 case AArch64::TBNZX:
426 return MI.getOperand(i: 2).getMBB();
427 case AArch64::CBZW:
428 case AArch64::CBNZW:
429 case AArch64::CBZX:
430 case AArch64::CBNZX:
431 case AArch64::Bcc:
432 return MI.getOperand(i: 1).getMBB();
433 case AArch64::CBWPri:
434 case AArch64::CBXPri:
435 case AArch64::CBBAssertExt:
436 case AArch64::CBHAssertExt:
437 case AArch64::CBWPrr:
438 case AArch64::CBXPrr:
439 return MI.getOperand(i: 3).getMBB();
440 }
441}
442
443void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
444 MachineBasicBlock &NewDestBB,
445 MachineBasicBlock &RestoreBB,
446 const DebugLoc &DL,
447 int64_t BrOffset,
448 RegScavenger *RS) const {
449 assert(RS && "RegScavenger required for long branching");
450 assert(MBB.empty() &&
451 "new block should be inserted for expanding unconditional branch");
452 assert(MBB.pred_size() == 1);
453 assert(RestoreBB.empty() &&
454 "restore block should be inserted for restoring clobbered registers");
455
456 auto buildIndirectBranch = [&](Register Reg, MachineBasicBlock &DestBB) {
457 // Offsets outside of the signed 33-bit range are not supported for ADRP +
458 // ADD.
459 if (!isInt<33>(x: BrOffset))
460 report_fatal_error(
461 reason: "Branch offsets outside of the signed 33-bit range not supported");
462
463 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
464 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGE);
465 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: Reg)
466 .addReg(RegNo: Reg)
467 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
468 .addImm(Val: 0);
469 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::BR)).addReg(RegNo: Reg);
470 };
471
472 RS->enterBasicBlockEnd(MBB);
473 // If X16 is unused, we can rely on the linker to insert a range extension
474 // thunk if NewDestBB is out of range of a single B instruction.
475 constexpr Register Reg = AArch64::X16;
476 if (!RS->isRegUsed(Reg)) {
477 insertUnconditionalBranch(MBB, DestBB: &NewDestBB, DL);
478 RS->setRegUsed(Reg);
479 return;
480 }
481
482 // In a cold block without BTI, insert the indirect branch if a register is
483 // free. Skip this if BTI is enabled to avoid inserting a BTI at the target,
484 // prioritizing a dynamic cost in cold code over a static cost in hot code.
485 AArch64FunctionInfo *AFI = MBB.getParent()->getInfo<AArch64FunctionInfo>();
486 bool HasBTI = AFI && AFI->branchTargetEnforcement();
487 if (MBB.getSectionID() == MBBSectionID::ColdSectionID && !HasBTI) {
488 Register Scavenged = RS->FindUnusedReg(RC: &AArch64::GPR64RegClass);
489 if (Scavenged != AArch64::NoRegister) {
490 buildIndirectBranch(Scavenged, NewDestBB);
491 RS->setRegUsed(Reg: Scavenged);
492 return;
493 }
494 }
495
496 // Note: Spilling X16 briefly moves the stack pointer, making it incompatible
497 // with red zones.
498 if (!AFI || AFI->hasRedZone().value_or(u: true))
499 report_fatal_error(
500 reason: "Unable to insert indirect branch inside function that has red zone");
501
502 // Otherwise, spill X16 and defer range extension to the linker.
503 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::STRXpre))
504 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
505 .addReg(RegNo: Reg)
506 .addReg(RegNo: AArch64::SP)
507 .addImm(Val: -16);
508
509 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: &RestoreBB);
510
511 BuildMI(BB&: RestoreBB, I: RestoreBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::LDRXpost))
512 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
513 .addReg(RegNo: Reg, Flags: RegState::Define)
514 .addReg(RegNo: AArch64::SP)
515 .addImm(Val: 16);
516}
517
518// Branch analysis.
519bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
520 MachineBasicBlock *&TBB,
521 MachineBasicBlock *&FBB,
522 SmallVectorImpl<MachineOperand> &Cond,
523 bool AllowModify) const {
524 // If the block has no terminators, it just falls into the block after it.
525 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
526 if (I == MBB.end())
527 return false;
528
529 // Skip over SpeculationBarrierEndBB terminators
530 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
531 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
532 --I;
533 }
534
535 if (!isUnpredicatedTerminator(MI: *I))
536 return false;
537
538 // Get the last instruction in the block.
539 MachineInstr *LastInst = &*I;
540
541 // If there is only one terminator instruction, process it.
542 unsigned LastOpc = LastInst->getOpcode();
543 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
544 if (isUncondBranchOpcode(Opc: LastOpc)) {
545 TBB = LastInst->getOperand(i: 0).getMBB();
546 return false;
547 }
548 if (isCondBranchOpcode(Opc: LastOpc)) {
549 // Block ends with fall-through condbranch.
550 parseCondBranch(LastInst, Target&: TBB, Cond);
551 return false;
552 }
553 return true; // Can't handle indirect branch.
554 }
555
556 // Get the instruction before it if it is a terminator.
557 MachineInstr *SecondLastInst = &*I;
558 unsigned SecondLastOpc = SecondLastInst->getOpcode();
559
560 // If AllowModify is true and the block ends with two or more unconditional
561 // branches, delete all but the first unconditional branch.
562 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc)) {
563 while (isUncondBranchOpcode(Opc: SecondLastOpc)) {
564 LastInst->eraseFromParent();
565 LastInst = SecondLastInst;
566 LastOpc = LastInst->getOpcode();
567 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
568 // Return now the only terminator is an unconditional branch.
569 TBB = LastInst->getOperand(i: 0).getMBB();
570 return false;
571 }
572 SecondLastInst = &*I;
573 SecondLastOpc = SecondLastInst->getOpcode();
574 }
575 }
576
577 // If we're allowed to modify and the block ends in a unconditional branch
578 // which could simply fallthrough, remove the branch. (Note: This case only
579 // matters when we can't understand the whole sequence, otherwise it's also
580 // handled by BranchFolding.cpp.)
581 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc) &&
582 MBB.isLayoutSuccessor(MBB: getBranchDestBlock(MI: *LastInst))) {
583 LastInst->eraseFromParent();
584 LastInst = SecondLastInst;
585 LastOpc = LastInst->getOpcode();
586 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
587 assert(!isUncondBranchOpcode(LastOpc) &&
588 "unreachable unconditional branches removed above");
589
590 if (isCondBranchOpcode(Opc: LastOpc)) {
591 // Block ends with fall-through condbranch.
592 parseCondBranch(LastInst, Target&: TBB, Cond);
593 return false;
594 }
595 return true; // Can't handle indirect branch.
596 }
597 SecondLastInst = &*I;
598 SecondLastOpc = SecondLastInst->getOpcode();
599 }
600
601 // If there are three terminators, we don't know what sort of block this is.
602 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(MI: *--I))
603 return true;
604
605 // If the block ends with a B and a Bcc, handle it.
606 if (isCondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
607 parseCondBranch(LastInst: SecondLastInst, Target&: TBB, Cond);
608 FBB = LastInst->getOperand(i: 0).getMBB();
609 return false;
610 }
611
612 // If the block ends with two unconditional branches, handle it. The second
613 // one is not executed, so remove it.
614 if (isUncondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
615 TBB = SecondLastInst->getOperand(i: 0).getMBB();
616 I = LastInst;
617 if (AllowModify)
618 I->eraseFromParent();
619 return false;
620 }
621
622 // ...likewise if it ends with an indirect branch followed by an unconditional
623 // branch.
624 if (isIndirectBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
625 I = LastInst;
626 if (AllowModify)
627 I->eraseFromParent();
628 return true;
629 }
630
631 // Otherwise, can't handle this.
632 return true;
633}
634
635bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
636 MachineBranchPredicate &MBP,
637 bool AllowModify) const {
638 // Use analyzeBranch to validate the branch pattern.
639 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
640 SmallVector<MachineOperand, 4> Cond;
641 if (analyzeBranch(MBB, TBB, FBB, Cond, AllowModify))
642 return true;
643
644 // analyzeBranch returns success with empty Cond for unconditional branches.
645 if (Cond.empty())
646 return true;
647
648 MBP.TrueDest = TBB;
649 assert(MBP.TrueDest && "expected!");
650 MBP.FalseDest = FBB ? FBB : MBB.getNextNode();
651
652 MBP.ConditionDef = nullptr;
653 MBP.SingleUseCondition = false;
654
655 // Find the conditional branch. After analyzeBranch succeeds with non-empty
656 // Cond, there's exactly one conditional branch - either last (fallthrough)
657 // or second-to-last (followed by unconditional B).
658 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
659 if (I == MBB.end())
660 return true;
661
662 if (isUncondBranchOpcode(Opc: I->getOpcode())) {
663 if (I == MBB.begin())
664 return true;
665 --I;
666 }
667
668 MachineInstr *CondBranch = &*I;
669 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
670
671 switch (CondBranch->getOpcode()) {
672 default:
673 return true;
674
675 case AArch64::Bcc:
676 // Bcc takes the NZCV flag as the operand to branch on, walk up the
677 // instruction stream to find the last instruction to define NZCV.
678 for (MachineInstr &MI : llvm::drop_begin(RangeOrContainer: llvm::reverse(C&: MBB))) {
679 if (MI.modifiesRegister(Reg: AArch64::NZCV, /*TRI=*/nullptr)) {
680 MBP.ConditionDef = &MI;
681 break;
682 }
683 }
684 return false;
685
686 case AArch64::CBZW:
687 case AArch64::CBZX:
688 case AArch64::CBNZW:
689 case AArch64::CBNZX: {
690 MBP.LHS = CondBranch->getOperand(i: 0);
691 MBP.RHS = MachineOperand::CreateImm(Val: 0);
692 unsigned Opc = CondBranch->getOpcode();
693 MBP.Predicate = (Opc == AArch64::CBNZX || Opc == AArch64::CBNZW)
694 ? MachineBranchPredicate::PRED_NE
695 : MachineBranchPredicate::PRED_EQ;
696 Register CondReg = MBP.LHS.getReg();
697 if (CondReg.isVirtual())
698 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
699 return false;
700 }
701
702 case AArch64::TBZW:
703 case AArch64::TBZX:
704 case AArch64::TBNZW:
705 case AArch64::TBNZX: {
706 Register CondReg = CondBranch->getOperand(i: 0).getReg();
707 if (CondReg.isVirtual())
708 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
709 return false;
710 }
711 }
712}
713
714bool AArch64InstrInfo::reverseBranchCondition(
715 SmallVectorImpl<MachineOperand> &Cond) const {
716 if (Cond[0].getImm() != -1) {
717 // Regular Bcc
718 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
719 Cond[0].setImm(AArch64CC::getInvertedCondCode(Code: CC));
720 } else {
721 // Folded compare-and-branch
722 switch (Cond[1].getImm()) {
723 default:
724 llvm_unreachable("Unknown conditional branch!");
725 case AArch64::CBZW:
726 Cond[1].setImm(AArch64::CBNZW);
727 break;
728 case AArch64::CBNZW:
729 Cond[1].setImm(AArch64::CBZW);
730 break;
731 case AArch64::CBZX:
732 Cond[1].setImm(AArch64::CBNZX);
733 break;
734 case AArch64::CBNZX:
735 Cond[1].setImm(AArch64::CBZX);
736 break;
737 case AArch64::TBZW:
738 Cond[1].setImm(AArch64::TBNZW);
739 break;
740 case AArch64::TBNZW:
741 Cond[1].setImm(AArch64::TBZW);
742 break;
743 case AArch64::TBZX:
744 Cond[1].setImm(AArch64::TBNZX);
745 break;
746 case AArch64::TBNZX:
747 Cond[1].setImm(AArch64::TBZX);
748 break;
749
750 // Cond is { -1, Opcode, CC, Op0, Op1, ... }
751 case AArch64::CBWPri:
752 case AArch64::CBXPri:
753 case AArch64::CBBAssertExt:
754 case AArch64::CBHAssertExt:
755 case AArch64::CBWPrr:
756 case AArch64::CBXPrr: {
757 // Pseudos using standard 4bit Arm condition codes
758 AArch64CC::CondCode CC =
759 static_cast<AArch64CC::CondCode>(Cond[2].getImm());
760 Cond[2].setImm(AArch64CC::getInvertedCondCode(Code: CC));
761 }
762 }
763 }
764
765 return false;
766}
767
768unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
769 int *BytesRemoved) const {
770 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
771 if (I == MBB.end())
772 return 0;
773
774 if (!isUncondBranchOpcode(Opc: I->getOpcode()) &&
775 !isCondBranchOpcode(Opc: I->getOpcode()))
776 return 0;
777
778 // Remove the branch.
779 I->eraseFromParent();
780
781 I = MBB.end();
782
783 if (I == MBB.begin()) {
784 if (BytesRemoved)
785 *BytesRemoved = 4;
786 return 1;
787 }
788 --I;
789 if (!isCondBranchOpcode(Opc: I->getOpcode())) {
790 if (BytesRemoved)
791 *BytesRemoved = 4;
792 return 1;
793 }
794
795 // Remove the branch.
796 I->eraseFromParent();
797 if (BytesRemoved)
798 *BytesRemoved = 8;
799
800 return 2;
801}
802
803void AArch64InstrInfo::instantiateCondBranch(
804 MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
805 ArrayRef<MachineOperand> Cond) const {
806 if (Cond[0].getImm() != -1) {
807 // Regular Bcc
808 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: Cond[0].getImm()).addMBB(MBB: TBB);
809 } else {
810 // Folded compare-and-branch
811 // Note that we use addOperand instead of addReg to keep the flags.
812
813 // cbz, cbnz
814 const MachineInstrBuilder MIB =
815 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[1].getImm())).add(MO: Cond[2]);
816
817 // tbz/tbnz
818 if (Cond.size() > 3)
819 MIB.add(MO: Cond[3]);
820
821 // cb
822 if (Cond.size() > 4)
823 MIB.add(MO: Cond[4]);
824
825 MIB.addMBB(MBB: TBB);
826
827 // cb[b,h]
828 if (Cond.size() > 5) {
829 MIB.addImm(Val: Cond[5].getImm());
830 MIB.addImm(Val: Cond[6].getImm());
831 }
832 }
833}
834
835unsigned AArch64InstrInfo::insertBranch(
836 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
837 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
838 // Shouldn't be a fall through.
839 assert(TBB && "insertBranch must not be told to insert a fallthrough");
840
841 if (!FBB) {
842 if (Cond.empty()) // Unconditional branch?
843 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: TBB);
844 else
845 instantiateCondBranch(MBB, DL, TBB, Cond);
846
847 if (BytesAdded)
848 *BytesAdded = 4;
849
850 return 1;
851 }
852
853 // Two-way conditional branch.
854 instantiateCondBranch(MBB, DL, TBB, Cond);
855 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: FBB);
856
857 if (BytesAdded)
858 *BytesAdded = 8;
859
860 return 2;
861}
862
863bool llvm::optimizeTerminators(MachineBasicBlock *MBB,
864 const TargetInstrInfo &TII) {
865 for (MachineInstr &MI : MBB->terminators()) {
866 unsigned Opc = MI.getOpcode();
867 switch (Opc) {
868 case AArch64::CBZW:
869 case AArch64::CBZX:
870 case AArch64::TBZW:
871 case AArch64::TBZX:
872 // CBZ/TBZ with WZR/XZR -> unconditional B
873 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
874 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
875 DEBUG_WITH_TYPE("optimizeTerminators",
876 dbgs() << "Removing always taken branch: " << MI);
877 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
878 SmallVector<MachineBasicBlock *> Succs(MBB->successors());
879 for (auto *S : Succs)
880 if (S != Target)
881 MBB->removeSuccessor(Succ: S);
882 DebugLoc DL = MI.getDebugLoc();
883 while (MBB->rbegin() != &MI)
884 MBB->rbegin()->eraseFromParent();
885 MI.eraseFromParent();
886 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: AArch64::B)).addMBB(MBB: Target);
887 return true;
888 }
889 break;
890 case AArch64::CBNZW:
891 case AArch64::CBNZX:
892 case AArch64::TBNZW:
893 case AArch64::TBNZX:
894 // CBNZ/TBNZ with WZR/XZR -> never taken, remove branch and successor
895 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
896 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
897 DEBUG_WITH_TYPE("optimizeTerminators",
898 dbgs() << "Removing never taken branch: " << MI);
899 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
900 MI.getParent()->removeSuccessor(Succ: Target);
901 MI.eraseFromParent();
902 return true;
903 }
904 break;
905 }
906 }
907 return false;
908}
909
910// Find the original register that VReg is copied from.
911static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
912 while (Register::isVirtualRegister(Reg: VReg)) {
913 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
914 if (!DefMI->isFullCopy())
915 return VReg;
916 VReg = DefMI->getOperand(i: 1).getReg();
917 }
918 return VReg;
919}
920
921// Determine if VReg is defined by an instruction that can be folded into a
922// csel instruction. If so, return the folded opcode, and the replacement
923// register.
924static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
925 unsigned *NewReg = nullptr) {
926 VReg = removeCopies(MRI, VReg);
927 if (!Register::isVirtualRegister(Reg: VReg))
928 return 0;
929
930 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(RC: MRI.getRegClass(Reg: VReg));
931 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
932 unsigned Opc = 0;
933 unsigned SrcReg = 0;
934 switch (DefMI->getOpcode()) {
935 case AArch64::SUBREG_TO_REG:
936 // Check for the following way to define an 64-bit immediate:
937 // %0:gpr32 = MOVi32imm 1
938 // %1:gpr64 = SUBREG_TO_REG %0:gpr32, %subreg.sub_32
939 if (!DefMI->getOperand(i: 1).isReg())
940 return 0;
941 if (!DefMI->getOperand(i: 2).isImm() ||
942 DefMI->getOperand(i: 2).getImm() != AArch64::sub_32)
943 return 0;
944 DefMI = MRI.getVRegDef(Reg: DefMI->getOperand(i: 1).getReg());
945 if (DefMI->getOpcode() != AArch64::MOVi32imm)
946 return 0;
947 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
948 return 0;
949 assert(Is64Bit);
950 SrcReg = AArch64::XZR;
951 Opc = AArch64::CSINCXr;
952 break;
953
954 case AArch64::MOVi32imm:
955 case AArch64::MOVi64imm:
956 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
957 return 0;
958 SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
959 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
960 break;
961
962 case AArch64::ADDSXri:
963 case AArch64::ADDSWri:
964 // if NZCV is used, do not fold.
965 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
966 isDead: true) == -1)
967 return 0;
968 // fall-through to ADDXri and ADDWri.
969 [[fallthrough]];
970 case AArch64::ADDXri:
971 case AArch64::ADDWri:
972 // add x, 1 -> csinc.
973 if (!DefMI->getOperand(i: 2).isImm() || DefMI->getOperand(i: 2).getImm() != 1 ||
974 DefMI->getOperand(i: 3).getImm() != 0)
975 return 0;
976 SrcReg = DefMI->getOperand(i: 1).getReg();
977 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
978 break;
979
980 case AArch64::ORNXrr:
981 case AArch64::ORNWrr: {
982 // not x -> csinv, represented as orn dst, xzr, src.
983 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
984 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
985 return 0;
986 SrcReg = DefMI->getOperand(i: 2).getReg();
987 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
988 break;
989 }
990
991 case AArch64::SUBSXrr:
992 case AArch64::SUBSWrr:
993 // if NZCV is used, do not fold.
994 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
995 isDead: true) == -1)
996 return 0;
997 // fall-through to SUBXrr and SUBWrr.
998 [[fallthrough]];
999 case AArch64::SUBXrr:
1000 case AArch64::SUBWrr: {
1001 // neg x -> csneg, represented as sub dst, xzr, src.
1002 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1003 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1004 return 0;
1005 SrcReg = DefMI->getOperand(i: 2).getReg();
1006 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
1007 break;
1008 }
1009 default:
1010 return 0;
1011 }
1012 assert(Opc && SrcReg && "Missing parameters");
1013
1014 if (NewReg)
1015 *NewReg = SrcReg;
1016 return Opc;
1017}
1018
1019bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1020 ArrayRef<MachineOperand> Cond,
1021 Register DstReg, Register TrueReg,
1022 Register FalseReg, int &CondCycles,
1023 int &TrueCycles,
1024 int &FalseCycles) const {
1025 // Check register classes.
1026 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1027 const TargetRegisterClass *RC =
1028 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1029 if (!RC)
1030 return false;
1031
1032 // Also need to check the dest regclass, in case we're trying to optimize
1033 // something like:
1034 // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
1035 if (!RI.getCommonSubClass(A: RC, B: MRI.getRegClass(Reg: DstReg)))
1036 return false;
1037
1038 // Expanding cbz/tbz requires an extra cycle of latency on the condition.
1039 unsigned ExtraCondLat = Cond.size() != 1;
1040
1041 // GPRs are handled by csel.
1042 // FIXME: Fold in x+1, -x, and ~x when applicable.
1043 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
1044 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1045 // Single-cycle csel, csinc, csinv, and csneg.
1046 CondCycles = 1 + ExtraCondLat;
1047 TrueCycles = FalseCycles = 1;
1048 if (canFoldIntoCSel(MRI, VReg: TrueReg))
1049 TrueCycles = 0;
1050 else if (canFoldIntoCSel(MRI, VReg: FalseReg))
1051 FalseCycles = 0;
1052 return true;
1053 }
1054
1055 // Scalar floating point is handled by fcsel.
1056 // FIXME: Form fabs, fmin, and fmax when applicable.
1057 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
1058 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
1059 CondCycles = 5 + ExtraCondLat;
1060 TrueCycles = FalseCycles = 2;
1061 return true;
1062 }
1063
1064 // Can't do vectors.
1065 return false;
1066}
1067
1068void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
1069 MachineBasicBlock::iterator I,
1070 const DebugLoc &DL, Register DstReg,
1071 ArrayRef<MachineOperand> Cond,
1072 Register TrueReg, Register FalseReg) const {
1073 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1074
1075 // Parse the condition code, see parseCondBranch() above.
1076 AArch64CC::CondCode CC;
1077 switch (Cond.size()) {
1078 default:
1079 llvm_unreachable("Unknown condition opcode in Cond");
1080 case 1: // b.cc
1081 CC = AArch64CC::CondCode(Cond[0].getImm());
1082 break;
1083 case 3: { // cbz/cbnz
1084 // We must insert a compare against 0.
1085 bool Is64Bit;
1086 switch (Cond[1].getImm()) {
1087 default:
1088 llvm_unreachable("Unknown branch opcode in Cond");
1089 case AArch64::CBZW:
1090 Is64Bit = false;
1091 CC = AArch64CC::EQ;
1092 break;
1093 case AArch64::CBZX:
1094 Is64Bit = true;
1095 CC = AArch64CC::EQ;
1096 break;
1097 case AArch64::CBNZW:
1098 Is64Bit = false;
1099 CC = AArch64CC::NE;
1100 break;
1101 case AArch64::CBNZX:
1102 Is64Bit = true;
1103 CC = AArch64CC::NE;
1104 break;
1105 }
1106 Register SrcReg = Cond[2].getReg();
1107 if (Is64Bit) {
1108 // cmp reg, #0 is actually subs xzr, reg, #0.
1109 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64spRegClass);
1110 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::SUBSXri), DestReg: AArch64::XZR)
1111 .addReg(RegNo: SrcReg)
1112 .addImm(Val: 0)
1113 .addImm(Val: 0);
1114 } else {
1115 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32spRegClass);
1116 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWri), DestReg: AArch64::WZR)
1117 .addReg(RegNo: SrcReg)
1118 .addImm(Val: 0)
1119 .addImm(Val: 0);
1120 }
1121 break;
1122 }
1123 case 4: { // tbz/tbnz
1124 // We must insert a tst instruction.
1125 switch (Cond[1].getImm()) {
1126 default:
1127 llvm_unreachable("Unknown branch opcode in Cond");
1128 case AArch64::TBZW:
1129 case AArch64::TBZX:
1130 CC = AArch64CC::EQ;
1131 break;
1132 case AArch64::TBNZW:
1133 case AArch64::TBNZX:
1134 CC = AArch64CC::NE;
1135 break;
1136 }
1137 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
1138 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
1139 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ANDSWri), DestReg: AArch64::WZR)
1140 .addReg(RegNo: Cond[2].getReg())
1141 .addImm(
1142 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 32));
1143 else
1144 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ANDSXri), DestReg: AArch64::XZR)
1145 .addReg(RegNo: Cond[2].getReg())
1146 .addImm(
1147 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 64));
1148 break;
1149 }
1150 case 5: { // cb
1151 // We must insert a cmp, that is a subs
1152 // 0 1 2 3 4
1153 // Cond is { -1, Opcode, CC, Op0, Op1 }
1154
1155 unsigned SubsOpc, SubsDestReg;
1156 bool IsImm = false;
1157 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
1158 switch (Cond[1].getImm()) {
1159 default:
1160 llvm_unreachable("Unknown branch opcode in Cond");
1161 case AArch64::CBWPri:
1162 SubsOpc = AArch64::SUBSWri;
1163 SubsDestReg = AArch64::WZR;
1164 IsImm = true;
1165 break;
1166 case AArch64::CBXPri:
1167 SubsOpc = AArch64::SUBSXri;
1168 SubsDestReg = AArch64::XZR;
1169 IsImm = true;
1170 break;
1171 case AArch64::CBWPrr:
1172 SubsOpc = AArch64::SUBSWrr;
1173 SubsDestReg = AArch64::WZR;
1174 IsImm = false;
1175 break;
1176 case AArch64::CBXPrr:
1177 SubsOpc = AArch64::SUBSXrr;
1178 SubsDestReg = AArch64::XZR;
1179 IsImm = false;
1180 break;
1181 }
1182
1183 if (IsImm)
1184 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
1185 .addReg(RegNo: Cond[3].getReg())
1186 .addImm(Val: Cond[4].getImm())
1187 .addImm(Val: 0);
1188 else
1189 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
1190 .addReg(RegNo: Cond[3].getReg())
1191 .addReg(RegNo: Cond[4].getReg());
1192 } break;
1193 case 7: { // cb[b,h]
1194 // We must insert a cmp, that is a subs, but also zero- or sign-extensions
1195 // that have been folded. For the first operand we codegen an explicit
1196 // extension, for the second operand we fold the extension into cmp.
1197 // 0 1 2 3 4 5 6
1198 // Cond is { -1, Opcode, CC, Op0, Op1, Ext0, Ext1 }
1199
1200 // We need a new register for the now explicitly extended register
1201 Register Reg = Cond[4].getReg();
1202 if (Cond[5].getImm() != AArch64_AM::InvalidShiftExtend) {
1203 unsigned ExtOpc;
1204 unsigned ExtBits;
1205 AArch64_AM::ShiftExtendType ExtendType =
1206 AArch64_AM::getExtendType(Imm: Cond[5].getImm());
1207 switch (ExtendType) {
1208 default:
1209 llvm_unreachable("Unknown shift-extend for CB instruction");
1210 case AArch64_AM::SXTB:
1211 assert(
1212 Cond[1].getImm() == AArch64::CBBAssertExt &&
1213 "Unexpected compare-and-branch instruction for SXTB shift-extend");
1214 ExtOpc = AArch64::SBFMWri;
1215 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1216 break;
1217 case AArch64_AM::SXTH:
1218 assert(
1219 Cond[1].getImm() == AArch64::CBHAssertExt &&
1220 "Unexpected compare-and-branch instruction for SXTH shift-extend");
1221 ExtOpc = AArch64::SBFMWri;
1222 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1223 break;
1224 case AArch64_AM::UXTB:
1225 assert(
1226 Cond[1].getImm() == AArch64::CBBAssertExt &&
1227 "Unexpected compare-and-branch instruction for UXTB shift-extend");
1228 ExtOpc = AArch64::ANDWri;
1229 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1230 break;
1231 case AArch64_AM::UXTH:
1232 assert(
1233 Cond[1].getImm() == AArch64::CBHAssertExt &&
1234 "Unexpected compare-and-branch instruction for UXTH shift-extend");
1235 ExtOpc = AArch64::ANDWri;
1236 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1237 break;
1238 }
1239
1240 // Build the explicit extension of the first operand
1241 Reg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32spRegClass);
1242 MachineInstrBuilder MBBI =
1243 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ExtOpc), DestReg: Reg).addReg(RegNo: Cond[4].getReg());
1244 if (ExtOpc != AArch64::ANDWri)
1245 MBBI.addImm(Val: 0);
1246 MBBI.addImm(Val: ExtBits);
1247 }
1248
1249 // Now, subs with an extended second operand
1250 if (Cond[6].getImm() != AArch64_AM::InvalidShiftExtend) {
1251 AArch64_AM::ShiftExtendType ExtendType =
1252 AArch64_AM::getExtendType(Imm: Cond[6].getImm());
1253 MRI.constrainRegClass(Reg, RC: MRI.getRegClass(Reg: Cond[3].getReg()));
1254 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: &AArch64::GPR32spRegClass);
1255 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrx), DestReg: AArch64::WZR)
1256 .addReg(RegNo: Cond[3].getReg())
1257 .addReg(RegNo: Reg)
1258 .addImm(Val: AArch64_AM::getArithExtendImm(ET: ExtendType, Imm: 0));
1259 } // If no extension is needed, just a regular subs
1260 else {
1261 MRI.constrainRegClass(Reg, RC: MRI.getRegClass(Reg: Cond[3].getReg()));
1262 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: &AArch64::GPR32spRegClass);
1263 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrr), DestReg: AArch64::WZR)
1264 .addReg(RegNo: Cond[3].getReg())
1265 .addReg(RegNo: Reg);
1266 }
1267
1268 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
1269 } break;
1270 }
1271
1272 unsigned Opc = 0;
1273 const TargetRegisterClass *RC = nullptr;
1274 bool TryFold = false;
1275 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass)) {
1276 RC = &AArch64::GPR64RegClass;
1277 Opc = AArch64::CSELXr;
1278 TryFold = true;
1279 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR32RegClass)) {
1280 RC = &AArch64::GPR32RegClass;
1281 Opc = AArch64::CSELWr;
1282 TryFold = true;
1283 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR64RegClass)) {
1284 RC = &AArch64::FPR64RegClass;
1285 Opc = AArch64::FCSELDrrr;
1286 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR32RegClass)) {
1287 RC = &AArch64::FPR32RegClass;
1288 Opc = AArch64::FCSELSrrr;
1289 }
1290 assert(RC && "Unsupported regclass");
1291
1292 // Try folding simple instructions into the csel.
1293 if (TryFold) {
1294 unsigned NewReg = 0;
1295 unsigned FoldedOpc = canFoldIntoCSel(MRI, VReg: TrueReg, NewReg: &NewReg);
1296 if (FoldedOpc) {
1297 // The folded opcodes csinc, csinc and csneg apply the operation to
1298 // FalseReg, so we need to invert the condition.
1299 CC = AArch64CC::getInvertedCondCode(Code: CC);
1300 TrueReg = FalseReg;
1301 } else
1302 FoldedOpc = canFoldIntoCSel(MRI, VReg: FalseReg, NewReg: &NewReg);
1303
1304 // Fold the operation. Leave any dead instructions for DCE to clean up.
1305 if (FoldedOpc) {
1306 FalseReg = NewReg;
1307 Opc = FoldedOpc;
1308 // Extend the live range of NewReg.
1309 MRI.clearKillFlags(Reg: NewReg);
1310 }
1311 }
1312
1313 // Pull all virtual register into the appropriate class.
1314 MRI.constrainRegClass(Reg: TrueReg, RC);
1315 // FalseReg might be WZR or XZR if the folded operand is a literal 1.
1316 assert(
1317 (FalseReg.isVirtual() || FalseReg == AArch64::WZR ||
1318 FalseReg == AArch64::XZR) &&
1319 "FalseReg was folded into a non-virtual register other than WZR or XZR");
1320 if (FalseReg.isVirtual())
1321 MRI.constrainRegClass(Reg: FalseReg, RC);
1322
1323 // Insert the csel.
1324 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
1325 .addReg(RegNo: TrueReg)
1326 .addReg(RegNo: FalseReg)
1327 .addImm(Val: CC);
1328}
1329
1330// Return true if Imm can be loaded into a register by a "cheap" sequence of
1331// instructions. For now, "cheap" means at most two instructions.
1332static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize) {
1333 if (BitSize == 32)
1334 return true;
1335
1336 assert(BitSize == 64 && "Only bit sizes of 32 or 64 allowed");
1337 uint64_t Imm = static_cast<uint64_t>(MI.getOperand(i: 1).getImm());
1338 SmallVector<AArch64_IMM::ImmInsnModel, 4> Is;
1339 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: Is);
1340
1341 return Is.size() <= 2;
1342}
1343
1344// Check if a COPY instruction is cheap.
1345static bool isCheapCopy(const MachineInstr &MI, const AArch64RegisterInfo &RI) {
1346 assert(MI.isCopy() && "Expected COPY instruction");
1347 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1348
1349 // Cross-bank copies (e.g., between GPR and FPR) are expensive on AArch64,
1350 // typically requiring an FMOV instruction with a 2-6 cycle latency.
1351 auto GetRegClass = [&](Register Reg) -> const TargetRegisterClass * {
1352 if (Reg.isVirtual())
1353 return MRI.getRegClass(Reg);
1354 if (Reg.isPhysical())
1355 return RI.getMinimalPhysRegClass(Reg);
1356 return nullptr;
1357 };
1358 const TargetRegisterClass *DstRC = GetRegClass(MI.getOperand(i: 0).getReg());
1359 const TargetRegisterClass *SrcRC = GetRegClass(MI.getOperand(i: 1).getReg());
1360 if (DstRC && SrcRC && !RI.getCommonSubClass(A: DstRC, B: SrcRC))
1361 return false;
1362
1363 return MI.isAsCheapAsAMove();
1364}
1365
1366// FIXME: this implementation should be micro-architecture dependent, so a
1367// micro-architecture target hook should be introduced here in future.
1368bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
1369 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1370 if (isExynosCheapAsMove(MI))
1371 return true;
1372 return MI.isAsCheapAsAMove();
1373 }
1374
1375 switch (MI.getOpcode()) {
1376 default:
1377 return MI.isAsCheapAsAMove();
1378
1379 case TargetOpcode::COPY:
1380 return isCheapCopy(MI, RI);
1381
1382 case AArch64::ADDWrs:
1383 case AArch64::ADDXrs:
1384 case AArch64::SUBWrs:
1385 case AArch64::SUBXrs:
1386 return Subtarget.hasALULSLFast() && MI.getOperand(i: 3).getImm() <= 4;
1387
1388 // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
1389 // ORRXri, it is as cheap as MOV.
1390 // Likewise if it can be expanded to MOVZ/MOVN/MOVK.
1391 case AArch64::MOVi32imm:
1392 return isCheapImmediate(MI, BitSize: 32);
1393 case AArch64::MOVi64imm:
1394 return isCheapImmediate(MI, BitSize: 64);
1395 }
1396}
1397
1398bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
1399 switch (MI.getOpcode()) {
1400 default:
1401 return false;
1402
1403 case AArch64::ADDWrs:
1404 case AArch64::ADDXrs:
1405 case AArch64::ADDSWrs:
1406 case AArch64::ADDSXrs: {
1407 unsigned Imm = MI.getOperand(i: 3).getImm();
1408 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1409 if (ShiftVal == 0)
1410 return true;
1411 return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
1412 }
1413
1414 case AArch64::ADDWrx:
1415 case AArch64::ADDXrx:
1416 case AArch64::ADDXrx64:
1417 case AArch64::ADDSWrx:
1418 case AArch64::ADDSXrx:
1419 case AArch64::ADDSXrx64: {
1420 unsigned Imm = MI.getOperand(i: 3).getImm();
1421 switch (AArch64_AM::getArithExtendType(Imm)) {
1422 default:
1423 return false;
1424 case AArch64_AM::UXTB:
1425 case AArch64_AM::UXTH:
1426 case AArch64_AM::UXTW:
1427 case AArch64_AM::UXTX:
1428 return AArch64_AM::getArithShiftValue(Imm) <= 4;
1429 }
1430 }
1431
1432 case AArch64::SUBWrs:
1433 case AArch64::SUBSWrs: {
1434 unsigned Imm = MI.getOperand(i: 3).getImm();
1435 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1436 return ShiftVal == 0 ||
1437 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
1438 }
1439
1440 case AArch64::SUBXrs:
1441 case AArch64::SUBSXrs: {
1442 unsigned Imm = MI.getOperand(i: 3).getImm();
1443 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1444 return ShiftVal == 0 ||
1445 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
1446 }
1447
1448 case AArch64::SUBWrx:
1449 case AArch64::SUBXrx:
1450 case AArch64::SUBXrx64:
1451 case AArch64::SUBSWrx:
1452 case AArch64::SUBSXrx:
1453 case AArch64::SUBSXrx64: {
1454 unsigned Imm = MI.getOperand(i: 3).getImm();
1455 switch (AArch64_AM::getArithExtendType(Imm)) {
1456 default:
1457 return false;
1458 case AArch64_AM::UXTB:
1459 case AArch64_AM::UXTH:
1460 case AArch64_AM::UXTW:
1461 case AArch64_AM::UXTX:
1462 return AArch64_AM::getArithShiftValue(Imm) == 0;
1463 }
1464 }
1465
1466 case AArch64::LDRBBroW:
1467 case AArch64::LDRBBroX:
1468 case AArch64::LDRBroW:
1469 case AArch64::LDRBroX:
1470 case AArch64::LDRDroW:
1471 case AArch64::LDRDroX:
1472 case AArch64::LDRHHroW:
1473 case AArch64::LDRHHroX:
1474 case AArch64::LDRHroW:
1475 case AArch64::LDRHroX:
1476 case AArch64::LDRQroW:
1477 case AArch64::LDRQroX:
1478 case AArch64::LDRSBWroW:
1479 case AArch64::LDRSBWroX:
1480 case AArch64::LDRSBXroW:
1481 case AArch64::LDRSBXroX:
1482 case AArch64::LDRSHWroW:
1483 case AArch64::LDRSHWroX:
1484 case AArch64::LDRSHXroW:
1485 case AArch64::LDRSHXroX:
1486 case AArch64::LDRSWroW:
1487 case AArch64::LDRSWroX:
1488 case AArch64::LDRSroW:
1489 case AArch64::LDRSroX:
1490 case AArch64::LDRWroW:
1491 case AArch64::LDRWroX:
1492 case AArch64::LDRXroW:
1493 case AArch64::LDRXroX:
1494 case AArch64::PRFMroW:
1495 case AArch64::PRFMroX:
1496 case AArch64::STRBBroW:
1497 case AArch64::STRBBroX:
1498 case AArch64::STRBroW:
1499 case AArch64::STRBroX:
1500 case AArch64::STRDroW:
1501 case AArch64::STRDroX:
1502 case AArch64::STRHHroW:
1503 case AArch64::STRHHroX:
1504 case AArch64::STRHroW:
1505 case AArch64::STRHroX:
1506 case AArch64::STRQroW:
1507 case AArch64::STRQroX:
1508 case AArch64::STRSroW:
1509 case AArch64::STRSroX:
1510 case AArch64::STRWroW:
1511 case AArch64::STRWroX:
1512 case AArch64::STRXroW:
1513 case AArch64::STRXroX: {
1514 unsigned IsSigned = MI.getOperand(i: 3).getImm();
1515 return !IsSigned;
1516 }
1517 }
1518}
1519
1520bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
1521 unsigned Opc = MI.getOpcode();
1522 switch (Opc) {
1523 default:
1524 return false;
1525 case AArch64::SEH_StackAlloc:
1526 case AArch64::SEH_SaveFPLR:
1527 case AArch64::SEH_SaveFPLR_X:
1528 case AArch64::SEH_SaveReg:
1529 case AArch64::SEH_SaveReg_X:
1530 case AArch64::SEH_SaveRegP:
1531 case AArch64::SEH_SaveRegP_X:
1532 case AArch64::SEH_SaveFReg:
1533 case AArch64::SEH_SaveFReg_X:
1534 case AArch64::SEH_SaveFRegP:
1535 case AArch64::SEH_SaveFRegP_X:
1536 case AArch64::SEH_SetFP:
1537 case AArch64::SEH_AddFP:
1538 case AArch64::SEH_Nop:
1539 case AArch64::SEH_PrologEnd:
1540 case AArch64::SEH_EpilogStart:
1541 case AArch64::SEH_EpilogEnd:
1542 case AArch64::SEH_PACSignLR:
1543 case AArch64::SEH_SaveAnyRegI:
1544 case AArch64::SEH_SaveAnyRegIP:
1545 case AArch64::SEH_SaveAnyRegQP:
1546 case AArch64::SEH_SaveAnyRegQPX:
1547 case AArch64::SEH_AllocZ:
1548 case AArch64::SEH_SaveZReg:
1549 case AArch64::SEH_SavePReg:
1550 return true;
1551 }
1552}
1553
1554bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1555 Register &SrcReg, Register &DstReg,
1556 unsigned &SubIdx) const {
1557 switch (MI.getOpcode()) {
1558 default:
1559 return false;
1560 case AArch64::SBFMXri: // aka sxtw
1561 case AArch64::UBFMXri: // aka uxtw
1562 // Check for the 32 -> 64 bit extension case, these instructions can do
1563 // much more.
1564 if (MI.getOperand(i: 2).getImm() != 0 || MI.getOperand(i: 3).getImm() != 31)
1565 return false;
1566 // This is a signed or unsigned 32 -> 64 bit extension.
1567 SrcReg = MI.getOperand(i: 1).getReg();
1568 DstReg = MI.getOperand(i: 0).getReg();
1569 SubIdx = AArch64::sub_32;
1570 return true;
1571 }
1572}
1573
1574bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
1575 const MachineInstr &MIa, const MachineInstr &MIb) const {
1576 const TargetRegisterInfo *TRI = &getRegisterInfo();
1577 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1578 int64_t OffsetA = 0, OffsetB = 0;
1579 TypeSize WidthA(0, false), WidthB(0, false);
1580 bool OffsetAIsScalable = false, OffsetBIsScalable = false;
1581
1582 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1583 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1584
1585 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
1586 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1587 return false;
1588
1589 // Retrieve the base, offset from the base and width. Width
1590 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
1591 // base are identical, and the offset of a lower memory access +
1592 // the width doesn't overlap the offset of a higher memory access,
1593 // then the memory accesses are different.
1594 // If OffsetAIsScalable and OffsetBIsScalable are both true, they
1595 // are assumed to have the same scale (vscale).
1596 if (getMemOperandWithOffsetWidth(MI: MIa, BaseOp&: BaseOpA, Offset&: OffsetA, OffsetIsScalable&: OffsetAIsScalable,
1597 Width&: WidthA, TRI) &&
1598 getMemOperandWithOffsetWidth(MI: MIb, BaseOp&: BaseOpB, Offset&: OffsetB, OffsetIsScalable&: OffsetBIsScalable,
1599 Width&: WidthB, TRI)) {
1600 if (BaseOpA->isIdenticalTo(Other: *BaseOpB) &&
1601 OffsetAIsScalable == OffsetBIsScalable) {
1602 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1603 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1604 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1605 if (LowWidth.isScalable() == OffsetAIsScalable &&
1606 LowOffset + (int)LowWidth.getKnownMinValue() <= HighOffset)
1607 return true;
1608 }
1609 }
1610 return false;
1611}
1612
1613bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1614 const MachineBasicBlock *MBB,
1615 const MachineFunction &MF) const {
1616 if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
1617 return true;
1618
1619 // Do not move an instruction that can be recognized as a branch target.
1620 if (hasBTISemantics(MI))
1621 return true;
1622
1623 switch (MI.getOpcode()) {
1624 case AArch64::HINT:
1625 // CSDB hints are scheduling barriers.
1626 if (MI.getOperand(i: 0).getImm() == 0x14)
1627 return true;
1628 break;
1629 case AArch64::DSB:
1630 case AArch64::ISB:
1631 // DSB and ISB also are scheduling barriers.
1632 return true;
1633 case AArch64::MSRpstatesvcrImm1:
1634 // SMSTART and SMSTOP are also scheduling barriers.
1635 return true;
1636 default:;
1637 }
1638 if (isSEHInstruction(MI))
1639 return true;
1640 auto Next = std::next(x: MI.getIterator());
1641 return Next != MBB->end() && Next->isCFIInstruction();
1642}
1643
1644/// analyzeCompare - For a comparison instruction, return the source registers
1645/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1646/// Return true if the comparison instruction can be analyzed.
1647bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1648 Register &SrcReg2, int64_t &CmpMask,
1649 int64_t &CmpValue) const {
1650 // The first operand can be a frame index where we'd normally expect a
1651 // register.
1652 // FIXME: Pass subregisters out of analyzeCompare
1653 assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1654 if (!MI.getOperand(i: 1).isReg() || MI.getOperand(i: 1).getSubReg())
1655 return false;
1656
1657 switch (MI.getOpcode()) {
1658 default:
1659 break;
1660 case AArch64::PTEST_PP:
1661 case AArch64::PTEST_PP_ANY:
1662 case AArch64::PTEST_PP_FIRST:
1663 SrcReg = MI.getOperand(i: 0).getReg();
1664 SrcReg2 = MI.getOperand(i: 1).getReg();
1665 if (MI.getOperand(i: 2).getSubReg())
1666 return false;
1667
1668 // Not sure about the mask and value for now...
1669 CmpMask = ~0;
1670 CmpValue = 0;
1671 return true;
1672 case AArch64::SUBSWrr:
1673 case AArch64::SUBSWrs:
1674 case AArch64::SUBSWrx:
1675 case AArch64::SUBSXrr:
1676 case AArch64::SUBSXrs:
1677 case AArch64::SUBSXrx:
1678 case AArch64::ADDSWrr:
1679 case AArch64::ADDSWrs:
1680 case AArch64::ADDSWrx:
1681 case AArch64::ADDSXrr:
1682 case AArch64::ADDSXrs:
1683 case AArch64::ADDSXrx:
1684 // Replace SUBSWrr with SUBWrr if NZCV is not used.
1685 SrcReg = MI.getOperand(i: 1).getReg();
1686 SrcReg2 = MI.getOperand(i: 2).getReg();
1687
1688 // FIXME: Pass subregisters out of analyzeCompare
1689 if (MI.getOperand(i: 2).getSubReg())
1690 return false;
1691
1692 CmpMask = ~0;
1693 CmpValue = 0;
1694 return true;
1695 case AArch64::SUBSWri:
1696 case AArch64::ADDSWri:
1697 case AArch64::SUBSXri:
1698 case AArch64::ADDSXri:
1699 SrcReg = MI.getOperand(i: 1).getReg();
1700 SrcReg2 = 0;
1701 CmpMask = ~0;
1702 CmpValue = MI.getOperand(i: 2).getImm();
1703 return true;
1704 case AArch64::ANDSWri:
1705 case AArch64::ANDSXri:
1706 // ANDS does not use the same encoding scheme as the others xxxS
1707 // instructions.
1708 SrcReg = MI.getOperand(i: 1).getReg();
1709 SrcReg2 = 0;
1710 CmpMask = ~0;
1711 CmpValue = AArch64_AM::decodeLogicalImmediate(
1712 val: MI.getOperand(i: 2).getImm(),
1713 regSize: MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1714 return true;
1715 }
1716
1717 return false;
1718}
1719
1720static bool UpdateOperandRegClass(MachineInstr &Instr) {
1721 MachineBasicBlock *MBB = Instr.getParent();
1722 assert(MBB && "Can't get MachineBasicBlock here");
1723 MachineFunction *MF = MBB->getParent();
1724 assert(MF && "Can't get MachineFunction here");
1725 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1726 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1727 MachineRegisterInfo *MRI = &MF->getRegInfo();
1728
1729 for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1730 ++OpIdx) {
1731 MachineOperand &MO = Instr.getOperand(i: OpIdx);
1732 const TargetRegisterClass *OpRegCstraints =
1733 Instr.getRegClassConstraint(OpIdx, TII, TRI);
1734
1735 // If there's no constraint, there's nothing to do.
1736 if (!OpRegCstraints)
1737 continue;
1738 // If the operand is a frame index, there's nothing to do here.
1739 // A frame index operand will resolve correctly during PEI.
1740 if (MO.isFI())
1741 continue;
1742
1743 assert(MO.isReg() &&
1744 "Operand has register constraints without being a register!");
1745
1746 Register Reg = MO.getReg();
1747 if (Reg.isPhysical()) {
1748 if (!OpRegCstraints->contains(Reg))
1749 return false;
1750 } else if (!OpRegCstraints->hasSubClassEq(RC: MRI->getRegClass(Reg)) &&
1751 !MRI->constrainRegClass(Reg, RC: OpRegCstraints))
1752 return false;
1753 }
1754
1755 return true;
1756}
1757
1758/// Return the opcode that does not set flags when possible - otherwise
1759/// return the original opcode. The caller is responsible to do the actual
1760/// substitution and legality checking.
1761static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1762 // Don't convert all compare instructions, because for some the zero register
1763 // encoding becomes the sp register.
1764 bool MIDefinesZeroReg = false;
1765 if (MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
1766 MI.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
1767 MIDefinesZeroReg = true;
1768
1769 switch (MI.getOpcode()) {
1770 default:
1771 return MI.getOpcode();
1772 case AArch64::ADDSWrr:
1773 return AArch64::ADDWrr;
1774 case AArch64::ADDSWri:
1775 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1776 case AArch64::ADDSWrs:
1777 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1778 case AArch64::ADDSWrx:
1779 return AArch64::ADDWrx;
1780 case AArch64::ADDSXrr:
1781 return AArch64::ADDXrr;
1782 case AArch64::ADDSXri:
1783 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1784 case AArch64::ADDSXrs:
1785 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1786 case AArch64::ADDSXrx:
1787 return AArch64::ADDXrx;
1788 case AArch64::SUBSWrr:
1789 return AArch64::SUBWrr;
1790 case AArch64::SUBSWri:
1791 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1792 case AArch64::SUBSWrs:
1793 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1794 case AArch64::SUBSWrx:
1795 return AArch64::SUBWrx;
1796 case AArch64::SUBSXrr:
1797 return AArch64::SUBXrr;
1798 case AArch64::SUBSXri:
1799 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1800 case AArch64::SUBSXrs:
1801 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1802 case AArch64::SUBSXrx:
1803 return AArch64::SUBXrx;
1804 }
1805}
1806
1807enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1808
1809/// True when condition flags are accessed (either by writing or reading)
1810/// on the instruction trace starting at From and ending at To.
1811///
1812/// Note: If From and To are from different blocks it's assumed CC are accessed
1813/// on the path.
1814static bool areCFlagsAccessedBetweenInstrs(
1815 MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1816 const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1817 // Early exit if To is at the beginning of the BB.
1818 if (To == To->getParent()->begin())
1819 return true;
1820
1821 // Check whether the instructions are in the same basic block
1822 // If not, assume the condition flags might get modified somewhere.
1823 if (To->getParent() != From->getParent())
1824 return true;
1825
1826 // From must be above To.
1827 assert(std::any_of(
1828 ++To.getReverse(), To->getParent()->rend(),
1829 [From](MachineInstr &MI) { return MI.getIterator() == From; }));
1830
1831 // We iterate backward starting at \p To until we hit \p From.
1832 for (const MachineInstr &Instr :
1833 instructionsWithoutDebug(It: ++To.getReverse(), End: From.getReverse())) {
1834 if (((AccessToCheck & AK_Write) &&
1835 Instr.modifiesRegister(Reg: AArch64::NZCV, TRI)) ||
1836 ((AccessToCheck & AK_Read) && Instr.readsRegister(Reg: AArch64::NZCV, TRI)))
1837 return true;
1838 }
1839 return false;
1840}
1841
1842std::optional<unsigned>
1843AArch64InstrInfo::canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
1844 MachineInstr *Pred,
1845 const MachineRegisterInfo *MRI) const {
1846 unsigned MaskOpcode = Mask->getOpcode();
1847 unsigned PredOpcode = Pred->getOpcode();
1848 bool PredIsPTestLike = isPTestLikeOpcode(Opc: PredOpcode);
1849 bool PredIsWhileLike = isWhileOpcode(Opc: PredOpcode);
1850
1851 if (PredIsWhileLike) {
1852 // For PTEST(PG, PG), PTEST is redundant when PG is the result of a WHILEcc
1853 // instruction and the condition is "any" since WHILcc does an implicit
1854 // PTEST(ALL, PG) check and PG is always a subset of ALL.
1855 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1856 return PredOpcode;
1857
1858 // For PTEST(PTRUE_ALL, WHILE), if the element size matches, the PTEST is
1859 // redundant since WHILE performs an implicit PTEST with an all active
1860 // mask.
1861 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1862 getElementSizeForOpcode(Opc: MaskOpcode) ==
1863 getElementSizeForOpcode(Opc: PredOpcode))
1864 return PredOpcode;
1865
1866 // For PTEST_FIRST(PTRUE_ALL, WHILE), the PTEST_FIRST is redundant since
1867 // WHILEcc performs an implicit PTEST with an all active mask, setting
1868 // the N flag as the PTEST_FIRST would.
1869 if (PTest->getOpcode() == AArch64::PTEST_PP_FIRST &&
1870 isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31)
1871 return PredOpcode;
1872
1873 return {};
1874 }
1875
1876 if (PredIsPTestLike) {
1877 // For PTEST(PG, PG), PTEST is redundant when PG is the result of an
1878 // instruction that sets the flags as PTEST would and the condition is
1879 // "any" since PG is always a subset of the governing predicate of the
1880 // ptest-like instruction.
1881 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1882 return PredOpcode;
1883
1884 auto PTestLikeMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1885
1886 // If the PTEST like instruction's general predicate is not `Mask`, attempt
1887 // to look through a copy and try again. This is because some instructions
1888 // take a predicate whose register class is a subset of its result class.
1889 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1890 PTestLikeMask->getOperand(i: 1).getReg().isVirtual())
1891 PTestLikeMask =
1892 MRI->getUniqueVRegDef(Reg: PTestLikeMask->getOperand(i: 1).getReg());
1893
1894 // For PTEST(PTRUE_ALL, PTEST_LIKE), the PTEST is redundant if the
1895 // the element size matches and either the PTEST_LIKE instruction uses
1896 // the same all active mask or the condition is "any".
1897 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1898 getElementSizeForOpcode(Opc: MaskOpcode) ==
1899 getElementSizeForOpcode(Opc: PredOpcode)) {
1900 if (Mask == PTestLikeMask || PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1901 return PredOpcode;
1902 }
1903
1904 // For PTEST(PG, PTEST_LIKE(PG, ...)), the PTEST is redundant since the
1905 // flags are set based on the same mask 'PG', but PTEST_LIKE must operate
1906 // on 8-bit predicates like the PTEST. Otherwise, for instructions like
1907 // compare that also support 16/32/64-bit predicates, the implicit PTEST
1908 // performed by the compare could consider fewer lanes for these element
1909 // sizes.
1910 //
1911 // For example, consider
1912 //
1913 // ptrue p0.b ; P0=1111-1111-1111-1111
1914 // index z0.s, #0, #1 ; Z0=<0,1,2,3>
1915 // index z1.s, #1, #1 ; Z1=<1,2,3,4>
1916 // cmphi p1.s, p0/z, z1.s, z0.s ; P1=0001-0001-0001-0001
1917 // ; ^ last active
1918 // ptest p0, p1.b ; P1=0001-0001-0001-0001
1919 // ; ^ last active
1920 //
1921 // where the compare generates a canonical all active 32-bit predicate
1922 // (equivalent to 'ptrue p1.s, all'). The implicit PTEST sets the last
1923 // active flag, whereas the PTEST instruction with the same mask doesn't.
1924 // For PTEST_ANY this doesn't apply as the flags in this case would be
1925 // identical regardless of element size.
1926 uint64_t PredElementSize = getElementSizeForOpcode(Opc: PredOpcode);
1927 if (Mask == PTestLikeMask && (PredElementSize == AArch64::ElementSizeB ||
1928 PTest->getOpcode() == AArch64::PTEST_PP_ANY))
1929 return PredOpcode;
1930
1931 return {};
1932 }
1933
1934 // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the
1935 // opcode so the PTEST becomes redundant.
1936 switch (PredOpcode) {
1937 case AArch64::AND_PPzPP:
1938 case AArch64::BIC_PPzPP:
1939 case AArch64::EOR_PPzPP:
1940 case AArch64::NAND_PPzPP:
1941 case AArch64::NOR_PPzPP:
1942 case AArch64::ORN_PPzPP:
1943 case AArch64::ORR_PPzPP:
1944 case AArch64::BRKA_PPzP:
1945 case AArch64::BRKPA_PPzPP:
1946 case AArch64::BRKB_PPzP:
1947 case AArch64::BRKPB_PPzPP:
1948 case AArch64::RDFFR_PPz: {
1949 // Check to see if our mask is the same. If not the resulting flag bits
1950 // may be different and we can't remove the ptest.
1951 auto *PredMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1952 if (Mask != PredMask)
1953 return {};
1954 break;
1955 }
1956 case AArch64::BRKN_PPzP: {
1957 // BRKN uses an all active implicit mask to set flags unlike the other
1958 // flag-setting instructions.
1959 // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B).
1960 if ((MaskOpcode != AArch64::PTRUE_B) ||
1961 (Mask->getOperand(i: 1).getImm() != 31))
1962 return {};
1963 break;
1964 }
1965 case AArch64::PTRUE_B:
1966 // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A)
1967 break;
1968 default:
1969 // Bail out if we don't recognize the input
1970 return {};
1971 }
1972
1973 return convertToFlagSettingOpc(Opc: PredOpcode);
1974}
1975
1976/// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
1977/// operation which could set the flags in an identical manner
1978bool AArch64InstrInfo::optimizePTestInstr(
1979 MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
1980 const MachineRegisterInfo *MRI) const {
1981 auto *Mask = MRI->getUniqueVRegDef(Reg: MaskReg);
1982 auto *Pred = MRI->getUniqueVRegDef(Reg: PredReg);
1983
1984 if (Pred->isCopy() && PTest->getOpcode() == AArch64::PTEST_PP_FIRST) {
1985 // Instructions which return a multi-vector (e.g. WHILECC_x2) require copies
1986 // before the branch to extract each subregister.
1987 auto Op = Pred->getOperand(i: 1);
1988 if (Op.isReg() && Op.getReg().isVirtual() &&
1989 Op.getSubReg() == AArch64::psub0)
1990 Pred = MRI->getUniqueVRegDef(Reg: Op.getReg());
1991 }
1992
1993 unsigned PredOpcode = Pred->getOpcode();
1994 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred, MRI);
1995 if (!NewOp)
1996 return false;
1997
1998 const TargetRegisterInfo *TRI = &getRegisterInfo();
1999
2000 // If another instruction between Pred and PTest accesses flags, don't remove
2001 // the ptest or update the earlier instruction to modify them.
2002 if (areCFlagsAccessedBetweenInstrs(From: Pred, To: PTest, TRI))
2003 return false;
2004
2005 // If we pass all the checks, it's safe to remove the PTEST and use the flags
2006 // as they are prior to PTEST. Sometimes this requires the tested PTEST
2007 // operand to be replaced with an equivalent instruction that also sets the
2008 // flags.
2009 PTest->eraseFromParent();
2010 if (*NewOp != PredOpcode) {
2011 Pred->setDesc(get(Opcode: *NewOp));
2012 bool succeeded = UpdateOperandRegClass(Instr&: *Pred);
2013 (void)succeeded;
2014 assert(succeeded && "Operands have incompatible register classes!");
2015 Pred->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: TRI);
2016 }
2017
2018 // Ensure that the flags def is live.
2019 if (Pred->registerDefIsDead(Reg: AArch64::NZCV, TRI)) {
2020 unsigned i = 0, e = Pred->getNumOperands();
2021 for (; i != e; ++i) {
2022 MachineOperand &MO = Pred->getOperand(i);
2023 if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::NZCV) {
2024 MO.setIsDead(false);
2025 break;
2026 }
2027 }
2028 }
2029 return true;
2030}
2031
2032/// Try to optimize a compare instruction. A compare instruction is an
2033/// instruction which produces AArch64::NZCV. It can be truly compare
2034/// instruction
2035/// when there are no uses of its destination register.
2036///
2037/// The following steps are tried in order:
2038/// 1. Convert CmpInstr into an unconditional version.
2039/// 2. Remove CmpInstr if above there is an instruction producing a needed
2040/// condition code or an instruction which can be converted into such an
2041/// instruction.
2042/// Only comparison with zero is supported.
2043bool AArch64InstrInfo::optimizeCompareInstr(
2044 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2045 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2046 assert(CmpInstr.getParent());
2047 assert(MRI);
2048
2049 // Replace SUBSWrr with SUBWrr if NZCV is not used.
2050 int DeadNZCVIdx =
2051 CmpInstr.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
2052 if (DeadNZCVIdx != -1) {
2053 if (CmpInstr.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
2054 CmpInstr.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr)) {
2055 CmpInstr.eraseFromParent();
2056 return true;
2057 }
2058 unsigned Opc = CmpInstr.getOpcode();
2059 unsigned NewOpc = convertToNonFlagSettingOpc(MI: CmpInstr);
2060 if (NewOpc == Opc)
2061 return false;
2062 const MCInstrDesc &MCID = get(Opcode: NewOpc);
2063 CmpInstr.setDesc(MCID);
2064 CmpInstr.removeOperand(OpNo: DeadNZCVIdx);
2065 bool succeeded = UpdateOperandRegClass(Instr&: CmpInstr);
2066 (void)succeeded;
2067 assert(succeeded && "Some operands reg class are incompatible!");
2068 return true;
2069 }
2070
2071 if (CmpInstr.getOpcode() == AArch64::PTEST_PP ||
2072 CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY ||
2073 CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST)
2074 return optimizePTestInstr(PTest: &CmpInstr, MaskReg: SrcReg, PredReg: SrcReg2, MRI);
2075
2076 if (SrcReg2 != 0)
2077 return false;
2078
2079 // CmpInstr is a Compare instruction if destination register is not used.
2080 if (!MRI->use_nodbg_empty(RegNo: CmpInstr.getOperand(i: 0).getReg()))
2081 return false;
2082
2083 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, MRI: *MRI))
2084 return true;
2085 return (CmpValue == 0 || CmpValue == 1) &&
2086 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, MRI: *MRI);
2087}
2088
2089/// Get opcode of S version of Instr.
2090/// If Instr is S version its opcode is returned.
2091/// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
2092/// or we are not interested in it.
2093static unsigned sForm(MachineInstr &Instr) {
2094 switch (Instr.getOpcode()) {
2095 default:
2096 return AArch64::INSTRUCTION_LIST_END;
2097
2098 case AArch64::ADDSWrr:
2099 case AArch64::ADDSWri:
2100 case AArch64::ADDSXrr:
2101 case AArch64::ADDSXri:
2102 case AArch64::ADDSWrx:
2103 case AArch64::ADDSXrx:
2104 case AArch64::ADDSWrs:
2105 case AArch64::ADDSXrs:
2106 case AArch64::SUBSWrr:
2107 case AArch64::SUBSWri:
2108 case AArch64::SUBSWrx:
2109 case AArch64::SUBSWrs:
2110 case AArch64::SUBSXrr:
2111 case AArch64::SUBSXri:
2112 case AArch64::SUBSXrx:
2113 case AArch64::SUBSXrs:
2114 case AArch64::ANDSWri:
2115 case AArch64::ANDSWrr:
2116 case AArch64::ANDSWrs:
2117 case AArch64::ANDSXri:
2118 case AArch64::ANDSXrr:
2119 case AArch64::ANDSXrs:
2120 case AArch64::BICSWrr:
2121 case AArch64::BICSXrr:
2122 case AArch64::BICSWrs:
2123 case AArch64::BICSXrs:
2124 case AArch64::ADCSWr:
2125 case AArch64::ADCSXr:
2126 case AArch64::SBCSWr:
2127 case AArch64::SBCSXr:
2128 return Instr.getOpcode();
2129
2130 case AArch64::ADDWrr:
2131 return AArch64::ADDSWrr;
2132 case AArch64::ADDWri:
2133 return AArch64::ADDSWri;
2134 case AArch64::ADDXrr:
2135 return AArch64::ADDSXrr;
2136 case AArch64::ADDXri:
2137 return AArch64::ADDSXri;
2138 case AArch64::ADDWrx:
2139 return AArch64::ADDSWrx;
2140 case AArch64::ADDXrx:
2141 return AArch64::ADDSXrx;
2142 case AArch64::ADDWrs:
2143 return AArch64::ADDSWrs;
2144 case AArch64::ADDXrs:
2145 return AArch64::ADDSXrs;
2146 case AArch64::ADCWr:
2147 return AArch64::ADCSWr;
2148 case AArch64::ADCXr:
2149 return AArch64::ADCSXr;
2150 case AArch64::SUBWrr:
2151 return AArch64::SUBSWrr;
2152 case AArch64::SUBWri:
2153 return AArch64::SUBSWri;
2154 case AArch64::SUBXrr:
2155 return AArch64::SUBSXrr;
2156 case AArch64::SUBXri:
2157 return AArch64::SUBSXri;
2158 case AArch64::SUBWrx:
2159 return AArch64::SUBSWrx;
2160 case AArch64::SUBXrx:
2161 return AArch64::SUBSXrx;
2162 case AArch64::SUBWrs:
2163 return AArch64::SUBSWrs;
2164 case AArch64::SUBXrs:
2165 return AArch64::SUBSXrs;
2166 case AArch64::SBCWr:
2167 return AArch64::SBCSWr;
2168 case AArch64::SBCXr:
2169 return AArch64::SBCSXr;
2170 case AArch64::ANDWri:
2171 return AArch64::ANDSWri;
2172 case AArch64::ANDXri:
2173 return AArch64::ANDSXri;
2174 case AArch64::ANDWrr:
2175 return AArch64::ANDSWrr;
2176 case AArch64::ANDWrs:
2177 return AArch64::ANDSWrs;
2178 case AArch64::ANDXrr:
2179 return AArch64::ANDSXrr;
2180 case AArch64::ANDXrs:
2181 return AArch64::ANDSXrs;
2182 case AArch64::BICWrr:
2183 return AArch64::BICSWrr;
2184 case AArch64::BICXrr:
2185 return AArch64::BICSXrr;
2186 case AArch64::BICWrs:
2187 return AArch64::BICSWrs;
2188 case AArch64::BICXrs:
2189 return AArch64::BICSXrs;
2190 }
2191}
2192
2193/// Check if AArch64::NZCV should be alive in successors of MBB.
2194static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB) {
2195 for (auto *BB : MBB->successors())
2196 if (BB->isLiveIn(Reg: AArch64::NZCV))
2197 return true;
2198 return false;
2199}
2200
2201/// \returns The condition code operand index for \p Instr if it is a branch
2202/// or select and -1 otherwise.
2203int AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(
2204 const MachineInstr &Instr) {
2205 switch (Instr.getOpcode()) {
2206 default:
2207 return -1;
2208
2209 case AArch64::Bcc: {
2210 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2211 assert(Idx >= 2);
2212 return Idx - 2;
2213 }
2214
2215 case AArch64::CSINVWr:
2216 case AArch64::CSINVXr:
2217 case AArch64::CSINCWr:
2218 case AArch64::CSINCXr:
2219 case AArch64::CSELWr:
2220 case AArch64::CSELXr:
2221 case AArch64::CSNEGWr:
2222 case AArch64::CSNEGXr:
2223 case AArch64::FCSELSrrr:
2224 case AArch64::FCSELDrrr: {
2225 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2226 assert(Idx >= 1);
2227 return Idx - 1;
2228 }
2229 }
2230}
2231
2232/// Find a condition code used by the instruction.
2233/// Returns AArch64CC::Invalid if either the instruction does not use condition
2234/// codes or we don't optimize CmpInstr in the presence of such instructions.
2235static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
2236 int CCIdx =
2237 AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(Instr);
2238 return CCIdx >= 0 ? static_cast<AArch64CC::CondCode>(
2239 Instr.getOperand(i: CCIdx).getImm())
2240 : AArch64CC::Invalid;
2241}
2242
2243static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
2244 assert(CC != AArch64CC::Invalid);
2245 UsedNZCV UsedFlags;
2246 switch (CC) {
2247 default:
2248 break;
2249
2250 case AArch64CC::EQ: // Z set
2251 case AArch64CC::NE: // Z clear
2252 UsedFlags.Z = true;
2253 break;
2254
2255 case AArch64CC::HI: // Z clear and C set
2256 case AArch64CC::LS: // Z set or C clear
2257 UsedFlags.Z = true;
2258 [[fallthrough]];
2259 case AArch64CC::HS: // C set
2260 case AArch64CC::LO: // C clear
2261 UsedFlags.C = true;
2262 break;
2263
2264 case AArch64CC::MI: // N set
2265 case AArch64CC::PL: // N clear
2266 UsedFlags.N = true;
2267 break;
2268
2269 case AArch64CC::VS: // V set
2270 case AArch64CC::VC: // V clear
2271 UsedFlags.V = true;
2272 break;
2273
2274 case AArch64CC::GT: // Z clear, N and V the same
2275 case AArch64CC::LE: // Z set, N and V differ
2276 UsedFlags.Z = true;
2277 [[fallthrough]];
2278 case AArch64CC::GE: // N and V the same
2279 case AArch64CC::LT: // N and V differ
2280 UsedFlags.N = true;
2281 UsedFlags.V = true;
2282 break;
2283 }
2284 return UsedFlags;
2285}
2286
2287/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
2288/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
2289/// \returns std::nullopt otherwise.
2290///
2291/// Collect instructions using that flags in \p CCUseInstrs if provided.
2292std::optional<UsedNZCV>
2293llvm::examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
2294 const TargetRegisterInfo &TRI,
2295 SmallVectorImpl<MachineInstr *> *CCUseInstrs) {
2296 MachineBasicBlock *CmpParent = CmpInstr.getParent();
2297 if (MI.getParent() != CmpParent)
2298 return std::nullopt;
2299
2300 if (areCFlagsAliveInSuccessors(MBB: CmpParent))
2301 return std::nullopt;
2302
2303 UsedNZCV NZCVUsedAfterCmp;
2304 for (MachineInstr &Instr : instructionsWithoutDebug(
2305 It: std::next(x: CmpInstr.getIterator()), End: CmpParent->instr_end())) {
2306 if (Instr.readsRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
2307 AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
2308 if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
2309 return std::nullopt;
2310 NZCVUsedAfterCmp |= getUsedNZCV(CC);
2311 if (CCUseInstrs)
2312 CCUseInstrs->push_back(Elt: &Instr);
2313 }
2314 if (Instr.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI))
2315 break;
2316 }
2317 return NZCVUsedAfterCmp;
2318}
2319
2320static bool isADDSRegImm(unsigned Opcode) {
2321 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
2322}
2323
2324static bool isSUBSRegImm(unsigned Opcode) {
2325 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
2326}
2327
2328static bool isANDOpcode(MachineInstr &MI) {
2329 unsigned Opc = sForm(Instr&: MI);
2330 switch (Opc) {
2331 case AArch64::ANDSWri:
2332 case AArch64::ANDSWrr:
2333 case AArch64::ANDSWrs:
2334 case AArch64::ANDSXri:
2335 case AArch64::ANDSXrr:
2336 case AArch64::ANDSXrs:
2337 case AArch64::BICSWrr:
2338 case AArch64::BICSXrr:
2339 case AArch64::BICSWrs:
2340 case AArch64::BICSXrs:
2341 return true;
2342 default:
2343 return false;
2344 }
2345}
2346
2347/// Check if CmpInstr can be substituted by MI.
2348///
2349/// CmpInstr can be substituted:
2350/// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2351/// - and, MI and CmpInstr are from the same MachineBB
2352/// - and, condition flags are not alive in successors of the CmpInstr parent
2353/// - and, if MI opcode is the S form there must be no defs of flags between
2354/// MI and CmpInstr
2355/// or if MI opcode is not the S form there must be neither defs of flags
2356/// nor uses of flags between MI and CmpInstr.
2357/// - and, C is not used after CmpInstr; CmpInstr's C is from adds/subs #0 on
2358/// SrcReg and can differ from MI (e.g. carry out of ADCS/SBCS).
2359/// - and, V is not used after CmpInstr unless MI is AND/BIC (V cleared) or MI
2360/// has NoSWrap (overflow is poison and the fold is still safe).
2361static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr,
2362 const TargetRegisterInfo &TRI) {
2363 // MI is an opcode sForm maps (add/sub/adc/sbc/and/bic and their S forms).
2364 assert(sForm(MI) != AArch64::INSTRUCTION_LIST_END);
2365
2366 const unsigned CmpOpcode = CmpInstr.getOpcode();
2367 if (!isADDSRegImm(Opcode: CmpOpcode) && !isSUBSRegImm(Opcode: CmpOpcode))
2368 return false;
2369
2370 assert((CmpInstr.getOperand(2).isImm() &&
2371 CmpInstr.getOperand(2).getImm() == 0) &&
2372 "Caller guarantees that CmpInstr compares with constant 0");
2373
2374 std::optional<UsedNZCV> NZVCUsed = examineCFlagsUse(MI, CmpInstr, TRI);
2375 if (!NZVCUsed || NZVCUsed->C)
2376 return false;
2377
2378 // CmpInstr is ADDS/SUBS with immediate 0 on SrcReg (compare SrcReg to zero).
2379 // After the fold, users see NZCV from MI (or its S form), not from CmpInstr.
2380 // N/Z match CmpInstr for the value in SrcReg; C/V need not match in general
2381 // (e.g. ADCS vs adds #0), so we require C unused after CmpInstr and gate V
2382 // as below. NoSWrap makes signed overflow poison; AND/BIC clear V.
2383 if (NZVCUsed->V && !MI.getFlag(Flag: MachineInstr::NoSWrap) && !isANDOpcode(MI))
2384 return false;
2385
2386 AccessKind AccessToCheck = AK_Write;
2387 if (sForm(Instr&: MI) != MI.getOpcode())
2388 AccessToCheck = AK_All;
2389 return !areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck);
2390}
2391
2392/// Substitute an instruction comparing to zero with another instruction
2393/// which produces needed condition flags.
2394///
2395/// Return true on success.
2396bool AArch64InstrInfo::substituteCmpToZero(
2397 MachineInstr &CmpInstr, unsigned SrcReg,
2398 const MachineRegisterInfo &MRI) const {
2399 // Get the unique definition of SrcReg.
2400 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2401 if (!MI)
2402 return false;
2403
2404 const TargetRegisterInfo &TRI = getRegisterInfo();
2405
2406 unsigned NewOpc = sForm(Instr&: *MI);
2407 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
2408 return false;
2409
2410 if (!canInstrSubstituteCmpInstr(MI&: *MI, CmpInstr, TRI))
2411 return false;
2412
2413 // Update the instruction to set NZCV.
2414 MI->setDesc(get(Opcode: NewOpc));
2415 CmpInstr.eraseFromParent();
2416 bool succeeded = UpdateOperandRegClass(Instr&: *MI);
2417 (void)succeeded;
2418 assert(succeeded && "Some operands reg class are incompatible!");
2419 MI->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: &TRI);
2420 return true;
2421}
2422
2423/// \returns True if \p CmpInstr can be removed.
2424///
2425/// \p IsInvertCC is true if, after removing \p CmpInstr, condition
2426/// codes used in \p CCUseInstrs must be inverted.
2427static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr,
2428 int CmpValue, const TargetRegisterInfo &TRI,
2429 SmallVectorImpl<MachineInstr *> &CCUseInstrs,
2430 bool &IsInvertCC) {
2431 assert((CmpValue == 0 || CmpValue == 1) &&
2432 "Only comparisons to 0 or 1 considered for removal!");
2433
2434 // MI is 'CSINCWr %vreg, wzr, wzr, <cc>' or 'CSINCXr %vreg, xzr, xzr, <cc>'
2435 unsigned MIOpc = MI.getOpcode();
2436 if (MIOpc == AArch64::CSINCWr) {
2437 if (MI.getOperand(i: 1).getReg() != AArch64::WZR ||
2438 MI.getOperand(i: 2).getReg() != AArch64::WZR)
2439 return false;
2440 } else if (MIOpc == AArch64::CSINCXr) {
2441 if (MI.getOperand(i: 1).getReg() != AArch64::XZR ||
2442 MI.getOperand(i: 2).getReg() != AArch64::XZR)
2443 return false;
2444 } else {
2445 return false;
2446 }
2447 AArch64CC::CondCode MICC = findCondCodeUsedByInstr(Instr: MI);
2448 if (MICC == AArch64CC::Invalid)
2449 return false;
2450
2451 // NZCV needs to be defined
2452 if (MI.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) != -1)
2453 return false;
2454
2455 // CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0' or 'SUBS %vreg, 1'
2456 const unsigned CmpOpcode = CmpInstr.getOpcode();
2457 bool IsSubsRegImm = isSUBSRegImm(Opcode: CmpOpcode);
2458 if (CmpValue && !IsSubsRegImm)
2459 return false;
2460 if (!CmpValue && !IsSubsRegImm && !isADDSRegImm(Opcode: CmpOpcode))
2461 return false;
2462
2463 // MI conditions allowed: eq, ne, mi, pl
2464 UsedNZCV MIUsedNZCV = getUsedNZCV(CC: MICC);
2465 if (MIUsedNZCV.C || MIUsedNZCV.V)
2466 return false;
2467
2468 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2469 examineCFlagsUse(MI, CmpInstr, TRI, CCUseInstrs: &CCUseInstrs);
2470 // Condition flags are not used in CmpInstr basic block successors and only
2471 // Z or N flags allowed to be used after CmpInstr within its basic block
2472 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2473 return false;
2474 // Z or N flag used after CmpInstr must correspond to the flag used in MI
2475 if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) ||
2476 (MIUsedNZCV.N && NZCVUsedAfterCmp->Z))
2477 return false;
2478 // If CmpInstr is comparison to zero MI conditions are limited to eq, ne
2479 if (MIUsedNZCV.N && !CmpValue)
2480 return false;
2481
2482 // There must be no defs of flags between MI and CmpInstr
2483 if (areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck: AK_Write))
2484 return false;
2485
2486 // Condition code is inverted in the following cases:
2487 // 1. MI condition is ne; CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2488 // 2. MI condition is eq, pl; CmpInstr is 'SUBS %vreg, 1'
2489 IsInvertCC = (CmpValue && (MICC == AArch64CC::EQ || MICC == AArch64CC::PL)) ||
2490 (!CmpValue && MICC == AArch64CC::NE);
2491 return true;
2492}
2493
2494/// Remove comparison in csinc-cmp sequence
2495///
2496/// Examples:
2497/// 1. \code
2498/// csinc w9, wzr, wzr, ne
2499/// cmp w9, #0
2500/// b.eq
2501/// \endcode
2502/// to
2503/// \code
2504/// csinc w9, wzr, wzr, ne
2505/// b.ne
2506/// \endcode
2507///
2508/// 2. \code
2509/// csinc x2, xzr, xzr, mi
2510/// cmp x2, #1
2511/// b.pl
2512/// \endcode
2513/// to
2514/// \code
2515/// csinc x2, xzr, xzr, mi
2516/// b.pl
2517/// \endcode
2518///
2519/// \param CmpInstr comparison instruction
2520/// \return True when comparison removed
2521bool AArch64InstrInfo::removeCmpToZeroOrOne(
2522 MachineInstr &CmpInstr, unsigned SrcReg, int CmpValue,
2523 const MachineRegisterInfo &MRI) const {
2524 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2525 if (!MI)
2526 return false;
2527 const TargetRegisterInfo &TRI = getRegisterInfo();
2528 SmallVector<MachineInstr *, 4> CCUseInstrs;
2529 bool IsInvertCC = false;
2530 if (!canCmpInstrBeRemoved(MI&: *MI, CmpInstr, CmpValue, TRI, CCUseInstrs,
2531 IsInvertCC))
2532 return false;
2533 // Make transformation
2534 CmpInstr.eraseFromParent();
2535 if (IsInvertCC) {
2536 // Invert condition codes in CmpInstr CC users
2537 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2538 int Idx = findCondCodeUseOperandIdxForBranchOrSelect(Instr: *CCUseInstr);
2539 assert(Idx >= 0 && "Unexpected instruction using CC.");
2540 MachineOperand &CCOperand = CCUseInstr->getOperand(i: Idx);
2541 AArch64CC::CondCode CCUse = AArch64CC::getInvertedCondCode(
2542 Code: static_cast<AArch64CC::CondCode>(CCOperand.getImm()));
2543 CCOperand.setImm(CCUse);
2544 }
2545 }
2546 return true;
2547}
2548
2549bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2550 if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2551 MI.getOpcode() != AArch64::CATCHRET &&
2552 MI.getOpcode() != AArch64::STACK_GUARD_UNMIX)
2553 return false;
2554
2555 MachineBasicBlock &MBB = *MI.getParent();
2556 auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
2557 auto TRI = Subtarget.getRegisterInfo();
2558 DebugLoc DL = MI.getDebugLoc();
2559
2560 if (MI.getOpcode() == AArch64::STACK_GUARD_UNMIX) {
2561 // Expand STACK_GUARD_UNMIX to: sub Rd, fp, Rs
2562 // This computes FP - stored_mixed_value to unmix the cookie
2563 Register DstReg = MI.getOperand(i: 0).getReg();
2564 Register SrcReg = MI.getOperand(i: 1).getReg();
2565
2566 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: DstReg)
2567 .addReg(RegNo: AArch64::FP)
2568 .addReg(RegNo: SrcReg);
2569
2570 MBB.erase(I: MI);
2571 return true;
2572 }
2573
2574 if (MI.getOpcode() == AArch64::CATCHRET) {
2575 // Skip to the first instruction before the epilog.
2576 const TargetInstrInfo *TII =
2577 MBB.getParent()->getSubtarget().getInstrInfo();
2578 MachineBasicBlock *TargetMBB = MI.getOperand(i: 0).getMBB();
2579 auto MBBI = MachineBasicBlock::iterator(MI);
2580 MachineBasicBlock::iterator FirstEpilogSEH = std::prev(x: MBBI);
2581 while (FirstEpilogSEH->getFlag(Flag: MachineInstr::FrameDestroy) &&
2582 FirstEpilogSEH != MBB.begin())
2583 FirstEpilogSEH = std::prev(x: FirstEpilogSEH);
2584 if (FirstEpilogSEH != MBB.begin())
2585 FirstEpilogSEH = std::next(x: FirstEpilogSEH);
2586 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADRP))
2587 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2588 .addMBB(MBB: TargetMBB);
2589 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri))
2590 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2591 .addReg(RegNo: AArch64::X0)
2592 .addMBB(MBB: TargetMBB)
2593 .addImm(Val: 0);
2594 TargetMBB->setMachineBlockAddressTaken();
2595 return true;
2596 }
2597
2598 Register Reg = MI.getOperand(i: 0).getReg();
2599 Module &M = *MBB.getParent()->getFunction().getParent();
2600 if (M.getStackProtectorGuard() == "sysreg") {
2601 const AArch64SysReg::SysReg *SrcReg =
2602 AArch64SysReg::lookupSysRegByName(Name: M.getStackProtectorGuardReg());
2603 if (!SrcReg)
2604 report_fatal_error(reason: "Unknown SysReg for Stack Protector Guard Register");
2605
2606 // mrs xN, sysreg
2607 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MRS))
2608 .addDef(RegNo: Reg, Flags: RegState::Renamable)
2609 .addImm(Val: SrcReg->Encoding);
2610 int Offset = M.getStackProtectorGuardOffset();
2611 if (Offset >= 0 && Offset <= 32760 && Offset % 8 == 0) {
2612 // ldr xN, [xN, #offset]
2613 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2614 .addDef(RegNo: Reg)
2615 .addUse(RegNo: Reg, Flags: RegState::Kill)
2616 .addImm(Val: Offset / 8);
2617 } else if (Offset >= -256 && Offset <= 255) {
2618 // ldur xN, [xN, #offset]
2619 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDURXi))
2620 .addDef(RegNo: Reg)
2621 .addUse(RegNo: Reg, Flags: RegState::Kill)
2622 .addImm(Val: Offset);
2623 } else if (Offset >= -4095 && Offset <= 4095) {
2624 if (Offset > 0) {
2625 // add xN, xN, #offset
2626 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri))
2627 .addDef(RegNo: Reg)
2628 .addUse(RegNo: Reg, Flags: RegState::Kill)
2629 .addImm(Val: Offset)
2630 .addImm(Val: 0);
2631 } else {
2632 // sub xN, xN, #offset
2633 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXri))
2634 .addDef(RegNo: Reg)
2635 .addUse(RegNo: Reg, Flags: RegState::Kill)
2636 .addImm(Val: -Offset)
2637 .addImm(Val: 0);
2638 }
2639 // ldr xN, [xN]
2640 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2641 .addDef(RegNo: Reg)
2642 .addUse(RegNo: Reg, Flags: RegState::Kill)
2643 .addImm(Val: 0);
2644 } else {
2645 // Cases that are larger than +/- 4095 and not a multiple of 8, or larger
2646 // than 23760.
2647 // It might be nice to use AArch64::MOVi32imm here, which would get
2648 // expanded in PreSched2 after PostRA, but our lone scratch Reg already
2649 // contains the MRS result. findScratchNonCalleeSaveRegister() in
2650 // AArch64FrameLowering might help us find such a scratch register
2651 // though. If we failed to find a scratch register, we could emit a
2652 // stream of add instructions to build up the immediate. Or, we could try
2653 // to insert a AArch64::MOVi32imm before register allocation so that we
2654 // didn't need to scavenge for a scratch register.
2655 report_fatal_error(reason: "Unable to encode Stack Protector Guard Offset");
2656 }
2657 MBB.erase(I: MI);
2658 return true;
2659 }
2660
2661 const GlobalValue *GV =
2662 cast<GlobalValue>(Val: (*MI.memoperands_begin())->getValue());
2663 const TargetMachine &TM = MBB.getParent()->getTarget();
2664 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2665 const unsigned char MO_NC = AArch64II::MO_NC;
2666
2667 unsigned GuardWidth = M.getStackProtectorGuardValueWidth().value_or(
2668 u: Subtarget.isTargetILP32() ? 4 : 8);
2669 if (GuardWidth != 4 && GuardWidth != 8)
2670 report_fatal_error(reason: "Unsupported stack protector value width");
2671 if ((OpFlags & AArch64II::MO_GOT) != 0) {
2672 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LOADgot), DestReg: Reg)
2673 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags);
2674 if (GuardWidth == 4) {
2675 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2676 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2677 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2678 .addUse(RegNo: Reg, Flags: RegState::Kill)
2679 .addImm(Val: 0)
2680 .addMemOperand(MMO: *MI.memoperands_begin())
2681 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2682 } else {
2683 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2684 .addReg(RegNo: Reg, Flags: RegState::Kill)
2685 .addImm(Val: 0)
2686 .addMemOperand(MMO: *MI.memoperands_begin());
2687 }
2688 } else if (TM.getCodeModel() == CodeModel::Large) {
2689 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg)
2690 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G0 | MO_NC)
2691 .addImm(Val: 0);
2692 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2693 .addReg(RegNo: Reg, Flags: RegState::Kill)
2694 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G1 | MO_NC)
2695 .addImm(Val: 16);
2696 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2697 .addReg(RegNo: Reg, Flags: RegState::Kill)
2698 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G2 | MO_NC)
2699 .addImm(Val: 32);
2700 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2701 .addReg(RegNo: Reg, Flags: RegState::Kill)
2702 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G3)
2703 .addImm(Val: 48);
2704 if (GuardWidth == 4) {
2705 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2706 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2707 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2708 .addUse(RegNo: Reg, Flags: RegState::Kill)
2709 .addImm(Val: 0)
2710 .addMemOperand(MMO: *MI.memoperands_begin())
2711 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2712 } else {
2713 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2714 .addReg(RegNo: Reg, Flags: RegState::Kill)
2715 .addImm(Val: 0)
2716 .addMemOperand(MMO: *MI.memoperands_begin());
2717 }
2718 } else {
2719 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
2720 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags | AArch64II::MO_PAGE);
2721 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
2722 if (GuardWidth == 4) {
2723 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2724 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2725 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2726 .addUse(RegNo: Reg, Flags: RegState::Kill)
2727 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2728 .addMemOperand(MMO: *MI.memoperands_begin())
2729 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2730 } else {
2731 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2732 .addReg(RegNo: Reg, Flags: RegState::Kill)
2733 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2734 .addMemOperand(MMO: *MI.memoperands_begin());
2735 }
2736 }
2737 // To match MSVC. Unlike x86_64 which uses xor instruction to mix the cookie,
2738 // we use sub instruction to mix the cookie on aarch64.
2739 // The mixing happens here in expandPostRAPseudo (after RA) to ensure we use
2740 // the final frame pointer value.
2741 if (Subtarget.getTargetTriple().isOSMSVCRT())
2742 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: Reg)
2743 .addReg(RegNo: AArch64::FP)
2744 .addReg(RegNo: Reg, Flags: RegState::Kill);
2745
2746 MBB.erase(I: MI);
2747
2748 return true;
2749}
2750
2751// Return true if this instruction simply sets its single destination register
2752// to zero. This is equivalent to a register rename of the zero-register.
2753bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
2754 switch (MI.getOpcode()) {
2755 default:
2756 break;
2757 case AArch64::MOVZWi:
2758 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
2759 if (MI.getOperand(i: 1).isImm() && MI.getOperand(i: 1).getImm() == 0) {
2760 assert(MI.getDesc().getNumOperands() == 3 &&
2761 MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
2762 return true;
2763 }
2764 break;
2765 case AArch64::ANDWri: // and Rd, Rzr, #imm
2766 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2767 case AArch64::ANDXri:
2768 return MI.getOperand(i: 1).getReg() == AArch64::XZR;
2769 case TargetOpcode::COPY:
2770 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2771 }
2772 return false;
2773}
2774
2775// Return true if this instruction simply renames a general register without
2776// modifying bits.
2777bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
2778 switch (MI.getOpcode()) {
2779 default:
2780 break;
2781 case TargetOpcode::COPY: {
2782 // GPR32 copies will by lowered to ORRXrs
2783 Register DstReg = MI.getOperand(i: 0).getReg();
2784 return (AArch64::GPR32RegClass.contains(Reg: DstReg) ||
2785 AArch64::GPR64RegClass.contains(Reg: DstReg));
2786 }
2787 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
2788 if (MI.getOperand(i: 1).getReg() == AArch64::XZR) {
2789 assert(MI.getDesc().getNumOperands() == 4 &&
2790 MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
2791 return true;
2792 }
2793 break;
2794 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
2795 if (MI.getOperand(i: 2).getImm() == 0) {
2796 assert(MI.getDesc().getNumOperands() == 4 &&
2797 MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
2798 return true;
2799 }
2800 break;
2801 }
2802 return false;
2803}
2804
2805// Return true if this instruction simply renames a general register without
2806// modifying bits.
2807bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
2808 switch (MI.getOpcode()) {
2809 default:
2810 break;
2811 case TargetOpcode::COPY: {
2812 Register DstReg = MI.getOperand(i: 0).getReg();
2813 return AArch64::FPR128RegClass.contains(Reg: DstReg);
2814 }
2815 case AArch64::ORRv16i8:
2816 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
2817 assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
2818 "invalid ORRv16i8 operands");
2819 return true;
2820 }
2821 break;
2822 }
2823 return false;
2824}
2825
2826static bool isFrameLoadOpcode(int Opcode) {
2827 switch (Opcode) {
2828 default:
2829 return false;
2830 case AArch64::LDRWui:
2831 case AArch64::LDRXui:
2832 case AArch64::LDRBui:
2833 case AArch64::LDRHui:
2834 case AArch64::LDRSui:
2835 case AArch64::LDRDui:
2836 case AArch64::LDRQui:
2837 case AArch64::LDR_PXI:
2838 return true;
2839 }
2840}
2841
2842Register AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
2843 int &FrameIndex) const {
2844 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2845 return Register();
2846
2847 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2848 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2849 FrameIndex = MI.getOperand(i: 1).getIndex();
2850 return MI.getOperand(i: 0).getReg();
2851 }
2852 return Register();
2853}
2854
2855static bool isFrameStoreOpcode(int Opcode) {
2856 switch (Opcode) {
2857 default:
2858 return false;
2859 case AArch64::STRWui:
2860 case AArch64::STRXui:
2861 case AArch64::STRBui:
2862 case AArch64::STRHui:
2863 case AArch64::STRSui:
2864 case AArch64::STRDui:
2865 case AArch64::STRQui:
2866 case AArch64::STR_PXI:
2867 return true;
2868 }
2869}
2870
2871Register AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
2872 int &FrameIndex) const {
2873 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2874 return Register();
2875
2876 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2877 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2878 FrameIndex = MI.getOperand(i: 1).getIndex();
2879 return MI.getOperand(i: 0).getReg();
2880 }
2881 return Register();
2882}
2883
2884Register AArch64InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
2885 int &FrameIndex) const {
2886 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2887 return Register();
2888
2889 if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
2890 return Reg;
2891
2892 SmallVector<const MachineMemOperand *, 1> Accesses;
2893 if (hasStoreToStackSlot(MI, Accesses)) {
2894 if (Accesses.size() > 1)
2895 return Register();
2896
2897 FrameIndex =
2898 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2899 ->getFrameIndex();
2900 return MI.getOperand(i: 0).getReg();
2901 }
2902 return Register();
2903}
2904
2905Register AArch64InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
2906 int &FrameIndex) const {
2907 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2908 return Register();
2909
2910 if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
2911 return Reg;
2912
2913 SmallVector<const MachineMemOperand *, 1> Accesses;
2914 if (hasLoadFromStackSlot(MI, Accesses)) {
2915 if (Accesses.size() > 1)
2916 return Register();
2917
2918 FrameIndex =
2919 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2920 ->getFrameIndex();
2921 return MI.getOperand(i: 0).getReg();
2922 }
2923 return Register();
2924}
2925
2926/// Check all MachineMemOperands for a hint to suppress pairing.
2927bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
2928 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2929 return MMO->getFlags() & MOSuppressPair;
2930 });
2931}
2932
2933/// Set a flag on the first MachineMemOperand to suppress pairing.
2934void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
2935 if (MI.memoperands_empty())
2936 return;
2937 (*MI.memoperands_begin())->setFlags(MOSuppressPair);
2938}
2939
2940/// Check all MachineMemOperands for a hint that the load/store is strided.
2941bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
2942 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2943 return MMO->getFlags() & MOStridedAccess;
2944 });
2945}
2946
2947bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
2948 switch (Opc) {
2949 default:
2950 return false;
2951 case AArch64::STURSi:
2952 case AArch64::STRSpre:
2953 case AArch64::STURDi:
2954 case AArch64::STRDpre:
2955 case AArch64::STURQi:
2956 case AArch64::STRQpre:
2957 case AArch64::STURBBi:
2958 case AArch64::STURHHi:
2959 case AArch64::STURWi:
2960 case AArch64::STRWpre:
2961 case AArch64::STURXi:
2962 case AArch64::STRXpre:
2963 case AArch64::LDURSi:
2964 case AArch64::LDRSpre:
2965 case AArch64::LDURDi:
2966 case AArch64::LDRDpre:
2967 case AArch64::LDURQi:
2968 case AArch64::LDRQpre:
2969 case AArch64::LDURWi:
2970 case AArch64::LDRWpre:
2971 case AArch64::LDURXi:
2972 case AArch64::LDRXpre:
2973 case AArch64::LDRSWpre:
2974 case AArch64::LDURSWi:
2975 case AArch64::LDURHHi:
2976 case AArch64::LDURBBi:
2977 case AArch64::LDURSBWi:
2978 case AArch64::LDURSHWi:
2979 return true;
2980 }
2981}
2982
2983std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
2984 switch (Opc) {
2985 default: return {};
2986 case AArch64::PRFMui: return AArch64::PRFUMi;
2987 case AArch64::LDRXui: return AArch64::LDURXi;
2988 case AArch64::LDRWui: return AArch64::LDURWi;
2989 case AArch64::LDRBui: return AArch64::LDURBi;
2990 case AArch64::LDRHui: return AArch64::LDURHi;
2991 case AArch64::LDRSui: return AArch64::LDURSi;
2992 case AArch64::LDRDui: return AArch64::LDURDi;
2993 case AArch64::LDRQui: return AArch64::LDURQi;
2994 case AArch64::LDRBBui: return AArch64::LDURBBi;
2995 case AArch64::LDRHHui: return AArch64::LDURHHi;
2996 case AArch64::LDRSBXui: return AArch64::LDURSBXi;
2997 case AArch64::LDRSBWui: return AArch64::LDURSBWi;
2998 case AArch64::LDRSHXui: return AArch64::LDURSHXi;
2999 case AArch64::LDRSHWui: return AArch64::LDURSHWi;
3000 case AArch64::LDRSWui: return AArch64::LDURSWi;
3001 case AArch64::STRXui: return AArch64::STURXi;
3002 case AArch64::STRWui: return AArch64::STURWi;
3003 case AArch64::STRBui: return AArch64::STURBi;
3004 case AArch64::STRHui: return AArch64::STURHi;
3005 case AArch64::STRSui: return AArch64::STURSi;
3006 case AArch64::STRDui: return AArch64::STURDi;
3007 case AArch64::STRQui: return AArch64::STURQi;
3008 case AArch64::STRBBui: return AArch64::STURBBi;
3009 case AArch64::STRHHui: return AArch64::STURHHi;
3010 }
3011}
3012
3013unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
3014 switch (Opc) {
3015 default:
3016 llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
3017 case AArch64::ADDG:
3018 case AArch64::LDAPURBi:
3019 case AArch64::LDAPURHi:
3020 case AArch64::LDAPURi:
3021 case AArch64::LDAPURSBWi:
3022 case AArch64::LDAPURSBXi:
3023 case AArch64::LDAPURSHWi:
3024 case AArch64::LDAPURSHXi:
3025 case AArch64::LDAPURSWi:
3026 case AArch64::LDAPURXi:
3027 case AArch64::LDR_PPXI:
3028 case AArch64::LDR_PXI:
3029 case AArch64::LDR_ZXI:
3030 case AArch64::LDR_ZZXI:
3031 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
3032 case AArch64::LDR_ZZZXI:
3033 case AArch64::LDR_ZZZZXI:
3034 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
3035 case AArch64::LDRBBui:
3036 case AArch64::LDRBui:
3037 case AArch64::LDRDui:
3038 case AArch64::LDRHHui:
3039 case AArch64::LDRHui:
3040 case AArch64::LDRQui:
3041 case AArch64::LDRSBWui:
3042 case AArch64::LDRSBXui:
3043 case AArch64::LDRSHWui:
3044 case AArch64::LDRSHXui:
3045 case AArch64::LDRSui:
3046 case AArch64::LDRSWui:
3047 case AArch64::LDRWui:
3048 case AArch64::LDRXui:
3049 case AArch64::LDURBBi:
3050 case AArch64::LDURBi:
3051 case AArch64::LDURDi:
3052 case AArch64::LDURHHi:
3053 case AArch64::LDURHi:
3054 case AArch64::LDURQi:
3055 case AArch64::LDURSBWi:
3056 case AArch64::LDURSBXi:
3057 case AArch64::LDURSHWi:
3058 case AArch64::LDURSHXi:
3059 case AArch64::LDURSi:
3060 case AArch64::LDURSWi:
3061 case AArch64::LDURWi:
3062 case AArch64::LDURXi:
3063 case AArch64::PRFMui:
3064 case AArch64::PRFUMi:
3065 case AArch64::ST2Gi:
3066 case AArch64::STGi:
3067 case AArch64::STLURBi:
3068 case AArch64::STLURHi:
3069 case AArch64::STLURWi:
3070 case AArch64::STLURXi:
3071 case AArch64::StoreSwiftAsyncContext:
3072 case AArch64::STR_PPXI:
3073 case AArch64::STR_PXI:
3074 case AArch64::STR_ZXI:
3075 case AArch64::STR_ZZXI:
3076 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
3077 case AArch64::STR_ZZZXI:
3078 case AArch64::STR_ZZZZXI:
3079 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
3080 case AArch64::STRBBui:
3081 case AArch64::STRBui:
3082 case AArch64::STRDui:
3083 case AArch64::STRHHui:
3084 case AArch64::STRHui:
3085 case AArch64::STRQui:
3086 case AArch64::STRSui:
3087 case AArch64::STRWui:
3088 case AArch64::STRXui:
3089 case AArch64::STURBBi:
3090 case AArch64::STURBi:
3091 case AArch64::STURDi:
3092 case AArch64::STURHHi:
3093 case AArch64::STURHi:
3094 case AArch64::STURQi:
3095 case AArch64::STURSi:
3096 case AArch64::STURWi:
3097 case AArch64::STURXi:
3098 case AArch64::STZ2Gi:
3099 case AArch64::STZGi:
3100 case AArch64::TAGPstack:
3101 return 2;
3102 case AArch64::LD1B_D_IMM:
3103 case AArch64::LD1B_H_IMM:
3104 case AArch64::LD1B_IMM:
3105 case AArch64::LD1B_S_IMM:
3106 case AArch64::LD1D_IMM:
3107 case AArch64::LD1H_D_IMM:
3108 case AArch64::LD1H_IMM:
3109 case AArch64::LD1H_S_IMM:
3110 case AArch64::LD1RB_D_IMM:
3111 case AArch64::LD1RB_H_IMM:
3112 case AArch64::LD1RB_IMM:
3113 case AArch64::LD1RB_S_IMM:
3114 case AArch64::LD1RD_IMM:
3115 case AArch64::LD1RH_D_IMM:
3116 case AArch64::LD1RH_IMM:
3117 case AArch64::LD1RH_S_IMM:
3118 case AArch64::LD1RSB_D_IMM:
3119 case AArch64::LD1RSB_H_IMM:
3120 case AArch64::LD1RSB_S_IMM:
3121 case AArch64::LD1RSH_D_IMM:
3122 case AArch64::LD1RSH_S_IMM:
3123 case AArch64::LD1RSW_IMM:
3124 case AArch64::LD1RW_D_IMM:
3125 case AArch64::LD1RW_IMM:
3126 case AArch64::LD1SB_D_IMM:
3127 case AArch64::LD1SB_H_IMM:
3128 case AArch64::LD1SB_S_IMM:
3129 case AArch64::LD1SH_D_IMM:
3130 case AArch64::LD1SH_S_IMM:
3131 case AArch64::LD1SW_D_IMM:
3132 case AArch64::LD1W_D_IMM:
3133 case AArch64::LD1W_IMM:
3134 case AArch64::LD2B_IMM:
3135 case AArch64::LD2D_IMM:
3136 case AArch64::LD2H_IMM:
3137 case AArch64::LD2W_IMM:
3138 case AArch64::LD3B_IMM:
3139 case AArch64::LD3D_IMM:
3140 case AArch64::LD3H_IMM:
3141 case AArch64::LD3W_IMM:
3142 case AArch64::LD4B_IMM:
3143 case AArch64::LD4D_IMM:
3144 case AArch64::LD4H_IMM:
3145 case AArch64::LD4W_IMM:
3146 case AArch64::LDG:
3147 case AArch64::LDNF1B_D_IMM:
3148 case AArch64::LDNF1B_H_IMM:
3149 case AArch64::LDNF1B_IMM:
3150 case AArch64::LDNF1B_S_IMM:
3151 case AArch64::LDNF1D_IMM:
3152 case AArch64::LDNF1H_D_IMM:
3153 case AArch64::LDNF1H_IMM:
3154 case AArch64::LDNF1H_S_IMM:
3155 case AArch64::LDNF1SB_D_IMM:
3156 case AArch64::LDNF1SB_H_IMM:
3157 case AArch64::LDNF1SB_S_IMM:
3158 case AArch64::LDNF1SH_D_IMM:
3159 case AArch64::LDNF1SH_S_IMM:
3160 case AArch64::LDNF1SW_D_IMM:
3161 case AArch64::LDNF1W_D_IMM:
3162 case AArch64::LDNF1W_IMM:
3163 case AArch64::LDNPDi:
3164 case AArch64::LDNPQi:
3165 case AArch64::LDNPSi:
3166 case AArch64::LDNPWi:
3167 case AArch64::LDNPXi:
3168 case AArch64::LDNT1B_ZRI:
3169 case AArch64::LDNT1D_ZRI:
3170 case AArch64::LDNT1H_ZRI:
3171 case AArch64::LDNT1W_ZRI:
3172 case AArch64::LDPDi:
3173 case AArch64::LDPQi:
3174 case AArch64::LDPSi:
3175 case AArch64::LDPWi:
3176 case AArch64::LDPXi:
3177 case AArch64::LDRBBpost:
3178 case AArch64::LDRBBpre:
3179 case AArch64::LDRBpost:
3180 case AArch64::LDRBpre:
3181 case AArch64::LDRDpost:
3182 case AArch64::LDRDpre:
3183 case AArch64::LDRHHpost:
3184 case AArch64::LDRHHpre:
3185 case AArch64::LDRHpost:
3186 case AArch64::LDRHpre:
3187 case AArch64::LDRQpost:
3188 case AArch64::LDRQpre:
3189 case AArch64::LDRSpost:
3190 case AArch64::LDRSpre:
3191 case AArch64::LDRWpost:
3192 case AArch64::LDRWpre:
3193 case AArch64::LDRXpost:
3194 case AArch64::LDRXpre:
3195 case AArch64::ST1B_D_IMM:
3196 case AArch64::ST1B_H_IMM:
3197 case AArch64::ST1B_IMM:
3198 case AArch64::ST1B_S_IMM:
3199 case AArch64::ST1D_IMM:
3200 case AArch64::ST1H_D_IMM:
3201 case AArch64::ST1H_IMM:
3202 case AArch64::ST1H_S_IMM:
3203 case AArch64::ST1W_D_IMM:
3204 case AArch64::ST1W_IMM:
3205 case AArch64::ST2B_IMM:
3206 case AArch64::ST2D_IMM:
3207 case AArch64::ST2H_IMM:
3208 case AArch64::ST2W_IMM:
3209 case AArch64::ST3B_IMM:
3210 case AArch64::ST3D_IMM:
3211 case AArch64::ST3H_IMM:
3212 case AArch64::ST3W_IMM:
3213 case AArch64::ST4B_IMM:
3214 case AArch64::ST4D_IMM:
3215 case AArch64::ST4H_IMM:
3216 case AArch64::ST4W_IMM:
3217 case AArch64::STGPi:
3218 case AArch64::STGPreIndex:
3219 case AArch64::STZGPreIndex:
3220 case AArch64::ST2GPreIndex:
3221 case AArch64::STZ2GPreIndex:
3222 case AArch64::STGPostIndex:
3223 case AArch64::STZGPostIndex:
3224 case AArch64::ST2GPostIndex:
3225 case AArch64::STZ2GPostIndex:
3226 case AArch64::STNPDi:
3227 case AArch64::STNPQi:
3228 case AArch64::STNPSi:
3229 case AArch64::STNPWi:
3230 case AArch64::STNPXi:
3231 case AArch64::STNT1B_ZRI:
3232 case AArch64::STNT1D_ZRI:
3233 case AArch64::STNT1H_ZRI:
3234 case AArch64::STNT1W_ZRI:
3235 case AArch64::STPDi:
3236 case AArch64::STPQi:
3237 case AArch64::STPSi:
3238 case AArch64::STPWi:
3239 case AArch64::STPXi:
3240 case AArch64::STRBBpost:
3241 case AArch64::STRBBpre:
3242 case AArch64::STRBpost:
3243 case AArch64::STRBpre:
3244 case AArch64::STRDpost:
3245 case AArch64::STRDpre:
3246 case AArch64::STRHHpost:
3247 case AArch64::STRHHpre:
3248 case AArch64::STRHpost:
3249 case AArch64::STRHpre:
3250 case AArch64::STRQpost:
3251 case AArch64::STRQpre:
3252 case AArch64::STRSpost:
3253 case AArch64::STRSpre:
3254 case AArch64::STRWpost:
3255 case AArch64::STRWpre:
3256 case AArch64::STRXpost:
3257 case AArch64::STRXpre:
3258 case AArch64::LD1B_2Z_IMM:
3259 case AArch64::LD1B_2Z_STRIDED_IMM:
3260 case AArch64::LD1H_2Z_IMM:
3261 case AArch64::LD1H_2Z_STRIDED_IMM:
3262 case AArch64::LD1W_2Z_IMM:
3263 case AArch64::LD1W_2Z_STRIDED_IMM:
3264 case AArch64::LD1D_2Z_IMM:
3265 case AArch64::LD1D_2Z_STRIDED_IMM:
3266 case AArch64::LD1B_4Z_IMM:
3267 case AArch64::LD1B_4Z_STRIDED_IMM:
3268 case AArch64::LD1H_4Z_IMM:
3269 case AArch64::LD1H_4Z_STRIDED_IMM:
3270 case AArch64::LD1W_4Z_IMM:
3271 case AArch64::LD1W_4Z_STRIDED_IMM:
3272 case AArch64::LD1D_4Z_IMM:
3273 case AArch64::LD1D_4Z_STRIDED_IMM:
3274 case AArch64::LD1B_2Z_IMM_PSEUDO:
3275 case AArch64::LD1H_2Z_IMM_PSEUDO:
3276 case AArch64::LD1W_2Z_IMM_PSEUDO:
3277 case AArch64::LD1D_2Z_IMM_PSEUDO:
3278 case AArch64::LD1B_4Z_IMM_PSEUDO:
3279 case AArch64::LD1H_4Z_IMM_PSEUDO:
3280 case AArch64::LD1W_4Z_IMM_PSEUDO:
3281 case AArch64::LD1D_4Z_IMM_PSEUDO:
3282 case AArch64::ST1B_2Z_IMM:
3283 case AArch64::ST1B_2Z_STRIDED_IMM:
3284 case AArch64::ST1H_2Z_IMM:
3285 case AArch64::ST1H_2Z_STRIDED_IMM:
3286 case AArch64::ST1W_2Z_IMM:
3287 case AArch64::ST1W_2Z_STRIDED_IMM:
3288 case AArch64::ST1D_2Z_IMM:
3289 case AArch64::ST1D_2Z_STRIDED_IMM:
3290 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
3291 case AArch64::LDNT1B_2Z_IMM:
3292 case AArch64::LDNT1B_2Z_STRIDED_IMM:
3293 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
3294 case AArch64::LDNT1H_2Z_IMM:
3295 case AArch64::LDNT1H_2Z_STRIDED_IMM:
3296 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
3297 case AArch64::LDNT1W_2Z_IMM:
3298 case AArch64::LDNT1W_2Z_STRIDED_IMM:
3299 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
3300 case AArch64::LDNT1D_2Z_IMM:
3301 case AArch64::LDNT1D_2Z_STRIDED_IMM:
3302 case AArch64::STNT1B_2Z_IMM:
3303 case AArch64::STNT1B_2Z_STRIDED_IMM:
3304 case AArch64::STNT1H_2Z_IMM:
3305 case AArch64::STNT1H_2Z_STRIDED_IMM:
3306 case AArch64::STNT1W_2Z_IMM:
3307 case AArch64::STNT1W_2Z_STRIDED_IMM:
3308 case AArch64::STNT1D_2Z_IMM:
3309 case AArch64::STNT1D_2Z_STRIDED_IMM:
3310 case AArch64::ST1B_4Z_IMM:
3311 case AArch64::ST1B_4Z_STRIDED_IMM:
3312 case AArch64::ST1H_4Z_IMM:
3313 case AArch64::ST1H_4Z_STRIDED_IMM:
3314 case AArch64::ST1W_4Z_IMM:
3315 case AArch64::ST1W_4Z_STRIDED_IMM:
3316 case AArch64::ST1D_4Z_IMM:
3317 case AArch64::ST1D_4Z_STRIDED_IMM:
3318 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
3319 case AArch64::LDNT1B_4Z_IMM:
3320 case AArch64::LDNT1B_4Z_STRIDED_IMM:
3321 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
3322 case AArch64::LDNT1H_4Z_IMM:
3323 case AArch64::LDNT1H_4Z_STRIDED_IMM:
3324 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
3325 case AArch64::LDNT1W_4Z_IMM:
3326 case AArch64::LDNT1W_4Z_STRIDED_IMM:
3327 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
3328 case AArch64::LDNT1D_4Z_IMM:
3329 case AArch64::LDNT1D_4Z_STRIDED_IMM:
3330 case AArch64::STNT1B_4Z_IMM:
3331 case AArch64::STNT1B_4Z_STRIDED_IMM:
3332 case AArch64::STNT1H_4Z_IMM:
3333 case AArch64::STNT1H_4Z_STRIDED_IMM:
3334 case AArch64::STNT1W_4Z_IMM:
3335 case AArch64::STNT1W_4Z_STRIDED_IMM:
3336 case AArch64::STNT1D_4Z_IMM:
3337 case AArch64::STNT1D_4Z_STRIDED_IMM:
3338 return 3;
3339 case AArch64::LDPDpost:
3340 case AArch64::LDPDpre:
3341 case AArch64::LDPQpost:
3342 case AArch64::LDPQpre:
3343 case AArch64::LDPSpost:
3344 case AArch64::LDPSpre:
3345 case AArch64::LDPWpost:
3346 case AArch64::LDPWpre:
3347 case AArch64::LDPXpost:
3348 case AArch64::LDPXpre:
3349 case AArch64::STGPpre:
3350 case AArch64::STGPpost:
3351 case AArch64::STPDpost:
3352 case AArch64::STPDpre:
3353 case AArch64::STPQpost:
3354 case AArch64::STPQpre:
3355 case AArch64::STPSpost:
3356 case AArch64::STPSpre:
3357 case AArch64::STPWpost:
3358 case AArch64::STPWpre:
3359 case AArch64::STPXpost:
3360 case AArch64::STPXpre:
3361 return 4;
3362 }
3363}
3364
3365bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
3366 switch (MI.getOpcode()) {
3367 default:
3368 return false;
3369 // Scaled instructions.
3370 case AArch64::STRSui:
3371 case AArch64::STRDui:
3372 case AArch64::STRQui:
3373 case AArch64::STRXui:
3374 case AArch64::STRWui:
3375 case AArch64::LDRSui:
3376 case AArch64::LDRDui:
3377 case AArch64::LDRQui:
3378 case AArch64::LDRXui:
3379 case AArch64::LDRWui:
3380 case AArch64::LDRSWui:
3381 // Unscaled instructions.
3382 case AArch64::STURSi:
3383 case AArch64::STRSpre:
3384 case AArch64::STURDi:
3385 case AArch64::STRDpre:
3386 case AArch64::STURQi:
3387 case AArch64::STRQpre:
3388 case AArch64::STURWi:
3389 case AArch64::STRWpre:
3390 case AArch64::STURXi:
3391 case AArch64::STRXpre:
3392 case AArch64::LDURSi:
3393 case AArch64::LDRSpre:
3394 case AArch64::LDURDi:
3395 case AArch64::LDRDpre:
3396 case AArch64::LDURQi:
3397 case AArch64::LDRQpre:
3398 case AArch64::LDURWi:
3399 case AArch64::LDRWpre:
3400 case AArch64::LDURXi:
3401 case AArch64::LDRXpre:
3402 case AArch64::LDURSWi:
3403 case AArch64::LDRSWpre:
3404 // SVE instructions.
3405 case AArch64::LDR_ZXI:
3406 case AArch64::STR_ZXI:
3407 return true;
3408 }
3409}
3410
3411bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
3412 switch (MI.getOpcode()) {
3413 default:
3414 assert((!MI.isCall() || !MI.isReturn()) &&
3415 "Unexpected instruction - was a new tail call opcode introduced?");
3416 return false;
3417 case AArch64::TCRETURNdi:
3418 case AArch64::TCRETURNri:
3419 case AArch64::TCRETURNrix16x17:
3420 case AArch64::TCRETURNrix17:
3421 case AArch64::TCRETURNrinotx16:
3422 case AArch64::TCRETURNriALL:
3423 case AArch64::AUTH_TCRETURN:
3424 case AArch64::AUTH_TCRETURN_BTI:
3425 return true;
3426 }
3427}
3428
3429unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) {
3430 switch (Opc) {
3431 default:
3432 llvm_unreachable("Opcode has no flag setting equivalent!");
3433 // 32-bit cases:
3434 case AArch64::ADDWri:
3435 return AArch64::ADDSWri;
3436 case AArch64::ADDWrr:
3437 return AArch64::ADDSWrr;
3438 case AArch64::ADDWrs:
3439 return AArch64::ADDSWrs;
3440 case AArch64::ADDWrx:
3441 return AArch64::ADDSWrx;
3442 case AArch64::ANDWri:
3443 return AArch64::ANDSWri;
3444 case AArch64::ANDWrr:
3445 return AArch64::ANDSWrr;
3446 case AArch64::ANDWrs:
3447 return AArch64::ANDSWrs;
3448 case AArch64::BICWrr:
3449 return AArch64::BICSWrr;
3450 case AArch64::BICWrs:
3451 return AArch64::BICSWrs;
3452 case AArch64::SUBWri:
3453 return AArch64::SUBSWri;
3454 case AArch64::SUBWrr:
3455 return AArch64::SUBSWrr;
3456 case AArch64::SUBWrs:
3457 return AArch64::SUBSWrs;
3458 case AArch64::SUBWrx:
3459 return AArch64::SUBSWrx;
3460 // 64-bit cases:
3461 case AArch64::ADDXri:
3462 return AArch64::ADDSXri;
3463 case AArch64::ADDXrr:
3464 return AArch64::ADDSXrr;
3465 case AArch64::ADDXrs:
3466 return AArch64::ADDSXrs;
3467 case AArch64::ADDXrx:
3468 return AArch64::ADDSXrx;
3469 case AArch64::ANDXri:
3470 return AArch64::ANDSXri;
3471 case AArch64::ANDXrr:
3472 return AArch64::ANDSXrr;
3473 case AArch64::ANDXrs:
3474 return AArch64::ANDSXrs;
3475 case AArch64::BICXrr:
3476 return AArch64::BICSXrr;
3477 case AArch64::BICXrs:
3478 return AArch64::BICSXrs;
3479 case AArch64::SUBXri:
3480 return AArch64::SUBSXri;
3481 case AArch64::SUBXrr:
3482 return AArch64::SUBSXrr;
3483 case AArch64::SUBXrs:
3484 return AArch64::SUBSXrs;
3485 case AArch64::SUBXrx:
3486 return AArch64::SUBSXrx;
3487 // SVE instructions:
3488 case AArch64::AND_PPzPP:
3489 return AArch64::ANDS_PPzPP;
3490 case AArch64::BIC_PPzPP:
3491 return AArch64::BICS_PPzPP;
3492 case AArch64::EOR_PPzPP:
3493 return AArch64::EORS_PPzPP;
3494 case AArch64::NAND_PPzPP:
3495 return AArch64::NANDS_PPzPP;
3496 case AArch64::NOR_PPzPP:
3497 return AArch64::NORS_PPzPP;
3498 case AArch64::ORN_PPzPP:
3499 return AArch64::ORNS_PPzPP;
3500 case AArch64::ORR_PPzPP:
3501 return AArch64::ORRS_PPzPP;
3502 case AArch64::BRKA_PPzP:
3503 return AArch64::BRKAS_PPzP;
3504 case AArch64::BRKPA_PPzPP:
3505 return AArch64::BRKPAS_PPzPP;
3506 case AArch64::BRKB_PPzP:
3507 return AArch64::BRKBS_PPzP;
3508 case AArch64::BRKPB_PPzPP:
3509 return AArch64::BRKPBS_PPzPP;
3510 case AArch64::BRKN_PPzP:
3511 return AArch64::BRKNS_PPzP;
3512 case AArch64::RDFFR_PPz:
3513 return AArch64::RDFFRS_PPz;
3514 case AArch64::PTRUE_B:
3515 return AArch64::PTRUES_B;
3516 }
3517}
3518
3519// Is this a candidate for ld/st merging or pairing? For example, we don't
3520// touch volatiles or load/stores that have a hint to avoid pair formation.
3521bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
3522
3523 bool IsPreLdSt = isPreLdSt(MI);
3524
3525 // If this is a volatile load/store, don't mess with it.
3526 if (MI.hasOrderedMemoryRef())
3527 return false;
3528
3529 // Make sure this is a reg/fi+imm (as opposed to an address reloc).
3530 // For Pre-inc LD/ST, the operand is shifted by one.
3531 assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
3532 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
3533 "Expected a reg or frame index operand.");
3534
3535 // For Pre-indexed addressing quadword instructions, the third operand is the
3536 // immediate value.
3537 bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(i: 3).isImm();
3538
3539 if (!MI.getOperand(i: 2).isImm() && !IsImmPreLdSt)
3540 return false;
3541
3542 // Can't merge/pair if the instruction modifies the base register.
3543 // e.g., ldr x0, [x0]
3544 // This case will never occur with an FI base.
3545 // However, if the instruction is an LDR<S,D,Q,W,X,SW>pre or
3546 // STR<S,D,Q,W,X>pre, it can be merged.
3547 // For example:
3548 // ldr q0, [x11, #32]!
3549 // ldr q1, [x11, #16]
3550 // to
3551 // ldp q0, q1, [x11, #32]!
3552 if (MI.getOperand(i: 1).isReg() && !IsPreLdSt) {
3553 Register BaseReg = MI.getOperand(i: 1).getReg();
3554 const TargetRegisterInfo *TRI = &getRegisterInfo();
3555 if (MI.modifiesRegister(Reg: BaseReg, TRI))
3556 return false;
3557 }
3558
3559 // Pairing SVE fills/spills is only valid for little-endian targets that
3560 // implement VLS 128.
3561 switch (MI.getOpcode()) {
3562 default:
3563 break;
3564 case AArch64::LDR_ZXI:
3565 case AArch64::STR_ZXI:
3566 if (!Subtarget.isLittleEndian() ||
3567 Subtarget.getSVEVectorSizeInBits() != 128)
3568 return false;
3569 }
3570
3571 // Check if this load/store has a hint to avoid pair formation.
3572 // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
3573 if (isLdStPairSuppressed(MI))
3574 return false;
3575
3576 // Do not pair any callee-save store/reload instructions in the
3577 // prologue/epilogue if the CFI information encoded the operations as separate
3578 // instructions, as that will cause the size of the actual prologue to mismatch
3579 // with the prologue size recorded in the Windows CFI.
3580 const MCAsmInfo &MAI = MI.getMF()->getTarget().getMCAsmInfo();
3581 bool NeedsWinCFI =
3582 MAI.usesWindowsCFI() && MI.getMF()->getFunction().needsUnwindTableEntry();
3583 if (NeedsWinCFI && (MI.getFlag(Flag: MachineInstr::FrameSetup) ||
3584 MI.getFlag(Flag: MachineInstr::FrameDestroy)))
3585 return false;
3586
3587 // On some CPUs quad load/store pairs are slower than two single load/stores.
3588 if (Subtarget.isPaired128Slow()) {
3589 switch (MI.getOpcode()) {
3590 default:
3591 break;
3592 case AArch64::LDURQi:
3593 case AArch64::STURQi:
3594 case AArch64::LDRQui:
3595 case AArch64::STRQui:
3596 return false;
3597 }
3598 }
3599
3600 return true;
3601}
3602
3603bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
3604 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3605 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
3606 const TargetRegisterInfo *TRI) const {
3607 if (!LdSt.mayLoadOrStore())
3608 return false;
3609
3610 const MachineOperand *BaseOp;
3611 TypeSize WidthN(0, false);
3612 if (!getMemOperandWithOffsetWidth(MI: LdSt, BaseOp, Offset, OffsetIsScalable,
3613 Width&: WidthN, TRI))
3614 return false;
3615 // The maximum vscale is 16 under AArch64, return the maximal extent for the
3616 // vector.
3617 Width = LocationSize::precise(Value: WidthN);
3618 BaseOps.push_back(Elt: BaseOp);
3619 return true;
3620}
3621
3622std::optional<ExtAddrMode>
3623AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
3624 const TargetRegisterInfo *TRI) const {
3625 const MachineOperand *Base; // Filled with the base operand of MI.
3626 int64_t Offset; // Filled with the offset of MI.
3627 bool OffsetIsScalable;
3628 if (!getMemOperandWithOffset(MI: MemI, BaseOp&: Base, Offset, OffsetIsScalable, TRI))
3629 return std::nullopt;
3630
3631 if (!Base->isReg())
3632 return std::nullopt;
3633 ExtAddrMode AM;
3634 AM.BaseReg = Base->getReg();
3635 AM.Displacement = Offset;
3636 AM.ScaledReg = 0;
3637 AM.Scale = 0;
3638 return AM;
3639}
3640
3641bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
3642 Register Reg,
3643 const MachineInstr &AddrI,
3644 ExtAddrMode &AM) const {
3645 // Filter out instructions into which we cannot fold.
3646 unsigned NumBytes;
3647 int64_t OffsetScale = 1;
3648 switch (MemI.getOpcode()) {
3649 default:
3650 return false;
3651
3652 case AArch64::LDURQi:
3653 case AArch64::STURQi:
3654 NumBytes = 16;
3655 break;
3656
3657 case AArch64::LDURDi:
3658 case AArch64::STURDi:
3659 case AArch64::LDURXi:
3660 case AArch64::STURXi:
3661 NumBytes = 8;
3662 break;
3663
3664 case AArch64::LDURWi:
3665 case AArch64::LDURSWi:
3666 case AArch64::STURWi:
3667 NumBytes = 4;
3668 break;
3669
3670 case AArch64::LDURHi:
3671 case AArch64::STURHi:
3672 case AArch64::LDURHHi:
3673 case AArch64::STURHHi:
3674 case AArch64::LDURSHXi:
3675 case AArch64::LDURSHWi:
3676 NumBytes = 2;
3677 break;
3678
3679 case AArch64::LDRBroX:
3680 case AArch64::LDRBBroX:
3681 case AArch64::LDRSBXroX:
3682 case AArch64::LDRSBWroX:
3683 case AArch64::STRBroX:
3684 case AArch64::STRBBroX:
3685 case AArch64::LDURBi:
3686 case AArch64::LDURBBi:
3687 case AArch64::LDURSBXi:
3688 case AArch64::LDURSBWi:
3689 case AArch64::STURBi:
3690 case AArch64::STURBBi:
3691 case AArch64::LDRBui:
3692 case AArch64::LDRBBui:
3693 case AArch64::LDRSBXui:
3694 case AArch64::LDRSBWui:
3695 case AArch64::STRBui:
3696 case AArch64::STRBBui:
3697 NumBytes = 1;
3698 break;
3699
3700 case AArch64::LDRQroX:
3701 case AArch64::STRQroX:
3702 case AArch64::LDRQui:
3703 case AArch64::STRQui:
3704 NumBytes = 16;
3705 OffsetScale = 16;
3706 break;
3707
3708 case AArch64::LDRDroX:
3709 case AArch64::STRDroX:
3710 case AArch64::LDRXroX:
3711 case AArch64::STRXroX:
3712 case AArch64::LDRDui:
3713 case AArch64::STRDui:
3714 case AArch64::LDRXui:
3715 case AArch64::STRXui:
3716 NumBytes = 8;
3717 OffsetScale = 8;
3718 break;
3719
3720 case AArch64::LDRWroX:
3721 case AArch64::LDRSWroX:
3722 case AArch64::STRWroX:
3723 case AArch64::LDRWui:
3724 case AArch64::LDRSWui:
3725 case AArch64::STRWui:
3726 NumBytes = 4;
3727 OffsetScale = 4;
3728 break;
3729
3730 case AArch64::LDRHroX:
3731 case AArch64::STRHroX:
3732 case AArch64::LDRHHroX:
3733 case AArch64::STRHHroX:
3734 case AArch64::LDRSHXroX:
3735 case AArch64::LDRSHWroX:
3736 case AArch64::LDRHui:
3737 case AArch64::STRHui:
3738 case AArch64::LDRHHui:
3739 case AArch64::STRHHui:
3740 case AArch64::LDRSHXui:
3741 case AArch64::LDRSHWui:
3742 NumBytes = 2;
3743 OffsetScale = 2;
3744 break;
3745 }
3746
3747 // Check the fold operand is not the loaded/stored value.
3748 const MachineOperand &BaseRegOp = MemI.getOperand(i: 0);
3749 if (BaseRegOp.isReg() && BaseRegOp.getReg() == Reg)
3750 return false;
3751
3752 // Handle memory instructions with a [Reg, Reg] addressing mode.
3753 if (MemI.getOperand(i: 2).isReg()) {
3754 // Bail if the addressing mode already includes extension of the offset
3755 // register.
3756 if (MemI.getOperand(i: 3).getImm())
3757 return false;
3758
3759 // Check if we actually have a scaled offset.
3760 if (MemI.getOperand(i: 4).getImm() == 0)
3761 OffsetScale = 1;
3762
3763 // If the address instructions is folded into the base register, then the
3764 // addressing mode must not have a scale. Then we can swap the base and the
3765 // scaled registers.
3766 if (MemI.getOperand(i: 1).getReg() == Reg && OffsetScale != 1)
3767 return false;
3768
3769 switch (AddrI.getOpcode()) {
3770 default:
3771 return false;
3772
3773 case AArch64::SBFMXri:
3774 // sxtw Xa, Wm
3775 // ldr Xd, [Xn, Xa, lsl #N]
3776 // ->
3777 // ldr Xd, [Xn, Wm, sxtw #N]
3778 if (AddrI.getOperand(i: 2).getImm() != 0 ||
3779 AddrI.getOperand(i: 3).getImm() != 31)
3780 return false;
3781
3782 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3783 if (AM.BaseReg == Reg)
3784 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3785 AM.ScaledReg = AddrI.getOperand(i: 1).getReg();
3786 AM.Scale = OffsetScale;
3787 AM.Displacement = 0;
3788 AM.Form = ExtAddrMode::Formula::SExtScaledReg;
3789 return true;
3790
3791 case TargetOpcode::SUBREG_TO_REG: {
3792 // mov Wa, Wm
3793 // ldr Xd, [Xn, Xa, lsl #N]
3794 // ->
3795 // ldr Xd, [Xn, Wm, uxtw #N]
3796
3797 // Zero-extension looks like an ORRWrs followed by a SUBREG_TO_REG.
3798 if (AddrI.getOperand(i: 2).getImm() != AArch64::sub_32)
3799 return false;
3800
3801 const MachineRegisterInfo &MRI = AddrI.getMF()->getRegInfo();
3802 Register OffsetReg = AddrI.getOperand(i: 1).getReg();
3803 if (!OffsetReg.isVirtual() || !MRI.hasOneNonDBGUse(RegNo: OffsetReg))
3804 return false;
3805
3806 const MachineInstr &DefMI = *MRI.getVRegDef(Reg: OffsetReg);
3807 if (DefMI.getOpcode() != AArch64::ORRWrs ||
3808 DefMI.getOperand(i: 1).getReg() != AArch64::WZR ||
3809 DefMI.getOperand(i: 3).getImm() != 0)
3810 return false;
3811
3812 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3813 if (AM.BaseReg == Reg)
3814 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3815 AM.ScaledReg = DefMI.getOperand(i: 2).getReg();
3816 AM.Scale = OffsetScale;
3817 AM.Displacement = 0;
3818 AM.Form = ExtAddrMode::Formula::ZExtScaledReg;
3819 return true;
3820 }
3821 }
3822 }
3823
3824 // Handle memory instructions with a [Reg, #Imm] addressing mode.
3825
3826 // Check we are not breaking a potential conversion to an LDP.
3827 auto validateOffsetForLDP = [](unsigned NumBytes, int64_t OldOffset,
3828 int64_t NewOffset) -> bool {
3829 int64_t MinOffset, MaxOffset;
3830 switch (NumBytes) {
3831 default:
3832 return true;
3833 case 4:
3834 MinOffset = -256;
3835 MaxOffset = 252;
3836 break;
3837 case 8:
3838 MinOffset = -512;
3839 MaxOffset = 504;
3840 break;
3841 case 16:
3842 MinOffset = -1024;
3843 MaxOffset = 1008;
3844 break;
3845 }
3846 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3847 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3848 };
3849 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) -> bool {
3850 int64_t OldOffset = MemI.getOperand(i: 2).getImm() * OffsetScale;
3851 int64_t NewOffset = OldOffset + Disp;
3852 if (!isLegalAddressingMode(NumBytes, Offset: NewOffset, /* Scale */ 0))
3853 return false;
3854 // If the old offset would fit into an LDP, but the new offset wouldn't,
3855 // bail out.
3856 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3857 return false;
3858 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3859 AM.ScaledReg = 0;
3860 AM.Scale = 0;
3861 AM.Displacement = NewOffset;
3862 AM.Form = ExtAddrMode::Formula::Basic;
3863 return true;
3864 };
3865
3866 auto canFoldAddRegIntoAddrMode =
3867 [&](int64_t Scale,
3868 ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool {
3869 if (MemI.getOperand(i: 2).getImm() != 0)
3870 return false;
3871 if ((unsigned)Scale != Scale)
3872 return false;
3873 if (!isLegalAddressingMode(NumBytes, /* Offset */ 0, Scale))
3874 return false;
3875 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3876 AM.ScaledReg = AddrI.getOperand(i: 2).getReg();
3877 AM.Scale = Scale;
3878 AM.Displacement = 0;
3879 AM.Form = Form;
3880 return true;
3881 };
3882
3883 auto avoidSlowSTRQ = [&](const MachineInstr &MemI) {
3884 unsigned Opcode = MemI.getOpcode();
3885 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3886 Subtarget.isSTRQroSlow();
3887 };
3888
3889 int64_t Disp = 0;
3890 const bool OptSize = MemI.getMF()->getFunction().hasOptSize();
3891 switch (AddrI.getOpcode()) {
3892 default:
3893 return false;
3894
3895 case AArch64::ADDXri:
3896 // add Xa, Xn, #N
3897 // ldr Xd, [Xa, #M]
3898 // ->
3899 // ldr Xd, [Xn, #N'+M]
3900 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3901 return canFoldAddSubImmIntoAddrMode(Disp);
3902
3903 case AArch64::SUBXri:
3904 // sub Xa, Xn, #N
3905 // ldr Xd, [Xa, #M]
3906 // ->
3907 // ldr Xd, [Xn, #N'+M]
3908 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3909 return canFoldAddSubImmIntoAddrMode(-Disp);
3910
3911 case AArch64::ADDXrs: {
3912 // add Xa, Xn, Xm, lsl #N
3913 // ldr Xd, [Xa]
3914 // ->
3915 // ldr Xd, [Xn, Xm, lsl #N]
3916
3917 // Don't fold the add if the result would be slower, unless optimising for
3918 // size.
3919 unsigned Shift = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
3920 if (AArch64_AM::getShiftType(Imm: Shift) != AArch64_AM::ShiftExtendType::LSL)
3921 return false;
3922 Shift = AArch64_AM::getShiftValue(Imm: Shift);
3923 if (!OptSize) {
3924 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3925 return false;
3926 if (avoidSlowSTRQ(MemI))
3927 return false;
3928 }
3929 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3930 }
3931
3932 case AArch64::ADDXrr:
3933 // add Xa, Xn, Xm
3934 // ldr Xd, [Xa]
3935 // ->
3936 // ldr Xd, [Xn, Xm, lsl #0]
3937
3938 // Don't fold the add if the result would be slower, unless optimising for
3939 // size.
3940 if (!OptSize && avoidSlowSTRQ(MemI))
3941 return false;
3942 return canFoldAddRegIntoAddrMode(1);
3943
3944 case AArch64::ADDXrx:
3945 // add Xa, Xn, Wm, {s,u}xtw #N
3946 // ldr Xd, [Xa]
3947 // ->
3948 // ldr Xd, [Xn, Wm, {s,u}xtw #N]
3949
3950 // Don't fold the add if the result would be slower, unless optimising for
3951 // size.
3952 if (!OptSize && avoidSlowSTRQ(MemI))
3953 return false;
3954
3955 // Can fold only sign-/zero-extend of a word.
3956 unsigned Imm = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
3957 AArch64_AM::ShiftExtendType Extend = AArch64_AM::getArithExtendType(Imm);
3958 if (Extend != AArch64_AM::UXTW && Extend != AArch64_AM::SXTW)
3959 return false;
3960
3961 return canFoldAddRegIntoAddrMode(
3962 1ULL << AArch64_AM::getArithShiftValue(Imm),
3963 (Extend == AArch64_AM::SXTW) ? ExtAddrMode::Formula::SExtScaledReg
3964 : ExtAddrMode::Formula::ZExtScaledReg);
3965 }
3966}
3967
3968// Given an opcode for an instruction with a [Reg, #Imm] addressing mode,
3969// return the opcode of an instruction performing the same operation, but using
3970// the [Reg, Reg] addressing mode.
3971static unsigned regOffsetOpcode(unsigned Opcode) {
3972 switch (Opcode) {
3973 default:
3974 llvm_unreachable("Address folding not implemented for instruction");
3975
3976 case AArch64::LDURQi:
3977 case AArch64::LDRQui:
3978 return AArch64::LDRQroX;
3979 case AArch64::STURQi:
3980 case AArch64::STRQui:
3981 return AArch64::STRQroX;
3982 case AArch64::LDURDi:
3983 case AArch64::LDRDui:
3984 return AArch64::LDRDroX;
3985 case AArch64::STURDi:
3986 case AArch64::STRDui:
3987 return AArch64::STRDroX;
3988 case AArch64::LDURXi:
3989 case AArch64::LDRXui:
3990 return AArch64::LDRXroX;
3991 case AArch64::STURXi:
3992 case AArch64::STRXui:
3993 return AArch64::STRXroX;
3994 case AArch64::LDURWi:
3995 case AArch64::LDRWui:
3996 return AArch64::LDRWroX;
3997 case AArch64::LDURSWi:
3998 case AArch64::LDRSWui:
3999 return AArch64::LDRSWroX;
4000 case AArch64::STURWi:
4001 case AArch64::STRWui:
4002 return AArch64::STRWroX;
4003 case AArch64::LDURHi:
4004 case AArch64::LDRHui:
4005 return AArch64::LDRHroX;
4006 case AArch64::STURHi:
4007 case AArch64::STRHui:
4008 return AArch64::STRHroX;
4009 case AArch64::LDURHHi:
4010 case AArch64::LDRHHui:
4011 return AArch64::LDRHHroX;
4012 case AArch64::STURHHi:
4013 case AArch64::STRHHui:
4014 return AArch64::STRHHroX;
4015 case AArch64::LDURSHXi:
4016 case AArch64::LDRSHXui:
4017 return AArch64::LDRSHXroX;
4018 case AArch64::LDURSHWi:
4019 case AArch64::LDRSHWui:
4020 return AArch64::LDRSHWroX;
4021 case AArch64::LDURBi:
4022 case AArch64::LDRBui:
4023 return AArch64::LDRBroX;
4024 case AArch64::LDURBBi:
4025 case AArch64::LDRBBui:
4026 return AArch64::LDRBBroX;
4027 case AArch64::LDURSBXi:
4028 case AArch64::LDRSBXui:
4029 return AArch64::LDRSBXroX;
4030 case AArch64::LDURSBWi:
4031 case AArch64::LDRSBWui:
4032 return AArch64::LDRSBWroX;
4033 case AArch64::STURBi:
4034 case AArch64::STRBui:
4035 return AArch64::STRBroX;
4036 case AArch64::STURBBi:
4037 case AArch64::STRBBui:
4038 return AArch64::STRBBroX;
4039 }
4040}
4041
4042// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4043// the opcode of an instruction performing the same operation, but using the
4044// [Reg, #Imm] addressing mode with scaled offset.
4045unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale) {
4046 switch (Opcode) {
4047 default:
4048 llvm_unreachable("Address folding not implemented for instruction");
4049
4050 case AArch64::LDURQi:
4051 Scale = 16;
4052 return AArch64::LDRQui;
4053 case AArch64::STURQi:
4054 Scale = 16;
4055 return AArch64::STRQui;
4056 case AArch64::LDURDi:
4057 Scale = 8;
4058 return AArch64::LDRDui;
4059 case AArch64::STURDi:
4060 Scale = 8;
4061 return AArch64::STRDui;
4062 case AArch64::LDURXi:
4063 Scale = 8;
4064 return AArch64::LDRXui;
4065 case AArch64::STURXi:
4066 Scale = 8;
4067 return AArch64::STRXui;
4068 case AArch64::LDURWi:
4069 Scale = 4;
4070 return AArch64::LDRWui;
4071 case AArch64::LDURSWi:
4072 Scale = 4;
4073 return AArch64::LDRSWui;
4074 case AArch64::STURWi:
4075 Scale = 4;
4076 return AArch64::STRWui;
4077 case AArch64::LDURHi:
4078 Scale = 2;
4079 return AArch64::LDRHui;
4080 case AArch64::STURHi:
4081 Scale = 2;
4082 return AArch64::STRHui;
4083 case AArch64::LDURHHi:
4084 Scale = 2;
4085 return AArch64::LDRHHui;
4086 case AArch64::STURHHi:
4087 Scale = 2;
4088 return AArch64::STRHHui;
4089 case AArch64::LDURSHXi:
4090 Scale = 2;
4091 return AArch64::LDRSHXui;
4092 case AArch64::LDURSHWi:
4093 Scale = 2;
4094 return AArch64::LDRSHWui;
4095 case AArch64::LDURBi:
4096 Scale = 1;
4097 return AArch64::LDRBui;
4098 case AArch64::LDURBBi:
4099 Scale = 1;
4100 return AArch64::LDRBBui;
4101 case AArch64::LDURSBXi:
4102 Scale = 1;
4103 return AArch64::LDRSBXui;
4104 case AArch64::LDURSBWi:
4105 Scale = 1;
4106 return AArch64::LDRSBWui;
4107 case AArch64::STURBi:
4108 Scale = 1;
4109 return AArch64::STRBui;
4110 case AArch64::STURBBi:
4111 Scale = 1;
4112 return AArch64::STRBBui;
4113 case AArch64::LDRQui:
4114 case AArch64::STRQui:
4115 Scale = 16;
4116 return Opcode;
4117 case AArch64::LDRDui:
4118 case AArch64::STRDui:
4119 case AArch64::LDRXui:
4120 case AArch64::STRXui:
4121 Scale = 8;
4122 return Opcode;
4123 case AArch64::LDRWui:
4124 case AArch64::LDRSWui:
4125 case AArch64::STRWui:
4126 Scale = 4;
4127 return Opcode;
4128 case AArch64::LDRHui:
4129 case AArch64::STRHui:
4130 case AArch64::LDRHHui:
4131 case AArch64::STRHHui:
4132 case AArch64::LDRSHXui:
4133 case AArch64::LDRSHWui:
4134 Scale = 2;
4135 return Opcode;
4136 case AArch64::LDRBui:
4137 case AArch64::LDRBBui:
4138 case AArch64::LDRSBXui:
4139 case AArch64::LDRSBWui:
4140 case AArch64::STRBui:
4141 case AArch64::STRBBui:
4142 Scale = 1;
4143 return Opcode;
4144 }
4145}
4146
4147// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4148// the opcode of an instruction performing the same operation, but using the
4149// [Reg, #Imm] addressing mode with unscaled offset.
4150unsigned unscaledOffsetOpcode(unsigned Opcode) {
4151 switch (Opcode) {
4152 default:
4153 llvm_unreachable("Address folding not implemented for instruction");
4154
4155 case AArch64::LDURQi:
4156 case AArch64::STURQi:
4157 case AArch64::LDURDi:
4158 case AArch64::STURDi:
4159 case AArch64::LDURXi:
4160 case AArch64::STURXi:
4161 case AArch64::LDURWi:
4162 case AArch64::LDURSWi:
4163 case AArch64::STURWi:
4164 case AArch64::LDURHi:
4165 case AArch64::STURHi:
4166 case AArch64::LDURHHi:
4167 case AArch64::STURHHi:
4168 case AArch64::LDURSHXi:
4169 case AArch64::LDURSHWi:
4170 case AArch64::LDURBi:
4171 case AArch64::STURBi:
4172 case AArch64::LDURBBi:
4173 case AArch64::STURBBi:
4174 case AArch64::LDURSBWi:
4175 case AArch64::LDURSBXi:
4176 return Opcode;
4177 case AArch64::LDRQui:
4178 return AArch64::LDURQi;
4179 case AArch64::STRQui:
4180 return AArch64::STURQi;
4181 case AArch64::LDRDui:
4182 return AArch64::LDURDi;
4183 case AArch64::STRDui:
4184 return AArch64::STURDi;
4185 case AArch64::LDRXui:
4186 return AArch64::LDURXi;
4187 case AArch64::STRXui:
4188 return AArch64::STURXi;
4189 case AArch64::LDRWui:
4190 return AArch64::LDURWi;
4191 case AArch64::LDRSWui:
4192 return AArch64::LDURSWi;
4193 case AArch64::STRWui:
4194 return AArch64::STURWi;
4195 case AArch64::LDRHui:
4196 return AArch64::LDURHi;
4197 case AArch64::STRHui:
4198 return AArch64::STURHi;
4199 case AArch64::LDRHHui:
4200 return AArch64::LDURHHi;
4201 case AArch64::STRHHui:
4202 return AArch64::STURHHi;
4203 case AArch64::LDRSHXui:
4204 return AArch64::LDURSHXi;
4205 case AArch64::LDRSHWui:
4206 return AArch64::LDURSHWi;
4207 case AArch64::LDRBBui:
4208 return AArch64::LDURBBi;
4209 case AArch64::LDRBui:
4210 return AArch64::LDURBi;
4211 case AArch64::STRBBui:
4212 return AArch64::STURBBi;
4213 case AArch64::STRBui:
4214 return AArch64::STURBi;
4215 case AArch64::LDRSBWui:
4216 return AArch64::LDURSBWi;
4217 case AArch64::LDRSBXui:
4218 return AArch64::LDURSBXi;
4219 }
4220}
4221
4222// Given the opcode of a memory load/store instruction, return the opcode of an
4223// instruction performing the same operation, but using
4224// the [Reg, Reg, {s,u}xtw #N] addressing mode with sign-/zero-extend of the
4225// offset register.
4226static unsigned offsetExtendOpcode(unsigned Opcode) {
4227 switch (Opcode) {
4228 default:
4229 llvm_unreachable("Address folding not implemented for instruction");
4230
4231 case AArch64::LDRQroX:
4232 case AArch64::LDURQi:
4233 case AArch64::LDRQui:
4234 return AArch64::LDRQroW;
4235 case AArch64::STRQroX:
4236 case AArch64::STURQi:
4237 case AArch64::STRQui:
4238 return AArch64::STRQroW;
4239 case AArch64::LDRDroX:
4240 case AArch64::LDURDi:
4241 case AArch64::LDRDui:
4242 return AArch64::LDRDroW;
4243 case AArch64::STRDroX:
4244 case AArch64::STURDi:
4245 case AArch64::STRDui:
4246 return AArch64::STRDroW;
4247 case AArch64::LDRXroX:
4248 case AArch64::LDURXi:
4249 case AArch64::LDRXui:
4250 return AArch64::LDRXroW;
4251 case AArch64::STRXroX:
4252 case AArch64::STURXi:
4253 case AArch64::STRXui:
4254 return AArch64::STRXroW;
4255 case AArch64::LDRWroX:
4256 case AArch64::LDURWi:
4257 case AArch64::LDRWui:
4258 return AArch64::LDRWroW;
4259 case AArch64::LDRSWroX:
4260 case AArch64::LDURSWi:
4261 case AArch64::LDRSWui:
4262 return AArch64::LDRSWroW;
4263 case AArch64::STRWroX:
4264 case AArch64::STURWi:
4265 case AArch64::STRWui:
4266 return AArch64::STRWroW;
4267 case AArch64::LDRHroX:
4268 case AArch64::LDURHi:
4269 case AArch64::LDRHui:
4270 return AArch64::LDRHroW;
4271 case AArch64::STRHroX:
4272 case AArch64::STURHi:
4273 case AArch64::STRHui:
4274 return AArch64::STRHroW;
4275 case AArch64::LDRHHroX:
4276 case AArch64::LDURHHi:
4277 case AArch64::LDRHHui:
4278 return AArch64::LDRHHroW;
4279 case AArch64::STRHHroX:
4280 case AArch64::STURHHi:
4281 case AArch64::STRHHui:
4282 return AArch64::STRHHroW;
4283 case AArch64::LDRSHXroX:
4284 case AArch64::LDURSHXi:
4285 case AArch64::LDRSHXui:
4286 return AArch64::LDRSHXroW;
4287 case AArch64::LDRSHWroX:
4288 case AArch64::LDURSHWi:
4289 case AArch64::LDRSHWui:
4290 return AArch64::LDRSHWroW;
4291 case AArch64::LDRBroX:
4292 case AArch64::LDURBi:
4293 case AArch64::LDRBui:
4294 return AArch64::LDRBroW;
4295 case AArch64::LDRBBroX:
4296 case AArch64::LDURBBi:
4297 case AArch64::LDRBBui:
4298 return AArch64::LDRBBroW;
4299 case AArch64::LDRSBXroX:
4300 case AArch64::LDURSBXi:
4301 case AArch64::LDRSBXui:
4302 return AArch64::LDRSBXroW;
4303 case AArch64::LDRSBWroX:
4304 case AArch64::LDURSBWi:
4305 case AArch64::LDRSBWui:
4306 return AArch64::LDRSBWroW;
4307 case AArch64::STRBroX:
4308 case AArch64::STURBi:
4309 case AArch64::STRBui:
4310 return AArch64::STRBroW;
4311 case AArch64::STRBBroX:
4312 case AArch64::STURBBi:
4313 case AArch64::STRBBui:
4314 return AArch64::STRBBroW;
4315 }
4316}
4317
4318MachineInstr *AArch64InstrInfo::emitLdStWithAddr(MachineInstr &MemI,
4319 const ExtAddrMode &AM) const {
4320
4321 const DebugLoc &DL = MemI.getDebugLoc();
4322 MachineBasicBlock &MBB = *MemI.getParent();
4323 MachineRegisterInfo &MRI = MemI.getMF()->getRegInfo();
4324
4325 if (AM.Form == ExtAddrMode::Formula::Basic) {
4326 if (AM.ScaledReg) {
4327 // The new instruction will be in the form `ldr Rt, [Xn, Xm, lsl #imm]`.
4328 unsigned Opcode = regOffsetOpcode(Opcode: MemI.getOpcode());
4329 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4330 auto B = BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4331 .addReg(RegNo: MemI.getOperand(i: 0).getReg(),
4332 Flags: getDefRegState(B: MemI.mayLoad()))
4333 .addReg(RegNo: AM.BaseReg)
4334 .addReg(RegNo: AM.ScaledReg)
4335 .addImm(Val: 0)
4336 .addImm(Val: AM.Scale > 1)
4337 .setMemRefs(MemI.memoperands())
4338 .setMIFlags(MemI.getFlags());
4339 return B.getInstr();
4340 }
4341
4342 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
4343 "Addressing mode not supported for folding");
4344
4345 // The new instruction will be in the form `ld[u]r Rt, [Xn, #imm]`.
4346 unsigned Scale = 1;
4347 unsigned Opcode = MemI.getOpcode();
4348 if (isInt<9>(x: AM.Displacement))
4349 Opcode = unscaledOffsetOpcode(Opcode);
4350 else
4351 Opcode = scaledOffsetOpcode(Opcode, Scale);
4352
4353 auto B =
4354 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4355 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4356 .addReg(RegNo: AM.BaseReg)
4357 .addImm(Val: AM.Displacement / Scale)
4358 .setMemRefs(MemI.memoperands())
4359 .setMIFlags(MemI.getFlags());
4360 return B.getInstr();
4361 }
4362
4363 if (AM.Form == ExtAddrMode::Formula::SExtScaledReg ||
4364 AM.Form == ExtAddrMode::Formula::ZExtScaledReg) {
4365 // The new instruction will be in the form `ldr Rt, [Xn, Wm, {s,u}xtw #N]`.
4366 assert(AM.ScaledReg && !AM.Displacement &&
4367 "Address offset can be a register or an immediate, but not both");
4368 unsigned Opcode = offsetExtendOpcode(Opcode: MemI.getOpcode());
4369 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4370 // Make sure the offset register is in the correct register class.
4371 Register OffsetReg = AM.ScaledReg;
4372 const TargetRegisterClass *RC = MRI.getRegClass(Reg: OffsetReg);
4373 if (RC->hasSuperClassEq(RC: &AArch64::GPR64RegClass)) {
4374 OffsetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4375 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: OffsetReg)
4376 .addReg(RegNo: AM.ScaledReg, Flags: {}, SubReg: AArch64::sub_32);
4377 }
4378 auto B =
4379 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4380 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4381 .addReg(RegNo: AM.BaseReg)
4382 .addReg(RegNo: OffsetReg)
4383 .addImm(Val: AM.Form == ExtAddrMode::Formula::SExtScaledReg)
4384 .addImm(Val: AM.Scale != 1)
4385 .setMemRefs(MemI.memoperands())
4386 .setMIFlags(MemI.getFlags());
4387
4388 return B.getInstr();
4389 }
4390
4391 llvm_unreachable(
4392 "Function must not be called with an addressing mode it can't handle");
4393}
4394
4395/// Return true if the opcode is a post-index ld/st instruction, which really
4396/// loads from base+0.
4397static bool isPostIndexLdStOpcode(unsigned Opcode) {
4398 switch (Opcode) {
4399 default:
4400 return false;
4401 case AArch64::LD1Fourv16b_POST:
4402 case AArch64::LD1Fourv1d_POST:
4403 case AArch64::LD1Fourv2d_POST:
4404 case AArch64::LD1Fourv2s_POST:
4405 case AArch64::LD1Fourv4h_POST:
4406 case AArch64::LD1Fourv4s_POST:
4407 case AArch64::LD1Fourv8b_POST:
4408 case AArch64::LD1Fourv8h_POST:
4409 case AArch64::LD1Onev16b_POST:
4410 case AArch64::LD1Onev1d_POST:
4411 case AArch64::LD1Onev2d_POST:
4412 case AArch64::LD1Onev2s_POST:
4413 case AArch64::LD1Onev4h_POST:
4414 case AArch64::LD1Onev4s_POST:
4415 case AArch64::LD1Onev8b_POST:
4416 case AArch64::LD1Onev8h_POST:
4417 case AArch64::LD1Rv16b_POST:
4418 case AArch64::LD1Rv1d_POST:
4419 case AArch64::LD1Rv2d_POST:
4420 case AArch64::LD1Rv2s_POST:
4421 case AArch64::LD1Rv4h_POST:
4422 case AArch64::LD1Rv4s_POST:
4423 case AArch64::LD1Rv8b_POST:
4424 case AArch64::LD1Rv8h_POST:
4425 case AArch64::LD1Threev16b_POST:
4426 case AArch64::LD1Threev1d_POST:
4427 case AArch64::LD1Threev2d_POST:
4428 case AArch64::LD1Threev2s_POST:
4429 case AArch64::LD1Threev4h_POST:
4430 case AArch64::LD1Threev4s_POST:
4431 case AArch64::LD1Threev8b_POST:
4432 case AArch64::LD1Threev8h_POST:
4433 case AArch64::LD1Twov16b_POST:
4434 case AArch64::LD1Twov1d_POST:
4435 case AArch64::LD1Twov2d_POST:
4436 case AArch64::LD1Twov2s_POST:
4437 case AArch64::LD1Twov4h_POST:
4438 case AArch64::LD1Twov4s_POST:
4439 case AArch64::LD1Twov8b_POST:
4440 case AArch64::LD1Twov8h_POST:
4441 case AArch64::LD1i16_POST:
4442 case AArch64::LD1i32_POST:
4443 case AArch64::LD1i64_POST:
4444 case AArch64::LD1i8_POST:
4445 case AArch64::LD2Rv16b_POST:
4446 case AArch64::LD2Rv1d_POST:
4447 case AArch64::LD2Rv2d_POST:
4448 case AArch64::LD2Rv2s_POST:
4449 case AArch64::LD2Rv4h_POST:
4450 case AArch64::LD2Rv4s_POST:
4451 case AArch64::LD2Rv8b_POST:
4452 case AArch64::LD2Rv8h_POST:
4453 case AArch64::LD2Twov16b_POST:
4454 case AArch64::LD2Twov2d_POST:
4455 case AArch64::LD2Twov2s_POST:
4456 case AArch64::LD2Twov4h_POST:
4457 case AArch64::LD2Twov4s_POST:
4458 case AArch64::LD2Twov8b_POST:
4459 case AArch64::LD2Twov8h_POST:
4460 case AArch64::LD2i16_POST:
4461 case AArch64::LD2i32_POST:
4462 case AArch64::LD2i64_POST:
4463 case AArch64::LD2i8_POST:
4464 case AArch64::LD3Rv16b_POST:
4465 case AArch64::LD3Rv1d_POST:
4466 case AArch64::LD3Rv2d_POST:
4467 case AArch64::LD3Rv2s_POST:
4468 case AArch64::LD3Rv4h_POST:
4469 case AArch64::LD3Rv4s_POST:
4470 case AArch64::LD3Rv8b_POST:
4471 case AArch64::LD3Rv8h_POST:
4472 case AArch64::LD3Threev16b_POST:
4473 case AArch64::LD3Threev2d_POST:
4474 case AArch64::LD3Threev2s_POST:
4475 case AArch64::LD3Threev4h_POST:
4476 case AArch64::LD3Threev4s_POST:
4477 case AArch64::LD3Threev8b_POST:
4478 case AArch64::LD3Threev8h_POST:
4479 case AArch64::LD3i16_POST:
4480 case AArch64::LD3i32_POST:
4481 case AArch64::LD3i64_POST:
4482 case AArch64::LD3i8_POST:
4483 case AArch64::LD4Fourv16b_POST:
4484 case AArch64::LD4Fourv2d_POST:
4485 case AArch64::LD4Fourv2s_POST:
4486 case AArch64::LD4Fourv4h_POST:
4487 case AArch64::LD4Fourv4s_POST:
4488 case AArch64::LD4Fourv8b_POST:
4489 case AArch64::LD4Fourv8h_POST:
4490 case AArch64::LD4Rv16b_POST:
4491 case AArch64::LD4Rv1d_POST:
4492 case AArch64::LD4Rv2d_POST:
4493 case AArch64::LD4Rv2s_POST:
4494 case AArch64::LD4Rv4h_POST:
4495 case AArch64::LD4Rv4s_POST:
4496 case AArch64::LD4Rv8b_POST:
4497 case AArch64::LD4Rv8h_POST:
4498 case AArch64::LD4i16_POST:
4499 case AArch64::LD4i32_POST:
4500 case AArch64::LD4i64_POST:
4501 case AArch64::LD4i8_POST:
4502 case AArch64::LDAPRWpost:
4503 case AArch64::LDAPRXpost:
4504 case AArch64::LDIAPPWpost:
4505 case AArch64::LDIAPPXpost:
4506 case AArch64::LDPDpost:
4507 case AArch64::LDPQpost:
4508 case AArch64::LDPSWpost:
4509 case AArch64::LDPSpost:
4510 case AArch64::LDPWpost:
4511 case AArch64::LDPXpost:
4512 case AArch64::LDRBBpost:
4513 case AArch64::LDRBpost:
4514 case AArch64::LDRDpost:
4515 case AArch64::LDRHHpost:
4516 case AArch64::LDRHpost:
4517 case AArch64::LDRQpost:
4518 case AArch64::LDRSBWpost:
4519 case AArch64::LDRSBXpost:
4520 case AArch64::LDRSHWpost:
4521 case AArch64::LDRSHXpost:
4522 case AArch64::LDRSWpost:
4523 case AArch64::LDRSpost:
4524 case AArch64::LDRWpost:
4525 case AArch64::LDRXpost:
4526 case AArch64::ST1Fourv16b_POST:
4527 case AArch64::ST1Fourv1d_POST:
4528 case AArch64::ST1Fourv2d_POST:
4529 case AArch64::ST1Fourv2s_POST:
4530 case AArch64::ST1Fourv4h_POST:
4531 case AArch64::ST1Fourv4s_POST:
4532 case AArch64::ST1Fourv8b_POST:
4533 case AArch64::ST1Fourv8h_POST:
4534 case AArch64::ST1Onev16b_POST:
4535 case AArch64::ST1Onev1d_POST:
4536 case AArch64::ST1Onev2d_POST:
4537 case AArch64::ST1Onev2s_POST:
4538 case AArch64::ST1Onev4h_POST:
4539 case AArch64::ST1Onev4s_POST:
4540 case AArch64::ST1Onev8b_POST:
4541 case AArch64::ST1Onev8h_POST:
4542 case AArch64::ST1Threev16b_POST:
4543 case AArch64::ST1Threev1d_POST:
4544 case AArch64::ST1Threev2d_POST:
4545 case AArch64::ST1Threev2s_POST:
4546 case AArch64::ST1Threev4h_POST:
4547 case AArch64::ST1Threev4s_POST:
4548 case AArch64::ST1Threev8b_POST:
4549 case AArch64::ST1Threev8h_POST:
4550 case AArch64::ST1Twov16b_POST:
4551 case AArch64::ST1Twov1d_POST:
4552 case AArch64::ST1Twov2d_POST:
4553 case AArch64::ST1Twov2s_POST:
4554 case AArch64::ST1Twov4h_POST:
4555 case AArch64::ST1Twov4s_POST:
4556 case AArch64::ST1Twov8b_POST:
4557 case AArch64::ST1Twov8h_POST:
4558 case AArch64::ST1i16_POST:
4559 case AArch64::ST1i32_POST:
4560 case AArch64::ST1i64_POST:
4561 case AArch64::ST1i8_POST:
4562 case AArch64::ST2GPostIndex:
4563 case AArch64::ST2Twov16b_POST:
4564 case AArch64::ST2Twov2d_POST:
4565 case AArch64::ST2Twov2s_POST:
4566 case AArch64::ST2Twov4h_POST:
4567 case AArch64::ST2Twov4s_POST:
4568 case AArch64::ST2Twov8b_POST:
4569 case AArch64::ST2Twov8h_POST:
4570 case AArch64::ST2i16_POST:
4571 case AArch64::ST2i32_POST:
4572 case AArch64::ST2i64_POST:
4573 case AArch64::ST2i8_POST:
4574 case AArch64::ST3Threev16b_POST:
4575 case AArch64::ST3Threev2d_POST:
4576 case AArch64::ST3Threev2s_POST:
4577 case AArch64::ST3Threev4h_POST:
4578 case AArch64::ST3Threev4s_POST:
4579 case AArch64::ST3Threev8b_POST:
4580 case AArch64::ST3Threev8h_POST:
4581 case AArch64::ST3i16_POST:
4582 case AArch64::ST3i32_POST:
4583 case AArch64::ST3i64_POST:
4584 case AArch64::ST3i8_POST:
4585 case AArch64::ST4Fourv16b_POST:
4586 case AArch64::ST4Fourv2d_POST:
4587 case AArch64::ST4Fourv2s_POST:
4588 case AArch64::ST4Fourv4h_POST:
4589 case AArch64::ST4Fourv4s_POST:
4590 case AArch64::ST4Fourv8b_POST:
4591 case AArch64::ST4Fourv8h_POST:
4592 case AArch64::ST4i16_POST:
4593 case AArch64::ST4i32_POST:
4594 case AArch64::ST4i64_POST:
4595 case AArch64::ST4i8_POST:
4596 case AArch64::STGPostIndex:
4597 case AArch64::STGPpost:
4598 case AArch64::STPDpost:
4599 case AArch64::STPQpost:
4600 case AArch64::STPSpost:
4601 case AArch64::STPWpost:
4602 case AArch64::STPXpost:
4603 case AArch64::STRBBpost:
4604 case AArch64::STRBpost:
4605 case AArch64::STRDpost:
4606 case AArch64::STRHHpost:
4607 case AArch64::STRHpost:
4608 case AArch64::STRQpost:
4609 case AArch64::STRSpost:
4610 case AArch64::STRWpost:
4611 case AArch64::STRXpost:
4612 case AArch64::STZ2GPostIndex:
4613 case AArch64::STZGPostIndex:
4614 return true;
4615 }
4616}
4617
4618bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
4619 const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
4620 bool &OffsetIsScalable, TypeSize &Width,
4621 const TargetRegisterInfo *TRI) const {
4622 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4623 // Handle only loads/stores with base register followed by immediate offset.
4624 if (LdSt.getNumExplicitOperands() == 3) {
4625 // Non-paired instruction (e.g., ldr x1, [x0, #8]).
4626 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
4627 !LdSt.getOperand(i: 2).isImm())
4628 return false;
4629 } else if (LdSt.getNumExplicitOperands() == 4) {
4630 // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
4631 if (!LdSt.getOperand(i: 1).isReg() ||
4632 (!LdSt.getOperand(i: 2).isReg() && !LdSt.getOperand(i: 2).isFI()) ||
4633 !LdSt.getOperand(i: 3).isImm())
4634 return false;
4635 } else
4636 return false;
4637
4638 // Get the scaling factor for the instruction and set the width for the
4639 // instruction.
4640 TypeSize Scale(0U, false);
4641 int64_t Dummy1, Dummy2;
4642
4643 // If this returns false, then it's an instruction we don't want to handle.
4644 if (!getMemOpInfo(Opcode: LdSt.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2))
4645 return false;
4646
4647 // Compute the offset. Offset is calculated as the immediate operand
4648 // multiplied by the scaling factor. Unscaled instructions have scaling factor
4649 // set to 1. Postindex are a special case which have an offset of 0.
4650 if (isPostIndexLdStOpcode(Opcode: LdSt.getOpcode())) {
4651 BaseOp = &LdSt.getOperand(i: 2);
4652 Offset = 0;
4653 } else if (LdSt.getNumExplicitOperands() == 3) {
4654 BaseOp = &LdSt.getOperand(i: 1);
4655 Offset = LdSt.getOperand(i: 2).getImm() * Scale.getKnownMinValue();
4656 } else {
4657 assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
4658 BaseOp = &LdSt.getOperand(i: 2);
4659 Offset = LdSt.getOperand(i: 3).getImm() * Scale.getKnownMinValue();
4660 }
4661 OffsetIsScalable = Scale.isScalable();
4662
4663 return BaseOp->isReg() || BaseOp->isFI();
4664}
4665
4666MachineOperand &
4667AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
4668 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4669 MachineOperand &OfsOp = LdSt.getOperand(i: LdSt.getNumExplicitOperands() - 1);
4670 assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
4671 return OfsOp;
4672}
4673
4674bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
4675 TypeSize &Width, int64_t &MinOffset,
4676 int64_t &MaxOffset) {
4677 switch (Opcode) {
4678 // Not a memory operation or something we want to handle.
4679 default:
4680 Scale = Width = TypeSize::getFixed(ExactSize: 0);
4681 MinOffset = MaxOffset = 0;
4682 return false;
4683 // LDR / STR
4684 case AArch64::LDRQui:
4685 case AArch64::STRQui:
4686 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4687 MinOffset = 0;
4688 MaxOffset = 4095;
4689 break;
4690 case AArch64::LDRXui:
4691 case AArch64::LDRDui:
4692 case AArch64::STRXui:
4693 case AArch64::STRDui:
4694 case AArch64::PRFMui:
4695 Scale = Width = TypeSize::getFixed(ExactSize: 8);
4696 MinOffset = 0;
4697 MaxOffset = 4095;
4698 break;
4699 case AArch64::LDRWui:
4700 case AArch64::LDRSui:
4701 case AArch64::LDRSWui:
4702 case AArch64::STRWui:
4703 case AArch64::STRSui:
4704 Scale = Width = TypeSize::getFixed(ExactSize: 4);
4705 MinOffset = 0;
4706 MaxOffset = 4095;
4707 break;
4708 case AArch64::LDRHui:
4709 case AArch64::LDRHHui:
4710 case AArch64::LDRSHWui:
4711 case AArch64::LDRSHXui:
4712 case AArch64::STRHui:
4713 case AArch64::STRHHui:
4714 Scale = Width = TypeSize::getFixed(ExactSize: 2);
4715 MinOffset = 0;
4716 MaxOffset = 4095;
4717 break;
4718 case AArch64::LDRBui:
4719 case AArch64::LDRBBui:
4720 case AArch64::LDRSBWui:
4721 case AArch64::LDRSBXui:
4722 case AArch64::STRBui:
4723 case AArch64::STRBBui:
4724 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4725 MinOffset = 0;
4726 MaxOffset = 4095;
4727 break;
4728 // post/pre inc
4729 case AArch64::STRQpre:
4730 case AArch64::LDRQpost:
4731 Scale = TypeSize::getFixed(ExactSize: 1);
4732 Width = TypeSize::getFixed(ExactSize: 16);
4733 MinOffset = -256;
4734 MaxOffset = 255;
4735 break;
4736 case AArch64::LDRDpost:
4737 case AArch64::LDRDpre:
4738 case AArch64::LDRXpost:
4739 case AArch64::LDRXpre:
4740 case AArch64::STRDpost:
4741 case AArch64::STRDpre:
4742 case AArch64::STRXpost:
4743 case AArch64::STRXpre:
4744 Scale = TypeSize::getFixed(ExactSize: 1);
4745 Width = TypeSize::getFixed(ExactSize: 8);
4746 MinOffset = -256;
4747 MaxOffset = 255;
4748 break;
4749 case AArch64::STRWpost:
4750 case AArch64::STRWpre:
4751 case AArch64::LDRWpost:
4752 case AArch64::LDRWpre:
4753 case AArch64::STRSpost:
4754 case AArch64::STRSpre:
4755 case AArch64::LDRSpost:
4756 case AArch64::LDRSpre:
4757 Scale = TypeSize::getFixed(ExactSize: 1);
4758 Width = TypeSize::getFixed(ExactSize: 4);
4759 MinOffset = -256;
4760 MaxOffset = 255;
4761 break;
4762 case AArch64::LDRHpost:
4763 case AArch64::LDRHpre:
4764 case AArch64::STRHpost:
4765 case AArch64::STRHpre:
4766 case AArch64::LDRHHpost:
4767 case AArch64::LDRHHpre:
4768 case AArch64::STRHHpost:
4769 case AArch64::STRHHpre:
4770 Scale = TypeSize::getFixed(ExactSize: 1);
4771 Width = TypeSize::getFixed(ExactSize: 2);
4772 MinOffset = -256;
4773 MaxOffset = 255;
4774 break;
4775 case AArch64::LDRBpost:
4776 case AArch64::LDRBpre:
4777 case AArch64::STRBpost:
4778 case AArch64::STRBpre:
4779 case AArch64::LDRBBpost:
4780 case AArch64::LDRBBpre:
4781 case AArch64::STRBBpost:
4782 case AArch64::STRBBpre:
4783 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4784 MinOffset = -256;
4785 MaxOffset = 255;
4786 break;
4787 // Unscaled
4788 case AArch64::LDURQi:
4789 case AArch64::STURQi:
4790 Scale = TypeSize::getFixed(ExactSize: 1);
4791 Width = TypeSize::getFixed(ExactSize: 16);
4792 MinOffset = -256;
4793 MaxOffset = 255;
4794 break;
4795 case AArch64::LDURXi:
4796 case AArch64::LDURDi:
4797 case AArch64::LDAPURXi:
4798 case AArch64::STURXi:
4799 case AArch64::STURDi:
4800 case AArch64::STLURXi:
4801 case AArch64::PRFUMi:
4802 Scale = TypeSize::getFixed(ExactSize: 1);
4803 Width = TypeSize::getFixed(ExactSize: 8);
4804 MinOffset = -256;
4805 MaxOffset = 255;
4806 break;
4807 case AArch64::LDURWi:
4808 case AArch64::LDURSi:
4809 case AArch64::LDURSWi:
4810 case AArch64::LDAPURi:
4811 case AArch64::LDAPURSWi:
4812 case AArch64::STURWi:
4813 case AArch64::STURSi:
4814 case AArch64::STLURWi:
4815 Scale = TypeSize::getFixed(ExactSize: 1);
4816 Width = TypeSize::getFixed(ExactSize: 4);
4817 MinOffset = -256;
4818 MaxOffset = 255;
4819 break;
4820 case AArch64::LDURHi:
4821 case AArch64::LDURHHi:
4822 case AArch64::LDURSHXi:
4823 case AArch64::LDURSHWi:
4824 case AArch64::LDAPURHi:
4825 case AArch64::LDAPURSHWi:
4826 case AArch64::LDAPURSHXi:
4827 case AArch64::STURHi:
4828 case AArch64::STURHHi:
4829 case AArch64::STLURHi:
4830 Scale = TypeSize::getFixed(ExactSize: 1);
4831 Width = TypeSize::getFixed(ExactSize: 2);
4832 MinOffset = -256;
4833 MaxOffset = 255;
4834 break;
4835 case AArch64::LDURBi:
4836 case AArch64::LDURBBi:
4837 case AArch64::LDURSBXi:
4838 case AArch64::LDURSBWi:
4839 case AArch64::LDAPURBi:
4840 case AArch64::LDAPURSBWi:
4841 case AArch64::LDAPURSBXi:
4842 case AArch64::STURBi:
4843 case AArch64::STURBBi:
4844 case AArch64::STLURBi:
4845 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4846 MinOffset = -256;
4847 MaxOffset = 255;
4848 break;
4849 // LDP / STP (including pre/post inc)
4850 case AArch64::LDPQi:
4851 case AArch64::LDNPQi:
4852 case AArch64::STPQi:
4853 case AArch64::STNPQi:
4854 case AArch64::LDPQpost:
4855 case AArch64::LDPQpre:
4856 case AArch64::STPQpost:
4857 case AArch64::STPQpre:
4858 Scale = TypeSize::getFixed(ExactSize: 16);
4859 Width = TypeSize::getFixed(ExactSize: 16 * 2);
4860 MinOffset = -64;
4861 MaxOffset = 63;
4862 break;
4863 case AArch64::LDPXi:
4864 case AArch64::LDPDi:
4865 case AArch64::LDNPXi:
4866 case AArch64::LDNPDi:
4867 case AArch64::STPXi:
4868 case AArch64::STPDi:
4869 case AArch64::STNPXi:
4870 case AArch64::STNPDi:
4871 case AArch64::LDPDpost:
4872 case AArch64::LDPDpre:
4873 case AArch64::LDPXpost:
4874 case AArch64::LDPXpre:
4875 case AArch64::STPDpost:
4876 case AArch64::STPDpre:
4877 case AArch64::STPXpost:
4878 case AArch64::STPXpre:
4879 Scale = TypeSize::getFixed(ExactSize: 8);
4880 Width = TypeSize::getFixed(ExactSize: 8 * 2);
4881 MinOffset = -64;
4882 MaxOffset = 63;
4883 break;
4884 case AArch64::LDPWi:
4885 case AArch64::LDPSi:
4886 case AArch64::LDNPWi:
4887 case AArch64::LDNPSi:
4888 case AArch64::STPWi:
4889 case AArch64::STPSi:
4890 case AArch64::STNPWi:
4891 case AArch64::STNPSi:
4892 case AArch64::LDPSpost:
4893 case AArch64::LDPSpre:
4894 case AArch64::LDPWpost:
4895 case AArch64::LDPWpre:
4896 case AArch64::STPSpost:
4897 case AArch64::STPSpre:
4898 case AArch64::STPWpost:
4899 case AArch64::STPWpre:
4900 Scale = TypeSize::getFixed(ExactSize: 4);
4901 Width = TypeSize::getFixed(ExactSize: 4 * 2);
4902 MinOffset = -64;
4903 MaxOffset = 63;
4904 break;
4905 case AArch64::StoreSwiftAsyncContext:
4906 // Store is an STRXui, but there might be an ADDXri in the expansion too.
4907 Scale = TypeSize::getFixed(ExactSize: 1);
4908 Width = TypeSize::getFixed(ExactSize: 8);
4909 MinOffset = 0;
4910 MaxOffset = 4095;
4911 break;
4912 case AArch64::ADDG:
4913 Scale = TypeSize::getFixed(ExactSize: 16);
4914 Width = TypeSize::getFixed(ExactSize: 0);
4915 MinOffset = 0;
4916 MaxOffset = 63;
4917 break;
4918 case AArch64::TAGPstack:
4919 Scale = TypeSize::getFixed(ExactSize: 16);
4920 Width = TypeSize::getFixed(ExactSize: 0);
4921 // TAGP with a negative offset turns into SUBP, which has a maximum offset
4922 // of 63 (not 64!).
4923 MinOffset = -63;
4924 MaxOffset = 63;
4925 break;
4926 case AArch64::LDG:
4927 case AArch64::STGi:
4928 case AArch64::STGPreIndex:
4929 case AArch64::STGPostIndex:
4930 case AArch64::STZGi:
4931 case AArch64::STZGPreIndex:
4932 case AArch64::STZGPostIndex:
4933 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4934 MinOffset = -256;
4935 MaxOffset = 255;
4936 break;
4937 // SVE
4938 case AArch64::STR_ZZZZXI:
4939 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
4940 case AArch64::LDR_ZZZZXI:
4941 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
4942 Scale = TypeSize::getScalable(MinimumSize: 16);
4943 Width = TypeSize::getScalable(MinimumSize: 16 * 4);
4944 MinOffset = -256;
4945 MaxOffset = 252;
4946 break;
4947 case AArch64::STR_ZZZXI:
4948 case AArch64::LDR_ZZZXI:
4949 Scale = TypeSize::getScalable(MinimumSize: 16);
4950 Width = TypeSize::getScalable(MinimumSize: 16 * 3);
4951 MinOffset = -256;
4952 MaxOffset = 253;
4953 break;
4954 case AArch64::STR_ZZXI:
4955 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
4956 case AArch64::LDR_ZZXI:
4957 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
4958 Scale = TypeSize::getScalable(MinimumSize: 16);
4959 Width = TypeSize::getScalable(MinimumSize: 16 * 2);
4960 MinOffset = -256;
4961 MaxOffset = 254;
4962 break;
4963 case AArch64::LDR_PXI:
4964 case AArch64::STR_PXI:
4965 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
4966 MinOffset = -256;
4967 MaxOffset = 255;
4968 break;
4969 case AArch64::LDR_PPXI:
4970 case AArch64::STR_PPXI:
4971 Scale = TypeSize::getScalable(MinimumSize: 2);
4972 Width = TypeSize::getScalable(MinimumSize: 2 * 2);
4973 MinOffset = -256;
4974 MaxOffset = 254;
4975 break;
4976 case AArch64::LDR_ZXI:
4977 case AArch64::STR_ZXI:
4978 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
4979 MinOffset = -256;
4980 MaxOffset = 255;
4981 break;
4982 case AArch64::LD1B_IMM:
4983 case AArch64::LD1H_IMM:
4984 case AArch64::LD1W_IMM:
4985 case AArch64::LD1D_IMM:
4986 case AArch64::LDNT1B_ZRI:
4987 case AArch64::LDNT1H_ZRI:
4988 case AArch64::LDNT1W_ZRI:
4989 case AArch64::LDNT1D_ZRI:
4990 case AArch64::ST1B_IMM:
4991 case AArch64::ST1H_IMM:
4992 case AArch64::ST1W_IMM:
4993 case AArch64::ST1D_IMM:
4994 case AArch64::STNT1B_ZRI:
4995 case AArch64::STNT1H_ZRI:
4996 case AArch64::STNT1W_ZRI:
4997 case AArch64::STNT1D_ZRI:
4998 case AArch64::LDNF1B_IMM:
4999 case AArch64::LDNF1H_IMM:
5000 case AArch64::LDNF1W_IMM:
5001 case AArch64::LDNF1D_IMM:
5002 // A full vectors worth of data
5003 // Width = mbytes * elements
5004 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5005 MinOffset = -8;
5006 MaxOffset = 7;
5007 break;
5008 case AArch64::LD2B_IMM:
5009 case AArch64::LD2H_IMM:
5010 case AArch64::LD2W_IMM:
5011 case AArch64::LD2D_IMM:
5012 case AArch64::ST2B_IMM:
5013 case AArch64::ST2H_IMM:
5014 case AArch64::ST2W_IMM:
5015 case AArch64::ST2D_IMM:
5016 case AArch64::LD1B_2Z_IMM:
5017 case AArch64::LD1B_2Z_STRIDED_IMM:
5018 case AArch64::LD1H_2Z_IMM:
5019 case AArch64::LD1H_2Z_STRIDED_IMM:
5020 case AArch64::LD1W_2Z_IMM:
5021 case AArch64::LD1W_2Z_STRIDED_IMM:
5022 case AArch64::LD1D_2Z_IMM:
5023 case AArch64::LD1D_2Z_STRIDED_IMM:
5024 case AArch64::LD1B_2Z_IMM_PSEUDO:
5025 case AArch64::LD1H_2Z_IMM_PSEUDO:
5026 case AArch64::LD1W_2Z_IMM_PSEUDO:
5027 case AArch64::LD1D_2Z_IMM_PSEUDO:
5028 case AArch64::ST1B_2Z_IMM:
5029 case AArch64::ST1B_2Z_STRIDED_IMM:
5030 case AArch64::ST1H_2Z_IMM:
5031 case AArch64::ST1H_2Z_STRIDED_IMM:
5032 case AArch64::ST1W_2Z_IMM:
5033 case AArch64::ST1W_2Z_STRIDED_IMM:
5034 case AArch64::ST1D_2Z_IMM:
5035 case AArch64::ST1D_2Z_STRIDED_IMM:
5036 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
5037 case AArch64::LDNT1B_2Z_IMM:
5038 case AArch64::LDNT1B_2Z_STRIDED_IMM:
5039 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
5040 case AArch64::LDNT1H_2Z_IMM:
5041 case AArch64::LDNT1H_2Z_STRIDED_IMM:
5042 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
5043 case AArch64::LDNT1W_2Z_IMM:
5044 case AArch64::LDNT1W_2Z_STRIDED_IMM:
5045 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
5046 case AArch64::LDNT1D_2Z_IMM:
5047 case AArch64::LDNT1D_2Z_STRIDED_IMM:
5048 case AArch64::STNT1B_2Z_IMM:
5049 case AArch64::STNT1B_2Z_STRIDED_IMM:
5050 case AArch64::STNT1H_2Z_IMM:
5051 case AArch64::STNT1H_2Z_STRIDED_IMM:
5052 case AArch64::STNT1W_2Z_IMM:
5053 case AArch64::STNT1W_2Z_STRIDED_IMM:
5054 case AArch64::STNT1D_2Z_IMM:
5055 case AArch64::STNT1D_2Z_STRIDED_IMM:
5056 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5057 MinOffset = -8;
5058 MaxOffset = 7;
5059 break;
5060 case AArch64::LD3B_IMM:
5061 case AArch64::LD3H_IMM:
5062 case AArch64::LD3W_IMM:
5063 case AArch64::LD3D_IMM:
5064 case AArch64::ST3B_IMM:
5065 case AArch64::ST3H_IMM:
5066 case AArch64::ST3W_IMM:
5067 case AArch64::ST3D_IMM:
5068 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5069 MinOffset = -8;
5070 MaxOffset = 7;
5071 break;
5072 case AArch64::LD4B_IMM:
5073 case AArch64::LD4H_IMM:
5074 case AArch64::LD4W_IMM:
5075 case AArch64::LD4D_IMM:
5076 case AArch64::ST4B_IMM:
5077 case AArch64::ST4H_IMM:
5078 case AArch64::ST4W_IMM:
5079 case AArch64::ST4D_IMM:
5080 case AArch64::LD1B_4Z_IMM:
5081 case AArch64::LD1B_4Z_STRIDED_IMM:
5082 case AArch64::LD1H_4Z_IMM:
5083 case AArch64::LD1H_4Z_STRIDED_IMM:
5084 case AArch64::LD1W_4Z_IMM:
5085 case AArch64::LD1W_4Z_STRIDED_IMM:
5086 case AArch64::LD1D_4Z_IMM:
5087 case AArch64::LD1D_4Z_STRIDED_IMM:
5088 case AArch64::LD1B_4Z_IMM_PSEUDO:
5089 case AArch64::LD1H_4Z_IMM_PSEUDO:
5090 case AArch64::LD1W_4Z_IMM_PSEUDO:
5091 case AArch64::LD1D_4Z_IMM_PSEUDO:
5092 case AArch64::ST1B_4Z_IMM:
5093 case AArch64::ST1B_4Z_STRIDED_IMM:
5094 case AArch64::ST1H_4Z_IMM:
5095 case AArch64::ST1H_4Z_STRIDED_IMM:
5096 case AArch64::ST1W_4Z_IMM:
5097 case AArch64::ST1W_4Z_STRIDED_IMM:
5098 case AArch64::ST1D_4Z_IMM:
5099 case AArch64::ST1D_4Z_STRIDED_IMM:
5100 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
5101 case AArch64::LDNT1B_4Z_IMM:
5102 case AArch64::LDNT1B_4Z_STRIDED_IMM:
5103 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
5104 case AArch64::LDNT1H_4Z_IMM:
5105 case AArch64::LDNT1H_4Z_STRIDED_IMM:
5106 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
5107 case AArch64::LDNT1W_4Z_IMM:
5108 case AArch64::LDNT1W_4Z_STRIDED_IMM:
5109 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
5110 case AArch64::LDNT1D_4Z_IMM:
5111 case AArch64::LDNT1D_4Z_STRIDED_IMM:
5112 case AArch64::STNT1B_4Z_IMM:
5113 case AArch64::STNT1B_4Z_STRIDED_IMM:
5114 case AArch64::STNT1H_4Z_IMM:
5115 case AArch64::STNT1H_4Z_STRIDED_IMM:
5116 case AArch64::STNT1W_4Z_IMM:
5117 case AArch64::STNT1W_4Z_STRIDED_IMM:
5118 case AArch64::STNT1D_4Z_IMM:
5119 case AArch64::STNT1D_4Z_STRIDED_IMM:
5120 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5121 MinOffset = -8;
5122 MaxOffset = 7;
5123 break;
5124 case AArch64::LD1B_H_IMM:
5125 case AArch64::LD1SB_H_IMM:
5126 case AArch64::LD1H_S_IMM:
5127 case AArch64::LD1SH_S_IMM:
5128 case AArch64::LD1W_D_IMM:
5129 case AArch64::LD1SW_D_IMM:
5130 case AArch64::ST1B_H_IMM:
5131 case AArch64::ST1H_S_IMM:
5132 case AArch64::ST1W_D_IMM:
5133 case AArch64::LDNF1B_H_IMM:
5134 case AArch64::LDNF1SB_H_IMM:
5135 case AArch64::LDNF1H_S_IMM:
5136 case AArch64::LDNF1SH_S_IMM:
5137 case AArch64::LDNF1W_D_IMM:
5138 case AArch64::LDNF1SW_D_IMM:
5139 // A half vector worth of data
5140 // Width = mbytes * elements
5141 Scale = Width = TypeSize::getScalable(MinimumSize: 8);
5142 MinOffset = -8;
5143 MaxOffset = 7;
5144 break;
5145 case AArch64::LD1B_S_IMM:
5146 case AArch64::LD1SB_S_IMM:
5147 case AArch64::LD1H_D_IMM:
5148 case AArch64::LD1SH_D_IMM:
5149 case AArch64::ST1B_S_IMM:
5150 case AArch64::ST1H_D_IMM:
5151 case AArch64::LDNF1B_S_IMM:
5152 case AArch64::LDNF1SB_S_IMM:
5153 case AArch64::LDNF1H_D_IMM:
5154 case AArch64::LDNF1SH_D_IMM:
5155 // A quarter vector worth of data
5156 // Width = mbytes * elements
5157 Scale = Width = TypeSize::getScalable(MinimumSize: 4);
5158 MinOffset = -8;
5159 MaxOffset = 7;
5160 break;
5161 case AArch64::LD1B_D_IMM:
5162 case AArch64::LD1SB_D_IMM:
5163 case AArch64::ST1B_D_IMM:
5164 case AArch64::LDNF1B_D_IMM:
5165 case AArch64::LDNF1SB_D_IMM:
5166 // A eighth vector worth of data
5167 // Width = mbytes * elements
5168 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5169 MinOffset = -8;
5170 MaxOffset = 7;
5171 break;
5172 case AArch64::ST2Gi:
5173 case AArch64::ST2GPreIndex:
5174 case AArch64::ST2GPostIndex:
5175 case AArch64::STZ2Gi:
5176 case AArch64::STZ2GPreIndex:
5177 case AArch64::STZ2GPostIndex:
5178 Scale = TypeSize::getFixed(ExactSize: 16);
5179 Width = TypeSize::getFixed(ExactSize: 32);
5180 MinOffset = -256;
5181 MaxOffset = 255;
5182 break;
5183 case AArch64::STGPi:
5184 case AArch64::STGPpost:
5185 case AArch64::STGPpre:
5186 Scale = Width = TypeSize::getFixed(ExactSize: 16);
5187 MinOffset = -64;
5188 MaxOffset = 63;
5189 break;
5190 case AArch64::LD1RB_IMM:
5191 case AArch64::LD1RB_H_IMM:
5192 case AArch64::LD1RB_S_IMM:
5193 case AArch64::LD1RB_D_IMM:
5194 case AArch64::LD1RSB_H_IMM:
5195 case AArch64::LD1RSB_S_IMM:
5196 case AArch64::LD1RSB_D_IMM:
5197 Scale = Width = TypeSize::getFixed(ExactSize: 1);
5198 MinOffset = 0;
5199 MaxOffset = 63;
5200 break;
5201 case AArch64::LD1RH_IMM:
5202 case AArch64::LD1RH_S_IMM:
5203 case AArch64::LD1RH_D_IMM:
5204 case AArch64::LD1RSH_S_IMM:
5205 case AArch64::LD1RSH_D_IMM:
5206 Scale = Width = TypeSize::getFixed(ExactSize: 2);
5207 MinOffset = 0;
5208 MaxOffset = 63;
5209 break;
5210 case AArch64::LD1RW_IMM:
5211 case AArch64::LD1RW_D_IMM:
5212 case AArch64::LD1RSW_IMM:
5213 Scale = Width = TypeSize::getFixed(ExactSize: 4);
5214 MinOffset = 0;
5215 MaxOffset = 63;
5216 break;
5217 case AArch64::LD1RD_IMM:
5218 Scale = Width = TypeSize::getFixed(ExactSize: 8);
5219 MinOffset = 0;
5220 MaxOffset = 63;
5221 break;
5222 }
5223
5224 return true;
5225}
5226
5227// Scaling factor for unscaled load or store.
5228int AArch64InstrInfo::getMemScale(unsigned Opc) {
5229 switch (Opc) {
5230 default:
5231 llvm_unreachable("Opcode has unknown scale!");
5232 case AArch64::LDRBui:
5233 case AArch64::LDRBBui:
5234 case AArch64::LDURBBi:
5235 case AArch64::LDRSBWui:
5236 case AArch64::LDURSBWi:
5237 case AArch64::STRBui:
5238 case AArch64::STRBBui:
5239 case AArch64::STURBBi:
5240 return 1;
5241 case AArch64::LDRHui:
5242 case AArch64::LDRHHui:
5243 case AArch64::LDURHHi:
5244 case AArch64::LDRSHWui:
5245 case AArch64::LDURSHWi:
5246 case AArch64::STRHui:
5247 case AArch64::STRHHui:
5248 case AArch64::STURHHi:
5249 return 2;
5250 case AArch64::LDRSui:
5251 case AArch64::LDURSi:
5252 case AArch64::LDRSpre:
5253 case AArch64::LDRSWui:
5254 case AArch64::LDURSWi:
5255 case AArch64::LDRSWpre:
5256 case AArch64::LDRWpre:
5257 case AArch64::LDRWui:
5258 case AArch64::LDURWi:
5259 case AArch64::STRSui:
5260 case AArch64::STURSi:
5261 case AArch64::STRSpre:
5262 case AArch64::STRWui:
5263 case AArch64::STURWi:
5264 case AArch64::STRWpre:
5265 case AArch64::LDPSi:
5266 case AArch64::LDPSWi:
5267 case AArch64::LDPWi:
5268 case AArch64::STPSi:
5269 case AArch64::STPWi:
5270 return 4;
5271 case AArch64::LDRDui:
5272 case AArch64::LDURDi:
5273 case AArch64::LDRDpre:
5274 case AArch64::LDRXui:
5275 case AArch64::LDURXi:
5276 case AArch64::LDRXpre:
5277 case AArch64::STRDui:
5278 case AArch64::STURDi:
5279 case AArch64::STRDpre:
5280 case AArch64::STRXui:
5281 case AArch64::STURXi:
5282 case AArch64::STRXpre:
5283 case AArch64::LDPDi:
5284 case AArch64::LDPXi:
5285 case AArch64::STPDi:
5286 case AArch64::STPXi:
5287 return 8;
5288 case AArch64::LDRQui:
5289 case AArch64::LDURQi:
5290 case AArch64::STRQui:
5291 case AArch64::STURQi:
5292 case AArch64::STRQpre:
5293 case AArch64::LDPQi:
5294 case AArch64::LDRQpre:
5295 case AArch64::STPQi:
5296 case AArch64::STGi:
5297 case AArch64::STZGi:
5298 case AArch64::ST2Gi:
5299 case AArch64::STZ2Gi:
5300 case AArch64::STGPi:
5301 return 16;
5302 }
5303}
5304
5305bool AArch64InstrInfo::isPreLd(const MachineInstr &MI) {
5306 switch (MI.getOpcode()) {
5307 default:
5308 return false;
5309 case AArch64::LDRWpre:
5310 case AArch64::LDRXpre:
5311 case AArch64::LDRSWpre:
5312 case AArch64::LDRSpre:
5313 case AArch64::LDRDpre:
5314 case AArch64::LDRQpre:
5315 return true;
5316 }
5317}
5318
5319bool AArch64InstrInfo::isPreSt(const MachineInstr &MI) {
5320 switch (MI.getOpcode()) {
5321 default:
5322 return false;
5323 case AArch64::STRWpre:
5324 case AArch64::STRXpre:
5325 case AArch64::STRSpre:
5326 case AArch64::STRDpre:
5327 case AArch64::STRQpre:
5328 return true;
5329 }
5330}
5331
5332bool AArch64InstrInfo::isPreLdSt(const MachineInstr &MI) {
5333 return isPreLd(MI) || isPreSt(MI);
5334}
5335
5336bool AArch64InstrInfo::isZExtLoad(const MachineInstr &MI) {
5337 switch (MI.getOpcode()) {
5338 default:
5339 return false;
5340 case AArch64::LDURBBi:
5341 case AArch64::LDURHHi:
5342 case AArch64::LDURWi:
5343 case AArch64::LDRBBui:
5344 case AArch64::LDRHHui:
5345 case AArch64::LDRWui:
5346 case AArch64::LDRBBroX:
5347 case AArch64::LDRHHroX:
5348 case AArch64::LDRWroX:
5349 case AArch64::LDRBBroW:
5350 case AArch64::LDRHHroW:
5351 case AArch64::LDRWroW:
5352 return true;
5353 }
5354}
5355
5356bool AArch64InstrInfo::isSExtLoad(const MachineInstr &MI) {
5357 switch (MI.getOpcode()) {
5358 default:
5359 return false;
5360 case AArch64::LDURSBWi:
5361 case AArch64::LDURSHWi:
5362 case AArch64::LDURSBXi:
5363 case AArch64::LDURSHXi:
5364 case AArch64::LDURSWi:
5365 case AArch64::LDRSBWui:
5366 case AArch64::LDRSHWui:
5367 case AArch64::LDRSBXui:
5368 case AArch64::LDRSHXui:
5369 case AArch64::LDRSWui:
5370 case AArch64::LDRSBWroX:
5371 case AArch64::LDRSHWroX:
5372 case AArch64::LDRSBXroX:
5373 case AArch64::LDRSHXroX:
5374 case AArch64::LDRSWroX:
5375 case AArch64::LDRSBWroW:
5376 case AArch64::LDRSHWroW:
5377 case AArch64::LDRSBXroW:
5378 case AArch64::LDRSHXroW:
5379 case AArch64::LDRSWroW:
5380 return true;
5381 }
5382}
5383
5384bool AArch64InstrInfo::isPairedLdSt(const MachineInstr &MI) {
5385 switch (MI.getOpcode()) {
5386 default:
5387 return false;
5388 case AArch64::LDPSi:
5389 case AArch64::LDPSWi:
5390 case AArch64::LDPDi:
5391 case AArch64::LDPQi:
5392 case AArch64::LDPWi:
5393 case AArch64::LDPXi:
5394 case AArch64::STPSi:
5395 case AArch64::STPDi:
5396 case AArch64::STPQi:
5397 case AArch64::STPWi:
5398 case AArch64::STPXi:
5399 case AArch64::STGPi:
5400 return true;
5401 }
5402}
5403
5404const MachineOperand &AArch64InstrInfo::getLdStBaseOp(const MachineInstr &MI) {
5405 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5406 unsigned Idx =
5407 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 2
5408 : 1;
5409 return MI.getOperand(i: Idx);
5410}
5411
5412const MachineOperand &
5413AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
5414 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5415 unsigned Idx =
5416 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 3
5417 : 2;
5418 return MI.getOperand(i: Idx);
5419}
5420
5421const MachineOperand &
5422AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
5423 switch (MI.getOpcode()) {
5424 default:
5425 llvm_unreachable("Unexpected opcode");
5426 case AArch64::LDRBroX:
5427 case AArch64::LDRBBroX:
5428 case AArch64::LDRSBXroX:
5429 case AArch64::LDRSBWroX:
5430 case AArch64::LDRHroX:
5431 case AArch64::LDRHHroX:
5432 case AArch64::LDRSHXroX:
5433 case AArch64::LDRSHWroX:
5434 case AArch64::LDRWroX:
5435 case AArch64::LDRSroX:
5436 case AArch64::LDRSWroX:
5437 case AArch64::LDRDroX:
5438 case AArch64::LDRXroX:
5439 case AArch64::LDRQroX:
5440 return MI.getOperand(i: 4);
5441 }
5442}
5443
5444static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
5445 Register Reg) {
5446 if (MI.getParent() == nullptr)
5447 return nullptr;
5448 const MachineFunction *MF = MI.getParent()->getParent();
5449 return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
5450}
5451
5452bool AArch64InstrInfo::isHForm(const MachineInstr &MI) {
5453 auto IsHFPR = [&](const MachineOperand &Op) {
5454 if (!Op.isReg())
5455 return false;
5456 auto Reg = Op.getReg();
5457 if (Reg.isPhysical())
5458 return AArch64::FPR16RegClass.contains(Reg);
5459 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5460 return TRC == &AArch64::FPR16RegClass ||
5461 TRC == &AArch64::FPR16_loRegClass;
5462 };
5463 return llvm::any_of(Range: MI.operands(), P: IsHFPR);
5464}
5465
5466bool AArch64InstrInfo::isQForm(const MachineInstr &MI) {
5467 auto IsQFPR = [&](const MachineOperand &Op) {
5468 if (!Op.isReg())
5469 return false;
5470 auto Reg = Op.getReg();
5471 if (Reg.isPhysical())
5472 return AArch64::FPR128RegClass.contains(Reg);
5473 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5474 return TRC == &AArch64::FPR128RegClass ||
5475 TRC == &AArch64::FPR128_loRegClass;
5476 };
5477 return llvm::any_of(Range: MI.operands(), P: IsQFPR);
5478}
5479
5480bool AArch64InstrInfo::hasBTISemantics(const MachineInstr &MI) {
5481 switch (MI.getOpcode()) {
5482 case AArch64::BRK:
5483 case AArch64::HLT:
5484 case AArch64::PACIASP:
5485 case AArch64::PACIBSP:
5486 // Implicit BTI behavior.
5487 return true;
5488 case AArch64::PAUTH_PROLOGUE:
5489 // PAUTH_PROLOGUE expands to PACI(A|B)SP.
5490 return true;
5491 case AArch64::HINT: {
5492 unsigned Imm = MI.getOperand(i: 0).getImm();
5493 // Explicit BTI instruction.
5494 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
5495 return true;
5496 // PACI(A|B)SP instructions.
5497 if (Imm == 25 || Imm == 27)
5498 return true;
5499 return false;
5500 }
5501 default:
5502 return false;
5503 }
5504}
5505
5506bool AArch64InstrInfo::isFpOrNEON(Register Reg) {
5507 if (Reg == 0)
5508 return false;
5509 assert(Reg.isPhysical() && "Expected physical register in isFpOrNEON");
5510 return AArch64::FPR128RegClass.contains(Reg) ||
5511 AArch64::FPR64RegClass.contains(Reg) ||
5512 AArch64::FPR32RegClass.contains(Reg) ||
5513 AArch64::FPR16RegClass.contains(Reg) ||
5514 AArch64::FPR8RegClass.contains(Reg);
5515}
5516
5517bool AArch64InstrInfo::isFpOrNEON(const MachineInstr &MI) {
5518 auto IsFPR = [&](const MachineOperand &Op) {
5519 if (!Op.isReg())
5520 return false;
5521 auto Reg = Op.getReg();
5522 if (Reg.isPhysical())
5523 return isFpOrNEON(Reg);
5524
5525 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5526 return TRC == &AArch64::FPR128RegClass ||
5527 TRC == &AArch64::FPR128_loRegClass ||
5528 TRC == &AArch64::FPR64RegClass ||
5529 TRC == &AArch64::FPR64_loRegClass ||
5530 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
5531 TRC == &AArch64::FPR8RegClass;
5532 };
5533 return llvm::any_of(Range: MI.operands(), P: IsFPR);
5534}
5535
5536// Scale the unscaled offsets. Returns false if the unscaled offset can't be
5537// scaled.
5538static bool scaleOffset(unsigned Opc, int64_t &Offset) {
5539 int Scale = AArch64InstrInfo::getMemScale(Opc);
5540
5541 // If the byte-offset isn't a multiple of the stride, we can't scale this
5542 // offset.
5543 if (Offset % Scale != 0)
5544 return false;
5545
5546 // Convert the byte-offset used by unscaled into an "element" offset used
5547 // by the scaled pair load/store instructions.
5548 Offset /= Scale;
5549 return true;
5550}
5551
5552static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
5553 if (FirstOpc == SecondOpc)
5554 return true;
5555 // We can also pair sign-ext and zero-ext instructions.
5556 switch (FirstOpc) {
5557 default:
5558 return false;
5559 case AArch64::STRSui:
5560 case AArch64::STURSi:
5561 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
5562 case AArch64::STRDui:
5563 case AArch64::STURDi:
5564 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
5565 case AArch64::STRQui:
5566 case AArch64::STURQi:
5567 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
5568 case AArch64::STRWui:
5569 case AArch64::STURWi:
5570 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
5571 case AArch64::STRXui:
5572 case AArch64::STURXi:
5573 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
5574 case AArch64::LDRSui:
5575 case AArch64::LDURSi:
5576 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
5577 case AArch64::LDRDui:
5578 case AArch64::LDURDi:
5579 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
5580 case AArch64::LDRQui:
5581 case AArch64::LDURQi:
5582 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
5583 case AArch64::LDRWui:
5584 case AArch64::LDURWi:
5585 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
5586 case AArch64::LDRSWui:
5587 case AArch64::LDURSWi:
5588 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
5589 case AArch64::LDRXui:
5590 case AArch64::LDURXi:
5591 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
5592 }
5593 // These instructions can't be paired based on their opcodes.
5594 return false;
5595}
5596
5597static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
5598 int64_t Offset1, unsigned Opcode1, int FI2,
5599 int64_t Offset2, unsigned Opcode2) {
5600 // Accesses through fixed stack object frame indices may access a different
5601 // fixed stack slot. Check that the object offsets + offsets match.
5602 if (MFI.isFixedObjectIndex(ObjectIdx: FI1) && MFI.isFixedObjectIndex(ObjectIdx: FI2)) {
5603 int64_t ObjectOffset1 = MFI.getObjectOffset(ObjectIdx: FI1);
5604 int64_t ObjectOffset2 = MFI.getObjectOffset(ObjectIdx: FI2);
5605 assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
5606 // Convert to scaled object offsets.
5607 int Scale1 = AArch64InstrInfo::getMemScale(Opc: Opcode1);
5608 if (ObjectOffset1 % Scale1 != 0)
5609 return false;
5610 ObjectOffset1 /= Scale1;
5611 int Scale2 = AArch64InstrInfo::getMemScale(Opc: Opcode2);
5612 if (ObjectOffset2 % Scale2 != 0)
5613 return false;
5614 ObjectOffset2 /= Scale2;
5615 ObjectOffset1 += Offset1;
5616 ObjectOffset2 += Offset2;
5617 return ObjectOffset1 + 1 == ObjectOffset2;
5618 }
5619
5620 return FI1 == FI2;
5621}
5622
5623/// Detect opportunities for ldp/stp formation.
5624///
5625/// Only called for LdSt for which getMemOperandWithOffset returns true.
5626bool AArch64InstrInfo::shouldClusterMemOps(
5627 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
5628 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
5629 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
5630 unsigned NumBytes) const {
5631 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
5632 const MachineOperand &BaseOp1 = *BaseOps1.front();
5633 const MachineOperand &BaseOp2 = *BaseOps2.front();
5634 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
5635 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
5636 if (BaseOp1.getType() != BaseOp2.getType())
5637 return false;
5638
5639 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
5640 "Only base registers and frame indices are supported.");
5641
5642 // Check for both base regs and base FI.
5643 if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
5644 return false;
5645
5646 // Only cluster up to a single pair.
5647 if (ClusterSize > 2)
5648 return false;
5649
5650 if (!isPairableLdStInst(MI: FirstLdSt) || !isPairableLdStInst(MI: SecondLdSt))
5651 return false;
5652
5653 // Can we pair these instructions based on their opcodes?
5654 unsigned FirstOpc = FirstLdSt.getOpcode();
5655 unsigned SecondOpc = SecondLdSt.getOpcode();
5656 if (!canPairLdStOpc(FirstOpc, SecondOpc))
5657 return false;
5658
5659 // Can't merge volatiles or load/stores that have a hint to avoid pair
5660 // formation, for example.
5661 if (!isCandidateToMergeOrPair(MI: FirstLdSt) ||
5662 !isCandidateToMergeOrPair(MI: SecondLdSt))
5663 return false;
5664
5665 // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
5666 int64_t Offset1 = FirstLdSt.getOperand(i: 2).getImm();
5667 if (hasUnscaledLdStOffset(Opc: FirstOpc) && !scaleOffset(Opc: FirstOpc, Offset&: Offset1))
5668 return false;
5669
5670 int64_t Offset2 = SecondLdSt.getOperand(i: 2).getImm();
5671 if (hasUnscaledLdStOffset(Opc: SecondOpc) && !scaleOffset(Opc: SecondOpc, Offset&: Offset2))
5672 return false;
5673
5674 // Pairwise instructions have a 7-bit signed offset field.
5675 if (Offset1 > 63 || Offset1 < -64)
5676 return false;
5677
5678 // The caller should already have ordered First/SecondLdSt by offset.
5679 // Note: except for non-equal frame index bases
5680 if (BaseOp1.isFI()) {
5681 assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
5682 "Caller should have ordered offsets.");
5683
5684 const MachineFrameInfo &MFI =
5685 FirstLdSt.getParent()->getParent()->getFrameInfo();
5686 return shouldClusterFI(MFI, FI1: BaseOp1.getIndex(), Offset1, Opcode1: FirstOpc,
5687 FI2: BaseOp2.getIndex(), Offset2, Opcode2: SecondOpc);
5688 }
5689
5690 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
5691
5692 return Offset1 + 1 == Offset2;
5693}
5694
5695static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
5696 MCRegister Reg, unsigned SubIdx,
5697 RegState State,
5698 const TargetRegisterInfo *TRI) {
5699 if (!SubIdx)
5700 return MIB.addReg(RegNo: Reg, Flags: State);
5701
5702 if (Reg.isPhysical())
5703 return MIB.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx), Flags: State);
5704 return MIB.addReg(RegNo: Reg, Flags: State, SubReg: SubIdx);
5705}
5706
5707static bool forwardCopyWillClobberTuple(unsigned DestReg, unsigned SrcReg,
5708 unsigned NumRegs) {
5709 // We really want the positive remainder mod 32 here, that happens to be
5710 // easily obtainable with a mask.
5711 return ((DestReg - SrcReg) & 0x1f) < NumRegs;
5712}
5713
5714void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
5715 MachineBasicBlock::iterator I,
5716 const DebugLoc &DL, MCRegister DestReg,
5717 MCRegister SrcReg, bool KillSrc,
5718 unsigned Opcode,
5719 ArrayRef<unsigned> Indices) const {
5720 assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
5721 const TargetRegisterInfo *TRI = &getRegisterInfo();
5722 uint16_t DestEncoding = TRI->getEncodingValue(Reg: DestReg);
5723 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
5724 unsigned NumRegs = Indices.size();
5725
5726 int SubReg = 0, End = NumRegs, Incr = 1;
5727 if (forwardCopyWillClobberTuple(DestReg: DestEncoding, SrcReg: SrcEncoding, NumRegs)) {
5728 SubReg = NumRegs - 1;
5729 End = -1;
5730 Incr = -1;
5731 }
5732
5733 for (; SubReg != End; SubReg += Incr) {
5734 const MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode));
5735 AddSubReg(MIB, Reg: DestReg, SubIdx: Indices[SubReg], State: RegState::Define, TRI);
5736 AddSubReg(MIB, Reg: SrcReg, SubIdx: Indices[SubReg], State: {}, TRI);
5737 AddSubReg(MIB, Reg: SrcReg, SubIdx: Indices[SubReg], State: getKillRegState(B: KillSrc), TRI);
5738 }
5739}
5740
5741void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
5742 MachineBasicBlock::iterator I,
5743 const DebugLoc &DL, MCRegister DestReg,
5744 MCRegister SrcReg, bool KillSrc,
5745 unsigned Opcode, unsigned ZeroReg,
5746 llvm::ArrayRef<unsigned> Indices) const {
5747 const TargetRegisterInfo *TRI = &getRegisterInfo();
5748 unsigned NumRegs = Indices.size();
5749
5750#ifndef NDEBUG
5751 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
5752 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
5753 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5754 "GPR reg sequences should not be able to overlap");
5755#endif
5756
5757 for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
5758 const MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode));
5759 AddSubReg(MIB, Reg: DestReg, SubIdx: Indices[SubReg], State: RegState::Define, TRI);
5760 MIB.addReg(RegNo: ZeroReg);
5761 AddSubReg(MIB, Reg: SrcReg, SubIdx: Indices[SubReg], State: getKillRegState(B: KillSrc), TRI);
5762 MIB.addImm(Val: 0);
5763 }
5764}
5765
5766/// Returns true if the instruction at I is in a streaming call site region,
5767/// within a single basic block.
5768/// A "call site streaming region" starts after smstart and ends at smstop
5769/// around a call to a streaming function. This walks backward from I.
5770static bool isInStreamingCallSiteRegion(MachineBasicBlock &MBB,
5771 MachineBasicBlock::iterator I) {
5772 MachineFunction &MF = *MBB.getParent();
5773 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
5774 if (!AFI->hasStreamingModeChanges())
5775 return false;
5776 // Walk backwards to find smstart/smstop
5777 for (MachineInstr &MI : reverse(C: make_range(x: MBB.begin(), y: I))) {
5778 unsigned Opc = MI.getOpcode();
5779 if (Opc == AArch64::MSRpstatesvcrImm1 || Opc == AArch64::MSRpstatePseudo) {
5780 // Check if this is SM change (not ZA)
5781 int64_t PState = MI.getOperand(i: 0).getImm();
5782 if (PState == AArch64SVCR::SVCRSM || PState == AArch64SVCR::SVCRSMZA) {
5783 // Operand 1 is 1 for start, 0 for stop
5784 return MI.getOperand(i: 1).getImm() == 1;
5785 }
5786 }
5787 }
5788 return false;
5789}
5790
5791/// Returns true if in a streaming call site region without SME-FA64.
5792static bool mustAvoidNeonAtMBBI(const AArch64Subtarget &Subtarget,
5793 MachineBasicBlock &MBB,
5794 MachineBasicBlock::iterator I) {
5795 return !Subtarget.hasSMEFA64() && isInStreamingCallSiteRegion(MBB, I);
5796}
5797
5798void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
5799 MachineBasicBlock::iterator I,
5800 const DebugLoc &DL, Register DestReg,
5801 Register SrcReg, bool KillSrc,
5802 bool RenamableDest,
5803 bool RenamableSrc) const {
5804 ++NumCopyInstrs;
5805 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) &&
5806 AArch64::GPR32spRegClass.contains(Reg: SrcReg)) {
5807 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5808 // If either operand is WSP, expand to ADD #0.
5809 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5810 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5811 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
5812 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5813 RC: &AArch64::GPR64spRegClass);
5814 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5815 RC: &AArch64::GPR64spRegClass);
5816 // This instruction is reading and writing X registers. This may upset
5817 // the register scavenger and machine verifier, so we need to indicate
5818 // that we are reading an undefined value from SrcRegX, but a proper
5819 // value from SrcReg.
5820 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: DestRegX)
5821 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5822 .addImm(Val: 0)
5823 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
5824 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5825 ++NumZCRegMoveInstrsGPR;
5826 } else {
5827 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDWri), DestReg)
5828 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5829 .addImm(Val: 0)
5830 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5831 if (Subtarget.hasZeroCycleRegMoveGPR32())
5832 ++NumZCRegMoveInstrsGPR;
5833 }
5834 } else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5835 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5836 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
5837 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5838 RC: &AArch64::GPR64spRegClass);
5839 assert(DestRegX.isValid() && "Destination super-reg not valid");
5840 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5841 RC: &AArch64::GPR64spRegClass);
5842 assert(SrcRegX.isValid() && "Source super-reg not valid");
5843 // This instruction is reading and writing X registers. This may upset
5844 // the register scavenger and machine verifier, so we need to indicate
5845 // that we are reading an undefined value from SrcRegX, but a proper
5846 // value from SrcReg.
5847 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg: DestRegX)
5848 .addReg(RegNo: AArch64::XZR)
5849 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5850 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5851 ++NumZCRegMoveInstrsGPR;
5852 } else {
5853 // Otherwise, expand to ORR WZR.
5854 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5855 .addReg(RegNo: AArch64::WZR)
5856 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5857 if (Subtarget.hasZeroCycleRegMoveGPR32())
5858 ++NumZCRegMoveInstrsGPR;
5859 }
5860 return;
5861 }
5862
5863 // GPR32 zeroing
5864 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::WZR) {
5865 if (Subtarget.hasZeroCycleZeroingGPR64() &&
5866 !Subtarget.hasZeroCycleZeroingGPR32()) {
5867 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5868 RC: &AArch64::GPR64spRegClass);
5869 assert(DestRegX.isValid() && "Destination super-reg not valid");
5870 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: DestRegX)
5871 .addImm(Val: 0)
5872 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5873 ++NumZCZeroingInstrsGPR;
5874 } else if (Subtarget.hasZeroCycleZeroingGPR32()) {
5875 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZWi), DestReg)
5876 .addImm(Val: 0)
5877 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5878 ++NumZCZeroingInstrsGPR;
5879 } else {
5880 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5881 .addReg(RegNo: AArch64::WZR)
5882 .addReg(RegNo: AArch64::WZR);
5883 }
5884 return;
5885 }
5886
5887 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) &&
5888 AArch64::GPR64spRegClass.contains(Reg: SrcReg)) {
5889 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5890 // If either operand is SP, expand to ADD #0.
5891 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg)
5892 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5893 .addImm(Val: 0)
5894 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5895 if (Subtarget.hasZeroCycleRegMoveGPR64())
5896 ++NumZCRegMoveInstrsGPR;
5897 } else {
5898 // Otherwise, expand to ORR XZR.
5899 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5900 .addReg(RegNo: AArch64::XZR)
5901 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5902 if (Subtarget.hasZeroCycleRegMoveGPR64())
5903 ++NumZCRegMoveInstrsGPR;
5904 }
5905 return;
5906 }
5907
5908 // GPR64 zeroing
5909 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::XZR) {
5910 if (Subtarget.hasZeroCycleZeroingGPR64()) {
5911 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg)
5912 .addImm(Val: 0)
5913 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5914 ++NumZCZeroingInstrsGPR;
5915 } else {
5916 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5917 .addReg(RegNo: AArch64::XZR)
5918 .addReg(RegNo: AArch64::XZR);
5919 }
5920 return;
5921 }
5922
5923 // Copy a Predicate register by ORRing with itself.
5924 if (AArch64::PPRRegClass.contains(Reg: DestReg) &&
5925 AArch64::PPRRegClass.contains(Reg: SrcReg)) {
5926 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5927 "Unexpected SVE register.");
5928 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg)
5929 .addReg(RegNo: SrcReg) // Pg
5930 .addReg(RegNo: SrcReg)
5931 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5932 return;
5933 }
5934
5935 // Copy a predicate-as-counter register by ORRing with itself as if it
5936 // were a regular predicate (mask) register.
5937 bool DestIsPNR = AArch64::PNRRegClass.contains(Reg: DestReg);
5938 bool SrcIsPNR = AArch64::PNRRegClass.contains(Reg: SrcReg);
5939 if (DestIsPNR || SrcIsPNR) {
5940 auto ToPPR = [](MCRegister R) -> MCRegister {
5941 return (R - AArch64::PN0) + AArch64::P0;
5942 };
5943 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg.asMCReg();
5944 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg.asMCReg();
5945
5946 if (PPRSrcReg != PPRDestReg) {
5947 auto NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg: PPRDestReg)
5948 .addReg(RegNo: PPRSrcReg) // Pg
5949 .addReg(RegNo: PPRSrcReg)
5950 .addReg(RegNo: PPRSrcReg, Flags: getKillRegState(B: KillSrc));
5951 if (DestIsPNR)
5952 NewMI.addDef(RegNo: DestReg, Flags: RegState::Implicit);
5953 }
5954 return;
5955 }
5956
5957 // Copy a Z register by ORRing with itself.
5958 if (AArch64::ZPRRegClass.contains(Reg: DestReg) &&
5959 AArch64::ZPRRegClass.contains(Reg: SrcReg)) {
5960 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5961 "Unexpected SVE register.");
5962 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ), DestReg)
5963 .addReg(RegNo: SrcReg)
5964 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5965 return;
5966 }
5967
5968 // Copy a Z register pair by copying the individual sub-registers.
5969 if ((AArch64::ZPR2RegClass.contains(Reg: DestReg) ||
5970 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
5971 (AArch64::ZPR2RegClass.contains(Reg: SrcReg) ||
5972 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
5973 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5974 "Unexpected SVE register.");
5975 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
5976 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORR_ZZZ,
5977 Indices);
5978 return;
5979 }
5980
5981 // Copy a Z register triple by copying the individual sub-registers.
5982 if (AArch64::ZPR3RegClass.contains(Reg: DestReg) &&
5983 AArch64::ZPR3RegClass.contains(Reg: SrcReg)) {
5984 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5985 "Unexpected SVE register.");
5986 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
5987 AArch64::zsub2};
5988 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORR_ZZZ,
5989 Indices);
5990 return;
5991 }
5992
5993 // Copy a Z register quad by copying the individual sub-registers.
5994 if ((AArch64::ZPR4RegClass.contains(Reg: DestReg) ||
5995 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
5996 (AArch64::ZPR4RegClass.contains(Reg: SrcReg) ||
5997 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
5998 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5999 "Unexpected SVE register.");
6000 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6001 AArch64::zsub2, AArch64::zsub3};
6002 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORR_ZZZ,
6003 Indices);
6004 return;
6005 }
6006
6007 // Copy a DDDD register quad by copying the individual sub-registers.
6008 if (AArch64::DDDDRegClass.contains(Reg: DestReg) &&
6009 AArch64::DDDDRegClass.contains(Reg: SrcReg)) {
6010 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6011 AArch64::dsub2, AArch64::dsub3};
6012 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRv8i8,
6013 Indices);
6014 return;
6015 }
6016
6017 // Copy a DDD register triple by copying the individual sub-registers.
6018 if (AArch64::DDDRegClass.contains(Reg: DestReg) &&
6019 AArch64::DDDRegClass.contains(Reg: SrcReg)) {
6020 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6021 AArch64::dsub2};
6022 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRv8i8,
6023 Indices);
6024 return;
6025 }
6026
6027 // Copy a DD register pair by copying the individual sub-registers.
6028 if (AArch64::DDRegClass.contains(Reg: DestReg) &&
6029 AArch64::DDRegClass.contains(Reg: SrcReg)) {
6030 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
6031 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRv8i8,
6032 Indices);
6033 return;
6034 }
6035
6036 // Copy a QQQQ register quad by copying the individual sub-registers.
6037 if (AArch64::QQQQRegClass.contains(Reg: DestReg) &&
6038 AArch64::QQQQRegClass.contains(Reg: SrcReg)) {
6039 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6040 AArch64::qsub2, AArch64::qsub3};
6041 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRv16i8,
6042 Indices);
6043 return;
6044 }
6045
6046 // Copy a QQQ register triple by copying the individual sub-registers.
6047 if (AArch64::QQQRegClass.contains(Reg: DestReg) &&
6048 AArch64::QQQRegClass.contains(Reg: SrcReg)) {
6049 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6050 AArch64::qsub2};
6051 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRv16i8,
6052 Indices);
6053 return;
6054 }
6055
6056 // Copy a QQ register pair by copying the individual sub-registers.
6057 if (AArch64::QQRegClass.contains(Reg: DestReg) &&
6058 AArch64::QQRegClass.contains(Reg: SrcReg)) {
6059 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
6060 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRv16i8,
6061 Indices);
6062 return;
6063 }
6064
6065 if (AArch64::XSeqPairsClassRegClass.contains(Reg: DestReg) &&
6066 AArch64::XSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6067 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
6068 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRXrs,
6069 ZeroReg: AArch64::XZR, Indices);
6070 return;
6071 }
6072
6073 if (AArch64::WSeqPairsClassRegClass.contains(Reg: DestReg) &&
6074 AArch64::WSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6075 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
6076 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRWrs,
6077 ZeroReg: AArch64::WZR, Indices);
6078 return;
6079 }
6080
6081 if (AArch64::FPR128RegClass.contains(Reg: DestReg) &&
6082 AArch64::FPR128RegClass.contains(Reg: SrcReg)) {
6083 // In streaming regions, NEON is illegal but streaming-SVE is available.
6084 // Use SVE for copies if we're in a streaming region and SME is available.
6085 // With +sme-fa64, NEON is legal in streaming mode so we can use it.
6086 if ((Subtarget.isSVEorStreamingSVEAvailable() &&
6087 !Subtarget.isNeonAvailable()) ||
6088 mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6089 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ))
6090 .addReg(RegNo: AArch64::Z0 + (DestReg - AArch64::Q0), Flags: RegState::Define)
6091 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0))
6092 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0));
6093 } else if (Subtarget.isNeonAvailable()) {
6094 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg)
6095 .addReg(RegNo: SrcReg)
6096 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6097 if (Subtarget.hasZeroCycleRegMoveFPR128())
6098 ++NumZCRegMoveInstrsFPR;
6099 } else {
6100 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::STRQpre))
6101 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6102 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6103 .addReg(RegNo: AArch64::SP)
6104 .addImm(Val: -16);
6105 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::LDRQpost))
6106 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6107 .addReg(RegNo: DestReg, Flags: RegState::Define)
6108 .addReg(RegNo: AArch64::SP)
6109 .addImm(Val: 16);
6110 }
6111 return;
6112 }
6113
6114 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6115 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6116 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6117 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6118 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6119 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6120 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
6121 RC: &AArch64::FPR128RegClass);
6122 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::dsub,
6123 RC: &AArch64::FPR128RegClass);
6124 // This instruction is reading and writing Q registers. This may upset
6125 // the register scavenger and machine verifier, so we need to indicate
6126 // that we are reading an undefined value from SrcRegQ, but a proper
6127 // value from SrcReg.
6128 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6129 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6130 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6131 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6132 ++NumZCRegMoveInstrsFPR;
6133 } else {
6134 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg)
6135 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6136 if (Subtarget.hasZeroCycleRegMoveFPR64())
6137 ++NumZCRegMoveInstrsFPR;
6138 }
6139 return;
6140 }
6141
6142 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6143 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6144 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6145 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6146 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6147 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6148 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6149 RC: &AArch64::FPR128RegClass);
6150 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6151 RC: &AArch64::FPR128RegClass);
6152 // This instruction is reading and writing Q registers. This may upset
6153 // the register scavenger and machine verifier, so we need to indicate
6154 // that we are reading an undefined value from SrcRegQ, but a proper
6155 // value from SrcReg.
6156 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6157 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6158 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6159 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6160 ++NumZCRegMoveInstrsFPR;
6161 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6162 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6163 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6164 RC: &AArch64::FPR64RegClass);
6165 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6166 RC: &AArch64::FPR64RegClass);
6167 // This instruction is reading and writing D registers. This may upset
6168 // the register scavenger and machine verifier, so we need to indicate
6169 // that we are reading an undefined value from SrcRegD, but a proper
6170 // value from SrcReg.
6171 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6172 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6173 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6174 ++NumZCRegMoveInstrsFPR;
6175 } else {
6176 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6177 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6178 if (Subtarget.hasZeroCycleRegMoveFPR32())
6179 ++NumZCRegMoveInstrsFPR;
6180 }
6181 return;
6182 }
6183
6184 if (AArch64::FPR16RegClass.contains(Reg: DestReg) &&
6185 AArch64::FPR16RegClass.contains(Reg: SrcReg)) {
6186 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6187 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6188 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6189 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6190 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6191 RC: &AArch64::FPR128RegClass);
6192 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6193 RC: &AArch64::FPR128RegClass);
6194 // This instruction is reading and writing Q registers. This may upset
6195 // the register scavenger and machine verifier, so we need to indicate
6196 // that we are reading an undefined value from SrcRegQ, but a proper
6197 // value from SrcReg.
6198 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6199 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6200 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6201 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6202 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6203 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6204 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6205 RC: &AArch64::FPR64RegClass);
6206 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6207 RC: &AArch64::FPR64RegClass);
6208 // This instruction is reading and writing D registers. This may upset
6209 // the register scavenger and machine verifier, so we need to indicate
6210 // that we are reading an undefined value from SrcRegD, but a proper
6211 // value from SrcReg.
6212 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6213 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6214 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6215 } else {
6216 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6217 RC: &AArch64::FPR32RegClass);
6218 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6219 RC: &AArch64::FPR32RegClass);
6220 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6221 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6222 }
6223 return;
6224 }
6225
6226 if (AArch64::FPR8RegClass.contains(Reg: DestReg) &&
6227 AArch64::FPR8RegClass.contains(Reg: SrcReg)) {
6228 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6229 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6230 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6231 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6232 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6233 RC: &AArch64::FPR128RegClass);
6234 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6235 RC: &AArch64::FPR128RegClass);
6236 // This instruction is reading and writing Q registers. This may upset
6237 // the register scavenger and machine verifier, so we need to indicate
6238 // that we are reading an undefined value from SrcRegQ, but a proper
6239 // value from SrcReg.
6240 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6241 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6242 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6243 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6244 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6245 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6246 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6247 RC: &AArch64::FPR64RegClass);
6248 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6249 RC: &AArch64::FPR64RegClass);
6250 // This instruction is reading and writing D registers. This may upset
6251 // the register scavenger and machine verifier, so we need to indicate
6252 // that we are reading an undefined value from SrcRegD, but a proper
6253 // value from SrcReg.
6254 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6255 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6256 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6257 } else {
6258 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6259 RC: &AArch64::FPR32RegClass);
6260 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6261 RC: &AArch64::FPR32RegClass);
6262 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6263 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6264 }
6265 return;
6266 }
6267
6268 // Copies between GPR64 and FPR64.
6269 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6270 AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
6271 if (AArch64::XZR == SrcReg) {
6272 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg);
6273 } else {
6274 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVXDr), DestReg)
6275 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6276 }
6277 return;
6278 }
6279 if (AArch64::GPR64RegClass.contains(Reg: DestReg) &&
6280 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6281 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDXr), DestReg)
6282 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6283 return;
6284 }
6285 // Copies between GPR32 and FPR32.
6286 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6287 AArch64::GPR32RegClass.contains(Reg: SrcReg)) {
6288 if (AArch64::WZR == SrcReg) {
6289 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVS0), DestReg);
6290 } else {
6291 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVWSr), DestReg)
6292 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6293 }
6294 return;
6295 }
6296 if (AArch64::GPR32RegClass.contains(Reg: DestReg) &&
6297 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6298 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSWr), DestReg)
6299 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6300 return;
6301 }
6302
6303 if (DestReg == AArch64::NZCV) {
6304 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
6305 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MSR))
6306 .addImm(Val: AArch64SysReg::NZCV)
6307 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6308 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Define);
6309 return;
6310 }
6311
6312 if (SrcReg == AArch64::NZCV) {
6313 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
6314 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MRS), DestReg)
6315 .addImm(Val: AArch64SysReg::NZCV)
6316 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6317 return;
6318 }
6319
6320#ifndef NDEBUG
6321 errs() << RI.getRegAsmName(DestReg) << " = COPY " << RI.getRegAsmName(SrcReg)
6322 << "\n";
6323#endif
6324 llvm_unreachable("unimplemented reg-to-reg copy");
6325}
6326
6327static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
6328 MachineBasicBlock &MBB,
6329 MachineBasicBlock::iterator InsertBefore,
6330 const MCInstrDesc &MCID,
6331 Register SrcReg, bool IsKill,
6332 unsigned SubIdx0, unsigned SubIdx1, int FI,
6333 MachineMemOperand *MMO) {
6334 Register SrcReg0 = SrcReg;
6335 Register SrcReg1 = SrcReg;
6336 if (SrcReg.isPhysical()) {
6337 SrcReg0 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx0);
6338 SubIdx0 = 0;
6339 SrcReg1 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx1);
6340 SubIdx1 = 0;
6341 }
6342 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6343 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: IsKill), SubReg: SubIdx0)
6344 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: IsKill), SubReg: SubIdx1)
6345 .addFrameIndex(Idx: FI)
6346 .addImm(Val: 0)
6347 .addMemOperand(MMO);
6348}
6349
6350void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
6351 MachineBasicBlock::iterator MBBI,
6352 Register SrcReg, bool isKill, int FI,
6353 const TargetRegisterClass *RC,
6354 Register VReg,
6355 MachineInstr::MIFlag Flags) const {
6356 MachineFunction &MF = *MBB.getParent();
6357 MachineFrameInfo &MFI = MF.getFrameInfo();
6358
6359 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6360 MachineMemOperand *MMO =
6361 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
6362 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6363 unsigned Opc = 0;
6364 bool Offset = true;
6365 MCRegister PNRReg = MCRegister::NoRegister;
6366 unsigned StackID = TargetStackID::Default;
6367 switch (RI.getSpillSize(RC: *RC)) {
6368 case 1:
6369 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6370 Opc = AArch64::STRBui;
6371 break;
6372 case 2: {
6373 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6374 Opc = AArch64::STRHui;
6375 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
6376 AArch64::PPRRegClass.hasSubClassEq(RC)) {
6377 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6378 "Unexpected register store without SVE store instructions");
6379 Opc = AArch64::STR_PXI;
6380 StackID = TargetStackID::ScalablePredicateVector;
6381 }
6382 break;
6383 }
6384 case 4:
6385 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6386 Opc = AArch64::STRWui;
6387 if (SrcReg.isVirtual())
6388 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32RegClass);
6389 else
6390 assert(SrcReg != AArch64::WSP);
6391 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6392 Opc = AArch64::STRSui;
6393 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6394 Opc = AArch64::STR_PPXI;
6395 StackID = TargetStackID::ScalablePredicateVector;
6396 }
6397 break;
6398 case 8:
6399 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6400 Opc = AArch64::STRXui;
6401 if (SrcReg.isVirtual())
6402 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
6403 else
6404 assert(SrcReg != AArch64::SP);
6405 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6406 Opc = AArch64::STRDui;
6407 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6408 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6409 MCID: get(Opcode: AArch64::STPWi), SrcReg, IsKill: isKill,
6410 SubIdx0: AArch64::sube32, SubIdx1: AArch64::subo32, FI, MMO);
6411 return;
6412 }
6413 break;
6414 case 16:
6415 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6416 Opc = AArch64::STRQui;
6417 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6418 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6419 Opc = AArch64::ST1Twov1d;
6420 Offset = false;
6421 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6422 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6423 MCID: get(Opcode: AArch64::STPXi), SrcReg, IsKill: isKill,
6424 SubIdx0: AArch64::sube64, SubIdx1: AArch64::subo64, FI, MMO);
6425 return;
6426 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6427 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6428 "Unexpected register store without SVE store instructions");
6429 Opc = AArch64::STR_ZXI;
6430 StackID = TargetStackID::ScalableVector;
6431 }
6432 break;
6433 case 24:
6434 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6435 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6436 Opc = AArch64::ST1Threev1d;
6437 Offset = false;
6438 }
6439 break;
6440 case 32:
6441 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6442 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6443 Opc = AArch64::ST1Fourv1d;
6444 Offset = false;
6445 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6446 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6447 Opc = AArch64::ST1Twov2d;
6448 Offset = false;
6449 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6450 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6451 "Unexpected register store without SVE store instructions");
6452 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
6453 StackID = TargetStackID::ScalableVector;
6454 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6455 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6456 "Unexpected register store without SVE store instructions");
6457 Opc = AArch64::STR_ZZXI;
6458 StackID = TargetStackID::ScalableVector;
6459 }
6460 break;
6461 case 48:
6462 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6463 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6464 Opc = AArch64::ST1Threev2d;
6465 Offset = false;
6466 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6467 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6468 "Unexpected register store without SVE store instructions");
6469 Opc = AArch64::STR_ZZZXI;
6470 StackID = TargetStackID::ScalableVector;
6471 }
6472 break;
6473 case 64:
6474 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6475 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6476 Opc = AArch64::ST1Fourv2d;
6477 Offset = false;
6478 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6479 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6480 "Unexpected register store without SVE store instructions");
6481 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
6482 StackID = TargetStackID::ScalableVector;
6483 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6484 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6485 "Unexpected register store without SVE store instructions");
6486 Opc = AArch64::STR_ZZZZXI;
6487 StackID = TargetStackID::ScalableVector;
6488 }
6489 break;
6490 }
6491 assert(Opc && "Unknown register class");
6492 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6493
6494 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6495 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
6496 .addFrameIndex(Idx: FI);
6497
6498 if (Offset)
6499 MI.addImm(Val: 0);
6500 if (PNRReg.isValid())
6501 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6502 MI.addMemOperand(MMO);
6503}
6504
6505static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
6506 MachineBasicBlock &MBB,
6507 MachineBasicBlock::iterator InsertBefore,
6508 const MCInstrDesc &MCID,
6509 Register DestReg, unsigned SubIdx0,
6510 unsigned SubIdx1, int FI,
6511 MachineMemOperand *MMO) {
6512 Register DestReg0 = DestReg;
6513 Register DestReg1 = DestReg;
6514 bool IsUndef = true;
6515 if (DestReg.isPhysical()) {
6516 DestReg0 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx0);
6517 SubIdx0 = 0;
6518 DestReg1 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx1);
6519 SubIdx1 = 0;
6520 IsUndef = false;
6521 }
6522 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6523 .addReg(RegNo: DestReg0, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx0)
6524 .addReg(RegNo: DestReg1, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx1)
6525 .addFrameIndex(Idx: FI)
6526 .addImm(Val: 0)
6527 .addMemOperand(MMO);
6528}
6529
6530void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
6531 MachineBasicBlock::iterator MBBI,
6532 Register DestReg, int FI,
6533 const TargetRegisterClass *RC,
6534 Register VReg, unsigned SubReg,
6535 MachineInstr::MIFlag Flags) const {
6536 MachineFunction &MF = *MBB.getParent();
6537 MachineFrameInfo &MFI = MF.getFrameInfo();
6538 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6539 MachineMemOperand *MMO =
6540 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
6541 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6542
6543 unsigned Opc = 0;
6544 bool Offset = true;
6545 unsigned StackID = TargetStackID::Default;
6546 Register PNRReg = MCRegister::NoRegister;
6547 switch (TRI.getSpillSize(RC: *RC)) {
6548 case 1:
6549 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6550 Opc = AArch64::LDRBui;
6551 break;
6552 case 2: {
6553 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
6554 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6555 Opc = AArch64::LDRHui;
6556 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
6557 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6558 "Unexpected register load without SVE load instructions");
6559 if (IsPNR)
6560 PNRReg = DestReg;
6561 Opc = AArch64::LDR_PXI;
6562 StackID = TargetStackID::ScalablePredicateVector;
6563 }
6564 break;
6565 }
6566 case 4:
6567 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6568 Opc = AArch64::LDRWui;
6569 if (DestReg.isVirtual())
6570 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR32RegClass);
6571 else
6572 assert(DestReg != AArch64::WSP);
6573 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6574 Opc = AArch64::LDRSui;
6575 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6576 Opc = AArch64::LDR_PPXI;
6577 StackID = TargetStackID::ScalablePredicateVector;
6578 }
6579 break;
6580 case 8:
6581 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6582 Opc = AArch64::LDRXui;
6583 if (DestReg.isVirtual())
6584 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR64RegClass);
6585 else
6586 assert(DestReg != AArch64::SP);
6587 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6588 Opc = AArch64::LDRDui;
6589 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6590 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6591 MCID: get(Opcode: AArch64::LDPWi), DestReg, SubIdx0: AArch64::sube32,
6592 SubIdx1: AArch64::subo32, FI, MMO);
6593 return;
6594 }
6595 break;
6596 case 16:
6597 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6598 Opc = AArch64::LDRQui;
6599 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6600 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6601 Opc = AArch64::LD1Twov1d;
6602 Offset = false;
6603 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6604 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6605 MCID: get(Opcode: AArch64::LDPXi), DestReg, SubIdx0: AArch64::sube64,
6606 SubIdx1: AArch64::subo64, FI, MMO);
6607 return;
6608 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6609 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6610 "Unexpected register load without SVE load instructions");
6611 Opc = AArch64::LDR_ZXI;
6612 StackID = TargetStackID::ScalableVector;
6613 }
6614 break;
6615 case 24:
6616 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6617 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6618 Opc = AArch64::LD1Threev1d;
6619 Offset = false;
6620 }
6621 break;
6622 case 32:
6623 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6624 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6625 Opc = AArch64::LD1Fourv1d;
6626 Offset = false;
6627 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6628 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6629 Opc = AArch64::LD1Twov2d;
6630 Offset = false;
6631 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6632 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6633 "Unexpected register load without SVE load instructions");
6634 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
6635 StackID = TargetStackID::ScalableVector;
6636 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6637 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6638 "Unexpected register load without SVE load instructions");
6639 Opc = AArch64::LDR_ZZXI;
6640 StackID = TargetStackID::ScalableVector;
6641 }
6642 break;
6643 case 48:
6644 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6645 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6646 Opc = AArch64::LD1Threev2d;
6647 Offset = false;
6648 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6649 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6650 "Unexpected register load without SVE load instructions");
6651 Opc = AArch64::LDR_ZZZXI;
6652 StackID = TargetStackID::ScalableVector;
6653 }
6654 break;
6655 case 64:
6656 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6657 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6658 Opc = AArch64::LD1Fourv2d;
6659 Offset = false;
6660 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6661 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6662 "Unexpected register load without SVE load instructions");
6663 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
6664 StackID = TargetStackID::ScalableVector;
6665 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6666 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6667 "Unexpected register load without SVE load instructions");
6668 Opc = AArch64::LDR_ZZZZXI;
6669 StackID = TargetStackID::ScalableVector;
6670 }
6671 break;
6672 }
6673
6674 assert(Opc && "Unknown register class");
6675 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6676
6677 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6678 .addReg(RegNo: DestReg, Flags: getDefRegState(B: true))
6679 .addFrameIndex(Idx: FI);
6680 if (Offset)
6681 MI.addImm(Val: 0);
6682 if (PNRReg.isValid() && !PNRReg.isVirtual())
6683 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6684 MI.addMemOperand(MMO);
6685}
6686
6687bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
6688 const MachineInstr &UseMI,
6689 const TargetRegisterInfo *TRI) {
6690 return any_of(Range: instructionsWithoutDebug(It: std::next(x: DefMI.getIterator()),
6691 End: UseMI.getIterator()),
6692 P: [TRI](const MachineInstr &I) {
6693 return I.modifiesRegister(Reg: AArch64::NZCV, TRI) ||
6694 I.readsRegister(Reg: AArch64::NZCV, TRI);
6695 });
6696}
6697
6698void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6699 const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized) {
6700 // The smallest scalable element supported by scaled SVE addressing
6701 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6702 // byte offset must always be a multiple of 2.
6703 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6704
6705 // VGSized offsets are divided by '2', because the VG register is the
6706 // the number of 64bit granules as opposed to 128bit vector chunks,
6707 // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
6708 // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
6709 // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
6710 ByteSized = Offset.getFixed();
6711 VGSized = Offset.getScalable() / 2;
6712}
6713
6714/// Returns the offset in parts to which this frame offset can be
6715/// decomposed for the purpose of describing a frame offset.
6716/// For non-scalable offsets this is simply its byte size.
6717void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6718 const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors,
6719 int64_t &NumDataVectors) {
6720 // The smallest scalable element supported by scaled SVE addressing
6721 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6722 // byte offset must always be a multiple of 2.
6723 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6724
6725 NumBytes = Offset.getFixed();
6726 NumDataVectors = 0;
6727 NumPredicateVectors = Offset.getScalable() / 2;
6728 // This method is used to get the offsets to adjust the frame offset.
6729 // If the function requires ADDPL to be used and needs more than two ADDPL
6730 // instructions, part of the offset is folded into NumDataVectors so that it
6731 // uses ADDVL for part of it, reducing the number of ADDPL instructions.
6732 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
6733 NumPredicateVectors > 62) {
6734 NumDataVectors = NumPredicateVectors / 8;
6735 NumPredicateVectors -= NumDataVectors * 8;
6736 }
6737}
6738
6739// Convenience function to create a DWARF expression for: Constant `Operation`.
6740// This helper emits compact sequences for common cases. For example, for`-15
6741// DW_OP_plus`, this helper would create DW_OP_lit15 DW_OP_minus.
6742static void appendConstantExpr(SmallVectorImpl<char> &Expr, int64_t Constant,
6743 dwarf::LocationAtom Operation) {
6744 if (Operation == dwarf::DW_OP_plus && Constant < 0 && -Constant <= 31) {
6745 // -Constant (1 to 31)
6746 Expr.push_back(Elt: dwarf::DW_OP_lit0 - Constant);
6747 Operation = dwarf::DW_OP_minus;
6748 } else if (Constant >= 0 && Constant <= 31) {
6749 // Literal value 0 to 31
6750 Expr.push_back(Elt: dwarf::DW_OP_lit0 + Constant);
6751 } else {
6752 // Signed constant
6753 Expr.push_back(Elt: dwarf::DW_OP_consts);
6754 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: Constant);
6755 }
6756 return Expr.push_back(Elt: Operation);
6757}
6758
6759// Convenience function to create a DWARF expression for a register.
6760static void appendReadRegExpr(SmallVectorImpl<char> &Expr, unsigned RegNum) {
6761 Expr.push_back(Elt: (char)dwarf::DW_OP_bregx);
6762 appendLEB128<LEB128Sign::Unsigned>(Buffer&: Expr, Value: RegNum);
6763 Expr.push_back(Elt: 0);
6764}
6765
6766// Convenience function to create a DWARF expression for loading a register from
6767// a CFA offset.
6768static void appendLoadRegExpr(SmallVectorImpl<char> &Expr,
6769 int64_t OffsetFromDefCFA) {
6770 // This assumes the top of the DWARF stack contains the CFA.
6771 Expr.push_back(Elt: dwarf::DW_OP_dup);
6772 // Add the offset to the register.
6773 appendConstantExpr(Expr, Constant: OffsetFromDefCFA, Operation: dwarf::DW_OP_plus);
6774 // Dereference the address (loads a 64 bit value)..
6775 Expr.push_back(Elt: dwarf::DW_OP_deref);
6776}
6777
6778// Convenience function to create a comment for
6779// (+/-) NumBytes (* RegScale)?
6780static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
6781 StringRef RegScale = {}) {
6782 if (NumBytes) {
6783 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(x: NumBytes);
6784 if (!RegScale.empty())
6785 Comment << ' ' << RegScale;
6786 }
6787}
6788
6789// Creates an MCCFIInstruction:
6790// { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
6791static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
6792 unsigned Reg,
6793 const StackOffset &Offset) {
6794 int64_t NumBytes, NumVGScaledBytes;
6795 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(Offset, ByteSized&: NumBytes,
6796 VGSized&: NumVGScaledBytes);
6797 std::string CommentBuffer;
6798 llvm::raw_string_ostream Comment(CommentBuffer);
6799
6800 if (Reg == AArch64::SP)
6801 Comment << "sp";
6802 else if (Reg == AArch64::FP)
6803 Comment << "fp";
6804 else
6805 Comment << printReg(Reg, TRI: &TRI);
6806
6807 // Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
6808 SmallString<64> Expr;
6809 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6810 assert(DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
6811 // Reg + NumBytes
6812 Expr.push_back(Elt: dwarf::DW_OP_breg0 + DwarfReg);
6813 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: NumBytes);
6814 appendOffsetComment(NumBytes, Comment);
6815 if (NumVGScaledBytes) {
6816 // + VG * NumVGScaledBytes
6817 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: "* VG");
6818 appendReadRegExpr(Expr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6819 appendConstantExpr(Expr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6820 Expr.push_back(Elt: dwarf::DW_OP_plus);
6821 }
6822
6823 // Wrap this into DW_CFA_def_cfa.
6824 SmallString<64> DefCfaExpr;
6825 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
6826 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
6827 DefCfaExpr.append(RHS: Expr.str());
6828 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
6829 Comment: Comment.str());
6830}
6831
6832MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
6833 unsigned FrameReg, unsigned Reg,
6834 const StackOffset &Offset,
6835 bool LastAdjustmentWasScalable) {
6836 if (Offset.getScalable())
6837 return createDefCFAExpression(TRI, Reg, Offset);
6838
6839 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6840 return MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: int(Offset.getFixed()));
6841
6842 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6843 return MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfReg, Offset: (int)Offset.getFixed());
6844}
6845
6846MCCFIInstruction
6847llvm::createCFAOffset(const TargetRegisterInfo &TRI, unsigned Reg,
6848 const StackOffset &OffsetFromDefCFA,
6849 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6850 int64_t NumBytes, NumVGScaledBytes;
6851 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6852 Offset: OffsetFromDefCFA, ByteSized&: NumBytes, VGSized&: NumVGScaledBytes);
6853
6854 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6855
6856 // Non-scalable offsets can use DW_CFA_offset directly.
6857 if (!NumVGScaledBytes)
6858 return MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: NumBytes);
6859
6860 std::string CommentBuffer;
6861 llvm::raw_string_ostream Comment(CommentBuffer);
6862 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
6863
6864 // Build up expression (CFA + VG * NumVGScaledBytes + NumBytes)
6865 assert(NumVGScaledBytes && "Expected scalable offset");
6866 SmallString<64> OffsetExpr;
6867 // + VG * NumVGScaledBytes
6868 StringRef VGRegScale;
6869 if (IncomingVGOffsetFromDefCFA) {
6870 appendLoadRegExpr(Expr&: OffsetExpr, OffsetFromDefCFA: *IncomingVGOffsetFromDefCFA);
6871 VGRegScale = "* IncomingVG";
6872 } else {
6873 appendReadRegExpr(Expr&: OffsetExpr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6874 VGRegScale = "* VG";
6875 }
6876 appendConstantExpr(Expr&: OffsetExpr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6877 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: VGRegScale);
6878 OffsetExpr.push_back(Elt: dwarf::DW_OP_plus);
6879 if (NumBytes) {
6880 // + NumBytes
6881 appendOffsetComment(NumBytes, Comment);
6882 appendConstantExpr(Expr&: OffsetExpr, Constant: NumBytes, Operation: dwarf::DW_OP_plus);
6883 }
6884
6885 // Wrap this into DW_CFA_expression
6886 SmallString<64> CfaExpr;
6887 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
6888 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: DwarfReg);
6889 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: OffsetExpr.size());
6890 CfaExpr.append(RHS: OffsetExpr.str());
6891
6892 return MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str(), Loc: SMLoc(),
6893 Comment: Comment.str());
6894}
6895
6896// Helper function to emit a frame offset adjustment from a given
6897// pointer (SrcReg), stored into DestReg. This function is explicit
6898// in that it requires the opcode.
6899static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
6900 MachineBasicBlock::iterator MBBI,
6901 const DebugLoc &DL, unsigned DestReg,
6902 unsigned SrcReg, int64_t Offset, unsigned Opc,
6903 const TargetInstrInfo *TII,
6904 MachineInstr::MIFlag Flag, bool NeedsWinCFI,
6905 bool *HasWinCFI, bool EmitCFAOffset,
6906 StackOffset CFAOffset, unsigned FrameReg) {
6907 int Sign = 1;
6908 unsigned MaxEncoding, ShiftSize;
6909 switch (Opc) {
6910 case AArch64::ADDXri:
6911 case AArch64::ADDSXri:
6912 case AArch64::SUBXri:
6913 case AArch64::SUBSXri:
6914 MaxEncoding = 0xfff;
6915 ShiftSize = 12;
6916 break;
6917 case AArch64::ADDVL_XXI:
6918 case AArch64::ADDPL_XXI:
6919 case AArch64::ADDSVL_XXI:
6920 case AArch64::ADDSPL_XXI:
6921 MaxEncoding = 31;
6922 ShiftSize = 0;
6923 if (Offset < 0) {
6924 MaxEncoding = 32;
6925 Sign = -1;
6926 Offset = -Offset;
6927 }
6928 break;
6929 default:
6930 llvm_unreachable("Unsupported opcode");
6931 }
6932
6933 // `Offset` can be in bytes or in "scalable bytes".
6934 int VScale = 1;
6935 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
6936 VScale = 16;
6937 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
6938 VScale = 2;
6939
6940 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
6941 // scratch register. If DestReg is a virtual register, use it as the
6942 // scratch register; otherwise, create a new virtual register (to be
6943 // replaced by the scavenger at the end of PEI). That case can be optimized
6944 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
6945 // register can be loaded with offset%8 and the add/sub can use an extending
6946 // instruction with LSL#3.
6947 // Currently the function handles any offsets but generates a poor sequence
6948 // of code.
6949 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
6950
6951 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
6952 Register TmpReg = DestReg;
6953 if (TmpReg == AArch64::XZR)
6954 TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
6955 RegClass: &AArch64::GPR64RegClass);
6956 do {
6957 uint64_t ThisVal = std::min<uint64_t>(a: Offset, b: MaxEncodableValue);
6958 unsigned LocalShiftSize = 0;
6959 if (ThisVal > MaxEncoding) {
6960 ThisVal = ThisVal >> ShiftSize;
6961 LocalShiftSize = ShiftSize;
6962 }
6963 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
6964 "Encoding cannot handle value that big");
6965
6966 Offset -= ThisVal << LocalShiftSize;
6967 if (Offset == 0)
6968 TmpReg = DestReg;
6969 auto MBI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: TmpReg)
6970 .addReg(RegNo: SrcReg)
6971 .addImm(Val: Sign * (int)ThisVal);
6972 if (ShiftSize)
6973 MBI = MBI.addImm(
6974 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LocalShiftSize));
6975 MBI = MBI.setMIFlag(Flag);
6976
6977 auto Change =
6978 VScale == 1
6979 ? StackOffset::getFixed(Fixed: ThisVal << LocalShiftSize)
6980 : StackOffset::getScalable(Scalable: VScale * (ThisVal << LocalShiftSize));
6981 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
6982 CFAOffset += Change;
6983 else
6984 CFAOffset -= Change;
6985 if (EmitCFAOffset && DestReg == TmpReg) {
6986 MachineFunction &MF = *MBB.getParent();
6987 const TargetSubtargetInfo &STI = MF.getSubtarget();
6988 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
6989
6990 unsigned CFIIndex = MF.addFrameInst(
6991 Inst: createDefCFA(TRI, FrameReg, Reg: DestReg, Offset: CFAOffset, LastAdjustmentWasScalable: VScale != 1));
6992 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
6993 .addCFIIndex(CFIIndex)
6994 .setMIFlags(Flag);
6995 }
6996
6997 if (NeedsWinCFI) {
6998 int Imm = (int)(ThisVal << LocalShiftSize);
6999 if (VScale != 1 && DestReg == AArch64::SP) {
7000 if (HasWinCFI)
7001 *HasWinCFI = true;
7002 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AllocZ))
7003 .addImm(Val: ThisVal)
7004 .setMIFlag(Flag);
7005 } else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
7006 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
7007 assert(VScale == 1 && "Expected non-scalable operation");
7008 if (HasWinCFI)
7009 *HasWinCFI = true;
7010 if (Imm == 0)
7011 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_SetFP)).setMIFlag(Flag);
7012 else
7013 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AddFP))
7014 .addImm(Val: Imm)
7015 .setMIFlag(Flag);
7016 assert(Offset == 0 && "Expected remaining offset to be zero to "
7017 "emit a single SEH directive");
7018 } else if (DestReg == AArch64::SP) {
7019 assert(VScale == 1 && "Expected non-scalable operation");
7020 if (HasWinCFI)
7021 *HasWinCFI = true;
7022 assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
7023 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_StackAlloc))
7024 .addImm(Val: Imm)
7025 .setMIFlag(Flag);
7026 }
7027 }
7028
7029 SrcReg = TmpReg;
7030 } while (Offset);
7031}
7032
7033void llvm::emitFrameOffset(MachineBasicBlock &MBB,
7034 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
7035 unsigned DestReg, unsigned SrcReg,
7036 StackOffset Offset, const TargetInstrInfo *TII,
7037 MachineInstr::MIFlag Flag, bool SetNZCV,
7038 bool NeedsWinCFI, bool *HasWinCFI,
7039 bool EmitCFAOffset, StackOffset CFAOffset,
7040 unsigned FrameReg) {
7041 // If a function is marked as arm_locally_streaming, then the runtime value of
7042 // vscale in the prologue/epilogue is different the runtime value of vscale
7043 // in the function's body. To avoid having to consider multiple vscales,
7044 // we can use `addsvl` to allocate any scalable stack-slots, which under
7045 // most circumstances will be only locals, not callee-save slots.
7046 const Function &F = MBB.getParent()->getFunction();
7047 bool UseSVL = F.hasFnAttribute(Kind: "aarch64_pstate_sm_body");
7048
7049 int64_t Bytes, NumPredicateVectors, NumDataVectors;
7050 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
7051 Offset, NumBytes&: Bytes, NumPredicateVectors, NumDataVectors);
7052
7053 // Insert ADDSXri for scalable offset at the end.
7054 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
7055 if (NeedsFinalDefNZCV)
7056 SetNZCV = false;
7057
7058 // First emit non-scalable frame offsets, or a simple 'mov'.
7059 if (Bytes || (!Offset && SrcReg != DestReg)) {
7060 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
7061 "SP increment/decrement not 8-byte aligned");
7062 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
7063 if (Bytes < 0) {
7064 Bytes = -Bytes;
7065 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
7066 }
7067 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: Bytes, Opc, TII, Flag,
7068 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7069 FrameReg);
7070 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
7071 ? StackOffset::getFixed(Fixed: -Bytes)
7072 : StackOffset::getFixed(Fixed: Bytes);
7073 SrcReg = DestReg;
7074 FrameReg = DestReg;
7075 }
7076
7077 assert(!(NeedsWinCFI && NumPredicateVectors) &&
7078 "WinCFI can't allocate fractions of an SVE data vector");
7079
7080 if (NumDataVectors) {
7081 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumDataVectors,
7082 Opc: UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI, TII,
7083 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7084 FrameReg);
7085 CFAOffset += StackOffset::getScalable(Scalable: -NumDataVectors * 16);
7086 SrcReg = DestReg;
7087 }
7088
7089 if (NumPredicateVectors) {
7090 assert(DestReg != AArch64::SP && "Unaligned access to SP");
7091 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumPredicateVectors,
7092 Opc: UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI, TII,
7093 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7094 FrameReg);
7095 }
7096
7097 if (NeedsFinalDefNZCV)
7098 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDSXri), DestReg)
7099 .addReg(RegNo: DestReg)
7100 .addImm(Val: 0)
7101 .addImm(Val: 0);
7102}
7103
7104MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
7105 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
7106 int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS,
7107 VirtRegMap *VRM) const {
7108 MachineBasicBlock::iterator InsertPt = MI;
7109 // This is a bit of a hack. Consider this instruction:
7110 //
7111 // %0 = COPY %sp; GPR64all:%0
7112 //
7113 // We explicitly chose GPR64all for the virtual register so such a copy might
7114 // be eliminated by RegisterCoalescer. However, that may not be possible, and
7115 // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
7116 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
7117 //
7118 // To prevent that, we are going to constrain the %0 register class here.
7119 if (MI.isFullCopy()) {
7120 Register DstReg = MI.getOperand(i: 0).getReg();
7121 Register SrcReg = MI.getOperand(i: 1).getReg();
7122 if (SrcReg == AArch64::SP && DstReg.isVirtual()) {
7123 MF.getRegInfo().constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass);
7124 return nullptr;
7125 }
7126 if (DstReg == AArch64::SP && SrcReg.isVirtual()) {
7127 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
7128 return nullptr;
7129 }
7130 // Nothing can folded with copy from/to NZCV.
7131 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
7132 return nullptr;
7133 }
7134
7135 // Handle the case where a copy is being spilled or filled but the source
7136 // and destination register class don't match. For example:
7137 //
7138 // %0 = COPY %xzr; GPR64common:%0
7139 //
7140 // In this case we can still safely fold away the COPY and generate the
7141 // following spill code:
7142 //
7143 // STRXui %xzr, %stack.0
7144 //
7145 // This also eliminates spilled cross register class COPYs (e.g. between x and
7146 // d regs) of the same size. For example:
7147 //
7148 // %0 = COPY %1; GPR64:%0, FPR64:%1
7149 //
7150 // will be filled as
7151 //
7152 // LDRDui %0, fi<#0>
7153 //
7154 // instead of
7155 //
7156 // LDRXui %Temp, fi<#0>
7157 // %0 = FMOV %Temp
7158 //
7159 if (MI.isCopy() && Ops.size() == 1 &&
7160 // Make sure we're only folding the explicit COPY defs/uses.
7161 (Ops[0] == 0 || Ops[0] == 1)) {
7162 bool IsSpill = Ops[0] == 0;
7163 bool IsFill = !IsSpill;
7164 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7165 const MachineRegisterInfo &MRI = MF.getRegInfo();
7166 MachineBasicBlock &MBB = *MI.getParent();
7167 const MachineOperand &DstMO = MI.getOperand(i: 0);
7168 const MachineOperand &SrcMO = MI.getOperand(i: 1);
7169 Register DstReg = DstMO.getReg();
7170 Register SrcReg = SrcMO.getReg();
7171 // This is slightly expensive to compute for physical regs since
7172 // getMinimalPhysRegClass is slow.
7173 auto getRegClass = [&](unsigned Reg) {
7174 return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
7175 : TRI.getMinimalPhysRegClass(Reg);
7176 };
7177
7178 if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
7179 assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
7180 TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
7181 "Mismatched register size in non subreg COPY");
7182 if (IsSpill)
7183 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg, isKill: SrcMO.isKill(), FI: FrameIndex,
7184 RC: getRegClass(SrcReg), VReg: Register());
7185 else
7186 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex,
7187 RC: getRegClass(DstReg), VReg: Register());
7188 return &*--InsertPt;
7189 }
7190
7191 // Handle cases like spilling def of:
7192 //
7193 // %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
7194 //
7195 // where the physical register source can be widened and stored to the full
7196 // virtual reg destination stack slot, in this case producing:
7197 //
7198 // STRXui %xzr, %stack.0
7199 //
7200 if (IsSpill && DstMO.isUndef() && SrcReg == AArch64::WZR &&
7201 TRI.getRegSizeInBits(RC: *getRegClass(DstReg)) == 64) {
7202 assert(SrcMO.getSubReg() == 0 &&
7203 "Unexpected subreg on physical register");
7204 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg: AArch64::XZR, isKill: SrcMO.isKill(),
7205 FI: FrameIndex, RC: &AArch64::GPR64RegClass, VReg: Register());
7206 return &*--InsertPt;
7207 }
7208
7209 // Handle cases like filling use of:
7210 //
7211 // %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
7212 //
7213 // where we can load the full virtual reg source stack slot, into the subreg
7214 // destination, in this case producing:
7215 //
7216 // LDRWui %0:sub_32<def,read-undef>, %stack.0
7217 //
7218 if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
7219 const TargetRegisterClass *FillRC = nullptr;
7220 switch (DstMO.getSubReg()) {
7221 default:
7222 break;
7223 case AArch64::sub_32:
7224 if (AArch64::GPR64RegClass.hasSubClassEq(RC: getRegClass(DstReg)))
7225 FillRC = &AArch64::GPR32RegClass;
7226 break;
7227 case AArch64::ssub:
7228 FillRC = &AArch64::FPR32RegClass;
7229 break;
7230 case AArch64::dsub:
7231 FillRC = &AArch64::FPR64RegClass;
7232 break;
7233 }
7234
7235 if (FillRC) {
7236 assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
7237 TRI.getRegSizeInBits(*FillRC) &&
7238 "Mismatched regclass size on folded subreg COPY");
7239 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex, RC: FillRC,
7240 VReg: Register());
7241 MachineInstr &LoadMI = *--InsertPt;
7242 MachineOperand &LoadDst = LoadMI.getOperand(i: 0);
7243 assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
7244 LoadDst.setSubReg(DstMO.getSubReg());
7245 LoadDst.setIsUndef();
7246 return &LoadMI;
7247 }
7248 }
7249 }
7250
7251 // Cannot fold.
7252 return nullptr;
7253}
7254
7255int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
7256 StackOffset &SOffset,
7257 bool *OutUseUnscaledOp,
7258 unsigned *OutUnscaledOp,
7259 int64_t *EmittableOffset) {
7260 // Set output values in case of early exit.
7261 if (EmittableOffset)
7262 *EmittableOffset = 0;
7263 if (OutUseUnscaledOp)
7264 *OutUseUnscaledOp = false;
7265 if (OutUnscaledOp)
7266 *OutUnscaledOp = 0;
7267
7268 // Exit early for structured vector spills/fills as they can't take an
7269 // immediate offset.
7270 switch (MI.getOpcode()) {
7271 default:
7272 break;
7273 case AArch64::LD1Rv1d:
7274 case AArch64::LD1Rv2s:
7275 case AArch64::LD1Rv2d:
7276 case AArch64::LD1Rv4h:
7277 case AArch64::LD1Rv4s:
7278 case AArch64::LD1Rv8b:
7279 case AArch64::LD1Rv8h:
7280 case AArch64::LD1Rv16b:
7281 case AArch64::LD1Twov2d:
7282 case AArch64::LD1Threev2d:
7283 case AArch64::LD1Fourv2d:
7284 case AArch64::LD1Twov1d:
7285 case AArch64::LD1Threev1d:
7286 case AArch64::LD1Fourv1d:
7287 case AArch64::ST1Twov2d:
7288 case AArch64::ST1Threev2d:
7289 case AArch64::ST1Fourv2d:
7290 case AArch64::ST1Twov1d:
7291 case AArch64::ST1Threev1d:
7292 case AArch64::ST1Fourv1d:
7293 case AArch64::ST1i8:
7294 case AArch64::ST1i16:
7295 case AArch64::ST1i32:
7296 case AArch64::ST1i64:
7297 case AArch64::IRG:
7298 case AArch64::IRGstack:
7299 case AArch64::STGloop:
7300 case AArch64::STZGloop:
7301 return AArch64FrameOffsetCannotUpdate;
7302 }
7303
7304 // Get the min/max offset and the scale.
7305 TypeSize ScaleValue(0U, false), Width(0U, false);
7306 int64_t MinOff, MaxOff;
7307 if (!AArch64InstrInfo::getMemOpInfo(Opcode: MI.getOpcode(), Scale&: ScaleValue, Width, MinOffset&: MinOff,
7308 MaxOffset&: MaxOff))
7309 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7310
7311 // Construct the complete offset.
7312 bool IsMulVL = ScaleValue.isScalable();
7313 unsigned Scale = ScaleValue.getKnownMinValue();
7314 int64_t Offset = IsMulVL ? SOffset.getScalable() : SOffset.getFixed();
7315
7316 const MachineOperand &ImmOpnd =
7317 MI.getOperand(i: AArch64InstrInfo::getLoadStoreImmIdx(Opc: MI.getOpcode()));
7318 Offset += ImmOpnd.getImm() * Scale;
7319
7320 // If the offset doesn't match the scale, we rewrite the instruction to
7321 // use the unscaled instruction instead. Likewise, if we have a negative
7322 // offset and there is an unscaled op to use.
7323 std::optional<unsigned> UnscaledOp =
7324 AArch64InstrInfo::getUnscaledLdSt(Opc: MI.getOpcode());
7325 bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
7326 if (useUnscaledOp &&
7327 !AArch64InstrInfo::getMemOpInfo(Opcode: *UnscaledOp, Scale&: ScaleValue, Width, MinOffset&: MinOff,
7328 MaxOffset&: MaxOff))
7329 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7330
7331 Scale = ScaleValue.getKnownMinValue();
7332 assert(IsMulVL == ScaleValue.isScalable() &&
7333 "Unscaled opcode has different value for scalable");
7334
7335 int64_t Remainder = Offset % Scale;
7336 assert(!(Remainder && useUnscaledOp) &&
7337 "Cannot have remainder when using unscaled op");
7338
7339 assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
7340 int64_t NewOffset = Offset / Scale;
7341 if (MinOff <= NewOffset && NewOffset <= MaxOff)
7342 Offset = Remainder;
7343 else {
7344 // Try to minimise the number of instructions required to materialise the
7345 // offset calculation. Specifically, for fixed offsets, if masking out the
7346 // low 12 bits leaves a legal add immediate, we can realise the offset
7347 // calculation with a single add instruction. Whenever this is possible,
7348 // prefer this split.
7349 int64_t HighPart = Offset & ~0xFFF;
7350 int64_t LowPart = Offset & 0xFFF;
7351 int64_t LowScaled = LowPart / Scale;
7352 if (!IsMulVL && NewOffset >= 0 && LowPart % Scale == 0 &&
7353 MinOff <= LowScaled && LowScaled <= MaxOff &&
7354 AArch64_AM::isLegalArithImmed(C: HighPart)) {
7355 NewOffset = LowScaled;
7356 Offset = HighPart;
7357 } else {
7358 // Default to a greedy split: take the memop immediate to be maximum /
7359 // minimum expressible offset and materialise the remainder.
7360 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
7361 Offset = Offset - (NewOffset * Scale);
7362 }
7363 }
7364
7365 if (EmittableOffset)
7366 *EmittableOffset = NewOffset;
7367 if (OutUseUnscaledOp)
7368 *OutUseUnscaledOp = useUnscaledOp;
7369 if (OutUnscaledOp && UnscaledOp)
7370 *OutUnscaledOp = *UnscaledOp;
7371
7372 if (IsMulVL)
7373 SOffset = StackOffset::get(Fixed: SOffset.getFixed(), Scalable: Offset);
7374 else
7375 SOffset = StackOffset::get(Fixed: Offset, Scalable: SOffset.getScalable());
7376 return AArch64FrameOffsetCanUpdate |
7377 (SOffset ? 0 : AArch64FrameOffsetIsLegal);
7378}
7379
7380bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
7381 unsigned FrameReg, StackOffset &Offset,
7382 const AArch64InstrInfo *TII) {
7383 unsigned Opcode = MI.getOpcode();
7384 unsigned ImmIdx = FrameRegIdx + 1;
7385
7386 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
7387 Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: ImmIdx).getImm());
7388 emitFrameOffset(MBB&: *MI.getParent(), MBBI: MI, DL: MI.getDebugLoc(),
7389 DestReg: MI.getOperand(i: 0).getReg(), SrcReg: FrameReg, Offset, TII,
7390 Flag: MachineInstr::NoFlags, SetNZCV: (Opcode == AArch64::ADDSXri));
7391 MI.eraseFromParent();
7392 Offset = StackOffset();
7393 return true;
7394 }
7395
7396 int64_t NewOffset;
7397 unsigned UnscaledOp;
7398 bool UseUnscaledOp;
7399 int Status = isAArch64FrameOffsetLegal(MI, SOffset&: Offset, OutUseUnscaledOp: &UseUnscaledOp,
7400 OutUnscaledOp: &UnscaledOp, EmittableOffset: &NewOffset);
7401 if (Status & AArch64FrameOffsetCanUpdate) {
7402 if (Status & AArch64FrameOffsetIsLegal)
7403 // Replace the FrameIndex with FrameReg.
7404 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
7405 if (UseUnscaledOp)
7406 MI.setDesc(TII->get(Opcode: UnscaledOp));
7407
7408 MI.getOperand(i: ImmIdx).ChangeToImmediate(ImmVal: NewOffset);
7409 return !Offset;
7410 }
7411
7412 return false;
7413}
7414
7415void AArch64InstrInfo::insertNoop(MachineBasicBlock &MBB,
7416 MachineBasicBlock::iterator MI) const {
7417 DebugLoc DL;
7418 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::NOP));
7419}
7420
7421MCInst AArch64InstrInfo::getNop() const { return MCInstBuilder(AArch64::NOP); }
7422
7423// AArch64 supports MachineCombiner.
7424bool AArch64InstrInfo::useMachineCombiner() const { return true; }
7425
7426// True when Opc sets flag
7427static bool isCombineInstrSettingFlag(unsigned Opc) {
7428 switch (Opc) {
7429 case AArch64::ADDSWrr:
7430 case AArch64::ADDSWri:
7431 case AArch64::ADDSXrr:
7432 case AArch64::ADDSXri:
7433 case AArch64::SUBSWrr:
7434 case AArch64::SUBSXrr:
7435 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7436 case AArch64::SUBSWri:
7437 case AArch64::SUBSXri:
7438 return true;
7439 default:
7440 break;
7441 }
7442 return false;
7443}
7444
7445// 32b Opcodes that can be combined with a MUL
7446static bool isCombineInstrCandidate32(unsigned Opc) {
7447 switch (Opc) {
7448 case AArch64::ADDWrr:
7449 case AArch64::ADDWri:
7450 case AArch64::SUBWrr:
7451 case AArch64::ADDSWrr:
7452 case AArch64::ADDSWri:
7453 case AArch64::SUBSWrr:
7454 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7455 case AArch64::SUBWri:
7456 case AArch64::SUBSWri:
7457 return true;
7458 default:
7459 break;
7460 }
7461 return false;
7462}
7463
7464// 64b Opcodes that can be combined with a MUL
7465static bool isCombineInstrCandidate64(unsigned Opc) {
7466 switch (Opc) {
7467 case AArch64::ADDXrr:
7468 case AArch64::ADDXri:
7469 case AArch64::SUBXrr:
7470 case AArch64::ADDSXrr:
7471 case AArch64::ADDSXri:
7472 case AArch64::SUBSXrr:
7473 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7474 case AArch64::SUBXri:
7475 case AArch64::SUBSXri:
7476 case AArch64::ADDv8i8:
7477 case AArch64::ADDv16i8:
7478 case AArch64::ADDv4i16:
7479 case AArch64::ADDv8i16:
7480 case AArch64::ADDv2i32:
7481 case AArch64::ADDv4i32:
7482 case AArch64::SUBv8i8:
7483 case AArch64::SUBv16i8:
7484 case AArch64::SUBv4i16:
7485 case AArch64::SUBv8i16:
7486 case AArch64::SUBv2i32:
7487 case AArch64::SUBv4i32:
7488 return true;
7489 default:
7490 break;
7491 }
7492 return false;
7493}
7494
7495// FP Opcodes that can be combined with a FMUL.
7496static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
7497 switch (Inst.getOpcode()) {
7498 default:
7499 break;
7500 case AArch64::FADDHrr:
7501 case AArch64::FADDSrr:
7502 case AArch64::FADDDrr:
7503 case AArch64::FADDv4f16:
7504 case AArch64::FADDv8f16:
7505 case AArch64::FADDv2f32:
7506 case AArch64::FADDv2f64:
7507 case AArch64::FADDv4f32:
7508 case AArch64::FSUBHrr:
7509 case AArch64::FSUBSrr:
7510 case AArch64::FSUBDrr:
7511 case AArch64::FSUBv4f16:
7512 case AArch64::FSUBv8f16:
7513 case AArch64::FSUBv2f32:
7514 case AArch64::FSUBv2f64:
7515 case AArch64::FSUBv4f32:
7516 TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options;
7517 // We can fuse FADD/FSUB with FMUL, if fusion is either allowed globally by
7518 // the target options or if FADD/FSUB has the contract fast-math flag.
7519 return Options.AllowFPOpFusion == FPOpFusion::Fast ||
7520 Inst.getFlag(Flag: MachineInstr::FmContract);
7521 }
7522 return false;
7523}
7524
7525// Opcodes that can be combined with a MUL
7526static bool isCombineInstrCandidate(unsigned Opc) {
7527 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
7528}
7529
7530//
7531// Utility routine that checks if \param MO is defined by an
7532// \param CombineOpc instruction in the basic block \param MBB
7533static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
7534 unsigned CombineOpc, unsigned ZeroReg = 0,
7535 bool CheckZeroReg = false) {
7536 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7537 MachineInstr *MI = nullptr;
7538
7539 if (MO.isReg() && MO.getReg().isVirtual())
7540 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
7541 // And it needs to be in the trace (otherwise, it won't have a depth).
7542 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
7543 return false;
7544 // Must only used by the user we combine with.
7545 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
7546 return false;
7547
7548 if (CheckZeroReg) {
7549 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
7550 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
7551 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
7552 // The third input reg must be zero.
7553 if (MI->getOperand(i: 3).getReg() != ZeroReg)
7554 return false;
7555 }
7556
7557 if (isCombineInstrSettingFlag(Opc: CombineOpc) &&
7558 MI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) == -1)
7559 return false;
7560
7561 return true;
7562}
7563
7564//
7565// Is \param MO defined by an integer multiply and can be combined?
7566static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7567 unsigned MulOpc, unsigned ZeroReg) {
7568 return canCombine(MBB, MO, CombineOpc: MulOpc, ZeroReg, CheckZeroReg: true);
7569}
7570
7571//
7572// Is \param MO defined by a floating-point multiply and can be combined?
7573static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7574 unsigned MulOpc) {
7575 return canCombine(MBB, MO, CombineOpc: MulOpc);
7576}
7577
7578// TODO: There are many more machine instruction opcodes to match:
7579// 1. Other data types (integer, vectors)
7580// 2. Other math / logic operations (xor, or)
7581// 3. Other forms of the same operation (intrinsics and other variants)
7582bool AArch64InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
7583 bool Invert) const {
7584 if (Invert)
7585 return false;
7586 switch (Inst.getOpcode()) {
7587 // == Floating-point types ==
7588 // -- Floating-point instructions --
7589 case AArch64::FADDHrr:
7590 case AArch64::FADDSrr:
7591 case AArch64::FADDDrr:
7592 case AArch64::FMULHrr:
7593 case AArch64::FMULSrr:
7594 case AArch64::FMULDrr:
7595 case AArch64::FMULX16:
7596 case AArch64::FMULX32:
7597 case AArch64::FMULX64:
7598 // -- Advanced SIMD instructions --
7599 case AArch64::FADDv4f16:
7600 case AArch64::FADDv8f16:
7601 case AArch64::FADDv2f32:
7602 case AArch64::FADDv4f32:
7603 case AArch64::FADDv2f64:
7604 case AArch64::FMULv4f16:
7605 case AArch64::FMULv8f16:
7606 case AArch64::FMULv2f32:
7607 case AArch64::FMULv4f32:
7608 case AArch64::FMULv2f64:
7609 case AArch64::FMULXv4f16:
7610 case AArch64::FMULXv8f16:
7611 case AArch64::FMULXv2f32:
7612 case AArch64::FMULXv4f32:
7613 case AArch64::FMULXv2f64:
7614 // -- SVE instructions --
7615 // Opcodes FMULX_ZZZ_? don't exist because there is no unpredicated FMULX
7616 // in the SVE instruction set (though there are predicated ones).
7617 case AArch64::FADD_ZZZ_H:
7618 case AArch64::FADD_ZZZ_S:
7619 case AArch64::FADD_ZZZ_D:
7620 case AArch64::FMUL_ZZZ_H:
7621 case AArch64::FMUL_ZZZ_S:
7622 case AArch64::FMUL_ZZZ_D:
7623 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
7624 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
7625
7626 // == Integer types ==
7627 // -- Base instructions --
7628 // Opcodes MULWrr and MULXrr don't exist because
7629 // `MUL <Wd>, <Wn>, <Wm>` and `MUL <Xd>, <Xn>, <Xm>` are aliases of
7630 // `MADD <Wd>, <Wn>, <Wm>, WZR` and `MADD <Xd>, <Xn>, <Xm>, XZR` respectively.
7631 // The machine-combiner does not support three-source-operands machine
7632 // instruction. So we cannot reassociate MULs.
7633 case AArch64::ADDWrr:
7634 case AArch64::ADDXrr:
7635 case AArch64::ANDWrr:
7636 case AArch64::ANDXrr:
7637 case AArch64::ORRWrr:
7638 case AArch64::ORRXrr:
7639 case AArch64::EORWrr:
7640 case AArch64::EORXrr:
7641 case AArch64::EONWrr:
7642 case AArch64::EONXrr:
7643 // -- Advanced SIMD instructions --
7644 // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL
7645 // in the Advanced SIMD instruction set.
7646 case AArch64::ADDv8i8:
7647 case AArch64::ADDv16i8:
7648 case AArch64::ADDv4i16:
7649 case AArch64::ADDv8i16:
7650 case AArch64::ADDv2i32:
7651 case AArch64::ADDv4i32:
7652 case AArch64::ADDv1i64:
7653 case AArch64::ADDv2i64:
7654 case AArch64::MULv8i8:
7655 case AArch64::MULv16i8:
7656 case AArch64::MULv4i16:
7657 case AArch64::MULv8i16:
7658 case AArch64::MULv2i32:
7659 case AArch64::MULv4i32:
7660 case AArch64::ANDv8i8:
7661 case AArch64::ANDv16i8:
7662 case AArch64::ORRv8i8:
7663 case AArch64::ORRv16i8:
7664 case AArch64::EORv8i8:
7665 case AArch64::EORv16i8:
7666 // -- SVE instructions --
7667 case AArch64::ADD_ZZZ_B:
7668 case AArch64::ADD_ZZZ_H:
7669 case AArch64::ADD_ZZZ_S:
7670 case AArch64::ADD_ZZZ_D:
7671 case AArch64::MUL_ZZZ_B:
7672 case AArch64::MUL_ZZZ_H:
7673 case AArch64::MUL_ZZZ_S:
7674 case AArch64::MUL_ZZZ_D:
7675 case AArch64::AND_ZZZ:
7676 case AArch64::ORR_ZZZ:
7677 case AArch64::EOR_ZZZ:
7678 return true;
7679
7680 default:
7681 return false;
7682 }
7683}
7684
7685/// Find instructions that can be turned into madd.
7686static bool getMaddPatterns(MachineInstr &Root,
7687 SmallVectorImpl<unsigned> &Patterns) {
7688 unsigned Opc = Root.getOpcode();
7689 MachineBasicBlock &MBB = *Root.getParent();
7690 bool Found = false;
7691
7692 if (!isCombineInstrCandidate(Opc))
7693 return false;
7694 if (isCombineInstrSettingFlag(Opc)) {
7695 int Cmp_NZCV =
7696 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
7697 // When NZCV is live bail out.
7698 if (Cmp_NZCV == -1)
7699 return false;
7700 unsigned NewOpc = convertToNonFlagSettingOpc(MI: Root);
7701 // When opcode can't change bail out.
7702 // CHECKME: do we miss any cases for opcode conversion?
7703 if (NewOpc == Opc)
7704 return false;
7705 Opc = NewOpc;
7706 }
7707
7708 auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
7709 unsigned Pattern) {
7710 if (canCombineWithMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode, ZeroReg)) {
7711 Patterns.push_back(Elt: Pattern);
7712 Found = true;
7713 }
7714 };
7715
7716 auto setVFound = [&](int Opcode, int Operand, unsigned Pattern) {
7717 if (canCombine(MBB, MO&: Root.getOperand(i: Operand), CombineOpc: Opcode)) {
7718 Patterns.push_back(Elt: Pattern);
7719 Found = true;
7720 }
7721 };
7722
7723 typedef AArch64MachineCombinerPattern MCP;
7724
7725 switch (Opc) {
7726 default:
7727 break;
7728 case AArch64::ADDWrr:
7729 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
7730 "ADDWrr does not have register operands");
7731 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
7732 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
7733 break;
7734 case AArch64::ADDXrr:
7735 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
7736 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
7737 break;
7738 case AArch64::SUBWrr:
7739 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
7740 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
7741 break;
7742 case AArch64::SUBXrr:
7743 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
7744 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
7745 break;
7746 case AArch64::ADDWri:
7747 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
7748 break;
7749 case AArch64::ADDXri:
7750 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
7751 break;
7752 case AArch64::SUBWri:
7753 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
7754 break;
7755 case AArch64::SUBXri:
7756 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
7757 break;
7758 case AArch64::ADDv8i8:
7759 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
7760 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
7761 break;
7762 case AArch64::ADDv16i8:
7763 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
7764 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
7765 break;
7766 case AArch64::ADDv4i16:
7767 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
7768 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
7769 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
7770 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
7771 break;
7772 case AArch64::ADDv8i16:
7773 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
7774 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
7775 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
7776 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
7777 break;
7778 case AArch64::ADDv2i32:
7779 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
7780 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
7781 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
7782 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
7783 break;
7784 case AArch64::ADDv4i32:
7785 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
7786 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
7787 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
7788 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
7789 break;
7790 case AArch64::SUBv8i8:
7791 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
7792 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
7793 break;
7794 case AArch64::SUBv16i8:
7795 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
7796 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
7797 break;
7798 case AArch64::SUBv4i16:
7799 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
7800 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7801 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7802 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7803 break;
7804 case AArch64::SUBv8i16:
7805 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7806 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7807 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7808 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7809 break;
7810 case AArch64::SUBv2i32:
7811 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7812 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7813 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7814 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7815 break;
7816 case AArch64::SUBv4i32:
7817 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7818 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7819 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7820 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7821 break;
7822 }
7823 return Found;
7824}
7825
7826bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
7827 switch (Opcode) {
7828 default:
7829 break;
7830 case AArch64::UABALB_ZZZ_D:
7831 case AArch64::UABALB_ZZZ_H:
7832 case AArch64::UABALB_ZZZ_S:
7833 case AArch64::UABALT_ZZZ_D:
7834 case AArch64::UABALT_ZZZ_H:
7835 case AArch64::UABALT_ZZZ_S:
7836 case AArch64::SABALB_ZZZ_D:
7837 case AArch64::SABALB_ZZZ_S:
7838 case AArch64::SABALB_ZZZ_H:
7839 case AArch64::SABALT_ZZZ_D:
7840 case AArch64::SABALT_ZZZ_S:
7841 case AArch64::SABALT_ZZZ_H:
7842 case AArch64::UABALv16i8_v8i16:
7843 case AArch64::UABALv2i32_v2i64:
7844 case AArch64::UABALv4i16_v4i32:
7845 case AArch64::UABALv4i32_v2i64:
7846 case AArch64::UABALv8i16_v4i32:
7847 case AArch64::UABALv8i8_v8i16:
7848 case AArch64::UABAv16i8:
7849 case AArch64::UABAv2i32:
7850 case AArch64::UABAv4i16:
7851 case AArch64::UABAv4i32:
7852 case AArch64::UABAv8i16:
7853 case AArch64::UABAv8i8:
7854 case AArch64::SABALv16i8_v8i16:
7855 case AArch64::SABALv2i32_v2i64:
7856 case AArch64::SABALv4i16_v4i32:
7857 case AArch64::SABALv4i32_v2i64:
7858 case AArch64::SABALv8i16_v4i32:
7859 case AArch64::SABALv8i8_v8i16:
7860 case AArch64::SABAv16i8:
7861 case AArch64::SABAv2i32:
7862 case AArch64::SABAv4i16:
7863 case AArch64::SABAv4i32:
7864 case AArch64::SABAv8i16:
7865 case AArch64::SABAv8i8:
7866 return true;
7867 }
7868
7869 return false;
7870}
7871
7872unsigned AArch64InstrInfo::getAccumulationStartOpcode(
7873 unsigned AccumulationOpcode) const {
7874 switch (AccumulationOpcode) {
7875 default:
7876 llvm_unreachable("Unsupported accumulation Opcode!");
7877 case AArch64::UABALB_ZZZ_D:
7878 return AArch64::UABDLB_ZZZ_D;
7879 case AArch64::UABALB_ZZZ_H:
7880 return AArch64::UABDLB_ZZZ_H;
7881 case AArch64::UABALB_ZZZ_S:
7882 return AArch64::UABDLB_ZZZ_S;
7883 case AArch64::UABALT_ZZZ_D:
7884 return AArch64::UABDLT_ZZZ_D;
7885 case AArch64::UABALT_ZZZ_H:
7886 return AArch64::UABDLT_ZZZ_H;
7887 case AArch64::UABALT_ZZZ_S:
7888 return AArch64::UABDLT_ZZZ_S;
7889 case AArch64::UABALv16i8_v8i16:
7890 return AArch64::UABDLv16i8_v8i16;
7891 case AArch64::UABALv2i32_v2i64:
7892 return AArch64::UABDLv2i32_v2i64;
7893 case AArch64::UABALv4i16_v4i32:
7894 return AArch64::UABDLv4i16_v4i32;
7895 case AArch64::UABALv4i32_v2i64:
7896 return AArch64::UABDLv4i32_v2i64;
7897 case AArch64::UABALv8i16_v4i32:
7898 return AArch64::UABDLv8i16_v4i32;
7899 case AArch64::UABALv8i8_v8i16:
7900 return AArch64::UABDLv8i8_v8i16;
7901 case AArch64::UABAv16i8:
7902 return AArch64::UABDv16i8;
7903 case AArch64::UABAv2i32:
7904 return AArch64::UABDv2i32;
7905 case AArch64::UABAv4i16:
7906 return AArch64::UABDv4i16;
7907 case AArch64::UABAv4i32:
7908 return AArch64::UABDv4i32;
7909 case AArch64::UABAv8i16:
7910 return AArch64::UABDv8i16;
7911 case AArch64::UABAv8i8:
7912 return AArch64::UABDv8i8;
7913 case AArch64::SABALB_ZZZ_D:
7914 return AArch64::SABDLB_ZZZ_D;
7915 case AArch64::SABALB_ZZZ_S:
7916 return AArch64::SABDLB_ZZZ_S;
7917 case AArch64::SABALB_ZZZ_H:
7918 return AArch64::SABDLB_ZZZ_H;
7919 case AArch64::SABALT_ZZZ_D:
7920 return AArch64::SABDLT_ZZZ_D;
7921 case AArch64::SABALT_ZZZ_S:
7922 return AArch64::SABDLT_ZZZ_S;
7923 case AArch64::SABALT_ZZZ_H:
7924 return AArch64::SABDLT_ZZZ_H;
7925 case AArch64::SABALv16i8_v8i16:
7926 return AArch64::SABDLv16i8_v8i16;
7927 case AArch64::SABALv2i32_v2i64:
7928 return AArch64::SABDLv2i32_v2i64;
7929 case AArch64::SABALv4i16_v4i32:
7930 return AArch64::SABDLv4i16_v4i32;
7931 case AArch64::SABALv4i32_v2i64:
7932 return AArch64::SABDLv4i32_v2i64;
7933 case AArch64::SABALv8i16_v4i32:
7934 return AArch64::SABDLv8i16_v4i32;
7935 case AArch64::SABALv8i8_v8i16:
7936 return AArch64::SABDLv8i8_v8i16;
7937 case AArch64::SABAv16i8:
7938 return AArch64::SABDv16i8;
7939 case AArch64::SABAv2i32:
7940 return AArch64::SABAv2i32;
7941 case AArch64::SABAv4i16:
7942 return AArch64::SABDv4i16;
7943 case AArch64::SABAv4i32:
7944 return AArch64::SABDv4i32;
7945 case AArch64::SABAv8i16:
7946 return AArch64::SABDv8i16;
7947 case AArch64::SABAv8i8:
7948 return AArch64::SABDv8i8;
7949 }
7950}
7951
7952/// Floating-Point Support
7953
7954/// Find instructions that can be turned into madd.
7955static bool getFMAPatterns(MachineInstr &Root,
7956 SmallVectorImpl<unsigned> &Patterns) {
7957
7958 if (!isCombineInstrCandidateFP(Inst: Root))
7959 return false;
7960
7961 MachineBasicBlock &MBB = *Root.getParent();
7962 bool Found = false;
7963
7964 auto Match = [&](int Opcode, int Operand, unsigned Pattern) -> bool {
7965 if (canCombineWithFMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode)) {
7966 Patterns.push_back(Elt: Pattern);
7967 return true;
7968 }
7969 return false;
7970 };
7971
7972 typedef AArch64MachineCombinerPattern MCP;
7973
7974 switch (Root.getOpcode()) {
7975 default:
7976 assert(false && "Unsupported FP instruction in combiner\n");
7977 break;
7978 case AArch64::FADDHrr:
7979 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
7980 "FADDHrr does not have register operands");
7981
7982 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
7983 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
7984 break;
7985 case AArch64::FADDSrr:
7986 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
7987 "FADDSrr does not have register operands");
7988
7989 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
7990 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
7991
7992 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
7993 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
7994 break;
7995 case AArch64::FADDDrr:
7996 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
7997 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
7998
7999 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
8000 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
8001 break;
8002 case AArch64::FADDv4f16:
8003 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
8004 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
8005
8006 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
8007 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
8008 break;
8009 case AArch64::FADDv8f16:
8010 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
8011 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
8012
8013 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
8014 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
8015 break;
8016 case AArch64::FADDv2f32:
8017 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
8018 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
8019
8020 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
8021 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
8022 break;
8023 case AArch64::FADDv2f64:
8024 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
8025 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
8026
8027 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
8028 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
8029 break;
8030 case AArch64::FADDv4f32:
8031 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
8032 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
8033
8034 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
8035 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
8036 break;
8037 case AArch64::FSUBHrr:
8038 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
8039 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
8040 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
8041 break;
8042 case AArch64::FSUBSrr:
8043 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
8044
8045 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
8046 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
8047
8048 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
8049 break;
8050 case AArch64::FSUBDrr:
8051 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
8052
8053 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
8054 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
8055
8056 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
8057 break;
8058 case AArch64::FSUBv4f16:
8059 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
8060 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
8061
8062 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
8063 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
8064 break;
8065 case AArch64::FSUBv8f16:
8066 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
8067 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
8068
8069 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
8070 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
8071 break;
8072 case AArch64::FSUBv2f32:
8073 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
8074 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
8075
8076 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
8077 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
8078 break;
8079 case AArch64::FSUBv2f64:
8080 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
8081 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
8082
8083 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
8084 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
8085 break;
8086 case AArch64::FSUBv4f32:
8087 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
8088 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
8089
8090 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
8091 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
8092 break;
8093 }
8094 return Found;
8095}
8096
8097static bool getFMULPatterns(MachineInstr &Root,
8098 SmallVectorImpl<unsigned> &Patterns) {
8099 MachineBasicBlock &MBB = *Root.getParent();
8100 bool Found = false;
8101
8102 auto Match = [&](unsigned Opcode, int Operand, unsigned Pattern) -> bool {
8103 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8104 MachineOperand &MO = Root.getOperand(i: Operand);
8105 MachineInstr *MI = nullptr;
8106 if (MO.isReg() && MO.getReg().isVirtual())
8107 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8108 // Ignore No-op COPYs in FMUL(COPY(DUP(..)))
8109 if (MI && MI->getOpcode() == TargetOpcode::COPY &&
8110 MI->getOperand(i: 1).getReg().isVirtual())
8111 MI = MRI.getUniqueVRegDef(Reg: MI->getOperand(i: 1).getReg());
8112 if (MI && MI->getOpcode() == Opcode) {
8113 Patterns.push_back(Elt: Pattern);
8114 return true;
8115 }
8116 return false;
8117 };
8118
8119 typedef AArch64MachineCombinerPattern MCP;
8120
8121 switch (Root.getOpcode()) {
8122 default:
8123 return false;
8124 case AArch64::FMULv2f32:
8125 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
8126 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
8127 break;
8128 case AArch64::FMULv2f64:
8129 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
8130 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
8131 break;
8132 case AArch64::FMULv4f16:
8133 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
8134 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
8135 break;
8136 case AArch64::FMULv4f32:
8137 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
8138 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
8139 break;
8140 case AArch64::FMULv8f16:
8141 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
8142 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
8143 break;
8144 }
8145
8146 return Found;
8147}
8148
8149static bool getFNEGPatterns(MachineInstr &Root,
8150 SmallVectorImpl<unsigned> &Patterns) {
8151 unsigned Opc = Root.getOpcode();
8152 MachineBasicBlock &MBB = *Root.getParent();
8153 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8154
8155 auto Match = [&](unsigned Opcode, unsigned Pattern) -> bool {
8156 MachineOperand &MO = Root.getOperand(i: 1);
8157 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8158 if (MI != nullptr && (MI->getOpcode() == Opcode) &&
8159 MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()) &&
8160 Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8161 Root.getFlag(Flag: MachineInstr::MIFlag::FmNsz) &&
8162 MI->getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8163 MI->getFlag(Flag: MachineInstr::MIFlag::FmNsz)) {
8164 Patterns.push_back(Elt: Pattern);
8165 return true;
8166 }
8167 return false;
8168 };
8169
8170 switch (Opc) {
8171 default:
8172 break;
8173 case AArch64::FNEGDr:
8174 return Match(AArch64::FMADDDrrr, AArch64MachineCombinerPattern::FNMADD);
8175 case AArch64::FNEGSr:
8176 return Match(AArch64::FMADDSrrr, AArch64MachineCombinerPattern::FNMADD);
8177 }
8178
8179 return false;
8180}
8181
8182/// Return true when a code sequence can improve throughput. It
8183/// should be called only for instructions in loops.
8184/// \param Pattern - combiner pattern
8185bool AArch64InstrInfo::isThroughputPattern(unsigned Pattern) const {
8186 switch (Pattern) {
8187 default:
8188 break;
8189 case AArch64MachineCombinerPattern::FMULADDH_OP1:
8190 case AArch64MachineCombinerPattern::FMULADDH_OP2:
8191 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
8192 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
8193 case AArch64MachineCombinerPattern::FMULADDS_OP1:
8194 case AArch64MachineCombinerPattern::FMULADDS_OP2:
8195 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
8196 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
8197 case AArch64MachineCombinerPattern::FMULADDD_OP1:
8198 case AArch64MachineCombinerPattern::FMULADDD_OP2:
8199 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
8200 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
8201 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
8202 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
8203 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
8204 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
8205 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
8206 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
8207 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
8208 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
8209 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
8210 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
8211 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
8212 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
8213 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
8214 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
8215 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
8216 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
8217 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
8218 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
8219 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
8220 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
8221 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
8222 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
8223 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
8224 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
8225 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
8226 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
8227 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
8228 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1:
8229 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
8230 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1:
8231 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
8232 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
8233 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
8234 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
8235 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
8236 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
8237 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
8238 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
8239 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
8240 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
8241 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
8242 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
8243 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
8244 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
8245 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2:
8246 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
8247 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2:
8248 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
8249 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2:
8250 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
8251 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2:
8252 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
8253 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2:
8254 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
8255 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
8256 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
8257 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
8258 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
8259 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
8260 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
8261 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
8262 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
8263 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
8264 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
8265 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
8266 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
8267 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
8268 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
8269 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
8270 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
8271 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
8272 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
8273 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
8274 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
8275 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
8276 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
8277 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
8278 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
8279 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
8280 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
8281 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
8282 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
8283 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
8284 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
8285 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
8286 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
8287 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
8288 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
8289 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
8290 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
8291 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
8292 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
8293 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
8294 return true;
8295 } // end switch (Pattern)
8296 return false;
8297}
8298
8299/// Find other MI combine patterns.
8300static bool getMiscPatterns(MachineInstr &Root,
8301 SmallVectorImpl<unsigned> &Patterns) {
8302 // A - (B + C) ==> (A - B) - C or (A - C) - B
8303 unsigned Opc = Root.getOpcode();
8304 MachineBasicBlock &MBB = *Root.getParent();
8305
8306 switch (Opc) {
8307 case AArch64::SUBWrr:
8308 case AArch64::SUBSWrr:
8309 case AArch64::SUBXrr:
8310 case AArch64::SUBSXrr:
8311 // Found candidate root.
8312 break;
8313 default:
8314 return false;
8315 }
8316
8317 if (isCombineInstrSettingFlag(Opc) &&
8318 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) ==
8319 -1)
8320 return false;
8321
8322 if (canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDWrr) ||
8323 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSWrr) ||
8324 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDXrr) ||
8325 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSXrr)) {
8326 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP1);
8327 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP2);
8328 return true;
8329 }
8330
8331 return false;
8332}
8333
8334/// Check if the given instruction forms a gather load pattern that can be
8335/// optimized for better Memory-Level Parallelism (MLP). This function
8336/// identifies chains of NEON lane load instructions that load data from
8337/// different memory addresses into individual lanes of a 128-bit vector
8338/// register, then attempts to split the pattern into parallel loads to break
8339/// the serial dependency between instructions.
8340///
8341/// Pattern Matched:
8342/// Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) ->
8343/// LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)
8344///
8345/// Transformed Into:
8346/// Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64
8347/// to combine the results, enabling better memory-level parallelism.
8348///
8349/// Supported Element Types:
8350/// - 32-bit elements (LD1i32, 4 lanes total)
8351/// - 16-bit elements (LD1i16, 8 lanes total)
8352/// - 8-bit elements (LD1i8, 16 lanes total)
8353static bool getGatherLanePattern(MachineInstr &Root,
8354 SmallVectorImpl<unsigned> &Patterns,
8355 unsigned LoadLaneOpCode, unsigned NumLanes) {
8356 const MachineFunction *MF = Root.getMF();
8357
8358 // Early exit if optimizing for size.
8359 if (MF->getFunction().hasMinSize())
8360 return false;
8361
8362 const MachineRegisterInfo &MRI = MF->getRegInfo();
8363 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
8364
8365 // The root of the pattern must load into the last lane of the vector.
8366 if (Root.getOperand(i: 2).getImm() != NumLanes - 1)
8367 return false;
8368
8369 // Check that we have load into all lanes except lane 0.
8370 // For each load we also want to check that:
8371 // 1. It has a single non-debug use (since we will be replacing the virtual
8372 // register)
8373 // 2. That the addressing mode only uses a single pointer operand
8374 auto *CurrInstr = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8375 auto Range = llvm::seq<unsigned>(Begin: 1, End: NumLanes - 1);
8376 SmallSet<unsigned, 16> RemainingLanes(Range.begin(), Range.end());
8377 SmallVector<const MachineInstr *, 16> LoadInstrs;
8378 while (!RemainingLanes.empty() && CurrInstr &&
8379 CurrInstr->getOpcode() == LoadLaneOpCode &&
8380 MRI.hasOneNonDBGUse(RegNo: CurrInstr->getOperand(i: 0).getReg()) &&
8381 CurrInstr->getNumOperands() == 4) {
8382 RemainingLanes.erase(V: CurrInstr->getOperand(i: 2).getImm());
8383 LoadInstrs.push_back(Elt: CurrInstr);
8384 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8385 }
8386
8387 // Check that we have found a match for lanes N-1.. 1.
8388 if (!RemainingLanes.empty())
8389 return false;
8390
8391 // Match the SUBREG_TO_REG sequence.
8392 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
8393 return false;
8394
8395 // Verify that the subreg to reg loads an integer into the first lane.
8396 auto Lane0LoadReg = CurrInstr->getOperand(i: 1).getReg();
8397 unsigned SingleLaneSizeInBits = 128 / NumLanes;
8398 if (TRI->getRegSizeInBits(Reg: Lane0LoadReg, MRI) != SingleLaneSizeInBits)
8399 return false;
8400
8401 // Verify that it also has a single non debug use.
8402 if (!MRI.hasOneNonDBGUse(RegNo: Lane0LoadReg))
8403 return false;
8404
8405 LoadInstrs.push_back(Elt: MRI.getUniqueVRegDef(Reg: Lane0LoadReg));
8406
8407 // If there is any chance of aliasing, do not apply the pattern.
8408 // Walk backward through the MBB starting from Root.
8409 // Exit early if we've encountered all load instructions or hit the search
8410 // limit.
8411 auto MBBItr = Root.getIterator();
8412 unsigned RemainingSteps = GatherOptSearchLimit;
8413 SmallPtrSet<const MachineInstr *, 16> RemainingLoadInstrs;
8414 RemainingLoadInstrs.insert(I: LoadInstrs.begin(), E: LoadInstrs.end());
8415 const MachineBasicBlock *MBB = Root.getParent();
8416
8417 for (; MBBItr != MBB->begin() && RemainingSteps > 0 &&
8418 !RemainingLoadInstrs.empty();
8419 --MBBItr, --RemainingSteps) {
8420 const MachineInstr &CurrInstr = *MBBItr;
8421
8422 // Remove this instruction from remaining loads if it's one we're tracking.
8423 RemainingLoadInstrs.erase(Ptr: &CurrInstr);
8424
8425 // Check for potential aliasing with any of the load instructions to
8426 // optimize.
8427 if (CurrInstr.isLoadFoldBarrier())
8428 return false;
8429 }
8430
8431 // If we hit the search limit without finding all load instructions,
8432 // don't match the pattern.
8433 if (RemainingSteps == 0 && !RemainingLoadInstrs.empty())
8434 return false;
8435
8436 switch (NumLanes) {
8437 case 4:
8438 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i32);
8439 break;
8440 case 8:
8441 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i16);
8442 break;
8443 case 16:
8444 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i8);
8445 break;
8446 default:
8447 llvm_unreachable("Got bad number of lanes for gather pattern.");
8448 }
8449
8450 return true;
8451}
8452
8453/// Search for patterns of LD instructions we can optimize.
8454static bool getLoadPatterns(MachineInstr &Root,
8455 SmallVectorImpl<unsigned> &Patterns) {
8456
8457 // The pattern searches for loads into single lanes.
8458 switch (Root.getOpcode()) {
8459 case AArch64::LD1i32:
8460 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 4);
8461 case AArch64::LD1i16:
8462 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 8);
8463 case AArch64::LD1i8:
8464 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 16);
8465 default:
8466 return false;
8467 }
8468}
8469
8470/// Generate optimized instruction sequence for gather load patterns to improve
8471/// Memory-Level Parallelism (MLP). This function transforms a chain of
8472/// sequential NEON lane loads into parallel vector loads that can execute
8473/// concurrently.
8474static void
8475generateGatherLanePattern(MachineInstr &Root,
8476 SmallVectorImpl<MachineInstr *> &InsInstrs,
8477 SmallVectorImpl<MachineInstr *> &DelInstrs,
8478 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8479 unsigned Pattern, unsigned NumLanes) {
8480 MachineFunction &MF = *Root.getParent()->getParent();
8481 MachineRegisterInfo &MRI = MF.getRegInfo();
8482 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8483
8484 // Gather the initial load instructions to build the pattern.
8485 SmallVector<MachineInstr *, 16> LoadToLaneInstrs;
8486 MachineInstr *CurrInstr = &Root;
8487 for (unsigned i = 0; i < NumLanes - 1; ++i) {
8488 LoadToLaneInstrs.push_back(Elt: CurrInstr);
8489 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8490 }
8491
8492 // Sort the load instructions according to the lane.
8493 llvm::sort(C&: LoadToLaneInstrs,
8494 Comp: [](const MachineInstr *A, const MachineInstr *B) {
8495 return A->getOperand(i: 2).getImm() > B->getOperand(i: 2).getImm();
8496 });
8497
8498 MachineInstr *SubregToReg = CurrInstr;
8499 LoadToLaneInstrs.push_back(
8500 Elt: MRI.getUniqueVRegDef(Reg: SubregToReg->getOperand(i: 1).getReg()));
8501 auto LoadToLaneInstrsAscending = llvm::reverse(C&: LoadToLaneInstrs);
8502
8503 const TargetRegisterClass *FPR128RegClass =
8504 MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg());
8505
8506 // Helper lambda to create a LD1 instruction.
8507 auto CreateLD1Instruction = [&](MachineInstr *OriginalInstr,
8508 Register SrcRegister, unsigned Lane,
8509 Register OffsetRegister,
8510 bool OffsetRegisterKillState) {
8511 auto NewRegister = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8512 MachineInstrBuilder LoadIndexIntoRegister =
8513 BuildMI(MF, MIMD: MIMetadata(*OriginalInstr), MCID: TII->get(Opcode: Root.getOpcode()),
8514 DestReg: NewRegister)
8515 .addReg(RegNo: SrcRegister)
8516 .addImm(Val: Lane)
8517 .addReg(RegNo: OffsetRegister, Flags: getKillRegState(B: OffsetRegisterKillState))
8518 .setMemRefs(OriginalInstr->memoperands());
8519 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewRegister, y: InsInstrs.size()));
8520 InsInstrs.push_back(Elt: LoadIndexIntoRegister);
8521 return NewRegister;
8522 };
8523
8524 // Helper to create load instruction based on the NumLanes in the NEON
8525 // register we are rewriting.
8526 auto CreateLDRInstruction =
8527 [&](unsigned NumLanes, Register DestReg, Register OffsetReg,
8528 ArrayRef<MachineMemOperand *> MMOs) -> MachineInstrBuilder {
8529 unsigned Opcode;
8530 switch (NumLanes) {
8531 case 4:
8532 Opcode = AArch64::LDRSui;
8533 break;
8534 case 8:
8535 Opcode = AArch64::LDRHui;
8536 break;
8537 case 16:
8538 Opcode = AArch64::LDRBui;
8539 break;
8540 default:
8541 llvm_unreachable(
8542 "Got unsupported number of lanes in machine-combiner gather pattern");
8543 }
8544 // Immediate offset load
8545 return BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg)
8546 .addReg(RegNo: OffsetReg)
8547 .addImm(Val: 0)
8548 .setMemRefs(MMOs);
8549 };
8550
8551 // Load the remaining lanes into register 0.
8552 auto LanesToLoadToReg0 =
8553 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + 1,
8554 y: LoadToLaneInstrsAscending.begin() + NumLanes / 2);
8555 Register PrevReg = SubregToReg->getOperand(i: 0).getReg();
8556 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg0)) {
8557 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8558 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8559 OffsetRegOperand.getReg(),
8560 OffsetRegOperand.isKill());
8561 DelInstrs.push_back(Elt: LoadInstr);
8562 }
8563 Register LastLoadReg0 = PrevReg;
8564
8565 // First load into register 1. Perform an integer load to zero out the upper
8566 // lanes in a single instruction.
8567 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
8568 MachineInstr *OriginalSplitLoad =
8569 *std::next(x: LoadToLaneInstrsAscending.begin(), n: NumLanes / 2);
8570 Register DestRegForMiddleIndex = MRI.createVirtualRegister(
8571 RegClass: MRI.getRegClass(Reg: Lane0Load->getOperand(i: 0).getReg()));
8572
8573 const MachineOperand &OriginalSplitToLoadOffsetOperand =
8574 OriginalSplitLoad->getOperand(i: 3);
8575 MachineInstrBuilder MiddleIndexLoadInstr =
8576 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
8577 OriginalSplitToLoadOffsetOperand.getReg(),
8578 OriginalSplitLoad->memoperands());
8579
8580 InstrIdxForVirtReg.insert(
8581 KV: std::make_pair(x&: DestRegForMiddleIndex, y: InsInstrs.size()));
8582 InsInstrs.push_back(Elt: MiddleIndexLoadInstr);
8583 DelInstrs.push_back(Elt: OriginalSplitLoad);
8584
8585 // Subreg To Reg instruction for register 1.
8586 Register DestRegForSubregToReg = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8587 unsigned SubregType;
8588 switch (NumLanes) {
8589 case 4:
8590 SubregType = AArch64::ssub;
8591 break;
8592 case 8:
8593 SubregType = AArch64::hsub;
8594 break;
8595 case 16:
8596 SubregType = AArch64::bsub;
8597 break;
8598 default:
8599 llvm_unreachable(
8600 "Got invalid NumLanes for machine-combiner gather pattern");
8601 }
8602
8603 auto SubRegToRegInstr =
8604 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubregToReg->getOpcode()),
8605 DestReg: DestRegForSubregToReg)
8606 .addReg(RegNo: DestRegForMiddleIndex, Flags: getKillRegState(B: true))
8607 .addImm(Val: SubregType);
8608 InstrIdxForVirtReg.insert(
8609 KV: std::make_pair(x&: DestRegForSubregToReg, y: InsInstrs.size()));
8610 InsInstrs.push_back(Elt: SubRegToRegInstr);
8611
8612 // Load remaining lanes into register 1.
8613 auto LanesToLoadToReg1 =
8614 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + NumLanes / 2 + 1,
8615 y: LoadToLaneInstrsAscending.end());
8616 PrevReg = SubRegToRegInstr->getOperand(i: 0).getReg();
8617 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg1)) {
8618 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8619 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8620 OffsetRegOperand.getReg(),
8621 OffsetRegOperand.isKill());
8622
8623 // Do not add the last reg to DelInstrs - it will be removed later.
8624 if (Index == NumLanes / 2 - 2) {
8625 break;
8626 }
8627 DelInstrs.push_back(Elt: LoadInstr);
8628 }
8629 Register LastLoadReg1 = PrevReg;
8630
8631 // Create the final zip instruction to combine the results.
8632 MachineInstrBuilder ZipInstr =
8633 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::ZIP1v2i64),
8634 DestReg: Root.getOperand(i: 0).getReg())
8635 .addReg(RegNo: LastLoadReg0)
8636 .addReg(RegNo: LastLoadReg1);
8637 InsInstrs.push_back(Elt: ZipInstr);
8638}
8639
8640CombinerObjective
8641AArch64InstrInfo::getCombinerObjective(unsigned Pattern) const {
8642 switch (Pattern) {
8643 case AArch64MachineCombinerPattern::SUBADD_OP1:
8644 case AArch64MachineCombinerPattern::SUBADD_OP2:
8645 case AArch64MachineCombinerPattern::GATHER_LANE_i32:
8646 case AArch64MachineCombinerPattern::GATHER_LANE_i16:
8647 case AArch64MachineCombinerPattern::GATHER_LANE_i8:
8648 return CombinerObjective::MustReduceDepth;
8649 default:
8650 return TargetInstrInfo::getCombinerObjective(Pattern);
8651 }
8652}
8653
8654/// Return true when there is potentially a faster code sequence for an
8655/// instruction chain ending in \p Root. All potential patterns are listed in
8656/// the \p Pattern vector. Pattern should be sorted in priority order since the
8657/// pattern evaluator stops checking as soon as it finds a faster sequence.
8658
8659bool AArch64InstrInfo::getMachineCombinerPatterns(
8660 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
8661 bool DoRegPressureReduce) const {
8662 // Integer patterns
8663 if (getMaddPatterns(Root, Patterns))
8664 return true;
8665 // Floating point patterns
8666 if (getFMULPatterns(Root, Patterns))
8667 return true;
8668 if (getFMAPatterns(Root, Patterns))
8669 return true;
8670 if (getFNEGPatterns(Root, Patterns))
8671 return true;
8672
8673 // Other patterns
8674 if (getMiscPatterns(Root, Patterns))
8675 return true;
8676
8677 // Load patterns
8678 if (getLoadPatterns(Root, Patterns))
8679 return true;
8680
8681 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
8682 DoRegPressureReduce);
8683}
8684
8685enum class FMAInstKind { Default, Indexed, Accumulator };
8686/// genFusedMultiply - Generate fused multiply instructions.
8687/// This function supports both integer and floating point instructions.
8688/// A typical example:
8689/// F|MUL I=A,B,0
8690/// F|ADD R,I,C
8691/// ==> F|MADD R,A,B,C
8692/// \param MF Containing MachineFunction
8693/// \param MRI Register information
8694/// \param TII Target information
8695/// \param Root is the F|ADD instruction
8696/// \param [out] InsInstrs is a vector of machine instructions and will
8697/// contain the generated madd instruction
8698/// \param IdxMulOpd is index of operand in Root that is the result of
8699/// the F|MUL. In the example above IdxMulOpd is 1.
8700/// \param MaddOpc the opcode fo the f|madd instruction
8701/// \param RC Register class of operands
8702/// \param kind of fma instruction (addressing mode) to be generated
8703/// \param ReplacedAddend is the result register from the instruction
8704/// replacing the non-combined operand, if any.
8705static MachineInstr *
8706genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
8707 const TargetInstrInfo *TII, MachineInstr &Root,
8708 SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
8709 unsigned MaddOpc, const TargetRegisterClass *RC,
8710 FMAInstKind kind = FMAInstKind::Default,
8711 const Register *ReplacedAddend = nullptr) {
8712 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8713
8714 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
8715 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
8716 Register ResultReg = Root.getOperand(i: 0).getReg();
8717 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8718 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
8719 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8720 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
8721
8722 Register SrcReg2;
8723 bool Src2IsKill;
8724 if (ReplacedAddend) {
8725 // If we just generated a new addend, we must be it's only use.
8726 SrcReg2 = *ReplacedAddend;
8727 Src2IsKill = true;
8728 } else {
8729 SrcReg2 = Root.getOperand(i: IdxOtherOpd).getReg();
8730 Src2IsKill = Root.getOperand(i: IdxOtherOpd).isKill();
8731 }
8732
8733 if (ResultReg.isVirtual())
8734 MRI.constrainRegClass(Reg: ResultReg, RC);
8735 if (SrcReg0.isVirtual())
8736 MRI.constrainRegClass(Reg: SrcReg0, RC);
8737 if (SrcReg1.isVirtual())
8738 MRI.constrainRegClass(Reg: SrcReg1, RC);
8739 if (SrcReg2.isVirtual())
8740 MRI.constrainRegClass(Reg: SrcReg2, RC);
8741
8742 MachineInstrBuilder MIB;
8743 if (kind == FMAInstKind::Default)
8744 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8745 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8746 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8747 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8748 else if (kind == FMAInstKind::Indexed)
8749 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8750 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8751 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8752 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8753 .addImm(Val: MUL->getOperand(i: 3).getImm());
8754 else if (kind == FMAInstKind::Accumulator)
8755 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8756 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8757 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8758 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill));
8759 else
8760 assert(false && "Invalid FMA instruction kind \n");
8761 // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
8762 InsInstrs.push_back(Elt: MIB);
8763 return MUL;
8764}
8765
8766static MachineInstr *
8767genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI,
8768 const TargetInstrInfo *TII, MachineInstr &Root,
8769 SmallVectorImpl<MachineInstr *> &InsInstrs) {
8770 MachineInstr *MAD = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8771
8772 unsigned Opc = 0;
8773 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MAD->getOperand(i: 0).getReg());
8774 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
8775 Opc = AArch64::FNMADDSrrr;
8776 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
8777 Opc = AArch64::FNMADDDrrr;
8778 else
8779 return nullptr;
8780
8781 Register ResultReg = Root.getOperand(i: 0).getReg();
8782 Register SrcReg0 = MAD->getOperand(i: 1).getReg();
8783 Register SrcReg1 = MAD->getOperand(i: 2).getReg();
8784 Register SrcReg2 = MAD->getOperand(i: 3).getReg();
8785 bool Src0IsKill = MAD->getOperand(i: 1).isKill();
8786 bool Src1IsKill = MAD->getOperand(i: 2).isKill();
8787 bool Src2IsKill = MAD->getOperand(i: 3).isKill();
8788 if (ResultReg.isVirtual())
8789 MRI.constrainRegClass(Reg: ResultReg, RC);
8790 if (SrcReg0.isVirtual())
8791 MRI.constrainRegClass(Reg: SrcReg0, RC);
8792 if (SrcReg1.isVirtual())
8793 MRI.constrainRegClass(Reg: SrcReg1, RC);
8794 if (SrcReg2.isVirtual())
8795 MRI.constrainRegClass(Reg: SrcReg2, RC);
8796
8797 MachineInstrBuilder MIB =
8798 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Opc), DestReg: ResultReg)
8799 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8800 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8801 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8802 InsInstrs.push_back(Elt: MIB);
8803
8804 return MAD;
8805}
8806
8807/// Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
8808static MachineInstr *
8809genIndexedMultiply(MachineInstr &Root,
8810 SmallVectorImpl<MachineInstr *> &InsInstrs,
8811 unsigned IdxDupOp, unsigned MulOpc,
8812 const TargetRegisterClass *RC, MachineRegisterInfo &MRI) {
8813 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
8814 "Invalid index of FMUL operand");
8815
8816 MachineFunction &MF = *Root.getMF();
8817 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8818
8819 MachineInstr *Dup =
8820 MF.getRegInfo().getUniqueVRegDef(Reg: Root.getOperand(i: IdxDupOp).getReg());
8821
8822 if (Dup->getOpcode() == TargetOpcode::COPY)
8823 Dup = MRI.getUniqueVRegDef(Reg: Dup->getOperand(i: 1).getReg());
8824
8825 Register DupSrcReg = Dup->getOperand(i: 1).getReg();
8826 MRI.clearKillFlags(Reg: DupSrcReg);
8827 MRI.constrainRegClass(Reg: DupSrcReg, RC);
8828
8829 unsigned DupSrcLane = Dup->getOperand(i: 2).getImm();
8830
8831 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
8832 MachineOperand &MulOp = Root.getOperand(i: IdxMulOp);
8833
8834 Register ResultReg = Root.getOperand(i: 0).getReg();
8835
8836 MachineInstrBuilder MIB;
8837 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MulOpc), DestReg: ResultReg)
8838 .add(MO: MulOp)
8839 .addReg(RegNo: DupSrcReg)
8840 .addImm(Val: DupSrcLane);
8841
8842 InsInstrs.push_back(Elt: MIB);
8843 return &Root;
8844}
8845
8846/// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
8847/// instructions.
8848///
8849/// \see genFusedMultiply
8850static MachineInstr *genFusedMultiplyAcc(
8851 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8852 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8853 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8854 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8855 kind: FMAInstKind::Accumulator);
8856}
8857
8858/// genNeg - Helper to generate an intermediate negation of the second operand
8859/// of Root
8860static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
8861 const TargetInstrInfo *TII, MachineInstr &Root,
8862 SmallVectorImpl<MachineInstr *> &InsInstrs,
8863 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8864 unsigned MnegOpc, const TargetRegisterClass *RC) {
8865 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
8866 MachineInstrBuilder MIB =
8867 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MnegOpc), DestReg: NewVR)
8868 .add(MO: Root.getOperand(i: 2));
8869 InsInstrs.push_back(Elt: MIB);
8870
8871 assert(InstrIdxForVirtReg.empty());
8872 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
8873
8874 return NewVR;
8875}
8876
8877/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8878/// instructions with an additional negation of the accumulator
8879static MachineInstr *genFusedMultiplyAccNeg(
8880 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8881 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8882 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8883 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8884 assert(IdxMulOpd == 1);
8885
8886 Register NewVR =
8887 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8888 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8889 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
8890}
8891
8892/// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
8893/// instructions.
8894///
8895/// \see genFusedMultiply
8896static MachineInstr *genFusedMultiplyIdx(
8897 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8898 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8899 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8900 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8901 kind: FMAInstKind::Indexed);
8902}
8903
8904/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8905/// instructions with an additional negation of the accumulator
8906static MachineInstr *genFusedMultiplyIdxNeg(
8907 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8908 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8909 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8910 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8911 assert(IdxMulOpd == 1);
8912
8913 Register NewVR =
8914 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8915
8916 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8917 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
8918}
8919
8920/// genMaddR - Generate madd instruction and combine mul and add using
8921/// an extra virtual register
8922/// Example - an ADD intermediate needs to be stored in a register:
8923/// MUL I=A,B,0
8924/// ADD R,I,Imm
8925/// ==> ORR V, ZR, Imm
8926/// ==> MADD R,A,B,V
8927/// \param MF Containing MachineFunction
8928/// \param MRI Register information
8929/// \param TII Target information
8930/// \param Root is the ADD instruction
8931/// \param [out] InsInstrs is a vector of machine instructions and will
8932/// contain the generated madd instruction
8933/// \param IdxMulOpd is index of operand in Root that is the result of
8934/// the MUL. In the example above IdxMulOpd is 1.
8935/// \param MaddOpc the opcode fo the madd instruction
8936/// \param VR is a virtual register that holds the value of an ADD operand
8937/// (V in the example above).
8938/// \param RC Register class of operands
8939static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
8940 const TargetInstrInfo *TII, MachineInstr &Root,
8941 SmallVectorImpl<MachineInstr *> &InsInstrs,
8942 unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
8943 const TargetRegisterClass *RC) {
8944 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8945
8946 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
8947 Register ResultReg = Root.getOperand(i: 0).getReg();
8948 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8949 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
8950 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8951 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
8952
8953 if (ResultReg.isVirtual())
8954 MRI.constrainRegClass(Reg: ResultReg, RC);
8955 if (SrcReg0.isVirtual())
8956 MRI.constrainRegClass(Reg: SrcReg0, RC);
8957 if (SrcReg1.isVirtual())
8958 MRI.constrainRegClass(Reg: SrcReg1, RC);
8959 if (Register::isVirtualRegister(Reg: VR))
8960 MRI.constrainRegClass(Reg: VR, RC);
8961
8962 MachineInstrBuilder MIB =
8963 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8964 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8965 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8966 .addReg(RegNo: VR);
8967 // Insert the MADD
8968 InsInstrs.push_back(Elt: MIB);
8969 return MUL;
8970}
8971
8972/// Do the following transformation
8973/// A - (B + C) ==> (A - B) - C
8974/// A - (B + C) ==> (A - C) - B
8975static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI,
8976 const TargetInstrInfo *TII, MachineInstr &Root,
8977 SmallVectorImpl<MachineInstr *> &InsInstrs,
8978 SmallVectorImpl<MachineInstr *> &DelInstrs,
8979 unsigned IdxOpd1,
8980 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
8981 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
8982 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
8983 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
8984
8985 Register ResultReg = Root.getOperand(i: 0).getReg();
8986 Register RegA = Root.getOperand(i: 1).getReg();
8987 bool RegAIsKill = Root.getOperand(i: 1).isKill();
8988 Register RegB = AddMI->getOperand(i: IdxOpd1).getReg();
8989 bool RegBIsKill = AddMI->getOperand(i: IdxOpd1).isKill();
8990 Register RegC = AddMI->getOperand(i: IdxOtherOpd).getReg();
8991 bool RegCIsKill = AddMI->getOperand(i: IdxOtherOpd).isKill();
8992 Register NewVR =
8993 MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 2).getReg()));
8994
8995 unsigned Opcode = Root.getOpcode();
8996 if (Opcode == AArch64::SUBSWrr)
8997 Opcode = AArch64::SUBWrr;
8998 else if (Opcode == AArch64::SUBSXrr)
8999 Opcode = AArch64::SUBXrr;
9000 else
9001 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
9002 "Unexpected instruction opcode.");
9003
9004 uint32_t Flags = Root.mergeFlagsWith(Other: *AddMI);
9005 Flags &= ~MachineInstr::NoSWrap;
9006 Flags &= ~MachineInstr::NoUWrap;
9007
9008 MachineInstrBuilder MIB1 =
9009 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: NewVR)
9010 .addReg(RegNo: RegA, Flags: getKillRegState(B: RegAIsKill))
9011 .addReg(RegNo: RegB, Flags: getKillRegState(B: RegBIsKill))
9012 .setMIFlags(Flags);
9013 MachineInstrBuilder MIB2 =
9014 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: ResultReg)
9015 .addReg(RegNo: NewVR, Flags: getKillRegState(B: true))
9016 .addReg(RegNo: RegC, Flags: getKillRegState(B: RegCIsKill))
9017 .setMIFlags(Flags);
9018
9019 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9020 InsInstrs.push_back(Elt: MIB1);
9021 InsInstrs.push_back(Elt: MIB2);
9022 DelInstrs.push_back(Elt: AddMI);
9023 DelInstrs.push_back(Elt: &Root);
9024}
9025
9026unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
9027 unsigned int AccumulatorOpCode) const {
9028 switch (AccumulatorOpCode) {
9029 case AArch64::UABALB_ZZZ_D:
9030 case AArch64::SABALB_ZZZ_D:
9031 case AArch64::UABALT_ZZZ_D:
9032 case AArch64::SABALT_ZZZ_D:
9033 return AArch64::ADD_ZZZ_D;
9034 case AArch64::UABALB_ZZZ_H:
9035 case AArch64::SABALB_ZZZ_H:
9036 case AArch64::UABALT_ZZZ_H:
9037 case AArch64::SABALT_ZZZ_H:
9038 return AArch64::ADD_ZZZ_H;
9039 case AArch64::UABALB_ZZZ_S:
9040 case AArch64::SABALB_ZZZ_S:
9041 case AArch64::UABALT_ZZZ_S:
9042 case AArch64::SABALT_ZZZ_S:
9043 return AArch64::ADD_ZZZ_S;
9044 case AArch64::UABALv16i8_v8i16:
9045 case AArch64::SABALv8i8_v8i16:
9046 case AArch64::SABAv8i16:
9047 case AArch64::UABAv8i16:
9048 return AArch64::ADDv8i16;
9049 case AArch64::SABALv2i32_v2i64:
9050 case AArch64::UABALv2i32_v2i64:
9051 case AArch64::SABALv4i32_v2i64:
9052 return AArch64::ADDv2i64;
9053 case AArch64::UABALv4i16_v4i32:
9054 case AArch64::SABALv4i16_v4i32:
9055 case AArch64::SABALv8i16_v4i32:
9056 case AArch64::SABAv4i32:
9057 case AArch64::UABAv4i32:
9058 return AArch64::ADDv4i32;
9059 case AArch64::UABALv4i32_v2i64:
9060 return AArch64::ADDv2i64;
9061 case AArch64::UABALv8i16_v4i32:
9062 return AArch64::ADDv4i32;
9063 case AArch64::UABALv8i8_v8i16:
9064 case AArch64::SABALv16i8_v8i16:
9065 return AArch64::ADDv8i16;
9066 case AArch64::UABAv16i8:
9067 case AArch64::SABAv16i8:
9068 return AArch64::ADDv16i8;
9069 case AArch64::UABAv4i16:
9070 case AArch64::SABAv4i16:
9071 return AArch64::ADDv4i16;
9072 case AArch64::UABAv2i32:
9073 case AArch64::SABAv2i32:
9074 return AArch64::ADDv2i32;
9075 case AArch64::UABAv8i8:
9076 case AArch64::SABAv8i8:
9077 return AArch64::ADDv8i8;
9078 default:
9079 llvm_unreachable("Unknown accumulator opcode");
9080 }
9081}
9082
9083/// When getMachineCombinerPatterns() finds potential patterns,
9084/// this function generates the instructions that could replace the
9085/// original code sequence
9086void AArch64InstrInfo::genAlternativeCodeSequence(
9087 MachineInstr &Root, unsigned Pattern,
9088 SmallVectorImpl<MachineInstr *> &InsInstrs,
9089 SmallVectorImpl<MachineInstr *> &DelInstrs,
9090 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
9091 MachineBasicBlock &MBB = *Root.getParent();
9092 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9093 MachineFunction &MF = *MBB.getParent();
9094 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
9095
9096 MachineInstr *MUL = nullptr;
9097 const TargetRegisterClass *RC;
9098 unsigned Opc;
9099 switch (Pattern) {
9100 default:
9101 // Reassociate instructions.
9102 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
9103 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
9104 return;
9105 case AArch64MachineCombinerPattern::SUBADD_OP1:
9106 // A - (B + C)
9107 // ==> (A - B) - C
9108 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 1,
9109 InstrIdxForVirtReg);
9110 return;
9111 case AArch64MachineCombinerPattern::SUBADD_OP2:
9112 // A - (B + C)
9113 // ==> (A - C) - B
9114 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 2,
9115 InstrIdxForVirtReg);
9116 return;
9117 case AArch64MachineCombinerPattern::MULADDW_OP1:
9118 case AArch64MachineCombinerPattern::MULADDX_OP1:
9119 // MUL I=A,B,0
9120 // ADD R,I,C
9121 // ==> MADD R,A,B,C
9122 // --- Create(MADD);
9123 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP1) {
9124 Opc = AArch64::MADDWrrr;
9125 RC = &AArch64::GPR32RegClass;
9126 } else {
9127 Opc = AArch64::MADDXrrr;
9128 RC = &AArch64::GPR64RegClass;
9129 }
9130 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9131 break;
9132 case AArch64MachineCombinerPattern::MULADDW_OP2:
9133 case AArch64MachineCombinerPattern::MULADDX_OP2:
9134 // MUL I=A,B,0
9135 // ADD R,C,I
9136 // ==> MADD R,A,B,C
9137 // --- Create(MADD);
9138 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP2) {
9139 Opc = AArch64::MADDWrrr;
9140 RC = &AArch64::GPR32RegClass;
9141 } else {
9142 Opc = AArch64::MADDXrrr;
9143 RC = &AArch64::GPR64RegClass;
9144 }
9145 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9146 break;
9147 case AArch64MachineCombinerPattern::MULADDWI_OP1:
9148 case AArch64MachineCombinerPattern::MULADDXI_OP1:
9149 case AArch64MachineCombinerPattern::MULSUBWI_OP1:
9150 case AArch64MachineCombinerPattern::MULSUBXI_OP1: {
9151 // MUL I=A,B,0
9152 // ADD/SUB R,I,Imm
9153 // ==> MOV V, Imm/-Imm
9154 // ==> MADD R,A,B,V
9155 // --- Create(MADD);
9156 const TargetRegisterClass *RC;
9157 unsigned BitSize, MovImm;
9158 if (Pattern == AArch64MachineCombinerPattern::MULADDWI_OP1 ||
9159 Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1) {
9160 MovImm = AArch64::MOVi32imm;
9161 RC = &AArch64::GPR32spRegClass;
9162 BitSize = 32;
9163 Opc = AArch64::MADDWrrr;
9164 RC = &AArch64::GPR32RegClass;
9165 } else {
9166 MovImm = AArch64::MOVi64imm;
9167 RC = &AArch64::GPR64spRegClass;
9168 BitSize = 64;
9169 Opc = AArch64::MADDXrrr;
9170 RC = &AArch64::GPR64RegClass;
9171 }
9172 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9173 uint64_t Imm = Root.getOperand(i: 2).getImm();
9174
9175 if (Root.getOperand(i: 3).isImm()) {
9176 unsigned Val = Root.getOperand(i: 3).getImm();
9177 Imm = Imm << Val;
9178 }
9179 bool IsSub = Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1 ||
9180 Pattern == AArch64MachineCombinerPattern::MULSUBXI_OP1;
9181 uint64_t UImm = SignExtend64(X: IsSub ? -Imm : Imm, B: BitSize);
9182 // Check that the immediate can be composed via a single instruction.
9183 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
9184 AArch64_IMM::expandMOVImm(Imm: UImm, BitSize, Insn);
9185 if (Insn.size() != 1)
9186 return;
9187 MachineInstrBuilder MIB1 =
9188 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MovImm), DestReg: NewVR)
9189 .addImm(Val: IsSub ? -Imm : Imm);
9190 InsInstrs.push_back(Elt: MIB1);
9191 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9192 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9193 break;
9194 }
9195 case AArch64MachineCombinerPattern::MULSUBW_OP1:
9196 case AArch64MachineCombinerPattern::MULSUBX_OP1: {
9197 // MUL I=A,B,0
9198 // SUB R,I, C
9199 // ==> SUB V, 0, C
9200 // ==> MADD R,A,B,V // = -C + A*B
9201 // --- Create(MADD);
9202 const TargetRegisterClass *SubRC;
9203 unsigned SubOpc, ZeroReg;
9204 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP1) {
9205 SubOpc = AArch64::SUBWrr;
9206 SubRC = &AArch64::GPR32spRegClass;
9207 ZeroReg = AArch64::WZR;
9208 Opc = AArch64::MADDWrrr;
9209 RC = &AArch64::GPR32RegClass;
9210 } else {
9211 SubOpc = AArch64::SUBXrr;
9212 SubRC = &AArch64::GPR64spRegClass;
9213 ZeroReg = AArch64::XZR;
9214 Opc = AArch64::MADDXrrr;
9215 RC = &AArch64::GPR64RegClass;
9216 }
9217 Register NewVR = MRI.createVirtualRegister(RegClass: SubRC);
9218 // SUB NewVR, 0, C
9219 MachineInstrBuilder MIB1 =
9220 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubOpc), DestReg: NewVR)
9221 .addReg(RegNo: ZeroReg)
9222 .add(MO: Root.getOperand(i: 2));
9223 InsInstrs.push_back(Elt: MIB1);
9224 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9225 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9226 break;
9227 }
9228 case AArch64MachineCombinerPattern::MULSUBW_OP2:
9229 case AArch64MachineCombinerPattern::MULSUBX_OP2:
9230 // MUL I=A,B,0
9231 // SUB R,C,I
9232 // ==> MSUB R,A,B,C (computes C - A*B)
9233 // --- Create(MSUB);
9234 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP2) {
9235 Opc = AArch64::MSUBWrrr;
9236 RC = &AArch64::GPR32RegClass;
9237 } else {
9238 Opc = AArch64::MSUBXrrr;
9239 RC = &AArch64::GPR64RegClass;
9240 }
9241 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9242 break;
9243 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
9244 Opc = AArch64::MLAv8i8;
9245 RC = &AArch64::FPR64RegClass;
9246 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9247 break;
9248 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
9249 Opc = AArch64::MLAv8i8;
9250 RC = &AArch64::FPR64RegClass;
9251 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9252 break;
9253 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
9254 Opc = AArch64::MLAv16i8;
9255 RC = &AArch64::FPR128RegClass;
9256 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9257 break;
9258 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
9259 Opc = AArch64::MLAv16i8;
9260 RC = &AArch64::FPR128RegClass;
9261 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9262 break;
9263 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
9264 Opc = AArch64::MLAv4i16;
9265 RC = &AArch64::FPR64RegClass;
9266 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9267 break;
9268 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
9269 Opc = AArch64::MLAv4i16;
9270 RC = &AArch64::FPR64RegClass;
9271 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9272 break;
9273 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
9274 Opc = AArch64::MLAv8i16;
9275 RC = &AArch64::FPR128RegClass;
9276 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9277 break;
9278 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
9279 Opc = AArch64::MLAv8i16;
9280 RC = &AArch64::FPR128RegClass;
9281 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9282 break;
9283 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
9284 Opc = AArch64::MLAv2i32;
9285 RC = &AArch64::FPR64RegClass;
9286 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9287 break;
9288 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
9289 Opc = AArch64::MLAv2i32;
9290 RC = &AArch64::FPR64RegClass;
9291 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9292 break;
9293 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
9294 Opc = AArch64::MLAv4i32;
9295 RC = &AArch64::FPR128RegClass;
9296 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9297 break;
9298 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
9299 Opc = AArch64::MLAv4i32;
9300 RC = &AArch64::FPR128RegClass;
9301 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9302 break;
9303
9304 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
9305 Opc = AArch64::MLAv8i8;
9306 RC = &AArch64::FPR64RegClass;
9307 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9308 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i8,
9309 RC);
9310 break;
9311 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
9312 Opc = AArch64::MLSv8i8;
9313 RC = &AArch64::FPR64RegClass;
9314 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9315 break;
9316 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
9317 Opc = AArch64::MLAv16i8;
9318 RC = &AArch64::FPR128RegClass;
9319 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9320 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv16i8,
9321 RC);
9322 break;
9323 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
9324 Opc = AArch64::MLSv16i8;
9325 RC = &AArch64::FPR128RegClass;
9326 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9327 break;
9328 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
9329 Opc = AArch64::MLAv4i16;
9330 RC = &AArch64::FPR64RegClass;
9331 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9332 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9333 RC);
9334 break;
9335 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
9336 Opc = AArch64::MLSv4i16;
9337 RC = &AArch64::FPR64RegClass;
9338 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9339 break;
9340 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
9341 Opc = AArch64::MLAv8i16;
9342 RC = &AArch64::FPR128RegClass;
9343 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9344 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9345 RC);
9346 break;
9347 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
9348 Opc = AArch64::MLSv8i16;
9349 RC = &AArch64::FPR128RegClass;
9350 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9351 break;
9352 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
9353 Opc = AArch64::MLAv2i32;
9354 RC = &AArch64::FPR64RegClass;
9355 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9356 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9357 RC);
9358 break;
9359 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
9360 Opc = AArch64::MLSv2i32;
9361 RC = &AArch64::FPR64RegClass;
9362 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9363 break;
9364 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
9365 Opc = AArch64::MLAv4i32;
9366 RC = &AArch64::FPR128RegClass;
9367 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9368 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9369 RC);
9370 break;
9371 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
9372 Opc = AArch64::MLSv4i32;
9373 RC = &AArch64::FPR128RegClass;
9374 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9375 break;
9376
9377 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
9378 Opc = AArch64::MLAv4i16_indexed;
9379 RC = &AArch64::FPR64RegClass;
9380 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9381 break;
9382 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
9383 Opc = AArch64::MLAv4i16_indexed;
9384 RC = &AArch64::FPR64RegClass;
9385 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9386 break;
9387 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
9388 Opc = AArch64::MLAv8i16_indexed;
9389 RC = &AArch64::FPR128RegClass;
9390 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9391 break;
9392 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
9393 Opc = AArch64::MLAv8i16_indexed;
9394 RC = &AArch64::FPR128RegClass;
9395 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9396 break;
9397 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
9398 Opc = AArch64::MLAv2i32_indexed;
9399 RC = &AArch64::FPR64RegClass;
9400 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9401 break;
9402 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
9403 Opc = AArch64::MLAv2i32_indexed;
9404 RC = &AArch64::FPR64RegClass;
9405 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9406 break;
9407 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
9408 Opc = AArch64::MLAv4i32_indexed;
9409 RC = &AArch64::FPR128RegClass;
9410 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9411 break;
9412 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
9413 Opc = AArch64::MLAv4i32_indexed;
9414 RC = &AArch64::FPR128RegClass;
9415 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9416 break;
9417
9418 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
9419 Opc = AArch64::MLAv4i16_indexed;
9420 RC = &AArch64::FPR64RegClass;
9421 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9422 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9423 RC);
9424 break;
9425 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
9426 Opc = AArch64::MLSv4i16_indexed;
9427 RC = &AArch64::FPR64RegClass;
9428 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9429 break;
9430 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
9431 Opc = AArch64::MLAv8i16_indexed;
9432 RC = &AArch64::FPR128RegClass;
9433 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9434 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9435 RC);
9436 break;
9437 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
9438 Opc = AArch64::MLSv8i16_indexed;
9439 RC = &AArch64::FPR128RegClass;
9440 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9441 break;
9442 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
9443 Opc = AArch64::MLAv2i32_indexed;
9444 RC = &AArch64::FPR64RegClass;
9445 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9446 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9447 RC);
9448 break;
9449 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
9450 Opc = AArch64::MLSv2i32_indexed;
9451 RC = &AArch64::FPR64RegClass;
9452 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9453 break;
9454 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
9455 Opc = AArch64::MLAv4i32_indexed;
9456 RC = &AArch64::FPR128RegClass;
9457 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9458 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9459 RC);
9460 break;
9461 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
9462 Opc = AArch64::MLSv4i32_indexed;
9463 RC = &AArch64::FPR128RegClass;
9464 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9465 break;
9466
9467 // Floating Point Support
9468 case AArch64MachineCombinerPattern::FMULADDH_OP1:
9469 Opc = AArch64::FMADDHrrr;
9470 RC = &AArch64::FPR16RegClass;
9471 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9472 break;
9473 case AArch64MachineCombinerPattern::FMULADDS_OP1:
9474 Opc = AArch64::FMADDSrrr;
9475 RC = &AArch64::FPR32RegClass;
9476 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9477 break;
9478 case AArch64MachineCombinerPattern::FMULADDD_OP1:
9479 Opc = AArch64::FMADDDrrr;
9480 RC = &AArch64::FPR64RegClass;
9481 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9482 break;
9483
9484 case AArch64MachineCombinerPattern::FMULADDH_OP2:
9485 Opc = AArch64::FMADDHrrr;
9486 RC = &AArch64::FPR16RegClass;
9487 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9488 break;
9489 case AArch64MachineCombinerPattern::FMULADDS_OP2:
9490 Opc = AArch64::FMADDSrrr;
9491 RC = &AArch64::FPR32RegClass;
9492 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9493 break;
9494 case AArch64MachineCombinerPattern::FMULADDD_OP2:
9495 Opc = AArch64::FMADDDrrr;
9496 RC = &AArch64::FPR64RegClass;
9497 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9498 break;
9499
9500 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
9501 Opc = AArch64::FMLAv1i32_indexed;
9502 RC = &AArch64::FPR32RegClass;
9503 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9504 kind: FMAInstKind::Indexed);
9505 break;
9506 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
9507 Opc = AArch64::FMLAv1i32_indexed;
9508 RC = &AArch64::FPR32RegClass;
9509 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9510 kind: FMAInstKind::Indexed);
9511 break;
9512
9513 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
9514 Opc = AArch64::FMLAv1i64_indexed;
9515 RC = &AArch64::FPR64RegClass;
9516 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9517 kind: FMAInstKind::Indexed);
9518 break;
9519 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
9520 Opc = AArch64::FMLAv1i64_indexed;
9521 RC = &AArch64::FPR64RegClass;
9522 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9523 kind: FMAInstKind::Indexed);
9524 break;
9525
9526 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
9527 RC = &AArch64::FPR64RegClass;
9528 Opc = AArch64::FMLAv4i16_indexed;
9529 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9530 kind: FMAInstKind::Indexed);
9531 break;
9532 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
9533 RC = &AArch64::FPR64RegClass;
9534 Opc = AArch64::FMLAv4f16;
9535 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9536 kind: FMAInstKind::Accumulator);
9537 break;
9538 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
9539 RC = &AArch64::FPR64RegClass;
9540 Opc = AArch64::FMLAv4i16_indexed;
9541 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9542 kind: FMAInstKind::Indexed);
9543 break;
9544 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
9545 RC = &AArch64::FPR64RegClass;
9546 Opc = AArch64::FMLAv4f16;
9547 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9548 kind: FMAInstKind::Accumulator);
9549 break;
9550
9551 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
9552 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
9553 RC = &AArch64::FPR64RegClass;
9554 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
9555 Opc = AArch64::FMLAv2i32_indexed;
9556 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9557 kind: FMAInstKind::Indexed);
9558 } else {
9559 Opc = AArch64::FMLAv2f32;
9560 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9561 kind: FMAInstKind::Accumulator);
9562 }
9563 break;
9564 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
9565 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
9566 RC = &AArch64::FPR64RegClass;
9567 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
9568 Opc = AArch64::FMLAv2i32_indexed;
9569 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9570 kind: FMAInstKind::Indexed);
9571 } else {
9572 Opc = AArch64::FMLAv2f32;
9573 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9574 kind: FMAInstKind::Accumulator);
9575 }
9576 break;
9577
9578 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
9579 RC = &AArch64::FPR128RegClass;
9580 Opc = AArch64::FMLAv8i16_indexed;
9581 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9582 kind: FMAInstKind::Indexed);
9583 break;
9584 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
9585 RC = &AArch64::FPR128RegClass;
9586 Opc = AArch64::FMLAv8f16;
9587 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9588 kind: FMAInstKind::Accumulator);
9589 break;
9590 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
9591 RC = &AArch64::FPR128RegClass;
9592 Opc = AArch64::FMLAv8i16_indexed;
9593 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9594 kind: FMAInstKind::Indexed);
9595 break;
9596 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
9597 RC = &AArch64::FPR128RegClass;
9598 Opc = AArch64::FMLAv8f16;
9599 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9600 kind: FMAInstKind::Accumulator);
9601 break;
9602
9603 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
9604 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
9605 RC = &AArch64::FPR128RegClass;
9606 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
9607 Opc = AArch64::FMLAv2i64_indexed;
9608 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9609 kind: FMAInstKind::Indexed);
9610 } else {
9611 Opc = AArch64::FMLAv2f64;
9612 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9613 kind: FMAInstKind::Accumulator);
9614 }
9615 break;
9616 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
9617 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
9618 RC = &AArch64::FPR128RegClass;
9619 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
9620 Opc = AArch64::FMLAv2i64_indexed;
9621 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9622 kind: FMAInstKind::Indexed);
9623 } else {
9624 Opc = AArch64::FMLAv2f64;
9625 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9626 kind: FMAInstKind::Accumulator);
9627 }
9628 break;
9629
9630 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
9631 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
9632 RC = &AArch64::FPR128RegClass;
9633 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
9634 Opc = AArch64::FMLAv4i32_indexed;
9635 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9636 kind: FMAInstKind::Indexed);
9637 } else {
9638 Opc = AArch64::FMLAv4f32;
9639 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9640 kind: FMAInstKind::Accumulator);
9641 }
9642 break;
9643
9644 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
9645 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
9646 RC = &AArch64::FPR128RegClass;
9647 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
9648 Opc = AArch64::FMLAv4i32_indexed;
9649 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9650 kind: FMAInstKind::Indexed);
9651 } else {
9652 Opc = AArch64::FMLAv4f32;
9653 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9654 kind: FMAInstKind::Accumulator);
9655 }
9656 break;
9657
9658 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
9659 Opc = AArch64::FNMSUBHrrr;
9660 RC = &AArch64::FPR16RegClass;
9661 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9662 break;
9663 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
9664 Opc = AArch64::FNMSUBSrrr;
9665 RC = &AArch64::FPR32RegClass;
9666 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9667 break;
9668 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
9669 Opc = AArch64::FNMSUBDrrr;
9670 RC = &AArch64::FPR64RegClass;
9671 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9672 break;
9673
9674 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
9675 Opc = AArch64::FNMADDHrrr;
9676 RC = &AArch64::FPR16RegClass;
9677 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9678 break;
9679 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
9680 Opc = AArch64::FNMADDSrrr;
9681 RC = &AArch64::FPR32RegClass;
9682 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9683 break;
9684 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
9685 Opc = AArch64::FNMADDDrrr;
9686 RC = &AArch64::FPR64RegClass;
9687 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9688 break;
9689
9690 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
9691 Opc = AArch64::FMSUBHrrr;
9692 RC = &AArch64::FPR16RegClass;
9693 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9694 break;
9695 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
9696 Opc = AArch64::FMSUBSrrr;
9697 RC = &AArch64::FPR32RegClass;
9698 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9699 break;
9700 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
9701 Opc = AArch64::FMSUBDrrr;
9702 RC = &AArch64::FPR64RegClass;
9703 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9704 break;
9705
9706 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
9707 Opc = AArch64::FMLSv1i32_indexed;
9708 RC = &AArch64::FPR32RegClass;
9709 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9710 kind: FMAInstKind::Indexed);
9711 break;
9712
9713 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
9714 Opc = AArch64::FMLSv1i64_indexed;
9715 RC = &AArch64::FPR64RegClass;
9716 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9717 kind: FMAInstKind::Indexed);
9718 break;
9719
9720 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
9721 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
9722 RC = &AArch64::FPR64RegClass;
9723 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9724 MachineInstrBuilder MIB1 =
9725 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f16), DestReg: NewVR)
9726 .add(MO: Root.getOperand(i: 2));
9727 InsInstrs.push_back(Elt: MIB1);
9728 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9729 if (Pattern == AArch64MachineCombinerPattern::FMLSv4f16_OP1) {
9730 Opc = AArch64::FMLAv4f16;
9731 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9732 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9733 } else {
9734 Opc = AArch64::FMLAv4i16_indexed;
9735 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9736 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9737 }
9738 break;
9739 }
9740 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
9741 RC = &AArch64::FPR64RegClass;
9742 Opc = AArch64::FMLSv4f16;
9743 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9744 kind: FMAInstKind::Accumulator);
9745 break;
9746 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
9747 RC = &AArch64::FPR64RegClass;
9748 Opc = AArch64::FMLSv4i16_indexed;
9749 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9750 kind: FMAInstKind::Indexed);
9751 break;
9752
9753 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
9754 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
9755 RC = &AArch64::FPR64RegClass;
9756 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
9757 Opc = AArch64::FMLSv2i32_indexed;
9758 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9759 kind: FMAInstKind::Indexed);
9760 } else {
9761 Opc = AArch64::FMLSv2f32;
9762 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9763 kind: FMAInstKind::Accumulator);
9764 }
9765 break;
9766
9767 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
9768 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
9769 RC = &AArch64::FPR128RegClass;
9770 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9771 MachineInstrBuilder MIB1 =
9772 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv8f16), DestReg: NewVR)
9773 .add(MO: Root.getOperand(i: 2));
9774 InsInstrs.push_back(Elt: MIB1);
9775 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9776 if (Pattern == AArch64MachineCombinerPattern::FMLSv8f16_OP1) {
9777 Opc = AArch64::FMLAv8f16;
9778 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9779 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9780 } else {
9781 Opc = AArch64::FMLAv8i16_indexed;
9782 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9783 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9784 }
9785 break;
9786 }
9787 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
9788 RC = &AArch64::FPR128RegClass;
9789 Opc = AArch64::FMLSv8f16;
9790 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9791 kind: FMAInstKind::Accumulator);
9792 break;
9793 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
9794 RC = &AArch64::FPR128RegClass;
9795 Opc = AArch64::FMLSv8i16_indexed;
9796 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9797 kind: FMAInstKind::Indexed);
9798 break;
9799
9800 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
9801 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
9802 RC = &AArch64::FPR128RegClass;
9803 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
9804 Opc = AArch64::FMLSv2i64_indexed;
9805 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9806 kind: FMAInstKind::Indexed);
9807 } else {
9808 Opc = AArch64::FMLSv2f64;
9809 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9810 kind: FMAInstKind::Accumulator);
9811 }
9812 break;
9813
9814 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
9815 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
9816 RC = &AArch64::FPR128RegClass;
9817 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
9818 Opc = AArch64::FMLSv4i32_indexed;
9819 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9820 kind: FMAInstKind::Indexed);
9821 } else {
9822 Opc = AArch64::FMLSv4f32;
9823 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9824 kind: FMAInstKind::Accumulator);
9825 }
9826 break;
9827 case AArch64MachineCombinerPattern::FMLSv2f32_OP1:
9828 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
9829 RC = &AArch64::FPR64RegClass;
9830 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9831 MachineInstrBuilder MIB1 =
9832 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f32), DestReg: NewVR)
9833 .add(MO: Root.getOperand(i: 2));
9834 InsInstrs.push_back(Elt: MIB1);
9835 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9836 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
9837 Opc = AArch64::FMLAv2i32_indexed;
9838 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9839 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9840 } else {
9841 Opc = AArch64::FMLAv2f32;
9842 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9843 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9844 }
9845 break;
9846 }
9847 case AArch64MachineCombinerPattern::FMLSv4f32_OP1:
9848 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
9849 RC = &AArch64::FPR128RegClass;
9850 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9851 MachineInstrBuilder MIB1 =
9852 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f32), DestReg: NewVR)
9853 .add(MO: Root.getOperand(i: 2));
9854 InsInstrs.push_back(Elt: MIB1);
9855 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9856 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
9857 Opc = AArch64::FMLAv4i32_indexed;
9858 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9859 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9860 } else {
9861 Opc = AArch64::FMLAv4f32;
9862 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9863 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9864 }
9865 break;
9866 }
9867 case AArch64MachineCombinerPattern::FMLSv2f64_OP1:
9868 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
9869 RC = &AArch64::FPR128RegClass;
9870 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9871 MachineInstrBuilder MIB1 =
9872 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f64), DestReg: NewVR)
9873 .add(MO: Root.getOperand(i: 2));
9874 InsInstrs.push_back(Elt: MIB1);
9875 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9876 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
9877 Opc = AArch64::FMLAv2i64_indexed;
9878 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9879 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9880 } else {
9881 Opc = AArch64::FMLAv2f64;
9882 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9883 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9884 }
9885 break;
9886 }
9887 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
9888 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2: {
9889 unsigned IdxDupOp =
9890 (Pattern == AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1) ? 1
9891 : 2;
9892 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i32_indexed,
9893 RC: &AArch64::FPR128RegClass, MRI);
9894 break;
9895 }
9896 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
9897 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2: {
9898 unsigned IdxDupOp =
9899 (Pattern == AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1) ? 1
9900 : 2;
9901 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i64_indexed,
9902 RC: &AArch64::FPR128RegClass, MRI);
9903 break;
9904 }
9905 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
9906 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2: {
9907 unsigned IdxDupOp =
9908 (Pattern == AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1) ? 1
9909 : 2;
9910 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i16_indexed,
9911 RC: &AArch64::FPR128_loRegClass, MRI);
9912 break;
9913 }
9914 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
9915 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2: {
9916 unsigned IdxDupOp =
9917 (Pattern == AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1) ? 1
9918 : 2;
9919 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i32_indexed,
9920 RC: &AArch64::FPR128RegClass, MRI);
9921 break;
9922 }
9923 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
9924 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2: {
9925 unsigned IdxDupOp =
9926 (Pattern == AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1) ? 1
9927 : 2;
9928 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv8i16_indexed,
9929 RC: &AArch64::FPR128_loRegClass, MRI);
9930 break;
9931 }
9932 case AArch64MachineCombinerPattern::FNMADD: {
9933 MUL = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs);
9934 break;
9935 }
9936 case AArch64MachineCombinerPattern::GATHER_LANE_i32: {
9937 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
9938 Pattern, NumLanes: 4);
9939 break;
9940 }
9941 case AArch64MachineCombinerPattern::GATHER_LANE_i16: {
9942 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
9943 Pattern, NumLanes: 8);
9944 break;
9945 }
9946 case AArch64MachineCombinerPattern::GATHER_LANE_i8: {
9947 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
9948 Pattern, NumLanes: 16);
9949 break;
9950 }
9951
9952 } // end switch (Pattern)
9953 // Record MUL and ADD/SUB for deletion
9954 if (MUL)
9955 DelInstrs.push_back(Elt: MUL);
9956 DelInstrs.push_back(Elt: &Root);
9957
9958 // Set the flags on the inserted instructions to be the merged flags of the
9959 // instructions that we have combined.
9960 uint32_t Flags = Root.getFlags();
9961 if (MUL)
9962 Flags = Root.mergeFlagsWith(Other: *MUL);
9963 for (auto *MI : InsInstrs)
9964 MI->setFlags(Flags);
9965}
9966
9967/// Replace csincr-branch sequence by simple conditional branch
9968///
9969/// Examples:
9970/// 1. \code
9971/// csinc w9, wzr, wzr, <condition code>
9972/// tbnz w9, #0, 0x44
9973/// \endcode
9974/// to
9975/// \code
9976/// b.<inverted condition code>
9977/// \endcode
9978///
9979/// 2. \code
9980/// csinc w9, wzr, wzr, <condition code>
9981/// tbz w9, #0, 0x44
9982/// \endcode
9983/// to
9984/// \code
9985/// b.<condition code>
9986/// \endcode
9987///
9988/// Replace compare and branch sequence by TBZ/TBNZ instruction when the
9989/// compare's constant operand is power of 2.
9990///
9991/// Examples:
9992/// \code
9993/// and w8, w8, #0x400
9994/// cbnz w8, L1
9995/// \endcode
9996/// to
9997/// \code
9998/// tbnz w8, #10, L1
9999/// \endcode
10000///
10001/// \param MI Conditional Branch
10002/// \return True when the simple conditional branch is generated
10003///
10004bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
10005 bool IsNegativeBranch = false;
10006 bool IsTestAndBranch = false;
10007 unsigned TargetBBInMI = 0;
10008 switch (MI.getOpcode()) {
10009 default:
10010 llvm_unreachable("Unknown branch instruction?");
10011 case AArch64::Bcc:
10012 case AArch64::CBWPri:
10013 case AArch64::CBXPri:
10014 case AArch64::CBBAssertExt:
10015 case AArch64::CBHAssertExt:
10016 case AArch64::CBWPrr:
10017 case AArch64::CBXPrr:
10018 return false;
10019 case AArch64::CBZW:
10020 case AArch64::CBZX:
10021 TargetBBInMI = 1;
10022 break;
10023 case AArch64::CBNZW:
10024 case AArch64::CBNZX:
10025 TargetBBInMI = 1;
10026 IsNegativeBranch = true;
10027 break;
10028 case AArch64::TBZW:
10029 case AArch64::TBZX:
10030 TargetBBInMI = 2;
10031 IsTestAndBranch = true;
10032 break;
10033 case AArch64::TBNZW:
10034 case AArch64::TBNZX:
10035 TargetBBInMI = 2;
10036 IsNegativeBranch = true;
10037 IsTestAndBranch = true;
10038 break;
10039 }
10040 // So we increment a zero register and test for bits other
10041 // than bit 0? Conservatively bail out in case the verifier
10042 // missed this case.
10043 if (IsTestAndBranch && MI.getOperand(i: 1).getImm())
10044 return false;
10045
10046 // Find Definition.
10047 assert(MI.getParent() && "Incomplete machine instruction\n");
10048 MachineBasicBlock *MBB = MI.getParent();
10049 MachineFunction *MF = MBB->getParent();
10050 MachineRegisterInfo *MRI = &MF->getRegInfo();
10051 Register VReg = MI.getOperand(i: 0).getReg();
10052 if (!VReg.isVirtual())
10053 return false;
10054
10055 MachineInstr *DefMI = MRI->getVRegDef(Reg: VReg);
10056
10057 // Look through COPY instructions to find definition.
10058 while (DefMI->isCopy()) {
10059 Register CopyVReg = DefMI->getOperand(i: 1).getReg();
10060 if (!MRI->hasOneNonDBGUse(RegNo: CopyVReg))
10061 return false;
10062 if (!MRI->hasOneDef(RegNo: CopyVReg))
10063 return false;
10064 DefMI = MRI->getVRegDef(Reg: CopyVReg);
10065 }
10066
10067 switch (DefMI->getOpcode()) {
10068 default:
10069 return false;
10070 // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
10071 case AArch64::ANDWri:
10072 case AArch64::ANDXri: {
10073 if (IsTestAndBranch)
10074 return false;
10075 if (DefMI->getParent() != MBB)
10076 return false;
10077 if (!MRI->hasOneNonDBGUse(RegNo: VReg))
10078 return false;
10079
10080 bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
10081 uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
10082 val: DefMI->getOperand(i: 2).getImm(), regSize: Is32Bit ? 32 : 64);
10083 if (!isPowerOf2_64(Value: Mask))
10084 return false;
10085
10086 MachineOperand &MO = DefMI->getOperand(i: 1);
10087 Register NewReg = MO.getReg();
10088 if (!NewReg.isVirtual())
10089 return false;
10090
10091 assert(!MRI->def_empty(NewReg) && "Register must be defined.");
10092
10093 MachineBasicBlock &RefToMBB = *MBB;
10094 MachineBasicBlock *TBB = MI.getOperand(i: 1).getMBB();
10095 DebugLoc DL = MI.getDebugLoc();
10096 unsigned Imm = Log2_64(Value: Mask);
10097 unsigned Opc = (Imm < 32)
10098 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
10099 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
10100 MachineInstr *NewMI = BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: Opc))
10101 .addReg(RegNo: NewReg)
10102 .addImm(Val: Imm)
10103 .addMBB(MBB: TBB);
10104 // Register lives on to the CBZ now.
10105 MO.setIsKill(false);
10106
10107 // For immediate smaller than 32, we need to use the 32-bit
10108 // variant (W) in all cases. Indeed the 64-bit variant does not
10109 // allow to encode them.
10110 // Therefore, if the input register is 64-bit, we need to take the
10111 // 32-bit sub-part.
10112 if (!Is32Bit && Imm < 32)
10113 NewMI->getOperand(i: 0).setSubReg(AArch64::sub_32);
10114 MI.eraseFromParent();
10115 return true;
10116 }
10117 // Look for CSINC
10118 case AArch64::CSINCWr:
10119 case AArch64::CSINCXr: {
10120 if (!(DefMI->getOperand(i: 1).getReg() == AArch64::WZR &&
10121 DefMI->getOperand(i: 2).getReg() == AArch64::WZR) &&
10122 !(DefMI->getOperand(i: 1).getReg() == AArch64::XZR &&
10123 DefMI->getOperand(i: 2).getReg() == AArch64::XZR))
10124 return false;
10125
10126 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
10127 isDead: true) != -1)
10128 return false;
10129
10130 AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(i: 3).getImm();
10131 // Convert only when the condition code is not modified between
10132 // the CSINC and the branch. The CC may be used by other
10133 // instructions in between.
10134 if (areCFlagsAccessedBetweenInstrs(From: DefMI, To: MI, TRI: &getRegisterInfo(), AccessToCheck: AK_Write))
10135 return false;
10136 MachineBasicBlock &RefToMBB = *MBB;
10137 MachineBasicBlock *TBB = MI.getOperand(i: TargetBBInMI).getMBB();
10138 DebugLoc DL = MI.getDebugLoc();
10139 if (IsNegativeBranch)
10140 CC = AArch64CC::getInvertedCondCode(Code: CC);
10141 BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: CC).addMBB(MBB: TBB);
10142 MI.eraseFromParent();
10143 return true;
10144 }
10145 }
10146}
10147
10148std::pair<unsigned, unsigned>
10149AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10150 const unsigned Mask = AArch64II::MO_FRAGMENT;
10151 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
10152}
10153
10154ArrayRef<std::pair<unsigned, const char *>>
10155AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10156 using namespace AArch64II;
10157
10158 static const std::pair<unsigned, const char *> TargetFlags[] = {
10159 {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
10160 {MO_G3, "aarch64-g3"}, {MO_G2, "aarch64-g2"},
10161 {MO_G1, "aarch64-g1"}, {MO_G0, "aarch64-g0"},
10162 {MO_HI12, "aarch64-hi12"}};
10163 return ArrayRef(TargetFlags);
10164}
10165
10166ArrayRef<std::pair<unsigned, const char *>>
10167AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
10168 using namespace AArch64II;
10169
10170 static const std::pair<unsigned, const char *> TargetFlags[] = {
10171 {MO_COFFSTUB, "aarch64-coffstub"},
10172 {MO_GOT, "aarch64-got"},
10173 {MO_NC, "aarch64-nc"},
10174 {MO_S, "aarch64-s"},
10175 {MO_TLS, "aarch64-tls"},
10176 {MO_DLLIMPORT, "aarch64-dllimport"},
10177 {MO_PREL, "aarch64-prel"},
10178 {MO_TAGGED, "aarch64-tagged"},
10179 {MO_ARM64EC_CALLMANGLE, "aarch64-arm64ec-callmangle"},
10180 };
10181 return ArrayRef(TargetFlags);
10182}
10183
10184ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10185AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10186 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10187 {{MOSuppressPair, "aarch64-suppress-pair"},
10188 {MOStridedAccess, "aarch64-strided-access"}};
10189 return ArrayRef(TargetFlags);
10190}
10191
10192/// Constants defining how certain sequences should be outlined.
10193/// This encompasses how an outlined function should be called, and what kind of
10194/// frame should be emitted for that outlined function.
10195///
10196/// \p MachineOutlinerDefault implies that the function should be called with
10197/// a save and restore of LR to the stack.
10198///
10199/// That is,
10200///
10201/// I1 Save LR OUTLINED_FUNCTION:
10202/// I2 --> BL OUTLINED_FUNCTION I1
10203/// I3 Restore LR I2
10204/// I3
10205/// RET
10206///
10207/// * Call construction overhead: 3 (save + BL + restore)
10208/// * Frame construction overhead: 1 (ret)
10209/// * Requires stack fixups? Yes
10210///
10211/// \p MachineOutlinerTailCall implies that the function is being created from
10212/// a sequence of instructions ending in a return.
10213///
10214/// That is,
10215///
10216/// I1 OUTLINED_FUNCTION:
10217/// I2 --> B OUTLINED_FUNCTION I1
10218/// RET I2
10219/// RET
10220///
10221/// * Call construction overhead: 1 (B)
10222/// * Frame construction overhead: 0 (Return included in sequence)
10223/// * Requires stack fixups? No
10224///
10225/// \p MachineOutlinerNoLRSave implies that the function should be called using
10226/// a BL instruction, but doesn't require LR to be saved and restored. This
10227/// happens when LR is known to be dead.
10228///
10229/// That is,
10230///
10231/// I1 OUTLINED_FUNCTION:
10232/// I2 --> BL OUTLINED_FUNCTION I1
10233/// I3 I2
10234/// I3
10235/// RET
10236///
10237/// * Call construction overhead: 1 (BL)
10238/// * Frame construction overhead: 1 (RET)
10239/// * Requires stack fixups? No
10240///
10241/// \p MachineOutlinerThunk implies that the function is being created from
10242/// a sequence of instructions ending in a call. The outlined function is
10243/// called with a BL instruction, and the outlined function tail-calls the
10244/// original call destination.
10245///
10246/// That is,
10247///
10248/// I1 OUTLINED_FUNCTION:
10249/// I2 --> BL OUTLINED_FUNCTION I1
10250/// BL f I2
10251/// B f
10252/// * Call construction overhead: 1 (BL)
10253/// * Frame construction overhead: 0
10254/// * Requires stack fixups? No
10255///
10256/// \p MachineOutlinerRegSave implies that the function should be called with a
10257/// save and restore of LR to an available register. This allows us to avoid
10258/// stack fixups. Note that this outlining variant is compatible with the
10259/// NoLRSave case.
10260///
10261/// That is,
10262///
10263/// I1 Save LR OUTLINED_FUNCTION:
10264/// I2 --> BL OUTLINED_FUNCTION I1
10265/// I3 Restore LR I2
10266/// I3
10267/// RET
10268///
10269/// * Call construction overhead: 3 (save + BL + restore)
10270/// * Frame construction overhead: 1 (ret)
10271/// * Requires stack fixups? No
10272enum MachineOutlinerClass {
10273 MachineOutlinerDefault, /// Emit a save, restore, call, and return.
10274 MachineOutlinerTailCall, /// Only emit a branch.
10275 MachineOutlinerNoLRSave, /// Emit a call and return.
10276 MachineOutlinerThunk, /// Emit a call and tail-call.
10277 MachineOutlinerRegSave /// Same as default, but save to a register.
10278};
10279
10280enum MachineOutlinerMBBFlags {
10281 LRUnavailableSomewhere = 0x2,
10282 HasCalls = 0x4,
10283 UnsafeRegsDead = 0x8
10284};
10285
10286Register
10287AArch64InstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
10288 MachineFunction *MF = C.getMF();
10289 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
10290 const AArch64RegisterInfo *ARI =
10291 static_cast<const AArch64RegisterInfo *>(&TRI);
10292 // Check if there is an available register across the sequence that we can
10293 // use.
10294 for (unsigned Reg : AArch64::GPR64RegClass) {
10295 if (!ARI->isReservedReg(MF: *MF, Reg) &&
10296 Reg != AArch64::LR && // LR is not reserved, but don't use it.
10297 Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
10298 Reg != AArch64::X17 && // Ditto for X17.
10299 C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
10300 C.isAvailableInsideSeq(Reg, TRI))
10301 return Reg;
10302 }
10303 return Register();
10304}
10305
10306static bool
10307outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
10308 const outliner::Candidate &b) {
10309 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10310 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10311
10312 return MFIa->getSignReturnAddressCondition() ==
10313 MFIb->getSignReturnAddressCondition();
10314}
10315
10316static bool
10317outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
10318 const outliner::Candidate &b) {
10319 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10320 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10321
10322 return MFIa->shouldSignWithBKey() == MFIb->shouldSignWithBKey();
10323}
10324
10325static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
10326 const outliner::Candidate &b) {
10327 const AArch64Subtarget &SubtargetA =
10328 a.getMF()->getSubtarget<AArch64Subtarget>();
10329 const AArch64Subtarget &SubtargetB =
10330 b.getMF()->getSubtarget<AArch64Subtarget>();
10331 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
10332}
10333
10334std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10335AArch64InstrInfo::getOutliningCandidateInfo(
10336 const MachineModuleInfo &MMI,
10337 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10338 unsigned MinRepeats) const {
10339 unsigned SequenceSize = 0;
10340 for (auto &MI : RepeatedSequenceLocs[0])
10341 SequenceSize += getInstSizeInBytes(MI);
10342
10343 unsigned NumBytesToCreateFrame = 0;
10344
10345 // Avoid splitting ADRP ADD/LDR pair into outlined functions.
10346 // These instructions are fused together by the scheduler.
10347 // Any candidate where ADRP is the last instruction should be rejected
10348 // as that will lead to splitting ADRP pair.
10349 MachineInstr &LastMI = RepeatedSequenceLocs[0].back();
10350 MachineInstr &FirstMI = RepeatedSequenceLocs[0].front();
10351 if (LastMI.getOpcode() == AArch64::ADRP &&
10352 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_PAGE) != 0 &&
10353 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10354 return std::nullopt;
10355 }
10356
10357 // Similarly any candidate where the first instruction is ADD/LDR with a
10358 // page offset should be rejected to avoid ADRP splitting.
10359 if ((FirstMI.getOpcode() == AArch64::ADDXri ||
10360 FirstMI.getOpcode() == AArch64::LDRXui) &&
10361 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_PAGEOFF) != 0 &&
10362 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10363 return std::nullopt;
10364 }
10365
10366 // We only allow outlining for functions having exactly matching return
10367 // address signing attributes, i.e., all share the same value for the
10368 // attribute "sign-return-address" and all share the same type of key they
10369 // are signed with.
10370 // Additionally we require all functions to simultaneously either support
10371 // v8.3a features or not. Otherwise an outlined function could get signed
10372 // using dedicated v8.3 instructions and a call from a function that doesn't
10373 // support v8.3 instructions would therefore be invalid.
10374 if (std::adjacent_find(
10375 first: RepeatedSequenceLocs.begin(), last: RepeatedSequenceLocs.end(),
10376 binary_pred: [](const outliner::Candidate &a, const outliner::Candidate &b) {
10377 // Return true if a and b are non-equal w.r.t. return address
10378 // signing or support of v8.3a features
10379 if (outliningCandidatesSigningScopeConsensus(a, b) &&
10380 outliningCandidatesSigningKeyConsensus(a, b) &&
10381 outliningCandidatesV8_3OpsConsensus(a, b)) {
10382 return false;
10383 }
10384 return true;
10385 }) != RepeatedSequenceLocs.end()) {
10386 return std::nullopt;
10387 }
10388
10389 // Since at this point all candidates agree on their return address signing
10390 // picking just one is fine. If the candidate functions potentially sign their
10391 // return addresses, the outlined function should do the same. Note that in
10392 // the case of "sign-return-address"="non-leaf" this is an assumption: It is
10393 // not certainly true that the outlined function will have to sign its return
10394 // address but this decision is made later, when the decision to outline
10395 // has already been made.
10396 // The same holds for the number of additional instructions we need: On
10397 // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
10398 // necessary. However, at this point we don't know if the outlined function
10399 // will have a RET instruction so we assume the worst.
10400 const TargetRegisterInfo &TRI = getRegisterInfo();
10401 // Performing a tail call may require extra checks when PAuth is enabled.
10402 // If PAuth is disabled, set it to zero for uniformity.
10403 unsigned NumBytesToCheckLRInTCEpilogue = 0;
10404 const auto RASignCondition = RepeatedSequenceLocs[0]
10405 .getMF()
10406 ->getInfo<AArch64FunctionInfo>()
10407 ->getSignReturnAddressCondition();
10408 if (RASignCondition != SignReturnAddress::None) {
10409 // One PAC and one AUT instructions
10410 NumBytesToCreateFrame += 8;
10411
10412 // PAuth is enabled - set extra tail call cost, if any.
10413 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
10414 MF: *RepeatedSequenceLocs[0].getMF());
10415 NumBytesToCheckLRInTCEpilogue =
10416 AArch64PAuth::getCheckerSizeInBytes(Method: LRCheckMethod);
10417 // Checking the authenticated LR value may significantly impact
10418 // SequenceSize, so account for it for more precise results.
10419 if (isTailCallReturnInst(MI: RepeatedSequenceLocs[0].back()))
10420 SequenceSize += NumBytesToCheckLRInTCEpilogue;
10421
10422 // We have to check if sp modifying instructions would get outlined.
10423 // If so we only allow outlining if sp is unchanged overall, so matching
10424 // sub and add instructions are okay to outline, all other sp modifications
10425 // are not
10426 auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
10427 int SPValue = 0;
10428 for (auto &MI : C) {
10429 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI)) {
10430 switch (MI.getOpcode()) {
10431 case AArch64::ADDXri:
10432 case AArch64::ADDWri:
10433 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10434 assert(MI.getOperand(2).isImm() &&
10435 "Expected operand to be immediate");
10436 assert(MI.getOperand(1).isReg() &&
10437 "Expected operand to be a register");
10438 // Check if the add just increments sp. If so, we search for
10439 // matching sub instructions that decrement sp. If not, the
10440 // modification is illegal
10441 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10442 SPValue += MI.getOperand(i: 2).getImm();
10443 else
10444 return true;
10445 break;
10446 case AArch64::SUBXri:
10447 case AArch64::SUBWri:
10448 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10449 assert(MI.getOperand(2).isImm() &&
10450 "Expected operand to be immediate");
10451 assert(MI.getOperand(1).isReg() &&
10452 "Expected operand to be a register");
10453 // Check if the sub just decrements sp. If so, we search for
10454 // matching add instructions that increment sp. If not, the
10455 // modification is illegal
10456 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10457 SPValue -= MI.getOperand(i: 2).getImm();
10458 else
10459 return true;
10460 break;
10461 default:
10462 return true;
10463 }
10464 }
10465 }
10466 if (SPValue)
10467 return true;
10468 return false;
10469 };
10470 // Remove candidates with illegal stack modifying instructions
10471 llvm::erase_if(C&: RepeatedSequenceLocs, P: hasIllegalSPModification);
10472
10473 // If the sequence doesn't have enough candidates left, then we're done.
10474 if (RepeatedSequenceLocs.size() < MinRepeats)
10475 return std::nullopt;
10476 }
10477
10478 // Properties about candidate MBBs that hold for all of them.
10479 unsigned FlagsSetInAll = 0xF;
10480
10481 // Compute liveness information for each candidate, and set FlagsSetInAll.
10482 for (outliner::Candidate &C : RepeatedSequenceLocs)
10483 FlagsSetInAll &= C.Flags;
10484
10485 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
10486
10487 // Helper lambda which sets call information for every candidate.
10488 auto SetCandidateCallInfo =
10489 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
10490 for (outliner::Candidate &C : RepeatedSequenceLocs)
10491 C.setCallInfo(CID: CallID, CO: NumBytesForCall);
10492 };
10493
10494 unsigned FrameID = MachineOutlinerDefault;
10495 NumBytesToCreateFrame += 4;
10496
10497 bool HasBTI = any_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10498 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
10499 });
10500
10501 // We check to see if CFI Instructions are present, and if they are
10502 // we find the number of CFI Instructions in the candidates.
10503 unsigned CFICount = 0;
10504 for (auto &I : RepeatedSequenceLocs[0]) {
10505 if (I.isCFIInstruction())
10506 CFICount++;
10507 }
10508
10509 // We compare the number of found CFI Instructions to the number of CFI
10510 // instructions in the parent function for each candidate. We must check this
10511 // since if we outline one of the CFI instructions in a function, we have to
10512 // outline them all for correctness. If we do not, the address offsets will be
10513 // incorrect between the two sections of the program.
10514 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10515 std::vector<MCCFIInstruction> CFIInstructions =
10516 C.getMF()->getFrameInstructions();
10517
10518 if (CFICount > 0 && CFICount != CFIInstructions.size())
10519 return std::nullopt;
10520 }
10521
10522 // Returns true if an instructions is safe to fix up, false otherwise.
10523 auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
10524 if (MI.isCall())
10525 return true;
10526
10527 if (!MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI) &&
10528 !MI.readsRegister(Reg: AArch64::SP, TRI: &TRI))
10529 return true;
10530
10531 // Any modification of SP will break our code to save/restore LR.
10532 // FIXME: We could handle some instructions which add a constant
10533 // offset to SP, with a bit more work.
10534 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI))
10535 return false;
10536
10537 // At this point, we have a stack instruction that we might need to
10538 // fix up. We'll handle it if it's a load or store.
10539 if (MI.mayLoadOrStore()) {
10540 const MachineOperand *Base; // Filled with the base operand of MI.
10541 int64_t Offset; // Filled with the offset of MI.
10542 bool OffsetIsScalable;
10543
10544 // Does it allow us to offset the base operand and is the base the
10545 // register SP?
10546 if (!getMemOperandWithOffset(MI, BaseOp&: Base, Offset, OffsetIsScalable, TRI: &TRI) ||
10547 !Base->isReg() || Base->getReg() != AArch64::SP)
10548 return false;
10549
10550 // Fixe-up code below assumes bytes.
10551 if (OffsetIsScalable)
10552 return false;
10553
10554 // Find the minimum/maximum offset for this instruction and check
10555 // if fixing it up would be in range.
10556 int64_t MinOffset,
10557 MaxOffset; // Unscaled offsets for the instruction.
10558 // The scale to multiply the offsets by.
10559 TypeSize Scale(0U, false), DummyWidth(0U, false);
10560 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width&: DummyWidth, MinOffset, MaxOffset);
10561
10562 Offset += 16; // Update the offset to what it would be if we outlined.
10563 if (Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
10564 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
10565 return false;
10566
10567 // It's in range, so we can outline it.
10568 return true;
10569 }
10570
10571 // FIXME: Add handling for instructions like "add x0, sp, #8".
10572
10573 // We can't fix it up, so don't outline it.
10574 return false;
10575 };
10576
10577 // True if it's possible to fix up each stack instruction in this sequence.
10578 // Important for frames/call variants that modify the stack.
10579 bool AllStackInstrsSafe =
10580 llvm::all_of(Range&: RepeatedSequenceLocs[0], P: IsSafeToFixup);
10581
10582 // If the last instruction in any candidate is a terminator, then we should
10583 // tail call all of the candidates.
10584 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10585 FrameID = MachineOutlinerTailCall;
10586 NumBytesToCreateFrame = 0;
10587 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
10588 SetCandidateCallInfo(MachineOutlinerTailCall, NumBytesForCall);
10589 }
10590
10591 else if (LastInstrOpcode == AArch64::BL ||
10592 ((LastInstrOpcode == AArch64::BLR ||
10593 LastInstrOpcode == AArch64::BLRNoIP) &&
10594 !HasBTI)) {
10595 // FIXME: Do we need to check if the code after this uses the value of LR?
10596 FrameID = MachineOutlinerThunk;
10597 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
10598 SetCandidateCallInfo(MachineOutlinerThunk, 4);
10599 }
10600
10601 else {
10602 // We need to decide how to emit calls + frames. We can always emit the same
10603 // frame if we don't need to save to the stack. If we have to save to the
10604 // stack, then we need a different frame.
10605 unsigned NumBytesNoStackCalls = 0;
10606 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
10607
10608 // Check if we have to save LR.
10609 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10610 bool LRAvailable =
10611 (C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
10612 ? C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI)
10613 : true;
10614 // If we have a noreturn caller, then we're going to be conservative and
10615 // say that we have to save LR. If we don't have a ret at the end of the
10616 // block, then we can't reason about liveness accurately.
10617 //
10618 // FIXME: We can probably do better than always disabling this in
10619 // noreturn functions by fixing up the liveness info.
10620 bool IsNoReturn =
10621 C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoReturn);
10622
10623 // Is LR available? If so, we don't need a save.
10624 if (LRAvailable && !IsNoReturn) {
10625 NumBytesNoStackCalls += 4;
10626 C.setCallInfo(CID: MachineOutlinerNoLRSave, CO: 4);
10627 CandidatesWithoutStackFixups.push_back(x: C);
10628 }
10629
10630 // Is an unused register available? If so, we won't modify the stack, so
10631 // we can outline with the same frame type as those that don't save LR.
10632 else if (findRegisterToSaveLRTo(C)) {
10633 NumBytesNoStackCalls += 12;
10634 C.setCallInfo(CID: MachineOutlinerRegSave, CO: 12);
10635 CandidatesWithoutStackFixups.push_back(x: C);
10636 }
10637
10638 // Is SP used in the sequence at all? If not, we don't have to modify
10639 // the stack, so we are guaranteed to get the same frame.
10640 else if (C.isAvailableInsideSeq(Reg: AArch64::SP, TRI)) {
10641 NumBytesNoStackCalls += 12;
10642 C.setCallInfo(CID: MachineOutlinerDefault, CO: 12);
10643 CandidatesWithoutStackFixups.push_back(x: C);
10644 }
10645
10646 // If we outline this, we need to modify the stack. Pretend we don't
10647 // outline this by saving all of its bytes.
10648 else {
10649 NumBytesNoStackCalls += SequenceSize;
10650 }
10651 }
10652
10653 // If there are no places where we have to save LR, then note that we
10654 // don't have to update the stack. Otherwise, give every candidate the
10655 // default call type, as long as it's safe to do so.
10656 if (!AllStackInstrsSafe ||
10657 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
10658 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
10659 FrameID = MachineOutlinerNoLRSave;
10660 if (RepeatedSequenceLocs.size() < MinRepeats)
10661 return std::nullopt;
10662 } else {
10663 SetCandidateCallInfo(MachineOutlinerDefault, 12);
10664
10665 // Bugzilla ID: 46767
10666 // TODO: Check if fixing up the stack more than once is safe so we can
10667 // outline these.
10668 //
10669 // An outline resulting in a caller that requires stack fixups at the
10670 // callsite to a callee that also requires stack fixups can happen when
10671 // there are no available registers at the candidate callsite for a
10672 // candidate that itself also has calls.
10673 //
10674 // In other words if function_containing_sequence in the following pseudo
10675 // assembly requires that we save LR at the point of the call, but there
10676 // are no available registers: in this case we save using SP and as a
10677 // result the SP offsets requires stack fixups by multiples of 16.
10678 //
10679 // function_containing_sequence:
10680 // ...
10681 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10682 // call OUTLINED_FUNCTION_N
10683 // restore LR from SP
10684 // ...
10685 //
10686 // OUTLINED_FUNCTION_N:
10687 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10688 // ...
10689 // bl foo
10690 // restore LR from SP
10691 // ret
10692 //
10693 // Because the code to handle more than one stack fixup does not
10694 // currently have the proper checks for legality, these cases will assert
10695 // in the AArch64 MachineOutliner. This is because the code to do this
10696 // needs more hardening, testing, better checks that generated code is
10697 // legal, etc and because it is only verified to handle a single pass of
10698 // stack fixup.
10699 //
10700 // The assert happens in AArch64InstrInfo::buildOutlinedFrame to catch
10701 // these cases until they are known to be handled. Bugzilla 46767 is
10702 // referenced in comments at the assert site.
10703 //
10704 // To avoid asserting (or generating non-legal code on noassert builds)
10705 // we remove all candidates which would need more than one stack fixup by
10706 // pruning the cases where the candidate has calls while also having no
10707 // available LR and having no available general purpose registers to copy
10708 // LR to (ie one extra stack save/restore).
10709 //
10710 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10711 erase_if(C&: RepeatedSequenceLocs, P: [this, &TRI](outliner::Candidate &C) {
10712 auto IsCall = [](const MachineInstr &MI) { return MI.isCall(); };
10713 return (llvm::any_of(Range&: C, P: IsCall)) &&
10714 (!C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI) ||
10715 !findRegisterToSaveLRTo(C));
10716 });
10717 }
10718 }
10719
10720 // If we dropped all of the candidates, bail out here.
10721 if (RepeatedSequenceLocs.size() < MinRepeats)
10722 return std::nullopt;
10723 }
10724
10725 // Does every candidate's MBB contain a call? If so, then we might have a call
10726 // in the range.
10727 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10728 // Check if the range contains a call. These require a save + restore of the
10729 // link register.
10730 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
10731 bool ModStackToSaveLR = false;
10732 if (any_of(Range: drop_end(RangeOrContainer&: FirstCand),
10733 P: [](const MachineInstr &MI) { return MI.isCall(); }))
10734 ModStackToSaveLR = true;
10735
10736 // Handle the last instruction separately. If this is a tail call, then the
10737 // last instruction is a call. We don't want to save + restore in this case.
10738 // However, it could be possible that the last instruction is a call without
10739 // it being valid to tail call this sequence. We should consider this as
10740 // well.
10741 else if (FrameID != MachineOutlinerThunk &&
10742 FrameID != MachineOutlinerTailCall && FirstCand.back().isCall())
10743 ModStackToSaveLR = true;
10744
10745 if (ModStackToSaveLR) {
10746 // We can't fix up the stack. Bail out.
10747 if (!AllStackInstrsSafe)
10748 return std::nullopt;
10749
10750 // Save + restore LR.
10751 NumBytesToCreateFrame += 8;
10752 }
10753 }
10754
10755 // If we have CFI instructions, we can only outline if the outlined section
10756 // can be a tail call
10757 if (FrameID != MachineOutlinerTailCall && CFICount > 0)
10758 return std::nullopt;
10759
10760 return std::make_unique<outliner::OutlinedFunction>(
10761 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: NumBytesToCreateFrame, args&: FrameID);
10762}
10763
10764void AArch64InstrInfo::mergeOutliningCandidateAttributes(
10765 Function &F, std::vector<outliner::Candidate> &Candidates) const {
10766 // If a bunch of candidates reach this point they must agree on their return
10767 // address signing. It is therefore enough to just consider the signing
10768 // behaviour of one of them
10769 const auto &CFn = Candidates.front().getMF()->getFunction();
10770
10771 if (CFn.hasFnAttribute(Kind: "ptrauth-returns"))
10772 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-returns"));
10773 if (CFn.hasFnAttribute(Kind: "ptrauth-auth-traps"))
10774 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-auth-traps"));
10775 // Since all candidates belong to the same module, just copy the
10776 // function-level attributes of an arbitrary function.
10777 if (CFn.hasFnAttribute(Kind: "sign-return-address"))
10778 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address"));
10779 if (CFn.hasFnAttribute(Kind: "sign-return-address-key"))
10780 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address-key"));
10781
10782 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
10783}
10784
10785bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
10786 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
10787 const Function &F = MF.getFunction();
10788
10789 // Can F be deduplicated by the linker? If it can, don't outline from it.
10790 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
10791 return false;
10792
10793 // Don't outline from functions with section markings; the program could
10794 // expect that all the code is in the named section.
10795 // FIXME: Allow outlining from multiple functions with the same section
10796 // marking.
10797 if (F.hasSection())
10798 return false;
10799
10800 // Outlining from functions with redzones is unsafe since the outliner may
10801 // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
10802 // outline from it.
10803 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
10804 if (!AFI || AFI->hasRedZone().value_or(u: true))
10805 return false;
10806
10807 // FIXME: Determine whether it is safe to outline from functions which contain
10808 // streaming-mode changes. We may need to ensure any smstart/smstop pairs are
10809 // outlined together and ensure it is safe to outline with async unwind info,
10810 // required for saving & restoring VG around calls.
10811 if (AFI->hasStreamingModeChanges())
10812 return false;
10813
10814 // FIXME: Teach the outliner to generate/handle Windows unwind info.
10815 if (MF.getTarget().getMCAsmInfo().usesWindowsCFI())
10816 return false;
10817
10818 // It's safe to outline from MF.
10819 return true;
10820}
10821
10822SmallVector<std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10823AArch64InstrInfo::getOutlinableRanges(MachineBasicBlock &MBB,
10824 unsigned &Flags) const {
10825 assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
10826 "Must track liveness!");
10827 SmallVector<
10828 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10829 Ranges;
10830 // According to the AArch64 Procedure Call Standard, the following are
10831 // undefined on entry/exit from a function call:
10832 //
10833 // * Registers x16, x17, (and thus w16, w17)
10834 // * Condition codes (and thus the NZCV register)
10835 //
10836 // If any of these registers are used inside or live across an outlined
10837 // function, then they may be modified later, either by the compiler or
10838 // some other tool (like the linker).
10839 //
10840 // To avoid outlining in these situations, partition each block into ranges
10841 // where these registers are dead. We will only outline from those ranges.
10842 LiveRegUnits LRU(getRegisterInfo());
10843 auto AreAllUnsafeRegsDead = [&LRU]() {
10844 return LRU.available(Reg: AArch64::W16) && LRU.available(Reg: AArch64::W17) &&
10845 LRU.available(Reg: AArch64::NZCV);
10846 };
10847
10848 // We need to know if LR is live across an outlining boundary later on in
10849 // order to decide how we'll create the outlined call, frame, etc.
10850 //
10851 // It's pretty expensive to check this for *every candidate* within a block.
10852 // That's some potentially n^2 behaviour, since in the worst case, we'd need
10853 // to compute liveness from the end of the block for O(n) candidates within
10854 // the block.
10855 //
10856 // So, to improve the average case, let's keep track of liveness from the end
10857 // of the block to the beginning of *every outlinable range*. If we know that
10858 // LR is available in every range we could outline from, then we know that
10859 // we don't need to check liveness for any candidate within that range.
10860 bool LRAvailableEverywhere = true;
10861 // Compute liveness bottom-up.
10862 LRU.addLiveOuts(MBB);
10863 // Update flags that require info about the entire MBB.
10864 auto UpdateWholeMBBFlags = [&Flags](const MachineInstr &MI) {
10865 if (MI.isCall() && !MI.isTerminator())
10866 Flags |= MachineOutlinerMBBFlags::HasCalls;
10867 };
10868 // Range: [RangeBegin, RangeEnd)
10869 MachineBasicBlock::instr_iterator RangeBegin, RangeEnd;
10870 unsigned RangeLen;
10871 auto CreateNewRangeStartingAt =
10872 [&RangeBegin, &RangeEnd,
10873 &RangeLen](MachineBasicBlock::instr_iterator NewBegin) {
10874 RangeBegin = NewBegin;
10875 RangeEnd = std::next(x: RangeBegin);
10876 RangeLen = 0;
10877 };
10878 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
10879 // At least one unsafe register is not dead. We do not want to outline at
10880 // this point. If it is long enough to outline from and does not cross a
10881 // bundle boundary, save the range [RangeBegin, RangeEnd).
10882 if (RangeLen <= 1)
10883 return;
10884 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
10885 return;
10886 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
10887 return;
10888 Ranges.emplace_back(Args&: RangeBegin, Args&: RangeEnd);
10889 };
10890 // Find the first point where all unsafe registers are dead.
10891 // FIND: <safe instr> <-- end of first potential range
10892 // SKIP: <unsafe def>
10893 // SKIP: ... everything between ...
10894 // SKIP: <unsafe use>
10895 auto FirstPossibleEndPt = MBB.instr_rbegin();
10896 for (; FirstPossibleEndPt != MBB.instr_rend(); ++FirstPossibleEndPt) {
10897 if (!FirstPossibleEndPt->isDebugInstr())
10898 LRU.stepBackward(MI: *FirstPossibleEndPt);
10899 // Update flags that impact how we outline across the entire block,
10900 // regardless of safety.
10901 UpdateWholeMBBFlags(*FirstPossibleEndPt);
10902 if (AreAllUnsafeRegsDead())
10903 break;
10904 }
10905 // If we exhausted the entire block, we have no safe ranges to outline.
10906 if (FirstPossibleEndPt == MBB.instr_rend())
10907 return Ranges;
10908 // Current range.
10909 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
10910 // StartPt points to the first place where all unsafe registers
10911 // are dead (if there is any such point). Begin partitioning the MBB into
10912 // ranges.
10913 for (auto &MI : make_range(x: FirstPossibleEndPt, y: MBB.instr_rend())) {
10914 if (!MI.isDebugInstr())
10915 LRU.stepBackward(MI);
10916 UpdateWholeMBBFlags(MI);
10917 if (!AreAllUnsafeRegsDead()) {
10918 SaveRangeIfNonEmpty();
10919 CreateNewRangeStartingAt(MI.getIterator());
10920 continue;
10921 }
10922 LRAvailableEverywhere &= LRU.available(Reg: AArch64::LR);
10923 RangeBegin = MI.getIterator();
10924 ++RangeLen;
10925 }
10926 // Above loop misses the last (or only) range. If we are still safe, then
10927 // let's save the range.
10928 if (AreAllUnsafeRegsDead())
10929 SaveRangeIfNonEmpty();
10930 if (Ranges.empty())
10931 return Ranges;
10932 // We found the ranges bottom-up. Mapping expects the top-down. Reverse
10933 // the order.
10934 std::reverse(first: Ranges.begin(), last: Ranges.end());
10935 // If there is at least one outlinable range where LR is unavailable
10936 // somewhere, remember that.
10937 if (!LRAvailableEverywhere)
10938 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
10939 return Ranges;
10940}
10941
10942outliner::InstrType
10943AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
10944 MachineBasicBlock::iterator &MIT,
10945 unsigned Flags) const {
10946 MachineInstr &MI = *MIT;
10947
10948 // Don't outline anything used for return address signing. The outlined
10949 // function will get signed later if needed
10950 switch (MI.getOpcode()) {
10951 case AArch64::PACM:
10952 case AArch64::PACIASP:
10953 case AArch64::PACIBSP:
10954 case AArch64::PACIASPPC:
10955 case AArch64::PACIBSPPC:
10956 case AArch64::AUTIASP:
10957 case AArch64::AUTIBSP:
10958 case AArch64::AUTIASPPCi:
10959 case AArch64::AUTIASPPCr:
10960 case AArch64::AUTIBSPPCi:
10961 case AArch64::AUTIBSPPCr:
10962 case AArch64::RETAA:
10963 case AArch64::RETAB:
10964 case AArch64::RETAASPPCi:
10965 case AArch64::RETAASPPCr:
10966 case AArch64::RETABSPPCi:
10967 case AArch64::RETABSPPCr:
10968 case AArch64::EMITBKEY:
10969 case AArch64::PAUTH_PROLOGUE:
10970 case AArch64::PAUTH_EPILOGUE:
10971 return outliner::InstrType::Illegal;
10972 }
10973
10974 // We can only outline these if we will tail call the outlined function, or
10975 // fix up the CFI offsets. Currently, CFI instructions are outlined only if
10976 // in a tail call.
10977 //
10978 // FIXME: If the proper fixups for the offset are implemented, this should be
10979 // possible.
10980 if (MI.isCFIInstruction())
10981 return outliner::InstrType::Legal;
10982
10983 // Is this a terminator for a basic block?
10984 if (MI.isTerminator())
10985 // TargetInstrInfo::getOutliningType has already filtered out anything
10986 // that would break this, so we can allow it here.
10987 return outliner::InstrType::Legal;
10988
10989 // Make sure none of the operands are un-outlinable.
10990 for (const MachineOperand &MOP : MI.operands()) {
10991 // A check preventing CFI indices was here before, but only CFI
10992 // instructions should have those.
10993 assert(!MOP.isCFIIndex());
10994
10995 // If it uses LR or W30 explicitly, then don't touch it.
10996 if (MOP.isReg() && !MOP.isImplicit() &&
10997 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
10998 return outliner::InstrType::Illegal;
10999 }
11000
11001 // Special cases for instructions that can always be outlined, but will fail
11002 // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
11003 // be outlined because they don't require a *specific* value to be in LR.
11004 if (MI.getOpcode() == AArch64::ADRP)
11005 return outliner::InstrType::Legal;
11006
11007 // If MI is a call we might be able to outline it. We don't want to outline
11008 // any calls that rely on the position of items on the stack. When we outline
11009 // something containing a call, we have to emit a save and restore of LR in
11010 // the outlined function. Currently, this always happens by saving LR to the
11011 // stack. Thus, if we outline, say, half the parameters for a function call
11012 // plus the call, then we'll break the callee's expectations for the layout
11013 // of the stack.
11014 //
11015 // FIXME: Allow calls to functions which construct a stack frame, as long
11016 // as they don't access arguments on the stack.
11017 // FIXME: Figure out some way to analyze functions defined in other modules.
11018 // We should be able to compute the memory usage based on the IR calling
11019 // convention, even if we can't see the definition.
11020 if (MI.isCall()) {
11021 // Get the function associated with the call. Look at each operand and find
11022 // the one that represents the callee and get its name.
11023 const Function *Callee = nullptr;
11024 for (const MachineOperand &MOP : MI.operands()) {
11025 if (MOP.isGlobal()) {
11026 Callee = dyn_cast<Function>(Val: MOP.getGlobal());
11027 break;
11028 }
11029 }
11030
11031 // Never outline calls to mcount. There isn't any rule that would require
11032 // this, but the Linux kernel's "ftrace" feature depends on it.
11033 if (Callee && Callee->getName() == "\01_mcount")
11034 return outliner::InstrType::Illegal;
11035
11036 // If we don't know anything about the callee, assume it depends on the
11037 // stack layout of the caller. In that case, it's only legal to outline
11038 // as a tail-call. Explicitly list the call instructions we know about so we
11039 // don't get unexpected results with call pseudo-instructions.
11040 auto UnknownCallOutlineType = outliner::InstrType::Illegal;
11041 if (MI.getOpcode() == AArch64::BLR ||
11042 MI.getOpcode() == AArch64::BLRNoIP || MI.getOpcode() == AArch64::BL)
11043 UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
11044
11045 if (!Callee)
11046 return UnknownCallOutlineType;
11047
11048 // We have a function we have information about. Check it if it's something
11049 // can safely outline.
11050 MachineFunction *CalleeMF = MMI.getMachineFunction(F: *Callee);
11051
11052 // We don't know what's going on with the callee at all. Don't touch it.
11053 if (!CalleeMF)
11054 return UnknownCallOutlineType;
11055
11056 // Check if we know anything about the callee saves on the function. If we
11057 // don't, then don't touch it, since that implies that we haven't
11058 // computed anything about its stack frame yet.
11059 MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
11060 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
11061 MFI.getNumObjects() > 0)
11062 return UnknownCallOutlineType;
11063
11064 // At this point, we can say that CalleeMF ought to not pass anything on the
11065 // stack. Therefore, we can outline it.
11066 return outliner::InstrType::Legal;
11067 }
11068
11069 // Don't touch the link register or W30.
11070 if (MI.readsRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()) ||
11071 MI.modifiesRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()))
11072 return outliner::InstrType::Illegal;
11073
11074 // Don't outline BTI instructions, because that will prevent the outlining
11075 // site from being indirectly callable.
11076 if (hasBTISemantics(MI))
11077 return outliner::InstrType::Illegal;
11078
11079 return outliner::InstrType::Legal;
11080}
11081
11082void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
11083 for (MachineInstr &MI : MBB) {
11084 const MachineOperand *Base;
11085 TypeSize Width(0, false);
11086 int64_t Offset;
11087 bool OffsetIsScalable;
11088
11089 // Is this a load or store with an immediate offset with SP as the base?
11090 if (!MI.mayLoadOrStore() ||
11091 !getMemOperandWithOffsetWidth(LdSt: MI, BaseOp&: Base, Offset, OffsetIsScalable, Width,
11092 TRI: &RI) ||
11093 (Base->isReg() && Base->getReg() != AArch64::SP))
11094 continue;
11095
11096 // It is, so we have to fix it up.
11097 TypeSize Scale(0U, false);
11098 int64_t Dummy1, Dummy2;
11099
11100 MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(LdSt&: MI);
11101 assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
11102 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2);
11103 assert(Scale != 0 && "Unexpected opcode!");
11104 assert(!OffsetIsScalable && "Expected offset to be a byte offset");
11105
11106 // We've pushed the return address to the stack, so add 16 to the offset.
11107 // This is safe, since we already checked if it would overflow when we
11108 // checked if this instruction was legal to outline.
11109 int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedValue();
11110 StackOffsetOperand.setImm(NewImm);
11111 }
11112}
11113
11114static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
11115 const AArch64InstrInfo *TII,
11116 bool ShouldSignReturnAddr) {
11117 if (!ShouldSignReturnAddr)
11118 return;
11119
11120 BuildMI(BB&: MBB, I: MBB.begin(), MIMD: DebugLoc(), MCID: TII->get(Opcode: AArch64::PAUTH_PROLOGUE))
11121 .setMIFlag(MachineInstr::FrameSetup);
11122 TII->createPauthEpilogueInstr(MBB, DL: DebugLoc());
11123}
11124
11125void AArch64InstrInfo::buildOutlinedFrame(
11126 MachineBasicBlock &MBB, MachineFunction &MF,
11127 const outliner::OutlinedFunction &OF) const {
11128
11129 AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
11130
11131 if (OF.FrameConstructionID == MachineOutlinerTailCall)
11132 FI->setOutliningStyle("Tail Call");
11133 else if (OF.FrameConstructionID == MachineOutlinerThunk) {
11134 // For thunk outlining, rewrite the last instruction from a call to a
11135 // tail-call.
11136 MachineInstr *Call = &*--MBB.instr_end();
11137 unsigned TailOpcode;
11138 if (Call->getOpcode() == AArch64::BL) {
11139 TailOpcode = AArch64::TCRETURNdi;
11140 } else {
11141 assert(Call->getOpcode() == AArch64::BLR ||
11142 Call->getOpcode() == AArch64::BLRNoIP);
11143 TailOpcode = AArch64::TCRETURNriALL;
11144 }
11145 MachineInstr *TC = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpcode))
11146 .add(MO: Call->getOperand(i: 0))
11147 .addImm(Val: 0);
11148 MBB.insert(I: MBB.end(), MI: TC);
11149 Call->eraseFromParent();
11150
11151 FI->setOutliningStyle("Thunk");
11152 }
11153
11154 bool IsLeafFunction = true;
11155
11156 // Is there a call in the outlined range?
11157 auto IsNonTailCall = [](const MachineInstr &MI) {
11158 return MI.isCall() && !MI.isReturn();
11159 };
11160
11161 if (llvm::any_of(Range: MBB.instrs(), P: IsNonTailCall)) {
11162 // Fix up the instructions in the range, since we're going to modify the
11163 // stack.
11164
11165 // Bugzilla ID: 46767
11166 // TODO: Check if fixing up twice is safe so we can outline these.
11167 assert(OF.FrameConstructionID != MachineOutlinerDefault &&
11168 "Can only fix up stack references once");
11169 fixupPostOutline(MBB);
11170
11171 IsLeafFunction = false;
11172
11173 // LR has to be a live in so that we can save it.
11174 if (!MBB.isLiveIn(Reg: AArch64::LR))
11175 MBB.addLiveIn(PhysReg: AArch64::LR);
11176
11177 MachineBasicBlock::iterator It = MBB.begin();
11178 MachineBasicBlock::iterator Et = MBB.end();
11179
11180 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11181 OF.FrameConstructionID == MachineOutlinerThunk)
11182 Et = std::prev(x: MBB.end());
11183
11184 // Insert a save before the outlined region
11185 MachineInstr *STRXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11186 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11187 .addReg(RegNo: AArch64::LR)
11188 .addReg(RegNo: AArch64::SP)
11189 .addImm(Val: -16);
11190 It = MBB.insert(I: It, MI: STRXpre);
11191
11192 if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
11193 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
11194
11195 // Add a CFI saying the stack was moved 16 B down.
11196 CFIBuilder.buildDefCFAOffset(Offset: 16);
11197
11198 // Add a CFI saying that the LR that we want to find is now 16 B higher
11199 // than before.
11200 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -16);
11201 }
11202
11203 // Insert a restore before the terminator for the function.
11204 MachineInstr *LDRXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11205 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11206 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11207 .addReg(RegNo: AArch64::SP)
11208 .addImm(Val: 16);
11209 Et = MBB.insert(I: Et, MI: LDRXpost);
11210 }
11211
11212 auto RASignCondition = FI->getSignReturnAddressCondition();
11213 bool ShouldSignReturnAddr = AArch64FunctionInfo::shouldSignReturnAddress(
11214 Condition: RASignCondition, IsLRSpilled: !IsLeafFunction);
11215
11216 // If this is a tail call outlined function, then there's already a return.
11217 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11218 OF.FrameConstructionID == MachineOutlinerThunk) {
11219 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11220 return;
11221 }
11222
11223 // It's not a tail call, so we have to insert the return ourselves.
11224
11225 // LR has to be a live in so that we can return to it.
11226 if (!MBB.isLiveIn(Reg: AArch64::LR))
11227 MBB.addLiveIn(PhysReg: AArch64::LR);
11228
11229 MachineInstr *ret = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::RET))
11230 .addReg(RegNo: AArch64::LR);
11231 MBB.insert(I: MBB.end(), MI: ret);
11232
11233 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11234
11235 FI->setOutliningStyle("Function");
11236
11237 // Did we have to modify the stack by saving the link register?
11238 if (OF.FrameConstructionID != MachineOutlinerDefault)
11239 return;
11240
11241 // We modified the stack.
11242 // Walk over the basic block and fix up all the stack accesses.
11243 fixupPostOutline(MBB);
11244}
11245
11246MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
11247 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
11248 MachineFunction &MF, outliner::Candidate &C) const {
11249
11250 // Are we tail calling?
11251 if (C.CallConstructionID == MachineOutlinerTailCall) {
11252 // If yes, then we can just branch to the label.
11253 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::TCRETURNdi))
11254 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()))
11255 .addImm(Val: 0));
11256 return It;
11257 }
11258
11259 // Are we saving the link register?
11260 if (C.CallConstructionID == MachineOutlinerNoLRSave ||
11261 C.CallConstructionID == MachineOutlinerThunk) {
11262 // No, so just insert the call.
11263 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11264 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11265 return It;
11266 }
11267
11268 // We want to return the spot where we inserted the call.
11269 MachineBasicBlock::iterator CallPt;
11270
11271 // Instructions for saving and restoring LR around the call instruction we're
11272 // going to insert.
11273 MachineInstr *Save;
11274 MachineInstr *Restore;
11275 // Can we save to a register?
11276 if (C.CallConstructionID == MachineOutlinerRegSave) {
11277 // FIXME: This logic should be sunk into a target-specific interface so that
11278 // we don't have to recompute the register.
11279 Register Reg = findRegisterToSaveLRTo(C);
11280 assert(Reg && "No callee-saved register available?");
11281
11282 // LR has to be a live in so that we can save it.
11283 if (!MBB.isLiveIn(Reg: AArch64::LR))
11284 MBB.addLiveIn(PhysReg: AArch64::LR);
11285
11286 // Save and restore LR from Reg.
11287 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: Reg)
11288 .addReg(RegNo: AArch64::XZR)
11289 .addReg(RegNo: AArch64::LR)
11290 .addImm(Val: 0);
11291 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: AArch64::LR)
11292 .addReg(RegNo: AArch64::XZR)
11293 .addReg(RegNo: Reg)
11294 .addImm(Val: 0);
11295 } else {
11296 // We have the default case. Save and restore from SP.
11297 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11298 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11299 .addReg(RegNo: AArch64::LR)
11300 .addReg(RegNo: AArch64::SP)
11301 .addImm(Val: -16);
11302 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11303 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11304 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11305 .addReg(RegNo: AArch64::SP)
11306 .addImm(Val: 16);
11307 }
11308
11309 It = MBB.insert(I: It, MI: Save);
11310 It++;
11311
11312 // Insert the call.
11313 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11314 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11315 CallPt = It;
11316 It++;
11317
11318 It = MBB.insert(I: It, MI: Restore);
11319 return CallPt;
11320}
11321
11322bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
11323 MachineFunction &MF) const {
11324 return MF.getFunction().hasMinSize();
11325}
11326
11327void AArch64InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
11328 MachineBasicBlock::iterator Iter,
11329 DebugLoc &DL,
11330 bool AllowSideEffects) const {
11331 const MachineFunction &MF = *MBB.getParent();
11332 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
11333 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
11334
11335 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
11336 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg).addImm(Val: 0).addImm(Val: 0);
11337 } else if (STI.isSVEorStreamingSVEAvailable()) {
11338 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::DUP_ZI_D), DestReg: Reg)
11339 .addImm(Val: 0)
11340 .addImm(Val: 0);
11341 } else if (STI.isNeonAvailable()) {
11342 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVIv2d_ns), DestReg: Reg)
11343 .addImm(Val: 0);
11344 } else {
11345 // This is a streaming-compatible function without SVE. We don't have full
11346 // Neon (just FPRs), so we can at most use the first 64-bit sub-register.
11347 // So given `movi v..` would be illegal use `fmov d..` instead.
11348 assert(STI.hasNEON() && "Expected to have NEON.");
11349 Register Reg64 = TRI.getSubReg(Reg, Idx: AArch64::dsub);
11350 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg: Reg64);
11351 }
11352}
11353
11354std::optional<DestSourcePair>
11355AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
11356
11357 // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
11358 // and zero immediate operands used as an alias for mov instruction.
11359 if (((MI.getOpcode() == AArch64::ORRWrs &&
11360 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11361 MI.getOperand(i: 3).getImm() == 0x0) ||
11362 (MI.getOpcode() == AArch64::ORRWrr &&
11363 MI.getOperand(i: 1).getReg() == AArch64::WZR)) &&
11364 // Check that the w->w move is not a zero-extending w->x mov.
11365 (!MI.getOperand(i: 0).getReg().isVirtual() ||
11366 MI.getOperand(i: 0).getSubReg() == 0) &&
11367 (!MI.getOperand(i: 0).getReg().isPhysical() ||
11368 MI.findRegisterDefOperandIdx(Reg: getXRegFromWReg(Reg: MI.getOperand(i: 0).getReg()),
11369 /*TRI=*/nullptr) == -1))
11370 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11371
11372 if (MI.getOpcode() == AArch64::ORRXrs &&
11373 MI.getOperand(i: 1).getReg() == AArch64::XZR &&
11374 MI.getOperand(i: 3).getImm() == 0x0)
11375 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11376
11377 return std::nullopt;
11378}
11379
11380std::optional<DestSourcePair>
11381AArch64InstrInfo::isCopyLikeInstrImpl(const MachineInstr &MI) const {
11382 if ((MI.getOpcode() == AArch64::ORRWrs &&
11383 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11384 MI.getOperand(i: 3).getImm() == 0x0) ||
11385 (MI.getOpcode() == AArch64::ORRWrr &&
11386 MI.getOperand(i: 1).getReg() == AArch64::WZR))
11387 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11388 return std::nullopt;
11389}
11390
11391std::optional<RegImmPair>
11392AArch64InstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
11393 int Sign = 1;
11394 int64_t Offset = 0;
11395
11396 // TODO: Handle cases where Reg is a super- or sub-register of the
11397 // destination register.
11398 const MachineOperand &Op0 = MI.getOperand(i: 0);
11399 if (!Op0.isReg() || Reg != Op0.getReg())
11400 return std::nullopt;
11401
11402 switch (MI.getOpcode()) {
11403 default:
11404 return std::nullopt;
11405 case AArch64::SUBWri:
11406 case AArch64::SUBXri:
11407 case AArch64::SUBSWri:
11408 case AArch64::SUBSXri:
11409 Sign *= -1;
11410 [[fallthrough]];
11411 case AArch64::ADDSWri:
11412 case AArch64::ADDSXri:
11413 case AArch64::ADDWri:
11414 case AArch64::ADDXri: {
11415 // TODO: Third operand can be global address (usually some string).
11416 if (!MI.getOperand(i: 0).isReg() || !MI.getOperand(i: 1).isReg() ||
11417 !MI.getOperand(i: 2).isImm())
11418 return std::nullopt;
11419 int Shift = MI.getOperand(i: 3).getImm();
11420 assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
11421 Offset = Sign * (MI.getOperand(i: 2).getImm() << Shift);
11422 }
11423 }
11424 return RegImmPair{MI.getOperand(i: 1).getReg(), Offset};
11425}
11426
11427/// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
11428/// the destination register then, if possible, describe the value in terms of
11429/// the source register.
11430static std::optional<ParamLoadedValue>
11431describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
11432 const TargetInstrInfo *TII,
11433 const TargetRegisterInfo *TRI) {
11434 auto DestSrc = TII->isCopyLikeInstr(MI);
11435 if (!DestSrc)
11436 return std::nullopt;
11437
11438 Register DestReg = DestSrc->Destination->getReg();
11439 Register SrcReg = DestSrc->Source->getReg();
11440
11441 if (!DestReg.isValid() || !SrcReg.isValid())
11442 return std::nullopt;
11443
11444 auto Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: {});
11445
11446 // If the described register is the destination, just return the source.
11447 if (DestReg == DescribedReg)
11448 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11449
11450 // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
11451 if (MI.getOpcode() == AArch64::ORRWrs &&
11452 TRI->isSuperRegister(RegA: DestReg, RegB: DescribedReg))
11453 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11454
11455 // We may need to describe the lower part of a ORRXrs move.
11456 if (MI.getOpcode() == AArch64::ORRXrs &&
11457 TRI->isSubRegister(RegA: DestReg, RegB: DescribedReg)) {
11458 Register SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: AArch64::sub_32);
11459 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcSubReg, isDef: false), Expr);
11460 }
11461
11462 assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
11463 "Unhandled ORR[XW]rs copy case");
11464
11465 return std::nullopt;
11466}
11467
11468bool AArch64InstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
11469 // Functions cannot be split to different sections on AArch64 if they have
11470 // a red zone. This is because relaxing a cross-section branch may require
11471 // incrementing the stack pointer to spill a register, which would overwrite
11472 // the red zone.
11473 if (MF.getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(u: true))
11474 return false;
11475
11476 return TargetInstrInfo::isFunctionSafeToSplit(MF);
11477}
11478
11479bool AArch64InstrInfo::isMBBSafeToSplitToCold(
11480 const MachineBasicBlock &MBB) const {
11481 // Asm Goto blocks can contain conditional branches to goto labels, which can
11482 // get moved out of range of the branch instruction.
11483 auto isAsmGoto = [](const MachineInstr &MI) {
11484 return MI.getOpcode() == AArch64::INLINEASM_BR;
11485 };
11486 if (llvm::any_of(Range: MBB, P: isAsmGoto) || MBB.isInlineAsmBrIndirectTarget())
11487 return false;
11488
11489 // Because jump tables are label-relative instead of table-relative, they all
11490 // must be in the same section or relocation fixup handling will fail.
11491
11492 // Check if MBB is a jump table target
11493 const MachineJumpTableInfo *MJTI = MBB.getParent()->getJumpTableInfo();
11494 auto containsMBB = [&MBB](const MachineJumpTableEntry &JTE) {
11495 return llvm::is_contained(Range: JTE.MBBs, Element: &MBB);
11496 };
11497 if (MJTI != nullptr && llvm::any_of(Range: MJTI->getJumpTables(), P: containsMBB))
11498 return false;
11499
11500 // Check if MBB contains a jump table lookup
11501 for (const MachineInstr &MI : MBB) {
11502 switch (MI.getOpcode()) {
11503 case TargetOpcode::G_BRJT:
11504 case AArch64::JumpTableDest32:
11505 case AArch64::JumpTableDest16:
11506 case AArch64::JumpTableDest8:
11507 return false;
11508 default:
11509 continue;
11510 }
11511 }
11512
11513 // MBB isn't a special case, so it's safe to be split to the cold section.
11514 return true;
11515}
11516
11517std::optional<ParamLoadedValue>
11518AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
11519 Register Reg) const {
11520 const MachineFunction *MF = MI.getMF();
11521 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
11522 switch (MI.getOpcode()) {
11523 case AArch64::MOVZWi:
11524 case AArch64::MOVZXi: {
11525 // MOVZWi may be used for producing zero-extended 32-bit immediates in
11526 // 64-bit parameters, so we need to consider super-registers.
11527 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
11528 return std::nullopt;
11529
11530 if (!MI.getOperand(i: 1).isImm())
11531 return std::nullopt;
11532 int64_t Immediate = MI.getOperand(i: 1).getImm();
11533 int Shift = MI.getOperand(i: 2).getImm();
11534 return ParamLoadedValue(MachineOperand::CreateImm(Val: Immediate << Shift),
11535 nullptr);
11536 }
11537 case AArch64::ORRWrs:
11538 case AArch64::ORRXrs:
11539 return describeORRLoadedValue(MI, DescribedReg: Reg, TII: this, TRI);
11540 }
11541
11542 return TargetInstrInfo::describeLoadedValue(MI, Reg);
11543}
11544
11545bool AArch64InstrInfo::isExtendLikelyToBeFolded(
11546 MachineInstr &ExtMI, MachineRegisterInfo &MRI) const {
11547 assert(ExtMI.getOpcode() == TargetOpcode::G_SEXT ||
11548 ExtMI.getOpcode() == TargetOpcode::G_ZEXT ||
11549 ExtMI.getOpcode() == TargetOpcode::G_ANYEXT);
11550
11551 // Anyexts are nops.
11552 if (ExtMI.getOpcode() == TargetOpcode::G_ANYEXT)
11553 return true;
11554
11555 Register DefReg = ExtMI.getOperand(i: 0).getReg();
11556 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
11557 return false;
11558
11559 // It's likely that a sext/zext as a G_PTR_ADD offset will be folded into an
11560 // addressing mode.
11561 auto *UserMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
11562 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
11563}
11564
11565uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
11566 return get(Opcode: Opc).TSFlags & AArch64::ElementSizeMask;
11567}
11568
11569bool AArch64InstrInfo::isPTestLikeOpcode(unsigned Opc) const {
11570 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsPTestLike;
11571}
11572
11573bool AArch64InstrInfo::isWhileOpcode(unsigned Opc) const {
11574 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsWhile;
11575}
11576
11577unsigned int
11578AArch64InstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
11579 return OptLevel >= CodeGenOptLevel::Aggressive ? 6 : 2;
11580}
11581
11582bool AArch64InstrInfo::isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
11583 unsigned Scale) const {
11584 if (Offset && Scale)
11585 return false;
11586
11587 // Check Reg + Imm
11588 if (!Scale) {
11589 // 9-bit signed offset
11590 if (isInt<9>(x: Offset))
11591 return true;
11592
11593 // 12-bit unsigned offset
11594 unsigned Shift = Log2_64(Value: NumBytes);
11595 if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
11596 // Must be a multiple of NumBytes (NumBytes is a power of 2)
11597 (Offset >> Shift) << Shift == Offset)
11598 return true;
11599 return false;
11600 }
11601
11602 // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
11603 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
11604}
11605
11606unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
11607 if (MF.getSubtarget<AArch64Subtarget>().hardenSlsBlr())
11608 return AArch64::BLRNoIP;
11609 else
11610 return AArch64::BLR;
11611}
11612
11613void AArch64InstrInfo::createPauthEpilogueInstr(MachineBasicBlock &MBB,
11614 DebugLoc DL) const {
11615 MachineBasicBlock::iterator InsertPt = MBB.getFirstTerminator();
11616 auto Builder = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: get(Opcode: AArch64::PAUTH_EPILOGUE))
11617 .setMIFlag(MachineInstr::FrameDestroy);
11618
11619 MachineFunction &MF = *MBB.getParent();
11620 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
11621 auto &AFL = *static_cast<const AArch64FrameLowering *>(
11622 MF.getSubtarget().getFrameLowering());
11623 if (AFL.getArgumentStackToRestore(MF, MBB)) {
11624 Builder.addReg(RegNo: AArch64::X17, Flags: RegState::ImplicitDefine);
11625 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11626 if (Subtarget.hasPAuthLR())
11627 Builder.addReg(RegNo: AArch64::X15, Flags: RegState::ImplicitDefine);
11628 return;
11629 }
11630
11631 if (AFI->branchProtectionPAuthLR() && !Subtarget.hasPAuthLR())
11632 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11633}
11634
11635MachineBasicBlock::iterator
11636AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI,
11637 Register TargetReg, bool FrameSetup) const {
11638 assert(TargetReg != AArch64::SP && "New top of stack cannot already be in SP");
11639
11640 MachineBasicBlock &MBB = *MBBI->getParent();
11641 MachineFunction &MF = *MBB.getParent();
11642 const AArch64InstrInfo *TII =
11643 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
11644 int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
11645 DebugLoc DL = MBB.findDebugLoc(MBBI);
11646
11647 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
11648 MachineBasicBlock *LoopTestMBB =
11649 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11650 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
11651 MachineBasicBlock *LoopBodyMBB =
11652 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11653 MF.insert(MBBI: MBBInsertPoint, MBB: LoopBodyMBB);
11654 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11655 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
11656 MachineInstr::MIFlag Flags =
11657 FrameSetup ? MachineInstr::FrameSetup : MachineInstr::NoFlags;
11658
11659 // LoopTest:
11660 // SUB SP, SP, #ProbeSize
11661 emitFrameOffset(MBB&: *LoopTestMBB, MBBI: LoopTestMBB->end(), DL, DestReg: AArch64::SP,
11662 SrcReg: AArch64::SP, Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII, Flag: Flags);
11663
11664 // CMP SP, TargetReg
11665 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrx64),
11666 DestReg: AArch64::XZR)
11667 .addReg(RegNo: AArch64::SP)
11668 .addReg(RegNo: TargetReg)
11669 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0))
11670 .setMIFlags(Flags);
11671
11672 // B.<Cond> LoopExit
11673 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
11674 .addImm(Val: AArch64CC::LE)
11675 .addMBB(MBB: ExitMBB)
11676 .setMIFlags(Flags);
11677
11678 // LDR XZR, [SP]
11679 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11680 .addDef(RegNo: AArch64::XZR)
11681 .addReg(RegNo: AArch64::SP)
11682 .addImm(Val: 0)
11683 .addMemOperand(MMO: MF.getMachineMemOperand(
11684 PtrInfo: MachinePointerInfo::getUnknownStack(MF),
11685 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 8,
11686 BaseAlignment: Align(8)))
11687 .setMIFlags(Flags);
11688
11689 // B loop
11690 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
11691 .addMBB(MBB: LoopTestMBB)
11692 .setMIFlags(Flags);
11693
11694 // LoopExit:
11695 // MOV SP, TargetReg
11696 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: AArch64::SP)
11697 .addReg(RegNo: TargetReg)
11698 .addImm(Val: 0)
11699 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
11700 .setMIFlags(Flags);
11701
11702 // LDR XZR, [SP]
11703 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11704 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define)
11705 .addReg(RegNo: AArch64::SP)
11706 .addImm(Val: 0)
11707 .setMIFlags(Flags);
11708
11709 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
11710 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
11711
11712 LoopTestMBB->addSuccessor(Succ: ExitMBB);
11713 LoopTestMBB->addSuccessor(Succ: LoopBodyMBB);
11714 LoopBodyMBB->addSuccessor(Succ: LoopTestMBB);
11715 MBB.addSuccessor(Succ: LoopTestMBB);
11716
11717 // Update liveins.
11718 if (MF.getRegInfo().reservedRegsFrozen())
11719 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopBodyMBB, LoopTestMBB});
11720
11721 return ExitMBB->begin();
11722}
11723
11724namespace {
11725class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
11726 MachineFunction *MF;
11727 const TargetInstrInfo *TII;
11728 const TargetRegisterInfo *TRI;
11729 MachineRegisterInfo &MRI;
11730
11731 /// The block of the loop
11732 MachineBasicBlock *LoopBB;
11733 /// The conditional branch of the loop
11734 MachineInstr *CondBranch;
11735 /// The compare instruction for loop control
11736 MachineInstr *Comp;
11737 /// The number of the operand of the loop counter value in Comp
11738 unsigned CompCounterOprNum;
11739 /// The instruction that updates the loop counter value
11740 MachineInstr *Update;
11741 /// The number of the operand of the loop counter value in Update
11742 unsigned UpdateCounterOprNum;
11743 /// The initial value of the loop counter
11744 Register Init;
11745 /// True iff Update is a predecessor of Comp
11746 bool IsUpdatePriorComp;
11747
11748 /// The normalized condition used by createTripCountGreaterCondition()
11749 SmallVector<MachineOperand, 4> Cond;
11750
11751public:
11752 AArch64PipelinerLoopInfo(MachineBasicBlock *LoopBB, MachineInstr *CondBranch,
11753 MachineInstr *Comp, unsigned CompCounterOprNum,
11754 MachineInstr *Update, unsigned UpdateCounterOprNum,
11755 Register Init, bool IsUpdatePriorComp,
11756 const SmallVectorImpl<MachineOperand> &Cond)
11757 : MF(Comp->getParent()->getParent()),
11758 TII(MF->getSubtarget().getInstrInfo()),
11759 TRI(MF->getSubtarget().getRegisterInfo()), MRI(MF->getRegInfo()),
11760 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
11761 CompCounterOprNum(CompCounterOprNum), Update(Update),
11762 UpdateCounterOprNum(UpdateCounterOprNum), Init(Init),
11763 IsUpdatePriorComp(IsUpdatePriorComp), Cond(Cond.begin(), Cond.end()) {}
11764
11765 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
11766 // Make the instructions for loop control be placed in stage 0.
11767 // The predecessors of Comp are considered by the caller.
11768 return MI == Comp;
11769 }
11770
11771 std::optional<bool> createTripCountGreaterCondition(
11772 int TC, MachineBasicBlock &MBB,
11773 SmallVectorImpl<MachineOperand> &CondParam) override {
11774 // A branch instruction will be inserted as "if (Cond) goto epilogue".
11775 // Cond is normalized for such use.
11776 // The predecessors of the branch are assumed to have already been inserted.
11777 CondParam = Cond;
11778 return {};
11779 }
11780
11781 void createRemainingIterationsGreaterCondition(
11782 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
11783 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) override;
11784
11785 void setPreheader(MachineBasicBlock *NewPreheader) override {}
11786
11787 void adjustTripCount(int TripCountAdjust) override {}
11788
11789 bool isMVEExpanderSupported() override { return true; }
11790};
11791} // namespace
11792
11793/// Clone an instruction from MI. The register of ReplaceOprNum-th operand
11794/// is replaced by ReplaceReg. The output register is newly created.
11795/// The other operands are unchanged from MI.
11796static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum,
11797 Register ReplaceReg, MachineBasicBlock &MBB,
11798 MachineBasicBlock::iterator InsertTo) {
11799 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
11800 const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
11801 MachineInstr *NewMI = MBB.getParent()->CloneMachineInstr(Orig: MI);
11802 Register Result = 0;
11803 for (unsigned I = 0; I < NewMI->getNumOperands(); ++I) {
11804 if (I == 0 && NewMI->getOperand(i: 0).getReg().isVirtual()) {
11805 Result = MRI.createVirtualRegister(
11806 RegClass: MRI.getRegClass(Reg: NewMI->getOperand(i: 0).getReg()));
11807 NewMI->getOperand(i: I).setReg(Result);
11808 } else if (I == ReplaceOprNum) {
11809 MRI.constrainRegClass(Reg: ReplaceReg, RC: TII->getRegClass(MCID: NewMI->getDesc(), OpNum: I));
11810 NewMI->getOperand(i: I).setReg(ReplaceReg);
11811 }
11812 }
11813 MBB.insert(I: InsertTo, MI: NewMI);
11814 return Result;
11815}
11816
11817void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
11818 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
11819 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) {
11820 // Create and accumulate conditions for next TC iterations.
11821 // Example:
11822 // SUBSXrr N, counter, implicit-def $nzcv # compare instruction for the last
11823 // # iteration of the kernel
11824 //
11825 // # insert the following instructions
11826 // cond = CSINCXr 0, 0, C, implicit $nzcv
11827 // counter = ADDXri counter, 1 # clone from this->Update
11828 // SUBSXrr n, counter, implicit-def $nzcv # clone from this->Comp
11829 // cond = CSINCXr cond, cond, C, implicit $nzcv
11830 // ... (repeat TC times)
11831 // SUBSXri cond, 0, implicit-def $nzcv
11832
11833 assert(CondBranch->getOpcode() == AArch64::Bcc);
11834 // CondCode to exit the loop
11835 AArch64CC::CondCode CC =
11836 (AArch64CC::CondCode)CondBranch->getOperand(i: 0).getImm();
11837 if (CondBranch->getOperand(i: 1).getMBB() == LoopBB)
11838 CC = AArch64CC::getInvertedCondCode(Code: CC);
11839
11840 // Accumulate conditions to exit the loop
11841 Register AccCond = AArch64::XZR;
11842
11843 // If CC holds, CurCond+1 is returned; otherwise CurCond is returned.
11844 auto AccumulateCond = [&](Register CurCond,
11845 AArch64CC::CondCode CC) -> Register {
11846 Register NewCond = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
11847 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::CSINCXr))
11848 .addReg(RegNo: NewCond, Flags: RegState::Define)
11849 .addReg(RegNo: CurCond)
11850 .addReg(RegNo: CurCond)
11851 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
11852 return NewCond;
11853 };
11854
11855 if (!LastStage0Insts.empty() && LastStage0Insts[Comp]->getParent() == &MBB) {
11856 // Update and Comp for I==0 are already exists in MBB
11857 // (MBB is an unrolled kernel)
11858 Register Counter;
11859 for (int I = 0; I <= TC; ++I) {
11860 Register NextCounter;
11861 if (I != 0)
11862 NextCounter =
11863 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
11864
11865 AccCond = AccumulateCond(AccCond, CC);
11866
11867 if (I != TC) {
11868 if (I == 0) {
11869 if (Update != Comp && IsUpdatePriorComp) {
11870 Counter =
11871 LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
11872 NextCounter = cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB,
11873 InsertTo: MBB.end());
11874 } else {
11875 // can use already calculated value
11876 NextCounter = LastStage0Insts[Update]->getOperand(i: 0).getReg();
11877 }
11878 } else if (Update != Comp) {
11879 NextCounter =
11880 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
11881 }
11882 }
11883 Counter = NextCounter;
11884 }
11885 } else {
11886 Register Counter;
11887 if (LastStage0Insts.empty()) {
11888 // use initial counter value (testing if the trip count is sufficient to
11889 // be executed by pipelined code)
11890 Counter = Init;
11891 if (IsUpdatePriorComp)
11892 Counter =
11893 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
11894 } else {
11895 // MBB is an epilogue block. LastStage0Insts[Comp] is in the kernel block.
11896 Counter = LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
11897 }
11898
11899 for (int I = 0; I <= TC; ++I) {
11900 Register NextCounter;
11901 NextCounter =
11902 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
11903 AccCond = AccumulateCond(AccCond, CC);
11904 if (I != TC && Update != Comp)
11905 NextCounter =
11906 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
11907 Counter = NextCounter;
11908 }
11909 }
11910
11911 // If AccCond == 0, the remainder is greater than TC.
11912 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::SUBSXri))
11913 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define | RegState::Dead)
11914 .addReg(RegNo: AccCond)
11915 .addImm(Val: 0)
11916 .addImm(Val: 0);
11917 Cond.clear();
11918 Cond.push_back(Elt: MachineOperand::CreateImm(Val: AArch64CC::EQ));
11919}
11920
11921static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB,
11922 Register &RegMBB, Register &RegOther) {
11923 assert(Phi.getNumOperands() == 5);
11924 if (Phi.getOperand(i: 2).getMBB() == MBB) {
11925 RegMBB = Phi.getOperand(i: 1).getReg();
11926 RegOther = Phi.getOperand(i: 3).getReg();
11927 } else {
11928 assert(Phi.getOperand(4).getMBB() == MBB);
11929 RegMBB = Phi.getOperand(i: 3).getReg();
11930 RegOther = Phi.getOperand(i: 1).getReg();
11931 }
11932}
11933
11934static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB) {
11935 if (!Reg.isVirtual())
11936 return false;
11937 const MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
11938 return MRI.getVRegDef(Reg)->getParent() != BB;
11939}
11940
11941/// If Reg is an induction variable, return true and set some parameters
11942static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
11943 MachineInstr *&UpdateInst,
11944 unsigned &UpdateCounterOprNum, Register &InitReg,
11945 bool &IsUpdatePriorComp) {
11946 // Example:
11947 //
11948 // Preheader:
11949 // InitReg = ...
11950 // LoopBB:
11951 // Reg0 = PHI (InitReg, Preheader), (Reg1, LoopBB)
11952 // Reg = COPY Reg0 ; COPY is ignored.
11953 // Reg1 = ADD Reg, #1; UpdateInst. Incremented by a loop invariant value.
11954 // ; Reg is the value calculated in the previous
11955 // ; iteration, so IsUpdatePriorComp == false.
11956
11957 if (LoopBB->pred_size() != 2)
11958 return false;
11959 if (!Reg.isVirtual())
11960 return false;
11961 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
11962 UpdateInst = nullptr;
11963 UpdateCounterOprNum = 0;
11964 InitReg = 0;
11965 IsUpdatePriorComp = true;
11966 Register CurReg = Reg;
11967 while (true) {
11968 MachineInstr *Def = MRI.getVRegDef(Reg: CurReg);
11969 if (Def->getParent() != LoopBB)
11970 return false;
11971 if (Def->isCopy()) {
11972 // Ignore copy instructions unless they contain subregisters
11973 if (Def->getOperand(i: 0).getSubReg() || Def->getOperand(i: 1).getSubReg())
11974 return false;
11975 CurReg = Def->getOperand(i: 1).getReg();
11976 } else if (Def->isPHI()) {
11977 if (InitReg != 0)
11978 return false;
11979 if (!UpdateInst)
11980 IsUpdatePriorComp = false;
11981 extractPhiReg(Phi: *Def, MBB: LoopBB, RegMBB&: CurReg, RegOther&: InitReg);
11982 } else {
11983 if (UpdateInst)
11984 return false;
11985 switch (Def->getOpcode()) {
11986 case AArch64::ADDSXri:
11987 case AArch64::ADDSWri:
11988 case AArch64::SUBSXri:
11989 case AArch64::SUBSWri:
11990 case AArch64::ADDXri:
11991 case AArch64::ADDWri:
11992 case AArch64::SUBXri:
11993 case AArch64::SUBWri:
11994 UpdateInst = Def;
11995 UpdateCounterOprNum = 1;
11996 break;
11997 case AArch64::ADDSXrr:
11998 case AArch64::ADDSWrr:
11999 case AArch64::SUBSXrr:
12000 case AArch64::SUBSWrr:
12001 case AArch64::ADDXrr:
12002 case AArch64::ADDWrr:
12003 case AArch64::SUBXrr:
12004 case AArch64::SUBWrr:
12005 UpdateInst = Def;
12006 if (isDefinedOutside(Reg: Def->getOperand(i: 2).getReg(), BB: LoopBB))
12007 UpdateCounterOprNum = 1;
12008 else if (isDefinedOutside(Reg: Def->getOperand(i: 1).getReg(), BB: LoopBB))
12009 UpdateCounterOprNum = 2;
12010 else
12011 return false;
12012 break;
12013 default:
12014 return false;
12015 }
12016 CurReg = Def->getOperand(i: UpdateCounterOprNum).getReg();
12017 }
12018
12019 if (!CurReg.isVirtual())
12020 return false;
12021 if (Reg == CurReg)
12022 break;
12023 }
12024
12025 if (!UpdateInst)
12026 return false;
12027
12028 return true;
12029}
12030
12031std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
12032AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
12033 // Accept loops that meet the following conditions
12034 // * The conditional branch is BCC
12035 // * The compare instruction is ADDS/SUBS/WHILEXX
12036 // * One operand of the compare is an induction variable and the other is a
12037 // loop invariant value
12038 // * The induction variable is incremented/decremented by a single instruction
12039 // * Does not contain CALL or instructions which have unmodeled side effects
12040
12041 for (MachineInstr &MI : *LoopBB)
12042 if (MI.isCall() || MI.hasUnmodeledSideEffects())
12043 // This instruction may use NZCV, which interferes with the instruction to
12044 // be inserted for loop control.
12045 return nullptr;
12046
12047 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
12048 SmallVector<MachineOperand, 4> Cond;
12049 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond))
12050 return nullptr;
12051
12052 // Infinite loops are not supported
12053 if (TBB == LoopBB && FBB == LoopBB)
12054 return nullptr;
12055
12056 // Must be conditional branch
12057 if (TBB != LoopBB && FBB == nullptr)
12058 return nullptr;
12059
12060 assert((TBB == LoopBB || FBB == LoopBB) &&
12061 "The Loop must be a single-basic-block loop");
12062
12063 MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
12064 const TargetRegisterInfo &TRI = getRegisterInfo();
12065
12066 if (CondBranch->getOpcode() != AArch64::Bcc)
12067 return nullptr;
12068
12069 // Normalization for createTripCountGreaterCondition()
12070 if (TBB == LoopBB)
12071 reverseBranchCondition(Cond);
12072
12073 MachineInstr *Comp = nullptr;
12074 unsigned CompCounterOprNum = 0;
12075 for (MachineInstr &MI : reverse(C&: *LoopBB)) {
12076 if (MI.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
12077 // Guarantee that the compare is SUBS/ADDS/WHILEXX and that one of the
12078 // operands is a loop invariant value
12079
12080 switch (MI.getOpcode()) {
12081 case AArch64::SUBSXri:
12082 case AArch64::SUBSWri:
12083 case AArch64::ADDSXri:
12084 case AArch64::ADDSWri:
12085 Comp = &MI;
12086 CompCounterOprNum = 1;
12087 break;
12088 case AArch64::ADDSWrr:
12089 case AArch64::ADDSXrr:
12090 case AArch64::SUBSWrr:
12091 case AArch64::SUBSXrr:
12092 Comp = &MI;
12093 break;
12094 default:
12095 if (isWhileOpcode(Opc: MI.getOpcode())) {
12096 Comp = &MI;
12097 break;
12098 }
12099 return nullptr;
12100 }
12101
12102 if (CompCounterOprNum == 0) {
12103 if (isDefinedOutside(Reg: Comp->getOperand(i: 1).getReg(), BB: LoopBB))
12104 CompCounterOprNum = 2;
12105 else if (isDefinedOutside(Reg: Comp->getOperand(i: 2).getReg(), BB: LoopBB))
12106 CompCounterOprNum = 1;
12107 else
12108 return nullptr;
12109 }
12110 break;
12111 }
12112 }
12113 if (!Comp)
12114 return nullptr;
12115
12116 MachineInstr *Update = nullptr;
12117 Register Init;
12118 bool IsUpdatePriorComp;
12119 unsigned UpdateCounterOprNum;
12120 if (!getIndVarInfo(Reg: Comp->getOperand(i: CompCounterOprNum).getReg(), LoopBB,
12121 UpdateInst&: Update, UpdateCounterOprNum, InitReg&: Init, IsUpdatePriorComp))
12122 return nullptr;
12123
12124 return std::make_unique<AArch64PipelinerLoopInfo>(
12125 args&: LoopBB, args&: CondBranch, args&: Comp, args&: CompCounterOprNum, args&: Update, args&: UpdateCounterOprNum,
12126 args&: Init, args&: IsUpdatePriorComp, args&: Cond);
12127}
12128
12129/// verifyInstruction - Perform target specific instruction verification.
12130bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI,
12131 StringRef &ErrInfo) const {
12132 // Verify that immediate offsets on load/store instructions are within range.
12133 // Stack objects with an FI operand are excluded as they can be fixed up
12134 // during PEI.
12135 TypeSize Scale(0U, false), Width(0U, false);
12136 int64_t MinOffset, MaxOffset;
12137 if (getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
12138 unsigned ImmIdx = getLoadStoreImmIdx(Opc: MI.getOpcode());
12139 if (MI.getOperand(i: ImmIdx).isImm() && !MI.getOperand(i: ImmIdx - 1).isFI()) {
12140 int64_t Imm = MI.getOperand(i: ImmIdx).getImm();
12141 if (Imm < MinOffset || Imm > MaxOffset) {
12142 ErrInfo = "Unexpected immediate on load/store instruction";
12143 return false;
12144 }
12145 }
12146 }
12147
12148 const MCInstrDesc &MCID = MI.getDesc();
12149 for (unsigned Op = 0; Op < MCID.getNumOperands(); Op++) {
12150 const MachineOperand &MO = MI.getOperand(i: Op);
12151 switch (MCID.operands()[Op].OperandType) {
12152 case AArch64::OPERAND_IMPLICIT_IMM_0:
12153 if (!MO.isImm() || MO.getImm() != 0) {
12154 ErrInfo = "OPERAND_IMPLICIT_IMM_0 should be 0";
12155 return false;
12156 }
12157 break;
12158 case AArch64::OPERAND_SHIFT_MSL:
12159 if (!MO.isImm() ||
12160 AArch64_AM::getShiftType(Imm: MO.getImm()) != AArch64_AM::MSL ||
12161 (AArch64_AM::getShiftValue(Imm: MO.getImm()) != 8 &&
12162 AArch64_AM::getShiftValue(Imm: MO.getImm()) != 16)) {
12163 ErrInfo = "OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
12164 return false;
12165 }
12166 break;
12167 case AArch64::OPERAND_IMM_UINT5:
12168 if (!MO.isImm() || !isUInt<5>(x: MO.getImm())) {
12169 ErrInfo = "OPERAND_IMM_UINT5 should be in the range 0 to 31";
12170 return false;
12171 }
12172 break;
12173 case AArch64::OPERAND_IMM_UINT8:
12174 if (!MO.isImm() || !isUInt<8>(x: MO.getImm())) {
12175 ErrInfo = "OPERAND_IMM_UINT8 should be in the range 0 to 255";
12176 return false;
12177 }
12178 break;
12179 default:
12180 break;
12181 }
12182 }
12183 return true;
12184}
12185
12186#define GET_INSTRINFO_HELPERS
12187#define GET_INSTRMAP_INFO
12188#include "AArch64GenInstrInfo.inc"
12189