1//===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64PointerAuth.h"
17#include "AArch64Subtarget.h"
18#include "MCTargetDesc/AArch64AddressingModes.h"
19#include "MCTargetDesc/AArch64MCLFIRewriter.h"
20#include "MCTargetDesc/AArch64MCTargetDesc.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/Analysis/AliasAnalysis.h"
28#include "llvm/CodeGen/CFIInstBuilder.h"
29#include "llvm/CodeGen/LivePhysRegs.h"
30#include "llvm/CodeGen/LiveRegUnits.h"
31#include "llvm/CodeGen/MachineBasicBlock.h"
32#include "llvm/CodeGen/MachineCombinerPattern.h"
33#include "llvm/CodeGen/MachineFrameInfo.h"
34#include "llvm/CodeGen/MachineFunction.h"
35#include "llvm/CodeGen/MachineInstr.h"
36#include "llvm/CodeGen/MachineInstrBuilder.h"
37#include "llvm/CodeGen/MachineMemOperand.h"
38#include "llvm/CodeGen/MachineModuleInfo.h"
39#include "llvm/CodeGen/MachineOperand.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/RegisterScavenging.h"
42#include "llvm/CodeGen/StackMaps.h"
43#include "llvm/CodeGen/TargetRegisterInfo.h"
44#include "llvm/CodeGen/TargetSubtargetInfo.h"
45#include "llvm/IR/DebugInfoMetadata.h"
46#include "llvm/IR/DebugLoc.h"
47#include "llvm/IR/GlobalValue.h"
48#include "llvm/IR/Module.h"
49#include "llvm/MC/MCAsmInfo.h"
50#include "llvm/MC/MCInst.h"
51#include "llvm/MC/MCInstBuilder.h"
52#include "llvm/MC/MCInstrDesc.h"
53#include "llvm/Support/Casting.h"
54#include "llvm/Support/CodeGen.h"
55#include "llvm/Support/CommandLine.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Support/LEB128.h"
58#include "llvm/Support/MathExtras.h"
59#include "llvm/Target/TargetMachine.h"
60#include "llvm/Target/TargetOptions.h"
61#include <cassert>
62#include <cstdint>
63#include <iterator>
64#include <utility>
65
66using namespace llvm;
67
68#define GET_INSTRINFO_CTOR_DTOR
69#include "AArch64GenInstrInfo.inc"
70
71#define DEBUG_TYPE "AArch64InstrInfo"
72
73STATISTIC(NumCopyInstrs, "Number of COPY instructions expanded");
74STATISTIC(NumZCRegMoveInstrsGPR, "Number of zero-cycle GPR register move "
75 "instructions expanded from canonical COPY");
76STATISTIC(NumZCRegMoveInstrsFPR, "Number of zero-cycle FPR register move "
77 "instructions expanded from canonical COPY");
78STATISTIC(NumZCZeroingInstrsGPR, "Number of zero-cycle GPR zeroing "
79 "instructions expanded from canonical COPY");
80// NumZCZeroingInstrsFPR is counted at AArch64AsmPrinter
81
82static cl::opt<unsigned>
83 CBDisplacementBits("aarch64-cb-offset-bits", cl::Hidden, cl::init(Val: 9),
84 cl::desc("Restrict range of CB instructions (DEBUG)"));
85
86static cl::opt<unsigned> TBZDisplacementBits(
87 "aarch64-tbz-offset-bits", cl::Hidden, cl::init(Val: 14),
88 cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
89
90static cl::opt<unsigned> CBZDisplacementBits(
91 "aarch64-cbz-offset-bits", cl::Hidden, cl::init(Val: 19),
92 cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
93
94static cl::opt<unsigned>
95 BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(Val: 19),
96 cl::desc("Restrict range of Bcc instructions (DEBUG)"));
97
98static cl::opt<unsigned>
99 BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(Val: 26),
100 cl::desc("Restrict range of B instructions (DEBUG)"));
101
102static cl::opt<unsigned> GatherOptSearchLimit(
103 "aarch64-search-limit", cl::Hidden, cl::init(Val: 2048),
104 cl::desc("Restrict range of instructions to search for the "
105 "machine-combiner gather pattern optimization"));
106
107static cl::opt<bool> UseCompactUnwindFrameRecordForOutlinedFunctions(
108 "aarch64-outliner-compact-unwind-frame", cl::Hidden, cl::init(Val: true),
109 cl::desc("Use a frame record for Mach-O non-leaf outlined functions"));
110
111AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
112 : AArch64GenInstrInfo(STI, RI, AArch64::ADJCALLSTACKDOWN,
113 AArch64::ADJCALLSTACKUP, AArch64::CATCHRET),
114 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
115
116/// Return the maximum number of bytes of code the specified instruction may be
117/// after LFI rewriting. If the instruction is not rewritten, std::nullopt is
118/// returned (use default sizing).
119///
120/// NOTE: the size estimates here must be kept in sync with the rewrites in
121/// AArch64MCLFIRewriter.cpp. Sizes may be overestimates of the rewritten
122/// instruction sequences.
123static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
124 switch (MI.getOpcode()) {
125 case AArch64::SVC:
126 // SVC expands to 4 instructions.
127 return 16;
128 case AArch64::BR:
129 case AArch64::BLR:
130 // Indirect branches/calls expand to 2 instructions (guard + br/blr).
131 return 8;
132 case AArch64::RET:
133 // RET through LR is not rewritten, but RET through another register
134 // expands to 2 instructions (guard + ret).
135 if (MI.getOperand(i: 0).getReg() != AArch64::LR)
136 return 8;
137 return 4;
138 case AArch64::RETAA:
139 case AArch64::RETAB:
140 // Authenticated returns expand to 3 instructions (authenticate + guard +
141 // ret).
142 return 12;
143 case AArch64::BRAA:
144 case AArch64::BRAAZ:
145 case AArch64::BRAB:
146 case AArch64::BRABZ:
147 case AArch64::BLRAA:
148 case AArch64::BLRAAZ:
149 case AArch64::BLRAB:
150 case AArch64::BLRABZ:
151 // Authenticated branches/calls expand to 3 instructions (authenticate +
152 // guard + branch).
153 return 12;
154 case AArch64::AUTIASP:
155 case AArch64::AUTIBSP:
156 case AArch64::AUTIAZ:
157 case AArch64::AUTIBZ:
158 case AArch64::XPACLRI:
159 // Authenticating LR expands to the instruction plus a deferred LR guard.
160 return 8;
161 case AArch64::SYSxt:
162 // VA-based DC/IC ops (op1=3, Cn=7, op2=1) expand to 2 instructions.
163 if (MI.getOperand(i: 0).getImm() == 3 && MI.getOperand(i: 1).getImm() == 7 &&
164 MI.getOperand(i: 3).getImm() == 1)
165 return 8;
166 return std::nullopt;
167 default:
168 break;
169 }
170
171 // Detect instructions that explicitly define SP or LR.
172 bool ModifiesLR = false;
173 bool ModifiesSP = false;
174 for (const MachineOperand &MO : MI.defs()) {
175 if (!MO.isReg())
176 continue;
177 if (MO.getReg() == AArch64::LR)
178 ModifiesLR = true;
179 else if (MO.getReg() == AArch64::SP)
180 ModifiesSP = true;
181 }
182
183 // Memory accesses expand to a base-register guard plus the rewritten access
184 // (8 bytes), with an extra base-register update for pre/post-index forms (12
185 // bytes total). If the access also defines LR, an LR mask is appended (+4
186 // bytes). Depending on additional optimizations that the rewriter performs,
187 // this may be an overestimate.
188 if (MI.mayLoadOrStore()) {
189 unsigned Size = isLFIPrePostMemAccess(Opcode: MI.getOpcode()) ? 12 : 8;
190 if (ModifiesLR)
191 Size += 4;
192 return Size;
193 }
194
195 // Non memory operations that modify LR or SP expand to 2 instructions.
196 if (ModifiesSP || ModifiesLR)
197 return 8;
198
199 // Default case: instructions that don't cause expansion.
200 // - TP accesses in LFI are a single load/store, so no expansion.
201 // - All remaining instructions are not rewritten.
202 return std::nullopt;
203}
204
205/// GetInstSize - Return the number of bytes of code the specified
206/// instruction may be. This returns the maximum number of bytes.
207unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
208 const MachineBasicBlock &MBB = *MI.getParent();
209 const MachineFunction *MF = MBB.getParent();
210 const Function &F = MF->getFunction();
211 const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
212
213 {
214 auto Op = MI.getOpcode();
215 if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
216 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(), MAI);
217 }
218
219 // Meta-instructions emit no code.
220 if (MI.isMetaInstruction())
221 return 0;
222
223 // FIXME: We currently only handle pseudoinstructions that don't get expanded
224 // before the assembly printer.
225 unsigned NumBytes = 0;
226 const MCInstrDesc &Desc = MI.getDesc();
227
228 // LFI rewriter expansions that supersede normal sizing.
229 const auto &STI = MF->getSubtarget<AArch64Subtarget>();
230 if (STI.isLFI())
231 if (auto Size = getLFIInstSizeInBytes(MI))
232 return *Size;
233
234 if (!MI.isBundle() && isTailCallReturnInst(MI)) {
235 NumBytes = Desc.getSize() ? Desc.getSize() : 4;
236
237 const auto *MFI = MF->getInfo<AArch64FunctionInfo>();
238 if (!MFI->shouldSignReturnAddress(MF: *MF))
239 return NumBytes;
240
241 auto Method = STI.getAuthenticatedLRCheckMethod(MF: *MF);
242 NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
243 return NumBytes;
244 }
245
246 // Size should be preferably set in
247 // llvm/lib/Target/AArch64/AArch64InstrInfo.td (default case).
248 // Specific cases handle instructions of variable sizes
249 switch (Desc.getOpcode()) {
250 default:
251 if (Desc.getSize())
252 return Desc.getSize();
253
254 // Anything not explicitly designated otherwise (i.e. pseudo-instructions
255 // with fixed constant size but not specified in .td file) is a normal
256 // 4-byte insn.
257 NumBytes = 4;
258 break;
259 case TargetOpcode::STACKMAP:
260 // The upper bound for a stackmap intrinsic is the full length of its shadow
261 NumBytes = StackMapOpers(&MI).getNumPatchBytes();
262 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
263 break;
264 case TargetOpcode::PATCHPOINT:
265 // The size of the patchpoint intrinsic is the number of bytes requested
266 NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
267 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
268 break;
269 case TargetOpcode::STATEPOINT:
270 NumBytes = StatepointOpers(&MI).getNumPatchBytes();
271 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
272 // No patch bytes means a normal call inst is emitted
273 if (NumBytes == 0)
274 NumBytes = 4;
275 break;
276 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
277 // If `patchable-function-entry` is set, PATCHABLE_FUNCTION_ENTER
278 // instructions are expanded to the specified number of NOPs. Otherwise,
279 // they are expanded to 36-byte XRay sleds.
280 NumBytes =
281 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry", Default: 9) * 4;
282 break;
283 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
284 case TargetOpcode::PATCHABLE_TAIL_CALL:
285 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
286 // An XRay sled can be 4 bytes of alignment plus a 32-byte block.
287 NumBytes = 36;
288 break;
289 case TargetOpcode::PATCHABLE_EVENT_CALL:
290 // EVENT_CALL XRay sleds are exactly 6 instructions long (no alignment).
291 NumBytes = 24;
292 break;
293
294 case AArch64::SPACE:
295 NumBytes = MI.getOperand(i: 1).getImm();
296 break;
297 case AArch64::MOVaddr:
298 case AArch64::MOVaddrJT:
299 case AArch64::MOVaddrCP:
300 case AArch64::MOVaddrBA:
301 case AArch64::MOVaddrTLS:
302 case AArch64::MOVaddrEXT: {
303 // Use the same logic as the pseudo expansion to count instructions.
304 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
305 AArch64_IMM::expandMOVAddr(Opcode: Desc.getOpcode(),
306 TargetFlags: MI.getOperand(i: 1).getTargetFlags(),
307 IsTargetMachO: Subtarget.isTargetMachO(), Insn);
308 NumBytes = Insn.size() * 4;
309 break;
310 }
311
312 case AArch64::MOVi32imm:
313 case AArch64::MOVi64imm: {
314 // Use the same logic as the pseudo expansion to count instructions.
315 unsigned BitSize = Desc.getOpcode() == AArch64::MOVi32imm ? 32 : 64;
316 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
317 AArch64_IMM::expandMOVImm(Imm: MI.getOperand(i: 1).getImm(), BitSize, Insn);
318 NumBytes = Insn.size() * 4;
319 break;
320 }
321
322 case TargetOpcode::BUNDLE:
323 NumBytes = getInstBundleSize(MI);
324 break;
325 }
326
327 return NumBytes;
328}
329
330static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
331 SmallVectorImpl<MachineOperand> &Cond) {
332 // Block ends with fall-through condbranch.
333 switch (LastInst->getOpcode()) {
334 default:
335 llvm_unreachable("Unknown branch instruction?");
336 case AArch64::Bcc:
337 Target = LastInst->getOperand(i: 1).getMBB();
338 Cond.push_back(Elt: LastInst->getOperand(i: 0));
339 break;
340 case AArch64::CBZW:
341 case AArch64::CBZX:
342 case AArch64::CBNZW:
343 case AArch64::CBNZX:
344 Target = LastInst->getOperand(i: 1).getMBB();
345 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
346 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
347 Cond.push_back(Elt: LastInst->getOperand(i: 0));
348 break;
349 case AArch64::TBZW:
350 case AArch64::TBZX:
351 case AArch64::TBNZW:
352 case AArch64::TBNZX:
353 Target = LastInst->getOperand(i: 2).getMBB();
354 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
355 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
356 Cond.push_back(Elt: LastInst->getOperand(i: 0));
357 Cond.push_back(Elt: LastInst->getOperand(i: 1));
358 break;
359 case AArch64::CBWPri:
360 case AArch64::CBXPri:
361 case AArch64::CBWPrr:
362 case AArch64::CBXPrr:
363 Target = LastInst->getOperand(i: 3).getMBB();
364 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
365 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
366 Cond.push_back(Elt: LastInst->getOperand(i: 0));
367 Cond.push_back(Elt: LastInst->getOperand(i: 1));
368 Cond.push_back(Elt: LastInst->getOperand(i: 2));
369 break;
370 case AArch64::CBBAssertExt:
371 case AArch64::CBHAssertExt:
372 Target = LastInst->getOperand(i: 3).getMBB();
373 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1)); // -1
374 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode())); // Opc
375 Cond.push_back(Elt: LastInst->getOperand(i: 0)); // Cond
376 Cond.push_back(Elt: LastInst->getOperand(i: 1)); // Op0
377 Cond.push_back(Elt: LastInst->getOperand(i: 2)); // Op1
378 Cond.push_back(Elt: LastInst->getOperand(i: 4)); // Ext0
379 Cond.push_back(Elt: LastInst->getOperand(i: 5)); // Ext1
380 break;
381 }
382}
383
384static unsigned getBranchDisplacementBits(unsigned Opc) {
385 switch (Opc) {
386 default:
387 llvm_unreachable("unexpected opcode!");
388 case AArch64::B:
389 return BDisplacementBits;
390 case AArch64::TBNZW:
391 case AArch64::TBZW:
392 case AArch64::TBNZX:
393 case AArch64::TBZX:
394 return TBZDisplacementBits;
395 case AArch64::CBNZW:
396 case AArch64::CBZW:
397 case AArch64::CBNZX:
398 case AArch64::CBZX:
399 return CBZDisplacementBits;
400 case AArch64::Bcc:
401 return BCCDisplacementBits;
402 case AArch64::CBWPri:
403 case AArch64::CBXPri:
404 case AArch64::CBBAssertExt:
405 case AArch64::CBHAssertExt:
406 case AArch64::CBWPrr:
407 case AArch64::CBXPrr:
408 return CBDisplacementBits;
409 }
410}
411
412bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
413 int64_t BrOffset) const {
414 unsigned Bits = getBranchDisplacementBits(Opc: BranchOp);
415 assert(Bits >= 3 && "max branch displacement must be enough to jump"
416 "over conditional branch expansion");
417 return isIntN(N: Bits, x: BrOffset / 4);
418}
419
420MachineBasicBlock *
421AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
422 switch (MI.getOpcode()) {
423 default:
424 llvm_unreachable("unexpected opcode!");
425 case AArch64::B:
426 return MI.getOperand(i: 0).getMBB();
427 case AArch64::TBZW:
428 case AArch64::TBNZW:
429 case AArch64::TBZX:
430 case AArch64::TBNZX:
431 return MI.getOperand(i: 2).getMBB();
432 case AArch64::CBZW:
433 case AArch64::CBNZW:
434 case AArch64::CBZX:
435 case AArch64::CBNZX:
436 case AArch64::Bcc:
437 return MI.getOperand(i: 1).getMBB();
438 case AArch64::CBWPri:
439 case AArch64::CBXPri:
440 case AArch64::CBBAssertExt:
441 case AArch64::CBHAssertExt:
442 case AArch64::CBWPrr:
443 case AArch64::CBXPrr:
444 return MI.getOperand(i: 3).getMBB();
445 }
446}
447
448void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
449 MachineBasicBlock &NewDestBB,
450 MachineBasicBlock &RestoreBB,
451 const DebugLoc &DL,
452 int64_t BrOffset,
453 RegScavenger *RS) const {
454 assert(RS && "RegScavenger required for long branching");
455 assert(MBB.empty() &&
456 "new block should be inserted for expanding unconditional branch");
457 assert(MBB.pred_size() == 1);
458 assert(RestoreBB.empty() &&
459 "restore block should be inserted for restoring clobbered registers");
460
461 auto buildIndirectBranch = [&](Register Reg, MachineBasicBlock &DestBB) {
462 // Offsets outside of the signed 33-bit range are not supported for ADRP +
463 // ADD.
464 if (!isInt<33>(x: BrOffset))
465 report_fatal_error(
466 reason: "Branch offsets outside of the signed 33-bit range not supported");
467
468 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
469 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGE);
470 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: Reg)
471 .addReg(RegNo: Reg)
472 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
473 .addImm(Val: 0);
474 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::BR)).addReg(RegNo: Reg);
475 };
476
477 RS->enterBasicBlockEnd(MBB);
478 // If X16 is unused, we can rely on the linker to insert a range extension
479 // thunk if NewDestBB is out of range of a single B instruction.
480 constexpr Register Reg = AArch64::X16;
481 if (!RS->isRegUsed(Reg)) {
482 insertUnconditionalBranch(MBB, DestBB: &NewDestBB, DL);
483 RS->setRegUsed(Reg);
484 return;
485 }
486
487 // In a cold block without BTI, insert the indirect branch if a register is
488 // free. Skip this if BTI is enabled to avoid inserting a BTI at the target,
489 // prioritizing a dynamic cost in cold code over a static cost in hot code.
490 AArch64FunctionInfo *AFI = MBB.getParent()->getInfo<AArch64FunctionInfo>();
491 bool HasBTI = AFI && AFI->branchTargetEnforcement();
492 if (MBB.getSectionID() == MBBSectionID::ColdSectionID && !HasBTI) {
493 Register Scavenged = RS->FindUnusedReg(RC: &AArch64::GPR64RegClass);
494 if (Scavenged != AArch64::NoRegister) {
495 buildIndirectBranch(Scavenged, NewDestBB);
496 RS->setRegUsed(Reg: Scavenged);
497 return;
498 }
499 }
500
501 // Note: Spilling X16 briefly moves the stack pointer, making it incompatible
502 // with red zones.
503 if (!AFI || AFI->hasRedZone().value_or(u: true))
504 report_fatal_error(
505 reason: "Unable to insert indirect branch inside function that has red zone");
506
507 // Otherwise, spill X16 and defer range extension to the linker.
508 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::STRXpre))
509 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
510 .addReg(RegNo: Reg)
511 .addReg(RegNo: AArch64::SP)
512 .addImm(Val: -16);
513
514 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: &RestoreBB);
515
516 BuildMI(BB&: RestoreBB, I: RestoreBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::LDRXpost))
517 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
518 .addReg(RegNo: Reg, Flags: RegState::Define)
519 .addReg(RegNo: AArch64::SP)
520 .addImm(Val: 16);
521}
522
523// Branch analysis.
524bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
525 MachineBasicBlock *&TBB,
526 MachineBasicBlock *&FBB,
527 SmallVectorImpl<MachineOperand> &Cond,
528 bool AllowModify) const {
529 // If the block has no terminators, it just falls into the block after it.
530 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
531 if (I == MBB.end())
532 return false;
533
534 // Skip over SpeculationBarrierEndBB terminators
535 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
536 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
537 --I;
538 }
539
540 if (!isUnpredicatedTerminator(MI: *I))
541 return false;
542
543 // Get the last instruction in the block.
544 MachineInstr *LastInst = &*I;
545
546 // If there is only one terminator instruction, process it.
547 unsigned LastOpc = LastInst->getOpcode();
548 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
549 if (isUncondBranchOpcode(Opc: LastOpc)) {
550 TBB = LastInst->getOperand(i: 0).getMBB();
551 return false;
552 }
553 if (isCondBranchOpcode(Opc: LastOpc)) {
554 // Block ends with fall-through condbranch.
555 parseCondBranch(LastInst, Target&: TBB, Cond);
556 return false;
557 }
558 return true; // Can't handle indirect branch.
559 }
560
561 // Get the instruction before it if it is a terminator.
562 MachineInstr *SecondLastInst = &*I;
563 unsigned SecondLastOpc = SecondLastInst->getOpcode();
564
565 // If AllowModify is true and the block ends with two or more unconditional
566 // branches, delete all but the first unconditional branch.
567 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc)) {
568 while (isUncondBranchOpcode(Opc: SecondLastOpc)) {
569 LastInst->eraseFromParent();
570 LastInst = SecondLastInst;
571 LastOpc = LastInst->getOpcode();
572 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
573 // Return now the only terminator is an unconditional branch.
574 TBB = LastInst->getOperand(i: 0).getMBB();
575 return false;
576 }
577 SecondLastInst = &*I;
578 SecondLastOpc = SecondLastInst->getOpcode();
579 }
580 }
581
582 // If we're allowed to modify and the block ends in a unconditional branch
583 // which could simply fallthrough, remove the branch. (Note: This case only
584 // matters when we can't understand the whole sequence, otherwise it's also
585 // handled by BranchFolding.cpp.)
586 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc) &&
587 MBB.isLayoutSuccessor(MBB: getBranchDestBlock(MI: *LastInst))) {
588 LastInst->eraseFromParent();
589 LastInst = SecondLastInst;
590 LastOpc = LastInst->getOpcode();
591 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
592 assert(!isUncondBranchOpcode(LastOpc) &&
593 "unreachable unconditional branches removed above");
594
595 if (isCondBranchOpcode(Opc: LastOpc)) {
596 // Block ends with fall-through condbranch.
597 parseCondBranch(LastInst, Target&: TBB, Cond);
598 return false;
599 }
600 return true; // Can't handle indirect branch.
601 }
602 SecondLastInst = &*I;
603 SecondLastOpc = SecondLastInst->getOpcode();
604 }
605
606 // If there are three terminators, we don't know what sort of block this is.
607 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(MI: *--I))
608 return true;
609
610 // If the block ends with a B and a Bcc, handle it.
611 if (isCondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
612 parseCondBranch(LastInst: SecondLastInst, Target&: TBB, Cond);
613 FBB = LastInst->getOperand(i: 0).getMBB();
614 return false;
615 }
616
617 // If the block ends with two unconditional branches, handle it. The second
618 // one is not executed, so remove it.
619 if (isUncondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
620 TBB = SecondLastInst->getOperand(i: 0).getMBB();
621 I = LastInst;
622 if (AllowModify)
623 I->eraseFromParent();
624 return false;
625 }
626
627 // ...likewise if it ends with an indirect branch followed by an unconditional
628 // branch.
629 if (isIndirectBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
630 I = LastInst;
631 if (AllowModify)
632 I->eraseFromParent();
633 return true;
634 }
635
636 // Otherwise, can't handle this.
637 return true;
638}
639
640bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
641 MachineBranchPredicate &MBP,
642 bool AllowModify) const {
643 // Use analyzeBranch to validate the branch pattern.
644 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
645 SmallVector<MachineOperand, 4> Cond;
646 if (analyzeBranch(MBB, TBB, FBB, Cond, AllowModify))
647 return true;
648
649 // analyzeBranch returns success with empty Cond for unconditional branches.
650 if (Cond.empty())
651 return true;
652
653 MBP.TrueDest = TBB;
654 assert(MBP.TrueDest && "expected!");
655 MBP.FalseDest = FBB ? FBB : MBB.getNextNode();
656
657 MBP.ConditionDef = nullptr;
658 MBP.SingleUseCondition = false;
659
660 // Find the conditional branch. After analyzeBranch succeeds with non-empty
661 // Cond, there's exactly one conditional branch - either last (fallthrough)
662 // or second-to-last (followed by unconditional B).
663 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
664 if (I == MBB.end())
665 return true;
666
667 if (isUncondBranchOpcode(Opc: I->getOpcode())) {
668 if (I == MBB.begin())
669 return true;
670 --I;
671 }
672
673 MachineInstr *CondBranch = &*I;
674 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
675
676 switch (CondBranch->getOpcode()) {
677 default:
678 return true;
679
680 case AArch64::Bcc:
681 // Bcc takes the NZCV flag as the operand to branch on, walk up the
682 // instruction stream to find the last instruction to define NZCV.
683 for (MachineInstr &MI : llvm::drop_begin(RangeOrContainer: llvm::reverse(C&: MBB))) {
684 if (MI.modifiesRegister(Reg: AArch64::NZCV, /*TRI=*/nullptr)) {
685 MBP.ConditionDef = &MI;
686 break;
687 }
688 }
689 return false;
690
691 case AArch64::CBZW:
692 case AArch64::CBZX:
693 case AArch64::CBNZW:
694 case AArch64::CBNZX: {
695 MBP.LHS = CondBranch->getOperand(i: 0);
696 MBP.RHS = MachineOperand::CreateImm(Val: 0);
697 unsigned Opc = CondBranch->getOpcode();
698 MBP.Predicate = (Opc == AArch64::CBNZX || Opc == AArch64::CBNZW)
699 ? MachineBranchPredicate::PRED_NE
700 : MachineBranchPredicate::PRED_EQ;
701 Register CondReg = MBP.LHS.getReg();
702 if (CondReg.isVirtual())
703 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
704 return false;
705 }
706
707 case AArch64::TBZW:
708 case AArch64::TBZX:
709 case AArch64::TBNZW:
710 case AArch64::TBNZX: {
711 Register CondReg = CondBranch->getOperand(i: 0).getReg();
712 if (CondReg.isVirtual())
713 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
714 return false;
715 }
716 }
717}
718
719bool AArch64InstrInfo::reverseBranchCondition(
720 SmallVectorImpl<MachineOperand> &Cond) const {
721 if (Cond[0].getImm() != -1) {
722 // Regular Bcc
723 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
724 Cond[0].setImm(AArch64CC::getInvertedCondCode(Code: CC));
725 } else {
726 // Folded compare-and-branch
727 switch (Cond[1].getImm()) {
728 default:
729 llvm_unreachable("Unknown conditional branch!");
730 case AArch64::CBZW:
731 Cond[1].setImm(AArch64::CBNZW);
732 break;
733 case AArch64::CBNZW:
734 Cond[1].setImm(AArch64::CBZW);
735 break;
736 case AArch64::CBZX:
737 Cond[1].setImm(AArch64::CBNZX);
738 break;
739 case AArch64::CBNZX:
740 Cond[1].setImm(AArch64::CBZX);
741 break;
742 case AArch64::TBZW:
743 Cond[1].setImm(AArch64::TBNZW);
744 break;
745 case AArch64::TBNZW:
746 Cond[1].setImm(AArch64::TBZW);
747 break;
748 case AArch64::TBZX:
749 Cond[1].setImm(AArch64::TBNZX);
750 break;
751 case AArch64::TBNZX:
752 Cond[1].setImm(AArch64::TBZX);
753 break;
754
755 // Cond is { -1, Opcode, CC, Op0, Op1, ... }
756 case AArch64::CBWPri:
757 case AArch64::CBXPri:
758 case AArch64::CBBAssertExt:
759 case AArch64::CBHAssertExt:
760 case AArch64::CBWPrr:
761 case AArch64::CBXPrr: {
762 // Pseudos using standard 4bit Arm condition codes
763 AArch64CC::CondCode CC =
764 static_cast<AArch64CC::CondCode>(Cond[2].getImm());
765 Cond[2].setImm(AArch64CC::getInvertedCondCode(Code: CC));
766 }
767 }
768 }
769
770 return false;
771}
772
773unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
774 int *BytesRemoved) const {
775 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
776 if (I == MBB.end())
777 return 0;
778
779 if (!isUncondBranchOpcode(Opc: I->getOpcode()) &&
780 !isCondBranchOpcode(Opc: I->getOpcode()))
781 return 0;
782
783 // Remove the branch.
784 I->eraseFromParent();
785
786 I = MBB.end();
787
788 if (I == MBB.begin()) {
789 if (BytesRemoved)
790 *BytesRemoved = 4;
791 return 1;
792 }
793 --I;
794 if (!isCondBranchOpcode(Opc: I->getOpcode())) {
795 if (BytesRemoved)
796 *BytesRemoved = 4;
797 return 1;
798 }
799
800 // Remove the branch.
801 I->eraseFromParent();
802 if (BytesRemoved)
803 *BytesRemoved = 8;
804
805 return 2;
806}
807
808void AArch64InstrInfo::instantiateCondBranch(
809 MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
810 ArrayRef<MachineOperand> Cond) const {
811 if (Cond[0].getImm() != -1) {
812 // Regular Bcc
813 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: Cond[0].getImm()).addMBB(MBB: TBB);
814 } else {
815 // Folded compare-and-branch
816 // Note that we use addOperand instead of addReg to keep the flags.
817
818 // cbz, cbnz
819 const MachineInstrBuilder MIB =
820 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[1].getImm())).add(MO: Cond[2]);
821
822 // tbz/tbnz
823 if (Cond.size() > 3)
824 MIB.add(MO: Cond[3]);
825
826 // cb
827 if (Cond.size() > 4)
828 MIB.add(MO: Cond[4]);
829
830 MIB.addMBB(MBB: TBB);
831
832 // cb[b,h]
833 if (Cond.size() > 5) {
834 MIB.addImm(Val: Cond[5].getImm());
835 MIB.addImm(Val: Cond[6].getImm());
836 }
837 }
838}
839
840unsigned AArch64InstrInfo::insertBranch(
841 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
842 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
843 // Shouldn't be a fall through.
844 assert(TBB && "insertBranch must not be told to insert a fallthrough");
845
846 if (!FBB) {
847 if (Cond.empty()) // Unconditional branch?
848 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: TBB);
849 else
850 instantiateCondBranch(MBB, DL, TBB, Cond);
851
852 if (BytesAdded)
853 *BytesAdded = 4;
854
855 return 1;
856 }
857
858 // Two-way conditional branch.
859 instantiateCondBranch(MBB, DL, TBB, Cond);
860 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: FBB);
861
862 if (BytesAdded)
863 *BytesAdded = 8;
864
865 return 2;
866}
867
868bool llvm::optimizeTerminators(MachineBasicBlock *MBB,
869 const TargetInstrInfo &TII) {
870 for (MachineInstr &MI : MBB->terminators()) {
871 unsigned Opc = MI.getOpcode();
872 switch (Opc) {
873 case AArch64::CBZW:
874 case AArch64::CBZX:
875 case AArch64::TBZW:
876 case AArch64::TBZX:
877 // CBZ/TBZ with WZR/XZR -> unconditional B
878 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
879 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
880 DEBUG_WITH_TYPE("optimizeTerminators",
881 dbgs() << "Removing always taken branch: " << MI);
882 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
883 SmallVector<MachineBasicBlock *> Succs(MBB->successors());
884 for (auto *S : Succs)
885 if (S != Target)
886 MBB->removeSuccessor(Succ: S);
887 DebugLoc DL = MI.getDebugLoc();
888 while (MBB->rbegin() != &MI)
889 MBB->rbegin()->eraseFromParent();
890 MI.eraseFromParent();
891 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: AArch64::B)).addMBB(MBB: Target);
892 return true;
893 }
894 break;
895 case AArch64::CBNZW:
896 case AArch64::CBNZX:
897 case AArch64::TBNZW:
898 case AArch64::TBNZX:
899 // CBNZ/TBNZ with WZR/XZR -> never taken, remove branch and successor
900 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
901 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
902 DEBUG_WITH_TYPE("optimizeTerminators",
903 dbgs() << "Removing never taken branch: " << MI);
904 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
905 MI.getParent()->removeSuccessor(Succ: Target);
906 MI.eraseFromParent();
907 return true;
908 }
909 break;
910 }
911 }
912 return false;
913}
914
915// Find the original register that VReg is copied from.
916static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
917 while (Register::isVirtualRegister(Reg: VReg)) {
918 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
919 if (!DefMI || !DefMI->isFullCopy())
920 return VReg;
921 VReg = DefMI->getOperand(i: 1).getReg();
922 }
923 return VReg;
924}
925
926// Determine if VReg is defined by an instruction that can be folded into a
927// csel instruction. If so, return the folded opcode, and the replacement
928// register.
929static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
930 unsigned *NewReg = nullptr) {
931 VReg = removeCopies(MRI, VReg);
932 if (!Register::isVirtualRegister(Reg: VReg))
933 return 0;
934
935 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(RC: MRI.getRegClass(Reg: VReg));
936 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
937 if (!DefMI)
938 return 0;
939 unsigned Opc = 0;
940 unsigned SrcReg = 0;
941 switch (DefMI->getOpcode()) {
942 case AArch64::SUBREG_TO_REG:
943 // Check for the following way to define an 64-bit immediate:
944 // %0:gpr32 = MOVi32imm 1
945 // %1:gpr64 = SUBREG_TO_REG %0:gpr32, %subreg.sub_32
946 if (!DefMI->getOperand(i: 1).isReg())
947 return 0;
948 if (!DefMI->getOperand(i: 2).isImm() ||
949 DefMI->getOperand(i: 2).getImm() != AArch64::sub_32)
950 return 0;
951 DefMI = MRI.getVRegDef(Reg: DefMI->getOperand(i: 1).getReg());
952 if (DefMI->getOpcode() != AArch64::MOVi32imm)
953 return 0;
954 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
955 return 0;
956 assert(Is64Bit);
957 SrcReg = AArch64::XZR;
958 Opc = AArch64::CSINCXr;
959 break;
960
961 case AArch64::MOVi32imm:
962 case AArch64::MOVi64imm:
963 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
964 return 0;
965 SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
966 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
967 break;
968
969 case AArch64::ADDSXri:
970 case AArch64::ADDSWri:
971 // if NZCV is used, do not fold.
972 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
973 isDead: true) == -1)
974 return 0;
975 // fall-through to ADDXri and ADDWri.
976 [[fallthrough]];
977 case AArch64::ADDXri:
978 case AArch64::ADDWri:
979 // add x, 1 -> csinc.
980 if (!DefMI->getOperand(i: 2).isImm() || DefMI->getOperand(i: 2).getImm() != 1 ||
981 DefMI->getOperand(i: 3).getImm() != 0)
982 return 0;
983 SrcReg = DefMI->getOperand(i: 1).getReg();
984 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
985 break;
986
987 case AArch64::ORNXrr:
988 case AArch64::ORNWrr: {
989 // not x -> csinv, represented as orn dst, xzr, src.
990 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
991 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
992 return 0;
993 SrcReg = DefMI->getOperand(i: 2).getReg();
994 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
995 break;
996 }
997
998 case AArch64::SUBSXrr:
999 case AArch64::SUBSWrr:
1000 // if NZCV is used, do not fold.
1001 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1002 isDead: true) == -1)
1003 return 0;
1004 // fall-through to SUBXrr and SUBWrr.
1005 [[fallthrough]];
1006 case AArch64::SUBXrr:
1007 case AArch64::SUBWrr: {
1008 // neg x -> csneg, represented as sub dst, xzr, src.
1009 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1010 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1011 return 0;
1012 SrcReg = DefMI->getOperand(i: 2).getReg();
1013 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
1014 break;
1015 }
1016 default:
1017 return 0;
1018 }
1019 assert(Opc && SrcReg && "Missing parameters");
1020
1021 if (NewReg)
1022 *NewReg = SrcReg;
1023 return Opc;
1024}
1025
1026bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1027 ArrayRef<MachineOperand> Cond,
1028 Register DstReg, Register TrueReg,
1029 Register FalseReg, int &CondCycles,
1030 int &TrueCycles,
1031 int &FalseCycles) const {
1032 // Check register classes.
1033 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1034 const TargetRegisterClass *RC =
1035 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1036 if (!RC)
1037 return false;
1038
1039 // Also need to check the dest regclass, in case we're trying to optimize
1040 // something like:
1041 // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
1042 if (!RI.getCommonSubClass(A: RC, B: MRI.getRegClass(Reg: DstReg)))
1043 return false;
1044
1045 // Expanding cbz/tbz requires an extra cycle of latency on the condition.
1046 unsigned ExtraCondLat = Cond.size() != 1;
1047
1048 // GPRs are handled by csel.
1049 // FIXME: Fold in x+1, -x, and ~x when applicable.
1050 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
1051 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1052 // Single-cycle csel, csinc, csinv, and csneg.
1053 CondCycles = 1 + ExtraCondLat;
1054 TrueCycles = FalseCycles = 1;
1055 if (canFoldIntoCSel(MRI, VReg: TrueReg))
1056 TrueCycles = 0;
1057 else if (canFoldIntoCSel(MRI, VReg: FalseReg))
1058 FalseCycles = 0;
1059 return true;
1060 }
1061
1062 // Scalar floating point is handled by fcsel.
1063 // FIXME: Form fabs, fmin, and fmax when applicable.
1064 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
1065 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
1066 CondCycles = 5 + ExtraCondLat;
1067 TrueCycles = FalseCycles = 2;
1068 return true;
1069 }
1070
1071 // Can't do vectors.
1072 return false;
1073}
1074
1075void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
1076 MachineBasicBlock::iterator I,
1077 const DebugLoc &DL, Register DstReg,
1078 ArrayRef<MachineOperand> Cond,
1079 Register TrueReg, Register FalseReg) const {
1080 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1081
1082 // Parse the condition code, see parseCondBranch() above.
1083 AArch64CC::CondCode CC;
1084 switch (Cond.size()) {
1085 default:
1086 llvm_unreachable("Unknown condition opcode in Cond");
1087 case 1: // b.cc
1088 CC = AArch64CC::CondCode(Cond[0].getImm());
1089 break;
1090 case 3: { // cbz/cbnz
1091 // We must insert a compare against 0.
1092 bool Is64Bit;
1093 switch (Cond[1].getImm()) {
1094 default:
1095 llvm_unreachable("Unknown branch opcode in Cond");
1096 case AArch64::CBZW:
1097 Is64Bit = false;
1098 CC = AArch64CC::EQ;
1099 break;
1100 case AArch64::CBZX:
1101 Is64Bit = true;
1102 CC = AArch64CC::EQ;
1103 break;
1104 case AArch64::CBNZW:
1105 Is64Bit = false;
1106 CC = AArch64CC::NE;
1107 break;
1108 case AArch64::CBNZX:
1109 Is64Bit = true;
1110 CC = AArch64CC::NE;
1111 break;
1112 }
1113 Register SrcReg = Cond[2].getReg();
1114 if (Is64Bit) {
1115 // cmp reg, #0 is actually subs xzr, reg, #0.
1116 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64spRegClass);
1117 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::SUBSXri), DestReg: AArch64::XZR)
1118 .addReg(RegNo: SrcReg)
1119 .addImm(Val: 0)
1120 .addImm(Val: 0);
1121 } else {
1122 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32spRegClass);
1123 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWri), DestReg: AArch64::WZR)
1124 .addReg(RegNo: SrcReg)
1125 .addImm(Val: 0)
1126 .addImm(Val: 0);
1127 }
1128 break;
1129 }
1130 case 4: { // tbz/tbnz
1131 // We must insert a tst instruction.
1132 switch (Cond[1].getImm()) {
1133 default:
1134 llvm_unreachable("Unknown branch opcode in Cond");
1135 case AArch64::TBZW:
1136 case AArch64::TBZX:
1137 CC = AArch64CC::EQ;
1138 break;
1139 case AArch64::TBNZW:
1140 case AArch64::TBNZX:
1141 CC = AArch64CC::NE;
1142 break;
1143 }
1144 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
1145 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
1146 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ANDSWri), DestReg: AArch64::WZR)
1147 .addReg(RegNo: Cond[2].getReg())
1148 .addImm(
1149 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 32));
1150 else
1151 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ANDSXri), DestReg: AArch64::XZR)
1152 .addReg(RegNo: Cond[2].getReg())
1153 .addImm(
1154 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 64));
1155 break;
1156 }
1157 case 5: { // cb
1158 // We must insert a cmp, that is a subs
1159 // 0 1 2 3 4
1160 // Cond is { -1, Opcode, CC, Op0, Op1 }
1161
1162 unsigned SubsOpc, SubsDestReg;
1163 bool IsImm = false;
1164 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
1165 switch (Cond[1].getImm()) {
1166 default:
1167 llvm_unreachable("Unknown branch opcode in Cond");
1168 case AArch64::CBWPri:
1169 SubsOpc = AArch64::SUBSWri;
1170 SubsDestReg = AArch64::WZR;
1171 IsImm = true;
1172 break;
1173 case AArch64::CBXPri:
1174 SubsOpc = AArch64::SUBSXri;
1175 SubsDestReg = AArch64::XZR;
1176 IsImm = true;
1177 break;
1178 case AArch64::CBWPrr:
1179 SubsOpc = AArch64::SUBSWrr;
1180 SubsDestReg = AArch64::WZR;
1181 IsImm = false;
1182 break;
1183 case AArch64::CBXPrr:
1184 SubsOpc = AArch64::SUBSXrr;
1185 SubsDestReg = AArch64::XZR;
1186 IsImm = false;
1187 break;
1188 }
1189
1190 if (IsImm)
1191 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
1192 .addReg(RegNo: Cond[3].getReg())
1193 .addImm(Val: Cond[4].getImm())
1194 .addImm(Val: 0);
1195 else
1196 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
1197 .addReg(RegNo: Cond[3].getReg())
1198 .addReg(RegNo: Cond[4].getReg());
1199 } break;
1200 case 7: { // cb[b,h]
1201 // We must insert a cmp, that is a subs, but also zero- or sign-extensions
1202 // that have been folded. For the first operand we codegen an explicit
1203 // extension, for the second operand we fold the extension into cmp.
1204 // 0 1 2 3 4 5 6
1205 // Cond is { -1, Opcode, CC, Op0, Op1, Ext0, Ext1 }
1206
1207 // We need a new register for the now explicitly extended register
1208 Register Reg = Cond[4].getReg();
1209 if (Cond[5].getImm() != AArch64_AM::InvalidShiftExtend) {
1210 unsigned ExtOpc;
1211 unsigned ExtBits;
1212 AArch64_AM::ShiftExtendType ExtendType =
1213 AArch64_AM::getExtendType(Imm: Cond[5].getImm());
1214 switch (ExtendType) {
1215 default:
1216 llvm_unreachable("Unknown shift-extend for CB instruction");
1217 case AArch64_AM::SXTB:
1218 assert(
1219 Cond[1].getImm() == AArch64::CBBAssertExt &&
1220 "Unexpected compare-and-branch instruction for SXTB shift-extend");
1221 ExtOpc = AArch64::SBFMWri;
1222 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1223 break;
1224 case AArch64_AM::SXTH:
1225 assert(
1226 Cond[1].getImm() == AArch64::CBHAssertExt &&
1227 "Unexpected compare-and-branch instruction for SXTH shift-extend");
1228 ExtOpc = AArch64::SBFMWri;
1229 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1230 break;
1231 case AArch64_AM::UXTB:
1232 assert(
1233 Cond[1].getImm() == AArch64::CBBAssertExt &&
1234 "Unexpected compare-and-branch instruction for UXTB shift-extend");
1235 ExtOpc = AArch64::ANDWri;
1236 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1237 break;
1238 case AArch64_AM::UXTH:
1239 assert(
1240 Cond[1].getImm() == AArch64::CBHAssertExt &&
1241 "Unexpected compare-and-branch instruction for UXTH shift-extend");
1242 ExtOpc = AArch64::ANDWri;
1243 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1244 break;
1245 }
1246
1247 // Build the explicit extension of the first operand
1248 Reg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32spRegClass);
1249 MachineInstrBuilder MBBI =
1250 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: ExtOpc), DestReg: Reg).addReg(RegNo: Cond[4].getReg());
1251 if (ExtOpc != AArch64::ANDWri)
1252 MBBI.addImm(Val: 0);
1253 MBBI.addImm(Val: ExtBits);
1254 }
1255
1256 // Now, subs with an extended second operand
1257 if (Cond[6].getImm() != AArch64_AM::InvalidShiftExtend) {
1258 AArch64_AM::ShiftExtendType ExtendType =
1259 AArch64_AM::getExtendType(Imm: Cond[6].getImm());
1260 MRI.constrainRegClass(Reg, RC: MRI.getRegClass(Reg: Cond[3].getReg()));
1261 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: &AArch64::GPR32spRegClass);
1262 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrx), DestReg: AArch64::WZR)
1263 .addReg(RegNo: Cond[3].getReg())
1264 .addReg(RegNo: Reg)
1265 .addImm(Val: AArch64_AM::getArithExtendImm(ET: ExtendType, Imm: 0));
1266 } // If no extension is needed, just a regular subs
1267 else {
1268 MRI.constrainRegClass(Reg, RC: MRI.getRegClass(Reg: Cond[3].getReg()));
1269 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: &AArch64::GPR32spRegClass);
1270 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrr), DestReg: AArch64::WZR)
1271 .addReg(RegNo: Cond[3].getReg())
1272 .addReg(RegNo: Reg);
1273 }
1274
1275 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
1276 } break;
1277 }
1278
1279 unsigned Opc = 0;
1280 const TargetRegisterClass *RC = nullptr;
1281 bool TryFold = false;
1282 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass)) {
1283 RC = &AArch64::GPR64RegClass;
1284 Opc = AArch64::CSELXr;
1285 TryFold = true;
1286 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR32RegClass)) {
1287 RC = &AArch64::GPR32RegClass;
1288 Opc = AArch64::CSELWr;
1289 TryFold = true;
1290 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR64RegClass)) {
1291 RC = &AArch64::FPR64RegClass;
1292 Opc = AArch64::FCSELDrrr;
1293 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR32RegClass)) {
1294 RC = &AArch64::FPR32RegClass;
1295 Opc = AArch64::FCSELSrrr;
1296 }
1297 assert(RC && "Unsupported regclass");
1298
1299 // Try folding simple instructions into the csel.
1300 if (TryFold) {
1301 unsigned NewReg = 0;
1302 unsigned FoldedOpc = canFoldIntoCSel(MRI, VReg: TrueReg, NewReg: &NewReg);
1303 if (FoldedOpc) {
1304 // The folded opcodes csinc, csinc and csneg apply the operation to
1305 // FalseReg, so we need to invert the condition.
1306 CC = AArch64CC::getInvertedCondCode(Code: CC);
1307 TrueReg = FalseReg;
1308 } else
1309 FoldedOpc = canFoldIntoCSel(MRI, VReg: FalseReg, NewReg: &NewReg);
1310
1311 // Fold the operation. Leave any dead instructions for DCE to clean up.
1312 if (FoldedOpc) {
1313 FalseReg = NewReg;
1314 Opc = FoldedOpc;
1315 // Extend the live range of NewReg.
1316 MRI.clearKillFlags(Reg: NewReg);
1317 }
1318 }
1319
1320 // Pull all virtual register into the appropriate class.
1321 MRI.constrainRegClass(Reg: TrueReg, RC);
1322 // FalseReg might be WZR or XZR if the folded operand is a literal 1.
1323 assert(
1324 (FalseReg.isVirtual() || FalseReg == AArch64::WZR ||
1325 FalseReg == AArch64::XZR) &&
1326 "FalseReg was folded into a non-virtual register other than WZR or XZR");
1327 if (FalseReg.isVirtual())
1328 MRI.constrainRegClass(Reg: FalseReg, RC);
1329
1330 // Insert the csel.
1331 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
1332 .addReg(RegNo: TrueReg)
1333 .addReg(RegNo: FalseReg)
1334 .addImm(Val: CC);
1335}
1336
1337// Return true if Imm can be loaded into a register by a "cheap" sequence of
1338// instructions. For now, "cheap" means at most two instructions.
1339static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize) {
1340 if (BitSize == 32)
1341 return true;
1342
1343 assert(BitSize == 64 && "Only bit sizes of 32 or 64 allowed");
1344 uint64_t Imm = static_cast<uint64_t>(MI.getOperand(i: 1).getImm());
1345 SmallVector<AArch64_IMM::ImmInsnModel, 4> Is;
1346 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: Is);
1347
1348 return Is.size() <= 2;
1349}
1350
1351// Check if a COPY instruction is cheap.
1352static bool isCheapCopy(const MachineInstr &MI, const AArch64RegisterInfo &RI) {
1353 assert(MI.isCopy() && "Expected COPY instruction");
1354 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1355
1356 // Cross-bank copies (e.g., between GPR and FPR) are expensive on AArch64,
1357 // typically requiring an FMOV instruction with a 2-6 cycle latency.
1358 auto GetRegClass = [&](Register Reg) -> const TargetRegisterClass * {
1359 if (Reg.isVirtual())
1360 return MRI.getRegClass(Reg);
1361 if (Reg.isPhysical())
1362 return RI.getMinimalPhysRegClass(Reg);
1363 return nullptr;
1364 };
1365 const TargetRegisterClass *DstRC = GetRegClass(MI.getOperand(i: 0).getReg());
1366 const TargetRegisterClass *SrcRC = GetRegClass(MI.getOperand(i: 1).getReg());
1367 if (DstRC && SrcRC && !RI.getCommonSubClass(A: DstRC, B: SrcRC))
1368 return false;
1369
1370 return MI.isAsCheapAsAMove();
1371}
1372
1373// FIXME: this implementation should be micro-architecture dependent, so a
1374// micro-architecture target hook should be introduced here in future.
1375bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
1376 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1377 if (isExynosCheapAsMove(MI))
1378 return true;
1379 return MI.isAsCheapAsAMove();
1380 }
1381
1382 switch (MI.getOpcode()) {
1383 default:
1384 return MI.isAsCheapAsAMove();
1385
1386 case TargetOpcode::COPY:
1387 return isCheapCopy(MI, RI);
1388
1389 case AArch64::ADDWrs:
1390 case AArch64::ADDXrs:
1391 case AArch64::SUBWrs:
1392 case AArch64::SUBXrs:
1393 return Subtarget.hasALULSLFast() && MI.getOperand(i: 3).getImm() <= 4;
1394
1395 // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
1396 // ORRXri, it is as cheap as MOV.
1397 // Likewise if it can be expanded to MOVZ/MOVN/MOVK.
1398 case AArch64::MOVi32imm:
1399 return isCheapImmediate(MI, BitSize: 32);
1400 case AArch64::MOVi64imm:
1401 return isCheapImmediate(MI, BitSize: 64);
1402 }
1403}
1404
1405bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
1406 switch (MI.getOpcode()) {
1407 default:
1408 return false;
1409
1410 case AArch64::ADDWrs:
1411 case AArch64::ADDXrs:
1412 case AArch64::ADDSWrs:
1413 case AArch64::ADDSXrs: {
1414 unsigned Imm = MI.getOperand(i: 3).getImm();
1415 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1416 if (ShiftVal == 0)
1417 return true;
1418 return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
1419 }
1420
1421 case AArch64::ADDWrx:
1422 case AArch64::ADDXrx:
1423 case AArch64::ADDXrx64:
1424 case AArch64::ADDSWrx:
1425 case AArch64::ADDSXrx:
1426 case AArch64::ADDSXrx64: {
1427 unsigned Imm = MI.getOperand(i: 3).getImm();
1428 switch (AArch64_AM::getArithExtendType(Imm)) {
1429 default:
1430 return false;
1431 case AArch64_AM::UXTB:
1432 case AArch64_AM::UXTH:
1433 case AArch64_AM::UXTW:
1434 case AArch64_AM::UXTX:
1435 return AArch64_AM::getArithShiftValue(Imm) <= 4;
1436 }
1437 }
1438
1439 case AArch64::SUBWrs:
1440 case AArch64::SUBSWrs: {
1441 unsigned Imm = MI.getOperand(i: 3).getImm();
1442 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1443 return ShiftVal == 0 ||
1444 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
1445 }
1446
1447 case AArch64::SUBXrs:
1448 case AArch64::SUBSXrs: {
1449 unsigned Imm = MI.getOperand(i: 3).getImm();
1450 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1451 return ShiftVal == 0 ||
1452 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
1453 }
1454
1455 case AArch64::SUBWrx:
1456 case AArch64::SUBXrx:
1457 case AArch64::SUBXrx64:
1458 case AArch64::SUBSWrx:
1459 case AArch64::SUBSXrx:
1460 case AArch64::SUBSXrx64: {
1461 unsigned Imm = MI.getOperand(i: 3).getImm();
1462 switch (AArch64_AM::getArithExtendType(Imm)) {
1463 default:
1464 return false;
1465 case AArch64_AM::UXTB:
1466 case AArch64_AM::UXTH:
1467 case AArch64_AM::UXTW:
1468 case AArch64_AM::UXTX:
1469 return AArch64_AM::getArithShiftValue(Imm) == 0;
1470 }
1471 }
1472
1473 case AArch64::LDRBBroW:
1474 case AArch64::LDRBBroX:
1475 case AArch64::LDRBroW:
1476 case AArch64::LDRBroX:
1477 case AArch64::LDRDroW:
1478 case AArch64::LDRDroX:
1479 case AArch64::LDRHHroW:
1480 case AArch64::LDRHHroX:
1481 case AArch64::LDRHroW:
1482 case AArch64::LDRHroX:
1483 case AArch64::LDRQroW:
1484 case AArch64::LDRQroX:
1485 case AArch64::LDRSBWroW:
1486 case AArch64::LDRSBWroX:
1487 case AArch64::LDRSBXroW:
1488 case AArch64::LDRSBXroX:
1489 case AArch64::LDRSHWroW:
1490 case AArch64::LDRSHWroX:
1491 case AArch64::LDRSHXroW:
1492 case AArch64::LDRSHXroX:
1493 case AArch64::LDRSWroW:
1494 case AArch64::LDRSWroX:
1495 case AArch64::LDRSroW:
1496 case AArch64::LDRSroX:
1497 case AArch64::LDRWroW:
1498 case AArch64::LDRWroX:
1499 case AArch64::LDRXroW:
1500 case AArch64::LDRXroX:
1501 case AArch64::PRFMroW:
1502 case AArch64::PRFMroX:
1503 case AArch64::STRBBroW:
1504 case AArch64::STRBBroX:
1505 case AArch64::STRBroW:
1506 case AArch64::STRBroX:
1507 case AArch64::STRDroW:
1508 case AArch64::STRDroX:
1509 case AArch64::STRHHroW:
1510 case AArch64::STRHHroX:
1511 case AArch64::STRHroW:
1512 case AArch64::STRHroX:
1513 case AArch64::STRQroW:
1514 case AArch64::STRQroX:
1515 case AArch64::STRSroW:
1516 case AArch64::STRSroX:
1517 case AArch64::STRWroW:
1518 case AArch64::STRWroX:
1519 case AArch64::STRXroW:
1520 case AArch64::STRXroX: {
1521 unsigned IsSigned = MI.getOperand(i: 3).getImm();
1522 return !IsSigned;
1523 }
1524 }
1525}
1526
1527bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
1528 unsigned Opc = MI.getOpcode();
1529 switch (Opc) {
1530 default:
1531 return false;
1532 case AArch64::SEH_StackAlloc:
1533 case AArch64::SEH_SaveFPLR:
1534 case AArch64::SEH_SaveFPLR_X:
1535 case AArch64::SEH_SaveReg:
1536 case AArch64::SEH_SaveReg_X:
1537 case AArch64::SEH_SaveRegP:
1538 case AArch64::SEH_SaveRegP_X:
1539 case AArch64::SEH_SaveFReg:
1540 case AArch64::SEH_SaveFReg_X:
1541 case AArch64::SEH_SaveFRegP:
1542 case AArch64::SEH_SaveFRegP_X:
1543 case AArch64::SEH_SetFP:
1544 case AArch64::SEH_AddFP:
1545 case AArch64::SEH_Nop:
1546 case AArch64::SEH_PrologEnd:
1547 case AArch64::SEH_EpilogStart:
1548 case AArch64::SEH_EpilogEnd:
1549 case AArch64::SEH_PACSignLR:
1550 case AArch64::SEH_SaveAnyRegI:
1551 case AArch64::SEH_SaveAnyRegIP:
1552 case AArch64::SEH_SaveAnyRegQP:
1553 case AArch64::SEH_SaveAnyRegQPX:
1554 case AArch64::SEH_AllocZ:
1555 case AArch64::SEH_SaveZReg:
1556 case AArch64::SEH_SavePReg:
1557 return true;
1558 }
1559}
1560
1561bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1562 Register &SrcReg, Register &DstReg,
1563 unsigned &SubIdx) const {
1564 switch (MI.getOpcode()) {
1565 default:
1566 return false;
1567 case AArch64::SBFMXri: // aka sxtw
1568 case AArch64::UBFMXri: // aka uxtw
1569 // Check for the 32 -> 64 bit extension case, these instructions can do
1570 // much more.
1571 if (MI.getOperand(i: 2).getImm() != 0 || MI.getOperand(i: 3).getImm() != 31)
1572 return false;
1573 // This is a signed or unsigned 32 -> 64 bit extension.
1574 SrcReg = MI.getOperand(i: 1).getReg();
1575 DstReg = MI.getOperand(i: 0).getReg();
1576 SubIdx = AArch64::sub_32;
1577 return true;
1578 }
1579}
1580
1581bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
1582 const MachineInstr &MIa, const MachineInstr &MIb) const {
1583 const TargetRegisterInfo *TRI = &getRegisterInfo();
1584 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1585 int64_t OffsetA = 0, OffsetB = 0;
1586 TypeSize WidthA(0, false), WidthB(0, false);
1587 bool OffsetAIsScalable = false, OffsetBIsScalable = false;
1588
1589 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1590 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1591
1592 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
1593 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1594 return false;
1595
1596 // Retrieve the base, offset from the base and width. Width
1597 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
1598 // base are identical, and the offset of a lower memory access +
1599 // the width doesn't overlap the offset of a higher memory access,
1600 // then the memory accesses are different.
1601 // If OffsetAIsScalable and OffsetBIsScalable are both true, they
1602 // are assumed to have the same scale (vscale).
1603 if (getMemOperandWithOffsetWidth(MI: MIa, BaseOp&: BaseOpA, Offset&: OffsetA, OffsetIsScalable&: OffsetAIsScalable,
1604 Width&: WidthA, TRI) &&
1605 getMemOperandWithOffsetWidth(MI: MIb, BaseOp&: BaseOpB, Offset&: OffsetB, OffsetIsScalable&: OffsetBIsScalable,
1606 Width&: WidthB, TRI)) {
1607 if (BaseOpA->isIdenticalTo(Other: *BaseOpB) &&
1608 OffsetAIsScalable == OffsetBIsScalable) {
1609 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1610 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1611 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1612 if (LowWidth.isScalable() == OffsetAIsScalable &&
1613 LowOffset + (int)LowWidth.getKnownMinValue() <= HighOffset)
1614 return true;
1615 }
1616 }
1617 return false;
1618}
1619
1620bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1621 const MachineBasicBlock *MBB,
1622 const MachineFunction &MF) const {
1623 if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
1624 return true;
1625
1626 // Do not move an instruction that can be recognized as a branch target.
1627 if (hasBTISemantics(MI))
1628 return true;
1629
1630 switch (MI.getOpcode()) {
1631 case AArch64::HINT:
1632 // CSDB hints are scheduling barriers.
1633 if (MI.getOperand(i: 0).getImm() == 0x14)
1634 return true;
1635 break;
1636 case AArch64::DSB:
1637 case AArch64::ISB:
1638 // DSB and ISB also are scheduling barriers.
1639 return true;
1640 case AArch64::MSRpstatesvcrImm1:
1641 // SMSTART and SMSTOP are also scheduling barriers.
1642 return true;
1643 default:;
1644 }
1645 if (isSEHInstruction(MI))
1646 return true;
1647 auto Next = std::next(x: MI.getIterator());
1648 return Next != MBB->end() && Next->isCFIInstruction();
1649}
1650
1651/// analyzeCompare - For a comparison instruction, return the source registers
1652/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1653/// Return true if the comparison instruction can be analyzed.
1654bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1655 Register &SrcReg2, int64_t &CmpMask,
1656 int64_t &CmpValue) const {
1657 // The first operand can be a frame index where we'd normally expect a
1658 // register.
1659 // FIXME: Pass subregisters out of analyzeCompare
1660 assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1661 if (!MI.getOperand(i: 1).isReg() || MI.getOperand(i: 1).getSubReg())
1662 return false;
1663
1664 switch (MI.getOpcode()) {
1665 default:
1666 break;
1667 case AArch64::PTEST_PP:
1668 case AArch64::PTEST_PP_ANY:
1669 case AArch64::PTEST_PP_FIRST:
1670 SrcReg = MI.getOperand(i: 0).getReg();
1671 SrcReg2 = MI.getOperand(i: 1).getReg();
1672 if (MI.getOperand(i: 2).getSubReg())
1673 return false;
1674
1675 // Not sure about the mask and value for now...
1676 CmpMask = ~0;
1677 CmpValue = 0;
1678 return true;
1679 case AArch64::SUBSWrr:
1680 case AArch64::SUBSWrs:
1681 case AArch64::SUBSWrx:
1682 case AArch64::SUBSXrr:
1683 case AArch64::SUBSXrs:
1684 case AArch64::SUBSXrx:
1685 case AArch64::ADDSWrr:
1686 case AArch64::ADDSWrs:
1687 case AArch64::ADDSWrx:
1688 case AArch64::ADDSXrr:
1689 case AArch64::ADDSXrs:
1690 case AArch64::ADDSXrx:
1691 // Replace SUBSWrr with SUBWrr if NZCV is not used.
1692 SrcReg = MI.getOperand(i: 1).getReg();
1693 SrcReg2 = MI.getOperand(i: 2).getReg();
1694
1695 // FIXME: Pass subregisters out of analyzeCompare
1696 if (MI.getOperand(i: 2).getSubReg())
1697 return false;
1698
1699 CmpMask = ~0;
1700 CmpValue = 0;
1701 return true;
1702 case AArch64::SUBSWri:
1703 case AArch64::ADDSWri:
1704 case AArch64::SUBSXri:
1705 case AArch64::ADDSXri:
1706 SrcReg = MI.getOperand(i: 1).getReg();
1707 SrcReg2 = 0;
1708 CmpMask = ~0;
1709 CmpValue = MI.getOperand(i: 2).getImm();
1710 return true;
1711 case AArch64::ANDSWri:
1712 case AArch64::ANDSXri:
1713 // ANDS does not use the same encoding scheme as the others xxxS
1714 // instructions.
1715 SrcReg = MI.getOperand(i: 1).getReg();
1716 SrcReg2 = 0;
1717 CmpMask = ~0;
1718 CmpValue = AArch64_AM::decodeLogicalImmediate(
1719 val: MI.getOperand(i: 2).getImm(),
1720 regSize: MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1721 return true;
1722 }
1723
1724 return false;
1725}
1726
1727static bool UpdateOperandRegClass(MachineInstr &Instr) {
1728 MachineBasicBlock *MBB = Instr.getParent();
1729 assert(MBB && "Can't get MachineBasicBlock here");
1730 MachineFunction *MF = MBB->getParent();
1731 assert(MF && "Can't get MachineFunction here");
1732 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1733 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1734 MachineRegisterInfo *MRI = &MF->getRegInfo();
1735
1736 for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1737 ++OpIdx) {
1738 MachineOperand &MO = Instr.getOperand(i: OpIdx);
1739 const TargetRegisterClass *OpRegCstraints =
1740 Instr.getRegClassConstraint(OpIdx, TII, TRI);
1741
1742 // If there's no constraint, there's nothing to do.
1743 if (!OpRegCstraints)
1744 continue;
1745 // If the operand is a frame index, there's nothing to do here.
1746 // A frame index operand will resolve correctly during PEI.
1747 if (MO.isFI())
1748 continue;
1749
1750 assert(MO.isReg() &&
1751 "Operand has register constraints without being a register!");
1752
1753 Register Reg = MO.getReg();
1754 if (Reg.isPhysical()) {
1755 if (!OpRegCstraints->contains(Reg))
1756 return false;
1757 } else if (!OpRegCstraints->hasSubClassEq(RC: MRI->getRegClass(Reg)) &&
1758 !MRI->constrainRegClass(Reg, RC: OpRegCstraints))
1759 return false;
1760 }
1761
1762 return true;
1763}
1764
1765/// Return the opcode that does not set flags when possible - otherwise
1766/// return the original opcode. The caller is responsible to do the actual
1767/// substitution and legality checking.
1768static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1769 // Don't convert all compare instructions, because for some the zero register
1770 // encoding becomes the sp register.
1771 bool MIDefinesZeroReg = false;
1772 if (MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
1773 MI.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
1774 MIDefinesZeroReg = true;
1775
1776 switch (MI.getOpcode()) {
1777 default:
1778 return MI.getOpcode();
1779 case AArch64::ADDSWrr:
1780 return AArch64::ADDWrr;
1781 case AArch64::ADDSWri:
1782 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1783 case AArch64::ADDSWrs:
1784 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1785 case AArch64::ADDSWrx:
1786 return AArch64::ADDWrx;
1787 case AArch64::ADDSXrr:
1788 return AArch64::ADDXrr;
1789 case AArch64::ADDSXri:
1790 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1791 case AArch64::ADDSXrs:
1792 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1793 case AArch64::ADDSXrx:
1794 return AArch64::ADDXrx;
1795 case AArch64::SUBSWrr:
1796 return AArch64::SUBWrr;
1797 case AArch64::SUBSWri:
1798 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1799 case AArch64::SUBSWrs:
1800 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1801 case AArch64::SUBSWrx:
1802 return AArch64::SUBWrx;
1803 case AArch64::SUBSXrr:
1804 return AArch64::SUBXrr;
1805 case AArch64::SUBSXri:
1806 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1807 case AArch64::SUBSXrs:
1808 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1809 case AArch64::SUBSXrx:
1810 return AArch64::SUBXrx;
1811 }
1812}
1813
1814enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1815
1816/// True when condition flags are accessed (either by writing or reading)
1817/// on the instruction trace starting at From and ending at To.
1818///
1819/// Note: If From and To are from different blocks it's assumed CC are accessed
1820/// on the path.
1821static bool areCFlagsAccessedBetweenInstrs(
1822 MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1823 const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1824 // Early exit if To is at the beginning of the BB.
1825 if (To == To->getParent()->begin())
1826 return true;
1827
1828 // Check whether the instructions are in the same basic block
1829 // If not, assume the condition flags might get modified somewhere.
1830 if (To->getParent() != From->getParent())
1831 return true;
1832
1833 // From must be above To.
1834 assert(std::any_of(
1835 ++To.getReverse(), To->getParent()->rend(),
1836 [From](MachineInstr &MI) { return MI.getIterator() == From; }));
1837
1838 // We iterate backward starting at \p To until we hit \p From.
1839 for (const MachineInstr &Instr :
1840 instructionsWithoutDebug(It: ++To.getReverse(), End: From.getReverse())) {
1841 if (((AccessToCheck & AK_Write) &&
1842 Instr.modifiesRegister(Reg: AArch64::NZCV, TRI)) ||
1843 ((AccessToCheck & AK_Read) && Instr.readsRegister(Reg: AArch64::NZCV, TRI)))
1844 return true;
1845 }
1846 return false;
1847}
1848
1849std::optional<unsigned>
1850AArch64InstrInfo::canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
1851 MachineInstr *Pred,
1852 const MachineRegisterInfo *MRI) const {
1853 unsigned MaskOpcode = Mask->getOpcode();
1854 unsigned PredOpcode = Pred->getOpcode();
1855 bool PredIsPTestLike = isPTestLikeOpcode(Opc: PredOpcode);
1856 bool PredIsWhileLike = isWhileOpcode(Opc: PredOpcode);
1857
1858 if (PredIsWhileLike) {
1859 // For PTEST(PG, PG), PTEST is redundant when PG is the result of a WHILEcc
1860 // instruction and the condition is "any" since WHILcc does an implicit
1861 // PTEST(ALL, PG) check and PG is always a subset of ALL.
1862 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1863 return PredOpcode;
1864
1865 // For PTEST(PTRUE_ALL, WHILE), if the element size matches, the PTEST is
1866 // redundant since WHILE performs an implicit PTEST with an all active
1867 // mask.
1868 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1869 getElementSizeForOpcode(Opc: MaskOpcode) ==
1870 getElementSizeForOpcode(Opc: PredOpcode))
1871 return PredOpcode;
1872
1873 // For PTEST_FIRST(PTRUE_ALL, WHILE), the PTEST_FIRST is redundant since
1874 // WHILEcc performs an implicit PTEST with an all active mask, setting
1875 // the N flag as the PTEST_FIRST would.
1876 if (PTest->getOpcode() == AArch64::PTEST_PP_FIRST &&
1877 isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31)
1878 return PredOpcode;
1879
1880 return {};
1881 }
1882
1883 if (PredIsPTestLike) {
1884 // For PTEST(PG, PG), PTEST is redundant when PG is the result of an
1885 // instruction that sets the flags as PTEST would and the condition is
1886 // "any" since PG is always a subset of the governing predicate of the
1887 // ptest-like instruction.
1888 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1889 return PredOpcode;
1890
1891 auto PTestLikeMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1892
1893 // If the PTEST like instruction's general predicate is not `Mask`, attempt
1894 // to look through a copy and try again. This is because some instructions
1895 // take a predicate whose register class is a subset of its result class.
1896 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1897 PTestLikeMask->getOperand(i: 1).getReg().isVirtual())
1898 PTestLikeMask =
1899 MRI->getUniqueVRegDef(Reg: PTestLikeMask->getOperand(i: 1).getReg());
1900
1901 // For PTEST(PTRUE_ALL, PTEST_LIKE), the PTEST is redundant if the
1902 // the element size matches and either the PTEST_LIKE instruction uses
1903 // the same all active mask or the condition is "any".
1904 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1905 getElementSizeForOpcode(Opc: MaskOpcode) ==
1906 getElementSizeForOpcode(Opc: PredOpcode)) {
1907 if (Mask == PTestLikeMask || PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1908 return PredOpcode;
1909 }
1910
1911 // For PTEST(PG, PTEST_LIKE(PG, ...)), the PTEST is redundant since the
1912 // flags are set based on the same mask 'PG', but PTEST_LIKE must operate
1913 // on 8-bit predicates like the PTEST. Otherwise, for instructions like
1914 // compare that also support 16/32/64-bit predicates, the implicit PTEST
1915 // performed by the compare could consider fewer lanes for these element
1916 // sizes.
1917 //
1918 // For example, consider
1919 //
1920 // ptrue p0.b ; P0=1111-1111-1111-1111
1921 // index z0.s, #0, #1 ; Z0=<0,1,2,3>
1922 // index z1.s, #1, #1 ; Z1=<1,2,3,4>
1923 // cmphi p1.s, p0/z, z1.s, z0.s ; P1=0001-0001-0001-0001
1924 // ; ^ last active
1925 // ptest p0, p1.b ; P1=0001-0001-0001-0001
1926 // ; ^ last active
1927 //
1928 // where the compare generates a canonical all active 32-bit predicate
1929 // (equivalent to 'ptrue p1.s, all'). The implicit PTEST sets the last
1930 // active flag, whereas the PTEST instruction with the same mask doesn't.
1931 // For PTEST_ANY this doesn't apply as the flags in this case would be
1932 // identical regardless of element size.
1933 uint64_t PredElementSize = getElementSizeForOpcode(Opc: PredOpcode);
1934 if (Mask == PTestLikeMask && (PredElementSize == AArch64::ElementSizeB ||
1935 PTest->getOpcode() == AArch64::PTEST_PP_ANY))
1936 return PredOpcode;
1937
1938 return {};
1939 }
1940
1941 // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the
1942 // opcode so the PTEST becomes redundant.
1943 switch (PredOpcode) {
1944 case AArch64::AND_PPzPP:
1945 case AArch64::BIC_PPzPP:
1946 case AArch64::EOR_PPzPP:
1947 case AArch64::NAND_PPzPP:
1948 case AArch64::NOR_PPzPP:
1949 case AArch64::ORN_PPzPP:
1950 case AArch64::ORR_PPzPP:
1951 case AArch64::BRKA_PPzP:
1952 case AArch64::BRKPA_PPzPP:
1953 case AArch64::BRKB_PPzP:
1954 case AArch64::BRKPB_PPzPP:
1955 case AArch64::RDFFR_PPz: {
1956 // Check to see if our mask is the same. If not the resulting flag bits
1957 // may be different and we can't remove the ptest.
1958 auto *PredMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1959 if (Mask != PredMask)
1960 return {};
1961 break;
1962 }
1963 case AArch64::BRKN_PPzP: {
1964 // BRKN uses an all active implicit mask to set flags unlike the other
1965 // flag-setting instructions.
1966 // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B).
1967 if ((MaskOpcode != AArch64::PTRUE_B) ||
1968 (Mask->getOperand(i: 1).getImm() != 31))
1969 return {};
1970 break;
1971 }
1972 case AArch64::PTRUE_B:
1973 // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A)
1974 break;
1975 default:
1976 // Bail out if we don't recognize the input
1977 return {};
1978 }
1979
1980 return convertToFlagSettingOpc(Opc: PredOpcode);
1981}
1982
1983/// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
1984/// operation which could set the flags in an identical manner
1985bool AArch64InstrInfo::optimizePTestInstr(
1986 MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
1987 const MachineRegisterInfo *MRI) const {
1988 auto *Mask = MRI->getUniqueVRegDef(Reg: MaskReg);
1989 auto *Pred = MRI->getUniqueVRegDef(Reg: PredReg);
1990
1991 if (Pred->isCopy() && PTest->getOpcode() == AArch64::PTEST_PP_FIRST) {
1992 // Instructions which return a multi-vector (e.g. WHILECC_x2) require copies
1993 // before the branch to extract each subregister.
1994 auto Op = Pred->getOperand(i: 1);
1995 if (Op.isReg() && Op.getReg().isVirtual() &&
1996 Op.getSubReg() == AArch64::psub0)
1997 Pred = MRI->getUniqueVRegDef(Reg: Op.getReg());
1998 }
1999
2000 unsigned PredOpcode = Pred->getOpcode();
2001 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred, MRI);
2002 if (!NewOp)
2003 return false;
2004
2005 const TargetRegisterInfo *TRI = &getRegisterInfo();
2006
2007 // If another instruction between Pred and PTest accesses flags, don't remove
2008 // the ptest or update the earlier instruction to modify them.
2009 if (areCFlagsAccessedBetweenInstrs(From: Pred, To: PTest, TRI))
2010 return false;
2011
2012 // If we pass all the checks, it's safe to remove the PTEST and use the flags
2013 // as they are prior to PTEST. Sometimes this requires the tested PTEST
2014 // operand to be replaced with an equivalent instruction that also sets the
2015 // flags.
2016 PTest->eraseFromParent();
2017 if (*NewOp != PredOpcode) {
2018 Pred->setDesc(get(Opcode: *NewOp));
2019 bool succeeded = UpdateOperandRegClass(Instr&: *Pred);
2020 (void)succeeded;
2021 assert(succeeded && "Operands have incompatible register classes!");
2022 Pred->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: TRI);
2023 }
2024
2025 // Ensure that the flags def is live.
2026 if (Pred->registerDefIsDead(Reg: AArch64::NZCV, TRI)) {
2027 unsigned i = 0, e = Pred->getNumOperands();
2028 for (; i != e; ++i) {
2029 MachineOperand &MO = Pred->getOperand(i);
2030 if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::NZCV) {
2031 MO.setIsDead(false);
2032 break;
2033 }
2034 }
2035 }
2036 return true;
2037}
2038
2039/// Try to optimize a compare instruction. A compare instruction is an
2040/// instruction which produces AArch64::NZCV. It can be truly compare
2041/// instruction
2042/// when there are no uses of its destination register.
2043///
2044/// The following steps are tried in order:
2045/// 1. Convert CmpInstr into an unconditional version.
2046/// 2. Remove CmpInstr if above there is an instruction producing a needed
2047/// condition code or an instruction which can be converted into such an
2048/// instruction.
2049/// Only comparison with zero is supported.
2050bool AArch64InstrInfo::optimizeCompareInstr(
2051 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2052 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2053 assert(CmpInstr.getParent());
2054 assert(MRI);
2055
2056 // Replace SUBSWrr with SUBWrr if NZCV is not used.
2057 int DeadNZCVIdx =
2058 CmpInstr.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
2059 if (DeadNZCVIdx != -1) {
2060 if (CmpInstr.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
2061 CmpInstr.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr)) {
2062 CmpInstr.eraseFromParent();
2063 return true;
2064 }
2065 unsigned Opc = CmpInstr.getOpcode();
2066 unsigned NewOpc = convertToNonFlagSettingOpc(MI: CmpInstr);
2067 if (NewOpc == Opc)
2068 return false;
2069 const MCInstrDesc &MCID = get(Opcode: NewOpc);
2070 CmpInstr.setDesc(MCID);
2071 CmpInstr.removeOperand(OpNo: DeadNZCVIdx);
2072 bool succeeded = UpdateOperandRegClass(Instr&: CmpInstr);
2073 (void)succeeded;
2074 assert(succeeded && "Some operands reg class are incompatible!");
2075 return true;
2076 }
2077
2078 if (CmpInstr.getOpcode() == AArch64::PTEST_PP ||
2079 CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY ||
2080 CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST)
2081 return optimizePTestInstr(PTest: &CmpInstr, MaskReg: SrcReg, PredReg: SrcReg2, MRI);
2082
2083 if (SrcReg2 != 0)
2084 return false;
2085
2086 // CmpInstr is a Compare instruction if destination register is not used.
2087 if (!MRI->use_nodbg_empty(RegNo: CmpInstr.getOperand(i: 0).getReg()))
2088 return false;
2089
2090 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, MRI: *MRI))
2091 return true;
2092 return (CmpValue == 0 || CmpValue == 1) &&
2093 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, MRI: *MRI);
2094}
2095
2096/// Get opcode of S version of Instr.
2097/// If Instr is S version its opcode is returned.
2098/// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
2099/// or we are not interested in it.
2100static unsigned sForm(MachineInstr &Instr) {
2101 switch (Instr.getOpcode()) {
2102 default:
2103 return AArch64::INSTRUCTION_LIST_END;
2104
2105 case AArch64::ADDSWrr:
2106 case AArch64::ADDSWri:
2107 case AArch64::ADDSXrr:
2108 case AArch64::ADDSXri:
2109 case AArch64::ADDSWrx:
2110 case AArch64::ADDSXrx:
2111 case AArch64::ADDSWrs:
2112 case AArch64::ADDSXrs:
2113 case AArch64::SUBSWrr:
2114 case AArch64::SUBSWri:
2115 case AArch64::SUBSWrx:
2116 case AArch64::SUBSWrs:
2117 case AArch64::SUBSXrr:
2118 case AArch64::SUBSXri:
2119 case AArch64::SUBSXrx:
2120 case AArch64::SUBSXrs:
2121 case AArch64::ANDSWri:
2122 case AArch64::ANDSWrr:
2123 case AArch64::ANDSWrs:
2124 case AArch64::ANDSXri:
2125 case AArch64::ANDSXrr:
2126 case AArch64::ANDSXrs:
2127 case AArch64::BICSWrr:
2128 case AArch64::BICSXrr:
2129 case AArch64::BICSWrs:
2130 case AArch64::BICSXrs:
2131 case AArch64::ADCSWr:
2132 case AArch64::ADCSXr:
2133 case AArch64::SBCSWr:
2134 case AArch64::SBCSXr:
2135 return Instr.getOpcode();
2136
2137 case AArch64::ADDWrr:
2138 return AArch64::ADDSWrr;
2139 case AArch64::ADDWri:
2140 return AArch64::ADDSWri;
2141 case AArch64::ADDXrr:
2142 return AArch64::ADDSXrr;
2143 case AArch64::ADDXri:
2144 return AArch64::ADDSXri;
2145 case AArch64::ADDWrx:
2146 return AArch64::ADDSWrx;
2147 case AArch64::ADDXrx:
2148 return AArch64::ADDSXrx;
2149 case AArch64::ADDWrs:
2150 return AArch64::ADDSWrs;
2151 case AArch64::ADDXrs:
2152 return AArch64::ADDSXrs;
2153 case AArch64::ADCWr:
2154 return AArch64::ADCSWr;
2155 case AArch64::ADCXr:
2156 return AArch64::ADCSXr;
2157 case AArch64::SUBWrr:
2158 return AArch64::SUBSWrr;
2159 case AArch64::SUBWri:
2160 return AArch64::SUBSWri;
2161 case AArch64::SUBXrr:
2162 return AArch64::SUBSXrr;
2163 case AArch64::SUBXri:
2164 return AArch64::SUBSXri;
2165 case AArch64::SUBWrx:
2166 return AArch64::SUBSWrx;
2167 case AArch64::SUBXrx:
2168 return AArch64::SUBSXrx;
2169 case AArch64::SUBWrs:
2170 return AArch64::SUBSWrs;
2171 case AArch64::SUBXrs:
2172 return AArch64::SUBSXrs;
2173 case AArch64::SBCWr:
2174 return AArch64::SBCSWr;
2175 case AArch64::SBCXr:
2176 return AArch64::SBCSXr;
2177 case AArch64::ANDWri:
2178 return AArch64::ANDSWri;
2179 case AArch64::ANDXri:
2180 return AArch64::ANDSXri;
2181 case AArch64::ANDWrr:
2182 return AArch64::ANDSWrr;
2183 case AArch64::ANDWrs:
2184 return AArch64::ANDSWrs;
2185 case AArch64::ANDXrr:
2186 return AArch64::ANDSXrr;
2187 case AArch64::ANDXrs:
2188 return AArch64::ANDSXrs;
2189 case AArch64::BICWrr:
2190 return AArch64::BICSWrr;
2191 case AArch64::BICXrr:
2192 return AArch64::BICSXrr;
2193 case AArch64::BICWrs:
2194 return AArch64::BICSWrs;
2195 case AArch64::BICXrs:
2196 return AArch64::BICSXrs;
2197 }
2198}
2199
2200/// Check if AArch64::NZCV should be alive in successors of MBB.
2201static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB) {
2202 for (auto *BB : MBB->successors())
2203 if (BB->isLiveIn(Reg: AArch64::NZCV))
2204 return true;
2205 return false;
2206}
2207
2208/// \returns The condition code operand index for \p Instr if it is a branch
2209/// or select and -1 otherwise.
2210int AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(
2211 const MachineInstr &Instr) {
2212 switch (Instr.getOpcode()) {
2213 default:
2214 return -1;
2215
2216 case AArch64::Bcc: {
2217 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2218 assert(Idx >= 2);
2219 return Idx - 2;
2220 }
2221
2222 case AArch64::CSINVWr:
2223 case AArch64::CSINVXr:
2224 case AArch64::CSINCWr:
2225 case AArch64::CSINCXr:
2226 case AArch64::CSELWr:
2227 case AArch64::CSELXr:
2228 case AArch64::CSNEGWr:
2229 case AArch64::CSNEGXr:
2230 case AArch64::FCSELSrrr:
2231 case AArch64::FCSELDrrr: {
2232 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2233 assert(Idx >= 1);
2234 return Idx - 1;
2235 }
2236 }
2237}
2238
2239/// Find a condition code used by the instruction.
2240/// Returns AArch64CC::Invalid if either the instruction does not use condition
2241/// codes or we don't optimize CmpInstr in the presence of such instructions.
2242static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
2243 int CCIdx =
2244 AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(Instr);
2245 return CCIdx >= 0 ? static_cast<AArch64CC::CondCode>(
2246 Instr.getOperand(i: CCIdx).getImm())
2247 : AArch64CC::Invalid;
2248}
2249
2250static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
2251 assert(CC != AArch64CC::Invalid);
2252 UsedNZCV UsedFlags;
2253 switch (CC) {
2254 default:
2255 break;
2256
2257 case AArch64CC::EQ: // Z set
2258 case AArch64CC::NE: // Z clear
2259 UsedFlags.Z = true;
2260 break;
2261
2262 case AArch64CC::HI: // Z clear and C set
2263 case AArch64CC::LS: // Z set or C clear
2264 UsedFlags.Z = true;
2265 [[fallthrough]];
2266 case AArch64CC::HS: // C set
2267 case AArch64CC::LO: // C clear
2268 UsedFlags.C = true;
2269 break;
2270
2271 case AArch64CC::MI: // N set
2272 case AArch64CC::PL: // N clear
2273 UsedFlags.N = true;
2274 break;
2275
2276 case AArch64CC::VS: // V set
2277 case AArch64CC::VC: // V clear
2278 UsedFlags.V = true;
2279 break;
2280
2281 case AArch64CC::GT: // Z clear, N and V the same
2282 case AArch64CC::LE: // Z set, N and V differ
2283 UsedFlags.Z = true;
2284 [[fallthrough]];
2285 case AArch64CC::GE: // N and V the same
2286 case AArch64CC::LT: // N and V differ
2287 UsedFlags.N = true;
2288 UsedFlags.V = true;
2289 break;
2290 }
2291 return UsedFlags;
2292}
2293
2294/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
2295/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
2296/// \returns std::nullopt otherwise.
2297///
2298/// Collect instructions using that flags in \p CCUseInstrs if provided.
2299std::optional<UsedNZCV>
2300llvm::examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
2301 const TargetRegisterInfo &TRI,
2302 SmallVectorImpl<MachineInstr *> *CCUseInstrs) {
2303 MachineBasicBlock *CmpParent = CmpInstr.getParent();
2304 if (MI.getParent() != CmpParent)
2305 return std::nullopt;
2306
2307 if (areCFlagsAliveInSuccessors(MBB: CmpParent))
2308 return std::nullopt;
2309
2310 UsedNZCV NZCVUsedAfterCmp;
2311 for (MachineInstr &Instr : instructionsWithoutDebug(
2312 It: std::next(x: CmpInstr.getIterator()), End: CmpParent->instr_end())) {
2313 if (Instr.readsRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
2314 AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
2315 if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
2316 return std::nullopt;
2317 NZCVUsedAfterCmp |= getUsedNZCV(CC);
2318 if (CCUseInstrs)
2319 CCUseInstrs->push_back(Elt: &Instr);
2320 }
2321 if (Instr.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI))
2322 break;
2323 }
2324 return NZCVUsedAfterCmp;
2325}
2326
2327static bool isADDSRegImm(unsigned Opcode) {
2328 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
2329}
2330
2331static bool isSUBSRegImm(unsigned Opcode) {
2332 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
2333}
2334
2335static bool isANDOpcode(MachineInstr &MI) {
2336 unsigned Opc = sForm(Instr&: MI);
2337 switch (Opc) {
2338 case AArch64::ANDSWri:
2339 case AArch64::ANDSWrr:
2340 case AArch64::ANDSWrs:
2341 case AArch64::ANDSXri:
2342 case AArch64::ANDSXrr:
2343 case AArch64::ANDSXrs:
2344 case AArch64::BICSWrr:
2345 case AArch64::BICSXrr:
2346 case AArch64::BICSWrs:
2347 case AArch64::BICSXrs:
2348 return true;
2349 default:
2350 return false;
2351 }
2352}
2353
2354/// Check if CmpInstr can be substituted by MI.
2355///
2356/// CmpInstr can be substituted:
2357/// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2358/// - and, MI and CmpInstr are from the same MachineBB
2359/// - and, condition flags are not alive in successors of the CmpInstr parent
2360/// - and, if MI opcode is the S form there must be no defs of flags between
2361/// MI and CmpInstr
2362/// or if MI opcode is not the S form there must be neither defs of flags
2363/// nor uses of flags between MI and CmpInstr.
2364/// - and, C is not used after CmpInstr; CmpInstr's C is from adds/subs #0 on
2365/// SrcReg and can differ from MI (e.g. carry out of ADCS/SBCS).
2366/// - and, V is not used after CmpInstr unless MI is AND/BIC (V cleared) or MI
2367/// has NoSWrap (overflow is poison and the fold is still safe).
2368static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr,
2369 const TargetRegisterInfo &TRI) {
2370 // MI is an opcode sForm maps (add/sub/adc/sbc/and/bic and their S forms).
2371 assert(sForm(MI) != AArch64::INSTRUCTION_LIST_END);
2372
2373 const unsigned CmpOpcode = CmpInstr.getOpcode();
2374 if (!isADDSRegImm(Opcode: CmpOpcode) && !isSUBSRegImm(Opcode: CmpOpcode))
2375 return false;
2376
2377 assert((CmpInstr.getOperand(2).isImm() &&
2378 CmpInstr.getOperand(2).getImm() == 0) &&
2379 "Caller guarantees that CmpInstr compares with constant 0");
2380
2381 std::optional<UsedNZCV> NZVCUsed = examineCFlagsUse(MI, CmpInstr, TRI);
2382 if (!NZVCUsed || NZVCUsed->C)
2383 return false;
2384
2385 // CmpInstr is ADDS/SUBS with immediate 0 on SrcReg (compare SrcReg to zero).
2386 // After the fold, users see NZCV from MI (or its S form), not from CmpInstr.
2387 // N/Z match CmpInstr for the value in SrcReg; C/V need not match in general
2388 // (e.g. ADCS vs adds #0), so we require C unused after CmpInstr and gate V
2389 // as below. NoSWrap makes signed overflow poison; AND/BIC clear V.
2390 if (NZVCUsed->V && !MI.getFlag(Flag: MachineInstr::NoSWrap) && !isANDOpcode(MI))
2391 return false;
2392
2393 AccessKind AccessToCheck = AK_Write;
2394 if (sForm(Instr&: MI) != MI.getOpcode())
2395 AccessToCheck = AK_All;
2396 return !areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck);
2397}
2398
2399/// Substitute an instruction comparing to zero with another instruction
2400/// which produces needed condition flags.
2401///
2402/// Return true on success.
2403bool AArch64InstrInfo::substituteCmpToZero(
2404 MachineInstr &CmpInstr, unsigned SrcReg,
2405 const MachineRegisterInfo &MRI) const {
2406 // Get the unique definition of SrcReg.
2407 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2408 if (!MI)
2409 return false;
2410
2411 const TargetRegisterInfo &TRI = getRegisterInfo();
2412
2413 unsigned NewOpc = sForm(Instr&: *MI);
2414 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
2415 return false;
2416
2417 if (!canInstrSubstituteCmpInstr(MI&: *MI, CmpInstr, TRI))
2418 return false;
2419
2420 // Update the instruction to set NZCV.
2421 MI->setDesc(get(Opcode: NewOpc));
2422 CmpInstr.eraseFromParent();
2423 bool succeeded = UpdateOperandRegClass(Instr&: *MI);
2424 (void)succeeded;
2425 assert(succeeded && "Some operands reg class are incompatible!");
2426 MI->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: &TRI);
2427 return true;
2428}
2429
2430/// \returns True if \p CmpInstr can be removed.
2431///
2432/// \p IsInvertCC is true if, after removing \p CmpInstr, condition
2433/// codes used in \p CCUseInstrs must be inverted.
2434static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr,
2435 int CmpValue, const TargetRegisterInfo &TRI,
2436 SmallVectorImpl<MachineInstr *> &CCUseInstrs,
2437 bool &IsInvertCC) {
2438 assert((CmpValue == 0 || CmpValue == 1) &&
2439 "Only comparisons to 0 or 1 considered for removal!");
2440
2441 // MI is 'CSINCWr %vreg, wzr, wzr, <cc>' or 'CSINCXr %vreg, xzr, xzr, <cc>'
2442 unsigned MIOpc = MI.getOpcode();
2443 if (MIOpc == AArch64::CSINCWr) {
2444 if (MI.getOperand(i: 1).getReg() != AArch64::WZR ||
2445 MI.getOperand(i: 2).getReg() != AArch64::WZR)
2446 return false;
2447 } else if (MIOpc == AArch64::CSINCXr) {
2448 if (MI.getOperand(i: 1).getReg() != AArch64::XZR ||
2449 MI.getOperand(i: 2).getReg() != AArch64::XZR)
2450 return false;
2451 } else {
2452 return false;
2453 }
2454 AArch64CC::CondCode MICC = findCondCodeUsedByInstr(Instr: MI);
2455 if (MICC == AArch64CC::Invalid)
2456 return false;
2457
2458 // NZCV needs to be defined
2459 if (MI.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) != -1)
2460 return false;
2461
2462 // CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0' or 'SUBS %vreg, 1'
2463 const unsigned CmpOpcode = CmpInstr.getOpcode();
2464 bool IsSubsRegImm = isSUBSRegImm(Opcode: CmpOpcode);
2465 if (CmpValue && !IsSubsRegImm)
2466 return false;
2467 if (!CmpValue && !IsSubsRegImm && !isADDSRegImm(Opcode: CmpOpcode))
2468 return false;
2469
2470 // MI conditions allowed: eq, ne, mi, pl
2471 UsedNZCV MIUsedNZCV = getUsedNZCV(CC: MICC);
2472 if (MIUsedNZCV.C || MIUsedNZCV.V)
2473 return false;
2474
2475 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2476 examineCFlagsUse(MI, CmpInstr, TRI, CCUseInstrs: &CCUseInstrs);
2477 // Condition flags are not used in CmpInstr basic block successors and only
2478 // Z or N flags allowed to be used after CmpInstr within its basic block
2479 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2480 return false;
2481 // Z or N flag used after CmpInstr must correspond to the flag used in MI
2482 if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) ||
2483 (MIUsedNZCV.N && NZCVUsedAfterCmp->Z))
2484 return false;
2485 // If CmpInstr is comparison to zero MI conditions are limited to eq, ne
2486 if (MIUsedNZCV.N && !CmpValue)
2487 return false;
2488
2489 // There must be no defs of flags between MI and CmpInstr
2490 if (areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck: AK_Write))
2491 return false;
2492
2493 // Condition code is inverted in the following cases:
2494 // 1. MI condition is ne; CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2495 // 2. MI condition is eq, pl; CmpInstr is 'SUBS %vreg, 1'
2496 IsInvertCC = (CmpValue && (MICC == AArch64CC::EQ || MICC == AArch64CC::PL)) ||
2497 (!CmpValue && MICC == AArch64CC::NE);
2498 return true;
2499}
2500
2501/// Remove comparison in csinc-cmp sequence
2502///
2503/// Examples:
2504/// 1. \code
2505/// csinc w9, wzr, wzr, ne
2506/// cmp w9, #0
2507/// b.eq
2508/// \endcode
2509/// to
2510/// \code
2511/// csinc w9, wzr, wzr, ne
2512/// b.ne
2513/// \endcode
2514///
2515/// 2. \code
2516/// csinc x2, xzr, xzr, mi
2517/// cmp x2, #1
2518/// b.pl
2519/// \endcode
2520/// to
2521/// \code
2522/// csinc x2, xzr, xzr, mi
2523/// b.pl
2524/// \endcode
2525///
2526/// \param CmpInstr comparison instruction
2527/// \return True when comparison removed
2528bool AArch64InstrInfo::removeCmpToZeroOrOne(
2529 MachineInstr &CmpInstr, unsigned SrcReg, int CmpValue,
2530 const MachineRegisterInfo &MRI) const {
2531 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2532 if (!MI)
2533 return false;
2534 const TargetRegisterInfo &TRI = getRegisterInfo();
2535 SmallVector<MachineInstr *, 4> CCUseInstrs;
2536 bool IsInvertCC = false;
2537 if (!canCmpInstrBeRemoved(MI&: *MI, CmpInstr, CmpValue, TRI, CCUseInstrs,
2538 IsInvertCC))
2539 return false;
2540 // Make transformation
2541 CmpInstr.eraseFromParent();
2542 if (IsInvertCC) {
2543 // Invert condition codes in CmpInstr CC users
2544 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2545 int Idx = findCondCodeUseOperandIdxForBranchOrSelect(Instr: *CCUseInstr);
2546 assert(Idx >= 0 && "Unexpected instruction using CC.");
2547 MachineOperand &CCOperand = CCUseInstr->getOperand(i: Idx);
2548 AArch64CC::CondCode CCUse = AArch64CC::getInvertedCondCode(
2549 Code: static_cast<AArch64CC::CondCode>(CCOperand.getImm()));
2550 CCOperand.setImm(CCUse);
2551 }
2552 }
2553 return true;
2554}
2555
2556bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2557 if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2558 MI.getOpcode() != AArch64::CATCHRET &&
2559 MI.getOpcode() != AArch64::STACK_GUARD_UNMIX)
2560 return false;
2561
2562 MachineBasicBlock &MBB = *MI.getParent();
2563 auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
2564 auto TRI = Subtarget.getRegisterInfo();
2565 DebugLoc DL = MI.getDebugLoc();
2566
2567 if (MI.getOpcode() == AArch64::STACK_GUARD_UNMIX) {
2568 // Expand STACK_GUARD_UNMIX to: sub Rd, fp, Rs
2569 // This computes FP - stored_mixed_value to unmix the cookie
2570 Register DstReg = MI.getOperand(i: 0).getReg();
2571 Register SrcReg = MI.getOperand(i: 1).getReg();
2572
2573 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: DstReg)
2574 .addReg(RegNo: AArch64::FP)
2575 .addReg(RegNo: SrcReg);
2576
2577 MBB.erase(I: MI);
2578 return true;
2579 }
2580
2581 if (MI.getOpcode() == AArch64::CATCHRET) {
2582 // Skip to the first instruction before the epilog.
2583 const TargetInstrInfo *TII =
2584 MBB.getParent()->getSubtarget().getInstrInfo();
2585 MachineBasicBlock *TargetMBB = MI.getOperand(i: 0).getMBB();
2586 auto MBBI = MachineBasicBlock::iterator(MI);
2587 MachineBasicBlock::iterator FirstEpilogSEH = std::prev(x: MBBI);
2588 while (FirstEpilogSEH->getFlag(Flag: MachineInstr::FrameDestroy) &&
2589 FirstEpilogSEH != MBB.begin())
2590 FirstEpilogSEH = std::prev(x: FirstEpilogSEH);
2591 if (FirstEpilogSEH != MBB.begin())
2592 FirstEpilogSEH = std::next(x: FirstEpilogSEH);
2593 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADRP))
2594 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2595 .addMBB(MBB: TargetMBB);
2596 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri))
2597 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2598 .addReg(RegNo: AArch64::X0)
2599 .addMBB(MBB: TargetMBB)
2600 .addImm(Val: 0);
2601 TargetMBB->setMachineBlockAddressTaken();
2602 return true;
2603 }
2604
2605 Register Reg = MI.getOperand(i: 0).getReg();
2606 Module &M = *MBB.getParent()->getFunction().getParent();
2607 if (M.getStackProtectorGuard() == "sysreg") {
2608 const AArch64SysReg::SysReg *SrcReg =
2609 AArch64SysReg::lookupSysRegByName(Name: M.getStackProtectorGuardReg());
2610 if (!SrcReg)
2611 report_fatal_error(reason: "Unknown SysReg for Stack Protector Guard Register");
2612
2613 // mrs xN, sysreg
2614 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MRS))
2615 .addDef(RegNo: Reg, Flags: RegState::Renamable)
2616 .addImm(Val: SrcReg->Encoding);
2617 int Offset = M.getStackProtectorGuardOffset();
2618 if (Offset >= 0 && Offset <= 32760 && Offset % 8 == 0) {
2619 // ldr xN, [xN, #offset]
2620 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2621 .addDef(RegNo: Reg)
2622 .addUse(RegNo: Reg, Flags: RegState::Kill)
2623 .addImm(Val: Offset / 8);
2624 } else if (Offset >= -256 && Offset <= 255) {
2625 // ldur xN, [xN, #offset]
2626 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDURXi))
2627 .addDef(RegNo: Reg)
2628 .addUse(RegNo: Reg, Flags: RegState::Kill)
2629 .addImm(Val: Offset);
2630 } else if (Offset >= -4095 && Offset <= 4095) {
2631 if (Offset > 0) {
2632 // add xN, xN, #offset
2633 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri))
2634 .addDef(RegNo: Reg)
2635 .addUse(RegNo: Reg, Flags: RegState::Kill)
2636 .addImm(Val: Offset)
2637 .addImm(Val: 0);
2638 } else {
2639 // sub xN, xN, #offset
2640 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXri))
2641 .addDef(RegNo: Reg)
2642 .addUse(RegNo: Reg, Flags: RegState::Kill)
2643 .addImm(Val: -Offset)
2644 .addImm(Val: 0);
2645 }
2646 // ldr xN, [xN]
2647 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2648 .addDef(RegNo: Reg)
2649 .addUse(RegNo: Reg, Flags: RegState::Kill)
2650 .addImm(Val: 0);
2651 } else {
2652 // Cases that are larger than +/- 4095 and not a multiple of 8, or larger
2653 // than 23760.
2654 // It might be nice to use AArch64::MOVi32imm here, which would get
2655 // expanded in PreSched2 after PostRA, but our lone scratch Reg already
2656 // contains the MRS result. findScratchNonCalleeSaveRegister() in
2657 // AArch64FrameLowering might help us find such a scratch register
2658 // though. If we failed to find a scratch register, we could emit a
2659 // stream of add instructions to build up the immediate. Or, we could try
2660 // to insert a AArch64::MOVi32imm before register allocation so that we
2661 // didn't need to scavenge for a scratch register.
2662 report_fatal_error(reason: "Unable to encode Stack Protector Guard Offset");
2663 }
2664 MBB.erase(I: MI);
2665 return true;
2666 }
2667
2668 const GlobalValue *GV =
2669 cast<GlobalValue>(Val: (*MI.memoperands_begin())->getValue());
2670 const TargetMachine &TM = MBB.getParent()->getTarget();
2671 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2672 const unsigned char MO_NC = AArch64II::MO_NC;
2673
2674 unsigned GuardWidth = M.getStackProtectorGuardValueWidth().value_or(
2675 u: Subtarget.isTargetILP32() ? 4 : 8);
2676 if (GuardWidth != 4 && GuardWidth != 8)
2677 report_fatal_error(reason: "Unsupported stack protector value width");
2678 if ((OpFlags & AArch64II::MO_GOT) != 0) {
2679 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LOADgot), DestReg: Reg)
2680 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags);
2681 if (GuardWidth == 4) {
2682 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2683 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2684 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2685 .addUse(RegNo: Reg, Flags: RegState::Kill)
2686 .addImm(Val: 0)
2687 .addMemOperand(MMO: *MI.memoperands_begin())
2688 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2689 } else {
2690 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2691 .addReg(RegNo: Reg, Flags: RegState::Kill)
2692 .addImm(Val: 0)
2693 .addMemOperand(MMO: *MI.memoperands_begin());
2694 }
2695 } else if (TM.getCodeModel() == CodeModel::Large) {
2696 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg)
2697 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G0 | MO_NC)
2698 .addImm(Val: 0);
2699 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2700 .addReg(RegNo: Reg, Flags: RegState::Kill)
2701 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G1 | MO_NC)
2702 .addImm(Val: 16);
2703 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2704 .addReg(RegNo: Reg, Flags: RegState::Kill)
2705 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G2 | MO_NC)
2706 .addImm(Val: 32);
2707 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2708 .addReg(RegNo: Reg, Flags: RegState::Kill)
2709 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G3)
2710 .addImm(Val: 48);
2711 if (GuardWidth == 4) {
2712 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2713 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2714 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2715 .addUse(RegNo: Reg, Flags: RegState::Kill)
2716 .addImm(Val: 0)
2717 .addMemOperand(MMO: *MI.memoperands_begin())
2718 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2719 } else {
2720 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2721 .addReg(RegNo: Reg, Flags: RegState::Kill)
2722 .addImm(Val: 0)
2723 .addMemOperand(MMO: *MI.memoperands_begin());
2724 }
2725 } else {
2726 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
2727 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags | AArch64II::MO_PAGE);
2728 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
2729 if (GuardWidth == 4) {
2730 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2731 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2732 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2733 .addUse(RegNo: Reg, Flags: RegState::Kill)
2734 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2735 .addMemOperand(MMO: *MI.memoperands_begin())
2736 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2737 } else {
2738 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2739 .addReg(RegNo: Reg, Flags: RegState::Kill)
2740 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2741 .addMemOperand(MMO: *MI.memoperands_begin());
2742 }
2743 }
2744 // To match MSVC. Unlike x86_64 which uses xor instruction to mix the cookie,
2745 // we use sub instruction to mix the cookie on aarch64.
2746 // The mixing happens here in expandPostRAPseudo (after RA) to ensure we use
2747 // the final frame pointer value.
2748 if (Subtarget.getTargetTriple().isOSMSVCRT())
2749 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: Reg)
2750 .addReg(RegNo: AArch64::FP)
2751 .addReg(RegNo: Reg, Flags: RegState::Kill);
2752
2753 MBB.erase(I: MI);
2754
2755 return true;
2756}
2757
2758// Return true if this instruction simply sets its single destination register
2759// to zero. This is equivalent to a register rename of the zero-register.
2760bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
2761 switch (MI.getOpcode()) {
2762 default:
2763 break;
2764 case AArch64::MOVZWi:
2765 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
2766 if (MI.getOperand(i: 1).isImm() && MI.getOperand(i: 1).getImm() == 0) {
2767 assert(MI.getDesc().getNumOperands() == 3 &&
2768 MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
2769 return true;
2770 }
2771 break;
2772 case AArch64::ANDWri: // and Rd, Rzr, #imm
2773 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2774 case AArch64::ANDXri:
2775 return MI.getOperand(i: 1).getReg() == AArch64::XZR;
2776 case TargetOpcode::COPY:
2777 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2778 }
2779 return false;
2780}
2781
2782// Return true if this instruction simply renames a general register without
2783// modifying bits.
2784bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
2785 switch (MI.getOpcode()) {
2786 default:
2787 break;
2788 case TargetOpcode::COPY: {
2789 // GPR32 copies will by lowered to ORRXrs
2790 Register DstReg = MI.getOperand(i: 0).getReg();
2791 return (AArch64::GPR32RegClass.contains(Reg: DstReg) ||
2792 AArch64::GPR64RegClass.contains(Reg: DstReg));
2793 }
2794 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
2795 if (MI.getOperand(i: 1).getReg() == AArch64::XZR) {
2796 assert(MI.getDesc().getNumOperands() == 4 &&
2797 MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
2798 return true;
2799 }
2800 break;
2801 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
2802 if (MI.getOperand(i: 2).getImm() == 0) {
2803 assert(MI.getDesc().getNumOperands() == 4 &&
2804 MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
2805 return true;
2806 }
2807 break;
2808 }
2809 return false;
2810}
2811
2812// Return true if this instruction simply renames a general register without
2813// modifying bits.
2814bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
2815 switch (MI.getOpcode()) {
2816 default:
2817 break;
2818 case TargetOpcode::COPY: {
2819 Register DstReg = MI.getOperand(i: 0).getReg();
2820 return AArch64::FPR128RegClass.contains(Reg: DstReg);
2821 }
2822 case AArch64::ORRv16i8:
2823 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
2824 assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
2825 "invalid ORRv16i8 operands");
2826 return true;
2827 }
2828 break;
2829 }
2830 return false;
2831}
2832
2833static bool isFrameLoadOpcode(int Opcode) {
2834 switch (Opcode) {
2835 default:
2836 return false;
2837 case AArch64::LDRWui:
2838 case AArch64::LDRXui:
2839 case AArch64::LDRBui:
2840 case AArch64::LDRHui:
2841 case AArch64::LDRSui:
2842 case AArch64::LDRDui:
2843 case AArch64::LDRQui:
2844 case AArch64::LDR_PXI:
2845 return true;
2846 }
2847}
2848
2849Register AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
2850 int &FrameIndex) const {
2851 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2852 return Register();
2853
2854 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2855 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2856 FrameIndex = MI.getOperand(i: 1).getIndex();
2857 return MI.getOperand(i: 0).getReg();
2858 }
2859 return Register();
2860}
2861
2862static bool isFrameStoreOpcode(int Opcode) {
2863 switch (Opcode) {
2864 default:
2865 return false;
2866 case AArch64::STRWui:
2867 case AArch64::STRXui:
2868 case AArch64::STRBui:
2869 case AArch64::STRHui:
2870 case AArch64::STRSui:
2871 case AArch64::STRDui:
2872 case AArch64::STRQui:
2873 case AArch64::STR_PXI:
2874 return true;
2875 }
2876}
2877
2878Register AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
2879 int &FrameIndex) const {
2880 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2881 return Register();
2882
2883 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2884 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2885 FrameIndex = MI.getOperand(i: 1).getIndex();
2886 return MI.getOperand(i: 0).getReg();
2887 }
2888 return Register();
2889}
2890
2891Register AArch64InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
2892 int &FrameIndex) const {
2893 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2894 return Register();
2895
2896 if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
2897 return Reg;
2898
2899 SmallVector<const MachineMemOperand *, 1> Accesses;
2900 if (hasStoreToStackSlot(MI, Accesses)) {
2901 if (Accesses.size() > 1)
2902 return Register();
2903
2904 FrameIndex =
2905 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2906 ->getFrameIndex();
2907 return MI.getOperand(i: 0).getReg();
2908 }
2909 return Register();
2910}
2911
2912Register AArch64InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
2913 int &FrameIndex) const {
2914 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2915 return Register();
2916
2917 if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
2918 return Reg;
2919
2920 SmallVector<const MachineMemOperand *, 1> Accesses;
2921 if (hasLoadFromStackSlot(MI, Accesses)) {
2922 if (Accesses.size() > 1)
2923 return Register();
2924
2925 FrameIndex =
2926 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2927 ->getFrameIndex();
2928 return MI.getOperand(i: 0).getReg();
2929 }
2930 return Register();
2931}
2932
2933/// Check all MachineMemOperands for a hint to suppress pairing.
2934bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
2935 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2936 return MMO->getFlags() & MOSuppressPair;
2937 });
2938}
2939
2940/// Set a flag on the first MachineMemOperand to suppress pairing.
2941void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
2942 if (MI.memoperands_empty())
2943 return;
2944 (*MI.memoperands_begin())->setFlags(MOSuppressPair);
2945}
2946
2947/// Check all MachineMemOperands for a hint that the load/store is strided.
2948bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
2949 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2950 return MMO->getFlags() & MOStridedAccess;
2951 });
2952}
2953
2954bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
2955 switch (Opc) {
2956 default:
2957 return false;
2958 case AArch64::STURSi:
2959 case AArch64::STRSpre:
2960 case AArch64::STURDi:
2961 case AArch64::STRDpre:
2962 case AArch64::STURQi:
2963 case AArch64::STRQpre:
2964 case AArch64::STURBBi:
2965 case AArch64::STURHHi:
2966 case AArch64::STURWi:
2967 case AArch64::STRWpre:
2968 case AArch64::STURXi:
2969 case AArch64::STRXpre:
2970 case AArch64::LDURSi:
2971 case AArch64::LDRSpre:
2972 case AArch64::LDURDi:
2973 case AArch64::LDRDpre:
2974 case AArch64::LDURQi:
2975 case AArch64::LDRQpre:
2976 case AArch64::LDURWi:
2977 case AArch64::LDRWpre:
2978 case AArch64::LDURXi:
2979 case AArch64::LDRXpre:
2980 case AArch64::LDRSWpre:
2981 case AArch64::LDURSWi:
2982 case AArch64::LDURHHi:
2983 case AArch64::LDURBBi:
2984 case AArch64::LDURSBWi:
2985 case AArch64::LDURSHWi:
2986 return true;
2987 }
2988}
2989
2990std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
2991 switch (Opc) {
2992 default: return {};
2993 case AArch64::PRFMui: return AArch64::PRFUMi;
2994 case AArch64::LDRXui: return AArch64::LDURXi;
2995 case AArch64::LDRWui: return AArch64::LDURWi;
2996 case AArch64::LDRBui: return AArch64::LDURBi;
2997 case AArch64::LDRHui: return AArch64::LDURHi;
2998 case AArch64::LDRSui: return AArch64::LDURSi;
2999 case AArch64::LDRDui: return AArch64::LDURDi;
3000 case AArch64::LDRQui: return AArch64::LDURQi;
3001 case AArch64::LDRBBui: return AArch64::LDURBBi;
3002 case AArch64::LDRHHui: return AArch64::LDURHHi;
3003 case AArch64::LDRSBXui: return AArch64::LDURSBXi;
3004 case AArch64::LDRSBWui: return AArch64::LDURSBWi;
3005 case AArch64::LDRSHXui: return AArch64::LDURSHXi;
3006 case AArch64::LDRSHWui: return AArch64::LDURSHWi;
3007 case AArch64::LDRSWui: return AArch64::LDURSWi;
3008 case AArch64::STRXui: return AArch64::STURXi;
3009 case AArch64::STRWui: return AArch64::STURWi;
3010 case AArch64::STRBui: return AArch64::STURBi;
3011 case AArch64::STRHui: return AArch64::STURHi;
3012 case AArch64::STRSui: return AArch64::STURSi;
3013 case AArch64::STRDui: return AArch64::STURDi;
3014 case AArch64::STRQui: return AArch64::STURQi;
3015 case AArch64::STRBBui: return AArch64::STURBBi;
3016 case AArch64::STRHHui: return AArch64::STURHHi;
3017 }
3018}
3019
3020unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
3021 switch (Opc) {
3022 default:
3023 llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
3024 case AArch64::ADDG:
3025 case AArch64::LDAPURBi:
3026 case AArch64::LDAPURHi:
3027 case AArch64::LDAPURi:
3028 case AArch64::LDAPURSBWi:
3029 case AArch64::LDAPURSBXi:
3030 case AArch64::LDAPURSHWi:
3031 case AArch64::LDAPURSHXi:
3032 case AArch64::LDAPURSWi:
3033 case AArch64::LDAPURXi:
3034 case AArch64::LDR_PPXI:
3035 case AArch64::LDR_PXI:
3036 case AArch64::LDR_ZXI:
3037 case AArch64::LDR_ZZXI:
3038 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
3039 case AArch64::LDR_ZZZXI:
3040 case AArch64::LDR_ZZZZXI:
3041 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
3042 case AArch64::LDRBBui:
3043 case AArch64::LDRBui:
3044 case AArch64::LDRDui:
3045 case AArch64::LDRHHui:
3046 case AArch64::LDRHui:
3047 case AArch64::LDRQui:
3048 case AArch64::LDRSBWui:
3049 case AArch64::LDRSBXui:
3050 case AArch64::LDRSHWui:
3051 case AArch64::LDRSHXui:
3052 case AArch64::LDRSui:
3053 case AArch64::LDRSWui:
3054 case AArch64::LDRWui:
3055 case AArch64::LDRXui:
3056 case AArch64::LDURBBi:
3057 case AArch64::LDURBi:
3058 case AArch64::LDURDi:
3059 case AArch64::LDURHHi:
3060 case AArch64::LDURHi:
3061 case AArch64::LDURQi:
3062 case AArch64::LDURSBWi:
3063 case AArch64::LDURSBXi:
3064 case AArch64::LDURSHWi:
3065 case AArch64::LDURSHXi:
3066 case AArch64::LDURSi:
3067 case AArch64::LDURSWi:
3068 case AArch64::LDURWi:
3069 case AArch64::LDURXi:
3070 case AArch64::PRFMui:
3071 case AArch64::PRFUMi:
3072 case AArch64::ST2Gi:
3073 case AArch64::STGi:
3074 case AArch64::STLURBi:
3075 case AArch64::STLURHi:
3076 case AArch64::STLURWi:
3077 case AArch64::STLURXi:
3078 case AArch64::StoreSwiftAsyncContext:
3079 case AArch64::STR_PPXI:
3080 case AArch64::STR_PXI:
3081 case AArch64::STR_ZXI:
3082 case AArch64::STR_ZZXI:
3083 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
3084 case AArch64::STR_ZZZXI:
3085 case AArch64::STR_ZZZZXI:
3086 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
3087 case AArch64::STRBBui:
3088 case AArch64::STRBui:
3089 case AArch64::STRDui:
3090 case AArch64::STRHHui:
3091 case AArch64::STRHui:
3092 case AArch64::STRQui:
3093 case AArch64::STRSui:
3094 case AArch64::STRWui:
3095 case AArch64::STRXui:
3096 case AArch64::STURBBi:
3097 case AArch64::STURBi:
3098 case AArch64::STURDi:
3099 case AArch64::STURHHi:
3100 case AArch64::STURHi:
3101 case AArch64::STURQi:
3102 case AArch64::STURSi:
3103 case AArch64::STURWi:
3104 case AArch64::STURXi:
3105 case AArch64::STZ2Gi:
3106 case AArch64::STZGi:
3107 case AArch64::TAGPstack:
3108 return 2;
3109 case AArch64::LD1B_D_IMM:
3110 case AArch64::LD1B_H_IMM:
3111 case AArch64::LD1B_IMM:
3112 case AArch64::LD1B_S_IMM:
3113 case AArch64::LD1D_IMM:
3114 case AArch64::LD1H_D_IMM:
3115 case AArch64::LD1H_IMM:
3116 case AArch64::LD1H_S_IMM:
3117 case AArch64::LD1RB_D_IMM:
3118 case AArch64::LD1RB_H_IMM:
3119 case AArch64::LD1RB_IMM:
3120 case AArch64::LD1RB_S_IMM:
3121 case AArch64::LD1RD_IMM:
3122 case AArch64::LD1RH_D_IMM:
3123 case AArch64::LD1RH_IMM:
3124 case AArch64::LD1RH_S_IMM:
3125 case AArch64::LD1RSB_D_IMM:
3126 case AArch64::LD1RSB_H_IMM:
3127 case AArch64::LD1RSB_S_IMM:
3128 case AArch64::LD1RSH_D_IMM:
3129 case AArch64::LD1RSH_S_IMM:
3130 case AArch64::LD1RSW_IMM:
3131 case AArch64::LD1RW_D_IMM:
3132 case AArch64::LD1RW_IMM:
3133 case AArch64::LD1SB_D_IMM:
3134 case AArch64::LD1SB_H_IMM:
3135 case AArch64::LD1SB_S_IMM:
3136 case AArch64::LD1SH_D_IMM:
3137 case AArch64::LD1SH_S_IMM:
3138 case AArch64::LD1SW_D_IMM:
3139 case AArch64::LD1W_D_IMM:
3140 case AArch64::LD1W_IMM:
3141 case AArch64::LD2B_IMM:
3142 case AArch64::LD2D_IMM:
3143 case AArch64::LD2H_IMM:
3144 case AArch64::LD2W_IMM:
3145 case AArch64::LD3B_IMM:
3146 case AArch64::LD3D_IMM:
3147 case AArch64::LD3H_IMM:
3148 case AArch64::LD3W_IMM:
3149 case AArch64::LD4B_IMM:
3150 case AArch64::LD4D_IMM:
3151 case AArch64::LD4H_IMM:
3152 case AArch64::LD4W_IMM:
3153 case AArch64::LDG:
3154 case AArch64::LDNF1B_D_IMM:
3155 case AArch64::LDNF1B_H_IMM:
3156 case AArch64::LDNF1B_IMM:
3157 case AArch64::LDNF1B_S_IMM:
3158 case AArch64::LDNF1D_IMM:
3159 case AArch64::LDNF1H_D_IMM:
3160 case AArch64::LDNF1H_IMM:
3161 case AArch64::LDNF1H_S_IMM:
3162 case AArch64::LDNF1SB_D_IMM:
3163 case AArch64::LDNF1SB_H_IMM:
3164 case AArch64::LDNF1SB_S_IMM:
3165 case AArch64::LDNF1SH_D_IMM:
3166 case AArch64::LDNF1SH_S_IMM:
3167 case AArch64::LDNF1SW_D_IMM:
3168 case AArch64::LDNF1W_D_IMM:
3169 case AArch64::LDNF1W_IMM:
3170 case AArch64::LDNPDi:
3171 case AArch64::LDNPQi:
3172 case AArch64::LDNPSi:
3173 case AArch64::LDNPWi:
3174 case AArch64::LDNPXi:
3175 case AArch64::LDNT1B_ZRI:
3176 case AArch64::LDNT1D_ZRI:
3177 case AArch64::LDNT1H_ZRI:
3178 case AArch64::LDNT1W_ZRI:
3179 case AArch64::LDPDi:
3180 case AArch64::LDPQi:
3181 case AArch64::LDPSi:
3182 case AArch64::LDPWi:
3183 case AArch64::LDPXi:
3184 case AArch64::LDRBBpost:
3185 case AArch64::LDRBBpre:
3186 case AArch64::LDRBpost:
3187 case AArch64::LDRBpre:
3188 case AArch64::LDRDpost:
3189 case AArch64::LDRDpre:
3190 case AArch64::LDRHHpost:
3191 case AArch64::LDRHHpre:
3192 case AArch64::LDRHpost:
3193 case AArch64::LDRHpre:
3194 case AArch64::LDRQpost:
3195 case AArch64::LDRQpre:
3196 case AArch64::LDRSpost:
3197 case AArch64::LDRSpre:
3198 case AArch64::LDRWpost:
3199 case AArch64::LDRWpre:
3200 case AArch64::LDRXpost:
3201 case AArch64::LDRXpre:
3202 case AArch64::ST1B_D_IMM:
3203 case AArch64::ST1B_H_IMM:
3204 case AArch64::ST1B_IMM:
3205 case AArch64::ST1B_S_IMM:
3206 case AArch64::ST1D_IMM:
3207 case AArch64::ST1H_D_IMM:
3208 case AArch64::ST1H_IMM:
3209 case AArch64::ST1H_S_IMM:
3210 case AArch64::ST1W_D_IMM:
3211 case AArch64::ST1W_IMM:
3212 case AArch64::ST2B_IMM:
3213 case AArch64::ST2D_IMM:
3214 case AArch64::ST2H_IMM:
3215 case AArch64::ST2W_IMM:
3216 case AArch64::ST3B_IMM:
3217 case AArch64::ST3D_IMM:
3218 case AArch64::ST3H_IMM:
3219 case AArch64::ST3W_IMM:
3220 case AArch64::ST4B_IMM:
3221 case AArch64::ST4D_IMM:
3222 case AArch64::ST4H_IMM:
3223 case AArch64::ST4W_IMM:
3224 case AArch64::STGPi:
3225 case AArch64::STGPreIndex:
3226 case AArch64::STZGPreIndex:
3227 case AArch64::ST2GPreIndex:
3228 case AArch64::STZ2GPreIndex:
3229 case AArch64::STGPostIndex:
3230 case AArch64::STZGPostIndex:
3231 case AArch64::ST2GPostIndex:
3232 case AArch64::STZ2GPostIndex:
3233 case AArch64::STNPDi:
3234 case AArch64::STNPQi:
3235 case AArch64::STNPSi:
3236 case AArch64::STNPWi:
3237 case AArch64::STNPXi:
3238 case AArch64::STNT1B_ZRI:
3239 case AArch64::STNT1D_ZRI:
3240 case AArch64::STNT1H_ZRI:
3241 case AArch64::STNT1W_ZRI:
3242 case AArch64::STPDi:
3243 case AArch64::STPQi:
3244 case AArch64::STPSi:
3245 case AArch64::STPWi:
3246 case AArch64::STPXi:
3247 case AArch64::STRBBpost:
3248 case AArch64::STRBBpre:
3249 case AArch64::STRBpost:
3250 case AArch64::STRBpre:
3251 case AArch64::STRDpost:
3252 case AArch64::STRDpre:
3253 case AArch64::STRHHpost:
3254 case AArch64::STRHHpre:
3255 case AArch64::STRHpost:
3256 case AArch64::STRHpre:
3257 case AArch64::STRQpost:
3258 case AArch64::STRQpre:
3259 case AArch64::STRSpost:
3260 case AArch64::STRSpre:
3261 case AArch64::STRWpost:
3262 case AArch64::STRWpre:
3263 case AArch64::STRXpost:
3264 case AArch64::STRXpre:
3265 case AArch64::LD1B_2Z_IMM:
3266 case AArch64::LD1B_2Z_STRIDED_IMM:
3267 case AArch64::LD1H_2Z_IMM:
3268 case AArch64::LD1H_2Z_STRIDED_IMM:
3269 case AArch64::LD1W_2Z_IMM:
3270 case AArch64::LD1W_2Z_STRIDED_IMM:
3271 case AArch64::LD1D_2Z_IMM:
3272 case AArch64::LD1D_2Z_STRIDED_IMM:
3273 case AArch64::LD1B_4Z_IMM:
3274 case AArch64::LD1B_4Z_STRIDED_IMM:
3275 case AArch64::LD1H_4Z_IMM:
3276 case AArch64::LD1H_4Z_STRIDED_IMM:
3277 case AArch64::LD1W_4Z_IMM:
3278 case AArch64::LD1W_4Z_STRIDED_IMM:
3279 case AArch64::LD1D_4Z_IMM:
3280 case AArch64::LD1D_4Z_STRIDED_IMM:
3281 case AArch64::LD1B_2Z_IMM_PSEUDO:
3282 case AArch64::LD1H_2Z_IMM_PSEUDO:
3283 case AArch64::LD1W_2Z_IMM_PSEUDO:
3284 case AArch64::LD1D_2Z_IMM_PSEUDO:
3285 case AArch64::LD1B_4Z_IMM_PSEUDO:
3286 case AArch64::LD1H_4Z_IMM_PSEUDO:
3287 case AArch64::LD1W_4Z_IMM_PSEUDO:
3288 case AArch64::LD1D_4Z_IMM_PSEUDO:
3289 case AArch64::ST1B_2Z_IMM:
3290 case AArch64::ST1B_2Z_STRIDED_IMM:
3291 case AArch64::ST1H_2Z_IMM:
3292 case AArch64::ST1H_2Z_STRIDED_IMM:
3293 case AArch64::ST1W_2Z_IMM:
3294 case AArch64::ST1W_2Z_STRIDED_IMM:
3295 case AArch64::ST1D_2Z_IMM:
3296 case AArch64::ST1D_2Z_STRIDED_IMM:
3297 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
3298 case AArch64::LDNT1B_2Z_IMM:
3299 case AArch64::LDNT1B_2Z_STRIDED_IMM:
3300 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
3301 case AArch64::LDNT1H_2Z_IMM:
3302 case AArch64::LDNT1H_2Z_STRIDED_IMM:
3303 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
3304 case AArch64::LDNT1W_2Z_IMM:
3305 case AArch64::LDNT1W_2Z_STRIDED_IMM:
3306 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
3307 case AArch64::LDNT1D_2Z_IMM:
3308 case AArch64::LDNT1D_2Z_STRIDED_IMM:
3309 case AArch64::STNT1B_2Z_IMM:
3310 case AArch64::STNT1B_2Z_STRIDED_IMM:
3311 case AArch64::STNT1H_2Z_IMM:
3312 case AArch64::STNT1H_2Z_STRIDED_IMM:
3313 case AArch64::STNT1W_2Z_IMM:
3314 case AArch64::STNT1W_2Z_STRIDED_IMM:
3315 case AArch64::STNT1D_2Z_IMM:
3316 case AArch64::STNT1D_2Z_STRIDED_IMM:
3317 case AArch64::ST1B_2Z_IMM_PSEUDO:
3318 case AArch64::ST1H_2Z_IMM_PSEUDO:
3319 case AArch64::ST1W_2Z_IMM_PSEUDO:
3320 case AArch64::ST1D_2Z_IMM_PSEUDO:
3321 case AArch64::STNT1B_2Z_IMM_PSEUDO:
3322 case AArch64::STNT1H_2Z_IMM_PSEUDO:
3323 case AArch64::STNT1W_2Z_IMM_PSEUDO:
3324 case AArch64::STNT1D_2Z_IMM_PSEUDO:
3325 case AArch64::ST1B_4Z_IMM:
3326 case AArch64::ST1B_4Z_STRIDED_IMM:
3327 case AArch64::ST1H_4Z_IMM:
3328 case AArch64::ST1H_4Z_STRIDED_IMM:
3329 case AArch64::ST1W_4Z_IMM:
3330 case AArch64::ST1W_4Z_STRIDED_IMM:
3331 case AArch64::ST1D_4Z_IMM:
3332 case AArch64::ST1D_4Z_STRIDED_IMM:
3333 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
3334 case AArch64::LDNT1B_4Z_IMM:
3335 case AArch64::LDNT1B_4Z_STRIDED_IMM:
3336 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
3337 case AArch64::LDNT1H_4Z_IMM:
3338 case AArch64::LDNT1H_4Z_STRIDED_IMM:
3339 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
3340 case AArch64::LDNT1W_4Z_IMM:
3341 case AArch64::LDNT1W_4Z_STRIDED_IMM:
3342 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
3343 case AArch64::LDNT1D_4Z_IMM:
3344 case AArch64::LDNT1D_4Z_STRIDED_IMM:
3345 case AArch64::STNT1B_4Z_IMM:
3346 case AArch64::STNT1B_4Z_STRIDED_IMM:
3347 case AArch64::STNT1H_4Z_IMM:
3348 case AArch64::STNT1H_4Z_STRIDED_IMM:
3349 case AArch64::STNT1W_4Z_IMM:
3350 case AArch64::STNT1W_4Z_STRIDED_IMM:
3351 case AArch64::STNT1D_4Z_IMM:
3352 case AArch64::STNT1D_4Z_STRIDED_IMM:
3353 case AArch64::ST1B_4Z_IMM_PSEUDO:
3354 case AArch64::ST1H_4Z_IMM_PSEUDO:
3355 case AArch64::ST1W_4Z_IMM_PSEUDO:
3356 case AArch64::ST1D_4Z_IMM_PSEUDO:
3357 case AArch64::STNT1B_4Z_IMM_PSEUDO:
3358 case AArch64::STNT1H_4Z_IMM_PSEUDO:
3359 case AArch64::STNT1W_4Z_IMM_PSEUDO:
3360 case AArch64::STNT1D_4Z_IMM_PSEUDO:
3361 return 3;
3362 case AArch64::LDPDpost:
3363 case AArch64::LDPDpre:
3364 case AArch64::LDPQpost:
3365 case AArch64::LDPQpre:
3366 case AArch64::LDPSpost:
3367 case AArch64::LDPSpre:
3368 case AArch64::LDPWpost:
3369 case AArch64::LDPWpre:
3370 case AArch64::LDPXpost:
3371 case AArch64::LDPXpre:
3372 case AArch64::STGPpre:
3373 case AArch64::STGPpost:
3374 case AArch64::STPDpost:
3375 case AArch64::STPDpre:
3376 case AArch64::STPQpost:
3377 case AArch64::STPQpre:
3378 case AArch64::STPSpost:
3379 case AArch64::STPSpre:
3380 case AArch64::STPWpost:
3381 case AArch64::STPWpre:
3382 case AArch64::STPXpost:
3383 case AArch64::STPXpre:
3384 return 4;
3385 }
3386}
3387
3388bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
3389 switch (MI.getOpcode()) {
3390 default:
3391 return false;
3392 // Scaled instructions.
3393 case AArch64::STRSui:
3394 case AArch64::STRDui:
3395 case AArch64::STRQui:
3396 case AArch64::STRXui:
3397 case AArch64::STRWui:
3398 case AArch64::LDRSui:
3399 case AArch64::LDRDui:
3400 case AArch64::LDRQui:
3401 case AArch64::LDRXui:
3402 case AArch64::LDRWui:
3403 case AArch64::LDRSWui:
3404 // Unscaled instructions.
3405 case AArch64::STURSi:
3406 case AArch64::STRSpre:
3407 case AArch64::STURDi:
3408 case AArch64::STRDpre:
3409 case AArch64::STURQi:
3410 case AArch64::STRQpre:
3411 case AArch64::STURWi:
3412 case AArch64::STRWpre:
3413 case AArch64::STURXi:
3414 case AArch64::STRXpre:
3415 case AArch64::LDURSi:
3416 case AArch64::LDRSpre:
3417 case AArch64::LDURDi:
3418 case AArch64::LDRDpre:
3419 case AArch64::LDURQi:
3420 case AArch64::LDRQpre:
3421 case AArch64::LDURWi:
3422 case AArch64::LDRWpre:
3423 case AArch64::LDURXi:
3424 case AArch64::LDRXpre:
3425 case AArch64::LDURSWi:
3426 case AArch64::LDRSWpre:
3427 // SVE instructions.
3428 case AArch64::LDR_ZXI:
3429 case AArch64::STR_ZXI:
3430 return true;
3431 }
3432}
3433
3434bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
3435 switch (MI.getOpcode()) {
3436 default:
3437 assert((!MI.isCall() || !MI.isReturn()) &&
3438 "Unexpected instruction - was a new tail call opcode introduced?");
3439 return false;
3440 case AArch64::TCRETURNdi:
3441 case AArch64::TCRETURNri:
3442 case AArch64::TCRETURNrix16x17:
3443 case AArch64::TCRETURNrix17:
3444 case AArch64::TCRETURNrinotx16:
3445 case AArch64::TCRETURNriALL:
3446 case AArch64::AUTH_TCRETURN:
3447 case AArch64::AUTH_TCRETURN_BTI:
3448 return true;
3449 }
3450}
3451
3452unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) {
3453 switch (Opc) {
3454 default:
3455 llvm_unreachable("Opcode has no flag setting equivalent!");
3456 // 32-bit cases:
3457 case AArch64::ADDWri:
3458 return AArch64::ADDSWri;
3459 case AArch64::ADDWrr:
3460 return AArch64::ADDSWrr;
3461 case AArch64::ADDWrs:
3462 return AArch64::ADDSWrs;
3463 case AArch64::ADDWrx:
3464 return AArch64::ADDSWrx;
3465 case AArch64::ANDWri:
3466 return AArch64::ANDSWri;
3467 case AArch64::ANDWrr:
3468 return AArch64::ANDSWrr;
3469 case AArch64::ANDWrs:
3470 return AArch64::ANDSWrs;
3471 case AArch64::BICWrr:
3472 return AArch64::BICSWrr;
3473 case AArch64::BICWrs:
3474 return AArch64::BICSWrs;
3475 case AArch64::SUBWri:
3476 return AArch64::SUBSWri;
3477 case AArch64::SUBWrr:
3478 return AArch64::SUBSWrr;
3479 case AArch64::SUBWrs:
3480 return AArch64::SUBSWrs;
3481 case AArch64::SUBWrx:
3482 return AArch64::SUBSWrx;
3483 // 64-bit cases:
3484 case AArch64::ADDXri:
3485 return AArch64::ADDSXri;
3486 case AArch64::ADDXrr:
3487 return AArch64::ADDSXrr;
3488 case AArch64::ADDXrs:
3489 return AArch64::ADDSXrs;
3490 case AArch64::ADDXrx:
3491 return AArch64::ADDSXrx;
3492 case AArch64::ANDXri:
3493 return AArch64::ANDSXri;
3494 case AArch64::ANDXrr:
3495 return AArch64::ANDSXrr;
3496 case AArch64::ANDXrs:
3497 return AArch64::ANDSXrs;
3498 case AArch64::BICXrr:
3499 return AArch64::BICSXrr;
3500 case AArch64::BICXrs:
3501 return AArch64::BICSXrs;
3502 case AArch64::SUBXri:
3503 return AArch64::SUBSXri;
3504 case AArch64::SUBXrr:
3505 return AArch64::SUBSXrr;
3506 case AArch64::SUBXrs:
3507 return AArch64::SUBSXrs;
3508 case AArch64::SUBXrx:
3509 return AArch64::SUBSXrx;
3510 // SVE instructions:
3511 case AArch64::AND_PPzPP:
3512 return AArch64::ANDS_PPzPP;
3513 case AArch64::BIC_PPzPP:
3514 return AArch64::BICS_PPzPP;
3515 case AArch64::EOR_PPzPP:
3516 return AArch64::EORS_PPzPP;
3517 case AArch64::NAND_PPzPP:
3518 return AArch64::NANDS_PPzPP;
3519 case AArch64::NOR_PPzPP:
3520 return AArch64::NORS_PPzPP;
3521 case AArch64::ORN_PPzPP:
3522 return AArch64::ORNS_PPzPP;
3523 case AArch64::ORR_PPzPP:
3524 return AArch64::ORRS_PPzPP;
3525 case AArch64::BRKA_PPzP:
3526 return AArch64::BRKAS_PPzP;
3527 case AArch64::BRKPA_PPzPP:
3528 return AArch64::BRKPAS_PPzPP;
3529 case AArch64::BRKB_PPzP:
3530 return AArch64::BRKBS_PPzP;
3531 case AArch64::BRKPB_PPzPP:
3532 return AArch64::BRKPBS_PPzPP;
3533 case AArch64::BRKN_PPzP:
3534 return AArch64::BRKNS_PPzP;
3535 case AArch64::RDFFR_PPz:
3536 return AArch64::RDFFRS_PPz;
3537 case AArch64::PTRUE_B:
3538 return AArch64::PTRUES_B;
3539 }
3540}
3541
3542// Is this a candidate for ld/st merging or pairing? For example, we don't
3543// touch volatiles or load/stores that have a hint to avoid pair formation.
3544bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
3545
3546 bool IsPreLdSt = isPreLdSt(MI);
3547
3548 // If this is a volatile load/store, don't mess with it.
3549 if (MI.hasOrderedMemoryRef())
3550 return false;
3551
3552 // Make sure this is a reg/fi+imm (as opposed to an address reloc).
3553 // For Pre-inc LD/ST, the operand is shifted by one.
3554 assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
3555 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
3556 "Expected a reg or frame index operand.");
3557
3558 // For Pre-indexed addressing quadword instructions, the third operand is the
3559 // immediate value.
3560 bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(i: 3).isImm();
3561
3562 if (!MI.getOperand(i: 2).isImm() && !IsImmPreLdSt)
3563 return false;
3564
3565 // Can't merge/pair if the instruction modifies the base register.
3566 // e.g., ldr x0, [x0]
3567 // This case will never occur with an FI base.
3568 // However, if the instruction is an LDR<S,D,Q,W,X,SW>pre or
3569 // STR<S,D,Q,W,X>pre, it can be merged.
3570 // For example:
3571 // ldr q0, [x11, #32]!
3572 // ldr q1, [x11, #16]
3573 // to
3574 // ldp q0, q1, [x11, #32]!
3575 if (MI.getOperand(i: 1).isReg() && !IsPreLdSt) {
3576 Register BaseReg = MI.getOperand(i: 1).getReg();
3577 const TargetRegisterInfo *TRI = &getRegisterInfo();
3578 if (MI.modifiesRegister(Reg: BaseReg, TRI))
3579 return false;
3580 }
3581
3582 // Pairing SVE fills/spills is only valid for little-endian targets that
3583 // implement VLS 128.
3584 switch (MI.getOpcode()) {
3585 default:
3586 break;
3587 case AArch64::LDR_ZXI:
3588 case AArch64::STR_ZXI:
3589 if (!Subtarget.isLittleEndian() ||
3590 Subtarget.getSVEVectorSizeInBits() != 128)
3591 return false;
3592 }
3593
3594 // Check if this load/store has a hint to avoid pair formation.
3595 // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
3596 if (isLdStPairSuppressed(MI))
3597 return false;
3598
3599 // Do not pair any callee-save store/reload instructions in the
3600 // prologue/epilogue if the CFI information encoded the operations as separate
3601 // instructions, as that will cause the size of the actual prologue to mismatch
3602 // with the prologue size recorded in the Windows CFI.
3603 const MCAsmInfo &MAI = MI.getMF()->getTarget().getMCAsmInfo();
3604 bool NeedsWinCFI =
3605 MAI.usesWindowsCFI() && MI.getMF()->getFunction().needsUnwindTableEntry();
3606 if (NeedsWinCFI && (MI.getFlag(Flag: MachineInstr::FrameSetup) ||
3607 MI.getFlag(Flag: MachineInstr::FrameDestroy)))
3608 return false;
3609
3610 // On some CPUs quad load/store pairs are slower than two single load/stores.
3611 if (Subtarget.isPaired128Slow()) {
3612 switch (MI.getOpcode()) {
3613 default:
3614 break;
3615 case AArch64::LDURQi:
3616 case AArch64::STURQi:
3617 case AArch64::LDRQui:
3618 case AArch64::STRQui:
3619 return false;
3620 }
3621 }
3622
3623 return true;
3624}
3625
3626bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
3627 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3628 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
3629 const TargetRegisterInfo *TRI) const {
3630 if (!LdSt.mayLoadOrStore())
3631 return false;
3632
3633 const MachineOperand *BaseOp;
3634 TypeSize WidthN(0, false);
3635 if (!getMemOperandWithOffsetWidth(MI: LdSt, BaseOp, Offset, OffsetIsScalable,
3636 Width&: WidthN, TRI))
3637 return false;
3638 // The maximum vscale is 16 under AArch64, return the maximal extent for the
3639 // vector.
3640 Width = LocationSize::precise(Value: WidthN);
3641 BaseOps.push_back(Elt: BaseOp);
3642 return true;
3643}
3644
3645std::optional<ExtAddrMode>
3646AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
3647 const TargetRegisterInfo *TRI) const {
3648 const MachineOperand *Base; // Filled with the base operand of MI.
3649 int64_t Offset; // Filled with the offset of MI.
3650 bool OffsetIsScalable;
3651 if (!getMemOperandWithOffset(MI: MemI, BaseOp&: Base, Offset, OffsetIsScalable, TRI))
3652 return std::nullopt;
3653
3654 if (!Base->isReg())
3655 return std::nullopt;
3656 ExtAddrMode AM;
3657 AM.BaseReg = Base->getReg();
3658 AM.Displacement = Offset;
3659 AM.ScaledReg = 0;
3660 AM.Scale = 0;
3661 return AM;
3662}
3663
3664bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
3665 Register Reg,
3666 const MachineInstr &AddrI,
3667 ExtAddrMode &AM) const {
3668 // Filter out instructions into which we cannot fold.
3669 unsigned NumBytes;
3670 int64_t OffsetScale = 1;
3671 switch (MemI.getOpcode()) {
3672 default:
3673 return false;
3674
3675 case AArch64::LDURQi:
3676 case AArch64::STURQi:
3677 NumBytes = 16;
3678 break;
3679
3680 case AArch64::LDURDi:
3681 case AArch64::STURDi:
3682 case AArch64::LDURXi:
3683 case AArch64::STURXi:
3684 NumBytes = 8;
3685 break;
3686
3687 case AArch64::LDURWi:
3688 case AArch64::LDURSWi:
3689 case AArch64::STURWi:
3690 NumBytes = 4;
3691 break;
3692
3693 case AArch64::LDURHi:
3694 case AArch64::STURHi:
3695 case AArch64::LDURHHi:
3696 case AArch64::STURHHi:
3697 case AArch64::LDURSHXi:
3698 case AArch64::LDURSHWi:
3699 NumBytes = 2;
3700 break;
3701
3702 case AArch64::LDRBroX:
3703 case AArch64::LDRBBroX:
3704 case AArch64::LDRSBXroX:
3705 case AArch64::LDRSBWroX:
3706 case AArch64::STRBroX:
3707 case AArch64::STRBBroX:
3708 case AArch64::LDURBi:
3709 case AArch64::LDURBBi:
3710 case AArch64::LDURSBXi:
3711 case AArch64::LDURSBWi:
3712 case AArch64::STURBi:
3713 case AArch64::STURBBi:
3714 case AArch64::LDRBui:
3715 case AArch64::LDRBBui:
3716 case AArch64::LDRSBXui:
3717 case AArch64::LDRSBWui:
3718 case AArch64::STRBui:
3719 case AArch64::STRBBui:
3720 NumBytes = 1;
3721 break;
3722
3723 case AArch64::LDRQroX:
3724 case AArch64::STRQroX:
3725 case AArch64::LDRQui:
3726 case AArch64::STRQui:
3727 NumBytes = 16;
3728 OffsetScale = 16;
3729 break;
3730
3731 case AArch64::LDRDroX:
3732 case AArch64::STRDroX:
3733 case AArch64::LDRXroX:
3734 case AArch64::STRXroX:
3735 case AArch64::LDRDui:
3736 case AArch64::STRDui:
3737 case AArch64::LDRXui:
3738 case AArch64::STRXui:
3739 NumBytes = 8;
3740 OffsetScale = 8;
3741 break;
3742
3743 case AArch64::LDRWroX:
3744 case AArch64::LDRSWroX:
3745 case AArch64::STRWroX:
3746 case AArch64::LDRWui:
3747 case AArch64::LDRSWui:
3748 case AArch64::STRWui:
3749 NumBytes = 4;
3750 OffsetScale = 4;
3751 break;
3752
3753 case AArch64::LDRHroX:
3754 case AArch64::STRHroX:
3755 case AArch64::LDRHHroX:
3756 case AArch64::STRHHroX:
3757 case AArch64::LDRSHXroX:
3758 case AArch64::LDRSHWroX:
3759 case AArch64::LDRHui:
3760 case AArch64::STRHui:
3761 case AArch64::LDRHHui:
3762 case AArch64::STRHHui:
3763 case AArch64::LDRSHXui:
3764 case AArch64::LDRSHWui:
3765 NumBytes = 2;
3766 OffsetScale = 2;
3767 break;
3768 }
3769
3770 // Check the fold operand is not the loaded/stored value.
3771 const MachineOperand &BaseRegOp = MemI.getOperand(i: 0);
3772 if (BaseRegOp.isReg() && BaseRegOp.getReg() == Reg)
3773 return false;
3774
3775 // Handle memory instructions with a [Reg, Reg] addressing mode.
3776 if (MemI.getOperand(i: 2).isReg()) {
3777 // Bail if the addressing mode already includes extension of the offset
3778 // register.
3779 if (MemI.getOperand(i: 3).getImm())
3780 return false;
3781
3782 // Check if we actually have a scaled offset.
3783 if (MemI.getOperand(i: 4).getImm() == 0)
3784 OffsetScale = 1;
3785
3786 // If the address instructions is folded into the base register, then the
3787 // addressing mode must not have a scale. Then we can swap the base and the
3788 // scaled registers.
3789 if (MemI.getOperand(i: 1).getReg() == Reg && OffsetScale != 1)
3790 return false;
3791
3792 switch (AddrI.getOpcode()) {
3793 default:
3794 return false;
3795
3796 case AArch64::SBFMXri:
3797 // sxtw Xa, Wm
3798 // ldr Xd, [Xn, Xa, lsl #N]
3799 // ->
3800 // ldr Xd, [Xn, Wm, sxtw #N]
3801 if (AddrI.getOperand(i: 2).getImm() != 0 ||
3802 AddrI.getOperand(i: 3).getImm() != 31)
3803 return false;
3804
3805 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3806 if (AM.BaseReg == Reg)
3807 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3808 AM.ScaledReg = AddrI.getOperand(i: 1).getReg();
3809 AM.Scale = OffsetScale;
3810 AM.Displacement = 0;
3811 AM.Form = ExtAddrMode::Formula::SExtScaledReg;
3812 return true;
3813
3814 case TargetOpcode::SUBREG_TO_REG: {
3815 // mov Wa, Wm
3816 // ldr Xd, [Xn, Xa, lsl #N]
3817 // ->
3818 // ldr Xd, [Xn, Wm, uxtw #N]
3819
3820 // Zero-extension looks like an ORRWrs followed by a SUBREG_TO_REG.
3821 if (AddrI.getOperand(i: 2).getImm() != AArch64::sub_32)
3822 return false;
3823
3824 const MachineRegisterInfo &MRI = AddrI.getMF()->getRegInfo();
3825 Register OffsetReg = AddrI.getOperand(i: 1).getReg();
3826 if (!OffsetReg.isVirtual() || !MRI.hasOneNonDBGUse(RegNo: OffsetReg))
3827 return false;
3828
3829 const MachineInstr &DefMI = *MRI.getVRegDef(Reg: OffsetReg);
3830 if (DefMI.getOpcode() != AArch64::ORRWrs ||
3831 DefMI.getOperand(i: 1).getReg() != AArch64::WZR ||
3832 DefMI.getOperand(i: 3).getImm() != 0)
3833 return false;
3834
3835 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3836 if (AM.BaseReg == Reg)
3837 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3838 AM.ScaledReg = DefMI.getOperand(i: 2).getReg();
3839 AM.Scale = OffsetScale;
3840 AM.Displacement = 0;
3841 AM.Form = ExtAddrMode::Formula::ZExtScaledReg;
3842 return true;
3843 }
3844 }
3845 }
3846
3847 // Handle memory instructions with a [Reg, #Imm] addressing mode.
3848
3849 // Check we are not breaking a potential conversion to an LDP.
3850 auto validateOffsetForLDP = [](unsigned NumBytes, int64_t OldOffset,
3851 int64_t NewOffset) -> bool {
3852 int64_t MinOffset, MaxOffset;
3853 switch (NumBytes) {
3854 default:
3855 return true;
3856 case 4:
3857 MinOffset = -256;
3858 MaxOffset = 252;
3859 break;
3860 case 8:
3861 MinOffset = -512;
3862 MaxOffset = 504;
3863 break;
3864 case 16:
3865 MinOffset = -1024;
3866 MaxOffset = 1008;
3867 break;
3868 }
3869 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3870 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3871 };
3872 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) -> bool {
3873 int64_t OldOffset = MemI.getOperand(i: 2).getImm() * OffsetScale;
3874 int64_t NewOffset = OldOffset + Disp;
3875 if (!isLegalAddressingMode(NumBytes, Offset: NewOffset, /* Scale */ 0))
3876 return false;
3877 // If the old offset would fit into an LDP, but the new offset wouldn't,
3878 // bail out.
3879 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3880 return false;
3881 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3882 AM.ScaledReg = 0;
3883 AM.Scale = 0;
3884 AM.Displacement = NewOffset;
3885 AM.Form = ExtAddrMode::Formula::Basic;
3886 return true;
3887 };
3888
3889 auto canFoldAddRegIntoAddrMode =
3890 [&](int64_t Scale,
3891 ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool {
3892 if (MemI.getOperand(i: 2).getImm() != 0)
3893 return false;
3894 if ((unsigned)Scale != Scale)
3895 return false;
3896 if (!isLegalAddressingMode(NumBytes, /* Offset */ 0, Scale))
3897 return false;
3898 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3899 AM.ScaledReg = AddrI.getOperand(i: 2).getReg();
3900 AM.Scale = Scale;
3901 AM.Displacement = 0;
3902 AM.Form = Form;
3903 return true;
3904 };
3905
3906 auto avoidSlowSTRQ = [&](const MachineInstr &MemI) {
3907 unsigned Opcode = MemI.getOpcode();
3908 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3909 Subtarget.isSTRQroSlow();
3910 };
3911
3912 int64_t Disp = 0;
3913 const bool OptSize = MemI.getMF()->getFunction().hasOptSize();
3914 switch (AddrI.getOpcode()) {
3915 default:
3916 return false;
3917
3918 case AArch64::ADDXri:
3919 // add Xa, Xn, #N
3920 // ldr Xd, [Xa, #M]
3921 // ->
3922 // ldr Xd, [Xn, #N'+M]
3923 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3924 return canFoldAddSubImmIntoAddrMode(Disp);
3925
3926 case AArch64::SUBXri:
3927 // sub Xa, Xn, #N
3928 // ldr Xd, [Xa, #M]
3929 // ->
3930 // ldr Xd, [Xn, #N'+M]
3931 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3932 return canFoldAddSubImmIntoAddrMode(-Disp);
3933
3934 case AArch64::ADDXrs: {
3935 // add Xa, Xn, Xm, lsl #N
3936 // ldr Xd, [Xa]
3937 // ->
3938 // ldr Xd, [Xn, Xm, lsl #N]
3939
3940 // Don't fold the add if the result would be slower, unless optimising for
3941 // size.
3942 unsigned Shift = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
3943 if (AArch64_AM::getShiftType(Imm: Shift) != AArch64_AM::ShiftExtendType::LSL)
3944 return false;
3945 Shift = AArch64_AM::getShiftValue(Imm: Shift);
3946 if (!OptSize) {
3947 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3948 return false;
3949 if (avoidSlowSTRQ(MemI))
3950 return false;
3951 }
3952 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3953 }
3954
3955 case AArch64::ADDXrr:
3956 // add Xa, Xn, Xm
3957 // ldr Xd, [Xa]
3958 // ->
3959 // ldr Xd, [Xn, Xm, lsl #0]
3960
3961 // Don't fold the add if the result would be slower, unless optimising for
3962 // size.
3963 if (!OptSize && avoidSlowSTRQ(MemI))
3964 return false;
3965 return canFoldAddRegIntoAddrMode(1);
3966
3967 case AArch64::ADDXrx:
3968 // add Xa, Xn, Wm, {s,u}xtw #N
3969 // ldr Xd, [Xa]
3970 // ->
3971 // ldr Xd, [Xn, Wm, {s,u}xtw #N]
3972
3973 // Don't fold the add if the result would be slower, unless optimising for
3974 // size.
3975 if (!OptSize && avoidSlowSTRQ(MemI))
3976 return false;
3977
3978 // Can fold only sign-/zero-extend of a word.
3979 unsigned Imm = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
3980 AArch64_AM::ShiftExtendType Extend = AArch64_AM::getArithExtendType(Imm);
3981 if (Extend != AArch64_AM::UXTW && Extend != AArch64_AM::SXTW)
3982 return false;
3983
3984 return canFoldAddRegIntoAddrMode(
3985 1ULL << AArch64_AM::getArithShiftValue(Imm),
3986 (Extend == AArch64_AM::SXTW) ? ExtAddrMode::Formula::SExtScaledReg
3987 : ExtAddrMode::Formula::ZExtScaledReg);
3988 }
3989}
3990
3991// Given an opcode for an instruction with a [Reg, #Imm] addressing mode,
3992// return the opcode of an instruction performing the same operation, but using
3993// the [Reg, Reg] addressing mode.
3994static unsigned regOffsetOpcode(unsigned Opcode) {
3995 switch (Opcode) {
3996 default:
3997 llvm_unreachable("Address folding not implemented for instruction");
3998
3999 case AArch64::LDURQi:
4000 case AArch64::LDRQui:
4001 return AArch64::LDRQroX;
4002 case AArch64::STURQi:
4003 case AArch64::STRQui:
4004 return AArch64::STRQroX;
4005 case AArch64::LDURDi:
4006 case AArch64::LDRDui:
4007 return AArch64::LDRDroX;
4008 case AArch64::STURDi:
4009 case AArch64::STRDui:
4010 return AArch64::STRDroX;
4011 case AArch64::LDURXi:
4012 case AArch64::LDRXui:
4013 return AArch64::LDRXroX;
4014 case AArch64::STURXi:
4015 case AArch64::STRXui:
4016 return AArch64::STRXroX;
4017 case AArch64::LDURWi:
4018 case AArch64::LDRWui:
4019 return AArch64::LDRWroX;
4020 case AArch64::LDURSWi:
4021 case AArch64::LDRSWui:
4022 return AArch64::LDRSWroX;
4023 case AArch64::STURWi:
4024 case AArch64::STRWui:
4025 return AArch64::STRWroX;
4026 case AArch64::LDURHi:
4027 case AArch64::LDRHui:
4028 return AArch64::LDRHroX;
4029 case AArch64::STURHi:
4030 case AArch64::STRHui:
4031 return AArch64::STRHroX;
4032 case AArch64::LDURHHi:
4033 case AArch64::LDRHHui:
4034 return AArch64::LDRHHroX;
4035 case AArch64::STURHHi:
4036 case AArch64::STRHHui:
4037 return AArch64::STRHHroX;
4038 case AArch64::LDURSHXi:
4039 case AArch64::LDRSHXui:
4040 return AArch64::LDRSHXroX;
4041 case AArch64::LDURSHWi:
4042 case AArch64::LDRSHWui:
4043 return AArch64::LDRSHWroX;
4044 case AArch64::LDURBi:
4045 case AArch64::LDRBui:
4046 return AArch64::LDRBroX;
4047 case AArch64::LDURBBi:
4048 case AArch64::LDRBBui:
4049 return AArch64::LDRBBroX;
4050 case AArch64::LDURSBXi:
4051 case AArch64::LDRSBXui:
4052 return AArch64::LDRSBXroX;
4053 case AArch64::LDURSBWi:
4054 case AArch64::LDRSBWui:
4055 return AArch64::LDRSBWroX;
4056 case AArch64::STURBi:
4057 case AArch64::STRBui:
4058 return AArch64::STRBroX;
4059 case AArch64::STURBBi:
4060 case AArch64::STRBBui:
4061 return AArch64::STRBBroX;
4062 }
4063}
4064
4065// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4066// the opcode of an instruction performing the same operation, but using the
4067// [Reg, #Imm] addressing mode with scaled offset.
4068unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale) {
4069 switch (Opcode) {
4070 default:
4071 llvm_unreachable("Address folding not implemented for instruction");
4072
4073 case AArch64::LDURQi:
4074 Scale = 16;
4075 return AArch64::LDRQui;
4076 case AArch64::STURQi:
4077 Scale = 16;
4078 return AArch64::STRQui;
4079 case AArch64::LDURDi:
4080 Scale = 8;
4081 return AArch64::LDRDui;
4082 case AArch64::STURDi:
4083 Scale = 8;
4084 return AArch64::STRDui;
4085 case AArch64::LDURXi:
4086 Scale = 8;
4087 return AArch64::LDRXui;
4088 case AArch64::STURXi:
4089 Scale = 8;
4090 return AArch64::STRXui;
4091 case AArch64::LDURWi:
4092 Scale = 4;
4093 return AArch64::LDRWui;
4094 case AArch64::LDURSWi:
4095 Scale = 4;
4096 return AArch64::LDRSWui;
4097 case AArch64::STURWi:
4098 Scale = 4;
4099 return AArch64::STRWui;
4100 case AArch64::LDURHi:
4101 Scale = 2;
4102 return AArch64::LDRHui;
4103 case AArch64::STURHi:
4104 Scale = 2;
4105 return AArch64::STRHui;
4106 case AArch64::LDURHHi:
4107 Scale = 2;
4108 return AArch64::LDRHHui;
4109 case AArch64::STURHHi:
4110 Scale = 2;
4111 return AArch64::STRHHui;
4112 case AArch64::LDURSHXi:
4113 Scale = 2;
4114 return AArch64::LDRSHXui;
4115 case AArch64::LDURSHWi:
4116 Scale = 2;
4117 return AArch64::LDRSHWui;
4118 case AArch64::LDURBi:
4119 Scale = 1;
4120 return AArch64::LDRBui;
4121 case AArch64::LDURBBi:
4122 Scale = 1;
4123 return AArch64::LDRBBui;
4124 case AArch64::LDURSBXi:
4125 Scale = 1;
4126 return AArch64::LDRSBXui;
4127 case AArch64::LDURSBWi:
4128 Scale = 1;
4129 return AArch64::LDRSBWui;
4130 case AArch64::STURBi:
4131 Scale = 1;
4132 return AArch64::STRBui;
4133 case AArch64::STURBBi:
4134 Scale = 1;
4135 return AArch64::STRBBui;
4136 case AArch64::LDRQui:
4137 case AArch64::STRQui:
4138 Scale = 16;
4139 return Opcode;
4140 case AArch64::LDRDui:
4141 case AArch64::STRDui:
4142 case AArch64::LDRXui:
4143 case AArch64::STRXui:
4144 Scale = 8;
4145 return Opcode;
4146 case AArch64::LDRWui:
4147 case AArch64::LDRSWui:
4148 case AArch64::STRWui:
4149 Scale = 4;
4150 return Opcode;
4151 case AArch64::LDRHui:
4152 case AArch64::STRHui:
4153 case AArch64::LDRHHui:
4154 case AArch64::STRHHui:
4155 case AArch64::LDRSHXui:
4156 case AArch64::LDRSHWui:
4157 Scale = 2;
4158 return Opcode;
4159 case AArch64::LDRBui:
4160 case AArch64::LDRBBui:
4161 case AArch64::LDRSBXui:
4162 case AArch64::LDRSBWui:
4163 case AArch64::STRBui:
4164 case AArch64::STRBBui:
4165 Scale = 1;
4166 return Opcode;
4167 }
4168}
4169
4170// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4171// the opcode of an instruction performing the same operation, but using the
4172// [Reg, #Imm] addressing mode with unscaled offset.
4173unsigned unscaledOffsetOpcode(unsigned Opcode) {
4174 switch (Opcode) {
4175 default:
4176 llvm_unreachable("Address folding not implemented for instruction");
4177
4178 case AArch64::LDURQi:
4179 case AArch64::STURQi:
4180 case AArch64::LDURDi:
4181 case AArch64::STURDi:
4182 case AArch64::LDURXi:
4183 case AArch64::STURXi:
4184 case AArch64::LDURWi:
4185 case AArch64::LDURSWi:
4186 case AArch64::STURWi:
4187 case AArch64::LDURHi:
4188 case AArch64::STURHi:
4189 case AArch64::LDURHHi:
4190 case AArch64::STURHHi:
4191 case AArch64::LDURSHXi:
4192 case AArch64::LDURSHWi:
4193 case AArch64::LDURBi:
4194 case AArch64::STURBi:
4195 case AArch64::LDURBBi:
4196 case AArch64::STURBBi:
4197 case AArch64::LDURSBWi:
4198 case AArch64::LDURSBXi:
4199 return Opcode;
4200 case AArch64::LDRQui:
4201 return AArch64::LDURQi;
4202 case AArch64::STRQui:
4203 return AArch64::STURQi;
4204 case AArch64::LDRDui:
4205 return AArch64::LDURDi;
4206 case AArch64::STRDui:
4207 return AArch64::STURDi;
4208 case AArch64::LDRXui:
4209 return AArch64::LDURXi;
4210 case AArch64::STRXui:
4211 return AArch64::STURXi;
4212 case AArch64::LDRWui:
4213 return AArch64::LDURWi;
4214 case AArch64::LDRSWui:
4215 return AArch64::LDURSWi;
4216 case AArch64::STRWui:
4217 return AArch64::STURWi;
4218 case AArch64::LDRHui:
4219 return AArch64::LDURHi;
4220 case AArch64::STRHui:
4221 return AArch64::STURHi;
4222 case AArch64::LDRHHui:
4223 return AArch64::LDURHHi;
4224 case AArch64::STRHHui:
4225 return AArch64::STURHHi;
4226 case AArch64::LDRSHXui:
4227 return AArch64::LDURSHXi;
4228 case AArch64::LDRSHWui:
4229 return AArch64::LDURSHWi;
4230 case AArch64::LDRBBui:
4231 return AArch64::LDURBBi;
4232 case AArch64::LDRBui:
4233 return AArch64::LDURBi;
4234 case AArch64::STRBBui:
4235 return AArch64::STURBBi;
4236 case AArch64::STRBui:
4237 return AArch64::STURBi;
4238 case AArch64::LDRSBWui:
4239 return AArch64::LDURSBWi;
4240 case AArch64::LDRSBXui:
4241 return AArch64::LDURSBXi;
4242 }
4243}
4244
4245// Given the opcode of a memory load/store instruction, return the opcode of an
4246// instruction performing the same operation, but using
4247// the [Reg, Reg, {s,u}xtw #N] addressing mode with sign-/zero-extend of the
4248// offset register.
4249static unsigned offsetExtendOpcode(unsigned Opcode) {
4250 switch (Opcode) {
4251 default:
4252 llvm_unreachable("Address folding not implemented for instruction");
4253
4254 case AArch64::LDRQroX:
4255 case AArch64::LDURQi:
4256 case AArch64::LDRQui:
4257 return AArch64::LDRQroW;
4258 case AArch64::STRQroX:
4259 case AArch64::STURQi:
4260 case AArch64::STRQui:
4261 return AArch64::STRQroW;
4262 case AArch64::LDRDroX:
4263 case AArch64::LDURDi:
4264 case AArch64::LDRDui:
4265 return AArch64::LDRDroW;
4266 case AArch64::STRDroX:
4267 case AArch64::STURDi:
4268 case AArch64::STRDui:
4269 return AArch64::STRDroW;
4270 case AArch64::LDRXroX:
4271 case AArch64::LDURXi:
4272 case AArch64::LDRXui:
4273 return AArch64::LDRXroW;
4274 case AArch64::STRXroX:
4275 case AArch64::STURXi:
4276 case AArch64::STRXui:
4277 return AArch64::STRXroW;
4278 case AArch64::LDRWroX:
4279 case AArch64::LDURWi:
4280 case AArch64::LDRWui:
4281 return AArch64::LDRWroW;
4282 case AArch64::LDRSWroX:
4283 case AArch64::LDURSWi:
4284 case AArch64::LDRSWui:
4285 return AArch64::LDRSWroW;
4286 case AArch64::STRWroX:
4287 case AArch64::STURWi:
4288 case AArch64::STRWui:
4289 return AArch64::STRWroW;
4290 case AArch64::LDRHroX:
4291 case AArch64::LDURHi:
4292 case AArch64::LDRHui:
4293 return AArch64::LDRHroW;
4294 case AArch64::STRHroX:
4295 case AArch64::STURHi:
4296 case AArch64::STRHui:
4297 return AArch64::STRHroW;
4298 case AArch64::LDRHHroX:
4299 case AArch64::LDURHHi:
4300 case AArch64::LDRHHui:
4301 return AArch64::LDRHHroW;
4302 case AArch64::STRHHroX:
4303 case AArch64::STURHHi:
4304 case AArch64::STRHHui:
4305 return AArch64::STRHHroW;
4306 case AArch64::LDRSHXroX:
4307 case AArch64::LDURSHXi:
4308 case AArch64::LDRSHXui:
4309 return AArch64::LDRSHXroW;
4310 case AArch64::LDRSHWroX:
4311 case AArch64::LDURSHWi:
4312 case AArch64::LDRSHWui:
4313 return AArch64::LDRSHWroW;
4314 case AArch64::LDRBroX:
4315 case AArch64::LDURBi:
4316 case AArch64::LDRBui:
4317 return AArch64::LDRBroW;
4318 case AArch64::LDRBBroX:
4319 case AArch64::LDURBBi:
4320 case AArch64::LDRBBui:
4321 return AArch64::LDRBBroW;
4322 case AArch64::LDRSBXroX:
4323 case AArch64::LDURSBXi:
4324 case AArch64::LDRSBXui:
4325 return AArch64::LDRSBXroW;
4326 case AArch64::LDRSBWroX:
4327 case AArch64::LDURSBWi:
4328 case AArch64::LDRSBWui:
4329 return AArch64::LDRSBWroW;
4330 case AArch64::STRBroX:
4331 case AArch64::STURBi:
4332 case AArch64::STRBui:
4333 return AArch64::STRBroW;
4334 case AArch64::STRBBroX:
4335 case AArch64::STURBBi:
4336 case AArch64::STRBBui:
4337 return AArch64::STRBBroW;
4338 }
4339}
4340
4341MachineInstr *AArch64InstrInfo::emitLdStWithAddr(MachineInstr &MemI,
4342 const ExtAddrMode &AM) const {
4343
4344 const DebugLoc &DL = MemI.getDebugLoc();
4345 MachineBasicBlock &MBB = *MemI.getParent();
4346 MachineRegisterInfo &MRI = MemI.getMF()->getRegInfo();
4347
4348 if (AM.Form == ExtAddrMode::Formula::Basic) {
4349 if (AM.ScaledReg) {
4350 // The new instruction will be in the form `ldr Rt, [Xn, Xm, lsl #imm]`.
4351 unsigned Opcode = regOffsetOpcode(Opcode: MemI.getOpcode());
4352 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4353 auto B = BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4354 .addReg(RegNo: MemI.getOperand(i: 0).getReg(),
4355 Flags: getDefRegState(B: MemI.mayLoad()))
4356 .addReg(RegNo: AM.BaseReg)
4357 .addReg(RegNo: AM.ScaledReg)
4358 .addImm(Val: 0)
4359 .addImm(Val: AM.Scale > 1)
4360 .setMemRefs(MemI.memoperands())
4361 .setMIFlags(MemI.getFlags());
4362 return B.getInstr();
4363 }
4364
4365 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
4366 "Addressing mode not supported for folding");
4367
4368 // The new instruction will be in the form `ld[u]r Rt, [Xn, #imm]`.
4369 unsigned Scale = 1;
4370 unsigned Opcode = MemI.getOpcode();
4371 if (isInt<9>(x: AM.Displacement))
4372 Opcode = unscaledOffsetOpcode(Opcode);
4373 else
4374 Opcode = scaledOffsetOpcode(Opcode, Scale);
4375
4376 auto B =
4377 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4378 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4379 .addReg(RegNo: AM.BaseReg)
4380 .addImm(Val: AM.Displacement / Scale)
4381 .setMemRefs(MemI.memoperands())
4382 .setMIFlags(MemI.getFlags());
4383 return B.getInstr();
4384 }
4385
4386 if (AM.Form == ExtAddrMode::Formula::SExtScaledReg ||
4387 AM.Form == ExtAddrMode::Formula::ZExtScaledReg) {
4388 // The new instruction will be in the form `ldr Rt, [Xn, Wm, {s,u}xtw #N]`.
4389 assert(AM.ScaledReg && !AM.Displacement &&
4390 "Address offset can be a register or an immediate, but not both");
4391 unsigned Opcode = offsetExtendOpcode(Opcode: MemI.getOpcode());
4392 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4393 // Make sure the offset register is in the correct register class.
4394 Register OffsetReg = AM.ScaledReg;
4395 const TargetRegisterClass *RC = MRI.getRegClass(Reg: OffsetReg);
4396 if (RC->hasSuperClassEq(RC: &AArch64::GPR64RegClass)) {
4397 OffsetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4398 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: OffsetReg)
4399 .addReg(RegNo: AM.ScaledReg, Flags: {}, SubReg: AArch64::sub_32);
4400 }
4401 auto B =
4402 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4403 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4404 .addReg(RegNo: AM.BaseReg)
4405 .addReg(RegNo: OffsetReg)
4406 .addImm(Val: AM.Form == ExtAddrMode::Formula::SExtScaledReg)
4407 .addImm(Val: AM.Scale != 1)
4408 .setMemRefs(MemI.memoperands())
4409 .setMIFlags(MemI.getFlags());
4410
4411 return B.getInstr();
4412 }
4413
4414 llvm_unreachable(
4415 "Function must not be called with an addressing mode it can't handle");
4416}
4417
4418/// Return true if the opcode is a post-index ld/st instruction, which really
4419/// loads from base+0.
4420static bool isPostIndexLdStOpcode(unsigned Opcode) {
4421 switch (Opcode) {
4422 default:
4423 return false;
4424 case AArch64::LD1Fourv16b_POST:
4425 case AArch64::LD1Fourv1d_POST:
4426 case AArch64::LD1Fourv2d_POST:
4427 case AArch64::LD1Fourv2s_POST:
4428 case AArch64::LD1Fourv4h_POST:
4429 case AArch64::LD1Fourv4s_POST:
4430 case AArch64::LD1Fourv8b_POST:
4431 case AArch64::LD1Fourv8h_POST:
4432 case AArch64::LD1Onev16b_POST:
4433 case AArch64::LD1Onev1d_POST:
4434 case AArch64::LD1Onev2d_POST:
4435 case AArch64::LD1Onev2s_POST:
4436 case AArch64::LD1Onev4h_POST:
4437 case AArch64::LD1Onev4s_POST:
4438 case AArch64::LD1Onev8b_POST:
4439 case AArch64::LD1Onev8h_POST:
4440 case AArch64::LD1Rv16b_POST:
4441 case AArch64::LD1Rv1d_POST:
4442 case AArch64::LD1Rv2d_POST:
4443 case AArch64::LD1Rv2s_POST:
4444 case AArch64::LD1Rv4h_POST:
4445 case AArch64::LD1Rv4s_POST:
4446 case AArch64::LD1Rv8b_POST:
4447 case AArch64::LD1Rv8h_POST:
4448 case AArch64::LD1Threev16b_POST:
4449 case AArch64::LD1Threev1d_POST:
4450 case AArch64::LD1Threev2d_POST:
4451 case AArch64::LD1Threev2s_POST:
4452 case AArch64::LD1Threev4h_POST:
4453 case AArch64::LD1Threev4s_POST:
4454 case AArch64::LD1Threev8b_POST:
4455 case AArch64::LD1Threev8h_POST:
4456 case AArch64::LD1Twov16b_POST:
4457 case AArch64::LD1Twov1d_POST:
4458 case AArch64::LD1Twov2d_POST:
4459 case AArch64::LD1Twov2s_POST:
4460 case AArch64::LD1Twov4h_POST:
4461 case AArch64::LD1Twov4s_POST:
4462 case AArch64::LD1Twov8b_POST:
4463 case AArch64::LD1Twov8h_POST:
4464 case AArch64::LD1i16_POST:
4465 case AArch64::LD1i32_POST:
4466 case AArch64::LD1i64_POST:
4467 case AArch64::LD1i8_POST:
4468 case AArch64::LD2Rv16b_POST:
4469 case AArch64::LD2Rv1d_POST:
4470 case AArch64::LD2Rv2d_POST:
4471 case AArch64::LD2Rv2s_POST:
4472 case AArch64::LD2Rv4h_POST:
4473 case AArch64::LD2Rv4s_POST:
4474 case AArch64::LD2Rv8b_POST:
4475 case AArch64::LD2Rv8h_POST:
4476 case AArch64::LD2Twov16b_POST:
4477 case AArch64::LD2Twov2d_POST:
4478 case AArch64::LD2Twov2s_POST:
4479 case AArch64::LD2Twov4h_POST:
4480 case AArch64::LD2Twov4s_POST:
4481 case AArch64::LD2Twov8b_POST:
4482 case AArch64::LD2Twov8h_POST:
4483 case AArch64::LD2i16_POST:
4484 case AArch64::LD2i32_POST:
4485 case AArch64::LD2i64_POST:
4486 case AArch64::LD2i8_POST:
4487 case AArch64::LD3Rv16b_POST:
4488 case AArch64::LD3Rv1d_POST:
4489 case AArch64::LD3Rv2d_POST:
4490 case AArch64::LD3Rv2s_POST:
4491 case AArch64::LD3Rv4h_POST:
4492 case AArch64::LD3Rv4s_POST:
4493 case AArch64::LD3Rv8b_POST:
4494 case AArch64::LD3Rv8h_POST:
4495 case AArch64::LD3Threev16b_POST:
4496 case AArch64::LD3Threev2d_POST:
4497 case AArch64::LD3Threev2s_POST:
4498 case AArch64::LD3Threev4h_POST:
4499 case AArch64::LD3Threev4s_POST:
4500 case AArch64::LD3Threev8b_POST:
4501 case AArch64::LD3Threev8h_POST:
4502 case AArch64::LD3i16_POST:
4503 case AArch64::LD3i32_POST:
4504 case AArch64::LD3i64_POST:
4505 case AArch64::LD3i8_POST:
4506 case AArch64::LD4Fourv16b_POST:
4507 case AArch64::LD4Fourv2d_POST:
4508 case AArch64::LD4Fourv2s_POST:
4509 case AArch64::LD4Fourv4h_POST:
4510 case AArch64::LD4Fourv4s_POST:
4511 case AArch64::LD4Fourv8b_POST:
4512 case AArch64::LD4Fourv8h_POST:
4513 case AArch64::LD4Rv16b_POST:
4514 case AArch64::LD4Rv1d_POST:
4515 case AArch64::LD4Rv2d_POST:
4516 case AArch64::LD4Rv2s_POST:
4517 case AArch64::LD4Rv4h_POST:
4518 case AArch64::LD4Rv4s_POST:
4519 case AArch64::LD4Rv8b_POST:
4520 case AArch64::LD4Rv8h_POST:
4521 case AArch64::LD4i16_POST:
4522 case AArch64::LD4i32_POST:
4523 case AArch64::LD4i64_POST:
4524 case AArch64::LD4i8_POST:
4525 case AArch64::LDAPRWpost:
4526 case AArch64::LDAPRXpost:
4527 case AArch64::LDIAPPWpost:
4528 case AArch64::LDIAPPXpost:
4529 case AArch64::LDPDpost:
4530 case AArch64::LDPQpost:
4531 case AArch64::LDPSWpost:
4532 case AArch64::LDPSpost:
4533 case AArch64::LDPWpost:
4534 case AArch64::LDPXpost:
4535 case AArch64::LDRBBpost:
4536 case AArch64::LDRBpost:
4537 case AArch64::LDRDpost:
4538 case AArch64::LDRHHpost:
4539 case AArch64::LDRHpost:
4540 case AArch64::LDRQpost:
4541 case AArch64::LDRSBWpost:
4542 case AArch64::LDRSBXpost:
4543 case AArch64::LDRSHWpost:
4544 case AArch64::LDRSHXpost:
4545 case AArch64::LDRSWpost:
4546 case AArch64::LDRSpost:
4547 case AArch64::LDRWpost:
4548 case AArch64::LDRXpost:
4549 case AArch64::ST1Fourv16b_POST:
4550 case AArch64::ST1Fourv1d_POST:
4551 case AArch64::ST1Fourv2d_POST:
4552 case AArch64::ST1Fourv2s_POST:
4553 case AArch64::ST1Fourv4h_POST:
4554 case AArch64::ST1Fourv4s_POST:
4555 case AArch64::ST1Fourv8b_POST:
4556 case AArch64::ST1Fourv8h_POST:
4557 case AArch64::ST1Onev16b_POST:
4558 case AArch64::ST1Onev1d_POST:
4559 case AArch64::ST1Onev2d_POST:
4560 case AArch64::ST1Onev2s_POST:
4561 case AArch64::ST1Onev4h_POST:
4562 case AArch64::ST1Onev4s_POST:
4563 case AArch64::ST1Onev8b_POST:
4564 case AArch64::ST1Onev8h_POST:
4565 case AArch64::ST1Threev16b_POST:
4566 case AArch64::ST1Threev1d_POST:
4567 case AArch64::ST1Threev2d_POST:
4568 case AArch64::ST1Threev2s_POST:
4569 case AArch64::ST1Threev4h_POST:
4570 case AArch64::ST1Threev4s_POST:
4571 case AArch64::ST1Threev8b_POST:
4572 case AArch64::ST1Threev8h_POST:
4573 case AArch64::ST1Twov16b_POST:
4574 case AArch64::ST1Twov1d_POST:
4575 case AArch64::ST1Twov2d_POST:
4576 case AArch64::ST1Twov2s_POST:
4577 case AArch64::ST1Twov4h_POST:
4578 case AArch64::ST1Twov4s_POST:
4579 case AArch64::ST1Twov8b_POST:
4580 case AArch64::ST1Twov8h_POST:
4581 case AArch64::ST1i16_POST:
4582 case AArch64::ST1i32_POST:
4583 case AArch64::ST1i64_POST:
4584 case AArch64::ST1i8_POST:
4585 case AArch64::ST2GPostIndex:
4586 case AArch64::ST2Twov16b_POST:
4587 case AArch64::ST2Twov2d_POST:
4588 case AArch64::ST2Twov2s_POST:
4589 case AArch64::ST2Twov4h_POST:
4590 case AArch64::ST2Twov4s_POST:
4591 case AArch64::ST2Twov8b_POST:
4592 case AArch64::ST2Twov8h_POST:
4593 case AArch64::ST2i16_POST:
4594 case AArch64::ST2i32_POST:
4595 case AArch64::ST2i64_POST:
4596 case AArch64::ST2i8_POST:
4597 case AArch64::ST3Threev16b_POST:
4598 case AArch64::ST3Threev2d_POST:
4599 case AArch64::ST3Threev2s_POST:
4600 case AArch64::ST3Threev4h_POST:
4601 case AArch64::ST3Threev4s_POST:
4602 case AArch64::ST3Threev8b_POST:
4603 case AArch64::ST3Threev8h_POST:
4604 case AArch64::ST3i16_POST:
4605 case AArch64::ST3i32_POST:
4606 case AArch64::ST3i64_POST:
4607 case AArch64::ST3i8_POST:
4608 case AArch64::ST4Fourv16b_POST:
4609 case AArch64::ST4Fourv2d_POST:
4610 case AArch64::ST4Fourv2s_POST:
4611 case AArch64::ST4Fourv4h_POST:
4612 case AArch64::ST4Fourv4s_POST:
4613 case AArch64::ST4Fourv8b_POST:
4614 case AArch64::ST4Fourv8h_POST:
4615 case AArch64::ST4i16_POST:
4616 case AArch64::ST4i32_POST:
4617 case AArch64::ST4i64_POST:
4618 case AArch64::ST4i8_POST:
4619 case AArch64::STGPostIndex:
4620 case AArch64::STGPpost:
4621 case AArch64::STPDpost:
4622 case AArch64::STPQpost:
4623 case AArch64::STPSpost:
4624 case AArch64::STPWpost:
4625 case AArch64::STPXpost:
4626 case AArch64::STRBBpost:
4627 case AArch64::STRBpost:
4628 case AArch64::STRDpost:
4629 case AArch64::STRHHpost:
4630 case AArch64::STRHpost:
4631 case AArch64::STRQpost:
4632 case AArch64::STRSpost:
4633 case AArch64::STRWpost:
4634 case AArch64::STRXpost:
4635 case AArch64::STZ2GPostIndex:
4636 case AArch64::STZGPostIndex:
4637 return true;
4638 }
4639}
4640
4641bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
4642 const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
4643 bool &OffsetIsScalable, TypeSize &Width,
4644 const TargetRegisterInfo *TRI) const {
4645 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4646 // Handle only loads/stores with base register followed by immediate offset.
4647 if (LdSt.getNumExplicitOperands() == 3) {
4648 // Non-paired instruction (e.g., ldr x1, [x0, #8]).
4649 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
4650 !LdSt.getOperand(i: 2).isImm())
4651 return false;
4652 } else if (LdSt.getNumExplicitOperands() == 4) {
4653 // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
4654 if (!LdSt.getOperand(i: 1).isReg() ||
4655 (!LdSt.getOperand(i: 2).isReg() && !LdSt.getOperand(i: 2).isFI()) ||
4656 !LdSt.getOperand(i: 3).isImm())
4657 return false;
4658 } else
4659 return false;
4660
4661 // Get the scaling factor for the instruction and set the width for the
4662 // instruction.
4663 TypeSize Scale(0U, false);
4664 int64_t Dummy1, Dummy2;
4665
4666 // If this returns false, then it's an instruction we don't want to handle.
4667 if (!getMemOpInfo(Opcode: LdSt.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2))
4668 return false;
4669
4670 // Compute the offset. Offset is calculated as the immediate operand
4671 // multiplied by the scaling factor. Unscaled instructions have scaling factor
4672 // set to 1. Postindex are a special case which have an offset of 0.
4673 if (isPostIndexLdStOpcode(Opcode: LdSt.getOpcode())) {
4674 BaseOp = &LdSt.getOperand(i: 2);
4675 Offset = 0;
4676 } else if (LdSt.getNumExplicitOperands() == 3) {
4677 BaseOp = &LdSt.getOperand(i: 1);
4678 Offset = LdSt.getOperand(i: 2).getImm() * Scale.getKnownMinValue();
4679 } else {
4680 assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
4681 BaseOp = &LdSt.getOperand(i: 2);
4682 Offset = LdSt.getOperand(i: 3).getImm() * Scale.getKnownMinValue();
4683 }
4684 OffsetIsScalable = Scale.isScalable();
4685
4686 return BaseOp->isReg() || BaseOp->isFI();
4687}
4688
4689MachineOperand &
4690AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
4691 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4692 MachineOperand &OfsOp = LdSt.getOperand(i: LdSt.getNumExplicitOperands() - 1);
4693 assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
4694 return OfsOp;
4695}
4696
4697bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
4698 TypeSize &Width, int64_t &MinOffset,
4699 int64_t &MaxOffset) {
4700 switch (Opcode) {
4701 // Not a memory operation or something we want to handle.
4702 default:
4703 Scale = Width = TypeSize::getFixed(ExactSize: 0);
4704 MinOffset = MaxOffset = 0;
4705 return false;
4706 // LDR / STR
4707 case AArch64::LDRQui:
4708 case AArch64::STRQui:
4709 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4710 MinOffset = 0;
4711 MaxOffset = 4095;
4712 break;
4713 case AArch64::LDRXui:
4714 case AArch64::LDRDui:
4715 case AArch64::STRXui:
4716 case AArch64::STRDui:
4717 case AArch64::PRFMui:
4718 Scale = Width = TypeSize::getFixed(ExactSize: 8);
4719 MinOffset = 0;
4720 MaxOffset = 4095;
4721 break;
4722 case AArch64::LDRWui:
4723 case AArch64::LDRSui:
4724 case AArch64::LDRSWui:
4725 case AArch64::STRWui:
4726 case AArch64::STRSui:
4727 Scale = Width = TypeSize::getFixed(ExactSize: 4);
4728 MinOffset = 0;
4729 MaxOffset = 4095;
4730 break;
4731 case AArch64::LDRHui:
4732 case AArch64::LDRHHui:
4733 case AArch64::LDRSHWui:
4734 case AArch64::LDRSHXui:
4735 case AArch64::STRHui:
4736 case AArch64::STRHHui:
4737 Scale = Width = TypeSize::getFixed(ExactSize: 2);
4738 MinOffset = 0;
4739 MaxOffset = 4095;
4740 break;
4741 case AArch64::LDRBui:
4742 case AArch64::LDRBBui:
4743 case AArch64::LDRSBWui:
4744 case AArch64::LDRSBXui:
4745 case AArch64::STRBui:
4746 case AArch64::STRBBui:
4747 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4748 MinOffset = 0;
4749 MaxOffset = 4095;
4750 break;
4751 // post/pre inc
4752 case AArch64::STRQpre:
4753 case AArch64::LDRQpost:
4754 Scale = TypeSize::getFixed(ExactSize: 1);
4755 Width = TypeSize::getFixed(ExactSize: 16);
4756 MinOffset = -256;
4757 MaxOffset = 255;
4758 break;
4759 case AArch64::LDRDpost:
4760 case AArch64::LDRDpre:
4761 case AArch64::LDRXpost:
4762 case AArch64::LDRXpre:
4763 case AArch64::STRDpost:
4764 case AArch64::STRDpre:
4765 case AArch64::STRXpost:
4766 case AArch64::STRXpre:
4767 Scale = TypeSize::getFixed(ExactSize: 1);
4768 Width = TypeSize::getFixed(ExactSize: 8);
4769 MinOffset = -256;
4770 MaxOffset = 255;
4771 break;
4772 case AArch64::STRWpost:
4773 case AArch64::STRWpre:
4774 case AArch64::LDRWpost:
4775 case AArch64::LDRWpre:
4776 case AArch64::STRSpost:
4777 case AArch64::STRSpre:
4778 case AArch64::LDRSpost:
4779 case AArch64::LDRSpre:
4780 Scale = TypeSize::getFixed(ExactSize: 1);
4781 Width = TypeSize::getFixed(ExactSize: 4);
4782 MinOffset = -256;
4783 MaxOffset = 255;
4784 break;
4785 case AArch64::LDRHpost:
4786 case AArch64::LDRHpre:
4787 case AArch64::STRHpost:
4788 case AArch64::STRHpre:
4789 case AArch64::LDRHHpost:
4790 case AArch64::LDRHHpre:
4791 case AArch64::STRHHpost:
4792 case AArch64::STRHHpre:
4793 Scale = TypeSize::getFixed(ExactSize: 1);
4794 Width = TypeSize::getFixed(ExactSize: 2);
4795 MinOffset = -256;
4796 MaxOffset = 255;
4797 break;
4798 case AArch64::LDRBpost:
4799 case AArch64::LDRBpre:
4800 case AArch64::STRBpost:
4801 case AArch64::STRBpre:
4802 case AArch64::LDRBBpost:
4803 case AArch64::LDRBBpre:
4804 case AArch64::STRBBpost:
4805 case AArch64::STRBBpre:
4806 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4807 MinOffset = -256;
4808 MaxOffset = 255;
4809 break;
4810 // Unscaled
4811 case AArch64::LDURQi:
4812 case AArch64::STURQi:
4813 Scale = TypeSize::getFixed(ExactSize: 1);
4814 Width = TypeSize::getFixed(ExactSize: 16);
4815 MinOffset = -256;
4816 MaxOffset = 255;
4817 break;
4818 case AArch64::LDURXi:
4819 case AArch64::LDURDi:
4820 case AArch64::LDAPURXi:
4821 case AArch64::STURXi:
4822 case AArch64::STURDi:
4823 case AArch64::STLURXi:
4824 case AArch64::PRFUMi:
4825 Scale = TypeSize::getFixed(ExactSize: 1);
4826 Width = TypeSize::getFixed(ExactSize: 8);
4827 MinOffset = -256;
4828 MaxOffset = 255;
4829 break;
4830 case AArch64::LDURWi:
4831 case AArch64::LDURSi:
4832 case AArch64::LDURSWi:
4833 case AArch64::LDAPURi:
4834 case AArch64::LDAPURSWi:
4835 case AArch64::STURWi:
4836 case AArch64::STURSi:
4837 case AArch64::STLURWi:
4838 Scale = TypeSize::getFixed(ExactSize: 1);
4839 Width = TypeSize::getFixed(ExactSize: 4);
4840 MinOffset = -256;
4841 MaxOffset = 255;
4842 break;
4843 case AArch64::LDURHi:
4844 case AArch64::LDURHHi:
4845 case AArch64::LDURSHXi:
4846 case AArch64::LDURSHWi:
4847 case AArch64::LDAPURHi:
4848 case AArch64::LDAPURSHWi:
4849 case AArch64::LDAPURSHXi:
4850 case AArch64::STURHi:
4851 case AArch64::STURHHi:
4852 case AArch64::STLURHi:
4853 Scale = TypeSize::getFixed(ExactSize: 1);
4854 Width = TypeSize::getFixed(ExactSize: 2);
4855 MinOffset = -256;
4856 MaxOffset = 255;
4857 break;
4858 case AArch64::LDURBi:
4859 case AArch64::LDURBBi:
4860 case AArch64::LDURSBXi:
4861 case AArch64::LDURSBWi:
4862 case AArch64::LDAPURBi:
4863 case AArch64::LDAPURSBWi:
4864 case AArch64::LDAPURSBXi:
4865 case AArch64::STURBi:
4866 case AArch64::STURBBi:
4867 case AArch64::STLURBi:
4868 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4869 MinOffset = -256;
4870 MaxOffset = 255;
4871 break;
4872 // LDP / STP (including pre/post inc)
4873 case AArch64::LDPQi:
4874 case AArch64::LDNPQi:
4875 case AArch64::STPQi:
4876 case AArch64::STNPQi:
4877 case AArch64::LDPQpost:
4878 case AArch64::LDPQpre:
4879 case AArch64::STPQpost:
4880 case AArch64::STPQpre:
4881 Scale = TypeSize::getFixed(ExactSize: 16);
4882 Width = TypeSize::getFixed(ExactSize: 16 * 2);
4883 MinOffset = -64;
4884 MaxOffset = 63;
4885 break;
4886 case AArch64::LDPXi:
4887 case AArch64::LDPDi:
4888 case AArch64::LDNPXi:
4889 case AArch64::LDNPDi:
4890 case AArch64::STPXi:
4891 case AArch64::STPDi:
4892 case AArch64::STNPXi:
4893 case AArch64::STNPDi:
4894 case AArch64::LDPDpost:
4895 case AArch64::LDPDpre:
4896 case AArch64::LDPXpost:
4897 case AArch64::LDPXpre:
4898 case AArch64::STPDpost:
4899 case AArch64::STPDpre:
4900 case AArch64::STPXpost:
4901 case AArch64::STPXpre:
4902 Scale = TypeSize::getFixed(ExactSize: 8);
4903 Width = TypeSize::getFixed(ExactSize: 8 * 2);
4904 MinOffset = -64;
4905 MaxOffset = 63;
4906 break;
4907 case AArch64::LDPWi:
4908 case AArch64::LDPSi:
4909 case AArch64::LDNPWi:
4910 case AArch64::LDNPSi:
4911 case AArch64::STPWi:
4912 case AArch64::STPSi:
4913 case AArch64::STNPWi:
4914 case AArch64::STNPSi:
4915 case AArch64::LDPSpost:
4916 case AArch64::LDPSpre:
4917 case AArch64::LDPWpost:
4918 case AArch64::LDPWpre:
4919 case AArch64::STPSpost:
4920 case AArch64::STPSpre:
4921 case AArch64::STPWpost:
4922 case AArch64::STPWpre:
4923 Scale = TypeSize::getFixed(ExactSize: 4);
4924 Width = TypeSize::getFixed(ExactSize: 4 * 2);
4925 MinOffset = -64;
4926 MaxOffset = 63;
4927 break;
4928 case AArch64::StoreSwiftAsyncContext:
4929 // Store is an STRXui, but there might be an ADDXri in the expansion too.
4930 Scale = TypeSize::getFixed(ExactSize: 1);
4931 Width = TypeSize::getFixed(ExactSize: 8);
4932 MinOffset = 0;
4933 MaxOffset = 4095;
4934 break;
4935 case AArch64::ADDG:
4936 Scale = TypeSize::getFixed(ExactSize: 16);
4937 Width = TypeSize::getFixed(ExactSize: 0);
4938 MinOffset = 0;
4939 MaxOffset = 63;
4940 break;
4941 case AArch64::TAGPstack:
4942 Scale = TypeSize::getFixed(ExactSize: 16);
4943 Width = TypeSize::getFixed(ExactSize: 0);
4944 // TAGP with a negative offset turns into SUBP, which has a maximum offset
4945 // of 63 (not 64!).
4946 MinOffset = -63;
4947 MaxOffset = 63;
4948 break;
4949 case AArch64::LDG:
4950 case AArch64::STGi:
4951 case AArch64::STGPreIndex:
4952 case AArch64::STGPostIndex:
4953 case AArch64::STZGi:
4954 case AArch64::STZGPreIndex:
4955 case AArch64::STZGPostIndex:
4956 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4957 MinOffset = -256;
4958 MaxOffset = 255;
4959 break;
4960 // SVE
4961 case AArch64::STR_ZZZZXI:
4962 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
4963 case AArch64::LDR_ZZZZXI:
4964 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
4965 Scale = TypeSize::getScalable(MinimumSize: 16);
4966 Width = TypeSize::getScalable(MinimumSize: 16 * 4);
4967 MinOffset = -256;
4968 MaxOffset = 252;
4969 break;
4970 case AArch64::STR_ZZZXI:
4971 case AArch64::LDR_ZZZXI:
4972 Scale = TypeSize::getScalable(MinimumSize: 16);
4973 Width = TypeSize::getScalable(MinimumSize: 16 * 3);
4974 MinOffset = -256;
4975 MaxOffset = 253;
4976 break;
4977 case AArch64::STR_ZZXI:
4978 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
4979 case AArch64::LDR_ZZXI:
4980 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
4981 Scale = TypeSize::getScalable(MinimumSize: 16);
4982 Width = TypeSize::getScalable(MinimumSize: 16 * 2);
4983 MinOffset = -256;
4984 MaxOffset = 254;
4985 break;
4986 case AArch64::LDR_PXI:
4987 case AArch64::STR_PXI:
4988 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
4989 MinOffset = -256;
4990 MaxOffset = 255;
4991 break;
4992 case AArch64::LDR_PPXI:
4993 case AArch64::STR_PPXI:
4994 Scale = TypeSize::getScalable(MinimumSize: 2);
4995 Width = TypeSize::getScalable(MinimumSize: 2 * 2);
4996 MinOffset = -256;
4997 MaxOffset = 254;
4998 break;
4999 case AArch64::LDR_ZXI:
5000 case AArch64::STR_ZXI:
5001 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5002 MinOffset = -256;
5003 MaxOffset = 255;
5004 break;
5005 case AArch64::LD1B_IMM:
5006 case AArch64::LD1H_IMM:
5007 case AArch64::LD1W_IMM:
5008 case AArch64::LD1D_IMM:
5009 case AArch64::LDNT1B_ZRI:
5010 case AArch64::LDNT1H_ZRI:
5011 case AArch64::LDNT1W_ZRI:
5012 case AArch64::LDNT1D_ZRI:
5013 case AArch64::ST1B_IMM:
5014 case AArch64::ST1H_IMM:
5015 case AArch64::ST1W_IMM:
5016 case AArch64::ST1D_IMM:
5017 case AArch64::STNT1B_ZRI:
5018 case AArch64::STNT1H_ZRI:
5019 case AArch64::STNT1W_ZRI:
5020 case AArch64::STNT1D_ZRI:
5021 case AArch64::LDNF1B_IMM:
5022 case AArch64::LDNF1H_IMM:
5023 case AArch64::LDNF1W_IMM:
5024 case AArch64::LDNF1D_IMM:
5025 // A full vectors worth of data
5026 // Width = mbytes * elements
5027 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5028 MinOffset = -8;
5029 MaxOffset = 7;
5030 break;
5031 case AArch64::LD2B_IMM:
5032 case AArch64::LD2H_IMM:
5033 case AArch64::LD2W_IMM:
5034 case AArch64::LD2D_IMM:
5035 case AArch64::ST2B_IMM:
5036 case AArch64::ST2H_IMM:
5037 case AArch64::ST2W_IMM:
5038 case AArch64::ST2D_IMM:
5039 case AArch64::LD1B_2Z_IMM:
5040 case AArch64::LD1B_2Z_STRIDED_IMM:
5041 case AArch64::LD1H_2Z_IMM:
5042 case AArch64::LD1H_2Z_STRIDED_IMM:
5043 case AArch64::LD1W_2Z_IMM:
5044 case AArch64::LD1W_2Z_STRIDED_IMM:
5045 case AArch64::LD1D_2Z_IMM:
5046 case AArch64::LD1D_2Z_STRIDED_IMM:
5047 case AArch64::LD1B_2Z_IMM_PSEUDO:
5048 case AArch64::LD1H_2Z_IMM_PSEUDO:
5049 case AArch64::LD1W_2Z_IMM_PSEUDO:
5050 case AArch64::LD1D_2Z_IMM_PSEUDO:
5051 case AArch64::ST1B_2Z_IMM:
5052 case AArch64::ST1B_2Z_STRIDED_IMM:
5053 case AArch64::ST1H_2Z_IMM:
5054 case AArch64::ST1H_2Z_STRIDED_IMM:
5055 case AArch64::ST1W_2Z_IMM:
5056 case AArch64::ST1W_2Z_STRIDED_IMM:
5057 case AArch64::ST1D_2Z_IMM:
5058 case AArch64::ST1D_2Z_STRIDED_IMM:
5059 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
5060 case AArch64::LDNT1B_2Z_IMM:
5061 case AArch64::LDNT1B_2Z_STRIDED_IMM:
5062 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
5063 case AArch64::LDNT1H_2Z_IMM:
5064 case AArch64::LDNT1H_2Z_STRIDED_IMM:
5065 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
5066 case AArch64::LDNT1W_2Z_IMM:
5067 case AArch64::LDNT1W_2Z_STRIDED_IMM:
5068 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
5069 case AArch64::LDNT1D_2Z_IMM:
5070 case AArch64::LDNT1D_2Z_STRIDED_IMM:
5071 case AArch64::STNT1B_2Z_IMM:
5072 case AArch64::STNT1B_2Z_STRIDED_IMM:
5073 case AArch64::STNT1H_2Z_IMM:
5074 case AArch64::STNT1H_2Z_STRIDED_IMM:
5075 case AArch64::STNT1W_2Z_IMM:
5076 case AArch64::STNT1W_2Z_STRIDED_IMM:
5077 case AArch64::STNT1D_2Z_IMM:
5078 case AArch64::STNT1D_2Z_STRIDED_IMM:
5079 case AArch64::ST1B_2Z_IMM_PSEUDO:
5080 case AArch64::ST1H_2Z_IMM_PSEUDO:
5081 case AArch64::ST1W_2Z_IMM_PSEUDO:
5082 case AArch64::ST1D_2Z_IMM_PSEUDO:
5083 case AArch64::STNT1B_2Z_IMM_PSEUDO:
5084 case AArch64::STNT1H_2Z_IMM_PSEUDO:
5085 case AArch64::STNT1W_2Z_IMM_PSEUDO:
5086 case AArch64::STNT1D_2Z_IMM_PSEUDO:
5087 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5088 MinOffset = -8;
5089 MaxOffset = 7;
5090 break;
5091 case AArch64::LD3B_IMM:
5092 case AArch64::LD3H_IMM:
5093 case AArch64::LD3W_IMM:
5094 case AArch64::LD3D_IMM:
5095 case AArch64::ST3B_IMM:
5096 case AArch64::ST3H_IMM:
5097 case AArch64::ST3W_IMM:
5098 case AArch64::ST3D_IMM:
5099 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5100 MinOffset = -8;
5101 MaxOffset = 7;
5102 break;
5103 case AArch64::LD4B_IMM:
5104 case AArch64::LD4H_IMM:
5105 case AArch64::LD4W_IMM:
5106 case AArch64::LD4D_IMM:
5107 case AArch64::ST4B_IMM:
5108 case AArch64::ST4H_IMM:
5109 case AArch64::ST4W_IMM:
5110 case AArch64::ST4D_IMM:
5111 case AArch64::LD1B_4Z_IMM:
5112 case AArch64::LD1B_4Z_STRIDED_IMM:
5113 case AArch64::LD1H_4Z_IMM:
5114 case AArch64::LD1H_4Z_STRIDED_IMM:
5115 case AArch64::LD1W_4Z_IMM:
5116 case AArch64::LD1W_4Z_STRIDED_IMM:
5117 case AArch64::LD1D_4Z_IMM:
5118 case AArch64::LD1D_4Z_STRIDED_IMM:
5119 case AArch64::LD1B_4Z_IMM_PSEUDO:
5120 case AArch64::LD1H_4Z_IMM_PSEUDO:
5121 case AArch64::LD1W_4Z_IMM_PSEUDO:
5122 case AArch64::LD1D_4Z_IMM_PSEUDO:
5123 case AArch64::ST1B_4Z_IMM:
5124 case AArch64::ST1B_4Z_STRIDED_IMM:
5125 case AArch64::ST1H_4Z_IMM:
5126 case AArch64::ST1H_4Z_STRIDED_IMM:
5127 case AArch64::ST1W_4Z_IMM:
5128 case AArch64::ST1W_4Z_STRIDED_IMM:
5129 case AArch64::ST1D_4Z_IMM:
5130 case AArch64::ST1D_4Z_STRIDED_IMM:
5131 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
5132 case AArch64::LDNT1B_4Z_IMM:
5133 case AArch64::LDNT1B_4Z_STRIDED_IMM:
5134 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
5135 case AArch64::LDNT1H_4Z_IMM:
5136 case AArch64::LDNT1H_4Z_STRIDED_IMM:
5137 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
5138 case AArch64::LDNT1W_4Z_IMM:
5139 case AArch64::LDNT1W_4Z_STRIDED_IMM:
5140 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
5141 case AArch64::LDNT1D_4Z_IMM:
5142 case AArch64::LDNT1D_4Z_STRIDED_IMM:
5143 case AArch64::STNT1B_4Z_IMM:
5144 case AArch64::STNT1B_4Z_STRIDED_IMM:
5145 case AArch64::STNT1H_4Z_IMM:
5146 case AArch64::STNT1H_4Z_STRIDED_IMM:
5147 case AArch64::STNT1W_4Z_IMM:
5148 case AArch64::STNT1W_4Z_STRIDED_IMM:
5149 case AArch64::STNT1D_4Z_IMM:
5150 case AArch64::STNT1D_4Z_STRIDED_IMM:
5151 case AArch64::ST1B_4Z_IMM_PSEUDO:
5152 case AArch64::ST1H_4Z_IMM_PSEUDO:
5153 case AArch64::ST1W_4Z_IMM_PSEUDO:
5154 case AArch64::ST1D_4Z_IMM_PSEUDO:
5155 case AArch64::STNT1B_4Z_IMM_PSEUDO:
5156 case AArch64::STNT1H_4Z_IMM_PSEUDO:
5157 case AArch64::STNT1W_4Z_IMM_PSEUDO:
5158 case AArch64::STNT1D_4Z_IMM_PSEUDO:
5159 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5160 MinOffset = -8;
5161 MaxOffset = 7;
5162 break;
5163 case AArch64::LD1B_H_IMM:
5164 case AArch64::LD1SB_H_IMM:
5165 case AArch64::LD1H_S_IMM:
5166 case AArch64::LD1SH_S_IMM:
5167 case AArch64::LD1W_D_IMM:
5168 case AArch64::LD1SW_D_IMM:
5169 case AArch64::ST1B_H_IMM:
5170 case AArch64::ST1H_S_IMM:
5171 case AArch64::ST1W_D_IMM:
5172 case AArch64::LDNF1B_H_IMM:
5173 case AArch64::LDNF1SB_H_IMM:
5174 case AArch64::LDNF1H_S_IMM:
5175 case AArch64::LDNF1SH_S_IMM:
5176 case AArch64::LDNF1W_D_IMM:
5177 case AArch64::LDNF1SW_D_IMM:
5178 // A half vector worth of data
5179 // Width = mbytes * elements
5180 Scale = Width = TypeSize::getScalable(MinimumSize: 8);
5181 MinOffset = -8;
5182 MaxOffset = 7;
5183 break;
5184 case AArch64::LD1B_S_IMM:
5185 case AArch64::LD1SB_S_IMM:
5186 case AArch64::LD1H_D_IMM:
5187 case AArch64::LD1SH_D_IMM:
5188 case AArch64::ST1B_S_IMM:
5189 case AArch64::ST1H_D_IMM:
5190 case AArch64::LDNF1B_S_IMM:
5191 case AArch64::LDNF1SB_S_IMM:
5192 case AArch64::LDNF1H_D_IMM:
5193 case AArch64::LDNF1SH_D_IMM:
5194 // A quarter vector worth of data
5195 // Width = mbytes * elements
5196 Scale = Width = TypeSize::getScalable(MinimumSize: 4);
5197 MinOffset = -8;
5198 MaxOffset = 7;
5199 break;
5200 case AArch64::LD1B_D_IMM:
5201 case AArch64::LD1SB_D_IMM:
5202 case AArch64::ST1B_D_IMM:
5203 case AArch64::LDNF1B_D_IMM:
5204 case AArch64::LDNF1SB_D_IMM:
5205 // A eighth vector worth of data
5206 // Width = mbytes * elements
5207 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5208 MinOffset = -8;
5209 MaxOffset = 7;
5210 break;
5211 case AArch64::ST2Gi:
5212 case AArch64::ST2GPreIndex:
5213 case AArch64::ST2GPostIndex:
5214 case AArch64::STZ2Gi:
5215 case AArch64::STZ2GPreIndex:
5216 case AArch64::STZ2GPostIndex:
5217 Scale = TypeSize::getFixed(ExactSize: 16);
5218 Width = TypeSize::getFixed(ExactSize: 32);
5219 MinOffset = -256;
5220 MaxOffset = 255;
5221 break;
5222 case AArch64::STGPi:
5223 case AArch64::STGPpost:
5224 case AArch64::STGPpre:
5225 Scale = Width = TypeSize::getFixed(ExactSize: 16);
5226 MinOffset = -64;
5227 MaxOffset = 63;
5228 break;
5229 case AArch64::LD1RB_IMM:
5230 case AArch64::LD1RB_H_IMM:
5231 case AArch64::LD1RB_S_IMM:
5232 case AArch64::LD1RB_D_IMM:
5233 case AArch64::LD1RSB_H_IMM:
5234 case AArch64::LD1RSB_S_IMM:
5235 case AArch64::LD1RSB_D_IMM:
5236 Scale = Width = TypeSize::getFixed(ExactSize: 1);
5237 MinOffset = 0;
5238 MaxOffset = 63;
5239 break;
5240 case AArch64::LD1RH_IMM:
5241 case AArch64::LD1RH_S_IMM:
5242 case AArch64::LD1RH_D_IMM:
5243 case AArch64::LD1RSH_S_IMM:
5244 case AArch64::LD1RSH_D_IMM:
5245 Scale = Width = TypeSize::getFixed(ExactSize: 2);
5246 MinOffset = 0;
5247 MaxOffset = 63;
5248 break;
5249 case AArch64::LD1RW_IMM:
5250 case AArch64::LD1RW_D_IMM:
5251 case AArch64::LD1RSW_IMM:
5252 Scale = Width = TypeSize::getFixed(ExactSize: 4);
5253 MinOffset = 0;
5254 MaxOffset = 63;
5255 break;
5256 case AArch64::LD1RD_IMM:
5257 Scale = Width = TypeSize::getFixed(ExactSize: 8);
5258 MinOffset = 0;
5259 MaxOffset = 63;
5260 break;
5261 }
5262
5263 return true;
5264}
5265
5266// Scaling factor for unscaled load or store.
5267int AArch64InstrInfo::getMemScale(unsigned Opc) {
5268 switch (Opc) {
5269 default:
5270 llvm_unreachable("Opcode has unknown scale!");
5271 case AArch64::LDRBui:
5272 case AArch64::LDRBBui:
5273 case AArch64::LDURBBi:
5274 case AArch64::LDRSBWui:
5275 case AArch64::LDURSBWi:
5276 case AArch64::STRBui:
5277 case AArch64::STRBBui:
5278 case AArch64::STURBBi:
5279 return 1;
5280 case AArch64::LDRHui:
5281 case AArch64::LDRHHui:
5282 case AArch64::LDURHHi:
5283 case AArch64::LDRSHWui:
5284 case AArch64::LDURSHWi:
5285 case AArch64::STRHui:
5286 case AArch64::STRHHui:
5287 case AArch64::STURHHi:
5288 return 2;
5289 case AArch64::LDRSui:
5290 case AArch64::LDURSi:
5291 case AArch64::LDRSpre:
5292 case AArch64::LDRSWui:
5293 case AArch64::LDURSWi:
5294 case AArch64::LDRSWpre:
5295 case AArch64::LDRWpre:
5296 case AArch64::LDRWui:
5297 case AArch64::LDURWi:
5298 case AArch64::STRSui:
5299 case AArch64::STURSi:
5300 case AArch64::STRSpre:
5301 case AArch64::STRWui:
5302 case AArch64::STURWi:
5303 case AArch64::STRWpre:
5304 case AArch64::LDPSi:
5305 case AArch64::LDPSWi:
5306 case AArch64::LDPWi:
5307 case AArch64::STPSi:
5308 case AArch64::STPWi:
5309 return 4;
5310 case AArch64::LDRDui:
5311 case AArch64::LDURDi:
5312 case AArch64::LDRDpre:
5313 case AArch64::LDRXui:
5314 case AArch64::LDURXi:
5315 case AArch64::LDRXpre:
5316 case AArch64::STRDui:
5317 case AArch64::STURDi:
5318 case AArch64::STRDpre:
5319 case AArch64::STRXui:
5320 case AArch64::STURXi:
5321 case AArch64::STRXpre:
5322 case AArch64::LDPDi:
5323 case AArch64::LDPXi:
5324 case AArch64::STPDi:
5325 case AArch64::STPXi:
5326 return 8;
5327 case AArch64::LDRQui:
5328 case AArch64::LDURQi:
5329 case AArch64::STRQui:
5330 case AArch64::STURQi:
5331 case AArch64::STRQpre:
5332 case AArch64::LDPQi:
5333 case AArch64::LDRQpre:
5334 case AArch64::STPQi:
5335 case AArch64::STGi:
5336 case AArch64::STZGi:
5337 case AArch64::ST2Gi:
5338 case AArch64::STZ2Gi:
5339 case AArch64::STGPi:
5340 return 16;
5341 }
5342}
5343
5344bool AArch64InstrInfo::isPreLd(const MachineInstr &MI) {
5345 switch (MI.getOpcode()) {
5346 default:
5347 return false;
5348 case AArch64::LDRWpre:
5349 case AArch64::LDRXpre:
5350 case AArch64::LDRSWpre:
5351 case AArch64::LDRSpre:
5352 case AArch64::LDRDpre:
5353 case AArch64::LDRQpre:
5354 return true;
5355 }
5356}
5357
5358bool AArch64InstrInfo::isPreSt(const MachineInstr &MI) {
5359 switch (MI.getOpcode()) {
5360 default:
5361 return false;
5362 case AArch64::STRWpre:
5363 case AArch64::STRXpre:
5364 case AArch64::STRSpre:
5365 case AArch64::STRDpre:
5366 case AArch64::STRQpre:
5367 return true;
5368 }
5369}
5370
5371bool AArch64InstrInfo::isPreLdSt(const MachineInstr &MI) {
5372 return isPreLd(MI) || isPreSt(MI);
5373}
5374
5375bool AArch64InstrInfo::isZExtLoad(const MachineInstr &MI) {
5376 switch (MI.getOpcode()) {
5377 default:
5378 return false;
5379 case AArch64::LDURBBi:
5380 case AArch64::LDURHHi:
5381 case AArch64::LDURWi:
5382 case AArch64::LDRBBui:
5383 case AArch64::LDRHHui:
5384 case AArch64::LDRWui:
5385 case AArch64::LDRBBroX:
5386 case AArch64::LDRHHroX:
5387 case AArch64::LDRWroX:
5388 case AArch64::LDRBBroW:
5389 case AArch64::LDRHHroW:
5390 case AArch64::LDRWroW:
5391 return true;
5392 }
5393}
5394
5395bool AArch64InstrInfo::isSExtLoad(const MachineInstr &MI) {
5396 switch (MI.getOpcode()) {
5397 default:
5398 return false;
5399 case AArch64::LDURSBWi:
5400 case AArch64::LDURSHWi:
5401 case AArch64::LDURSBXi:
5402 case AArch64::LDURSHXi:
5403 case AArch64::LDURSWi:
5404 case AArch64::LDRSBWui:
5405 case AArch64::LDRSHWui:
5406 case AArch64::LDRSBXui:
5407 case AArch64::LDRSHXui:
5408 case AArch64::LDRSWui:
5409 case AArch64::LDRSBWroX:
5410 case AArch64::LDRSHWroX:
5411 case AArch64::LDRSBXroX:
5412 case AArch64::LDRSHXroX:
5413 case AArch64::LDRSWroX:
5414 case AArch64::LDRSBWroW:
5415 case AArch64::LDRSHWroW:
5416 case AArch64::LDRSBXroW:
5417 case AArch64::LDRSHXroW:
5418 case AArch64::LDRSWroW:
5419 return true;
5420 }
5421}
5422
5423bool AArch64InstrInfo::isPairedLdSt(const MachineInstr &MI) {
5424 switch (MI.getOpcode()) {
5425 default:
5426 return false;
5427 case AArch64::LDPSi:
5428 case AArch64::LDPSWi:
5429 case AArch64::LDPDi:
5430 case AArch64::LDPQi:
5431 case AArch64::LDPWi:
5432 case AArch64::LDPXi:
5433 case AArch64::STPSi:
5434 case AArch64::STPDi:
5435 case AArch64::STPQi:
5436 case AArch64::STPWi:
5437 case AArch64::STPXi:
5438 case AArch64::STGPi:
5439 return true;
5440 }
5441}
5442
5443const MachineOperand &AArch64InstrInfo::getLdStBaseOp(const MachineInstr &MI) {
5444 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5445 unsigned Idx =
5446 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 2
5447 : 1;
5448 return MI.getOperand(i: Idx);
5449}
5450
5451const MachineOperand &
5452AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
5453 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5454 unsigned Idx =
5455 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 3
5456 : 2;
5457 return MI.getOperand(i: Idx);
5458}
5459
5460const MachineOperand &
5461AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
5462 switch (MI.getOpcode()) {
5463 default:
5464 llvm_unreachable("Unexpected opcode");
5465 case AArch64::LDRBroX:
5466 case AArch64::LDRBBroX:
5467 case AArch64::LDRSBXroX:
5468 case AArch64::LDRSBWroX:
5469 case AArch64::LDRHroX:
5470 case AArch64::LDRHHroX:
5471 case AArch64::LDRSHXroX:
5472 case AArch64::LDRSHWroX:
5473 case AArch64::LDRWroX:
5474 case AArch64::LDRSroX:
5475 case AArch64::LDRSWroX:
5476 case AArch64::LDRDroX:
5477 case AArch64::LDRXroX:
5478 case AArch64::LDRQroX:
5479 return MI.getOperand(i: 4);
5480 }
5481}
5482
5483static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
5484 Register Reg) {
5485 if (MI.getParent() == nullptr)
5486 return nullptr;
5487 const MachineFunction *MF = MI.getParent()->getParent();
5488 return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
5489}
5490
5491bool AArch64InstrInfo::isHForm(const MachineInstr &MI) {
5492 auto IsHFPR = [&](const MachineOperand &Op) {
5493 if (!Op.isReg())
5494 return false;
5495 auto Reg = Op.getReg();
5496 if (Reg.isPhysical())
5497 return AArch64::FPR16RegClass.contains(Reg);
5498 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5499 return TRC == &AArch64::FPR16RegClass ||
5500 TRC == &AArch64::FPR16_loRegClass;
5501 };
5502 return llvm::any_of(Range: MI.operands(), P: IsHFPR);
5503}
5504
5505bool AArch64InstrInfo::isQForm(const MachineInstr &MI) {
5506 auto IsQFPR = [&](const MachineOperand &Op) {
5507 if (!Op.isReg())
5508 return false;
5509 auto Reg = Op.getReg();
5510 if (Reg.isPhysical())
5511 return AArch64::FPR128RegClass.contains(Reg);
5512 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5513 return TRC == &AArch64::FPR128RegClass ||
5514 TRC == &AArch64::FPR128_loRegClass;
5515 };
5516 return llvm::any_of(Range: MI.operands(), P: IsQFPR);
5517}
5518
5519bool AArch64InstrInfo::hasBTISemantics(const MachineInstr &MI) {
5520 switch (MI.getOpcode()) {
5521 case AArch64::BRK:
5522 case AArch64::HLT:
5523 case AArch64::PACIASP:
5524 case AArch64::PACIBSP:
5525 // Implicit BTI behavior.
5526 return true;
5527 case AArch64::PAUTH_PROLOGUE:
5528 // PAUTH_PROLOGUE expands to PACI(A|B)SP.
5529 return true;
5530 case AArch64::HINT: {
5531 unsigned Imm = MI.getOperand(i: 0).getImm();
5532 // Explicit BTI instruction.
5533 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
5534 return true;
5535 // PACI(A|B)SP instructions.
5536 if (Imm == 25 || Imm == 27)
5537 return true;
5538 return false;
5539 }
5540 default:
5541 return false;
5542 }
5543}
5544
5545bool AArch64InstrInfo::isFpOrNEON(Register Reg) {
5546 if (Reg == 0)
5547 return false;
5548 assert(Reg.isPhysical() && "Expected physical register in isFpOrNEON");
5549 return AArch64::FPR128RegClass.contains(Reg) ||
5550 AArch64::FPR64RegClass.contains(Reg) ||
5551 AArch64::FPR32RegClass.contains(Reg) ||
5552 AArch64::FPR16RegClass.contains(Reg) ||
5553 AArch64::FPR8RegClass.contains(Reg);
5554}
5555
5556bool AArch64InstrInfo::isFpOrNEON(const MachineInstr &MI) {
5557 auto IsFPR = [&](const MachineOperand &Op) {
5558 if (!Op.isReg())
5559 return false;
5560 auto Reg = Op.getReg();
5561 if (Reg.isPhysical())
5562 return isFpOrNEON(Reg);
5563
5564 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5565 return TRC == &AArch64::FPR128RegClass ||
5566 TRC == &AArch64::FPR128_loRegClass ||
5567 TRC == &AArch64::FPR64RegClass ||
5568 TRC == &AArch64::FPR64_loRegClass ||
5569 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
5570 TRC == &AArch64::FPR8RegClass;
5571 };
5572 return llvm::any_of(Range: MI.operands(), P: IsFPR);
5573}
5574
5575// Scale the unscaled offsets. Returns false if the unscaled offset can't be
5576// scaled.
5577static bool scaleOffset(unsigned Opc, int64_t &Offset) {
5578 int Scale = AArch64InstrInfo::getMemScale(Opc);
5579
5580 // If the byte-offset isn't a multiple of the stride, we can't scale this
5581 // offset.
5582 if (Offset % Scale != 0)
5583 return false;
5584
5585 // Convert the byte-offset used by unscaled into an "element" offset used
5586 // by the scaled pair load/store instructions.
5587 Offset /= Scale;
5588 return true;
5589}
5590
5591static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
5592 if (FirstOpc == SecondOpc)
5593 return true;
5594 // We can also pair sign-ext and zero-ext instructions.
5595 switch (FirstOpc) {
5596 default:
5597 return false;
5598 case AArch64::STRSui:
5599 case AArch64::STURSi:
5600 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
5601 case AArch64::STRDui:
5602 case AArch64::STURDi:
5603 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
5604 case AArch64::STRQui:
5605 case AArch64::STURQi:
5606 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
5607 case AArch64::STRWui:
5608 case AArch64::STURWi:
5609 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
5610 case AArch64::STRXui:
5611 case AArch64::STURXi:
5612 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
5613 case AArch64::LDRSui:
5614 case AArch64::LDURSi:
5615 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
5616 case AArch64::LDRDui:
5617 case AArch64::LDURDi:
5618 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
5619 case AArch64::LDRQui:
5620 case AArch64::LDURQi:
5621 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
5622 case AArch64::LDRWui:
5623 case AArch64::LDURWi:
5624 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
5625 case AArch64::LDRSWui:
5626 case AArch64::LDURSWi:
5627 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
5628 case AArch64::LDRXui:
5629 case AArch64::LDURXi:
5630 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
5631 }
5632 // These instructions can't be paired based on their opcodes.
5633 return false;
5634}
5635
5636static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
5637 int64_t Offset1, unsigned Opcode1, int FI2,
5638 int64_t Offset2, unsigned Opcode2) {
5639 // Accesses through fixed stack object frame indices may access a different
5640 // fixed stack slot. Check that the object offsets + offsets match.
5641 if (MFI.isFixedObjectIndex(ObjectIdx: FI1) && MFI.isFixedObjectIndex(ObjectIdx: FI2)) {
5642 int64_t ObjectOffset1 = MFI.getObjectOffset(ObjectIdx: FI1);
5643 int64_t ObjectOffset2 = MFI.getObjectOffset(ObjectIdx: FI2);
5644 assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
5645 // Convert to scaled object offsets.
5646 int Scale1 = AArch64InstrInfo::getMemScale(Opc: Opcode1);
5647 if (ObjectOffset1 % Scale1 != 0)
5648 return false;
5649 ObjectOffset1 /= Scale1;
5650 int Scale2 = AArch64InstrInfo::getMemScale(Opc: Opcode2);
5651 if (ObjectOffset2 % Scale2 != 0)
5652 return false;
5653 ObjectOffset2 /= Scale2;
5654 ObjectOffset1 += Offset1;
5655 ObjectOffset2 += Offset2;
5656 return ObjectOffset1 + 1 == ObjectOffset2;
5657 }
5658
5659 return FI1 == FI2;
5660}
5661
5662/// Detect opportunities for ldp/stp formation.
5663///
5664/// Only called for LdSt for which getMemOperandWithOffset returns true.
5665bool AArch64InstrInfo::shouldClusterMemOps(
5666 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
5667 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
5668 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
5669 unsigned NumBytes) const {
5670 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
5671 const MachineOperand &BaseOp1 = *BaseOps1.front();
5672 const MachineOperand &BaseOp2 = *BaseOps2.front();
5673 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
5674 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
5675 if (BaseOp1.getType() != BaseOp2.getType())
5676 return false;
5677
5678 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
5679 "Only base registers and frame indices are supported.");
5680
5681 // Check for both base regs and base FI.
5682 if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
5683 return false;
5684
5685 // Only cluster up to a single pair.
5686 if (ClusterSize > 2)
5687 return false;
5688
5689 if (!isPairableLdStInst(MI: FirstLdSt) || !isPairableLdStInst(MI: SecondLdSt))
5690 return false;
5691
5692 // Can we pair these instructions based on their opcodes?
5693 unsigned FirstOpc = FirstLdSt.getOpcode();
5694 unsigned SecondOpc = SecondLdSt.getOpcode();
5695 if (!canPairLdStOpc(FirstOpc, SecondOpc))
5696 return false;
5697
5698 // Can't merge volatiles or load/stores that have a hint to avoid pair
5699 // formation, for example.
5700 if (!isCandidateToMergeOrPair(MI: FirstLdSt) ||
5701 !isCandidateToMergeOrPair(MI: SecondLdSt))
5702 return false;
5703
5704 // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
5705 int64_t Offset1 = FirstLdSt.getOperand(i: 2).getImm();
5706 if (hasUnscaledLdStOffset(Opc: FirstOpc) && !scaleOffset(Opc: FirstOpc, Offset&: Offset1))
5707 return false;
5708
5709 int64_t Offset2 = SecondLdSt.getOperand(i: 2).getImm();
5710 if (hasUnscaledLdStOffset(Opc: SecondOpc) && !scaleOffset(Opc: SecondOpc, Offset&: Offset2))
5711 return false;
5712
5713 // Pairwise instructions have a 7-bit signed offset field.
5714 if (Offset1 > 63 || Offset1 < -64)
5715 return false;
5716
5717 // The caller should already have ordered First/SecondLdSt by offset.
5718 // Note: except for non-equal frame index bases
5719 if (BaseOp1.isFI()) {
5720 assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
5721 "Caller should have ordered offsets.");
5722
5723 const MachineFrameInfo &MFI =
5724 FirstLdSt.getParent()->getParent()->getFrameInfo();
5725 return shouldClusterFI(MFI, FI1: BaseOp1.getIndex(), Offset1, Opcode1: FirstOpc,
5726 FI2: BaseOp2.getIndex(), Offset2, Opcode2: SecondOpc);
5727 }
5728
5729 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
5730
5731 return Offset1 + 1 == Offset2;
5732}
5733
5734static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
5735 MCRegister Reg, unsigned SubIdx,
5736 RegState State,
5737 const TargetRegisterInfo *TRI) {
5738 if (!SubIdx)
5739 return MIB.addReg(RegNo: Reg, Flags: State);
5740
5741 if (Reg.isPhysical())
5742 return MIB.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx), Flags: State);
5743 return MIB.addReg(RegNo: Reg, Flags: State, SubReg: SubIdx);
5744}
5745
5746void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
5747 MachineBasicBlock::iterator I,
5748 const DebugLoc &DL, MCRegister DestReg,
5749 MCRegister SrcReg, bool KillSrc,
5750 ArrayRef<unsigned> Indices) const {
5751 assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
5752 const TargetRegisterInfo *TRI = &getRegisterInfo();
5753 uint16_t DestEncoding = TRI->getEncodingValue(Reg: DestReg);
5754 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
5755 unsigned NumRegs = Indices.size();
5756 MCRegister DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[0]);
5757 assert(!AArch64::PNRRegClass.contains(DestSubReg) &&
5758 "Unexpected predicate tuple copy");
5759 unsigned MaxRegs = AArch64::PPRRegClass.contains(Reg: DestSubReg) ? 15 : 31;
5760
5761 int SubReg = 0, End = NumRegs, Incr = 1;
5762 // Copy in reverse if a forward copy will clobber the tuple
5763 if (((DestEncoding - SrcEncoding) & MaxRegs) < NumRegs) {
5764 SubReg = NumRegs - 1;
5765 End = -1;
5766 Incr = -1;
5767 }
5768
5769 for (; SubReg != End; SubReg += Incr) {
5770 DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[SubReg]);
5771 MCRegister SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: Indices[SubReg]);
5772 copyPhysRegImpl(MBB, I, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc);
5773 }
5774}
5775
5776void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
5777 MachineBasicBlock::iterator I,
5778 const DebugLoc &DL, MCRegister DestReg,
5779 MCRegister SrcReg, bool KillSrc,
5780 unsigned Opcode, unsigned ZeroReg,
5781 llvm::ArrayRef<unsigned> Indices) const {
5782 const TargetRegisterInfo *TRI = &getRegisterInfo();
5783 unsigned NumRegs = Indices.size();
5784
5785#ifndef NDEBUG
5786 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
5787 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
5788 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5789 "GPR reg sequences should not be able to overlap");
5790#endif
5791
5792 for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
5793 const MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode));
5794 AddSubReg(MIB, Reg: DestReg, SubIdx: Indices[SubReg], State: RegState::Define, TRI);
5795 MIB.addReg(RegNo: ZeroReg);
5796 AddSubReg(MIB, Reg: SrcReg, SubIdx: Indices[SubReg], State: getKillRegState(B: KillSrc), TRI);
5797 MIB.addImm(Val: 0);
5798 }
5799}
5800
5801/// Returns true if the instruction at I is in a streaming call site region,
5802/// within a single basic block.
5803/// A "call site streaming region" starts after smstart and ends at smstop
5804/// around a call to a streaming function. This walks backward from I.
5805static bool isInStreamingCallSiteRegion(MachineBasicBlock &MBB,
5806 MachineBasicBlock::iterator I) {
5807 MachineFunction &MF = *MBB.getParent();
5808 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
5809 if (!AFI->hasStreamingModeChanges())
5810 return false;
5811 // Walk backwards to find smstart/smstop
5812 for (MachineInstr &MI : reverse(C: make_range(x: MBB.begin(), y: I))) {
5813 unsigned Opc = MI.getOpcode();
5814 if (Opc == AArch64::MSRpstatesvcrImm1 || Opc == AArch64::MSRpstatePseudo) {
5815 // Check if this is SM change (not ZA)
5816 int64_t PState = MI.getOperand(i: 0).getImm();
5817 if (PState == AArch64SVCR::SVCRSM || PState == AArch64SVCR::SVCRSMZA) {
5818 // Operand 1 is 1 for start, 0 for stop
5819 return MI.getOperand(i: 1).getImm() == 1;
5820 }
5821 }
5822 }
5823 return false;
5824}
5825
5826/// Returns true if in a streaming call site region without SME-FA64.
5827static bool mustAvoidNeonAtMBBI(const AArch64Subtarget &Subtarget,
5828 MachineBasicBlock &MBB,
5829 MachineBasicBlock::iterator I) {
5830 return !Subtarget.hasSMEFA64() && isInStreamingCallSiteRegion(MBB, I);
5831}
5832
5833void AArch64InstrInfo::copyPhysRegImpl(MachineBasicBlock &MBB,
5834 MachineBasicBlock::iterator I,
5835 const DebugLoc &DL, Register DestReg,
5836 Register SrcReg, bool KillSrc,
5837 bool RenamableDest,
5838 bool RenamableSrc) const {
5839 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) &&
5840 AArch64::GPR32spRegClass.contains(Reg: SrcReg)) {
5841 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5842 // If either operand is WSP, expand to ADD #0.
5843 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5844 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5845 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
5846 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5847 RC: &AArch64::GPR64spRegClass);
5848 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5849 RC: &AArch64::GPR64spRegClass);
5850 // This instruction is reading and writing X registers. This may upset
5851 // the register scavenger and machine verifier, so we need to indicate
5852 // that we are reading an undefined value from SrcRegX, but a proper
5853 // value from SrcReg.
5854 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: DestRegX)
5855 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5856 .addImm(Val: 0)
5857 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
5858 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5859 ++NumZCRegMoveInstrsGPR;
5860 } else {
5861 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDWri), DestReg)
5862 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5863 .addImm(Val: 0)
5864 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5865 if (Subtarget.hasZeroCycleRegMoveGPR32())
5866 ++NumZCRegMoveInstrsGPR;
5867 }
5868 } else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5869 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5870 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
5871 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5872 RC: &AArch64::GPR64spRegClass);
5873 assert(DestRegX.isValid() && "Destination super-reg not valid");
5874 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5875 RC: &AArch64::GPR64spRegClass);
5876 assert(SrcRegX.isValid() && "Source super-reg not valid");
5877 // This instruction is reading and writing X registers. This may upset
5878 // the register scavenger and machine verifier, so we need to indicate
5879 // that we are reading an undefined value from SrcRegX, but a proper
5880 // value from SrcReg.
5881 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg: DestRegX)
5882 .addReg(RegNo: AArch64::XZR)
5883 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5884 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5885 ++NumZCRegMoveInstrsGPR;
5886 } else {
5887 // Otherwise, expand to ORR WZR.
5888 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5889 .addReg(RegNo: AArch64::WZR)
5890 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5891 if (Subtarget.hasZeroCycleRegMoveGPR32())
5892 ++NumZCRegMoveInstrsGPR;
5893 }
5894 return;
5895 }
5896
5897 // GPR32 zeroing
5898 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::WZR) {
5899 if (Subtarget.hasZeroCycleZeroingGPR64() &&
5900 !Subtarget.hasZeroCycleZeroingGPR32()) {
5901 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5902 RC: &AArch64::GPR64spRegClass);
5903 assert(DestRegX.isValid() && "Destination super-reg not valid");
5904 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: DestRegX)
5905 .addImm(Val: 0)
5906 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5907 ++NumZCZeroingInstrsGPR;
5908 } else if (Subtarget.hasZeroCycleZeroingGPR32()) {
5909 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZWi), DestReg)
5910 .addImm(Val: 0)
5911 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5912 ++NumZCZeroingInstrsGPR;
5913 } else {
5914 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5915 .addReg(RegNo: AArch64::WZR)
5916 .addReg(RegNo: AArch64::WZR);
5917 }
5918 return;
5919 }
5920
5921 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) &&
5922 AArch64::GPR64spRegClass.contains(Reg: SrcReg)) {
5923 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5924 // If either operand is SP, expand to ADD #0.
5925 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg)
5926 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5927 .addImm(Val: 0)
5928 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5929 if (Subtarget.hasZeroCycleRegMoveGPR64())
5930 ++NumZCRegMoveInstrsGPR;
5931 } else {
5932 // Otherwise, expand to ORR XZR.
5933 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5934 .addReg(RegNo: AArch64::XZR)
5935 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5936 if (Subtarget.hasZeroCycleRegMoveGPR64())
5937 ++NumZCRegMoveInstrsGPR;
5938 }
5939 return;
5940 }
5941
5942 // GPR64 zeroing
5943 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::XZR) {
5944 if (Subtarget.hasZeroCycleZeroingGPR64()) {
5945 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg)
5946 .addImm(Val: 0)
5947 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5948 ++NumZCZeroingInstrsGPR;
5949 } else {
5950 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5951 .addReg(RegNo: AArch64::XZR)
5952 .addReg(RegNo: AArch64::XZR);
5953 }
5954 return;
5955 }
5956
5957 // Copy a Predicate register by ORRing with itself.
5958 if (AArch64::PPRRegClass.contains(Reg: DestReg) &&
5959 AArch64::PPRRegClass.contains(Reg: SrcReg)) {
5960 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5961 "Unexpected SVE register.");
5962 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg)
5963 .addReg(RegNo: SrcReg) // Pg
5964 .addReg(RegNo: SrcReg)
5965 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5966 return;
5967 }
5968
5969 // Copy a predicate-as-counter register by ORRing with itself as if it
5970 // were a regular predicate (mask) register.
5971 bool DestIsPNR = AArch64::PNRRegClass.contains(Reg: DestReg);
5972 bool SrcIsPNR = AArch64::PNRRegClass.contains(Reg: SrcReg);
5973 if (DestIsPNR || SrcIsPNR) {
5974 auto ToPPR = [](MCRegister R) -> MCRegister {
5975 return (R - AArch64::PN0) + AArch64::P0;
5976 };
5977 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg.asMCReg();
5978 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg.asMCReg();
5979
5980 if (PPRSrcReg != PPRDestReg) {
5981 auto NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg: PPRDestReg)
5982 .addReg(RegNo: PPRSrcReg) // Pg
5983 .addReg(RegNo: PPRSrcReg)
5984 .addReg(RegNo: PPRSrcReg, Flags: getKillRegState(B: KillSrc));
5985 if (DestIsPNR)
5986 NewMI.addDef(RegNo: DestReg, Flags: RegState::Implicit);
5987 }
5988 return;
5989 }
5990
5991 // Copy a predicate register pair by copying the individual sub-registers.
5992 if (AArch64::PPR2RegClass.contains(Reg: DestReg) &&
5993 AArch64::PPR2RegClass.contains(Reg: SrcReg)) {
5994 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5995 "Unexpected SVE predicate register.");
5996 static const unsigned Indices[] = {AArch64::psub0, AArch64::psub1};
5997 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
5998 return;
5999 }
6000
6001 // Copy a Z register by ORRing with itself.
6002 if (AArch64::ZPRRegClass.contains(Reg: DestReg) &&
6003 AArch64::ZPRRegClass.contains(Reg: SrcReg)) {
6004 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6005 "Unexpected SVE register.");
6006 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ), DestReg)
6007 .addReg(RegNo: SrcReg)
6008 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6009 return;
6010 }
6011
6012 // Copy a Z register pair by copying the individual sub-registers.
6013 if ((AArch64::ZPR2RegClass.contains(Reg: DestReg) ||
6014 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6015 (AArch64::ZPR2RegClass.contains(Reg: SrcReg) ||
6016 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6017 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6018 "Unexpected SVE register.");
6019 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
6020 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6021 return;
6022 }
6023
6024 // Copy a Z register triple by copying the individual sub-registers.
6025 if (AArch64::ZPR3RegClass.contains(Reg: DestReg) &&
6026 AArch64::ZPR3RegClass.contains(Reg: SrcReg)) {
6027 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6028 "Unexpected SVE register.");
6029 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6030 AArch64::zsub2};
6031 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6032 return;
6033 }
6034
6035 // Copy a Z register quad by copying the individual sub-registers.
6036 if ((AArch64::ZPR4RegClass.contains(Reg: DestReg) ||
6037 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6038 (AArch64::ZPR4RegClass.contains(Reg: SrcReg) ||
6039 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6040 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6041 "Unexpected SVE register.");
6042 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6043 AArch64::zsub2, AArch64::zsub3};
6044 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6045 return;
6046 }
6047
6048 // Copy a DDDD register quad by copying the individual sub-registers.
6049 if (AArch64::DDDDRegClass.contains(Reg: DestReg) &&
6050 AArch64::DDDDRegClass.contains(Reg: SrcReg)) {
6051 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6052 AArch64::dsub2, AArch64::dsub3};
6053 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6054 return;
6055 }
6056
6057 // Copy a DDD register triple by copying the individual sub-registers.
6058 if (AArch64::DDDRegClass.contains(Reg: DestReg) &&
6059 AArch64::DDDRegClass.contains(Reg: SrcReg)) {
6060 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6061 AArch64::dsub2};
6062 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6063 return;
6064 }
6065
6066 // Copy a DD register pair by copying the individual sub-registers.
6067 if (AArch64::DDRegClass.contains(Reg: DestReg) &&
6068 AArch64::DDRegClass.contains(Reg: SrcReg)) {
6069 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
6070 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6071 return;
6072 }
6073
6074 // Copy a QQQQ register quad by copying the individual sub-registers.
6075 if (AArch64::QQQQRegClass.contains(Reg: DestReg) &&
6076 AArch64::QQQQRegClass.contains(Reg: SrcReg)) {
6077 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6078 AArch64::qsub2, AArch64::qsub3};
6079 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6080 return;
6081 }
6082
6083 // Copy a QQQ register triple by copying the individual sub-registers.
6084 if (AArch64::QQQRegClass.contains(Reg: DestReg) &&
6085 AArch64::QQQRegClass.contains(Reg: SrcReg)) {
6086 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6087 AArch64::qsub2};
6088 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6089 return;
6090 }
6091
6092 // Copy a QQ register pair by copying the individual sub-registers.
6093 if (AArch64::QQRegClass.contains(Reg: DestReg) &&
6094 AArch64::QQRegClass.contains(Reg: SrcReg)) {
6095 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
6096 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6097 return;
6098 }
6099
6100 if (AArch64::XSeqPairsClassRegClass.contains(Reg: DestReg) &&
6101 AArch64::XSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6102 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
6103 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRXrs,
6104 ZeroReg: AArch64::XZR, Indices);
6105 return;
6106 }
6107
6108 if (AArch64::WSeqPairsClassRegClass.contains(Reg: DestReg) &&
6109 AArch64::WSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6110 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
6111 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRWrs,
6112 ZeroReg: AArch64::WZR, Indices);
6113 return;
6114 }
6115
6116 if (AArch64::FPR128RegClass.contains(Reg: DestReg) &&
6117 AArch64::FPR128RegClass.contains(Reg: SrcReg)) {
6118 // In streaming regions, NEON is illegal but streaming-SVE is available.
6119 // Use SVE for copies if we're in a streaming region and SME is available.
6120 // With +sme-fa64, NEON is legal in streaming mode so we can use it.
6121 if ((Subtarget.isSVEorStreamingSVEAvailable() &&
6122 !Subtarget.isNeonAvailable()) ||
6123 mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6124 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ))
6125 .addReg(RegNo: AArch64::Z0 + (DestReg - AArch64::Q0), Flags: RegState::Define)
6126 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0))
6127 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0));
6128 } else if (Subtarget.isNeonAvailable()) {
6129 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg)
6130 .addReg(RegNo: SrcReg)
6131 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6132 if (Subtarget.hasZeroCycleRegMoveFPR128())
6133 ++NumZCRegMoveInstrsFPR;
6134 } else {
6135 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::STRQpre))
6136 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6137 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6138 .addReg(RegNo: AArch64::SP)
6139 .addImm(Val: -16);
6140 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::LDRQpost))
6141 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6142 .addReg(RegNo: DestReg, Flags: RegState::Define)
6143 .addReg(RegNo: AArch64::SP)
6144 .addImm(Val: 16);
6145 }
6146 return;
6147 }
6148
6149 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6150 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6151 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6152 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6153 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6154 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6155 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
6156 RC: &AArch64::FPR128RegClass);
6157 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::dsub,
6158 RC: &AArch64::FPR128RegClass);
6159 // This instruction is reading and writing Q registers. This may upset
6160 // the register scavenger and machine verifier, so we need to indicate
6161 // that we are reading an undefined value from SrcRegQ, but a proper
6162 // value from SrcReg.
6163 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6164 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6165 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6166 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6167 ++NumZCRegMoveInstrsFPR;
6168 } else {
6169 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg)
6170 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6171 if (Subtarget.hasZeroCycleRegMoveFPR64())
6172 ++NumZCRegMoveInstrsFPR;
6173 }
6174 return;
6175 }
6176
6177 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6178 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6179 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6180 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6181 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6182 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6183 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6184 RC: &AArch64::FPR128RegClass);
6185 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6186 RC: &AArch64::FPR128RegClass);
6187 // This instruction is reading and writing Q registers. This may upset
6188 // the register scavenger and machine verifier, so we need to indicate
6189 // that we are reading an undefined value from SrcRegQ, but a proper
6190 // value from SrcReg.
6191 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6192 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6193 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6194 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6195 ++NumZCRegMoveInstrsFPR;
6196 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6197 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6198 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6199 RC: &AArch64::FPR64RegClass);
6200 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6201 RC: &AArch64::FPR64RegClass);
6202 // This instruction is reading and writing D registers. This may upset
6203 // the register scavenger and machine verifier, so we need to indicate
6204 // that we are reading an undefined value from SrcRegD, but a proper
6205 // value from SrcReg.
6206 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6207 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6208 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6209 ++NumZCRegMoveInstrsFPR;
6210 } else {
6211 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6212 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6213 if (Subtarget.hasZeroCycleRegMoveFPR32())
6214 ++NumZCRegMoveInstrsFPR;
6215 }
6216 return;
6217 }
6218
6219 if (AArch64::FPR16RegClass.contains(Reg: DestReg) &&
6220 AArch64::FPR16RegClass.contains(Reg: SrcReg)) {
6221 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6222 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6223 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6224 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6225 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6226 RC: &AArch64::FPR128RegClass);
6227 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6228 RC: &AArch64::FPR128RegClass);
6229 // This instruction is reading and writing Q registers. This may upset
6230 // the register scavenger and machine verifier, so we need to indicate
6231 // that we are reading an undefined value from SrcRegQ, but a proper
6232 // value from SrcReg.
6233 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6234 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6235 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6236 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6237 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6238 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6239 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6240 RC: &AArch64::FPR64RegClass);
6241 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6242 RC: &AArch64::FPR64RegClass);
6243 // This instruction is reading and writing D registers. This may upset
6244 // the register scavenger and machine verifier, so we need to indicate
6245 // that we are reading an undefined value from SrcRegD, but a proper
6246 // value from SrcReg.
6247 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6248 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6249 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6250 } else {
6251 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6252 RC: &AArch64::FPR32RegClass);
6253 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6254 RC: &AArch64::FPR32RegClass);
6255 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6256 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6257 }
6258 return;
6259 }
6260
6261 if (AArch64::FPR8RegClass.contains(Reg: DestReg) &&
6262 AArch64::FPR8RegClass.contains(Reg: SrcReg)) {
6263 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6264 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6265 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6266 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6267 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6268 RC: &AArch64::FPR128RegClass);
6269 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6270 RC: &AArch64::FPR128RegClass);
6271 // This instruction is reading and writing Q registers. This may upset
6272 // the register scavenger and machine verifier, so we need to indicate
6273 // that we are reading an undefined value from SrcRegQ, but a proper
6274 // value from SrcReg.
6275 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6276 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6277 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6278 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6279 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6280 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6281 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6282 RC: &AArch64::FPR64RegClass);
6283 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6284 RC: &AArch64::FPR64RegClass);
6285 // This instruction is reading and writing D registers. This may upset
6286 // the register scavenger and machine verifier, so we need to indicate
6287 // that we are reading an undefined value from SrcRegD, but a proper
6288 // value from SrcReg.
6289 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6290 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6291 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6292 } else {
6293 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6294 RC: &AArch64::FPR32RegClass);
6295 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6296 RC: &AArch64::FPR32RegClass);
6297 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6298 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6299 }
6300 return;
6301 }
6302
6303 // Copies between GPR64 and FPR64.
6304 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6305 AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
6306 if (AArch64::XZR == SrcReg) {
6307 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg);
6308 } else {
6309 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVXDr), DestReg)
6310 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6311 }
6312 return;
6313 }
6314 if (AArch64::GPR64RegClass.contains(Reg: DestReg) &&
6315 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6316 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDXr), DestReg)
6317 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6318 return;
6319 }
6320 // Copies between GPR32 and FPR32.
6321 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6322 AArch64::GPR32RegClass.contains(Reg: SrcReg)) {
6323 if (AArch64::WZR == SrcReg) {
6324 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVS0), DestReg);
6325 } else {
6326 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVWSr), DestReg)
6327 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6328 }
6329 return;
6330 }
6331 if (AArch64::GPR32RegClass.contains(Reg: DestReg) &&
6332 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6333 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSWr), DestReg)
6334 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6335 return;
6336 }
6337
6338 if (DestReg == AArch64::NZCV) {
6339 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
6340 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MSR))
6341 .addImm(Val: AArch64SysReg::NZCV)
6342 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6343 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Define);
6344 return;
6345 }
6346
6347 if (SrcReg == AArch64::NZCV) {
6348 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
6349 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MRS), DestReg)
6350 .addImm(Val: AArch64SysReg::NZCV)
6351 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6352 return;
6353 }
6354
6355#ifndef NDEBUG
6356 errs() << RI.getRegAsmName(DestReg) << " = COPY " << RI.getRegAsmName(SrcReg)
6357 << "\n";
6358#endif
6359 llvm_unreachable("unimplemented reg-to-reg copy");
6360}
6361
6362void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
6363 MachineBasicBlock::iterator I,
6364 const DebugLoc &DL, Register DestReg,
6365 Register SrcReg, bool KillSrc,
6366 bool RenamableDest,
6367 bool RenamableSrc) const {
6368 ++NumCopyInstrs;
6369 copyPhysRegImpl(MBB, I, DL, DestReg, SrcReg, KillSrc, RenamableDest,
6370 RenamableSrc);
6371 return;
6372}
6373
6374static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
6375 MachineBasicBlock &MBB,
6376 MachineBasicBlock::iterator InsertBefore,
6377 const MCInstrDesc &MCID,
6378 Register SrcReg, bool IsKill,
6379 unsigned SubIdx0, unsigned SubIdx1, int FI,
6380 MachineMemOperand *MMO) {
6381 Register SrcReg0 = SrcReg;
6382 Register SrcReg1 = SrcReg;
6383 if (SrcReg.isPhysical()) {
6384 SrcReg0 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx0);
6385 SubIdx0 = 0;
6386 SrcReg1 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx1);
6387 SubIdx1 = 0;
6388 }
6389 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6390 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: IsKill), SubReg: SubIdx0)
6391 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: IsKill), SubReg: SubIdx1)
6392 .addFrameIndex(Idx: FI)
6393 .addImm(Val: 0)
6394 .addMemOperand(MMO);
6395}
6396
6397void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
6398 MachineBasicBlock::iterator MBBI,
6399 Register SrcReg, bool isKill, int FI,
6400 const TargetRegisterClass *RC,
6401 Register VReg,
6402 MachineInstr::MIFlag Flags) const {
6403 MachineFunction &MF = *MBB.getParent();
6404 MachineFrameInfo &MFI = MF.getFrameInfo();
6405
6406 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6407 MachineMemOperand *MMO =
6408 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
6409 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6410 unsigned Opc = 0;
6411 bool Offset = true;
6412 MCRegister PNRReg = MCRegister::NoRegister;
6413 unsigned StackID = TargetStackID::Default;
6414 switch (RI.getSpillSize(RC: *RC)) {
6415 case 1:
6416 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6417 Opc = AArch64::STRBui;
6418 break;
6419 case 2: {
6420 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6421 Opc = AArch64::STRHui;
6422 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
6423 AArch64::PPRRegClass.hasSubClassEq(RC)) {
6424 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6425 "Unexpected register store without SVE store instructions");
6426 Opc = AArch64::STR_PXI;
6427 StackID = TargetStackID::ScalablePredicateVector;
6428 }
6429 break;
6430 }
6431 case 4:
6432 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6433 Opc = AArch64::STRWui;
6434 if (SrcReg.isVirtual())
6435 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32RegClass);
6436 else
6437 assert(SrcReg != AArch64::WSP);
6438 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6439 Opc = AArch64::STRSui;
6440 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6441 Opc = AArch64::STR_PPXI;
6442 StackID = TargetStackID::ScalablePredicateVector;
6443 }
6444 break;
6445 case 8:
6446 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6447 Opc = AArch64::STRXui;
6448 if (SrcReg.isVirtual())
6449 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
6450 else
6451 assert(SrcReg != AArch64::SP);
6452 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6453 Opc = AArch64::STRDui;
6454 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6455 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6456 MCID: get(Opcode: AArch64::STPWi), SrcReg, IsKill: isKill,
6457 SubIdx0: AArch64::sube32, SubIdx1: AArch64::subo32, FI, MMO);
6458 return;
6459 }
6460 break;
6461 case 16:
6462 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6463 Opc = AArch64::STRQui;
6464 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6465 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6466 Opc = AArch64::ST1Twov1d;
6467 Offset = false;
6468 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6469 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6470 MCID: get(Opcode: AArch64::STPXi), SrcReg, IsKill: isKill,
6471 SubIdx0: AArch64::sube64, SubIdx1: AArch64::subo64, FI, MMO);
6472 return;
6473 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6474 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6475 "Unexpected register store without SVE store instructions");
6476 Opc = AArch64::STR_ZXI;
6477 StackID = TargetStackID::ScalableVector;
6478 }
6479 break;
6480 case 24:
6481 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6482 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6483 Opc = AArch64::ST1Threev1d;
6484 Offset = false;
6485 }
6486 break;
6487 case 32:
6488 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6489 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6490 Opc = AArch64::ST1Fourv1d;
6491 Offset = false;
6492 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6493 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6494 Opc = AArch64::ST1Twov2d;
6495 Offset = false;
6496 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6497 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6498 "Unexpected register store without SVE store instructions");
6499 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
6500 StackID = TargetStackID::ScalableVector;
6501 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6502 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6503 "Unexpected register store without SVE store instructions");
6504 Opc = AArch64::STR_ZZXI;
6505 StackID = TargetStackID::ScalableVector;
6506 }
6507 break;
6508 case 48:
6509 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6510 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6511 Opc = AArch64::ST1Threev2d;
6512 Offset = false;
6513 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6514 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6515 "Unexpected register store without SVE store instructions");
6516 Opc = AArch64::STR_ZZZXI;
6517 StackID = TargetStackID::ScalableVector;
6518 }
6519 break;
6520 case 64:
6521 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6522 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6523 Opc = AArch64::ST1Fourv2d;
6524 Offset = false;
6525 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6526 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6527 "Unexpected register store without SVE store instructions");
6528 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
6529 StackID = TargetStackID::ScalableVector;
6530 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6531 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6532 "Unexpected register store without SVE store instructions");
6533 Opc = AArch64::STR_ZZZZXI;
6534 StackID = TargetStackID::ScalableVector;
6535 }
6536 break;
6537 }
6538 assert(Opc && "Unknown register class");
6539 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6540
6541 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6542 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
6543 .addFrameIndex(Idx: FI);
6544
6545 if (Offset)
6546 MI.addImm(Val: 0);
6547 if (PNRReg.isValid())
6548 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6549 MI.addMemOperand(MMO);
6550}
6551
6552static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
6553 MachineBasicBlock &MBB,
6554 MachineBasicBlock::iterator InsertBefore,
6555 const MCInstrDesc &MCID,
6556 Register DestReg, unsigned SubIdx0,
6557 unsigned SubIdx1, int FI,
6558 MachineMemOperand *MMO) {
6559 Register DestReg0 = DestReg;
6560 Register DestReg1 = DestReg;
6561 bool IsUndef = true;
6562 if (DestReg.isPhysical()) {
6563 DestReg0 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx0);
6564 SubIdx0 = 0;
6565 DestReg1 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx1);
6566 SubIdx1 = 0;
6567 IsUndef = false;
6568 }
6569 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6570 .addReg(RegNo: DestReg0, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx0)
6571 .addReg(RegNo: DestReg1, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx1)
6572 .addFrameIndex(Idx: FI)
6573 .addImm(Val: 0)
6574 .addMemOperand(MMO);
6575}
6576
6577void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
6578 MachineBasicBlock::iterator MBBI,
6579 Register DestReg, int FI,
6580 const TargetRegisterClass *RC,
6581 Register VReg, unsigned SubReg,
6582 MachineInstr::MIFlag Flags) const {
6583 MachineFunction &MF = *MBB.getParent();
6584 MachineFrameInfo &MFI = MF.getFrameInfo();
6585 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6586 MachineMemOperand *MMO =
6587 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
6588 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6589
6590 unsigned Opc = 0;
6591 bool Offset = true;
6592 unsigned StackID = TargetStackID::Default;
6593 Register PNRReg = MCRegister::NoRegister;
6594 switch (TRI.getSpillSize(RC: *RC)) {
6595 case 1:
6596 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6597 Opc = AArch64::LDRBui;
6598 break;
6599 case 2: {
6600 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
6601 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6602 Opc = AArch64::LDRHui;
6603 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
6604 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6605 "Unexpected register load without SVE load instructions");
6606 if (IsPNR)
6607 PNRReg = DestReg;
6608 Opc = AArch64::LDR_PXI;
6609 StackID = TargetStackID::ScalablePredicateVector;
6610 }
6611 break;
6612 }
6613 case 4:
6614 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6615 Opc = AArch64::LDRWui;
6616 if (DestReg.isVirtual())
6617 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR32RegClass);
6618 else
6619 assert(DestReg != AArch64::WSP);
6620 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6621 Opc = AArch64::LDRSui;
6622 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6623 Opc = AArch64::LDR_PPXI;
6624 StackID = TargetStackID::ScalablePredicateVector;
6625 }
6626 break;
6627 case 8:
6628 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6629 Opc = AArch64::LDRXui;
6630 if (DestReg.isVirtual())
6631 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR64RegClass);
6632 else
6633 assert(DestReg != AArch64::SP);
6634 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6635 Opc = AArch64::LDRDui;
6636 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6637 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6638 MCID: get(Opcode: AArch64::LDPWi), DestReg, SubIdx0: AArch64::sube32,
6639 SubIdx1: AArch64::subo32, FI, MMO);
6640 return;
6641 }
6642 break;
6643 case 16:
6644 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6645 Opc = AArch64::LDRQui;
6646 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6647 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6648 Opc = AArch64::LD1Twov1d;
6649 Offset = false;
6650 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6651 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6652 MCID: get(Opcode: AArch64::LDPXi), DestReg, SubIdx0: AArch64::sube64,
6653 SubIdx1: AArch64::subo64, FI, MMO);
6654 return;
6655 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6656 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6657 "Unexpected register load without SVE load instructions");
6658 Opc = AArch64::LDR_ZXI;
6659 StackID = TargetStackID::ScalableVector;
6660 }
6661 break;
6662 case 24:
6663 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6664 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6665 Opc = AArch64::LD1Threev1d;
6666 Offset = false;
6667 }
6668 break;
6669 case 32:
6670 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6671 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6672 Opc = AArch64::LD1Fourv1d;
6673 Offset = false;
6674 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6675 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6676 Opc = AArch64::LD1Twov2d;
6677 Offset = false;
6678 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6679 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6680 "Unexpected register load without SVE load instructions");
6681 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
6682 StackID = TargetStackID::ScalableVector;
6683 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6684 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6685 "Unexpected register load without SVE load instructions");
6686 Opc = AArch64::LDR_ZZXI;
6687 StackID = TargetStackID::ScalableVector;
6688 }
6689 break;
6690 case 48:
6691 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6692 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6693 Opc = AArch64::LD1Threev2d;
6694 Offset = false;
6695 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6696 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6697 "Unexpected register load without SVE load instructions");
6698 Opc = AArch64::LDR_ZZZXI;
6699 StackID = TargetStackID::ScalableVector;
6700 }
6701 break;
6702 case 64:
6703 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6704 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6705 Opc = AArch64::LD1Fourv2d;
6706 Offset = false;
6707 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6708 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6709 "Unexpected register load without SVE load instructions");
6710 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
6711 StackID = TargetStackID::ScalableVector;
6712 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6713 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6714 "Unexpected register load without SVE load instructions");
6715 Opc = AArch64::LDR_ZZZZXI;
6716 StackID = TargetStackID::ScalableVector;
6717 }
6718 break;
6719 }
6720
6721 assert(Opc && "Unknown register class");
6722 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6723
6724 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6725 .addReg(RegNo: DestReg, Flags: getDefRegState(B: true))
6726 .addFrameIndex(Idx: FI);
6727 if (Offset)
6728 MI.addImm(Val: 0);
6729 if (PNRReg.isValid() && !PNRReg.isVirtual())
6730 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6731 MI.addMemOperand(MMO);
6732}
6733
6734bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
6735 const MachineInstr &UseMI,
6736 const TargetRegisterInfo *TRI) {
6737 return any_of(Range: instructionsWithoutDebug(It: std::next(x: DefMI.getIterator()),
6738 End: UseMI.getIterator()),
6739 P: [TRI](const MachineInstr &I) {
6740 return I.modifiesRegister(Reg: AArch64::NZCV, TRI) ||
6741 I.readsRegister(Reg: AArch64::NZCV, TRI);
6742 });
6743}
6744
6745void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6746 const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized) {
6747 // The smallest scalable element supported by scaled SVE addressing
6748 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6749 // byte offset must always be a multiple of 2.
6750 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6751
6752 // VGSized offsets are divided by '2', because the VG register is the
6753 // the number of 64bit granules as opposed to 128bit vector chunks,
6754 // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
6755 // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
6756 // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
6757 ByteSized = Offset.getFixed();
6758 VGSized = Offset.getScalable() / 2;
6759}
6760
6761/// Returns the offset in parts to which this frame offset can be
6762/// decomposed for the purpose of describing a frame offset.
6763/// For non-scalable offsets this is simply its byte size.
6764void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6765 const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors,
6766 int64_t &NumDataVectors) {
6767 // The smallest scalable element supported by scaled SVE addressing
6768 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6769 // byte offset must always be a multiple of 2.
6770 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6771
6772 NumBytes = Offset.getFixed();
6773 NumDataVectors = 0;
6774 NumPredicateVectors = Offset.getScalable() / 2;
6775 // This method is used to get the offsets to adjust the frame offset.
6776 // If the function requires ADDPL to be used and needs more than two ADDPL
6777 // instructions, part of the offset is folded into NumDataVectors so that it
6778 // uses ADDVL for part of it, reducing the number of ADDPL instructions.
6779 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
6780 NumPredicateVectors > 62) {
6781 NumDataVectors = NumPredicateVectors / 8;
6782 NumPredicateVectors -= NumDataVectors * 8;
6783 }
6784}
6785
6786// Convenience function to create a DWARF expression for: Constant `Operation`.
6787// This helper emits compact sequences for common cases. For example, for`-15
6788// DW_OP_plus`, this helper would create DW_OP_lit15 DW_OP_minus.
6789static void appendConstantExpr(SmallVectorImpl<char> &Expr, int64_t Constant,
6790 dwarf::LocationAtom Operation) {
6791 if (Operation == dwarf::DW_OP_plus && Constant < 0 && -Constant <= 31) {
6792 // -Constant (1 to 31)
6793 Expr.push_back(Elt: dwarf::DW_OP_lit0 - Constant);
6794 Operation = dwarf::DW_OP_minus;
6795 } else if (Constant >= 0 && Constant <= 31) {
6796 // Literal value 0 to 31
6797 Expr.push_back(Elt: dwarf::DW_OP_lit0 + Constant);
6798 } else {
6799 // Signed constant
6800 Expr.push_back(Elt: dwarf::DW_OP_consts);
6801 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: Constant);
6802 }
6803 return Expr.push_back(Elt: Operation);
6804}
6805
6806// Convenience function to create a DWARF expression for a register.
6807static void appendReadRegExpr(SmallVectorImpl<char> &Expr, unsigned RegNum) {
6808 Expr.push_back(Elt: (char)dwarf::DW_OP_bregx);
6809 appendLEB128<LEB128Sign::Unsigned>(Buffer&: Expr, Value: RegNum);
6810 Expr.push_back(Elt: 0);
6811}
6812
6813// Convenience function to create a DWARF expression for loading a register from
6814// a CFA offset.
6815static void appendLoadRegExpr(SmallVectorImpl<char> &Expr,
6816 int64_t OffsetFromDefCFA) {
6817 // This assumes the top of the DWARF stack contains the CFA.
6818 Expr.push_back(Elt: dwarf::DW_OP_dup);
6819 // Add the offset to the register.
6820 appendConstantExpr(Expr, Constant: OffsetFromDefCFA, Operation: dwarf::DW_OP_plus);
6821 // Dereference the address (loads a 64 bit value)..
6822 Expr.push_back(Elt: dwarf::DW_OP_deref);
6823}
6824
6825// Convenience function to create a comment for
6826// (+/-) NumBytes (* RegScale)?
6827static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
6828 StringRef RegScale = {}) {
6829 if (NumBytes) {
6830 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(x: NumBytes);
6831 if (!RegScale.empty())
6832 Comment << ' ' << RegScale;
6833 }
6834}
6835
6836// Creates an MCCFIInstruction:
6837// { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
6838static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
6839 unsigned Reg,
6840 const StackOffset &Offset) {
6841 int64_t NumBytes, NumVGScaledBytes;
6842 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(Offset, ByteSized&: NumBytes,
6843 VGSized&: NumVGScaledBytes);
6844 std::string CommentBuffer;
6845 llvm::raw_string_ostream Comment(CommentBuffer);
6846
6847 if (Reg == AArch64::SP)
6848 Comment << "sp";
6849 else if (Reg == AArch64::FP)
6850 Comment << "fp";
6851 else
6852 Comment << printReg(Reg, TRI: &TRI);
6853
6854 // Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
6855 SmallString<64> Expr;
6856 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6857 assert(DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
6858 // Reg + NumBytes
6859 Expr.push_back(Elt: dwarf::DW_OP_breg0 + DwarfReg);
6860 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: NumBytes);
6861 appendOffsetComment(NumBytes, Comment);
6862 if (NumVGScaledBytes) {
6863 // + VG * NumVGScaledBytes
6864 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: "* VG");
6865 appendReadRegExpr(Expr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6866 appendConstantExpr(Expr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6867 Expr.push_back(Elt: dwarf::DW_OP_plus);
6868 }
6869
6870 // Wrap this into DW_CFA_def_cfa.
6871 SmallString<64> DefCfaExpr;
6872 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
6873 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
6874 DefCfaExpr.append(RHS: Expr.str());
6875 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
6876 Comment: Comment.str());
6877}
6878
6879MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
6880 unsigned FrameReg, unsigned Reg,
6881 const StackOffset &Offset,
6882 bool LastAdjustmentWasScalable) {
6883 if (Offset.getScalable())
6884 return createDefCFAExpression(TRI, Reg, Offset);
6885
6886 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6887 return MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: int(Offset.getFixed()));
6888
6889 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6890 return MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfReg, Offset: (int)Offset.getFixed());
6891}
6892
6893MCCFIInstruction
6894llvm::createCFAOffset(const TargetRegisterInfo &TRI, unsigned Reg,
6895 const StackOffset &OffsetFromDefCFA,
6896 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6897 int64_t NumBytes, NumVGScaledBytes;
6898 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6899 Offset: OffsetFromDefCFA, ByteSized&: NumBytes, VGSized&: NumVGScaledBytes);
6900
6901 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6902
6903 // Non-scalable offsets can use DW_CFA_offset directly.
6904 if (!NumVGScaledBytes)
6905 return MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: NumBytes);
6906
6907 std::string CommentBuffer;
6908 llvm::raw_string_ostream Comment(CommentBuffer);
6909 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
6910
6911 // Build up expression (CFA + VG * NumVGScaledBytes + NumBytes)
6912 assert(NumVGScaledBytes && "Expected scalable offset");
6913 SmallString<64> OffsetExpr;
6914 // + VG * NumVGScaledBytes
6915 StringRef VGRegScale;
6916 if (IncomingVGOffsetFromDefCFA) {
6917 appendLoadRegExpr(Expr&: OffsetExpr, OffsetFromDefCFA: *IncomingVGOffsetFromDefCFA);
6918 VGRegScale = "* IncomingVG";
6919 } else {
6920 appendReadRegExpr(Expr&: OffsetExpr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6921 VGRegScale = "* VG";
6922 }
6923 appendConstantExpr(Expr&: OffsetExpr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6924 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: VGRegScale);
6925 OffsetExpr.push_back(Elt: dwarf::DW_OP_plus);
6926 if (NumBytes) {
6927 // + NumBytes
6928 appendOffsetComment(NumBytes, Comment);
6929 appendConstantExpr(Expr&: OffsetExpr, Constant: NumBytes, Operation: dwarf::DW_OP_plus);
6930 }
6931
6932 // Wrap this into DW_CFA_expression
6933 SmallString<64> CfaExpr;
6934 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
6935 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: DwarfReg);
6936 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: OffsetExpr.size());
6937 CfaExpr.append(RHS: OffsetExpr.str());
6938
6939 return MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str(), Loc: SMLoc(),
6940 Comment: Comment.str());
6941}
6942
6943// Helper function to emit a frame offset adjustment from a given
6944// pointer (SrcReg), stored into DestReg. This function is explicit
6945// in that it requires the opcode.
6946static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
6947 MachineBasicBlock::iterator MBBI,
6948 const DebugLoc &DL, unsigned DestReg,
6949 unsigned SrcReg, int64_t Offset, unsigned Opc,
6950 const TargetInstrInfo *TII,
6951 MachineInstr::MIFlag Flag, bool NeedsWinCFI,
6952 bool *HasWinCFI, bool EmitCFAOffset,
6953 StackOffset CFAOffset, unsigned FrameReg) {
6954 int Sign = 1;
6955 unsigned MaxEncoding, ShiftSize;
6956 switch (Opc) {
6957 case AArch64::ADDXri:
6958 case AArch64::ADDSXri:
6959 case AArch64::SUBXri:
6960 case AArch64::SUBSXri:
6961 MaxEncoding = 0xfff;
6962 ShiftSize = 12;
6963 break;
6964 case AArch64::ADDVL_XXI:
6965 case AArch64::ADDPL_XXI:
6966 case AArch64::ADDSVL_XXI:
6967 case AArch64::ADDSPL_XXI:
6968 MaxEncoding = 31;
6969 ShiftSize = 0;
6970 if (Offset < 0) {
6971 MaxEncoding = 32;
6972 Sign = -1;
6973 Offset = -Offset;
6974 }
6975 break;
6976 default:
6977 llvm_unreachable("Unsupported opcode");
6978 }
6979
6980 // `Offset` can be in bytes or in "scalable bytes".
6981 int VScale = 1;
6982 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
6983 VScale = 16;
6984 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
6985 VScale = 2;
6986
6987 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
6988 // scratch register. If DestReg is a virtual register, use it as the
6989 // scratch register; otherwise, create a new virtual register (to be
6990 // replaced by the scavenger at the end of PEI). That case can be optimized
6991 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
6992 // register can be loaded with offset%8 and the add/sub can use an extending
6993 // instruction with LSL#3.
6994 // Currently the function handles any offsets but generates a poor sequence
6995 // of code.
6996 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
6997
6998 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
6999 Register TmpReg = DestReg;
7000 if (TmpReg == AArch64::XZR)
7001 TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
7002 RegClass: &AArch64::GPR64RegClass);
7003 do {
7004 uint64_t ThisVal = std::min<uint64_t>(a: Offset, b: MaxEncodableValue);
7005 unsigned LocalShiftSize = 0;
7006 if (ThisVal > MaxEncoding) {
7007 ThisVal = ThisVal >> ShiftSize;
7008 LocalShiftSize = ShiftSize;
7009 }
7010 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
7011 "Encoding cannot handle value that big");
7012
7013 Offset -= ThisVal << LocalShiftSize;
7014 if (Offset == 0)
7015 TmpReg = DestReg;
7016 auto MBI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: TmpReg)
7017 .addReg(RegNo: SrcReg)
7018 .addImm(Val: Sign * (int)ThisVal);
7019 if (ShiftSize)
7020 MBI = MBI.addImm(
7021 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LocalShiftSize));
7022 MBI = MBI.setMIFlag(Flag);
7023
7024 auto Change =
7025 VScale == 1
7026 ? StackOffset::getFixed(Fixed: ThisVal << LocalShiftSize)
7027 : StackOffset::getScalable(Scalable: VScale * (ThisVal << LocalShiftSize));
7028 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
7029 CFAOffset += Change;
7030 else
7031 CFAOffset -= Change;
7032 if (EmitCFAOffset && DestReg == TmpReg) {
7033 MachineFunction &MF = *MBB.getParent();
7034 const TargetSubtargetInfo &STI = MF.getSubtarget();
7035 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
7036
7037 unsigned CFIIndex = MF.addFrameInst(
7038 Inst: createDefCFA(TRI, FrameReg, Reg: DestReg, Offset: CFAOffset, LastAdjustmentWasScalable: VScale != 1));
7039 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
7040 .addCFIIndex(CFIIndex)
7041 .setMIFlags(Flag);
7042 }
7043
7044 if (NeedsWinCFI) {
7045 int Imm = (int)(ThisVal << LocalShiftSize);
7046 if (VScale != 1 && DestReg == AArch64::SP) {
7047 if (HasWinCFI)
7048 *HasWinCFI = true;
7049 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AllocZ))
7050 .addImm(Val: ThisVal)
7051 .setMIFlag(Flag);
7052 } else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
7053 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
7054 assert(VScale == 1 && "Expected non-scalable operation");
7055 if (HasWinCFI)
7056 *HasWinCFI = true;
7057 if (Imm == 0)
7058 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_SetFP)).setMIFlag(Flag);
7059 else
7060 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AddFP))
7061 .addImm(Val: Imm)
7062 .setMIFlag(Flag);
7063 assert(Offset == 0 && "Expected remaining offset to be zero to "
7064 "emit a single SEH directive");
7065 } else if (DestReg == AArch64::SP) {
7066 assert(VScale == 1 && "Expected non-scalable operation");
7067 if (HasWinCFI)
7068 *HasWinCFI = true;
7069 assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
7070 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_StackAlloc))
7071 .addImm(Val: Imm)
7072 .setMIFlag(Flag);
7073 }
7074 }
7075
7076 SrcReg = TmpReg;
7077 } while (Offset);
7078}
7079
7080void llvm::emitFrameOffset(MachineBasicBlock &MBB,
7081 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
7082 unsigned DestReg, unsigned SrcReg,
7083 StackOffset Offset, const TargetInstrInfo *TII,
7084 MachineInstr::MIFlag Flag, bool SetNZCV,
7085 bool NeedsWinCFI, bool *HasWinCFI,
7086 bool EmitCFAOffset, StackOffset CFAOffset,
7087 unsigned FrameReg) {
7088 // If a function is marked as arm_locally_streaming, then the runtime value of
7089 // vscale in the prologue/epilogue is different the runtime value of vscale
7090 // in the function's body. To avoid having to consider multiple vscales,
7091 // we can use `addsvl` to allocate any scalable stack-slots, which under
7092 // most circumstances will be only locals, not callee-save slots.
7093 const Function &F = MBB.getParent()->getFunction();
7094 bool UseSVL = F.hasFnAttribute(Kind: "aarch64_pstate_sm_body");
7095
7096 int64_t Bytes, NumPredicateVectors, NumDataVectors;
7097 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
7098 Offset, NumBytes&: Bytes, NumPredicateVectors, NumDataVectors);
7099
7100 // Insert ADDSXri for scalable offset at the end.
7101 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
7102 if (NeedsFinalDefNZCV)
7103 SetNZCV = false;
7104
7105 // First emit non-scalable frame offsets, or a simple 'mov'.
7106 if (Bytes || (!Offset && SrcReg != DestReg)) {
7107 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
7108 "SP increment/decrement not 8-byte aligned");
7109 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
7110 if (Bytes < 0) {
7111 Bytes = -Bytes;
7112 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
7113 }
7114 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: Bytes, Opc, TII, Flag,
7115 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7116 FrameReg);
7117 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
7118 ? StackOffset::getFixed(Fixed: -Bytes)
7119 : StackOffset::getFixed(Fixed: Bytes);
7120 SrcReg = DestReg;
7121 FrameReg = DestReg;
7122 }
7123
7124 assert(!(NeedsWinCFI && NumPredicateVectors) &&
7125 "WinCFI can't allocate fractions of an SVE data vector");
7126
7127 if (NumDataVectors) {
7128 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumDataVectors,
7129 Opc: UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI, TII,
7130 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7131 FrameReg);
7132 CFAOffset += StackOffset::getScalable(Scalable: -NumDataVectors * 16);
7133 SrcReg = DestReg;
7134 }
7135
7136 if (NumPredicateVectors) {
7137 assert(DestReg != AArch64::SP && "Unaligned access to SP");
7138 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumPredicateVectors,
7139 Opc: UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI, TII,
7140 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7141 FrameReg);
7142 }
7143
7144 if (NeedsFinalDefNZCV)
7145 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDSXri), DestReg)
7146 .addReg(RegNo: DestReg)
7147 .addImm(Val: 0)
7148 .addImm(Val: 0);
7149}
7150
7151MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
7152 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
7153 int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS,
7154 VirtRegMap *VRM) const {
7155 MachineBasicBlock::iterator InsertPt = MI;
7156 // This is a bit of a hack. Consider this instruction:
7157 //
7158 // %0 = COPY %sp; GPR64all:%0
7159 //
7160 // We explicitly chose GPR64all for the virtual register so such a copy might
7161 // be eliminated by RegisterCoalescer. However, that may not be possible, and
7162 // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
7163 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
7164 //
7165 // To prevent that, we are going to constrain the %0 register class here.
7166 if (MI.isFullCopy()) {
7167 Register DstReg = MI.getOperand(i: 0).getReg();
7168 Register SrcReg = MI.getOperand(i: 1).getReg();
7169 if (SrcReg == AArch64::SP && DstReg.isVirtual()) {
7170 MF.getRegInfo().constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass);
7171 return nullptr;
7172 }
7173 if (DstReg == AArch64::SP && SrcReg.isVirtual()) {
7174 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
7175 return nullptr;
7176 }
7177 // Nothing can folded with copy from/to NZCV.
7178 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
7179 return nullptr;
7180 }
7181
7182 // Handle the case where a copy is being spilled or filled but the source
7183 // and destination register class don't match. For example:
7184 //
7185 // %0 = COPY %xzr; GPR64common:%0
7186 //
7187 // In this case we can still safely fold away the COPY and generate the
7188 // following spill code:
7189 //
7190 // STRXui %xzr, %stack.0
7191 //
7192 // This also eliminates spilled cross register class COPYs (e.g. between x and
7193 // d regs) of the same size. For example:
7194 //
7195 // %0 = COPY %1; GPR64:%0, FPR64:%1
7196 //
7197 // will be filled as
7198 //
7199 // LDRDui %0, fi<#0>
7200 //
7201 // instead of
7202 //
7203 // LDRXui %Temp, fi<#0>
7204 // %0 = FMOV %Temp
7205 //
7206 if (MI.isCopy() && Ops.size() == 1 &&
7207 // Make sure we're only folding the explicit COPY defs/uses.
7208 (Ops[0] == 0 || Ops[0] == 1)) {
7209 bool IsSpill = Ops[0] == 0;
7210 bool IsFill = !IsSpill;
7211 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7212 const MachineRegisterInfo &MRI = MF.getRegInfo();
7213 MachineBasicBlock &MBB = *MI.getParent();
7214 const MachineOperand &DstMO = MI.getOperand(i: 0);
7215 const MachineOperand &SrcMO = MI.getOperand(i: 1);
7216 Register DstReg = DstMO.getReg();
7217 Register SrcReg = SrcMO.getReg();
7218 // This is slightly expensive to compute for physical regs since
7219 // getMinimalPhysRegClass is slow.
7220 auto getRegClass = [&](unsigned Reg) {
7221 return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
7222 : TRI.getMinimalPhysRegClass(Reg);
7223 };
7224
7225 if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
7226 assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
7227 TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
7228 "Mismatched register size in non subreg COPY");
7229 if (IsSpill)
7230 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg, isKill: SrcMO.isKill(), FI: FrameIndex,
7231 RC: getRegClass(SrcReg), VReg: Register());
7232 else
7233 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex,
7234 RC: getRegClass(DstReg), VReg: Register());
7235 return &*--InsertPt;
7236 }
7237
7238 // Handle cases like spilling def of:
7239 //
7240 // %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
7241 //
7242 // where the physical register source can be widened and stored to the full
7243 // virtual reg destination stack slot, in this case producing:
7244 //
7245 // STRXui %xzr, %stack.0
7246 //
7247 if (IsSpill && DstMO.isUndef() && SrcReg == AArch64::WZR &&
7248 TRI.getRegSizeInBits(RC: *getRegClass(DstReg)) == 64) {
7249 assert(SrcMO.getSubReg() == 0 &&
7250 "Unexpected subreg on physical register");
7251 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg: AArch64::XZR, isKill: SrcMO.isKill(),
7252 FI: FrameIndex, RC: &AArch64::GPR64RegClass, VReg: Register());
7253 return &*--InsertPt;
7254 }
7255
7256 // Handle cases like filling use of:
7257 //
7258 // %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
7259 //
7260 // where we can load the full virtual reg source stack slot, into the subreg
7261 // destination, in this case producing:
7262 //
7263 // LDRWui %0:sub_32<def,read-undef>, %stack.0
7264 //
7265 if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
7266 const TargetRegisterClass *FillRC = nullptr;
7267 switch (DstMO.getSubReg()) {
7268 default:
7269 break;
7270 case AArch64::sub_32:
7271 if (AArch64::GPR64RegClass.hasSubClassEq(RC: getRegClass(DstReg)))
7272 FillRC = &AArch64::GPR32RegClass;
7273 break;
7274 case AArch64::ssub:
7275 FillRC = &AArch64::FPR32RegClass;
7276 break;
7277 case AArch64::dsub:
7278 FillRC = &AArch64::FPR64RegClass;
7279 break;
7280 }
7281
7282 if (FillRC) {
7283 assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
7284 TRI.getRegSizeInBits(*FillRC) &&
7285 "Mismatched regclass size on folded subreg COPY");
7286 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex, RC: FillRC,
7287 VReg: Register());
7288 MachineInstr &LoadMI = *--InsertPt;
7289 MachineOperand &LoadDst = LoadMI.getOperand(i: 0);
7290 assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
7291 LoadDst.setSubReg(DstMO.getSubReg());
7292 LoadDst.setIsUndef();
7293 return &LoadMI;
7294 }
7295 }
7296 }
7297
7298 // Cannot fold.
7299 return nullptr;
7300}
7301
7302int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
7303 StackOffset &SOffset,
7304 bool *OutUseUnscaledOp,
7305 unsigned *OutUnscaledOp,
7306 int64_t *EmittableOffset) {
7307 // Set output values in case of early exit.
7308 if (EmittableOffset)
7309 *EmittableOffset = 0;
7310 if (OutUseUnscaledOp)
7311 *OutUseUnscaledOp = false;
7312 if (OutUnscaledOp)
7313 *OutUnscaledOp = 0;
7314
7315 // Exit early for structured vector spills/fills as they can't take an
7316 // immediate offset.
7317 switch (MI.getOpcode()) {
7318 default:
7319 break;
7320 case AArch64::LD1Rv1d:
7321 case AArch64::LD1Rv2s:
7322 case AArch64::LD1Rv2d:
7323 case AArch64::LD1Rv4h:
7324 case AArch64::LD1Rv4s:
7325 case AArch64::LD1Rv8b:
7326 case AArch64::LD1Rv8h:
7327 case AArch64::LD1Rv16b:
7328 case AArch64::LD1Twov2d:
7329 case AArch64::LD1Threev2d:
7330 case AArch64::LD1Fourv2d:
7331 case AArch64::LD1Twov1d:
7332 case AArch64::LD1Threev1d:
7333 case AArch64::LD1Fourv1d:
7334 case AArch64::ST1Twov2d:
7335 case AArch64::ST1Threev2d:
7336 case AArch64::ST1Fourv2d:
7337 case AArch64::ST1Twov1d:
7338 case AArch64::ST1Threev1d:
7339 case AArch64::ST1Fourv1d:
7340 case AArch64::ST1i8:
7341 case AArch64::ST1i16:
7342 case AArch64::ST1i32:
7343 case AArch64::ST1i64:
7344 case AArch64::IRG:
7345 case AArch64::IRGstack:
7346 case AArch64::STGloop:
7347 case AArch64::STZGloop:
7348 return AArch64FrameOffsetCannotUpdate;
7349 }
7350
7351 // Get the min/max offset and the scale.
7352 TypeSize ScaleValue(0U, false), Width(0U, false);
7353 int64_t MinOff, MaxOff;
7354 if (!AArch64InstrInfo::getMemOpInfo(Opcode: MI.getOpcode(), Scale&: ScaleValue, Width, MinOffset&: MinOff,
7355 MaxOffset&: MaxOff))
7356 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7357
7358 // Construct the complete offset.
7359 bool IsMulVL = ScaleValue.isScalable();
7360 unsigned Scale = ScaleValue.getKnownMinValue();
7361 int64_t Offset = IsMulVL ? SOffset.getScalable() : SOffset.getFixed();
7362
7363 const MachineOperand &ImmOpnd =
7364 MI.getOperand(i: AArch64InstrInfo::getLoadStoreImmIdx(Opc: MI.getOpcode()));
7365 Offset += ImmOpnd.getImm() * Scale;
7366
7367 // If the offset doesn't match the scale, we rewrite the instruction to
7368 // use the unscaled instruction instead. Likewise, if we have a negative
7369 // offset and there is an unscaled op to use.
7370 std::optional<unsigned> UnscaledOp =
7371 AArch64InstrInfo::getUnscaledLdSt(Opc: MI.getOpcode());
7372 bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
7373 if (useUnscaledOp &&
7374 !AArch64InstrInfo::getMemOpInfo(Opcode: *UnscaledOp, Scale&: ScaleValue, Width, MinOffset&: MinOff,
7375 MaxOffset&: MaxOff))
7376 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7377
7378 Scale = ScaleValue.getKnownMinValue();
7379 assert(IsMulVL == ScaleValue.isScalable() &&
7380 "Unscaled opcode has different value for scalable");
7381
7382 int64_t Remainder = Offset % Scale;
7383 assert(!(Remainder && useUnscaledOp) &&
7384 "Cannot have remainder when using unscaled op");
7385
7386 assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
7387 int64_t NewOffset = Offset / Scale;
7388 if (MinOff <= NewOffset && NewOffset <= MaxOff)
7389 Offset = Remainder;
7390 else {
7391 // Try to minimise the number of instructions required to materialise the
7392 // offset calculation. Specifically, for fixed offsets, if masking out the
7393 // low 12 bits leaves a legal add immediate, we can realise the offset
7394 // calculation with a single add instruction. Whenever this is possible,
7395 // prefer this split.
7396 int64_t HighPart = Offset & ~0xFFF;
7397 int64_t LowPart = Offset & 0xFFF;
7398 int64_t LowScaled = LowPart / Scale;
7399 if (!IsMulVL && NewOffset >= 0 && LowPart % Scale == 0 &&
7400 MinOff <= LowScaled && LowScaled <= MaxOff &&
7401 AArch64_AM::isLegalArithImmed(C: HighPart)) {
7402 NewOffset = LowScaled;
7403 Offset = HighPart;
7404 } else {
7405 // Default to a greedy split: take the memop immediate to be maximum /
7406 // minimum expressible offset and materialise the remainder.
7407 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
7408 Offset = Offset - (NewOffset * Scale);
7409 }
7410 }
7411
7412 if (EmittableOffset)
7413 *EmittableOffset = NewOffset;
7414 if (OutUseUnscaledOp)
7415 *OutUseUnscaledOp = useUnscaledOp;
7416 if (OutUnscaledOp && UnscaledOp)
7417 *OutUnscaledOp = *UnscaledOp;
7418
7419 if (IsMulVL)
7420 SOffset = StackOffset::get(Fixed: SOffset.getFixed(), Scalable: Offset);
7421 else
7422 SOffset = StackOffset::get(Fixed: Offset, Scalable: SOffset.getScalable());
7423 return AArch64FrameOffsetCanUpdate |
7424 (SOffset ? 0 : AArch64FrameOffsetIsLegal);
7425}
7426
7427bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
7428 unsigned FrameReg, StackOffset &Offset,
7429 const AArch64InstrInfo *TII) {
7430 unsigned Opcode = MI.getOpcode();
7431 unsigned ImmIdx = FrameRegIdx + 1;
7432
7433 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
7434 Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: ImmIdx).getImm());
7435 emitFrameOffset(MBB&: *MI.getParent(), MBBI: MI, DL: MI.getDebugLoc(),
7436 DestReg: MI.getOperand(i: 0).getReg(), SrcReg: FrameReg, Offset, TII,
7437 Flag: MachineInstr::NoFlags, SetNZCV: (Opcode == AArch64::ADDSXri));
7438 MI.eraseFromParent();
7439 Offset = StackOffset();
7440 return true;
7441 }
7442
7443 int64_t NewOffset;
7444 unsigned UnscaledOp;
7445 bool UseUnscaledOp;
7446 int Status = isAArch64FrameOffsetLegal(MI, SOffset&: Offset, OutUseUnscaledOp: &UseUnscaledOp,
7447 OutUnscaledOp: &UnscaledOp, EmittableOffset: &NewOffset);
7448 if (Status & AArch64FrameOffsetCanUpdate) {
7449 if (Status & AArch64FrameOffsetIsLegal)
7450 // Replace the FrameIndex with FrameReg.
7451 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
7452 if (UseUnscaledOp)
7453 MI.setDesc(TII->get(Opcode: UnscaledOp));
7454
7455 MI.getOperand(i: ImmIdx).ChangeToImmediate(ImmVal: NewOffset);
7456 return !Offset;
7457 }
7458
7459 return false;
7460}
7461
7462void AArch64InstrInfo::insertNoop(MachineBasicBlock &MBB,
7463 MachineBasicBlock::iterator MI) const {
7464 DebugLoc DL;
7465 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::NOP));
7466}
7467
7468MCInst AArch64InstrInfo::getNop() const { return MCInstBuilder(AArch64::NOP); }
7469
7470// AArch64 supports MachineCombiner.
7471bool AArch64InstrInfo::useMachineCombiner() const { return true; }
7472
7473// True when Opc sets flag
7474static bool isCombineInstrSettingFlag(unsigned Opc) {
7475 switch (Opc) {
7476 case AArch64::ADDSWrr:
7477 case AArch64::ADDSWri:
7478 case AArch64::ADDSXrr:
7479 case AArch64::ADDSXri:
7480 case AArch64::SUBSWrr:
7481 case AArch64::SUBSXrr:
7482 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7483 case AArch64::SUBSWri:
7484 case AArch64::SUBSXri:
7485 return true;
7486 default:
7487 break;
7488 }
7489 return false;
7490}
7491
7492// 32b Opcodes that can be combined with a MUL
7493static bool isCombineInstrCandidate32(unsigned Opc) {
7494 switch (Opc) {
7495 case AArch64::ADDWrr:
7496 case AArch64::ADDWri:
7497 case AArch64::SUBWrr:
7498 case AArch64::ADDSWrr:
7499 case AArch64::ADDSWri:
7500 case AArch64::SUBSWrr:
7501 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7502 case AArch64::SUBWri:
7503 case AArch64::SUBSWri:
7504 return true;
7505 default:
7506 break;
7507 }
7508 return false;
7509}
7510
7511// 64b Opcodes that can be combined with a MUL
7512static bool isCombineInstrCandidate64(unsigned Opc) {
7513 switch (Opc) {
7514 case AArch64::ADDXrr:
7515 case AArch64::ADDXri:
7516 case AArch64::SUBXrr:
7517 case AArch64::ADDSXrr:
7518 case AArch64::ADDSXri:
7519 case AArch64::SUBSXrr:
7520 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7521 case AArch64::SUBXri:
7522 case AArch64::SUBSXri:
7523 case AArch64::ADDv8i8:
7524 case AArch64::ADDv16i8:
7525 case AArch64::ADDv4i16:
7526 case AArch64::ADDv8i16:
7527 case AArch64::ADDv2i32:
7528 case AArch64::ADDv4i32:
7529 case AArch64::SUBv8i8:
7530 case AArch64::SUBv16i8:
7531 case AArch64::SUBv4i16:
7532 case AArch64::SUBv8i16:
7533 case AArch64::SUBv2i32:
7534 case AArch64::SUBv4i32:
7535 return true;
7536 default:
7537 break;
7538 }
7539 return false;
7540}
7541
7542// FP Opcodes that can be combined with a FMUL.
7543static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
7544 switch (Inst.getOpcode()) {
7545 default:
7546 break;
7547 case AArch64::FADDHrr:
7548 case AArch64::FADDSrr:
7549 case AArch64::FADDDrr:
7550 case AArch64::FADDv4f16:
7551 case AArch64::FADDv8f16:
7552 case AArch64::FADDv2f32:
7553 case AArch64::FADDv2f64:
7554 case AArch64::FADDv4f32:
7555 case AArch64::FSUBHrr:
7556 case AArch64::FSUBSrr:
7557 case AArch64::FSUBDrr:
7558 case AArch64::FSUBv4f16:
7559 case AArch64::FSUBv8f16:
7560 case AArch64::FSUBv2f32:
7561 case AArch64::FSUBv2f64:
7562 case AArch64::FSUBv4f32:
7563 TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options;
7564 // We can fuse FADD/FSUB with FMUL, if fusion is either allowed globally by
7565 // the target options or if FADD/FSUB has the contract fast-math flag.
7566 return Options.AllowFPOpFusion == FPOpFusion::Fast ||
7567 Inst.getFlag(Flag: MachineInstr::FmContract);
7568 }
7569 return false;
7570}
7571
7572// Opcodes that can be combined with a MUL
7573static bool isCombineInstrCandidate(unsigned Opc) {
7574 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
7575}
7576
7577//
7578// Utility routine that checks if \param MO is defined by an
7579// \param CombineOpc instruction in the basic block \param MBB
7580static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
7581 unsigned CombineOpc, unsigned ZeroReg = 0,
7582 bool CheckZeroReg = false) {
7583 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7584 MachineInstr *MI = nullptr;
7585
7586 if (MO.isReg() && MO.getReg().isVirtual())
7587 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
7588 // And it needs to be in the trace (otherwise, it won't have a depth).
7589 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
7590 return false;
7591 // Must only used by the user we combine with.
7592 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
7593 return false;
7594
7595 if (CheckZeroReg) {
7596 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
7597 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
7598 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
7599 // The third input reg must be zero.
7600 if (MI->getOperand(i: 3).getReg() != ZeroReg)
7601 return false;
7602 }
7603
7604 if (isCombineInstrSettingFlag(Opc: CombineOpc) &&
7605 MI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) == -1)
7606 return false;
7607
7608 return true;
7609}
7610
7611//
7612// Is \param MO defined by an integer multiply and can be combined?
7613static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7614 unsigned MulOpc, unsigned ZeroReg) {
7615 return canCombine(MBB, MO, CombineOpc: MulOpc, ZeroReg, CheckZeroReg: true);
7616}
7617
7618//
7619// Is \param MO defined by a floating-point multiply and can be combined?
7620static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7621 unsigned MulOpc) {
7622 return canCombine(MBB, MO, CombineOpc: MulOpc);
7623}
7624
7625// TODO: There are many more machine instruction opcodes to match:
7626// 1. Other data types (integer, vectors)
7627// 2. Other math / logic operations (xor, or)
7628// 3. Other forms of the same operation (intrinsics and other variants)
7629bool AArch64InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
7630 bool Invert) const {
7631 if (Invert)
7632 return false;
7633 switch (Inst.getOpcode()) {
7634 // == Floating-point types ==
7635 // -- Floating-point instructions --
7636 case AArch64::FADDHrr:
7637 case AArch64::FADDSrr:
7638 case AArch64::FADDDrr:
7639 case AArch64::FMULHrr:
7640 case AArch64::FMULSrr:
7641 case AArch64::FMULDrr:
7642 case AArch64::FMULX16:
7643 case AArch64::FMULX32:
7644 case AArch64::FMULX64:
7645 // -- Advanced SIMD instructions --
7646 case AArch64::FADDv4f16:
7647 case AArch64::FADDv8f16:
7648 case AArch64::FADDv2f32:
7649 case AArch64::FADDv4f32:
7650 case AArch64::FADDv2f64:
7651 case AArch64::FMULv4f16:
7652 case AArch64::FMULv8f16:
7653 case AArch64::FMULv2f32:
7654 case AArch64::FMULv4f32:
7655 case AArch64::FMULv2f64:
7656 case AArch64::FMULXv4f16:
7657 case AArch64::FMULXv8f16:
7658 case AArch64::FMULXv2f32:
7659 case AArch64::FMULXv4f32:
7660 case AArch64::FMULXv2f64:
7661 // -- SVE instructions --
7662 // Opcodes FMULX_ZZZ_? don't exist because there is no unpredicated FMULX
7663 // in the SVE instruction set (though there are predicated ones).
7664 case AArch64::FADD_ZZZ_H:
7665 case AArch64::FADD_ZZZ_S:
7666 case AArch64::FADD_ZZZ_D:
7667 case AArch64::FMUL_ZZZ_H:
7668 case AArch64::FMUL_ZZZ_S:
7669 case AArch64::FMUL_ZZZ_D:
7670 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
7671 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
7672
7673 // == Integer types ==
7674 // -- Base instructions --
7675 // Opcodes MULWrr and MULXrr don't exist because
7676 // `MUL <Wd>, <Wn>, <Wm>` and `MUL <Xd>, <Xn>, <Xm>` are aliases of
7677 // `MADD <Wd>, <Wn>, <Wm>, WZR` and `MADD <Xd>, <Xn>, <Xm>, XZR` respectively.
7678 // The machine-combiner does not support three-source-operands machine
7679 // instruction. So we cannot reassociate MULs.
7680 case AArch64::ADDWrr:
7681 case AArch64::ADDXrr:
7682 case AArch64::ANDWrr:
7683 case AArch64::ANDXrr:
7684 case AArch64::ORRWrr:
7685 case AArch64::ORRXrr:
7686 case AArch64::EORWrr:
7687 case AArch64::EORXrr:
7688 case AArch64::EONWrr:
7689 case AArch64::EONXrr:
7690 // -- Advanced SIMD instructions --
7691 // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL
7692 // in the Advanced SIMD instruction set.
7693 case AArch64::ADDv8i8:
7694 case AArch64::ADDv16i8:
7695 case AArch64::ADDv4i16:
7696 case AArch64::ADDv8i16:
7697 case AArch64::ADDv2i32:
7698 case AArch64::ADDv4i32:
7699 case AArch64::ADDv1i64:
7700 case AArch64::ADDv2i64:
7701 case AArch64::MULv8i8:
7702 case AArch64::MULv16i8:
7703 case AArch64::MULv4i16:
7704 case AArch64::MULv8i16:
7705 case AArch64::MULv2i32:
7706 case AArch64::MULv4i32:
7707 case AArch64::ANDv8i8:
7708 case AArch64::ANDv16i8:
7709 case AArch64::ORRv8i8:
7710 case AArch64::ORRv16i8:
7711 case AArch64::EORv8i8:
7712 case AArch64::EORv16i8:
7713 // -- SVE instructions --
7714 case AArch64::ADD_ZZZ_B:
7715 case AArch64::ADD_ZZZ_H:
7716 case AArch64::ADD_ZZZ_S:
7717 case AArch64::ADD_ZZZ_D:
7718 case AArch64::MUL_ZZZ_B:
7719 case AArch64::MUL_ZZZ_H:
7720 case AArch64::MUL_ZZZ_S:
7721 case AArch64::MUL_ZZZ_D:
7722 case AArch64::AND_ZZZ:
7723 case AArch64::ORR_ZZZ:
7724 case AArch64::EOR_ZZZ:
7725 return true;
7726
7727 default:
7728 return false;
7729 }
7730}
7731
7732/// Find instructions that can be turned into madd.
7733static bool getMaddPatterns(MachineInstr &Root,
7734 SmallVectorImpl<unsigned> &Patterns) {
7735 unsigned Opc = Root.getOpcode();
7736 MachineBasicBlock &MBB = *Root.getParent();
7737 bool Found = false;
7738
7739 if (!isCombineInstrCandidate(Opc))
7740 return false;
7741 if (isCombineInstrSettingFlag(Opc)) {
7742 int Cmp_NZCV =
7743 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
7744 // When NZCV is live bail out.
7745 if (Cmp_NZCV == -1)
7746 return false;
7747 unsigned NewOpc = convertToNonFlagSettingOpc(MI: Root);
7748 // When opcode can't change bail out.
7749 // CHECKME: do we miss any cases for opcode conversion?
7750 if (NewOpc == Opc)
7751 return false;
7752 Opc = NewOpc;
7753 }
7754
7755 auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
7756 unsigned Pattern) {
7757 if (canCombineWithMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode, ZeroReg)) {
7758 Patterns.push_back(Elt: Pattern);
7759 Found = true;
7760 }
7761 };
7762
7763 auto setVFound = [&](int Opcode, int Operand, unsigned Pattern) {
7764 if (canCombine(MBB, MO&: Root.getOperand(i: Operand), CombineOpc: Opcode)) {
7765 Patterns.push_back(Elt: Pattern);
7766 Found = true;
7767 }
7768 };
7769
7770 typedef AArch64MachineCombinerPattern MCP;
7771
7772 switch (Opc) {
7773 default:
7774 break;
7775 case AArch64::ADDWrr:
7776 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
7777 "ADDWrr does not have register operands");
7778 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
7779 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
7780 break;
7781 case AArch64::ADDXrr:
7782 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
7783 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
7784 break;
7785 case AArch64::SUBWrr:
7786 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
7787 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
7788 break;
7789 case AArch64::SUBXrr:
7790 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
7791 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
7792 break;
7793 case AArch64::ADDWri:
7794 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
7795 break;
7796 case AArch64::ADDXri:
7797 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
7798 break;
7799 case AArch64::SUBWri:
7800 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
7801 break;
7802 case AArch64::SUBXri:
7803 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
7804 break;
7805 case AArch64::ADDv8i8:
7806 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
7807 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
7808 break;
7809 case AArch64::ADDv16i8:
7810 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
7811 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
7812 break;
7813 case AArch64::ADDv4i16:
7814 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
7815 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
7816 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
7817 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
7818 break;
7819 case AArch64::ADDv8i16:
7820 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
7821 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
7822 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
7823 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
7824 break;
7825 case AArch64::ADDv2i32:
7826 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
7827 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
7828 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
7829 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
7830 break;
7831 case AArch64::ADDv4i32:
7832 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
7833 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
7834 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
7835 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
7836 break;
7837 case AArch64::SUBv8i8:
7838 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
7839 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
7840 break;
7841 case AArch64::SUBv16i8:
7842 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
7843 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
7844 break;
7845 case AArch64::SUBv4i16:
7846 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
7847 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7848 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7849 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7850 break;
7851 case AArch64::SUBv8i16:
7852 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7853 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7854 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7855 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7856 break;
7857 case AArch64::SUBv2i32:
7858 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7859 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7860 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7861 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7862 break;
7863 case AArch64::SUBv4i32:
7864 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7865 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7866 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7867 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7868 break;
7869 }
7870 return Found;
7871}
7872
7873bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
7874 switch (Opcode) {
7875 default:
7876 break;
7877 case AArch64::UABALB_ZZZ_D:
7878 case AArch64::UABALB_ZZZ_H:
7879 case AArch64::UABALB_ZZZ_S:
7880 case AArch64::UABALT_ZZZ_D:
7881 case AArch64::UABALT_ZZZ_H:
7882 case AArch64::UABALT_ZZZ_S:
7883 case AArch64::SABALB_ZZZ_D:
7884 case AArch64::SABALB_ZZZ_S:
7885 case AArch64::SABALB_ZZZ_H:
7886 case AArch64::SABALT_ZZZ_D:
7887 case AArch64::SABALT_ZZZ_S:
7888 case AArch64::SABALT_ZZZ_H:
7889 case AArch64::UABALv16i8_v8i16:
7890 case AArch64::UABALv2i32_v2i64:
7891 case AArch64::UABALv4i16_v4i32:
7892 case AArch64::UABALv4i32_v2i64:
7893 case AArch64::UABALv8i16_v4i32:
7894 case AArch64::UABALv8i8_v8i16:
7895 case AArch64::UABAv16i8:
7896 case AArch64::UABAv2i32:
7897 case AArch64::UABAv4i16:
7898 case AArch64::UABAv4i32:
7899 case AArch64::UABAv8i16:
7900 case AArch64::UABAv8i8:
7901 case AArch64::SABALv16i8_v8i16:
7902 case AArch64::SABALv2i32_v2i64:
7903 case AArch64::SABALv4i16_v4i32:
7904 case AArch64::SABALv4i32_v2i64:
7905 case AArch64::SABALv8i16_v4i32:
7906 case AArch64::SABALv8i8_v8i16:
7907 case AArch64::SABAv16i8:
7908 case AArch64::SABAv2i32:
7909 case AArch64::SABAv4i16:
7910 case AArch64::SABAv4i32:
7911 case AArch64::SABAv8i16:
7912 case AArch64::SABAv8i8:
7913 return true;
7914 }
7915
7916 return false;
7917}
7918
7919unsigned AArch64InstrInfo::getAccumulationStartOpcode(
7920 unsigned AccumulationOpcode) const {
7921 switch (AccumulationOpcode) {
7922 default:
7923 llvm_unreachable("Unsupported accumulation Opcode!");
7924 case AArch64::UABALB_ZZZ_D:
7925 return AArch64::UABDLB_ZZZ_D;
7926 case AArch64::UABALB_ZZZ_H:
7927 return AArch64::UABDLB_ZZZ_H;
7928 case AArch64::UABALB_ZZZ_S:
7929 return AArch64::UABDLB_ZZZ_S;
7930 case AArch64::UABALT_ZZZ_D:
7931 return AArch64::UABDLT_ZZZ_D;
7932 case AArch64::UABALT_ZZZ_H:
7933 return AArch64::UABDLT_ZZZ_H;
7934 case AArch64::UABALT_ZZZ_S:
7935 return AArch64::UABDLT_ZZZ_S;
7936 case AArch64::UABALv16i8_v8i16:
7937 return AArch64::UABDLv16i8_v8i16;
7938 case AArch64::UABALv2i32_v2i64:
7939 return AArch64::UABDLv2i32_v2i64;
7940 case AArch64::UABALv4i16_v4i32:
7941 return AArch64::UABDLv4i16_v4i32;
7942 case AArch64::UABALv4i32_v2i64:
7943 return AArch64::UABDLv4i32_v2i64;
7944 case AArch64::UABALv8i16_v4i32:
7945 return AArch64::UABDLv8i16_v4i32;
7946 case AArch64::UABALv8i8_v8i16:
7947 return AArch64::UABDLv8i8_v8i16;
7948 case AArch64::UABAv16i8:
7949 return AArch64::UABDv16i8;
7950 case AArch64::UABAv2i32:
7951 return AArch64::UABDv2i32;
7952 case AArch64::UABAv4i16:
7953 return AArch64::UABDv4i16;
7954 case AArch64::UABAv4i32:
7955 return AArch64::UABDv4i32;
7956 case AArch64::UABAv8i16:
7957 return AArch64::UABDv8i16;
7958 case AArch64::UABAv8i8:
7959 return AArch64::UABDv8i8;
7960 case AArch64::SABALB_ZZZ_D:
7961 return AArch64::SABDLB_ZZZ_D;
7962 case AArch64::SABALB_ZZZ_S:
7963 return AArch64::SABDLB_ZZZ_S;
7964 case AArch64::SABALB_ZZZ_H:
7965 return AArch64::SABDLB_ZZZ_H;
7966 case AArch64::SABALT_ZZZ_D:
7967 return AArch64::SABDLT_ZZZ_D;
7968 case AArch64::SABALT_ZZZ_S:
7969 return AArch64::SABDLT_ZZZ_S;
7970 case AArch64::SABALT_ZZZ_H:
7971 return AArch64::SABDLT_ZZZ_H;
7972 case AArch64::SABALv16i8_v8i16:
7973 return AArch64::SABDLv16i8_v8i16;
7974 case AArch64::SABALv2i32_v2i64:
7975 return AArch64::SABDLv2i32_v2i64;
7976 case AArch64::SABALv4i16_v4i32:
7977 return AArch64::SABDLv4i16_v4i32;
7978 case AArch64::SABALv4i32_v2i64:
7979 return AArch64::SABDLv4i32_v2i64;
7980 case AArch64::SABALv8i16_v4i32:
7981 return AArch64::SABDLv8i16_v4i32;
7982 case AArch64::SABALv8i8_v8i16:
7983 return AArch64::SABDLv8i8_v8i16;
7984 case AArch64::SABAv16i8:
7985 return AArch64::SABDv16i8;
7986 case AArch64::SABAv2i32:
7987 return AArch64::SABAv2i32;
7988 case AArch64::SABAv4i16:
7989 return AArch64::SABDv4i16;
7990 case AArch64::SABAv4i32:
7991 return AArch64::SABDv4i32;
7992 case AArch64::SABAv8i16:
7993 return AArch64::SABDv8i16;
7994 case AArch64::SABAv8i8:
7995 return AArch64::SABDv8i8;
7996 }
7997}
7998
7999/// Floating-Point Support
8000
8001/// Find instructions that can be turned into madd.
8002static bool getFMAPatterns(MachineInstr &Root,
8003 SmallVectorImpl<unsigned> &Patterns) {
8004
8005 if (!isCombineInstrCandidateFP(Inst: Root))
8006 return false;
8007
8008 MachineBasicBlock &MBB = *Root.getParent();
8009 bool Found = false;
8010
8011 auto Match = [&](int Opcode, int Operand, unsigned Pattern) -> bool {
8012 if (canCombineWithFMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode)) {
8013 Patterns.push_back(Elt: Pattern);
8014 return true;
8015 }
8016 return false;
8017 };
8018
8019 typedef AArch64MachineCombinerPattern MCP;
8020
8021 switch (Root.getOpcode()) {
8022 default:
8023 assert(false && "Unsupported FP instruction in combiner\n");
8024 break;
8025 case AArch64::FADDHrr:
8026 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8027 "FADDHrr does not have register operands");
8028
8029 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
8030 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
8031 break;
8032 case AArch64::FADDSrr:
8033 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8034 "FADDSrr does not have register operands");
8035
8036 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
8037 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
8038
8039 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
8040 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
8041 break;
8042 case AArch64::FADDDrr:
8043 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
8044 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
8045
8046 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
8047 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
8048 break;
8049 case AArch64::FADDv4f16:
8050 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
8051 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
8052
8053 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
8054 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
8055 break;
8056 case AArch64::FADDv8f16:
8057 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
8058 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
8059
8060 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
8061 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
8062 break;
8063 case AArch64::FADDv2f32:
8064 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
8065 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
8066
8067 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
8068 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
8069 break;
8070 case AArch64::FADDv2f64:
8071 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
8072 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
8073
8074 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
8075 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
8076 break;
8077 case AArch64::FADDv4f32:
8078 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
8079 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
8080
8081 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
8082 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
8083 break;
8084 case AArch64::FSUBHrr:
8085 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
8086 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
8087 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
8088 break;
8089 case AArch64::FSUBSrr:
8090 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
8091
8092 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
8093 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
8094
8095 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
8096 break;
8097 case AArch64::FSUBDrr:
8098 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
8099
8100 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
8101 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
8102
8103 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
8104 break;
8105 case AArch64::FSUBv4f16:
8106 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
8107 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
8108
8109 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
8110 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
8111 break;
8112 case AArch64::FSUBv8f16:
8113 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
8114 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
8115
8116 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
8117 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
8118 break;
8119 case AArch64::FSUBv2f32:
8120 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
8121 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
8122
8123 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
8124 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
8125 break;
8126 case AArch64::FSUBv2f64:
8127 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
8128 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
8129
8130 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
8131 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
8132 break;
8133 case AArch64::FSUBv4f32:
8134 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
8135 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
8136
8137 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
8138 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
8139 break;
8140 }
8141 return Found;
8142}
8143
8144static bool getFMULPatterns(MachineInstr &Root,
8145 SmallVectorImpl<unsigned> &Patterns) {
8146 MachineBasicBlock &MBB = *Root.getParent();
8147 bool Found = false;
8148
8149 auto Match = [&](unsigned Opcode, int Operand, unsigned Pattern) -> bool {
8150 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8151 MachineOperand &MO = Root.getOperand(i: Operand);
8152 MachineInstr *MI = nullptr;
8153 if (MO.isReg() && MO.getReg().isVirtual())
8154 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8155 // Ignore No-op COPYs in FMUL(COPY(DUP(..)))
8156 if (MI && MI->getOpcode() == TargetOpcode::COPY &&
8157 MI->getOperand(i: 1).getReg().isVirtual())
8158 MI = MRI.getUniqueVRegDef(Reg: MI->getOperand(i: 1).getReg());
8159 if (MI && MI->getOpcode() == Opcode) {
8160 Patterns.push_back(Elt: Pattern);
8161 return true;
8162 }
8163 return false;
8164 };
8165
8166 typedef AArch64MachineCombinerPattern MCP;
8167
8168 switch (Root.getOpcode()) {
8169 default:
8170 return false;
8171 case AArch64::FMULv2f32:
8172 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
8173 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
8174 break;
8175 case AArch64::FMULv2f64:
8176 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
8177 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
8178 break;
8179 case AArch64::FMULv4f16:
8180 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
8181 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
8182 break;
8183 case AArch64::FMULv4f32:
8184 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
8185 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
8186 break;
8187 case AArch64::FMULv8f16:
8188 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
8189 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
8190 break;
8191 }
8192
8193 return Found;
8194}
8195
8196static bool getFNEGPatterns(MachineInstr &Root,
8197 SmallVectorImpl<unsigned> &Patterns) {
8198 unsigned Opc = Root.getOpcode();
8199 MachineBasicBlock &MBB = *Root.getParent();
8200 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8201
8202 auto Match = [&](unsigned Opcode, unsigned Pattern) -> bool {
8203 MachineOperand &MO = Root.getOperand(i: 1);
8204 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8205 if (MI != nullptr && (MI->getOpcode() == Opcode) &&
8206 MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()) &&
8207 Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8208 Root.getFlag(Flag: MachineInstr::MIFlag::FmNsz) &&
8209 MI->getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8210 MI->getFlag(Flag: MachineInstr::MIFlag::FmNsz)) {
8211 Patterns.push_back(Elt: Pattern);
8212 return true;
8213 }
8214 return false;
8215 };
8216
8217 switch (Opc) {
8218 default:
8219 break;
8220 case AArch64::FNEGDr:
8221 return Match(AArch64::FMADDDrrr, AArch64MachineCombinerPattern::FNMADD);
8222 case AArch64::FNEGSr:
8223 return Match(AArch64::FMADDSrrr, AArch64MachineCombinerPattern::FNMADD);
8224 }
8225
8226 return false;
8227}
8228
8229/// Return true when a code sequence can improve throughput. It
8230/// should be called only for instructions in loops.
8231/// \param Pattern - combiner pattern
8232bool AArch64InstrInfo::isThroughputPattern(unsigned Pattern) const {
8233 switch (Pattern) {
8234 default:
8235 break;
8236 case AArch64MachineCombinerPattern::FMULADDH_OP1:
8237 case AArch64MachineCombinerPattern::FMULADDH_OP2:
8238 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
8239 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
8240 case AArch64MachineCombinerPattern::FMULADDS_OP1:
8241 case AArch64MachineCombinerPattern::FMULADDS_OP2:
8242 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
8243 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
8244 case AArch64MachineCombinerPattern::FMULADDD_OP1:
8245 case AArch64MachineCombinerPattern::FMULADDD_OP2:
8246 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
8247 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
8248 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
8249 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
8250 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
8251 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
8252 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
8253 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
8254 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
8255 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
8256 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
8257 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
8258 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
8259 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
8260 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
8261 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
8262 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
8263 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
8264 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
8265 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
8266 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
8267 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
8268 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
8269 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
8270 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
8271 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
8272 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
8273 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
8274 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
8275 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1:
8276 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
8277 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1:
8278 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
8279 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
8280 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
8281 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
8282 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
8283 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
8284 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
8285 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
8286 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
8287 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
8288 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
8289 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
8290 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
8291 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
8292 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2:
8293 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
8294 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2:
8295 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
8296 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2:
8297 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
8298 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2:
8299 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
8300 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2:
8301 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
8302 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
8303 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
8304 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
8305 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
8306 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
8307 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
8308 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
8309 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
8310 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
8311 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
8312 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
8313 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
8314 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
8315 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
8316 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
8317 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
8318 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
8319 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
8320 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
8321 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
8322 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
8323 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
8324 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
8325 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
8326 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
8327 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
8328 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
8329 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
8330 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
8331 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
8332 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
8333 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
8334 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
8335 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
8336 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
8337 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
8338 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
8339 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
8340 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
8341 return true;
8342 } // end switch (Pattern)
8343 return false;
8344}
8345
8346/// Find other MI combine patterns.
8347static bool getMiscPatterns(MachineInstr &Root,
8348 SmallVectorImpl<unsigned> &Patterns) {
8349 // A - (B + C) ==> (A - B) - C or (A - C) - B
8350 unsigned Opc = Root.getOpcode();
8351 MachineBasicBlock &MBB = *Root.getParent();
8352
8353 switch (Opc) {
8354 case AArch64::SUBWrr:
8355 case AArch64::SUBSWrr:
8356 case AArch64::SUBXrr:
8357 case AArch64::SUBSXrr:
8358 // Found candidate root.
8359 break;
8360 default:
8361 return false;
8362 }
8363
8364 if (isCombineInstrSettingFlag(Opc) &&
8365 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) ==
8366 -1)
8367 return false;
8368
8369 if (canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDWrr) ||
8370 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSWrr) ||
8371 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDXrr) ||
8372 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSXrr)) {
8373 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP1);
8374 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP2);
8375 return true;
8376 }
8377
8378 return false;
8379}
8380
8381/// Check if the given instruction forms a gather load pattern that can be
8382/// optimized for better Memory-Level Parallelism (MLP). This function
8383/// identifies chains of NEON lane load instructions that load data from
8384/// different memory addresses into individual lanes of a 128-bit vector
8385/// register, then attempts to split the pattern into parallel loads to break
8386/// the serial dependency between instructions.
8387///
8388/// Pattern Matched:
8389/// Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) ->
8390/// LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)
8391///
8392/// Transformed Into:
8393/// Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64
8394/// to combine the results, enabling better memory-level parallelism.
8395///
8396/// Supported Element Types:
8397/// - 32-bit elements (LD1i32, 4 lanes total)
8398/// - 16-bit elements (LD1i16, 8 lanes total)
8399/// - 8-bit elements (LD1i8, 16 lanes total)
8400static bool getGatherLanePattern(MachineInstr &Root,
8401 SmallVectorImpl<unsigned> &Patterns,
8402 unsigned LoadLaneOpCode, unsigned NumLanes) {
8403 const MachineFunction *MF = Root.getMF();
8404
8405 // Early exit if optimizing for size.
8406 if (MF->getFunction().hasMinSize())
8407 return false;
8408
8409 const MachineRegisterInfo &MRI = MF->getRegInfo();
8410 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
8411
8412 // The root of the pattern must load into the last lane of the vector.
8413 if (Root.getOperand(i: 2).getImm() != NumLanes - 1)
8414 return false;
8415
8416 // Check that we have load into all lanes except lane 0.
8417 // For each load we also want to check that:
8418 // 1. It has a single non-debug use (since we will be replacing the virtual
8419 // register)
8420 // 2. That the addressing mode only uses a single pointer operand
8421 auto *CurrInstr = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8422 auto Range = llvm::seq<unsigned>(Begin: 1, End: NumLanes - 1);
8423 SmallSet<unsigned, 16> RemainingLanes(Range.begin(), Range.end());
8424 SmallVector<const MachineInstr *, 16> LoadInstrs;
8425 while (!RemainingLanes.empty() && CurrInstr &&
8426 CurrInstr->getOpcode() == LoadLaneOpCode &&
8427 MRI.hasOneNonDBGUse(RegNo: CurrInstr->getOperand(i: 0).getReg()) &&
8428 CurrInstr->getNumOperands() == 4) {
8429 RemainingLanes.erase(V: CurrInstr->getOperand(i: 2).getImm());
8430 LoadInstrs.push_back(Elt: CurrInstr);
8431 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8432 }
8433
8434 // Check that we have found a match for lanes N-1.. 1.
8435 if (!RemainingLanes.empty())
8436 return false;
8437
8438 // Match the SUBREG_TO_REG sequence.
8439 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
8440 return false;
8441
8442 // Verify that the subreg to reg loads an integer into the first lane.
8443 auto Lane0LoadReg = CurrInstr->getOperand(i: 1).getReg();
8444 unsigned SingleLaneSizeInBits = 128 / NumLanes;
8445 if (TRI->getRegSizeInBits(Reg: Lane0LoadReg, MRI) != SingleLaneSizeInBits)
8446 return false;
8447
8448 // Verify that it also has a single non debug use.
8449 if (!MRI.hasOneNonDBGUse(RegNo: Lane0LoadReg))
8450 return false;
8451
8452 LoadInstrs.push_back(Elt: MRI.getUniqueVRegDef(Reg: Lane0LoadReg));
8453
8454 // If there is any chance of aliasing, do not apply the pattern.
8455 // Walk backward through the MBB starting from Root.
8456 // Exit early if we've encountered all load instructions or hit the search
8457 // limit.
8458 auto MBBItr = Root.getIterator();
8459 unsigned RemainingSteps = GatherOptSearchLimit;
8460 SmallPtrSet<const MachineInstr *, 16> RemainingLoadInstrs;
8461 RemainingLoadInstrs.insert(I: LoadInstrs.begin(), E: LoadInstrs.end());
8462 const MachineBasicBlock *MBB = Root.getParent();
8463
8464 for (; MBBItr != MBB->begin() && RemainingSteps > 0 &&
8465 !RemainingLoadInstrs.empty();
8466 --MBBItr, --RemainingSteps) {
8467 const MachineInstr &CurrInstr = *MBBItr;
8468
8469 // Remove this instruction from remaining loads if it's one we're tracking.
8470 RemainingLoadInstrs.erase(Ptr: &CurrInstr);
8471
8472 // Check for potential aliasing with any of the load instructions to
8473 // optimize.
8474 if (CurrInstr.isLoadFoldBarrier())
8475 return false;
8476 }
8477
8478 // If we hit the search limit without finding all load instructions,
8479 // don't match the pattern.
8480 if (RemainingSteps == 0 && !RemainingLoadInstrs.empty())
8481 return false;
8482
8483 switch (NumLanes) {
8484 case 4:
8485 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i32);
8486 break;
8487 case 8:
8488 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i16);
8489 break;
8490 case 16:
8491 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i8);
8492 break;
8493 default:
8494 llvm_unreachable("Got bad number of lanes for gather pattern.");
8495 }
8496
8497 return true;
8498}
8499
8500/// Search for patterns of LD instructions we can optimize.
8501static bool getLoadPatterns(MachineInstr &Root,
8502 SmallVectorImpl<unsigned> &Patterns) {
8503
8504 // The pattern searches for loads into single lanes.
8505 switch (Root.getOpcode()) {
8506 case AArch64::LD1i32:
8507 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 4);
8508 case AArch64::LD1i16:
8509 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 8);
8510 case AArch64::LD1i8:
8511 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 16);
8512 default:
8513 return false;
8514 }
8515}
8516
8517/// Generate optimized instruction sequence for gather load patterns to improve
8518/// Memory-Level Parallelism (MLP). This function transforms a chain of
8519/// sequential NEON lane loads into parallel vector loads that can execute
8520/// concurrently.
8521static void
8522generateGatherLanePattern(MachineInstr &Root,
8523 SmallVectorImpl<MachineInstr *> &InsInstrs,
8524 SmallVectorImpl<MachineInstr *> &DelInstrs,
8525 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8526 unsigned Pattern, unsigned NumLanes) {
8527 MachineFunction &MF = *Root.getParent()->getParent();
8528 MachineRegisterInfo &MRI = MF.getRegInfo();
8529 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8530
8531 // Gather the initial load instructions to build the pattern.
8532 SmallVector<MachineInstr *, 16> LoadToLaneInstrs;
8533 MachineInstr *CurrInstr = &Root;
8534 for (unsigned i = 0; i < NumLanes - 1; ++i) {
8535 LoadToLaneInstrs.push_back(Elt: CurrInstr);
8536 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8537 }
8538
8539 // Sort the load instructions according to the lane.
8540 llvm::sort(C&: LoadToLaneInstrs,
8541 Comp: [](const MachineInstr *A, const MachineInstr *B) {
8542 return A->getOperand(i: 2).getImm() > B->getOperand(i: 2).getImm();
8543 });
8544
8545 MachineInstr *SubregToReg = CurrInstr;
8546 LoadToLaneInstrs.push_back(
8547 Elt: MRI.getUniqueVRegDef(Reg: SubregToReg->getOperand(i: 1).getReg()));
8548 auto LoadToLaneInstrsAscending = llvm::reverse(C&: LoadToLaneInstrs);
8549
8550 const TargetRegisterClass *FPR128RegClass =
8551 MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg());
8552
8553 // Helper lambda to create a LD1 instruction.
8554 auto CreateLD1Instruction = [&](MachineInstr *OriginalInstr,
8555 Register SrcRegister, unsigned Lane,
8556 Register OffsetRegister,
8557 bool OffsetRegisterKillState) {
8558 auto NewRegister = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8559 MachineInstrBuilder LoadIndexIntoRegister =
8560 BuildMI(MF, MIMD: MIMetadata(*OriginalInstr), MCID: TII->get(Opcode: Root.getOpcode()),
8561 DestReg: NewRegister)
8562 .addReg(RegNo: SrcRegister)
8563 .addImm(Val: Lane)
8564 .addReg(RegNo: OffsetRegister, Flags: getKillRegState(B: OffsetRegisterKillState))
8565 .setMemRefs(OriginalInstr->memoperands());
8566 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewRegister, y: InsInstrs.size()));
8567 InsInstrs.push_back(Elt: LoadIndexIntoRegister);
8568 return NewRegister;
8569 };
8570
8571 // Helper to create load instruction based on the NumLanes in the NEON
8572 // register we are rewriting.
8573 auto CreateLDRInstruction =
8574 [&](unsigned NumLanes, Register DestReg, Register OffsetReg,
8575 ArrayRef<MachineMemOperand *> MMOs) -> MachineInstrBuilder {
8576 unsigned Opcode;
8577 switch (NumLanes) {
8578 case 4:
8579 Opcode = AArch64::LDRSui;
8580 break;
8581 case 8:
8582 Opcode = AArch64::LDRHui;
8583 break;
8584 case 16:
8585 Opcode = AArch64::LDRBui;
8586 break;
8587 default:
8588 llvm_unreachable(
8589 "Got unsupported number of lanes in machine-combiner gather pattern");
8590 }
8591 // Immediate offset load
8592 return BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg)
8593 .addReg(RegNo: OffsetReg)
8594 .addImm(Val: 0)
8595 .setMemRefs(MMOs);
8596 };
8597
8598 // Load the remaining lanes into register 0.
8599 auto LanesToLoadToReg0 =
8600 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + 1,
8601 y: LoadToLaneInstrsAscending.begin() + NumLanes / 2);
8602 Register PrevReg = SubregToReg->getOperand(i: 0).getReg();
8603 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg0)) {
8604 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8605 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8606 OffsetRegOperand.getReg(),
8607 OffsetRegOperand.isKill());
8608 DelInstrs.push_back(Elt: LoadInstr);
8609 }
8610 Register LastLoadReg0 = PrevReg;
8611
8612 // First load into register 1. Perform an integer load to zero out the upper
8613 // lanes in a single instruction.
8614 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
8615 MachineInstr *OriginalSplitLoad =
8616 *std::next(x: LoadToLaneInstrsAscending.begin(), n: NumLanes / 2);
8617 Register DestRegForMiddleIndex = MRI.createVirtualRegister(
8618 RegClass: MRI.getRegClass(Reg: Lane0Load->getOperand(i: 0).getReg()));
8619
8620 const MachineOperand &OriginalSplitToLoadOffsetOperand =
8621 OriginalSplitLoad->getOperand(i: 3);
8622 MachineInstrBuilder MiddleIndexLoadInstr =
8623 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
8624 OriginalSplitToLoadOffsetOperand.getReg(),
8625 OriginalSplitLoad->memoperands());
8626
8627 InstrIdxForVirtReg.insert(
8628 KV: std::make_pair(x&: DestRegForMiddleIndex, y: InsInstrs.size()));
8629 InsInstrs.push_back(Elt: MiddleIndexLoadInstr);
8630 DelInstrs.push_back(Elt: OriginalSplitLoad);
8631
8632 // Subreg To Reg instruction for register 1.
8633 Register DestRegForSubregToReg = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8634 unsigned SubregType;
8635 switch (NumLanes) {
8636 case 4:
8637 SubregType = AArch64::ssub;
8638 break;
8639 case 8:
8640 SubregType = AArch64::hsub;
8641 break;
8642 case 16:
8643 SubregType = AArch64::bsub;
8644 break;
8645 default:
8646 llvm_unreachable(
8647 "Got invalid NumLanes for machine-combiner gather pattern");
8648 }
8649
8650 auto SubRegToRegInstr =
8651 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubregToReg->getOpcode()),
8652 DestReg: DestRegForSubregToReg)
8653 .addReg(RegNo: DestRegForMiddleIndex, Flags: getKillRegState(B: true))
8654 .addImm(Val: SubregType);
8655 InstrIdxForVirtReg.insert(
8656 KV: std::make_pair(x&: DestRegForSubregToReg, y: InsInstrs.size()));
8657 InsInstrs.push_back(Elt: SubRegToRegInstr);
8658
8659 // Load remaining lanes into register 1.
8660 auto LanesToLoadToReg1 =
8661 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + NumLanes / 2 + 1,
8662 y: LoadToLaneInstrsAscending.end());
8663 PrevReg = SubRegToRegInstr->getOperand(i: 0).getReg();
8664 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg1)) {
8665 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8666 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8667 OffsetRegOperand.getReg(),
8668 OffsetRegOperand.isKill());
8669
8670 // Do not add the last reg to DelInstrs - it will be removed later.
8671 if (Index == NumLanes / 2 - 2) {
8672 break;
8673 }
8674 DelInstrs.push_back(Elt: LoadInstr);
8675 }
8676 Register LastLoadReg1 = PrevReg;
8677
8678 // Create the final zip instruction to combine the results.
8679 MachineInstrBuilder ZipInstr =
8680 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::ZIP1v2i64),
8681 DestReg: Root.getOperand(i: 0).getReg())
8682 .addReg(RegNo: LastLoadReg0)
8683 .addReg(RegNo: LastLoadReg1);
8684 InsInstrs.push_back(Elt: ZipInstr);
8685}
8686
8687CombinerObjective
8688AArch64InstrInfo::getCombinerObjective(unsigned Pattern) const {
8689 switch (Pattern) {
8690 case AArch64MachineCombinerPattern::SUBADD_OP1:
8691 case AArch64MachineCombinerPattern::SUBADD_OP2:
8692 case AArch64MachineCombinerPattern::GATHER_LANE_i32:
8693 case AArch64MachineCombinerPattern::GATHER_LANE_i16:
8694 case AArch64MachineCombinerPattern::GATHER_LANE_i8:
8695 return CombinerObjective::MustReduceDepth;
8696 default:
8697 return TargetInstrInfo::getCombinerObjective(Pattern);
8698 }
8699}
8700
8701/// Return true when there is potentially a faster code sequence for an
8702/// instruction chain ending in \p Root. All potential patterns are listed in
8703/// the \p Pattern vector. Pattern should be sorted in priority order since the
8704/// pattern evaluator stops checking as soon as it finds a faster sequence.
8705
8706bool AArch64InstrInfo::getMachineCombinerPatterns(
8707 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
8708 bool DoRegPressureReduce) const {
8709 // Integer patterns
8710 if (getMaddPatterns(Root, Patterns))
8711 return true;
8712 // Floating point patterns
8713 if (getFMULPatterns(Root, Patterns))
8714 return true;
8715 if (getFMAPatterns(Root, Patterns))
8716 return true;
8717 if (getFNEGPatterns(Root, Patterns))
8718 return true;
8719
8720 // Other patterns
8721 if (getMiscPatterns(Root, Patterns))
8722 return true;
8723
8724 // Load patterns
8725 if (getLoadPatterns(Root, Patterns))
8726 return true;
8727
8728 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
8729 DoRegPressureReduce);
8730}
8731
8732enum class FMAInstKind { Default, Indexed, Accumulator };
8733/// genFusedMultiply - Generate fused multiply instructions.
8734/// This function supports both integer and floating point instructions.
8735/// A typical example:
8736/// F|MUL I=A,B,0
8737/// F|ADD R,I,C
8738/// ==> F|MADD R,A,B,C
8739/// \param MF Containing MachineFunction
8740/// \param MRI Register information
8741/// \param TII Target information
8742/// \param Root is the F|ADD instruction
8743/// \param [out] InsInstrs is a vector of machine instructions and will
8744/// contain the generated madd instruction
8745/// \param IdxMulOpd is index of operand in Root that is the result of
8746/// the F|MUL. In the example above IdxMulOpd is 1.
8747/// \param MaddOpc the opcode fo the f|madd instruction
8748/// \param RC Register class of operands
8749/// \param kind of fma instruction (addressing mode) to be generated
8750/// \param ReplacedAddend is the result register from the instruction
8751/// replacing the non-combined operand, if any.
8752static MachineInstr *
8753genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
8754 const TargetInstrInfo *TII, MachineInstr &Root,
8755 SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
8756 unsigned MaddOpc, const TargetRegisterClass *RC,
8757 FMAInstKind kind = FMAInstKind::Default,
8758 const Register *ReplacedAddend = nullptr) {
8759 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8760
8761 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
8762 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
8763 Register ResultReg = Root.getOperand(i: 0).getReg();
8764 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8765 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
8766 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8767 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
8768
8769 Register SrcReg2;
8770 bool Src2IsKill;
8771 if (ReplacedAddend) {
8772 // If we just generated a new addend, we must be it's only use.
8773 SrcReg2 = *ReplacedAddend;
8774 Src2IsKill = true;
8775 } else {
8776 SrcReg2 = Root.getOperand(i: IdxOtherOpd).getReg();
8777 Src2IsKill = Root.getOperand(i: IdxOtherOpd).isKill();
8778 }
8779
8780 if (ResultReg.isVirtual())
8781 MRI.constrainRegClass(Reg: ResultReg, RC);
8782 if (SrcReg0.isVirtual())
8783 MRI.constrainRegClass(Reg: SrcReg0, RC);
8784 if (SrcReg1.isVirtual())
8785 MRI.constrainRegClass(Reg: SrcReg1, RC);
8786 if (SrcReg2.isVirtual())
8787 MRI.constrainRegClass(Reg: SrcReg2, RC);
8788
8789 MachineInstrBuilder MIB;
8790 if (kind == FMAInstKind::Default)
8791 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8792 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8793 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8794 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8795 else if (kind == FMAInstKind::Indexed)
8796 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8797 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8798 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8799 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8800 .addImm(Val: MUL->getOperand(i: 3).getImm());
8801 else if (kind == FMAInstKind::Accumulator)
8802 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8803 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8804 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8805 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill));
8806 else
8807 assert(false && "Invalid FMA instruction kind \n");
8808 // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
8809 InsInstrs.push_back(Elt: MIB);
8810 return MUL;
8811}
8812
8813static MachineInstr *
8814genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI,
8815 const TargetInstrInfo *TII, MachineInstr &Root,
8816 SmallVectorImpl<MachineInstr *> &InsInstrs) {
8817 MachineInstr *MAD = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8818
8819 unsigned Opc = 0;
8820 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MAD->getOperand(i: 0).getReg());
8821 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
8822 Opc = AArch64::FNMADDSrrr;
8823 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
8824 Opc = AArch64::FNMADDDrrr;
8825 else
8826 return nullptr;
8827
8828 Register ResultReg = Root.getOperand(i: 0).getReg();
8829 Register SrcReg0 = MAD->getOperand(i: 1).getReg();
8830 Register SrcReg1 = MAD->getOperand(i: 2).getReg();
8831 Register SrcReg2 = MAD->getOperand(i: 3).getReg();
8832 bool Src0IsKill = MAD->getOperand(i: 1).isKill();
8833 bool Src1IsKill = MAD->getOperand(i: 2).isKill();
8834 bool Src2IsKill = MAD->getOperand(i: 3).isKill();
8835 if (ResultReg.isVirtual())
8836 MRI.constrainRegClass(Reg: ResultReg, RC);
8837 if (SrcReg0.isVirtual())
8838 MRI.constrainRegClass(Reg: SrcReg0, RC);
8839 if (SrcReg1.isVirtual())
8840 MRI.constrainRegClass(Reg: SrcReg1, RC);
8841 if (SrcReg2.isVirtual())
8842 MRI.constrainRegClass(Reg: SrcReg2, RC);
8843
8844 MachineInstrBuilder MIB =
8845 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Opc), DestReg: ResultReg)
8846 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8847 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8848 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8849 InsInstrs.push_back(Elt: MIB);
8850
8851 return MAD;
8852}
8853
8854/// Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
8855static MachineInstr *
8856genIndexedMultiply(MachineInstr &Root,
8857 SmallVectorImpl<MachineInstr *> &InsInstrs,
8858 unsigned IdxDupOp, unsigned MulOpc,
8859 const TargetRegisterClass *RC, MachineRegisterInfo &MRI) {
8860 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
8861 "Invalid index of FMUL operand");
8862
8863 MachineFunction &MF = *Root.getMF();
8864 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8865
8866 MachineInstr *Dup =
8867 MF.getRegInfo().getUniqueVRegDef(Reg: Root.getOperand(i: IdxDupOp).getReg());
8868
8869 if (Dup->getOpcode() == TargetOpcode::COPY)
8870 Dup = MRI.getUniqueVRegDef(Reg: Dup->getOperand(i: 1).getReg());
8871
8872 Register DupSrcReg = Dup->getOperand(i: 1).getReg();
8873 MRI.clearKillFlags(Reg: DupSrcReg);
8874 MRI.constrainRegClass(Reg: DupSrcReg, RC);
8875
8876 unsigned DupSrcLane = Dup->getOperand(i: 2).getImm();
8877
8878 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
8879 MachineOperand &MulOp = Root.getOperand(i: IdxMulOp);
8880
8881 Register ResultReg = Root.getOperand(i: 0).getReg();
8882
8883 MachineInstrBuilder MIB;
8884 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MulOpc), DestReg: ResultReg)
8885 .add(MO: MulOp)
8886 .addReg(RegNo: DupSrcReg)
8887 .addImm(Val: DupSrcLane);
8888
8889 InsInstrs.push_back(Elt: MIB);
8890 return &Root;
8891}
8892
8893/// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
8894/// instructions.
8895///
8896/// \see genFusedMultiply
8897static MachineInstr *genFusedMultiplyAcc(
8898 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8899 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8900 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8901 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8902 kind: FMAInstKind::Accumulator);
8903}
8904
8905/// genNeg - Helper to generate an intermediate negation of the second operand
8906/// of Root
8907static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
8908 const TargetInstrInfo *TII, MachineInstr &Root,
8909 SmallVectorImpl<MachineInstr *> &InsInstrs,
8910 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8911 unsigned MnegOpc, const TargetRegisterClass *RC) {
8912 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
8913 MachineInstrBuilder MIB =
8914 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MnegOpc), DestReg: NewVR)
8915 .add(MO: Root.getOperand(i: 2));
8916 InsInstrs.push_back(Elt: MIB);
8917
8918 assert(InstrIdxForVirtReg.empty());
8919 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
8920
8921 return NewVR;
8922}
8923
8924/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8925/// instructions with an additional negation of the accumulator
8926static MachineInstr *genFusedMultiplyAccNeg(
8927 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8928 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8929 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8930 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8931 assert(IdxMulOpd == 1);
8932
8933 Register NewVR =
8934 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8935 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8936 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
8937}
8938
8939/// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
8940/// instructions.
8941///
8942/// \see genFusedMultiply
8943static MachineInstr *genFusedMultiplyIdx(
8944 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8945 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8946 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8947 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8948 kind: FMAInstKind::Indexed);
8949}
8950
8951/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8952/// instructions with an additional negation of the accumulator
8953static MachineInstr *genFusedMultiplyIdxNeg(
8954 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8955 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8956 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8957 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8958 assert(IdxMulOpd == 1);
8959
8960 Register NewVR =
8961 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8962
8963 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8964 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
8965}
8966
8967/// genMaddR - Generate madd instruction and combine mul and add using
8968/// an extra virtual register
8969/// Example - an ADD intermediate needs to be stored in a register:
8970/// MUL I=A,B,0
8971/// ADD R,I,Imm
8972/// ==> ORR V, ZR, Imm
8973/// ==> MADD R,A,B,V
8974/// \param MF Containing MachineFunction
8975/// \param MRI Register information
8976/// \param TII Target information
8977/// \param Root is the ADD instruction
8978/// \param [out] InsInstrs is a vector of machine instructions and will
8979/// contain the generated madd instruction
8980/// \param IdxMulOpd is index of operand in Root that is the result of
8981/// the MUL. In the example above IdxMulOpd is 1.
8982/// \param MaddOpc the opcode fo the madd instruction
8983/// \param VR is a virtual register that holds the value of an ADD operand
8984/// (V in the example above).
8985/// \param RC Register class of operands
8986static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
8987 const TargetInstrInfo *TII, MachineInstr &Root,
8988 SmallVectorImpl<MachineInstr *> &InsInstrs,
8989 unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
8990 const TargetRegisterClass *RC) {
8991 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8992
8993 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
8994 Register ResultReg = Root.getOperand(i: 0).getReg();
8995 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8996 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
8997 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8998 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
8999
9000 if (ResultReg.isVirtual())
9001 MRI.constrainRegClass(Reg: ResultReg, RC);
9002 if (SrcReg0.isVirtual())
9003 MRI.constrainRegClass(Reg: SrcReg0, RC);
9004 if (SrcReg1.isVirtual())
9005 MRI.constrainRegClass(Reg: SrcReg1, RC);
9006 if (Register::isVirtualRegister(Reg: VR))
9007 MRI.constrainRegClass(Reg: VR, RC);
9008
9009 MachineInstrBuilder MIB =
9010 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
9011 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
9012 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
9013 .addReg(RegNo: VR);
9014 // Insert the MADD
9015 InsInstrs.push_back(Elt: MIB);
9016 return MUL;
9017}
9018
9019/// Do the following transformation
9020/// A - (B + C) ==> (A - B) - C
9021/// A - (B + C) ==> (A - C) - B
9022static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI,
9023 const TargetInstrInfo *TII, MachineInstr &Root,
9024 SmallVectorImpl<MachineInstr *> &InsInstrs,
9025 SmallVectorImpl<MachineInstr *> &DelInstrs,
9026 unsigned IdxOpd1,
9027 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
9028 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
9029 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
9030 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
9031
9032 Register ResultReg = Root.getOperand(i: 0).getReg();
9033 Register RegA = Root.getOperand(i: 1).getReg();
9034 bool RegAIsKill = Root.getOperand(i: 1).isKill();
9035 Register RegB = AddMI->getOperand(i: IdxOpd1).getReg();
9036 bool RegBIsKill = AddMI->getOperand(i: IdxOpd1).isKill();
9037 Register RegC = AddMI->getOperand(i: IdxOtherOpd).getReg();
9038 bool RegCIsKill = AddMI->getOperand(i: IdxOtherOpd).isKill();
9039 Register NewVR =
9040 MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 2).getReg()));
9041
9042 unsigned Opcode = Root.getOpcode();
9043 if (Opcode == AArch64::SUBSWrr)
9044 Opcode = AArch64::SUBWrr;
9045 else if (Opcode == AArch64::SUBSXrr)
9046 Opcode = AArch64::SUBXrr;
9047 else
9048 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
9049 "Unexpected instruction opcode.");
9050
9051 uint32_t Flags = Root.mergeFlagsWith(Other: *AddMI);
9052 Flags &= ~MachineInstr::NoSWrap;
9053 Flags &= ~MachineInstr::NoUWrap;
9054
9055 MachineInstrBuilder MIB1 =
9056 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: NewVR)
9057 .addReg(RegNo: RegA, Flags: getKillRegState(B: RegAIsKill))
9058 .addReg(RegNo: RegB, Flags: getKillRegState(B: RegBIsKill))
9059 .setMIFlags(Flags);
9060 MachineInstrBuilder MIB2 =
9061 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: ResultReg)
9062 .addReg(RegNo: NewVR, Flags: getKillRegState(B: true))
9063 .addReg(RegNo: RegC, Flags: getKillRegState(B: RegCIsKill))
9064 .setMIFlags(Flags);
9065
9066 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9067 InsInstrs.push_back(Elt: MIB1);
9068 InsInstrs.push_back(Elt: MIB2);
9069 DelInstrs.push_back(Elt: AddMI);
9070 DelInstrs.push_back(Elt: &Root);
9071}
9072
9073unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
9074 unsigned int AccumulatorOpCode) const {
9075 switch (AccumulatorOpCode) {
9076 case AArch64::UABALB_ZZZ_D:
9077 case AArch64::SABALB_ZZZ_D:
9078 case AArch64::UABALT_ZZZ_D:
9079 case AArch64::SABALT_ZZZ_D:
9080 return AArch64::ADD_ZZZ_D;
9081 case AArch64::UABALB_ZZZ_H:
9082 case AArch64::SABALB_ZZZ_H:
9083 case AArch64::UABALT_ZZZ_H:
9084 case AArch64::SABALT_ZZZ_H:
9085 return AArch64::ADD_ZZZ_H;
9086 case AArch64::UABALB_ZZZ_S:
9087 case AArch64::SABALB_ZZZ_S:
9088 case AArch64::UABALT_ZZZ_S:
9089 case AArch64::SABALT_ZZZ_S:
9090 return AArch64::ADD_ZZZ_S;
9091 case AArch64::UABALv16i8_v8i16:
9092 case AArch64::SABALv8i8_v8i16:
9093 case AArch64::SABAv8i16:
9094 case AArch64::UABAv8i16:
9095 return AArch64::ADDv8i16;
9096 case AArch64::SABALv2i32_v2i64:
9097 case AArch64::UABALv2i32_v2i64:
9098 case AArch64::SABALv4i32_v2i64:
9099 return AArch64::ADDv2i64;
9100 case AArch64::UABALv4i16_v4i32:
9101 case AArch64::SABALv4i16_v4i32:
9102 case AArch64::SABALv8i16_v4i32:
9103 case AArch64::SABAv4i32:
9104 case AArch64::UABAv4i32:
9105 return AArch64::ADDv4i32;
9106 case AArch64::UABALv4i32_v2i64:
9107 return AArch64::ADDv2i64;
9108 case AArch64::UABALv8i16_v4i32:
9109 return AArch64::ADDv4i32;
9110 case AArch64::UABALv8i8_v8i16:
9111 case AArch64::SABALv16i8_v8i16:
9112 return AArch64::ADDv8i16;
9113 case AArch64::UABAv16i8:
9114 case AArch64::SABAv16i8:
9115 return AArch64::ADDv16i8;
9116 case AArch64::UABAv4i16:
9117 case AArch64::SABAv4i16:
9118 return AArch64::ADDv4i16;
9119 case AArch64::UABAv2i32:
9120 case AArch64::SABAv2i32:
9121 return AArch64::ADDv2i32;
9122 case AArch64::UABAv8i8:
9123 case AArch64::SABAv8i8:
9124 return AArch64::ADDv8i8;
9125 default:
9126 llvm_unreachable("Unknown accumulator opcode");
9127 }
9128}
9129
9130/// When getMachineCombinerPatterns() finds potential patterns,
9131/// this function generates the instructions that could replace the
9132/// original code sequence
9133void AArch64InstrInfo::genAlternativeCodeSequence(
9134 MachineInstr &Root, unsigned Pattern,
9135 SmallVectorImpl<MachineInstr *> &InsInstrs,
9136 SmallVectorImpl<MachineInstr *> &DelInstrs,
9137 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
9138 MachineBasicBlock &MBB = *Root.getParent();
9139 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9140 MachineFunction &MF = *MBB.getParent();
9141 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
9142
9143 MachineInstr *MUL = nullptr;
9144 const TargetRegisterClass *RC;
9145 unsigned Opc;
9146 switch (Pattern) {
9147 default:
9148 // Reassociate instructions.
9149 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
9150 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
9151 return;
9152 case AArch64MachineCombinerPattern::SUBADD_OP1:
9153 // A - (B + C)
9154 // ==> (A - B) - C
9155 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 1,
9156 InstrIdxForVirtReg);
9157 return;
9158 case AArch64MachineCombinerPattern::SUBADD_OP2:
9159 // A - (B + C)
9160 // ==> (A - C) - B
9161 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 2,
9162 InstrIdxForVirtReg);
9163 return;
9164 case AArch64MachineCombinerPattern::MULADDW_OP1:
9165 case AArch64MachineCombinerPattern::MULADDX_OP1:
9166 // MUL I=A,B,0
9167 // ADD R,I,C
9168 // ==> MADD R,A,B,C
9169 // --- Create(MADD);
9170 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP1) {
9171 Opc = AArch64::MADDWrrr;
9172 RC = &AArch64::GPR32RegClass;
9173 } else {
9174 Opc = AArch64::MADDXrrr;
9175 RC = &AArch64::GPR64RegClass;
9176 }
9177 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9178 break;
9179 case AArch64MachineCombinerPattern::MULADDW_OP2:
9180 case AArch64MachineCombinerPattern::MULADDX_OP2:
9181 // MUL I=A,B,0
9182 // ADD R,C,I
9183 // ==> MADD R,A,B,C
9184 // --- Create(MADD);
9185 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP2) {
9186 Opc = AArch64::MADDWrrr;
9187 RC = &AArch64::GPR32RegClass;
9188 } else {
9189 Opc = AArch64::MADDXrrr;
9190 RC = &AArch64::GPR64RegClass;
9191 }
9192 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9193 break;
9194 case AArch64MachineCombinerPattern::MULADDWI_OP1:
9195 case AArch64MachineCombinerPattern::MULADDXI_OP1:
9196 case AArch64MachineCombinerPattern::MULSUBWI_OP1:
9197 case AArch64MachineCombinerPattern::MULSUBXI_OP1: {
9198 // MUL I=A,B,0
9199 // ADD/SUB R,I,Imm
9200 // ==> MOV V, Imm/-Imm
9201 // ==> MADD R,A,B,V
9202 // --- Create(MADD);
9203 const TargetRegisterClass *RC;
9204 unsigned BitSize, MovImm;
9205 if (Pattern == AArch64MachineCombinerPattern::MULADDWI_OP1 ||
9206 Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1) {
9207 MovImm = AArch64::MOVi32imm;
9208 RC = &AArch64::GPR32spRegClass;
9209 BitSize = 32;
9210 Opc = AArch64::MADDWrrr;
9211 RC = &AArch64::GPR32RegClass;
9212 } else {
9213 MovImm = AArch64::MOVi64imm;
9214 RC = &AArch64::GPR64spRegClass;
9215 BitSize = 64;
9216 Opc = AArch64::MADDXrrr;
9217 RC = &AArch64::GPR64RegClass;
9218 }
9219 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9220 uint64_t Imm = Root.getOperand(i: 2).getImm();
9221
9222 if (Root.getOperand(i: 3).isImm()) {
9223 unsigned Val = Root.getOperand(i: 3).getImm();
9224 Imm = Imm << Val;
9225 }
9226 bool IsSub = Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1 ||
9227 Pattern == AArch64MachineCombinerPattern::MULSUBXI_OP1;
9228 uint64_t UImm = SignExtend64(X: IsSub ? -Imm : Imm, B: BitSize);
9229 // Check that the immediate can be composed via a single instruction.
9230 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
9231 AArch64_IMM::expandMOVImm(Imm: UImm, BitSize, Insn);
9232 if (Insn.size() != 1)
9233 return;
9234 MachineInstrBuilder MIB1 =
9235 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MovImm), DestReg: NewVR)
9236 .addImm(Val: IsSub ? -Imm : Imm);
9237 InsInstrs.push_back(Elt: MIB1);
9238 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9239 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9240 break;
9241 }
9242 case AArch64MachineCombinerPattern::MULSUBW_OP1:
9243 case AArch64MachineCombinerPattern::MULSUBX_OP1: {
9244 // MUL I=A,B,0
9245 // SUB R,I, C
9246 // ==> SUB V, 0, C
9247 // ==> MADD R,A,B,V // = -C + A*B
9248 // --- Create(MADD);
9249 const TargetRegisterClass *SubRC;
9250 unsigned SubOpc, ZeroReg;
9251 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP1) {
9252 SubOpc = AArch64::SUBWrr;
9253 SubRC = &AArch64::GPR32spRegClass;
9254 ZeroReg = AArch64::WZR;
9255 Opc = AArch64::MADDWrrr;
9256 RC = &AArch64::GPR32RegClass;
9257 } else {
9258 SubOpc = AArch64::SUBXrr;
9259 SubRC = &AArch64::GPR64spRegClass;
9260 ZeroReg = AArch64::XZR;
9261 Opc = AArch64::MADDXrrr;
9262 RC = &AArch64::GPR64RegClass;
9263 }
9264 Register NewVR = MRI.createVirtualRegister(RegClass: SubRC);
9265 // SUB NewVR, 0, C
9266 MachineInstrBuilder MIB1 =
9267 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubOpc), DestReg: NewVR)
9268 .addReg(RegNo: ZeroReg)
9269 .add(MO: Root.getOperand(i: 2));
9270 InsInstrs.push_back(Elt: MIB1);
9271 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9272 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9273 break;
9274 }
9275 case AArch64MachineCombinerPattern::MULSUBW_OP2:
9276 case AArch64MachineCombinerPattern::MULSUBX_OP2:
9277 // MUL I=A,B,0
9278 // SUB R,C,I
9279 // ==> MSUB R,A,B,C (computes C - A*B)
9280 // --- Create(MSUB);
9281 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP2) {
9282 Opc = AArch64::MSUBWrrr;
9283 RC = &AArch64::GPR32RegClass;
9284 } else {
9285 Opc = AArch64::MSUBXrrr;
9286 RC = &AArch64::GPR64RegClass;
9287 }
9288 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9289 break;
9290 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
9291 Opc = AArch64::MLAv8i8;
9292 RC = &AArch64::FPR64RegClass;
9293 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9294 break;
9295 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
9296 Opc = AArch64::MLAv8i8;
9297 RC = &AArch64::FPR64RegClass;
9298 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9299 break;
9300 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
9301 Opc = AArch64::MLAv16i8;
9302 RC = &AArch64::FPR128RegClass;
9303 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9304 break;
9305 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
9306 Opc = AArch64::MLAv16i8;
9307 RC = &AArch64::FPR128RegClass;
9308 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9309 break;
9310 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
9311 Opc = AArch64::MLAv4i16;
9312 RC = &AArch64::FPR64RegClass;
9313 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9314 break;
9315 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
9316 Opc = AArch64::MLAv4i16;
9317 RC = &AArch64::FPR64RegClass;
9318 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9319 break;
9320 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
9321 Opc = AArch64::MLAv8i16;
9322 RC = &AArch64::FPR128RegClass;
9323 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9324 break;
9325 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
9326 Opc = AArch64::MLAv8i16;
9327 RC = &AArch64::FPR128RegClass;
9328 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9329 break;
9330 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
9331 Opc = AArch64::MLAv2i32;
9332 RC = &AArch64::FPR64RegClass;
9333 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9334 break;
9335 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
9336 Opc = AArch64::MLAv2i32;
9337 RC = &AArch64::FPR64RegClass;
9338 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9339 break;
9340 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
9341 Opc = AArch64::MLAv4i32;
9342 RC = &AArch64::FPR128RegClass;
9343 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9344 break;
9345 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
9346 Opc = AArch64::MLAv4i32;
9347 RC = &AArch64::FPR128RegClass;
9348 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9349 break;
9350
9351 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
9352 Opc = AArch64::MLAv8i8;
9353 RC = &AArch64::FPR64RegClass;
9354 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9355 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i8,
9356 RC);
9357 break;
9358 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
9359 Opc = AArch64::MLSv8i8;
9360 RC = &AArch64::FPR64RegClass;
9361 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9362 break;
9363 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
9364 Opc = AArch64::MLAv16i8;
9365 RC = &AArch64::FPR128RegClass;
9366 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9367 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv16i8,
9368 RC);
9369 break;
9370 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
9371 Opc = AArch64::MLSv16i8;
9372 RC = &AArch64::FPR128RegClass;
9373 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9374 break;
9375 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
9376 Opc = AArch64::MLAv4i16;
9377 RC = &AArch64::FPR64RegClass;
9378 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9379 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9380 RC);
9381 break;
9382 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
9383 Opc = AArch64::MLSv4i16;
9384 RC = &AArch64::FPR64RegClass;
9385 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9386 break;
9387 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
9388 Opc = AArch64::MLAv8i16;
9389 RC = &AArch64::FPR128RegClass;
9390 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9391 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9392 RC);
9393 break;
9394 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
9395 Opc = AArch64::MLSv8i16;
9396 RC = &AArch64::FPR128RegClass;
9397 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9398 break;
9399 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
9400 Opc = AArch64::MLAv2i32;
9401 RC = &AArch64::FPR64RegClass;
9402 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9403 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9404 RC);
9405 break;
9406 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
9407 Opc = AArch64::MLSv2i32;
9408 RC = &AArch64::FPR64RegClass;
9409 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9410 break;
9411 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
9412 Opc = AArch64::MLAv4i32;
9413 RC = &AArch64::FPR128RegClass;
9414 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9415 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9416 RC);
9417 break;
9418 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
9419 Opc = AArch64::MLSv4i32;
9420 RC = &AArch64::FPR128RegClass;
9421 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9422 break;
9423
9424 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
9425 Opc = AArch64::MLAv4i16_indexed;
9426 RC = &AArch64::FPR64RegClass;
9427 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9428 break;
9429 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
9430 Opc = AArch64::MLAv4i16_indexed;
9431 RC = &AArch64::FPR64RegClass;
9432 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9433 break;
9434 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
9435 Opc = AArch64::MLAv8i16_indexed;
9436 RC = &AArch64::FPR128RegClass;
9437 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9438 break;
9439 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
9440 Opc = AArch64::MLAv8i16_indexed;
9441 RC = &AArch64::FPR128RegClass;
9442 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9443 break;
9444 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
9445 Opc = AArch64::MLAv2i32_indexed;
9446 RC = &AArch64::FPR64RegClass;
9447 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9448 break;
9449 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
9450 Opc = AArch64::MLAv2i32_indexed;
9451 RC = &AArch64::FPR64RegClass;
9452 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9453 break;
9454 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
9455 Opc = AArch64::MLAv4i32_indexed;
9456 RC = &AArch64::FPR128RegClass;
9457 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9458 break;
9459 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
9460 Opc = AArch64::MLAv4i32_indexed;
9461 RC = &AArch64::FPR128RegClass;
9462 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9463 break;
9464
9465 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
9466 Opc = AArch64::MLAv4i16_indexed;
9467 RC = &AArch64::FPR64RegClass;
9468 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9469 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9470 RC);
9471 break;
9472 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
9473 Opc = AArch64::MLSv4i16_indexed;
9474 RC = &AArch64::FPR64RegClass;
9475 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9476 break;
9477 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
9478 Opc = AArch64::MLAv8i16_indexed;
9479 RC = &AArch64::FPR128RegClass;
9480 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9481 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9482 RC);
9483 break;
9484 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
9485 Opc = AArch64::MLSv8i16_indexed;
9486 RC = &AArch64::FPR128RegClass;
9487 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9488 break;
9489 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
9490 Opc = AArch64::MLAv2i32_indexed;
9491 RC = &AArch64::FPR64RegClass;
9492 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9493 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9494 RC);
9495 break;
9496 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
9497 Opc = AArch64::MLSv2i32_indexed;
9498 RC = &AArch64::FPR64RegClass;
9499 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9500 break;
9501 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
9502 Opc = AArch64::MLAv4i32_indexed;
9503 RC = &AArch64::FPR128RegClass;
9504 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9505 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9506 RC);
9507 break;
9508 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
9509 Opc = AArch64::MLSv4i32_indexed;
9510 RC = &AArch64::FPR128RegClass;
9511 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9512 break;
9513
9514 // Floating Point Support
9515 case AArch64MachineCombinerPattern::FMULADDH_OP1:
9516 Opc = AArch64::FMADDHrrr;
9517 RC = &AArch64::FPR16RegClass;
9518 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9519 break;
9520 case AArch64MachineCombinerPattern::FMULADDS_OP1:
9521 Opc = AArch64::FMADDSrrr;
9522 RC = &AArch64::FPR32RegClass;
9523 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9524 break;
9525 case AArch64MachineCombinerPattern::FMULADDD_OP1:
9526 Opc = AArch64::FMADDDrrr;
9527 RC = &AArch64::FPR64RegClass;
9528 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9529 break;
9530
9531 case AArch64MachineCombinerPattern::FMULADDH_OP2:
9532 Opc = AArch64::FMADDHrrr;
9533 RC = &AArch64::FPR16RegClass;
9534 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9535 break;
9536 case AArch64MachineCombinerPattern::FMULADDS_OP2:
9537 Opc = AArch64::FMADDSrrr;
9538 RC = &AArch64::FPR32RegClass;
9539 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9540 break;
9541 case AArch64MachineCombinerPattern::FMULADDD_OP2:
9542 Opc = AArch64::FMADDDrrr;
9543 RC = &AArch64::FPR64RegClass;
9544 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9545 break;
9546
9547 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
9548 Opc = AArch64::FMLAv1i32_indexed;
9549 RC = &AArch64::FPR32RegClass;
9550 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9551 kind: FMAInstKind::Indexed);
9552 break;
9553 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
9554 Opc = AArch64::FMLAv1i32_indexed;
9555 RC = &AArch64::FPR32RegClass;
9556 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9557 kind: FMAInstKind::Indexed);
9558 break;
9559
9560 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
9561 Opc = AArch64::FMLAv1i64_indexed;
9562 RC = &AArch64::FPR64RegClass;
9563 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9564 kind: FMAInstKind::Indexed);
9565 break;
9566 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
9567 Opc = AArch64::FMLAv1i64_indexed;
9568 RC = &AArch64::FPR64RegClass;
9569 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9570 kind: FMAInstKind::Indexed);
9571 break;
9572
9573 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
9574 RC = &AArch64::FPR64RegClass;
9575 Opc = AArch64::FMLAv4i16_indexed;
9576 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9577 kind: FMAInstKind::Indexed);
9578 break;
9579 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
9580 RC = &AArch64::FPR64RegClass;
9581 Opc = AArch64::FMLAv4f16;
9582 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9583 kind: FMAInstKind::Accumulator);
9584 break;
9585 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
9586 RC = &AArch64::FPR64RegClass;
9587 Opc = AArch64::FMLAv4i16_indexed;
9588 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9589 kind: FMAInstKind::Indexed);
9590 break;
9591 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
9592 RC = &AArch64::FPR64RegClass;
9593 Opc = AArch64::FMLAv4f16;
9594 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9595 kind: FMAInstKind::Accumulator);
9596 break;
9597
9598 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
9599 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
9600 RC = &AArch64::FPR64RegClass;
9601 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
9602 Opc = AArch64::FMLAv2i32_indexed;
9603 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9604 kind: FMAInstKind::Indexed);
9605 } else {
9606 Opc = AArch64::FMLAv2f32;
9607 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9608 kind: FMAInstKind::Accumulator);
9609 }
9610 break;
9611 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
9612 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
9613 RC = &AArch64::FPR64RegClass;
9614 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
9615 Opc = AArch64::FMLAv2i32_indexed;
9616 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9617 kind: FMAInstKind::Indexed);
9618 } else {
9619 Opc = AArch64::FMLAv2f32;
9620 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9621 kind: FMAInstKind::Accumulator);
9622 }
9623 break;
9624
9625 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
9626 RC = &AArch64::FPR128RegClass;
9627 Opc = AArch64::FMLAv8i16_indexed;
9628 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9629 kind: FMAInstKind::Indexed);
9630 break;
9631 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
9632 RC = &AArch64::FPR128RegClass;
9633 Opc = AArch64::FMLAv8f16;
9634 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9635 kind: FMAInstKind::Accumulator);
9636 break;
9637 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
9638 RC = &AArch64::FPR128RegClass;
9639 Opc = AArch64::FMLAv8i16_indexed;
9640 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9641 kind: FMAInstKind::Indexed);
9642 break;
9643 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
9644 RC = &AArch64::FPR128RegClass;
9645 Opc = AArch64::FMLAv8f16;
9646 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9647 kind: FMAInstKind::Accumulator);
9648 break;
9649
9650 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
9651 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
9652 RC = &AArch64::FPR128RegClass;
9653 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
9654 Opc = AArch64::FMLAv2i64_indexed;
9655 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9656 kind: FMAInstKind::Indexed);
9657 } else {
9658 Opc = AArch64::FMLAv2f64;
9659 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9660 kind: FMAInstKind::Accumulator);
9661 }
9662 break;
9663 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
9664 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
9665 RC = &AArch64::FPR128RegClass;
9666 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
9667 Opc = AArch64::FMLAv2i64_indexed;
9668 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9669 kind: FMAInstKind::Indexed);
9670 } else {
9671 Opc = AArch64::FMLAv2f64;
9672 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9673 kind: FMAInstKind::Accumulator);
9674 }
9675 break;
9676
9677 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
9678 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
9679 RC = &AArch64::FPR128RegClass;
9680 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
9681 Opc = AArch64::FMLAv4i32_indexed;
9682 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9683 kind: FMAInstKind::Indexed);
9684 } else {
9685 Opc = AArch64::FMLAv4f32;
9686 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9687 kind: FMAInstKind::Accumulator);
9688 }
9689 break;
9690
9691 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
9692 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
9693 RC = &AArch64::FPR128RegClass;
9694 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
9695 Opc = AArch64::FMLAv4i32_indexed;
9696 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9697 kind: FMAInstKind::Indexed);
9698 } else {
9699 Opc = AArch64::FMLAv4f32;
9700 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9701 kind: FMAInstKind::Accumulator);
9702 }
9703 break;
9704
9705 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
9706 Opc = AArch64::FNMSUBHrrr;
9707 RC = &AArch64::FPR16RegClass;
9708 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9709 break;
9710 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
9711 Opc = AArch64::FNMSUBSrrr;
9712 RC = &AArch64::FPR32RegClass;
9713 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9714 break;
9715 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
9716 Opc = AArch64::FNMSUBDrrr;
9717 RC = &AArch64::FPR64RegClass;
9718 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9719 break;
9720
9721 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
9722 Opc = AArch64::FNMADDHrrr;
9723 RC = &AArch64::FPR16RegClass;
9724 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9725 break;
9726 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
9727 Opc = AArch64::FNMADDSrrr;
9728 RC = &AArch64::FPR32RegClass;
9729 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9730 break;
9731 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
9732 Opc = AArch64::FNMADDDrrr;
9733 RC = &AArch64::FPR64RegClass;
9734 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9735 break;
9736
9737 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
9738 Opc = AArch64::FMSUBHrrr;
9739 RC = &AArch64::FPR16RegClass;
9740 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9741 break;
9742 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
9743 Opc = AArch64::FMSUBSrrr;
9744 RC = &AArch64::FPR32RegClass;
9745 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9746 break;
9747 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
9748 Opc = AArch64::FMSUBDrrr;
9749 RC = &AArch64::FPR64RegClass;
9750 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9751 break;
9752
9753 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
9754 Opc = AArch64::FMLSv1i32_indexed;
9755 RC = &AArch64::FPR32RegClass;
9756 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9757 kind: FMAInstKind::Indexed);
9758 break;
9759
9760 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
9761 Opc = AArch64::FMLSv1i64_indexed;
9762 RC = &AArch64::FPR64RegClass;
9763 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9764 kind: FMAInstKind::Indexed);
9765 break;
9766
9767 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
9768 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
9769 RC = &AArch64::FPR64RegClass;
9770 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9771 MachineInstrBuilder MIB1 =
9772 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f16), DestReg: NewVR)
9773 .add(MO: Root.getOperand(i: 2));
9774 InsInstrs.push_back(Elt: MIB1);
9775 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9776 if (Pattern == AArch64MachineCombinerPattern::FMLSv4f16_OP1) {
9777 Opc = AArch64::FMLAv4f16;
9778 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9779 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9780 } else {
9781 Opc = AArch64::FMLAv4i16_indexed;
9782 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9783 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9784 }
9785 break;
9786 }
9787 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
9788 RC = &AArch64::FPR64RegClass;
9789 Opc = AArch64::FMLSv4f16;
9790 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9791 kind: FMAInstKind::Accumulator);
9792 break;
9793 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
9794 RC = &AArch64::FPR64RegClass;
9795 Opc = AArch64::FMLSv4i16_indexed;
9796 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9797 kind: FMAInstKind::Indexed);
9798 break;
9799
9800 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
9801 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
9802 RC = &AArch64::FPR64RegClass;
9803 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
9804 Opc = AArch64::FMLSv2i32_indexed;
9805 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9806 kind: FMAInstKind::Indexed);
9807 } else {
9808 Opc = AArch64::FMLSv2f32;
9809 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9810 kind: FMAInstKind::Accumulator);
9811 }
9812 break;
9813
9814 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
9815 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
9816 RC = &AArch64::FPR128RegClass;
9817 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9818 MachineInstrBuilder MIB1 =
9819 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv8f16), DestReg: NewVR)
9820 .add(MO: Root.getOperand(i: 2));
9821 InsInstrs.push_back(Elt: MIB1);
9822 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9823 if (Pattern == AArch64MachineCombinerPattern::FMLSv8f16_OP1) {
9824 Opc = AArch64::FMLAv8f16;
9825 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9826 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9827 } else {
9828 Opc = AArch64::FMLAv8i16_indexed;
9829 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9830 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9831 }
9832 break;
9833 }
9834 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
9835 RC = &AArch64::FPR128RegClass;
9836 Opc = AArch64::FMLSv8f16;
9837 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9838 kind: FMAInstKind::Accumulator);
9839 break;
9840 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
9841 RC = &AArch64::FPR128RegClass;
9842 Opc = AArch64::FMLSv8i16_indexed;
9843 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9844 kind: FMAInstKind::Indexed);
9845 break;
9846
9847 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
9848 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
9849 RC = &AArch64::FPR128RegClass;
9850 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
9851 Opc = AArch64::FMLSv2i64_indexed;
9852 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9853 kind: FMAInstKind::Indexed);
9854 } else {
9855 Opc = AArch64::FMLSv2f64;
9856 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9857 kind: FMAInstKind::Accumulator);
9858 }
9859 break;
9860
9861 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
9862 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
9863 RC = &AArch64::FPR128RegClass;
9864 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
9865 Opc = AArch64::FMLSv4i32_indexed;
9866 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9867 kind: FMAInstKind::Indexed);
9868 } else {
9869 Opc = AArch64::FMLSv4f32;
9870 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9871 kind: FMAInstKind::Accumulator);
9872 }
9873 break;
9874 case AArch64MachineCombinerPattern::FMLSv2f32_OP1:
9875 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
9876 RC = &AArch64::FPR64RegClass;
9877 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9878 MachineInstrBuilder MIB1 =
9879 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f32), DestReg: NewVR)
9880 .add(MO: Root.getOperand(i: 2));
9881 InsInstrs.push_back(Elt: MIB1);
9882 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9883 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
9884 Opc = AArch64::FMLAv2i32_indexed;
9885 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9886 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9887 } else {
9888 Opc = AArch64::FMLAv2f32;
9889 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9890 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9891 }
9892 break;
9893 }
9894 case AArch64MachineCombinerPattern::FMLSv4f32_OP1:
9895 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
9896 RC = &AArch64::FPR128RegClass;
9897 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9898 MachineInstrBuilder MIB1 =
9899 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f32), DestReg: NewVR)
9900 .add(MO: Root.getOperand(i: 2));
9901 InsInstrs.push_back(Elt: MIB1);
9902 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9903 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
9904 Opc = AArch64::FMLAv4i32_indexed;
9905 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9906 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9907 } else {
9908 Opc = AArch64::FMLAv4f32;
9909 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9910 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9911 }
9912 break;
9913 }
9914 case AArch64MachineCombinerPattern::FMLSv2f64_OP1:
9915 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
9916 RC = &AArch64::FPR128RegClass;
9917 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9918 MachineInstrBuilder MIB1 =
9919 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f64), DestReg: NewVR)
9920 .add(MO: Root.getOperand(i: 2));
9921 InsInstrs.push_back(Elt: MIB1);
9922 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9923 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
9924 Opc = AArch64::FMLAv2i64_indexed;
9925 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9926 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9927 } else {
9928 Opc = AArch64::FMLAv2f64;
9929 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9930 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9931 }
9932 break;
9933 }
9934 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
9935 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2: {
9936 unsigned IdxDupOp =
9937 (Pattern == AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1) ? 1
9938 : 2;
9939 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i32_indexed,
9940 RC: &AArch64::FPR128RegClass, MRI);
9941 break;
9942 }
9943 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
9944 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2: {
9945 unsigned IdxDupOp =
9946 (Pattern == AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1) ? 1
9947 : 2;
9948 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i64_indexed,
9949 RC: &AArch64::FPR128RegClass, MRI);
9950 break;
9951 }
9952 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
9953 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2: {
9954 unsigned IdxDupOp =
9955 (Pattern == AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1) ? 1
9956 : 2;
9957 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i16_indexed,
9958 RC: &AArch64::FPR128_loRegClass, MRI);
9959 break;
9960 }
9961 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
9962 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2: {
9963 unsigned IdxDupOp =
9964 (Pattern == AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1) ? 1
9965 : 2;
9966 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i32_indexed,
9967 RC: &AArch64::FPR128RegClass, MRI);
9968 break;
9969 }
9970 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
9971 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2: {
9972 unsigned IdxDupOp =
9973 (Pattern == AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1) ? 1
9974 : 2;
9975 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv8i16_indexed,
9976 RC: &AArch64::FPR128_loRegClass, MRI);
9977 break;
9978 }
9979 case AArch64MachineCombinerPattern::FNMADD: {
9980 MUL = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs);
9981 break;
9982 }
9983 case AArch64MachineCombinerPattern::GATHER_LANE_i32: {
9984 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
9985 Pattern, NumLanes: 4);
9986 break;
9987 }
9988 case AArch64MachineCombinerPattern::GATHER_LANE_i16: {
9989 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
9990 Pattern, NumLanes: 8);
9991 break;
9992 }
9993 case AArch64MachineCombinerPattern::GATHER_LANE_i8: {
9994 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
9995 Pattern, NumLanes: 16);
9996 break;
9997 }
9998
9999 } // end switch (Pattern)
10000 // Record MUL and ADD/SUB for deletion
10001 if (MUL)
10002 DelInstrs.push_back(Elt: MUL);
10003 DelInstrs.push_back(Elt: &Root);
10004
10005 // Set the flags on the inserted instructions to be the merged flags of the
10006 // instructions that we have combined.
10007 uint32_t Flags = Root.getFlags();
10008 if (MUL)
10009 Flags = Root.mergeFlagsWith(Other: *MUL);
10010 for (auto *MI : InsInstrs)
10011 MI->setFlags(Flags);
10012}
10013
10014/// Replace csincr-branch sequence by simple conditional branch
10015///
10016/// Examples:
10017/// 1. \code
10018/// csinc w9, wzr, wzr, <condition code>
10019/// tbnz w9, #0, 0x44
10020/// \endcode
10021/// to
10022/// \code
10023/// b.<inverted condition code>
10024/// \endcode
10025///
10026/// 2. \code
10027/// csinc w9, wzr, wzr, <condition code>
10028/// tbz w9, #0, 0x44
10029/// \endcode
10030/// to
10031/// \code
10032/// b.<condition code>
10033/// \endcode
10034///
10035/// Replace compare and branch sequence by TBZ/TBNZ instruction when the
10036/// compare's constant operand is power of 2.
10037///
10038/// Examples:
10039/// \code
10040/// and w8, w8, #0x400
10041/// cbnz w8, L1
10042/// \endcode
10043/// to
10044/// \code
10045/// tbnz w8, #10, L1
10046/// \endcode
10047///
10048/// \param MI Conditional Branch
10049/// \return True when the simple conditional branch is generated
10050///
10051bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
10052 bool IsNegativeBranch = false;
10053 bool IsTestAndBranch = false;
10054 unsigned TargetBBInMI = 0;
10055 switch (MI.getOpcode()) {
10056 default:
10057 llvm_unreachable("Unknown branch instruction?");
10058 case AArch64::Bcc:
10059 case AArch64::CBWPri:
10060 case AArch64::CBXPri:
10061 case AArch64::CBBAssertExt:
10062 case AArch64::CBHAssertExt:
10063 case AArch64::CBWPrr:
10064 case AArch64::CBXPrr:
10065 return false;
10066 case AArch64::CBZW:
10067 case AArch64::CBZX:
10068 TargetBBInMI = 1;
10069 break;
10070 case AArch64::CBNZW:
10071 case AArch64::CBNZX:
10072 TargetBBInMI = 1;
10073 IsNegativeBranch = true;
10074 break;
10075 case AArch64::TBZW:
10076 case AArch64::TBZX:
10077 TargetBBInMI = 2;
10078 IsTestAndBranch = true;
10079 break;
10080 case AArch64::TBNZW:
10081 case AArch64::TBNZX:
10082 TargetBBInMI = 2;
10083 IsNegativeBranch = true;
10084 IsTestAndBranch = true;
10085 break;
10086 }
10087 // So we increment a zero register and test for bits other
10088 // than bit 0? Conservatively bail out in case the verifier
10089 // missed this case.
10090 if (IsTestAndBranch && MI.getOperand(i: 1).getImm())
10091 return false;
10092
10093 // Find Definition.
10094 assert(MI.getParent() && "Incomplete machine instruction\n");
10095 MachineBasicBlock *MBB = MI.getParent();
10096 MachineFunction *MF = MBB->getParent();
10097 MachineRegisterInfo *MRI = &MF->getRegInfo();
10098 Register VReg = MI.getOperand(i: 0).getReg();
10099 if (!VReg.isVirtual())
10100 return false;
10101
10102 MachineInstr *DefMI = MRI->getVRegDef(Reg: VReg);
10103 if (!DefMI)
10104 return false;
10105
10106 // Look through COPY instructions to find definition.
10107 while (DefMI->isCopy()) {
10108 Register CopyVReg = DefMI->getOperand(i: 1).getReg();
10109 if (!CopyVReg.isVirtual())
10110 return false;
10111 if (!MRI->hasOneNonDBGUse(RegNo: CopyVReg))
10112 return false;
10113 DefMI = MRI->getVRegDef(Reg: CopyVReg);
10114 if (!DefMI)
10115 return false;
10116 }
10117
10118 switch (DefMI->getOpcode()) {
10119 default:
10120 return false;
10121 // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
10122 case AArch64::ANDWri:
10123 case AArch64::ANDXri: {
10124 if (IsTestAndBranch)
10125 return false;
10126 if (DefMI->getParent() != MBB)
10127 return false;
10128 if (!MRI->hasOneNonDBGUse(RegNo: VReg))
10129 return false;
10130
10131 bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
10132 uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
10133 val: DefMI->getOperand(i: 2).getImm(), regSize: Is32Bit ? 32 : 64);
10134 if (!isPowerOf2_64(Value: Mask))
10135 return false;
10136
10137 MachineOperand &MO = DefMI->getOperand(i: 1);
10138 Register NewReg = MO.getReg();
10139 if (!NewReg.isVirtual())
10140 return false;
10141
10142 if (!MRI->getVRegDef(Reg: NewReg))
10143 return false;
10144
10145 MachineBasicBlock &RefToMBB = *MBB;
10146 MachineBasicBlock *TBB = MI.getOperand(i: 1).getMBB();
10147 DebugLoc DL = MI.getDebugLoc();
10148 unsigned Imm = Log2_64(Value: Mask);
10149 unsigned Opc = (Imm < 32)
10150 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
10151 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
10152 MachineInstr *NewMI = BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: Opc))
10153 .addReg(RegNo: NewReg)
10154 .addImm(Val: Imm)
10155 .addMBB(MBB: TBB);
10156 // Register lives on to the CBZ now.
10157 MO.setIsKill(false);
10158
10159 // For immediate smaller than 32, we need to use the 32-bit
10160 // variant (W) in all cases. Indeed the 64-bit variant does not
10161 // allow to encode them.
10162 // Therefore, if the input register is 64-bit, we need to take the
10163 // 32-bit sub-part.
10164 if (!Is32Bit && Imm < 32)
10165 NewMI->getOperand(i: 0).setSubReg(AArch64::sub_32);
10166 MI.eraseFromParent();
10167 return true;
10168 }
10169 // Look for CSINC
10170 case AArch64::CSINCWr:
10171 case AArch64::CSINCXr: {
10172 if (!(DefMI->getOperand(i: 1).getReg() == AArch64::WZR &&
10173 DefMI->getOperand(i: 2).getReg() == AArch64::WZR) &&
10174 !(DefMI->getOperand(i: 1).getReg() == AArch64::XZR &&
10175 DefMI->getOperand(i: 2).getReg() == AArch64::XZR))
10176 return false;
10177
10178 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
10179 isDead: true) != -1)
10180 return false;
10181
10182 AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(i: 3).getImm();
10183 // Convert only when the condition code is not modified between
10184 // the CSINC and the branch. The CC may be used by other
10185 // instructions in between.
10186 if (areCFlagsAccessedBetweenInstrs(From: DefMI, To: MI, TRI: &getRegisterInfo(), AccessToCheck: AK_Write))
10187 return false;
10188 MachineBasicBlock &RefToMBB = *MBB;
10189 MachineBasicBlock *TBB = MI.getOperand(i: TargetBBInMI).getMBB();
10190 DebugLoc DL = MI.getDebugLoc();
10191 if (IsNegativeBranch)
10192 CC = AArch64CC::getInvertedCondCode(Code: CC);
10193 BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: CC).addMBB(MBB: TBB);
10194 MI.eraseFromParent();
10195 return true;
10196 }
10197 }
10198}
10199
10200std::pair<unsigned, unsigned>
10201AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10202 const unsigned Mask = AArch64II::MO_FRAGMENT;
10203 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
10204}
10205
10206ArrayRef<std::pair<unsigned, const char *>>
10207AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10208 using namespace AArch64II;
10209
10210 static const std::pair<unsigned, const char *> TargetFlags[] = {
10211 {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
10212 {MO_G3, "aarch64-g3"}, {MO_G2, "aarch64-g2"},
10213 {MO_G1, "aarch64-g1"}, {MO_G0, "aarch64-g0"},
10214 {MO_HI12, "aarch64-hi12"}};
10215 return ArrayRef(TargetFlags);
10216}
10217
10218ArrayRef<std::pair<unsigned, const char *>>
10219AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
10220 using namespace AArch64II;
10221
10222 static const std::pair<unsigned, const char *> TargetFlags[] = {
10223 {MO_COFFSTUB, "aarch64-coffstub"},
10224 {MO_GOT, "aarch64-got"},
10225 {MO_NC, "aarch64-nc"},
10226 {MO_S, "aarch64-s"},
10227 {MO_TLS, "aarch64-tls"},
10228 {MO_DLLIMPORT, "aarch64-dllimport"},
10229 {MO_PREL, "aarch64-prel"},
10230 {MO_TAGGED, "aarch64-tagged"},
10231 {MO_ARM64EC_CALLMANGLE, "aarch64-arm64ec-callmangle"},
10232 };
10233 return ArrayRef(TargetFlags);
10234}
10235
10236ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10237AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10238 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10239 {{MOSuppressPair, "aarch64-suppress-pair"},
10240 {MOStridedAccess, "aarch64-strided-access"}};
10241 return ArrayRef(TargetFlags);
10242}
10243
10244/// Constants defining how certain sequences should be outlined.
10245/// This encompasses how an outlined function should be called, and what kind of
10246/// frame should be emitted for that outlined function.
10247///
10248/// \p MachineOutlinerDefault implies that the function should be called with
10249/// a save and restore of LR to the stack.
10250///
10251/// That is,
10252///
10253/// I1 Save LR OUTLINED_FUNCTION:
10254/// I2 --> BL OUTLINED_FUNCTION I1
10255/// I3 Restore LR I2
10256/// I3
10257/// RET
10258///
10259/// * Call construction overhead: 3 (save + BL + restore)
10260/// * Frame construction overhead: 1 (ret)
10261/// * Requires stack fixups? Yes
10262///
10263/// \p MachineOutlinerTailCall implies that the function is being created from
10264/// a sequence of instructions ending in a return.
10265///
10266/// That is,
10267///
10268/// I1 OUTLINED_FUNCTION:
10269/// I2 --> B OUTLINED_FUNCTION I1
10270/// RET I2
10271/// RET
10272///
10273/// * Call construction overhead: 1 (B)
10274/// * Frame construction overhead: 0 (Return included in sequence)
10275/// * Requires stack fixups? No
10276///
10277/// \p MachineOutlinerNoLRSave implies that the function should be called using
10278/// a BL instruction, but doesn't require LR to be saved and restored. This
10279/// happens when LR is known to be dead.
10280///
10281/// That is,
10282///
10283/// I1 OUTLINED_FUNCTION:
10284/// I2 --> BL OUTLINED_FUNCTION I1
10285/// I3 I2
10286/// I3
10287/// RET
10288///
10289/// * Call construction overhead: 1 (BL)
10290/// * Frame construction overhead: 1 (RET)
10291/// * Requires stack fixups? No
10292///
10293/// \p MachineOutlinerThunk implies that the function is being created from
10294/// a sequence of instructions ending in a call. The outlined function is
10295/// called with a BL instruction, and the outlined function tail-calls the
10296/// original call destination.
10297///
10298/// That is,
10299///
10300/// I1 OUTLINED_FUNCTION:
10301/// I2 --> BL OUTLINED_FUNCTION I1
10302/// BL f I2
10303/// B f
10304/// * Call construction overhead: 1 (BL)
10305/// * Frame construction overhead: 0
10306/// * Requires stack fixups? No
10307///
10308/// \p MachineOutlinerRegSave implies that the function should be called with a
10309/// save and restore of LR to an available register. This allows us to avoid
10310/// stack fixups. Note that this outlining variant is compatible with the
10311/// NoLRSave case.
10312///
10313/// That is,
10314///
10315/// I1 Save LR OUTLINED_FUNCTION:
10316/// I2 --> BL OUTLINED_FUNCTION I1
10317/// I3 Restore LR I2
10318/// I3
10319/// RET
10320///
10321/// * Call construction overhead: 3 (save + BL + restore)
10322/// * Frame construction overhead: 1 (ret)
10323/// * Requires stack fixups? No
10324enum MachineOutlinerClass {
10325 MachineOutlinerDefault, /// Emit a save, restore, call, and return.
10326 MachineOutlinerTailCall, /// Only emit a branch.
10327 MachineOutlinerNoLRSave, /// Emit a call and return.
10328 MachineOutlinerThunk, /// Emit a call and tail-call.
10329 MachineOutlinerRegSave /// Same as default, but save to a register.
10330};
10331
10332enum MachineOutlinerMBBFlags {
10333 LRUnavailableSomewhere = 0x2,
10334 HasCalls = 0x4,
10335 UnsafeRegsDead = 0x8
10336};
10337
10338/// Return true if the frame-record form of the outlined prologue is enabled for
10339/// the target of \p MF.
10340///
10341/// A non-leaf outlined function must save LR. On MachO, saving LR alone
10342/// (str x30) has no compact unwind encoding, so we get a large DWARF FDE
10343/// instead. Saving FP and LR as a frame record (stp x29, x30 ; mov x29, sp)
10344/// gets the small FRAME encoding, and costs one extra instruction.
10345static bool isCompactUnwindFrameRecordEnabled(const MachineFunction &MF) {
10346 return UseCompactUnwindFrameRecordForOutlinedFunctions &&
10347 MF.getTarget().getTargetTriple().isOSBinFormatMachO();
10348}
10349
10350/// Return true if the outlined function in \p MBB should save FP and LR as a
10351/// frame record instead of saving LR alone.
10352static bool shouldUseCompactUnwindFrameRecordForOutlinedFunction(
10353 const MachineBasicBlock &MBB) {
10354 const MachineFunction &MF = *MBB.getParent();
10355
10356 // Only worth it if the function has unwind info to shrink.
10357 if (!isCompactUnwindFrameRecordEnabled(MF) ||
10358 !MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF))
10359 return false;
10360
10361 // Only safe if the outlined code never touches FP, since we overwrite it.
10362 LiveRegUnits LRU(*MF.getSubtarget().getRegisterInfo());
10363 for (const MachineInstr &MI : MBB.instrs())
10364 LRU.accumulate(MI);
10365 return LRU.available(Reg: AArch64::FP);
10366}
10367
10368/// Predict what the above will answer, for use while costing candidates. The
10369/// outlined function does not exist yet, so answer from \p RepeatedSequenceLocs
10370/// instead. This is only an estimate; buildOutlinedFrame() makes the call.
10371static bool predictCompactUnwindFrameRecordForOutlinedFunction(
10372 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10373 const TargetRegisterInfo &TRI) {
10374 if (!isCompactUnwindFrameRecordEnabled(MF: *RepeatedSequenceLocs.front().getMF()))
10375 return false;
10376
10377 // The outlined function is nounwind only if every candidate is, so it has
10378 // unwind info if any candidate does.
10379 if (llvm::none_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10380 const MachineFunction &MF = *C.getMF();
10381 return MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF);
10382 }))
10383 return false;
10384
10385 // FP is free in the outlined function only if it is free in every candidate.
10386 return llvm::all_of(Range&: RepeatedSequenceLocs, P: [&TRI](outliner::Candidate &C) {
10387 return C.isAvailableInsideSeq(Reg: AArch64::FP, TRI);
10388 });
10389}
10390
10391Register
10392AArch64InstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
10393 MachineFunction *MF = C.getMF();
10394 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
10395 const AArch64RegisterInfo *ARI =
10396 static_cast<const AArch64RegisterInfo *>(&TRI);
10397 // Check if there is an available register across the sequence that we can
10398 // use.
10399 for (unsigned Reg : AArch64::GPR64RegClass) {
10400 if (!ARI->isReservedReg(MF: *MF, Reg) &&
10401 Reg != AArch64::LR && // LR is not reserved, but don't use it.
10402 Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
10403 Reg != AArch64::X17 && // Ditto for X17.
10404 C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
10405 C.isAvailableInsideSeq(Reg, TRI))
10406 return Reg;
10407 }
10408 return Register();
10409}
10410
10411static bool
10412outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
10413 const outliner::Candidate &b) {
10414 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10415 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10416
10417 return MFIa->getSignReturnAddressCondition() ==
10418 MFIb->getSignReturnAddressCondition();
10419}
10420
10421static bool
10422outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
10423 const outliner::Candidate &b) {
10424 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10425 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10426
10427 return MFIa->shouldSignWithBKey() == MFIb->shouldSignWithBKey();
10428}
10429
10430static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
10431 const outliner::Candidate &b) {
10432 const AArch64Subtarget &SubtargetA =
10433 a.getMF()->getSubtarget<AArch64Subtarget>();
10434 const AArch64Subtarget &SubtargetB =
10435 b.getMF()->getSubtarget<AArch64Subtarget>();
10436 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
10437}
10438
10439std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10440AArch64InstrInfo::getOutliningCandidateInfo(
10441 const MachineModuleInfo &MMI,
10442 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10443 unsigned MinRepeats) const {
10444 unsigned SequenceSize = 0;
10445 for (auto &MI : RepeatedSequenceLocs[0])
10446 SequenceSize += getInstSizeInBytes(MI);
10447
10448 unsigned NumBytesToCreateFrame = 0;
10449
10450 // Avoid splitting ADRP ADD/LDR pair into outlined functions.
10451 // These instructions are fused together by the scheduler.
10452 // Any candidate where ADRP is the last instruction should be rejected
10453 // as that will lead to splitting ADRP pair.
10454 MachineInstr &LastMI = RepeatedSequenceLocs[0].back();
10455 MachineInstr &FirstMI = RepeatedSequenceLocs[0].front();
10456 if (LastMI.getOpcode() == AArch64::ADRP &&
10457 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_PAGE) != 0 &&
10458 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10459 return std::nullopt;
10460 }
10461
10462 // Similarly any candidate where the first instruction is ADD/LDR with a
10463 // page offset should be rejected to avoid ADRP splitting.
10464 if ((FirstMI.getOpcode() == AArch64::ADDXri ||
10465 FirstMI.getOpcode() == AArch64::LDRXui) &&
10466 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_PAGEOFF) != 0 &&
10467 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10468 return std::nullopt;
10469 }
10470
10471 // We only allow outlining for functions having exactly matching return
10472 // address signing attributes, i.e., all share the same value for the
10473 // attribute "sign-return-address" and all share the same type of key they
10474 // are signed with.
10475 // Additionally we require all functions to simultaneously either support
10476 // v8.3a features or not. Otherwise an outlined function could get signed
10477 // using dedicated v8.3 instructions and a call from a function that doesn't
10478 // support v8.3 instructions would therefore be invalid.
10479 if (std::adjacent_find(
10480 first: RepeatedSequenceLocs.begin(), last: RepeatedSequenceLocs.end(),
10481 binary_pred: [](const outliner::Candidate &a, const outliner::Candidate &b) {
10482 // Return true if a and b are non-equal w.r.t. return address
10483 // signing or support of v8.3a features
10484 if (outliningCandidatesSigningScopeConsensus(a, b) &&
10485 outliningCandidatesSigningKeyConsensus(a, b) &&
10486 outliningCandidatesV8_3OpsConsensus(a, b)) {
10487 return false;
10488 }
10489 return true;
10490 }) != RepeatedSequenceLocs.end()) {
10491 return std::nullopt;
10492 }
10493
10494 // Since at this point all candidates agree on their return address signing
10495 // picking just one is fine. If the candidate functions potentially sign their
10496 // return addresses, the outlined function should do the same. Note that in
10497 // the case of "sign-return-address"="non-leaf" this is an assumption: It is
10498 // not certainly true that the outlined function will have to sign its return
10499 // address but this decision is made later, when the decision to outline
10500 // has already been made.
10501 // The same holds for the number of additional instructions we need: On
10502 // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
10503 // necessary. However, at this point we don't know if the outlined function
10504 // will have a RET instruction so we assume the worst.
10505 const TargetRegisterInfo &TRI = getRegisterInfo();
10506 // Performing a tail call may require extra checks when PAuth is enabled.
10507 // If PAuth is disabled, set it to zero for uniformity.
10508 unsigned NumBytesToCheckLRInTCEpilogue = 0;
10509 const auto RASignCondition = RepeatedSequenceLocs[0]
10510 .getMF()
10511 ->getInfo<AArch64FunctionInfo>()
10512 ->getSignReturnAddressCondition();
10513 if (RASignCondition != SignReturnAddress::None) {
10514 // One PAC and one AUT instructions
10515 NumBytesToCreateFrame += 8;
10516
10517 // PAuth is enabled - set extra tail call cost, if any.
10518 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
10519 MF: *RepeatedSequenceLocs[0].getMF());
10520 NumBytesToCheckLRInTCEpilogue =
10521 AArch64PAuth::getCheckerSizeInBytes(Method: LRCheckMethod);
10522 // Checking the authenticated LR value may significantly impact
10523 // SequenceSize, so account for it for more precise results.
10524 if (isTailCallReturnInst(MI: RepeatedSequenceLocs[0].back()))
10525 SequenceSize += NumBytesToCheckLRInTCEpilogue;
10526
10527 // We have to check if sp modifying instructions would get outlined.
10528 // If so we only allow outlining if sp is unchanged overall, so matching
10529 // sub and add instructions are okay to outline, all other sp modifications
10530 // are not
10531 auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
10532 int SPValue = 0;
10533 for (auto &MI : C) {
10534 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI)) {
10535 switch (MI.getOpcode()) {
10536 case AArch64::ADDXri:
10537 case AArch64::ADDWri:
10538 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10539 assert(MI.getOperand(2).isImm() &&
10540 "Expected operand to be immediate");
10541 assert(MI.getOperand(1).isReg() &&
10542 "Expected operand to be a register");
10543 // Check if the add just increments sp. If so, we search for
10544 // matching sub instructions that decrement sp. If not, the
10545 // modification is illegal
10546 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10547 SPValue += MI.getOperand(i: 2).getImm();
10548 else
10549 return true;
10550 break;
10551 case AArch64::SUBXri:
10552 case AArch64::SUBWri:
10553 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10554 assert(MI.getOperand(2).isImm() &&
10555 "Expected operand to be immediate");
10556 assert(MI.getOperand(1).isReg() &&
10557 "Expected operand to be a register");
10558 // Check if the sub just decrements sp. If so, we search for
10559 // matching add instructions that increment sp. If not, the
10560 // modification is illegal
10561 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10562 SPValue -= MI.getOperand(i: 2).getImm();
10563 else
10564 return true;
10565 break;
10566 default:
10567 return true;
10568 }
10569 }
10570 }
10571 if (SPValue)
10572 return true;
10573 return false;
10574 };
10575 // Remove candidates with illegal stack modifying instructions
10576 llvm::erase_if(C&: RepeatedSequenceLocs, P: hasIllegalSPModification);
10577
10578 // If the sequence doesn't have enough candidates left, then we're done.
10579 if (RepeatedSequenceLocs.size() < MinRepeats)
10580 return std::nullopt;
10581 }
10582
10583 // Properties about candidate MBBs that hold for all of them.
10584 unsigned FlagsSetInAll = 0xF;
10585
10586 // Compute liveness information for each candidate, and set FlagsSetInAll.
10587 for (outliner::Candidate &C : RepeatedSequenceLocs)
10588 FlagsSetInAll &= C.Flags;
10589
10590 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
10591
10592 // Helper lambda which sets call information for every candidate.
10593 auto SetCandidateCallInfo =
10594 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
10595 for (outliner::Candidate &C : RepeatedSequenceLocs)
10596 C.setCallInfo(CID: CallID, CO: NumBytesForCall);
10597 };
10598
10599 unsigned FrameID = MachineOutlinerDefault;
10600 NumBytesToCreateFrame += 4;
10601
10602 bool HasBTI = any_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10603 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
10604 });
10605
10606 // We check to see if CFI Instructions are present, and if they are
10607 // we find the number of CFI Instructions in the candidates.
10608 unsigned CFICount = 0;
10609 for (auto &I : RepeatedSequenceLocs[0]) {
10610 if (I.isCFIInstruction())
10611 CFICount++;
10612 }
10613
10614 // We compare the number of found CFI Instructions to the number of CFI
10615 // instructions in the parent function for each candidate. We must check this
10616 // since if we outline one of the CFI instructions in a function, we have to
10617 // outline them all for correctness. If we do not, the address offsets will be
10618 // incorrect between the two sections of the program.
10619 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10620 std::vector<MCCFIInstruction> CFIInstructions =
10621 C.getMF()->getFrameInstructions();
10622
10623 if (CFICount > 0 && CFICount != CFIInstructions.size())
10624 return std::nullopt;
10625 }
10626
10627 // Returns true if an instructions is safe to fix up, false otherwise.
10628 auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
10629 if (MI.isCall())
10630 return true;
10631
10632 if (!MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI) &&
10633 !MI.readsRegister(Reg: AArch64::SP, TRI: &TRI))
10634 return true;
10635
10636 // Any modification of SP will break our code to save/restore LR.
10637 // FIXME: We could handle some instructions which add a constant
10638 // offset to SP, with a bit more work.
10639 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI))
10640 return false;
10641
10642 // At this point, we have a stack instruction that we might need to
10643 // fix up. We'll handle it if it's a load or store.
10644 if (MI.mayLoadOrStore()) {
10645 const MachineOperand *Base; // Filled with the base operand of MI.
10646 int64_t Offset; // Filled with the offset of MI.
10647 bool OffsetIsScalable;
10648
10649 // Does it allow us to offset the base operand and is the base the
10650 // register SP?
10651 if (!getMemOperandWithOffset(MI, BaseOp&: Base, Offset, OffsetIsScalable, TRI: &TRI) ||
10652 !Base->isReg() || Base->getReg() != AArch64::SP)
10653 return false;
10654
10655 // Fixe-up code below assumes bytes.
10656 if (OffsetIsScalable)
10657 return false;
10658
10659 // Find the minimum/maximum offset for this instruction and check
10660 // if fixing it up would be in range.
10661 int64_t MinOffset,
10662 MaxOffset; // Unscaled offsets for the instruction.
10663 // The scale to multiply the offsets by.
10664 TypeSize Scale(0U, false), DummyWidth(0U, false);
10665 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width&: DummyWidth, MinOffset, MaxOffset);
10666
10667 Offset += 16; // Update the offset to what it would be if we outlined.
10668 if (Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
10669 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
10670 return false;
10671
10672 // It's in range, so we can outline it.
10673 return true;
10674 }
10675
10676 // FIXME: Add handling for instructions like "add x0, sp, #8".
10677
10678 // We can't fix it up, so don't outline it.
10679 return false;
10680 };
10681
10682 // True if it's possible to fix up each stack instruction in this sequence.
10683 // Important for frames/call variants that modify the stack.
10684 bool AllStackInstrsSafe =
10685 llvm::all_of(Range&: RepeatedSequenceLocs[0], P: IsSafeToFixup);
10686
10687 // If the last instruction in any candidate is a terminator, then we should
10688 // tail call all of the candidates.
10689 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10690 FrameID = MachineOutlinerTailCall;
10691 NumBytesToCreateFrame = 0;
10692 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
10693 SetCandidateCallInfo(MachineOutlinerTailCall, NumBytesForCall);
10694 }
10695
10696 else if (LastInstrOpcode == AArch64::BL ||
10697 ((LastInstrOpcode == AArch64::BLR ||
10698 LastInstrOpcode == AArch64::BLRNoIP) &&
10699 !HasBTI)) {
10700 // FIXME: Do we need to check if the code after this uses the value of LR?
10701 FrameID = MachineOutlinerThunk;
10702 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
10703 SetCandidateCallInfo(MachineOutlinerThunk, 4);
10704 }
10705
10706 else {
10707 // We need to decide how to emit calls + frames. We can always emit the same
10708 // frame if we don't need to save to the stack. If we have to save to the
10709 // stack, then we need a different frame.
10710 unsigned NumBytesNoStackCalls = 0;
10711 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
10712
10713 // Check if we have to save LR.
10714 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10715 bool LRAvailable =
10716 (C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
10717 ? C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI)
10718 : true;
10719 // If we have a noreturn caller, then we're going to be conservative and
10720 // say that we have to save LR. If we don't have a ret at the end of the
10721 // block, then we can't reason about liveness accurately.
10722 //
10723 // FIXME: We can probably do better than always disabling this in
10724 // noreturn functions by fixing up the liveness info.
10725 bool IsNoReturn =
10726 C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoReturn);
10727
10728 // Is LR available? If so, we don't need a save.
10729 if (LRAvailable && !IsNoReturn) {
10730 NumBytesNoStackCalls += 4;
10731 C.setCallInfo(CID: MachineOutlinerNoLRSave, CO: 4);
10732 CandidatesWithoutStackFixups.push_back(x: C);
10733 }
10734
10735 // Is an unused register available? If so, we won't modify the stack, so
10736 // we can outline with the same frame type as those that don't save LR.
10737 else if (findRegisterToSaveLRTo(C)) {
10738 NumBytesNoStackCalls += 12;
10739 C.setCallInfo(CID: MachineOutlinerRegSave, CO: 12);
10740 CandidatesWithoutStackFixups.push_back(x: C);
10741 }
10742
10743 // Is SP used in the sequence at all? If not, we don't have to modify
10744 // the stack, so we are guaranteed to get the same frame.
10745 else if (C.isAvailableInsideSeq(Reg: AArch64::SP, TRI)) {
10746 NumBytesNoStackCalls += 12;
10747 C.setCallInfo(CID: MachineOutlinerDefault, CO: 12);
10748 CandidatesWithoutStackFixups.push_back(x: C);
10749 }
10750
10751 // If we outline this, we need to modify the stack. Pretend we don't
10752 // outline this by saving all of its bytes.
10753 else {
10754 NumBytesNoStackCalls += SequenceSize;
10755 }
10756 }
10757
10758 // If there are no places where we have to save LR, then note that we
10759 // don't have to update the stack. Otherwise, give every candidate the
10760 // default call type, as long as it's safe to do so.
10761 if (!AllStackInstrsSafe ||
10762 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
10763 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
10764 FrameID = MachineOutlinerNoLRSave;
10765 if (RepeatedSequenceLocs.size() < MinRepeats)
10766 return std::nullopt;
10767 } else {
10768 SetCandidateCallInfo(MachineOutlinerDefault, 12);
10769
10770 // Bugzilla ID: 46767
10771 // TODO: Check if fixing up the stack more than once is safe so we can
10772 // outline these.
10773 //
10774 // An outline resulting in a caller that requires stack fixups at the
10775 // callsite to a callee that also requires stack fixups can happen when
10776 // there are no available registers at the candidate callsite for a
10777 // candidate that itself also has calls.
10778 //
10779 // In other words if function_containing_sequence in the following pseudo
10780 // assembly requires that we save LR at the point of the call, but there
10781 // are no available registers: in this case we save using SP and as a
10782 // result the SP offsets requires stack fixups by multiples of 16.
10783 //
10784 // function_containing_sequence:
10785 // ...
10786 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10787 // call OUTLINED_FUNCTION_N
10788 // restore LR from SP
10789 // ...
10790 //
10791 // OUTLINED_FUNCTION_N:
10792 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10793 // ...
10794 // bl foo
10795 // restore LR from SP
10796 // ret
10797 //
10798 // Because the code to handle more than one stack fixup does not
10799 // currently have the proper checks for legality, these cases will assert
10800 // in the AArch64 MachineOutliner. This is because the code to do this
10801 // needs more hardening, testing, better checks that generated code is
10802 // legal, etc and because it is only verified to handle a single pass of
10803 // stack fixup.
10804 //
10805 // The assert happens in AArch64InstrInfo::buildOutlinedFrame to catch
10806 // these cases until they are known to be handled. Bugzilla 46767 is
10807 // referenced in comments at the assert site.
10808 //
10809 // To avoid asserting (or generating non-legal code on noassert builds)
10810 // we remove all candidates which would need more than one stack fixup by
10811 // pruning the cases where the candidate has calls while also having no
10812 // available LR and having no available general purpose registers to copy
10813 // LR to (ie one extra stack save/restore).
10814 //
10815 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10816 erase_if(C&: RepeatedSequenceLocs, P: [this, &TRI](outliner::Candidate &C) {
10817 auto IsCall = [](const MachineInstr &MI) { return MI.isCall(); };
10818 return (llvm::any_of(Range&: C, P: IsCall)) &&
10819 (!C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI) ||
10820 !findRegisterToSaveLRTo(C));
10821 });
10822 }
10823 }
10824
10825 // If we dropped all of the candidates, bail out here.
10826 if (RepeatedSequenceLocs.size() < MinRepeats)
10827 return std::nullopt;
10828 }
10829
10830 // Does every candidate's MBB contain a call? If so, then we might have a call
10831 // in the range.
10832 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10833 // Check if the range contains a call. These require a save + restore of the
10834 // link register.
10835 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
10836 bool ModStackToSaveLR = false;
10837 if (any_of(Range: drop_end(RangeOrContainer&: FirstCand),
10838 P: [](const MachineInstr &MI) { return MI.isCall(); }))
10839 ModStackToSaveLR = true;
10840
10841 // Handle the last instruction separately. If this is a tail call, then the
10842 // last instruction is a call. We don't want to save + restore in this case.
10843 // However, it could be possible that the last instruction is a call without
10844 // it being valid to tail call this sequence. We should consider this as
10845 // well.
10846 else if (FrameID != MachineOutlinerThunk &&
10847 FrameID != MachineOutlinerTailCall && FirstCand.back().isCall())
10848 ModStackToSaveLR = true;
10849
10850 if (ModStackToSaveLR) {
10851 // We can't fix up the stack. Bail out.
10852 if (!AllStackInstrsSafe)
10853 return std::nullopt;
10854
10855 // Save + restore LR.
10856 NumBytesToCreateFrame += 8;
10857
10858 // Add the extra mov if we will save a frame record instead of just LR.
10859 if (predictCompactUnwindFrameRecordForOutlinedFunction(
10860 RepeatedSequenceLocs, TRI))
10861 NumBytesToCreateFrame += 4;
10862 }
10863 }
10864
10865 // If we have CFI instructions, we can only outline if the outlined section
10866 // can be a tail call
10867 if (FrameID != MachineOutlinerTailCall && CFICount > 0)
10868 return std::nullopt;
10869
10870 return std::make_unique<outliner::OutlinedFunction>(
10871 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: NumBytesToCreateFrame, args&: FrameID);
10872}
10873
10874void AArch64InstrInfo::mergeOutliningCandidateAttributes(
10875 Function &F, std::vector<outliner::Candidate> &Candidates) const {
10876 // If a bunch of candidates reach this point they must agree on their return
10877 // address signing. It is therefore enough to just consider the signing
10878 // behaviour of one of them
10879 const auto &CFn = Candidates.front().getMF()->getFunction();
10880
10881 if (CFn.hasFnAttribute(Kind: "ptrauth-returns"))
10882 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-returns"));
10883 if (CFn.hasFnAttribute(Kind: "ptrauth-auth-traps"))
10884 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-auth-traps"));
10885 // Since all candidates belong to the same module, just copy the
10886 // function-level attributes of an arbitrary function.
10887 if (CFn.hasFnAttribute(Kind: "sign-return-address"))
10888 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address"));
10889 if (CFn.hasFnAttribute(Kind: "sign-return-address-key"))
10890 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address-key"));
10891
10892 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
10893}
10894
10895bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
10896 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
10897 const Function &F = MF.getFunction();
10898
10899 // Can F be deduplicated by the linker? If it can, don't outline from it.
10900 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
10901 return false;
10902
10903 // Don't outline from functions with section markings; the program could
10904 // expect that all the code is in the named section.
10905 // FIXME: Allow outlining from multiple functions with the same section
10906 // marking.
10907 if (F.hasSection())
10908 return false;
10909
10910 // Outlining from functions with redzones is unsafe since the outliner may
10911 // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
10912 // outline from it.
10913 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
10914 if (!AFI || AFI->hasRedZone().value_or(u: true))
10915 return false;
10916
10917 // FIXME: Determine whether it is safe to outline from functions which contain
10918 // streaming-mode changes. We may need to ensure any smstart/smstop pairs are
10919 // outlined together and ensure it is safe to outline with async unwind info,
10920 // required for saving & restoring VG around calls.
10921 if (AFI->hasStreamingModeChanges())
10922 return false;
10923
10924 // FIXME: Teach the outliner to generate/handle Windows unwind info.
10925 if (MF.getTarget().getMCAsmInfo().usesWindowsCFI())
10926 return false;
10927
10928 // It's safe to outline from MF.
10929 return true;
10930}
10931
10932SmallVector<std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10933AArch64InstrInfo::getOutlinableRanges(MachineBasicBlock &MBB,
10934 unsigned &Flags) const {
10935 assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
10936 "Must track liveness!");
10937 SmallVector<
10938 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10939 Ranges;
10940 // According to the AArch64 Procedure Call Standard, the following are
10941 // undefined on entry/exit from a function call:
10942 //
10943 // * Registers x16, x17, (and thus w16, w17)
10944 // * Condition codes (and thus the NZCV register)
10945 //
10946 // If any of these registers are used inside or live across an outlined
10947 // function, then they may be modified later, either by the compiler or
10948 // some other tool (like the linker).
10949 //
10950 // To avoid outlining in these situations, partition each block into ranges
10951 // where these registers are dead. We will only outline from those ranges.
10952 LiveRegUnits LRU(getRegisterInfo());
10953 auto AreAllUnsafeRegsDead = [&LRU]() {
10954 return LRU.available(Reg: AArch64::W16) && LRU.available(Reg: AArch64::W17) &&
10955 LRU.available(Reg: AArch64::NZCV);
10956 };
10957
10958 // We need to know if LR is live across an outlining boundary later on in
10959 // order to decide how we'll create the outlined call, frame, etc.
10960 //
10961 // It's pretty expensive to check this for *every candidate* within a block.
10962 // That's some potentially n^2 behaviour, since in the worst case, we'd need
10963 // to compute liveness from the end of the block for O(n) candidates within
10964 // the block.
10965 //
10966 // So, to improve the average case, let's keep track of liveness from the end
10967 // of the block to the beginning of *every outlinable range*. If we know that
10968 // LR is available in every range we could outline from, then we know that
10969 // we don't need to check liveness for any candidate within that range.
10970 bool LRAvailableEverywhere = true;
10971 // Compute liveness bottom-up.
10972 LRU.addLiveOuts(MBB);
10973 // Update flags that require info about the entire MBB.
10974 auto UpdateWholeMBBFlags = [&Flags](const MachineInstr &MI) {
10975 if (MI.isCall() && !MI.isTerminator())
10976 Flags |= MachineOutlinerMBBFlags::HasCalls;
10977 };
10978 // Range: [RangeBegin, RangeEnd)
10979 MachineBasicBlock::instr_iterator RangeBegin, RangeEnd;
10980 unsigned RangeLen;
10981 auto CreateNewRangeStartingAt =
10982 [&RangeBegin, &RangeEnd,
10983 &RangeLen](MachineBasicBlock::instr_iterator NewBegin) {
10984 RangeBegin = NewBegin;
10985 RangeEnd = std::next(x: RangeBegin);
10986 RangeLen = 0;
10987 };
10988 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
10989 // At least one unsafe register is not dead. We do not want to outline at
10990 // this point. If it is long enough to outline from and does not cross a
10991 // bundle boundary, save the range [RangeBegin, RangeEnd).
10992 if (RangeLen <= 1)
10993 return;
10994 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
10995 return;
10996 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
10997 return;
10998 Ranges.emplace_back(Args&: RangeBegin, Args&: RangeEnd);
10999 };
11000 // Find the first point where all unsafe registers are dead.
11001 // FIND: <safe instr> <-- end of first potential range
11002 // SKIP: <unsafe def>
11003 // SKIP: ... everything between ...
11004 // SKIP: <unsafe use>
11005 auto FirstPossibleEndPt = MBB.instr_rbegin();
11006 for (; FirstPossibleEndPt != MBB.instr_rend(); ++FirstPossibleEndPt) {
11007 if (!FirstPossibleEndPt->isDebugInstr())
11008 LRU.stepBackward(MI: *FirstPossibleEndPt);
11009 // Update flags that impact how we outline across the entire block,
11010 // regardless of safety.
11011 UpdateWholeMBBFlags(*FirstPossibleEndPt);
11012 if (AreAllUnsafeRegsDead())
11013 break;
11014 }
11015 // If we exhausted the entire block, we have no safe ranges to outline.
11016 if (FirstPossibleEndPt == MBB.instr_rend())
11017 return Ranges;
11018 // Current range.
11019 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
11020 // StartPt points to the first place where all unsafe registers
11021 // are dead (if there is any such point). Begin partitioning the MBB into
11022 // ranges.
11023 for (auto &MI : make_range(x: FirstPossibleEndPt, y: MBB.instr_rend())) {
11024 if (!MI.isDebugInstr())
11025 LRU.stepBackward(MI);
11026 UpdateWholeMBBFlags(MI);
11027 if (!AreAllUnsafeRegsDead()) {
11028 SaveRangeIfNonEmpty();
11029 CreateNewRangeStartingAt(MI.getIterator());
11030 continue;
11031 }
11032 LRAvailableEverywhere &= LRU.available(Reg: AArch64::LR);
11033 RangeBegin = MI.getIterator();
11034 ++RangeLen;
11035 }
11036 // Above loop misses the last (or only) range. If we are still safe, then
11037 // let's save the range.
11038 if (AreAllUnsafeRegsDead())
11039 SaveRangeIfNonEmpty();
11040 if (Ranges.empty())
11041 return Ranges;
11042 // We found the ranges bottom-up. Mapping expects the top-down. Reverse
11043 // the order.
11044 std::reverse(first: Ranges.begin(), last: Ranges.end());
11045 // If there is at least one outlinable range where LR is unavailable
11046 // somewhere, remember that.
11047 if (!LRAvailableEverywhere)
11048 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
11049 return Ranges;
11050}
11051
11052outliner::InstrType
11053AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
11054 MachineBasicBlock::iterator &MIT,
11055 unsigned Flags) const {
11056 MachineInstr &MI = *MIT;
11057
11058 // Don't outline anything used for return address signing. The outlined
11059 // function will get signed later if needed
11060 switch (MI.getOpcode()) {
11061 case AArch64::PACM:
11062 case AArch64::PACIASP:
11063 case AArch64::PACIBSP:
11064 case AArch64::PACIASPPC:
11065 case AArch64::PACIBSPPC:
11066 case AArch64::AUTIASP:
11067 case AArch64::AUTIBSP:
11068 case AArch64::AUTIASPPCi:
11069 case AArch64::AUTIASPPCr:
11070 case AArch64::AUTIBSPPCi:
11071 case AArch64::AUTIBSPPCr:
11072 case AArch64::RETAA:
11073 case AArch64::RETAB:
11074 case AArch64::RETAASPPCi:
11075 case AArch64::RETAASPPCr:
11076 case AArch64::RETABSPPCi:
11077 case AArch64::RETABSPPCr:
11078 case AArch64::EMITBKEY:
11079 case AArch64::PAUTH_PROLOGUE:
11080 case AArch64::PAUTH_EPILOGUE:
11081 return outliner::InstrType::Illegal;
11082 }
11083
11084 // We can only outline these if we will tail call the outlined function, or
11085 // fix up the CFI offsets. Currently, CFI instructions are outlined only if
11086 // in a tail call.
11087 //
11088 // FIXME: If the proper fixups for the offset are implemented, this should be
11089 // possible.
11090 if (MI.isCFIInstruction())
11091 return outliner::InstrType::Legal;
11092
11093 // Is this a terminator for a basic block?
11094 if (MI.isTerminator())
11095 // TargetInstrInfo::getOutliningType has already filtered out anything
11096 // that would break this, so we can allow it here.
11097 return outliner::InstrType::Legal;
11098
11099 // Make sure none of the operands are un-outlinable.
11100 for (const MachineOperand &MOP : MI.operands()) {
11101 // A check preventing CFI indices was here before, but only CFI
11102 // instructions should have those.
11103 assert(!MOP.isCFIIndex());
11104
11105 // If it uses LR or W30 explicitly, then don't touch it.
11106 if (MOP.isReg() && !MOP.isImplicit() &&
11107 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
11108 return outliner::InstrType::Illegal;
11109 }
11110
11111 // Special cases for instructions that can always be outlined, but will fail
11112 // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
11113 // be outlined because they don't require a *specific* value to be in LR.
11114 if (MI.getOpcode() == AArch64::ADRP)
11115 return outliner::InstrType::Legal;
11116
11117 // If MI is a call we might be able to outline it. We don't want to outline
11118 // any calls that rely on the position of items on the stack. When we outline
11119 // something containing a call, we have to emit a save and restore of LR in
11120 // the outlined function. Currently, this always happens by saving LR to the
11121 // stack. Thus, if we outline, say, half the parameters for a function call
11122 // plus the call, then we'll break the callee's expectations for the layout
11123 // of the stack.
11124 //
11125 // FIXME: Allow calls to functions which construct a stack frame, as long
11126 // as they don't access arguments on the stack.
11127 // FIXME: Figure out some way to analyze functions defined in other modules.
11128 // We should be able to compute the memory usage based on the IR calling
11129 // convention, even if we can't see the definition.
11130 if (MI.isCall()) {
11131 // Get the function associated with the call. Look at each operand and find
11132 // the one that represents the callee and get its name.
11133 const Function *Callee = nullptr;
11134 for (const MachineOperand &MOP : MI.operands()) {
11135 if (MOP.isGlobal()) {
11136 Callee = dyn_cast<Function>(Val: MOP.getGlobal());
11137 break;
11138 }
11139 }
11140
11141 // Never outline calls to mcount. There isn't any rule that would require
11142 // this, but the Linux kernel's "ftrace" feature depends on it.
11143 if (Callee && Callee->getName() == "\01_mcount")
11144 return outliner::InstrType::Illegal;
11145
11146 // If we don't know anything about the callee, assume it depends on the
11147 // stack layout of the caller. In that case, it's only legal to outline
11148 // as a tail-call. Explicitly list the call instructions we know about so we
11149 // don't get unexpected results with call pseudo-instructions.
11150 auto UnknownCallOutlineType = outliner::InstrType::Illegal;
11151 if (MI.getOpcode() == AArch64::BLR ||
11152 MI.getOpcode() == AArch64::BLRNoIP || MI.getOpcode() == AArch64::BL)
11153 UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
11154
11155 if (!Callee)
11156 return UnknownCallOutlineType;
11157
11158 // We have a function we have information about. Check it if it's something
11159 // can safely outline.
11160 MachineFunction *CalleeMF = MMI.getMachineFunction(F: *Callee);
11161
11162 // We don't know what's going on with the callee at all. Don't touch it.
11163 if (!CalleeMF)
11164 return UnknownCallOutlineType;
11165
11166 // Check if we know anything about the callee saves on the function. If we
11167 // don't, then don't touch it, since that implies that we haven't
11168 // computed anything about its stack frame yet.
11169 MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
11170 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
11171 MFI.getNumObjects() > 0)
11172 return UnknownCallOutlineType;
11173
11174 // At this point, we can say that CalleeMF ought to not pass anything on the
11175 // stack. Therefore, we can outline it.
11176 return outliner::InstrType::Legal;
11177 }
11178
11179 // Don't touch the link register or W30.
11180 if (MI.readsRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()) ||
11181 MI.modifiesRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()))
11182 return outliner::InstrType::Illegal;
11183
11184 // Don't outline BTI instructions, because that will prevent the outlining
11185 // site from being indirectly callable.
11186 if (hasBTISemantics(MI))
11187 return outliner::InstrType::Illegal;
11188
11189 return outliner::InstrType::Legal;
11190}
11191
11192void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
11193 for (MachineInstr &MI : MBB) {
11194 const MachineOperand *Base;
11195 TypeSize Width(0, false);
11196 int64_t Offset;
11197 bool OffsetIsScalable;
11198
11199 // Is this a load or store with an immediate offset with SP as the base?
11200 if (!MI.mayLoadOrStore() ||
11201 !getMemOperandWithOffsetWidth(LdSt: MI, BaseOp&: Base, Offset, OffsetIsScalable, Width,
11202 TRI: &RI) ||
11203 (Base->isReg() && Base->getReg() != AArch64::SP))
11204 continue;
11205
11206 // It is, so we have to fix it up.
11207 TypeSize Scale(0U, false);
11208 int64_t Dummy1, Dummy2;
11209
11210 MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(LdSt&: MI);
11211 assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
11212 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2);
11213 assert(Scale != 0 && "Unexpected opcode!");
11214 assert(!OffsetIsScalable && "Expected offset to be a byte offset");
11215
11216 // We've pushed the return address to the stack, so add 16 to the offset.
11217 // This is safe, since we already checked if it would overflow when we
11218 // checked if this instruction was legal to outline.
11219 int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedValue();
11220 StackOffsetOperand.setImm(NewImm);
11221 }
11222}
11223
11224static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
11225 const AArch64InstrInfo *TII,
11226 bool ShouldSignReturnAddr) {
11227 if (!ShouldSignReturnAddr)
11228 return;
11229
11230 BuildMI(BB&: MBB, I: MBB.begin(), MIMD: DebugLoc(), MCID: TII->get(Opcode: AArch64::PAUTH_PROLOGUE))
11231 .setMIFlag(MachineInstr::FrameSetup);
11232 TII->createPauthEpilogueInstr(MBB, DL: DebugLoc());
11233}
11234
11235void AArch64InstrInfo::buildOutlinedFrame(
11236 MachineBasicBlock &MBB, MachineFunction &MF,
11237 const outliner::OutlinedFunction &OF) const {
11238
11239 AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
11240
11241 if (OF.FrameConstructionID == MachineOutlinerTailCall)
11242 FI->setOutliningStyle("Tail Call");
11243 else if (OF.FrameConstructionID == MachineOutlinerThunk) {
11244 // For thunk outlining, rewrite the last instruction from a call to a
11245 // tail-call.
11246 MachineInstr *Call = &*--MBB.instr_end();
11247 unsigned TailOpcode;
11248 if (Call->getOpcode() == AArch64::BL) {
11249 TailOpcode = AArch64::TCRETURNdi;
11250 } else {
11251 assert(Call->getOpcode() == AArch64::BLR ||
11252 Call->getOpcode() == AArch64::BLRNoIP);
11253 TailOpcode = AArch64::TCRETURNriALL;
11254 }
11255 MachineInstr *TC = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpcode))
11256 .add(MO: Call->getOperand(i: 0))
11257 .addImm(Val: 0);
11258 MBB.insert(I: MBB.end(), MI: TC);
11259 Call->eraseFromParent();
11260
11261 FI->setOutliningStyle("Thunk");
11262 }
11263
11264 bool IsLeafFunction = true;
11265
11266 // Is there a call in the outlined range?
11267 auto IsNonTailCall = [](const MachineInstr &MI) {
11268 return MI.isCall() && !MI.isReturn();
11269 };
11270
11271 if (llvm::any_of(Range: MBB.instrs(), P: IsNonTailCall)) {
11272 // Fix up the instructions in the range, since we're going to modify the
11273 // stack.
11274
11275 // Bugzilla ID: 46767
11276 // TODO: Check if fixing up twice is safe so we can outline these.
11277 assert(OF.FrameConstructionID != MachineOutlinerDefault &&
11278 "Can only fix up stack references once");
11279 fixupPostOutline(MBB);
11280
11281 IsLeafFunction = false;
11282
11283 // LR has to be a live in so that we can save it.
11284 if (!MBB.isLiveIn(Reg: AArch64::LR))
11285 MBB.addLiveIn(PhysReg: AArch64::LR);
11286
11287 MachineBasicBlock::iterator It = MBB.begin();
11288 MachineBasicBlock::iterator Et = MBB.end();
11289
11290 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11291 OF.FrameConstructionID == MachineOutlinerThunk)
11292 Et = std::prev(x: MBB.end());
11293
11294 // There is a call in the range, so we must save LR. Save it as part of a
11295 // frame record when that gives us a smaller compact unwind encoding.
11296 if (shouldUseCompactUnwindFrameRecordForOutlinedFunction(MBB)) {
11297 // FP is saved here, so it must be live-in.
11298 if (!MBB.isLiveIn(Reg: AArch64::FP))
11299 MBB.addLiveIn(PhysReg: AArch64::FP);
11300
11301 // stp x29, x30, [sp, #-16]! (the pre-index imm is scaled by 8: -2 * 8)
11302 MachineInstr *STPXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STPXpre))
11303 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11304 .addReg(RegNo: AArch64::FP)
11305 .addReg(RegNo: AArch64::LR)
11306 .addReg(RegNo: AArch64::SP)
11307 .addImm(Val: -2);
11308 It = MBB.insert(I: It, MI: STPXpre);
11309
11310 // mov x29, sp (add x29, sp, #0), so x29 points at the frame record.
11311 MachineInstr *SetFP = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ADDXri))
11312 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11313 .addReg(RegNo: AArch64::SP)
11314 .addImm(Val: 0)
11315 .addImm(Val: 0);
11316 MBB.insertAfter(I: It, MI: SetFP);
11317
11318 // Describe the frame record with FP as the CFA. The encoder needs all
11319 // three to pick FRAME. No need to check for unwind info here: we only
11320 // get here if the function has it.
11321 CFIInstBuilder CFIBuilder(MBB, std::next(x: SetFP->getIterator()),
11322 MachineInstr::FrameSetup);
11323 CFIBuilder.buildDefCFA(Reg: AArch64::FP, Offset: 16);
11324 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -8);
11325 CFIBuilder.buildOffset(Reg: AArch64::FP, Offset: -16);
11326
11327 // ldp x29, x30, [sp], #16
11328 MachineInstr *LDPXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDPXpost))
11329 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11330 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11331 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11332 .addReg(RegNo: AArch64::SP)
11333 .addImm(Val: 2);
11334 Et = MBB.insert(I: Et, MI: LDPXpost);
11335 } else {
11336 // Insert a save before the outlined region
11337 MachineInstr *STRXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11338 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11339 .addReg(RegNo: AArch64::LR)
11340 .addReg(RegNo: AArch64::SP)
11341 .addImm(Val: -16);
11342 It = MBB.insert(I: It, MI: STRXpre);
11343
11344 if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
11345 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
11346
11347 // Add a CFI saying the stack was moved 16 B down.
11348 CFIBuilder.buildDefCFAOffset(Offset: 16);
11349
11350 // Add a CFI saying that the LR that we want to find is now 16 B higher
11351 // than before.
11352 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -16);
11353 }
11354
11355 // Insert a restore before the terminator for the function.
11356 MachineInstr *LDRXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11357 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11358 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11359 .addReg(RegNo: AArch64::SP)
11360 .addImm(Val: 16);
11361 Et = MBB.insert(I: Et, MI: LDRXpost);
11362 }
11363 }
11364
11365 auto RASignCondition = FI->getSignReturnAddressCondition();
11366 bool ShouldSignReturnAddr = AArch64FunctionInfo::shouldSignReturnAddress(
11367 Condition: RASignCondition, IsLRSpilled: !IsLeafFunction);
11368
11369 // If this is a tail call outlined function, then there's already a return.
11370 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11371 OF.FrameConstructionID == MachineOutlinerThunk) {
11372 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11373 return;
11374 }
11375
11376 // It's not a tail call, so we have to insert the return ourselves.
11377
11378 // LR has to be a live in so that we can return to it.
11379 if (!MBB.isLiveIn(Reg: AArch64::LR))
11380 MBB.addLiveIn(PhysReg: AArch64::LR);
11381
11382 MachineInstr *ret = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::RET))
11383 .addReg(RegNo: AArch64::LR);
11384 MBB.insert(I: MBB.end(), MI: ret);
11385
11386 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11387
11388 FI->setOutliningStyle("Function");
11389
11390 // Did we have to modify the stack by saving the link register?
11391 if (OF.FrameConstructionID != MachineOutlinerDefault)
11392 return;
11393
11394 // We modified the stack.
11395 // Walk over the basic block and fix up all the stack accesses.
11396 fixupPostOutline(MBB);
11397}
11398
11399MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
11400 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
11401 MachineFunction &MF, outliner::Candidate &C) const {
11402
11403 // Are we tail calling?
11404 if (C.CallConstructionID == MachineOutlinerTailCall) {
11405 // If yes, then we can just branch to the label.
11406 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::TCRETURNdi))
11407 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()))
11408 .addImm(Val: 0));
11409 return It;
11410 }
11411
11412 // Are we saving the link register?
11413 if (C.CallConstructionID == MachineOutlinerNoLRSave ||
11414 C.CallConstructionID == MachineOutlinerThunk) {
11415 // No, so just insert the call.
11416 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11417 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11418 return It;
11419 }
11420
11421 // We want to return the spot where we inserted the call.
11422 MachineBasicBlock::iterator CallPt;
11423
11424 // Instructions for saving and restoring LR around the call instruction we're
11425 // going to insert.
11426 MachineInstr *Save;
11427 MachineInstr *Restore;
11428 // Can we save to a register?
11429 if (C.CallConstructionID == MachineOutlinerRegSave) {
11430 // FIXME: This logic should be sunk into a target-specific interface so that
11431 // we don't have to recompute the register.
11432 Register Reg = findRegisterToSaveLRTo(C);
11433 assert(Reg && "No callee-saved register available?");
11434
11435 // LR has to be a live in so that we can save it.
11436 if (!MBB.isLiveIn(Reg: AArch64::LR))
11437 MBB.addLiveIn(PhysReg: AArch64::LR);
11438
11439 // Save and restore LR from Reg.
11440 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: Reg)
11441 .addReg(RegNo: AArch64::XZR)
11442 .addReg(RegNo: AArch64::LR)
11443 .addImm(Val: 0);
11444 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: AArch64::LR)
11445 .addReg(RegNo: AArch64::XZR)
11446 .addReg(RegNo: Reg)
11447 .addImm(Val: 0);
11448 } else {
11449 // We have the default case. Save and restore from SP.
11450 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11451 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11452 .addReg(RegNo: AArch64::LR)
11453 .addReg(RegNo: AArch64::SP)
11454 .addImm(Val: -16);
11455 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11456 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11457 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11458 .addReg(RegNo: AArch64::SP)
11459 .addImm(Val: 16);
11460 }
11461
11462 It = MBB.insert(I: It, MI: Save);
11463 It++;
11464
11465 // Insert the call.
11466 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11467 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11468 CallPt = It;
11469 It++;
11470
11471 It = MBB.insert(I: It, MI: Restore);
11472 return CallPt;
11473}
11474
11475bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
11476 MachineFunction &MF) const {
11477 return MF.getFunction().hasMinSize();
11478}
11479
11480void AArch64InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
11481 MachineBasicBlock::iterator Iter,
11482 DebugLoc &DL,
11483 bool AllowSideEffects) const {
11484 const MachineFunction &MF = *MBB.getParent();
11485 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
11486 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
11487
11488 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
11489 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg).addImm(Val: 0).addImm(Val: 0);
11490 } else if (STI.isSVEorStreamingSVEAvailable()) {
11491 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::DUP_ZI_D), DestReg: Reg)
11492 .addImm(Val: 0)
11493 .addImm(Val: 0);
11494 } else if (STI.isNeonAvailable()) {
11495 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVIv2d_ns), DestReg: Reg)
11496 .addImm(Val: 0);
11497 } else {
11498 // No Advanced SIMD (streaming-compatible without SVE, or +nosimd), so use
11499 // `fmov d...` instead of `movi v...`; writing `d` also clears the upper
11500 // 64 bits.
11501 assert(STI.hasFPARMv8() && "Expected FP to be available.");
11502 Register Reg64 = TRI.getSubReg(Reg, Idx: AArch64::dsub);
11503 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg: Reg64);
11504 }
11505}
11506
11507std::optional<DestSourcePair>
11508AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
11509
11510 // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
11511 // and zero immediate operands used as an alias for mov instruction.
11512 if ((MI.getOpcode() == AArch64::ORRWrs &&
11513 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11514 MI.getOperand(i: 3).getImm() == 0x0) ||
11515 (MI.getOpcode() == AArch64::ORRWrr &&
11516 MI.getOperand(i: 1).getReg() == AArch64::WZR)) {
11517 // Check that the w->w move is not a zero-extending w->x mov.
11518 if ((MI.getOperand(i: 0).getReg().isPhysical() &&
11519 MI.findRegisterDefOperandIdx(
11520 Reg: getXRegFromWReg(Reg: MI.getOperand(i: 0).getReg()),
11521 /*TRI=*/nullptr) == -1) ||
11522 (MI.getOperand(i: 0).getReg().isVirtual() &&
11523 !MI.getOperand(i: 0).getSubReg()))
11524 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11525 }
11526
11527 if (MI.getOpcode() == AArch64::ORRXrs &&
11528 MI.getOperand(i: 1).getReg() == AArch64::XZR &&
11529 MI.getOperand(i: 3).getImm() == 0x0)
11530 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11531
11532 return std::nullopt;
11533}
11534
11535std::optional<DestSourcePair>
11536AArch64InstrInfo::isCopyLikeInstrImpl(const MachineInstr &MI) const {
11537 if ((MI.getOpcode() == AArch64::ORRWrs &&
11538 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11539 MI.getOperand(i: 3).getImm() == 0x0) ||
11540 (MI.getOpcode() == AArch64::ORRWrr &&
11541 MI.getOperand(i: 1).getReg() == AArch64::WZR))
11542 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11543 return std::nullopt;
11544}
11545
11546std::optional<RegImmPair>
11547AArch64InstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
11548 int Sign = 1;
11549 int64_t Offset = 0;
11550
11551 // TODO: Handle cases where Reg is a super- or sub-register of the
11552 // destination register.
11553 const MachineOperand &Op0 = MI.getOperand(i: 0);
11554 if (!Op0.isReg() || Reg != Op0.getReg())
11555 return std::nullopt;
11556
11557 switch (MI.getOpcode()) {
11558 default:
11559 return std::nullopt;
11560 case AArch64::SUBWri:
11561 case AArch64::SUBXri:
11562 case AArch64::SUBSWri:
11563 case AArch64::SUBSXri:
11564 Sign *= -1;
11565 [[fallthrough]];
11566 case AArch64::ADDSWri:
11567 case AArch64::ADDSXri:
11568 case AArch64::ADDWri:
11569 case AArch64::ADDXri: {
11570 // TODO: Third operand can be global address (usually some string).
11571 if (!MI.getOperand(i: 0).isReg() || !MI.getOperand(i: 1).isReg() ||
11572 !MI.getOperand(i: 2).isImm())
11573 return std::nullopt;
11574 int Shift = MI.getOperand(i: 3).getImm();
11575 assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
11576 Offset = Sign * (MI.getOperand(i: 2).getImm() << Shift);
11577 }
11578 }
11579 return RegImmPair{MI.getOperand(i: 1).getReg(), Offset};
11580}
11581
11582/// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
11583/// the destination register then, if possible, describe the value in terms of
11584/// the source register.
11585static std::optional<ParamLoadedValue>
11586describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
11587 const TargetInstrInfo *TII,
11588 const TargetRegisterInfo *TRI) {
11589 auto DestSrc = TII->isCopyLikeInstr(MI);
11590 if (!DestSrc)
11591 return std::nullopt;
11592
11593 Register DestReg = DestSrc->Destination->getReg();
11594 Register SrcReg = DestSrc->Source->getReg();
11595
11596 if (!DestReg.isValid() || !SrcReg.isValid())
11597 return std::nullopt;
11598
11599 auto Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: {});
11600
11601 // If the described register is the destination, just return the source.
11602 if (DestReg == DescribedReg)
11603 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11604
11605 // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
11606 if (MI.getOpcode() == AArch64::ORRWrs &&
11607 TRI->isSuperRegister(RegA: DestReg, RegB: DescribedReg))
11608 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11609
11610 // We may need to describe the lower part of a ORRXrs move.
11611 if (MI.getOpcode() == AArch64::ORRXrs &&
11612 TRI->isSubRegister(RegA: DestReg, RegB: DescribedReg)) {
11613 Register SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: AArch64::sub_32);
11614 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcSubReg, isDef: false), Expr);
11615 }
11616
11617 assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
11618 "Unhandled ORR[XW]rs copy case");
11619
11620 return std::nullopt;
11621}
11622
11623bool AArch64InstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
11624 // Functions cannot be split to different sections on AArch64 if they have
11625 // a red zone. This is because relaxing a cross-section branch may require
11626 // incrementing the stack pointer to spill a register, which would overwrite
11627 // the red zone.
11628 if (MF.getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(u: true))
11629 return false;
11630
11631 return TargetInstrInfo::isFunctionSafeToSplit(MF);
11632}
11633
11634bool AArch64InstrInfo::isMBBSafeToSplitToCold(
11635 const MachineBasicBlock &MBB) const {
11636 // Asm Goto blocks can contain conditional branches to goto labels, which can
11637 // get moved out of range of the branch instruction.
11638 auto isAsmGoto = [](const MachineInstr &MI) {
11639 return MI.getOpcode() == AArch64::INLINEASM_BR;
11640 };
11641 if (llvm::any_of(Range: MBB, P: isAsmGoto) || MBB.isInlineAsmBrIndirectTarget())
11642 return false;
11643
11644 // Because jump tables are label-relative instead of table-relative, they all
11645 // must be in the same section or relocation fixup handling will fail.
11646
11647 // Check if MBB is a jump table target
11648 const MachineJumpTableInfo *MJTI = MBB.getParent()->getJumpTableInfo();
11649 auto containsMBB = [&MBB](const MachineJumpTableEntry &JTE) {
11650 return llvm::is_contained(Range: JTE.MBBs, Element: &MBB);
11651 };
11652 if (MJTI != nullptr && llvm::any_of(Range: MJTI->getJumpTables(), P: containsMBB))
11653 return false;
11654
11655 // Check if MBB contains a jump table lookup
11656 for (const MachineInstr &MI : MBB) {
11657 switch (MI.getOpcode()) {
11658 case TargetOpcode::G_BRJT:
11659 case AArch64::JumpTableDest32:
11660 case AArch64::JumpTableDest16:
11661 case AArch64::JumpTableDest8:
11662 return false;
11663 default:
11664 continue;
11665 }
11666 }
11667
11668 // MBB isn't a special case, so it's safe to be split to the cold section.
11669 return true;
11670}
11671
11672std::optional<ParamLoadedValue>
11673AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
11674 Register Reg) const {
11675 const MachineFunction *MF = MI.getMF();
11676 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
11677 switch (MI.getOpcode()) {
11678 case AArch64::MOVZWi:
11679 case AArch64::MOVZXi: {
11680 // MOVZWi may be used for producing zero-extended 32-bit immediates in
11681 // 64-bit parameters, so we need to consider super-registers.
11682 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
11683 return std::nullopt;
11684
11685 if (!MI.getOperand(i: 1).isImm())
11686 return std::nullopt;
11687 int64_t Immediate = MI.getOperand(i: 1).getImm();
11688 int Shift = MI.getOperand(i: 2).getImm();
11689 return ParamLoadedValue(MachineOperand::CreateImm(Val: Immediate << Shift),
11690 nullptr);
11691 }
11692 case AArch64::ORRWrs:
11693 case AArch64::ORRXrs:
11694 return describeORRLoadedValue(MI, DescribedReg: Reg, TII: this, TRI);
11695 }
11696
11697 return TargetInstrInfo::describeLoadedValue(MI, Reg);
11698}
11699
11700bool AArch64InstrInfo::isExtendLikelyToBeFolded(
11701 MachineInstr &ExtMI, MachineRegisterInfo &MRI) const {
11702 assert(ExtMI.getOpcode() == TargetOpcode::G_SEXT ||
11703 ExtMI.getOpcode() == TargetOpcode::G_ZEXT ||
11704 ExtMI.getOpcode() == TargetOpcode::G_ANYEXT);
11705
11706 // Anyexts are nops.
11707 if (ExtMI.getOpcode() == TargetOpcode::G_ANYEXT)
11708 return true;
11709
11710 Register DefReg = ExtMI.getOperand(i: 0).getReg();
11711 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
11712 return false;
11713
11714 // It's likely that a sext/zext as a G_PTR_ADD offset will be folded into an
11715 // addressing mode.
11716 auto *UserMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
11717 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
11718}
11719
11720uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
11721 return get(Opcode: Opc).TSFlags & AArch64::ElementSizeMask;
11722}
11723
11724bool AArch64InstrInfo::isPTestLikeOpcode(unsigned Opc) const {
11725 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsPTestLike;
11726}
11727
11728bool AArch64InstrInfo::isWhileOpcode(unsigned Opc) const {
11729 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsWhile;
11730}
11731
11732unsigned int
11733AArch64InstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
11734 return OptLevel >= CodeGenOptLevel::Aggressive ? 6 : 2;
11735}
11736
11737bool AArch64InstrInfo::isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
11738 unsigned Scale) const {
11739 if (Offset && Scale)
11740 return false;
11741
11742 // Check Reg + Imm
11743 if (!Scale) {
11744 // 9-bit signed offset
11745 if (isInt<9>(x: Offset))
11746 return true;
11747
11748 // 12-bit unsigned offset
11749 unsigned Shift = Log2_64(Value: NumBytes);
11750 if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
11751 // Must be a multiple of NumBytes (NumBytes is a power of 2)
11752 (Offset >> Shift) << Shift == Offset)
11753 return true;
11754 return false;
11755 }
11756
11757 // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
11758 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
11759}
11760
11761unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
11762 if (MF.getSubtarget<AArch64Subtarget>().hardenSlsBlr())
11763 return AArch64::BLRNoIP;
11764 else
11765 return AArch64::BLR;
11766}
11767
11768void AArch64InstrInfo::createPauthEpilogueInstr(MachineBasicBlock &MBB,
11769 DebugLoc DL) const {
11770 MachineBasicBlock::iterator InsertPt = MBB.getFirstTerminator();
11771 auto Builder = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: get(Opcode: AArch64::PAUTH_EPILOGUE))
11772 .setMIFlag(MachineInstr::FrameDestroy);
11773
11774 MachineFunction &MF = *MBB.getParent();
11775 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
11776 auto &AFL = *static_cast<const AArch64FrameLowering *>(
11777 MF.getSubtarget().getFrameLowering());
11778 if (AFL.getArgumentStackToRestore(MF, MBB)) {
11779 Builder.addReg(RegNo: AArch64::X17, Flags: RegState::ImplicitDefine);
11780 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11781 if (Subtarget.hasPAuthLR())
11782 Builder.addReg(RegNo: AArch64::X15, Flags: RegState::ImplicitDefine);
11783 return;
11784 }
11785
11786 if (AFI->branchProtectionPAuthLR() && !Subtarget.hasPAuthLR())
11787 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11788}
11789
11790MachineBasicBlock::iterator
11791AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI,
11792 Register TargetReg, bool FrameSetup) const {
11793 assert(TargetReg != AArch64::SP && "New top of stack cannot already be in SP");
11794
11795 MachineBasicBlock &MBB = *MBBI->getParent();
11796 MachineFunction &MF = *MBB.getParent();
11797 const AArch64InstrInfo *TII =
11798 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
11799 int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
11800 DebugLoc DL = MBB.findDebugLoc(MBBI);
11801
11802 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
11803 MachineBasicBlock *LoopTestMBB =
11804 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11805 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
11806 MachineBasicBlock *LoopBodyMBB =
11807 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11808 MF.insert(MBBI: MBBInsertPoint, MBB: LoopBodyMBB);
11809 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11810 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
11811 MachineInstr::MIFlag Flags =
11812 FrameSetup ? MachineInstr::FrameSetup : MachineInstr::NoFlags;
11813
11814 // LoopTest:
11815 // SUB SP, SP, #ProbeSize
11816 emitFrameOffset(MBB&: *LoopTestMBB, MBBI: LoopTestMBB->end(), DL, DestReg: AArch64::SP,
11817 SrcReg: AArch64::SP, Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII, Flag: Flags);
11818
11819 // CMP SP, TargetReg
11820 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrx64),
11821 DestReg: AArch64::XZR)
11822 .addReg(RegNo: AArch64::SP)
11823 .addReg(RegNo: TargetReg)
11824 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0))
11825 .setMIFlags(Flags);
11826
11827 // B.<Cond> LoopExit
11828 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
11829 .addImm(Val: AArch64CC::LE)
11830 .addMBB(MBB: ExitMBB)
11831 .setMIFlags(Flags);
11832
11833 // LDR XZR, [SP]
11834 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11835 .addDef(RegNo: AArch64::XZR)
11836 .addReg(RegNo: AArch64::SP)
11837 .addImm(Val: 0)
11838 .addMemOperand(MMO: MF.getMachineMemOperand(
11839 PtrInfo: MachinePointerInfo::getUnknownStack(MF),
11840 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 8,
11841 BaseAlignment: Align(8)))
11842 .setMIFlags(Flags);
11843
11844 // B loop
11845 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
11846 .addMBB(MBB: LoopTestMBB)
11847 .setMIFlags(Flags);
11848
11849 // LoopExit:
11850 // MOV SP, TargetReg
11851 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: AArch64::SP)
11852 .addReg(RegNo: TargetReg)
11853 .addImm(Val: 0)
11854 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
11855 .setMIFlags(Flags);
11856
11857 // LDR XZR, [SP]
11858 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11859 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define)
11860 .addReg(RegNo: AArch64::SP)
11861 .addImm(Val: 0)
11862 .setMIFlags(Flags);
11863
11864 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
11865 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
11866
11867 LoopTestMBB->addSuccessor(Succ: ExitMBB);
11868 LoopTestMBB->addSuccessor(Succ: LoopBodyMBB);
11869 LoopBodyMBB->addSuccessor(Succ: LoopTestMBB);
11870 MBB.addSuccessor(Succ: LoopTestMBB);
11871
11872 // Update liveins.
11873 if (MF.getRegInfo().reservedRegsFrozen())
11874 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopBodyMBB, LoopTestMBB});
11875
11876 return ExitMBB->begin();
11877}
11878
11879namespace {
11880class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
11881 MachineFunction *MF;
11882 const TargetInstrInfo *TII;
11883 const TargetRegisterInfo *TRI;
11884 MachineRegisterInfo &MRI;
11885
11886 /// The block of the loop
11887 MachineBasicBlock *LoopBB;
11888 /// The conditional branch of the loop
11889 MachineInstr *CondBranch;
11890 /// The compare instruction for loop control
11891 MachineInstr *Comp;
11892 /// The number of the operand of the loop counter value in Comp
11893 unsigned CompCounterOprNum;
11894 /// The instruction that updates the loop counter value
11895 MachineInstr *Update;
11896 /// The number of the operand of the loop counter value in Update
11897 unsigned UpdateCounterOprNum;
11898 /// The initial value of the loop counter
11899 Register Init;
11900 /// True iff Update is a predecessor of Comp
11901 bool IsUpdatePriorComp;
11902
11903 /// The normalized condition used by createTripCountGreaterCondition()
11904 SmallVector<MachineOperand, 4> Cond;
11905
11906public:
11907 AArch64PipelinerLoopInfo(MachineBasicBlock *LoopBB, MachineInstr *CondBranch,
11908 MachineInstr *Comp, unsigned CompCounterOprNum,
11909 MachineInstr *Update, unsigned UpdateCounterOprNum,
11910 Register Init, bool IsUpdatePriorComp,
11911 const SmallVectorImpl<MachineOperand> &Cond)
11912 : MF(Comp->getParent()->getParent()),
11913 TII(MF->getSubtarget().getInstrInfo()),
11914 TRI(MF->getSubtarget().getRegisterInfo()), MRI(MF->getRegInfo()),
11915 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
11916 CompCounterOprNum(CompCounterOprNum), Update(Update),
11917 UpdateCounterOprNum(UpdateCounterOprNum), Init(Init),
11918 IsUpdatePriorComp(IsUpdatePriorComp), Cond(Cond.begin(), Cond.end()) {}
11919
11920 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
11921 // Make the instructions for loop control be placed in stage 0.
11922 // The predecessors of Comp are considered by the caller.
11923 return MI == Comp;
11924 }
11925
11926 std::optional<bool> createTripCountGreaterCondition(
11927 int TC, MachineBasicBlock &MBB,
11928 SmallVectorImpl<MachineOperand> &CondParam) override {
11929 // A branch instruction will be inserted as "if (Cond) goto epilogue".
11930 // Cond is normalized for such use.
11931 // The predecessors of the branch are assumed to have already been inserted.
11932 CondParam = Cond;
11933 return {};
11934 }
11935
11936 void createRemainingIterationsGreaterCondition(
11937 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
11938 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) override;
11939
11940 void setPreheader(MachineBasicBlock *NewPreheader) override {}
11941
11942 void adjustTripCount(int TripCountAdjust) override {}
11943
11944 bool isMVEExpanderSupported() override { return true; }
11945};
11946} // namespace
11947
11948/// Clone an instruction from MI. The register of ReplaceOprNum-th operand
11949/// is replaced by ReplaceReg. The output register is newly created.
11950/// The other operands are unchanged from MI.
11951static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum,
11952 Register ReplaceReg, MachineBasicBlock &MBB,
11953 MachineBasicBlock::iterator InsertTo) {
11954 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
11955 const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
11956 MachineInstr *NewMI = MBB.getParent()->CloneMachineInstr(Orig: MI);
11957 Register Result = 0;
11958 for (unsigned I = 0; I < NewMI->getNumOperands(); ++I) {
11959 if (I == 0 && NewMI->getOperand(i: 0).getReg().isVirtual()) {
11960 Result = MRI.createVirtualRegister(
11961 RegClass: MRI.getRegClass(Reg: NewMI->getOperand(i: 0).getReg()));
11962 NewMI->getOperand(i: I).setReg(Result);
11963 } else if (I == ReplaceOprNum) {
11964 MRI.constrainRegClass(Reg: ReplaceReg, RC: TII->getRegClass(MCID: NewMI->getDesc(), OpNum: I));
11965 NewMI->getOperand(i: I).setReg(ReplaceReg);
11966 }
11967 }
11968 MBB.insert(I: InsertTo, MI: NewMI);
11969 return Result;
11970}
11971
11972void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
11973 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
11974 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) {
11975 // Create and accumulate conditions for next TC iterations.
11976 // Example:
11977 // SUBSXrr N, counter, implicit-def $nzcv # compare instruction for the last
11978 // # iteration of the kernel
11979 //
11980 // # insert the following instructions
11981 // cond = CSINCXr 0, 0, C, implicit $nzcv
11982 // counter = ADDXri counter, 1 # clone from this->Update
11983 // SUBSXrr n, counter, implicit-def $nzcv # clone from this->Comp
11984 // cond = CSINCXr cond, cond, C, implicit $nzcv
11985 // ... (repeat TC times)
11986 // SUBSXri cond, 0, implicit-def $nzcv
11987
11988 assert(CondBranch->getOpcode() == AArch64::Bcc);
11989 // CondCode to exit the loop
11990 AArch64CC::CondCode CC =
11991 (AArch64CC::CondCode)CondBranch->getOperand(i: 0).getImm();
11992 if (CondBranch->getOperand(i: 1).getMBB() == LoopBB)
11993 CC = AArch64CC::getInvertedCondCode(Code: CC);
11994
11995 // Accumulate conditions to exit the loop
11996 Register AccCond = AArch64::XZR;
11997
11998 // If CC holds, CurCond+1 is returned; otherwise CurCond is returned.
11999 auto AccumulateCond = [&](Register CurCond,
12000 AArch64CC::CondCode CC) -> Register {
12001 Register NewCond = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
12002 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::CSINCXr))
12003 .addReg(RegNo: NewCond, Flags: RegState::Define)
12004 .addReg(RegNo: CurCond)
12005 .addReg(RegNo: CurCond)
12006 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
12007 return NewCond;
12008 };
12009
12010 if (!LastStage0Insts.empty() && LastStage0Insts[Comp]->getParent() == &MBB) {
12011 // Update and Comp for I==0 are already exists in MBB
12012 // (MBB is an unrolled kernel)
12013 Register Counter;
12014 for (int I = 0; I <= TC; ++I) {
12015 Register NextCounter;
12016 if (I != 0)
12017 NextCounter =
12018 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12019
12020 AccCond = AccumulateCond(AccCond, CC);
12021
12022 if (I != TC) {
12023 if (I == 0) {
12024 if (Update != Comp && IsUpdatePriorComp) {
12025 Counter =
12026 LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12027 NextCounter = cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB,
12028 InsertTo: MBB.end());
12029 } else {
12030 // can use already calculated value
12031 NextCounter = LastStage0Insts[Update]->getOperand(i: 0).getReg();
12032 }
12033 } else if (Update != Comp) {
12034 NextCounter =
12035 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12036 }
12037 }
12038 Counter = NextCounter;
12039 }
12040 } else {
12041 Register Counter;
12042 if (LastStage0Insts.empty()) {
12043 // use initial counter value (testing if the trip count is sufficient to
12044 // be executed by pipelined code)
12045 Counter = Init;
12046 if (IsUpdatePriorComp)
12047 Counter =
12048 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12049 } else {
12050 // MBB is an epilogue block. LastStage0Insts[Comp] is in the kernel block.
12051 Counter = LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12052 }
12053
12054 for (int I = 0; I <= TC; ++I) {
12055 Register NextCounter;
12056 NextCounter =
12057 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12058 AccCond = AccumulateCond(AccCond, CC);
12059 if (I != TC && Update != Comp)
12060 NextCounter =
12061 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12062 Counter = NextCounter;
12063 }
12064 }
12065
12066 // If AccCond == 0, the remainder is greater than TC.
12067 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::SUBSXri))
12068 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define | RegState::Dead)
12069 .addReg(RegNo: AccCond)
12070 .addImm(Val: 0)
12071 .addImm(Val: 0);
12072 Cond.clear();
12073 Cond.push_back(Elt: MachineOperand::CreateImm(Val: AArch64CC::EQ));
12074}
12075
12076static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB,
12077 Register &RegMBB, Register &RegOther) {
12078 assert(Phi.getNumOperands() == 5);
12079 if (Phi.getOperand(i: 2).getMBB() == MBB) {
12080 RegMBB = Phi.getOperand(i: 1).getReg();
12081 RegOther = Phi.getOperand(i: 3).getReg();
12082 } else {
12083 assert(Phi.getOperand(4).getMBB() == MBB);
12084 RegMBB = Phi.getOperand(i: 3).getReg();
12085 RegOther = Phi.getOperand(i: 1).getReg();
12086 }
12087}
12088
12089static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB) {
12090 if (!Reg.isVirtual())
12091 return false;
12092 const MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
12093 return MRI.getDefBlock(Reg) != BB;
12094}
12095
12096/// If Reg is an induction variable, return true and set some parameters
12097static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
12098 MachineInstr *&UpdateInst,
12099 unsigned &UpdateCounterOprNum, Register &InitReg,
12100 bool &IsUpdatePriorComp) {
12101 // Example:
12102 //
12103 // Preheader:
12104 // InitReg = ...
12105 // LoopBB:
12106 // Reg0 = PHI (InitReg, Preheader), (Reg1, LoopBB)
12107 // Reg = COPY Reg0 ; COPY is ignored.
12108 // Reg1 = ADD Reg, #1; UpdateInst. Incremented by a loop invariant value.
12109 // ; Reg is the value calculated in the previous
12110 // ; iteration, so IsUpdatePriorComp == false.
12111
12112 if (LoopBB->pred_size() != 2)
12113 return false;
12114 if (!Reg.isVirtual())
12115 return false;
12116 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
12117 UpdateInst = nullptr;
12118 UpdateCounterOprNum = 0;
12119 InitReg = 0;
12120 IsUpdatePriorComp = true;
12121 Register CurReg = Reg;
12122 while (true) {
12123 MachineInstr *Def = MRI.getVRegDef(Reg: CurReg);
12124 if (Def->getParent() != LoopBB)
12125 return false;
12126 if (Def->isCopy()) {
12127 // Ignore copy instructions unless they contain subregisters
12128 if (Def->getOperand(i: 0).getSubReg() || Def->getOperand(i: 1).getSubReg())
12129 return false;
12130 CurReg = Def->getOperand(i: 1).getReg();
12131 } else if (Def->isPHI()) {
12132 if (InitReg != 0)
12133 return false;
12134 if (!UpdateInst)
12135 IsUpdatePriorComp = false;
12136 extractPhiReg(Phi: *Def, MBB: LoopBB, RegMBB&: CurReg, RegOther&: InitReg);
12137 } else {
12138 if (UpdateInst)
12139 return false;
12140 switch (Def->getOpcode()) {
12141 case AArch64::ADDSXri:
12142 case AArch64::ADDSWri:
12143 case AArch64::SUBSXri:
12144 case AArch64::SUBSWri:
12145 case AArch64::ADDXri:
12146 case AArch64::ADDWri:
12147 case AArch64::SUBXri:
12148 case AArch64::SUBWri:
12149 UpdateInst = Def;
12150 UpdateCounterOprNum = 1;
12151 break;
12152 case AArch64::ADDSXrr:
12153 case AArch64::ADDSWrr:
12154 case AArch64::SUBSXrr:
12155 case AArch64::SUBSWrr:
12156 case AArch64::ADDXrr:
12157 case AArch64::ADDWrr:
12158 case AArch64::SUBXrr:
12159 case AArch64::SUBWrr:
12160 UpdateInst = Def;
12161 if (isDefinedOutside(Reg: Def->getOperand(i: 2).getReg(), BB: LoopBB))
12162 UpdateCounterOprNum = 1;
12163 else if (isDefinedOutside(Reg: Def->getOperand(i: 1).getReg(), BB: LoopBB))
12164 UpdateCounterOprNum = 2;
12165 else
12166 return false;
12167 break;
12168 default:
12169 return false;
12170 }
12171 CurReg = Def->getOperand(i: UpdateCounterOprNum).getReg();
12172 }
12173
12174 if (!CurReg.isVirtual())
12175 return false;
12176 if (Reg == CurReg)
12177 break;
12178 }
12179
12180 if (!UpdateInst)
12181 return false;
12182
12183 return true;
12184}
12185
12186std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
12187AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
12188 // Accept loops that meet the following conditions
12189 // * The conditional branch is BCC
12190 // * The compare instruction is ADDS/SUBS/WHILEXX
12191 // * One operand of the compare is an induction variable and the other is a
12192 // loop invariant value
12193 // * The induction variable is incremented/decremented by a single instruction
12194 // * Does not contain CALL or instructions which have unmodeled side effects
12195
12196 for (MachineInstr &MI : *LoopBB)
12197 if (MI.isCall() || MI.hasUnmodeledSideEffects())
12198 // This instruction may use NZCV, which interferes with the instruction to
12199 // be inserted for loop control.
12200 return nullptr;
12201
12202 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
12203 SmallVector<MachineOperand, 4> Cond;
12204 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond))
12205 return nullptr;
12206
12207 // Infinite loops are not supported
12208 if (TBB == LoopBB && FBB == LoopBB)
12209 return nullptr;
12210
12211 // Must be conditional branch
12212 if (TBB != LoopBB && FBB == nullptr)
12213 return nullptr;
12214
12215 assert((TBB == LoopBB || FBB == LoopBB) &&
12216 "The Loop must be a single-basic-block loop");
12217
12218 MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
12219 const TargetRegisterInfo &TRI = getRegisterInfo();
12220
12221 if (CondBranch->getOpcode() != AArch64::Bcc)
12222 return nullptr;
12223
12224 // Normalization for createTripCountGreaterCondition()
12225 if (TBB == LoopBB)
12226 reverseBranchCondition(Cond);
12227
12228 MachineInstr *Comp = nullptr;
12229 unsigned CompCounterOprNum = 0;
12230 for (MachineInstr &MI : reverse(C&: *LoopBB)) {
12231 if (MI.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
12232 // Guarantee that the compare is SUBS/ADDS/WHILEXX and that one of the
12233 // operands is a loop invariant value
12234
12235 switch (MI.getOpcode()) {
12236 case AArch64::SUBSXri:
12237 case AArch64::SUBSWri:
12238 case AArch64::ADDSXri:
12239 case AArch64::ADDSWri:
12240 Comp = &MI;
12241 CompCounterOprNum = 1;
12242 break;
12243 case AArch64::ADDSWrr:
12244 case AArch64::ADDSXrr:
12245 case AArch64::SUBSWrr:
12246 case AArch64::SUBSXrr:
12247 Comp = &MI;
12248 break;
12249 default:
12250 if (isWhileOpcode(Opc: MI.getOpcode())) {
12251 Comp = &MI;
12252 break;
12253 }
12254 return nullptr;
12255 }
12256
12257 if (CompCounterOprNum == 0) {
12258 if (isDefinedOutside(Reg: Comp->getOperand(i: 1).getReg(), BB: LoopBB))
12259 CompCounterOprNum = 2;
12260 else if (isDefinedOutside(Reg: Comp->getOperand(i: 2).getReg(), BB: LoopBB))
12261 CompCounterOprNum = 1;
12262 else
12263 return nullptr;
12264 }
12265 break;
12266 }
12267 }
12268 if (!Comp)
12269 return nullptr;
12270
12271 MachineInstr *Update = nullptr;
12272 Register Init;
12273 bool IsUpdatePriorComp;
12274 unsigned UpdateCounterOprNum;
12275 if (!getIndVarInfo(Reg: Comp->getOperand(i: CompCounterOprNum).getReg(), LoopBB,
12276 UpdateInst&: Update, UpdateCounterOprNum, InitReg&: Init, IsUpdatePriorComp))
12277 return nullptr;
12278
12279 return std::make_unique<AArch64PipelinerLoopInfo>(
12280 args&: LoopBB, args&: CondBranch, args&: Comp, args&: CompCounterOprNum, args&: Update, args&: UpdateCounterOprNum,
12281 args&: Init, args&: IsUpdatePriorComp, args&: Cond);
12282}
12283
12284/// verifyInstruction - Perform target specific instruction verification.
12285bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI,
12286 StringRef &ErrInfo) const {
12287 // Verify that immediate offsets on load/store instructions are within range.
12288 // Stack objects with an FI operand are excluded as they can be fixed up
12289 // during PEI.
12290 TypeSize Scale(0U, false), Width(0U, false);
12291 int64_t MinOffset, MaxOffset;
12292 if (getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
12293 unsigned ImmIdx = getLoadStoreImmIdx(Opc: MI.getOpcode());
12294 if (MI.getOperand(i: ImmIdx).isImm() && !MI.getOperand(i: ImmIdx - 1).isFI()) {
12295 int64_t Imm = MI.getOperand(i: ImmIdx).getImm();
12296 if (Imm < MinOffset || Imm > MaxOffset) {
12297 ErrInfo = "Unexpected immediate on load/store instruction";
12298 return false;
12299 }
12300 }
12301 }
12302
12303 const MCInstrDesc &MCID = MI.getDesc();
12304 for (unsigned Op = 0; Op < MCID.getNumOperands(); Op++) {
12305 const MachineOperand &MO = MI.getOperand(i: Op);
12306 switch (MCID.operands()[Op].OperandType) {
12307 case AArch64::OPERAND_IMPLICIT_IMM_0:
12308 if (!MO.isImm() || MO.getImm() != 0) {
12309 ErrInfo = "OPERAND_IMPLICIT_IMM_0 should be 0";
12310 return false;
12311 }
12312 break;
12313 case AArch64::OPERAND_SHIFT_MSL:
12314 if (!MO.isImm() ||
12315 AArch64_AM::getShiftType(Imm: MO.getImm()) != AArch64_AM::MSL ||
12316 (AArch64_AM::getShiftValue(Imm: MO.getImm()) != 8 &&
12317 AArch64_AM::getShiftValue(Imm: MO.getImm()) != 16)) {
12318 ErrInfo = "OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
12319 return false;
12320 }
12321 break;
12322 case AArch64::OPERAND_IMM_UINT1:
12323 if (!MO.isImm() || (MO.getImm() != 0 && MO.getImm() != 1)) {
12324 ErrInfo = "OPERAND_IMM_UINT1 should be 0 or 1";
12325 return false;
12326 }
12327 break;
12328 case AArch64::OPERAND_IMM_UINT4plus1:
12329 if (!MO.isImm() || MO.getImm() <= 0 || MO.getImm() > 16) {
12330 ErrInfo = "OPERAND_IMM_UINT4plus1 should be in the range 1 to 16";
12331 return false;
12332 }
12333 break;
12334 case AArch64::OPERAND_IMM_UINT5:
12335 if (!MO.isImm() || !isUInt<5>(x: MO.getImm())) {
12336 ErrInfo = "OPERAND_IMM_UINT5 should be in the range 0 to 31";
12337 return false;
12338 }
12339 break;
12340 case AArch64::OPERAND_IMM_UINT8:
12341 if (!MO.isImm() || !isUInt<8>(x: MO.getImm())) {
12342 ErrInfo = "OPERAND_IMM_UINT8 should be in the range 0 to 255";
12343 return false;
12344 }
12345 break;
12346 default:
12347 break;
12348 }
12349 }
12350 return true;
12351}
12352
12353#define GET_INSTRINFO_HELPERS
12354#define GET_INSTRMAP_INFO
12355#include "AArch64GenInstrInfo.inc"
12356