1//===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64PointerAuth.h"
17#include "AArch64Subtarget.h"
18#include "MCTargetDesc/AArch64AddressingModes.h"
19#include "MCTargetDesc/AArch64MCLFIRewriter.h"
20#include "MCTargetDesc/AArch64MCTargetDesc.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/CodeGen/CFIInstBuilder.h"
28#include "llvm/CodeGen/LivePhysRegs.h"
29#include "llvm/CodeGen/LiveRegUnits.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineCombinerPattern.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstr.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/RegisterScavenging.h"
41#include "llvm/CodeGen/StackMaps.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/CodeGen/TargetSubtargetInfo.h"
44#include "llvm/IR/DebugInfoMetadata.h"
45#include "llvm/IR/DebugLoc.h"
46#include "llvm/IR/GlobalValue.h"
47#include "llvm/IR/Module.h"
48#include "llvm/MC/MCAsmInfo.h"
49#include "llvm/MC/MCInst.h"
50#include "llvm/MC/MCInstBuilder.h"
51#include "llvm/MC/MCInstrDesc.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CodeGen.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/ErrorHandling.h"
56#include "llvm/Support/LEB128.h"
57#include "llvm/Support/MathExtras.h"
58#include "llvm/Target/TargetMachine.h"
59#include "llvm/Target/TargetOptions.h"
60#include <cassert>
61#include <cstdint>
62#include <iterator>
63#include <utility>
64
65using namespace llvm;
66
67#define GET_INSTRINFO_CTOR_DTOR
68#include "AArch64GenInstrInfo.inc"
69
70#define DEBUG_TYPE "AArch64InstrInfo"
71
72STATISTIC(NumCopyInstrs, "Number of COPY instructions expanded");
73STATISTIC(NumZCRegMoveInstrsGPR, "Number of zero-cycle GPR register move "
74 "instructions expanded from canonical COPY");
75STATISTIC(NumZCRegMoveInstrsFPR, "Number of zero-cycle FPR register move "
76 "instructions expanded from canonical COPY");
77STATISTIC(NumZCZeroingInstrsGPR, "Number of zero-cycle GPR zeroing "
78 "instructions expanded from canonical COPY");
79// NumZCZeroingInstrsFPR is counted at AArch64AsmPrinter
80
81static cl::opt<unsigned>
82 CBDisplacementBits("aarch64-cb-offset-bits", cl::Hidden, cl::init(Val: 9),
83 cl::desc("Restrict range of CB instructions (DEBUG)"));
84
85static cl::opt<unsigned> TBZDisplacementBits(
86 "aarch64-tbz-offset-bits", cl::Hidden, cl::init(Val: 14),
87 cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"));
88
89static cl::opt<unsigned> CBZDisplacementBits(
90 "aarch64-cbz-offset-bits", cl::Hidden, cl::init(Val: 19),
91 cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"));
92
93static cl::opt<unsigned>
94 BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(Val: 19),
95 cl::desc("Restrict range of Bcc instructions (DEBUG)"));
96
97static cl::opt<unsigned>
98 BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(Val: 26),
99 cl::desc("Restrict range of B instructions (DEBUG)"));
100
101static cl::opt<unsigned> GatherOptSearchLimit(
102 "aarch64-search-limit", cl::Hidden, cl::init(Val: 2048),
103 cl::desc("Restrict range of instructions to search for the "
104 "machine-combiner gather pattern optimization"));
105
106static cl::opt<bool> UseCompactUnwindFrameRecordForOutlinedFunctions(
107 "aarch64-outliner-compact-unwind-frame", cl::Hidden, cl::init(Val: true),
108 cl::desc("Use a frame record for Mach-O non-leaf outlined functions"));
109
110AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
111 : AArch64GenInstrInfo(STI, RI, AArch64::ADJCALLSTACKDOWN,
112 AArch64::ADJCALLSTACKUP, AArch64::CATCHRET),
113 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
114
115/// Return the maximum number of bytes of code the specified instruction may be
116/// after LFI rewriting. If the instruction is not rewritten, std::nullopt is
117/// returned (use default sizing).
118///
119/// NOTE: the size estimates here must be kept in sync with the rewrites in
120/// AArch64MCLFIRewriter.cpp. Sizes may be overestimates of the rewritten
121/// instruction sequences.
122static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
123 switch (MI.getOpcode()) {
124 case AArch64::SVC:
125 // SVC expands to 4 instructions.
126 return 16;
127 case AArch64::BR:
128 case AArch64::BLR:
129 // Indirect branches/calls expand to 2 instructions (guard + br/blr).
130 return 8;
131 case AArch64::RET:
132 // RET through another register expands to 2 instructions (guard + ret).
133 // RET through LR may also expand to 2 instructions if a deferred LR guard
134 // is flushed before the return.
135 return 8;
136 case AArch64::RETAA:
137 case AArch64::RETAB:
138 // Authenticated returns expand to 3 instructions (authenticate + guard +
139 // ret).
140 return 12;
141 case AArch64::BRAA:
142 case AArch64::BRAAZ:
143 case AArch64::BRAB:
144 case AArch64::BRABZ:
145 case AArch64::BLRAA:
146 case AArch64::BLRAAZ:
147 case AArch64::BLRAB:
148 case AArch64::BLRABZ:
149 // Authenticated branches/calls expand to 3 instructions (authenticate +
150 // guard + branch).
151 return 12;
152 case AArch64::AUTIASP:
153 case AArch64::AUTIBSP:
154 case AArch64::AUTIAZ:
155 case AArch64::AUTIBZ:
156 case AArch64::XPACLRI:
157 // Authenticating LR expands to the instruction plus a deferred LR guard.
158 return 8;
159 case AArch64::SYSxt:
160 // VA-based DC/IC ops (op1=3, Cn=7, op2=1) expand to 2 instructions.
161 if (MI.getOperand(i: 0).getImm() == 3 && MI.getOperand(i: 1).getImm() == 7 &&
162 MI.getOperand(i: 3).getImm() == 1)
163 return 8;
164 return std::nullopt;
165 default:
166 break;
167 }
168
169 // Detect instructions that explicitly define SP or LR.
170 bool ModifiesLR = false;
171 bool ModifiesSP = false;
172 for (const MachineOperand &MO : MI.defs()) {
173 if (!MO.isReg())
174 continue;
175 if (MO.getReg() == AArch64::LR)
176 ModifiesLR = true;
177 else if (MO.getReg() == AArch64::SP)
178 ModifiesSP = true;
179 }
180
181 // Memory accesses expand to a base-register guard plus the rewritten access
182 // (8 bytes), with an extra base-register update for pre/post-index forms (12
183 // bytes total). If the access also defines LR, an LR mask is appended (+4
184 // bytes). Depending on additional optimizations that the rewriter performs,
185 // this may be an overestimate.
186 if (MI.mayLoadOrStore()) {
187 unsigned Size = isLFIPrePostMemAccess(Opcode: MI.getOpcode()) ? 12 : 8;
188 if (ModifiesLR)
189 Size += 4;
190 return Size;
191 }
192
193 // Non memory operations that modify LR or SP expand to 2 instructions.
194 if (ModifiesSP || ModifiesLR)
195 return 8;
196
197 // Default case: instructions that don't cause expansion.
198 // - TP accesses in LFI are a single load/store, so no expansion.
199 // - All remaining instructions are not rewritten.
200 return std::nullopt;
201}
202
203/// GetInstSize - Return the number of bytes of code the specified
204/// instruction may be. This returns the maximum number of bytes.
205unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
206 const MCInstrDesc &Desc = MI.getDesc();
207 if (!Desc.isPseudo() && !Subtarget.isLFI()) {
208 assert(Desc.getSize() == 4 && "Unexpected instruction size");
209 return 4;
210 }
211
212 const MachineBasicBlock &MBB = *MI.getParent();
213 const MachineFunction *MF = MBB.getParent();
214 const Function &F = MF->getFunction();
215 const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
216
217 {
218 auto Op = MI.getOpcode();
219 if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
220 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(), MAI);
221 }
222
223 // Meta-instructions emit no code.
224 if (MI.isMetaInstruction())
225 return 0;
226
227 // FIXME: We currently only handle pseudoinstructions that don't get expanded
228 // before the assembly printer.
229 unsigned NumBytes = 0;
230
231 // LFI rewriter expansions that supersede normal sizing.
232 const auto &STI = MF->getSubtarget<AArch64Subtarget>();
233 if (STI.isLFI())
234 if (auto Size = getLFIInstSizeInBytes(MI))
235 return *Size;
236
237 if (!MI.isBundle() && isTailCallReturnInst(MI)) {
238 NumBytes = Desc.getSize() ? Desc.getSize() : 4;
239
240 const auto *MFI = MF->getInfo<AArch64FunctionInfo>();
241 if (!MFI->shouldSignReturnAddress(MF: *MF))
242 return NumBytes;
243
244 auto Method = STI.getAuthenticatedLRCheckMethod(MF: *MF);
245 NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
246 return NumBytes;
247 }
248
249 // Size should be preferably set in
250 // llvm/lib/Target/AArch64/AArch64InstrInfo.td (default case).
251 // Specific cases handle instructions of variable sizes
252 switch (Desc.getOpcode()) {
253 default:
254 if (Desc.getSize())
255 return Desc.getSize();
256
257 // Anything not explicitly designated otherwise (i.e. pseudo-instructions
258 // with fixed constant size but not specified in .td file) is a normal
259 // 4-byte insn.
260 NumBytes = 4;
261 break;
262 case TargetOpcode::STACKMAP:
263 // The upper bound for a stackmap intrinsic is the full length of its shadow
264 NumBytes = StackMapOpers(&MI).getNumPatchBytes();
265 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
266 break;
267 case TargetOpcode::PATCHPOINT:
268 // The size of the patchpoint intrinsic is the number of bytes requested
269 NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
270 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
271 break;
272 case TargetOpcode::STATEPOINT:
273 NumBytes = StatepointOpers(&MI).getNumPatchBytes();
274 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
275 // No patch bytes means a normal call inst is emitted
276 if (NumBytes == 0)
277 NumBytes = 4;
278 break;
279 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
280 // If `patchable-function-entry` is set, PATCHABLE_FUNCTION_ENTER
281 // instructions are expanded to the specified number of NOPs. Otherwise,
282 // they are expanded to 36-byte XRay sleds.
283 NumBytes =
284 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry", Default: 9) * 4;
285 break;
286 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
287 case TargetOpcode::PATCHABLE_TAIL_CALL:
288 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
289 // An XRay sled can be 4 bytes of alignment plus a 32-byte block.
290 NumBytes = 36;
291 break;
292 case TargetOpcode::PATCHABLE_EVENT_CALL:
293 // EVENT_CALL XRay sleds are exactly 6 instructions long (no alignment).
294 NumBytes = 24;
295 break;
296
297 case AArch64::SPACE:
298 NumBytes = MI.getOperand(i: 1).getImm();
299 break;
300 case AArch64::MOVaddr:
301 case AArch64::MOVaddrJT:
302 case AArch64::MOVaddrCP:
303 case AArch64::MOVaddrBA:
304 case AArch64::MOVaddrTLS:
305 case AArch64::MOVaddrEXT: {
306 // Use the same logic as the pseudo expansion to count instructions.
307 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
308 AArch64_IMM::expandMOVAddr(Opcode: Desc.getOpcode(),
309 TargetFlags: MI.getOperand(i: 1).getTargetFlags(),
310 IsTargetMachO: Subtarget.isTargetMachO(), Insn);
311 NumBytes = Insn.size() * 4;
312 break;
313 }
314
315 case AArch64::MOVi32imm:
316 case AArch64::MOVi64imm: {
317 // Use the same logic as the pseudo expansion to count instructions.
318 unsigned BitSize = Desc.getOpcode() == AArch64::MOVi32imm ? 32 : 64;
319 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
320 AArch64_IMM::expandMOVImm(Imm: MI.getOperand(i: 1).getImm(), BitSize, Insn);
321 NumBytes = Insn.size() * 4;
322 break;
323 }
324
325 case TargetOpcode::BUNDLE:
326 NumBytes = getInstBundleSize(MI);
327 break;
328 }
329
330 return NumBytes;
331}
332
333static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
334 SmallVectorImpl<MachineOperand> &Cond) {
335 // Block ends with fall-through condbranch.
336 switch (LastInst->getOpcode()) {
337 default:
338 llvm_unreachable("Unknown branch instruction?");
339 case AArch64::Bcc:
340 Target = LastInst->getOperand(i: 1).getMBB();
341 Cond.push_back(Elt: LastInst->getOperand(i: 0));
342 break;
343 case AArch64::CBZW:
344 case AArch64::CBZX:
345 case AArch64::CBNZW:
346 case AArch64::CBNZX:
347 Target = LastInst->getOperand(i: 1).getMBB();
348 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
349 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
350 Cond.push_back(Elt: LastInst->getOperand(i: 0));
351 break;
352 case AArch64::TBZW:
353 case AArch64::TBZX:
354 case AArch64::TBNZW:
355 case AArch64::TBNZX:
356 Target = LastInst->getOperand(i: 2).getMBB();
357 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
358 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
359 Cond.push_back(Elt: LastInst->getOperand(i: 0));
360 Cond.push_back(Elt: LastInst->getOperand(i: 1));
361 break;
362 case AArch64::CBWPri:
363 case AArch64::CBXPri:
364 case AArch64::CBWPrr:
365 case AArch64::CBXPrr:
366 Target = LastInst->getOperand(i: 3).getMBB();
367 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
368 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
369 Cond.push_back(Elt: LastInst->getOperand(i: 0));
370 Cond.push_back(Elt: LastInst->getOperand(i: 1));
371 Cond.push_back(Elt: LastInst->getOperand(i: 2));
372 break;
373 case AArch64::CBBAssertExt:
374 case AArch64::CBHAssertExt:
375 Target = LastInst->getOperand(i: 3).getMBB();
376 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1)); // -1
377 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode())); // Opc
378 Cond.push_back(Elt: LastInst->getOperand(i: 0)); // Cond
379 Cond.push_back(Elt: LastInst->getOperand(i: 1)); // Op0
380 Cond.push_back(Elt: LastInst->getOperand(i: 2)); // Op1
381 Cond.push_back(Elt: LastInst->getOperand(i: 4)); // Ext0
382 Cond.push_back(Elt: LastInst->getOperand(i: 5)); // Ext1
383 break;
384 }
385}
386
387static unsigned getBranchDisplacementBits(unsigned Opc) {
388 switch (Opc) {
389 default:
390 llvm_unreachable("unexpected opcode!");
391 case AArch64::B:
392 return BDisplacementBits;
393 case AArch64::TBNZW:
394 case AArch64::TBZW:
395 case AArch64::TBNZX:
396 case AArch64::TBZX:
397 return TBZDisplacementBits;
398 case AArch64::CBNZW:
399 case AArch64::CBZW:
400 case AArch64::CBNZX:
401 case AArch64::CBZX:
402 return CBZDisplacementBits;
403 case AArch64::Bcc:
404 return BCCDisplacementBits;
405 case AArch64::CBWPri:
406 case AArch64::CBXPri:
407 case AArch64::CBBAssertExt:
408 case AArch64::CBHAssertExt:
409 case AArch64::CBWPrr:
410 case AArch64::CBXPrr:
411 return CBDisplacementBits;
412 }
413}
414
415bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
416 int64_t BrOffset) const {
417 unsigned Bits = getBranchDisplacementBits(Opc: BranchOp);
418 assert(Bits >= 3 && "max branch displacement must be enough to jump"
419 "over conditional branch expansion");
420 return isIntN(N: Bits, x: BrOffset / 4);
421}
422
423MachineBasicBlock *
424AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
425 switch (MI.getOpcode()) {
426 default:
427 llvm_unreachable("unexpected opcode!");
428 case AArch64::B:
429 return MI.getOperand(i: 0).getMBB();
430 case AArch64::TBZW:
431 case AArch64::TBNZW:
432 case AArch64::TBZX:
433 case AArch64::TBNZX:
434 return MI.getOperand(i: 2).getMBB();
435 case AArch64::CBZW:
436 case AArch64::CBNZW:
437 case AArch64::CBZX:
438 case AArch64::CBNZX:
439 case AArch64::Bcc:
440 return MI.getOperand(i: 1).getMBB();
441 case AArch64::CBWPri:
442 case AArch64::CBXPri:
443 case AArch64::CBBAssertExt:
444 case AArch64::CBHAssertExt:
445 case AArch64::CBWPrr:
446 case AArch64::CBXPrr:
447 return MI.getOperand(i: 3).getMBB();
448 }
449}
450
451void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
452 MachineBasicBlock &NewDestBB,
453 MachineBasicBlock &RestoreBB,
454 const DebugLoc &DL,
455 int64_t BrOffset,
456 RegScavenger *RS) const {
457 assert(RS && "RegScavenger required for long branching");
458 assert(MBB.empty() &&
459 "new block should be inserted for expanding unconditional branch");
460 assert(MBB.pred_size() == 1);
461 assert(RestoreBB.empty() &&
462 "restore block should be inserted for restoring clobbered registers");
463
464 auto buildIndirectBranch = [&](Register Reg, MachineBasicBlock &DestBB) {
465 // Offsets outside of the signed 33-bit range are not supported for ADRP +
466 // ADD.
467 if (!isInt<33>(x: BrOffset))
468 report_fatal_error(
469 reason: "Branch offsets outside of the signed 33-bit range not supported");
470
471 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
472 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGE);
473 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: Reg)
474 .addReg(RegNo: Reg)
475 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
476 .addImm(Val: 0);
477 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::BR)).addReg(RegNo: Reg);
478 };
479
480 RS->enterBasicBlockEnd(MBB);
481 // If X16 is unused, we can rely on the linker to insert a range extension
482 // thunk if NewDestBB is out of range of a single B instruction.
483 constexpr Register Reg = AArch64::X16;
484 if (!RS->isRegUsed(Reg)) {
485 insertUnconditionalBranch(MBB, DestBB: &NewDestBB, DL);
486 RS->setRegUsed(Reg);
487 return;
488 }
489
490 // In a cold block without BTI, insert the indirect branch if a register is
491 // free. Skip this if BTI is enabled to avoid inserting a BTI at the target,
492 // prioritizing a dynamic cost in cold code over a static cost in hot code.
493 AArch64FunctionInfo *AFI = MBB.getParent()->getInfo<AArch64FunctionInfo>();
494 bool HasBTI = AFI && AFI->branchTargetEnforcement();
495 if (MBB.getSectionID() == MBBSectionID::ColdSectionID && !HasBTI) {
496 Register Scavenged = RS->FindUnusedReg(RC: &AArch64::GPR64RegClass);
497 if (Scavenged.isValid()) {
498 buildIndirectBranch(Scavenged, NewDestBB);
499 RS->setRegUsed(Reg: Scavenged);
500 return;
501 }
502 }
503
504 // Note: Spilling X16 briefly moves the stack pointer, making it incompatible
505 // with red zones.
506 if (!AFI || AFI->hasRedZone().value_or(u: true))
507 report_fatal_error(
508 reason: "Unable to insert indirect branch inside function that has red zone");
509
510 // Otherwise, spill X16 and defer range extension to the linker.
511 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::STRXpre))
512 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
513 .addReg(RegNo: Reg)
514 .addReg(RegNo: AArch64::SP)
515 .addImm(Val: -16);
516
517 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: &RestoreBB);
518
519 BuildMI(BB&: RestoreBB, I: RestoreBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::LDRXpost))
520 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
521 .addReg(RegNo: Reg, Flags: RegState::Define)
522 .addReg(RegNo: AArch64::SP)
523 .addImm(Val: 16);
524}
525
526// Branch analysis.
527bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
528 MachineBasicBlock *&TBB,
529 MachineBasicBlock *&FBB,
530 SmallVectorImpl<MachineOperand> &Cond,
531 bool AllowModify) const {
532 // If the block has no terminators, it just falls into the block after it.
533 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
534 if (I == MBB.end())
535 return false;
536
537 // Skip over SpeculationBarrierEndBB terminators
538 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
539 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
540 --I;
541 }
542
543 if (!isUnpredicatedTerminator(MI: *I))
544 return false;
545
546 // Get the last instruction in the block.
547 MachineInstr *LastInst = &*I;
548
549 // If there is only one terminator instruction, process it.
550 unsigned LastOpc = LastInst->getOpcode();
551 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
552 if (isUncondBranchOpcode(Opc: LastOpc)) {
553 TBB = LastInst->getOperand(i: 0).getMBB();
554 return false;
555 }
556 if (isCondBranchOpcode(Opc: LastOpc)) {
557 // Block ends with fall-through condbranch.
558 parseCondBranch(LastInst, Target&: TBB, Cond);
559 return false;
560 }
561 return true; // Can't handle indirect branch.
562 }
563
564 // Get the instruction before it if it is a terminator.
565 MachineInstr *SecondLastInst = &*I;
566 unsigned SecondLastOpc = SecondLastInst->getOpcode();
567
568 // If AllowModify is true and the block ends with two or more unconditional
569 // branches, delete all but the first unconditional branch.
570 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc)) {
571 while (isUncondBranchOpcode(Opc: SecondLastOpc)) {
572 LastInst->eraseFromParent();
573 LastInst = SecondLastInst;
574 LastOpc = LastInst->getOpcode();
575 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
576 // Return now the only terminator is an unconditional branch.
577 TBB = LastInst->getOperand(i: 0).getMBB();
578 return false;
579 }
580 SecondLastInst = &*I;
581 SecondLastOpc = SecondLastInst->getOpcode();
582 }
583 }
584
585 // If we're allowed to modify and the block ends in a unconditional branch
586 // which could simply fallthrough, remove the branch. (Note: This case only
587 // matters when we can't understand the whole sequence, otherwise it's also
588 // handled by BranchFolding.cpp.)
589 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc) &&
590 MBB.isLayoutSuccessor(MBB: getBranchDestBlock(MI: *LastInst))) {
591 LastInst->eraseFromParent();
592 LastInst = SecondLastInst;
593 LastOpc = LastInst->getOpcode();
594 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
595 assert(!isUncondBranchOpcode(LastOpc) &&
596 "unreachable unconditional branches removed above");
597
598 if (isCondBranchOpcode(Opc: LastOpc)) {
599 // Block ends with fall-through condbranch.
600 parseCondBranch(LastInst, Target&: TBB, Cond);
601 return false;
602 }
603 return true; // Can't handle indirect branch.
604 }
605 SecondLastInst = &*I;
606 SecondLastOpc = SecondLastInst->getOpcode();
607 }
608
609 // If there are three terminators, we don't know what sort of block this is.
610 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(MI: *--I))
611 return true;
612
613 // If the block ends with a B and a Bcc, handle it.
614 if (isCondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
615 parseCondBranch(LastInst: SecondLastInst, Target&: TBB, Cond);
616 FBB = LastInst->getOperand(i: 0).getMBB();
617 return false;
618 }
619
620 // If the block ends with two unconditional branches, handle it. The second
621 // one is not executed, so remove it.
622 if (isUncondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
623 TBB = SecondLastInst->getOperand(i: 0).getMBB();
624 I = LastInst;
625 if (AllowModify)
626 I->eraseFromParent();
627 return false;
628 }
629
630 // ...likewise if it ends with an indirect branch followed by an unconditional
631 // branch.
632 if (isIndirectBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
633 I = LastInst;
634 if (AllowModify)
635 I->eraseFromParent();
636 return true;
637 }
638
639 // Otherwise, can't handle this.
640 return true;
641}
642
643bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
644 MachineBranchPredicate &MBP,
645 bool AllowModify) const {
646 // Use analyzeBranch to validate the branch pattern.
647 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
648 SmallVector<MachineOperand, 4> Cond;
649 if (analyzeBranch(MBB, TBB, FBB, Cond, AllowModify))
650 return true;
651
652 // analyzeBranch returns success with empty Cond for unconditional branches.
653 if (Cond.empty())
654 return true;
655
656 MBP.TrueDest = TBB;
657 assert(MBP.TrueDest && "expected!");
658 MBP.FalseDest = FBB ? FBB : MBB.getNextNode();
659
660 MBP.ConditionDef = nullptr;
661 MBP.SingleUseCondition = false;
662
663 // Find the conditional branch. After analyzeBranch succeeds with non-empty
664 // Cond, there's exactly one conditional branch - either last (fallthrough)
665 // or second-to-last (followed by unconditional B).
666 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
667 if (I == MBB.end())
668 return true;
669
670 if (isUncondBranchOpcode(Opc: I->getOpcode())) {
671 if (I == MBB.begin())
672 return true;
673 --I;
674 }
675
676 MachineInstr *CondBranch = &*I;
677 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
678
679 switch (CondBranch->getOpcode()) {
680 default:
681 return true;
682
683 case AArch64::Bcc:
684 // Bcc takes the NZCV flag as the operand to branch on, walk up the
685 // instruction stream to find the last instruction to define NZCV.
686 for (MachineInstr &MI : llvm::drop_begin(RangeOrContainer: llvm::reverse(C&: MBB))) {
687 if (MI.modifiesRegister(Reg: AArch64::NZCV, /*TRI=*/nullptr)) {
688 MBP.ConditionDef = &MI;
689 break;
690 }
691 }
692 return false;
693
694 case AArch64::CBZW:
695 case AArch64::CBZX:
696 case AArch64::CBNZW:
697 case AArch64::CBNZX: {
698 MBP.LHS = CondBranch->getOperand(i: 0);
699 MBP.RHS = MachineOperand::CreateImm(Val: 0);
700 unsigned Opc = CondBranch->getOpcode();
701 MBP.Predicate = (Opc == AArch64::CBNZX || Opc == AArch64::CBNZW)
702 ? MachineBranchPredicate::PRED_NE
703 : MachineBranchPredicate::PRED_EQ;
704 Register CondReg = MBP.LHS.getReg();
705 if (CondReg.isVirtual())
706 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
707 return false;
708 }
709
710 case AArch64::TBZW:
711 case AArch64::TBZX:
712 case AArch64::TBNZW:
713 case AArch64::TBNZX: {
714 Register CondReg = CondBranch->getOperand(i: 0).getReg();
715 if (CondReg.isVirtual())
716 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
717 return false;
718 }
719 }
720}
721
722bool AArch64InstrInfo::reverseBranchCondition(
723 SmallVectorImpl<MachineOperand> &Cond) const {
724 if (Cond[0].getImm() != -1) {
725 // Regular Bcc
726 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
727 Cond[0].setImm(AArch64CC::getInvertedCondCode(Code: CC));
728 } else {
729 // Folded compare-and-branch
730 switch (Cond[1].getImm()) {
731 default:
732 llvm_unreachable("Unknown conditional branch!");
733 case AArch64::CBZW:
734 Cond[1].setImm(AArch64::CBNZW);
735 break;
736 case AArch64::CBNZW:
737 Cond[1].setImm(AArch64::CBZW);
738 break;
739 case AArch64::CBZX:
740 Cond[1].setImm(AArch64::CBNZX);
741 break;
742 case AArch64::CBNZX:
743 Cond[1].setImm(AArch64::CBZX);
744 break;
745 case AArch64::TBZW:
746 Cond[1].setImm(AArch64::TBNZW);
747 break;
748 case AArch64::TBNZW:
749 Cond[1].setImm(AArch64::TBZW);
750 break;
751 case AArch64::TBZX:
752 Cond[1].setImm(AArch64::TBNZX);
753 break;
754 case AArch64::TBNZX:
755 Cond[1].setImm(AArch64::TBZX);
756 break;
757
758 // Cond is { -1, Opcode, CC, Op0, Op1, ... }
759 case AArch64::CBWPri:
760 case AArch64::CBXPri:
761 case AArch64::CBBAssertExt:
762 case AArch64::CBHAssertExt:
763 case AArch64::CBWPrr:
764 case AArch64::CBXPrr: {
765 // Pseudos using standard 4bit Arm condition codes
766 AArch64CC::CondCode CC =
767 static_cast<AArch64CC::CondCode>(Cond[2].getImm());
768 Cond[2].setImm(AArch64CC::getInvertedCondCode(Code: CC));
769 }
770 }
771 }
772
773 return false;
774}
775
776unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
777 int *BytesRemoved) const {
778 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
779 if (I == MBB.end())
780 return 0;
781
782 if (!isUncondBranchOpcode(Opc: I->getOpcode()) &&
783 !isCondBranchOpcode(Opc: I->getOpcode()))
784 return 0;
785
786 // Remove the branch.
787 I->eraseFromParent();
788
789 I = MBB.end();
790
791 if (I == MBB.begin()) {
792 if (BytesRemoved)
793 *BytesRemoved = 4;
794 return 1;
795 }
796 --I;
797 if (!isCondBranchOpcode(Opc: I->getOpcode())) {
798 if (BytesRemoved)
799 *BytesRemoved = 4;
800 return 1;
801 }
802
803 // Remove the branch.
804 I->eraseFromParent();
805 if (BytesRemoved)
806 *BytesRemoved = 8;
807
808 return 2;
809}
810
811void AArch64InstrInfo::instantiateCondBranch(
812 MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
813 ArrayRef<MachineOperand> Cond) const {
814 if (Cond[0].getImm() != -1) {
815 // Regular Bcc
816 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: Cond[0].getImm()).addMBB(MBB: TBB);
817 } else {
818 // Folded compare-and-branch
819 // Note that we use addOperand instead of addReg to keep the flags.
820
821 // cbz, cbnz
822 const MachineInstrBuilder MIB =
823 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[1].getImm())).add(MO: Cond[2]);
824
825 // tbz/tbnz
826 if (Cond.size() > 3)
827 MIB.add(MO: Cond[3]);
828
829 // cb
830 if (Cond.size() > 4)
831 MIB.add(MO: Cond[4]);
832
833 MIB.addMBB(MBB: TBB);
834
835 // cb[b,h]
836 if (Cond.size() > 5) {
837 MIB.addImm(Val: Cond[5].getImm());
838 MIB.addImm(Val: Cond[6].getImm());
839 }
840 }
841}
842
843unsigned AArch64InstrInfo::insertBranch(
844 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
845 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
846 // Shouldn't be a fall through.
847 assert(TBB && "insertBranch must not be told to insert a fallthrough");
848
849 if (!FBB) {
850 if (Cond.empty()) // Unconditional branch?
851 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: TBB);
852 else
853 instantiateCondBranch(MBB, DL, TBB, Cond);
854
855 if (BytesAdded)
856 *BytesAdded = 4;
857
858 return 1;
859 }
860
861 // Two-way conditional branch.
862 instantiateCondBranch(MBB, DL, TBB, Cond);
863 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: FBB);
864
865 if (BytesAdded)
866 *BytesAdded = 8;
867
868 return 2;
869}
870
871#ifndef NDEBUG
872static bool isValidCBExtend(int64_t Opc, AArch64_AM::ShiftExtendType Ext) {
873 switch (Ext) {
874 default:
875 return false;
876 case AArch64_AM::UXTB:
877 case AArch64_AM::SXTB:
878 return Opc == AArch64::CBBAssertExt;
879 case AArch64_AM::UXTH:
880 case AArch64_AM::SXTH:
881 return Opc == AArch64::CBHAssertExt;
882 }
883}
884#endif
885
886AArch64CC::CondCode AArch64InstrInfo::insertCmpForCondBr(
887 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL,
888 ArrayRef<MachineOperand> Cond) const {
889 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
890
891 // Parse the condition code, see parseCondBranch() above.
892 AArch64CC::CondCode CC;
893 switch (Cond.size()) {
894 default:
895 llvm_unreachable("Unknown condition opcode in Cond");
896 case 1: // b.cc
897 CC = AArch64CC::CondCode(Cond[0].getImm());
898 break;
899 case 3: { // cbz/cbnz
900 // We must insert a compare against 0.
901 bool Is64Bit;
902 switch (Cond[1].getImm()) {
903 default:
904 llvm_unreachable("Unknown branch opcode in Cond");
905 case AArch64::CBZW:
906 Is64Bit = false;
907 CC = AArch64CC::EQ;
908 break;
909 case AArch64::CBZX:
910 Is64Bit = true;
911 CC = AArch64CC::EQ;
912 break;
913 case AArch64::CBNZW:
914 Is64Bit = false;
915 CC = AArch64CC::NE;
916 break;
917 case AArch64::CBNZX:
918 Is64Bit = true;
919 CC = AArch64CC::NE;
920 break;
921 }
922 Register SrcReg = Cond[2].getReg();
923 if (Is64Bit) {
924 // cmp reg, #0 is actually subs xzr, reg, #0.
925 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64spRegClass);
926 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSXri), DestReg: AArch64::XZR)
927 .addReg(RegNo: SrcReg)
928 .addImm(Val: 0)
929 .addImm(Val: 0);
930 } else {
931 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32spRegClass);
932 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWri), DestReg: AArch64::WZR)
933 .addReg(RegNo: SrcReg)
934 .addImm(Val: 0)
935 .addImm(Val: 0);
936 }
937 } break;
938 case 4: { // tbz/tbnz
939 // We must insert a tst instruction.
940 switch (Cond[1].getImm()) {
941 default:
942 llvm_unreachable("Unknown branch opcode in Cond");
943 case AArch64::TBZW:
944 case AArch64::TBZX:
945 CC = AArch64CC::EQ;
946 break;
947 case AArch64::TBNZW:
948 case AArch64::TBNZX:
949 CC = AArch64CC::NE;
950 break;
951 }
952 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
953 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
954 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::ANDSWri), DestReg: AArch64::WZR)
955 .addReg(RegNo: Cond[2].getReg())
956 .addImm(
957 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 32));
958 else
959 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::ANDSXri), DestReg: AArch64::XZR)
960 .addReg(RegNo: Cond[2].getReg())
961 .addImm(
962 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 64));
963 } break;
964 case 5: { // cb
965 // We must insert a cmp, that is a subs
966 // 0 1 2 3 4
967 // Cond is { -1, Opcode, CC, Op0, Op1 }
968 unsigned SubsOpc, SubsDestReg;
969 bool IsImm = false;
970 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
971 switch (Cond[1].getImm()) {
972 default:
973 llvm_unreachable("Unknown branch opcode in Cond");
974 case AArch64::CBWPri:
975 SubsOpc = AArch64::SUBSWri;
976 SubsDestReg = AArch64::WZR;
977 IsImm = true;
978 break;
979 case AArch64::CBXPri:
980 SubsOpc = AArch64::SUBSXri;
981 SubsDestReg = AArch64::XZR;
982 IsImm = true;
983 break;
984 case AArch64::CBWPrr:
985 SubsOpc = AArch64::SUBSWrr;
986 SubsDestReg = AArch64::WZR;
987 IsImm = false;
988 break;
989 case AArch64::CBXPrr:
990 SubsOpc = AArch64::SUBSXrr;
991 SubsDestReg = AArch64::XZR;
992 IsImm = false;
993 break;
994 }
995
996 if (IsImm) {
997 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 1));
998 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
999 .addReg(RegNo: Cond[3].getReg())
1000 .addImm(Val: Cond[4].getImm())
1001 .addImm(Val: 0);
1002 } else {
1003 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 1));
1004 MRI.constrainRegClass(Reg: Cond[4].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 2));
1005 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
1006 .addReg(RegNo: Cond[3].getReg())
1007 .addReg(RegNo: Cond[4].getReg());
1008 }
1009 } break;
1010 case 7: { // cb[b,h]
1011 // We must insert a cmp, that is a subs, but also zero- or sign-extensions
1012 // that have been folded. For the first operand we codegen an explicit
1013 // extension, for the second operand we fold the extension into cmp.
1014 // 0 1 2 3 4 5 6
1015 // Cond is { -1, Opcode, CC, Op0, Op1, Ext0, Ext1 }
1016
1017 // We need a new register for the now explicitly extended register
1018 Register Reg = Cond[3].getReg();
1019 if (Cond[5].getImm() != AArch64_AM::InvalidShiftExtend) {
1020 unsigned ExtOpc;
1021 unsigned ExtBits;
1022 AArch64_AM::ShiftExtendType ExtendType =
1023 AArch64_AM::getExtendType(Imm: Cond[5].getImm());
1024 assert(isValidCBExtend(Cond[1].getImm(), ExtendType) &&
1025 "Unexpected compare-and-branch instruction for extend type");
1026 switch (ExtendType) {
1027 default:
1028 llvm_unreachable("Unknown shift-extend for CB instruction");
1029 case AArch64_AM::SXTB:
1030 ExtOpc = AArch64::SBFMWri;
1031 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1032 break;
1033 case AArch64_AM::SXTH:
1034 ExtOpc = AArch64::SBFMWri;
1035 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1036 break;
1037 case AArch64_AM::UXTB:
1038 ExtOpc = AArch64::ANDWri;
1039 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1040 break;
1041 case AArch64_AM::UXTH:
1042 ExtOpc = AArch64::ANDWri;
1043 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1044 break;
1045 }
1046
1047 // Build the explicit extension of the first operand
1048 Reg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32commonRegClass);
1049 MachineInstrBuilder MBBI =
1050 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ExtOpc), DestReg: Reg).addReg(RegNo: Cond[3].getReg());
1051 if (ExtOpc != AArch64::ANDWri)
1052 MBBI.addImm(Val: 0);
1053 MBBI.addImm(Val: ExtBits);
1054 }
1055
1056 // Now, subs with an extended second operand
1057 if (Cond[6].getImm() != AArch64_AM::InvalidShiftExtend) {
1058 MRI.constrainRegClass(Reg, RC: &AArch64::GPR32commonRegClass);
1059 AArch64_AM::ShiftExtendType ExtendType =
1060 AArch64_AM::getExtendType(Imm: Cond[6].getImm());
1061 assert(isValidCBExtend(Cond[1].getImm(), ExtendType) &&
1062 "Unexpected compare-and-branch instruction for extend type");
1063 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrx), DestReg: AArch64::WZR)
1064 .addReg(RegNo: Reg)
1065 .addReg(RegNo: Cond[4].getReg())
1066 .addImm(Val: AArch64_AM::getArithExtendImm(ET: ExtendType, Imm: 0));
1067 } // If no extension is needed, just a regular subs
1068 else {
1069 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrr), DestReg: AArch64::WZR)
1070 .addReg(RegNo: Reg)
1071 .addReg(RegNo: Cond[4].getReg());
1072 }
1073
1074 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
1075 } break;
1076 }
1077 return CC;
1078}
1079
1080bool llvm::optimizeTerminators(MachineBasicBlock *MBB,
1081 const TargetInstrInfo &TII) {
1082 for (MachineInstr &MI : MBB->terminators()) {
1083 unsigned Opc = MI.getOpcode();
1084 switch (Opc) {
1085 case AArch64::CBZW:
1086 case AArch64::CBZX:
1087 case AArch64::TBZW:
1088 case AArch64::TBZX:
1089 // CBZ/TBZ with WZR/XZR -> unconditional B
1090 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
1091 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
1092 DEBUG_WITH_TYPE("optimizeTerminators",
1093 dbgs() << "Removing always taken branch: " << MI);
1094 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
1095 SmallVector<MachineBasicBlock *> Succs(MBB->successors());
1096 for (auto *S : Succs)
1097 if (S != Target)
1098 MBB->removeSuccessor(Succ: S);
1099 DebugLoc DL = MI.getDebugLoc();
1100 while (MBB->rbegin() != &MI)
1101 MBB->rbegin()->eraseFromParent();
1102 MI.eraseFromParent();
1103 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: AArch64::B)).addMBB(MBB: Target);
1104 return true;
1105 }
1106 break;
1107 case AArch64::CBNZW:
1108 case AArch64::CBNZX:
1109 case AArch64::TBNZW:
1110 case AArch64::TBNZX:
1111 // CBNZ/TBNZ with WZR/XZR -> never taken, remove branch and successor
1112 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
1113 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
1114 DEBUG_WITH_TYPE("optimizeTerminators",
1115 dbgs() << "Removing never taken branch: " << MI);
1116 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
1117 MI.getParent()->removeSuccessor(Succ: Target);
1118 MI.eraseFromParent();
1119 return true;
1120 }
1121 break;
1122 }
1123 }
1124 return false;
1125}
1126
1127// Find the original register that VReg is copied from.
1128static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
1129 while (Register::isVirtualRegister(Reg: VReg)) {
1130 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1131 if (!DefMI || !DefMI->isFullCopy())
1132 return VReg;
1133 VReg = DefMI->getOperand(i: 1).getReg();
1134 }
1135 return VReg;
1136}
1137
1138// Determine if VReg is defined by an instruction that can be folded into a
1139// csel instruction. If so, return the folded opcode, and the replacement
1140// register.
1141static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
1142 unsigned *NewReg = nullptr) {
1143 VReg = removeCopies(MRI, VReg);
1144 if (!Register::isVirtualRegister(Reg: VReg))
1145 return 0;
1146
1147 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(RC: MRI.getRegClass(Reg: VReg));
1148 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1149 if (!DefMI)
1150 return 0;
1151 unsigned Opc = 0;
1152 unsigned SrcReg = 0;
1153 switch (DefMI->getOpcode()) {
1154 case AArch64::SUBREG_TO_REG:
1155 // Check for the following way to define an 64-bit immediate:
1156 // %0:gpr32 = MOVi32imm 1
1157 // %1:gpr64 = SUBREG_TO_REG %0:gpr32, %subreg.sub_32
1158 if (!DefMI->getOperand(i: 1).isReg())
1159 return 0;
1160 if (!DefMI->getOperand(i: 2).isImm() ||
1161 DefMI->getOperand(i: 2).getImm() != AArch64::sub_32)
1162 return 0;
1163 DefMI = MRI.getVRegDef(Reg: DefMI->getOperand(i: 1).getReg());
1164 if (DefMI->getOpcode() != AArch64::MOVi32imm)
1165 return 0;
1166 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
1167 return 0;
1168 assert(Is64Bit);
1169 SrcReg = AArch64::XZR;
1170 Opc = AArch64::CSINCXr;
1171 break;
1172
1173 case AArch64::MOVi32imm:
1174 case AArch64::MOVi64imm:
1175 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
1176 return 0;
1177 SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1178 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
1179 break;
1180
1181 case AArch64::ADDSXri:
1182 case AArch64::ADDSWri:
1183 // if NZCV is used, do not fold.
1184 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1185 isDead: true) == -1)
1186 return 0;
1187 // fall-through to ADDXri and ADDWri.
1188 [[fallthrough]];
1189 case AArch64::ADDXri:
1190 case AArch64::ADDWri:
1191 // add x, 1 -> csinc.
1192 if (!DefMI->getOperand(i: 2).isImm() || DefMI->getOperand(i: 2).getImm() != 1 ||
1193 DefMI->getOperand(i: 3).getImm() != 0)
1194 return 0;
1195 SrcReg = DefMI->getOperand(i: 1).getReg();
1196 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
1197 break;
1198
1199 case AArch64::ORNXrr:
1200 case AArch64::ORNWrr: {
1201 // not x -> csinv, represented as orn dst, xzr, src.
1202 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1203 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1204 return 0;
1205 SrcReg = DefMI->getOperand(i: 2).getReg();
1206 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
1207 break;
1208 }
1209
1210 case AArch64::SUBSXrr:
1211 case AArch64::SUBSWrr:
1212 // if NZCV is used, do not fold.
1213 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1214 isDead: true) == -1)
1215 return 0;
1216 // fall-through to SUBXrr and SUBWrr.
1217 [[fallthrough]];
1218 case AArch64::SUBXrr:
1219 case AArch64::SUBWrr: {
1220 // neg x -> csneg, represented as sub dst, xzr, src.
1221 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1222 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1223 return 0;
1224 SrcReg = DefMI->getOperand(i: 2).getReg();
1225 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
1226 break;
1227 }
1228 default:
1229 return 0;
1230 }
1231 assert(Opc && SrcReg && "Missing parameters");
1232
1233 if (NewReg)
1234 *NewReg = SrcReg;
1235 return Opc;
1236}
1237
1238bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1239 ArrayRef<MachineOperand> Cond,
1240 Register DstReg, Register TrueReg,
1241 Register FalseReg, int &CondCycles,
1242 int &TrueCycles,
1243 int &FalseCycles) const {
1244 // Check register classes.
1245 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1246 const TargetRegisterClass *RC =
1247 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1248 if (!RC)
1249 return false;
1250
1251 // Also need to check the dest regclass, in case we're trying to optimize
1252 // something like:
1253 // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
1254 if (!RI.getCommonSubClass(A: RC, B: MRI.getRegClass(Reg: DstReg)))
1255 return false;
1256
1257 // Expanding cbz/tbz requires an extra cycle of latency on the condition.
1258 unsigned ExtraCondLat = Cond.size() != 1;
1259
1260 // GPRs are handled by csel.
1261 // FIXME: Fold in x+1, -x, and ~x when applicable.
1262 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
1263 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1264 // Single-cycle csel, csinc, csinv, and csneg.
1265 CondCycles = 1 + ExtraCondLat;
1266 TrueCycles = FalseCycles = 1;
1267 if (canFoldIntoCSel(MRI, VReg: TrueReg))
1268 TrueCycles = 0;
1269 else if (canFoldIntoCSel(MRI, VReg: FalseReg))
1270 FalseCycles = 0;
1271 return true;
1272 }
1273
1274 // Scalar floating point is handled by fcsel.
1275 // FIXME: Form fabs, fmin, and fmax when applicable.
1276 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
1277 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
1278 CondCycles = 5 + ExtraCondLat;
1279 TrueCycles = FalseCycles = 2;
1280 return true;
1281 }
1282
1283 // No single conditional move for a 128-bit vector, but we can emit a sequence
1284 // of csetm (~1), dup (~5, cross domain), bsl (~2).
1285 if (AArch64::FPR128RegClass.hasSubClassEq(RC) &&
1286 Subtarget.isNeonAvailable() &&
1287 !MBB.getParent()->getFunction().hasMinSize()) {
1288 CondCycles = 8 + ExtraCondLat;
1289 TrueCycles = FalseCycles = 2;
1290 return true;
1291 }
1292
1293 return false;
1294}
1295
1296void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
1297 MachineBasicBlock::iterator I,
1298 const DebugLoc &DL, Register DstReg,
1299 ArrayRef<MachineOperand> Cond,
1300 Register TrueReg, Register FalseReg) const {
1301
1302 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1303 AArch64CC::CondCode CC = insertCmpForCondBr(MBB, MI: I, DL, Cond);
1304
1305 // A 128-bit vector has no conditional move so blend the operands with a mask
1306 // built from the flags.
1307 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR128RegClass)) {
1308 assert(Subtarget.isNeonAvailable() && "Expected NEON for a vector select");
1309 MRI.constrainRegClass(Reg: TrueReg, RC: &AArch64::FPR128RegClass);
1310 MRI.constrainRegClass(Reg: FalseReg, RC: &AArch64::FPR128RegClass);
1311 Register CondSet = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1312 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::CSINVXr), DestReg: CondSet)
1313 .addReg(RegNo: AArch64::XZR)
1314 .addReg(RegNo: AArch64::XZR)
1315 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
1316 Register Mask = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
1317 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::DUPv2i64gpr), DestReg: Mask).addReg(RegNo: CondSet);
1318 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::BSPv16i8), DestReg: DstReg)
1319 .addReg(RegNo: Mask)
1320 .addReg(RegNo: TrueReg)
1321 .addReg(RegNo: FalseReg);
1322 return;
1323 }
1324
1325 unsigned Opc = 0;
1326 const TargetRegisterClass *RC = nullptr;
1327 bool TryFold = false;
1328 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass)) {
1329 RC = &AArch64::GPR64RegClass;
1330 Opc = AArch64::CSELXr;
1331 TryFold = true;
1332 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR32RegClass)) {
1333 RC = &AArch64::GPR32RegClass;
1334 Opc = AArch64::CSELWr;
1335 TryFold = true;
1336 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR64RegClass)) {
1337 RC = &AArch64::FPR64RegClass;
1338 Opc = AArch64::FCSELDrrr;
1339 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR32RegClass)) {
1340 RC = &AArch64::FPR32RegClass;
1341 Opc = AArch64::FCSELSrrr;
1342 }
1343 assert(RC && "Unsupported regclass");
1344
1345 // Try folding simple instructions into the csel.
1346 if (TryFold) {
1347 unsigned NewReg = 0;
1348 unsigned FoldedOpc = canFoldIntoCSel(MRI, VReg: TrueReg, NewReg: &NewReg);
1349 if (FoldedOpc) {
1350 // The folded opcodes csinc, csinc and csneg apply the operation to
1351 // FalseReg, so we need to invert the condition.
1352 CC = AArch64CC::getInvertedCondCode(Code: CC);
1353 TrueReg = FalseReg;
1354 } else
1355 FoldedOpc = canFoldIntoCSel(MRI, VReg: FalseReg, NewReg: &NewReg);
1356
1357 // Fold the operation. Leave any dead instructions for DCE to clean up.
1358 if (FoldedOpc) {
1359 FalseReg = NewReg;
1360 Opc = FoldedOpc;
1361 // Extend the live range of NewReg.
1362 MRI.clearKillFlags(Reg: NewReg);
1363 }
1364 }
1365
1366 // Pull all virtual register into the appropriate class.
1367 MRI.constrainRegClass(Reg: TrueReg, RC);
1368 // FalseReg might be WZR or XZR if the folded operand is a literal 1.
1369 assert(
1370 (FalseReg.isVirtual() || FalseReg == AArch64::WZR ||
1371 FalseReg == AArch64::XZR) &&
1372 "FalseReg was folded into a non-virtual register other than WZR or XZR");
1373 if (FalseReg.isVirtual())
1374 MRI.constrainRegClass(Reg: FalseReg, RC);
1375
1376 // Insert the csel.
1377 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
1378 .addReg(RegNo: TrueReg)
1379 .addReg(RegNo: FalseReg)
1380 .addImm(Val: CC);
1381}
1382
1383// Return true if Imm can be loaded into a register by a "cheap" sequence of
1384// instructions. For now, "cheap" means at most two instructions.
1385static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize) {
1386 if (BitSize == 32)
1387 return true;
1388
1389 assert(BitSize == 64 && "Only bit sizes of 32 or 64 allowed");
1390 uint64_t Imm = static_cast<uint64_t>(MI.getOperand(i: 1).getImm());
1391 SmallVector<AArch64_IMM::ImmInsnModel, 4> Is;
1392 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: Is);
1393
1394 return Is.size() <= 2;
1395}
1396
1397// Check if a COPY instruction is cheap.
1398static bool isCheapCopy(const MachineInstr &MI, const AArch64RegisterInfo &RI) {
1399 assert(MI.isCopy() && "Expected COPY instruction");
1400 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1401
1402 // Cross-bank copies (e.g., between GPR and FPR) are expensive on AArch64,
1403 // typically requiring an FMOV instruction with a 2-6 cycle latency.
1404 auto GetRegClass = [&](Register Reg) -> const TargetRegisterClass * {
1405 if (Reg.isVirtual())
1406 return MRI.getRegClass(Reg);
1407 if (Reg.isPhysical())
1408 return RI.getMinimalPhysRegClass(Reg);
1409 return nullptr;
1410 };
1411 const TargetRegisterClass *DstRC = GetRegClass(MI.getOperand(i: 0).getReg());
1412 const TargetRegisterClass *SrcRC = GetRegClass(MI.getOperand(i: 1).getReg());
1413 if (DstRC && SrcRC && !RI.getCommonSubClass(A: DstRC, B: SrcRC))
1414 return false;
1415
1416 return MI.isAsCheapAsAMove();
1417}
1418
1419// FIXME: this implementation should be micro-architecture dependent, so a
1420// micro-architecture target hook should be introduced here in future.
1421bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
1422 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1423 if (isExynosCheapAsMove(MI))
1424 return true;
1425 return MI.isAsCheapAsAMove();
1426 }
1427
1428 switch (MI.getOpcode()) {
1429 default:
1430 return MI.isAsCheapAsAMove();
1431
1432 case TargetOpcode::COPY:
1433 return isCheapCopy(MI, RI);
1434
1435 case AArch64::ADDWrs:
1436 case AArch64::ADDXrs:
1437 case AArch64::SUBWrs:
1438 case AArch64::SUBXrs:
1439 return Subtarget.hasALULSLFast() && MI.getOperand(i: 3).getImm() <= 4;
1440
1441 // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
1442 // ORRXri, it is as cheap as MOV.
1443 // Likewise if it can be expanded to MOVZ/MOVN/MOVK.
1444 case AArch64::MOVi32imm:
1445 return isCheapImmediate(MI, BitSize: 32);
1446 case AArch64::MOVi64imm:
1447 return isCheapImmediate(MI, BitSize: 64);
1448 }
1449}
1450
1451bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
1452 switch (MI.getOpcode()) {
1453 default:
1454 return false;
1455
1456 case AArch64::ADDWrs:
1457 case AArch64::ADDXrs:
1458 case AArch64::ADDSWrs:
1459 case AArch64::ADDSXrs: {
1460 unsigned Imm = MI.getOperand(i: 3).getImm();
1461 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1462 if (ShiftVal == 0)
1463 return true;
1464 return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
1465 }
1466
1467 case AArch64::ADDWrx:
1468 case AArch64::ADDXrx:
1469 case AArch64::ADDXrx64:
1470 case AArch64::ADDSWrx:
1471 case AArch64::ADDSXrx:
1472 case AArch64::ADDSXrx64: {
1473 unsigned Imm = MI.getOperand(i: 3).getImm();
1474 switch (AArch64_AM::getArithExtendType(Imm)) {
1475 default:
1476 return false;
1477 case AArch64_AM::UXTB:
1478 case AArch64_AM::UXTH:
1479 case AArch64_AM::UXTW:
1480 case AArch64_AM::UXTX:
1481 return AArch64_AM::getArithShiftValue(Imm) <= 4;
1482 }
1483 }
1484
1485 case AArch64::SUBWrs:
1486 case AArch64::SUBSWrs: {
1487 unsigned Imm = MI.getOperand(i: 3).getImm();
1488 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1489 return ShiftVal == 0 ||
1490 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
1491 }
1492
1493 case AArch64::SUBXrs:
1494 case AArch64::SUBSXrs: {
1495 unsigned Imm = MI.getOperand(i: 3).getImm();
1496 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1497 return ShiftVal == 0 ||
1498 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
1499 }
1500
1501 case AArch64::SUBWrx:
1502 case AArch64::SUBXrx:
1503 case AArch64::SUBXrx64:
1504 case AArch64::SUBSWrx:
1505 case AArch64::SUBSXrx:
1506 case AArch64::SUBSXrx64: {
1507 unsigned Imm = MI.getOperand(i: 3).getImm();
1508 switch (AArch64_AM::getArithExtendType(Imm)) {
1509 default:
1510 return false;
1511 case AArch64_AM::UXTB:
1512 case AArch64_AM::UXTH:
1513 case AArch64_AM::UXTW:
1514 case AArch64_AM::UXTX:
1515 return AArch64_AM::getArithShiftValue(Imm) == 0;
1516 }
1517 }
1518
1519 case AArch64::LDRBBroW:
1520 case AArch64::LDRBBroX:
1521 case AArch64::LDRBroW:
1522 case AArch64::LDRBroX:
1523 case AArch64::LDRDroW:
1524 case AArch64::LDRDroX:
1525 case AArch64::LDRHHroW:
1526 case AArch64::LDRHHroX:
1527 case AArch64::LDRHroW:
1528 case AArch64::LDRHroX:
1529 case AArch64::LDRQroW:
1530 case AArch64::LDRQroX:
1531 case AArch64::LDRSBWroW:
1532 case AArch64::LDRSBWroX:
1533 case AArch64::LDRSBXroW:
1534 case AArch64::LDRSBXroX:
1535 case AArch64::LDRSHWroW:
1536 case AArch64::LDRSHWroX:
1537 case AArch64::LDRSHXroW:
1538 case AArch64::LDRSHXroX:
1539 case AArch64::LDRSWroW:
1540 case AArch64::LDRSWroX:
1541 case AArch64::LDRSroW:
1542 case AArch64::LDRSroX:
1543 case AArch64::LDRWroW:
1544 case AArch64::LDRWroX:
1545 case AArch64::LDRXroW:
1546 case AArch64::LDRXroX:
1547 case AArch64::PRFMroW:
1548 case AArch64::PRFMroX:
1549 case AArch64::STRBBroW:
1550 case AArch64::STRBBroX:
1551 case AArch64::STRBroW:
1552 case AArch64::STRBroX:
1553 case AArch64::STRDroW:
1554 case AArch64::STRDroX:
1555 case AArch64::STRHHroW:
1556 case AArch64::STRHHroX:
1557 case AArch64::STRHroW:
1558 case AArch64::STRHroX:
1559 case AArch64::STRQroW:
1560 case AArch64::STRQroX:
1561 case AArch64::STRSroW:
1562 case AArch64::STRSroX:
1563 case AArch64::STRWroW:
1564 case AArch64::STRWroX:
1565 case AArch64::STRXroW:
1566 case AArch64::STRXroX: {
1567 unsigned IsSigned = MI.getOperand(i: 3).getImm();
1568 return !IsSigned;
1569 }
1570 }
1571}
1572
1573bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
1574 unsigned Opc = MI.getOpcode();
1575 switch (Opc) {
1576 default:
1577 return false;
1578 case AArch64::SEH_StackAlloc:
1579 case AArch64::SEH_SaveFPLR:
1580 case AArch64::SEH_SaveFPLR_X:
1581 case AArch64::SEH_SaveReg:
1582 case AArch64::SEH_SaveReg_X:
1583 case AArch64::SEH_SaveRegP:
1584 case AArch64::SEH_SaveRegP_X:
1585 case AArch64::SEH_SaveFReg:
1586 case AArch64::SEH_SaveFReg_X:
1587 case AArch64::SEH_SaveFRegP:
1588 case AArch64::SEH_SaveFRegP_X:
1589 case AArch64::SEH_SetFP:
1590 case AArch64::SEH_AddFP:
1591 case AArch64::SEH_Nop:
1592 case AArch64::SEH_PrologEnd:
1593 case AArch64::SEH_EpilogStart:
1594 case AArch64::SEH_EpilogEnd:
1595 case AArch64::SEH_PACSignLR:
1596 case AArch64::SEH_SaveAnyRegI:
1597 case AArch64::SEH_SaveAnyRegIP:
1598 case AArch64::SEH_SaveAnyRegQP:
1599 case AArch64::SEH_SaveAnyRegQPX:
1600 case AArch64::SEH_AllocZ:
1601 case AArch64::SEH_SaveZReg:
1602 case AArch64::SEH_SavePReg:
1603 return true;
1604 }
1605}
1606
1607bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1608 Register &SrcReg, Register &DstReg,
1609 unsigned &SubIdx) const {
1610 switch (MI.getOpcode()) {
1611 default:
1612 return false;
1613 case AArch64::SBFMXri: // aka sxtw
1614 case AArch64::UBFMXri: // aka uxtw
1615 // Check for the 32 -> 64 bit extension case, these instructions can do
1616 // much more.
1617 if (MI.getOperand(i: 2).getImm() != 0 || MI.getOperand(i: 3).getImm() != 31)
1618 return false;
1619 // This is a signed or unsigned 32 -> 64 bit extension.
1620 SrcReg = MI.getOperand(i: 1).getReg();
1621 DstReg = MI.getOperand(i: 0).getReg();
1622 SubIdx = AArch64::sub_32;
1623 return true;
1624 }
1625}
1626
1627bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
1628 const MachineInstr &MIa, const MachineInstr &MIb) const {
1629 const TargetRegisterInfo *TRI = &getRegisterInfo();
1630 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1631 int64_t OffsetA = 0, OffsetB = 0;
1632 TypeSize WidthA(0, false), WidthB(0, false);
1633 bool OffsetAIsScalable = false, OffsetBIsScalable = false;
1634
1635 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1636 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1637
1638 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
1639 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1640 return false;
1641
1642 // Retrieve the base, offset from the base and width. Width
1643 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
1644 // base are identical, and the offset of a lower memory access +
1645 // the width doesn't overlap the offset of a higher memory access,
1646 // then the memory accesses are different.
1647 // If OffsetAIsScalable and OffsetBIsScalable are both true, they
1648 // are assumed to have the same scale (vscale).
1649 if (getMemOperandWithOffsetWidth(MI: MIa, BaseOp&: BaseOpA, Offset&: OffsetA, OffsetIsScalable&: OffsetAIsScalable,
1650 Width&: WidthA, TRI) &&
1651 getMemOperandWithOffsetWidth(MI: MIb, BaseOp&: BaseOpB, Offset&: OffsetB, OffsetIsScalable&: OffsetBIsScalable,
1652 Width&: WidthB, TRI)) {
1653 if (BaseOpA->isIdenticalTo(Other: *BaseOpB) &&
1654 OffsetAIsScalable == OffsetBIsScalable) {
1655 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1656 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1657 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1658 if (LowWidth.isScalable() == OffsetAIsScalable &&
1659 LowOffset + (int)LowWidth.getKnownMinValue() <= HighOffset)
1660 return true;
1661 }
1662 }
1663 return false;
1664}
1665
1666bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1667 const MachineBasicBlock *MBB,
1668 const MachineFunction &MF) const {
1669 if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
1670 return true;
1671
1672 // Do not move an instruction that can be recognized as a branch target.
1673 if (hasBTISemantics(MI))
1674 return true;
1675
1676 switch (MI.getOpcode()) {
1677 case AArch64::HINT:
1678 // CSDB hints are scheduling barriers.
1679 if (MI.getOperand(i: 0).getImm() == 0x14)
1680 return true;
1681 break;
1682 case AArch64::DSB:
1683 case AArch64::ISB:
1684 // DSB and ISB also are scheduling barriers.
1685 return true;
1686 case AArch64::MSRpstatesvcrImm1:
1687 // SMSTART and SMSTOP are also scheduling barriers.
1688 return true;
1689 default:;
1690 }
1691 if (isSEHInstruction(MI))
1692 return true;
1693 auto Next = std::next(x: MI.getIterator());
1694 return Next != MBB->end() && Next->isCFIInstruction();
1695}
1696
1697/// analyzeCompare - For a comparison instruction, return the source registers
1698/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1699/// Return true if the comparison instruction can be analyzed.
1700bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1701 Register &SrcReg2, int64_t &CmpMask,
1702 int64_t &CmpValue) const {
1703 // The first operand can be a frame index where we'd normally expect a
1704 // register.
1705 // FIXME: Pass subregisters out of analyzeCompare
1706 assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1707 if (!MI.getOperand(i: 1).isReg() || MI.getOperand(i: 1).getSubReg())
1708 return false;
1709
1710 switch (MI.getOpcode()) {
1711 default:
1712 break;
1713 case AArch64::PTEST_PP:
1714 case AArch64::PTEST_PP_ANY:
1715 case AArch64::PTEST_PP_FIRST:
1716 SrcReg = MI.getOperand(i: 0).getReg();
1717 SrcReg2 = MI.getOperand(i: 1).getReg();
1718 if (MI.getOperand(i: 2).getSubReg())
1719 return false;
1720
1721 // Not sure about the mask and value for now...
1722 CmpMask = ~0;
1723 CmpValue = 0;
1724 return true;
1725 case AArch64::SUBSWrr:
1726 case AArch64::SUBSWrs:
1727 case AArch64::SUBSWrx:
1728 case AArch64::SUBSXrr:
1729 case AArch64::SUBSXrs:
1730 case AArch64::SUBSXrx:
1731 case AArch64::ADDSWrr:
1732 case AArch64::ADDSWrs:
1733 case AArch64::ADDSWrx:
1734 case AArch64::ADDSXrr:
1735 case AArch64::ADDSXrs:
1736 case AArch64::ADDSXrx:
1737 // Replace SUBSWrr with SUBWrr if NZCV is not used.
1738 SrcReg = MI.getOperand(i: 1).getReg();
1739 SrcReg2 = MI.getOperand(i: 2).getReg();
1740
1741 // FIXME: Pass subregisters out of analyzeCompare
1742 if (MI.getOperand(i: 2).getSubReg())
1743 return false;
1744
1745 CmpMask = ~0;
1746 CmpValue = 0;
1747 return true;
1748 case AArch64::SUBSWri:
1749 case AArch64::ADDSWri:
1750 case AArch64::SUBSXri:
1751 case AArch64::ADDSXri:
1752 SrcReg = MI.getOperand(i: 1).getReg();
1753 SrcReg2 = 0;
1754 CmpMask = ~0;
1755 CmpValue = MI.getOperand(i: 2).getImm();
1756 return true;
1757 case AArch64::ANDSWri:
1758 case AArch64::ANDSXri:
1759 // ANDS does not use the same encoding scheme as the others xxxS
1760 // instructions.
1761 SrcReg = MI.getOperand(i: 1).getReg();
1762 SrcReg2 = 0;
1763 CmpMask = ~0;
1764 CmpValue = AArch64_AM::decodeLogicalImmediate(
1765 val: MI.getOperand(i: 2).getImm(),
1766 regSize: MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1767 return true;
1768 }
1769
1770 return false;
1771}
1772
1773static bool UpdateOperandRegClass(MachineInstr &Instr) {
1774 MachineBasicBlock *MBB = Instr.getParent();
1775 assert(MBB && "Can't get MachineBasicBlock here");
1776 MachineFunction *MF = MBB->getParent();
1777 assert(MF && "Can't get MachineFunction here");
1778 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1779 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1780 MachineRegisterInfo *MRI = &MF->getRegInfo();
1781
1782 for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1783 ++OpIdx) {
1784 MachineOperand &MO = Instr.getOperand(i: OpIdx);
1785 const TargetRegisterClass *OpRegCstraints =
1786 Instr.getRegClassConstraint(OpIdx, TII, TRI);
1787
1788 // If there's no constraint, there's nothing to do.
1789 if (!OpRegCstraints)
1790 continue;
1791 // If the operand is a frame index, there's nothing to do here.
1792 // A frame index operand will resolve correctly during PEI.
1793 if (MO.isFI())
1794 continue;
1795
1796 assert(MO.isReg() &&
1797 "Operand has register constraints without being a register!");
1798
1799 Register Reg = MO.getReg();
1800 if (Reg.isPhysical()) {
1801 if (!OpRegCstraints->contains(Reg))
1802 return false;
1803 } else if (!OpRegCstraints->hasSubClassEq(RC: MRI->getRegClass(Reg)) &&
1804 !MRI->constrainRegClass(Reg, RC: OpRegCstraints))
1805 return false;
1806 }
1807
1808 return true;
1809}
1810
1811/// Return the opcode that does not set flags when possible - otherwise
1812/// return the original opcode. The caller is responsible to do the actual
1813/// substitution and legality checking.
1814static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1815 // Don't convert all compare instructions, because for some the zero register
1816 // encoding becomes the sp register.
1817 bool MIDefinesZeroReg = false;
1818 if (MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
1819 MI.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
1820 MIDefinesZeroReg = true;
1821
1822 switch (MI.getOpcode()) {
1823 default:
1824 return MI.getOpcode();
1825 case AArch64::ADDSWrr:
1826 return AArch64::ADDWrr;
1827 case AArch64::ADDSWri:
1828 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1829 case AArch64::ADDSWrs:
1830 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1831 case AArch64::ADDSWrx:
1832 return AArch64::ADDWrx;
1833 case AArch64::ADDSXrr:
1834 return AArch64::ADDXrr;
1835 case AArch64::ADDSXri:
1836 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1837 case AArch64::ADDSXrs:
1838 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1839 case AArch64::ADDSXrx:
1840 return AArch64::ADDXrx;
1841 case AArch64::SUBSWrr:
1842 return AArch64::SUBWrr;
1843 case AArch64::SUBSWri:
1844 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1845 case AArch64::SUBSWrs:
1846 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1847 case AArch64::SUBSWrx:
1848 return AArch64::SUBWrx;
1849 case AArch64::SUBSXrr:
1850 return AArch64::SUBXrr;
1851 case AArch64::SUBSXri:
1852 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1853 case AArch64::SUBSXrs:
1854 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1855 case AArch64::SUBSXrx:
1856 return AArch64::SUBXrx;
1857 }
1858}
1859
1860enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1861
1862/// True when condition flags are accessed (either by writing or reading)
1863/// on the instruction trace starting at From and ending at To.
1864///
1865/// Note: If From and To are from different blocks it's assumed CC are accessed
1866/// on the path.
1867static bool areCFlagsAccessedBetweenInstrs(
1868 MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1869 const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1870 // Early exit if To is at the beginning of the BB.
1871 if (To == To->getParent()->begin())
1872 return true;
1873
1874 // Check whether the instructions are in the same basic block
1875 // If not, assume the condition flags might get modified somewhere.
1876 if (To->getParent() != From->getParent())
1877 return true;
1878
1879 // From must be above To.
1880 assert(std::any_of(
1881 ++To.getReverse(), To->getParent()->rend(),
1882 [From](MachineInstr &MI) { return MI.getIterator() == From; }));
1883
1884 // We iterate backward starting at \p To until we hit \p From.
1885 for (const MachineInstr &Instr :
1886 instructionsWithoutDebug(It: ++To.getReverse(), End: From.getReverse())) {
1887 if (((AccessToCheck & AK_Write) &&
1888 Instr.modifiesRegister(Reg: AArch64::NZCV, TRI)) ||
1889 ((AccessToCheck & AK_Read) && Instr.readsRegister(Reg: AArch64::NZCV, TRI)))
1890 return true;
1891 }
1892 return false;
1893}
1894
1895std::optional<unsigned>
1896AArch64InstrInfo::canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
1897 MachineInstr *Pred,
1898 const MachineRegisterInfo *MRI) const {
1899 unsigned MaskOpcode = Mask->getOpcode();
1900 unsigned PredOpcode = Pred->getOpcode();
1901 bool PredIsPTestLike = isPTestLikeOpcode(Opc: PredOpcode);
1902 bool PredIsWhileLike = isWhileOpcode(Opc: PredOpcode);
1903
1904 if (PredIsWhileLike) {
1905 // For PTEST(PG, PG), PTEST is redundant when PG is the result of a WHILEcc
1906 // instruction and the condition is "any" since WHILcc does an implicit
1907 // PTEST(ALL, PG) check and PG is always a subset of ALL.
1908 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1909 return PredOpcode;
1910
1911 // For PTEST(PTRUE_ALL, WHILE), if the element size matches, the PTEST is
1912 // redundant since WHILE performs an implicit PTEST with an all active
1913 // mask.
1914 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1915 getElementSizeForOpcode(Opc: MaskOpcode) ==
1916 getElementSizeForOpcode(Opc: PredOpcode))
1917 return PredOpcode;
1918
1919 // For PTEST_FIRST(PTRUE_ALL, WHILE), the PTEST_FIRST is redundant since
1920 // WHILEcc performs an implicit PTEST with an all active mask, setting
1921 // the N flag as the PTEST_FIRST would.
1922 if (PTest->getOpcode() == AArch64::PTEST_PP_FIRST &&
1923 isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31)
1924 return PredOpcode;
1925
1926 return {};
1927 }
1928
1929 if (PredIsPTestLike) {
1930 // For PTEST(PG, PG), PTEST is redundant when PG is the result of an
1931 // instruction that sets the flags as PTEST would and the condition is
1932 // "any" since PG is always a subset of the governing predicate of the
1933 // ptest-like instruction.
1934 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1935 return PredOpcode;
1936
1937 auto PTestLikeMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1938
1939 // If the PTEST like instruction's general predicate is not `Mask`, attempt
1940 // to look through a copy and try again. This is because some instructions
1941 // take a predicate whose register class is a subset of its result class.
1942 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1943 PTestLikeMask->getOperand(i: 1).getReg().isVirtual())
1944 PTestLikeMask =
1945 MRI->getUniqueVRegDef(Reg: PTestLikeMask->getOperand(i: 1).getReg());
1946
1947 // For PTEST(PTRUE_ALL, PTEST_LIKE), the PTEST is redundant if the
1948 // the element size matches and either the PTEST_LIKE instruction uses
1949 // the same all active mask or the condition is "any".
1950 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1951 getElementSizeForOpcode(Opc: MaskOpcode) ==
1952 getElementSizeForOpcode(Opc: PredOpcode)) {
1953 if (Mask == PTestLikeMask || PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1954 return PredOpcode;
1955 }
1956
1957 // For PTEST(PG, PTEST_LIKE(PG, ...)), the PTEST is redundant since the
1958 // flags are set based on the same mask 'PG', but PTEST_LIKE must operate
1959 // on 8-bit predicates like the PTEST. Otherwise, for instructions like
1960 // compare that also support 16/32/64-bit predicates, the implicit PTEST
1961 // performed by the compare could consider fewer lanes for these element
1962 // sizes.
1963 //
1964 // For example, consider
1965 //
1966 // ptrue p0.b ; P0=1111-1111-1111-1111
1967 // index z0.s, #0, #1 ; Z0=<0,1,2,3>
1968 // index z1.s, #1, #1 ; Z1=<1,2,3,4>
1969 // cmphi p1.s, p0/z, z1.s, z0.s ; P1=0001-0001-0001-0001
1970 // ; ^ last active
1971 // ptest p0, p1.b ; P1=0001-0001-0001-0001
1972 // ; ^ last active
1973 //
1974 // where the compare generates a canonical all active 32-bit predicate
1975 // (equivalent to 'ptrue p1.s, all'). The implicit PTEST sets the last
1976 // active flag, whereas the PTEST instruction with the same mask doesn't.
1977 // For PTEST_ANY this doesn't apply as the flags in this case would be
1978 // identical regardless of element size.
1979 uint64_t PredElementSize = getElementSizeForOpcode(Opc: PredOpcode);
1980 if (Mask == PTestLikeMask && (PredElementSize == AArch64::ElementSizeB ||
1981 PTest->getOpcode() == AArch64::PTEST_PP_ANY))
1982 return PredOpcode;
1983
1984 return {};
1985 }
1986
1987 // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the
1988 // opcode so the PTEST becomes redundant.
1989 switch (PredOpcode) {
1990 case AArch64::AND_PPzPP:
1991 case AArch64::BIC_PPzPP:
1992 case AArch64::EOR_PPzPP:
1993 case AArch64::NAND_PPzPP:
1994 case AArch64::NOR_PPzPP:
1995 case AArch64::ORN_PPzPP:
1996 case AArch64::ORR_PPzPP:
1997 case AArch64::BRKA_PPzP:
1998 case AArch64::BRKPA_PPzPP:
1999 case AArch64::BRKB_PPzP:
2000 case AArch64::BRKPB_PPzPP:
2001 case AArch64::RDFFR_PPz: {
2002 // Check to see if our mask is the same. If not the resulting flag bits
2003 // may be different and we can't remove the ptest.
2004 auto *PredMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
2005 if (Mask != PredMask)
2006 return {};
2007 break;
2008 }
2009 case AArch64::BRKN_PPzP: {
2010 // BRKN uses an all active implicit mask to set flags unlike the other
2011 // flag-setting instructions.
2012 // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B).
2013 if ((MaskOpcode != AArch64::PTRUE_B) ||
2014 (Mask->getOperand(i: 1).getImm() != 31))
2015 return {};
2016 break;
2017 }
2018 case AArch64::PTRUE_B:
2019 // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A)
2020 break;
2021 default:
2022 // Bail out if we don't recognize the input
2023 return {};
2024 }
2025
2026 return convertToFlagSettingOpc(Opc: PredOpcode);
2027}
2028
2029/// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
2030/// operation which could set the flags in an identical manner
2031bool AArch64InstrInfo::optimizePTestInstr(
2032 MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
2033 const MachineRegisterInfo *MRI) const {
2034 auto *Mask = MRI->getUniqueVRegDef(Reg: MaskReg);
2035 auto *Pred = MRI->getUniqueVRegDef(Reg: PredReg);
2036
2037 if (Pred->isCopy() && PTest->getOpcode() == AArch64::PTEST_PP_FIRST) {
2038 // Instructions which return a multi-vector (e.g. WHILECC_x2) require copies
2039 // before the branch to extract each subregister.
2040 auto Op = Pred->getOperand(i: 1);
2041 if (Op.isReg() && Op.getReg().isVirtual() &&
2042 Op.getSubReg() == AArch64::psub0)
2043 Pred = MRI->getUniqueVRegDef(Reg: Op.getReg());
2044 }
2045
2046 unsigned PredOpcode = Pred->getOpcode();
2047 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred, MRI);
2048 if (!NewOp)
2049 return false;
2050
2051 const TargetRegisterInfo *TRI = &getRegisterInfo();
2052
2053 // If another instruction between Pred and PTest accesses flags, don't remove
2054 // the ptest or update the earlier instruction to modify them.
2055 if (areCFlagsAccessedBetweenInstrs(From: Pred, To: PTest, TRI))
2056 return false;
2057
2058 // If we pass all the checks, it's safe to remove the PTEST and use the flags
2059 // as they are prior to PTEST. Sometimes this requires the tested PTEST
2060 // operand to be replaced with an equivalent instruction that also sets the
2061 // flags.
2062 PTest->eraseFromParent();
2063 if (*NewOp != PredOpcode) {
2064 Pred->setDesc(get(Opcode: *NewOp));
2065 bool succeeded = UpdateOperandRegClass(Instr&: *Pred);
2066 (void)succeeded;
2067 assert(succeeded && "Operands have incompatible register classes!");
2068 Pred->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: TRI);
2069 }
2070
2071 // Ensure that the flags def is live.
2072 if (Pred->registerDefIsDead(Reg: AArch64::NZCV, TRI)) {
2073 unsigned i = 0, e = Pred->getNumOperands();
2074 for (; i != e; ++i) {
2075 MachineOperand &MO = Pred->getOperand(i);
2076 if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::NZCV) {
2077 MO.setIsDead(false);
2078 break;
2079 }
2080 }
2081 }
2082 return true;
2083}
2084
2085/// Try to optimize a compare instruction. A compare instruction is an
2086/// instruction which produces AArch64::NZCV. It can be truly compare
2087/// instruction
2088/// when there are no uses of its destination register.
2089///
2090/// The following steps are tried in order:
2091/// 1. Convert CmpInstr into an unconditional version.
2092/// 2. Remove CmpInstr if above there is an instruction producing a needed
2093/// condition code or an instruction which can be converted into such an
2094/// instruction.
2095/// Only comparison with zero is supported.
2096bool AArch64InstrInfo::optimizeCompareInstr(
2097 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2098 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2099 assert(CmpInstr.getParent());
2100 assert(MRI);
2101
2102 // Replace SUBSWrr with SUBWrr if NZCV is not used.
2103 int DeadNZCVIdx =
2104 CmpInstr.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
2105 if (DeadNZCVIdx != -1) {
2106 if (CmpInstr.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
2107 CmpInstr.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr)) {
2108 CmpInstr.eraseFromParent();
2109 return true;
2110 }
2111 unsigned Opc = CmpInstr.getOpcode();
2112 unsigned NewOpc = convertToNonFlagSettingOpc(MI: CmpInstr);
2113 if (NewOpc == Opc)
2114 return false;
2115 const MCInstrDesc &MCID = get(Opcode: NewOpc);
2116 CmpInstr.setDesc(MCID);
2117 CmpInstr.removeOperand(OpNo: DeadNZCVIdx);
2118 bool succeeded = UpdateOperandRegClass(Instr&: CmpInstr);
2119 (void)succeeded;
2120 assert(succeeded && "Some operands reg class are incompatible!");
2121 return true;
2122 }
2123
2124 if (CmpInstr.getOpcode() == AArch64::PTEST_PP ||
2125 CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY ||
2126 CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST)
2127 return optimizePTestInstr(PTest: &CmpInstr, MaskReg: SrcReg, PredReg: SrcReg2, MRI);
2128
2129 if (SrcReg2 != 0)
2130 return false;
2131
2132 // CmpInstr is a Compare instruction if destination register is not used.
2133 if (!MRI->use_nodbg_empty(RegNo: CmpInstr.getOperand(i: 0).getReg()))
2134 return false;
2135
2136 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, MRI: *MRI))
2137 return true;
2138 return (CmpValue == 0 || CmpValue == 1) &&
2139 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, MRI: *MRI);
2140}
2141
2142/// Get opcode of S version of Instr.
2143/// If Instr is S version its opcode is returned.
2144/// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
2145/// or we are not interested in it.
2146static unsigned sForm(MachineInstr &Instr) {
2147 switch (Instr.getOpcode()) {
2148 default:
2149 return AArch64::INSTRUCTION_LIST_END;
2150
2151 case AArch64::ADDSWrr:
2152 case AArch64::ADDSWri:
2153 case AArch64::ADDSXrr:
2154 case AArch64::ADDSXri:
2155 case AArch64::ADDSWrx:
2156 case AArch64::ADDSXrx:
2157 case AArch64::ADDSWrs:
2158 case AArch64::ADDSXrs:
2159 case AArch64::SUBSWrr:
2160 case AArch64::SUBSWri:
2161 case AArch64::SUBSWrx:
2162 case AArch64::SUBSWrs:
2163 case AArch64::SUBSXrr:
2164 case AArch64::SUBSXri:
2165 case AArch64::SUBSXrx:
2166 case AArch64::SUBSXrs:
2167 case AArch64::ANDSWri:
2168 case AArch64::ANDSWrr:
2169 case AArch64::ANDSWrs:
2170 case AArch64::ANDSXri:
2171 case AArch64::ANDSXrr:
2172 case AArch64::ANDSXrs:
2173 case AArch64::BICSWrr:
2174 case AArch64::BICSXrr:
2175 case AArch64::BICSWrs:
2176 case AArch64::BICSXrs:
2177 case AArch64::ADCSWr:
2178 case AArch64::ADCSXr:
2179 case AArch64::SBCSWr:
2180 case AArch64::SBCSXr:
2181 return Instr.getOpcode();
2182
2183 case AArch64::ADDWrr:
2184 return AArch64::ADDSWrr;
2185 case AArch64::ADDWri:
2186 return AArch64::ADDSWri;
2187 case AArch64::ADDXrr:
2188 return AArch64::ADDSXrr;
2189 case AArch64::ADDXri:
2190 return AArch64::ADDSXri;
2191 case AArch64::ADDWrx:
2192 return AArch64::ADDSWrx;
2193 case AArch64::ADDXrx:
2194 return AArch64::ADDSXrx;
2195 case AArch64::ADDWrs:
2196 return AArch64::ADDSWrs;
2197 case AArch64::ADDXrs:
2198 return AArch64::ADDSXrs;
2199 case AArch64::ADCWr:
2200 return AArch64::ADCSWr;
2201 case AArch64::ADCXr:
2202 return AArch64::ADCSXr;
2203 case AArch64::SUBWrr:
2204 return AArch64::SUBSWrr;
2205 case AArch64::SUBWri:
2206 return AArch64::SUBSWri;
2207 case AArch64::SUBXrr:
2208 return AArch64::SUBSXrr;
2209 case AArch64::SUBXri:
2210 return AArch64::SUBSXri;
2211 case AArch64::SUBWrx:
2212 return AArch64::SUBSWrx;
2213 case AArch64::SUBXrx:
2214 return AArch64::SUBSXrx;
2215 case AArch64::SUBWrs:
2216 return AArch64::SUBSWrs;
2217 case AArch64::SUBXrs:
2218 return AArch64::SUBSXrs;
2219 case AArch64::SBCWr:
2220 return AArch64::SBCSWr;
2221 case AArch64::SBCXr:
2222 return AArch64::SBCSXr;
2223 case AArch64::ANDWri:
2224 return AArch64::ANDSWri;
2225 case AArch64::ANDXri:
2226 return AArch64::ANDSXri;
2227 case AArch64::ANDWrr:
2228 return AArch64::ANDSWrr;
2229 case AArch64::ANDWrs:
2230 return AArch64::ANDSWrs;
2231 case AArch64::ANDXrr:
2232 return AArch64::ANDSXrr;
2233 case AArch64::ANDXrs:
2234 return AArch64::ANDSXrs;
2235 case AArch64::BICWrr:
2236 return AArch64::BICSWrr;
2237 case AArch64::BICXrr:
2238 return AArch64::BICSXrr;
2239 case AArch64::BICWrs:
2240 return AArch64::BICSWrs;
2241 case AArch64::BICXrs:
2242 return AArch64::BICSXrs;
2243 }
2244}
2245
2246/// Check if AArch64::NZCV should be alive in successors of MBB.
2247static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB) {
2248 for (auto *BB : MBB->successors())
2249 if (BB->isLiveIn(Reg: AArch64::NZCV))
2250 return true;
2251 return false;
2252}
2253
2254/// \returns The condition code operand index for \p Instr if it is a branch
2255/// or select and -1 otherwise.
2256int AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(
2257 const MachineInstr &Instr) {
2258 switch (Instr.getOpcode()) {
2259 default:
2260 return -1;
2261
2262 case AArch64::Bcc: {
2263 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2264 assert(Idx >= 2);
2265 return Idx - 2;
2266 }
2267
2268 case AArch64::CSINVWr:
2269 case AArch64::CSINVXr:
2270 case AArch64::CSINCWr:
2271 case AArch64::CSINCXr:
2272 case AArch64::CSELWr:
2273 case AArch64::CSELXr:
2274 case AArch64::CSNEGWr:
2275 case AArch64::CSNEGXr:
2276 case AArch64::FCSELSrrr:
2277 case AArch64::FCSELDrrr: {
2278 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2279 assert(Idx >= 1);
2280 return Idx - 1;
2281 }
2282 }
2283}
2284
2285/// Find a condition code used by the instruction.
2286/// Returns AArch64CC::Invalid if either the instruction does not use condition
2287/// codes or we don't optimize CmpInstr in the presence of such instructions.
2288static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
2289 int CCIdx =
2290 AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(Instr);
2291 return CCIdx >= 0 ? static_cast<AArch64CC::CondCode>(
2292 Instr.getOperand(i: CCIdx).getImm())
2293 : AArch64CC::Invalid;
2294}
2295
2296static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
2297 assert(CC != AArch64CC::Invalid);
2298 UsedNZCV UsedFlags;
2299 switch (CC) {
2300 default:
2301 break;
2302
2303 case AArch64CC::EQ: // Z set
2304 case AArch64CC::NE: // Z clear
2305 UsedFlags.Z = true;
2306 break;
2307
2308 case AArch64CC::HI: // Z clear and C set
2309 case AArch64CC::LS: // Z set or C clear
2310 UsedFlags.Z = true;
2311 [[fallthrough]];
2312 case AArch64CC::HS: // C set
2313 case AArch64CC::LO: // C clear
2314 UsedFlags.C = true;
2315 break;
2316
2317 case AArch64CC::MI: // N set
2318 case AArch64CC::PL: // N clear
2319 UsedFlags.N = true;
2320 break;
2321
2322 case AArch64CC::VS: // V set
2323 case AArch64CC::VC: // V clear
2324 UsedFlags.V = true;
2325 break;
2326
2327 case AArch64CC::GT: // Z clear, N and V the same
2328 case AArch64CC::LE: // Z set, N and V differ
2329 UsedFlags.Z = true;
2330 [[fallthrough]];
2331 case AArch64CC::GE: // N and V the same
2332 case AArch64CC::LT: // N and V differ
2333 UsedFlags.N = true;
2334 UsedFlags.V = true;
2335 break;
2336 }
2337 return UsedFlags;
2338}
2339
2340/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
2341/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
2342/// \returns std::nullopt otherwise.
2343///
2344/// Collect instructions using that flags in \p CCUseInstrs if provided.
2345std::optional<UsedNZCV>
2346llvm::examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
2347 const TargetRegisterInfo &TRI,
2348 SmallVectorImpl<MachineInstr *> *CCUseInstrs) {
2349 MachineBasicBlock *CmpParent = CmpInstr.getParent();
2350 if (MI.getParent() != CmpParent)
2351 return std::nullopt;
2352
2353 if (areCFlagsAliveInSuccessors(MBB: CmpParent))
2354 return std::nullopt;
2355
2356 UsedNZCV NZCVUsedAfterCmp;
2357 for (MachineInstr &Instr : instructionsWithoutDebug(
2358 It: std::next(x: CmpInstr.getIterator()), End: CmpParent->instr_end())) {
2359 if (Instr.readsRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
2360 AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
2361 if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
2362 return std::nullopt;
2363 NZCVUsedAfterCmp |= getUsedNZCV(CC);
2364 if (CCUseInstrs)
2365 CCUseInstrs->push_back(Elt: &Instr);
2366 }
2367 if (Instr.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI))
2368 break;
2369 }
2370 return NZCVUsedAfterCmp;
2371}
2372
2373static bool isADDSRegImm(unsigned Opcode) {
2374 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
2375}
2376
2377static bool isSUBSRegImm(unsigned Opcode) {
2378 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
2379}
2380
2381static bool isANDOpcode(MachineInstr &MI) {
2382 unsigned Opc = sForm(Instr&: MI);
2383 switch (Opc) {
2384 case AArch64::ANDSWri:
2385 case AArch64::ANDSWrr:
2386 case AArch64::ANDSWrs:
2387 case AArch64::ANDSXri:
2388 case AArch64::ANDSXrr:
2389 case AArch64::ANDSXrs:
2390 case AArch64::BICSWrr:
2391 case AArch64::BICSXrr:
2392 case AArch64::BICSWrs:
2393 case AArch64::BICSXrs:
2394 return true;
2395 default:
2396 return false;
2397 }
2398}
2399
2400/// Check if CmpInstr can be substituted by MI.
2401///
2402/// CmpInstr can be substituted:
2403/// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2404/// - and, MI and CmpInstr are from the same MachineBB
2405/// - and, condition flags are not alive in successors of the CmpInstr parent
2406/// - and, if MI opcode is the S form there must be no defs of flags between
2407/// MI and CmpInstr
2408/// or if MI opcode is not the S form there must be neither defs of flags
2409/// nor uses of flags between MI and CmpInstr.
2410/// - and, C is not used after CmpInstr; CmpInstr's C is from adds/subs #0 on
2411/// SrcReg and can differ from MI (e.g. carry out of ADCS/SBCS).
2412/// - and, V is not used after CmpInstr unless MI is AND/BIC (V cleared) or MI
2413/// has NoSWrap (overflow is poison and the fold is still safe).
2414static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr,
2415 const TargetRegisterInfo &TRI) {
2416 // MI is an opcode sForm maps (add/sub/adc/sbc/and/bic and their S forms).
2417 assert(sForm(MI) != AArch64::INSTRUCTION_LIST_END);
2418
2419 const unsigned CmpOpcode = CmpInstr.getOpcode();
2420 if (!isADDSRegImm(Opcode: CmpOpcode) && !isSUBSRegImm(Opcode: CmpOpcode))
2421 return false;
2422
2423 assert((CmpInstr.getOperand(2).isImm() &&
2424 CmpInstr.getOperand(2).getImm() == 0) &&
2425 "Caller guarantees that CmpInstr compares with constant 0");
2426
2427 std::optional<UsedNZCV> NZVCUsed = examineCFlagsUse(MI, CmpInstr, TRI);
2428 if (!NZVCUsed || NZVCUsed->C)
2429 return false;
2430
2431 // CmpInstr is ADDS/SUBS with immediate 0 on SrcReg (compare SrcReg to zero).
2432 // After the fold, users see NZCV from MI (or its S form), not from CmpInstr.
2433 // N/Z match CmpInstr for the value in SrcReg; C/V need not match in general
2434 // (e.g. ADCS vs adds #0), so we require C unused after CmpInstr and gate V
2435 // as below. NoSWrap makes signed overflow poison; AND/BIC clear V.
2436 if (NZVCUsed->V && !MI.getFlag(Flag: MachineInstr::NoSWrap) && !isANDOpcode(MI))
2437 return false;
2438
2439 AccessKind AccessToCheck = AK_Write;
2440 if (sForm(Instr&: MI) != MI.getOpcode())
2441 AccessToCheck = AK_All;
2442 return !areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck);
2443}
2444
2445/// Substitute an instruction comparing to zero with another instruction
2446/// which produces needed condition flags.
2447///
2448/// Return true on success.
2449bool AArch64InstrInfo::substituteCmpToZero(
2450 MachineInstr &CmpInstr, unsigned SrcReg,
2451 const MachineRegisterInfo &MRI) const {
2452 // Get the unique definition of SrcReg.
2453 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2454 if (!MI)
2455 return false;
2456
2457 const TargetRegisterInfo &TRI = getRegisterInfo();
2458
2459 unsigned NewOpc = sForm(Instr&: *MI);
2460 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
2461 return false;
2462
2463 if (!canInstrSubstituteCmpInstr(MI&: *MI, CmpInstr, TRI))
2464 return false;
2465
2466 // Update the instruction to set NZCV.
2467 MI->setDesc(get(Opcode: NewOpc));
2468 CmpInstr.eraseFromParent();
2469 bool succeeded = UpdateOperandRegClass(Instr&: *MI);
2470 (void)succeeded;
2471 assert(succeeded && "Some operands reg class are incompatible!");
2472 MI->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: &TRI);
2473 return true;
2474}
2475
2476/// \returns True if \p CmpInstr can be removed.
2477///
2478/// \p IsInvertCC is true if, after removing \p CmpInstr, condition
2479/// codes used in \p CCUseInstrs must be inverted.
2480static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr,
2481 int CmpValue, const TargetRegisterInfo &TRI,
2482 SmallVectorImpl<MachineInstr *> &CCUseInstrs,
2483 bool &IsInvertCC) {
2484 assert((CmpValue == 0 || CmpValue == 1) &&
2485 "Only comparisons to 0 or 1 considered for removal!");
2486
2487 // MI is 'CSINCWr %vreg, wzr, wzr, <cc>' or 'CSINCXr %vreg, xzr, xzr, <cc>'
2488 unsigned MIOpc = MI.getOpcode();
2489 if (MIOpc == AArch64::CSINCWr) {
2490 if (MI.getOperand(i: 1).getReg() != AArch64::WZR ||
2491 MI.getOperand(i: 2).getReg() != AArch64::WZR)
2492 return false;
2493 } else if (MIOpc == AArch64::CSINCXr) {
2494 if (MI.getOperand(i: 1).getReg() != AArch64::XZR ||
2495 MI.getOperand(i: 2).getReg() != AArch64::XZR)
2496 return false;
2497 } else {
2498 return false;
2499 }
2500 AArch64CC::CondCode MICC = findCondCodeUsedByInstr(Instr: MI);
2501 if (MICC == AArch64CC::Invalid)
2502 return false;
2503
2504 // NZCV needs to be defined
2505 if (MI.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) != -1)
2506 return false;
2507
2508 // CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0' or 'SUBS %vreg, 1'
2509 const unsigned CmpOpcode = CmpInstr.getOpcode();
2510 bool IsSubsRegImm = isSUBSRegImm(Opcode: CmpOpcode);
2511 if (CmpValue && !IsSubsRegImm)
2512 return false;
2513 if (!CmpValue && !IsSubsRegImm && !isADDSRegImm(Opcode: CmpOpcode))
2514 return false;
2515
2516 // MI conditions allowed: eq, ne, mi, pl
2517 UsedNZCV MIUsedNZCV = getUsedNZCV(CC: MICC);
2518 if (MIUsedNZCV.C || MIUsedNZCV.V)
2519 return false;
2520
2521 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2522 examineCFlagsUse(MI, CmpInstr, TRI, CCUseInstrs: &CCUseInstrs);
2523 // Condition flags are not used in CmpInstr basic block successors and only
2524 // Z or N flags allowed to be used after CmpInstr within its basic block
2525 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2526 return false;
2527 // Z or N flag used after CmpInstr must correspond to the flag used in MI
2528 if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) ||
2529 (MIUsedNZCV.N && NZCVUsedAfterCmp->Z))
2530 return false;
2531 // If CmpInstr is comparison to zero MI conditions are limited to eq, ne
2532 if (MIUsedNZCV.N && !CmpValue)
2533 return false;
2534
2535 // There must be no defs of flags between MI and CmpInstr
2536 if (areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck: AK_Write))
2537 return false;
2538
2539 // Condition code is inverted in the following cases:
2540 // 1. MI condition is ne; CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2541 // 2. MI condition is eq, pl; CmpInstr is 'SUBS %vreg, 1'
2542 IsInvertCC = (CmpValue && (MICC == AArch64CC::EQ || MICC == AArch64CC::PL)) ||
2543 (!CmpValue && MICC == AArch64CC::NE);
2544 return true;
2545}
2546
2547/// Remove comparison in csinc-cmp sequence
2548///
2549/// Examples:
2550/// 1. \code
2551/// csinc w9, wzr, wzr, ne
2552/// cmp w9, #0
2553/// b.eq
2554/// \endcode
2555/// to
2556/// \code
2557/// csinc w9, wzr, wzr, ne
2558/// b.ne
2559/// \endcode
2560///
2561/// 2. \code
2562/// csinc x2, xzr, xzr, mi
2563/// cmp x2, #1
2564/// b.pl
2565/// \endcode
2566/// to
2567/// \code
2568/// csinc x2, xzr, xzr, mi
2569/// b.pl
2570/// \endcode
2571///
2572/// \param CmpInstr comparison instruction
2573/// \return True when comparison removed
2574bool AArch64InstrInfo::removeCmpToZeroOrOne(
2575 MachineInstr &CmpInstr, unsigned SrcReg, int CmpValue,
2576 const MachineRegisterInfo &MRI) const {
2577 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2578 if (!MI)
2579 return false;
2580 const TargetRegisterInfo &TRI = getRegisterInfo();
2581 SmallVector<MachineInstr *, 4> CCUseInstrs;
2582 bool IsInvertCC = false;
2583 if (!canCmpInstrBeRemoved(MI&: *MI, CmpInstr, CmpValue, TRI, CCUseInstrs,
2584 IsInvertCC))
2585 return false;
2586 // Make transformation
2587 CmpInstr.eraseFromParent();
2588 if (IsInvertCC) {
2589 // Invert condition codes in CmpInstr CC users
2590 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2591 int Idx = findCondCodeUseOperandIdxForBranchOrSelect(Instr: *CCUseInstr);
2592 assert(Idx >= 0 && "Unexpected instruction using CC.");
2593 MachineOperand &CCOperand = CCUseInstr->getOperand(i: Idx);
2594 AArch64CC::CondCode CCUse = AArch64CC::getInvertedCondCode(
2595 Code: static_cast<AArch64CC::CondCode>(CCOperand.getImm()));
2596 CCOperand.setImm(CCUse);
2597 }
2598 }
2599 return true;
2600}
2601
2602bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2603 if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2604 MI.getOpcode() != AArch64::CATCHRET &&
2605 MI.getOpcode() != AArch64::STACK_GUARD_UNMIX)
2606 return false;
2607
2608 MachineBasicBlock &MBB = *MI.getParent();
2609 auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
2610 auto TRI = Subtarget.getRegisterInfo();
2611 DebugLoc DL = MI.getDebugLoc();
2612
2613 if (MI.getOpcode() == AArch64::STACK_GUARD_UNMIX) {
2614 // Expand STACK_GUARD_UNMIX to: sub Rd, fp, Rs
2615 // This computes FP - stored_mixed_value to unmix the cookie
2616 Register DstReg = MI.getOperand(i: 0).getReg();
2617 Register SrcReg = MI.getOperand(i: 1).getReg();
2618
2619 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: DstReg)
2620 .addReg(RegNo: AArch64::FP)
2621 .addReg(RegNo: SrcReg);
2622
2623 MBB.erase(I: MI);
2624 return true;
2625 }
2626
2627 if (MI.getOpcode() == AArch64::CATCHRET) {
2628 // Skip to the first instruction before the epilog.
2629 const TargetInstrInfo *TII =
2630 MBB.getParent()->getSubtarget().getInstrInfo();
2631 MachineBasicBlock *TargetMBB = MI.getOperand(i: 0).getMBB();
2632 auto MBBI = MachineBasicBlock::iterator(MI);
2633 MachineBasicBlock::iterator FirstEpilogSEH = std::prev(x: MBBI);
2634 while (FirstEpilogSEH->getFlag(Flag: MachineInstr::FrameDestroy) &&
2635 FirstEpilogSEH != MBB.begin())
2636 FirstEpilogSEH = std::prev(x: FirstEpilogSEH);
2637 if (FirstEpilogSEH != MBB.begin())
2638 FirstEpilogSEH = std::next(x: FirstEpilogSEH);
2639 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADRP))
2640 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2641 .addMBB(MBB: TargetMBB, TargetFlags: AArch64II::MO_PAGE);
2642 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri))
2643 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2644 .addReg(RegNo: AArch64::X0)
2645 .addMBB(MBB: TargetMBB, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
2646 .addImm(Val: 0);
2647 TargetMBB->setMachineBlockAddressTaken();
2648 return true;
2649 }
2650
2651 Register Reg = MI.getOperand(i: 0).getReg();
2652 Module &M = *MBB.getParent()->getFunction().getParent();
2653 if (M.getStackProtectorGuard() == "sysreg") {
2654 const AArch64SysReg::SysReg *SrcReg =
2655 AArch64SysReg::lookupSysRegByName(Name: M.getStackProtectorGuardReg());
2656 if (!SrcReg)
2657 report_fatal_error(reason: "Unknown SysReg for Stack Protector Guard Register");
2658
2659 // mrs xN, sysreg
2660 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MRS))
2661 .addDef(RegNo: Reg, Flags: RegState::Renamable)
2662 .addImm(Val: SrcReg->Encoding);
2663 int Offset = M.getStackProtectorGuardOffset();
2664 if (Offset >= 0 && Offset <= 32760 && Offset % 8 == 0) {
2665 // ldr xN, [xN, #offset]
2666 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2667 .addDef(RegNo: Reg)
2668 .addUse(RegNo: Reg, Flags: RegState::Kill)
2669 .addImm(Val: Offset / 8);
2670 } else if (Offset >= -256 && Offset <= 255) {
2671 // ldur xN, [xN, #offset]
2672 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDURXi))
2673 .addDef(RegNo: Reg)
2674 .addUse(RegNo: Reg, Flags: RegState::Kill)
2675 .addImm(Val: Offset);
2676 } else if (Offset >= -4095 && Offset <= 4095) {
2677 if (Offset > 0) {
2678 // add xN, xN, #offset
2679 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri))
2680 .addDef(RegNo: Reg)
2681 .addUse(RegNo: Reg, Flags: RegState::Kill)
2682 .addImm(Val: Offset)
2683 .addImm(Val: 0);
2684 } else {
2685 // sub xN, xN, #offset
2686 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXri))
2687 .addDef(RegNo: Reg)
2688 .addUse(RegNo: Reg, Flags: RegState::Kill)
2689 .addImm(Val: -Offset)
2690 .addImm(Val: 0);
2691 }
2692 // ldr xN, [xN]
2693 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2694 .addDef(RegNo: Reg)
2695 .addUse(RegNo: Reg, Flags: RegState::Kill)
2696 .addImm(Val: 0);
2697 } else {
2698 // Cases that are larger than +/- 4095 and not a multiple of 8, or larger
2699 // than 23760.
2700 // It might be nice to use AArch64::MOVi32imm here, which would get
2701 // expanded in PreSched2 after PostRA, but our lone scratch Reg already
2702 // contains the MRS result. findScratchNonCalleeSaveRegister() in
2703 // AArch64FrameLowering might help us find such a scratch register
2704 // though. If we failed to find a scratch register, we could emit a
2705 // stream of add instructions to build up the immediate. Or, we could try
2706 // to insert a AArch64::MOVi32imm before register allocation so that we
2707 // didn't need to scavenge for a scratch register.
2708 report_fatal_error(reason: "Unable to encode Stack Protector Guard Offset");
2709 }
2710 MBB.erase(I: MI);
2711 return true;
2712 }
2713
2714 const GlobalValue *GV =
2715 cast<GlobalValue>(Val: (*MI.memoperands_begin())->getValue());
2716 const TargetMachine &TM = MBB.getParent()->getTarget();
2717 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2718 const unsigned char MO_NC = AArch64II::MO_NC;
2719
2720 unsigned GuardWidth = M.getStackProtectorGuardValueWidth().value_or(
2721 u: Subtarget.isTargetILP32() ? 4 : 8);
2722 if (GuardWidth != 4 && GuardWidth != 8)
2723 report_fatal_error(reason: "Unsupported stack protector value width");
2724 if ((OpFlags & AArch64II::MO_GOT) != 0) {
2725 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LOADgot), DestReg: Reg)
2726 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags);
2727 if (GuardWidth == 4) {
2728 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2729 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2730 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2731 .addUse(RegNo: Reg, Flags: RegState::Kill)
2732 .addImm(Val: 0)
2733 .addMemOperand(MMO: *MI.memoperands_begin())
2734 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2735 } else {
2736 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2737 .addReg(RegNo: Reg, Flags: RegState::Kill)
2738 .addImm(Val: 0)
2739 .addMemOperand(MMO: *MI.memoperands_begin());
2740 }
2741 } else if (TM.getCodeModel() == CodeModel::Large) {
2742 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg)
2743 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G0 | MO_NC)
2744 .addImm(Val: 0);
2745 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2746 .addReg(RegNo: Reg, Flags: RegState::Kill)
2747 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G1 | MO_NC)
2748 .addImm(Val: 16);
2749 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2750 .addReg(RegNo: Reg, Flags: RegState::Kill)
2751 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G2 | MO_NC)
2752 .addImm(Val: 32);
2753 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2754 .addReg(RegNo: Reg, Flags: RegState::Kill)
2755 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G3)
2756 .addImm(Val: 48);
2757 if (GuardWidth == 4) {
2758 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2759 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2760 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2761 .addUse(RegNo: Reg, Flags: RegState::Kill)
2762 .addImm(Val: 0)
2763 .addMemOperand(MMO: *MI.memoperands_begin())
2764 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2765 } else {
2766 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2767 .addReg(RegNo: Reg, Flags: RegState::Kill)
2768 .addImm(Val: 0)
2769 .addMemOperand(MMO: *MI.memoperands_begin());
2770 }
2771 } else {
2772 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
2773 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags | AArch64II::MO_PAGE);
2774 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
2775 if (GuardWidth == 4) {
2776 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2777 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2778 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2779 .addUse(RegNo: Reg, Flags: RegState::Kill)
2780 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2781 .addMemOperand(MMO: *MI.memoperands_begin())
2782 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2783 } else {
2784 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2785 .addReg(RegNo: Reg, Flags: RegState::Kill)
2786 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2787 .addMemOperand(MMO: *MI.memoperands_begin());
2788 }
2789 }
2790 // To match MSVC. Unlike x86_64 which uses xor instruction to mix the cookie,
2791 // we use sub instruction to mix the cookie on aarch64.
2792 // The mixing happens here in expandPostRAPseudo (after RA) to ensure we use
2793 // the final frame pointer value.
2794 if (Subtarget.getTargetTriple().isOSMSVCRT())
2795 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: Reg)
2796 .addReg(RegNo: AArch64::FP)
2797 .addReg(RegNo: Reg, Flags: RegState::Kill);
2798
2799 MBB.erase(I: MI);
2800
2801 return true;
2802}
2803
2804// Return true if this instruction simply sets its single destination register
2805// to zero. This is equivalent to a register rename of the zero-register.
2806bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
2807 switch (MI.getOpcode()) {
2808 default:
2809 break;
2810 case AArch64::MOVZWi:
2811 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
2812 if (MI.getOperand(i: 1).isImm() && MI.getOperand(i: 1).getImm() == 0) {
2813 assert(MI.getDesc().getNumOperands() == 3 &&
2814 MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
2815 return true;
2816 }
2817 break;
2818 case AArch64::ANDWri: // and Rd, Rzr, #imm
2819 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2820 case AArch64::ANDXri:
2821 return MI.getOperand(i: 1).getReg() == AArch64::XZR;
2822 case TargetOpcode::COPY:
2823 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2824 }
2825 return false;
2826}
2827
2828// Return true if this instruction simply renames a general register without
2829// modifying bits.
2830bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
2831 switch (MI.getOpcode()) {
2832 default:
2833 break;
2834 case TargetOpcode::COPY: {
2835 // GPR32 copies will by lowered to ORRXrs
2836 Register DstReg = MI.getOperand(i: 0).getReg();
2837 return (AArch64::GPR32RegClass.contains(Reg: DstReg) ||
2838 AArch64::GPR64RegClass.contains(Reg: DstReg));
2839 }
2840 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
2841 if (MI.getOperand(i: 1).getReg() == AArch64::XZR) {
2842 assert(MI.getDesc().getNumOperands() == 4 &&
2843 MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
2844 return true;
2845 }
2846 break;
2847 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
2848 if (MI.getOperand(i: 2).getImm() == 0) {
2849 assert(MI.getDesc().getNumOperands() == 4 &&
2850 MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
2851 return true;
2852 }
2853 break;
2854 }
2855 return false;
2856}
2857
2858// Return true if this instruction simply renames a general register without
2859// modifying bits.
2860bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
2861 switch (MI.getOpcode()) {
2862 default:
2863 break;
2864 case TargetOpcode::COPY: {
2865 Register DstReg = MI.getOperand(i: 0).getReg();
2866 return AArch64::FPR128RegClass.contains(Reg: DstReg);
2867 }
2868 case AArch64::ORRv16i8:
2869 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
2870 assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
2871 "invalid ORRv16i8 operands");
2872 return true;
2873 }
2874 break;
2875 }
2876 return false;
2877}
2878
2879static bool isFrameLoadOpcode(int Opcode) {
2880 switch (Opcode) {
2881 default:
2882 return false;
2883 case AArch64::LDRWui:
2884 case AArch64::LDRXui:
2885 case AArch64::LDRBui:
2886 case AArch64::LDRHui:
2887 case AArch64::LDRSui:
2888 case AArch64::LDRDui:
2889 case AArch64::LDRQui:
2890 case AArch64::LDR_PXI:
2891 return true;
2892 }
2893}
2894
2895Register AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
2896 int &FrameIndex) const {
2897 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2898 return Register();
2899
2900 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2901 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2902 FrameIndex = MI.getOperand(i: 1).getIndex();
2903 return MI.getOperand(i: 0).getReg();
2904 }
2905 return Register();
2906}
2907
2908static bool isFrameStoreOpcode(int Opcode) {
2909 switch (Opcode) {
2910 default:
2911 return false;
2912 case AArch64::STRWui:
2913 case AArch64::STRXui:
2914 case AArch64::STRBui:
2915 case AArch64::STRHui:
2916 case AArch64::STRSui:
2917 case AArch64::STRDui:
2918 case AArch64::STRQui:
2919 case AArch64::STR_PXI:
2920 return true;
2921 }
2922}
2923
2924Register AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
2925 int &FrameIndex) const {
2926 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2927 return Register();
2928
2929 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2930 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2931 FrameIndex = MI.getOperand(i: 1).getIndex();
2932 return MI.getOperand(i: 0).getReg();
2933 }
2934 return Register();
2935}
2936
2937Register AArch64InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
2938 int &FrameIndex) const {
2939 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2940 return Register();
2941
2942 if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
2943 return Reg;
2944
2945 SmallVector<const MachineMemOperand *, 1> Accesses;
2946 if (hasStoreToStackSlot(MI, Accesses)) {
2947 if (Accesses.size() > 1)
2948 return Register();
2949
2950 FrameIndex =
2951 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2952 ->getFrameIndex();
2953 return MI.getOperand(i: 0).getReg();
2954 }
2955 return Register();
2956}
2957
2958Register AArch64InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
2959 int &FrameIndex) const {
2960 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2961 return Register();
2962
2963 if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
2964 return Reg;
2965
2966 SmallVector<const MachineMemOperand *, 1> Accesses;
2967 if (hasLoadFromStackSlot(MI, Accesses)) {
2968 if (Accesses.size() > 1)
2969 return Register();
2970
2971 FrameIndex =
2972 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2973 ->getFrameIndex();
2974 return MI.getOperand(i: 0).getReg();
2975 }
2976 return Register();
2977}
2978
2979/// Check all MachineMemOperands for a hint to suppress pairing.
2980bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
2981 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2982 return MMO->getFlags() & MOSuppressPair;
2983 });
2984}
2985
2986/// Set a flag on the first MachineMemOperand to suppress pairing.
2987void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
2988 if (MI.memoperands_empty())
2989 return;
2990 (*MI.memoperands_begin())->setFlags(MOSuppressPair);
2991}
2992
2993/// Check all MachineMemOperands for a hint that the load/store is strided.
2994bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
2995 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2996 return MMO->getFlags() & MOStridedAccess;
2997 });
2998}
2999
3000bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
3001 switch (Opc) {
3002 default:
3003 return false;
3004 case AArch64::STURSi:
3005 case AArch64::STRSpre:
3006 case AArch64::STURDi:
3007 case AArch64::STRDpre:
3008 case AArch64::STURQi:
3009 case AArch64::STRQpre:
3010 case AArch64::STURBBi:
3011 case AArch64::STURHHi:
3012 case AArch64::STURWi:
3013 case AArch64::STRWpre:
3014 case AArch64::STURXi:
3015 case AArch64::STRXpre:
3016 case AArch64::LDURSi:
3017 case AArch64::LDRSpre:
3018 case AArch64::LDURDi:
3019 case AArch64::LDRDpre:
3020 case AArch64::LDURQi:
3021 case AArch64::LDRQpre:
3022 case AArch64::LDURWi:
3023 case AArch64::LDRWpre:
3024 case AArch64::LDURXi:
3025 case AArch64::LDRXpre:
3026 case AArch64::LDRSWpre:
3027 case AArch64::LDURSWi:
3028 case AArch64::LDURHHi:
3029 case AArch64::LDURBBi:
3030 case AArch64::LDURSBWi:
3031 case AArch64::LDURSHWi:
3032 return true;
3033 }
3034}
3035
3036std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
3037 switch (Opc) {
3038 default: return {};
3039 case AArch64::PRFMui: return AArch64::PRFUMi;
3040 case AArch64::LDRXui: return AArch64::LDURXi;
3041 case AArch64::LDRWui: return AArch64::LDURWi;
3042 case AArch64::LDRBui: return AArch64::LDURBi;
3043 case AArch64::LDRHui: return AArch64::LDURHi;
3044 case AArch64::LDRSui: return AArch64::LDURSi;
3045 case AArch64::LDRDui: return AArch64::LDURDi;
3046 case AArch64::LDRQui: return AArch64::LDURQi;
3047 case AArch64::LDRBBui: return AArch64::LDURBBi;
3048 case AArch64::LDRHHui: return AArch64::LDURHHi;
3049 case AArch64::LDRSBXui: return AArch64::LDURSBXi;
3050 case AArch64::LDRSBWui: return AArch64::LDURSBWi;
3051 case AArch64::LDRSHXui: return AArch64::LDURSHXi;
3052 case AArch64::LDRSHWui: return AArch64::LDURSHWi;
3053 case AArch64::LDRSWui: return AArch64::LDURSWi;
3054 case AArch64::STRXui: return AArch64::STURXi;
3055 case AArch64::STRWui: return AArch64::STURWi;
3056 case AArch64::STRBui: return AArch64::STURBi;
3057 case AArch64::STRHui: return AArch64::STURHi;
3058 case AArch64::STRSui: return AArch64::STURSi;
3059 case AArch64::STRDui: return AArch64::STURDi;
3060 case AArch64::STRQui: return AArch64::STURQi;
3061 case AArch64::STRBBui: return AArch64::STURBBi;
3062 case AArch64::STRHHui: return AArch64::STURHHi;
3063 }
3064}
3065
3066unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
3067 switch (Opc) {
3068 default:
3069 llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
3070 case AArch64::ADDG:
3071 case AArch64::LDAPURBi:
3072 case AArch64::LDAPURHi:
3073 case AArch64::LDAPURi:
3074 case AArch64::LDAPURSBWi:
3075 case AArch64::LDAPURSBXi:
3076 case AArch64::LDAPURSHWi:
3077 case AArch64::LDAPURSHXi:
3078 case AArch64::LDAPURSWi:
3079 case AArch64::LDAPURXi:
3080 case AArch64::LDR_PPXI:
3081 case AArch64::LDR_PXI:
3082 case AArch64::LDR_ZXI:
3083 case AArch64::LDR_ZZXI:
3084 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
3085 case AArch64::LDR_ZZZXI:
3086 case AArch64::LDR_ZZZZXI:
3087 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
3088 case AArch64::LDRBBui:
3089 case AArch64::LDRBui:
3090 case AArch64::LDRDui:
3091 case AArch64::LDRHHui:
3092 case AArch64::LDRHui:
3093 case AArch64::LDRQui:
3094 case AArch64::LDRSBWui:
3095 case AArch64::LDRSBXui:
3096 case AArch64::LDRSHWui:
3097 case AArch64::LDRSHXui:
3098 case AArch64::LDRSui:
3099 case AArch64::LDRSWui:
3100 case AArch64::LDRWui:
3101 case AArch64::LDRXui:
3102 case AArch64::LDURBBi:
3103 case AArch64::LDURBi:
3104 case AArch64::LDURDi:
3105 case AArch64::LDURHHi:
3106 case AArch64::LDURHi:
3107 case AArch64::LDURQi:
3108 case AArch64::LDURSBWi:
3109 case AArch64::LDURSBXi:
3110 case AArch64::LDURSHWi:
3111 case AArch64::LDURSHXi:
3112 case AArch64::LDURSi:
3113 case AArch64::LDURSWi:
3114 case AArch64::LDURWi:
3115 case AArch64::LDURXi:
3116 case AArch64::PRFMui:
3117 case AArch64::PRFUMi:
3118 case AArch64::ST2Gi:
3119 case AArch64::STGi:
3120 case AArch64::STLURBi:
3121 case AArch64::STLURHi:
3122 case AArch64::STLURWi:
3123 case AArch64::STLURXi:
3124 case AArch64::StoreSwiftAsyncContext:
3125 case AArch64::STR_PPXI:
3126 case AArch64::STR_PXI:
3127 case AArch64::STR_ZXI:
3128 case AArch64::STR_ZZXI:
3129 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
3130 case AArch64::STR_ZZZXI:
3131 case AArch64::STR_ZZZZXI:
3132 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
3133 case AArch64::STRBBui:
3134 case AArch64::STRBui:
3135 case AArch64::STRDui:
3136 case AArch64::STRHHui:
3137 case AArch64::STRHui:
3138 case AArch64::STRQui:
3139 case AArch64::STRSui:
3140 case AArch64::STRWui:
3141 case AArch64::STRXui:
3142 case AArch64::STURBBi:
3143 case AArch64::STURBi:
3144 case AArch64::STURDi:
3145 case AArch64::STURHHi:
3146 case AArch64::STURHi:
3147 case AArch64::STURQi:
3148 case AArch64::STURSi:
3149 case AArch64::STURWi:
3150 case AArch64::STURXi:
3151 case AArch64::STZ2Gi:
3152 case AArch64::STZGi:
3153 case AArch64::TAGPstack:
3154 return 2;
3155 case AArch64::LD1B_D_IMM:
3156 case AArch64::LD1B_H_IMM:
3157 case AArch64::LD1B_IMM:
3158 case AArch64::LD1B_S_IMM:
3159 case AArch64::LD1D_IMM:
3160 case AArch64::LD1H_D_IMM:
3161 case AArch64::LD1H_IMM:
3162 case AArch64::LD1H_S_IMM:
3163 case AArch64::LD1RB_D_IMM:
3164 case AArch64::LD1RB_H_IMM:
3165 case AArch64::LD1RB_IMM:
3166 case AArch64::LD1RB_S_IMM:
3167 case AArch64::LD1RD_IMM:
3168 case AArch64::LD1RH_D_IMM:
3169 case AArch64::LD1RH_IMM:
3170 case AArch64::LD1RH_S_IMM:
3171 case AArch64::LD1RSB_D_IMM:
3172 case AArch64::LD1RSB_H_IMM:
3173 case AArch64::LD1RSB_S_IMM:
3174 case AArch64::LD1RSH_D_IMM:
3175 case AArch64::LD1RSH_S_IMM:
3176 case AArch64::LD1RSW_IMM:
3177 case AArch64::LD1RW_D_IMM:
3178 case AArch64::LD1RW_IMM:
3179 case AArch64::LD1SB_D_IMM:
3180 case AArch64::LD1SB_H_IMM:
3181 case AArch64::LD1SB_S_IMM:
3182 case AArch64::LD1SH_D_IMM:
3183 case AArch64::LD1SH_S_IMM:
3184 case AArch64::LD1SW_D_IMM:
3185 case AArch64::LD1W_D_IMM:
3186 case AArch64::LD1W_IMM:
3187 case AArch64::LD2B_IMM:
3188 case AArch64::LD2D_IMM:
3189 case AArch64::LD2H_IMM:
3190 case AArch64::LD2W_IMM:
3191 case AArch64::LD3B_IMM:
3192 case AArch64::LD3D_IMM:
3193 case AArch64::LD3H_IMM:
3194 case AArch64::LD3W_IMM:
3195 case AArch64::LD4B_IMM:
3196 case AArch64::LD4D_IMM:
3197 case AArch64::LD4H_IMM:
3198 case AArch64::LD4W_IMM:
3199 case AArch64::LDG:
3200 case AArch64::LDNF1B_D_IMM:
3201 case AArch64::LDNF1B_H_IMM:
3202 case AArch64::LDNF1B_IMM:
3203 case AArch64::LDNF1B_S_IMM:
3204 case AArch64::LDNF1D_IMM:
3205 case AArch64::LDNF1H_D_IMM:
3206 case AArch64::LDNF1H_IMM:
3207 case AArch64::LDNF1H_S_IMM:
3208 case AArch64::LDNF1SB_D_IMM:
3209 case AArch64::LDNF1SB_H_IMM:
3210 case AArch64::LDNF1SB_S_IMM:
3211 case AArch64::LDNF1SH_D_IMM:
3212 case AArch64::LDNF1SH_S_IMM:
3213 case AArch64::LDNF1SW_D_IMM:
3214 case AArch64::LDNF1W_D_IMM:
3215 case AArch64::LDNF1W_IMM:
3216 case AArch64::LDNPDi:
3217 case AArch64::LDNPQi:
3218 case AArch64::LDNPSi:
3219 case AArch64::LDNPWi:
3220 case AArch64::LDNPXi:
3221 case AArch64::LDNT1B_ZRI:
3222 case AArch64::LDNT1D_ZRI:
3223 case AArch64::LDNT1H_ZRI:
3224 case AArch64::LDNT1W_ZRI:
3225 case AArch64::LDPDi:
3226 case AArch64::LDPQi:
3227 case AArch64::LDPSi:
3228 case AArch64::LDPWi:
3229 case AArch64::LDPXi:
3230 case AArch64::LDRBBpost:
3231 case AArch64::LDRBBpre:
3232 case AArch64::LDRBpost:
3233 case AArch64::LDRBpre:
3234 case AArch64::LDRDpost:
3235 case AArch64::LDRDpre:
3236 case AArch64::LDRHHpost:
3237 case AArch64::LDRHHpre:
3238 case AArch64::LDRHpost:
3239 case AArch64::LDRHpre:
3240 case AArch64::LDRQpost:
3241 case AArch64::LDRQpre:
3242 case AArch64::LDRSpost:
3243 case AArch64::LDRSpre:
3244 case AArch64::LDRWpost:
3245 case AArch64::LDRWpre:
3246 case AArch64::LDRXpost:
3247 case AArch64::LDRXpre:
3248 case AArch64::ST1B_D_IMM:
3249 case AArch64::ST1B_H_IMM:
3250 case AArch64::ST1B_IMM:
3251 case AArch64::ST1B_S_IMM:
3252 case AArch64::ST1D_IMM:
3253 case AArch64::ST1H_D_IMM:
3254 case AArch64::ST1H_IMM:
3255 case AArch64::ST1H_S_IMM:
3256 case AArch64::ST1W_D_IMM:
3257 case AArch64::ST1W_IMM:
3258 case AArch64::ST2B_IMM:
3259 case AArch64::ST2D_IMM:
3260 case AArch64::ST2H_IMM:
3261 case AArch64::ST2W_IMM:
3262 case AArch64::ST3B_IMM:
3263 case AArch64::ST3D_IMM:
3264 case AArch64::ST3H_IMM:
3265 case AArch64::ST3W_IMM:
3266 case AArch64::ST4B_IMM:
3267 case AArch64::ST4D_IMM:
3268 case AArch64::ST4H_IMM:
3269 case AArch64::ST4W_IMM:
3270 case AArch64::STGPi:
3271 case AArch64::STGPreIndex:
3272 case AArch64::STZGPreIndex:
3273 case AArch64::ST2GPreIndex:
3274 case AArch64::STZ2GPreIndex:
3275 case AArch64::STGPostIndex:
3276 case AArch64::STZGPostIndex:
3277 case AArch64::ST2GPostIndex:
3278 case AArch64::STZ2GPostIndex:
3279 case AArch64::STNPDi:
3280 case AArch64::STNPQi:
3281 case AArch64::STNPSi:
3282 case AArch64::STNPWi:
3283 case AArch64::STNPXi:
3284 case AArch64::STNT1B_ZRI:
3285 case AArch64::STNT1D_ZRI:
3286 case AArch64::STNT1H_ZRI:
3287 case AArch64::STNT1W_ZRI:
3288 case AArch64::STPDi:
3289 case AArch64::STPQi:
3290 case AArch64::STPSi:
3291 case AArch64::STPWi:
3292 case AArch64::STPXi:
3293 case AArch64::STRBBpost:
3294 case AArch64::STRBBpre:
3295 case AArch64::STRBpost:
3296 case AArch64::STRBpre:
3297 case AArch64::STRDpost:
3298 case AArch64::STRDpre:
3299 case AArch64::STRHHpost:
3300 case AArch64::STRHHpre:
3301 case AArch64::STRHpost:
3302 case AArch64::STRHpre:
3303 case AArch64::STRQpost:
3304 case AArch64::STRQpre:
3305 case AArch64::STRSpost:
3306 case AArch64::STRSpre:
3307 case AArch64::STRWpost:
3308 case AArch64::STRWpre:
3309 case AArch64::STRXpost:
3310 case AArch64::STRXpre:
3311 case AArch64::LD1B_2Z_IMM:
3312 case AArch64::LD1B_2Z_STRIDED_IMM:
3313 case AArch64::LD1H_2Z_IMM:
3314 case AArch64::LD1H_2Z_STRIDED_IMM:
3315 case AArch64::LD1W_2Z_IMM:
3316 case AArch64::LD1W_2Z_STRIDED_IMM:
3317 case AArch64::LD1D_2Z_IMM:
3318 case AArch64::LD1D_2Z_STRIDED_IMM:
3319 case AArch64::LD1B_4Z_IMM:
3320 case AArch64::LD1B_4Z_STRIDED_IMM:
3321 case AArch64::LD1H_4Z_IMM:
3322 case AArch64::LD1H_4Z_STRIDED_IMM:
3323 case AArch64::LD1W_4Z_IMM:
3324 case AArch64::LD1W_4Z_STRIDED_IMM:
3325 case AArch64::LD1D_4Z_IMM:
3326 case AArch64::LD1D_4Z_STRIDED_IMM:
3327 case AArch64::LD1B_2Z_IMM_PSEUDO:
3328 case AArch64::LD1H_2Z_IMM_PSEUDO:
3329 case AArch64::LD1W_2Z_IMM_PSEUDO:
3330 case AArch64::LD1D_2Z_IMM_PSEUDO:
3331 case AArch64::LD1B_4Z_IMM_PSEUDO:
3332 case AArch64::LD1H_4Z_IMM_PSEUDO:
3333 case AArch64::LD1W_4Z_IMM_PSEUDO:
3334 case AArch64::LD1D_4Z_IMM_PSEUDO:
3335 case AArch64::ST1B_2Z_IMM:
3336 case AArch64::ST1B_2Z_STRIDED_IMM:
3337 case AArch64::ST1H_2Z_IMM:
3338 case AArch64::ST1H_2Z_STRIDED_IMM:
3339 case AArch64::ST1W_2Z_IMM:
3340 case AArch64::ST1W_2Z_STRIDED_IMM:
3341 case AArch64::ST1D_2Z_IMM:
3342 case AArch64::ST1D_2Z_STRIDED_IMM:
3343 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
3344 case AArch64::LDNT1B_2Z_IMM:
3345 case AArch64::LDNT1B_2Z_STRIDED_IMM:
3346 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
3347 case AArch64::LDNT1H_2Z_IMM:
3348 case AArch64::LDNT1H_2Z_STRIDED_IMM:
3349 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
3350 case AArch64::LDNT1W_2Z_IMM:
3351 case AArch64::LDNT1W_2Z_STRIDED_IMM:
3352 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
3353 case AArch64::LDNT1D_2Z_IMM:
3354 case AArch64::LDNT1D_2Z_STRIDED_IMM:
3355 case AArch64::STNT1B_2Z_IMM:
3356 case AArch64::STNT1B_2Z_STRIDED_IMM:
3357 case AArch64::STNT1H_2Z_IMM:
3358 case AArch64::STNT1H_2Z_STRIDED_IMM:
3359 case AArch64::STNT1W_2Z_IMM:
3360 case AArch64::STNT1W_2Z_STRIDED_IMM:
3361 case AArch64::STNT1D_2Z_IMM:
3362 case AArch64::STNT1D_2Z_STRIDED_IMM:
3363 case AArch64::ST1B_2Z_IMM_PSEUDO:
3364 case AArch64::ST1H_2Z_IMM_PSEUDO:
3365 case AArch64::ST1W_2Z_IMM_PSEUDO:
3366 case AArch64::ST1D_2Z_IMM_PSEUDO:
3367 case AArch64::STNT1B_2Z_IMM_PSEUDO:
3368 case AArch64::STNT1H_2Z_IMM_PSEUDO:
3369 case AArch64::STNT1W_2Z_IMM_PSEUDO:
3370 case AArch64::STNT1D_2Z_IMM_PSEUDO:
3371 case AArch64::ST1B_4Z_IMM:
3372 case AArch64::ST1B_4Z_STRIDED_IMM:
3373 case AArch64::ST1H_4Z_IMM:
3374 case AArch64::ST1H_4Z_STRIDED_IMM:
3375 case AArch64::ST1W_4Z_IMM:
3376 case AArch64::ST1W_4Z_STRIDED_IMM:
3377 case AArch64::ST1D_4Z_IMM:
3378 case AArch64::ST1D_4Z_STRIDED_IMM:
3379 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
3380 case AArch64::LDNT1B_4Z_IMM:
3381 case AArch64::LDNT1B_4Z_STRIDED_IMM:
3382 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
3383 case AArch64::LDNT1H_4Z_IMM:
3384 case AArch64::LDNT1H_4Z_STRIDED_IMM:
3385 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
3386 case AArch64::LDNT1W_4Z_IMM:
3387 case AArch64::LDNT1W_4Z_STRIDED_IMM:
3388 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
3389 case AArch64::LDNT1D_4Z_IMM:
3390 case AArch64::LDNT1D_4Z_STRIDED_IMM:
3391 case AArch64::STNT1B_4Z_IMM:
3392 case AArch64::STNT1B_4Z_STRIDED_IMM:
3393 case AArch64::STNT1H_4Z_IMM:
3394 case AArch64::STNT1H_4Z_STRIDED_IMM:
3395 case AArch64::STNT1W_4Z_IMM:
3396 case AArch64::STNT1W_4Z_STRIDED_IMM:
3397 case AArch64::STNT1D_4Z_IMM:
3398 case AArch64::STNT1D_4Z_STRIDED_IMM:
3399 case AArch64::ST1B_4Z_IMM_PSEUDO:
3400 case AArch64::ST1H_4Z_IMM_PSEUDO:
3401 case AArch64::ST1W_4Z_IMM_PSEUDO:
3402 case AArch64::ST1D_4Z_IMM_PSEUDO:
3403 case AArch64::STNT1B_4Z_IMM_PSEUDO:
3404 case AArch64::STNT1H_4Z_IMM_PSEUDO:
3405 case AArch64::STNT1W_4Z_IMM_PSEUDO:
3406 case AArch64::STNT1D_4Z_IMM_PSEUDO:
3407 return 3;
3408 case AArch64::LDPDpost:
3409 case AArch64::LDPDpre:
3410 case AArch64::LDPQpost:
3411 case AArch64::LDPQpre:
3412 case AArch64::LDPSpost:
3413 case AArch64::LDPSpre:
3414 case AArch64::LDPWpost:
3415 case AArch64::LDPWpre:
3416 case AArch64::LDPXpost:
3417 case AArch64::LDPXpre:
3418 case AArch64::STGPpre:
3419 case AArch64::STGPpost:
3420 case AArch64::STPDpost:
3421 case AArch64::STPDpre:
3422 case AArch64::STPQpost:
3423 case AArch64::STPQpre:
3424 case AArch64::STPSpost:
3425 case AArch64::STPSpre:
3426 case AArch64::STPWpost:
3427 case AArch64::STPWpre:
3428 case AArch64::STPXpost:
3429 case AArch64::STPXpre:
3430 return 4;
3431 }
3432}
3433
3434bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
3435 switch (MI.getOpcode()) {
3436 default:
3437 return false;
3438 // Scaled instructions.
3439 case AArch64::STRSui:
3440 case AArch64::STRDui:
3441 case AArch64::STRQui:
3442 case AArch64::STRXui:
3443 case AArch64::STRWui:
3444 case AArch64::LDRSui:
3445 case AArch64::LDRDui:
3446 case AArch64::LDRQui:
3447 case AArch64::LDRXui:
3448 case AArch64::LDRWui:
3449 case AArch64::LDRSWui:
3450 // Unscaled instructions.
3451 case AArch64::STURSi:
3452 case AArch64::STRSpre:
3453 case AArch64::STURDi:
3454 case AArch64::STRDpre:
3455 case AArch64::STURQi:
3456 case AArch64::STRQpre:
3457 case AArch64::STURWi:
3458 case AArch64::STRWpre:
3459 case AArch64::STURXi:
3460 case AArch64::STRXpre:
3461 case AArch64::LDURSi:
3462 case AArch64::LDRSpre:
3463 case AArch64::LDURDi:
3464 case AArch64::LDRDpre:
3465 case AArch64::LDURQi:
3466 case AArch64::LDRQpre:
3467 case AArch64::LDURWi:
3468 case AArch64::LDRWpre:
3469 case AArch64::LDURXi:
3470 case AArch64::LDRXpre:
3471 case AArch64::LDURSWi:
3472 case AArch64::LDRSWpre:
3473 // SVE instructions.
3474 case AArch64::LDR_ZXI:
3475 case AArch64::STR_ZXI:
3476 return true;
3477 }
3478}
3479
3480bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
3481 switch (MI.getOpcode()) {
3482 default:
3483 assert((!MI.isCall() || !MI.isReturn()) &&
3484 "Unexpected instruction - was a new tail call opcode introduced?");
3485 return false;
3486 case AArch64::TCRETURNdi:
3487 case AArch64::TCRETURNri:
3488 case AArch64::TCRETURNrix16x17:
3489 case AArch64::TCRETURNrix17:
3490 case AArch64::TCRETURNrinotx16:
3491 case AArch64::TCRETURNriALL:
3492 case AArch64::AUTH_TCRETURN:
3493 case AArch64::AUTH_TCRETURN_BTI:
3494 return true;
3495 }
3496}
3497
3498unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) {
3499 switch (Opc) {
3500 default:
3501 llvm_unreachable("Opcode has no flag setting equivalent!");
3502 // 32-bit cases:
3503 case AArch64::ADDWri:
3504 return AArch64::ADDSWri;
3505 case AArch64::ADDWrr:
3506 return AArch64::ADDSWrr;
3507 case AArch64::ADDWrs:
3508 return AArch64::ADDSWrs;
3509 case AArch64::ADDWrx:
3510 return AArch64::ADDSWrx;
3511 case AArch64::ANDWri:
3512 return AArch64::ANDSWri;
3513 case AArch64::ANDWrr:
3514 return AArch64::ANDSWrr;
3515 case AArch64::ANDWrs:
3516 return AArch64::ANDSWrs;
3517 case AArch64::BICWrr:
3518 return AArch64::BICSWrr;
3519 case AArch64::BICWrs:
3520 return AArch64::BICSWrs;
3521 case AArch64::SUBWri:
3522 return AArch64::SUBSWri;
3523 case AArch64::SUBWrr:
3524 return AArch64::SUBSWrr;
3525 case AArch64::SUBWrs:
3526 return AArch64::SUBSWrs;
3527 case AArch64::SUBWrx:
3528 return AArch64::SUBSWrx;
3529 // 64-bit cases:
3530 case AArch64::ADDXri:
3531 return AArch64::ADDSXri;
3532 case AArch64::ADDXrr:
3533 return AArch64::ADDSXrr;
3534 case AArch64::ADDXrs:
3535 return AArch64::ADDSXrs;
3536 case AArch64::ADDXrx:
3537 return AArch64::ADDSXrx;
3538 case AArch64::ANDXri:
3539 return AArch64::ANDSXri;
3540 case AArch64::ANDXrr:
3541 return AArch64::ANDSXrr;
3542 case AArch64::ANDXrs:
3543 return AArch64::ANDSXrs;
3544 case AArch64::BICXrr:
3545 return AArch64::BICSXrr;
3546 case AArch64::BICXrs:
3547 return AArch64::BICSXrs;
3548 case AArch64::SUBXri:
3549 return AArch64::SUBSXri;
3550 case AArch64::SUBXrr:
3551 return AArch64::SUBSXrr;
3552 case AArch64::SUBXrs:
3553 return AArch64::SUBSXrs;
3554 case AArch64::SUBXrx:
3555 return AArch64::SUBSXrx;
3556 // SVE instructions:
3557 case AArch64::AND_PPzPP:
3558 return AArch64::ANDS_PPzPP;
3559 case AArch64::BIC_PPzPP:
3560 return AArch64::BICS_PPzPP;
3561 case AArch64::EOR_PPzPP:
3562 return AArch64::EORS_PPzPP;
3563 case AArch64::NAND_PPzPP:
3564 return AArch64::NANDS_PPzPP;
3565 case AArch64::NOR_PPzPP:
3566 return AArch64::NORS_PPzPP;
3567 case AArch64::ORN_PPzPP:
3568 return AArch64::ORNS_PPzPP;
3569 case AArch64::ORR_PPzPP:
3570 return AArch64::ORRS_PPzPP;
3571 case AArch64::BRKA_PPzP:
3572 return AArch64::BRKAS_PPzP;
3573 case AArch64::BRKPA_PPzPP:
3574 return AArch64::BRKPAS_PPzPP;
3575 case AArch64::BRKB_PPzP:
3576 return AArch64::BRKBS_PPzP;
3577 case AArch64::BRKPB_PPzPP:
3578 return AArch64::BRKPBS_PPzPP;
3579 case AArch64::BRKN_PPzP:
3580 return AArch64::BRKNS_PPzP;
3581 case AArch64::RDFFR_PPz:
3582 return AArch64::RDFFRS_PPz;
3583 case AArch64::PTRUE_B:
3584 return AArch64::PTRUES_B;
3585 }
3586}
3587
3588// Is this a candidate for ld/st merging or pairing? For example, we don't
3589// touch volatiles or load/stores that have a hint to avoid pair formation.
3590bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
3591
3592 bool IsPreLdSt = isPreLdSt(MI);
3593
3594 // If this is a volatile load/store, don't mess with it.
3595 if (MI.hasOrderedMemoryRef())
3596 return false;
3597
3598 // Make sure this is a reg/fi+imm (as opposed to an address reloc).
3599 // For Pre-inc LD/ST, the operand is shifted by one.
3600 assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
3601 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
3602 "Expected a reg or frame index operand.");
3603
3604 // For Pre-indexed addressing quadword instructions, the third operand is the
3605 // immediate value.
3606 bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(i: 3).isImm();
3607
3608 if (!MI.getOperand(i: 2).isImm() && !IsImmPreLdSt)
3609 return false;
3610
3611 // Can't merge/pair if the instruction modifies the base register.
3612 // e.g., ldr x0, [x0]
3613 // This case will never occur with an FI base.
3614 // However, if the instruction is an LDR<S,D,Q,W,X,SW>pre or
3615 // STR<S,D,Q,W,X>pre, it can be merged.
3616 // For example:
3617 // ldr q0, [x11, #32]!
3618 // ldr q1, [x11, #16]
3619 // to
3620 // ldp q0, q1, [x11, #32]!
3621 if (MI.getOperand(i: 1).isReg() && !IsPreLdSt) {
3622 Register BaseReg = MI.getOperand(i: 1).getReg();
3623 const TargetRegisterInfo *TRI = &getRegisterInfo();
3624 if (MI.modifiesRegister(Reg: BaseReg, TRI))
3625 return false;
3626 }
3627
3628 // Pairing SVE fills/spills is only valid for little-endian targets that
3629 // implement VLS 128.
3630 switch (MI.getOpcode()) {
3631 default:
3632 break;
3633 case AArch64::LDR_ZXI:
3634 case AArch64::STR_ZXI:
3635 if (!Subtarget.isLittleEndian() ||
3636 Subtarget.getSVEVectorSizeInBits() != 128)
3637 return false;
3638 }
3639
3640 // Check if this load/store has a hint to avoid pair formation.
3641 // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
3642 if (isLdStPairSuppressed(MI))
3643 return false;
3644
3645 // Do not pair any callee-save store/reload instructions in the
3646 // prologue/epilogue if the CFI information encoded the operations as separate
3647 // instructions, as that will cause the size of the actual prologue to mismatch
3648 // with the prologue size recorded in the Windows CFI.
3649 const MCAsmInfo &MAI = MI.getMF()->getTarget().getMCAsmInfo();
3650 bool NeedsWinCFI =
3651 MAI.usesWindowsCFI() && MI.getMF()->getFunction().needsUnwindTableEntry();
3652 if (NeedsWinCFI && (MI.getFlag(Flag: MachineInstr::FrameSetup) ||
3653 MI.getFlag(Flag: MachineInstr::FrameDestroy)))
3654 return false;
3655
3656 // On some CPUs quad load/store pairs are slower than two single load/stores.
3657 if (Subtarget.isPaired128Slow()) {
3658 switch (MI.getOpcode()) {
3659 default:
3660 break;
3661 case AArch64::LDURQi:
3662 case AArch64::STURQi:
3663 case AArch64::LDRQui:
3664 case AArch64::STRQui:
3665 return false;
3666 }
3667 }
3668
3669 return true;
3670}
3671
3672bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
3673 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3674 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
3675 const TargetRegisterInfo *TRI) const {
3676 if (!LdSt.mayLoadOrStore())
3677 return false;
3678
3679 const MachineOperand *BaseOp;
3680 TypeSize WidthN(0, false);
3681 if (!getMemOperandWithOffsetWidth(MI: LdSt, BaseOp, Offset, OffsetIsScalable,
3682 Width&: WidthN, TRI))
3683 return false;
3684 // The maximum vscale is 16 under AArch64, return the maximal extent for the
3685 // vector.
3686 Width = LocationSize::precise(Value: WidthN);
3687 BaseOps.push_back(Elt: BaseOp);
3688 return true;
3689}
3690
3691std::optional<ExtAddrMode>
3692AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI,
3693 const TargetRegisterInfo *TRI) const {
3694 const MachineOperand *Base; // Filled with the base operand of MI.
3695 int64_t Offset; // Filled with the offset of MI.
3696 bool OffsetIsScalable;
3697 if (!getMemOperandWithOffset(MI: MemI, BaseOp&: Base, Offset, OffsetIsScalable, TRI))
3698 return std::nullopt;
3699
3700 if (!Base->isReg())
3701 return std::nullopt;
3702 ExtAddrMode AM;
3703 AM.BaseReg = Base->getReg();
3704 AM.Displacement = Offset;
3705 AM.ScaledReg = 0;
3706 AM.Scale = 0;
3707 return AM;
3708}
3709
3710bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
3711 Register Reg,
3712 const MachineInstr &AddrI,
3713 ExtAddrMode &AM) const {
3714 // Filter out instructions into which we cannot fold.
3715 unsigned NumBytes;
3716 int64_t OffsetScale = 1;
3717 switch (MemI.getOpcode()) {
3718 default:
3719 return false;
3720
3721 case AArch64::LDURQi:
3722 case AArch64::STURQi:
3723 NumBytes = 16;
3724 break;
3725
3726 case AArch64::LDURDi:
3727 case AArch64::STURDi:
3728 case AArch64::LDURXi:
3729 case AArch64::STURXi:
3730 NumBytes = 8;
3731 break;
3732
3733 case AArch64::LDURWi:
3734 case AArch64::LDURSWi:
3735 case AArch64::STURWi:
3736 NumBytes = 4;
3737 break;
3738
3739 case AArch64::LDURHi:
3740 case AArch64::STURHi:
3741 case AArch64::LDURHHi:
3742 case AArch64::STURHHi:
3743 case AArch64::LDURSHXi:
3744 case AArch64::LDURSHWi:
3745 NumBytes = 2;
3746 break;
3747
3748 case AArch64::LDRBroX:
3749 case AArch64::LDRBBroX:
3750 case AArch64::LDRSBXroX:
3751 case AArch64::LDRSBWroX:
3752 case AArch64::STRBroX:
3753 case AArch64::STRBBroX:
3754 case AArch64::LDURBi:
3755 case AArch64::LDURBBi:
3756 case AArch64::LDURSBXi:
3757 case AArch64::LDURSBWi:
3758 case AArch64::STURBi:
3759 case AArch64::STURBBi:
3760 case AArch64::LDRBui:
3761 case AArch64::LDRBBui:
3762 case AArch64::LDRSBXui:
3763 case AArch64::LDRSBWui:
3764 case AArch64::STRBui:
3765 case AArch64::STRBBui:
3766 NumBytes = 1;
3767 break;
3768
3769 case AArch64::LDRQroX:
3770 case AArch64::STRQroX:
3771 case AArch64::LDRQui:
3772 case AArch64::STRQui:
3773 NumBytes = 16;
3774 OffsetScale = 16;
3775 break;
3776
3777 case AArch64::LDRDroX:
3778 case AArch64::STRDroX:
3779 case AArch64::LDRXroX:
3780 case AArch64::STRXroX:
3781 case AArch64::LDRDui:
3782 case AArch64::STRDui:
3783 case AArch64::LDRXui:
3784 case AArch64::STRXui:
3785 NumBytes = 8;
3786 OffsetScale = 8;
3787 break;
3788
3789 case AArch64::LDRWroX:
3790 case AArch64::LDRSWroX:
3791 case AArch64::STRWroX:
3792 case AArch64::LDRWui:
3793 case AArch64::LDRSWui:
3794 case AArch64::STRWui:
3795 NumBytes = 4;
3796 OffsetScale = 4;
3797 break;
3798
3799 case AArch64::LDRHroX:
3800 case AArch64::STRHroX:
3801 case AArch64::LDRHHroX:
3802 case AArch64::STRHHroX:
3803 case AArch64::LDRSHXroX:
3804 case AArch64::LDRSHWroX:
3805 case AArch64::LDRHui:
3806 case AArch64::STRHui:
3807 case AArch64::LDRHHui:
3808 case AArch64::STRHHui:
3809 case AArch64::LDRSHXui:
3810 case AArch64::LDRSHWui:
3811 NumBytes = 2;
3812 OffsetScale = 2;
3813 break;
3814 }
3815
3816 // Check the fold operand is not the loaded/stored value.
3817 const MachineOperand &BaseRegOp = MemI.getOperand(i: 0);
3818 if (BaseRegOp.isReg() && BaseRegOp.getReg() == Reg)
3819 return false;
3820
3821 // Handle memory instructions with a [Reg, Reg] addressing mode.
3822 if (MemI.getOperand(i: 2).isReg()) {
3823 // Bail if the addressing mode already includes extension of the offset
3824 // register.
3825 if (MemI.getOperand(i: 3).getImm())
3826 return false;
3827
3828 // Check if we actually have a scaled offset.
3829 if (MemI.getOperand(i: 4).getImm() == 0)
3830 OffsetScale = 1;
3831
3832 // If the address instructions is folded into the base register, then the
3833 // addressing mode must not have a scale. Then we can swap the base and the
3834 // scaled registers.
3835 if (MemI.getOperand(i: 1).getReg() == Reg && OffsetScale != 1)
3836 return false;
3837
3838 switch (AddrI.getOpcode()) {
3839 default:
3840 return false;
3841
3842 case AArch64::SBFMXri:
3843 // sxtw Xa, Wm
3844 // ldr Xd, [Xn, Xa, lsl #N]
3845 // ->
3846 // ldr Xd, [Xn, Wm, sxtw #N]
3847 if (AddrI.getOperand(i: 2).getImm() != 0 ||
3848 AddrI.getOperand(i: 3).getImm() != 31)
3849 return false;
3850
3851 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3852 if (AM.BaseReg == Reg)
3853 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3854 AM.ScaledReg = AddrI.getOperand(i: 1).getReg();
3855 AM.Scale = OffsetScale;
3856 AM.Displacement = 0;
3857 AM.Form = ExtAddrMode::Formula::SExtScaledReg;
3858 return true;
3859
3860 case TargetOpcode::SUBREG_TO_REG: {
3861 // mov Wa, Wm
3862 // ldr Xd, [Xn, Xa, lsl #N]
3863 // ->
3864 // ldr Xd, [Xn, Wm, uxtw #N]
3865
3866 // Zero-extension looks like an ORRWrs followed by a SUBREG_TO_REG.
3867 if (AddrI.getOperand(i: 2).getImm() != AArch64::sub_32)
3868 return false;
3869
3870 const MachineRegisterInfo &MRI = AddrI.getMF()->getRegInfo();
3871 Register OffsetReg = AddrI.getOperand(i: 1).getReg();
3872 if (!OffsetReg.isVirtual() || !MRI.hasOneNonDBGUse(RegNo: OffsetReg))
3873 return false;
3874
3875 const MachineInstr &DefMI = *MRI.getVRegDef(Reg: OffsetReg);
3876 if (DefMI.getOpcode() != AArch64::ORRWrs ||
3877 DefMI.getOperand(i: 1).getReg() != AArch64::WZR ||
3878 DefMI.getOperand(i: 3).getImm() != 0)
3879 return false;
3880
3881 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3882 if (AM.BaseReg == Reg)
3883 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3884 AM.ScaledReg = DefMI.getOperand(i: 2).getReg();
3885 AM.Scale = OffsetScale;
3886 AM.Displacement = 0;
3887 AM.Form = ExtAddrMode::Formula::ZExtScaledReg;
3888 return true;
3889 }
3890 }
3891 }
3892
3893 // Handle memory instructions with a [Reg, #Imm] addressing mode.
3894
3895 // Check we are not breaking a potential conversion to an LDP.
3896 auto validateOffsetForLDP = [](unsigned NumBytes, int64_t OldOffset,
3897 int64_t NewOffset) -> bool {
3898 int64_t MinOffset, MaxOffset;
3899 switch (NumBytes) {
3900 default:
3901 return true;
3902 case 4:
3903 MinOffset = -256;
3904 MaxOffset = 252;
3905 break;
3906 case 8:
3907 MinOffset = -512;
3908 MaxOffset = 504;
3909 break;
3910 case 16:
3911 MinOffset = -1024;
3912 MaxOffset = 1008;
3913 break;
3914 }
3915 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3916 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3917 };
3918 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) -> bool {
3919 int64_t OldOffset = MemI.getOperand(i: 2).getImm() * OffsetScale;
3920 int64_t NewOffset = OldOffset + Disp;
3921 if (!isLegalAddressingMode(NumBytes, Offset: NewOffset, /* Scale */ 0))
3922 return false;
3923 // If the old offset would fit into an LDP, but the new offset wouldn't,
3924 // bail out.
3925 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3926 return false;
3927 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3928 AM.ScaledReg = 0;
3929 AM.Scale = 0;
3930 AM.Displacement = NewOffset;
3931 AM.Form = ExtAddrMode::Formula::Basic;
3932 return true;
3933 };
3934
3935 auto canFoldAddRegIntoAddrMode =
3936 [&](int64_t Scale,
3937 ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool {
3938 if (MemI.getOperand(i: 2).getImm() != 0)
3939 return false;
3940 if ((unsigned)Scale != Scale)
3941 return false;
3942 if (!isLegalAddressingMode(NumBytes, /* Offset */ 0, Scale))
3943 return false;
3944 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3945 AM.ScaledReg = AddrI.getOperand(i: 2).getReg();
3946 AM.Scale = Scale;
3947 AM.Displacement = 0;
3948 AM.Form = Form;
3949 return true;
3950 };
3951
3952 auto avoidSlowSTRQ = [&](const MachineInstr &MemI) {
3953 unsigned Opcode = MemI.getOpcode();
3954 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3955 Subtarget.isSTRQroSlow();
3956 };
3957
3958 int64_t Disp = 0;
3959 const bool OptSize = MemI.getMF()->getFunction().hasOptSize();
3960 switch (AddrI.getOpcode()) {
3961 default:
3962 return false;
3963
3964 case AArch64::ADDXri:
3965 // add Xa, Xn, #N
3966 // ldr Xd, [Xa, #M]
3967 // ->
3968 // ldr Xd, [Xn, #N'+M]
3969 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3970 return canFoldAddSubImmIntoAddrMode(Disp);
3971
3972 case AArch64::SUBXri:
3973 // sub Xa, Xn, #N
3974 // ldr Xd, [Xa, #M]
3975 // ->
3976 // ldr Xd, [Xn, #N'+M]
3977 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3978 return canFoldAddSubImmIntoAddrMode(-Disp);
3979
3980 case AArch64::ADDXrs: {
3981 // add Xa, Xn, Xm, lsl #N
3982 // ldr Xd, [Xa]
3983 // ->
3984 // ldr Xd, [Xn, Xm, lsl #N]
3985
3986 // Don't fold the add if the result would be slower, unless optimising for
3987 // size.
3988 unsigned Shift = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
3989 if (AArch64_AM::getShiftType(Imm: Shift) != AArch64_AM::ShiftExtendType::LSL)
3990 return false;
3991 Shift = AArch64_AM::getShiftValue(Imm: Shift);
3992 if (!OptSize) {
3993 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3994 return false;
3995 if (avoidSlowSTRQ(MemI))
3996 return false;
3997 }
3998 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3999 }
4000
4001 case AArch64::ADDXrr:
4002 // add Xa, Xn, Xm
4003 // ldr Xd, [Xa]
4004 // ->
4005 // ldr Xd, [Xn, Xm, lsl #0]
4006
4007 // Don't fold the add if the result would be slower, unless optimising for
4008 // size.
4009 if (!OptSize && avoidSlowSTRQ(MemI))
4010 return false;
4011 return canFoldAddRegIntoAddrMode(1);
4012
4013 case AArch64::ADDXrx:
4014 // add Xa, Xn, Wm, {s,u}xtw #N
4015 // ldr Xd, [Xa]
4016 // ->
4017 // ldr Xd, [Xn, Wm, {s,u}xtw #N]
4018
4019 // Don't fold the add if the result would be slower, unless optimising for
4020 // size.
4021 if (!OptSize && avoidSlowSTRQ(MemI))
4022 return false;
4023
4024 // Can fold only sign-/zero-extend of a word.
4025 unsigned Imm = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
4026 AArch64_AM::ShiftExtendType Extend = AArch64_AM::getArithExtendType(Imm);
4027 if (Extend != AArch64_AM::UXTW && Extend != AArch64_AM::SXTW)
4028 return false;
4029
4030 return canFoldAddRegIntoAddrMode(
4031 1ULL << AArch64_AM::getArithShiftValue(Imm),
4032 (Extend == AArch64_AM::SXTW) ? ExtAddrMode::Formula::SExtScaledReg
4033 : ExtAddrMode::Formula::ZExtScaledReg);
4034 }
4035}
4036
4037// Given an opcode for an instruction with a [Reg, #Imm] addressing mode,
4038// return the opcode of an instruction performing the same operation, but using
4039// the [Reg, Reg] addressing mode.
4040static unsigned regOffsetOpcode(unsigned Opcode) {
4041 switch (Opcode) {
4042 default:
4043 llvm_unreachable("Address folding not implemented for instruction");
4044
4045 case AArch64::LDURQi:
4046 case AArch64::LDRQui:
4047 return AArch64::LDRQroX;
4048 case AArch64::STURQi:
4049 case AArch64::STRQui:
4050 return AArch64::STRQroX;
4051 case AArch64::LDURDi:
4052 case AArch64::LDRDui:
4053 return AArch64::LDRDroX;
4054 case AArch64::STURDi:
4055 case AArch64::STRDui:
4056 return AArch64::STRDroX;
4057 case AArch64::LDURXi:
4058 case AArch64::LDRXui:
4059 return AArch64::LDRXroX;
4060 case AArch64::STURXi:
4061 case AArch64::STRXui:
4062 return AArch64::STRXroX;
4063 case AArch64::LDURWi:
4064 case AArch64::LDRWui:
4065 return AArch64::LDRWroX;
4066 case AArch64::LDURSWi:
4067 case AArch64::LDRSWui:
4068 return AArch64::LDRSWroX;
4069 case AArch64::STURWi:
4070 case AArch64::STRWui:
4071 return AArch64::STRWroX;
4072 case AArch64::LDURHi:
4073 case AArch64::LDRHui:
4074 return AArch64::LDRHroX;
4075 case AArch64::STURHi:
4076 case AArch64::STRHui:
4077 return AArch64::STRHroX;
4078 case AArch64::LDURHHi:
4079 case AArch64::LDRHHui:
4080 return AArch64::LDRHHroX;
4081 case AArch64::STURHHi:
4082 case AArch64::STRHHui:
4083 return AArch64::STRHHroX;
4084 case AArch64::LDURSHXi:
4085 case AArch64::LDRSHXui:
4086 return AArch64::LDRSHXroX;
4087 case AArch64::LDURSHWi:
4088 case AArch64::LDRSHWui:
4089 return AArch64::LDRSHWroX;
4090 case AArch64::LDURBi:
4091 case AArch64::LDRBui:
4092 return AArch64::LDRBroX;
4093 case AArch64::LDURBBi:
4094 case AArch64::LDRBBui:
4095 return AArch64::LDRBBroX;
4096 case AArch64::LDURSBXi:
4097 case AArch64::LDRSBXui:
4098 return AArch64::LDRSBXroX;
4099 case AArch64::LDURSBWi:
4100 case AArch64::LDRSBWui:
4101 return AArch64::LDRSBWroX;
4102 case AArch64::STURBi:
4103 case AArch64::STRBui:
4104 return AArch64::STRBroX;
4105 case AArch64::STURBBi:
4106 case AArch64::STRBBui:
4107 return AArch64::STRBBroX;
4108 }
4109}
4110
4111// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4112// the opcode of an instruction performing the same operation, but using the
4113// [Reg, #Imm] addressing mode with scaled offset.
4114unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale) {
4115 switch (Opcode) {
4116 default:
4117 llvm_unreachable("Address folding not implemented for instruction");
4118
4119 case AArch64::LDURQi:
4120 Scale = 16;
4121 return AArch64::LDRQui;
4122 case AArch64::STURQi:
4123 Scale = 16;
4124 return AArch64::STRQui;
4125 case AArch64::LDURDi:
4126 Scale = 8;
4127 return AArch64::LDRDui;
4128 case AArch64::STURDi:
4129 Scale = 8;
4130 return AArch64::STRDui;
4131 case AArch64::LDURXi:
4132 Scale = 8;
4133 return AArch64::LDRXui;
4134 case AArch64::STURXi:
4135 Scale = 8;
4136 return AArch64::STRXui;
4137 case AArch64::LDURWi:
4138 Scale = 4;
4139 return AArch64::LDRWui;
4140 case AArch64::LDURSWi:
4141 Scale = 4;
4142 return AArch64::LDRSWui;
4143 case AArch64::STURWi:
4144 Scale = 4;
4145 return AArch64::STRWui;
4146 case AArch64::LDURHi:
4147 Scale = 2;
4148 return AArch64::LDRHui;
4149 case AArch64::STURHi:
4150 Scale = 2;
4151 return AArch64::STRHui;
4152 case AArch64::LDURHHi:
4153 Scale = 2;
4154 return AArch64::LDRHHui;
4155 case AArch64::STURHHi:
4156 Scale = 2;
4157 return AArch64::STRHHui;
4158 case AArch64::LDURSHXi:
4159 Scale = 2;
4160 return AArch64::LDRSHXui;
4161 case AArch64::LDURSHWi:
4162 Scale = 2;
4163 return AArch64::LDRSHWui;
4164 case AArch64::LDURBi:
4165 Scale = 1;
4166 return AArch64::LDRBui;
4167 case AArch64::LDURBBi:
4168 Scale = 1;
4169 return AArch64::LDRBBui;
4170 case AArch64::LDURSBXi:
4171 Scale = 1;
4172 return AArch64::LDRSBXui;
4173 case AArch64::LDURSBWi:
4174 Scale = 1;
4175 return AArch64::LDRSBWui;
4176 case AArch64::STURBi:
4177 Scale = 1;
4178 return AArch64::STRBui;
4179 case AArch64::STURBBi:
4180 Scale = 1;
4181 return AArch64::STRBBui;
4182 case AArch64::LDRQui:
4183 case AArch64::STRQui:
4184 Scale = 16;
4185 return Opcode;
4186 case AArch64::LDRDui:
4187 case AArch64::STRDui:
4188 case AArch64::LDRXui:
4189 case AArch64::STRXui:
4190 Scale = 8;
4191 return Opcode;
4192 case AArch64::LDRWui:
4193 case AArch64::LDRSWui:
4194 case AArch64::STRWui:
4195 Scale = 4;
4196 return Opcode;
4197 case AArch64::LDRHui:
4198 case AArch64::STRHui:
4199 case AArch64::LDRHHui:
4200 case AArch64::STRHHui:
4201 case AArch64::LDRSHXui:
4202 case AArch64::LDRSHWui:
4203 Scale = 2;
4204 return Opcode;
4205 case AArch64::LDRBui:
4206 case AArch64::LDRBBui:
4207 case AArch64::LDRSBXui:
4208 case AArch64::LDRSBWui:
4209 case AArch64::STRBui:
4210 case AArch64::STRBBui:
4211 Scale = 1;
4212 return Opcode;
4213 }
4214}
4215
4216// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4217// the opcode of an instruction performing the same operation, but using the
4218// [Reg, #Imm] addressing mode with unscaled offset.
4219unsigned unscaledOffsetOpcode(unsigned Opcode) {
4220 switch (Opcode) {
4221 default:
4222 llvm_unreachable("Address folding not implemented for instruction");
4223
4224 case AArch64::LDURQi:
4225 case AArch64::STURQi:
4226 case AArch64::LDURDi:
4227 case AArch64::STURDi:
4228 case AArch64::LDURXi:
4229 case AArch64::STURXi:
4230 case AArch64::LDURWi:
4231 case AArch64::LDURSWi:
4232 case AArch64::STURWi:
4233 case AArch64::LDURHi:
4234 case AArch64::STURHi:
4235 case AArch64::LDURHHi:
4236 case AArch64::STURHHi:
4237 case AArch64::LDURSHXi:
4238 case AArch64::LDURSHWi:
4239 case AArch64::LDURBi:
4240 case AArch64::STURBi:
4241 case AArch64::LDURBBi:
4242 case AArch64::STURBBi:
4243 case AArch64::LDURSBWi:
4244 case AArch64::LDURSBXi:
4245 return Opcode;
4246 case AArch64::LDRQui:
4247 return AArch64::LDURQi;
4248 case AArch64::STRQui:
4249 return AArch64::STURQi;
4250 case AArch64::LDRDui:
4251 return AArch64::LDURDi;
4252 case AArch64::STRDui:
4253 return AArch64::STURDi;
4254 case AArch64::LDRXui:
4255 return AArch64::LDURXi;
4256 case AArch64::STRXui:
4257 return AArch64::STURXi;
4258 case AArch64::LDRWui:
4259 return AArch64::LDURWi;
4260 case AArch64::LDRSWui:
4261 return AArch64::LDURSWi;
4262 case AArch64::STRWui:
4263 return AArch64::STURWi;
4264 case AArch64::LDRHui:
4265 return AArch64::LDURHi;
4266 case AArch64::STRHui:
4267 return AArch64::STURHi;
4268 case AArch64::LDRHHui:
4269 return AArch64::LDURHHi;
4270 case AArch64::STRHHui:
4271 return AArch64::STURHHi;
4272 case AArch64::LDRSHXui:
4273 return AArch64::LDURSHXi;
4274 case AArch64::LDRSHWui:
4275 return AArch64::LDURSHWi;
4276 case AArch64::LDRBBui:
4277 return AArch64::LDURBBi;
4278 case AArch64::LDRBui:
4279 return AArch64::LDURBi;
4280 case AArch64::STRBBui:
4281 return AArch64::STURBBi;
4282 case AArch64::STRBui:
4283 return AArch64::STURBi;
4284 case AArch64::LDRSBWui:
4285 return AArch64::LDURSBWi;
4286 case AArch64::LDRSBXui:
4287 return AArch64::LDURSBXi;
4288 }
4289}
4290
4291// Given the opcode of a memory load/store instruction, return the opcode of an
4292// instruction performing the same operation, but using
4293// the [Reg, Reg, {s,u}xtw #N] addressing mode with sign-/zero-extend of the
4294// offset register.
4295static unsigned offsetExtendOpcode(unsigned Opcode) {
4296 switch (Opcode) {
4297 default:
4298 llvm_unreachable("Address folding not implemented for instruction");
4299
4300 case AArch64::LDRQroX:
4301 case AArch64::LDURQi:
4302 case AArch64::LDRQui:
4303 return AArch64::LDRQroW;
4304 case AArch64::STRQroX:
4305 case AArch64::STURQi:
4306 case AArch64::STRQui:
4307 return AArch64::STRQroW;
4308 case AArch64::LDRDroX:
4309 case AArch64::LDURDi:
4310 case AArch64::LDRDui:
4311 return AArch64::LDRDroW;
4312 case AArch64::STRDroX:
4313 case AArch64::STURDi:
4314 case AArch64::STRDui:
4315 return AArch64::STRDroW;
4316 case AArch64::LDRXroX:
4317 case AArch64::LDURXi:
4318 case AArch64::LDRXui:
4319 return AArch64::LDRXroW;
4320 case AArch64::STRXroX:
4321 case AArch64::STURXi:
4322 case AArch64::STRXui:
4323 return AArch64::STRXroW;
4324 case AArch64::LDRWroX:
4325 case AArch64::LDURWi:
4326 case AArch64::LDRWui:
4327 return AArch64::LDRWroW;
4328 case AArch64::LDRSWroX:
4329 case AArch64::LDURSWi:
4330 case AArch64::LDRSWui:
4331 return AArch64::LDRSWroW;
4332 case AArch64::STRWroX:
4333 case AArch64::STURWi:
4334 case AArch64::STRWui:
4335 return AArch64::STRWroW;
4336 case AArch64::LDRHroX:
4337 case AArch64::LDURHi:
4338 case AArch64::LDRHui:
4339 return AArch64::LDRHroW;
4340 case AArch64::STRHroX:
4341 case AArch64::STURHi:
4342 case AArch64::STRHui:
4343 return AArch64::STRHroW;
4344 case AArch64::LDRHHroX:
4345 case AArch64::LDURHHi:
4346 case AArch64::LDRHHui:
4347 return AArch64::LDRHHroW;
4348 case AArch64::STRHHroX:
4349 case AArch64::STURHHi:
4350 case AArch64::STRHHui:
4351 return AArch64::STRHHroW;
4352 case AArch64::LDRSHXroX:
4353 case AArch64::LDURSHXi:
4354 case AArch64::LDRSHXui:
4355 return AArch64::LDRSHXroW;
4356 case AArch64::LDRSHWroX:
4357 case AArch64::LDURSHWi:
4358 case AArch64::LDRSHWui:
4359 return AArch64::LDRSHWroW;
4360 case AArch64::LDRBroX:
4361 case AArch64::LDURBi:
4362 case AArch64::LDRBui:
4363 return AArch64::LDRBroW;
4364 case AArch64::LDRBBroX:
4365 case AArch64::LDURBBi:
4366 case AArch64::LDRBBui:
4367 return AArch64::LDRBBroW;
4368 case AArch64::LDRSBXroX:
4369 case AArch64::LDURSBXi:
4370 case AArch64::LDRSBXui:
4371 return AArch64::LDRSBXroW;
4372 case AArch64::LDRSBWroX:
4373 case AArch64::LDURSBWi:
4374 case AArch64::LDRSBWui:
4375 return AArch64::LDRSBWroW;
4376 case AArch64::STRBroX:
4377 case AArch64::STURBi:
4378 case AArch64::STRBui:
4379 return AArch64::STRBroW;
4380 case AArch64::STRBBroX:
4381 case AArch64::STURBBi:
4382 case AArch64::STRBBui:
4383 return AArch64::STRBBroW;
4384 }
4385}
4386
4387MachineInstr *AArch64InstrInfo::emitLdStWithAddr(MachineInstr &MemI,
4388 const ExtAddrMode &AM) const {
4389
4390 const DebugLoc &DL = MemI.getDebugLoc();
4391 MachineBasicBlock &MBB = *MemI.getParent();
4392 MachineRegisterInfo &MRI = MemI.getMF()->getRegInfo();
4393
4394 if (AM.Form == ExtAddrMode::Formula::Basic) {
4395 if (AM.ScaledReg) {
4396 // The new instruction will be in the form `ldr Rt, [Xn, Xm, lsl #imm]`.
4397 unsigned Opcode = regOffsetOpcode(Opcode: MemI.getOpcode());
4398 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4399 auto B = BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4400 .addReg(RegNo: MemI.getOperand(i: 0).getReg(),
4401 Flags: getDefRegState(B: MemI.mayLoad()))
4402 .addReg(RegNo: AM.BaseReg)
4403 .addReg(RegNo: AM.ScaledReg)
4404 .addImm(Val: 0)
4405 .addImm(Val: AM.Scale > 1)
4406 .setMemRefs(MemI.memoperands())
4407 .setMIFlags(MemI.getFlags());
4408 return B.getInstr();
4409 }
4410
4411 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
4412 "Addressing mode not supported for folding");
4413
4414 // The new instruction will be in the form `ld[u]r Rt, [Xn, #imm]`.
4415 unsigned Scale = 1;
4416 unsigned Opcode = MemI.getOpcode();
4417 if (isInt<9>(x: AM.Displacement))
4418 Opcode = unscaledOffsetOpcode(Opcode);
4419 else
4420 Opcode = scaledOffsetOpcode(Opcode, Scale);
4421
4422 auto B =
4423 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4424 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4425 .addReg(RegNo: AM.BaseReg)
4426 .addImm(Val: AM.Displacement / Scale)
4427 .setMemRefs(MemI.memoperands())
4428 .setMIFlags(MemI.getFlags());
4429 return B.getInstr();
4430 }
4431
4432 if (AM.Form == ExtAddrMode::Formula::SExtScaledReg ||
4433 AM.Form == ExtAddrMode::Formula::ZExtScaledReg) {
4434 // The new instruction will be in the form `ldr Rt, [Xn, Wm, {s,u}xtw #N]`.
4435 assert(AM.ScaledReg && !AM.Displacement &&
4436 "Address offset can be a register or an immediate, but not both");
4437 unsigned Opcode = offsetExtendOpcode(Opcode: MemI.getOpcode());
4438 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4439 // Make sure the offset register is in the correct register class.
4440 Register OffsetReg = AM.ScaledReg;
4441 const TargetRegisterClass *RC = MRI.getRegClass(Reg: OffsetReg);
4442 if (RC->hasSuperClassEq(RC: &AArch64::GPR64RegClass)) {
4443 OffsetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4444 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: OffsetReg)
4445 .addReg(RegNo: AM.ScaledReg, Flags: {}, SubReg: AArch64::sub_32);
4446 }
4447 auto B =
4448 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4449 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4450 .addReg(RegNo: AM.BaseReg)
4451 .addReg(RegNo: OffsetReg)
4452 .addImm(Val: AM.Form == ExtAddrMode::Formula::SExtScaledReg)
4453 .addImm(Val: AM.Scale != 1)
4454 .setMemRefs(MemI.memoperands())
4455 .setMIFlags(MemI.getFlags());
4456
4457 return B.getInstr();
4458 }
4459
4460 llvm_unreachable(
4461 "Function must not be called with an addressing mode it can't handle");
4462}
4463
4464/// Return true if the opcode is a post-index ld/st instruction, which really
4465/// loads from base+0.
4466static bool isPostIndexLdStOpcode(unsigned Opcode) {
4467 switch (Opcode) {
4468 default:
4469 return false;
4470 case AArch64::LD1Fourv16b_POST:
4471 case AArch64::LD1Fourv1d_POST:
4472 case AArch64::LD1Fourv2d_POST:
4473 case AArch64::LD1Fourv2s_POST:
4474 case AArch64::LD1Fourv4h_POST:
4475 case AArch64::LD1Fourv4s_POST:
4476 case AArch64::LD1Fourv8b_POST:
4477 case AArch64::LD1Fourv8h_POST:
4478 case AArch64::LD1Onev16b_POST:
4479 case AArch64::LD1Onev1d_POST:
4480 case AArch64::LD1Onev2d_POST:
4481 case AArch64::LD1Onev2s_POST:
4482 case AArch64::LD1Onev4h_POST:
4483 case AArch64::LD1Onev4s_POST:
4484 case AArch64::LD1Onev8b_POST:
4485 case AArch64::LD1Onev8h_POST:
4486 case AArch64::LD1Rv16b_POST:
4487 case AArch64::LD1Rv1d_POST:
4488 case AArch64::LD1Rv2d_POST:
4489 case AArch64::LD1Rv2s_POST:
4490 case AArch64::LD1Rv4h_POST:
4491 case AArch64::LD1Rv4s_POST:
4492 case AArch64::LD1Rv8b_POST:
4493 case AArch64::LD1Rv8h_POST:
4494 case AArch64::LD1Threev16b_POST:
4495 case AArch64::LD1Threev1d_POST:
4496 case AArch64::LD1Threev2d_POST:
4497 case AArch64::LD1Threev2s_POST:
4498 case AArch64::LD1Threev4h_POST:
4499 case AArch64::LD1Threev4s_POST:
4500 case AArch64::LD1Threev8b_POST:
4501 case AArch64::LD1Threev8h_POST:
4502 case AArch64::LD1Twov16b_POST:
4503 case AArch64::LD1Twov1d_POST:
4504 case AArch64::LD1Twov2d_POST:
4505 case AArch64::LD1Twov2s_POST:
4506 case AArch64::LD1Twov4h_POST:
4507 case AArch64::LD1Twov4s_POST:
4508 case AArch64::LD1Twov8b_POST:
4509 case AArch64::LD1Twov8h_POST:
4510 case AArch64::LD1i16_POST:
4511 case AArch64::LD1i32_POST:
4512 case AArch64::LD1i64_POST:
4513 case AArch64::LD1i8_POST:
4514 case AArch64::LD2Rv16b_POST:
4515 case AArch64::LD2Rv1d_POST:
4516 case AArch64::LD2Rv2d_POST:
4517 case AArch64::LD2Rv2s_POST:
4518 case AArch64::LD2Rv4h_POST:
4519 case AArch64::LD2Rv4s_POST:
4520 case AArch64::LD2Rv8b_POST:
4521 case AArch64::LD2Rv8h_POST:
4522 case AArch64::LD2Twov16b_POST:
4523 case AArch64::LD2Twov2d_POST:
4524 case AArch64::LD2Twov2s_POST:
4525 case AArch64::LD2Twov4h_POST:
4526 case AArch64::LD2Twov4s_POST:
4527 case AArch64::LD2Twov8b_POST:
4528 case AArch64::LD2Twov8h_POST:
4529 case AArch64::LD2i16_POST:
4530 case AArch64::LD2i32_POST:
4531 case AArch64::LD2i64_POST:
4532 case AArch64::LD2i8_POST:
4533 case AArch64::LD3Rv16b_POST:
4534 case AArch64::LD3Rv1d_POST:
4535 case AArch64::LD3Rv2d_POST:
4536 case AArch64::LD3Rv2s_POST:
4537 case AArch64::LD3Rv4h_POST:
4538 case AArch64::LD3Rv4s_POST:
4539 case AArch64::LD3Rv8b_POST:
4540 case AArch64::LD3Rv8h_POST:
4541 case AArch64::LD3Threev16b_POST:
4542 case AArch64::LD3Threev2d_POST:
4543 case AArch64::LD3Threev2s_POST:
4544 case AArch64::LD3Threev4h_POST:
4545 case AArch64::LD3Threev4s_POST:
4546 case AArch64::LD3Threev8b_POST:
4547 case AArch64::LD3Threev8h_POST:
4548 case AArch64::LD3i16_POST:
4549 case AArch64::LD3i32_POST:
4550 case AArch64::LD3i64_POST:
4551 case AArch64::LD3i8_POST:
4552 case AArch64::LD4Fourv16b_POST:
4553 case AArch64::LD4Fourv2d_POST:
4554 case AArch64::LD4Fourv2s_POST:
4555 case AArch64::LD4Fourv4h_POST:
4556 case AArch64::LD4Fourv4s_POST:
4557 case AArch64::LD4Fourv8b_POST:
4558 case AArch64::LD4Fourv8h_POST:
4559 case AArch64::LD4Rv16b_POST:
4560 case AArch64::LD4Rv1d_POST:
4561 case AArch64::LD4Rv2d_POST:
4562 case AArch64::LD4Rv2s_POST:
4563 case AArch64::LD4Rv4h_POST:
4564 case AArch64::LD4Rv4s_POST:
4565 case AArch64::LD4Rv8b_POST:
4566 case AArch64::LD4Rv8h_POST:
4567 case AArch64::LD4i16_POST:
4568 case AArch64::LD4i32_POST:
4569 case AArch64::LD4i64_POST:
4570 case AArch64::LD4i8_POST:
4571 case AArch64::LDAPRWpost:
4572 case AArch64::LDAPRXpost:
4573 case AArch64::LDIAPPWpost:
4574 case AArch64::LDIAPPXpost:
4575 case AArch64::LDPDpost:
4576 case AArch64::LDPQpost:
4577 case AArch64::LDPSWpost:
4578 case AArch64::LDPSpost:
4579 case AArch64::LDPWpost:
4580 case AArch64::LDPXpost:
4581 case AArch64::LDRBBpost:
4582 case AArch64::LDRBpost:
4583 case AArch64::LDRDpost:
4584 case AArch64::LDRHHpost:
4585 case AArch64::LDRHpost:
4586 case AArch64::LDRQpost:
4587 case AArch64::LDRSBWpost:
4588 case AArch64::LDRSBXpost:
4589 case AArch64::LDRSHWpost:
4590 case AArch64::LDRSHXpost:
4591 case AArch64::LDRSWpost:
4592 case AArch64::LDRSpost:
4593 case AArch64::LDRWpost:
4594 case AArch64::LDRXpost:
4595 case AArch64::ST1Fourv16b_POST:
4596 case AArch64::ST1Fourv1d_POST:
4597 case AArch64::ST1Fourv2d_POST:
4598 case AArch64::ST1Fourv2s_POST:
4599 case AArch64::ST1Fourv4h_POST:
4600 case AArch64::ST1Fourv4s_POST:
4601 case AArch64::ST1Fourv8b_POST:
4602 case AArch64::ST1Fourv8h_POST:
4603 case AArch64::ST1Onev16b_POST:
4604 case AArch64::ST1Onev1d_POST:
4605 case AArch64::ST1Onev2d_POST:
4606 case AArch64::ST1Onev2s_POST:
4607 case AArch64::ST1Onev4h_POST:
4608 case AArch64::ST1Onev4s_POST:
4609 case AArch64::ST1Onev8b_POST:
4610 case AArch64::ST1Onev8h_POST:
4611 case AArch64::ST1Threev16b_POST:
4612 case AArch64::ST1Threev1d_POST:
4613 case AArch64::ST1Threev2d_POST:
4614 case AArch64::ST1Threev2s_POST:
4615 case AArch64::ST1Threev4h_POST:
4616 case AArch64::ST1Threev4s_POST:
4617 case AArch64::ST1Threev8b_POST:
4618 case AArch64::ST1Threev8h_POST:
4619 case AArch64::ST1Twov16b_POST:
4620 case AArch64::ST1Twov1d_POST:
4621 case AArch64::ST1Twov2d_POST:
4622 case AArch64::ST1Twov2s_POST:
4623 case AArch64::ST1Twov4h_POST:
4624 case AArch64::ST1Twov4s_POST:
4625 case AArch64::ST1Twov8b_POST:
4626 case AArch64::ST1Twov8h_POST:
4627 case AArch64::ST1i16_POST:
4628 case AArch64::ST1i32_POST:
4629 case AArch64::ST1i64_POST:
4630 case AArch64::ST1i8_POST:
4631 case AArch64::ST2GPostIndex:
4632 case AArch64::ST2Twov16b_POST:
4633 case AArch64::ST2Twov2d_POST:
4634 case AArch64::ST2Twov2s_POST:
4635 case AArch64::ST2Twov4h_POST:
4636 case AArch64::ST2Twov4s_POST:
4637 case AArch64::ST2Twov8b_POST:
4638 case AArch64::ST2Twov8h_POST:
4639 case AArch64::ST2i16_POST:
4640 case AArch64::ST2i32_POST:
4641 case AArch64::ST2i64_POST:
4642 case AArch64::ST2i8_POST:
4643 case AArch64::ST3Threev16b_POST:
4644 case AArch64::ST3Threev2d_POST:
4645 case AArch64::ST3Threev2s_POST:
4646 case AArch64::ST3Threev4h_POST:
4647 case AArch64::ST3Threev4s_POST:
4648 case AArch64::ST3Threev8b_POST:
4649 case AArch64::ST3Threev8h_POST:
4650 case AArch64::ST3i16_POST:
4651 case AArch64::ST3i32_POST:
4652 case AArch64::ST3i64_POST:
4653 case AArch64::ST3i8_POST:
4654 case AArch64::ST4Fourv16b_POST:
4655 case AArch64::ST4Fourv2d_POST:
4656 case AArch64::ST4Fourv2s_POST:
4657 case AArch64::ST4Fourv4h_POST:
4658 case AArch64::ST4Fourv4s_POST:
4659 case AArch64::ST4Fourv8b_POST:
4660 case AArch64::ST4Fourv8h_POST:
4661 case AArch64::ST4i16_POST:
4662 case AArch64::ST4i32_POST:
4663 case AArch64::ST4i64_POST:
4664 case AArch64::ST4i8_POST:
4665 case AArch64::STGPostIndex:
4666 case AArch64::STGPpost:
4667 case AArch64::STPDpost:
4668 case AArch64::STPQpost:
4669 case AArch64::STPSpost:
4670 case AArch64::STPWpost:
4671 case AArch64::STPXpost:
4672 case AArch64::STRBBpost:
4673 case AArch64::STRBpost:
4674 case AArch64::STRDpost:
4675 case AArch64::STRHHpost:
4676 case AArch64::STRHpost:
4677 case AArch64::STRQpost:
4678 case AArch64::STRSpost:
4679 case AArch64::STRWpost:
4680 case AArch64::STRXpost:
4681 case AArch64::STZ2GPostIndex:
4682 case AArch64::STZGPostIndex:
4683 return true;
4684 }
4685}
4686
4687bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
4688 const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
4689 bool &OffsetIsScalable, TypeSize &Width,
4690 const TargetRegisterInfo *TRI) const {
4691 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4692 // Handle only loads/stores with base register followed by immediate offset.
4693 if (LdSt.getNumExplicitOperands() == 3) {
4694 // Non-paired instruction (e.g., ldr x1, [x0, #8]).
4695 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
4696 !LdSt.getOperand(i: 2).isImm())
4697 return false;
4698 } else if (LdSt.getNumExplicitOperands() == 4) {
4699 // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
4700 if (!LdSt.getOperand(i: 1).isReg() ||
4701 (!LdSt.getOperand(i: 2).isReg() && !LdSt.getOperand(i: 2).isFI()) ||
4702 !LdSt.getOperand(i: 3).isImm())
4703 return false;
4704 } else
4705 return false;
4706
4707 // Get the scaling factor for the instruction and set the width for the
4708 // instruction.
4709 TypeSize Scale(0U, false);
4710 int64_t Dummy1, Dummy2;
4711
4712 // If this returns false, then it's an instruction we don't want to handle.
4713 if (!getMemOpInfo(Opcode: LdSt.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2))
4714 return false;
4715
4716 // Compute the offset. Offset is calculated as the immediate operand
4717 // multiplied by the scaling factor. Unscaled instructions have scaling factor
4718 // set to 1. Postindex are a special case which have an offset of 0.
4719 if (isPostIndexLdStOpcode(Opcode: LdSt.getOpcode())) {
4720 BaseOp = &LdSt.getOperand(i: 2);
4721 Offset = 0;
4722 } else if (LdSt.getNumExplicitOperands() == 3) {
4723 BaseOp = &LdSt.getOperand(i: 1);
4724 Offset = LdSt.getOperand(i: 2).getImm() * Scale.getKnownMinValue();
4725 } else {
4726 assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
4727 BaseOp = &LdSt.getOperand(i: 2);
4728 Offset = LdSt.getOperand(i: 3).getImm() * Scale.getKnownMinValue();
4729 }
4730 OffsetIsScalable = Scale.isScalable();
4731
4732 return BaseOp->isReg() || BaseOp->isFI();
4733}
4734
4735MachineOperand &
4736AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
4737 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4738 MachineOperand &OfsOp = LdSt.getOperand(i: LdSt.getNumExplicitOperands() - 1);
4739 assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
4740 return OfsOp;
4741}
4742
4743bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
4744 TypeSize &Width, int64_t &MinOffset,
4745 int64_t &MaxOffset) {
4746 switch (Opcode) {
4747 // Not a memory operation or something we want to handle.
4748 default:
4749 Scale = Width = TypeSize::getFixed(ExactSize: 0);
4750 MinOffset = MaxOffset = 0;
4751 return false;
4752 // LDR / STR
4753 case AArch64::LDRQui:
4754 case AArch64::STRQui:
4755 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4756 MinOffset = 0;
4757 MaxOffset = 4095;
4758 break;
4759 case AArch64::LDRXui:
4760 case AArch64::LDRDui:
4761 case AArch64::STRXui:
4762 case AArch64::STRDui:
4763 case AArch64::PRFMui:
4764 Scale = Width = TypeSize::getFixed(ExactSize: 8);
4765 MinOffset = 0;
4766 MaxOffset = 4095;
4767 break;
4768 case AArch64::LDRWui:
4769 case AArch64::LDRSui:
4770 case AArch64::LDRSWui:
4771 case AArch64::STRWui:
4772 case AArch64::STRSui:
4773 Scale = Width = TypeSize::getFixed(ExactSize: 4);
4774 MinOffset = 0;
4775 MaxOffset = 4095;
4776 break;
4777 case AArch64::LDRHui:
4778 case AArch64::LDRHHui:
4779 case AArch64::LDRSHWui:
4780 case AArch64::LDRSHXui:
4781 case AArch64::STRHui:
4782 case AArch64::STRHHui:
4783 Scale = Width = TypeSize::getFixed(ExactSize: 2);
4784 MinOffset = 0;
4785 MaxOffset = 4095;
4786 break;
4787 case AArch64::LDRBui:
4788 case AArch64::LDRBBui:
4789 case AArch64::LDRSBWui:
4790 case AArch64::LDRSBXui:
4791 case AArch64::STRBui:
4792 case AArch64::STRBBui:
4793 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4794 MinOffset = 0;
4795 MaxOffset = 4095;
4796 break;
4797 // post/pre inc
4798 case AArch64::STRQpre:
4799 case AArch64::LDRQpost:
4800 Scale = TypeSize::getFixed(ExactSize: 1);
4801 Width = TypeSize::getFixed(ExactSize: 16);
4802 MinOffset = -256;
4803 MaxOffset = 255;
4804 break;
4805 case AArch64::LDRDpost:
4806 case AArch64::LDRDpre:
4807 case AArch64::LDRXpost:
4808 case AArch64::LDRXpre:
4809 case AArch64::STRDpost:
4810 case AArch64::STRDpre:
4811 case AArch64::STRXpost:
4812 case AArch64::STRXpre:
4813 Scale = TypeSize::getFixed(ExactSize: 1);
4814 Width = TypeSize::getFixed(ExactSize: 8);
4815 MinOffset = -256;
4816 MaxOffset = 255;
4817 break;
4818 case AArch64::STRWpost:
4819 case AArch64::STRWpre:
4820 case AArch64::LDRWpost:
4821 case AArch64::LDRWpre:
4822 case AArch64::STRSpost:
4823 case AArch64::STRSpre:
4824 case AArch64::LDRSpost:
4825 case AArch64::LDRSpre:
4826 Scale = TypeSize::getFixed(ExactSize: 1);
4827 Width = TypeSize::getFixed(ExactSize: 4);
4828 MinOffset = -256;
4829 MaxOffset = 255;
4830 break;
4831 case AArch64::LDRHpost:
4832 case AArch64::LDRHpre:
4833 case AArch64::STRHpost:
4834 case AArch64::STRHpre:
4835 case AArch64::LDRHHpost:
4836 case AArch64::LDRHHpre:
4837 case AArch64::STRHHpost:
4838 case AArch64::STRHHpre:
4839 Scale = TypeSize::getFixed(ExactSize: 1);
4840 Width = TypeSize::getFixed(ExactSize: 2);
4841 MinOffset = -256;
4842 MaxOffset = 255;
4843 break;
4844 case AArch64::LDRBpost:
4845 case AArch64::LDRBpre:
4846 case AArch64::STRBpost:
4847 case AArch64::STRBpre:
4848 case AArch64::LDRBBpost:
4849 case AArch64::LDRBBpre:
4850 case AArch64::STRBBpost:
4851 case AArch64::STRBBpre:
4852 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4853 MinOffset = -256;
4854 MaxOffset = 255;
4855 break;
4856 // Unscaled
4857 case AArch64::LDURQi:
4858 case AArch64::STURQi:
4859 Scale = TypeSize::getFixed(ExactSize: 1);
4860 Width = TypeSize::getFixed(ExactSize: 16);
4861 MinOffset = -256;
4862 MaxOffset = 255;
4863 break;
4864 case AArch64::LDURXi:
4865 case AArch64::LDURDi:
4866 case AArch64::LDAPURXi:
4867 case AArch64::STURXi:
4868 case AArch64::STURDi:
4869 case AArch64::STLURXi:
4870 case AArch64::PRFUMi:
4871 Scale = TypeSize::getFixed(ExactSize: 1);
4872 Width = TypeSize::getFixed(ExactSize: 8);
4873 MinOffset = -256;
4874 MaxOffset = 255;
4875 break;
4876 case AArch64::LDURWi:
4877 case AArch64::LDURSi:
4878 case AArch64::LDURSWi:
4879 case AArch64::LDAPURi:
4880 case AArch64::LDAPURSWi:
4881 case AArch64::STURWi:
4882 case AArch64::STURSi:
4883 case AArch64::STLURWi:
4884 Scale = TypeSize::getFixed(ExactSize: 1);
4885 Width = TypeSize::getFixed(ExactSize: 4);
4886 MinOffset = -256;
4887 MaxOffset = 255;
4888 break;
4889 case AArch64::LDURHi:
4890 case AArch64::LDURHHi:
4891 case AArch64::LDURSHXi:
4892 case AArch64::LDURSHWi:
4893 case AArch64::LDAPURHi:
4894 case AArch64::LDAPURSHWi:
4895 case AArch64::LDAPURSHXi:
4896 case AArch64::STURHi:
4897 case AArch64::STURHHi:
4898 case AArch64::STLURHi:
4899 Scale = TypeSize::getFixed(ExactSize: 1);
4900 Width = TypeSize::getFixed(ExactSize: 2);
4901 MinOffset = -256;
4902 MaxOffset = 255;
4903 break;
4904 case AArch64::LDURBi:
4905 case AArch64::LDURBBi:
4906 case AArch64::LDURSBXi:
4907 case AArch64::LDURSBWi:
4908 case AArch64::LDAPURBi:
4909 case AArch64::LDAPURSBWi:
4910 case AArch64::LDAPURSBXi:
4911 case AArch64::STURBi:
4912 case AArch64::STURBBi:
4913 case AArch64::STLURBi:
4914 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4915 MinOffset = -256;
4916 MaxOffset = 255;
4917 break;
4918 // LDP / STP (including pre/post inc)
4919 case AArch64::LDPQi:
4920 case AArch64::LDNPQi:
4921 case AArch64::STPQi:
4922 case AArch64::STNPQi:
4923 case AArch64::LDPQpost:
4924 case AArch64::LDPQpre:
4925 case AArch64::STPQpost:
4926 case AArch64::STPQpre:
4927 Scale = TypeSize::getFixed(ExactSize: 16);
4928 Width = TypeSize::getFixed(ExactSize: 16 * 2);
4929 MinOffset = -64;
4930 MaxOffset = 63;
4931 break;
4932 case AArch64::LDPXi:
4933 case AArch64::LDPDi:
4934 case AArch64::LDNPXi:
4935 case AArch64::LDNPDi:
4936 case AArch64::STPXi:
4937 case AArch64::STPDi:
4938 case AArch64::STNPXi:
4939 case AArch64::STNPDi:
4940 case AArch64::LDPDpost:
4941 case AArch64::LDPDpre:
4942 case AArch64::LDPXpost:
4943 case AArch64::LDPXpre:
4944 case AArch64::STPDpost:
4945 case AArch64::STPDpre:
4946 case AArch64::STPXpost:
4947 case AArch64::STPXpre:
4948 Scale = TypeSize::getFixed(ExactSize: 8);
4949 Width = TypeSize::getFixed(ExactSize: 8 * 2);
4950 MinOffset = -64;
4951 MaxOffset = 63;
4952 break;
4953 case AArch64::LDPWi:
4954 case AArch64::LDPSi:
4955 case AArch64::LDNPWi:
4956 case AArch64::LDNPSi:
4957 case AArch64::STPWi:
4958 case AArch64::STPSi:
4959 case AArch64::STNPWi:
4960 case AArch64::STNPSi:
4961 case AArch64::LDPSpost:
4962 case AArch64::LDPSpre:
4963 case AArch64::LDPWpost:
4964 case AArch64::LDPWpre:
4965 case AArch64::STPSpost:
4966 case AArch64::STPSpre:
4967 case AArch64::STPWpost:
4968 case AArch64::STPWpre:
4969 Scale = TypeSize::getFixed(ExactSize: 4);
4970 Width = TypeSize::getFixed(ExactSize: 4 * 2);
4971 MinOffset = -64;
4972 MaxOffset = 63;
4973 break;
4974 case AArch64::StoreSwiftAsyncContext:
4975 // Store is an STRXui, but there might be an ADDXri in the expansion too.
4976 Scale = TypeSize::getFixed(ExactSize: 1);
4977 Width = TypeSize::getFixed(ExactSize: 8);
4978 MinOffset = 0;
4979 MaxOffset = 4095;
4980 break;
4981 case AArch64::ADDG:
4982 Scale = TypeSize::getFixed(ExactSize: 16);
4983 Width = TypeSize::getFixed(ExactSize: 0);
4984 MinOffset = 0;
4985 MaxOffset = 63;
4986 break;
4987 case AArch64::TAGPstack:
4988 Scale = TypeSize::getFixed(ExactSize: 16);
4989 Width = TypeSize::getFixed(ExactSize: 0);
4990 // TAGP with a negative offset turns into SUBP, which has a maximum offset
4991 // of 63 (not 64!).
4992 MinOffset = -63;
4993 MaxOffset = 63;
4994 break;
4995 case AArch64::LDG:
4996 case AArch64::STGi:
4997 case AArch64::STGPreIndex:
4998 case AArch64::STGPostIndex:
4999 case AArch64::STZGi:
5000 case AArch64::STZGPreIndex:
5001 case AArch64::STZGPostIndex:
5002 Scale = Width = TypeSize::getFixed(ExactSize: 16);
5003 MinOffset = -256;
5004 MaxOffset = 255;
5005 break;
5006 // SVE
5007 case AArch64::STR_ZZZZXI:
5008 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
5009 case AArch64::LDR_ZZZZXI:
5010 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
5011 Scale = TypeSize::getScalable(MinimumSize: 16);
5012 Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5013 MinOffset = -256;
5014 MaxOffset = 252;
5015 break;
5016 case AArch64::STR_ZZZXI:
5017 case AArch64::LDR_ZZZXI:
5018 Scale = TypeSize::getScalable(MinimumSize: 16);
5019 Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5020 MinOffset = -256;
5021 MaxOffset = 253;
5022 break;
5023 case AArch64::STR_ZZXI:
5024 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
5025 case AArch64::LDR_ZZXI:
5026 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
5027 Scale = TypeSize::getScalable(MinimumSize: 16);
5028 Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5029 MinOffset = -256;
5030 MaxOffset = 254;
5031 break;
5032 case AArch64::LDR_PXI:
5033 case AArch64::STR_PXI:
5034 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5035 MinOffset = -256;
5036 MaxOffset = 255;
5037 break;
5038 case AArch64::LDR_PPXI:
5039 case AArch64::STR_PPXI:
5040 Scale = TypeSize::getScalable(MinimumSize: 2);
5041 Width = TypeSize::getScalable(MinimumSize: 2 * 2);
5042 MinOffset = -256;
5043 MaxOffset = 254;
5044 break;
5045 case AArch64::LDR_ZXI:
5046 case AArch64::STR_ZXI:
5047 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5048 MinOffset = -256;
5049 MaxOffset = 255;
5050 break;
5051 case AArch64::LD1B_IMM:
5052 case AArch64::LD1H_IMM:
5053 case AArch64::LD1W_IMM:
5054 case AArch64::LD1D_IMM:
5055 case AArch64::LDNT1B_ZRI:
5056 case AArch64::LDNT1H_ZRI:
5057 case AArch64::LDNT1W_ZRI:
5058 case AArch64::LDNT1D_ZRI:
5059 case AArch64::ST1B_IMM:
5060 case AArch64::ST1H_IMM:
5061 case AArch64::ST1W_IMM:
5062 case AArch64::ST1D_IMM:
5063 case AArch64::STNT1B_ZRI:
5064 case AArch64::STNT1H_ZRI:
5065 case AArch64::STNT1W_ZRI:
5066 case AArch64::STNT1D_ZRI:
5067 case AArch64::LDNF1B_IMM:
5068 case AArch64::LDNF1H_IMM:
5069 case AArch64::LDNF1W_IMM:
5070 case AArch64::LDNF1D_IMM:
5071 // A full vectors worth of data
5072 // Width = mbytes * elements
5073 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5074 MinOffset = -8;
5075 MaxOffset = 7;
5076 break;
5077 case AArch64::LD2B_IMM:
5078 case AArch64::LD2H_IMM:
5079 case AArch64::LD2W_IMM:
5080 case AArch64::LD2D_IMM:
5081 case AArch64::ST2B_IMM:
5082 case AArch64::ST2H_IMM:
5083 case AArch64::ST2W_IMM:
5084 case AArch64::ST2D_IMM:
5085 case AArch64::LD1B_2Z_IMM:
5086 case AArch64::LD1B_2Z_STRIDED_IMM:
5087 case AArch64::LD1H_2Z_IMM:
5088 case AArch64::LD1H_2Z_STRIDED_IMM:
5089 case AArch64::LD1W_2Z_IMM:
5090 case AArch64::LD1W_2Z_STRIDED_IMM:
5091 case AArch64::LD1D_2Z_IMM:
5092 case AArch64::LD1D_2Z_STRIDED_IMM:
5093 case AArch64::LD1B_2Z_IMM_PSEUDO:
5094 case AArch64::LD1H_2Z_IMM_PSEUDO:
5095 case AArch64::LD1W_2Z_IMM_PSEUDO:
5096 case AArch64::LD1D_2Z_IMM_PSEUDO:
5097 case AArch64::ST1B_2Z_IMM:
5098 case AArch64::ST1B_2Z_STRIDED_IMM:
5099 case AArch64::ST1H_2Z_IMM:
5100 case AArch64::ST1H_2Z_STRIDED_IMM:
5101 case AArch64::ST1W_2Z_IMM:
5102 case AArch64::ST1W_2Z_STRIDED_IMM:
5103 case AArch64::ST1D_2Z_IMM:
5104 case AArch64::ST1D_2Z_STRIDED_IMM:
5105 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
5106 case AArch64::LDNT1B_2Z_IMM:
5107 case AArch64::LDNT1B_2Z_STRIDED_IMM:
5108 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
5109 case AArch64::LDNT1H_2Z_IMM:
5110 case AArch64::LDNT1H_2Z_STRIDED_IMM:
5111 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
5112 case AArch64::LDNT1W_2Z_IMM:
5113 case AArch64::LDNT1W_2Z_STRIDED_IMM:
5114 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
5115 case AArch64::LDNT1D_2Z_IMM:
5116 case AArch64::LDNT1D_2Z_STRIDED_IMM:
5117 case AArch64::STNT1B_2Z_IMM:
5118 case AArch64::STNT1B_2Z_STRIDED_IMM:
5119 case AArch64::STNT1H_2Z_IMM:
5120 case AArch64::STNT1H_2Z_STRIDED_IMM:
5121 case AArch64::STNT1W_2Z_IMM:
5122 case AArch64::STNT1W_2Z_STRIDED_IMM:
5123 case AArch64::STNT1D_2Z_IMM:
5124 case AArch64::STNT1D_2Z_STRIDED_IMM:
5125 case AArch64::ST1B_2Z_IMM_PSEUDO:
5126 case AArch64::ST1H_2Z_IMM_PSEUDO:
5127 case AArch64::ST1W_2Z_IMM_PSEUDO:
5128 case AArch64::ST1D_2Z_IMM_PSEUDO:
5129 case AArch64::STNT1B_2Z_IMM_PSEUDO:
5130 case AArch64::STNT1H_2Z_IMM_PSEUDO:
5131 case AArch64::STNT1W_2Z_IMM_PSEUDO:
5132 case AArch64::STNT1D_2Z_IMM_PSEUDO:
5133 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5134 MinOffset = -8;
5135 MaxOffset = 7;
5136 break;
5137 case AArch64::LD3B_IMM:
5138 case AArch64::LD3H_IMM:
5139 case AArch64::LD3W_IMM:
5140 case AArch64::LD3D_IMM:
5141 case AArch64::ST3B_IMM:
5142 case AArch64::ST3H_IMM:
5143 case AArch64::ST3W_IMM:
5144 case AArch64::ST3D_IMM:
5145 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5146 MinOffset = -8;
5147 MaxOffset = 7;
5148 break;
5149 case AArch64::LD4B_IMM:
5150 case AArch64::LD4H_IMM:
5151 case AArch64::LD4W_IMM:
5152 case AArch64::LD4D_IMM:
5153 case AArch64::ST4B_IMM:
5154 case AArch64::ST4H_IMM:
5155 case AArch64::ST4W_IMM:
5156 case AArch64::ST4D_IMM:
5157 case AArch64::LD1B_4Z_IMM:
5158 case AArch64::LD1B_4Z_STRIDED_IMM:
5159 case AArch64::LD1H_4Z_IMM:
5160 case AArch64::LD1H_4Z_STRIDED_IMM:
5161 case AArch64::LD1W_4Z_IMM:
5162 case AArch64::LD1W_4Z_STRIDED_IMM:
5163 case AArch64::LD1D_4Z_IMM:
5164 case AArch64::LD1D_4Z_STRIDED_IMM:
5165 case AArch64::LD1B_4Z_IMM_PSEUDO:
5166 case AArch64::LD1H_4Z_IMM_PSEUDO:
5167 case AArch64::LD1W_4Z_IMM_PSEUDO:
5168 case AArch64::LD1D_4Z_IMM_PSEUDO:
5169 case AArch64::ST1B_4Z_IMM:
5170 case AArch64::ST1B_4Z_STRIDED_IMM:
5171 case AArch64::ST1H_4Z_IMM:
5172 case AArch64::ST1H_4Z_STRIDED_IMM:
5173 case AArch64::ST1W_4Z_IMM:
5174 case AArch64::ST1W_4Z_STRIDED_IMM:
5175 case AArch64::ST1D_4Z_IMM:
5176 case AArch64::ST1D_4Z_STRIDED_IMM:
5177 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
5178 case AArch64::LDNT1B_4Z_IMM:
5179 case AArch64::LDNT1B_4Z_STRIDED_IMM:
5180 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
5181 case AArch64::LDNT1H_4Z_IMM:
5182 case AArch64::LDNT1H_4Z_STRIDED_IMM:
5183 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
5184 case AArch64::LDNT1W_4Z_IMM:
5185 case AArch64::LDNT1W_4Z_STRIDED_IMM:
5186 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
5187 case AArch64::LDNT1D_4Z_IMM:
5188 case AArch64::LDNT1D_4Z_STRIDED_IMM:
5189 case AArch64::STNT1B_4Z_IMM:
5190 case AArch64::STNT1B_4Z_STRIDED_IMM:
5191 case AArch64::STNT1H_4Z_IMM:
5192 case AArch64::STNT1H_4Z_STRIDED_IMM:
5193 case AArch64::STNT1W_4Z_IMM:
5194 case AArch64::STNT1W_4Z_STRIDED_IMM:
5195 case AArch64::STNT1D_4Z_IMM:
5196 case AArch64::STNT1D_4Z_STRIDED_IMM:
5197 case AArch64::ST1B_4Z_IMM_PSEUDO:
5198 case AArch64::ST1H_4Z_IMM_PSEUDO:
5199 case AArch64::ST1W_4Z_IMM_PSEUDO:
5200 case AArch64::ST1D_4Z_IMM_PSEUDO:
5201 case AArch64::STNT1B_4Z_IMM_PSEUDO:
5202 case AArch64::STNT1H_4Z_IMM_PSEUDO:
5203 case AArch64::STNT1W_4Z_IMM_PSEUDO:
5204 case AArch64::STNT1D_4Z_IMM_PSEUDO:
5205 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5206 MinOffset = -8;
5207 MaxOffset = 7;
5208 break;
5209 case AArch64::LD1B_H_IMM:
5210 case AArch64::LD1SB_H_IMM:
5211 case AArch64::LD1H_S_IMM:
5212 case AArch64::LD1SH_S_IMM:
5213 case AArch64::LD1W_D_IMM:
5214 case AArch64::LD1SW_D_IMM:
5215 case AArch64::ST1B_H_IMM:
5216 case AArch64::ST1H_S_IMM:
5217 case AArch64::ST1W_D_IMM:
5218 case AArch64::LDNF1B_H_IMM:
5219 case AArch64::LDNF1SB_H_IMM:
5220 case AArch64::LDNF1H_S_IMM:
5221 case AArch64::LDNF1SH_S_IMM:
5222 case AArch64::LDNF1W_D_IMM:
5223 case AArch64::LDNF1SW_D_IMM:
5224 // A half vector worth of data
5225 // Width = mbytes * elements
5226 Scale = Width = TypeSize::getScalable(MinimumSize: 8);
5227 MinOffset = -8;
5228 MaxOffset = 7;
5229 break;
5230 case AArch64::LD1B_S_IMM:
5231 case AArch64::LD1SB_S_IMM:
5232 case AArch64::LD1H_D_IMM:
5233 case AArch64::LD1SH_D_IMM:
5234 case AArch64::ST1B_S_IMM:
5235 case AArch64::ST1H_D_IMM:
5236 case AArch64::LDNF1B_S_IMM:
5237 case AArch64::LDNF1SB_S_IMM:
5238 case AArch64::LDNF1H_D_IMM:
5239 case AArch64::LDNF1SH_D_IMM:
5240 // A quarter vector worth of data
5241 // Width = mbytes * elements
5242 Scale = Width = TypeSize::getScalable(MinimumSize: 4);
5243 MinOffset = -8;
5244 MaxOffset = 7;
5245 break;
5246 case AArch64::LD1B_D_IMM:
5247 case AArch64::LD1SB_D_IMM:
5248 case AArch64::ST1B_D_IMM:
5249 case AArch64::LDNF1B_D_IMM:
5250 case AArch64::LDNF1SB_D_IMM:
5251 // A eighth vector worth of data
5252 // Width = mbytes * elements
5253 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5254 MinOffset = -8;
5255 MaxOffset = 7;
5256 break;
5257 case AArch64::ST2Gi:
5258 case AArch64::ST2GPreIndex:
5259 case AArch64::ST2GPostIndex:
5260 case AArch64::STZ2Gi:
5261 case AArch64::STZ2GPreIndex:
5262 case AArch64::STZ2GPostIndex:
5263 Scale = TypeSize::getFixed(ExactSize: 16);
5264 Width = TypeSize::getFixed(ExactSize: 32);
5265 MinOffset = -256;
5266 MaxOffset = 255;
5267 break;
5268 case AArch64::STGPi:
5269 case AArch64::STGPpost:
5270 case AArch64::STGPpre:
5271 Scale = Width = TypeSize::getFixed(ExactSize: 16);
5272 MinOffset = -64;
5273 MaxOffset = 63;
5274 break;
5275 case AArch64::LD1RB_IMM:
5276 case AArch64::LD1RB_H_IMM:
5277 case AArch64::LD1RB_S_IMM:
5278 case AArch64::LD1RB_D_IMM:
5279 case AArch64::LD1RSB_H_IMM:
5280 case AArch64::LD1RSB_S_IMM:
5281 case AArch64::LD1RSB_D_IMM:
5282 Scale = Width = TypeSize::getFixed(ExactSize: 1);
5283 MinOffset = 0;
5284 MaxOffset = 63;
5285 break;
5286 case AArch64::LD1RH_IMM:
5287 case AArch64::LD1RH_S_IMM:
5288 case AArch64::LD1RH_D_IMM:
5289 case AArch64::LD1RSH_S_IMM:
5290 case AArch64::LD1RSH_D_IMM:
5291 Scale = Width = TypeSize::getFixed(ExactSize: 2);
5292 MinOffset = 0;
5293 MaxOffset = 63;
5294 break;
5295 case AArch64::LD1RW_IMM:
5296 case AArch64::LD1RW_D_IMM:
5297 case AArch64::LD1RSW_IMM:
5298 Scale = Width = TypeSize::getFixed(ExactSize: 4);
5299 MinOffset = 0;
5300 MaxOffset = 63;
5301 break;
5302 case AArch64::LD1RD_IMM:
5303 Scale = Width = TypeSize::getFixed(ExactSize: 8);
5304 MinOffset = 0;
5305 MaxOffset = 63;
5306 break;
5307 }
5308
5309 return true;
5310}
5311
5312// Scaling factor for unscaled load or store.
5313int AArch64InstrInfo::getMemScale(unsigned Opc) {
5314 switch (Opc) {
5315 default:
5316 llvm_unreachable("Opcode has unknown scale!");
5317 case AArch64::LDRBui:
5318 case AArch64::LDRBBui:
5319 case AArch64::LDURBBi:
5320 case AArch64::LDRSBWui:
5321 case AArch64::LDURSBWi:
5322 case AArch64::STRBui:
5323 case AArch64::STRBBui:
5324 case AArch64::STURBBi:
5325 return 1;
5326 case AArch64::LDRHui:
5327 case AArch64::LDRHHui:
5328 case AArch64::LDURHHi:
5329 case AArch64::LDRSHWui:
5330 case AArch64::LDURSHWi:
5331 case AArch64::STRHui:
5332 case AArch64::STRHHui:
5333 case AArch64::STURHHi:
5334 return 2;
5335 case AArch64::LDRSui:
5336 case AArch64::LDURSi:
5337 case AArch64::LDRSpre:
5338 case AArch64::LDRSWui:
5339 case AArch64::LDURSWi:
5340 case AArch64::LDRSWpre:
5341 case AArch64::LDRWpre:
5342 case AArch64::LDRWui:
5343 case AArch64::LDURWi:
5344 case AArch64::STRSui:
5345 case AArch64::STURSi:
5346 case AArch64::STRSpre:
5347 case AArch64::STRWui:
5348 case AArch64::STURWi:
5349 case AArch64::STRWpre:
5350 case AArch64::LDPSi:
5351 case AArch64::LDPSWi:
5352 case AArch64::LDPWi:
5353 case AArch64::STPSi:
5354 case AArch64::STPWi:
5355 return 4;
5356 case AArch64::LDRDui:
5357 case AArch64::LDURDi:
5358 case AArch64::LDRDpre:
5359 case AArch64::LDRXui:
5360 case AArch64::LDURXi:
5361 case AArch64::LDRXpre:
5362 case AArch64::STRDui:
5363 case AArch64::STURDi:
5364 case AArch64::STRDpre:
5365 case AArch64::STRXui:
5366 case AArch64::STURXi:
5367 case AArch64::STRXpre:
5368 case AArch64::LDPDi:
5369 case AArch64::LDPXi:
5370 case AArch64::STPDi:
5371 case AArch64::STPXi:
5372 return 8;
5373 case AArch64::LDRQui:
5374 case AArch64::LDURQi:
5375 case AArch64::STRQui:
5376 case AArch64::STURQi:
5377 case AArch64::STRQpre:
5378 case AArch64::LDPQi:
5379 case AArch64::LDRQpre:
5380 case AArch64::STPQi:
5381 case AArch64::STGi:
5382 case AArch64::STZGi:
5383 case AArch64::ST2Gi:
5384 case AArch64::STZ2Gi:
5385 case AArch64::STGPi:
5386 return 16;
5387 }
5388}
5389
5390bool AArch64InstrInfo::isPreLd(const MachineInstr &MI) {
5391 switch (MI.getOpcode()) {
5392 default:
5393 return false;
5394 case AArch64::LDRWpre:
5395 case AArch64::LDRXpre:
5396 case AArch64::LDRSWpre:
5397 case AArch64::LDRSpre:
5398 case AArch64::LDRDpre:
5399 case AArch64::LDRQpre:
5400 return true;
5401 }
5402}
5403
5404bool AArch64InstrInfo::isPreSt(const MachineInstr &MI) {
5405 switch (MI.getOpcode()) {
5406 default:
5407 return false;
5408 case AArch64::STRWpre:
5409 case AArch64::STRXpre:
5410 case AArch64::STRSpre:
5411 case AArch64::STRDpre:
5412 case AArch64::STRQpre:
5413 return true;
5414 }
5415}
5416
5417bool AArch64InstrInfo::isPreLdSt(const MachineInstr &MI) {
5418 return isPreLd(MI) || isPreSt(MI);
5419}
5420
5421bool AArch64InstrInfo::isZExtLoad(const MachineInstr &MI) {
5422 switch (MI.getOpcode()) {
5423 default:
5424 return false;
5425 case AArch64::LDURBBi:
5426 case AArch64::LDURHHi:
5427 case AArch64::LDURWi:
5428 case AArch64::LDRBBui:
5429 case AArch64::LDRHHui:
5430 case AArch64::LDRWui:
5431 case AArch64::LDRBBroX:
5432 case AArch64::LDRHHroX:
5433 case AArch64::LDRWroX:
5434 case AArch64::LDRBBroW:
5435 case AArch64::LDRHHroW:
5436 case AArch64::LDRWroW:
5437 return true;
5438 }
5439}
5440
5441bool AArch64InstrInfo::isSExtLoad(const MachineInstr &MI) {
5442 switch (MI.getOpcode()) {
5443 default:
5444 return false;
5445 case AArch64::LDURSBWi:
5446 case AArch64::LDURSHWi:
5447 case AArch64::LDURSBXi:
5448 case AArch64::LDURSHXi:
5449 case AArch64::LDURSWi:
5450 case AArch64::LDRSBWui:
5451 case AArch64::LDRSHWui:
5452 case AArch64::LDRSBXui:
5453 case AArch64::LDRSHXui:
5454 case AArch64::LDRSWui:
5455 case AArch64::LDRSBWroX:
5456 case AArch64::LDRSHWroX:
5457 case AArch64::LDRSBXroX:
5458 case AArch64::LDRSHXroX:
5459 case AArch64::LDRSWroX:
5460 case AArch64::LDRSBWroW:
5461 case AArch64::LDRSHWroW:
5462 case AArch64::LDRSBXroW:
5463 case AArch64::LDRSHXroW:
5464 case AArch64::LDRSWroW:
5465 return true;
5466 }
5467}
5468
5469bool AArch64InstrInfo::isPairedLdSt(const MachineInstr &MI) {
5470 switch (MI.getOpcode()) {
5471 default:
5472 return false;
5473 case AArch64::LDPSi:
5474 case AArch64::LDPSWi:
5475 case AArch64::LDPDi:
5476 case AArch64::LDPQi:
5477 case AArch64::LDPWi:
5478 case AArch64::LDPXi:
5479 case AArch64::STPSi:
5480 case AArch64::STPDi:
5481 case AArch64::STPQi:
5482 case AArch64::STPWi:
5483 case AArch64::STPXi:
5484 case AArch64::STGPi:
5485 return true;
5486 }
5487}
5488
5489const MachineOperand &AArch64InstrInfo::getLdStBaseOp(const MachineInstr &MI) {
5490 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5491 unsigned Idx =
5492 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 2
5493 : 1;
5494 return MI.getOperand(i: Idx);
5495}
5496
5497const MachineOperand &
5498AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
5499 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5500 unsigned Idx =
5501 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 3
5502 : 2;
5503 return MI.getOperand(i: Idx);
5504}
5505
5506const MachineOperand &
5507AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
5508 switch (MI.getOpcode()) {
5509 default:
5510 llvm_unreachable("Unexpected opcode");
5511 case AArch64::LDRBroX:
5512 case AArch64::LDRBBroX:
5513 case AArch64::LDRSBXroX:
5514 case AArch64::LDRSBWroX:
5515 case AArch64::LDRHroX:
5516 case AArch64::LDRHHroX:
5517 case AArch64::LDRSHXroX:
5518 case AArch64::LDRSHWroX:
5519 case AArch64::LDRWroX:
5520 case AArch64::LDRSroX:
5521 case AArch64::LDRSWroX:
5522 case AArch64::LDRDroX:
5523 case AArch64::LDRXroX:
5524 case AArch64::LDRQroX:
5525 return MI.getOperand(i: 4);
5526 }
5527}
5528
5529static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
5530 Register Reg) {
5531 if (MI.getParent() == nullptr)
5532 return nullptr;
5533 const MachineFunction *MF = MI.getParent()->getParent();
5534 return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
5535}
5536
5537bool AArch64InstrInfo::isHForm(const MachineInstr &MI) {
5538 auto IsHFPR = [&](const MachineOperand &Op) {
5539 if (!Op.isReg())
5540 return false;
5541 auto Reg = Op.getReg();
5542 if (Reg.isPhysical())
5543 return AArch64::FPR16RegClass.contains(Reg);
5544 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5545 return TRC == &AArch64::FPR16RegClass ||
5546 TRC == &AArch64::FPR16_loRegClass;
5547 };
5548 return llvm::any_of(Range: MI.operands(), P: IsHFPR);
5549}
5550
5551bool AArch64InstrInfo::isQForm(const MachineInstr &MI) {
5552 auto IsQFPR = [&](const MachineOperand &Op) {
5553 if (!Op.isReg())
5554 return false;
5555 auto Reg = Op.getReg();
5556 if (Reg.isPhysical())
5557 return AArch64::FPR128RegClass.contains(Reg);
5558 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5559 return TRC == &AArch64::FPR128RegClass ||
5560 TRC == &AArch64::FPR128_loRegClass;
5561 };
5562 return llvm::any_of(Range: MI.operands(), P: IsQFPR);
5563}
5564
5565bool AArch64InstrInfo::hasBTISemantics(const MachineInstr &MI) {
5566 switch (MI.getOpcode()) {
5567 case AArch64::BRK:
5568 case AArch64::HLT:
5569 case AArch64::PACIASP:
5570 case AArch64::PACIBSP:
5571 // Implicit BTI behavior.
5572 return true;
5573 case AArch64::PAUTH_PROLOGUE:
5574 // PAUTH_PROLOGUE expands to PACI(A|B)SP.
5575 return true;
5576 case AArch64::HINT: {
5577 unsigned Imm = MI.getOperand(i: 0).getImm();
5578 // Explicit BTI instruction.
5579 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
5580 return true;
5581 // PACI(A|B)SP instructions.
5582 if (Imm == 25 || Imm == 27)
5583 return true;
5584 return false;
5585 }
5586 default:
5587 return false;
5588 }
5589}
5590
5591bool AArch64InstrInfo::isFpOrNEON(Register Reg) {
5592 if (Reg == 0)
5593 return false;
5594 assert(Reg.isPhysical() && "Expected physical register in isFpOrNEON");
5595 return AArch64::FPR128RegClass.contains(Reg) ||
5596 AArch64::FPR64RegClass.contains(Reg) ||
5597 AArch64::FPR32RegClass.contains(Reg) ||
5598 AArch64::FPR16RegClass.contains(Reg) ||
5599 AArch64::FPR8RegClass.contains(Reg);
5600}
5601
5602bool AArch64InstrInfo::isFpOrNEON(const MachineInstr &MI) {
5603 auto IsFPR = [&](const MachineOperand &Op) {
5604 if (!Op.isReg())
5605 return false;
5606 auto Reg = Op.getReg();
5607 if (Reg.isPhysical())
5608 return isFpOrNEON(Reg);
5609
5610 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5611 return TRC == &AArch64::FPR128RegClass ||
5612 TRC == &AArch64::FPR128_loRegClass ||
5613 TRC == &AArch64::FPR64RegClass ||
5614 TRC == &AArch64::FPR64_loRegClass ||
5615 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
5616 TRC == &AArch64::FPR8RegClass;
5617 };
5618 return llvm::any_of(Range: MI.operands(), P: IsFPR);
5619}
5620
5621// Scale the unscaled offsets. Returns false if the unscaled offset can't be
5622// scaled.
5623static bool scaleOffset(unsigned Opc, int64_t &Offset) {
5624 int Scale = AArch64InstrInfo::getMemScale(Opc);
5625
5626 // If the byte-offset isn't a multiple of the stride, we can't scale this
5627 // offset.
5628 if (Offset % Scale != 0)
5629 return false;
5630
5631 // Convert the byte-offset used by unscaled into an "element" offset used
5632 // by the scaled pair load/store instructions.
5633 Offset /= Scale;
5634 return true;
5635}
5636
5637static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
5638 if (FirstOpc == SecondOpc)
5639 return true;
5640 // We can also pair sign-ext and zero-ext instructions.
5641 switch (FirstOpc) {
5642 default:
5643 return false;
5644 case AArch64::STRSui:
5645 case AArch64::STURSi:
5646 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
5647 case AArch64::STRDui:
5648 case AArch64::STURDi:
5649 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
5650 case AArch64::STRQui:
5651 case AArch64::STURQi:
5652 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
5653 case AArch64::STRWui:
5654 case AArch64::STURWi:
5655 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
5656 case AArch64::STRXui:
5657 case AArch64::STURXi:
5658 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
5659 case AArch64::LDRSui:
5660 case AArch64::LDURSi:
5661 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
5662 case AArch64::LDRDui:
5663 case AArch64::LDURDi:
5664 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
5665 case AArch64::LDRQui:
5666 case AArch64::LDURQi:
5667 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
5668 case AArch64::LDRWui:
5669 case AArch64::LDURWi:
5670 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
5671 case AArch64::LDRSWui:
5672 case AArch64::LDURSWi:
5673 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
5674 case AArch64::LDRXui:
5675 case AArch64::LDURXi:
5676 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
5677 }
5678 // These instructions can't be paired based on their opcodes.
5679 return false;
5680}
5681
5682static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
5683 int64_t Offset1, unsigned Opcode1, int FI2,
5684 int64_t Offset2, unsigned Opcode2) {
5685 // Accesses through fixed stack object frame indices may access a different
5686 // fixed stack slot. Check that the object offsets + offsets match.
5687 if (MFI.isFixedObjectIndex(ObjectIdx: FI1) && MFI.isFixedObjectIndex(ObjectIdx: FI2)) {
5688 int64_t ObjectOffset1 = MFI.getObjectOffset(ObjectIdx: FI1);
5689 int64_t ObjectOffset2 = MFI.getObjectOffset(ObjectIdx: FI2);
5690 assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
5691 // Convert to scaled object offsets.
5692 int Scale1 = AArch64InstrInfo::getMemScale(Opc: Opcode1);
5693 if (ObjectOffset1 % Scale1 != 0)
5694 return false;
5695 ObjectOffset1 /= Scale1;
5696 int Scale2 = AArch64InstrInfo::getMemScale(Opc: Opcode2);
5697 if (ObjectOffset2 % Scale2 != 0)
5698 return false;
5699 ObjectOffset2 /= Scale2;
5700 ObjectOffset1 += Offset1;
5701 ObjectOffset2 += Offset2;
5702 return ObjectOffset1 + 1 == ObjectOffset2;
5703 }
5704
5705 return FI1 == FI2;
5706}
5707
5708/// Detect opportunities for ldp/stp formation.
5709///
5710/// Only called for LdSt for which getMemOperandWithOffset returns true.
5711bool AArch64InstrInfo::shouldClusterMemOps(
5712 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
5713 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
5714 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
5715 unsigned NumBytes) const {
5716 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
5717 const MachineOperand &BaseOp1 = *BaseOps1.front();
5718 const MachineOperand &BaseOp2 = *BaseOps2.front();
5719 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
5720 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
5721 if (BaseOp1.getType() != BaseOp2.getType())
5722 return false;
5723
5724 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
5725 "Only base registers and frame indices are supported.");
5726
5727 // Check for both base regs and base FI.
5728 if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
5729 return false;
5730
5731 // Only cluster up to a single pair.
5732 if (ClusterSize > 2)
5733 return false;
5734
5735 if (!isPairableLdStInst(MI: FirstLdSt) || !isPairableLdStInst(MI: SecondLdSt))
5736 return false;
5737
5738 // Can we pair these instructions based on their opcodes?
5739 unsigned FirstOpc = FirstLdSt.getOpcode();
5740 unsigned SecondOpc = SecondLdSt.getOpcode();
5741 if (!canPairLdStOpc(FirstOpc, SecondOpc))
5742 return false;
5743
5744 // Can't merge volatiles or load/stores that have a hint to avoid pair
5745 // formation, for example.
5746 if (!isCandidateToMergeOrPair(MI: FirstLdSt) ||
5747 !isCandidateToMergeOrPair(MI: SecondLdSt))
5748 return false;
5749
5750 // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
5751 int64_t Offset1 = FirstLdSt.getOperand(i: 2).getImm();
5752 if (hasUnscaledLdStOffset(Opc: FirstOpc) && !scaleOffset(Opc: FirstOpc, Offset&: Offset1))
5753 return false;
5754
5755 int64_t Offset2 = SecondLdSt.getOperand(i: 2).getImm();
5756 if (hasUnscaledLdStOffset(Opc: SecondOpc) && !scaleOffset(Opc: SecondOpc, Offset&: Offset2))
5757 return false;
5758
5759 // Pairwise instructions have a 7-bit signed offset field.
5760 if (Offset1 > 63 || Offset1 < -64)
5761 return false;
5762
5763 // The caller should already have ordered First/SecondLdSt by offset.
5764 // Note: except for non-equal frame index bases
5765 if (BaseOp1.isFI()) {
5766 assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
5767 "Caller should have ordered offsets.");
5768
5769 const MachineFrameInfo &MFI =
5770 FirstLdSt.getParent()->getParent()->getFrameInfo();
5771 return shouldClusterFI(MFI, FI1: BaseOp1.getIndex(), Offset1, Opcode1: FirstOpc,
5772 FI2: BaseOp2.getIndex(), Offset2, Opcode2: SecondOpc);
5773 }
5774
5775 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
5776
5777 return Offset1 + 1 == Offset2;
5778}
5779
5780static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
5781 MCRegister Reg, unsigned SubIdx,
5782 RegState State,
5783 const TargetRegisterInfo *TRI) {
5784 if (!SubIdx)
5785 return MIB.addReg(RegNo: Reg, Flags: State);
5786
5787 if (Reg.isPhysical())
5788 return MIB.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx), Flags: State);
5789 return MIB.addReg(RegNo: Reg, Flags: State, SubReg: SubIdx);
5790}
5791
5792void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
5793 MachineBasicBlock::iterator I,
5794 const DebugLoc &DL, MCRegister DestReg,
5795 MCRegister SrcReg, bool KillSrc,
5796 ArrayRef<unsigned> Indices) const {
5797 assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
5798 const TargetRegisterInfo *TRI = &getRegisterInfo();
5799 uint16_t DestEncoding = TRI->getEncodingValue(Reg: DestReg);
5800 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
5801 unsigned NumRegs = Indices.size();
5802 MCRegister DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[0]);
5803 assert(!AArch64::PNRRegClass.contains(DestSubReg) &&
5804 "Unexpected predicate tuple copy");
5805 unsigned MaxRegs = AArch64::PPRRegClass.contains(Reg: DestSubReg) ? 15 : 31;
5806
5807 int SubReg = 0, End = NumRegs, Incr = 1;
5808 // Copy in reverse if a forward copy will clobber the tuple
5809 if (((DestEncoding - SrcEncoding) & MaxRegs) < NumRegs) {
5810 SubReg = NumRegs - 1;
5811 End = -1;
5812 Incr = -1;
5813 }
5814
5815 for (; SubReg != End; SubReg += Incr) {
5816 DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[SubReg]);
5817 MCRegister SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: Indices[SubReg]);
5818 copyPhysRegImpl(MBB, I, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc);
5819 }
5820}
5821
5822void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
5823 MachineBasicBlock::iterator I,
5824 const DebugLoc &DL, MCRegister DestReg,
5825 MCRegister SrcReg, bool KillSrc,
5826 unsigned Opcode, unsigned ZeroReg,
5827 llvm::ArrayRef<unsigned> Indices) const {
5828 const TargetRegisterInfo *TRI = &getRegisterInfo();
5829 unsigned NumRegs = Indices.size();
5830
5831#ifndef NDEBUG
5832 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
5833 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
5834 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5835 "GPR reg sequences should not be able to overlap");
5836#endif
5837
5838 for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
5839 const MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode));
5840 AddSubReg(MIB, Reg: DestReg, SubIdx: Indices[SubReg], State: RegState::Define, TRI);
5841 MIB.addReg(RegNo: ZeroReg);
5842 AddSubReg(MIB, Reg: SrcReg, SubIdx: Indices[SubReg], State: getKillRegState(B: KillSrc), TRI);
5843 MIB.addImm(Val: 0);
5844 }
5845}
5846
5847/// Returns true if the instruction at I is in a streaming call site region,
5848/// within a single basic block.
5849/// A "call site streaming region" starts after smstart and ends at smstop
5850/// around a call to a streaming function. This walks backward from I.
5851static bool isInStreamingCallSiteRegion(MachineBasicBlock &MBB,
5852 MachineBasicBlock::iterator I) {
5853 MachineFunction &MF = *MBB.getParent();
5854 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
5855 if (!AFI->hasStreamingModeChanges())
5856 return false;
5857 // Walk backwards to find smstart/smstop
5858 for (MachineInstr &MI : reverse(C: make_range(x: MBB.begin(), y: I))) {
5859 unsigned Opc = MI.getOpcode();
5860 if (Opc == AArch64::MSRpstatesvcrImm1 || Opc == AArch64::MSRpstatePseudo) {
5861 // Check if this is SM change (not ZA)
5862 int64_t PState = MI.getOperand(i: 0).getImm();
5863 if (PState == AArch64SVCR::SVCRSM || PState == AArch64SVCR::SVCRSMZA) {
5864 // Operand 1 is 1 for start, 0 for stop
5865 return MI.getOperand(i: 1).getImm() == 1;
5866 }
5867 }
5868 }
5869 return false;
5870}
5871
5872/// Returns true if in a streaming call site region without SME-FA64.
5873static bool mustAvoidNeonAtMBBI(const AArch64Subtarget &Subtarget,
5874 MachineBasicBlock &MBB,
5875 MachineBasicBlock::iterator I) {
5876 return !Subtarget.hasSMEFA64() && isInStreamingCallSiteRegion(MBB, I);
5877}
5878
5879void AArch64InstrInfo::copyPhysRegImpl(MachineBasicBlock &MBB,
5880 MachineBasicBlock::iterator I,
5881 const DebugLoc &DL, Register DestReg,
5882 Register SrcReg, bool KillSrc,
5883 bool RenamableDest,
5884 bool RenamableSrc) const {
5885 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) &&
5886 AArch64::GPR32spRegClass.contains(Reg: SrcReg)) {
5887 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5888 // If either operand is WSP, expand to ADD #0.
5889 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5890 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5891 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
5892 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5893 RC: &AArch64::GPR64spRegClass);
5894 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5895 RC: &AArch64::GPR64spRegClass);
5896 // This instruction is reading and writing X registers. This may upset
5897 // the register scavenger and machine verifier, so we need to indicate
5898 // that we are reading an undefined value from SrcRegX, but a proper
5899 // value from SrcReg.
5900 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: DestRegX)
5901 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5902 .addImm(Val: 0)
5903 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
5904 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5905 ++NumZCRegMoveInstrsGPR;
5906 } else {
5907 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDWri), DestReg)
5908 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5909 .addImm(Val: 0)
5910 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5911 if (Subtarget.hasZeroCycleRegMoveGPR32())
5912 ++NumZCRegMoveInstrsGPR;
5913 }
5914 } else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5915 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5916 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
5917 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5918 RC: &AArch64::GPR64spRegClass);
5919 assert(DestRegX.isValid() && "Destination super-reg not valid");
5920 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5921 RC: &AArch64::GPR64spRegClass);
5922 assert(SrcRegX.isValid() && "Source super-reg not valid");
5923 // This instruction is reading and writing X registers. This may upset
5924 // the register scavenger and machine verifier, so we need to indicate
5925 // that we are reading an undefined value from SrcRegX, but a proper
5926 // value from SrcReg.
5927 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg: DestRegX)
5928 .addReg(RegNo: AArch64::XZR)
5929 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5930 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5931 ++NumZCRegMoveInstrsGPR;
5932 } else {
5933 // Otherwise, expand to ORR WZR.
5934 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5935 .addReg(RegNo: AArch64::WZR)
5936 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5937 if (Subtarget.hasZeroCycleRegMoveGPR32())
5938 ++NumZCRegMoveInstrsGPR;
5939 }
5940 return;
5941 }
5942
5943 // GPR32 zeroing
5944 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::WZR) {
5945 if (Subtarget.hasZeroCycleZeroingGPR64() &&
5946 !Subtarget.hasZeroCycleZeroingGPR32()) {
5947 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5948 RC: &AArch64::GPR64spRegClass);
5949 assert(DestRegX.isValid() && "Destination super-reg not valid");
5950 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: DestRegX)
5951 .addImm(Val: 0)
5952 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5953 ++NumZCZeroingInstrsGPR;
5954 } else if (Subtarget.hasZeroCycleZeroingGPR32()) {
5955 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZWi), DestReg)
5956 .addImm(Val: 0)
5957 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5958 ++NumZCZeroingInstrsGPR;
5959 } else {
5960 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5961 .addReg(RegNo: AArch64::WZR)
5962 .addReg(RegNo: AArch64::WZR);
5963 }
5964 return;
5965 }
5966
5967 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) &&
5968 AArch64::GPR64spRegClass.contains(Reg: SrcReg)) {
5969 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5970 // If either operand is SP, expand to ADD #0.
5971 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg)
5972 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5973 .addImm(Val: 0)
5974 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5975 if (Subtarget.hasZeroCycleRegMoveGPR64())
5976 ++NumZCRegMoveInstrsGPR;
5977 } else {
5978 // Otherwise, expand to ORR XZR.
5979 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5980 .addReg(RegNo: AArch64::XZR)
5981 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5982 if (Subtarget.hasZeroCycleRegMoveGPR64())
5983 ++NumZCRegMoveInstrsGPR;
5984 }
5985 return;
5986 }
5987
5988 // GPR64 zeroing
5989 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::XZR) {
5990 if (Subtarget.hasZeroCycleZeroingGPR64()) {
5991 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg)
5992 .addImm(Val: 0)
5993 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5994 ++NumZCZeroingInstrsGPR;
5995 } else {
5996 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5997 .addReg(RegNo: AArch64::XZR)
5998 .addReg(RegNo: AArch64::XZR);
5999 }
6000 return;
6001 }
6002
6003 // Copy a Predicate register by ORRing with itself.
6004 if (AArch64::PPRRegClass.contains(Reg: DestReg) &&
6005 AArch64::PPRRegClass.contains(Reg: SrcReg)) {
6006 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6007 "Unexpected SVE register.");
6008 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg)
6009 .addReg(RegNo: SrcReg) // Pg
6010 .addReg(RegNo: SrcReg)
6011 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6012 return;
6013 }
6014
6015 // Copy a predicate-as-counter register by ORRing with itself as if it
6016 // were a regular predicate (mask) register.
6017 bool DestIsPNR = AArch64::PNRRegClass.contains(Reg: DestReg);
6018 bool SrcIsPNR = AArch64::PNRRegClass.contains(Reg: SrcReg);
6019 if (DestIsPNR || SrcIsPNR) {
6020 auto ToPPR = [](MCRegister R) -> MCRegister {
6021 return (R - AArch64::PN0) + AArch64::P0;
6022 };
6023 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg.asMCReg();
6024 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg.asMCReg();
6025
6026 if (PPRSrcReg != PPRDestReg) {
6027 auto NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg: PPRDestReg)
6028 .addReg(RegNo: PPRSrcReg) // Pg
6029 .addReg(RegNo: PPRSrcReg)
6030 .addReg(RegNo: PPRSrcReg, Flags: getKillRegState(B: KillSrc));
6031 if (DestIsPNR)
6032 NewMI.addDef(RegNo: DestReg, Flags: RegState::Implicit);
6033 }
6034 return;
6035 }
6036
6037 // Copy a predicate register pair by copying the individual sub-registers.
6038 if (AArch64::PPR2RegClass.contains(Reg: DestReg) &&
6039 AArch64::PPR2RegClass.contains(Reg: SrcReg)) {
6040 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6041 "Unexpected SVE predicate register.");
6042 static const unsigned Indices[] = {AArch64::psub0, AArch64::psub1};
6043 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6044 return;
6045 }
6046
6047 // Copy a Z register by ORRing with itself.
6048 if (AArch64::ZPRRegClass.contains(Reg: DestReg) &&
6049 AArch64::ZPRRegClass.contains(Reg: SrcReg)) {
6050 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6051 "Unexpected SVE register.");
6052 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ), DestReg)
6053 .addReg(RegNo: SrcReg)
6054 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6055 return;
6056 }
6057
6058 // Copy a Z register pair by copying the individual sub-registers.
6059 if ((AArch64::ZPR2RegClass.contains(Reg: DestReg) ||
6060 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6061 (AArch64::ZPR2RegClass.contains(Reg: SrcReg) ||
6062 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6063 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6064 "Unexpected SVE register.");
6065 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
6066 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6067 return;
6068 }
6069
6070 // Copy a Z register triple by copying the individual sub-registers.
6071 if (AArch64::ZPR3RegClass.contains(Reg: DestReg) &&
6072 AArch64::ZPR3RegClass.contains(Reg: SrcReg)) {
6073 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6074 "Unexpected SVE register.");
6075 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6076 AArch64::zsub2};
6077 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6078 return;
6079 }
6080
6081 // Copy a Z register quad by copying the individual sub-registers.
6082 if ((AArch64::ZPR4RegClass.contains(Reg: DestReg) ||
6083 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6084 (AArch64::ZPR4RegClass.contains(Reg: SrcReg) ||
6085 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6086 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6087 "Unexpected SVE register.");
6088 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6089 AArch64::zsub2, AArch64::zsub3};
6090 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6091 return;
6092 }
6093
6094 // Copy a DDDD register quad by copying the individual sub-registers.
6095 if (AArch64::DDDDRegClass.contains(Reg: DestReg) &&
6096 AArch64::DDDDRegClass.contains(Reg: SrcReg)) {
6097 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6098 AArch64::dsub2, AArch64::dsub3};
6099 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6100 return;
6101 }
6102
6103 // Copy a DDD register triple by copying the individual sub-registers.
6104 if (AArch64::DDDRegClass.contains(Reg: DestReg) &&
6105 AArch64::DDDRegClass.contains(Reg: SrcReg)) {
6106 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6107 AArch64::dsub2};
6108 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6109 return;
6110 }
6111
6112 // Copy a DD register pair by copying the individual sub-registers.
6113 if (AArch64::DDRegClass.contains(Reg: DestReg) &&
6114 AArch64::DDRegClass.contains(Reg: SrcReg)) {
6115 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
6116 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6117 return;
6118 }
6119
6120 // Copy a QQQQ register quad by copying the individual sub-registers.
6121 if (AArch64::QQQQRegClass.contains(Reg: DestReg) &&
6122 AArch64::QQQQRegClass.contains(Reg: SrcReg)) {
6123 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6124 AArch64::qsub2, AArch64::qsub3};
6125 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6126 return;
6127 }
6128
6129 // Copy a QQQ register triple by copying the individual sub-registers.
6130 if (AArch64::QQQRegClass.contains(Reg: DestReg) &&
6131 AArch64::QQQRegClass.contains(Reg: SrcReg)) {
6132 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6133 AArch64::qsub2};
6134 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6135 return;
6136 }
6137
6138 // Copy a QQ register pair by copying the individual sub-registers.
6139 if (AArch64::QQRegClass.contains(Reg: DestReg) &&
6140 AArch64::QQRegClass.contains(Reg: SrcReg)) {
6141 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
6142 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6143 return;
6144 }
6145
6146 if (AArch64::XSeqPairsClassRegClass.contains(Reg: DestReg) &&
6147 AArch64::XSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6148 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
6149 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRXrs,
6150 ZeroReg: AArch64::XZR, Indices);
6151 return;
6152 }
6153
6154 if (AArch64::WSeqPairsClassRegClass.contains(Reg: DestReg) &&
6155 AArch64::WSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6156 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
6157 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRWrs,
6158 ZeroReg: AArch64::WZR, Indices);
6159 return;
6160 }
6161
6162 if (AArch64::FPR128RegClass.contains(Reg: DestReg) &&
6163 AArch64::FPR128RegClass.contains(Reg: SrcReg)) {
6164 // In streaming regions, NEON is illegal but streaming-SVE is available.
6165 // Use SVE for copies if we're in a streaming region and SME is available.
6166 // With +sme-fa64, NEON is legal in streaming mode so we can use it.
6167 if ((Subtarget.isSVEorStreamingSVEAvailable() &&
6168 !Subtarget.isNeonAvailable()) ||
6169 mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6170 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ))
6171 .addReg(RegNo: AArch64::Z0 + (DestReg - AArch64::Q0), Flags: RegState::Define)
6172 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0))
6173 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0));
6174 } else if (Subtarget.isNeonAvailable()) {
6175 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg)
6176 .addReg(RegNo: SrcReg)
6177 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6178 if (Subtarget.hasZeroCycleRegMoveFPR128())
6179 ++NumZCRegMoveInstrsFPR;
6180 } else {
6181 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::STRQpre))
6182 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6183 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6184 .addReg(RegNo: AArch64::SP)
6185 .addImm(Val: -16);
6186 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::LDRQpost))
6187 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6188 .addReg(RegNo: DestReg, Flags: RegState::Define)
6189 .addReg(RegNo: AArch64::SP)
6190 .addImm(Val: 16);
6191 }
6192 return;
6193 }
6194
6195 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6196 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6197 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6198 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6199 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6200 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6201 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
6202 RC: &AArch64::FPR128RegClass);
6203 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::dsub,
6204 RC: &AArch64::FPR128RegClass);
6205 // This instruction is reading and writing Q registers. This may upset
6206 // the register scavenger and machine verifier, so we need to indicate
6207 // that we are reading an undefined value from SrcRegQ, but a proper
6208 // value from SrcReg.
6209 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6210 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6211 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6212 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6213 ++NumZCRegMoveInstrsFPR;
6214 } else {
6215 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg)
6216 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6217 if (Subtarget.hasZeroCycleRegMoveFPR64())
6218 ++NumZCRegMoveInstrsFPR;
6219 }
6220 return;
6221 }
6222
6223 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6224 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6225 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6226 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6227 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6228 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6229 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6230 RC: &AArch64::FPR128RegClass);
6231 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6232 RC: &AArch64::FPR128RegClass);
6233 // This instruction is reading and writing Q registers. This may upset
6234 // the register scavenger and machine verifier, so we need to indicate
6235 // that we are reading an undefined value from SrcRegQ, but a proper
6236 // value from SrcReg.
6237 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6238 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6239 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6240 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6241 ++NumZCRegMoveInstrsFPR;
6242 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6243 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6244 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6245 RC: &AArch64::FPR64RegClass);
6246 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6247 RC: &AArch64::FPR64RegClass);
6248 // This instruction is reading and writing D registers. This may upset
6249 // the register scavenger and machine verifier, so we need to indicate
6250 // that we are reading an undefined value from SrcRegD, but a proper
6251 // value from SrcReg.
6252 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6253 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6254 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6255 ++NumZCRegMoveInstrsFPR;
6256 } else {
6257 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6258 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6259 if (Subtarget.hasZeroCycleRegMoveFPR32())
6260 ++NumZCRegMoveInstrsFPR;
6261 }
6262 return;
6263 }
6264
6265 if (AArch64::FPR16RegClass.contains(Reg: DestReg) &&
6266 AArch64::FPR16RegClass.contains(Reg: SrcReg)) {
6267 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6268 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6269 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6270 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6271 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6272 RC: &AArch64::FPR128RegClass);
6273 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6274 RC: &AArch64::FPR128RegClass);
6275 // This instruction is reading and writing Q registers. This may upset
6276 // the register scavenger and machine verifier, so we need to indicate
6277 // that we are reading an undefined value from SrcRegQ, but a proper
6278 // value from SrcReg.
6279 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6280 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6281 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6282 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6283 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6284 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6285 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6286 RC: &AArch64::FPR64RegClass);
6287 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6288 RC: &AArch64::FPR64RegClass);
6289 // This instruction is reading and writing D registers. This may upset
6290 // the register scavenger and machine verifier, so we need to indicate
6291 // that we are reading an undefined value from SrcRegD, but a proper
6292 // value from SrcReg.
6293 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6294 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6295 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6296 } else {
6297 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6298 RC: &AArch64::FPR32RegClass);
6299 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6300 RC: &AArch64::FPR32RegClass);
6301 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6302 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6303 }
6304 return;
6305 }
6306
6307 if (AArch64::FPR8RegClass.contains(Reg: DestReg) &&
6308 AArch64::FPR8RegClass.contains(Reg: SrcReg)) {
6309 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6310 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6311 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6312 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6313 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6314 RC: &AArch64::FPR128RegClass);
6315 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6316 RC: &AArch64::FPR128RegClass);
6317 // This instruction is reading and writing Q registers. This may upset
6318 // the register scavenger and machine verifier, so we need to indicate
6319 // that we are reading an undefined value from SrcRegQ, but a proper
6320 // value from SrcReg.
6321 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6322 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6323 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6324 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6325 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6326 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6327 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6328 RC: &AArch64::FPR64RegClass);
6329 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6330 RC: &AArch64::FPR64RegClass);
6331 // This instruction is reading and writing D registers. This may upset
6332 // the register scavenger and machine verifier, so we need to indicate
6333 // that we are reading an undefined value from SrcRegD, but a proper
6334 // value from SrcReg.
6335 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6336 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6337 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6338 } else {
6339 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6340 RC: &AArch64::FPR32RegClass);
6341 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6342 RC: &AArch64::FPR32RegClass);
6343 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6344 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6345 }
6346 return;
6347 }
6348
6349 // Copies between GPR64 and FPR64.
6350 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6351 AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
6352 if (AArch64::XZR == SrcReg) {
6353 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg);
6354 } else {
6355 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVXDr), DestReg)
6356 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6357 }
6358 return;
6359 }
6360 if (AArch64::GPR64RegClass.contains(Reg: DestReg) &&
6361 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6362 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDXr), DestReg)
6363 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6364 return;
6365 }
6366 // Copies between GPR32 and FPR32.
6367 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6368 AArch64::GPR32RegClass.contains(Reg: SrcReg)) {
6369 if (AArch64::WZR == SrcReg) {
6370 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVS0), DestReg);
6371 } else {
6372 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVWSr), DestReg)
6373 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6374 }
6375 return;
6376 }
6377 if (AArch64::GPR32RegClass.contains(Reg: DestReg) &&
6378 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6379 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSWr), DestReg)
6380 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6381 return;
6382 }
6383
6384 if (DestReg == AArch64::NZCV) {
6385 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
6386 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MSR))
6387 .addImm(Val: AArch64SysReg::NZCV)
6388 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6389 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Define);
6390 return;
6391 }
6392
6393 if (SrcReg == AArch64::NZCV) {
6394 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
6395 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MRS), DestReg)
6396 .addImm(Val: AArch64SysReg::NZCV)
6397 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6398 return;
6399 }
6400
6401#ifndef NDEBUG
6402 errs() << RI.getRegAsmName(DestReg) << " = COPY " << RI.getRegAsmName(SrcReg)
6403 << "\n";
6404#endif
6405 llvm_unreachable("unimplemented reg-to-reg copy");
6406}
6407
6408void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
6409 MachineBasicBlock::iterator I,
6410 const DebugLoc &DL, Register DestReg,
6411 Register SrcReg, bool KillSrc,
6412 bool RenamableDest,
6413 bool RenamableSrc) const {
6414 ++NumCopyInstrs;
6415 copyPhysRegImpl(MBB, I, DL, DestReg, SrcReg, KillSrc, RenamableDest,
6416 RenamableSrc);
6417 return;
6418}
6419
6420static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
6421 MachineBasicBlock &MBB,
6422 MachineBasicBlock::iterator InsertBefore,
6423 const MCInstrDesc &MCID,
6424 Register SrcReg, bool IsKill,
6425 unsigned SubIdx0, unsigned SubIdx1, int FI,
6426 MachineMemOperand *MMO) {
6427 Register SrcReg0 = SrcReg;
6428 Register SrcReg1 = SrcReg;
6429 if (SrcReg.isPhysical()) {
6430 SrcReg0 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx0);
6431 SubIdx0 = 0;
6432 SrcReg1 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx1);
6433 SubIdx1 = 0;
6434 }
6435 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6436 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: IsKill), SubReg: SubIdx0)
6437 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: IsKill), SubReg: SubIdx1)
6438 .addFrameIndex(Idx: FI)
6439 .addImm(Val: 0)
6440 .addMemOperand(MMO);
6441}
6442
6443void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
6444 MachineBasicBlock::iterator MBBI,
6445 Register SrcReg, bool isKill, int FI,
6446 const TargetRegisterClass *RC,
6447 Register VReg,
6448 MachineInstr::MIFlag Flags) const {
6449 MachineFunction &MF = *MBB.getParent();
6450 MachineFrameInfo &MFI = MF.getFrameInfo();
6451
6452 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6453 MachineMemOperand *MMO =
6454 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
6455 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6456 unsigned Opc = 0;
6457 bool Offset = true;
6458 MCRegister PNRReg = MCRegister::NoRegister;
6459 unsigned StackID = TargetStackID::Default;
6460 switch (RI.getSpillSize(RC: *RC)) {
6461 case 1:
6462 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6463 Opc = AArch64::STRBui;
6464 break;
6465 case 2: {
6466 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6467 Opc = AArch64::STRHui;
6468 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
6469 AArch64::PPRRegClass.hasSubClassEq(RC)) {
6470 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6471 "Unexpected register store without SVE store instructions");
6472 Opc = AArch64::STR_PXI;
6473 StackID = TargetStackID::ScalablePredicateVector;
6474 }
6475 break;
6476 }
6477 case 4:
6478 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6479 Opc = AArch64::STRWui;
6480 if (SrcReg.isVirtual())
6481 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32RegClass);
6482 else
6483 assert(SrcReg != AArch64::WSP);
6484 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6485 Opc = AArch64::STRSui;
6486 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6487 Opc = AArch64::STR_PPXI;
6488 StackID = TargetStackID::ScalablePredicateVector;
6489 }
6490 break;
6491 case 8:
6492 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6493 Opc = AArch64::STRXui;
6494 if (SrcReg.isVirtual())
6495 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
6496 else
6497 assert(SrcReg != AArch64::SP);
6498 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6499 Opc = AArch64::STRDui;
6500 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6501 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6502 MCID: get(Opcode: AArch64::STPWi), SrcReg, IsKill: isKill,
6503 SubIdx0: AArch64::sube32, SubIdx1: AArch64::subo32, FI, MMO);
6504 return;
6505 }
6506 break;
6507 case 16:
6508 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6509 Opc = AArch64::STRQui;
6510 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6511 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6512 Opc = AArch64::ST1Twov1d;
6513 Offset = false;
6514 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6515 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6516 MCID: get(Opcode: AArch64::STPXi), SrcReg, IsKill: isKill,
6517 SubIdx0: AArch64::sube64, SubIdx1: AArch64::subo64, FI, MMO);
6518 return;
6519 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6520 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6521 "Unexpected register store without SVE store instructions");
6522 Opc = AArch64::STR_ZXI;
6523 StackID = TargetStackID::ScalableVector;
6524 }
6525 break;
6526 case 24:
6527 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6528 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6529 Opc = AArch64::ST1Threev1d;
6530 Offset = false;
6531 }
6532 break;
6533 case 32:
6534 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6535 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6536 Opc = AArch64::ST1Fourv1d;
6537 Offset = false;
6538 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6539 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6540 Opc = AArch64::ST1Twov2d;
6541 Offset = false;
6542 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6543 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6544 "Unexpected register store without SVE store instructions");
6545 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
6546 StackID = TargetStackID::ScalableVector;
6547 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6548 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6549 "Unexpected register store without SVE store instructions");
6550 Opc = AArch64::STR_ZZXI;
6551 StackID = TargetStackID::ScalableVector;
6552 }
6553 break;
6554 case 48:
6555 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6556 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6557 Opc = AArch64::ST1Threev2d;
6558 Offset = false;
6559 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6560 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6561 "Unexpected register store without SVE store instructions");
6562 Opc = AArch64::STR_ZZZXI;
6563 StackID = TargetStackID::ScalableVector;
6564 }
6565 break;
6566 case 64:
6567 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6568 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6569 Opc = AArch64::ST1Fourv2d;
6570 Offset = false;
6571 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6572 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6573 "Unexpected register store without SVE store instructions");
6574 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
6575 StackID = TargetStackID::ScalableVector;
6576 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6577 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6578 "Unexpected register store without SVE store instructions");
6579 Opc = AArch64::STR_ZZZZXI;
6580 StackID = TargetStackID::ScalableVector;
6581 }
6582 break;
6583 }
6584 assert(Opc && "Unknown register class");
6585 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6586
6587 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6588 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
6589 .addFrameIndex(Idx: FI);
6590
6591 if (Offset)
6592 MI.addImm(Val: 0);
6593 if (PNRReg.isValid())
6594 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6595 MI.addMemOperand(MMO);
6596}
6597
6598static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
6599 MachineBasicBlock &MBB,
6600 MachineBasicBlock::iterator InsertBefore,
6601 const MCInstrDesc &MCID,
6602 Register DestReg, unsigned SubIdx0,
6603 unsigned SubIdx1, int FI,
6604 MachineMemOperand *MMO) {
6605 Register DestReg0 = DestReg;
6606 Register DestReg1 = DestReg;
6607 bool IsUndef = true;
6608 if (DestReg.isPhysical()) {
6609 DestReg0 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx0);
6610 SubIdx0 = 0;
6611 DestReg1 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx1);
6612 SubIdx1 = 0;
6613 IsUndef = false;
6614 }
6615 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6616 .addReg(RegNo: DestReg0, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx0)
6617 .addReg(RegNo: DestReg1, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx1)
6618 .addFrameIndex(Idx: FI)
6619 .addImm(Val: 0)
6620 .addMemOperand(MMO);
6621}
6622
6623void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
6624 MachineBasicBlock::iterator MBBI,
6625 Register DestReg, int FI,
6626 const TargetRegisterClass *RC,
6627 Register VReg, unsigned SubReg,
6628 MachineInstr::MIFlag Flags) const {
6629 MachineFunction &MF = *MBB.getParent();
6630 MachineFrameInfo &MFI = MF.getFrameInfo();
6631 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6632 MachineMemOperand *MMO =
6633 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
6634 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6635
6636 unsigned Opc = 0;
6637 bool Offset = true;
6638 unsigned StackID = TargetStackID::Default;
6639 Register PNRReg;
6640 switch (TRI.getSpillSize(RC: *RC)) {
6641 case 1:
6642 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6643 Opc = AArch64::LDRBui;
6644 break;
6645 case 2: {
6646 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
6647 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6648 Opc = AArch64::LDRHui;
6649 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
6650 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6651 "Unexpected register load without SVE load instructions");
6652 if (IsPNR)
6653 PNRReg = DestReg;
6654 Opc = AArch64::LDR_PXI;
6655 StackID = TargetStackID::ScalablePredicateVector;
6656 }
6657 break;
6658 }
6659 case 4:
6660 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6661 Opc = AArch64::LDRWui;
6662 if (DestReg.isVirtual())
6663 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR32RegClass);
6664 else
6665 assert(DestReg != AArch64::WSP);
6666 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6667 Opc = AArch64::LDRSui;
6668 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6669 Opc = AArch64::LDR_PPXI;
6670 StackID = TargetStackID::ScalablePredicateVector;
6671 }
6672 break;
6673 case 8:
6674 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6675 Opc = AArch64::LDRXui;
6676 if (DestReg.isVirtual())
6677 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR64RegClass);
6678 else
6679 assert(DestReg != AArch64::SP);
6680 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6681 Opc = AArch64::LDRDui;
6682 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6683 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6684 MCID: get(Opcode: AArch64::LDPWi), DestReg, SubIdx0: AArch64::sube32,
6685 SubIdx1: AArch64::subo32, FI, MMO);
6686 return;
6687 }
6688 break;
6689 case 16:
6690 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6691 Opc = AArch64::LDRQui;
6692 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6693 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6694 Opc = AArch64::LD1Twov1d;
6695 Offset = false;
6696 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6697 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6698 MCID: get(Opcode: AArch64::LDPXi), DestReg, SubIdx0: AArch64::sube64,
6699 SubIdx1: AArch64::subo64, FI, MMO);
6700 return;
6701 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6702 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6703 "Unexpected register load without SVE load instructions");
6704 Opc = AArch64::LDR_ZXI;
6705 StackID = TargetStackID::ScalableVector;
6706 }
6707 break;
6708 case 24:
6709 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6710 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6711 Opc = AArch64::LD1Threev1d;
6712 Offset = false;
6713 }
6714 break;
6715 case 32:
6716 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6717 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6718 Opc = AArch64::LD1Fourv1d;
6719 Offset = false;
6720 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6721 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6722 Opc = AArch64::LD1Twov2d;
6723 Offset = false;
6724 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6725 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6726 "Unexpected register load without SVE load instructions");
6727 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
6728 StackID = TargetStackID::ScalableVector;
6729 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6730 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6731 "Unexpected register load without SVE load instructions");
6732 Opc = AArch64::LDR_ZZXI;
6733 StackID = TargetStackID::ScalableVector;
6734 }
6735 break;
6736 case 48:
6737 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6738 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6739 Opc = AArch64::LD1Threev2d;
6740 Offset = false;
6741 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6742 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6743 "Unexpected register load without SVE load instructions");
6744 Opc = AArch64::LDR_ZZZXI;
6745 StackID = TargetStackID::ScalableVector;
6746 }
6747 break;
6748 case 64:
6749 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6750 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6751 Opc = AArch64::LD1Fourv2d;
6752 Offset = false;
6753 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6754 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6755 "Unexpected register load without SVE load instructions");
6756 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
6757 StackID = TargetStackID::ScalableVector;
6758 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6759 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6760 "Unexpected register load without SVE load instructions");
6761 Opc = AArch64::LDR_ZZZZXI;
6762 StackID = TargetStackID::ScalableVector;
6763 }
6764 break;
6765 }
6766
6767 assert(Opc && "Unknown register class");
6768 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6769
6770 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6771 .addReg(RegNo: DestReg, Flags: getDefRegState(B: true))
6772 .addFrameIndex(Idx: FI);
6773 if (Offset)
6774 MI.addImm(Val: 0);
6775 if (PNRReg.isValid() && !PNRReg.isVirtual())
6776 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6777 MI.addMemOperand(MMO);
6778}
6779
6780bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
6781 const MachineInstr &UseMI,
6782 const TargetRegisterInfo *TRI) {
6783 return any_of(Range: instructionsWithoutDebug(It: std::next(x: DefMI.getIterator()),
6784 End: UseMI.getIterator()),
6785 P: [TRI](const MachineInstr &I) {
6786 return I.modifiesRegister(Reg: AArch64::NZCV, TRI) ||
6787 I.readsRegister(Reg: AArch64::NZCV, TRI);
6788 });
6789}
6790
6791void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6792 const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized) {
6793 // The smallest scalable element supported by scaled SVE addressing
6794 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6795 // byte offset must always be a multiple of 2.
6796 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6797
6798 // VGSized offsets are divided by '2', because the VG register is the
6799 // the number of 64bit granules as opposed to 128bit vector chunks,
6800 // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
6801 // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
6802 // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
6803 ByteSized = Offset.getFixed();
6804 VGSized = Offset.getScalable() / 2;
6805}
6806
6807/// Returns the offset in parts to which this frame offset can be
6808/// decomposed for the purpose of describing a frame offset.
6809/// For non-scalable offsets this is simply its byte size.
6810void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6811 const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors,
6812 int64_t &NumDataVectors) {
6813 // The smallest scalable element supported by scaled SVE addressing
6814 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6815 // byte offset must always be a multiple of 2.
6816 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6817
6818 NumBytes = Offset.getFixed();
6819 NumDataVectors = 0;
6820 NumPredicateVectors = Offset.getScalable() / 2;
6821 // This method is used to get the offsets to adjust the frame offset.
6822 // If the function requires ADDPL to be used and needs more than two ADDPL
6823 // instructions, part of the offset is folded into NumDataVectors so that it
6824 // uses ADDVL for part of it, reducing the number of ADDPL instructions.
6825 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
6826 NumPredicateVectors > 62) {
6827 NumDataVectors = NumPredicateVectors / 8;
6828 NumPredicateVectors -= NumDataVectors * 8;
6829 }
6830}
6831
6832// Convenience function to create a DWARF expression for: Constant `Operation`.
6833// This helper emits compact sequences for common cases. For example, for`-15
6834// DW_OP_plus`, this helper would create DW_OP_lit15 DW_OP_minus.
6835static void appendConstantExpr(SmallVectorImpl<char> &Expr, int64_t Constant,
6836 dwarf::LocationAtom Operation) {
6837 if (Operation == dwarf::DW_OP_plus && Constant < 0 && -Constant <= 31) {
6838 // -Constant (1 to 31)
6839 Expr.push_back(Elt: dwarf::DW_OP_lit0 - Constant);
6840 Operation = dwarf::DW_OP_minus;
6841 } else if (Constant >= 0 && Constant <= 31) {
6842 // Literal value 0 to 31
6843 Expr.push_back(Elt: dwarf::DW_OP_lit0 + Constant);
6844 } else {
6845 // Signed constant
6846 Expr.push_back(Elt: dwarf::DW_OP_consts);
6847 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: Constant);
6848 }
6849 return Expr.push_back(Elt: Operation);
6850}
6851
6852// Convenience function to create a DWARF expression for a register.
6853static void appendReadRegExpr(SmallVectorImpl<char> &Expr, unsigned RegNum) {
6854 Expr.push_back(Elt: (char)dwarf::DW_OP_bregx);
6855 appendLEB128<LEB128Sign::Unsigned>(Buffer&: Expr, Value: RegNum);
6856 Expr.push_back(Elt: 0);
6857}
6858
6859// Convenience function to create a DWARF expression for loading a register from
6860// a CFA offset.
6861static void appendLoadRegExpr(SmallVectorImpl<char> &Expr,
6862 int64_t OffsetFromDefCFA) {
6863 // This assumes the top of the DWARF stack contains the CFA.
6864 Expr.push_back(Elt: dwarf::DW_OP_dup);
6865 // Add the offset to the register.
6866 appendConstantExpr(Expr, Constant: OffsetFromDefCFA, Operation: dwarf::DW_OP_plus);
6867 // Dereference the address (loads a 64 bit value)..
6868 Expr.push_back(Elt: dwarf::DW_OP_deref);
6869}
6870
6871// Convenience function to create a comment for
6872// (+/-) NumBytes (* RegScale)?
6873static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
6874 StringRef RegScale = {}) {
6875 if (NumBytes) {
6876 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(x: NumBytes);
6877 if (!RegScale.empty())
6878 Comment << ' ' << RegScale;
6879 }
6880}
6881
6882// Creates an MCCFIInstruction:
6883// { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
6884static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
6885 unsigned Reg,
6886 const StackOffset &Offset) {
6887 int64_t NumBytes, NumVGScaledBytes;
6888 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(Offset, ByteSized&: NumBytes,
6889 VGSized&: NumVGScaledBytes);
6890 std::string CommentBuffer;
6891 llvm::raw_string_ostream Comment(CommentBuffer);
6892
6893 if (Reg == AArch64::SP)
6894 Comment << "sp";
6895 else if (Reg == AArch64::FP)
6896 Comment << "fp";
6897 else
6898 Comment << printReg(Reg, TRI: &TRI);
6899
6900 // Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
6901 SmallString<64> Expr;
6902 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6903 assert(DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
6904 // Reg + NumBytes
6905 Expr.push_back(Elt: dwarf::DW_OP_breg0 + DwarfReg);
6906 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: NumBytes);
6907 appendOffsetComment(NumBytes, Comment);
6908 if (NumVGScaledBytes) {
6909 // + VG * NumVGScaledBytes
6910 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: "* VG");
6911 appendReadRegExpr(Expr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6912 appendConstantExpr(Expr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6913 Expr.push_back(Elt: dwarf::DW_OP_plus);
6914 }
6915
6916 // Wrap this into DW_CFA_def_cfa.
6917 SmallString<64> DefCfaExpr;
6918 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
6919 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
6920 DefCfaExpr.append(RHS: Expr.str());
6921 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
6922 Comment: Comment.str());
6923}
6924
6925MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
6926 unsigned FrameReg, unsigned Reg,
6927 const StackOffset &Offset,
6928 bool LastAdjustmentWasScalable) {
6929 if (Offset.getScalable())
6930 return createDefCFAExpression(TRI, Reg, Offset);
6931
6932 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6933 return MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: int(Offset.getFixed()));
6934
6935 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6936 return MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfReg, Offset: (int)Offset.getFixed());
6937}
6938
6939MCCFIInstruction
6940llvm::createCFAOffset(const TargetRegisterInfo &TRI, unsigned Reg,
6941 const StackOffset &OffsetFromDefCFA,
6942 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6943 int64_t NumBytes, NumVGScaledBytes;
6944 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6945 Offset: OffsetFromDefCFA, ByteSized&: NumBytes, VGSized&: NumVGScaledBytes);
6946
6947 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6948
6949 // Non-scalable offsets can use DW_CFA_offset directly.
6950 if (!NumVGScaledBytes)
6951 return MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: NumBytes);
6952
6953 std::string CommentBuffer;
6954 llvm::raw_string_ostream Comment(CommentBuffer);
6955 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
6956
6957 // Build up expression (CFA + VG * NumVGScaledBytes + NumBytes)
6958 assert(NumVGScaledBytes && "Expected scalable offset");
6959 SmallString<64> OffsetExpr;
6960 // + VG * NumVGScaledBytes
6961 StringRef VGRegScale;
6962 if (IncomingVGOffsetFromDefCFA) {
6963 appendLoadRegExpr(Expr&: OffsetExpr, OffsetFromDefCFA: *IncomingVGOffsetFromDefCFA);
6964 VGRegScale = "* IncomingVG";
6965 } else {
6966 appendReadRegExpr(Expr&: OffsetExpr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6967 VGRegScale = "* VG";
6968 }
6969 appendConstantExpr(Expr&: OffsetExpr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6970 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: VGRegScale);
6971 OffsetExpr.push_back(Elt: dwarf::DW_OP_plus);
6972 if (NumBytes) {
6973 // + NumBytes
6974 appendOffsetComment(NumBytes, Comment);
6975 appendConstantExpr(Expr&: OffsetExpr, Constant: NumBytes, Operation: dwarf::DW_OP_plus);
6976 }
6977
6978 // Wrap this into DW_CFA_expression
6979 SmallString<64> CfaExpr;
6980 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
6981 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: DwarfReg);
6982 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: OffsetExpr.size());
6983 CfaExpr.append(RHS: OffsetExpr.str());
6984
6985 return MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str(), Loc: SMLoc(),
6986 Comment: Comment.str());
6987}
6988
6989// Helper function to emit a frame offset adjustment from a given
6990// pointer (SrcReg), stored into DestReg. This function is explicit
6991// in that it requires the opcode.
6992static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
6993 MachineBasicBlock::iterator MBBI,
6994 const DebugLoc &DL, unsigned DestReg,
6995 unsigned SrcReg, int64_t Offset, unsigned Opc,
6996 const TargetInstrInfo *TII,
6997 MachineInstr::MIFlag Flag, bool NeedsWinCFI,
6998 bool *HasWinCFI, bool EmitCFAOffset,
6999 StackOffset CFAOffset, unsigned FrameReg) {
7000 int Sign = 1;
7001 unsigned MaxEncoding, ShiftSize;
7002 switch (Opc) {
7003 case AArch64::ADDXri:
7004 case AArch64::ADDSXri:
7005 case AArch64::SUBXri:
7006 case AArch64::SUBSXri:
7007 MaxEncoding = 0xfff;
7008 ShiftSize = 12;
7009 break;
7010 case AArch64::ADDVL_XXI:
7011 case AArch64::ADDPL_XXI:
7012 case AArch64::ADDSVL_XXI:
7013 case AArch64::ADDSPL_XXI:
7014 MaxEncoding = 31;
7015 ShiftSize = 0;
7016 if (Offset < 0) {
7017 MaxEncoding = 32;
7018 Sign = -1;
7019 Offset = -Offset;
7020 }
7021 break;
7022 default:
7023 llvm_unreachable("Unsupported opcode");
7024 }
7025
7026 // `Offset` can be in bytes or in "scalable bytes".
7027 int VScale = 1;
7028 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
7029 VScale = 16;
7030 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
7031 VScale = 2;
7032
7033 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
7034 // scratch register. If DestReg is a virtual register, use it as the
7035 // scratch register; otherwise, create a new virtual register (to be
7036 // replaced by the scavenger at the end of PEI). That case can be optimized
7037 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
7038 // register can be loaded with offset%8 and the add/sub can use an extending
7039 // instruction with LSL#3.
7040 // Currently the function handles any offsets but generates a poor sequence
7041 // of code.
7042 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
7043
7044 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
7045 Register TmpReg = DestReg;
7046 if (TmpReg == AArch64::XZR)
7047 TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
7048 RegClass: &AArch64::GPR64RegClass);
7049 do {
7050 uint64_t ThisVal = std::min<uint64_t>(a: Offset, b: MaxEncodableValue);
7051 unsigned LocalShiftSize = 0;
7052 if (ThisVal > MaxEncoding) {
7053 ThisVal = ThisVal >> ShiftSize;
7054 LocalShiftSize = ShiftSize;
7055 }
7056 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
7057 "Encoding cannot handle value that big");
7058
7059 Offset -= ThisVal << LocalShiftSize;
7060 if (Offset == 0)
7061 TmpReg = DestReg;
7062 auto MBI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: TmpReg)
7063 .addReg(RegNo: SrcReg)
7064 .addImm(Val: Sign * (int)ThisVal);
7065 if (ShiftSize)
7066 MBI = MBI.addImm(
7067 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LocalShiftSize));
7068 MBI = MBI.setMIFlag(Flag);
7069
7070 auto Change =
7071 VScale == 1
7072 ? StackOffset::getFixed(Fixed: ThisVal << LocalShiftSize)
7073 : StackOffset::getScalable(Scalable: VScale * (ThisVal << LocalShiftSize));
7074 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
7075 CFAOffset += Change;
7076 else
7077 CFAOffset -= Change;
7078 if (EmitCFAOffset && DestReg == TmpReg) {
7079 MachineFunction &MF = *MBB.getParent();
7080 const TargetSubtargetInfo &STI = MF.getSubtarget();
7081 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
7082
7083 unsigned CFIIndex = MF.addFrameInst(
7084 Inst: createDefCFA(TRI, FrameReg, Reg: DestReg, Offset: CFAOffset, LastAdjustmentWasScalable: VScale != 1));
7085 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
7086 .addCFIIndex(CFIIndex)
7087 .setMIFlags(Flag);
7088 }
7089
7090 if (NeedsWinCFI) {
7091 int Imm = (int)(ThisVal << LocalShiftSize);
7092 if (VScale != 1 && DestReg == AArch64::SP) {
7093 if (HasWinCFI)
7094 *HasWinCFI = true;
7095 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AllocZ))
7096 .addImm(Val: ThisVal)
7097 .setMIFlag(Flag);
7098 } else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
7099 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
7100 assert(VScale == 1 && "Expected non-scalable operation");
7101 if (HasWinCFI)
7102 *HasWinCFI = true;
7103 if (Imm == 0)
7104 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_SetFP)).setMIFlag(Flag);
7105 else
7106 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AddFP))
7107 .addImm(Val: Imm)
7108 .setMIFlag(Flag);
7109 assert(Offset == 0 && "Expected remaining offset to be zero to "
7110 "emit a single SEH directive");
7111 } else if (DestReg == AArch64::SP) {
7112 assert(VScale == 1 && "Expected non-scalable operation");
7113 if (HasWinCFI)
7114 *HasWinCFI = true;
7115 assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
7116 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_StackAlloc))
7117 .addImm(Val: Imm)
7118 .setMIFlag(Flag);
7119 }
7120 }
7121
7122 SrcReg = TmpReg;
7123 } while (Offset);
7124}
7125
7126void llvm::emitFrameOffset(MachineBasicBlock &MBB,
7127 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
7128 unsigned DestReg, unsigned SrcReg,
7129 StackOffset Offset, const TargetInstrInfo *TII,
7130 MachineInstr::MIFlag Flag, bool SetNZCV,
7131 bool NeedsWinCFI, bool *HasWinCFI,
7132 bool EmitCFAOffset, StackOffset CFAOffset,
7133 unsigned FrameReg) {
7134 // If a function is marked as arm_locally_streaming, then the runtime value of
7135 // vscale in the prologue/epilogue is different the runtime value of vscale
7136 // in the function's body. To avoid having to consider multiple vscales,
7137 // we can use `addsvl` to allocate any scalable stack-slots, which under
7138 // most circumstances will be only locals, not callee-save slots.
7139 const Function &F = MBB.getParent()->getFunction();
7140 bool UseSVL = F.hasFnAttribute(Kind: "aarch64_pstate_sm_body");
7141
7142 int64_t Bytes, NumPredicateVectors, NumDataVectors;
7143 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
7144 Offset, NumBytes&: Bytes, NumPredicateVectors, NumDataVectors);
7145
7146 // Insert ADDSXri for scalable offset at the end.
7147 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
7148 if (NeedsFinalDefNZCV)
7149 SetNZCV = false;
7150
7151 // First emit non-scalable frame offsets, or a simple 'mov'.
7152 if (Bytes || (!Offset && SrcReg != DestReg)) {
7153 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
7154 "SP increment/decrement not 8-byte aligned");
7155 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
7156 if (Bytes < 0) {
7157 Bytes = -Bytes;
7158 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
7159 }
7160 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: Bytes, Opc, TII, Flag,
7161 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7162 FrameReg);
7163 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
7164 ? StackOffset::getFixed(Fixed: -Bytes)
7165 : StackOffset::getFixed(Fixed: Bytes);
7166 SrcReg = DestReg;
7167 FrameReg = DestReg;
7168 }
7169
7170 assert(!(NeedsWinCFI && NumPredicateVectors) &&
7171 "WinCFI can't allocate fractions of an SVE data vector");
7172
7173 if (NumDataVectors) {
7174 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumDataVectors,
7175 Opc: UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI, TII,
7176 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7177 FrameReg);
7178 CFAOffset += StackOffset::getScalable(Scalable: -NumDataVectors * 16);
7179 SrcReg = DestReg;
7180 }
7181
7182 if (NumPredicateVectors) {
7183 assert(DestReg != AArch64::SP && "Unaligned access to SP");
7184 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumPredicateVectors,
7185 Opc: UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI, TII,
7186 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7187 FrameReg);
7188 }
7189
7190 if (NeedsFinalDefNZCV)
7191 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDSXri), DestReg)
7192 .addReg(RegNo: DestReg)
7193 .addImm(Val: 0)
7194 .addImm(Val: 0);
7195}
7196
7197MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
7198 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
7199 int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS,
7200 VirtRegMap *VRM) const {
7201 MachineBasicBlock::iterator InsertPt = MI;
7202 // This is a bit of a hack. Consider this instruction:
7203 //
7204 // %0 = COPY %sp; GPR64all:%0
7205 //
7206 // We explicitly chose GPR64all for the virtual register so such a copy might
7207 // be eliminated by RegisterCoalescer. However, that may not be possible, and
7208 // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
7209 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
7210 //
7211 // To prevent that, we are going to constrain the %0 register class here.
7212 if (MI.isFullCopy()) {
7213 Register DstReg = MI.getOperand(i: 0).getReg();
7214 Register SrcReg = MI.getOperand(i: 1).getReg();
7215 if (SrcReg == AArch64::SP && DstReg.isVirtual()) {
7216 MF.getRegInfo().constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass);
7217 return nullptr;
7218 }
7219 if (DstReg == AArch64::SP && SrcReg.isVirtual()) {
7220 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
7221 return nullptr;
7222 }
7223 // Nothing can folded with copy from/to NZCV.
7224 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
7225 return nullptr;
7226 }
7227
7228 // Handle the case where a copy is being spilled or filled but the source
7229 // and destination register class don't match. For example:
7230 //
7231 // %0 = COPY %xzr; GPR64common:%0
7232 //
7233 // In this case we can still safely fold away the COPY and generate the
7234 // following spill code:
7235 //
7236 // STRXui %xzr, %stack.0
7237 //
7238 // This also eliminates spilled cross register class COPYs (e.g. between x and
7239 // d regs) of the same size. For example:
7240 //
7241 // %0 = COPY %1; GPR64:%0, FPR64:%1
7242 //
7243 // will be filled as
7244 //
7245 // LDRDui %0, fi<#0>
7246 //
7247 // instead of
7248 //
7249 // LDRXui %Temp, fi<#0>
7250 // %0 = FMOV %Temp
7251 //
7252 if (MI.isCopy() && Ops.size() == 1 &&
7253 // Make sure we're only folding the explicit COPY defs/uses.
7254 (Ops[0] == 0 || Ops[0] == 1)) {
7255 bool IsSpill = Ops[0] == 0;
7256 bool IsFill = !IsSpill;
7257 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7258 const MachineRegisterInfo &MRI = MF.getRegInfo();
7259 MachineBasicBlock &MBB = *MI.getParent();
7260 const MachineOperand &DstMO = MI.getOperand(i: 0);
7261 const MachineOperand &SrcMO = MI.getOperand(i: 1);
7262 Register DstReg = DstMO.getReg();
7263 Register SrcReg = SrcMO.getReg();
7264 // This is slightly expensive to compute for physical regs since
7265 // getMinimalPhysRegClass is slow.
7266 auto getRegClass = [&](unsigned Reg) {
7267 return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
7268 : TRI.getMinimalPhysRegClass(Reg);
7269 };
7270
7271 if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
7272 assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
7273 TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
7274 "Mismatched register size in non subreg COPY");
7275 if (IsSpill)
7276 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg, isKill: SrcMO.isKill(), FI: FrameIndex,
7277 RC: getRegClass(SrcReg), VReg: Register());
7278 else
7279 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex,
7280 RC: getRegClass(DstReg), VReg: Register());
7281 return &*--InsertPt;
7282 }
7283
7284 // Handle cases like spilling def of:
7285 //
7286 // %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
7287 //
7288 // where the physical register source can be widened and stored to the full
7289 // virtual reg destination stack slot, in this case producing:
7290 //
7291 // STRXui %xzr, %stack.0
7292 //
7293 if (IsSpill && DstMO.isUndef() && SrcReg == AArch64::WZR &&
7294 TRI.getRegSizeInBits(RC: *getRegClass(DstReg)) == 64) {
7295 assert(SrcMO.getSubReg() == 0 &&
7296 "Unexpected subreg on physical register");
7297 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg: AArch64::XZR, isKill: SrcMO.isKill(),
7298 FI: FrameIndex, RC: &AArch64::GPR64RegClass, VReg: Register());
7299 return &*--InsertPt;
7300 }
7301
7302 // Handle cases like filling use of:
7303 //
7304 // %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
7305 //
7306 // where we can load the full virtual reg source stack slot, into the subreg
7307 // destination, in this case producing:
7308 //
7309 // LDRWui %0:sub_32<def,read-undef>, %stack.0
7310 //
7311 if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
7312 const TargetRegisterClass *FillRC = nullptr;
7313 switch (DstMO.getSubReg()) {
7314 default:
7315 break;
7316 case AArch64::sub_32:
7317 if (AArch64::GPR64RegClass.hasSubClassEq(RC: getRegClass(DstReg)))
7318 FillRC = &AArch64::GPR32RegClass;
7319 break;
7320 case AArch64::ssub:
7321 FillRC = &AArch64::FPR32RegClass;
7322 break;
7323 case AArch64::dsub:
7324 FillRC = &AArch64::FPR64RegClass;
7325 break;
7326 }
7327
7328 if (FillRC) {
7329 assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
7330 TRI.getRegSizeInBits(*FillRC) &&
7331 "Mismatched regclass size on folded subreg COPY");
7332 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex, RC: FillRC,
7333 VReg: Register());
7334 MachineInstr &LoadMI = *--InsertPt;
7335 MachineOperand &LoadDst = LoadMI.getOperand(i: 0);
7336 assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
7337 LoadDst.setSubReg(DstMO.getSubReg());
7338 LoadDst.setIsUndef();
7339 return &LoadMI;
7340 }
7341 }
7342 }
7343
7344 // Cannot fold.
7345 return nullptr;
7346}
7347
7348int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
7349 StackOffset &SOffset,
7350 bool *OutUseUnscaledOp,
7351 unsigned *OutUnscaledOp,
7352 int64_t *EmittableOffset) {
7353 // Set output values in case of early exit.
7354 if (EmittableOffset)
7355 *EmittableOffset = 0;
7356 if (OutUseUnscaledOp)
7357 *OutUseUnscaledOp = false;
7358 if (OutUnscaledOp)
7359 *OutUnscaledOp = 0;
7360
7361 // Exit early for structured vector spills/fills as they can't take an
7362 // immediate offset.
7363 switch (MI.getOpcode()) {
7364 default:
7365 break;
7366 case AArch64::LD1Rv1d:
7367 case AArch64::LD1Rv2s:
7368 case AArch64::LD1Rv2d:
7369 case AArch64::LD1Rv4h:
7370 case AArch64::LD1Rv4s:
7371 case AArch64::LD1Rv8b:
7372 case AArch64::LD1Rv8h:
7373 case AArch64::LD1Rv16b:
7374 case AArch64::LD1Twov2d:
7375 case AArch64::LD1Threev2d:
7376 case AArch64::LD1Fourv2d:
7377 case AArch64::LD1Twov1d:
7378 case AArch64::LD1Threev1d:
7379 case AArch64::LD1Fourv1d:
7380 case AArch64::ST1Twov2d:
7381 case AArch64::ST1Threev2d:
7382 case AArch64::ST1Fourv2d:
7383 case AArch64::ST1Twov1d:
7384 case AArch64::ST1Threev1d:
7385 case AArch64::ST1Fourv1d:
7386 case AArch64::ST1i8:
7387 case AArch64::ST1i16:
7388 case AArch64::ST1i32:
7389 case AArch64::ST1i64:
7390 case AArch64::IRG:
7391 case AArch64::IRGstack:
7392 case AArch64::STGloop:
7393 case AArch64::STZGloop:
7394 return AArch64FrameOffsetCannotUpdate;
7395 }
7396
7397 // Get the min/max offset and the scale.
7398 TypeSize ScaleValue(0U, false), Width(0U, false);
7399 int64_t MinOff, MaxOff;
7400 if (!AArch64InstrInfo::getMemOpInfo(Opcode: MI.getOpcode(), Scale&: ScaleValue, Width, MinOffset&: MinOff,
7401 MaxOffset&: MaxOff))
7402 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7403
7404 // Construct the complete offset.
7405 bool IsMulVL = ScaleValue.isScalable();
7406 unsigned Scale = ScaleValue.getKnownMinValue();
7407 int64_t Offset = IsMulVL ? SOffset.getScalable() : SOffset.getFixed();
7408
7409 const MachineOperand &ImmOpnd =
7410 MI.getOperand(i: AArch64InstrInfo::getLoadStoreImmIdx(Opc: MI.getOpcode()));
7411 Offset += ImmOpnd.getImm() * Scale;
7412
7413 // If the offset doesn't match the scale, we rewrite the instruction to
7414 // use the unscaled instruction instead. Likewise, if we have a negative
7415 // offset and there is an unscaled op to use.
7416 std::optional<unsigned> UnscaledOp =
7417 AArch64InstrInfo::getUnscaledLdSt(Opc: MI.getOpcode());
7418 bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
7419 if (useUnscaledOp &&
7420 !AArch64InstrInfo::getMemOpInfo(Opcode: *UnscaledOp, Scale&: ScaleValue, Width, MinOffset&: MinOff,
7421 MaxOffset&: MaxOff))
7422 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7423
7424 Scale = ScaleValue.getKnownMinValue();
7425 assert(IsMulVL == ScaleValue.isScalable() &&
7426 "Unscaled opcode has different value for scalable");
7427
7428 int64_t Remainder = Offset % Scale;
7429 assert(!(Remainder && useUnscaledOp) &&
7430 "Cannot have remainder when using unscaled op");
7431
7432 assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
7433 int64_t NewOffset = Offset / Scale;
7434 if (MinOff <= NewOffset && NewOffset <= MaxOff)
7435 Offset = Remainder;
7436 else {
7437 // Try to minimise the number of instructions required to materialise the
7438 // offset calculation. Specifically, for fixed offsets, if masking out the
7439 // low 12 bits leaves a legal add immediate, we can realise the offset
7440 // calculation with a single add instruction. Whenever this is possible,
7441 // prefer this split.
7442 int64_t HighPart = Offset & ~0xFFF;
7443 int64_t LowPart = Offset & 0xFFF;
7444 int64_t LowScaled = LowPart / Scale;
7445 if (!IsMulVL && NewOffset >= 0 && LowPart % Scale == 0 &&
7446 MinOff <= LowScaled && LowScaled <= MaxOff &&
7447 AArch64_AM::isLegalArithImmed(C: HighPart)) {
7448 NewOffset = LowScaled;
7449 Offset = HighPart;
7450 } else {
7451 // Default to a greedy split: take the memop immediate to be maximum /
7452 // minimum expressible offset and materialise the remainder.
7453 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
7454 Offset = Offset - (NewOffset * Scale);
7455 }
7456 }
7457
7458 if (EmittableOffset)
7459 *EmittableOffset = NewOffset;
7460 if (OutUseUnscaledOp)
7461 *OutUseUnscaledOp = useUnscaledOp;
7462 if (OutUnscaledOp && UnscaledOp)
7463 *OutUnscaledOp = *UnscaledOp;
7464
7465 if (IsMulVL)
7466 SOffset = StackOffset::get(Fixed: SOffset.getFixed(), Scalable: Offset);
7467 else
7468 SOffset = StackOffset::get(Fixed: Offset, Scalable: SOffset.getScalable());
7469 return AArch64FrameOffsetCanUpdate |
7470 (SOffset ? 0 : AArch64FrameOffsetIsLegal);
7471}
7472
7473bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
7474 unsigned FrameReg, StackOffset &Offset,
7475 const AArch64InstrInfo *TII) {
7476 unsigned Opcode = MI.getOpcode();
7477 unsigned ImmIdx = FrameRegIdx + 1;
7478
7479 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
7480 Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: ImmIdx).getImm());
7481 emitFrameOffset(MBB&: *MI.getParent(), MBBI: MI, DL: MI.getDebugLoc(),
7482 DestReg: MI.getOperand(i: 0).getReg(), SrcReg: FrameReg, Offset, TII,
7483 Flag: MachineInstr::NoFlags, SetNZCV: (Opcode == AArch64::ADDSXri));
7484 MI.eraseFromParent();
7485 Offset = StackOffset();
7486 return true;
7487 }
7488
7489 int64_t NewOffset;
7490 unsigned UnscaledOp;
7491 bool UseUnscaledOp;
7492 int Status = isAArch64FrameOffsetLegal(MI, SOffset&: Offset, OutUseUnscaledOp: &UseUnscaledOp,
7493 OutUnscaledOp: &UnscaledOp, EmittableOffset: &NewOffset);
7494 if (Status & AArch64FrameOffsetCanUpdate) {
7495 if (Status & AArch64FrameOffsetIsLegal)
7496 // Replace the FrameIndex with FrameReg.
7497 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
7498 if (UseUnscaledOp)
7499 MI.setDesc(TII->get(Opcode: UnscaledOp));
7500
7501 MI.getOperand(i: ImmIdx).ChangeToImmediate(ImmVal: NewOffset);
7502 return !Offset;
7503 }
7504
7505 return false;
7506}
7507
7508void AArch64InstrInfo::insertNoop(MachineBasicBlock &MBB,
7509 MachineBasicBlock::iterator MI) const {
7510 DebugLoc DL;
7511 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::NOP));
7512}
7513
7514MCInst AArch64InstrInfo::getNop() const { return MCInstBuilder(AArch64::NOP); }
7515
7516// AArch64 supports MachineCombiner.
7517bool AArch64InstrInfo::useMachineCombiner() const { return true; }
7518
7519// True when Opc sets flag
7520static bool isCombineInstrSettingFlag(unsigned Opc) {
7521 switch (Opc) {
7522 case AArch64::ADDSWrr:
7523 case AArch64::ADDSWri:
7524 case AArch64::ADDSXrr:
7525 case AArch64::ADDSXri:
7526 case AArch64::SUBSWrr:
7527 case AArch64::SUBSXrr:
7528 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7529 case AArch64::SUBSWri:
7530 case AArch64::SUBSXri:
7531 return true;
7532 default:
7533 break;
7534 }
7535 return false;
7536}
7537
7538// 32b Opcodes that can be combined with a MUL
7539static bool isCombineInstrCandidate32(unsigned Opc) {
7540 switch (Opc) {
7541 case AArch64::ADDWrr:
7542 case AArch64::ADDWri:
7543 case AArch64::SUBWrr:
7544 case AArch64::ADDSWrr:
7545 case AArch64::ADDSWri:
7546 case AArch64::SUBSWrr:
7547 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7548 case AArch64::SUBWri:
7549 case AArch64::SUBSWri:
7550 return true;
7551 default:
7552 break;
7553 }
7554 return false;
7555}
7556
7557// 64b Opcodes that can be combined with a MUL
7558static bool isCombineInstrCandidate64(unsigned Opc) {
7559 switch (Opc) {
7560 case AArch64::ADDXrr:
7561 case AArch64::ADDXri:
7562 case AArch64::SUBXrr:
7563 case AArch64::ADDSXrr:
7564 case AArch64::ADDSXri:
7565 case AArch64::SUBSXrr:
7566 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7567 case AArch64::SUBXri:
7568 case AArch64::SUBSXri:
7569 case AArch64::ADDv8i8:
7570 case AArch64::ADDv16i8:
7571 case AArch64::ADDv4i16:
7572 case AArch64::ADDv8i16:
7573 case AArch64::ADDv2i32:
7574 case AArch64::ADDv4i32:
7575 case AArch64::SUBv8i8:
7576 case AArch64::SUBv16i8:
7577 case AArch64::SUBv4i16:
7578 case AArch64::SUBv8i16:
7579 case AArch64::SUBv2i32:
7580 case AArch64::SUBv4i32:
7581 return true;
7582 default:
7583 break;
7584 }
7585 return false;
7586}
7587
7588// FP Opcodes that can be combined with a FMUL.
7589static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
7590 switch (Inst.getOpcode()) {
7591 default:
7592 break;
7593 case AArch64::FADDHrr:
7594 case AArch64::FADDSrr:
7595 case AArch64::FADDDrr:
7596 case AArch64::FADDv4f16:
7597 case AArch64::FADDv8f16:
7598 case AArch64::FADDv2f32:
7599 case AArch64::FADDv2f64:
7600 case AArch64::FADDv4f32:
7601 case AArch64::FSUBHrr:
7602 case AArch64::FSUBSrr:
7603 case AArch64::FSUBDrr:
7604 case AArch64::FSUBv4f16:
7605 case AArch64::FSUBv8f16:
7606 case AArch64::FSUBv2f32:
7607 case AArch64::FSUBv2f64:
7608 case AArch64::FSUBv4f32:
7609 // We can fuse FADD/FSUB with FMUL, if FADD/FSUB has the contract fast-math
7610 // flag.
7611 return Inst.getFlag(Flag: MachineInstr::FmContract);
7612 }
7613 return false;
7614}
7615
7616// Opcodes that can be combined with a MUL
7617static bool isCombineInstrCandidate(unsigned Opc) {
7618 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
7619}
7620
7621//
7622// Utility routine that checks if \param MO is defined by an
7623// \param CombineOpc instruction in the basic block \param MBB
7624static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
7625 unsigned CombineOpc, unsigned ZeroReg = 0,
7626 bool CheckZeroReg = false) {
7627 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7628 MachineInstr *MI = nullptr;
7629
7630 if (MO.isReg() && MO.getReg().isVirtual())
7631 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
7632 // And it needs to be in the trace (otherwise, it won't have a depth).
7633 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
7634 return false;
7635 // Must only used by the user we combine with.
7636 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
7637 return false;
7638
7639 if (CheckZeroReg) {
7640 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
7641 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
7642 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
7643 // The third input reg must be zero.
7644 if (MI->getOperand(i: 3).getReg() != ZeroReg)
7645 return false;
7646 }
7647
7648 if (isCombineInstrSettingFlag(Opc: CombineOpc) &&
7649 MI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) == -1)
7650 return false;
7651
7652 return true;
7653}
7654
7655//
7656// Is \param MO defined by an integer multiply and can be combined?
7657static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7658 unsigned MulOpc, unsigned ZeroReg) {
7659 return canCombine(MBB, MO, CombineOpc: MulOpc, ZeroReg, CheckZeroReg: true);
7660}
7661
7662//
7663// Is \param MO defined by a floating-point multiply and can be combined?
7664static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7665 unsigned MulOpc) {
7666 return canCombine(MBB, MO, CombineOpc: MulOpc);
7667}
7668
7669// TODO: There are many more machine instruction opcodes to match:
7670// 1. Other data types (integer, vectors)
7671// 2. Other math / logic operations (xor, or)
7672// 3. Other forms of the same operation (intrinsics and other variants)
7673bool AArch64InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
7674 bool Invert) const {
7675 if (Invert)
7676 return false;
7677 switch (Inst.getOpcode()) {
7678 // == Floating-point types ==
7679 // -- Floating-point instructions --
7680 case AArch64::FADDHrr:
7681 case AArch64::FADDSrr:
7682 case AArch64::FADDDrr:
7683 case AArch64::FMULHrr:
7684 case AArch64::FMULSrr:
7685 case AArch64::FMULDrr:
7686 case AArch64::FMULX16:
7687 case AArch64::FMULX32:
7688 case AArch64::FMULX64:
7689 // -- Advanced SIMD instructions --
7690 case AArch64::FADDv4f16:
7691 case AArch64::FADDv8f16:
7692 case AArch64::FADDv2f32:
7693 case AArch64::FADDv4f32:
7694 case AArch64::FADDv2f64:
7695 case AArch64::FMULv4f16:
7696 case AArch64::FMULv8f16:
7697 case AArch64::FMULv2f32:
7698 case AArch64::FMULv4f32:
7699 case AArch64::FMULv2f64:
7700 case AArch64::FMULXv4f16:
7701 case AArch64::FMULXv8f16:
7702 case AArch64::FMULXv2f32:
7703 case AArch64::FMULXv4f32:
7704 case AArch64::FMULXv2f64:
7705 // -- SVE instructions --
7706 // Opcodes FMULX_ZZZ_? don't exist because there is no unpredicated FMULX
7707 // in the SVE instruction set (though there are predicated ones).
7708 case AArch64::FADD_ZZZ_H:
7709 case AArch64::FADD_ZZZ_S:
7710 case AArch64::FADD_ZZZ_D:
7711 case AArch64::FMUL_ZZZ_H:
7712 case AArch64::FMUL_ZZZ_S:
7713 case AArch64::FMUL_ZZZ_D:
7714 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
7715 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
7716
7717 // == Integer types ==
7718 // -- Base instructions --
7719 // Opcodes MULWrr and MULXrr don't exist because
7720 // `MUL <Wd>, <Wn>, <Wm>` and `MUL <Xd>, <Xn>, <Xm>` are aliases of
7721 // `MADD <Wd>, <Wn>, <Wm>, WZR` and `MADD <Xd>, <Xn>, <Xm>, XZR` respectively.
7722 // The machine-combiner does not support three-source-operands machine
7723 // instruction. So we cannot reassociate MULs.
7724 case AArch64::ADDWrr:
7725 case AArch64::ADDXrr:
7726 case AArch64::ANDWrr:
7727 case AArch64::ANDXrr:
7728 case AArch64::ORRWrr:
7729 case AArch64::ORRXrr:
7730 case AArch64::EORWrr:
7731 case AArch64::EORXrr:
7732 case AArch64::EONWrr:
7733 case AArch64::EONXrr:
7734 // -- Advanced SIMD instructions --
7735 // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL
7736 // in the Advanced SIMD instruction set.
7737 case AArch64::ADDv8i8:
7738 case AArch64::ADDv16i8:
7739 case AArch64::ADDv4i16:
7740 case AArch64::ADDv8i16:
7741 case AArch64::ADDv2i32:
7742 case AArch64::ADDv4i32:
7743 case AArch64::ADDv1i64:
7744 case AArch64::ADDv2i64:
7745 case AArch64::MULv8i8:
7746 case AArch64::MULv16i8:
7747 case AArch64::MULv4i16:
7748 case AArch64::MULv8i16:
7749 case AArch64::MULv2i32:
7750 case AArch64::MULv4i32:
7751 case AArch64::ANDv8i8:
7752 case AArch64::ANDv16i8:
7753 case AArch64::ORRv8i8:
7754 case AArch64::ORRv16i8:
7755 case AArch64::EORv8i8:
7756 case AArch64::EORv16i8:
7757 // -- SVE instructions --
7758 case AArch64::ADD_ZZZ_B:
7759 case AArch64::ADD_ZZZ_H:
7760 case AArch64::ADD_ZZZ_S:
7761 case AArch64::ADD_ZZZ_D:
7762 case AArch64::MUL_ZZZ_B:
7763 case AArch64::MUL_ZZZ_H:
7764 case AArch64::MUL_ZZZ_S:
7765 case AArch64::MUL_ZZZ_D:
7766 case AArch64::AND_ZZZ:
7767 case AArch64::ORR_ZZZ:
7768 case AArch64::EOR_ZZZ:
7769 return true;
7770
7771 default:
7772 return false;
7773 }
7774}
7775
7776/// Find instructions that can be turned into madd.
7777static bool getMaddPatterns(MachineInstr &Root,
7778 SmallVectorImpl<unsigned> &Patterns) {
7779 unsigned Opc = Root.getOpcode();
7780 MachineBasicBlock &MBB = *Root.getParent();
7781 bool Found = false;
7782
7783 if (!isCombineInstrCandidate(Opc))
7784 return false;
7785 if (isCombineInstrSettingFlag(Opc)) {
7786 int Cmp_NZCV =
7787 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
7788 // When NZCV is live bail out.
7789 if (Cmp_NZCV == -1)
7790 return false;
7791 unsigned NewOpc = convertToNonFlagSettingOpc(MI: Root);
7792 // When opcode can't change bail out.
7793 // CHECKME: do we miss any cases for opcode conversion?
7794 if (NewOpc == Opc)
7795 return false;
7796 Opc = NewOpc;
7797 }
7798
7799 auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
7800 unsigned Pattern) {
7801 if (canCombineWithMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode, ZeroReg)) {
7802 Patterns.push_back(Elt: Pattern);
7803 Found = true;
7804 }
7805 };
7806
7807 auto setVFound = [&](int Opcode, int Operand, unsigned Pattern) {
7808 if (canCombine(MBB, MO&: Root.getOperand(i: Operand), CombineOpc: Opcode)) {
7809 Patterns.push_back(Elt: Pattern);
7810 Found = true;
7811 }
7812 };
7813
7814 typedef AArch64MachineCombinerPattern MCP;
7815
7816 switch (Opc) {
7817 default:
7818 break;
7819 case AArch64::ADDWrr:
7820 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
7821 "ADDWrr does not have register operands");
7822 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
7823 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
7824 break;
7825 case AArch64::ADDXrr:
7826 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
7827 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
7828 break;
7829 case AArch64::SUBWrr:
7830 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
7831 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
7832 break;
7833 case AArch64::SUBXrr:
7834 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
7835 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
7836 break;
7837 case AArch64::ADDWri:
7838 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
7839 break;
7840 case AArch64::ADDXri:
7841 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
7842 break;
7843 case AArch64::SUBWri:
7844 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
7845 break;
7846 case AArch64::SUBXri:
7847 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
7848 break;
7849 case AArch64::ADDv8i8:
7850 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
7851 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
7852 break;
7853 case AArch64::ADDv16i8:
7854 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
7855 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
7856 break;
7857 case AArch64::ADDv4i16:
7858 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
7859 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
7860 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
7861 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
7862 break;
7863 case AArch64::ADDv8i16:
7864 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
7865 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
7866 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
7867 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
7868 break;
7869 case AArch64::ADDv2i32:
7870 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
7871 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
7872 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
7873 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
7874 break;
7875 case AArch64::ADDv4i32:
7876 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
7877 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
7878 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
7879 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
7880 break;
7881 case AArch64::SUBv8i8:
7882 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
7883 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
7884 break;
7885 case AArch64::SUBv16i8:
7886 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
7887 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
7888 break;
7889 case AArch64::SUBv4i16:
7890 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
7891 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7892 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7893 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7894 break;
7895 case AArch64::SUBv8i16:
7896 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7897 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7898 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7899 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7900 break;
7901 case AArch64::SUBv2i32:
7902 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7903 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7904 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7905 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7906 break;
7907 case AArch64::SUBv4i32:
7908 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7909 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7910 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7911 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7912 break;
7913 }
7914 return Found;
7915}
7916
7917bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
7918 switch (Opcode) {
7919 default:
7920 break;
7921 case AArch64::UABALB_ZZZ_D:
7922 case AArch64::UABALB_ZZZ_H:
7923 case AArch64::UABALB_ZZZ_S:
7924 case AArch64::UABALT_ZZZ_D:
7925 case AArch64::UABALT_ZZZ_H:
7926 case AArch64::UABALT_ZZZ_S:
7927 case AArch64::SABALB_ZZZ_D:
7928 case AArch64::SABALB_ZZZ_S:
7929 case AArch64::SABALB_ZZZ_H:
7930 case AArch64::SABALT_ZZZ_D:
7931 case AArch64::SABALT_ZZZ_S:
7932 case AArch64::SABALT_ZZZ_H:
7933 case AArch64::UABALv16i8_v8i16:
7934 case AArch64::UABALv2i32_v2i64:
7935 case AArch64::UABALv4i16_v4i32:
7936 case AArch64::UABALv4i32_v2i64:
7937 case AArch64::UABALv8i16_v4i32:
7938 case AArch64::UABALv8i8_v8i16:
7939 case AArch64::UABAv16i8:
7940 case AArch64::UABAv2i32:
7941 case AArch64::UABAv4i16:
7942 case AArch64::UABAv4i32:
7943 case AArch64::UABAv8i16:
7944 case AArch64::UABAv8i8:
7945 case AArch64::SABALv16i8_v8i16:
7946 case AArch64::SABALv2i32_v2i64:
7947 case AArch64::SABALv4i16_v4i32:
7948 case AArch64::SABALv4i32_v2i64:
7949 case AArch64::SABALv8i16_v4i32:
7950 case AArch64::SABALv8i8_v8i16:
7951 case AArch64::SABAv16i8:
7952 case AArch64::SABAv2i32:
7953 case AArch64::SABAv4i16:
7954 case AArch64::SABAv4i32:
7955 case AArch64::SABAv8i16:
7956 case AArch64::SABAv8i8:
7957 return true;
7958 }
7959
7960 return false;
7961}
7962
7963unsigned AArch64InstrInfo::getAccumulationStartOpcode(
7964 unsigned AccumulationOpcode) const {
7965 switch (AccumulationOpcode) {
7966 default:
7967 llvm_unreachable("Unsupported accumulation Opcode!");
7968 case AArch64::UABALB_ZZZ_D:
7969 return AArch64::UABDLB_ZZZ_D;
7970 case AArch64::UABALB_ZZZ_H:
7971 return AArch64::UABDLB_ZZZ_H;
7972 case AArch64::UABALB_ZZZ_S:
7973 return AArch64::UABDLB_ZZZ_S;
7974 case AArch64::UABALT_ZZZ_D:
7975 return AArch64::UABDLT_ZZZ_D;
7976 case AArch64::UABALT_ZZZ_H:
7977 return AArch64::UABDLT_ZZZ_H;
7978 case AArch64::UABALT_ZZZ_S:
7979 return AArch64::UABDLT_ZZZ_S;
7980 case AArch64::UABALv16i8_v8i16:
7981 return AArch64::UABDLv16i8_v8i16;
7982 case AArch64::UABALv2i32_v2i64:
7983 return AArch64::UABDLv2i32_v2i64;
7984 case AArch64::UABALv4i16_v4i32:
7985 return AArch64::UABDLv4i16_v4i32;
7986 case AArch64::UABALv4i32_v2i64:
7987 return AArch64::UABDLv4i32_v2i64;
7988 case AArch64::UABALv8i16_v4i32:
7989 return AArch64::UABDLv8i16_v4i32;
7990 case AArch64::UABALv8i8_v8i16:
7991 return AArch64::UABDLv8i8_v8i16;
7992 case AArch64::UABAv16i8:
7993 return AArch64::UABDv16i8;
7994 case AArch64::UABAv2i32:
7995 return AArch64::UABDv2i32;
7996 case AArch64::UABAv4i16:
7997 return AArch64::UABDv4i16;
7998 case AArch64::UABAv4i32:
7999 return AArch64::UABDv4i32;
8000 case AArch64::UABAv8i16:
8001 return AArch64::UABDv8i16;
8002 case AArch64::UABAv8i8:
8003 return AArch64::UABDv8i8;
8004 case AArch64::SABALB_ZZZ_D:
8005 return AArch64::SABDLB_ZZZ_D;
8006 case AArch64::SABALB_ZZZ_S:
8007 return AArch64::SABDLB_ZZZ_S;
8008 case AArch64::SABALB_ZZZ_H:
8009 return AArch64::SABDLB_ZZZ_H;
8010 case AArch64::SABALT_ZZZ_D:
8011 return AArch64::SABDLT_ZZZ_D;
8012 case AArch64::SABALT_ZZZ_S:
8013 return AArch64::SABDLT_ZZZ_S;
8014 case AArch64::SABALT_ZZZ_H:
8015 return AArch64::SABDLT_ZZZ_H;
8016 case AArch64::SABALv16i8_v8i16:
8017 return AArch64::SABDLv16i8_v8i16;
8018 case AArch64::SABALv2i32_v2i64:
8019 return AArch64::SABDLv2i32_v2i64;
8020 case AArch64::SABALv4i16_v4i32:
8021 return AArch64::SABDLv4i16_v4i32;
8022 case AArch64::SABALv4i32_v2i64:
8023 return AArch64::SABDLv4i32_v2i64;
8024 case AArch64::SABALv8i16_v4i32:
8025 return AArch64::SABDLv8i16_v4i32;
8026 case AArch64::SABALv8i8_v8i16:
8027 return AArch64::SABDLv8i8_v8i16;
8028 case AArch64::SABAv16i8:
8029 return AArch64::SABDv16i8;
8030 case AArch64::SABAv2i32:
8031 return AArch64::SABAv2i32;
8032 case AArch64::SABAv4i16:
8033 return AArch64::SABDv4i16;
8034 case AArch64::SABAv4i32:
8035 return AArch64::SABDv4i32;
8036 case AArch64::SABAv8i16:
8037 return AArch64::SABDv8i16;
8038 case AArch64::SABAv8i8:
8039 return AArch64::SABDv8i8;
8040 }
8041}
8042
8043/// Floating-Point Support
8044
8045/// Find instructions that can be turned into madd.
8046static bool getFMAPatterns(MachineInstr &Root,
8047 SmallVectorImpl<unsigned> &Patterns) {
8048
8049 if (!isCombineInstrCandidateFP(Inst: Root))
8050 return false;
8051
8052 MachineBasicBlock &MBB = *Root.getParent();
8053 bool Found = false;
8054
8055 auto Match = [&](int Opcode, int Operand, unsigned Pattern) -> bool {
8056 if (canCombineWithFMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode)) {
8057 Patterns.push_back(Elt: Pattern);
8058 return true;
8059 }
8060 return false;
8061 };
8062
8063 typedef AArch64MachineCombinerPattern MCP;
8064
8065 switch (Root.getOpcode()) {
8066 default:
8067 assert(false && "Unsupported FP instruction in combiner\n");
8068 break;
8069 case AArch64::FADDHrr:
8070 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8071 "FADDHrr does not have register operands");
8072
8073 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
8074 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
8075 break;
8076 case AArch64::FADDSrr:
8077 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8078 "FADDSrr does not have register operands");
8079
8080 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
8081 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
8082
8083 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
8084 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
8085 break;
8086 case AArch64::FADDDrr:
8087 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
8088 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
8089
8090 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
8091 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
8092 break;
8093 case AArch64::FADDv4f16:
8094 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
8095 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
8096
8097 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
8098 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
8099 break;
8100 case AArch64::FADDv8f16:
8101 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
8102 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
8103
8104 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
8105 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
8106 break;
8107 case AArch64::FADDv2f32:
8108 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
8109 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
8110
8111 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
8112 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
8113 break;
8114 case AArch64::FADDv2f64:
8115 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
8116 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
8117
8118 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
8119 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
8120 break;
8121 case AArch64::FADDv4f32:
8122 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
8123 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
8124
8125 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
8126 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
8127 break;
8128 case AArch64::FSUBHrr:
8129 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
8130 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
8131 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
8132 break;
8133 case AArch64::FSUBSrr:
8134 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
8135
8136 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
8137 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
8138
8139 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
8140 break;
8141 case AArch64::FSUBDrr:
8142 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
8143
8144 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
8145 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
8146
8147 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
8148 break;
8149 case AArch64::FSUBv4f16:
8150 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
8151 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
8152
8153 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
8154 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
8155 break;
8156 case AArch64::FSUBv8f16:
8157 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
8158 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
8159
8160 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
8161 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
8162 break;
8163 case AArch64::FSUBv2f32:
8164 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
8165 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
8166
8167 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
8168 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
8169 break;
8170 case AArch64::FSUBv2f64:
8171 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
8172 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
8173
8174 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
8175 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
8176 break;
8177 case AArch64::FSUBv4f32:
8178 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
8179 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
8180
8181 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
8182 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
8183 break;
8184 }
8185 return Found;
8186}
8187
8188static bool getFMULPatterns(MachineInstr &Root,
8189 SmallVectorImpl<unsigned> &Patterns) {
8190 MachineBasicBlock &MBB = *Root.getParent();
8191 bool Found = false;
8192
8193 auto Match = [&](unsigned Opcode, int Operand, unsigned Pattern) -> bool {
8194 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8195 MachineOperand &MO = Root.getOperand(i: Operand);
8196 MachineInstr *MI = nullptr;
8197 if (MO.isReg() && MO.getReg().isVirtual())
8198 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8199 // Ignore No-op COPYs in FMUL(COPY(DUP(..)))
8200 if (MI && MI->getOpcode() == TargetOpcode::COPY &&
8201 MI->getOperand(i: 1).getReg().isVirtual())
8202 MI = MRI.getUniqueVRegDef(Reg: MI->getOperand(i: 1).getReg());
8203 if (MI && MI->getOpcode() == Opcode) {
8204 Patterns.push_back(Elt: Pattern);
8205 return true;
8206 }
8207 return false;
8208 };
8209
8210 typedef AArch64MachineCombinerPattern MCP;
8211
8212 switch (Root.getOpcode()) {
8213 default:
8214 return false;
8215 case AArch64::FMULv2f32:
8216 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
8217 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
8218 break;
8219 case AArch64::FMULv2f64:
8220 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
8221 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
8222 break;
8223 case AArch64::FMULv4f16:
8224 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
8225 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
8226 break;
8227 case AArch64::FMULv4f32:
8228 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
8229 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
8230 break;
8231 case AArch64::FMULv8f16:
8232 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
8233 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
8234 break;
8235 }
8236
8237 return Found;
8238}
8239
8240static bool getFNEGPatterns(MachineInstr &Root,
8241 SmallVectorImpl<unsigned> &Patterns) {
8242 unsigned Opc = Root.getOpcode();
8243 MachineBasicBlock &MBB = *Root.getParent();
8244 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8245
8246 auto Match = [&](unsigned Opcode, unsigned Pattern) -> bool {
8247 MachineOperand &MO = Root.getOperand(i: 1);
8248 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8249 if (MI != nullptr && (MI->getOpcode() == Opcode) &&
8250 MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()) &&
8251 Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8252 Root.getFlag(Flag: MachineInstr::MIFlag::FmNsz) &&
8253 MI->getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8254 MI->getFlag(Flag: MachineInstr::MIFlag::FmNsz)) {
8255 Patterns.push_back(Elt: Pattern);
8256 return true;
8257 }
8258 return false;
8259 };
8260
8261 switch (Opc) {
8262 default:
8263 break;
8264 case AArch64::FNEGDr:
8265 return Match(AArch64::FMADDDrrr, AArch64MachineCombinerPattern::FNMADD);
8266 case AArch64::FNEGSr:
8267 return Match(AArch64::FMADDSrrr, AArch64MachineCombinerPattern::FNMADD);
8268 }
8269
8270 return false;
8271}
8272
8273/// Return true when a code sequence can improve throughput. It
8274/// should be called only for instructions in loops.
8275/// \param Pattern - combiner pattern
8276bool AArch64InstrInfo::isThroughputPattern(unsigned Pattern) const {
8277 switch (Pattern) {
8278 default:
8279 break;
8280 case AArch64MachineCombinerPattern::FMULADDH_OP1:
8281 case AArch64MachineCombinerPattern::FMULADDH_OP2:
8282 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
8283 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
8284 case AArch64MachineCombinerPattern::FMULADDS_OP1:
8285 case AArch64MachineCombinerPattern::FMULADDS_OP2:
8286 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
8287 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
8288 case AArch64MachineCombinerPattern::FMULADDD_OP1:
8289 case AArch64MachineCombinerPattern::FMULADDD_OP2:
8290 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
8291 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
8292 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
8293 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
8294 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
8295 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
8296 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
8297 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
8298 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
8299 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
8300 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
8301 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
8302 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
8303 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
8304 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
8305 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
8306 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
8307 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
8308 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
8309 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
8310 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
8311 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
8312 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
8313 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
8314 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
8315 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
8316 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
8317 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
8318 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
8319 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1:
8320 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
8321 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1:
8322 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
8323 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
8324 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
8325 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
8326 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
8327 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
8328 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
8329 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
8330 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
8331 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
8332 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
8333 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
8334 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
8335 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
8336 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2:
8337 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
8338 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2:
8339 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
8340 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2:
8341 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
8342 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2:
8343 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
8344 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2:
8345 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
8346 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
8347 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
8348 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
8349 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
8350 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
8351 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
8352 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
8353 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
8354 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
8355 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
8356 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
8357 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
8358 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
8359 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
8360 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
8361 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
8362 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
8363 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
8364 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
8365 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
8366 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
8367 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
8368 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
8369 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
8370 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
8371 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
8372 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
8373 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
8374 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
8375 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
8376 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
8377 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
8378 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
8379 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
8380 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
8381 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
8382 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
8383 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
8384 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
8385 return true;
8386 } // end switch (Pattern)
8387 return false;
8388}
8389
8390/// Find other MI combine patterns.
8391static bool getMiscPatterns(MachineInstr &Root,
8392 SmallVectorImpl<unsigned> &Patterns) {
8393 // A - (B + C) ==> (A - B) - C or (A - C) - B
8394 unsigned Opc = Root.getOpcode();
8395 MachineBasicBlock &MBB = *Root.getParent();
8396
8397 switch (Opc) {
8398 case AArch64::SUBWrr:
8399 case AArch64::SUBSWrr:
8400 case AArch64::SUBXrr:
8401 case AArch64::SUBSXrr:
8402 // Found candidate root.
8403 break;
8404 default:
8405 return false;
8406 }
8407
8408 if (isCombineInstrSettingFlag(Opc) &&
8409 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) ==
8410 -1)
8411 return false;
8412
8413 if (canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDWrr) ||
8414 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSWrr) ||
8415 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDXrr) ||
8416 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSXrr)) {
8417 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP1);
8418 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP2);
8419 return true;
8420 }
8421
8422 return false;
8423}
8424
8425/// Check if the given instruction forms a gather load pattern that can be
8426/// optimized for better Memory-Level Parallelism (MLP). This function
8427/// identifies chains of NEON lane load instructions that load data from
8428/// different memory addresses into individual lanes of a 128-bit vector
8429/// register, then attempts to split the pattern into parallel loads to break
8430/// the serial dependency between instructions.
8431///
8432/// Pattern Matched:
8433/// Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) ->
8434/// LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)
8435///
8436/// Transformed Into:
8437/// Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64
8438/// to combine the results, enabling better memory-level parallelism.
8439///
8440/// Supported Element Types:
8441/// - 32-bit elements (LD1i32, 4 lanes total)
8442/// - 16-bit elements (LD1i16, 8 lanes total)
8443/// - 8-bit elements (LD1i8, 16 lanes total)
8444static bool getGatherLanePattern(MachineInstr &Root,
8445 SmallVectorImpl<unsigned> &Patterns,
8446 unsigned LoadLaneOpCode, unsigned NumLanes) {
8447 const MachineFunction *MF = Root.getMF();
8448
8449 // Early exit if optimizing for size.
8450 if (MF->getFunction().hasMinSize())
8451 return false;
8452
8453 const MachineRegisterInfo &MRI = MF->getRegInfo();
8454 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
8455
8456 // The root of the pattern must load into the last lane of the vector.
8457 if (Root.getOperand(i: 2).getImm() != NumLanes - 1)
8458 return false;
8459
8460 // Check that we have load into all lanes except lane 0.
8461 // For each load we also want to check that:
8462 // 1. It has a single non-debug use (since we will be replacing the virtual
8463 // register)
8464 // 2. That the addressing mode only uses a single pointer operand
8465 auto *CurrInstr = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8466 auto Range = llvm::seq<unsigned>(Begin: 1, End: NumLanes - 1);
8467 SmallSet<unsigned, 16> RemainingLanes(Range.begin(), Range.end());
8468 SmallVector<const MachineInstr *, 16> LoadInstrs;
8469 while (!RemainingLanes.empty() && CurrInstr &&
8470 CurrInstr->getOpcode() == LoadLaneOpCode &&
8471 MRI.hasOneNonDBGUse(RegNo: CurrInstr->getOperand(i: 0).getReg()) &&
8472 CurrInstr->getNumOperands() == 4) {
8473 RemainingLanes.erase(V: CurrInstr->getOperand(i: 2).getImm());
8474 LoadInstrs.push_back(Elt: CurrInstr);
8475 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8476 }
8477
8478 // Check that we have found a match for lanes N-1.. 1.
8479 if (!RemainingLanes.empty())
8480 return false;
8481
8482 // Match the SUBREG_TO_REG sequence.
8483 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
8484 return false;
8485
8486 // Verify that the subreg to reg loads an integer into the first lane.
8487 auto Lane0LoadReg = CurrInstr->getOperand(i: 1).getReg();
8488 unsigned SingleLaneSizeInBits = 128 / NumLanes;
8489 if (TRI->getRegSizeInBits(Reg: Lane0LoadReg, MRI) != SingleLaneSizeInBits)
8490 return false;
8491
8492 // Verify that it also has a single non debug use.
8493 if (!MRI.hasOneNonDBGUse(RegNo: Lane0LoadReg))
8494 return false;
8495
8496 LoadInstrs.push_back(Elt: MRI.getUniqueVRegDef(Reg: Lane0LoadReg));
8497
8498 // If there is any chance of aliasing, do not apply the pattern.
8499 // Walk backward through the MBB starting from Root.
8500 // Exit early if we've encountered all load instructions or hit the search
8501 // limit.
8502 auto MBBItr = Root.getIterator();
8503 unsigned RemainingSteps = GatherOptSearchLimit;
8504 SmallPtrSet<const MachineInstr *, 16> RemainingLoadInstrs;
8505 RemainingLoadInstrs.insert(I: LoadInstrs.begin(), E: LoadInstrs.end());
8506 const MachineBasicBlock *MBB = Root.getParent();
8507
8508 for (; MBBItr != MBB->begin() && RemainingSteps > 0 &&
8509 !RemainingLoadInstrs.empty();
8510 --MBBItr, --RemainingSteps) {
8511 const MachineInstr &CurrInstr = *MBBItr;
8512
8513 // Remove this instruction from remaining loads if it's one we're tracking.
8514 RemainingLoadInstrs.erase(Ptr: &CurrInstr);
8515
8516 // Check for potential aliasing with any of the load instructions to
8517 // optimize.
8518 if (CurrInstr.isLoadFoldBarrier())
8519 return false;
8520 }
8521
8522 // If we hit the search limit without finding all load instructions,
8523 // don't match the pattern.
8524 if (RemainingSteps == 0 && !RemainingLoadInstrs.empty())
8525 return false;
8526
8527 switch (NumLanes) {
8528 case 4:
8529 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i32);
8530 break;
8531 case 8:
8532 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i16);
8533 break;
8534 case 16:
8535 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i8);
8536 break;
8537 default:
8538 llvm_unreachable("Got bad number of lanes for gather pattern.");
8539 }
8540
8541 return true;
8542}
8543
8544/// Search for patterns of LD instructions we can optimize.
8545static bool getLoadPatterns(MachineInstr &Root,
8546 SmallVectorImpl<unsigned> &Patterns) {
8547
8548 // The pattern searches for loads into single lanes.
8549 switch (Root.getOpcode()) {
8550 case AArch64::LD1i32:
8551 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 4);
8552 case AArch64::LD1i16:
8553 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 8);
8554 case AArch64::LD1i8:
8555 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 16);
8556 default:
8557 return false;
8558 }
8559}
8560
8561/// Generate optimized instruction sequence for gather load patterns to improve
8562/// Memory-Level Parallelism (MLP). This function transforms a chain of
8563/// sequential NEON lane loads into parallel vector loads that can execute
8564/// concurrently.
8565static void
8566generateGatherLanePattern(MachineInstr &Root,
8567 SmallVectorImpl<MachineInstr *> &InsInstrs,
8568 SmallVectorImpl<MachineInstr *> &DelInstrs,
8569 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8570 unsigned Pattern, unsigned NumLanes) {
8571 MachineFunction &MF = *Root.getParent()->getParent();
8572 MachineRegisterInfo &MRI = MF.getRegInfo();
8573 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8574
8575 // Gather the initial load instructions to build the pattern.
8576 SmallVector<MachineInstr *, 16> LoadToLaneInstrs;
8577 MachineInstr *CurrInstr = &Root;
8578 for (unsigned i = 0; i < NumLanes - 1; ++i) {
8579 LoadToLaneInstrs.push_back(Elt: CurrInstr);
8580 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8581 }
8582
8583 // Sort the load instructions according to the lane.
8584 llvm::sort(C&: LoadToLaneInstrs,
8585 Comp: [](const MachineInstr *A, const MachineInstr *B) {
8586 return A->getOperand(i: 2).getImm() > B->getOperand(i: 2).getImm();
8587 });
8588
8589 MachineInstr *SubregToReg = CurrInstr;
8590 LoadToLaneInstrs.push_back(
8591 Elt: MRI.getUniqueVRegDef(Reg: SubregToReg->getOperand(i: 1).getReg()));
8592 auto LoadToLaneInstrsAscending = llvm::reverse(C&: LoadToLaneInstrs);
8593
8594 const TargetRegisterClass *FPR128RegClass =
8595 MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg());
8596
8597 // Helper lambda to create a LD1 instruction.
8598 auto CreateLD1Instruction = [&](MachineInstr *OriginalInstr,
8599 Register SrcRegister, unsigned Lane,
8600 Register OffsetRegister,
8601 bool OffsetRegisterKillState) {
8602 auto NewRegister = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8603 MachineInstrBuilder LoadIndexIntoRegister =
8604 BuildMI(MF, MIMD: MIMetadata(*OriginalInstr), MCID: TII->get(Opcode: Root.getOpcode()),
8605 DestReg: NewRegister)
8606 .addReg(RegNo: SrcRegister)
8607 .addImm(Val: Lane)
8608 .addReg(RegNo: OffsetRegister, Flags: getKillRegState(B: OffsetRegisterKillState))
8609 .setMemRefs(OriginalInstr->memoperands());
8610 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewRegister, y: InsInstrs.size()));
8611 InsInstrs.push_back(Elt: LoadIndexIntoRegister);
8612 return NewRegister;
8613 };
8614
8615 // Helper to create load instruction based on the NumLanes in the NEON
8616 // register we are rewriting.
8617 auto CreateLDRInstruction =
8618 [&](unsigned NumLanes, Register DestReg, Register OffsetReg,
8619 ArrayRef<MachineMemOperand *> MMOs) -> MachineInstrBuilder {
8620 unsigned Opcode;
8621 switch (NumLanes) {
8622 case 4:
8623 Opcode = AArch64::LDRSui;
8624 break;
8625 case 8:
8626 Opcode = AArch64::LDRHui;
8627 break;
8628 case 16:
8629 Opcode = AArch64::LDRBui;
8630 break;
8631 default:
8632 llvm_unreachable(
8633 "Got unsupported number of lanes in machine-combiner gather pattern");
8634 }
8635 // Immediate offset load
8636 return BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg)
8637 .addReg(RegNo: OffsetReg)
8638 .addImm(Val: 0)
8639 .setMemRefs(MMOs);
8640 };
8641
8642 // Load the remaining lanes into register 0.
8643 auto LanesToLoadToReg0 =
8644 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + 1,
8645 y: LoadToLaneInstrsAscending.begin() + NumLanes / 2);
8646 Register PrevReg = SubregToReg->getOperand(i: 0).getReg();
8647 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg0)) {
8648 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8649 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8650 OffsetRegOperand.getReg(),
8651 OffsetRegOperand.isKill());
8652 DelInstrs.push_back(Elt: LoadInstr);
8653 }
8654 Register LastLoadReg0 = PrevReg;
8655
8656 // First load into register 1. Perform an integer load to zero out the upper
8657 // lanes in a single instruction.
8658 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
8659 MachineInstr *OriginalSplitLoad =
8660 *std::next(x: LoadToLaneInstrsAscending.begin(), n: NumLanes / 2);
8661 Register DestRegForMiddleIndex = MRI.createVirtualRegister(
8662 RegClass: MRI.getRegClass(Reg: Lane0Load->getOperand(i: 0).getReg()));
8663
8664 const MachineOperand &OriginalSplitToLoadOffsetOperand =
8665 OriginalSplitLoad->getOperand(i: 3);
8666 MachineInstrBuilder MiddleIndexLoadInstr =
8667 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
8668 OriginalSplitToLoadOffsetOperand.getReg(),
8669 OriginalSplitLoad->memoperands());
8670
8671 InstrIdxForVirtReg.insert(
8672 KV: std::make_pair(x&: DestRegForMiddleIndex, y: InsInstrs.size()));
8673 InsInstrs.push_back(Elt: MiddleIndexLoadInstr);
8674 DelInstrs.push_back(Elt: OriginalSplitLoad);
8675
8676 // Subreg To Reg instruction for register 1.
8677 Register DestRegForSubregToReg = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8678 unsigned SubregType;
8679 switch (NumLanes) {
8680 case 4:
8681 SubregType = AArch64::ssub;
8682 break;
8683 case 8:
8684 SubregType = AArch64::hsub;
8685 break;
8686 case 16:
8687 SubregType = AArch64::bsub;
8688 break;
8689 default:
8690 llvm_unreachable(
8691 "Got invalid NumLanes for machine-combiner gather pattern");
8692 }
8693
8694 auto SubRegToRegInstr =
8695 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubregToReg->getOpcode()),
8696 DestReg: DestRegForSubregToReg)
8697 .addReg(RegNo: DestRegForMiddleIndex, Flags: getKillRegState(B: true))
8698 .addImm(Val: SubregType);
8699 InstrIdxForVirtReg.insert(
8700 KV: std::make_pair(x&: DestRegForSubregToReg, y: InsInstrs.size()));
8701 InsInstrs.push_back(Elt: SubRegToRegInstr);
8702
8703 // Load remaining lanes into register 1.
8704 auto LanesToLoadToReg1 =
8705 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + NumLanes / 2 + 1,
8706 y: LoadToLaneInstrsAscending.end());
8707 PrevReg = SubRegToRegInstr->getOperand(i: 0).getReg();
8708 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg1)) {
8709 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8710 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8711 OffsetRegOperand.getReg(),
8712 OffsetRegOperand.isKill());
8713
8714 // Do not add the last reg to DelInstrs - it will be removed later.
8715 if (Index == NumLanes / 2 - 2) {
8716 break;
8717 }
8718 DelInstrs.push_back(Elt: LoadInstr);
8719 }
8720 Register LastLoadReg1 = PrevReg;
8721
8722 // Create the final zip instruction to combine the results.
8723 MachineInstrBuilder ZipInstr =
8724 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::ZIP1v2i64),
8725 DestReg: Root.getOperand(i: 0).getReg())
8726 .addReg(RegNo: LastLoadReg0)
8727 .addReg(RegNo: LastLoadReg1);
8728 InsInstrs.push_back(Elt: ZipInstr);
8729}
8730
8731CombinerObjective
8732AArch64InstrInfo::getCombinerObjective(unsigned Pattern) const {
8733 switch (Pattern) {
8734 case AArch64MachineCombinerPattern::SUBADD_OP1:
8735 case AArch64MachineCombinerPattern::SUBADD_OP2:
8736 case AArch64MachineCombinerPattern::GATHER_LANE_i32:
8737 case AArch64MachineCombinerPattern::GATHER_LANE_i16:
8738 case AArch64MachineCombinerPattern::GATHER_LANE_i8:
8739 return CombinerObjective::MustReduceDepth;
8740 default:
8741 return TargetInstrInfo::getCombinerObjective(Pattern);
8742 }
8743}
8744
8745/// Return true when there is potentially a faster code sequence for an
8746/// instruction chain ending in \p Root. All potential patterns are listed in
8747/// the \p Pattern vector. Pattern should be sorted in priority order since the
8748/// pattern evaluator stops checking as soon as it finds a faster sequence.
8749
8750bool AArch64InstrInfo::getMachineCombinerPatterns(
8751 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
8752 bool DoRegPressureReduce) const {
8753 // Integer patterns
8754 if (getMaddPatterns(Root, Patterns))
8755 return true;
8756 // Floating point patterns
8757 if (getFMULPatterns(Root, Patterns))
8758 return true;
8759 if (getFMAPatterns(Root, Patterns))
8760 return true;
8761 if (getFNEGPatterns(Root, Patterns))
8762 return true;
8763
8764 // Other patterns
8765 if (getMiscPatterns(Root, Patterns))
8766 return true;
8767
8768 // Load patterns
8769 if (getLoadPatterns(Root, Patterns))
8770 return true;
8771
8772 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
8773 DoRegPressureReduce);
8774}
8775
8776enum class FMAInstKind { Default, Indexed, Accumulator };
8777/// genFusedMultiply - Generate fused multiply instructions.
8778/// This function supports both integer and floating point instructions.
8779/// A typical example:
8780/// F|MUL I=A,B,0
8781/// F|ADD R,I,C
8782/// ==> F|MADD R,A,B,C
8783/// \param MF Containing MachineFunction
8784/// \param MRI Register information
8785/// \param TII Target information
8786/// \param Root is the F|ADD instruction
8787/// \param [out] InsInstrs is a vector of machine instructions and will
8788/// contain the generated madd instruction
8789/// \param IdxMulOpd is index of operand in Root that is the result of
8790/// the F|MUL. In the example above IdxMulOpd is 1.
8791/// \param MaddOpc the opcode fo the f|madd instruction
8792/// \param RC Register class of operands
8793/// \param kind of fma instruction (addressing mode) to be generated
8794/// \param ReplacedAddend is the result register from the instruction
8795/// replacing the non-combined operand, if any.
8796static MachineInstr *
8797genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
8798 const TargetInstrInfo *TII, MachineInstr &Root,
8799 SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
8800 unsigned MaddOpc, const TargetRegisterClass *RC,
8801 FMAInstKind kind = FMAInstKind::Default,
8802 const Register *ReplacedAddend = nullptr) {
8803 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8804
8805 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
8806 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
8807 Register ResultReg = Root.getOperand(i: 0).getReg();
8808 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8809 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
8810 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8811 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
8812
8813 Register SrcReg2;
8814 bool Src2IsKill;
8815 if (ReplacedAddend) {
8816 // If we just generated a new addend, we must be it's only use.
8817 SrcReg2 = *ReplacedAddend;
8818 Src2IsKill = true;
8819 } else {
8820 SrcReg2 = Root.getOperand(i: IdxOtherOpd).getReg();
8821 Src2IsKill = Root.getOperand(i: IdxOtherOpd).isKill();
8822 }
8823
8824 if (ResultReg.isVirtual())
8825 MRI.constrainRegClass(Reg: ResultReg, RC);
8826 if (SrcReg0.isVirtual())
8827 MRI.constrainRegClass(Reg: SrcReg0, RC);
8828 if (SrcReg1.isVirtual())
8829 MRI.constrainRegClass(Reg: SrcReg1, RC);
8830 if (SrcReg2.isVirtual())
8831 MRI.constrainRegClass(Reg: SrcReg2, RC);
8832
8833 MachineInstrBuilder MIB;
8834 if (kind == FMAInstKind::Default)
8835 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8836 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8837 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8838 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8839 else if (kind == FMAInstKind::Indexed)
8840 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8841 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8842 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8843 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8844 .addImm(Val: MUL->getOperand(i: 3).getImm());
8845 else if (kind == FMAInstKind::Accumulator)
8846 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8847 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8848 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8849 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill));
8850 else
8851 assert(false && "Invalid FMA instruction kind \n");
8852 // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
8853 InsInstrs.push_back(Elt: MIB);
8854 return MUL;
8855}
8856
8857static MachineInstr *
8858genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI,
8859 const TargetInstrInfo *TII, MachineInstr &Root,
8860 SmallVectorImpl<MachineInstr *> &InsInstrs) {
8861 MachineInstr *MAD = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8862
8863 unsigned Opc = 0;
8864 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MAD->getOperand(i: 0).getReg());
8865 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
8866 Opc = AArch64::FNMADDSrrr;
8867 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
8868 Opc = AArch64::FNMADDDrrr;
8869 else
8870 return nullptr;
8871
8872 Register ResultReg = Root.getOperand(i: 0).getReg();
8873 Register SrcReg0 = MAD->getOperand(i: 1).getReg();
8874 Register SrcReg1 = MAD->getOperand(i: 2).getReg();
8875 Register SrcReg2 = MAD->getOperand(i: 3).getReg();
8876 bool Src0IsKill = MAD->getOperand(i: 1).isKill();
8877 bool Src1IsKill = MAD->getOperand(i: 2).isKill();
8878 bool Src2IsKill = MAD->getOperand(i: 3).isKill();
8879 if (ResultReg.isVirtual())
8880 MRI.constrainRegClass(Reg: ResultReg, RC);
8881 if (SrcReg0.isVirtual())
8882 MRI.constrainRegClass(Reg: SrcReg0, RC);
8883 if (SrcReg1.isVirtual())
8884 MRI.constrainRegClass(Reg: SrcReg1, RC);
8885 if (SrcReg2.isVirtual())
8886 MRI.constrainRegClass(Reg: SrcReg2, RC);
8887
8888 MachineInstrBuilder MIB =
8889 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Opc), DestReg: ResultReg)
8890 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8891 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8892 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8893 InsInstrs.push_back(Elt: MIB);
8894
8895 return MAD;
8896}
8897
8898/// Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
8899static MachineInstr *
8900genIndexedMultiply(MachineInstr &Root,
8901 SmallVectorImpl<MachineInstr *> &InsInstrs,
8902 unsigned IdxDupOp, unsigned MulOpc,
8903 const TargetRegisterClass *RC, MachineRegisterInfo &MRI) {
8904 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
8905 "Invalid index of FMUL operand");
8906
8907 MachineFunction &MF = *Root.getMF();
8908 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8909
8910 MachineInstr *Dup =
8911 MF.getRegInfo().getUniqueVRegDef(Reg: Root.getOperand(i: IdxDupOp).getReg());
8912
8913 if (Dup->getOpcode() == TargetOpcode::COPY)
8914 Dup = MRI.getUniqueVRegDef(Reg: Dup->getOperand(i: 1).getReg());
8915
8916 Register DupSrcReg = Dup->getOperand(i: 1).getReg();
8917 MRI.clearKillFlags(Reg: DupSrcReg);
8918 MRI.constrainRegClass(Reg: DupSrcReg, RC);
8919
8920 unsigned DupSrcLane = Dup->getOperand(i: 2).getImm();
8921
8922 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
8923 MachineOperand &MulOp = Root.getOperand(i: IdxMulOp);
8924
8925 Register ResultReg = Root.getOperand(i: 0).getReg();
8926
8927 MachineInstrBuilder MIB;
8928 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MulOpc), DestReg: ResultReg)
8929 .add(MO: MulOp)
8930 .addReg(RegNo: DupSrcReg)
8931 .addImm(Val: DupSrcLane);
8932
8933 InsInstrs.push_back(Elt: MIB);
8934 return &Root;
8935}
8936
8937/// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
8938/// instructions.
8939///
8940/// \see genFusedMultiply
8941static MachineInstr *genFusedMultiplyAcc(
8942 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8943 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8944 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8945 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8946 kind: FMAInstKind::Accumulator);
8947}
8948
8949/// genNeg - Helper to generate an intermediate negation of the second operand
8950/// of Root
8951static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
8952 const TargetInstrInfo *TII, MachineInstr &Root,
8953 SmallVectorImpl<MachineInstr *> &InsInstrs,
8954 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8955 unsigned MnegOpc, const TargetRegisterClass *RC) {
8956 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
8957 MachineInstrBuilder MIB =
8958 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MnegOpc), DestReg: NewVR)
8959 .add(MO: Root.getOperand(i: 2));
8960 InsInstrs.push_back(Elt: MIB);
8961
8962 assert(InstrIdxForVirtReg.empty());
8963 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
8964
8965 return NewVR;
8966}
8967
8968/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8969/// instructions with an additional negation of the accumulator
8970static MachineInstr *genFusedMultiplyAccNeg(
8971 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8972 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8973 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8974 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8975 assert(IdxMulOpd == 1);
8976
8977 Register NewVR =
8978 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8979 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8980 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
8981}
8982
8983/// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
8984/// instructions.
8985///
8986/// \see genFusedMultiply
8987static MachineInstr *genFusedMultiplyIdx(
8988 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8989 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8990 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8991 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8992 kind: FMAInstKind::Indexed);
8993}
8994
8995/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8996/// instructions with an additional negation of the accumulator
8997static MachineInstr *genFusedMultiplyIdxNeg(
8998 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8999 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
9000 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
9001 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
9002 assert(IdxMulOpd == 1);
9003
9004 Register NewVR =
9005 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
9006
9007 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
9008 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9009}
9010
9011/// genMaddR - Generate madd instruction and combine mul and add using
9012/// an extra virtual register
9013/// Example - an ADD intermediate needs to be stored in a register:
9014/// MUL I=A,B,0
9015/// ADD R,I,Imm
9016/// ==> ORR V, ZR, Imm
9017/// ==> MADD R,A,B,V
9018/// \param MF Containing MachineFunction
9019/// \param MRI Register information
9020/// \param TII Target information
9021/// \param Root is the ADD instruction
9022/// \param [out] InsInstrs is a vector of machine instructions and will
9023/// contain the generated madd instruction
9024/// \param IdxMulOpd is index of operand in Root that is the result of
9025/// the MUL. In the example above IdxMulOpd is 1.
9026/// \param MaddOpc the opcode fo the madd instruction
9027/// \param VR is a virtual register that holds the value of an ADD operand
9028/// (V in the example above).
9029/// \param RC Register class of operands
9030static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
9031 const TargetInstrInfo *TII, MachineInstr &Root,
9032 SmallVectorImpl<MachineInstr *> &InsInstrs,
9033 unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
9034 const TargetRegisterClass *RC) {
9035 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
9036
9037 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
9038 Register ResultReg = Root.getOperand(i: 0).getReg();
9039 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
9040 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
9041 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
9042 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
9043
9044 if (ResultReg.isVirtual())
9045 MRI.constrainRegClass(Reg: ResultReg, RC);
9046 if (SrcReg0.isVirtual())
9047 MRI.constrainRegClass(Reg: SrcReg0, RC);
9048 if (SrcReg1.isVirtual())
9049 MRI.constrainRegClass(Reg: SrcReg1, RC);
9050 if (Register::isVirtualRegister(Reg: VR))
9051 MRI.constrainRegClass(Reg: VR, RC);
9052
9053 MachineInstrBuilder MIB =
9054 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
9055 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
9056 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
9057 .addReg(RegNo: VR);
9058 // Insert the MADD
9059 InsInstrs.push_back(Elt: MIB);
9060 return MUL;
9061}
9062
9063/// Do the following transformation
9064/// A - (B + C) ==> (A - B) - C
9065/// A - (B + C) ==> (A - C) - B
9066static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI,
9067 const TargetInstrInfo *TII, MachineInstr &Root,
9068 SmallVectorImpl<MachineInstr *> &InsInstrs,
9069 SmallVectorImpl<MachineInstr *> &DelInstrs,
9070 unsigned IdxOpd1,
9071 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
9072 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
9073 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
9074 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
9075
9076 Register ResultReg = Root.getOperand(i: 0).getReg();
9077 Register RegA = Root.getOperand(i: 1).getReg();
9078 bool RegAIsKill = Root.getOperand(i: 1).isKill();
9079 Register RegB = AddMI->getOperand(i: IdxOpd1).getReg();
9080 bool RegBIsKill = AddMI->getOperand(i: IdxOpd1).isKill();
9081 Register RegC = AddMI->getOperand(i: IdxOtherOpd).getReg();
9082 bool RegCIsKill = AddMI->getOperand(i: IdxOtherOpd).isKill();
9083 Register NewVR =
9084 MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 2).getReg()));
9085
9086 unsigned Opcode = Root.getOpcode();
9087 if (Opcode == AArch64::SUBSWrr)
9088 Opcode = AArch64::SUBWrr;
9089 else if (Opcode == AArch64::SUBSXrr)
9090 Opcode = AArch64::SUBXrr;
9091 else
9092 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
9093 "Unexpected instruction opcode.");
9094
9095 uint32_t Flags = Root.mergeFlagsWith(Other: *AddMI);
9096 Flags &= ~MachineInstr::NoSWrap;
9097 Flags &= ~MachineInstr::NoUWrap;
9098
9099 MachineInstrBuilder MIB1 =
9100 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: NewVR)
9101 .addReg(RegNo: RegA, Flags: getKillRegState(B: RegAIsKill))
9102 .addReg(RegNo: RegB, Flags: getKillRegState(B: RegBIsKill))
9103 .setMIFlags(Flags);
9104 MachineInstrBuilder MIB2 =
9105 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: ResultReg)
9106 .addReg(RegNo: NewVR, Flags: getKillRegState(B: true))
9107 .addReg(RegNo: RegC, Flags: getKillRegState(B: RegCIsKill))
9108 .setMIFlags(Flags);
9109
9110 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9111 InsInstrs.push_back(Elt: MIB1);
9112 InsInstrs.push_back(Elt: MIB2);
9113 DelInstrs.push_back(Elt: AddMI);
9114 DelInstrs.push_back(Elt: &Root);
9115}
9116
9117unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
9118 unsigned int AccumulatorOpCode) const {
9119 switch (AccumulatorOpCode) {
9120 case AArch64::UABALB_ZZZ_D:
9121 case AArch64::SABALB_ZZZ_D:
9122 case AArch64::UABALT_ZZZ_D:
9123 case AArch64::SABALT_ZZZ_D:
9124 return AArch64::ADD_ZZZ_D;
9125 case AArch64::UABALB_ZZZ_H:
9126 case AArch64::SABALB_ZZZ_H:
9127 case AArch64::UABALT_ZZZ_H:
9128 case AArch64::SABALT_ZZZ_H:
9129 return AArch64::ADD_ZZZ_H;
9130 case AArch64::UABALB_ZZZ_S:
9131 case AArch64::SABALB_ZZZ_S:
9132 case AArch64::UABALT_ZZZ_S:
9133 case AArch64::SABALT_ZZZ_S:
9134 return AArch64::ADD_ZZZ_S;
9135 case AArch64::UABALv16i8_v8i16:
9136 case AArch64::SABALv8i8_v8i16:
9137 case AArch64::SABAv8i16:
9138 case AArch64::UABAv8i16:
9139 return AArch64::ADDv8i16;
9140 case AArch64::SABALv2i32_v2i64:
9141 case AArch64::UABALv2i32_v2i64:
9142 case AArch64::SABALv4i32_v2i64:
9143 return AArch64::ADDv2i64;
9144 case AArch64::UABALv4i16_v4i32:
9145 case AArch64::SABALv4i16_v4i32:
9146 case AArch64::SABALv8i16_v4i32:
9147 case AArch64::SABAv4i32:
9148 case AArch64::UABAv4i32:
9149 return AArch64::ADDv4i32;
9150 case AArch64::UABALv4i32_v2i64:
9151 return AArch64::ADDv2i64;
9152 case AArch64::UABALv8i16_v4i32:
9153 return AArch64::ADDv4i32;
9154 case AArch64::UABALv8i8_v8i16:
9155 case AArch64::SABALv16i8_v8i16:
9156 return AArch64::ADDv8i16;
9157 case AArch64::UABAv16i8:
9158 case AArch64::SABAv16i8:
9159 return AArch64::ADDv16i8;
9160 case AArch64::UABAv4i16:
9161 case AArch64::SABAv4i16:
9162 return AArch64::ADDv4i16;
9163 case AArch64::UABAv2i32:
9164 case AArch64::SABAv2i32:
9165 return AArch64::ADDv2i32;
9166 case AArch64::UABAv8i8:
9167 case AArch64::SABAv8i8:
9168 return AArch64::ADDv8i8;
9169 default:
9170 llvm_unreachable("Unknown accumulator opcode");
9171 }
9172}
9173
9174/// When getMachineCombinerPatterns() finds potential patterns,
9175/// this function generates the instructions that could replace the
9176/// original code sequence
9177void AArch64InstrInfo::genAlternativeCodeSequence(
9178 MachineInstr &Root, unsigned Pattern,
9179 SmallVectorImpl<MachineInstr *> &InsInstrs,
9180 SmallVectorImpl<MachineInstr *> &DelInstrs,
9181 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
9182 MachineBasicBlock &MBB = *Root.getParent();
9183 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9184 MachineFunction &MF = *MBB.getParent();
9185 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
9186
9187 MachineInstr *MUL = nullptr;
9188 const TargetRegisterClass *RC;
9189 unsigned Opc;
9190 switch (Pattern) {
9191 default:
9192 // Reassociate instructions.
9193 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
9194 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
9195 return;
9196 case AArch64MachineCombinerPattern::SUBADD_OP1:
9197 // A - (B + C)
9198 // ==> (A - B) - C
9199 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 1,
9200 InstrIdxForVirtReg);
9201 return;
9202 case AArch64MachineCombinerPattern::SUBADD_OP2:
9203 // A - (B + C)
9204 // ==> (A - C) - B
9205 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 2,
9206 InstrIdxForVirtReg);
9207 return;
9208 case AArch64MachineCombinerPattern::MULADDW_OP1:
9209 case AArch64MachineCombinerPattern::MULADDX_OP1:
9210 // MUL I=A,B,0
9211 // ADD R,I,C
9212 // ==> MADD R,A,B,C
9213 // --- Create(MADD);
9214 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP1) {
9215 Opc = AArch64::MADDWrrr;
9216 RC = &AArch64::GPR32RegClass;
9217 } else {
9218 Opc = AArch64::MADDXrrr;
9219 RC = &AArch64::GPR64RegClass;
9220 }
9221 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9222 break;
9223 case AArch64MachineCombinerPattern::MULADDW_OP2:
9224 case AArch64MachineCombinerPattern::MULADDX_OP2:
9225 // MUL I=A,B,0
9226 // ADD R,C,I
9227 // ==> MADD R,A,B,C
9228 // --- Create(MADD);
9229 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP2) {
9230 Opc = AArch64::MADDWrrr;
9231 RC = &AArch64::GPR32RegClass;
9232 } else {
9233 Opc = AArch64::MADDXrrr;
9234 RC = &AArch64::GPR64RegClass;
9235 }
9236 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9237 break;
9238 case AArch64MachineCombinerPattern::MULADDWI_OP1:
9239 case AArch64MachineCombinerPattern::MULADDXI_OP1:
9240 case AArch64MachineCombinerPattern::MULSUBWI_OP1:
9241 case AArch64MachineCombinerPattern::MULSUBXI_OP1: {
9242 // MUL I=A,B,0
9243 // ADD/SUB R,I,Imm
9244 // ==> MOV V, Imm/-Imm
9245 // ==> MADD R,A,B,V
9246 // --- Create(MADD);
9247 const TargetRegisterClass *RC;
9248 unsigned BitSize, MovImm;
9249 if (Pattern == AArch64MachineCombinerPattern::MULADDWI_OP1 ||
9250 Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1) {
9251 MovImm = AArch64::MOVi32imm;
9252 RC = &AArch64::GPR32spRegClass;
9253 BitSize = 32;
9254 Opc = AArch64::MADDWrrr;
9255 RC = &AArch64::GPR32RegClass;
9256 } else {
9257 MovImm = AArch64::MOVi64imm;
9258 RC = &AArch64::GPR64spRegClass;
9259 BitSize = 64;
9260 Opc = AArch64::MADDXrrr;
9261 RC = &AArch64::GPR64RegClass;
9262 }
9263 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9264 uint64_t Imm = Root.getOperand(i: 2).getImm();
9265
9266 if (Root.getOperand(i: 3).isImm()) {
9267 unsigned Val = Root.getOperand(i: 3).getImm();
9268 Imm = Imm << Val;
9269 }
9270 bool IsSub = Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1 ||
9271 Pattern == AArch64MachineCombinerPattern::MULSUBXI_OP1;
9272 uint64_t UImm = SignExtend64(X: IsSub ? -Imm : Imm, B: BitSize);
9273 // Check that the immediate can be composed via a single instruction.
9274 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
9275 AArch64_IMM::expandMOVImm(Imm: UImm, BitSize, Insn);
9276 if (Insn.size() != 1)
9277 return;
9278 MachineInstrBuilder MIB1 =
9279 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MovImm), DestReg: NewVR)
9280 .addImm(Val: IsSub ? -Imm : Imm);
9281 InsInstrs.push_back(Elt: MIB1);
9282 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9283 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9284 break;
9285 }
9286 case AArch64MachineCombinerPattern::MULSUBW_OP1:
9287 case AArch64MachineCombinerPattern::MULSUBX_OP1: {
9288 // MUL I=A,B,0
9289 // SUB R,I, C
9290 // ==> SUB V, 0, C
9291 // ==> MADD R,A,B,V // = -C + A*B
9292 // --- Create(MADD);
9293 const TargetRegisterClass *SubRC;
9294 unsigned SubOpc, ZeroReg;
9295 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP1) {
9296 SubOpc = AArch64::SUBWrr;
9297 SubRC = &AArch64::GPR32spRegClass;
9298 ZeroReg = AArch64::WZR;
9299 Opc = AArch64::MADDWrrr;
9300 RC = &AArch64::GPR32RegClass;
9301 } else {
9302 SubOpc = AArch64::SUBXrr;
9303 SubRC = &AArch64::GPR64spRegClass;
9304 ZeroReg = AArch64::XZR;
9305 Opc = AArch64::MADDXrrr;
9306 RC = &AArch64::GPR64RegClass;
9307 }
9308 Register NewVR = MRI.createVirtualRegister(RegClass: SubRC);
9309 // SUB NewVR, 0, C
9310 MachineInstrBuilder MIB1 =
9311 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubOpc), DestReg: NewVR)
9312 .addReg(RegNo: ZeroReg)
9313 .add(MO: Root.getOperand(i: 2));
9314 InsInstrs.push_back(Elt: MIB1);
9315 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9316 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9317 break;
9318 }
9319 case AArch64MachineCombinerPattern::MULSUBW_OP2:
9320 case AArch64MachineCombinerPattern::MULSUBX_OP2:
9321 // MUL I=A,B,0
9322 // SUB R,C,I
9323 // ==> MSUB R,A,B,C (computes C - A*B)
9324 // --- Create(MSUB);
9325 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP2) {
9326 Opc = AArch64::MSUBWrrr;
9327 RC = &AArch64::GPR32RegClass;
9328 } else {
9329 Opc = AArch64::MSUBXrrr;
9330 RC = &AArch64::GPR64RegClass;
9331 }
9332 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9333 break;
9334 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
9335 Opc = AArch64::MLAv8i8;
9336 RC = &AArch64::FPR64RegClass;
9337 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9338 break;
9339 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
9340 Opc = AArch64::MLAv8i8;
9341 RC = &AArch64::FPR64RegClass;
9342 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9343 break;
9344 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
9345 Opc = AArch64::MLAv16i8;
9346 RC = &AArch64::FPR128RegClass;
9347 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9348 break;
9349 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
9350 Opc = AArch64::MLAv16i8;
9351 RC = &AArch64::FPR128RegClass;
9352 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9353 break;
9354 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
9355 Opc = AArch64::MLAv4i16;
9356 RC = &AArch64::FPR64RegClass;
9357 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9358 break;
9359 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
9360 Opc = AArch64::MLAv4i16;
9361 RC = &AArch64::FPR64RegClass;
9362 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9363 break;
9364 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
9365 Opc = AArch64::MLAv8i16;
9366 RC = &AArch64::FPR128RegClass;
9367 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9368 break;
9369 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
9370 Opc = AArch64::MLAv8i16;
9371 RC = &AArch64::FPR128RegClass;
9372 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9373 break;
9374 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
9375 Opc = AArch64::MLAv2i32;
9376 RC = &AArch64::FPR64RegClass;
9377 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9378 break;
9379 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
9380 Opc = AArch64::MLAv2i32;
9381 RC = &AArch64::FPR64RegClass;
9382 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9383 break;
9384 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
9385 Opc = AArch64::MLAv4i32;
9386 RC = &AArch64::FPR128RegClass;
9387 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9388 break;
9389 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
9390 Opc = AArch64::MLAv4i32;
9391 RC = &AArch64::FPR128RegClass;
9392 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9393 break;
9394
9395 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
9396 Opc = AArch64::MLAv8i8;
9397 RC = &AArch64::FPR64RegClass;
9398 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9399 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i8,
9400 RC);
9401 break;
9402 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
9403 Opc = AArch64::MLSv8i8;
9404 RC = &AArch64::FPR64RegClass;
9405 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9406 break;
9407 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
9408 Opc = AArch64::MLAv16i8;
9409 RC = &AArch64::FPR128RegClass;
9410 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9411 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv16i8,
9412 RC);
9413 break;
9414 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
9415 Opc = AArch64::MLSv16i8;
9416 RC = &AArch64::FPR128RegClass;
9417 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9418 break;
9419 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
9420 Opc = AArch64::MLAv4i16;
9421 RC = &AArch64::FPR64RegClass;
9422 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9423 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9424 RC);
9425 break;
9426 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
9427 Opc = AArch64::MLSv4i16;
9428 RC = &AArch64::FPR64RegClass;
9429 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9430 break;
9431 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
9432 Opc = AArch64::MLAv8i16;
9433 RC = &AArch64::FPR128RegClass;
9434 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9435 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9436 RC);
9437 break;
9438 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
9439 Opc = AArch64::MLSv8i16;
9440 RC = &AArch64::FPR128RegClass;
9441 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9442 break;
9443 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
9444 Opc = AArch64::MLAv2i32;
9445 RC = &AArch64::FPR64RegClass;
9446 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9447 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9448 RC);
9449 break;
9450 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
9451 Opc = AArch64::MLSv2i32;
9452 RC = &AArch64::FPR64RegClass;
9453 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9454 break;
9455 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
9456 Opc = AArch64::MLAv4i32;
9457 RC = &AArch64::FPR128RegClass;
9458 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9459 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9460 RC);
9461 break;
9462 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
9463 Opc = AArch64::MLSv4i32;
9464 RC = &AArch64::FPR128RegClass;
9465 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9466 break;
9467
9468 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
9469 Opc = AArch64::MLAv4i16_indexed;
9470 RC = &AArch64::FPR64RegClass;
9471 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9472 break;
9473 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
9474 Opc = AArch64::MLAv4i16_indexed;
9475 RC = &AArch64::FPR64RegClass;
9476 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9477 break;
9478 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
9479 Opc = AArch64::MLAv8i16_indexed;
9480 RC = &AArch64::FPR128RegClass;
9481 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9482 break;
9483 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
9484 Opc = AArch64::MLAv8i16_indexed;
9485 RC = &AArch64::FPR128RegClass;
9486 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9487 break;
9488 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
9489 Opc = AArch64::MLAv2i32_indexed;
9490 RC = &AArch64::FPR64RegClass;
9491 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9492 break;
9493 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
9494 Opc = AArch64::MLAv2i32_indexed;
9495 RC = &AArch64::FPR64RegClass;
9496 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9497 break;
9498 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
9499 Opc = AArch64::MLAv4i32_indexed;
9500 RC = &AArch64::FPR128RegClass;
9501 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9502 break;
9503 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
9504 Opc = AArch64::MLAv4i32_indexed;
9505 RC = &AArch64::FPR128RegClass;
9506 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9507 break;
9508
9509 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
9510 Opc = AArch64::MLAv4i16_indexed;
9511 RC = &AArch64::FPR64RegClass;
9512 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9513 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9514 RC);
9515 break;
9516 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
9517 Opc = AArch64::MLSv4i16_indexed;
9518 RC = &AArch64::FPR64RegClass;
9519 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9520 break;
9521 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
9522 Opc = AArch64::MLAv8i16_indexed;
9523 RC = &AArch64::FPR128RegClass;
9524 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9525 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9526 RC);
9527 break;
9528 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
9529 Opc = AArch64::MLSv8i16_indexed;
9530 RC = &AArch64::FPR128RegClass;
9531 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9532 break;
9533 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
9534 Opc = AArch64::MLAv2i32_indexed;
9535 RC = &AArch64::FPR64RegClass;
9536 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9537 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9538 RC);
9539 break;
9540 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
9541 Opc = AArch64::MLSv2i32_indexed;
9542 RC = &AArch64::FPR64RegClass;
9543 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9544 break;
9545 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
9546 Opc = AArch64::MLAv4i32_indexed;
9547 RC = &AArch64::FPR128RegClass;
9548 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9549 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9550 RC);
9551 break;
9552 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
9553 Opc = AArch64::MLSv4i32_indexed;
9554 RC = &AArch64::FPR128RegClass;
9555 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9556 break;
9557
9558 // Floating Point Support
9559 case AArch64MachineCombinerPattern::FMULADDH_OP1:
9560 Opc = AArch64::FMADDHrrr;
9561 RC = &AArch64::FPR16RegClass;
9562 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9563 break;
9564 case AArch64MachineCombinerPattern::FMULADDS_OP1:
9565 Opc = AArch64::FMADDSrrr;
9566 RC = &AArch64::FPR32RegClass;
9567 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9568 break;
9569 case AArch64MachineCombinerPattern::FMULADDD_OP1:
9570 Opc = AArch64::FMADDDrrr;
9571 RC = &AArch64::FPR64RegClass;
9572 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9573 break;
9574
9575 case AArch64MachineCombinerPattern::FMULADDH_OP2:
9576 Opc = AArch64::FMADDHrrr;
9577 RC = &AArch64::FPR16RegClass;
9578 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9579 break;
9580 case AArch64MachineCombinerPattern::FMULADDS_OP2:
9581 Opc = AArch64::FMADDSrrr;
9582 RC = &AArch64::FPR32RegClass;
9583 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9584 break;
9585 case AArch64MachineCombinerPattern::FMULADDD_OP2:
9586 Opc = AArch64::FMADDDrrr;
9587 RC = &AArch64::FPR64RegClass;
9588 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9589 break;
9590
9591 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
9592 Opc = AArch64::FMLAv1i32_indexed;
9593 RC = &AArch64::FPR32RegClass;
9594 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9595 kind: FMAInstKind::Indexed);
9596 break;
9597 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
9598 Opc = AArch64::FMLAv1i32_indexed;
9599 RC = &AArch64::FPR32RegClass;
9600 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9601 kind: FMAInstKind::Indexed);
9602 break;
9603
9604 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
9605 Opc = AArch64::FMLAv1i64_indexed;
9606 RC = &AArch64::FPR64RegClass;
9607 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9608 kind: FMAInstKind::Indexed);
9609 break;
9610 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
9611 Opc = AArch64::FMLAv1i64_indexed;
9612 RC = &AArch64::FPR64RegClass;
9613 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9614 kind: FMAInstKind::Indexed);
9615 break;
9616
9617 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
9618 RC = &AArch64::FPR64RegClass;
9619 Opc = AArch64::FMLAv4i16_indexed;
9620 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9621 kind: FMAInstKind::Indexed);
9622 break;
9623 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
9624 RC = &AArch64::FPR64RegClass;
9625 Opc = AArch64::FMLAv4f16;
9626 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9627 kind: FMAInstKind::Accumulator);
9628 break;
9629 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
9630 RC = &AArch64::FPR64RegClass;
9631 Opc = AArch64::FMLAv4i16_indexed;
9632 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9633 kind: FMAInstKind::Indexed);
9634 break;
9635 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
9636 RC = &AArch64::FPR64RegClass;
9637 Opc = AArch64::FMLAv4f16;
9638 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9639 kind: FMAInstKind::Accumulator);
9640 break;
9641
9642 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
9643 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
9644 RC = &AArch64::FPR64RegClass;
9645 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
9646 Opc = AArch64::FMLAv2i32_indexed;
9647 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9648 kind: FMAInstKind::Indexed);
9649 } else {
9650 Opc = AArch64::FMLAv2f32;
9651 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9652 kind: FMAInstKind::Accumulator);
9653 }
9654 break;
9655 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
9656 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
9657 RC = &AArch64::FPR64RegClass;
9658 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
9659 Opc = AArch64::FMLAv2i32_indexed;
9660 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9661 kind: FMAInstKind::Indexed);
9662 } else {
9663 Opc = AArch64::FMLAv2f32;
9664 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9665 kind: FMAInstKind::Accumulator);
9666 }
9667 break;
9668
9669 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
9670 RC = &AArch64::FPR128RegClass;
9671 Opc = AArch64::FMLAv8i16_indexed;
9672 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9673 kind: FMAInstKind::Indexed);
9674 break;
9675 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
9676 RC = &AArch64::FPR128RegClass;
9677 Opc = AArch64::FMLAv8f16;
9678 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9679 kind: FMAInstKind::Accumulator);
9680 break;
9681 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
9682 RC = &AArch64::FPR128RegClass;
9683 Opc = AArch64::FMLAv8i16_indexed;
9684 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9685 kind: FMAInstKind::Indexed);
9686 break;
9687 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
9688 RC = &AArch64::FPR128RegClass;
9689 Opc = AArch64::FMLAv8f16;
9690 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9691 kind: FMAInstKind::Accumulator);
9692 break;
9693
9694 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
9695 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
9696 RC = &AArch64::FPR128RegClass;
9697 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
9698 Opc = AArch64::FMLAv2i64_indexed;
9699 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9700 kind: FMAInstKind::Indexed);
9701 } else {
9702 Opc = AArch64::FMLAv2f64;
9703 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9704 kind: FMAInstKind::Accumulator);
9705 }
9706 break;
9707 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
9708 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
9709 RC = &AArch64::FPR128RegClass;
9710 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
9711 Opc = AArch64::FMLAv2i64_indexed;
9712 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9713 kind: FMAInstKind::Indexed);
9714 } else {
9715 Opc = AArch64::FMLAv2f64;
9716 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9717 kind: FMAInstKind::Accumulator);
9718 }
9719 break;
9720
9721 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
9722 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
9723 RC = &AArch64::FPR128RegClass;
9724 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
9725 Opc = AArch64::FMLAv4i32_indexed;
9726 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9727 kind: FMAInstKind::Indexed);
9728 } else {
9729 Opc = AArch64::FMLAv4f32;
9730 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9731 kind: FMAInstKind::Accumulator);
9732 }
9733 break;
9734
9735 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
9736 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
9737 RC = &AArch64::FPR128RegClass;
9738 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
9739 Opc = AArch64::FMLAv4i32_indexed;
9740 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9741 kind: FMAInstKind::Indexed);
9742 } else {
9743 Opc = AArch64::FMLAv4f32;
9744 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9745 kind: FMAInstKind::Accumulator);
9746 }
9747 break;
9748
9749 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
9750 Opc = AArch64::FNMSUBHrrr;
9751 RC = &AArch64::FPR16RegClass;
9752 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9753 break;
9754 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
9755 Opc = AArch64::FNMSUBSrrr;
9756 RC = &AArch64::FPR32RegClass;
9757 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9758 break;
9759 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
9760 Opc = AArch64::FNMSUBDrrr;
9761 RC = &AArch64::FPR64RegClass;
9762 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9763 break;
9764
9765 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
9766 Opc = AArch64::FNMADDHrrr;
9767 RC = &AArch64::FPR16RegClass;
9768 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9769 break;
9770 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
9771 Opc = AArch64::FNMADDSrrr;
9772 RC = &AArch64::FPR32RegClass;
9773 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9774 break;
9775 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
9776 Opc = AArch64::FNMADDDrrr;
9777 RC = &AArch64::FPR64RegClass;
9778 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9779 break;
9780
9781 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
9782 Opc = AArch64::FMSUBHrrr;
9783 RC = &AArch64::FPR16RegClass;
9784 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9785 break;
9786 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
9787 Opc = AArch64::FMSUBSrrr;
9788 RC = &AArch64::FPR32RegClass;
9789 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9790 break;
9791 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
9792 Opc = AArch64::FMSUBDrrr;
9793 RC = &AArch64::FPR64RegClass;
9794 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9795 break;
9796
9797 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
9798 Opc = AArch64::FMLSv1i32_indexed;
9799 RC = &AArch64::FPR32RegClass;
9800 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9801 kind: FMAInstKind::Indexed);
9802 break;
9803
9804 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
9805 Opc = AArch64::FMLSv1i64_indexed;
9806 RC = &AArch64::FPR64RegClass;
9807 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9808 kind: FMAInstKind::Indexed);
9809 break;
9810
9811 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
9812 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
9813 RC = &AArch64::FPR64RegClass;
9814 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9815 MachineInstrBuilder MIB1 =
9816 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f16), DestReg: NewVR)
9817 .add(MO: Root.getOperand(i: 2));
9818 InsInstrs.push_back(Elt: MIB1);
9819 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9820 if (Pattern == AArch64MachineCombinerPattern::FMLSv4f16_OP1) {
9821 Opc = AArch64::FMLAv4f16;
9822 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9823 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9824 } else {
9825 Opc = AArch64::FMLAv4i16_indexed;
9826 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9827 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9828 }
9829 break;
9830 }
9831 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
9832 RC = &AArch64::FPR64RegClass;
9833 Opc = AArch64::FMLSv4f16;
9834 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9835 kind: FMAInstKind::Accumulator);
9836 break;
9837 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
9838 RC = &AArch64::FPR64RegClass;
9839 Opc = AArch64::FMLSv4i16_indexed;
9840 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9841 kind: FMAInstKind::Indexed);
9842 break;
9843
9844 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
9845 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
9846 RC = &AArch64::FPR64RegClass;
9847 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
9848 Opc = AArch64::FMLSv2i32_indexed;
9849 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9850 kind: FMAInstKind::Indexed);
9851 } else {
9852 Opc = AArch64::FMLSv2f32;
9853 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9854 kind: FMAInstKind::Accumulator);
9855 }
9856 break;
9857
9858 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
9859 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
9860 RC = &AArch64::FPR128RegClass;
9861 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9862 MachineInstrBuilder MIB1 =
9863 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv8f16), DestReg: NewVR)
9864 .add(MO: Root.getOperand(i: 2));
9865 InsInstrs.push_back(Elt: MIB1);
9866 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9867 if (Pattern == AArch64MachineCombinerPattern::FMLSv8f16_OP1) {
9868 Opc = AArch64::FMLAv8f16;
9869 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9870 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9871 } else {
9872 Opc = AArch64::FMLAv8i16_indexed;
9873 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9874 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9875 }
9876 break;
9877 }
9878 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
9879 RC = &AArch64::FPR128RegClass;
9880 Opc = AArch64::FMLSv8f16;
9881 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9882 kind: FMAInstKind::Accumulator);
9883 break;
9884 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
9885 RC = &AArch64::FPR128RegClass;
9886 Opc = AArch64::FMLSv8i16_indexed;
9887 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9888 kind: FMAInstKind::Indexed);
9889 break;
9890
9891 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
9892 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
9893 RC = &AArch64::FPR128RegClass;
9894 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
9895 Opc = AArch64::FMLSv2i64_indexed;
9896 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9897 kind: FMAInstKind::Indexed);
9898 } else {
9899 Opc = AArch64::FMLSv2f64;
9900 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9901 kind: FMAInstKind::Accumulator);
9902 }
9903 break;
9904
9905 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
9906 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
9907 RC = &AArch64::FPR128RegClass;
9908 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
9909 Opc = AArch64::FMLSv4i32_indexed;
9910 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9911 kind: FMAInstKind::Indexed);
9912 } else {
9913 Opc = AArch64::FMLSv4f32;
9914 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9915 kind: FMAInstKind::Accumulator);
9916 }
9917 break;
9918 case AArch64MachineCombinerPattern::FMLSv2f32_OP1:
9919 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
9920 RC = &AArch64::FPR64RegClass;
9921 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9922 MachineInstrBuilder MIB1 =
9923 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f32), DestReg: NewVR)
9924 .add(MO: Root.getOperand(i: 2));
9925 InsInstrs.push_back(Elt: MIB1);
9926 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9927 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
9928 Opc = AArch64::FMLAv2i32_indexed;
9929 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9930 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9931 } else {
9932 Opc = AArch64::FMLAv2f32;
9933 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9934 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9935 }
9936 break;
9937 }
9938 case AArch64MachineCombinerPattern::FMLSv4f32_OP1:
9939 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
9940 RC = &AArch64::FPR128RegClass;
9941 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9942 MachineInstrBuilder MIB1 =
9943 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f32), DestReg: NewVR)
9944 .add(MO: Root.getOperand(i: 2));
9945 InsInstrs.push_back(Elt: MIB1);
9946 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9947 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
9948 Opc = AArch64::FMLAv4i32_indexed;
9949 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9950 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9951 } else {
9952 Opc = AArch64::FMLAv4f32;
9953 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9954 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9955 }
9956 break;
9957 }
9958 case AArch64MachineCombinerPattern::FMLSv2f64_OP1:
9959 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
9960 RC = &AArch64::FPR128RegClass;
9961 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9962 MachineInstrBuilder MIB1 =
9963 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f64), DestReg: NewVR)
9964 .add(MO: Root.getOperand(i: 2));
9965 InsInstrs.push_back(Elt: MIB1);
9966 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9967 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
9968 Opc = AArch64::FMLAv2i64_indexed;
9969 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9970 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9971 } else {
9972 Opc = AArch64::FMLAv2f64;
9973 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9974 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9975 }
9976 break;
9977 }
9978 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
9979 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2: {
9980 unsigned IdxDupOp =
9981 (Pattern == AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1) ? 1
9982 : 2;
9983 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i32_indexed,
9984 RC: &AArch64::FPR128RegClass, MRI);
9985 break;
9986 }
9987 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
9988 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2: {
9989 unsigned IdxDupOp =
9990 (Pattern == AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1) ? 1
9991 : 2;
9992 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i64_indexed,
9993 RC: &AArch64::FPR128RegClass, MRI);
9994 break;
9995 }
9996 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
9997 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2: {
9998 unsigned IdxDupOp =
9999 (Pattern == AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1) ? 1
10000 : 2;
10001 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i16_indexed,
10002 RC: &AArch64::FPR128_loRegClass, MRI);
10003 break;
10004 }
10005 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
10006 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2: {
10007 unsigned IdxDupOp =
10008 (Pattern == AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1) ? 1
10009 : 2;
10010 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i32_indexed,
10011 RC: &AArch64::FPR128RegClass, MRI);
10012 break;
10013 }
10014 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
10015 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2: {
10016 unsigned IdxDupOp =
10017 (Pattern == AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1) ? 1
10018 : 2;
10019 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv8i16_indexed,
10020 RC: &AArch64::FPR128_loRegClass, MRI);
10021 break;
10022 }
10023 case AArch64MachineCombinerPattern::FNMADD: {
10024 MUL = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs);
10025 break;
10026 }
10027 case AArch64MachineCombinerPattern::GATHER_LANE_i32: {
10028 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10029 Pattern, NumLanes: 4);
10030 break;
10031 }
10032 case AArch64MachineCombinerPattern::GATHER_LANE_i16: {
10033 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10034 Pattern, NumLanes: 8);
10035 break;
10036 }
10037 case AArch64MachineCombinerPattern::GATHER_LANE_i8: {
10038 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10039 Pattern, NumLanes: 16);
10040 break;
10041 }
10042
10043 } // end switch (Pattern)
10044 // Record MUL and ADD/SUB for deletion
10045 if (MUL)
10046 DelInstrs.push_back(Elt: MUL);
10047 DelInstrs.push_back(Elt: &Root);
10048
10049 // Set the flags on the inserted instructions to be the merged flags of the
10050 // instructions that we have combined.
10051 uint32_t Flags = Root.getFlags();
10052 if (MUL)
10053 Flags = Root.mergeFlagsWith(Other: *MUL);
10054 for (auto *MI : InsInstrs)
10055 MI->setFlags(Flags);
10056}
10057
10058/// Replace csincr-branch sequence by simple conditional branch
10059///
10060/// Examples:
10061/// 1. \code
10062/// csinc w9, wzr, wzr, <condition code>
10063/// tbnz w9, #0, 0x44
10064/// \endcode
10065/// to
10066/// \code
10067/// b.<inverted condition code>
10068/// \endcode
10069///
10070/// 2. \code
10071/// csinc w9, wzr, wzr, <condition code>
10072/// tbz w9, #0, 0x44
10073/// \endcode
10074/// to
10075/// \code
10076/// b.<condition code>
10077/// \endcode
10078///
10079/// Replace compare and branch sequence by TBZ/TBNZ instruction when the
10080/// compare's constant operand is power of 2.
10081///
10082/// Examples:
10083/// \code
10084/// and w8, w8, #0x400
10085/// cbnz w8, L1
10086/// \endcode
10087/// to
10088/// \code
10089/// tbnz w8, #10, L1
10090/// \endcode
10091///
10092/// \param MI Conditional Branch
10093/// \return True when the simple conditional branch is generated
10094///
10095bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
10096 bool IsNegativeBranch = false;
10097 bool IsTestAndBranch = false;
10098 unsigned TargetBBInMI = 0;
10099 switch (MI.getOpcode()) {
10100 default:
10101 llvm_unreachable("Unknown branch instruction?");
10102 case AArch64::Bcc:
10103 case AArch64::CBWPri:
10104 case AArch64::CBXPri:
10105 case AArch64::CBBAssertExt:
10106 case AArch64::CBHAssertExt:
10107 case AArch64::CBWPrr:
10108 case AArch64::CBXPrr:
10109 return false;
10110 case AArch64::CBZW:
10111 case AArch64::CBZX:
10112 TargetBBInMI = 1;
10113 break;
10114 case AArch64::CBNZW:
10115 case AArch64::CBNZX:
10116 TargetBBInMI = 1;
10117 IsNegativeBranch = true;
10118 break;
10119 case AArch64::TBZW:
10120 case AArch64::TBZX:
10121 TargetBBInMI = 2;
10122 IsTestAndBranch = true;
10123 break;
10124 case AArch64::TBNZW:
10125 case AArch64::TBNZX:
10126 TargetBBInMI = 2;
10127 IsNegativeBranch = true;
10128 IsTestAndBranch = true;
10129 break;
10130 }
10131 // So we increment a zero register and test for bits other
10132 // than bit 0? Conservatively bail out in case the verifier
10133 // missed this case.
10134 if (IsTestAndBranch && MI.getOperand(i: 1).getImm())
10135 return false;
10136
10137 // Find Definition.
10138 assert(MI.getParent() && "Incomplete machine instruction\n");
10139 MachineBasicBlock *MBB = MI.getParent();
10140 MachineFunction *MF = MBB->getParent();
10141 MachineRegisterInfo *MRI = &MF->getRegInfo();
10142 Register VReg = MI.getOperand(i: 0).getReg();
10143 if (!VReg.isVirtual())
10144 return false;
10145
10146 MachineInstr *DefMI = MRI->getVRegDef(Reg: VReg);
10147 if (!DefMI)
10148 return false;
10149
10150 // Look through COPY instructions to find definition.
10151 while (DefMI->isCopy()) {
10152 Register CopyVReg = DefMI->getOperand(i: 1).getReg();
10153 if (!CopyVReg.isVirtual())
10154 return false;
10155 if (!MRI->hasOneNonDBGUse(RegNo: CopyVReg))
10156 return false;
10157 DefMI = MRI->getVRegDef(Reg: CopyVReg);
10158 if (!DefMI)
10159 return false;
10160 }
10161
10162 switch (DefMI->getOpcode()) {
10163 default:
10164 return false;
10165 // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
10166 case AArch64::ANDWri:
10167 case AArch64::ANDXri: {
10168 if (IsTestAndBranch)
10169 return false;
10170 if (DefMI->getParent() != MBB)
10171 return false;
10172 if (!MRI->hasOneNonDBGUse(RegNo: VReg))
10173 return false;
10174
10175 bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
10176 uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
10177 val: DefMI->getOperand(i: 2).getImm(), regSize: Is32Bit ? 32 : 64);
10178 if (!isPowerOf2_64(Value: Mask))
10179 return false;
10180
10181 MachineOperand &MO = DefMI->getOperand(i: 1);
10182 Register NewReg = MO.getReg();
10183 if (!NewReg.isVirtual())
10184 return false;
10185
10186 if (!MRI->getVRegDef(Reg: NewReg))
10187 return false;
10188
10189 MachineBasicBlock &RefToMBB = *MBB;
10190 MachineBasicBlock *TBB = MI.getOperand(i: 1).getMBB();
10191 DebugLoc DL = MI.getDebugLoc();
10192 unsigned Imm = Log2_64(Value: Mask);
10193 unsigned Opc = (Imm < 32)
10194 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
10195 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
10196 MachineInstr *NewMI = BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: Opc))
10197 .addReg(RegNo: NewReg)
10198 .addImm(Val: Imm)
10199 .addMBB(MBB: TBB);
10200 // Register lives on to the CBZ now.
10201 MO.setIsKill(false);
10202
10203 // For immediate smaller than 32, we need to use the 32-bit
10204 // variant (W) in all cases. Indeed the 64-bit variant does not
10205 // allow to encode them.
10206 // Therefore, if the input register is 64-bit, we need to take the
10207 // 32-bit sub-part.
10208 if (!Is32Bit && Imm < 32)
10209 NewMI->getOperand(i: 0).setSubReg(AArch64::sub_32);
10210 MI.eraseFromParent();
10211 return true;
10212 }
10213 // Look for CSINC
10214 case AArch64::CSINCWr:
10215 case AArch64::CSINCXr: {
10216 if (!(DefMI->getOperand(i: 1).getReg() == AArch64::WZR &&
10217 DefMI->getOperand(i: 2).getReg() == AArch64::WZR) &&
10218 !(DefMI->getOperand(i: 1).getReg() == AArch64::XZR &&
10219 DefMI->getOperand(i: 2).getReg() == AArch64::XZR))
10220 return false;
10221
10222 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
10223 isDead: true) != -1)
10224 return false;
10225
10226 AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(i: 3).getImm();
10227 // Convert only when the condition code is not modified between
10228 // the CSINC and the branch. The CC may be used by other
10229 // instructions in between.
10230 if (areCFlagsAccessedBetweenInstrs(From: DefMI, To: MI, TRI: &getRegisterInfo(), AccessToCheck: AK_Write))
10231 return false;
10232 MachineBasicBlock &RefToMBB = *MBB;
10233 MachineBasicBlock *TBB = MI.getOperand(i: TargetBBInMI).getMBB();
10234 DebugLoc DL = MI.getDebugLoc();
10235 if (IsNegativeBranch)
10236 CC = AArch64CC::getInvertedCondCode(Code: CC);
10237 BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: CC).addMBB(MBB: TBB);
10238 MI.eraseFromParent();
10239 return true;
10240 }
10241 }
10242}
10243
10244std::pair<unsigned, unsigned>
10245AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10246 const unsigned Mask = AArch64II::MO_FRAGMENT;
10247 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
10248}
10249
10250ArrayRef<std::pair<unsigned, const char *>>
10251AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10252 using namespace AArch64II;
10253
10254 static const std::pair<unsigned, const char *> TargetFlags[] = {
10255 {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
10256 {MO_G3, "aarch64-g3"}, {MO_G2, "aarch64-g2"},
10257 {MO_G1, "aarch64-g1"}, {MO_G0, "aarch64-g0"},
10258 {MO_HI12, "aarch64-hi12"}};
10259 return ArrayRef(TargetFlags);
10260}
10261
10262ArrayRef<std::pair<unsigned, const char *>>
10263AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
10264 using namespace AArch64II;
10265
10266 static const std::pair<unsigned, const char *> TargetFlags[] = {
10267 {MO_COFFSTUB, "aarch64-coffstub"},
10268 {MO_GOT, "aarch64-got"},
10269 {MO_NC, "aarch64-nc"},
10270 {MO_S, "aarch64-s"},
10271 {MO_TLS, "aarch64-tls"},
10272 {MO_DLLIMPORT, "aarch64-dllimport"},
10273 {MO_PREL, "aarch64-prel"},
10274 {MO_TAGGED, "aarch64-tagged"},
10275 {MO_ARM64EC_CALLMANGLE, "aarch64-arm64ec-callmangle"},
10276 };
10277 return ArrayRef(TargetFlags);
10278}
10279
10280ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10281AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10282 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10283 {{MOSuppressPair, "aarch64-suppress-pair"},
10284 {MOStridedAccess, "aarch64-strided-access"}};
10285 return ArrayRef(TargetFlags);
10286}
10287
10288/// Constants defining how certain sequences should be outlined.
10289/// This encompasses how an outlined function should be called, and what kind of
10290/// frame should be emitted for that outlined function.
10291///
10292/// \p MachineOutlinerDefault implies that the function should be called with
10293/// a save and restore of LR to the stack.
10294///
10295/// That is,
10296///
10297/// I1 Save LR OUTLINED_FUNCTION:
10298/// I2 --> BL OUTLINED_FUNCTION I1
10299/// I3 Restore LR I2
10300/// I3
10301/// RET
10302///
10303/// * Call construction overhead: 3 (save + BL + restore)
10304/// * Frame construction overhead: 1 (ret)
10305/// * Requires stack fixups? Yes
10306///
10307/// \p MachineOutlinerTailCall implies that the function is being created from
10308/// a sequence of instructions ending in a return.
10309///
10310/// That is,
10311///
10312/// I1 OUTLINED_FUNCTION:
10313/// I2 --> B OUTLINED_FUNCTION I1
10314/// RET I2
10315/// RET
10316///
10317/// * Call construction overhead: 1 (B)
10318/// * Frame construction overhead: 0 (Return included in sequence)
10319/// * Requires stack fixups? No
10320///
10321/// \p MachineOutlinerNoLRSave implies that the function should be called using
10322/// a BL instruction, but doesn't require LR to be saved and restored. This
10323/// happens when LR is known to be dead.
10324///
10325/// That is,
10326///
10327/// I1 OUTLINED_FUNCTION:
10328/// I2 --> BL OUTLINED_FUNCTION I1
10329/// I3 I2
10330/// I3
10331/// RET
10332///
10333/// * Call construction overhead: 1 (BL)
10334/// * Frame construction overhead: 1 (RET)
10335/// * Requires stack fixups? No
10336///
10337/// \p MachineOutlinerThunk implies that the function is being created from
10338/// a sequence of instructions ending in a call. The outlined function is
10339/// called with a BL instruction, and the outlined function tail-calls the
10340/// original call destination.
10341///
10342/// That is,
10343///
10344/// I1 OUTLINED_FUNCTION:
10345/// I2 --> BL OUTLINED_FUNCTION I1
10346/// BL f I2
10347/// B f
10348/// * Call construction overhead: 1 (BL)
10349/// * Frame construction overhead: 0
10350/// * Requires stack fixups? No
10351///
10352/// \p MachineOutlinerRegSave implies that the function should be called with a
10353/// save and restore of LR to an available register. This allows us to avoid
10354/// stack fixups. Note that this outlining variant is compatible with the
10355/// NoLRSave case.
10356///
10357/// That is,
10358///
10359/// I1 Save LR OUTLINED_FUNCTION:
10360/// I2 --> BL OUTLINED_FUNCTION I1
10361/// I3 Restore LR I2
10362/// I3
10363/// RET
10364///
10365/// * Call construction overhead: 3 (save + BL + restore)
10366/// * Frame construction overhead: 1 (ret)
10367/// * Requires stack fixups? No
10368enum MachineOutlinerClass {
10369 MachineOutlinerDefault, /// Emit a save, restore, call, and return.
10370 MachineOutlinerTailCall, /// Only emit a branch.
10371 MachineOutlinerNoLRSave, /// Emit a call and return.
10372 MachineOutlinerThunk, /// Emit a call and tail-call.
10373 MachineOutlinerRegSave /// Same as default, but save to a register.
10374};
10375
10376enum MachineOutlinerMBBFlags {
10377 LRUnavailableSomewhere = 0x2,
10378 HasCalls = 0x4,
10379 UnsafeRegsDead = 0x8
10380};
10381
10382/// Return true if the frame-record form of the outlined prologue is enabled for
10383/// the target of \p MF.
10384///
10385/// A non-leaf outlined function must save LR. On MachO, saving LR alone
10386/// (str x30) has no compact unwind encoding, so we get a large DWARF FDE
10387/// instead. Saving FP and LR as a frame record (stp x29, x30 ; mov x29, sp)
10388/// gets the small FRAME encoding, and costs one extra instruction.
10389static bool isCompactUnwindFrameRecordEnabled(const MachineFunction &MF) {
10390 return UseCompactUnwindFrameRecordForOutlinedFunctions &&
10391 MF.getTarget().getTargetTriple().isOSBinFormatMachO();
10392}
10393
10394/// Return true if the outlined function in \p MBB should save FP and LR as a
10395/// frame record instead of saving LR alone.
10396static bool shouldUseCompactUnwindFrameRecordForOutlinedFunction(
10397 const MachineBasicBlock &MBB) {
10398 const MachineFunction &MF = *MBB.getParent();
10399
10400 // Only worth it if the function has unwind info to shrink.
10401 if (!isCompactUnwindFrameRecordEnabled(MF) ||
10402 !MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF))
10403 return false;
10404
10405 // Only safe if the outlined code never touches FP, since we overwrite it.
10406 LiveRegUnits LRU(*MF.getSubtarget().getRegisterInfo());
10407 for (const MachineInstr &MI : MBB.instrs())
10408 LRU.accumulate(MI);
10409 return LRU.available(Reg: AArch64::FP);
10410}
10411
10412/// Predict what the above will answer, for use while costing candidates. The
10413/// outlined function does not exist yet, so answer from \p RepeatedSequenceLocs
10414/// instead. This is only an estimate; buildOutlinedFrame() makes the call.
10415static bool predictCompactUnwindFrameRecordForOutlinedFunction(
10416 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10417 const TargetRegisterInfo &TRI) {
10418 if (!isCompactUnwindFrameRecordEnabled(MF: *RepeatedSequenceLocs.front().getMF()))
10419 return false;
10420
10421 // The outlined function is nounwind only if every candidate is, so it has
10422 // unwind info if any candidate does.
10423 if (llvm::none_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10424 const MachineFunction &MF = *C.getMF();
10425 return MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF);
10426 }))
10427 return false;
10428
10429 // FP is free in the outlined function only if it is free in every candidate.
10430 return llvm::all_of(Range&: RepeatedSequenceLocs, P: [&TRI](outliner::Candidate &C) {
10431 return C.isAvailableInsideSeq(Reg: AArch64::FP, TRI);
10432 });
10433}
10434
10435Register
10436AArch64InstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
10437 MachineFunction *MF = C.getMF();
10438 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
10439 const AArch64RegisterInfo *ARI =
10440 static_cast<const AArch64RegisterInfo *>(&TRI);
10441 // Check if there is an available register across the sequence that we can
10442 // use.
10443 for (unsigned Reg : AArch64::GPR64RegClass) {
10444 if (!ARI->isReservedReg(MF: *MF, Reg) &&
10445 Reg != AArch64::LR && // LR is not reserved, but don't use it.
10446 Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
10447 Reg != AArch64::X17 && // Ditto for X17.
10448 C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
10449 C.isAvailableInsideSeq(Reg, TRI))
10450 return Reg;
10451 }
10452 return Register();
10453}
10454
10455static bool
10456outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
10457 const outliner::Candidate &b) {
10458 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10459 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10460
10461 return MFIa->getSignReturnAddressCondition() ==
10462 MFIb->getSignReturnAddressCondition();
10463}
10464
10465static bool
10466outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
10467 const outliner::Candidate &b) {
10468 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10469 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10470
10471 return MFIa->shouldSignWithBKey() == MFIb->shouldSignWithBKey();
10472}
10473
10474static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
10475 const outliner::Candidate &b) {
10476 const AArch64Subtarget &SubtargetA =
10477 a.getMF()->getSubtarget<AArch64Subtarget>();
10478 const AArch64Subtarget &SubtargetB =
10479 b.getMF()->getSubtarget<AArch64Subtarget>();
10480 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
10481}
10482
10483std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10484AArch64InstrInfo::getOutliningCandidateInfo(
10485 const MachineModuleInfo &MMI,
10486 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10487 unsigned MinRepeats) const {
10488 unsigned SequenceSize = 0;
10489 for (auto &MI : RepeatedSequenceLocs[0])
10490 SequenceSize += getInstSizeInBytes(MI);
10491
10492 unsigned NumBytesToCreateFrame = 0;
10493
10494 // Avoid splitting ADRP ADD/LDR pair into outlined functions.
10495 // These instructions are fused together by the scheduler.
10496 // Any candidate where ADRP is the last instruction should be rejected
10497 // as that will lead to splitting ADRP pair.
10498 MachineInstr &LastMI = RepeatedSequenceLocs[0].back();
10499 MachineInstr &FirstMI = RepeatedSequenceLocs[0].front();
10500 if (LastMI.getOpcode() == AArch64::ADRP &&
10501 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_PAGE) != 0 &&
10502 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10503 return std::nullopt;
10504 }
10505
10506 // Similarly any candidate where the first instruction is ADD/LDR with a
10507 // page offset should be rejected to avoid ADRP splitting.
10508 if ((FirstMI.getOpcode() == AArch64::ADDXri ||
10509 FirstMI.getOpcode() == AArch64::LDRXui) &&
10510 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_PAGEOFF) != 0 &&
10511 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10512 return std::nullopt;
10513 }
10514
10515 // We only allow outlining for functions having exactly matching return
10516 // address signing attributes, i.e., all share the same value for the
10517 // attribute "sign-return-address" and all share the same type of key they
10518 // are signed with.
10519 // Additionally we require all functions to simultaneously either support
10520 // v8.3a features or not. Otherwise an outlined function could get signed
10521 // using dedicated v8.3 instructions and a call from a function that doesn't
10522 // support v8.3 instructions would therefore be invalid.
10523 if (std::adjacent_find(
10524 first: RepeatedSequenceLocs.begin(), last: RepeatedSequenceLocs.end(),
10525 binary_pred: [](const outliner::Candidate &a, const outliner::Candidate &b) {
10526 // Return true if a and b are non-equal w.r.t. return address
10527 // signing or support of v8.3a features
10528 if (outliningCandidatesSigningScopeConsensus(a, b) &&
10529 outliningCandidatesSigningKeyConsensus(a, b) &&
10530 outliningCandidatesV8_3OpsConsensus(a, b)) {
10531 return false;
10532 }
10533 return true;
10534 }) != RepeatedSequenceLocs.end()) {
10535 return std::nullopt;
10536 }
10537
10538 // Since at this point all candidates agree on their return address signing
10539 // picking just one is fine. If the candidate functions potentially sign their
10540 // return addresses, the outlined function should do the same. Note that in
10541 // the case of "sign-return-address"="non-leaf" this is an assumption: It is
10542 // not certainly true that the outlined function will have to sign its return
10543 // address but this decision is made later, when the decision to outline
10544 // has already been made.
10545 // The same holds for the number of additional instructions we need: On
10546 // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
10547 // necessary. However, at this point we don't know if the outlined function
10548 // will have a RET instruction so we assume the worst.
10549 const TargetRegisterInfo &TRI = getRegisterInfo();
10550 // Performing a tail call may require extra checks when PAuth is enabled.
10551 // If PAuth is disabled, set it to zero for uniformity.
10552 unsigned NumBytesToCheckLRInTCEpilogue = 0;
10553 const auto RASignCondition = RepeatedSequenceLocs[0]
10554 .getMF()
10555 ->getInfo<AArch64FunctionInfo>()
10556 ->getSignReturnAddressCondition();
10557 if (RASignCondition != SignReturnAddress::None) {
10558 // One PAC and one AUT instructions
10559 NumBytesToCreateFrame += 8;
10560
10561 // PAuth is enabled - set extra tail call cost, if any.
10562 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
10563 MF: *RepeatedSequenceLocs[0].getMF());
10564 NumBytesToCheckLRInTCEpilogue =
10565 AArch64PAuth::getCheckerSizeInBytes(Method: LRCheckMethod);
10566 // Checking the authenticated LR value may significantly impact
10567 // SequenceSize, so account for it for more precise results.
10568 if (isTailCallReturnInst(MI: RepeatedSequenceLocs[0].back()))
10569 SequenceSize += NumBytesToCheckLRInTCEpilogue;
10570
10571 // We have to check if sp modifying instructions would get outlined.
10572 // If so we only allow outlining if sp is unchanged overall, so matching
10573 // sub and add instructions are okay to outline, all other sp modifications
10574 // are not
10575 auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
10576 int SPValue = 0;
10577 for (auto &MI : C) {
10578 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI)) {
10579 switch (MI.getOpcode()) {
10580 case AArch64::ADDXri:
10581 case AArch64::ADDWri:
10582 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10583 assert(MI.getOperand(2).isImm() &&
10584 "Expected operand to be immediate");
10585 assert(MI.getOperand(1).isReg() &&
10586 "Expected operand to be a register");
10587 // Check if the add just increments sp. If so, we search for
10588 // matching sub instructions that decrement sp. If not, the
10589 // modification is illegal
10590 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10591 SPValue += MI.getOperand(i: 2).getImm();
10592 else
10593 return true;
10594 break;
10595 case AArch64::SUBXri:
10596 case AArch64::SUBWri:
10597 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10598 assert(MI.getOperand(2).isImm() &&
10599 "Expected operand to be immediate");
10600 assert(MI.getOperand(1).isReg() &&
10601 "Expected operand to be a register");
10602 // Check if the sub just decrements sp. If so, we search for
10603 // matching add instructions that increment sp. If not, the
10604 // modification is illegal
10605 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10606 SPValue -= MI.getOperand(i: 2).getImm();
10607 else
10608 return true;
10609 break;
10610 default:
10611 return true;
10612 }
10613 }
10614 }
10615 if (SPValue)
10616 return true;
10617 return false;
10618 };
10619 // Remove candidates with illegal stack modifying instructions
10620 llvm::erase_if(C&: RepeatedSequenceLocs, P: hasIllegalSPModification);
10621
10622 // If the sequence doesn't have enough candidates left, then we're done.
10623 if (RepeatedSequenceLocs.size() < MinRepeats)
10624 return std::nullopt;
10625 }
10626
10627 // Properties about candidate MBBs that hold for all of them.
10628 unsigned FlagsSetInAll = 0xF;
10629
10630 // Compute liveness information for each candidate, and set FlagsSetInAll.
10631 for (outliner::Candidate &C : RepeatedSequenceLocs)
10632 FlagsSetInAll &= C.Flags;
10633
10634 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
10635
10636 // Helper lambda which sets call information for every candidate.
10637 auto SetCandidateCallInfo =
10638 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
10639 for (outliner::Candidate &C : RepeatedSequenceLocs)
10640 C.setCallInfo(CID: CallID, CO: NumBytesForCall);
10641 };
10642
10643 unsigned FrameID = MachineOutlinerDefault;
10644 NumBytesToCreateFrame += 4;
10645
10646 bool HasBTI = any_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10647 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
10648 });
10649
10650 // We check to see if CFI Instructions are present, and if they are
10651 // we find the number of CFI Instructions in the candidates.
10652 unsigned CFICount = 0;
10653 for (auto &I : RepeatedSequenceLocs[0]) {
10654 if (I.isCFIInstruction())
10655 CFICount++;
10656 }
10657
10658 // We compare the number of found CFI Instructions to the number of CFI
10659 // instructions in the parent function for each candidate. We must check this
10660 // since if we outline one of the CFI instructions in a function, we have to
10661 // outline them all for correctness. If we do not, the address offsets will be
10662 // incorrect between the two sections of the program.
10663 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10664 std::vector<MCCFIInstruction> CFIInstructions =
10665 C.getMF()->getFrameInstructions();
10666
10667 if (CFICount > 0 && CFICount != CFIInstructions.size())
10668 return std::nullopt;
10669 }
10670
10671 // Returns true if an instructions is safe to fix up, false otherwise.
10672 auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
10673 if (MI.isCall())
10674 return true;
10675
10676 if (!MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI) &&
10677 !MI.readsRegister(Reg: AArch64::SP, TRI: &TRI))
10678 return true;
10679
10680 // Any modification of SP will break our code to save/restore LR.
10681 // FIXME: We could handle some instructions which add a constant
10682 // offset to SP, with a bit more work.
10683 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI))
10684 return false;
10685
10686 // At this point, we have a stack instruction that we might need to
10687 // fix up. We'll handle it if it's a load or store.
10688 if (MI.mayLoadOrStore()) {
10689 const MachineOperand *Base; // Filled with the base operand of MI.
10690 int64_t Offset; // Filled with the offset of MI.
10691 bool OffsetIsScalable;
10692
10693 // Does it allow us to offset the base operand and is the base the
10694 // register SP?
10695 if (!getMemOperandWithOffset(MI, BaseOp&: Base, Offset, OffsetIsScalable, TRI: &TRI) ||
10696 !Base->isReg() || Base->getReg() != AArch64::SP)
10697 return false;
10698
10699 // Fixe-up code below assumes bytes.
10700 if (OffsetIsScalable)
10701 return false;
10702
10703 // Find the minimum/maximum offset for this instruction and check
10704 // if fixing it up would be in range.
10705 int64_t MinOffset,
10706 MaxOffset; // Unscaled offsets for the instruction.
10707 // The scale to multiply the offsets by.
10708 TypeSize Scale(0U, false), DummyWidth(0U, false);
10709 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width&: DummyWidth, MinOffset, MaxOffset);
10710
10711 Offset += 16; // Update the offset to what it would be if we outlined.
10712 if (Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
10713 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
10714 return false;
10715
10716 // It's in range, so we can outline it.
10717 return true;
10718 }
10719
10720 // FIXME: Add handling for instructions like "add x0, sp, #8".
10721
10722 // We can't fix it up, so don't outline it.
10723 return false;
10724 };
10725
10726 // True if it's possible to fix up each stack instruction in this sequence.
10727 // Important for frames/call variants that modify the stack.
10728 bool AllStackInstrsSafe =
10729 llvm::all_of(Range&: RepeatedSequenceLocs[0], P: IsSafeToFixup);
10730
10731 // If the last instruction in any candidate is a terminator, then we should
10732 // tail call all of the candidates.
10733 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10734 FrameID = MachineOutlinerTailCall;
10735 NumBytesToCreateFrame = 0;
10736 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
10737 SetCandidateCallInfo(MachineOutlinerTailCall, NumBytesForCall);
10738 }
10739
10740 else if (LastInstrOpcode == AArch64::BL ||
10741 ((LastInstrOpcode == AArch64::BLR ||
10742 LastInstrOpcode == AArch64::BLRNoIP) &&
10743 !HasBTI)) {
10744 // FIXME: Do we need to check if the code after this uses the value of LR?
10745 FrameID = MachineOutlinerThunk;
10746 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
10747 SetCandidateCallInfo(MachineOutlinerThunk, 4);
10748 }
10749
10750 else {
10751 // We need to decide how to emit calls + frames. We can always emit the same
10752 // frame if we don't need to save to the stack. If we have to save to the
10753 // stack, then we need a different frame.
10754 unsigned NumBytesNoStackCalls = 0;
10755 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
10756
10757 // Check if we have to save LR.
10758 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10759 bool LRAvailable =
10760 (C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
10761 ? C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI)
10762 : true;
10763 // If we have a noreturn caller, then we're going to be conservative and
10764 // say that we have to save LR. If we don't have a ret at the end of the
10765 // block, then we can't reason about liveness accurately.
10766 //
10767 // FIXME: We can probably do better than always disabling this in
10768 // noreturn functions by fixing up the liveness info.
10769 bool IsNoReturn =
10770 C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoReturn);
10771
10772 // Is LR available? If so, we don't need a save.
10773 if (LRAvailable && !IsNoReturn) {
10774 NumBytesNoStackCalls += 4;
10775 C.setCallInfo(CID: MachineOutlinerNoLRSave, CO: 4);
10776 CandidatesWithoutStackFixups.push_back(x: C);
10777 }
10778
10779 // Is an unused register available? If so, we won't modify the stack, so
10780 // we can outline with the same frame type as those that don't save LR.
10781 else if (findRegisterToSaveLRTo(C)) {
10782 NumBytesNoStackCalls += 12;
10783 C.setCallInfo(CID: MachineOutlinerRegSave, CO: 12);
10784 CandidatesWithoutStackFixups.push_back(x: C);
10785 }
10786
10787 // Is SP used in the sequence at all? If not, we don't have to modify
10788 // the stack, so we are guaranteed to get the same frame.
10789 else if (C.isAvailableInsideSeq(Reg: AArch64::SP, TRI)) {
10790 NumBytesNoStackCalls += 12;
10791 C.setCallInfo(CID: MachineOutlinerDefault, CO: 12);
10792 CandidatesWithoutStackFixups.push_back(x: C);
10793 }
10794
10795 // If we outline this, we need to modify the stack. Pretend we don't
10796 // outline this by saving all of its bytes.
10797 else {
10798 NumBytesNoStackCalls += SequenceSize;
10799 }
10800 }
10801
10802 // If there are no places where we have to save LR, then note that we
10803 // don't have to update the stack. Otherwise, give every candidate the
10804 // default call type, as long as it's safe to do so.
10805 if (!AllStackInstrsSafe ||
10806 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
10807 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
10808 FrameID = MachineOutlinerNoLRSave;
10809 if (RepeatedSequenceLocs.size() < MinRepeats)
10810 return std::nullopt;
10811 } else {
10812 SetCandidateCallInfo(MachineOutlinerDefault, 12);
10813
10814 // Bugzilla ID: 46767
10815 // TODO: Check if fixing up the stack more than once is safe so we can
10816 // outline these.
10817 //
10818 // An outline resulting in a caller that requires stack fixups at the
10819 // callsite to a callee that also requires stack fixups can happen when
10820 // there are no available registers at the candidate callsite for a
10821 // candidate that itself also has calls.
10822 //
10823 // In other words if function_containing_sequence in the following pseudo
10824 // assembly requires that we save LR at the point of the call, but there
10825 // are no available registers: in this case we save using SP and as a
10826 // result the SP offsets requires stack fixups by multiples of 16.
10827 //
10828 // function_containing_sequence:
10829 // ...
10830 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10831 // call OUTLINED_FUNCTION_N
10832 // restore LR from SP
10833 // ...
10834 //
10835 // OUTLINED_FUNCTION_N:
10836 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10837 // ...
10838 // bl foo
10839 // restore LR from SP
10840 // ret
10841 //
10842 // Because the code to handle more than one stack fixup does not
10843 // currently have the proper checks for legality, these cases will assert
10844 // in the AArch64 MachineOutliner. This is because the code to do this
10845 // needs more hardening, testing, better checks that generated code is
10846 // legal, etc and because it is only verified to handle a single pass of
10847 // stack fixup.
10848 //
10849 // The assert happens in AArch64InstrInfo::buildOutlinedFrame to catch
10850 // these cases until they are known to be handled. Bugzilla 46767 is
10851 // referenced in comments at the assert site.
10852 //
10853 // To avoid asserting (or generating non-legal code on noassert builds)
10854 // we remove all candidates which would need more than one stack fixup by
10855 // pruning the cases where the candidate has calls while also having no
10856 // available LR and having no available general purpose registers to copy
10857 // LR to (ie one extra stack save/restore).
10858 //
10859 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10860 erase_if(C&: RepeatedSequenceLocs, P: [this, &TRI](outliner::Candidate &C) {
10861 auto IsCall = [](const MachineInstr &MI) { return MI.isCall(); };
10862 return (llvm::any_of(Range&: C, P: IsCall)) &&
10863 (!C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI) ||
10864 !findRegisterToSaveLRTo(C));
10865 });
10866 }
10867 }
10868
10869 // If we dropped all of the candidates, bail out here.
10870 if (RepeatedSequenceLocs.size() < MinRepeats)
10871 return std::nullopt;
10872 }
10873
10874 // Does every candidate's MBB contain a call? If so, then we might have a call
10875 // in the range.
10876 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10877 // Check if the range contains a call. These require a save + restore of the
10878 // link register.
10879 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
10880 bool ModStackToSaveLR = false;
10881 if (any_of(Range: drop_end(RangeOrContainer&: FirstCand),
10882 P: [](const MachineInstr &MI) { return MI.isCall(); }))
10883 ModStackToSaveLR = true;
10884
10885 // Handle the last instruction separately. If this is a tail call, then the
10886 // last instruction is a call. We don't want to save + restore in this case.
10887 // However, it could be possible that the last instruction is a call without
10888 // it being valid to tail call this sequence. We should consider this as
10889 // well.
10890 else if (FrameID != MachineOutlinerThunk &&
10891 FrameID != MachineOutlinerTailCall && FirstCand.back().isCall())
10892 ModStackToSaveLR = true;
10893
10894 if (ModStackToSaveLR) {
10895 // We can't fix up the stack. Bail out.
10896 if (!AllStackInstrsSafe)
10897 return std::nullopt;
10898
10899 // Save + restore LR.
10900 NumBytesToCreateFrame += 8;
10901
10902 // Add the extra mov if we will save a frame record instead of just LR.
10903 if (predictCompactUnwindFrameRecordForOutlinedFunction(
10904 RepeatedSequenceLocs, TRI))
10905 NumBytesToCreateFrame += 4;
10906 }
10907 }
10908
10909 // If we have CFI instructions, we can only outline if the outlined section
10910 // can be a tail call
10911 if (FrameID != MachineOutlinerTailCall && CFICount > 0)
10912 return std::nullopt;
10913
10914 return std::make_unique<outliner::OutlinedFunction>(
10915 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: NumBytesToCreateFrame, args&: FrameID);
10916}
10917
10918void AArch64InstrInfo::mergeOutliningCandidateAttributes(
10919 Function &F, std::vector<outliner::Candidate> &Candidates) const {
10920 // If a bunch of candidates reach this point they must agree on their return
10921 // address signing. It is therefore enough to just consider the signing
10922 // behaviour of one of them
10923 const auto &CFn = Candidates.front().getMF()->getFunction();
10924
10925 if (CFn.hasFnAttribute(Kind: "ptrauth-returns"))
10926 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-returns"));
10927 if (CFn.hasFnAttribute(Kind: "ptrauth-auth-traps"))
10928 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-auth-traps"));
10929 // Since all candidates belong to the same module, just copy the
10930 // function-level attributes of an arbitrary function.
10931 if (CFn.hasFnAttribute(Kind: "sign-return-address"))
10932 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address"));
10933 if (CFn.hasFnAttribute(Kind: "sign-return-address-key"))
10934 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address-key"));
10935
10936 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
10937}
10938
10939bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
10940 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
10941 const Function &F = MF.getFunction();
10942
10943 // Can F be deduplicated by the linker? If it can, don't outline from it.
10944 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
10945 return false;
10946
10947 // Don't outline from functions with section markings; the program could
10948 // expect that all the code is in the named section.
10949 // FIXME: Allow outlining from multiple functions with the same section
10950 // marking.
10951 if (F.hasSection())
10952 return false;
10953
10954 // Outlining from functions with redzones is unsafe since the outliner may
10955 // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
10956 // outline from it.
10957 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
10958 if (!AFI || AFI->hasRedZone().value_or(u: true))
10959 return false;
10960
10961 // FIXME: Determine whether it is safe to outline from functions which contain
10962 // streaming-mode changes. We may need to ensure any smstart/smstop pairs are
10963 // outlined together and ensure it is safe to outline with async unwind info,
10964 // required for saving & restoring VG around calls.
10965 if (AFI->hasStreamingModeChanges())
10966 return false;
10967
10968 // FIXME: Teach the outliner to generate/handle Windows unwind info.
10969 if (MF.getTarget().getMCAsmInfo().usesWindowsCFI())
10970 return false;
10971
10972 // It's safe to outline from MF.
10973 return true;
10974}
10975
10976SmallVector<std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10977AArch64InstrInfo::getOutlinableRanges(MachineBasicBlock &MBB,
10978 unsigned &Flags) const {
10979 assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
10980 "Must track liveness!");
10981 SmallVector<
10982 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10983 Ranges;
10984 // According to the AArch64 Procedure Call Standard, the following are
10985 // undefined on entry/exit from a function call:
10986 //
10987 // * Registers x16, x17, (and thus w16, w17)
10988 // * Condition codes (and thus the NZCV register)
10989 //
10990 // If any of these registers are used inside or live across an outlined
10991 // function, then they may be modified later, either by the compiler or
10992 // some other tool (like the linker).
10993 //
10994 // To avoid outlining in these situations, partition each block into ranges
10995 // where these registers are dead. We will only outline from those ranges.
10996 LiveRegUnits LRU(getRegisterInfo());
10997 auto AreAllUnsafeRegsDead = [&LRU]() {
10998 return LRU.available(Reg: AArch64::W16) && LRU.available(Reg: AArch64::W17) &&
10999 LRU.available(Reg: AArch64::NZCV);
11000 };
11001
11002 // We need to know if LR is live across an outlining boundary later on in
11003 // order to decide how we'll create the outlined call, frame, etc.
11004 //
11005 // It's pretty expensive to check this for *every candidate* within a block.
11006 // That's some potentially n^2 behaviour, since in the worst case, we'd need
11007 // to compute liveness from the end of the block for O(n) candidates within
11008 // the block.
11009 //
11010 // So, to improve the average case, let's keep track of liveness from the end
11011 // of the block to the beginning of *every outlinable range*. If we know that
11012 // LR is available in every range we could outline from, then we know that
11013 // we don't need to check liveness for any candidate within that range.
11014 bool LRAvailableEverywhere = true;
11015 // Compute liveness bottom-up.
11016 LRU.addLiveOuts(MBB);
11017 // Update flags that require info about the entire MBB.
11018 auto UpdateWholeMBBFlags = [&Flags](const MachineInstr &MI) {
11019 if (MI.isCall() && !MI.isTerminator())
11020 Flags |= MachineOutlinerMBBFlags::HasCalls;
11021 };
11022 // Range: [RangeBegin, RangeEnd)
11023 MachineBasicBlock::instr_iterator RangeBegin, RangeEnd;
11024 unsigned RangeLen;
11025 auto CreateNewRangeStartingAt =
11026 [&RangeBegin, &RangeEnd,
11027 &RangeLen](MachineBasicBlock::instr_iterator NewBegin) {
11028 RangeBegin = NewBegin;
11029 RangeEnd = std::next(x: RangeBegin);
11030 RangeLen = 0;
11031 };
11032 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
11033 // At least one unsafe register is not dead. We do not want to outline at
11034 // this point. If it is long enough to outline from and does not cross a
11035 // bundle boundary, save the range [RangeBegin, RangeEnd).
11036 if (RangeLen <= 1)
11037 return;
11038 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
11039 return;
11040 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
11041 return;
11042 Ranges.emplace_back(Args&: RangeBegin, Args&: RangeEnd);
11043 };
11044 // Find the first point where all unsafe registers are dead.
11045 // FIND: <safe instr> <-- end of first potential range
11046 // SKIP: <unsafe def>
11047 // SKIP: ... everything between ...
11048 // SKIP: <unsafe use>
11049 auto FirstPossibleEndPt = MBB.instr_rbegin();
11050 for (; FirstPossibleEndPt != MBB.instr_rend(); ++FirstPossibleEndPt) {
11051 if (!FirstPossibleEndPt->isDebugInstr())
11052 LRU.stepBackward(MI: *FirstPossibleEndPt);
11053 // Update flags that impact how we outline across the entire block,
11054 // regardless of safety.
11055 UpdateWholeMBBFlags(*FirstPossibleEndPt);
11056 if (AreAllUnsafeRegsDead())
11057 break;
11058 }
11059 // If we exhausted the entire block, we have no safe ranges to outline.
11060 if (FirstPossibleEndPt == MBB.instr_rend())
11061 return Ranges;
11062 // Current range.
11063 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
11064 // StartPt points to the first place where all unsafe registers
11065 // are dead (if there is any such point). Begin partitioning the MBB into
11066 // ranges.
11067 for (auto &MI : make_range(x: FirstPossibleEndPt, y: MBB.instr_rend())) {
11068 if (!MI.isDebugInstr())
11069 LRU.stepBackward(MI);
11070 UpdateWholeMBBFlags(MI);
11071 if (!AreAllUnsafeRegsDead()) {
11072 SaveRangeIfNonEmpty();
11073 CreateNewRangeStartingAt(MI.getIterator());
11074 continue;
11075 }
11076 LRAvailableEverywhere &= LRU.available(Reg: AArch64::LR);
11077 // RangeBegin may point at a debug instruction because the mapper ignores
11078 // debug instructions wherever they appear. Only count non-debug
11079 // instructions so debug info cannot make a short range outlinable.
11080 RangeBegin = MI.getIterator();
11081 if (!MI.isDebugInstr())
11082 ++RangeLen;
11083 }
11084 // Above loop misses the last (or only) range. If we are still safe, then
11085 // let's save the range.
11086 if (AreAllUnsafeRegsDead())
11087 SaveRangeIfNonEmpty();
11088 if (Ranges.empty())
11089 return Ranges;
11090 // We found the ranges bottom-up. Mapping expects the top-down. Reverse
11091 // the order.
11092 std::reverse(first: Ranges.begin(), last: Ranges.end());
11093 // If there is at least one outlinable range where LR is unavailable
11094 // somewhere, remember that.
11095 if (!LRAvailableEverywhere)
11096 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
11097 return Ranges;
11098}
11099
11100outliner::InstrType
11101AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
11102 MachineBasicBlock::iterator &MIT,
11103 unsigned Flags) const {
11104 MachineInstr &MI = *MIT;
11105
11106 // Don't outline anything used for return address signing. The outlined
11107 // function will get signed later if needed
11108 switch (MI.getOpcode()) {
11109 case AArch64::PACM:
11110 case AArch64::PACIASP:
11111 case AArch64::PACIBSP:
11112 case AArch64::PACIASPPC:
11113 case AArch64::PACIBSPPC:
11114 case AArch64::AUTIASP:
11115 case AArch64::AUTIBSP:
11116 case AArch64::AUTIASPPCi:
11117 case AArch64::AUTIASPPCr:
11118 case AArch64::AUTIBSPPCi:
11119 case AArch64::AUTIBSPPCr:
11120 case AArch64::RETAA:
11121 case AArch64::RETAB:
11122 case AArch64::RETAASPPCi:
11123 case AArch64::RETAASPPCr:
11124 case AArch64::RETABSPPCi:
11125 case AArch64::RETABSPPCr:
11126 case AArch64::EMITBKEY:
11127 case AArch64::PAUTH_PROLOGUE:
11128 case AArch64::PAUTH_EPILOGUE:
11129 return outliner::InstrType::Illegal;
11130 }
11131
11132 // We can only outline these if we will tail call the outlined function, or
11133 // fix up the CFI offsets. Currently, CFI instructions are outlined only if
11134 // in a tail call.
11135 //
11136 // FIXME: If the proper fixups for the offset are implemented, this should be
11137 // possible.
11138 if (MI.isCFIInstruction())
11139 return outliner::InstrType::Legal;
11140
11141 // Is this a terminator for a basic block?
11142 if (MI.isTerminator())
11143 // TargetInstrInfo::getOutliningType has already filtered out anything
11144 // that would break this, so we can allow it here.
11145 return outliner::InstrType::Legal;
11146
11147 // Make sure none of the operands are un-outlinable.
11148 for (const MachineOperand &MOP : MI.operands()) {
11149 // A check preventing CFI indices was here before, but only CFI
11150 // instructions should have those.
11151 assert(!MOP.isCFIIndex());
11152
11153 // If it uses LR or W30 explicitly, then don't touch it.
11154 if (MOP.isReg() && !MOP.isImplicit() &&
11155 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
11156 return outliner::InstrType::Illegal;
11157 }
11158
11159 // Special cases for instructions that can always be outlined, but will fail
11160 // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
11161 // be outlined because they don't require a *specific* value to be in LR.
11162 if (MI.getOpcode() == AArch64::ADRP)
11163 return outliner::InstrType::Legal;
11164
11165 // If MI is a call we might be able to outline it. We don't want to outline
11166 // any calls that rely on the position of items on the stack. When we outline
11167 // something containing a call, we have to emit a save and restore of LR in
11168 // the outlined function. Currently, this always happens by saving LR to the
11169 // stack. Thus, if we outline, say, half the parameters for a function call
11170 // plus the call, then we'll break the callee's expectations for the layout
11171 // of the stack.
11172 //
11173 // FIXME: Allow calls to functions which construct a stack frame, as long
11174 // as they don't access arguments on the stack.
11175 // FIXME: Figure out some way to analyze functions defined in other modules.
11176 // We should be able to compute the memory usage based on the IR calling
11177 // convention, even if we can't see the definition.
11178 if (MI.isCall()) {
11179 // Get the function associated with the call. Look at each operand and find
11180 // the one that represents the callee and get its name.
11181 const Function *Callee = nullptr;
11182 for (const MachineOperand &MOP : MI.operands()) {
11183 if (MOP.isGlobal()) {
11184 Callee = dyn_cast<Function>(Val: MOP.getGlobal());
11185 break;
11186 }
11187 }
11188
11189 // Never outline calls to mcount. There isn't any rule that would require
11190 // this, but the Linux kernel's "ftrace" feature depends on it.
11191 if (Callee && Callee->getName() == "\01_mcount")
11192 return outliner::InstrType::Illegal;
11193
11194 // If we don't know anything about the callee, assume it depends on the
11195 // stack layout of the caller. In that case, it's only legal to outline
11196 // as a tail-call. Explicitly list the call instructions we know about so we
11197 // don't get unexpected results with call pseudo-instructions.
11198 auto UnknownCallOutlineType = outliner::InstrType::Illegal;
11199 if (MI.getOpcode() == AArch64::BLR ||
11200 MI.getOpcode() == AArch64::BLRNoIP || MI.getOpcode() == AArch64::BL)
11201 UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
11202
11203 if (!Callee)
11204 return UnknownCallOutlineType;
11205
11206 // We have a function we have information about. Check it if it's something
11207 // can safely outline.
11208 MachineFunction *CalleeMF = MMI.getMachineFunction(F: *Callee);
11209
11210 // We don't know what's going on with the callee at all. Don't touch it.
11211 if (!CalleeMF)
11212 return UnknownCallOutlineType;
11213
11214 // Check if we know anything about the callee saves on the function. If we
11215 // don't, then don't touch it, since that implies that we haven't
11216 // computed anything about its stack frame yet.
11217 MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
11218 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
11219 MFI.getNumObjects() > 0)
11220 return UnknownCallOutlineType;
11221
11222 // At this point, we can say that CalleeMF ought to not pass anything on the
11223 // stack. Therefore, we can outline it.
11224 return outliner::InstrType::Legal;
11225 }
11226
11227 // Don't touch the link register or W30.
11228 if (MI.readsRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()) ||
11229 MI.modifiesRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()))
11230 return outliner::InstrType::Illegal;
11231
11232 // Don't outline BTI instructions, because that will prevent the outlining
11233 // site from being indirectly callable.
11234 if (hasBTISemantics(MI))
11235 return outliner::InstrType::Illegal;
11236
11237 return outliner::InstrType::Legal;
11238}
11239
11240void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
11241 for (MachineInstr &MI : MBB) {
11242 const MachineOperand *Base;
11243 TypeSize Width(0, false);
11244 int64_t Offset;
11245 bool OffsetIsScalable;
11246
11247 // Is this a load or store with an immediate offset with SP as the base?
11248 if (!MI.mayLoadOrStore() ||
11249 !getMemOperandWithOffsetWidth(LdSt: MI, BaseOp&: Base, Offset, OffsetIsScalable, Width,
11250 TRI: &RI) ||
11251 (Base->isReg() && Base->getReg() != AArch64::SP))
11252 continue;
11253
11254 // It is, so we have to fix it up.
11255 TypeSize Scale(0U, false);
11256 int64_t Dummy1, Dummy2;
11257
11258 MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(LdSt&: MI);
11259 assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
11260 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2);
11261 assert(Scale != 0 && "Unexpected opcode!");
11262 assert(!OffsetIsScalable && "Expected offset to be a byte offset");
11263
11264 // We've pushed the return address to the stack, so add 16 to the offset.
11265 // This is safe, since we already checked if it would overflow when we
11266 // checked if this instruction was legal to outline.
11267 int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedValue();
11268 StackOffsetOperand.setImm(NewImm);
11269 }
11270}
11271
11272static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
11273 const AArch64InstrInfo *TII,
11274 bool ShouldSignReturnAddr) {
11275 if (!ShouldSignReturnAddr)
11276 return;
11277
11278 BuildMI(BB&: MBB, I: MBB.begin(), MIMD: DebugLoc(), MCID: TII->get(Opcode: AArch64::PAUTH_PROLOGUE))
11279 .setMIFlag(MachineInstr::FrameSetup);
11280 TII->createPauthEpilogueInstr(MBB, DL: DebugLoc());
11281}
11282
11283void AArch64InstrInfo::buildOutlinedFrame(
11284 MachineBasicBlock &MBB, MachineFunction &MF,
11285 const outliner::OutlinedFunction &OF) const {
11286
11287 AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
11288
11289 if (OF.FrameConstructionID == MachineOutlinerTailCall)
11290 FI->setOutliningStyle("Tail Call");
11291 else if (OF.FrameConstructionID == MachineOutlinerThunk) {
11292 // For thunk outlining, rewrite the last instruction from a call to a
11293 // tail-call.
11294 MachineInstr *Call = &*--MBB.instr_end();
11295 unsigned TailOpcode;
11296 if (Call->getOpcode() == AArch64::BL) {
11297 TailOpcode = AArch64::TCRETURNdi;
11298 } else {
11299 assert(Call->getOpcode() == AArch64::BLR ||
11300 Call->getOpcode() == AArch64::BLRNoIP);
11301 TailOpcode = AArch64::TCRETURNriALL;
11302 }
11303 MachineInstr *TC = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpcode))
11304 .add(MO: Call->getOperand(i: 0))
11305 .addImm(Val: 0);
11306 MBB.insert(I: MBB.end(), MI: TC);
11307 Call->eraseFromParent();
11308
11309 FI->setOutliningStyle("Thunk");
11310 }
11311
11312 bool IsLeafFunction = true;
11313
11314 // Is there a call in the outlined range?
11315 auto IsNonTailCall = [](const MachineInstr &MI) {
11316 return MI.isCall() && !MI.isReturn();
11317 };
11318
11319 if (llvm::any_of(Range: MBB.instrs(), P: IsNonTailCall)) {
11320 // Fix up the instructions in the range, since we're going to modify the
11321 // stack.
11322
11323 // Bugzilla ID: 46767
11324 // TODO: Check if fixing up twice is safe so we can outline these.
11325 assert(OF.FrameConstructionID != MachineOutlinerDefault &&
11326 "Can only fix up stack references once");
11327 fixupPostOutline(MBB);
11328
11329 IsLeafFunction = false;
11330
11331 // LR has to be a live in so that we can save it.
11332 if (!MBB.isLiveIn(Reg: AArch64::LR))
11333 MBB.addLiveIn(PhysReg: AArch64::LR);
11334
11335 MachineBasicBlock::iterator It = MBB.begin();
11336 MachineBasicBlock::iterator Et = MBB.end();
11337
11338 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11339 OF.FrameConstructionID == MachineOutlinerThunk)
11340 Et = std::prev(x: MBB.end());
11341
11342 // There is a call in the range, so we must save LR. Save it as part of a
11343 // frame record when that gives us a smaller compact unwind encoding.
11344 if (shouldUseCompactUnwindFrameRecordForOutlinedFunction(MBB)) {
11345 // FP is saved here, so it must be live-in.
11346 if (!MBB.isLiveIn(Reg: AArch64::FP))
11347 MBB.addLiveIn(PhysReg: AArch64::FP);
11348
11349 // stp x29, x30, [sp, #-16]! (the pre-index imm is scaled by 8: -2 * 8)
11350 MachineInstr *STPXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STPXpre))
11351 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11352 .addReg(RegNo: AArch64::FP)
11353 .addReg(RegNo: AArch64::LR)
11354 .addReg(RegNo: AArch64::SP)
11355 .addImm(Val: -2);
11356 It = MBB.insert(I: It, MI: STPXpre);
11357
11358 // mov x29, sp (add x29, sp, #0), so x29 points at the frame record.
11359 MachineInstr *SetFP = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ADDXri))
11360 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11361 .addReg(RegNo: AArch64::SP)
11362 .addImm(Val: 0)
11363 .addImm(Val: 0);
11364 MBB.insertAfter(I: It, MI: SetFP);
11365
11366 // Describe the frame record with FP as the CFA. The encoder needs all
11367 // three to pick FRAME. No need to check for unwind info here: we only
11368 // get here if the function has it.
11369 CFIInstBuilder CFIBuilder(MBB, std::next(x: SetFP->getIterator()),
11370 MachineInstr::FrameSetup);
11371 CFIBuilder.buildDefCFA(Reg: AArch64::FP, Offset: 16);
11372 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -8);
11373 CFIBuilder.buildOffset(Reg: AArch64::FP, Offset: -16);
11374
11375 // ldp x29, x30, [sp], #16
11376 MachineInstr *LDPXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDPXpost))
11377 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11378 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11379 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11380 .addReg(RegNo: AArch64::SP)
11381 .addImm(Val: 2);
11382 Et = MBB.insert(I: Et, MI: LDPXpost);
11383 } else {
11384 // Insert a save before the outlined region
11385 MachineInstr *STRXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11386 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11387 .addReg(RegNo: AArch64::LR)
11388 .addReg(RegNo: AArch64::SP)
11389 .addImm(Val: -16);
11390 It = MBB.insert(I: It, MI: STRXpre);
11391
11392 if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
11393 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
11394
11395 // Add a CFI saying the stack was moved 16 B down.
11396 CFIBuilder.buildDefCFAOffset(Offset: 16);
11397
11398 // Add a CFI saying that the LR that we want to find is now 16 B higher
11399 // than before.
11400 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -16);
11401 }
11402
11403 // Insert a restore before the terminator for the function.
11404 MachineInstr *LDRXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11405 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11406 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11407 .addReg(RegNo: AArch64::SP)
11408 .addImm(Val: 16);
11409 Et = MBB.insert(I: Et, MI: LDRXpost);
11410 }
11411 }
11412
11413 auto RASignCondition = FI->getSignReturnAddressCondition();
11414 bool ShouldSignReturnAddr = AArch64FunctionInfo::shouldSignReturnAddress(
11415 Condition: RASignCondition, IsLRSpilled: !IsLeafFunction);
11416
11417 // If this is a tail call outlined function, then there's already a return.
11418 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11419 OF.FrameConstructionID == MachineOutlinerThunk) {
11420 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11421 return;
11422 }
11423
11424 // It's not a tail call, so we have to insert the return ourselves.
11425
11426 // LR has to be a live in so that we can return to it.
11427 if (!MBB.isLiveIn(Reg: AArch64::LR))
11428 MBB.addLiveIn(PhysReg: AArch64::LR);
11429
11430 MachineInstr *ret = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::RET))
11431 .addReg(RegNo: AArch64::LR);
11432 MBB.insert(I: MBB.end(), MI: ret);
11433
11434 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11435
11436 FI->setOutliningStyle("Function");
11437
11438 // Did we have to modify the stack by saving the link register?
11439 if (OF.FrameConstructionID != MachineOutlinerDefault)
11440 return;
11441
11442 // We modified the stack.
11443 // Walk over the basic block and fix up all the stack accesses.
11444 fixupPostOutline(MBB);
11445}
11446
11447MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
11448 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
11449 MachineFunction &MF, outliner::Candidate &C) const {
11450
11451 // Are we tail calling?
11452 if (C.CallConstructionID == MachineOutlinerTailCall) {
11453 // If yes, then we can just branch to the label.
11454 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::TCRETURNdi))
11455 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()))
11456 .addImm(Val: 0));
11457 return It;
11458 }
11459
11460 // Are we saving the link register?
11461 if (C.CallConstructionID == MachineOutlinerNoLRSave ||
11462 C.CallConstructionID == MachineOutlinerThunk) {
11463 // No, so just insert the call.
11464 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11465 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11466 return It;
11467 }
11468
11469 // We want to return the spot where we inserted the call.
11470 MachineBasicBlock::iterator CallPt;
11471
11472 // Instructions for saving and restoring LR around the call instruction we're
11473 // going to insert.
11474 MachineInstr *Save;
11475 MachineInstr *Restore;
11476 // Can we save to a register?
11477 if (C.CallConstructionID == MachineOutlinerRegSave) {
11478 // FIXME: This logic should be sunk into a target-specific interface so that
11479 // we don't have to recompute the register.
11480 Register Reg = findRegisterToSaveLRTo(C);
11481 assert(Reg && "No callee-saved register available?");
11482
11483 // LR has to be a live in so that we can save it.
11484 if (!MBB.isLiveIn(Reg: AArch64::LR))
11485 MBB.addLiveIn(PhysReg: AArch64::LR);
11486
11487 // Save and restore LR from Reg.
11488 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: Reg)
11489 .addReg(RegNo: AArch64::XZR)
11490 .addReg(RegNo: AArch64::LR)
11491 .addImm(Val: 0);
11492 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: AArch64::LR)
11493 .addReg(RegNo: AArch64::XZR)
11494 .addReg(RegNo: Reg)
11495 .addImm(Val: 0);
11496 } else {
11497 // We have the default case. Save and restore from SP.
11498 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11499 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11500 .addReg(RegNo: AArch64::LR)
11501 .addReg(RegNo: AArch64::SP)
11502 .addImm(Val: -16);
11503 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11504 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11505 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11506 .addReg(RegNo: AArch64::SP)
11507 .addImm(Val: 16);
11508 }
11509
11510 It = MBB.insert(I: It, MI: Save);
11511 It++;
11512
11513 // Insert the call.
11514 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11515 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11516 CallPt = It;
11517 It++;
11518
11519 It = MBB.insert(I: It, MI: Restore);
11520 return CallPt;
11521}
11522
11523bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
11524 MachineFunction &MF) const {
11525 return MF.getFunction().hasMinSize();
11526}
11527
11528void AArch64InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
11529 MachineBasicBlock::iterator Iter,
11530 DebugLoc &DL,
11531 bool AllowSideEffects) const {
11532 const MachineFunction &MF = *MBB.getParent();
11533 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
11534 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
11535
11536 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
11537 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg).addImm(Val: 0).addImm(Val: 0);
11538 } else if (STI.isSVEorStreamingSVEAvailable()) {
11539 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::DUP_ZI_D), DestReg: Reg)
11540 .addImm(Val: 0)
11541 .addImm(Val: 0);
11542 } else if (STI.isNeonAvailable()) {
11543 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVIv2d_ns), DestReg: Reg)
11544 .addImm(Val: 0);
11545 } else {
11546 // No Advanced SIMD (streaming-compatible without SVE, or +nosimd), so use
11547 // `fmov d...` instead of `movi v...`; writing `d` also clears the upper
11548 // 64 bits.
11549 assert(STI.hasFPARMv8() && "Expected FP to be available.");
11550 Register Reg64 = TRI.getSubReg(Reg, Idx: AArch64::dsub);
11551 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg: Reg64);
11552 }
11553}
11554
11555std::optional<DestSourcePair>
11556AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
11557
11558 // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
11559 // and zero immediate operands used as an alias for mov instruction.
11560 if ((MI.getOpcode() == AArch64::ORRWrs &&
11561 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11562 MI.getOperand(i: 3).getImm() == 0x0) ||
11563 (MI.getOpcode() == AArch64::ORRWrr &&
11564 MI.getOperand(i: 1).getReg() == AArch64::WZR)) {
11565 // Check that the w->w move is not a zero-extending w->x mov.
11566 if ((MI.getOperand(i: 0).getReg().isPhysical() &&
11567 MI.findRegisterDefOperandIdx(
11568 Reg: getXRegFromWReg(Reg: MI.getOperand(i: 0).getReg()),
11569 /*TRI=*/nullptr) == -1) ||
11570 (MI.getOperand(i: 0).getReg().isVirtual() &&
11571 !MI.getOperand(i: 0).getSubReg()))
11572 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11573 }
11574
11575 if (MI.getOpcode() == AArch64::ORRXrs &&
11576 MI.getOperand(i: 1).getReg() == AArch64::XZR &&
11577 MI.getOperand(i: 3).getImm() == 0x0)
11578 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11579
11580 return std::nullopt;
11581}
11582
11583std::optional<DestSourcePair>
11584AArch64InstrInfo::isCopyLikeInstrImpl(const MachineInstr &MI) const {
11585 if ((MI.getOpcode() == AArch64::ORRWrs &&
11586 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11587 MI.getOperand(i: 3).getImm() == 0x0) ||
11588 (MI.getOpcode() == AArch64::ORRWrr &&
11589 MI.getOperand(i: 1).getReg() == AArch64::WZR))
11590 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11591 return std::nullopt;
11592}
11593
11594std::optional<RegImmPair>
11595AArch64InstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
11596 int Sign = 1;
11597 int64_t Offset = 0;
11598
11599 // TODO: Handle cases where Reg is a super- or sub-register of the
11600 // destination register.
11601 const MachineOperand &Op0 = MI.getOperand(i: 0);
11602 if (!Op0.isReg() || Reg != Op0.getReg())
11603 return std::nullopt;
11604
11605 switch (MI.getOpcode()) {
11606 default:
11607 return std::nullopt;
11608 case AArch64::SUBWri:
11609 case AArch64::SUBXri:
11610 case AArch64::SUBSWri:
11611 case AArch64::SUBSXri:
11612 Sign *= -1;
11613 [[fallthrough]];
11614 case AArch64::ADDSWri:
11615 case AArch64::ADDSXri:
11616 case AArch64::ADDWri:
11617 case AArch64::ADDXri: {
11618 // TODO: Third operand can be global address (usually some string).
11619 if (!MI.getOperand(i: 0).isReg() || !MI.getOperand(i: 1).isReg() ||
11620 !MI.getOperand(i: 2).isImm())
11621 return std::nullopt;
11622 int Shift = MI.getOperand(i: 3).getImm();
11623 assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
11624 Offset = Sign * (MI.getOperand(i: 2).getImm() << Shift);
11625 }
11626 }
11627 return RegImmPair{MI.getOperand(i: 1).getReg(), Offset};
11628}
11629
11630/// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
11631/// the destination register then, if possible, describe the value in terms of
11632/// the source register.
11633static std::optional<ParamLoadedValue>
11634describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
11635 const TargetInstrInfo *TII,
11636 const TargetRegisterInfo *TRI) {
11637 auto DestSrc = TII->isCopyLikeInstr(MI);
11638 if (!DestSrc)
11639 return std::nullopt;
11640
11641 Register DestReg = DestSrc->Destination->getReg();
11642 Register SrcReg = DestSrc->Source->getReg();
11643
11644 if (!DestReg.isValid() || !SrcReg.isValid())
11645 return std::nullopt;
11646
11647 auto Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: {});
11648
11649 // If the described register is the destination, just return the source.
11650 if (DestReg == DescribedReg)
11651 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11652
11653 // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
11654 if (MI.getOpcode() == AArch64::ORRWrs &&
11655 TRI->isSuperRegister(RegA: DestReg, RegB: DescribedReg))
11656 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11657
11658 // We may need to describe the lower part of a ORRXrs move.
11659 if (MI.getOpcode() == AArch64::ORRXrs &&
11660 TRI->isSubRegister(RegA: DestReg, RegB: DescribedReg)) {
11661 Register SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: AArch64::sub_32);
11662 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcSubReg, isDef: false), Expr);
11663 }
11664
11665 assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
11666 "Unhandled ORR[XW]rs copy case");
11667
11668 return std::nullopt;
11669}
11670
11671bool AArch64InstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
11672 // Functions cannot be split to different sections on AArch64 if they have
11673 // a red zone. This is because relaxing a cross-section branch may require
11674 // incrementing the stack pointer to spill a register, which would overwrite
11675 // the red zone.
11676 if (MF.getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(u: true))
11677 return false;
11678
11679 return TargetInstrInfo::isFunctionSafeToSplit(MF);
11680}
11681
11682bool AArch64InstrInfo::isMBBSafeToSplitToCold(
11683 const MachineBasicBlock &MBB) const {
11684 // Asm Goto blocks can contain conditional branches to goto labels, which can
11685 // get moved out of range of the branch instruction.
11686 auto isAsmGoto = [](const MachineInstr &MI) {
11687 return MI.getOpcode() == AArch64::INLINEASM_BR;
11688 };
11689 if (llvm::any_of(Range: MBB, P: isAsmGoto) || MBB.isInlineAsmBrIndirectTarget())
11690 return false;
11691
11692 // Because jump tables are label-relative instead of table-relative, they all
11693 // must be in the same section or relocation fixup handling will fail.
11694
11695 // Check if MBB is a jump table target
11696 const MachineJumpTableInfo *MJTI = MBB.getParent()->getJumpTableInfo();
11697 auto containsMBB = [&MBB](const MachineJumpTableEntry &JTE) {
11698 return llvm::is_contained(Range: JTE.MBBs, Element: &MBB);
11699 };
11700 if (MJTI != nullptr && llvm::any_of(Range: MJTI->getJumpTables(), P: containsMBB))
11701 return false;
11702
11703 // Check if MBB contains a jump table lookup
11704 for (const MachineInstr &MI : MBB) {
11705 switch (MI.getOpcode()) {
11706 case TargetOpcode::G_BRJT:
11707 case AArch64::JumpTableDest32:
11708 case AArch64::JumpTableDest16:
11709 case AArch64::JumpTableDest8:
11710 return false;
11711 default:
11712 continue;
11713 }
11714 }
11715
11716 // MBB isn't a special case, so it's safe to be split to the cold section.
11717 return true;
11718}
11719
11720std::optional<ParamLoadedValue>
11721AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
11722 Register Reg) const {
11723 const MachineFunction *MF = MI.getMF();
11724 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
11725 switch (MI.getOpcode()) {
11726 case AArch64::MOVZWi:
11727 case AArch64::MOVZXi: {
11728 // MOVZWi may be used for producing zero-extended 32-bit immediates in
11729 // 64-bit parameters, so we need to consider super-registers.
11730 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
11731 return std::nullopt;
11732
11733 if (!MI.getOperand(i: 1).isImm())
11734 return std::nullopt;
11735 int64_t Immediate = MI.getOperand(i: 1).getImm();
11736 int Shift = MI.getOperand(i: 2).getImm();
11737 return ParamLoadedValue(MachineOperand::CreateImm(Val: Immediate << Shift),
11738 nullptr);
11739 }
11740 case AArch64::ORRWrs:
11741 case AArch64::ORRXrs:
11742 return describeORRLoadedValue(MI, DescribedReg: Reg, TII: this, TRI);
11743 }
11744
11745 return TargetInstrInfo::describeLoadedValue(MI, Reg);
11746}
11747
11748bool AArch64InstrInfo::isExtendLikelyToBeFolded(
11749 MachineInstr &ExtMI, MachineRegisterInfo &MRI) const {
11750 assert(ExtMI.getOpcode() == TargetOpcode::G_SEXT ||
11751 ExtMI.getOpcode() == TargetOpcode::G_ZEXT ||
11752 ExtMI.getOpcode() == TargetOpcode::G_ANYEXT);
11753
11754 // Anyexts are nops.
11755 if (ExtMI.getOpcode() == TargetOpcode::G_ANYEXT)
11756 return true;
11757
11758 Register DefReg = ExtMI.getOperand(i: 0).getReg();
11759 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
11760 return false;
11761
11762 // It's likely that a sext/zext as a G_PTR_ADD offset will be folded into an
11763 // addressing mode.
11764 auto *UserMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
11765 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
11766}
11767
11768uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
11769 return get(Opcode: Opc).TSFlags & AArch64::ElementSizeMask;
11770}
11771
11772bool AArch64InstrInfo::isPTestLikeOpcode(unsigned Opc) const {
11773 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsPTestLike;
11774}
11775
11776bool AArch64InstrInfo::isWhileOpcode(unsigned Opc) const {
11777 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsWhile;
11778}
11779
11780unsigned int
11781AArch64InstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
11782 return OptLevel >= CodeGenOptLevel::Aggressive ? 6 : 2;
11783}
11784
11785bool AArch64InstrInfo::isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
11786 unsigned Scale) const {
11787 if (Offset && Scale)
11788 return false;
11789
11790 // Check Reg + Imm
11791 if (!Scale) {
11792 // 9-bit signed offset
11793 if (isInt<9>(x: Offset))
11794 return true;
11795
11796 // 12-bit unsigned offset
11797 unsigned Shift = Log2_64(Value: NumBytes);
11798 if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
11799 // Must be a multiple of NumBytes (NumBytes is a power of 2)
11800 (Offset >> Shift) << Shift == Offset)
11801 return true;
11802 return false;
11803 }
11804
11805 // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
11806 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
11807}
11808
11809unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
11810 if (MF.getSubtarget<AArch64Subtarget>().hardenSlsBlr())
11811 return AArch64::BLRNoIP;
11812 else
11813 return AArch64::BLR;
11814}
11815
11816void AArch64InstrInfo::createPauthEpilogueInstr(MachineBasicBlock &MBB,
11817 DebugLoc DL) const {
11818 MachineBasicBlock::iterator InsertPt = MBB.getFirstTerminator();
11819 auto Builder = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: get(Opcode: AArch64::PAUTH_EPILOGUE))
11820 .setMIFlag(MachineInstr::FrameDestroy);
11821
11822 MachineFunction &MF = *MBB.getParent();
11823 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
11824 auto &AFL = *static_cast<const AArch64FrameLowering *>(
11825 MF.getSubtarget().getFrameLowering());
11826 if (AFL.getArgumentStackToRestore(MF, MBB)) {
11827 Builder.addReg(RegNo: AArch64::X17, Flags: RegState::ImplicitDefine);
11828 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11829 if (AFI->branchProtectionPAuthLR())
11830 Builder.addReg(RegNo: AArch64::X15, Flags: RegState::ImplicitDefine);
11831 return;
11832 }
11833
11834 if (AFI->branchProtectionPAuthLR() && !Subtarget.hasPAuthLR())
11835 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11836}
11837
11838MachineBasicBlock::iterator
11839AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI,
11840 Register TargetReg, bool FrameSetup) const {
11841 assert(TargetReg != AArch64::SP && "New top of stack cannot already be in SP");
11842
11843 MachineBasicBlock &MBB = *MBBI->getParent();
11844 MachineFunction &MF = *MBB.getParent();
11845 const AArch64InstrInfo *TII =
11846 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
11847 int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
11848 DebugLoc DL = MBB.findDebugLoc(MBBI);
11849
11850 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
11851 MachineBasicBlock *LoopTestMBB =
11852 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11853 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
11854 MachineBasicBlock *LoopBodyMBB =
11855 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11856 MF.insert(MBBI: MBBInsertPoint, MBB: LoopBodyMBB);
11857 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11858 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
11859 MachineInstr::MIFlag Flags =
11860 FrameSetup ? MachineInstr::FrameSetup : MachineInstr::NoFlags;
11861
11862 // LoopTest:
11863 // SUB SP, SP, #ProbeSize
11864 emitFrameOffset(MBB&: *LoopTestMBB, MBBI: LoopTestMBB->end(), DL, DestReg: AArch64::SP,
11865 SrcReg: AArch64::SP, Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII, Flag: Flags);
11866
11867 // CMP SP, TargetReg
11868 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrx64),
11869 DestReg: AArch64::XZR)
11870 .addReg(RegNo: AArch64::SP)
11871 .addReg(RegNo: TargetReg)
11872 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0))
11873 .setMIFlags(Flags);
11874
11875 // B.<Cond> LoopExit
11876 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
11877 .addImm(Val: AArch64CC::LE)
11878 .addMBB(MBB: ExitMBB)
11879 .setMIFlags(Flags);
11880
11881 // LDR XZR, [SP]
11882 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11883 .addDef(RegNo: AArch64::XZR)
11884 .addReg(RegNo: AArch64::SP)
11885 .addImm(Val: 0)
11886 .addMemOperand(MMO: MF.getMachineMemOperand(
11887 PtrInfo: MachinePointerInfo::getUnknownStack(MF),
11888 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 8,
11889 BaseAlignment: Align(8)))
11890 .setMIFlags(Flags);
11891
11892 // B loop
11893 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
11894 .addMBB(MBB: LoopTestMBB)
11895 .setMIFlags(Flags);
11896
11897 // LoopExit:
11898 // MOV SP, TargetReg
11899 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: AArch64::SP)
11900 .addReg(RegNo: TargetReg)
11901 .addImm(Val: 0)
11902 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
11903 .setMIFlags(Flags);
11904
11905 // LDR XZR, [SP]
11906 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11907 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define)
11908 .addReg(RegNo: AArch64::SP)
11909 .addImm(Val: 0)
11910 .setMIFlags(Flags);
11911
11912 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
11913 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
11914
11915 LoopTestMBB->addSuccessor(Succ: ExitMBB);
11916 LoopTestMBB->addSuccessor(Succ: LoopBodyMBB);
11917 LoopBodyMBB->addSuccessor(Succ: LoopTestMBB);
11918 MBB.addSuccessor(Succ: LoopTestMBB);
11919
11920 // Update liveins.
11921 if (MF.getRegInfo().reservedRegsFrozen())
11922 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopBodyMBB, LoopTestMBB});
11923
11924 return ExitMBB->begin();
11925}
11926
11927namespace {
11928class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
11929 MachineFunction *MF;
11930 const TargetInstrInfo *TII;
11931 const TargetRegisterInfo *TRI;
11932 MachineRegisterInfo &MRI;
11933
11934 /// The block of the loop
11935 MachineBasicBlock *LoopBB;
11936 /// The conditional branch of the loop
11937 MachineInstr *CondBranch;
11938 /// The compare instruction for loop control
11939 MachineInstr *Comp;
11940 /// The number of the operand of the loop counter value in Comp
11941 unsigned CompCounterOprNum;
11942 /// The instruction that updates the loop counter value
11943 MachineInstr *Update;
11944 /// The number of the operand of the loop counter value in Update
11945 unsigned UpdateCounterOprNum;
11946 /// The initial value of the loop counter
11947 Register Init;
11948 /// True iff Update is a predecessor of Comp
11949 bool IsUpdatePriorComp;
11950
11951 /// The normalized condition used by createTripCountGreaterCondition()
11952 SmallVector<MachineOperand, 4> Cond;
11953
11954public:
11955 AArch64PipelinerLoopInfo(MachineBasicBlock *LoopBB, MachineInstr *CondBranch,
11956 MachineInstr *Comp, unsigned CompCounterOprNum,
11957 MachineInstr *Update, unsigned UpdateCounterOprNum,
11958 Register Init, bool IsUpdatePriorComp,
11959 const SmallVectorImpl<MachineOperand> &Cond)
11960 : MF(Comp->getParent()->getParent()),
11961 TII(MF->getSubtarget().getInstrInfo()),
11962 TRI(MF->getSubtarget().getRegisterInfo()), MRI(MF->getRegInfo()),
11963 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
11964 CompCounterOprNum(CompCounterOprNum), Update(Update),
11965 UpdateCounterOprNum(UpdateCounterOprNum), Init(Init),
11966 IsUpdatePriorComp(IsUpdatePriorComp), Cond(Cond.begin(), Cond.end()) {}
11967
11968 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
11969 // Make the instructions for loop control be placed in stage 0.
11970 // The predecessors of Comp are considered by the caller.
11971 return MI == Comp;
11972 }
11973
11974 std::optional<bool> createTripCountGreaterCondition(
11975 int TC, MachineBasicBlock &MBB,
11976 SmallVectorImpl<MachineOperand> &CondParam) override {
11977 // A branch instruction will be inserted as "if (Cond) goto epilogue".
11978 // Cond is normalized for such use.
11979 // The predecessors of the branch are assumed to have already been inserted.
11980 CondParam = Cond;
11981 return {};
11982 }
11983
11984 void createRemainingIterationsGreaterCondition(
11985 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
11986 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) override;
11987
11988 void setPreheader(MachineBasicBlock *NewPreheader) override {}
11989
11990 void adjustTripCount(int TripCountAdjust) override {}
11991
11992 bool isMVEExpanderSupported() override { return true; }
11993};
11994} // namespace
11995
11996/// Clone an instruction from MI. The register of ReplaceOprNum-th operand
11997/// is replaced by ReplaceReg. The output register is newly created.
11998/// The other operands are unchanged from MI.
11999static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum,
12000 Register ReplaceReg, MachineBasicBlock &MBB,
12001 MachineBasicBlock::iterator InsertTo) {
12002 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
12003 const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
12004 MachineInstr *NewMI = MBB.getParent()->CloneMachineInstr(Orig: MI);
12005 Register Result = 0;
12006 for (unsigned I = 0; I < NewMI->getNumOperands(); ++I) {
12007 if (I == 0 && NewMI->getOperand(i: 0).getReg().isVirtual()) {
12008 Result = MRI.createVirtualRegister(
12009 RegClass: MRI.getRegClass(Reg: NewMI->getOperand(i: 0).getReg()));
12010 NewMI->getOperand(i: I).setReg(Result);
12011 } else if (I == ReplaceOprNum) {
12012 MRI.constrainRegClass(Reg: ReplaceReg, RC: TII->getRegClass(MCID: NewMI->getDesc(), OpNum: I));
12013 NewMI->getOperand(i: I).setReg(ReplaceReg);
12014 }
12015 }
12016 MBB.insert(I: InsertTo, MI: NewMI);
12017 return Result;
12018}
12019
12020void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
12021 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
12022 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) {
12023 // Create and accumulate conditions for next TC iterations.
12024 // Example:
12025 // SUBSXrr N, counter, implicit-def $nzcv # compare instruction for the last
12026 // # iteration of the kernel
12027 //
12028 // # insert the following instructions
12029 // cond = CSINCXr 0, 0, C, implicit $nzcv
12030 // counter = ADDXri counter, 1 # clone from this->Update
12031 // SUBSXrr n, counter, implicit-def $nzcv # clone from this->Comp
12032 // cond = CSINCXr cond, cond, C, implicit $nzcv
12033 // ... (repeat TC times)
12034 // SUBSXri cond, 0, implicit-def $nzcv
12035
12036 assert(CondBranch->getOpcode() == AArch64::Bcc);
12037 // CondCode to exit the loop
12038 AArch64CC::CondCode CC =
12039 (AArch64CC::CondCode)CondBranch->getOperand(i: 0).getImm();
12040 if (CondBranch->getOperand(i: 1).getMBB() == LoopBB)
12041 CC = AArch64CC::getInvertedCondCode(Code: CC);
12042
12043 // Accumulate conditions to exit the loop
12044 Register AccCond = AArch64::XZR;
12045
12046 // If CC holds, CurCond+1 is returned; otherwise CurCond is returned.
12047 auto AccumulateCond = [&](Register CurCond,
12048 AArch64CC::CondCode CC) -> Register {
12049 Register NewCond = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
12050 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::CSINCXr))
12051 .addReg(RegNo: NewCond, Flags: RegState::Define)
12052 .addReg(RegNo: CurCond)
12053 .addReg(RegNo: CurCond)
12054 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
12055 return NewCond;
12056 };
12057
12058 if (!LastStage0Insts.empty() && LastStage0Insts[Comp]->getParent() == &MBB) {
12059 // Update and Comp for I==0 are already exists in MBB
12060 // (MBB is an unrolled kernel)
12061 Register Counter;
12062 for (int I = 0; I <= TC; ++I) {
12063 Register NextCounter;
12064 if (I != 0)
12065 NextCounter =
12066 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12067
12068 AccCond = AccumulateCond(AccCond, CC);
12069
12070 if (I != TC) {
12071 if (I == 0) {
12072 if (Update != Comp && IsUpdatePriorComp) {
12073 Counter =
12074 LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12075 NextCounter = cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB,
12076 InsertTo: MBB.end());
12077 } else {
12078 // can use already calculated value
12079 NextCounter = LastStage0Insts[Update]->getOperand(i: 0).getReg();
12080 }
12081 } else if (Update != Comp) {
12082 NextCounter =
12083 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12084 }
12085 }
12086 Counter = NextCounter;
12087 }
12088 } else {
12089 Register Counter;
12090 if (LastStage0Insts.empty()) {
12091 // use initial counter value (testing if the trip count is sufficient to
12092 // be executed by pipelined code)
12093 Counter = Init;
12094 if (IsUpdatePriorComp)
12095 Counter =
12096 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12097 } else {
12098 // MBB is an epilogue block. LastStage0Insts[Comp] is in the kernel block.
12099 Counter = LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12100 }
12101
12102 for (int I = 0; I <= TC; ++I) {
12103 Register NextCounter;
12104 NextCounter =
12105 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12106 AccCond = AccumulateCond(AccCond, CC);
12107 if (I != TC && Update != Comp)
12108 NextCounter =
12109 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12110 Counter = NextCounter;
12111 }
12112 }
12113
12114 // If AccCond == 0, the remainder is greater than TC.
12115 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::SUBSXri))
12116 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define | RegState::Dead)
12117 .addReg(RegNo: AccCond)
12118 .addImm(Val: 0)
12119 .addImm(Val: 0);
12120 Cond.clear();
12121 Cond.push_back(Elt: MachineOperand::CreateImm(Val: AArch64CC::EQ));
12122}
12123
12124static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB,
12125 Register &RegMBB, Register &RegOther) {
12126 assert(Phi.getNumOperands() == 5);
12127 if (Phi.getOperand(i: 2).getMBB() == MBB) {
12128 RegMBB = Phi.getOperand(i: 1).getReg();
12129 RegOther = Phi.getOperand(i: 3).getReg();
12130 } else {
12131 assert(Phi.getOperand(4).getMBB() == MBB);
12132 RegMBB = Phi.getOperand(i: 3).getReg();
12133 RegOther = Phi.getOperand(i: 1).getReg();
12134 }
12135}
12136
12137static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB) {
12138 if (!Reg.isVirtual())
12139 return false;
12140 const MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
12141 return MRI.getDefBlock(Reg) != BB;
12142}
12143
12144/// If Reg is an induction variable, return true and set some parameters
12145static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
12146 MachineInstr *&UpdateInst,
12147 unsigned &UpdateCounterOprNum, Register &InitReg,
12148 bool &IsUpdatePriorComp) {
12149 // Example:
12150 //
12151 // Preheader:
12152 // InitReg = ...
12153 // LoopBB:
12154 // Reg0 = PHI (InitReg, Preheader), (Reg1, LoopBB)
12155 // Reg = COPY Reg0 ; COPY is ignored.
12156 // Reg1 = ADD Reg, #1; UpdateInst. Incremented by a loop invariant value.
12157 // ; Reg is the value calculated in the previous
12158 // ; iteration, so IsUpdatePriorComp == false.
12159
12160 if (LoopBB->pred_size() != 2)
12161 return false;
12162 if (!Reg.isVirtual())
12163 return false;
12164 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
12165 UpdateInst = nullptr;
12166 UpdateCounterOprNum = 0;
12167 InitReg = 0;
12168 IsUpdatePriorComp = true;
12169 Register CurReg = Reg;
12170 while (true) {
12171 MachineInstr *Def = MRI.getVRegDef(Reg: CurReg);
12172 if (Def->getParent() != LoopBB)
12173 return false;
12174 if (Def->isCopy()) {
12175 // Ignore copy instructions unless they contain subregisters
12176 if (Def->getOperand(i: 0).getSubReg() || Def->getOperand(i: 1).getSubReg())
12177 return false;
12178 CurReg = Def->getOperand(i: 1).getReg();
12179 } else if (Def->isPHI()) {
12180 if (InitReg != 0)
12181 return false;
12182 if (!UpdateInst)
12183 IsUpdatePriorComp = false;
12184 extractPhiReg(Phi: *Def, MBB: LoopBB, RegMBB&: CurReg, RegOther&: InitReg);
12185 } else {
12186 if (UpdateInst)
12187 return false;
12188 switch (Def->getOpcode()) {
12189 case AArch64::ADDSXri:
12190 case AArch64::ADDSWri:
12191 case AArch64::SUBSXri:
12192 case AArch64::SUBSWri:
12193 case AArch64::ADDXri:
12194 case AArch64::ADDWri:
12195 case AArch64::SUBXri:
12196 case AArch64::SUBWri:
12197 UpdateInst = Def;
12198 UpdateCounterOprNum = 1;
12199 break;
12200 case AArch64::ADDSXrr:
12201 case AArch64::ADDSWrr:
12202 case AArch64::SUBSXrr:
12203 case AArch64::SUBSWrr:
12204 case AArch64::ADDXrr:
12205 case AArch64::ADDWrr:
12206 case AArch64::SUBXrr:
12207 case AArch64::SUBWrr:
12208 UpdateInst = Def;
12209 if (isDefinedOutside(Reg: Def->getOperand(i: 2).getReg(), BB: LoopBB))
12210 UpdateCounterOprNum = 1;
12211 else if (isDefinedOutside(Reg: Def->getOperand(i: 1).getReg(), BB: LoopBB))
12212 UpdateCounterOprNum = 2;
12213 else
12214 return false;
12215 break;
12216 default:
12217 return false;
12218 }
12219 CurReg = Def->getOperand(i: UpdateCounterOprNum).getReg();
12220 }
12221
12222 if (!CurReg.isVirtual())
12223 return false;
12224 if (Reg == CurReg)
12225 break;
12226 }
12227
12228 if (!UpdateInst)
12229 return false;
12230
12231 return true;
12232}
12233
12234std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
12235AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
12236 // Accept loops that meet the following conditions
12237 // * The conditional branch is BCC
12238 // * The compare instruction is ADDS/SUBS/WHILEXX
12239 // * One operand of the compare is an induction variable and the other is a
12240 // loop invariant value
12241 // * The induction variable is incremented/decremented by a single instruction
12242 // * Does not contain CALL or instructions which have unmodeled side effects
12243
12244 for (MachineInstr &MI : *LoopBB)
12245 if (MI.isCall() || MI.hasUnmodeledSideEffects())
12246 // This instruction may use NZCV, which interferes with the instruction to
12247 // be inserted for loop control.
12248 return nullptr;
12249
12250 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
12251 SmallVector<MachineOperand, 4> Cond;
12252 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond))
12253 return nullptr;
12254
12255 // Infinite loops are not supported
12256 if (TBB == LoopBB && FBB == LoopBB)
12257 return nullptr;
12258
12259 // Must be conditional branch
12260 if (TBB != LoopBB && FBB == nullptr)
12261 return nullptr;
12262
12263 assert((TBB == LoopBB || FBB == LoopBB) &&
12264 "The Loop must be a single-basic-block loop");
12265
12266 MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
12267 const TargetRegisterInfo &TRI = getRegisterInfo();
12268
12269 if (CondBranch->getOpcode() != AArch64::Bcc)
12270 return nullptr;
12271
12272 // Normalization for createTripCountGreaterCondition()
12273 if (TBB == LoopBB)
12274 reverseBranchCondition(Cond);
12275
12276 MachineInstr *Comp = nullptr;
12277 unsigned CompCounterOprNum = 0;
12278 for (MachineInstr &MI : reverse(C&: *LoopBB)) {
12279 if (MI.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
12280 // Guarantee that the compare is SUBS/ADDS/WHILEXX and that one of the
12281 // operands is a loop invariant value
12282
12283 switch (MI.getOpcode()) {
12284 case AArch64::SUBSXri:
12285 case AArch64::SUBSWri:
12286 case AArch64::ADDSXri:
12287 case AArch64::ADDSWri:
12288 Comp = &MI;
12289 CompCounterOprNum = 1;
12290 break;
12291 case AArch64::ADDSWrr:
12292 case AArch64::ADDSXrr:
12293 case AArch64::SUBSWrr:
12294 case AArch64::SUBSXrr:
12295 Comp = &MI;
12296 break;
12297 default:
12298 if (isWhileOpcode(Opc: MI.getOpcode())) {
12299 Comp = &MI;
12300 break;
12301 }
12302 return nullptr;
12303 }
12304
12305 if (CompCounterOprNum == 0) {
12306 if (isDefinedOutside(Reg: Comp->getOperand(i: 1).getReg(), BB: LoopBB))
12307 CompCounterOprNum = 2;
12308 else if (isDefinedOutside(Reg: Comp->getOperand(i: 2).getReg(), BB: LoopBB))
12309 CompCounterOprNum = 1;
12310 else
12311 return nullptr;
12312 }
12313 break;
12314 }
12315 }
12316 if (!Comp)
12317 return nullptr;
12318
12319 MachineInstr *Update = nullptr;
12320 Register Init;
12321 bool IsUpdatePriorComp;
12322 unsigned UpdateCounterOprNum;
12323 if (!getIndVarInfo(Reg: Comp->getOperand(i: CompCounterOprNum).getReg(), LoopBB,
12324 UpdateInst&: Update, UpdateCounterOprNum, InitReg&: Init, IsUpdatePriorComp))
12325 return nullptr;
12326
12327 return std::make_unique<AArch64PipelinerLoopInfo>(
12328 args&: LoopBB, args&: CondBranch, args&: Comp, args&: CompCounterOprNum, args&: Update, args&: UpdateCounterOprNum,
12329 args&: Init, args&: IsUpdatePriorComp, args&: Cond);
12330}
12331
12332/// verifyInstruction - Perform target specific instruction verification.
12333bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI,
12334 StringRef &ErrInfo) const {
12335 // Verify that immediate offsets on load/store instructions are within range.
12336 // Stack objects with an FI operand are excluded as they can be fixed up
12337 // during PEI.
12338 TypeSize Scale(0U, false), Width(0U, false);
12339 int64_t MinOffset, MaxOffset;
12340 if (getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
12341 unsigned ImmIdx = getLoadStoreImmIdx(Opc: MI.getOpcode());
12342 if (MI.getOperand(i: ImmIdx).isImm() && !MI.getOperand(i: ImmIdx - 1).isFI()) {
12343 int64_t Imm = MI.getOperand(i: ImmIdx).getImm();
12344 if (Imm < MinOffset || Imm > MaxOffset) {
12345 ErrInfo = "Unexpected immediate on load/store instruction";
12346 return false;
12347 }
12348 }
12349 }
12350
12351 const MCInstrDesc &MCID = MI.getDesc();
12352 for (unsigned Op = 0; Op < MCID.getNumOperands(); Op++) {
12353 const MachineOperand &MO = MI.getOperand(i: Op);
12354 switch (MCID.operands()[Op].OperandType) {
12355 case AArch64::OPERAND_IMPLICIT_IMM_0:
12356 if (!MO.isImm() || MO.getImm() != 0) {
12357 ErrInfo = "OPERAND_IMPLICIT_IMM_0 should be 0";
12358 return false;
12359 }
12360 break;
12361 case AArch64::OPERAND_SHIFT_MSL:
12362 if (!MO.isImm() ||
12363 AArch64_AM::getShiftType(Imm: MO.getImm()) != AArch64_AM::MSL ||
12364 (AArch64_AM::getShiftValue(Imm: MO.getImm()) != 8 &&
12365 AArch64_AM::getShiftValue(Imm: MO.getImm()) != 16)) {
12366 ErrInfo = "OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
12367 return false;
12368 }
12369 break;
12370 case AArch64::OPERAND_IMM_UINT1:
12371 if (!MO.isImm() || (MO.getImm() != 0 && MO.getImm() != 1)) {
12372 ErrInfo = "OPERAND_IMM_UINT1 should be 0 or 1";
12373 return false;
12374 }
12375 break;
12376 case AArch64::OPERAND_IMM_UINT4plus1:
12377 if (!MO.isImm() || MO.getImm() <= 0 || MO.getImm() > 16) {
12378 ErrInfo = "OPERAND_IMM_UINT4plus1 should be in the range 1 to 16";
12379 return false;
12380 }
12381 break;
12382 case AArch64::OPERAND_IMM_UINT5:
12383 if (!MO.isImm() || !isUInt<5>(x: MO.getImm())) {
12384 ErrInfo = "OPERAND_IMM_UINT5 should be in the range 0 to 31";
12385 return false;
12386 }
12387 break;
12388 case AArch64::OPERAND_IMM_UINT8:
12389 if (!MO.isImm() || !isUInt<8>(x: MO.getImm())) {
12390 ErrInfo = "OPERAND_IMM_UINT8 should be in the range 0 to 255";
12391 return false;
12392 }
12393 break;
12394 default:
12395 break;
12396 }
12397 }
12398 return true;
12399}
12400
12401#define GET_INSTRINFO_HELPERS
12402#define GET_INSTRMAP_INFO
12403#include "AArch64GenInstrInfo.inc"
12404