1//===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64PointerAuth.h"
17#include "AArch64Subtarget.h"
18#include "MCTargetDesc/AArch64AddressingModes.h"
19#include "MCTargetDesc/AArch64MCLFIRewriter.h"
20#include "MCTargetDesc/AArch64MCTargetDesc.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/CodeGen/CFIInstBuilder.h"
28#include "llvm/CodeGen/LivePhysRegs.h"
29#include "llvm/CodeGen/LiveRegUnits.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineCombinerPattern.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstr.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/RegisterScavenging.h"
41#include "llvm/CodeGen/StackMaps.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/CodeGen/TargetSubtargetInfo.h"
44#include "llvm/IR/DebugInfoMetadata.h"
45#include "llvm/IR/DebugLoc.h"
46#include "llvm/IR/GlobalValue.h"
47#include "llvm/IR/Module.h"
48#include "llvm/MC/MCAsmInfo.h"
49#include "llvm/MC/MCInst.h"
50#include "llvm/MC/MCInstBuilder.h"
51#include "llvm/MC/MCInstrDesc.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CodeGen.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/LEB128.h"
56#include "llvm/Support/MathExtras.h"
57#include "llvm/Target/TargetMachine.h"
58#include "llvm/Target/TargetOptions.h"
59#include <cassert>
60#include <cstdint>
61#include <iterator>
62#include <utility>
63
64using namespace llvm;
65
66#define GET_INSTRINFO_CTOR_DTOR
67#include "AArch64GenInstrInfo.inc"
68
69#define DEBUG_TYPE "AArch64InstrInfo"
70
71STATISTIC(NumCopyInstrs, "Number of COPY instructions expanded");
72STATISTIC(NumZCRegMoveInstrsGPR, "Number of zero-cycle GPR register move "
73 "instructions expanded from canonical COPY");
74STATISTIC(NumZCRegMoveInstrsFPR, "Number of zero-cycle FPR register move "
75 "instructions expanded from canonical COPY");
76STATISTIC(NumZCZeroingInstrsGPR, "Number of zero-cycle GPR zeroing "
77 "instructions expanded from canonical COPY");
78// NumZCZeroingInstrsFPR is counted at AArch64AsmPrinter
79
80AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
81 : AArch64GenInstrInfo(STI, RI, AArch64::ADJCALLSTACKDOWN,
82 AArch64::ADJCALLSTACKUP, AArch64::CATCHRET),
83 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
84
85/// Return the maximum number of bytes of code the specified instruction may be
86/// after LFI rewriting. If the instruction is not rewritten, std::nullopt is
87/// returned (use default sizing).
88///
89/// NOTE: the size estimates here must be kept in sync with the rewrites in
90/// AArch64MCLFIRewriter.cpp. Sizes may be overestimates of the rewritten
91/// instruction sequences.
92static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
93 switch (MI.getOpcode()) {
94 case AArch64::SVC:
95 // SVC expands to 4 instructions.
96 return 16;
97 case AArch64::BR:
98 case AArch64::BLR:
99 // Indirect branches/calls expand to 2 instructions (guard + br/blr).
100 return 8;
101 case AArch64::RET:
102 // RET through another register expands to 2 instructions (guard + ret).
103 if (MI.getOperand(i: 0).getReg() != AArch64::LR)
104 return 8;
105 return std::nullopt;
106 case AArch64::RETAA:
107 case AArch64::RETAB:
108 // Authenticated returns expand to 3 instructions (authenticate + guard +
109 // ret). Any deferred LR guard is discarded, so the flush allowance added
110 // by getInstSizeInBytes is an overestimate here.
111 return 12;
112 case AArch64::BRAA:
113 case AArch64::BRAAZ:
114 case AArch64::BRAB:
115 case AArch64::BRABZ:
116 case AArch64::BLRAA:
117 case AArch64::BLRAAZ:
118 case AArch64::BLRAB:
119 case AArch64::BLRABZ:
120 // Authenticated branches/calls expand to 3 instructions (authenticate +
121 // guard + branch).
122 return 12;
123 case AArch64::BRA:
124 case AArch64::BLRA:
125 // Up to 2 instructions to materialize the discriminator (mov + movk),
126 // followed by an authenticated branch/call that expands to 3 instructions
127 // (authenticate + guard + branch).
128 return 20;
129 case AArch64::TLSDESC_CALLSEQ:
130 // adrp + ldr + add + blr, where the ldr and blr each gain a guard.
131 return 24;
132 case AArch64::TLSDESC_AUTH_CALLSEQ:
133 // adrp + ldr + add + blraa, where the ldr gains a guard and the blraa
134 // expands to 3 instructions (authenticate + guard + blr).
135 return 28;
136 case AArch64::AUTIASP:
137 case AArch64::AUTIBSP:
138 case AArch64::AUTIAZ:
139 case AArch64::AUTIBZ:
140 case AArch64::XPACLRI:
141 // Authenticating LR expands to the instruction plus a deferred LR guard.
142 return 8;
143 case AArch64::SYSxt:
144 // VA-based DC/IC ops (op1=3, Cn=7, op2=1) expand to 2 instructions.
145 if (MI.getOperand(i: 0).getImm() == 3 && MI.getOperand(i: 1).getImm() == 7 &&
146 MI.getOperand(i: 3).getImm() == 1)
147 return 8;
148 return std::nullopt;
149 default:
150 break;
151 }
152
153 // Detect instructions that explicitly define SP or LR.
154 bool ModifiesLR = false;
155 bool ModifiesSP = false;
156 for (const MachineOperand &MO : MI.defs()) {
157 if (!MO.isReg())
158 continue;
159 if (MO.getReg() == AArch64::LR)
160 ModifiesLR = true;
161 else if (MO.getReg() == AArch64::SP)
162 ModifiesSP = true;
163 }
164
165 // Memory accesses expand to a base-register guard plus the rewritten access
166 // (8 bytes), with an extra base-register update for pre/post-index forms (12
167 // bytes total). If the access also defines LR, an LR mask is appended (+4
168 // bytes). Depending on additional optimizations that the rewriter performs,
169 // this may be an overestimate.
170 if (MI.mayLoadOrStore()) {
171 unsigned Size = isLFIPrePostMemAccess(Opcode: MI.getOpcode()) ? 12 : 8;
172 if (ModifiesLR)
173 Size += 4;
174 return Size;
175 }
176
177 // Non memory operations that modify LR or SP expand to 2 instructions.
178 if (ModifiesSP || ModifiesLR)
179 return 8;
180
181 // Default case: instructions that don't cause expansion.
182 // - TP accesses in LFI are a single load/store, so no expansion.
183 // - All remaining instructions are not rewritten.
184 return std::nullopt;
185}
186
187/// GetInstSize - Return the number of bytes of code the specified
188/// instruction may be. This returns the maximum number of bytes.
189unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
190 const MCInstrDesc &Desc = MI.getDesc();
191 if (!Desc.isPseudo() && !Subtarget.isLFI()) {
192 assert(Desc.getSize() == 4 && "Unexpected instruction size");
193 return 4;
194 }
195
196 unsigned NumBytes = getInstSizeInBytesNoGuard(MI);
197
198 // The LFI rewriter may flush a deferred LR guard (1 instruction) before any
199 // control-flow instruction. Bundles are skipped because their size is the
200 // sum of the bundled instructions, which already include this.
201 if (Subtarget.isLFI() && !MI.isBundle() &&
202 (MI.isCall() || MI.isBranch() || MI.isReturn()))
203 NumBytes += 4;
204
205 return NumBytes;
206}
207
208unsigned
209AArch64InstrInfo::getInstSizeInBytesNoGuard(const MachineInstr &MI) const {
210 const MCInstrDesc &Desc = MI.getDesc();
211 const MachineBasicBlock &MBB = *MI.getParent();
212 const MachineFunction *MF = MBB.getParent();
213 const Function &F = MF->getFunction();
214 const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
215
216 {
217 auto Op = MI.getOpcode();
218 if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
219 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(), MAI);
220 }
221
222 // Meta-instructions emit no code.
223 if (MI.isMetaInstruction())
224 return 0;
225
226 // FIXME: We currently only handle pseudoinstructions that don't get expanded
227 // before the assembly printer.
228 unsigned NumBytes = 0;
229
230 // LFI rewriter expansions that supersede normal sizing.
231 const auto &STI = MF->getSubtarget<AArch64Subtarget>();
232 if (STI.isLFI())
233 if (auto Size = getLFIInstSizeInBytes(MI))
234 return *Size;
235
236 if (!MI.isBundle() && isTailCallReturnInst(MI)) {
237 NumBytes = Desc.getSize() ? Desc.getSize() : 4;
238
239 if (STI.isLFI()) {
240 unsigned Opc = MI.getOpcode();
241 if (Opc == AArch64::AUTH_TCRETURN || Opc == AArch64::AUTH_TCRETURN_BTI) {
242 // The braa/brab expands to 3 instructions (authenticate + guard + br).
243 NumBytes += 8;
244 } else if (MI.getOperand(i: 0).isReg()) {
245 // Indirect tail calls expand to 2 instructions (guard + br).
246 NumBytes += 4;
247 }
248 }
249
250 const auto *MFI = MF->getInfo<AArch64FunctionInfo>();
251 if (!MFI->shouldSignReturnAddress(MF: *MF))
252 return NumBytes;
253
254 auto Method = STI.getAuthenticatedLRCheckMethod(MF: *MF);
255 NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
256 return NumBytes;
257 }
258
259 // Size should be preferably set in
260 // llvm/lib/Target/AArch64/AArch64InstrInfo.td (default case).
261 // Specific cases handle instructions of variable sizes
262 switch (Desc.getOpcode()) {
263 default:
264 if (Desc.getSize())
265 return Desc.getSize();
266
267 // Anything not explicitly designated otherwise (i.e. pseudo-instructions
268 // with fixed constant size but not specified in .td file) is a normal
269 // 4-byte insn.
270 NumBytes = 4;
271 break;
272 case TargetOpcode::STACKMAP:
273 // The upper bound for a stackmap intrinsic is the full length of its shadow
274 NumBytes = StackMapOpers(&MI).getNumPatchBytes();
275 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
276 break;
277 case TargetOpcode::PATCHPOINT:
278 // The size of the patchpoint intrinsic is the number of bytes requested
279 NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
280 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
281 break;
282 case TargetOpcode::STATEPOINT:
283 NumBytes = StatepointOpers(&MI).getNumPatchBytes();
284 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
285 // No patch bytes means a normal call inst is emitted
286 if (NumBytes == 0)
287 NumBytes = 4;
288 break;
289 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
290 // If `patchable-function-entry` is set, PATCHABLE_FUNCTION_ENTER
291 // instructions are expanded to the specified number of NOPs. Otherwise,
292 // they are expanded to 36-byte XRay sleds.
293 NumBytes =
294 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry", Default: 9) * 4;
295 break;
296 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
297 case TargetOpcode::PATCHABLE_TAIL_CALL:
298 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
299 // An XRay sled can be 4 bytes of alignment plus a 32-byte block.
300 NumBytes = 36;
301 break;
302 case TargetOpcode::PATCHABLE_EVENT_CALL:
303 // EVENT_CALL XRay sleds are exactly 6 instructions long (no alignment).
304 NumBytes = 24;
305 break;
306
307 case AArch64::SPACE:
308 NumBytes = MI.getOperand(i: 1).getImm();
309 break;
310 case AArch64::MOVaddr:
311 case AArch64::MOVaddrJT:
312 case AArch64::MOVaddrCP:
313 case AArch64::MOVaddrBA:
314 case AArch64::MOVaddrTLS:
315 case AArch64::MOVaddrEXT: {
316 // Use the same logic as the pseudo expansion to count instructions.
317 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
318 AArch64_IMM::expandMOVAddr(Opcode: Desc.getOpcode(),
319 TargetFlags: MI.getOperand(i: 1).getTargetFlags(),
320 IsTargetMachO: Subtarget.isTargetMachO(), Insn);
321 NumBytes = Insn.size() * 4;
322 break;
323 }
324
325 case AArch64::MOVi32imm:
326 case AArch64::MOVi64imm: {
327 // Use the same logic as the pseudo expansion to count instructions.
328 unsigned BitSize = Desc.getOpcode() == AArch64::MOVi32imm ? 32 : 64;
329 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
330 AArch64_IMM::expandMOVImm(Imm: MI.getOperand(i: 1).getImm(), BitSize, Insn);
331 NumBytes = Insn.size() * 4;
332 break;
333 }
334
335 case TargetOpcode::BUNDLE:
336 NumBytes = getInstBundleSize(MI);
337 break;
338 }
339
340 return NumBytes;
341}
342
343static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
344 SmallVectorImpl<MachineOperand> &Cond) {
345 // Block ends with fall-through condbranch.
346 switch (LastInst->getOpcode()) {
347 default:
348 llvm_unreachable("Unknown branch instruction?");
349 case AArch64::Bcc:
350 Target = LastInst->getOperand(i: 1).getMBB();
351 Cond.push_back(Elt: LastInst->getOperand(i: 0));
352 break;
353 case AArch64::CBZW:
354 case AArch64::CBZX:
355 case AArch64::CBNZW:
356 case AArch64::CBNZX:
357 Target = LastInst->getOperand(i: 1).getMBB();
358 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
359 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
360 Cond.push_back(Elt: LastInst->getOperand(i: 0));
361 break;
362 case AArch64::TBZW:
363 case AArch64::TBZX:
364 case AArch64::TBNZW:
365 case AArch64::TBNZX:
366 Target = LastInst->getOperand(i: 2).getMBB();
367 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
368 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
369 Cond.push_back(Elt: LastInst->getOperand(i: 0));
370 Cond.push_back(Elt: LastInst->getOperand(i: 1));
371 break;
372 case AArch64::CBWPri:
373 case AArch64::CBXPri:
374 case AArch64::CBWPrr:
375 case AArch64::CBXPrr:
376 Target = LastInst->getOperand(i: 3).getMBB();
377 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
378 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
379 Cond.push_back(Elt: LastInst->getOperand(i: 0));
380 Cond.push_back(Elt: LastInst->getOperand(i: 1));
381 Cond.push_back(Elt: LastInst->getOperand(i: 2));
382 break;
383 case AArch64::CBBAssertExt:
384 case AArch64::CBHAssertExt:
385 Target = LastInst->getOperand(i: 3).getMBB();
386 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1)); // -1
387 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode())); // Opc
388 Cond.push_back(Elt: LastInst->getOperand(i: 0)); // Cond
389 Cond.push_back(Elt: LastInst->getOperand(i: 1)); // Op0
390 Cond.push_back(Elt: LastInst->getOperand(i: 2)); // Op1
391 Cond.push_back(Elt: LastInst->getOperand(i: 4)); // Ext0
392 Cond.push_back(Elt: LastInst->getOperand(i: 5)); // Ext1
393 break;
394 }
395}
396
397static unsigned getBranchDisplacementBits(const AArch64Options &CLOpts,
398 unsigned Opc) {
399 switch (Opc) {
400 default:
401 llvm_unreachable("unexpected opcode!");
402 case AArch64::B:
403 return CLOpts.b_offset_bits;
404 case AArch64::TBNZW:
405 case AArch64::TBZW:
406 case AArch64::TBNZX:
407 case AArch64::TBZX:
408 return CLOpts.tbz_offset_bits;
409 case AArch64::CBNZW:
410 case AArch64::CBZW:
411 case AArch64::CBNZX:
412 case AArch64::CBZX:
413 return CLOpts.cbz_offset_bits;
414 case AArch64::Bcc:
415 return CLOpts.bcc_offset_bits;
416 case AArch64::CBWPri:
417 case AArch64::CBXPri:
418 case AArch64::CBBAssertExt:
419 case AArch64::CBHAssertExt:
420 case AArch64::CBWPrr:
421 case AArch64::CBXPrr:
422 return CLOpts.cb_offset_bits;
423 }
424}
425
426bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
427 int64_t BrOffset) const {
428 unsigned Bits = getBranchDisplacementBits(CLOpts: Subtarget.getCLOpts(), Opc: BranchOp);
429 assert(Bits >= 3 && "max branch displacement must be enough to jump"
430 "over conditional branch expansion");
431 return isIntN(N: Bits, x: BrOffset / 4);
432}
433
434MachineBasicBlock *
435AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
436 switch (MI.getOpcode()) {
437 default:
438 llvm_unreachable("unexpected opcode!");
439 case AArch64::B:
440 return MI.getOperand(i: 0).getMBB();
441 case AArch64::TBZW:
442 case AArch64::TBNZW:
443 case AArch64::TBZX:
444 case AArch64::TBNZX:
445 return MI.getOperand(i: 2).getMBB();
446 case AArch64::CBZW:
447 case AArch64::CBNZW:
448 case AArch64::CBZX:
449 case AArch64::CBNZX:
450 case AArch64::Bcc:
451 return MI.getOperand(i: 1).getMBB();
452 case AArch64::CBWPri:
453 case AArch64::CBXPri:
454 case AArch64::CBBAssertExt:
455 case AArch64::CBHAssertExt:
456 case AArch64::CBWPrr:
457 case AArch64::CBXPrr:
458 return MI.getOperand(i: 3).getMBB();
459 }
460}
461
462void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
463 MachineBasicBlock &NewDestBB,
464 MachineBasicBlock &RestoreBB,
465 const DebugLoc &DL,
466 int64_t BrOffset,
467 RegScavenger *RS) const {
468 assert(RS && "RegScavenger required for long branching");
469 assert(MBB.empty() &&
470 "new block should be inserted for expanding unconditional branch");
471 assert(MBB.pred_size() == 1);
472 assert(RestoreBB.empty() &&
473 "restore block should be inserted for restoring clobbered registers");
474
475 auto buildIndirectBranch = [&](Register Reg, MachineBasicBlock &DestBB) {
476 // Offsets outside of the signed 33-bit range are not supported for ADRP +
477 // ADD.
478 if (!isInt<33>(x: BrOffset))
479 report_fatal_error(
480 reason: "Branch offsets outside of the signed 33-bit range not supported");
481
482 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
483 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGE);
484 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: Reg)
485 .addReg(RegNo: Reg)
486 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
487 .addImm(Val: 0);
488 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::BR)).addReg(RegNo: Reg);
489 };
490
491 RS->enterBasicBlockEnd(MBB);
492 // If X16 is unused, we can rely on the linker to insert a range extension
493 // thunk if NewDestBB is out of range of a single B instruction.
494 constexpr Register Reg = AArch64::X16;
495 if (!RS->isRegUsed(Reg)) {
496 insertUnconditionalBranch(MBB, DestBB: &NewDestBB, DL);
497 RS->setRegUsed(Reg);
498 return;
499 }
500
501 // In a cold block without BTI, insert the indirect branch if a register is
502 // free. Skip this if BTI is enabled to avoid inserting a BTI at the target,
503 // prioritizing a dynamic cost in cold code over a static cost in hot code.
504 AArch64FunctionInfo *AFI = MBB.getParent()->getInfo<AArch64FunctionInfo>();
505 bool HasBTI = AFI && AFI->branchTargetEnforcement();
506 if (MBB.getSectionID() == MBBSectionID::ColdSectionID && !HasBTI) {
507 Register Scavenged = RS->FindUnusedReg(RC: &AArch64::GPR64RegClass);
508 if (Scavenged.isValid()) {
509 buildIndirectBranch(Scavenged, NewDestBB);
510 RS->setRegUsed(Reg: Scavenged);
511 return;
512 }
513 }
514
515 // Note: Spilling X16 briefly moves the stack pointer, making it incompatible
516 // with red zones.
517 if (!AFI || AFI->hasRedZone().value_or(u: true))
518 report_fatal_error(
519 reason: "Unable to insert indirect branch inside function that has red zone");
520
521 // Otherwise, spill X16 and defer range extension to the linker.
522 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::STRXpre))
523 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
524 .addReg(RegNo: Reg)
525 .addReg(RegNo: AArch64::SP)
526 .addImm(Val: -16);
527
528 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: &RestoreBB);
529
530 BuildMI(BB&: RestoreBB, I: RestoreBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::LDRXpost))
531 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
532 .addReg(RegNo: Reg, Flags: RegState::Define)
533 .addReg(RegNo: AArch64::SP)
534 .addImm(Val: 16);
535}
536
537// Branch analysis.
538bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
539 MachineBasicBlock *&TBB,
540 MachineBasicBlock *&FBB,
541 SmallVectorImpl<MachineOperand> &Cond,
542 bool AllowModify) const {
543 // If the block has no terminators, it just falls into the block after it.
544 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
545 if (I == MBB.end())
546 return false;
547
548 // Skip over SpeculationBarrierEndBB terminators
549 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
550 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
551 --I;
552 }
553
554 if (!isUnpredicatedTerminator(MI: *I))
555 return false;
556
557 // Get the last instruction in the block.
558 MachineInstr *LastInst = &*I;
559
560 // If there is only one terminator instruction, process it.
561 unsigned LastOpc = LastInst->getOpcode();
562 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
563 if (isUncondBranchOpcode(Opc: LastOpc)) {
564 TBB = LastInst->getOperand(i: 0).getMBB();
565 return false;
566 }
567 if (isCondBranchOpcode(Opc: LastOpc)) {
568 // Block ends with fall-through condbranch.
569 parseCondBranch(LastInst, Target&: TBB, Cond);
570 return false;
571 }
572 return true; // Can't handle indirect branch.
573 }
574
575 // Get the instruction before it if it is a terminator.
576 MachineInstr *SecondLastInst = &*I;
577 unsigned SecondLastOpc = SecondLastInst->getOpcode();
578
579 // If AllowModify is true and the block ends with two or more unconditional
580 // branches, delete all but the first unconditional branch.
581 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc)) {
582 while (isUncondBranchOpcode(Opc: SecondLastOpc)) {
583 LastInst->eraseFromParent();
584 LastInst = SecondLastInst;
585 LastOpc = LastInst->getOpcode();
586 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
587 // Return now the only terminator is an unconditional branch.
588 TBB = LastInst->getOperand(i: 0).getMBB();
589 return false;
590 }
591 SecondLastInst = &*I;
592 SecondLastOpc = SecondLastInst->getOpcode();
593 }
594 }
595
596 // If we're allowed to modify and the block ends in a unconditional branch
597 // which could simply fallthrough, remove the branch. (Note: This case only
598 // matters when we can't understand the whole sequence, otherwise it's also
599 // handled by BranchFolding.cpp.)
600 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc) &&
601 MBB.isLayoutSuccessor(MBB: getBranchDestBlock(MI: *LastInst))) {
602 LastInst->eraseFromParent();
603 LastInst = SecondLastInst;
604 LastOpc = LastInst->getOpcode();
605 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
606 assert(!isUncondBranchOpcode(LastOpc) &&
607 "unreachable unconditional branches removed above");
608
609 if (isCondBranchOpcode(Opc: LastOpc)) {
610 // Block ends with fall-through condbranch.
611 parseCondBranch(LastInst, Target&: TBB, Cond);
612 return false;
613 }
614 return true; // Can't handle indirect branch.
615 }
616 SecondLastInst = &*I;
617 SecondLastOpc = SecondLastInst->getOpcode();
618 }
619
620 // If there are three terminators, we don't know what sort of block this is.
621 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(MI: *--I))
622 return true;
623
624 // If the block ends with a B and a Bcc, handle it.
625 if (isCondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
626 parseCondBranch(LastInst: SecondLastInst, Target&: TBB, Cond);
627 FBB = LastInst->getOperand(i: 0).getMBB();
628 return false;
629 }
630
631 // If the block ends with two unconditional branches, handle it. The second
632 // one is not executed, so remove it.
633 if (isUncondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
634 TBB = SecondLastInst->getOperand(i: 0).getMBB();
635 I = LastInst;
636 if (AllowModify)
637 I->eraseFromParent();
638 return false;
639 }
640
641 // ...likewise if it ends with an indirect branch followed by an unconditional
642 // branch.
643 if (isIndirectBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
644 I = LastInst;
645 if (AllowModify)
646 I->eraseFromParent();
647 return true;
648 }
649
650 // Otherwise, can't handle this.
651 return true;
652}
653
654bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
655 MachineBranchPredicate &MBP,
656 bool AllowModify) const {
657 // Use analyzeBranch to validate the branch pattern.
658 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
659 SmallVector<MachineOperand, 4> Cond;
660 if (analyzeBranch(MBB, TBB, FBB, Cond, AllowModify))
661 return true;
662
663 // analyzeBranch returns success with empty Cond for unconditional branches.
664 if (Cond.empty())
665 return true;
666
667 MBP.TrueDest = TBB;
668 assert(MBP.TrueDest && "expected!");
669 MBP.FalseDest = FBB ? FBB : MBB.getNextNode();
670
671 MBP.ConditionDef = nullptr;
672 MBP.SingleUseCondition = false;
673
674 // Find the conditional branch. After analyzeBranch succeeds with non-empty
675 // Cond, there's exactly one conditional branch - either last (fallthrough)
676 // or second-to-last (followed by unconditional B).
677 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
678 if (I == MBB.end())
679 return true;
680
681 if (isUncondBranchOpcode(Opc: I->getOpcode())) {
682 if (I == MBB.begin())
683 return true;
684 --I;
685 }
686
687 MachineInstr *CondBranch = &*I;
688 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
689
690 switch (CondBranch->getOpcode()) {
691 default:
692 return true;
693
694 case AArch64::Bcc:
695 // Bcc takes the NZCV flag as the operand to branch on, walk up the
696 // instruction stream to find the last instruction to define NZCV.
697 for (MachineInstr &MI : llvm::drop_begin(RangeOrContainer: llvm::reverse(C&: MBB))) {
698 if (MI.modifiesRegister(Reg: AArch64::NZCV, /*TRI=*/nullptr)) {
699 MBP.ConditionDef = &MI;
700 break;
701 }
702 }
703 return false;
704
705 case AArch64::CBZW:
706 case AArch64::CBZX:
707 case AArch64::CBNZW:
708 case AArch64::CBNZX: {
709 MBP.LHS = CondBranch->getOperand(i: 0);
710 MBP.RHS = MachineOperand::CreateImm(Val: 0);
711 unsigned Opc = CondBranch->getOpcode();
712 MBP.Predicate = (Opc == AArch64::CBNZX || Opc == AArch64::CBNZW)
713 ? MachineBranchPredicate::PRED_NE
714 : MachineBranchPredicate::PRED_EQ;
715 Register CondReg = MBP.LHS.getReg();
716 if (CondReg.isVirtual())
717 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
718 return false;
719 }
720
721 case AArch64::TBZW:
722 case AArch64::TBZX:
723 case AArch64::TBNZW:
724 case AArch64::TBNZX: {
725 Register CondReg = CondBranch->getOperand(i: 0).getReg();
726 if (CondReg.isVirtual())
727 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
728 return false;
729 }
730 }
731}
732
733bool AArch64InstrInfo::reverseBranchCondition(
734 SmallVectorImpl<MachineOperand> &Cond) const {
735 if (Cond[0].getImm() != -1) {
736 // Regular Bcc
737 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
738 Cond[0].setImm(AArch64CC::getInvertedCondCode(Code: CC));
739 } else {
740 // Folded compare-and-branch
741 switch (Cond[1].getImm()) {
742 default:
743 llvm_unreachable("Unknown conditional branch!");
744 case AArch64::CBZW:
745 Cond[1].setImm(AArch64::CBNZW);
746 break;
747 case AArch64::CBNZW:
748 Cond[1].setImm(AArch64::CBZW);
749 break;
750 case AArch64::CBZX:
751 Cond[1].setImm(AArch64::CBNZX);
752 break;
753 case AArch64::CBNZX:
754 Cond[1].setImm(AArch64::CBZX);
755 break;
756 case AArch64::TBZW:
757 Cond[1].setImm(AArch64::TBNZW);
758 break;
759 case AArch64::TBNZW:
760 Cond[1].setImm(AArch64::TBZW);
761 break;
762 case AArch64::TBZX:
763 Cond[1].setImm(AArch64::TBNZX);
764 break;
765 case AArch64::TBNZX:
766 Cond[1].setImm(AArch64::TBZX);
767 break;
768
769 // Cond is { -1, Opcode, CC, Op0, Op1, ... }
770 case AArch64::CBWPri:
771 case AArch64::CBXPri:
772 case AArch64::CBBAssertExt:
773 case AArch64::CBHAssertExt:
774 case AArch64::CBWPrr:
775 case AArch64::CBXPrr: {
776 // Pseudos using standard 4bit Arm condition codes
777 AArch64CC::CondCode CC =
778 static_cast<AArch64CC::CondCode>(Cond[2].getImm());
779 Cond[2].setImm(AArch64CC::getInvertedCondCode(Code: CC));
780 }
781 }
782 }
783
784 return false;
785}
786
787unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
788 int *BytesRemoved) const {
789 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
790 if (I == MBB.end())
791 return 0;
792
793 if (!isUncondBranchOpcode(Opc: I->getOpcode()) &&
794 !isCondBranchOpcode(Opc: I->getOpcode()))
795 return 0;
796
797 // Remove the branch.
798 I->eraseFromParent();
799
800 I = MBB.end();
801
802 if (I == MBB.begin()) {
803 if (BytesRemoved)
804 *BytesRemoved = 4;
805 return 1;
806 }
807 --I;
808 if (!isCondBranchOpcode(Opc: I->getOpcode())) {
809 if (BytesRemoved)
810 *BytesRemoved = 4;
811 return 1;
812 }
813
814 // Remove the branch.
815 I->eraseFromParent();
816 if (BytesRemoved)
817 *BytesRemoved = 8;
818
819 return 2;
820}
821
822void AArch64InstrInfo::instantiateCondBranch(
823 MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
824 ArrayRef<MachineOperand> Cond) const {
825 if (Cond[0].getImm() != -1) {
826 // Regular Bcc
827 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: Cond[0].getImm()).addMBB(MBB: TBB);
828 } else {
829 // Folded compare-and-branch
830 // Note that we use addOperand instead of addReg to keep the flags.
831
832 // cbz, cbnz
833 const MachineInstrBuilder MIB =
834 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[1].getImm())).add(MO: Cond[2]);
835
836 // tbz/tbnz
837 if (Cond.size() > 3)
838 MIB.add(MO: Cond[3]);
839
840 // cb
841 if (Cond.size() > 4)
842 MIB.add(MO: Cond[4]);
843
844 MIB.addMBB(MBB: TBB);
845
846 // cb[b,h]
847 if (Cond.size() > 5) {
848 MIB.addImm(Val: Cond[5].getImm());
849 MIB.addImm(Val: Cond[6].getImm());
850 }
851 }
852}
853
854unsigned AArch64InstrInfo::insertBranch(
855 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
856 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
857 // Shouldn't be a fall through.
858 assert(TBB && "insertBranch must not be told to insert a fallthrough");
859
860 if (!FBB) {
861 if (Cond.empty()) // Unconditional branch?
862 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: TBB);
863 else
864 instantiateCondBranch(MBB, DL, TBB, Cond);
865
866 if (BytesAdded)
867 *BytesAdded = 4;
868
869 return 1;
870 }
871
872 // Two-way conditional branch.
873 instantiateCondBranch(MBB, DL, TBB, Cond);
874 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: FBB);
875
876 if (BytesAdded)
877 *BytesAdded = 8;
878
879 return 2;
880}
881
882#ifndef NDEBUG
883static bool isValidCBExtend(int64_t Opc, AArch64_AM::ShiftExtendType Ext) {
884 switch (Ext) {
885 default:
886 return false;
887 case AArch64_AM::UXTB:
888 case AArch64_AM::SXTB:
889 return Opc == AArch64::CBBAssertExt;
890 case AArch64_AM::UXTH:
891 case AArch64_AM::SXTH:
892 return Opc == AArch64::CBHAssertExt;
893 }
894}
895#endif
896
897AArch64CC::CondCode AArch64InstrInfo::insertCmpForCondBr(
898 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL,
899 ArrayRef<MachineOperand> Cond) const {
900 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
901
902 // Parse the condition code, see parseCondBranch() above.
903 AArch64CC::CondCode CC;
904 switch (Cond.size()) {
905 default:
906 llvm_unreachable("Unknown condition opcode in Cond");
907 case 1: // b.cc
908 CC = AArch64CC::CondCode(Cond[0].getImm());
909 break;
910 case 3: { // cbz/cbnz
911 // We must insert a compare against 0.
912 bool Is64Bit;
913 switch (Cond[1].getImm()) {
914 default:
915 llvm_unreachable("Unknown branch opcode in Cond");
916 case AArch64::CBZW:
917 Is64Bit = false;
918 CC = AArch64CC::EQ;
919 break;
920 case AArch64::CBZX:
921 Is64Bit = true;
922 CC = AArch64CC::EQ;
923 break;
924 case AArch64::CBNZW:
925 Is64Bit = false;
926 CC = AArch64CC::NE;
927 break;
928 case AArch64::CBNZX:
929 Is64Bit = true;
930 CC = AArch64CC::NE;
931 break;
932 }
933 Register SrcReg = Cond[2].getReg();
934 if (Is64Bit) {
935 // cmp reg, #0 is actually subs xzr, reg, #0.
936 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64spRegClass);
937 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSXri), DestReg: AArch64::XZR)
938 .addReg(RegNo: SrcReg)
939 .addImm(Val: 0)
940 .addImm(Val: 0);
941 } else {
942 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32spRegClass);
943 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWri), DestReg: AArch64::WZR)
944 .addReg(RegNo: SrcReg)
945 .addImm(Val: 0)
946 .addImm(Val: 0);
947 }
948 } break;
949 case 4: { // tbz/tbnz
950 // We must insert a tst instruction.
951 switch (Cond[1].getImm()) {
952 default:
953 llvm_unreachable("Unknown branch opcode in Cond");
954 case AArch64::TBZW:
955 case AArch64::TBZX:
956 CC = AArch64CC::EQ;
957 break;
958 case AArch64::TBNZW:
959 case AArch64::TBNZX:
960 CC = AArch64CC::NE;
961 break;
962 }
963 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
964 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
965 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::ANDSWri), DestReg: AArch64::WZR)
966 .addReg(RegNo: Cond[2].getReg())
967 .addImm(
968 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 32));
969 else
970 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::ANDSXri), DestReg: AArch64::XZR)
971 .addReg(RegNo: Cond[2].getReg())
972 .addImm(
973 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 64));
974 } break;
975 case 5: { // cb
976 // We must insert a cmp, that is a subs
977 // 0 1 2 3 4
978 // Cond is { -1, Opcode, CC, Op0, Op1 }
979 unsigned SubsOpc, SubsDestReg;
980 bool IsImm = false;
981 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
982 switch (Cond[1].getImm()) {
983 default:
984 llvm_unreachable("Unknown branch opcode in Cond");
985 case AArch64::CBWPri:
986 SubsOpc = AArch64::SUBSWri;
987 SubsDestReg = AArch64::WZR;
988 IsImm = true;
989 break;
990 case AArch64::CBXPri:
991 SubsOpc = AArch64::SUBSXri;
992 SubsDestReg = AArch64::XZR;
993 IsImm = true;
994 break;
995 case AArch64::CBWPrr:
996 SubsOpc = AArch64::SUBSWrr;
997 SubsDestReg = AArch64::WZR;
998 IsImm = false;
999 break;
1000 case AArch64::CBXPrr:
1001 SubsOpc = AArch64::SUBSXrr;
1002 SubsDestReg = AArch64::XZR;
1003 IsImm = false;
1004 break;
1005 }
1006
1007 if (IsImm) {
1008 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 1));
1009 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
1010 .addReg(RegNo: Cond[3].getReg())
1011 .addImm(Val: Cond[4].getImm())
1012 .addImm(Val: 0);
1013 } else {
1014 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 1));
1015 MRI.constrainRegClass(Reg: Cond[4].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 2));
1016 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
1017 .addReg(RegNo: Cond[3].getReg())
1018 .addReg(RegNo: Cond[4].getReg());
1019 }
1020 } break;
1021 case 7: { // cb[b,h]
1022 // We must insert a cmp, that is a subs, but also zero- or sign-extensions
1023 // that have been folded. For the first operand we codegen an explicit
1024 // extension, for the second operand we fold the extension into cmp.
1025 // 0 1 2 3 4 5 6
1026 // Cond is { -1, Opcode, CC, Op0, Op1, Ext0, Ext1 }
1027
1028 // We need a new register for the now explicitly extended register
1029 Register Reg = Cond[3].getReg();
1030 if (Cond[5].getImm() != AArch64_AM::InvalidShiftExtend) {
1031 unsigned ExtOpc;
1032 unsigned ExtBits;
1033 AArch64_AM::ShiftExtendType ExtendType =
1034 AArch64_AM::getExtendType(Imm: Cond[5].getImm());
1035 assert(isValidCBExtend(Cond[1].getImm(), ExtendType) &&
1036 "Unexpected compare-and-branch instruction for extend type");
1037 switch (ExtendType) {
1038 default:
1039 llvm_unreachable("Unknown shift-extend for CB instruction");
1040 case AArch64_AM::SXTB:
1041 ExtOpc = AArch64::SBFMWri;
1042 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1043 break;
1044 case AArch64_AM::SXTH:
1045 ExtOpc = AArch64::SBFMWri;
1046 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1047 break;
1048 case AArch64_AM::UXTB:
1049 ExtOpc = AArch64::ANDWri;
1050 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1051 break;
1052 case AArch64_AM::UXTH:
1053 ExtOpc = AArch64::ANDWri;
1054 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1055 break;
1056 }
1057
1058 // Build the explicit extension of the first operand
1059 Reg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32commonRegClass);
1060 MachineInstrBuilder MBBI =
1061 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ExtOpc), DestReg: Reg).addReg(RegNo: Cond[3].getReg());
1062 if (ExtOpc != AArch64::ANDWri)
1063 MBBI.addImm(Val: 0);
1064 MBBI.addImm(Val: ExtBits);
1065 }
1066
1067 // Now, subs with an extended second operand
1068 if (Cond[6].getImm() != AArch64_AM::InvalidShiftExtend) {
1069 MRI.constrainRegClass(Reg, RC: &AArch64::GPR32commonRegClass);
1070 AArch64_AM::ShiftExtendType ExtendType =
1071 AArch64_AM::getExtendType(Imm: Cond[6].getImm());
1072 assert(isValidCBExtend(Cond[1].getImm(), ExtendType) &&
1073 "Unexpected compare-and-branch instruction for extend type");
1074 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrx), DestReg: AArch64::WZR)
1075 .addReg(RegNo: Reg)
1076 .addReg(RegNo: Cond[4].getReg())
1077 .addImm(Val: AArch64_AM::getArithExtendImm(ET: ExtendType, Imm: 0));
1078 } // If no extension is needed, just a regular subs
1079 else {
1080 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrr), DestReg: AArch64::WZR)
1081 .addReg(RegNo: Reg)
1082 .addReg(RegNo: Cond[4].getReg());
1083 }
1084
1085 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
1086 } break;
1087 }
1088 return CC;
1089}
1090
1091bool llvm::optimizeTerminators(MachineBasicBlock *MBB,
1092 const TargetInstrInfo &TII) {
1093 for (MachineInstr &MI : MBB->terminators()) {
1094 unsigned Opc = MI.getOpcode();
1095 switch (Opc) {
1096 case AArch64::CBZW:
1097 case AArch64::CBZX:
1098 case AArch64::TBZW:
1099 case AArch64::TBZX:
1100 // CBZ/TBZ with WZR/XZR -> unconditional B
1101 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
1102 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
1103 DEBUG_WITH_TYPE("optimizeTerminators",
1104 dbgs() << "Removing always taken branch: " << MI);
1105 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
1106 SmallVector<MachineBasicBlock *> Succs(MBB->successors());
1107 for (auto *S : Succs)
1108 if (S != Target)
1109 MBB->removeSuccessor(Succ: S);
1110 DebugLoc DL = MI.getDebugLoc();
1111 while (MBB->rbegin() != &MI)
1112 MBB->rbegin()->eraseFromParent();
1113 MI.eraseFromParent();
1114 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: AArch64::B)).addMBB(MBB: Target);
1115 return true;
1116 }
1117 break;
1118 case AArch64::CBNZW:
1119 case AArch64::CBNZX:
1120 case AArch64::TBNZW:
1121 case AArch64::TBNZX:
1122 // CBNZ/TBNZ with WZR/XZR -> never taken, remove branch and successor
1123 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
1124 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
1125 DEBUG_WITH_TYPE("optimizeTerminators",
1126 dbgs() << "Removing never taken branch: " << MI);
1127 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
1128 MI.getParent()->removeSuccessor(Succ: Target);
1129 MI.eraseFromParent();
1130 return true;
1131 }
1132 break;
1133 }
1134 }
1135 return false;
1136}
1137
1138// Find the original register that VReg is copied from.
1139static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
1140 while (Register::isVirtualRegister(Reg: VReg)) {
1141 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1142 if (!DefMI || !DefMI->isFullCopy())
1143 return VReg;
1144 VReg = DefMI->getOperand(i: 1).getReg();
1145 }
1146 return VReg;
1147}
1148
1149// Determine if VReg is defined by an instruction that can be folded into a
1150// csel instruction. If so, return the folded opcode, and the replacement
1151// register.
1152static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
1153 unsigned *NewReg = nullptr) {
1154 VReg = removeCopies(MRI, VReg);
1155 if (!Register::isVirtualRegister(Reg: VReg))
1156 return 0;
1157
1158 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(RC: MRI.getRegClass(Reg: VReg));
1159 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1160 if (!DefMI)
1161 return 0;
1162 unsigned Opc = 0;
1163 unsigned SrcReg = 0;
1164 switch (DefMI->getOpcode()) {
1165 case AArch64::SUBREG_TO_REG:
1166 // Check for the following way to define an 64-bit immediate:
1167 // %0:gpr32 = MOVi32imm 1
1168 // %1:gpr64 = SUBREG_TO_REG %0:gpr32, %subreg.sub_32
1169 if (!DefMI->getOperand(i: 1).isReg())
1170 return 0;
1171 if (!DefMI->getOperand(i: 2).isImm() ||
1172 DefMI->getOperand(i: 2).getImm() != AArch64::sub_32)
1173 return 0;
1174 DefMI = MRI.getVRegDef(Reg: DefMI->getOperand(i: 1).getReg());
1175 if (DefMI->getOpcode() != AArch64::MOVi32imm)
1176 return 0;
1177 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
1178 return 0;
1179 assert(Is64Bit);
1180 SrcReg = AArch64::XZR;
1181 Opc = AArch64::CSINCXr;
1182 break;
1183
1184 case AArch64::MOVi32imm:
1185 case AArch64::MOVi64imm:
1186 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
1187 return 0;
1188 SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1189 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
1190 break;
1191
1192 case AArch64::ADDSXri:
1193 case AArch64::ADDSWri:
1194 // if NZCV is used, do not fold.
1195 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1196 isDead: true) == -1)
1197 return 0;
1198 // fall-through to ADDXri and ADDWri.
1199 [[fallthrough]];
1200 case AArch64::ADDXri:
1201 case AArch64::ADDWri:
1202 // add x, 1 -> csinc.
1203 if (!DefMI->getOperand(i: 2).isImm() || DefMI->getOperand(i: 2).getImm() != 1 ||
1204 DefMI->getOperand(i: 3).getImm() != 0)
1205 return 0;
1206 SrcReg = DefMI->getOperand(i: 1).getReg();
1207 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
1208 break;
1209
1210 case AArch64::ORNXrr:
1211 case AArch64::ORNWrr: {
1212 // not x -> csinv, represented as orn dst, xzr, src.
1213 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1214 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1215 return 0;
1216 SrcReg = DefMI->getOperand(i: 2).getReg();
1217 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
1218 break;
1219 }
1220
1221 case AArch64::SUBSXrr:
1222 case AArch64::SUBSWrr:
1223 // if NZCV is used, do not fold.
1224 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1225 isDead: true) == -1)
1226 return 0;
1227 // fall-through to SUBXrr and SUBWrr.
1228 [[fallthrough]];
1229 case AArch64::SUBXrr:
1230 case AArch64::SUBWrr: {
1231 // neg x -> csneg, represented as sub dst, xzr, src.
1232 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1233 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1234 return 0;
1235 SrcReg = DefMI->getOperand(i: 2).getReg();
1236 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
1237 break;
1238 }
1239 default:
1240 return 0;
1241 }
1242 assert(Opc && SrcReg && "Missing parameters");
1243
1244 if (NewReg)
1245 *NewReg = SrcReg;
1246 return Opc;
1247}
1248
1249bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1250 ArrayRef<MachineOperand> Cond,
1251 Register DstReg, Register TrueReg,
1252 Register FalseReg,
1253 SelectExpansion &Exp) const {
1254 // Check register classes.
1255 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1256 const TargetRegisterClass *RC =
1257 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1258 if (!RC)
1259 return false;
1260
1261 // Also need to check the dest regclass, in case we're trying to optimize
1262 // something like:
1263 // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
1264 if (!RI.getCommonSubClass(A: RC, B: MRI.getRegClass(Reg: DstReg)))
1265 return false;
1266
1267 // Expanding cbz/tbz requires a flag setting instruction in addition to the
1268 // select, adding an extra cycle of latency on the condition.
1269 unsigned ExtraCondLat = Cond.size() != 1;
1270 Exp.NumInsts = 1 + ExtraCondLat;
1271 if (ExtraCondLat)
1272 Exp.ClobberedPhysRegs.push_back(Elt: AArch64::NZCV);
1273
1274 // GPRs are handled by csel.
1275 // FIXME: Fold in x+1, -x, and ~x when applicable.
1276 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
1277 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1278 // Single-cycle csel, csinc, csinv, and csneg.
1279 Exp.CondCycles = 1 + ExtraCondLat;
1280 Exp.TrueCycles = Exp.FalseCycles = 1;
1281 if (canFoldIntoCSel(MRI, VReg: TrueReg))
1282 Exp.TrueCycles = 0;
1283 else if (canFoldIntoCSel(MRI, VReg: FalseReg))
1284 Exp.FalseCycles = 0;
1285 return true;
1286 }
1287
1288 // Scalar floating point is handled by fcsel.
1289 // FIXME: Form fabs, fmin, and fmax when applicable.
1290 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
1291 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
1292 Exp.CondCycles = 5 + ExtraCondLat;
1293 Exp.TrueCycles = Exp.FalseCycles = 2;
1294 return true;
1295 }
1296
1297 // No single conditional move for a 128-bit vector, but we can emit a sequence
1298 // of csetm (~1), dup (~5, cross domain), bsl (~2).
1299 if (AArch64::FPR128RegClass.hasSubClassEq(RC) &&
1300 Subtarget.isNeonAvailable() &&
1301 !MBB.getParent()->getFunction().hasMinSize()) {
1302 Exp.CondCycles = 8 + ExtraCondLat;
1303 Exp.TrueCycles = Exp.FalseCycles = 2;
1304 Exp.NumInsts = 3 + ExtraCondLat;
1305 return true;
1306 }
1307
1308 return false;
1309}
1310
1311void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
1312 MachineBasicBlock::iterator I,
1313 const DebugLoc &DL, Register DstReg,
1314 ArrayRef<MachineOperand> Cond,
1315 Register TrueReg, Register FalseReg) const {
1316
1317 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1318 AArch64CC::CondCode CC = insertCmpForCondBr(MBB, MI: I, DL, Cond);
1319
1320 // A 128-bit vector has no conditional move so blend the operands with a mask
1321 // built from the flags.
1322 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR128RegClass)) {
1323 assert(Subtarget.isNeonAvailable() && "Expected NEON for a vector select");
1324 MRI.constrainRegClass(Reg: TrueReg, RC: &AArch64::FPR128RegClass);
1325 MRI.constrainRegClass(Reg: FalseReg, RC: &AArch64::FPR128RegClass);
1326 Register CondSet = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1327 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::CSINVXr), DestReg: CondSet)
1328 .addReg(RegNo: AArch64::XZR)
1329 .addReg(RegNo: AArch64::XZR)
1330 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
1331 Register Mask = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
1332 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::DUPv2i64gpr), DestReg: Mask).addReg(RegNo: CondSet);
1333 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::BSPv16i8), DestReg: DstReg)
1334 .addReg(RegNo: Mask)
1335 .addReg(RegNo: TrueReg)
1336 .addReg(RegNo: FalseReg);
1337 return;
1338 }
1339
1340 unsigned Opc = 0;
1341 const TargetRegisterClass *RC = nullptr;
1342 bool TryFold = false;
1343 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass)) {
1344 RC = &AArch64::GPR64RegClass;
1345 Opc = AArch64::CSELXr;
1346 TryFold = true;
1347 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR32RegClass)) {
1348 RC = &AArch64::GPR32RegClass;
1349 Opc = AArch64::CSELWr;
1350 TryFold = true;
1351 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR64RegClass)) {
1352 RC = &AArch64::FPR64RegClass;
1353 Opc = AArch64::FCSELDrrr;
1354 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR32RegClass)) {
1355 RC = &AArch64::FPR32RegClass;
1356 Opc = AArch64::FCSELSrrr;
1357 }
1358 assert(RC && "Unsupported regclass");
1359
1360 // Try folding simple instructions into the csel.
1361 if (TryFold) {
1362 unsigned NewReg = 0;
1363 unsigned FoldedOpc = canFoldIntoCSel(MRI, VReg: TrueReg, NewReg: &NewReg);
1364 if (FoldedOpc) {
1365 // The folded opcodes csinc, csinc and csneg apply the operation to
1366 // FalseReg, so we need to invert the condition.
1367 CC = AArch64CC::getInvertedCondCode(Code: CC);
1368 TrueReg = FalseReg;
1369 } else
1370 FoldedOpc = canFoldIntoCSel(MRI, VReg: FalseReg, NewReg: &NewReg);
1371
1372 // Fold the operation. Leave any dead instructions for DCE to clean up.
1373 if (FoldedOpc) {
1374 FalseReg = NewReg;
1375 Opc = FoldedOpc;
1376 }
1377 }
1378
1379 // Pull all virtual register into the appropriate class.
1380 MRI.constrainRegClass(Reg: TrueReg, RC);
1381 // FalseReg might be WZR or XZR if the folded operand is a literal 1.
1382 assert(
1383 (FalseReg.isVirtual() || FalseReg == AArch64::WZR ||
1384 FalseReg == AArch64::XZR) &&
1385 "FalseReg was folded into a non-virtual register other than WZR or XZR");
1386 if (FalseReg.isVirtual())
1387 MRI.constrainRegClass(Reg: FalseReg, RC);
1388
1389 // Insert the csel.
1390 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
1391 .addReg(RegNo: TrueReg)
1392 .addReg(RegNo: FalseReg)
1393 .addImm(Val: CC);
1394}
1395
1396// Return true if Imm can be loaded into a register by a "cheap" sequence of
1397// instructions. For now, "cheap" means at most two instructions.
1398static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize) {
1399 if (BitSize == 32)
1400 return true;
1401
1402 assert(BitSize == 64 && "Only bit sizes of 32 or 64 allowed");
1403 uint64_t Imm = static_cast<uint64_t>(MI.getOperand(i: 1).getImm());
1404 SmallVector<AArch64_IMM::ImmInsnModel, 4> Is;
1405 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: Is);
1406
1407 return Is.size() <= 2;
1408}
1409
1410// Check if a COPY instruction is cheap.
1411static bool isCheapCopy(const MachineInstr &MI, const AArch64RegisterInfo &RI) {
1412 assert(MI.isCopy() && "Expected COPY instruction");
1413 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1414
1415 // Cross-bank copies (e.g., between GPR and FPR) are expensive on AArch64,
1416 // typically requiring an FMOV instruction with a 2-6 cycle latency.
1417 auto GetRegClass = [&](Register Reg) -> const TargetRegisterClass * {
1418 if (Reg.isVirtual())
1419 return MRI.getRegClass(Reg);
1420 if (Reg.isPhysical())
1421 return RI.getMinimalPhysRegClass(Reg);
1422 return nullptr;
1423 };
1424 const TargetRegisterClass *DstRC = GetRegClass(MI.getOperand(i: 0).getReg());
1425 const TargetRegisterClass *SrcRC = GetRegClass(MI.getOperand(i: 1).getReg());
1426 if (DstRC && SrcRC && !RI.getCommonSubClass(A: DstRC, B: SrcRC))
1427 return false;
1428
1429 return MI.isAsCheapAsAMove();
1430}
1431
1432// FIXME: this implementation should be micro-architecture dependent, so a
1433// micro-architecture target hook should be introduced here in future.
1434bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
1435 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1436 if (isExynosCheapAsMove(MI))
1437 return true;
1438 return MI.isAsCheapAsAMove();
1439 }
1440
1441 switch (MI.getOpcode()) {
1442 default:
1443 return MI.isAsCheapAsAMove();
1444
1445 case TargetOpcode::COPY:
1446 return isCheapCopy(MI, RI);
1447
1448 case AArch64::ADDWrs:
1449 case AArch64::ADDXrs:
1450 case AArch64::SUBWrs:
1451 case AArch64::SUBXrs:
1452 return Subtarget.hasALULSLFast() && MI.getOperand(i: 3).getImm() <= 4;
1453
1454 // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
1455 // ORRXri, it is as cheap as MOV.
1456 // Likewise if it can be expanded to MOVZ/MOVN/MOVK.
1457 case AArch64::MOVi32imm:
1458 return isCheapImmediate(MI, BitSize: 32);
1459 case AArch64::MOVi64imm:
1460 return isCheapImmediate(MI, BitSize: 64);
1461 }
1462}
1463
1464bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
1465 switch (MI.getOpcode()) {
1466 default:
1467 return false;
1468
1469 case AArch64::ADDWrs:
1470 case AArch64::ADDXrs:
1471 case AArch64::ADDSWrs:
1472 case AArch64::ADDSXrs: {
1473 unsigned Imm = MI.getOperand(i: 3).getImm();
1474 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1475 if (ShiftVal == 0)
1476 return true;
1477 return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
1478 }
1479
1480 case AArch64::ADDWrx:
1481 case AArch64::ADDXrx:
1482 case AArch64::ADDXrx64:
1483 case AArch64::ADDSWrx:
1484 case AArch64::ADDSXrx:
1485 case AArch64::ADDSXrx64: {
1486 unsigned Imm = MI.getOperand(i: 3).getImm();
1487 switch (AArch64_AM::getArithExtendType(Imm)) {
1488 default:
1489 return false;
1490 case AArch64_AM::UXTB:
1491 case AArch64_AM::UXTH:
1492 case AArch64_AM::UXTW:
1493 case AArch64_AM::UXTX:
1494 return AArch64_AM::getArithShiftValue(Imm) <= 4;
1495 }
1496 }
1497
1498 case AArch64::SUBWrs:
1499 case AArch64::SUBSWrs: {
1500 unsigned Imm = MI.getOperand(i: 3).getImm();
1501 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1502 return ShiftVal == 0 ||
1503 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
1504 }
1505
1506 case AArch64::SUBXrs:
1507 case AArch64::SUBSXrs: {
1508 unsigned Imm = MI.getOperand(i: 3).getImm();
1509 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1510 return ShiftVal == 0 ||
1511 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
1512 }
1513
1514 case AArch64::SUBWrx:
1515 case AArch64::SUBXrx:
1516 case AArch64::SUBXrx64:
1517 case AArch64::SUBSWrx:
1518 case AArch64::SUBSXrx:
1519 case AArch64::SUBSXrx64: {
1520 unsigned Imm = MI.getOperand(i: 3).getImm();
1521 switch (AArch64_AM::getArithExtendType(Imm)) {
1522 default:
1523 return false;
1524 case AArch64_AM::UXTB:
1525 case AArch64_AM::UXTH:
1526 case AArch64_AM::UXTW:
1527 case AArch64_AM::UXTX:
1528 return AArch64_AM::getArithShiftValue(Imm) == 0;
1529 }
1530 }
1531
1532 case AArch64::LDRBBroW:
1533 case AArch64::LDRBBroX:
1534 case AArch64::LDRBroW:
1535 case AArch64::LDRBroX:
1536 case AArch64::LDRDroW:
1537 case AArch64::LDRDroX:
1538 case AArch64::LDRHHroW:
1539 case AArch64::LDRHHroX:
1540 case AArch64::LDRHroW:
1541 case AArch64::LDRHroX:
1542 case AArch64::LDRQroW:
1543 case AArch64::LDRQroX:
1544 case AArch64::LDRSBWroW:
1545 case AArch64::LDRSBWroX:
1546 case AArch64::LDRSBXroW:
1547 case AArch64::LDRSBXroX:
1548 case AArch64::LDRSHWroW:
1549 case AArch64::LDRSHWroX:
1550 case AArch64::LDRSHXroW:
1551 case AArch64::LDRSHXroX:
1552 case AArch64::LDRSWroW:
1553 case AArch64::LDRSWroX:
1554 case AArch64::LDRSroW:
1555 case AArch64::LDRSroX:
1556 case AArch64::LDRWroW:
1557 case AArch64::LDRWroX:
1558 case AArch64::LDRXroW:
1559 case AArch64::LDRXroX:
1560 case AArch64::PRFMroW:
1561 case AArch64::PRFMroX:
1562 case AArch64::STRBBroW:
1563 case AArch64::STRBBroX:
1564 case AArch64::STRBroW:
1565 case AArch64::STRBroX:
1566 case AArch64::STRDroW:
1567 case AArch64::STRDroX:
1568 case AArch64::STRHHroW:
1569 case AArch64::STRHHroX:
1570 case AArch64::STRHroW:
1571 case AArch64::STRHroX:
1572 case AArch64::STRQroW:
1573 case AArch64::STRQroX:
1574 case AArch64::STRSroW:
1575 case AArch64::STRSroX:
1576 case AArch64::STRWroW:
1577 case AArch64::STRWroX:
1578 case AArch64::STRXroW:
1579 case AArch64::STRXroX: {
1580 unsigned IsSigned = MI.getOperand(i: 3).getImm();
1581 return !IsSigned;
1582 }
1583 }
1584}
1585
1586bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
1587 unsigned Opc = MI.getOpcode();
1588 switch (Opc) {
1589 default:
1590 return false;
1591 case AArch64::SEH_EndC:
1592 case AArch64::SEH_StackAlloc:
1593 case AArch64::SEH_SaveFPLR:
1594 case AArch64::SEH_SaveFPLR_X:
1595 case AArch64::SEH_SaveReg:
1596 case AArch64::SEH_SaveReg_X:
1597 case AArch64::SEH_SaveRegP:
1598 case AArch64::SEH_SaveRegP_X:
1599 case AArch64::SEH_SaveFReg:
1600 case AArch64::SEH_SaveFReg_X:
1601 case AArch64::SEH_SaveFRegP:
1602 case AArch64::SEH_SaveFRegP_X:
1603 case AArch64::SEH_SetFP:
1604 case AArch64::SEH_AddFP:
1605 case AArch64::SEH_Nop:
1606 case AArch64::SEH_PrologEnd:
1607 case AArch64::SEH_EpilogStart:
1608 case AArch64::SEH_EpilogEnd:
1609 case AArch64::SEH_PACSignLR:
1610 case AArch64::SEH_SaveAnyRegI:
1611 case AArch64::SEH_SaveAnyRegIP:
1612 case AArch64::SEH_SaveAnyRegQ:
1613 case AArch64::SEH_SaveAnyRegQX:
1614 case AArch64::SEH_SaveAnyRegQP:
1615 case AArch64::SEH_SaveAnyRegQPX:
1616 case AArch64::SEH_AllocZ:
1617 case AArch64::SEH_SaveZReg:
1618 case AArch64::SEH_SavePReg:
1619 return true;
1620 }
1621}
1622
1623bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1624 Register &SrcReg, Register &DstReg,
1625 unsigned &SubIdx) const {
1626 switch (MI.getOpcode()) {
1627 default:
1628 return false;
1629 case AArch64::SBFMXri: // aka sxtw
1630 case AArch64::UBFMXri: // aka uxtw
1631 // Check for the 32 -> 64 bit extension case, these instructions can do
1632 // much more.
1633 if (MI.getOperand(i: 2).getImm() != 0 || MI.getOperand(i: 3).getImm() != 31)
1634 return false;
1635 // This is a signed or unsigned 32 -> 64 bit extension.
1636 SrcReg = MI.getOperand(i: 1).getReg();
1637 DstReg = MI.getOperand(i: 0).getReg();
1638 SubIdx = AArch64::sub_32;
1639 return true;
1640 }
1641}
1642
1643bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
1644 const MachineInstr &MIa, const MachineInstr &MIb) const {
1645 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1646 int64_t OffsetA = 0, OffsetB = 0;
1647 TypeSize WidthA(0, false), WidthB(0, false);
1648 bool OffsetAIsScalable = false, OffsetBIsScalable = false;
1649
1650 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1651 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1652
1653 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
1654 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1655 return false;
1656
1657 // Retrieve the base, offset from the base and width. Width
1658 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
1659 // base are identical, and the offset of a lower memory access +
1660 // the width doesn't overlap the offset of a higher memory access,
1661 // then the memory accesses are different.
1662 // If OffsetAIsScalable and OffsetBIsScalable are both true, they
1663 // are assumed to have the same scale (vscale).
1664 if (getMemOperandWithOffsetWidth(MI: MIa, BaseOp&: BaseOpA, Offset&: OffsetA, OffsetIsScalable&: OffsetAIsScalable,
1665 Width&: WidthA) &&
1666 getMemOperandWithOffsetWidth(MI: MIb, BaseOp&: BaseOpB, Offset&: OffsetB, OffsetIsScalable&: OffsetBIsScalable,
1667 Width&: WidthB)) {
1668 if (BaseOpA->isIdenticalTo(Other: *BaseOpB) &&
1669 OffsetAIsScalable == OffsetBIsScalable) {
1670 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1671 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1672 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1673 if (LowWidth.isScalable() == OffsetAIsScalable &&
1674 LowOffset + (int)LowWidth.getKnownMinValue() <= HighOffset)
1675 return true;
1676 }
1677 }
1678 return false;
1679}
1680
1681bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1682 const MachineBasicBlock *MBB,
1683 const MachineFunction &MF) const {
1684 if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
1685 return true;
1686
1687 // Do not move an instruction that can be recognized as a branch target.
1688 if (hasBTISemantics(MI))
1689 return true;
1690
1691 switch (MI.getOpcode()) {
1692 case AArch64::PAUTH_EPILOGUE:
1693 // Keep authentication after the other epilogue instructions and close to
1694 // the return, allowing them to combine into an authenticated return.
1695 return true;
1696 case AArch64::HINT:
1697 // CSDB hints are scheduling barriers.
1698 if (MI.getOperand(i: 0).getImm() == 0x14)
1699 return true;
1700 break;
1701 case AArch64::DSB:
1702 case AArch64::ISB:
1703 // DSB and ISB also are scheduling barriers.
1704 return true;
1705 case AArch64::MSRpstatesvcrImm1:
1706 // SMSTART and SMSTOP are also scheduling barriers.
1707 return true;
1708 default:;
1709 }
1710 if (isSEHInstruction(MI))
1711 return true;
1712 // Skip debug instructions and pseudo probes so they do not change
1713 // scheduling regions.
1714 auto Next = next_nodbg(It: MI.getIterator(), End: MBB->instr_end());
1715 return Next != MBB->instr_end() && Next->isCFIInstruction();
1716}
1717
1718/// analyzeCompare - For a comparison instruction, return the source registers
1719/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1720/// Return true if the comparison instruction can be analyzed.
1721bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1722 Register &SrcReg2, int64_t &CmpMask,
1723 int64_t &CmpValue) const {
1724 // The first operand can be a frame index where we'd normally expect a
1725 // register.
1726 // FIXME: Pass subregisters out of analyzeCompare
1727 assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1728 if (!MI.getOperand(i: 1).isReg() || MI.getOperand(i: 1).getSubReg())
1729 return false;
1730
1731 switch (MI.getOpcode()) {
1732 default:
1733 break;
1734 case AArch64::PTEST_PP:
1735 case AArch64::PTEST_PP_ANY:
1736 case AArch64::PTEST_PP_FIRST:
1737 SrcReg = MI.getOperand(i: 0).getReg();
1738 SrcReg2 = MI.getOperand(i: 1).getReg();
1739 if (MI.getOperand(i: 2).getSubReg())
1740 return false;
1741
1742 // Not sure about the mask and value for now...
1743 CmpMask = ~0;
1744 CmpValue = 0;
1745 return true;
1746 case AArch64::SUBSWrr:
1747 case AArch64::SUBSWrs:
1748 case AArch64::SUBSWrx:
1749 case AArch64::SUBSXrr:
1750 case AArch64::SUBSXrs:
1751 case AArch64::SUBSXrx:
1752 case AArch64::ADDSWrr:
1753 case AArch64::ADDSWrs:
1754 case AArch64::ADDSWrx:
1755 case AArch64::ADDSXrr:
1756 case AArch64::ADDSXrs:
1757 case AArch64::ADDSXrx:
1758 // Replace SUBSWrr with SUBWrr if NZCV is not used.
1759 SrcReg = MI.getOperand(i: 1).getReg();
1760 SrcReg2 = MI.getOperand(i: 2).getReg();
1761
1762 // FIXME: Pass subregisters out of analyzeCompare
1763 if (MI.getOperand(i: 2).getSubReg())
1764 return false;
1765
1766 CmpMask = ~0;
1767 CmpValue = 0;
1768 return true;
1769 case AArch64::SUBSWri:
1770 case AArch64::ADDSWri:
1771 case AArch64::SUBSXri:
1772 case AArch64::ADDSXri:
1773 SrcReg = MI.getOperand(i: 1).getReg();
1774 SrcReg2 = 0;
1775 CmpMask = ~0;
1776 CmpValue = MI.getOperand(i: 2).getImm();
1777 return true;
1778 case AArch64::ANDSWri:
1779 case AArch64::ANDSXri:
1780 // ANDS does not use the same encoding scheme as the others xxxS
1781 // instructions.
1782 SrcReg = MI.getOperand(i: 1).getReg();
1783 SrcReg2 = 0;
1784 CmpMask = ~0;
1785 CmpValue = AArch64_AM::decodeLogicalImmediate(
1786 val: MI.getOperand(i: 2).getImm(),
1787 regSize: MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1788 return true;
1789 }
1790
1791 return false;
1792}
1793
1794static void updateOperandRegClass(MachineInstr &Instr) {
1795 MachineBasicBlock *MBB = Instr.getParent();
1796 assert(MBB && "Can't get MachineBasicBlock here");
1797 MachineFunction *MF = MBB->getParent();
1798 assert(MF && "Can't get MachineFunction here");
1799 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1800 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1801 MachineRegisterInfo *MRI = &MF->getRegInfo();
1802
1803 for (const auto &[OpIdx, MO] : enumerate(First: Instr.operands())) {
1804 if (!MO.isReg() || !MO.getReg().isVirtual())
1805 continue;
1806
1807 Register Reg = MO.getReg();
1808 const TargetRegisterClass *NewRC = Instr.getRegClassConstraintEffect(
1809 OpIdx, CurRC: MRI->getRegClass(Reg), TII, TRI);
1810 if (!NewRC || !MRI->constrainRegClass(Reg, RC: NewRC))
1811 llvm_unreachable("Operand register classes are incompatible");
1812 }
1813}
1814
1815/// Return the opcode that does not set flags when possible - otherwise
1816/// return the original opcode. The caller is responsible to do the actual
1817/// substitution and legality checking.
1818static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1819 // Don't convert all compare instructions, because for some the zero register
1820 // encoding becomes the sp register.
1821 bool MIDefinesZeroReg = false;
1822 if (MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
1823 MI.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
1824 MIDefinesZeroReg = true;
1825
1826 switch (MI.getOpcode()) {
1827 default:
1828 return MI.getOpcode();
1829 case AArch64::ADDSWrr:
1830 return AArch64::ADDWrr;
1831 case AArch64::ADDSWri:
1832 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1833 case AArch64::ADDSWrs:
1834 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1835 case AArch64::ADDSWrx:
1836 return AArch64::ADDWrx;
1837 case AArch64::ADDSXrr:
1838 return AArch64::ADDXrr;
1839 case AArch64::ADDSXri:
1840 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1841 case AArch64::ADDSXrs:
1842 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1843 case AArch64::ADDSXrx:
1844 return AArch64::ADDXrx;
1845 case AArch64::SUBSWrr:
1846 return AArch64::SUBWrr;
1847 case AArch64::SUBSWri:
1848 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1849 case AArch64::SUBSWrs:
1850 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1851 case AArch64::SUBSWrx:
1852 return AArch64::SUBWrx;
1853 case AArch64::SUBSXrr:
1854 return AArch64::SUBXrr;
1855 case AArch64::SUBSXri:
1856 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1857 case AArch64::SUBSXrs:
1858 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1859 case AArch64::SUBSXrx:
1860 return AArch64::SUBXrx;
1861 }
1862}
1863
1864enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1865
1866/// True when condition flags are accessed (either by writing or reading)
1867/// on the instruction trace starting at From and ending at To.
1868///
1869/// Note: If From and To are from different blocks it's assumed CC are accessed
1870/// on the path.
1871static bool areCFlagsAccessedBetweenInstrs(
1872 MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1873 const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1874 // Early exit if To is at the beginning of the BB.
1875 if (To == To->getParent()->begin())
1876 return true;
1877
1878 // Check whether the instructions are in the same basic block
1879 // If not, assume the condition flags might get modified somewhere.
1880 if (To->getParent() != From->getParent())
1881 return true;
1882
1883 // From must be above To.
1884 assert(std::any_of(
1885 ++To.getReverse(), To->getParent()->rend(),
1886 [From](MachineInstr &MI) { return MI.getIterator() == From; }));
1887
1888 // We iterate backward starting at \p To until we hit \p From.
1889 for (const MachineInstr &Instr :
1890 instructionsWithoutDebug(It: ++To.getReverse(), End: From.getReverse())) {
1891 if (((AccessToCheck & AK_Write) &&
1892 Instr.modifiesRegister(Reg: AArch64::NZCV, TRI)) ||
1893 ((AccessToCheck & AK_Read) && Instr.readsRegister(Reg: AArch64::NZCV, TRI)))
1894 return true;
1895 }
1896 return false;
1897}
1898
1899std::optional<unsigned>
1900AArch64InstrInfo::canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
1901 MachineInstr *Pred,
1902 const MachineRegisterInfo *MRI) const {
1903 unsigned MaskOpcode = Mask->getOpcode();
1904 unsigned PredOpcode = Pred->getOpcode();
1905 bool PredIsPTestLike = isPTestLikeOpcode(Opc: PredOpcode);
1906 bool PredIsWhileLike = isWhileOpcode(Opc: PredOpcode);
1907
1908 if (PredIsWhileLike) {
1909 // For PTEST(PG, PG), PTEST is redundant when PG is the result of a WHILEcc
1910 // instruction and the condition is "any" since WHILcc does an implicit
1911 // PTEST(ALL, PG) check and PG is always a subset of ALL.
1912 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1913 return PredOpcode;
1914
1915 // For PTEST(PTRUE_ALL, WHILE), if the element size matches, the PTEST is
1916 // redundant since WHILE performs an implicit PTEST with an all active
1917 // mask.
1918 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1919 getElementSizeForOpcode(Opc: MaskOpcode) ==
1920 getElementSizeForOpcode(Opc: PredOpcode))
1921 return PredOpcode;
1922
1923 // For PTEST_FIRST(PTRUE_ALL, WHILE), the PTEST_FIRST is redundant since
1924 // WHILEcc performs an implicit PTEST with an all active mask, setting
1925 // the N flag as the PTEST_FIRST would.
1926 if (PTest->getOpcode() == AArch64::PTEST_PP_FIRST &&
1927 isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31)
1928 return PredOpcode;
1929
1930 return {};
1931 }
1932
1933 if (PredIsPTestLike) {
1934 // For PTEST(PG, PG), PTEST is redundant when PG is the result of an
1935 // instruction that sets the flags as PTEST would and the condition is
1936 // "any" since PG is always a subset of the governing predicate of the
1937 // ptest-like instruction.
1938 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1939 return PredOpcode;
1940
1941 auto PTestLikeMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1942
1943 // If the PTEST like instruction's general predicate is not `Mask`, attempt
1944 // to look through a copy and try again. This is because some instructions
1945 // take a predicate whose register class is a subset of its result class.
1946 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1947 PTestLikeMask->getOperand(i: 1).getReg().isVirtual())
1948 PTestLikeMask =
1949 MRI->getUniqueVRegDef(Reg: PTestLikeMask->getOperand(i: 1).getReg());
1950
1951 // For PTEST(PTRUE_ALL, PTEST_LIKE), the PTEST is redundant if the
1952 // the element size matches and either the PTEST_LIKE instruction uses
1953 // the same all active mask or the condition is "any".
1954 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1955 getElementSizeForOpcode(Opc: MaskOpcode) ==
1956 getElementSizeForOpcode(Opc: PredOpcode)) {
1957 if (Mask == PTestLikeMask || PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1958 return PredOpcode;
1959 }
1960
1961 // For PTEST(PG, PTEST_LIKE(PG, ...)), the PTEST is redundant since the
1962 // flags are set based on the same mask 'PG', but PTEST_LIKE must operate
1963 // on 8-bit predicates like the PTEST. Otherwise, for instructions like
1964 // compare that also support 16/32/64-bit predicates, the implicit PTEST
1965 // performed by the compare could consider fewer lanes for these element
1966 // sizes.
1967 //
1968 // For example, consider
1969 //
1970 // ptrue p0.b ; P0=1111-1111-1111-1111
1971 // index z0.s, #0, #1 ; Z0=<0,1,2,3>
1972 // index z1.s, #1, #1 ; Z1=<1,2,3,4>
1973 // cmphi p1.s, p0/z, z1.s, z0.s ; P1=0001-0001-0001-0001
1974 // ; ^ last active
1975 // ptest p0, p1.b ; P1=0001-0001-0001-0001
1976 // ; ^ last active
1977 //
1978 // where the compare generates a canonical all active 32-bit predicate
1979 // (equivalent to 'ptrue p1.s, all'). The implicit PTEST sets the last
1980 // active flag, whereas the PTEST instruction with the same mask doesn't.
1981 // For PTEST_ANY this doesn't apply as the flags in this case would be
1982 // identical regardless of element size.
1983 uint64_t PredElementSize = getElementSizeForOpcode(Opc: PredOpcode);
1984 if (Mask == PTestLikeMask && (PredElementSize == AArch64::ElementSizeB ||
1985 PTest->getOpcode() == AArch64::PTEST_PP_ANY))
1986 return PredOpcode;
1987
1988 return {};
1989 }
1990
1991 // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the
1992 // opcode so the PTEST becomes redundant.
1993 switch (PredOpcode) {
1994 case AArch64::AND_PPzPP:
1995 case AArch64::BIC_PPzPP:
1996 case AArch64::EOR_PPzPP:
1997 case AArch64::NAND_PPzPP:
1998 case AArch64::NOR_PPzPP:
1999 case AArch64::ORN_PPzPP:
2000 case AArch64::ORR_PPzPP:
2001 case AArch64::BRKA_PPzP:
2002 case AArch64::BRKPA_PPzPP:
2003 case AArch64::BRKB_PPzP:
2004 case AArch64::BRKPB_PPzPP:
2005 case AArch64::RDFFR_PPz: {
2006 // Check to see if our mask is the same. If not the resulting flag bits
2007 // may be different and we can't remove the ptest.
2008 auto *PredMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
2009 if (Mask != PredMask)
2010 return {};
2011 break;
2012 }
2013 case AArch64::BRKN_PPzP: {
2014 // BRKN uses an all active implicit mask to set flags unlike the other
2015 // flag-setting instructions.
2016 // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B).
2017 if ((MaskOpcode != AArch64::PTRUE_B) ||
2018 (Mask->getOperand(i: 1).getImm() != 31))
2019 return {};
2020 break;
2021 }
2022 case AArch64::PTRUE_B:
2023 // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A)
2024 break;
2025 default:
2026 // Bail out if we don't recognize the input
2027 return {};
2028 }
2029
2030 return convertToFlagSettingOpc(Opc: PredOpcode);
2031}
2032
2033/// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
2034/// operation which could set the flags in an identical manner
2035bool AArch64InstrInfo::optimizePTestInstr(
2036 MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
2037 const MachineRegisterInfo *MRI) const {
2038 auto *Mask = MRI->getUniqueVRegDef(Reg: MaskReg);
2039 auto *Pred = MRI->getUniqueVRegDef(Reg: PredReg);
2040
2041 if (Pred->isCopy() && PTest->getOpcode() == AArch64::PTEST_PP_FIRST) {
2042 // Instructions which return a multi-vector (e.g. WHILECC_x2) require copies
2043 // before the branch to extract each subregister.
2044 auto Op = Pred->getOperand(i: 1);
2045 if (Op.isReg() && Op.getReg().isVirtual() &&
2046 Op.getSubReg() == AArch64::psub0)
2047 Pred = MRI->getUniqueVRegDef(Reg: Op.getReg());
2048 }
2049
2050 unsigned PredOpcode = Pred->getOpcode();
2051 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred, MRI);
2052 if (!NewOp)
2053 return false;
2054
2055 const TargetRegisterInfo *TRI = &getRegisterInfo();
2056
2057 // If another instruction between Pred and PTest accesses flags, don't remove
2058 // the ptest or update the earlier instruction to modify them.
2059 if (areCFlagsAccessedBetweenInstrs(From: Pred, To: PTest, TRI))
2060 return false;
2061
2062 // If we pass all the checks, it's safe to remove the PTEST and use the flags
2063 // as they are prior to PTEST. Sometimes this requires the tested PTEST
2064 // operand to be replaced with an equivalent instruction that also sets the
2065 // flags.
2066 PTest->eraseFromParent();
2067 if (*NewOp != PredOpcode) {
2068 Pred->setDesc(get(Opcode: *NewOp));
2069 updateOperandRegClass(Instr&: *Pred);
2070 Pred->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: TRI);
2071 }
2072
2073 // Ensure that the flags def is live.
2074 if (Pred->registerDefIsDead(Reg: AArch64::NZCV, TRI)) {
2075 unsigned i = 0, e = Pred->getNumOperands();
2076 for (; i != e; ++i) {
2077 MachineOperand &MO = Pred->getOperand(i);
2078 if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::NZCV) {
2079 MO.setIsDead(false);
2080 break;
2081 }
2082 }
2083 }
2084 return true;
2085}
2086
2087/// Try to optimize a compare instruction. A compare instruction is an
2088/// instruction which produces AArch64::NZCV. It can be truly compare
2089/// instruction
2090/// when there are no uses of its destination register.
2091///
2092/// The following steps are tried in order:
2093/// 1. Convert CmpInstr into an unconditional version.
2094/// 2. Remove CmpInstr if above there is an instruction producing a needed
2095/// condition code or an instruction which can be converted into such an
2096/// instruction.
2097/// Only comparison with zero is supported.
2098bool AArch64InstrInfo::optimizeCompareInstr(
2099 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2100 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2101 assert(CmpInstr.getParent());
2102 assert(MRI);
2103
2104 // Replace SUBSWrr with SUBWrr if NZCV is not used.
2105 int DeadNZCVIdx =
2106 CmpInstr.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
2107 if (DeadNZCVIdx != -1) {
2108 if (CmpInstr.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
2109 CmpInstr.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr)) {
2110 CmpInstr.eraseFromParent();
2111 return true;
2112 }
2113 unsigned Opc = CmpInstr.getOpcode();
2114 unsigned NewOpc = convertToNonFlagSettingOpc(MI: CmpInstr);
2115 if (NewOpc == Opc)
2116 return false;
2117 const MCInstrDesc &MCID = get(Opcode: NewOpc);
2118 CmpInstr.setDesc(MCID);
2119 CmpInstr.removeOperand(OpNo: DeadNZCVIdx);
2120 updateOperandRegClass(Instr&: CmpInstr);
2121 return true;
2122 }
2123
2124 if (CmpInstr.getOpcode() == AArch64::PTEST_PP ||
2125 CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY ||
2126 CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST)
2127 return optimizePTestInstr(PTest: &CmpInstr, MaskReg: SrcReg, PredReg: SrcReg2, MRI);
2128
2129 if (SrcReg2 != 0)
2130 return false;
2131
2132 // CmpInstr is a Compare instruction if destination register is not used.
2133 if (!MRI->use_nodbg_empty(RegNo: CmpInstr.getOperand(i: 0).getReg()))
2134 return false;
2135
2136 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, MRI: *MRI))
2137 return true;
2138 return (CmpValue == 0 || CmpValue == 1) &&
2139 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, MRI: *MRI);
2140}
2141
2142/// Get opcode of S version of Instr.
2143/// If Instr is S version its opcode is returned.
2144/// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
2145/// or we are not interested in it.
2146static unsigned sForm(MachineInstr &Instr) {
2147 switch (Instr.getOpcode()) {
2148 default:
2149 return AArch64::INSTRUCTION_LIST_END;
2150
2151 case AArch64::ADDSWrr:
2152 case AArch64::ADDSWri:
2153 case AArch64::ADDSXrr:
2154 case AArch64::ADDSXri:
2155 case AArch64::ADDSWrx:
2156 case AArch64::ADDSXrx:
2157 case AArch64::ADDSWrs:
2158 case AArch64::ADDSXrs:
2159 case AArch64::SUBSWrr:
2160 case AArch64::SUBSWri:
2161 case AArch64::SUBSWrx:
2162 case AArch64::SUBSWrs:
2163 case AArch64::SUBSXrr:
2164 case AArch64::SUBSXri:
2165 case AArch64::SUBSXrx:
2166 case AArch64::SUBSXrs:
2167 case AArch64::ANDSWri:
2168 case AArch64::ANDSWrr:
2169 case AArch64::ANDSWrs:
2170 case AArch64::ANDSXri:
2171 case AArch64::ANDSXrr:
2172 case AArch64::ANDSXrs:
2173 case AArch64::BICSWrr:
2174 case AArch64::BICSXrr:
2175 case AArch64::BICSWrs:
2176 case AArch64::BICSXrs:
2177 case AArch64::ADCSWr:
2178 case AArch64::ADCSXr:
2179 case AArch64::SBCSWr:
2180 case AArch64::SBCSXr:
2181 return Instr.getOpcode();
2182
2183 case AArch64::ADDWrr:
2184 return AArch64::ADDSWrr;
2185 case AArch64::ADDWri:
2186 return AArch64::ADDSWri;
2187 case AArch64::ADDXrr:
2188 return AArch64::ADDSXrr;
2189 case AArch64::ADDXri:
2190 return AArch64::ADDSXri;
2191 case AArch64::ADDWrx:
2192 return AArch64::ADDSWrx;
2193 case AArch64::ADDXrx:
2194 return AArch64::ADDSXrx;
2195 case AArch64::ADDWrs:
2196 return AArch64::ADDSWrs;
2197 case AArch64::ADDXrs:
2198 return AArch64::ADDSXrs;
2199 case AArch64::ADCWr:
2200 return AArch64::ADCSWr;
2201 case AArch64::ADCXr:
2202 return AArch64::ADCSXr;
2203 case AArch64::SUBWrr:
2204 return AArch64::SUBSWrr;
2205 case AArch64::SUBWri:
2206 return AArch64::SUBSWri;
2207 case AArch64::SUBXrr:
2208 return AArch64::SUBSXrr;
2209 case AArch64::SUBXri:
2210 return AArch64::SUBSXri;
2211 case AArch64::SUBWrx:
2212 return AArch64::SUBSWrx;
2213 case AArch64::SUBXrx:
2214 return AArch64::SUBSXrx;
2215 case AArch64::SUBWrs:
2216 return AArch64::SUBSWrs;
2217 case AArch64::SUBXrs:
2218 return AArch64::SUBSXrs;
2219 case AArch64::SBCWr:
2220 return AArch64::SBCSWr;
2221 case AArch64::SBCXr:
2222 return AArch64::SBCSXr;
2223 case AArch64::ANDWri:
2224 return AArch64::ANDSWri;
2225 case AArch64::ANDXri:
2226 return AArch64::ANDSXri;
2227 case AArch64::ANDWrr:
2228 return AArch64::ANDSWrr;
2229 case AArch64::ANDWrs:
2230 return AArch64::ANDSWrs;
2231 case AArch64::ANDXrr:
2232 return AArch64::ANDSXrr;
2233 case AArch64::ANDXrs:
2234 return AArch64::ANDSXrs;
2235 case AArch64::BICWrr:
2236 return AArch64::BICSWrr;
2237 case AArch64::BICXrr:
2238 return AArch64::BICSXrr;
2239 case AArch64::BICWrs:
2240 return AArch64::BICSWrs;
2241 case AArch64::BICXrs:
2242 return AArch64::BICSXrs;
2243 }
2244}
2245
2246/// Check if AArch64::NZCV should be alive in successors of MBB.
2247static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB) {
2248 for (auto *BB : MBB->successors())
2249 if (BB->isLiveIn(Reg: AArch64::NZCV))
2250 return true;
2251 return false;
2252}
2253
2254/// \returns The condition code operand index for \p Instr if it is a branch
2255/// or select and -1 otherwise.
2256int AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(
2257 const MachineInstr &Instr) {
2258 switch (Instr.getOpcode()) {
2259 default:
2260 return -1;
2261
2262 case AArch64::Bcc: {
2263 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2264 assert(Idx >= 2);
2265 return Idx - 2;
2266 }
2267
2268 case AArch64::CSINVWr:
2269 case AArch64::CSINVXr:
2270 case AArch64::CSINCWr:
2271 case AArch64::CSINCXr:
2272 case AArch64::CSELWr:
2273 case AArch64::CSELXr:
2274 case AArch64::CSNEGWr:
2275 case AArch64::CSNEGXr:
2276 case AArch64::FCSELSrrr:
2277 case AArch64::FCSELDrrr: {
2278 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2279 assert(Idx >= 1);
2280 return Idx - 1;
2281 }
2282 }
2283}
2284
2285/// Find a condition code used by the instruction.
2286/// Returns AArch64CC::Invalid if either the instruction does not use condition
2287/// codes or we don't optimize CmpInstr in the presence of such instructions.
2288static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
2289 int CCIdx =
2290 AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(Instr);
2291 return CCIdx >= 0 ? static_cast<AArch64CC::CondCode>(
2292 Instr.getOperand(i: CCIdx).getImm())
2293 : AArch64CC::Invalid;
2294}
2295
2296static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
2297 assert(CC != AArch64CC::Invalid);
2298 UsedNZCV UsedFlags;
2299 switch (CC) {
2300 default:
2301 break;
2302
2303 case AArch64CC::EQ: // Z set
2304 case AArch64CC::NE: // Z clear
2305 UsedFlags.Z = true;
2306 break;
2307
2308 case AArch64CC::HI: // Z clear and C set
2309 case AArch64CC::LS: // Z set or C clear
2310 UsedFlags.Z = true;
2311 [[fallthrough]];
2312 case AArch64CC::HS: // C set
2313 case AArch64CC::LO: // C clear
2314 UsedFlags.C = true;
2315 break;
2316
2317 case AArch64CC::MI: // N set
2318 case AArch64CC::PL: // N clear
2319 UsedFlags.N = true;
2320 break;
2321
2322 case AArch64CC::VS: // V set
2323 case AArch64CC::VC: // V clear
2324 UsedFlags.V = true;
2325 break;
2326
2327 case AArch64CC::GT: // Z clear, N and V the same
2328 case AArch64CC::LE: // Z set, N and V differ
2329 UsedFlags.Z = true;
2330 [[fallthrough]];
2331 case AArch64CC::GE: // N and V the same
2332 case AArch64CC::LT: // N and V differ
2333 UsedFlags.N = true;
2334 UsedFlags.V = true;
2335 break;
2336 }
2337 return UsedFlags;
2338}
2339
2340/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
2341/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
2342/// \returns std::nullopt otherwise.
2343///
2344/// Collect instructions using that flags in \p CCUseInstrs if provided.
2345std::optional<UsedNZCV>
2346llvm::examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
2347 const TargetRegisterInfo &TRI,
2348 SmallVectorImpl<MachineInstr *> *CCUseInstrs) {
2349 MachineBasicBlock *CmpParent = CmpInstr.getParent();
2350 if (MI.getParent() != CmpParent)
2351 return std::nullopt;
2352
2353 if (areCFlagsAliveInSuccessors(MBB: CmpParent))
2354 return std::nullopt;
2355
2356 UsedNZCV NZCVUsedAfterCmp;
2357 for (MachineInstr &Instr : instructionsWithoutDebug(
2358 It: std::next(x: CmpInstr.getIterator()), End: CmpParent->instr_end())) {
2359 if (Instr.readsRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
2360 AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
2361 if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
2362 return std::nullopt;
2363 NZCVUsedAfterCmp |= getUsedNZCV(CC);
2364 if (CCUseInstrs)
2365 CCUseInstrs->push_back(Elt: &Instr);
2366 }
2367 if (Instr.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI))
2368 break;
2369 }
2370 return NZCVUsedAfterCmp;
2371}
2372
2373static bool isADDSRegImm(unsigned Opcode) {
2374 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
2375}
2376
2377static bool isSUBSRegImm(unsigned Opcode) {
2378 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
2379}
2380
2381static bool isANDOpcode(MachineInstr &MI) {
2382 unsigned Opc = sForm(Instr&: MI);
2383 switch (Opc) {
2384 case AArch64::ANDSWri:
2385 case AArch64::ANDSWrr:
2386 case AArch64::ANDSWrs:
2387 case AArch64::ANDSXri:
2388 case AArch64::ANDSXrr:
2389 case AArch64::ANDSXrs:
2390 case AArch64::BICSWrr:
2391 case AArch64::BICSXrr:
2392 case AArch64::BICSWrs:
2393 case AArch64::BICSXrs:
2394 return true;
2395 default:
2396 return false;
2397 }
2398}
2399
2400/// Check if CmpInstr can be substituted by MI.
2401///
2402/// CmpInstr can be substituted:
2403/// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2404/// - and, MI and CmpInstr are from the same MachineBB
2405/// - and, condition flags are not alive in successors of the CmpInstr parent
2406/// - and, if MI opcode is the S form there must be no defs of flags between
2407/// MI and CmpInstr
2408/// or if MI opcode is not the S form there must be neither defs of flags
2409/// nor uses of flags between MI and CmpInstr.
2410/// - and, C is not used after CmpInstr; CmpInstr's C is from adds/subs #0 on
2411/// SrcReg and can differ from MI (e.g. carry out of ADCS/SBCS).
2412/// - and, V is not used after CmpInstr unless MI is AND/BIC (V cleared) or MI
2413/// has NoSWrap (overflow is poison and the fold is still safe).
2414static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr,
2415 const TargetRegisterInfo &TRI) {
2416 // MI is an opcode sForm maps (add/sub/adc/sbc/and/bic and their S forms).
2417 assert(sForm(MI) != AArch64::INSTRUCTION_LIST_END);
2418
2419 const unsigned CmpOpcode = CmpInstr.getOpcode();
2420 if (!isADDSRegImm(Opcode: CmpOpcode) && !isSUBSRegImm(Opcode: CmpOpcode))
2421 return false;
2422
2423 assert((CmpInstr.getOperand(2).isImm() &&
2424 CmpInstr.getOperand(2).getImm() == 0) &&
2425 "Caller guarantees that CmpInstr compares with constant 0");
2426
2427 std::optional<UsedNZCV> NZVCUsed = examineCFlagsUse(MI, CmpInstr, TRI);
2428 if (!NZVCUsed || NZVCUsed->C)
2429 return false;
2430
2431 // CmpInstr is ADDS/SUBS with immediate 0 on SrcReg (compare SrcReg to zero).
2432 // After the fold, users see NZCV from MI (or its S form), not from CmpInstr.
2433 // N/Z match CmpInstr for the value in SrcReg; C/V need not match in general
2434 // (e.g. ADCS vs adds #0), so we require C unused after CmpInstr and gate V
2435 // as below. NoSWrap makes signed overflow poison; AND/BIC clear V.
2436 if (NZVCUsed->V && !MI.getFlag(Flag: MachineInstr::NoSWrap) && !isANDOpcode(MI))
2437 return false;
2438
2439 AccessKind AccessToCheck = AK_Write;
2440 if (sForm(Instr&: MI) != MI.getOpcode())
2441 AccessToCheck = AK_All;
2442 return !areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck);
2443}
2444
2445/// Substitute an instruction comparing to zero with another instruction
2446/// which produces needed condition flags.
2447///
2448/// Return true on success.
2449bool AArch64InstrInfo::substituteCmpToZero(
2450 MachineInstr &CmpInstr, unsigned SrcReg,
2451 const MachineRegisterInfo &MRI) const {
2452 // Get the unique definition of SrcReg.
2453 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2454 if (!MI)
2455 return false;
2456
2457 const TargetRegisterInfo &TRI = getRegisterInfo();
2458
2459 unsigned NewOpc = sForm(Instr&: *MI);
2460 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
2461 return false;
2462
2463 if (!canInstrSubstituteCmpInstr(MI&: *MI, CmpInstr, TRI))
2464 return false;
2465
2466 // Update the instruction to set NZCV.
2467 MI->setDesc(get(Opcode: NewOpc));
2468 CmpInstr.eraseFromParent();
2469 updateOperandRegClass(Instr&: *MI);
2470 MI->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: &TRI);
2471 return true;
2472}
2473
2474/// \returns True if \p CmpInstr can be removed.
2475///
2476/// \p IsInvertCC is true if, after removing \p CmpInstr, condition
2477/// codes used in \p CCUseInstrs must be inverted.
2478static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr,
2479 int CmpValue, const TargetRegisterInfo &TRI,
2480 SmallVectorImpl<MachineInstr *> &CCUseInstrs,
2481 bool &IsInvertCC) {
2482 assert((CmpValue == 0 || CmpValue == 1) &&
2483 "Only comparisons to 0 or 1 considered for removal!");
2484
2485 // MI is 'CSINCWr %vreg, wzr, wzr, <cc>' or 'CSINCXr %vreg, xzr, xzr, <cc>'
2486 unsigned MIOpc = MI.getOpcode();
2487 if (MIOpc == AArch64::CSINCWr) {
2488 if (MI.getOperand(i: 1).getReg() != AArch64::WZR ||
2489 MI.getOperand(i: 2).getReg() != AArch64::WZR)
2490 return false;
2491 } else if (MIOpc == AArch64::CSINCXr) {
2492 if (MI.getOperand(i: 1).getReg() != AArch64::XZR ||
2493 MI.getOperand(i: 2).getReg() != AArch64::XZR)
2494 return false;
2495 } else {
2496 return false;
2497 }
2498 AArch64CC::CondCode MICC = findCondCodeUsedByInstr(Instr: MI);
2499 if (MICC == AArch64CC::Invalid)
2500 return false;
2501
2502 // NZCV needs to be defined
2503 if (MI.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) != -1)
2504 return false;
2505
2506 // CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0' or 'SUBS %vreg, 1'
2507 const unsigned CmpOpcode = CmpInstr.getOpcode();
2508 bool IsSubsRegImm = isSUBSRegImm(Opcode: CmpOpcode);
2509 if (CmpValue && !IsSubsRegImm)
2510 return false;
2511 if (!CmpValue && !IsSubsRegImm && !isADDSRegImm(Opcode: CmpOpcode))
2512 return false;
2513
2514 // MI conditions allowed: eq, ne, mi, pl
2515 UsedNZCV MIUsedNZCV = getUsedNZCV(CC: MICC);
2516 if (MIUsedNZCV.C || MIUsedNZCV.V)
2517 return false;
2518
2519 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2520 examineCFlagsUse(MI, CmpInstr, TRI, CCUseInstrs: &CCUseInstrs);
2521 // Condition flags are not used in CmpInstr basic block successors and only
2522 // Z or N flags allowed to be used after CmpInstr within its basic block
2523 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2524 return false;
2525 // Z or N flag used after CmpInstr must correspond to the flag used in MI
2526 if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) ||
2527 (MIUsedNZCV.N && NZCVUsedAfterCmp->Z))
2528 return false;
2529 // If CmpInstr is comparison to zero MI conditions are limited to eq, ne
2530 if (MIUsedNZCV.N && !CmpValue)
2531 return false;
2532
2533 // There must be no defs of flags between MI and CmpInstr
2534 if (areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck: AK_Write))
2535 return false;
2536
2537 // Condition code is inverted in the following cases:
2538 // 1. MI condition is ne; CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2539 // 2. MI condition is eq, pl; CmpInstr is 'SUBS %vreg, 1'
2540 IsInvertCC = (CmpValue && (MICC == AArch64CC::EQ || MICC == AArch64CC::PL)) ||
2541 (!CmpValue && MICC == AArch64CC::NE);
2542 return true;
2543}
2544
2545/// Remove comparison in csinc-cmp sequence
2546///
2547/// Examples:
2548/// 1. \code
2549/// csinc w9, wzr, wzr, ne
2550/// cmp w9, #0
2551/// b.eq
2552/// \endcode
2553/// to
2554/// \code
2555/// csinc w9, wzr, wzr, ne
2556/// b.ne
2557/// \endcode
2558///
2559/// 2. \code
2560/// csinc x2, xzr, xzr, mi
2561/// cmp x2, #1
2562/// b.pl
2563/// \endcode
2564/// to
2565/// \code
2566/// csinc x2, xzr, xzr, mi
2567/// b.pl
2568/// \endcode
2569///
2570/// \param CmpInstr comparison instruction
2571/// \return True when comparison removed
2572bool AArch64InstrInfo::removeCmpToZeroOrOne(
2573 MachineInstr &CmpInstr, unsigned SrcReg, int CmpValue,
2574 const MachineRegisterInfo &MRI) const {
2575 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2576 if (!MI)
2577 return false;
2578 const TargetRegisterInfo &TRI = getRegisterInfo();
2579 SmallVector<MachineInstr *, 4> CCUseInstrs;
2580 bool IsInvertCC = false;
2581 if (!canCmpInstrBeRemoved(MI&: *MI, CmpInstr, CmpValue, TRI, CCUseInstrs,
2582 IsInvertCC))
2583 return false;
2584 // Make transformation
2585 CmpInstr.eraseFromParent();
2586 if (IsInvertCC) {
2587 // Invert condition codes in CmpInstr CC users
2588 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2589 int Idx = findCondCodeUseOperandIdxForBranchOrSelect(Instr: *CCUseInstr);
2590 assert(Idx >= 0 && "Unexpected instruction using CC.");
2591 MachineOperand &CCOperand = CCUseInstr->getOperand(i: Idx);
2592 AArch64CC::CondCode CCUse = AArch64CC::getInvertedCondCode(
2593 Code: static_cast<AArch64CC::CondCode>(CCOperand.getImm()));
2594 CCOperand.setImm(CCUse);
2595 }
2596 }
2597 return true;
2598}
2599
2600bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2601 if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2602 MI.getOpcode() != AArch64::CATCHRET &&
2603 MI.getOpcode() != AArch64::STACK_GUARD_UNMIX)
2604 return false;
2605
2606 MachineBasicBlock &MBB = *MI.getParent();
2607 auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
2608 auto TRI = Subtarget.getRegisterInfo();
2609 DebugLoc DL = MI.getDebugLoc();
2610
2611 if (MI.getOpcode() == AArch64::STACK_GUARD_UNMIX) {
2612 // Expand STACK_GUARD_UNMIX to: sub Rd, fp, Rs
2613 // This computes FP - stored_mixed_value to unmix the cookie
2614 Register DstReg = MI.getOperand(i: 0).getReg();
2615 Register SrcReg = MI.getOperand(i: 1).getReg();
2616
2617 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: DstReg)
2618 .addReg(RegNo: AArch64::FP)
2619 .addReg(RegNo: SrcReg);
2620
2621 MBB.erase(I: MI);
2622 return true;
2623 }
2624
2625 if (MI.getOpcode() == AArch64::CATCHRET) {
2626 // Skip to the first instruction before the epilog.
2627 const TargetInstrInfo *TII =
2628 MBB.getParent()->getSubtarget().getInstrInfo();
2629 MachineBasicBlock *TargetMBB = MI.getOperand(i: 0).getMBB();
2630 auto MBBI = MachineBasicBlock::iterator(MI);
2631 MachineBasicBlock::iterator FirstEpilogSEH = std::prev(x: MBBI);
2632 while (FirstEpilogSEH->getFlag(Flag: MachineInstr::FrameDestroy) &&
2633 FirstEpilogSEH != MBB.begin())
2634 FirstEpilogSEH = std::prev(x: FirstEpilogSEH);
2635 if (FirstEpilogSEH != MBB.begin())
2636 FirstEpilogSEH = std::next(x: FirstEpilogSEH);
2637 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADRP))
2638 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2639 .addMBB(MBB: TargetMBB, TargetFlags: AArch64II::MO_PAGE);
2640 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri))
2641 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2642 .addReg(RegNo: AArch64::X0)
2643 .addMBB(MBB: TargetMBB, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
2644 .addImm(Val: 0);
2645 TargetMBB->setMachineBlockAddressTaken();
2646 return true;
2647 }
2648
2649 Register Reg = MI.getOperand(i: 0).getReg();
2650 Module &M = *MBB.getParent()->getFunction().getParent();
2651 if (M.getStackProtectorGuard() == "sysreg") {
2652 const AArch64SysReg::SysReg *SrcReg =
2653 AArch64SysReg::lookupSysRegByName(Name: M.getStackProtectorGuardReg());
2654 if (!SrcReg)
2655 report_fatal_error(reason: "Unknown SysReg for Stack Protector Guard Register");
2656
2657 // mrs xN, sysreg
2658 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MRS))
2659 .addDef(RegNo: Reg, Flags: RegState::Renamable)
2660 .addImm(Val: SrcReg->Encoding);
2661 int Offset = M.getStackProtectorGuardOffset();
2662 if (Offset >= 0 && Offset <= 32760 && Offset % 8 == 0) {
2663 // ldr xN, [xN, #offset]
2664 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2665 .addDef(RegNo: Reg)
2666 .addUse(RegNo: Reg, Flags: RegState::Kill)
2667 .addImm(Val: Offset / 8);
2668 } else if (Offset >= -256 && Offset <= 255) {
2669 // ldur xN, [xN, #offset]
2670 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDURXi))
2671 .addDef(RegNo: Reg)
2672 .addUse(RegNo: Reg, Flags: RegState::Kill)
2673 .addImm(Val: Offset);
2674 } else if (Offset >= -4095 && Offset <= 4095) {
2675 if (Offset > 0) {
2676 // add xN, xN, #offset
2677 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri))
2678 .addDef(RegNo: Reg)
2679 .addUse(RegNo: Reg, Flags: RegState::Kill)
2680 .addImm(Val: Offset)
2681 .addImm(Val: 0);
2682 } else {
2683 // sub xN, xN, #offset
2684 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXri))
2685 .addDef(RegNo: Reg)
2686 .addUse(RegNo: Reg, Flags: RegState::Kill)
2687 .addImm(Val: -Offset)
2688 .addImm(Val: 0);
2689 }
2690 // ldr xN, [xN]
2691 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2692 .addDef(RegNo: Reg)
2693 .addUse(RegNo: Reg, Flags: RegState::Kill)
2694 .addImm(Val: 0);
2695 } else {
2696 // Cases that are larger than +/- 4095 and not a multiple of 8, or larger
2697 // than 23760.
2698 // It might be nice to use AArch64::MOVi32imm here, which would get
2699 // expanded in PreSched2 after PostRA, but our lone scratch Reg already
2700 // contains the MRS result. findScratchNonCalleeSaveRegister() in
2701 // AArch64FrameLowering might help us find such a scratch register
2702 // though. If we failed to find a scratch register, we could emit a
2703 // stream of add instructions to build up the immediate. Or, we could try
2704 // to insert a AArch64::MOVi32imm before register allocation so that we
2705 // didn't need to scavenge for a scratch register.
2706 report_fatal_error(reason: "Unable to encode Stack Protector Guard Offset");
2707 }
2708 MBB.erase(I: MI);
2709 return true;
2710 }
2711
2712 const GlobalValue *GV =
2713 cast<GlobalValue>(Val: (*MI.memoperands_begin())->getValue());
2714 const TargetMachine &TM = MBB.getParent()->getTarget();
2715 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2716 const unsigned char MO_NC = AArch64II::MO_NC;
2717
2718 unsigned GuardWidth = M.getStackProtectorGuardValueWidth().value_or(
2719 u: Subtarget.isTargetILP32() ? 4 : 8);
2720 if (GuardWidth != 4 && GuardWidth != 8)
2721 report_fatal_error(reason: "Unsupported stack protector value width");
2722 if ((OpFlags & AArch64II::MO_GOT) != 0) {
2723 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LOADgot), DestReg: Reg)
2724 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags);
2725 if (GuardWidth == 4) {
2726 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2727 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2728 .addDef(RegNo: Reg32)
2729 .addUse(RegNo: Reg, Flags: RegState::Kill)
2730 .addImm(Val: 0)
2731 .addMemOperand(MMO: *MI.memoperands_begin())
2732 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2733 } else {
2734 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2735 .addReg(RegNo: Reg, Flags: RegState::Kill)
2736 .addImm(Val: 0)
2737 .addMemOperand(MMO: *MI.memoperands_begin());
2738 }
2739 } else if (TM.getCodeModel() == CodeModel::Large) {
2740 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg)
2741 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G0 | MO_NC)
2742 .addImm(Val: 0);
2743 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2744 .addReg(RegNo: Reg, Flags: RegState::Kill)
2745 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G1 | MO_NC)
2746 .addImm(Val: 16);
2747 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2748 .addReg(RegNo: Reg, Flags: RegState::Kill)
2749 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G2 | MO_NC)
2750 .addImm(Val: 32);
2751 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2752 .addReg(RegNo: Reg, Flags: RegState::Kill)
2753 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G3)
2754 .addImm(Val: 48);
2755 if (GuardWidth == 4) {
2756 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2757 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2758 .addDef(RegNo: Reg32)
2759 .addUse(RegNo: Reg, Flags: RegState::Kill)
2760 .addImm(Val: 0)
2761 .addMemOperand(MMO: *MI.memoperands_begin())
2762 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2763 } else {
2764 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2765 .addReg(RegNo: Reg, Flags: RegState::Kill)
2766 .addImm(Val: 0)
2767 .addMemOperand(MMO: *MI.memoperands_begin());
2768 }
2769 } else {
2770 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
2771 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags | AArch64II::MO_PAGE);
2772 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
2773 if (GuardWidth == 4) {
2774 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2775 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2776 .addDef(RegNo: Reg32)
2777 .addUse(RegNo: Reg, Flags: RegState::Kill)
2778 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2779 .addMemOperand(MMO: *MI.memoperands_begin())
2780 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2781 } else {
2782 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2783 .addReg(RegNo: Reg, Flags: RegState::Kill)
2784 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2785 .addMemOperand(MMO: *MI.memoperands_begin());
2786 }
2787 }
2788 // To match MSVC. Unlike x86_64 which uses xor instruction to mix the cookie,
2789 // we use sub instruction to mix the cookie on aarch64.
2790 // The mixing happens here in expandPostRAPseudo (after RA) to ensure we use
2791 // the final frame pointer value.
2792 if (Subtarget.getTargetTriple().isOSMSVCRT())
2793 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: Reg)
2794 .addReg(RegNo: AArch64::FP)
2795 .addReg(RegNo: Reg, Flags: RegState::Kill);
2796
2797 MBB.erase(I: MI);
2798
2799 return true;
2800}
2801
2802// Return true if this instruction simply sets its single destination register
2803// to zero. This is equivalent to a register rename of the zero-register.
2804bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
2805 switch (MI.getOpcode()) {
2806 default:
2807 break;
2808 case AArch64::MOVZWi:
2809 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
2810 if (MI.getOperand(i: 1).isImm() && MI.getOperand(i: 1).getImm() == 0) {
2811 assert(MI.getDesc().getNumOperands() == 3 &&
2812 MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
2813 return true;
2814 }
2815 break;
2816 case AArch64::ANDWri: // and Rd, Rzr, #imm
2817 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2818 case AArch64::ANDXri:
2819 return MI.getOperand(i: 1).getReg() == AArch64::XZR;
2820 case TargetOpcode::COPY:
2821 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2822 }
2823 return false;
2824}
2825
2826// Return true if this instruction simply renames a general register without
2827// modifying bits.
2828bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
2829 switch (MI.getOpcode()) {
2830 default:
2831 break;
2832 case TargetOpcode::COPY: {
2833 // GPR32 copies will by lowered to ORRXrs
2834 Register DstReg = MI.getOperand(i: 0).getReg();
2835 return DstReg.isPhysical() && (AArch64::GPR32RegClass.contains(Reg: DstReg) ||
2836 AArch64::GPR64RegClass.contains(Reg: DstReg));
2837 }
2838 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
2839 if (MI.getOperand(i: 1).getReg() == AArch64::XZR) {
2840 assert(MI.getDesc().getNumOperands() == 4 &&
2841 MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
2842 return true;
2843 }
2844 break;
2845 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
2846 if (MI.getOperand(i: 2).getImm() == 0) {
2847 assert(MI.getDesc().getNumOperands() == 4 &&
2848 MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
2849 return true;
2850 }
2851 break;
2852 }
2853 return false;
2854}
2855
2856// Return true if this instruction simply renames a general register without
2857// modifying bits.
2858bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
2859 switch (MI.getOpcode()) {
2860 default:
2861 break;
2862 case TargetOpcode::COPY: {
2863 Register DstReg = MI.getOperand(i: 0).getReg();
2864 return DstReg.isPhysical() && AArch64::FPR128RegClass.contains(Reg: DstReg);
2865 }
2866 case AArch64::ORRv16i8:
2867 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
2868 assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
2869 "invalid ORRv16i8 operands");
2870 return true;
2871 }
2872 break;
2873 }
2874 return false;
2875}
2876
2877static bool isFrameLoadOpcode(int Opcode) {
2878 switch (Opcode) {
2879 default:
2880 return false;
2881 case AArch64::LDRWui:
2882 case AArch64::LDRXui:
2883 case AArch64::LDRBui:
2884 case AArch64::LDRHui:
2885 case AArch64::LDRSui:
2886 case AArch64::LDRDui:
2887 case AArch64::LDRQui:
2888 case AArch64::LDR_PXI:
2889 return true;
2890 }
2891}
2892
2893Register AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
2894 int &FrameIndex) const {
2895 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2896 return Register();
2897
2898 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2899 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2900 FrameIndex = MI.getOperand(i: 1).getIndex();
2901 return MI.getOperand(i: 0).getReg();
2902 }
2903 return Register();
2904}
2905
2906static bool isFrameStoreOpcode(int Opcode) {
2907 switch (Opcode) {
2908 default:
2909 return false;
2910 case AArch64::STRWui:
2911 case AArch64::STRXui:
2912 case AArch64::STRBui:
2913 case AArch64::STRHui:
2914 case AArch64::STRSui:
2915 case AArch64::STRDui:
2916 case AArch64::STRQui:
2917 case AArch64::STR_PXI:
2918 return true;
2919 }
2920}
2921
2922Register AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
2923 int &FrameIndex) const {
2924 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2925 return Register();
2926
2927 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2928 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2929 FrameIndex = MI.getOperand(i: 1).getIndex();
2930 return MI.getOperand(i: 0).getReg();
2931 }
2932 return Register();
2933}
2934
2935Register AArch64InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
2936 int &FrameIndex) const {
2937 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2938 return Register();
2939
2940 if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
2941 return Reg;
2942
2943 SmallVector<const MachineMemOperand *, 1> Accesses;
2944 if (hasStoreToStackSlot(MI, Accesses)) {
2945 if (Accesses.size() > 1)
2946 return Register();
2947
2948 FrameIndex =
2949 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2950 ->getFrameIndex();
2951 return MI.getOperand(i: 0).getReg();
2952 }
2953 return Register();
2954}
2955
2956Register AArch64InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
2957 int &FrameIndex) const {
2958 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2959 return Register();
2960
2961 if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
2962 return Reg;
2963
2964 SmallVector<const MachineMemOperand *, 1> Accesses;
2965 if (hasLoadFromStackSlot(MI, Accesses)) {
2966 if (Accesses.size() > 1)
2967 return Register();
2968
2969 FrameIndex =
2970 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2971 ->getFrameIndex();
2972 return MI.getOperand(i: 0).getReg();
2973 }
2974 return Register();
2975}
2976
2977/// Check all MachineMemOperands for a hint to suppress pairing.
2978bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
2979 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2980 return MMO->getFlags() & MOSuppressPair;
2981 });
2982}
2983
2984/// Set a flag on the first MachineMemOperand to suppress pairing.
2985void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
2986 if (MI.memoperands_empty())
2987 return;
2988 (*MI.memoperands_begin())->setFlags(MOSuppressPair);
2989}
2990
2991/// Check all MachineMemOperands for a hint that the load/store is strided.
2992bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
2993 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2994 return MMO->getFlags() & MOStridedAccess;
2995 });
2996}
2997
2998bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
2999 switch (Opc) {
3000 default:
3001 return false;
3002 case AArch64::STURSi:
3003 case AArch64::STRSpre:
3004 case AArch64::STURDi:
3005 case AArch64::STRDpre:
3006 case AArch64::STURQi:
3007 case AArch64::STRQpre:
3008 case AArch64::STURBBi:
3009 case AArch64::STURHHi:
3010 case AArch64::STURWi:
3011 case AArch64::STRWpre:
3012 case AArch64::STURXi:
3013 case AArch64::STRXpre:
3014 case AArch64::LDURSi:
3015 case AArch64::LDRSpre:
3016 case AArch64::LDURDi:
3017 case AArch64::LDRDpre:
3018 case AArch64::LDURQi:
3019 case AArch64::LDRQpre:
3020 case AArch64::LDURWi:
3021 case AArch64::LDRWpre:
3022 case AArch64::LDURXi:
3023 case AArch64::LDRXpre:
3024 case AArch64::LDRSWpre:
3025 case AArch64::LDURSWi:
3026 case AArch64::LDURHHi:
3027 case AArch64::LDURBBi:
3028 case AArch64::LDURSBWi:
3029 case AArch64::LDURSHWi:
3030 return true;
3031 }
3032}
3033
3034std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
3035 switch (Opc) {
3036 default: return {};
3037 case AArch64::PRFMui: return AArch64::PRFUMi;
3038 case AArch64::LDRXui: return AArch64::LDURXi;
3039 case AArch64::LDRWui: return AArch64::LDURWi;
3040 case AArch64::LDRBui: return AArch64::LDURBi;
3041 case AArch64::LDRHui: return AArch64::LDURHi;
3042 case AArch64::LDRSui: return AArch64::LDURSi;
3043 case AArch64::LDRDui: return AArch64::LDURDi;
3044 case AArch64::LDRQui: return AArch64::LDURQi;
3045 case AArch64::LDRBBui: return AArch64::LDURBBi;
3046 case AArch64::LDRHHui: return AArch64::LDURHHi;
3047 case AArch64::LDRSBXui: return AArch64::LDURSBXi;
3048 case AArch64::LDRSBWui: return AArch64::LDURSBWi;
3049 case AArch64::LDRSHXui: return AArch64::LDURSHXi;
3050 case AArch64::LDRSHWui: return AArch64::LDURSHWi;
3051 case AArch64::LDRSWui: return AArch64::LDURSWi;
3052 case AArch64::STRXui: return AArch64::STURXi;
3053 case AArch64::STRWui: return AArch64::STURWi;
3054 case AArch64::STRBui: return AArch64::STURBi;
3055 case AArch64::STRHui: return AArch64::STURHi;
3056 case AArch64::STRSui: return AArch64::STURSi;
3057 case AArch64::STRDui: return AArch64::STURDi;
3058 case AArch64::STRQui: return AArch64::STURQi;
3059 case AArch64::STRBBui: return AArch64::STURBBi;
3060 case AArch64::STRHHui: return AArch64::STURHHi;
3061 }
3062}
3063
3064unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
3065 switch (Opc) {
3066 default:
3067 llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
3068 case AArch64::ADDG:
3069 case AArch64::LDAPURBi:
3070 case AArch64::LDAPURHi:
3071 case AArch64::LDAPURi:
3072 case AArch64::LDAPURSBWi:
3073 case AArch64::LDAPURSBXi:
3074 case AArch64::LDAPURSHWi:
3075 case AArch64::LDAPURSHXi:
3076 case AArch64::LDAPURSWi:
3077 case AArch64::LDAPURXi:
3078 case AArch64::LDR_PPXI:
3079 case AArch64::LDR_PXI:
3080 case AArch64::LDR_ZXI:
3081 case AArch64::LDR_ZZXI:
3082 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
3083 case AArch64::LDR_ZZZXI:
3084 case AArch64::LDR_ZZZZXI:
3085 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
3086 case AArch64::LDRBBui:
3087 case AArch64::LDRBui:
3088 case AArch64::LDRDui:
3089 case AArch64::LDRHHui:
3090 case AArch64::LDRHui:
3091 case AArch64::LDRQui:
3092 case AArch64::LDRSBWui:
3093 case AArch64::LDRSBXui:
3094 case AArch64::LDRSHWui:
3095 case AArch64::LDRSHXui:
3096 case AArch64::LDRSui:
3097 case AArch64::LDRSWui:
3098 case AArch64::LDRWui:
3099 case AArch64::LDRXui:
3100 case AArch64::LDURBBi:
3101 case AArch64::LDURBi:
3102 case AArch64::LDURDi:
3103 case AArch64::LDURHHi:
3104 case AArch64::LDURHi:
3105 case AArch64::LDURQi:
3106 case AArch64::LDURSBWi:
3107 case AArch64::LDURSBXi:
3108 case AArch64::LDURSHWi:
3109 case AArch64::LDURSHXi:
3110 case AArch64::LDURSi:
3111 case AArch64::LDURSWi:
3112 case AArch64::LDURWi:
3113 case AArch64::LDURXi:
3114 case AArch64::PRFMui:
3115 case AArch64::PRFUMi:
3116 case AArch64::ST2Gi:
3117 case AArch64::STGi:
3118 case AArch64::STLURBi:
3119 case AArch64::STLURHi:
3120 case AArch64::STLURWi:
3121 case AArch64::STLURXi:
3122 case AArch64::StoreSwiftAsyncContext:
3123 case AArch64::STR_PPXI:
3124 case AArch64::STR_PXI:
3125 case AArch64::STR_ZXI:
3126 case AArch64::STR_ZZXI:
3127 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
3128 case AArch64::STR_ZZZXI:
3129 case AArch64::STR_ZZZZXI:
3130 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
3131 case AArch64::STRBBui:
3132 case AArch64::STRBui:
3133 case AArch64::STRDui:
3134 case AArch64::STRHHui:
3135 case AArch64::STRHui:
3136 case AArch64::STRQui:
3137 case AArch64::STRSui:
3138 case AArch64::STRWui:
3139 case AArch64::STRXui:
3140 case AArch64::STURBBi:
3141 case AArch64::STURBi:
3142 case AArch64::STURDi:
3143 case AArch64::STURHHi:
3144 case AArch64::STURHi:
3145 case AArch64::STURQi:
3146 case AArch64::STURSi:
3147 case AArch64::STURWi:
3148 case AArch64::STURXi:
3149 case AArch64::STZ2Gi:
3150 case AArch64::STZGi:
3151 case AArch64::TAGPstack:
3152 case AArch64::ATOMIC_STORE_HINT_Bi:
3153 case AArch64::ATOMIC_STORE_HINT_Hi:
3154 case AArch64::ATOMIC_STORE_HINT_Wi:
3155 case AArch64::ATOMIC_STORE_HINT_Si:
3156 case AArch64::ATOMIC_STORE_HINT_Xi:
3157 case AArch64::ATOMIC_STORE_HINT_Di:
3158 case AArch64::ATOMIC_STORE_HINT_Bui:
3159 case AArch64::ATOMIC_STORE_HINT_Hui:
3160 case AArch64::ATOMIC_STORE_HINT_Wui:
3161 case AArch64::ATOMIC_STORE_HINT_Sui:
3162 case AArch64::ATOMIC_STORE_HINT_Xui:
3163 case AArch64::ATOMIC_STORE_HINT_Dui:
3164 return 2;
3165 case AArch64::LD1B_D_IMM:
3166 case AArch64::LD1B_H_IMM:
3167 case AArch64::LD1B_IMM:
3168 case AArch64::LD1B_S_IMM:
3169 case AArch64::LD1D_IMM:
3170 case AArch64::LD1H_D_IMM:
3171 case AArch64::LD1H_IMM:
3172 case AArch64::LD1H_S_IMM:
3173 case AArch64::LD1RB_D_IMM:
3174 case AArch64::LD1RB_H_IMM:
3175 case AArch64::LD1RB_IMM:
3176 case AArch64::LD1RB_S_IMM:
3177 case AArch64::LD1RD_IMM:
3178 case AArch64::LD1RH_D_IMM:
3179 case AArch64::LD1RH_IMM:
3180 case AArch64::LD1RH_S_IMM:
3181 case AArch64::LD1RSB_D_IMM:
3182 case AArch64::LD1RSB_H_IMM:
3183 case AArch64::LD1RSB_S_IMM:
3184 case AArch64::LD1RSH_D_IMM:
3185 case AArch64::LD1RSH_S_IMM:
3186 case AArch64::LD1RSW_IMM:
3187 case AArch64::LD1RW_D_IMM:
3188 case AArch64::LD1RW_IMM:
3189 case AArch64::LD1SB_D_IMM:
3190 case AArch64::LD1SB_H_IMM:
3191 case AArch64::LD1SB_S_IMM:
3192 case AArch64::LD1SH_D_IMM:
3193 case AArch64::LD1SH_S_IMM:
3194 case AArch64::LD1SW_D_IMM:
3195 case AArch64::LD1W_D_IMM:
3196 case AArch64::LD1W_IMM:
3197 case AArch64::LD2B_IMM:
3198 case AArch64::LD2D_IMM:
3199 case AArch64::LD2H_IMM:
3200 case AArch64::LD2W_IMM:
3201 case AArch64::LD3B_IMM:
3202 case AArch64::LD3D_IMM:
3203 case AArch64::LD3H_IMM:
3204 case AArch64::LD3W_IMM:
3205 case AArch64::LD4B_IMM:
3206 case AArch64::LD4D_IMM:
3207 case AArch64::LD4H_IMM:
3208 case AArch64::LD4W_IMM:
3209 case AArch64::LDG:
3210 case AArch64::LDNF1B_D_IMM:
3211 case AArch64::LDNF1B_H_IMM:
3212 case AArch64::LDNF1B_IMM:
3213 case AArch64::LDNF1B_S_IMM:
3214 case AArch64::LDNF1D_IMM:
3215 case AArch64::LDNF1H_D_IMM:
3216 case AArch64::LDNF1H_IMM:
3217 case AArch64::LDNF1H_S_IMM:
3218 case AArch64::LDNF1SB_D_IMM:
3219 case AArch64::LDNF1SB_H_IMM:
3220 case AArch64::LDNF1SB_S_IMM:
3221 case AArch64::LDNF1SH_D_IMM:
3222 case AArch64::LDNF1SH_S_IMM:
3223 case AArch64::LDNF1SW_D_IMM:
3224 case AArch64::LDNF1W_D_IMM:
3225 case AArch64::LDNF1W_IMM:
3226 case AArch64::LDNPDi:
3227 case AArch64::LDNPQi:
3228 case AArch64::LDNPSi:
3229 case AArch64::LDNPWi:
3230 case AArch64::LDNPXi:
3231 case AArch64::LDNT1B_ZRI:
3232 case AArch64::LDNT1D_ZRI:
3233 case AArch64::LDNT1H_ZRI:
3234 case AArch64::LDNT1W_ZRI:
3235 case AArch64::LDPDi:
3236 case AArch64::LDPQi:
3237 case AArch64::LDPSi:
3238 case AArch64::LDPWi:
3239 case AArch64::LDPXi:
3240 case AArch64::LDRBBpost:
3241 case AArch64::LDRBBpre:
3242 case AArch64::LDRBpost:
3243 case AArch64::LDRBpre:
3244 case AArch64::LDRDpost:
3245 case AArch64::LDRDpre:
3246 case AArch64::LDRHHpost:
3247 case AArch64::LDRHHpre:
3248 case AArch64::LDRHpost:
3249 case AArch64::LDRHpre:
3250 case AArch64::LDRQpost:
3251 case AArch64::LDRQpre:
3252 case AArch64::LDRSpost:
3253 case AArch64::LDRSpre:
3254 case AArch64::LDRWpost:
3255 case AArch64::LDRWpre:
3256 case AArch64::LDRXpost:
3257 case AArch64::LDRXpre:
3258 case AArch64::ST1B_D_IMM:
3259 case AArch64::ST1B_H_IMM:
3260 case AArch64::ST1B_IMM:
3261 case AArch64::ST1B_S_IMM:
3262 case AArch64::ST1D_IMM:
3263 case AArch64::ST1H_D_IMM:
3264 case AArch64::ST1H_IMM:
3265 case AArch64::ST1H_S_IMM:
3266 case AArch64::ST1W_D_IMM:
3267 case AArch64::ST1W_IMM:
3268 case AArch64::ST2B_IMM:
3269 case AArch64::ST2D_IMM:
3270 case AArch64::ST2H_IMM:
3271 case AArch64::ST2W_IMM:
3272 case AArch64::ST3B_IMM:
3273 case AArch64::ST3D_IMM:
3274 case AArch64::ST3H_IMM:
3275 case AArch64::ST3W_IMM:
3276 case AArch64::ST4B_IMM:
3277 case AArch64::ST4D_IMM:
3278 case AArch64::ST4H_IMM:
3279 case AArch64::ST4W_IMM:
3280 case AArch64::STGPi:
3281 case AArch64::STGPreIndex:
3282 case AArch64::STZGPreIndex:
3283 case AArch64::ST2GPreIndex:
3284 case AArch64::STZ2GPreIndex:
3285 case AArch64::STGPostIndex:
3286 case AArch64::STZGPostIndex:
3287 case AArch64::ST2GPostIndex:
3288 case AArch64::STZ2GPostIndex:
3289 case AArch64::STNPDi:
3290 case AArch64::STNPQi:
3291 case AArch64::STNPSi:
3292 case AArch64::STNPWi:
3293 case AArch64::STNPXi:
3294 case AArch64::STNT1B_ZRI:
3295 case AArch64::STNT1D_ZRI:
3296 case AArch64::STNT1H_ZRI:
3297 case AArch64::STNT1W_ZRI:
3298 case AArch64::STPDi:
3299 case AArch64::STPQi:
3300 case AArch64::STPSi:
3301 case AArch64::STPWi:
3302 case AArch64::STPXi:
3303 case AArch64::STRBBpost:
3304 case AArch64::STRBBpre:
3305 case AArch64::STRBpost:
3306 case AArch64::STRBpre:
3307 case AArch64::STRDpost:
3308 case AArch64::STRDpre:
3309 case AArch64::STRHHpost:
3310 case AArch64::STRHHpre:
3311 case AArch64::STRHpost:
3312 case AArch64::STRHpre:
3313 case AArch64::STRQpost:
3314 case AArch64::STRQpre:
3315 case AArch64::STRSpost:
3316 case AArch64::STRSpre:
3317 case AArch64::STRWpost:
3318 case AArch64::STRWpre:
3319 case AArch64::STRXpost:
3320 case AArch64::STRXpre:
3321 case AArch64::LD1B_2Z_IMM:
3322 case AArch64::LD1B_2Z_STRIDED_IMM:
3323 case AArch64::LD1H_2Z_IMM:
3324 case AArch64::LD1H_2Z_STRIDED_IMM:
3325 case AArch64::LD1W_2Z_IMM:
3326 case AArch64::LD1W_2Z_STRIDED_IMM:
3327 case AArch64::LD1D_2Z_IMM:
3328 case AArch64::LD1D_2Z_STRIDED_IMM:
3329 case AArch64::LD1B_4Z_IMM:
3330 case AArch64::LD1B_4Z_STRIDED_IMM:
3331 case AArch64::LD1H_4Z_IMM:
3332 case AArch64::LD1H_4Z_STRIDED_IMM:
3333 case AArch64::LD1W_4Z_IMM:
3334 case AArch64::LD1W_4Z_STRIDED_IMM:
3335 case AArch64::LD1D_4Z_IMM:
3336 case AArch64::LD1D_4Z_STRIDED_IMM:
3337 case AArch64::LD1B_2Z_IMM_PSEUDO:
3338 case AArch64::LD1H_2Z_IMM_PSEUDO:
3339 case AArch64::LD1W_2Z_IMM_PSEUDO:
3340 case AArch64::LD1D_2Z_IMM_PSEUDO:
3341 case AArch64::LD1B_4Z_IMM_PSEUDO:
3342 case AArch64::LD1H_4Z_IMM_PSEUDO:
3343 case AArch64::LD1W_4Z_IMM_PSEUDO:
3344 case AArch64::LD1D_4Z_IMM_PSEUDO:
3345 case AArch64::ST1B_2Z_IMM:
3346 case AArch64::ST1B_2Z_STRIDED_IMM:
3347 case AArch64::ST1H_2Z_IMM:
3348 case AArch64::ST1H_2Z_STRIDED_IMM:
3349 case AArch64::ST1W_2Z_IMM:
3350 case AArch64::ST1W_2Z_STRIDED_IMM:
3351 case AArch64::ST1D_2Z_IMM:
3352 case AArch64::ST1D_2Z_STRIDED_IMM:
3353 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
3354 case AArch64::LDNT1B_2Z_IMM:
3355 case AArch64::LDNT1B_2Z_STRIDED_IMM:
3356 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
3357 case AArch64::LDNT1H_2Z_IMM:
3358 case AArch64::LDNT1H_2Z_STRIDED_IMM:
3359 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
3360 case AArch64::LDNT1W_2Z_IMM:
3361 case AArch64::LDNT1W_2Z_STRIDED_IMM:
3362 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
3363 case AArch64::LDNT1D_2Z_IMM:
3364 case AArch64::LDNT1D_2Z_STRIDED_IMM:
3365 case AArch64::STNT1B_2Z_IMM:
3366 case AArch64::STNT1B_2Z_STRIDED_IMM:
3367 case AArch64::STNT1H_2Z_IMM:
3368 case AArch64::STNT1H_2Z_STRIDED_IMM:
3369 case AArch64::STNT1W_2Z_IMM:
3370 case AArch64::STNT1W_2Z_STRIDED_IMM:
3371 case AArch64::STNT1D_2Z_IMM:
3372 case AArch64::STNT1D_2Z_STRIDED_IMM:
3373 case AArch64::ST1B_2Z_IMM_PSEUDO:
3374 case AArch64::ST1H_2Z_IMM_PSEUDO:
3375 case AArch64::ST1W_2Z_IMM_PSEUDO:
3376 case AArch64::ST1D_2Z_IMM_PSEUDO:
3377 case AArch64::STNT1B_2Z_IMM_PSEUDO:
3378 case AArch64::STNT1H_2Z_IMM_PSEUDO:
3379 case AArch64::STNT1W_2Z_IMM_PSEUDO:
3380 case AArch64::STNT1D_2Z_IMM_PSEUDO:
3381 case AArch64::ST1B_4Z_IMM:
3382 case AArch64::ST1B_4Z_STRIDED_IMM:
3383 case AArch64::ST1H_4Z_IMM:
3384 case AArch64::ST1H_4Z_STRIDED_IMM:
3385 case AArch64::ST1W_4Z_IMM:
3386 case AArch64::ST1W_4Z_STRIDED_IMM:
3387 case AArch64::ST1D_4Z_IMM:
3388 case AArch64::ST1D_4Z_STRIDED_IMM:
3389 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
3390 case AArch64::LDNT1B_4Z_IMM:
3391 case AArch64::LDNT1B_4Z_STRIDED_IMM:
3392 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
3393 case AArch64::LDNT1H_4Z_IMM:
3394 case AArch64::LDNT1H_4Z_STRIDED_IMM:
3395 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
3396 case AArch64::LDNT1W_4Z_IMM:
3397 case AArch64::LDNT1W_4Z_STRIDED_IMM:
3398 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
3399 case AArch64::LDNT1D_4Z_IMM:
3400 case AArch64::LDNT1D_4Z_STRIDED_IMM:
3401 case AArch64::STNT1B_4Z_IMM:
3402 case AArch64::STNT1B_4Z_STRIDED_IMM:
3403 case AArch64::STNT1H_4Z_IMM:
3404 case AArch64::STNT1H_4Z_STRIDED_IMM:
3405 case AArch64::STNT1W_4Z_IMM:
3406 case AArch64::STNT1W_4Z_STRIDED_IMM:
3407 case AArch64::STNT1D_4Z_IMM:
3408 case AArch64::STNT1D_4Z_STRIDED_IMM:
3409 case AArch64::ST1B_4Z_IMM_PSEUDO:
3410 case AArch64::ST1H_4Z_IMM_PSEUDO:
3411 case AArch64::ST1W_4Z_IMM_PSEUDO:
3412 case AArch64::ST1D_4Z_IMM_PSEUDO:
3413 case AArch64::STNT1B_4Z_IMM_PSEUDO:
3414 case AArch64::STNT1H_4Z_IMM_PSEUDO:
3415 case AArch64::STNT1W_4Z_IMM_PSEUDO:
3416 case AArch64::STNT1D_4Z_IMM_PSEUDO:
3417 return 3;
3418 case AArch64::LDPDpost:
3419 case AArch64::LDPDpre:
3420 case AArch64::LDPQpost:
3421 case AArch64::LDPQpre:
3422 case AArch64::LDPSpost:
3423 case AArch64::LDPSpre:
3424 case AArch64::LDPWpost:
3425 case AArch64::LDPWpre:
3426 case AArch64::LDPXpost:
3427 case AArch64::LDPXpre:
3428 case AArch64::STGPpre:
3429 case AArch64::STGPpost:
3430 case AArch64::STPDpost:
3431 case AArch64::STPDpre:
3432 case AArch64::STPQpost:
3433 case AArch64::STPQpre:
3434 case AArch64::STPSpost:
3435 case AArch64::STPSpre:
3436 case AArch64::STPWpost:
3437 case AArch64::STPWpre:
3438 case AArch64::STPXpost:
3439 case AArch64::STPXpre:
3440 return 4;
3441 }
3442}
3443
3444bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
3445 switch (MI.getOpcode()) {
3446 default:
3447 return false;
3448 // Scaled instructions.
3449 case AArch64::STRSui:
3450 case AArch64::STRDui:
3451 case AArch64::STRQui:
3452 case AArch64::STRXui:
3453 case AArch64::STRWui:
3454 case AArch64::LDRSui:
3455 case AArch64::LDRDui:
3456 case AArch64::LDRQui:
3457 case AArch64::LDRXui:
3458 case AArch64::LDRWui:
3459 case AArch64::LDRSWui:
3460 // Unscaled instructions.
3461 case AArch64::STURSi:
3462 case AArch64::STRSpre:
3463 case AArch64::STURDi:
3464 case AArch64::STRDpre:
3465 case AArch64::STURQi:
3466 case AArch64::STRQpre:
3467 case AArch64::STURWi:
3468 case AArch64::STRWpre:
3469 case AArch64::STURXi:
3470 case AArch64::STRXpre:
3471 case AArch64::LDURSi:
3472 case AArch64::LDRSpre:
3473 case AArch64::LDURDi:
3474 case AArch64::LDRDpre:
3475 case AArch64::LDURQi:
3476 case AArch64::LDRQpre:
3477 case AArch64::LDURWi:
3478 case AArch64::LDRWpre:
3479 case AArch64::LDURXi:
3480 case AArch64::LDRXpre:
3481 case AArch64::LDURSWi:
3482 case AArch64::LDRSWpre:
3483 // SVE instructions.
3484 case AArch64::LDR_ZXI:
3485 case AArch64::STR_ZXI:
3486 return true;
3487 }
3488}
3489
3490bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
3491 switch (MI.getOpcode()) {
3492 default:
3493 assert((!MI.isCall() || !MI.isReturn()) &&
3494 "Unexpected instruction - was a new tail call opcode introduced?");
3495 return false;
3496 case AArch64::TCRETURNdi:
3497 case AArch64::TCRETURNri:
3498 case AArch64::TCRETURNrix16x17:
3499 case AArch64::TCRETURNrix17:
3500 case AArch64::TCRETURNrinotx16:
3501 case AArch64::TCRETURNriALL:
3502 case AArch64::AUTH_TCRETURN:
3503 case AArch64::AUTH_TCRETURN_BTI:
3504 return true;
3505 }
3506}
3507
3508unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) {
3509 switch (Opc) {
3510 default:
3511 llvm_unreachable("Opcode has no flag setting equivalent!");
3512 // 32-bit cases:
3513 case AArch64::ADDWri:
3514 return AArch64::ADDSWri;
3515 case AArch64::ADDWrr:
3516 return AArch64::ADDSWrr;
3517 case AArch64::ADDWrs:
3518 return AArch64::ADDSWrs;
3519 case AArch64::ADDWrx:
3520 return AArch64::ADDSWrx;
3521 case AArch64::ANDWri:
3522 return AArch64::ANDSWri;
3523 case AArch64::ANDWrr:
3524 return AArch64::ANDSWrr;
3525 case AArch64::ANDWrs:
3526 return AArch64::ANDSWrs;
3527 case AArch64::BICWrr:
3528 return AArch64::BICSWrr;
3529 case AArch64::BICWrs:
3530 return AArch64::BICSWrs;
3531 case AArch64::SUBWri:
3532 return AArch64::SUBSWri;
3533 case AArch64::SUBWrr:
3534 return AArch64::SUBSWrr;
3535 case AArch64::SUBWrs:
3536 return AArch64::SUBSWrs;
3537 case AArch64::SUBWrx:
3538 return AArch64::SUBSWrx;
3539 // 64-bit cases:
3540 case AArch64::ADDXri:
3541 return AArch64::ADDSXri;
3542 case AArch64::ADDXrr:
3543 return AArch64::ADDSXrr;
3544 case AArch64::ADDXrs:
3545 return AArch64::ADDSXrs;
3546 case AArch64::ADDXrx:
3547 return AArch64::ADDSXrx;
3548 case AArch64::ANDXri:
3549 return AArch64::ANDSXri;
3550 case AArch64::ANDXrr:
3551 return AArch64::ANDSXrr;
3552 case AArch64::ANDXrs:
3553 return AArch64::ANDSXrs;
3554 case AArch64::BICXrr:
3555 return AArch64::BICSXrr;
3556 case AArch64::BICXrs:
3557 return AArch64::BICSXrs;
3558 case AArch64::SUBXri:
3559 return AArch64::SUBSXri;
3560 case AArch64::SUBXrr:
3561 return AArch64::SUBSXrr;
3562 case AArch64::SUBXrs:
3563 return AArch64::SUBSXrs;
3564 case AArch64::SUBXrx:
3565 return AArch64::SUBSXrx;
3566 // SVE instructions:
3567 case AArch64::AND_PPzPP:
3568 return AArch64::ANDS_PPzPP;
3569 case AArch64::BIC_PPzPP:
3570 return AArch64::BICS_PPzPP;
3571 case AArch64::EOR_PPzPP:
3572 return AArch64::EORS_PPzPP;
3573 case AArch64::NAND_PPzPP:
3574 return AArch64::NANDS_PPzPP;
3575 case AArch64::NOR_PPzPP:
3576 return AArch64::NORS_PPzPP;
3577 case AArch64::ORN_PPzPP:
3578 return AArch64::ORNS_PPzPP;
3579 case AArch64::ORR_PPzPP:
3580 return AArch64::ORRS_PPzPP;
3581 case AArch64::BRKA_PPzP:
3582 return AArch64::BRKAS_PPzP;
3583 case AArch64::BRKPA_PPzPP:
3584 return AArch64::BRKPAS_PPzPP;
3585 case AArch64::BRKB_PPzP:
3586 return AArch64::BRKBS_PPzP;
3587 case AArch64::BRKPB_PPzPP:
3588 return AArch64::BRKPBS_PPzPP;
3589 case AArch64::BRKN_PPzP:
3590 return AArch64::BRKNS_PPzP;
3591 case AArch64::RDFFR_PPz:
3592 return AArch64::RDFFRS_PPz;
3593 case AArch64::PTRUE_B:
3594 return AArch64::PTRUES_B;
3595 }
3596}
3597
3598// Is this a candidate for ld/st merging or pairing? For example, we don't
3599// touch volatiles or load/stores that have a hint to avoid pair formation.
3600bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
3601
3602 bool IsPreLdSt = isPreLdSt(MI);
3603
3604 // If this is a volatile load/store, don't mess with it.
3605 if (MI.hasOrderedMemoryRef())
3606 return false;
3607
3608 // Make sure this is a reg/fi+imm (as opposed to an address reloc).
3609 // For Pre-inc LD/ST, the operand is shifted by one.
3610 assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
3611 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
3612 "Expected a reg or frame index operand.");
3613
3614 // For Pre-indexed addressing quadword instructions, the third operand is the
3615 // immediate value.
3616 bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(i: 3).isImm();
3617
3618 if (!MI.getOperand(i: 2).isImm() && !IsImmPreLdSt)
3619 return false;
3620
3621 // Can't merge/pair if the instruction modifies the base register.
3622 // e.g., ldr x0, [x0]
3623 // This case will never occur with an FI base.
3624 // However, if the instruction is an LDR<S,D,Q,W,X,SW>pre or
3625 // STR<S,D,Q,W,X>pre, it can be merged.
3626 // For example:
3627 // ldr q0, [x11, #32]!
3628 // ldr q1, [x11, #16]
3629 // to
3630 // ldp q0, q1, [x11, #32]!
3631 if (MI.getOperand(i: 1).isReg() && !IsPreLdSt) {
3632 Register BaseReg = MI.getOperand(i: 1).getReg();
3633 const TargetRegisterInfo *TRI = &getRegisterInfo();
3634 if (MI.modifiesRegister(Reg: BaseReg, TRI))
3635 return false;
3636 }
3637
3638 // Pairing SVE fills/spills is only valid for little-endian targets that
3639 // implement VLS 128.
3640 switch (MI.getOpcode()) {
3641 default:
3642 break;
3643 case AArch64::LDR_ZXI:
3644 case AArch64::STR_ZXI:
3645 if (!Subtarget.isLittleEndian() ||
3646 Subtarget.getSVEVectorSizeInBits() != 128)
3647 return false;
3648 }
3649
3650 // Check if this load/store has a hint to avoid pair formation.
3651 // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
3652 if (isLdStPairSuppressed(MI))
3653 return false;
3654
3655 // Do not pair any callee-save store/reload instructions in the
3656 // prologue/epilogue if the CFI information encoded the operations as separate
3657 // instructions, as that will cause the size of the actual prologue to mismatch
3658 // with the prologue size recorded in the Windows CFI.
3659 if ((MI.getFlag(Flag: MachineInstr::FrameSetup) ||
3660 MI.getFlag(Flag: MachineInstr::FrameDestroy)) &&
3661 MI.getMF()->needsWinCFI())
3662 return false;
3663
3664 // On some CPUs quad load/store pairs are slower than two single load/stores.
3665 if (Subtarget.isPaired128Slow()) {
3666 switch (MI.getOpcode()) {
3667 default:
3668 break;
3669 case AArch64::LDURQi:
3670 case AArch64::STURQi:
3671 case AArch64::LDRQui:
3672 case AArch64::STRQui:
3673 return false;
3674 }
3675 }
3676
3677 return true;
3678}
3679
3680bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
3681 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3682 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
3683 if (!LdSt.mayLoadOrStore())
3684 return false;
3685
3686 const MachineOperand *BaseOp;
3687 TypeSize WidthN(0, false);
3688 if (!getMemOperandWithOffsetWidth(MI: LdSt, BaseOp, Offset, OffsetIsScalable,
3689 Width&: WidthN))
3690 return false;
3691 // The maximum vscale is 16 under AArch64, return the maximal extent for the
3692 // vector.
3693 Width = LocationSize::precise(Value: WidthN);
3694 BaseOps.push_back(Elt: BaseOp);
3695 return true;
3696}
3697
3698std::optional<ExtAddrMode>
3699AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI) const {
3700 const MachineOperand *Base; // Filled with the base operand of MI.
3701 int64_t Offset; // Filled with the offset of MI.
3702 bool OffsetIsScalable;
3703 if (!getMemOperandWithOffset(MI: MemI, BaseOp&: Base, Offset, OffsetIsScalable))
3704 return std::nullopt;
3705
3706 if (!Base->isReg())
3707 return std::nullopt;
3708 ExtAddrMode AM;
3709 AM.BaseReg = Base->getReg();
3710 AM.Displacement = Offset;
3711 AM.ScaledReg = 0;
3712 AM.Scale = 0;
3713 return AM;
3714}
3715
3716bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
3717 Register Reg,
3718 const MachineInstr &AddrI,
3719 ExtAddrMode &AM) const {
3720 // Filter out instructions into which we cannot fold.
3721 unsigned NumBytes;
3722 int64_t OffsetScale = 1;
3723 switch (MemI.getOpcode()) {
3724 default:
3725 return false;
3726
3727 case AArch64::LDURQi:
3728 case AArch64::STURQi:
3729 NumBytes = 16;
3730 break;
3731
3732 case AArch64::LDURDi:
3733 case AArch64::STURDi:
3734 case AArch64::LDURXi:
3735 case AArch64::STURXi:
3736 NumBytes = 8;
3737 break;
3738
3739 case AArch64::LDURSi:
3740 case AArch64::STURSi:
3741 case AArch64::LDURWi:
3742 case AArch64::LDURSWi:
3743 case AArch64::STURWi:
3744 NumBytes = 4;
3745 break;
3746
3747 case AArch64::LDURHi:
3748 case AArch64::STURHi:
3749 case AArch64::LDURHHi:
3750 case AArch64::STURHHi:
3751 case AArch64::LDURSHXi:
3752 case AArch64::LDURSHWi:
3753 NumBytes = 2;
3754 break;
3755
3756 case AArch64::LDRBroX:
3757 case AArch64::LDRBBroX:
3758 case AArch64::LDRSBXroX:
3759 case AArch64::LDRSBWroX:
3760 case AArch64::STRBroX:
3761 case AArch64::STRBBroX:
3762 case AArch64::LDURBi:
3763 case AArch64::LDURBBi:
3764 case AArch64::LDURSBXi:
3765 case AArch64::LDURSBWi:
3766 case AArch64::STURBi:
3767 case AArch64::STURBBi:
3768 case AArch64::LDRBui:
3769 case AArch64::LDRBBui:
3770 case AArch64::LDRSBXui:
3771 case AArch64::LDRSBWui:
3772 case AArch64::STRBui:
3773 case AArch64::STRBBui:
3774 NumBytes = 1;
3775 break;
3776
3777 case AArch64::LDRQroX:
3778 case AArch64::STRQroX:
3779 case AArch64::LDRQui:
3780 case AArch64::STRQui:
3781 NumBytes = 16;
3782 OffsetScale = 16;
3783 break;
3784
3785 case AArch64::LDRDroX:
3786 case AArch64::STRDroX:
3787 case AArch64::LDRXroX:
3788 case AArch64::STRXroX:
3789 case AArch64::LDRDui:
3790 case AArch64::STRDui:
3791 case AArch64::LDRXui:
3792 case AArch64::STRXui:
3793 NumBytes = 8;
3794 OffsetScale = 8;
3795 break;
3796
3797 case AArch64::LDRSroX:
3798 case AArch64::STRSroX:
3799 case AArch64::LDRWroX:
3800 case AArch64::LDRSWroX:
3801 case AArch64::STRWroX:
3802 case AArch64::LDRSui:
3803 case AArch64::STRSui:
3804 case AArch64::LDRWui:
3805 case AArch64::LDRSWui:
3806 case AArch64::STRWui:
3807 NumBytes = 4;
3808 OffsetScale = 4;
3809 break;
3810
3811 case AArch64::LDRHroX:
3812 case AArch64::STRHroX:
3813 case AArch64::LDRHHroX:
3814 case AArch64::STRHHroX:
3815 case AArch64::LDRSHXroX:
3816 case AArch64::LDRSHWroX:
3817 case AArch64::LDRHui:
3818 case AArch64::STRHui:
3819 case AArch64::LDRHHui:
3820 case AArch64::STRHHui:
3821 case AArch64::LDRSHXui:
3822 case AArch64::LDRSHWui:
3823 NumBytes = 2;
3824 OffsetScale = 2;
3825 break;
3826 }
3827
3828 // Check the fold operand is not the loaded/stored value.
3829 const MachineOperand &BaseRegOp = MemI.getOperand(i: 0);
3830 if (BaseRegOp.isReg() && BaseRegOp.getReg() == Reg)
3831 return false;
3832
3833 // Handle memory instructions with a [Reg, Reg] addressing mode.
3834 if (MemI.getOperand(i: 2).isReg()) {
3835 // Bail if the addressing mode already includes extension of the offset
3836 // register.
3837 if (MemI.getOperand(i: 3).getImm())
3838 return false;
3839
3840 // Check if we actually have a scaled offset.
3841 if (MemI.getOperand(i: 4).getImm() == 0)
3842 OffsetScale = 1;
3843
3844 // If the address instructions is folded into the base register, then the
3845 // addressing mode must not have a scale. Then we can swap the base and the
3846 // scaled registers.
3847 if (MemI.getOperand(i: 1).getReg() == Reg && OffsetScale != 1)
3848 return false;
3849
3850 switch (AddrI.getOpcode()) {
3851 default:
3852 return false;
3853
3854 case AArch64::SBFMXri:
3855 // sxtw Xa, Wm
3856 // ldr Xd, [Xn, Xa, lsl #N]
3857 // ->
3858 // ldr Xd, [Xn, Wm, sxtw #N]
3859 if (AddrI.getOperand(i: 2).getImm() != 0 ||
3860 AddrI.getOperand(i: 3).getImm() != 31)
3861 return false;
3862
3863 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3864 if (AM.BaseReg == Reg)
3865 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3866 AM.ScaledReg = AddrI.getOperand(i: 1).getReg();
3867 AM.Scale = OffsetScale;
3868 AM.Displacement = 0;
3869 AM.Form = ExtAddrMode::Formula::SExtScaledReg;
3870 return true;
3871
3872 case TargetOpcode::SUBREG_TO_REG: {
3873 // mov Wa, Wm
3874 // ldr Xd, [Xn, Xa, lsl #N]
3875 // ->
3876 // ldr Xd, [Xn, Wm, uxtw #N]
3877
3878 // Zero-extension looks like an ORRWrs followed by a SUBREG_TO_REG.
3879 if (AddrI.getOperand(i: 2).getImm() != AArch64::sub_32)
3880 return false;
3881
3882 const MachineRegisterInfo &MRI = AddrI.getMF()->getRegInfo();
3883 Register OffsetReg = AddrI.getOperand(i: 1).getReg();
3884 if (!OffsetReg.isVirtual() || !MRI.hasOneNonDBGUse(RegNo: OffsetReg))
3885 return false;
3886
3887 const MachineInstr &DefMI = *MRI.getVRegDef(Reg: OffsetReg);
3888 if (DefMI.getOpcode() != AArch64::ORRWrs ||
3889 DefMI.getOperand(i: 1).getReg() != AArch64::WZR ||
3890 DefMI.getOperand(i: 3).getImm() != 0)
3891 return false;
3892
3893 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3894 if (AM.BaseReg == Reg)
3895 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3896 AM.ScaledReg = DefMI.getOperand(i: 2).getReg();
3897 AM.Scale = OffsetScale;
3898 AM.Displacement = 0;
3899 AM.Form = ExtAddrMode::Formula::ZExtScaledReg;
3900 return true;
3901 }
3902 }
3903 }
3904
3905 // Handle memory instructions with a [Reg, #Imm] addressing mode.
3906
3907 // Check we are not breaking a potential conversion to an LDP.
3908 auto validateOffsetForLDP = [](unsigned NumBytes, int64_t OldOffset,
3909 int64_t NewOffset) -> bool {
3910 int64_t MinOffset, MaxOffset;
3911 switch (NumBytes) {
3912 default:
3913 return true;
3914 case 4:
3915 MinOffset = -256;
3916 MaxOffset = 252;
3917 break;
3918 case 8:
3919 MinOffset = -512;
3920 MaxOffset = 504;
3921 break;
3922 case 16:
3923 MinOffset = -1024;
3924 MaxOffset = 1008;
3925 break;
3926 }
3927 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3928 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3929 };
3930 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) -> bool {
3931 int64_t OldOffset = MemI.getOperand(i: 2).getImm() * OffsetScale;
3932 int64_t NewOffset = OldOffset + Disp;
3933 if (!isLegalAddressingMode(NumBytes, Offset: NewOffset, /* Scale */ 0))
3934 return false;
3935 // If the old offset would fit into an LDP, but the new offset wouldn't,
3936 // bail out.
3937 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3938 return false;
3939 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3940 AM.ScaledReg = 0;
3941 AM.Scale = 0;
3942 AM.Displacement = NewOffset;
3943 AM.Form = ExtAddrMode::Formula::Basic;
3944 return true;
3945 };
3946
3947 auto canFoldAddRegIntoAddrMode =
3948 [&](int64_t Scale,
3949 ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool {
3950 if (MemI.getOperand(i: 2).getImm() != 0)
3951 return false;
3952 if ((unsigned)Scale != Scale)
3953 return false;
3954 if (!isLegalAddressingMode(NumBytes, /* Offset */ 0, Scale))
3955 return false;
3956 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3957 AM.ScaledReg = AddrI.getOperand(i: 2).getReg();
3958 AM.Scale = Scale;
3959 AM.Displacement = 0;
3960 AM.Form = Form;
3961 return true;
3962 };
3963
3964 auto avoidSlowSTRQ = [&](const MachineInstr &MemI) {
3965 unsigned Opcode = MemI.getOpcode();
3966 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3967 Subtarget.isSTRQroSlow();
3968 };
3969
3970 int64_t Disp = 0;
3971 const bool OptSize = MemI.getMF()->getFunction().hasOptSize();
3972 switch (AddrI.getOpcode()) {
3973 default:
3974 return false;
3975
3976 case AArch64::ADDXri:
3977 // add Xa, Xn, #N
3978 // ldr Xd, [Xa, #M]
3979 // ->
3980 // ldr Xd, [Xn, #N'+M]
3981 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3982 return canFoldAddSubImmIntoAddrMode(Disp);
3983
3984 case AArch64::SUBXri:
3985 // sub Xa, Xn, #N
3986 // ldr Xd, [Xa, #M]
3987 // ->
3988 // ldr Xd, [Xn, #N'+M]
3989 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3990 return canFoldAddSubImmIntoAddrMode(-Disp);
3991
3992 case AArch64::ADDXrs: {
3993 // add Xa, Xn, Xm, lsl #N
3994 // ldr Xd, [Xa]
3995 // ->
3996 // ldr Xd, [Xn, Xm, lsl #N]
3997
3998 // Don't fold the add if the result would be slower, unless optimising for
3999 // size.
4000 unsigned Shift = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
4001 if (AArch64_AM::getShiftType(Imm: Shift) != AArch64_AM::ShiftExtendType::LSL)
4002 return false;
4003 Shift = AArch64_AM::getShiftValue(Imm: Shift);
4004 if (!OptSize) {
4005 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
4006 return false;
4007 if (avoidSlowSTRQ(MemI))
4008 return false;
4009 }
4010 return canFoldAddRegIntoAddrMode(1ULL << Shift);
4011 }
4012
4013 case AArch64::ADDXrr:
4014 // add Xa, Xn, Xm
4015 // ldr Xd, [Xa]
4016 // ->
4017 // ldr Xd, [Xn, Xm, lsl #0]
4018
4019 // Don't fold the add if the result would be slower, unless optimising for
4020 // size.
4021 if (!OptSize && avoidSlowSTRQ(MemI))
4022 return false;
4023 return canFoldAddRegIntoAddrMode(1);
4024
4025 case AArch64::ADDXrx:
4026 // add Xa, Xn, Wm, {s,u}xtw #N
4027 // ldr Xd, [Xa]
4028 // ->
4029 // ldr Xd, [Xn, Wm, {s,u}xtw #N]
4030
4031 // Don't fold the add if the result would be slower, unless optimising for
4032 // size.
4033 if (!OptSize && avoidSlowSTRQ(MemI))
4034 return false;
4035
4036 // Can fold only sign-/zero-extend of a word.
4037 unsigned Imm = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
4038 AArch64_AM::ShiftExtendType Extend = AArch64_AM::getArithExtendType(Imm);
4039 if (Extend != AArch64_AM::UXTW && Extend != AArch64_AM::SXTW)
4040 return false;
4041
4042 return canFoldAddRegIntoAddrMode(
4043 1ULL << AArch64_AM::getArithShiftValue(Imm),
4044 (Extend == AArch64_AM::SXTW) ? ExtAddrMode::Formula::SExtScaledReg
4045 : ExtAddrMode::Formula::ZExtScaledReg);
4046 }
4047}
4048
4049// Given an opcode for an instruction with a [Reg, #Imm] addressing mode,
4050// return the opcode of an instruction performing the same operation, but using
4051// the [Reg, Reg] addressing mode.
4052static unsigned regOffsetOpcode(unsigned Opcode) {
4053 switch (Opcode) {
4054 default:
4055 llvm_unreachable("Address folding not implemented for instruction");
4056
4057 case AArch64::LDURQi:
4058 case AArch64::LDRQui:
4059 return AArch64::LDRQroX;
4060 case AArch64::STURQi:
4061 case AArch64::STRQui:
4062 return AArch64::STRQroX;
4063 case AArch64::LDURDi:
4064 case AArch64::LDRDui:
4065 return AArch64::LDRDroX;
4066 case AArch64::STURDi:
4067 case AArch64::STRDui:
4068 return AArch64::STRDroX;
4069 case AArch64::LDURXi:
4070 case AArch64::LDRXui:
4071 return AArch64::LDRXroX;
4072 case AArch64::STURXi:
4073 case AArch64::STRXui:
4074 return AArch64::STRXroX;
4075 case AArch64::LDURSi:
4076 case AArch64::LDRSui:
4077 return AArch64::LDRSroX;
4078 case AArch64::STURSi:
4079 case AArch64::STRSui:
4080 return AArch64::STRSroX;
4081 case AArch64::LDURWi:
4082 case AArch64::LDRWui:
4083 return AArch64::LDRWroX;
4084 case AArch64::LDURSWi:
4085 case AArch64::LDRSWui:
4086 return AArch64::LDRSWroX;
4087 case AArch64::STURWi:
4088 case AArch64::STRWui:
4089 return AArch64::STRWroX;
4090 case AArch64::LDURHi:
4091 case AArch64::LDRHui:
4092 return AArch64::LDRHroX;
4093 case AArch64::STURHi:
4094 case AArch64::STRHui:
4095 return AArch64::STRHroX;
4096 case AArch64::LDURHHi:
4097 case AArch64::LDRHHui:
4098 return AArch64::LDRHHroX;
4099 case AArch64::STURHHi:
4100 case AArch64::STRHHui:
4101 return AArch64::STRHHroX;
4102 case AArch64::LDURSHXi:
4103 case AArch64::LDRSHXui:
4104 return AArch64::LDRSHXroX;
4105 case AArch64::LDURSHWi:
4106 case AArch64::LDRSHWui:
4107 return AArch64::LDRSHWroX;
4108 case AArch64::LDURBi:
4109 case AArch64::LDRBui:
4110 return AArch64::LDRBroX;
4111 case AArch64::LDURBBi:
4112 case AArch64::LDRBBui:
4113 return AArch64::LDRBBroX;
4114 case AArch64::LDURSBXi:
4115 case AArch64::LDRSBXui:
4116 return AArch64::LDRSBXroX;
4117 case AArch64::LDURSBWi:
4118 case AArch64::LDRSBWui:
4119 return AArch64::LDRSBWroX;
4120 case AArch64::STURBi:
4121 case AArch64::STRBui:
4122 return AArch64::STRBroX;
4123 case AArch64::STURBBi:
4124 case AArch64::STRBBui:
4125 return AArch64::STRBBroX;
4126 }
4127}
4128
4129// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4130// the opcode of an instruction performing the same operation, but using the
4131// [Reg, #Imm] addressing mode with scaled offset.
4132unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale) {
4133 switch (Opcode) {
4134 default:
4135 llvm_unreachable("Address folding not implemented for instruction");
4136
4137 case AArch64::LDURQi:
4138 Scale = 16;
4139 return AArch64::LDRQui;
4140 case AArch64::STURQi:
4141 Scale = 16;
4142 return AArch64::STRQui;
4143 case AArch64::LDURDi:
4144 Scale = 8;
4145 return AArch64::LDRDui;
4146 case AArch64::STURDi:
4147 Scale = 8;
4148 return AArch64::STRDui;
4149 case AArch64::LDURXi:
4150 Scale = 8;
4151 return AArch64::LDRXui;
4152 case AArch64::STURXi:
4153 Scale = 8;
4154 return AArch64::STRXui;
4155 case AArch64::LDURSi:
4156 Scale = 4;
4157 return AArch64::LDRSui;
4158 case AArch64::STURSi:
4159 Scale = 4;
4160 return AArch64::STRSui;
4161 case AArch64::LDURWi:
4162 Scale = 4;
4163 return AArch64::LDRWui;
4164 case AArch64::LDURSWi:
4165 Scale = 4;
4166 return AArch64::LDRSWui;
4167 case AArch64::STURWi:
4168 Scale = 4;
4169 return AArch64::STRWui;
4170 case AArch64::LDURHi:
4171 Scale = 2;
4172 return AArch64::LDRHui;
4173 case AArch64::STURHi:
4174 Scale = 2;
4175 return AArch64::STRHui;
4176 case AArch64::LDURHHi:
4177 Scale = 2;
4178 return AArch64::LDRHHui;
4179 case AArch64::STURHHi:
4180 Scale = 2;
4181 return AArch64::STRHHui;
4182 case AArch64::LDURSHXi:
4183 Scale = 2;
4184 return AArch64::LDRSHXui;
4185 case AArch64::LDURSHWi:
4186 Scale = 2;
4187 return AArch64::LDRSHWui;
4188 case AArch64::LDURBi:
4189 Scale = 1;
4190 return AArch64::LDRBui;
4191 case AArch64::LDURBBi:
4192 Scale = 1;
4193 return AArch64::LDRBBui;
4194 case AArch64::LDURSBXi:
4195 Scale = 1;
4196 return AArch64::LDRSBXui;
4197 case AArch64::LDURSBWi:
4198 Scale = 1;
4199 return AArch64::LDRSBWui;
4200 case AArch64::STURBi:
4201 Scale = 1;
4202 return AArch64::STRBui;
4203 case AArch64::STURBBi:
4204 Scale = 1;
4205 return AArch64::STRBBui;
4206 case AArch64::LDRQui:
4207 case AArch64::STRQui:
4208 Scale = 16;
4209 return Opcode;
4210 case AArch64::LDRDui:
4211 case AArch64::STRDui:
4212 case AArch64::LDRXui:
4213 case AArch64::STRXui:
4214 Scale = 8;
4215 return Opcode;
4216 case AArch64::LDRSui:
4217 case AArch64::STRSui:
4218 case AArch64::LDRWui:
4219 case AArch64::LDRSWui:
4220 case AArch64::STRWui:
4221 Scale = 4;
4222 return Opcode;
4223 case AArch64::LDRHui:
4224 case AArch64::STRHui:
4225 case AArch64::LDRHHui:
4226 case AArch64::STRHHui:
4227 case AArch64::LDRSHXui:
4228 case AArch64::LDRSHWui:
4229 Scale = 2;
4230 return Opcode;
4231 case AArch64::LDRBui:
4232 case AArch64::LDRBBui:
4233 case AArch64::LDRSBXui:
4234 case AArch64::LDRSBWui:
4235 case AArch64::STRBui:
4236 case AArch64::STRBBui:
4237 Scale = 1;
4238 return Opcode;
4239 }
4240}
4241
4242// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4243// the opcode of an instruction performing the same operation, but using the
4244// [Reg, #Imm] addressing mode with unscaled offset.
4245unsigned unscaledOffsetOpcode(unsigned Opcode) {
4246 switch (Opcode) {
4247 default:
4248 llvm_unreachable("Address folding not implemented for instruction");
4249
4250 case AArch64::LDURQi:
4251 case AArch64::STURQi:
4252 case AArch64::LDURDi:
4253 case AArch64::STURDi:
4254 case AArch64::LDURXi:
4255 case AArch64::STURXi:
4256 case AArch64::LDURSi:
4257 case AArch64::STURSi:
4258 case AArch64::LDURWi:
4259 case AArch64::LDURSWi:
4260 case AArch64::STURWi:
4261 case AArch64::LDURHi:
4262 case AArch64::STURHi:
4263 case AArch64::LDURHHi:
4264 case AArch64::STURHHi:
4265 case AArch64::LDURSHXi:
4266 case AArch64::LDURSHWi:
4267 case AArch64::LDURBi:
4268 case AArch64::STURBi:
4269 case AArch64::LDURBBi:
4270 case AArch64::STURBBi:
4271 case AArch64::LDURSBWi:
4272 case AArch64::LDURSBXi:
4273 return Opcode;
4274 case AArch64::LDRQui:
4275 return AArch64::LDURQi;
4276 case AArch64::STRQui:
4277 return AArch64::STURQi;
4278 case AArch64::LDRDui:
4279 return AArch64::LDURDi;
4280 case AArch64::STRDui:
4281 return AArch64::STURDi;
4282 case AArch64::LDRXui:
4283 return AArch64::LDURXi;
4284 case AArch64::STRXui:
4285 return AArch64::STURXi;
4286 case AArch64::LDRSui:
4287 return AArch64::LDURSi;
4288 case AArch64::STRSui:
4289 return AArch64::STURSi;
4290 case AArch64::LDRWui:
4291 return AArch64::LDURWi;
4292 case AArch64::LDRSWui:
4293 return AArch64::LDURSWi;
4294 case AArch64::STRWui:
4295 return AArch64::STURWi;
4296 case AArch64::LDRHui:
4297 return AArch64::LDURHi;
4298 case AArch64::STRHui:
4299 return AArch64::STURHi;
4300 case AArch64::LDRHHui:
4301 return AArch64::LDURHHi;
4302 case AArch64::STRHHui:
4303 return AArch64::STURHHi;
4304 case AArch64::LDRSHXui:
4305 return AArch64::LDURSHXi;
4306 case AArch64::LDRSHWui:
4307 return AArch64::LDURSHWi;
4308 case AArch64::LDRBBui:
4309 return AArch64::LDURBBi;
4310 case AArch64::LDRBui:
4311 return AArch64::LDURBi;
4312 case AArch64::STRBBui:
4313 return AArch64::STURBBi;
4314 case AArch64::STRBui:
4315 return AArch64::STURBi;
4316 case AArch64::LDRSBWui:
4317 return AArch64::LDURSBWi;
4318 case AArch64::LDRSBXui:
4319 return AArch64::LDURSBXi;
4320 }
4321}
4322
4323// Given the opcode of a memory load/store instruction, return the opcode of an
4324// instruction performing the same operation, but using
4325// the [Reg, Reg, {s,u}xtw #N] addressing mode with sign-/zero-extend of the
4326// offset register.
4327static unsigned offsetExtendOpcode(unsigned Opcode) {
4328 switch (Opcode) {
4329 default:
4330 llvm_unreachable("Address folding not implemented for instruction");
4331
4332 case AArch64::LDRQroX:
4333 case AArch64::LDURQi:
4334 case AArch64::LDRQui:
4335 return AArch64::LDRQroW;
4336 case AArch64::STRQroX:
4337 case AArch64::STURQi:
4338 case AArch64::STRQui:
4339 return AArch64::STRQroW;
4340 case AArch64::LDRDroX:
4341 case AArch64::LDURDi:
4342 case AArch64::LDRDui:
4343 return AArch64::LDRDroW;
4344 case AArch64::STRDroX:
4345 case AArch64::STURDi:
4346 case AArch64::STRDui:
4347 return AArch64::STRDroW;
4348 case AArch64::LDRXroX:
4349 case AArch64::LDURXi:
4350 case AArch64::LDRXui:
4351 return AArch64::LDRXroW;
4352 case AArch64::STRXroX:
4353 case AArch64::STURXi:
4354 case AArch64::STRXui:
4355 return AArch64::STRXroW;
4356 case AArch64::LDRSroX:
4357 case AArch64::LDURSi:
4358 case AArch64::LDRSui:
4359 return AArch64::LDRSroW;
4360 case AArch64::STRSroX:
4361 case AArch64::STURSi:
4362 case AArch64::STRSui:
4363 return AArch64::STRSroW;
4364 case AArch64::LDRWroX:
4365 case AArch64::LDURWi:
4366 case AArch64::LDRWui:
4367 return AArch64::LDRWroW;
4368 case AArch64::LDRSWroX:
4369 case AArch64::LDURSWi:
4370 case AArch64::LDRSWui:
4371 return AArch64::LDRSWroW;
4372 case AArch64::STRWroX:
4373 case AArch64::STURWi:
4374 case AArch64::STRWui:
4375 return AArch64::STRWroW;
4376 case AArch64::LDRHroX:
4377 case AArch64::LDURHi:
4378 case AArch64::LDRHui:
4379 return AArch64::LDRHroW;
4380 case AArch64::STRHroX:
4381 case AArch64::STURHi:
4382 case AArch64::STRHui:
4383 return AArch64::STRHroW;
4384 case AArch64::LDRHHroX:
4385 case AArch64::LDURHHi:
4386 case AArch64::LDRHHui:
4387 return AArch64::LDRHHroW;
4388 case AArch64::STRHHroX:
4389 case AArch64::STURHHi:
4390 case AArch64::STRHHui:
4391 return AArch64::STRHHroW;
4392 case AArch64::LDRSHXroX:
4393 case AArch64::LDURSHXi:
4394 case AArch64::LDRSHXui:
4395 return AArch64::LDRSHXroW;
4396 case AArch64::LDRSHWroX:
4397 case AArch64::LDURSHWi:
4398 case AArch64::LDRSHWui:
4399 return AArch64::LDRSHWroW;
4400 case AArch64::LDRBroX:
4401 case AArch64::LDURBi:
4402 case AArch64::LDRBui:
4403 return AArch64::LDRBroW;
4404 case AArch64::LDRBBroX:
4405 case AArch64::LDURBBi:
4406 case AArch64::LDRBBui:
4407 return AArch64::LDRBBroW;
4408 case AArch64::LDRSBXroX:
4409 case AArch64::LDURSBXi:
4410 case AArch64::LDRSBXui:
4411 return AArch64::LDRSBXroW;
4412 case AArch64::LDRSBWroX:
4413 case AArch64::LDURSBWi:
4414 case AArch64::LDRSBWui:
4415 return AArch64::LDRSBWroW;
4416 case AArch64::STRBroX:
4417 case AArch64::STURBi:
4418 case AArch64::STRBui:
4419 return AArch64::STRBroW;
4420 case AArch64::STRBBroX:
4421 case AArch64::STURBBi:
4422 case AArch64::STRBBui:
4423 return AArch64::STRBBroW;
4424 }
4425}
4426
4427MachineInstr *AArch64InstrInfo::emitLdStWithAddr(MachineInstr &MemI,
4428 const ExtAddrMode &AM) const {
4429
4430 const DebugLoc &DL = MemI.getDebugLoc();
4431 MachineBasicBlock &MBB = *MemI.getParent();
4432 MachineRegisterInfo &MRI = MemI.getMF()->getRegInfo();
4433
4434 if (AM.Form == ExtAddrMode::Formula::Basic) {
4435 if (AM.ScaledReg) {
4436 // The new instruction will be in the form `ldr Rt, [Xn, Xm, lsl #imm]`.
4437 unsigned Opcode = regOffsetOpcode(Opcode: MemI.getOpcode());
4438 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4439 auto B = BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4440 .addReg(RegNo: MemI.getOperand(i: 0).getReg(),
4441 Flags: getDefRegState(B: MemI.mayLoad()))
4442 .addReg(RegNo: AM.BaseReg)
4443 .addReg(RegNo: AM.ScaledReg)
4444 .addImm(Val: 0)
4445 .addImm(Val: AM.Scale > 1)
4446 .setMemRefs(MemI.memoperands())
4447 .setMIFlags(MemI.getFlags());
4448 return B.getInstr();
4449 }
4450
4451 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
4452 "Addressing mode not supported for folding");
4453
4454 // The new instruction will be in the form `ld[u]r Rt, [Xn, #imm]`.
4455 unsigned Scale = 1;
4456 unsigned Opcode = MemI.getOpcode();
4457 if (isInt<9>(x: AM.Displacement))
4458 Opcode = unscaledOffsetOpcode(Opcode);
4459 else
4460 Opcode = scaledOffsetOpcode(Opcode, Scale);
4461
4462 auto B =
4463 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4464 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4465 .addReg(RegNo: AM.BaseReg)
4466 .addImm(Val: AM.Displacement / Scale)
4467 .setMemRefs(MemI.memoperands())
4468 .setMIFlags(MemI.getFlags());
4469 return B.getInstr();
4470 }
4471
4472 if (AM.Form == ExtAddrMode::Formula::SExtScaledReg ||
4473 AM.Form == ExtAddrMode::Formula::ZExtScaledReg) {
4474 // The new instruction will be in the form `ldr Rt, [Xn, Wm, {s,u}xtw #N]`.
4475 assert(AM.ScaledReg && !AM.Displacement &&
4476 "Address offset can be a register or an immediate, but not both");
4477 unsigned Opcode = offsetExtendOpcode(Opcode: MemI.getOpcode());
4478 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4479 // Make sure the offset register is in the correct register class.
4480 Register OffsetReg = AM.ScaledReg;
4481 const TargetRegisterClass *RC = MRI.getRegClass(Reg: OffsetReg);
4482 if (RC->hasSuperClassEq(RC: &AArch64::GPR64RegClass)) {
4483 OffsetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4484 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: OffsetReg)
4485 .addReg(RegNo: AM.ScaledReg, Flags: {}, SubReg: AArch64::sub_32);
4486 }
4487 auto B =
4488 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4489 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4490 .addReg(RegNo: AM.BaseReg)
4491 .addReg(RegNo: OffsetReg)
4492 .addImm(Val: AM.Form == ExtAddrMode::Formula::SExtScaledReg)
4493 .addImm(Val: AM.Scale != 1)
4494 .setMemRefs(MemI.memoperands())
4495 .setMIFlags(MemI.getFlags());
4496
4497 return B.getInstr();
4498 }
4499
4500 llvm_unreachable(
4501 "Function must not be called with an addressing mode it can't handle");
4502}
4503
4504/// Return true if the opcode is a post-index ld/st instruction, which really
4505/// loads from base+0.
4506static bool isPostIndexLdStOpcode(unsigned Opcode) {
4507 switch (Opcode) {
4508 default:
4509 return false;
4510 case AArch64::LD1Fourv16b_POST:
4511 case AArch64::LD1Fourv1d_POST:
4512 case AArch64::LD1Fourv2d_POST:
4513 case AArch64::LD1Fourv2s_POST:
4514 case AArch64::LD1Fourv4h_POST:
4515 case AArch64::LD1Fourv4s_POST:
4516 case AArch64::LD1Fourv8b_POST:
4517 case AArch64::LD1Fourv8h_POST:
4518 case AArch64::LD1Onev16b_POST:
4519 case AArch64::LD1Onev1d_POST:
4520 case AArch64::LD1Onev2d_POST:
4521 case AArch64::LD1Onev2s_POST:
4522 case AArch64::LD1Onev4h_POST:
4523 case AArch64::LD1Onev4s_POST:
4524 case AArch64::LD1Onev8b_POST:
4525 case AArch64::LD1Onev8h_POST:
4526 case AArch64::LD1Rv16b_POST:
4527 case AArch64::LD1Rv1d_POST:
4528 case AArch64::LD1Rv2d_POST:
4529 case AArch64::LD1Rv2s_POST:
4530 case AArch64::LD1Rv4h_POST:
4531 case AArch64::LD1Rv4s_POST:
4532 case AArch64::LD1Rv8b_POST:
4533 case AArch64::LD1Rv8h_POST:
4534 case AArch64::LD1Threev16b_POST:
4535 case AArch64::LD1Threev1d_POST:
4536 case AArch64::LD1Threev2d_POST:
4537 case AArch64::LD1Threev2s_POST:
4538 case AArch64::LD1Threev4h_POST:
4539 case AArch64::LD1Threev4s_POST:
4540 case AArch64::LD1Threev8b_POST:
4541 case AArch64::LD1Threev8h_POST:
4542 case AArch64::LD1Twov16b_POST:
4543 case AArch64::LD1Twov1d_POST:
4544 case AArch64::LD1Twov2d_POST:
4545 case AArch64::LD1Twov2s_POST:
4546 case AArch64::LD1Twov4h_POST:
4547 case AArch64::LD1Twov4s_POST:
4548 case AArch64::LD1Twov8b_POST:
4549 case AArch64::LD1Twov8h_POST:
4550 case AArch64::LD1i16_POST:
4551 case AArch64::LD1i32_POST:
4552 case AArch64::LD1i64_POST:
4553 case AArch64::LD1i8_POST:
4554 case AArch64::LD2Rv16b_POST:
4555 case AArch64::LD2Rv1d_POST:
4556 case AArch64::LD2Rv2d_POST:
4557 case AArch64::LD2Rv2s_POST:
4558 case AArch64::LD2Rv4h_POST:
4559 case AArch64::LD2Rv4s_POST:
4560 case AArch64::LD2Rv8b_POST:
4561 case AArch64::LD2Rv8h_POST:
4562 case AArch64::LD2Twov16b_POST:
4563 case AArch64::LD2Twov2d_POST:
4564 case AArch64::LD2Twov2s_POST:
4565 case AArch64::LD2Twov4h_POST:
4566 case AArch64::LD2Twov4s_POST:
4567 case AArch64::LD2Twov8b_POST:
4568 case AArch64::LD2Twov8h_POST:
4569 case AArch64::LD2i16_POST:
4570 case AArch64::LD2i32_POST:
4571 case AArch64::LD2i64_POST:
4572 case AArch64::LD2i8_POST:
4573 case AArch64::LD3Rv16b_POST:
4574 case AArch64::LD3Rv1d_POST:
4575 case AArch64::LD3Rv2d_POST:
4576 case AArch64::LD3Rv2s_POST:
4577 case AArch64::LD3Rv4h_POST:
4578 case AArch64::LD3Rv4s_POST:
4579 case AArch64::LD3Rv8b_POST:
4580 case AArch64::LD3Rv8h_POST:
4581 case AArch64::LD3Threev16b_POST:
4582 case AArch64::LD3Threev2d_POST:
4583 case AArch64::LD3Threev2s_POST:
4584 case AArch64::LD3Threev4h_POST:
4585 case AArch64::LD3Threev4s_POST:
4586 case AArch64::LD3Threev8b_POST:
4587 case AArch64::LD3Threev8h_POST:
4588 case AArch64::LD3i16_POST:
4589 case AArch64::LD3i32_POST:
4590 case AArch64::LD3i64_POST:
4591 case AArch64::LD3i8_POST:
4592 case AArch64::LD4Fourv16b_POST:
4593 case AArch64::LD4Fourv2d_POST:
4594 case AArch64::LD4Fourv2s_POST:
4595 case AArch64::LD4Fourv4h_POST:
4596 case AArch64::LD4Fourv4s_POST:
4597 case AArch64::LD4Fourv8b_POST:
4598 case AArch64::LD4Fourv8h_POST:
4599 case AArch64::LD4Rv16b_POST:
4600 case AArch64::LD4Rv1d_POST:
4601 case AArch64::LD4Rv2d_POST:
4602 case AArch64::LD4Rv2s_POST:
4603 case AArch64::LD4Rv4h_POST:
4604 case AArch64::LD4Rv4s_POST:
4605 case AArch64::LD4Rv8b_POST:
4606 case AArch64::LD4Rv8h_POST:
4607 case AArch64::LD4i16_POST:
4608 case AArch64::LD4i32_POST:
4609 case AArch64::LD4i64_POST:
4610 case AArch64::LD4i8_POST:
4611 case AArch64::LDAPRWpost:
4612 case AArch64::LDAPRXpost:
4613 case AArch64::LDIAPPWpost:
4614 case AArch64::LDIAPPXpost:
4615 case AArch64::LDPDpost:
4616 case AArch64::LDPQpost:
4617 case AArch64::LDPSWpost:
4618 case AArch64::LDPSpost:
4619 case AArch64::LDPWpost:
4620 case AArch64::LDPXpost:
4621 case AArch64::LDRBBpost:
4622 case AArch64::LDRBpost:
4623 case AArch64::LDRDpost:
4624 case AArch64::LDRHHpost:
4625 case AArch64::LDRHpost:
4626 case AArch64::LDRQpost:
4627 case AArch64::LDRSBWpost:
4628 case AArch64::LDRSBXpost:
4629 case AArch64::LDRSHWpost:
4630 case AArch64::LDRSHXpost:
4631 case AArch64::LDRSWpost:
4632 case AArch64::LDRSpost:
4633 case AArch64::LDRWpost:
4634 case AArch64::LDRXpost:
4635 case AArch64::ST1Fourv16b_POST:
4636 case AArch64::ST1Fourv1d_POST:
4637 case AArch64::ST1Fourv2d_POST:
4638 case AArch64::ST1Fourv2s_POST:
4639 case AArch64::ST1Fourv4h_POST:
4640 case AArch64::ST1Fourv4s_POST:
4641 case AArch64::ST1Fourv8b_POST:
4642 case AArch64::ST1Fourv8h_POST:
4643 case AArch64::ST1Onev16b_POST:
4644 case AArch64::ST1Onev1d_POST:
4645 case AArch64::ST1Onev2d_POST:
4646 case AArch64::ST1Onev2s_POST:
4647 case AArch64::ST1Onev4h_POST:
4648 case AArch64::ST1Onev4s_POST:
4649 case AArch64::ST1Onev8b_POST:
4650 case AArch64::ST1Onev8h_POST:
4651 case AArch64::ST1Threev16b_POST:
4652 case AArch64::ST1Threev1d_POST:
4653 case AArch64::ST1Threev2d_POST:
4654 case AArch64::ST1Threev2s_POST:
4655 case AArch64::ST1Threev4h_POST:
4656 case AArch64::ST1Threev4s_POST:
4657 case AArch64::ST1Threev8b_POST:
4658 case AArch64::ST1Threev8h_POST:
4659 case AArch64::ST1Twov16b_POST:
4660 case AArch64::ST1Twov1d_POST:
4661 case AArch64::ST1Twov2d_POST:
4662 case AArch64::ST1Twov2s_POST:
4663 case AArch64::ST1Twov4h_POST:
4664 case AArch64::ST1Twov4s_POST:
4665 case AArch64::ST1Twov8b_POST:
4666 case AArch64::ST1Twov8h_POST:
4667 case AArch64::ST1i16_POST:
4668 case AArch64::ST1i32_POST:
4669 case AArch64::ST1i64_POST:
4670 case AArch64::ST1i8_POST:
4671 case AArch64::ST2GPostIndex:
4672 case AArch64::ST2Twov16b_POST:
4673 case AArch64::ST2Twov2d_POST:
4674 case AArch64::ST2Twov2s_POST:
4675 case AArch64::ST2Twov4h_POST:
4676 case AArch64::ST2Twov4s_POST:
4677 case AArch64::ST2Twov8b_POST:
4678 case AArch64::ST2Twov8h_POST:
4679 case AArch64::ST2i16_POST:
4680 case AArch64::ST2i32_POST:
4681 case AArch64::ST2i64_POST:
4682 case AArch64::ST2i8_POST:
4683 case AArch64::ST3Threev16b_POST:
4684 case AArch64::ST3Threev2d_POST:
4685 case AArch64::ST3Threev2s_POST:
4686 case AArch64::ST3Threev4h_POST:
4687 case AArch64::ST3Threev4s_POST:
4688 case AArch64::ST3Threev8b_POST:
4689 case AArch64::ST3Threev8h_POST:
4690 case AArch64::ST3i16_POST:
4691 case AArch64::ST3i32_POST:
4692 case AArch64::ST3i64_POST:
4693 case AArch64::ST3i8_POST:
4694 case AArch64::ST4Fourv16b_POST:
4695 case AArch64::ST4Fourv2d_POST:
4696 case AArch64::ST4Fourv2s_POST:
4697 case AArch64::ST4Fourv4h_POST:
4698 case AArch64::ST4Fourv4s_POST:
4699 case AArch64::ST4Fourv8b_POST:
4700 case AArch64::ST4Fourv8h_POST:
4701 case AArch64::ST4i16_POST:
4702 case AArch64::ST4i32_POST:
4703 case AArch64::ST4i64_POST:
4704 case AArch64::ST4i8_POST:
4705 case AArch64::STGPostIndex:
4706 case AArch64::STGPpost:
4707 case AArch64::STPDpost:
4708 case AArch64::STPQpost:
4709 case AArch64::STPSpost:
4710 case AArch64::STPWpost:
4711 case AArch64::STPXpost:
4712 case AArch64::STRBBpost:
4713 case AArch64::STRBpost:
4714 case AArch64::STRDpost:
4715 case AArch64::STRHHpost:
4716 case AArch64::STRHpost:
4717 case AArch64::STRQpost:
4718 case AArch64::STRSpost:
4719 case AArch64::STRWpost:
4720 case AArch64::STRXpost:
4721 case AArch64::STZ2GPostIndex:
4722 case AArch64::STZGPostIndex:
4723 return true;
4724 }
4725}
4726
4727bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
4728 const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
4729 bool &OffsetIsScalable, TypeSize &Width) const {
4730 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4731 // Handle only loads/stores with base register followed by immediate offset.
4732 if (LdSt.getNumExplicitOperands() == 3) {
4733 // Non-paired instruction (e.g., ldr x1, [x0, #8]).
4734 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
4735 !LdSt.getOperand(i: 2).isImm())
4736 return false;
4737 } else if (LdSt.getNumExplicitOperands() == 4) {
4738 // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
4739 if (!LdSt.getOperand(i: 1).isReg() ||
4740 (!LdSt.getOperand(i: 2).isReg() && !LdSt.getOperand(i: 2).isFI()) ||
4741 !LdSt.getOperand(i: 3).isImm())
4742 return false;
4743 } else
4744 return false;
4745
4746 // Get the scaling factor for the instruction and set the width for the
4747 // instruction.
4748 TypeSize Scale(0U, false);
4749 int64_t Dummy1, Dummy2;
4750
4751 // If this returns false, then it's an instruction we don't want to handle.
4752 if (!getMemOpInfo(Opcode: LdSt.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2))
4753 return false;
4754
4755 // Compute the offset. Offset is calculated as the immediate operand
4756 // multiplied by the scaling factor. Unscaled instructions have scaling factor
4757 // set to 1. Postindex are a special case which have an offset of 0.
4758 if (isPostIndexLdStOpcode(Opcode: LdSt.getOpcode())) {
4759 BaseOp = &LdSt.getOperand(i: 2);
4760 Offset = 0;
4761 } else if (LdSt.getNumExplicitOperands() == 3) {
4762 BaseOp = &LdSt.getOperand(i: 1);
4763 Offset = LdSt.getOperand(i: 2).getImm() * Scale.getKnownMinValue();
4764 } else {
4765 assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
4766 BaseOp = &LdSt.getOperand(i: 2);
4767 Offset = LdSt.getOperand(i: 3).getImm() * Scale.getKnownMinValue();
4768 }
4769 OffsetIsScalable = Scale.isScalable();
4770
4771 return BaseOp->isReg() || BaseOp->isFI();
4772}
4773
4774MachineOperand &
4775AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
4776 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4777 MachineOperand &OfsOp = LdSt.getOperand(i: LdSt.getNumExplicitOperands() - 1);
4778 assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
4779 return OfsOp;
4780}
4781
4782bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
4783 TypeSize &Width, int64_t &MinOffset,
4784 int64_t &MaxOffset) {
4785 switch (Opcode) {
4786 // Not a memory operation or something we want to handle.
4787 default:
4788 Scale = Width = TypeSize::getFixed(ExactSize: 0);
4789 MinOffset = MaxOffset = 0;
4790 return false;
4791 // LDR / STR
4792 case AArch64::LDRQui:
4793 case AArch64::STRQui:
4794 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4795 MinOffset = 0;
4796 MaxOffset = 4095;
4797 break;
4798 case AArch64::LDRXui:
4799 case AArch64::LDRDui:
4800 case AArch64::STRXui:
4801 case AArch64::STRDui:
4802 case AArch64::PRFMui:
4803 case AArch64::ATOMIC_STORE_HINT_Xui:
4804 case AArch64::ATOMIC_STORE_HINT_Dui:
4805 Scale = Width = TypeSize::getFixed(ExactSize: 8);
4806 MinOffset = 0;
4807 MaxOffset = 4095;
4808 break;
4809 case AArch64::LDRWui:
4810 case AArch64::LDRSui:
4811 case AArch64::LDRSWui:
4812 case AArch64::STRWui:
4813 case AArch64::STRSui:
4814 case AArch64::ATOMIC_STORE_HINT_Wui:
4815 case AArch64::ATOMIC_STORE_HINT_Sui:
4816 Scale = Width = TypeSize::getFixed(ExactSize: 4);
4817 MinOffset = 0;
4818 MaxOffset = 4095;
4819 break;
4820 case AArch64::LDRHui:
4821 case AArch64::LDRHHui:
4822 case AArch64::LDRSHWui:
4823 case AArch64::LDRSHXui:
4824 case AArch64::STRHui:
4825 case AArch64::STRHHui:
4826 case AArch64::ATOMIC_STORE_HINT_Hui:
4827 Scale = Width = TypeSize::getFixed(ExactSize: 2);
4828 MinOffset = 0;
4829 MaxOffset = 4095;
4830 break;
4831 case AArch64::LDRBui:
4832 case AArch64::LDRBBui:
4833 case AArch64::LDRSBWui:
4834 case AArch64::LDRSBXui:
4835 case AArch64::STRBui:
4836 case AArch64::STRBBui:
4837 case AArch64::ATOMIC_STORE_HINT_Bui:
4838 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4839 MinOffset = 0;
4840 MaxOffset = 4095;
4841 break;
4842 // post/pre inc
4843 case AArch64::STRQpre:
4844 case AArch64::LDRQpost:
4845 Scale = TypeSize::getFixed(ExactSize: 1);
4846 Width = TypeSize::getFixed(ExactSize: 16);
4847 MinOffset = -256;
4848 MaxOffset = 255;
4849 break;
4850 case AArch64::LDRDpost:
4851 case AArch64::LDRDpre:
4852 case AArch64::LDRXpost:
4853 case AArch64::LDRXpre:
4854 case AArch64::STRDpost:
4855 case AArch64::STRDpre:
4856 case AArch64::STRXpost:
4857 case AArch64::STRXpre:
4858 Scale = TypeSize::getFixed(ExactSize: 1);
4859 Width = TypeSize::getFixed(ExactSize: 8);
4860 MinOffset = -256;
4861 MaxOffset = 255;
4862 break;
4863 case AArch64::STRWpost:
4864 case AArch64::STRWpre:
4865 case AArch64::LDRWpost:
4866 case AArch64::LDRWpre:
4867 case AArch64::STRSpost:
4868 case AArch64::STRSpre:
4869 case AArch64::LDRSpost:
4870 case AArch64::LDRSpre:
4871 Scale = TypeSize::getFixed(ExactSize: 1);
4872 Width = TypeSize::getFixed(ExactSize: 4);
4873 MinOffset = -256;
4874 MaxOffset = 255;
4875 break;
4876 case AArch64::LDRHpost:
4877 case AArch64::LDRHpre:
4878 case AArch64::STRHpost:
4879 case AArch64::STRHpre:
4880 case AArch64::LDRHHpost:
4881 case AArch64::LDRHHpre:
4882 case AArch64::STRHHpost:
4883 case AArch64::STRHHpre:
4884 Scale = TypeSize::getFixed(ExactSize: 1);
4885 Width = TypeSize::getFixed(ExactSize: 2);
4886 MinOffset = -256;
4887 MaxOffset = 255;
4888 break;
4889 case AArch64::LDRBpost:
4890 case AArch64::LDRBpre:
4891 case AArch64::STRBpost:
4892 case AArch64::STRBpre:
4893 case AArch64::LDRBBpost:
4894 case AArch64::LDRBBpre:
4895 case AArch64::STRBBpost:
4896 case AArch64::STRBBpre:
4897 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4898 MinOffset = -256;
4899 MaxOffset = 255;
4900 break;
4901 // Unscaled
4902 case AArch64::LDURQi:
4903 case AArch64::STURQi:
4904 Scale = TypeSize::getFixed(ExactSize: 1);
4905 Width = TypeSize::getFixed(ExactSize: 16);
4906 MinOffset = -256;
4907 MaxOffset = 255;
4908 break;
4909 case AArch64::LDURXi:
4910 case AArch64::LDURDi:
4911 case AArch64::LDAPURXi:
4912 case AArch64::STURXi:
4913 case AArch64::STURDi:
4914 case AArch64::STLURXi:
4915 case AArch64::PRFUMi:
4916 case AArch64::ATOMIC_STORE_HINT_Xi:
4917 case AArch64::ATOMIC_STORE_HINT_Di:
4918 Scale = TypeSize::getFixed(ExactSize: 1);
4919 Width = TypeSize::getFixed(ExactSize: 8);
4920 MinOffset = -256;
4921 MaxOffset = 255;
4922 break;
4923 case AArch64::LDURWi:
4924 case AArch64::LDURSi:
4925 case AArch64::LDURSWi:
4926 case AArch64::LDAPURi:
4927 case AArch64::LDAPURSWi:
4928 case AArch64::STURWi:
4929 case AArch64::STURSi:
4930 case AArch64::STLURWi:
4931 case AArch64::ATOMIC_STORE_HINT_Wi:
4932 case AArch64::ATOMIC_STORE_HINT_Si:
4933 Scale = TypeSize::getFixed(ExactSize: 1);
4934 Width = TypeSize::getFixed(ExactSize: 4);
4935 MinOffset = -256;
4936 MaxOffset = 255;
4937 break;
4938 case AArch64::LDURHi:
4939 case AArch64::LDURHHi:
4940 case AArch64::LDURSHXi:
4941 case AArch64::LDURSHWi:
4942 case AArch64::LDAPURHi:
4943 case AArch64::LDAPURSHWi:
4944 case AArch64::LDAPURSHXi:
4945 case AArch64::STURHi:
4946 case AArch64::STURHHi:
4947 case AArch64::STLURHi:
4948 case AArch64::ATOMIC_STORE_HINT_Hi:
4949 Scale = TypeSize::getFixed(ExactSize: 1);
4950 Width = TypeSize::getFixed(ExactSize: 2);
4951 MinOffset = -256;
4952 MaxOffset = 255;
4953 break;
4954 case AArch64::LDURBi:
4955 case AArch64::LDURBBi:
4956 case AArch64::LDURSBXi:
4957 case AArch64::LDURSBWi:
4958 case AArch64::LDAPURBi:
4959 case AArch64::LDAPURSBWi:
4960 case AArch64::LDAPURSBXi:
4961 case AArch64::STURBi:
4962 case AArch64::STURBBi:
4963 case AArch64::STLURBi:
4964 case AArch64::ATOMIC_STORE_HINT_Bi:
4965 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4966 MinOffset = -256;
4967 MaxOffset = 255;
4968 break;
4969 // LDP / STP (including pre/post inc)
4970 case AArch64::LDPQi:
4971 case AArch64::LDNPQi:
4972 case AArch64::STPQi:
4973 case AArch64::STNPQi:
4974 case AArch64::LDPQpost:
4975 case AArch64::LDPQpre:
4976 case AArch64::STPQpost:
4977 case AArch64::STPQpre:
4978 Scale = TypeSize::getFixed(ExactSize: 16);
4979 Width = TypeSize::getFixed(ExactSize: 16 * 2);
4980 MinOffset = -64;
4981 MaxOffset = 63;
4982 break;
4983 case AArch64::LDPXi:
4984 case AArch64::LDPDi:
4985 case AArch64::LDNPXi:
4986 case AArch64::LDNPDi:
4987 case AArch64::STPXi:
4988 case AArch64::STPDi:
4989 case AArch64::STNPXi:
4990 case AArch64::STNPDi:
4991 case AArch64::LDPDpost:
4992 case AArch64::LDPDpre:
4993 case AArch64::LDPXpost:
4994 case AArch64::LDPXpre:
4995 case AArch64::STPDpost:
4996 case AArch64::STPDpre:
4997 case AArch64::STPXpost:
4998 case AArch64::STPXpre:
4999 Scale = TypeSize::getFixed(ExactSize: 8);
5000 Width = TypeSize::getFixed(ExactSize: 8 * 2);
5001 MinOffset = -64;
5002 MaxOffset = 63;
5003 break;
5004 case AArch64::LDPWi:
5005 case AArch64::LDPSi:
5006 case AArch64::LDNPWi:
5007 case AArch64::LDNPSi:
5008 case AArch64::STPWi:
5009 case AArch64::STPSi:
5010 case AArch64::STNPWi:
5011 case AArch64::STNPSi:
5012 case AArch64::LDPSpost:
5013 case AArch64::LDPSpre:
5014 case AArch64::LDPWpost:
5015 case AArch64::LDPWpre:
5016 case AArch64::STPSpost:
5017 case AArch64::STPSpre:
5018 case AArch64::STPWpost:
5019 case AArch64::STPWpre:
5020 Scale = TypeSize::getFixed(ExactSize: 4);
5021 Width = TypeSize::getFixed(ExactSize: 4 * 2);
5022 MinOffset = -64;
5023 MaxOffset = 63;
5024 break;
5025 case AArch64::StoreSwiftAsyncContext:
5026 // Store is an STRXui, but there might be an ADDXri in the expansion too.
5027 Scale = TypeSize::getFixed(ExactSize: 1);
5028 Width = TypeSize::getFixed(ExactSize: 8);
5029 MinOffset = 0;
5030 MaxOffset = 4095;
5031 break;
5032 case AArch64::ADDG:
5033 Scale = TypeSize::getFixed(ExactSize: 16);
5034 Width = TypeSize::getFixed(ExactSize: 0);
5035 MinOffset = 0;
5036 MaxOffset = 63;
5037 break;
5038 case AArch64::TAGPstack:
5039 Scale = TypeSize::getFixed(ExactSize: 16);
5040 Width = TypeSize::getFixed(ExactSize: 0);
5041 // TAGP with a negative offset turns into SUBP, which has a maximum offset
5042 // of 63 (not 64!).
5043 MinOffset = -63;
5044 MaxOffset = 63;
5045 break;
5046 case AArch64::LDG:
5047 case AArch64::STGi:
5048 case AArch64::STGPreIndex:
5049 case AArch64::STGPostIndex:
5050 case AArch64::STZGi:
5051 case AArch64::STZGPreIndex:
5052 case AArch64::STZGPostIndex:
5053 Scale = Width = TypeSize::getFixed(ExactSize: 16);
5054 MinOffset = -256;
5055 MaxOffset = 255;
5056 break;
5057 // SVE
5058 case AArch64::STR_ZZZZXI:
5059 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
5060 case AArch64::LDR_ZZZZXI:
5061 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
5062 Scale = TypeSize::getScalable(MinimumSize: 16);
5063 Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5064 MinOffset = -256;
5065 MaxOffset = 252;
5066 break;
5067 case AArch64::STR_ZZZXI:
5068 case AArch64::LDR_ZZZXI:
5069 Scale = TypeSize::getScalable(MinimumSize: 16);
5070 Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5071 MinOffset = -256;
5072 MaxOffset = 253;
5073 break;
5074 case AArch64::STR_ZZXI:
5075 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
5076 case AArch64::LDR_ZZXI:
5077 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
5078 Scale = TypeSize::getScalable(MinimumSize: 16);
5079 Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5080 MinOffset = -256;
5081 MaxOffset = 254;
5082 break;
5083 case AArch64::LDR_PXI:
5084 case AArch64::STR_PXI:
5085 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5086 MinOffset = -256;
5087 MaxOffset = 255;
5088 break;
5089 case AArch64::LDR_PPXI:
5090 case AArch64::STR_PPXI:
5091 Scale = TypeSize::getScalable(MinimumSize: 2);
5092 Width = TypeSize::getScalable(MinimumSize: 2 * 2);
5093 MinOffset = -256;
5094 MaxOffset = 254;
5095 break;
5096 case AArch64::LDR_ZXI:
5097 case AArch64::STR_ZXI:
5098 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5099 MinOffset = -256;
5100 MaxOffset = 255;
5101 break;
5102 case AArch64::LD1B_IMM:
5103 case AArch64::LD1H_IMM:
5104 case AArch64::LD1W_IMM:
5105 case AArch64::LD1D_IMM:
5106 case AArch64::LDNT1B_ZRI:
5107 case AArch64::LDNT1H_ZRI:
5108 case AArch64::LDNT1W_ZRI:
5109 case AArch64::LDNT1D_ZRI:
5110 case AArch64::ST1B_IMM:
5111 case AArch64::ST1H_IMM:
5112 case AArch64::ST1W_IMM:
5113 case AArch64::ST1D_IMM:
5114 case AArch64::STNT1B_ZRI:
5115 case AArch64::STNT1H_ZRI:
5116 case AArch64::STNT1W_ZRI:
5117 case AArch64::STNT1D_ZRI:
5118 case AArch64::LDNF1B_IMM:
5119 case AArch64::LDNF1H_IMM:
5120 case AArch64::LDNF1W_IMM:
5121 case AArch64::LDNF1D_IMM:
5122 // A full vectors worth of data
5123 // Width = mbytes * elements
5124 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5125 MinOffset = -8;
5126 MaxOffset = 7;
5127 break;
5128 case AArch64::LD2B_IMM:
5129 case AArch64::LD2H_IMM:
5130 case AArch64::LD2W_IMM:
5131 case AArch64::LD2D_IMM:
5132 case AArch64::ST2B_IMM:
5133 case AArch64::ST2H_IMM:
5134 case AArch64::ST2W_IMM:
5135 case AArch64::ST2D_IMM:
5136 case AArch64::LD1B_2Z_IMM:
5137 case AArch64::LD1B_2Z_STRIDED_IMM:
5138 case AArch64::LD1H_2Z_IMM:
5139 case AArch64::LD1H_2Z_STRIDED_IMM:
5140 case AArch64::LD1W_2Z_IMM:
5141 case AArch64::LD1W_2Z_STRIDED_IMM:
5142 case AArch64::LD1D_2Z_IMM:
5143 case AArch64::LD1D_2Z_STRIDED_IMM:
5144 case AArch64::LD1B_2Z_IMM_PSEUDO:
5145 case AArch64::LD1H_2Z_IMM_PSEUDO:
5146 case AArch64::LD1W_2Z_IMM_PSEUDO:
5147 case AArch64::LD1D_2Z_IMM_PSEUDO:
5148 case AArch64::ST1B_2Z_IMM:
5149 case AArch64::ST1B_2Z_STRIDED_IMM:
5150 case AArch64::ST1H_2Z_IMM:
5151 case AArch64::ST1H_2Z_STRIDED_IMM:
5152 case AArch64::ST1W_2Z_IMM:
5153 case AArch64::ST1W_2Z_STRIDED_IMM:
5154 case AArch64::ST1D_2Z_IMM:
5155 case AArch64::ST1D_2Z_STRIDED_IMM:
5156 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
5157 case AArch64::LDNT1B_2Z_IMM:
5158 case AArch64::LDNT1B_2Z_STRIDED_IMM:
5159 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
5160 case AArch64::LDNT1H_2Z_IMM:
5161 case AArch64::LDNT1H_2Z_STRIDED_IMM:
5162 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
5163 case AArch64::LDNT1W_2Z_IMM:
5164 case AArch64::LDNT1W_2Z_STRIDED_IMM:
5165 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
5166 case AArch64::LDNT1D_2Z_IMM:
5167 case AArch64::LDNT1D_2Z_STRIDED_IMM:
5168 case AArch64::STNT1B_2Z_IMM:
5169 case AArch64::STNT1B_2Z_STRIDED_IMM:
5170 case AArch64::STNT1H_2Z_IMM:
5171 case AArch64::STNT1H_2Z_STRIDED_IMM:
5172 case AArch64::STNT1W_2Z_IMM:
5173 case AArch64::STNT1W_2Z_STRIDED_IMM:
5174 case AArch64::STNT1D_2Z_IMM:
5175 case AArch64::STNT1D_2Z_STRIDED_IMM:
5176 case AArch64::ST1B_2Z_IMM_PSEUDO:
5177 case AArch64::ST1H_2Z_IMM_PSEUDO:
5178 case AArch64::ST1W_2Z_IMM_PSEUDO:
5179 case AArch64::ST1D_2Z_IMM_PSEUDO:
5180 case AArch64::STNT1B_2Z_IMM_PSEUDO:
5181 case AArch64::STNT1H_2Z_IMM_PSEUDO:
5182 case AArch64::STNT1W_2Z_IMM_PSEUDO:
5183 case AArch64::STNT1D_2Z_IMM_PSEUDO:
5184 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5185 MinOffset = -8;
5186 MaxOffset = 7;
5187 break;
5188 case AArch64::LD3B_IMM:
5189 case AArch64::LD3H_IMM:
5190 case AArch64::LD3W_IMM:
5191 case AArch64::LD3D_IMM:
5192 case AArch64::ST3B_IMM:
5193 case AArch64::ST3H_IMM:
5194 case AArch64::ST3W_IMM:
5195 case AArch64::ST3D_IMM:
5196 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5197 MinOffset = -8;
5198 MaxOffset = 7;
5199 break;
5200 case AArch64::LD4B_IMM:
5201 case AArch64::LD4H_IMM:
5202 case AArch64::LD4W_IMM:
5203 case AArch64::LD4D_IMM:
5204 case AArch64::ST4B_IMM:
5205 case AArch64::ST4H_IMM:
5206 case AArch64::ST4W_IMM:
5207 case AArch64::ST4D_IMM:
5208 case AArch64::LD1B_4Z_IMM:
5209 case AArch64::LD1B_4Z_STRIDED_IMM:
5210 case AArch64::LD1H_4Z_IMM:
5211 case AArch64::LD1H_4Z_STRIDED_IMM:
5212 case AArch64::LD1W_4Z_IMM:
5213 case AArch64::LD1W_4Z_STRIDED_IMM:
5214 case AArch64::LD1D_4Z_IMM:
5215 case AArch64::LD1D_4Z_STRIDED_IMM:
5216 case AArch64::LD1B_4Z_IMM_PSEUDO:
5217 case AArch64::LD1H_4Z_IMM_PSEUDO:
5218 case AArch64::LD1W_4Z_IMM_PSEUDO:
5219 case AArch64::LD1D_4Z_IMM_PSEUDO:
5220 case AArch64::ST1B_4Z_IMM:
5221 case AArch64::ST1B_4Z_STRIDED_IMM:
5222 case AArch64::ST1H_4Z_IMM:
5223 case AArch64::ST1H_4Z_STRIDED_IMM:
5224 case AArch64::ST1W_4Z_IMM:
5225 case AArch64::ST1W_4Z_STRIDED_IMM:
5226 case AArch64::ST1D_4Z_IMM:
5227 case AArch64::ST1D_4Z_STRIDED_IMM:
5228 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
5229 case AArch64::LDNT1B_4Z_IMM:
5230 case AArch64::LDNT1B_4Z_STRIDED_IMM:
5231 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
5232 case AArch64::LDNT1H_4Z_IMM:
5233 case AArch64::LDNT1H_4Z_STRIDED_IMM:
5234 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
5235 case AArch64::LDNT1W_4Z_IMM:
5236 case AArch64::LDNT1W_4Z_STRIDED_IMM:
5237 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
5238 case AArch64::LDNT1D_4Z_IMM:
5239 case AArch64::LDNT1D_4Z_STRIDED_IMM:
5240 case AArch64::STNT1B_4Z_IMM:
5241 case AArch64::STNT1B_4Z_STRIDED_IMM:
5242 case AArch64::STNT1H_4Z_IMM:
5243 case AArch64::STNT1H_4Z_STRIDED_IMM:
5244 case AArch64::STNT1W_4Z_IMM:
5245 case AArch64::STNT1W_4Z_STRIDED_IMM:
5246 case AArch64::STNT1D_4Z_IMM:
5247 case AArch64::STNT1D_4Z_STRIDED_IMM:
5248 case AArch64::ST1B_4Z_IMM_PSEUDO:
5249 case AArch64::ST1H_4Z_IMM_PSEUDO:
5250 case AArch64::ST1W_4Z_IMM_PSEUDO:
5251 case AArch64::ST1D_4Z_IMM_PSEUDO:
5252 case AArch64::STNT1B_4Z_IMM_PSEUDO:
5253 case AArch64::STNT1H_4Z_IMM_PSEUDO:
5254 case AArch64::STNT1W_4Z_IMM_PSEUDO:
5255 case AArch64::STNT1D_4Z_IMM_PSEUDO:
5256 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5257 MinOffset = -8;
5258 MaxOffset = 7;
5259 break;
5260 case AArch64::LD1B_H_IMM:
5261 case AArch64::LD1SB_H_IMM:
5262 case AArch64::LD1H_S_IMM:
5263 case AArch64::LD1SH_S_IMM:
5264 case AArch64::LD1W_D_IMM:
5265 case AArch64::LD1SW_D_IMM:
5266 case AArch64::ST1B_H_IMM:
5267 case AArch64::ST1H_S_IMM:
5268 case AArch64::ST1W_D_IMM:
5269 case AArch64::LDNF1B_H_IMM:
5270 case AArch64::LDNF1SB_H_IMM:
5271 case AArch64::LDNF1H_S_IMM:
5272 case AArch64::LDNF1SH_S_IMM:
5273 case AArch64::LDNF1W_D_IMM:
5274 case AArch64::LDNF1SW_D_IMM:
5275 // A half vector worth of data
5276 // Width = mbytes * elements
5277 Scale = Width = TypeSize::getScalable(MinimumSize: 8);
5278 MinOffset = -8;
5279 MaxOffset = 7;
5280 break;
5281 case AArch64::LD1B_S_IMM:
5282 case AArch64::LD1SB_S_IMM:
5283 case AArch64::LD1H_D_IMM:
5284 case AArch64::LD1SH_D_IMM:
5285 case AArch64::ST1B_S_IMM:
5286 case AArch64::ST1H_D_IMM:
5287 case AArch64::LDNF1B_S_IMM:
5288 case AArch64::LDNF1SB_S_IMM:
5289 case AArch64::LDNF1H_D_IMM:
5290 case AArch64::LDNF1SH_D_IMM:
5291 // A quarter vector worth of data
5292 // Width = mbytes * elements
5293 Scale = Width = TypeSize::getScalable(MinimumSize: 4);
5294 MinOffset = -8;
5295 MaxOffset = 7;
5296 break;
5297 case AArch64::LD1B_D_IMM:
5298 case AArch64::LD1SB_D_IMM:
5299 case AArch64::ST1B_D_IMM:
5300 case AArch64::LDNF1B_D_IMM:
5301 case AArch64::LDNF1SB_D_IMM:
5302 // A eighth vector worth of data
5303 // Width = mbytes * elements
5304 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5305 MinOffset = -8;
5306 MaxOffset = 7;
5307 break;
5308 case AArch64::ST2Gi:
5309 case AArch64::ST2GPreIndex:
5310 case AArch64::ST2GPostIndex:
5311 case AArch64::STZ2Gi:
5312 case AArch64::STZ2GPreIndex:
5313 case AArch64::STZ2GPostIndex:
5314 Scale = TypeSize::getFixed(ExactSize: 16);
5315 Width = TypeSize::getFixed(ExactSize: 32);
5316 MinOffset = -256;
5317 MaxOffset = 255;
5318 break;
5319 case AArch64::STGPi:
5320 case AArch64::STGPpost:
5321 case AArch64::STGPpre:
5322 Scale = Width = TypeSize::getFixed(ExactSize: 16);
5323 MinOffset = -64;
5324 MaxOffset = 63;
5325 break;
5326 case AArch64::LD1RB_IMM:
5327 case AArch64::LD1RB_H_IMM:
5328 case AArch64::LD1RB_S_IMM:
5329 case AArch64::LD1RB_D_IMM:
5330 case AArch64::LD1RSB_H_IMM:
5331 case AArch64::LD1RSB_S_IMM:
5332 case AArch64::LD1RSB_D_IMM:
5333 Scale = Width = TypeSize::getFixed(ExactSize: 1);
5334 MinOffset = 0;
5335 MaxOffset = 63;
5336 break;
5337 case AArch64::LD1RH_IMM:
5338 case AArch64::LD1RH_S_IMM:
5339 case AArch64::LD1RH_D_IMM:
5340 case AArch64::LD1RSH_S_IMM:
5341 case AArch64::LD1RSH_D_IMM:
5342 Scale = Width = TypeSize::getFixed(ExactSize: 2);
5343 MinOffset = 0;
5344 MaxOffset = 63;
5345 break;
5346 case AArch64::LD1RW_IMM:
5347 case AArch64::LD1RW_D_IMM:
5348 case AArch64::LD1RSW_IMM:
5349 Scale = Width = TypeSize::getFixed(ExactSize: 4);
5350 MinOffset = 0;
5351 MaxOffset = 63;
5352 break;
5353 case AArch64::LD1RD_IMM:
5354 Scale = Width = TypeSize::getFixed(ExactSize: 8);
5355 MinOffset = 0;
5356 MaxOffset = 63;
5357 break;
5358 }
5359
5360 return true;
5361}
5362
5363// Scaling factor for unscaled load or store.
5364int AArch64InstrInfo::getMemScale(unsigned Opc) {
5365 switch (Opc) {
5366 default:
5367 llvm_unreachable("Opcode has unknown scale!");
5368 case AArch64::LDRBui:
5369 case AArch64::LDRBBui:
5370 case AArch64::LDURBBi:
5371 case AArch64::LDRSBWui:
5372 case AArch64::LDURSBWi:
5373 case AArch64::STRBui:
5374 case AArch64::STRBBui:
5375 case AArch64::STURBBi:
5376 return 1;
5377 case AArch64::LDRHui:
5378 case AArch64::LDRHHui:
5379 case AArch64::LDURHHi:
5380 case AArch64::LDRSHWui:
5381 case AArch64::LDURSHWi:
5382 case AArch64::STRHui:
5383 case AArch64::STRHHui:
5384 case AArch64::STURHHi:
5385 return 2;
5386 case AArch64::LDRSui:
5387 case AArch64::LDURSi:
5388 case AArch64::LDRSpre:
5389 case AArch64::LDRSWui:
5390 case AArch64::LDURSWi:
5391 case AArch64::LDRSWpre:
5392 case AArch64::LDRWpre:
5393 case AArch64::LDRWui:
5394 case AArch64::LDURWi:
5395 case AArch64::STRSui:
5396 case AArch64::STURSi:
5397 case AArch64::STRSpre:
5398 case AArch64::STRWui:
5399 case AArch64::STURWi:
5400 case AArch64::STRWpre:
5401 case AArch64::LDPSi:
5402 case AArch64::LDPSWi:
5403 case AArch64::LDPWi:
5404 case AArch64::STPSi:
5405 case AArch64::STPWi:
5406 return 4;
5407 case AArch64::LDRDui:
5408 case AArch64::LDURDi:
5409 case AArch64::LDRDpre:
5410 case AArch64::LDRXui:
5411 case AArch64::LDURXi:
5412 case AArch64::LDRXpre:
5413 case AArch64::STRDui:
5414 case AArch64::STURDi:
5415 case AArch64::STRDpre:
5416 case AArch64::STRXui:
5417 case AArch64::STURXi:
5418 case AArch64::STRXpre:
5419 case AArch64::LDPDi:
5420 case AArch64::LDPXi:
5421 case AArch64::STPDi:
5422 case AArch64::STPXi:
5423 return 8;
5424 case AArch64::LDRQui:
5425 case AArch64::LDURQi:
5426 case AArch64::STRQui:
5427 case AArch64::STURQi:
5428 case AArch64::STRQpre:
5429 case AArch64::LDPQi:
5430 case AArch64::LDRQpre:
5431 case AArch64::STPQi:
5432 case AArch64::STGi:
5433 case AArch64::STZGi:
5434 case AArch64::ST2Gi:
5435 case AArch64::STZ2Gi:
5436 case AArch64::STGPi:
5437 return 16;
5438 }
5439}
5440
5441bool AArch64InstrInfo::isPreLd(const MachineInstr &MI) {
5442 switch (MI.getOpcode()) {
5443 default:
5444 return false;
5445 case AArch64::LDRWpre:
5446 case AArch64::LDRXpre:
5447 case AArch64::LDRSWpre:
5448 case AArch64::LDRSpre:
5449 case AArch64::LDRDpre:
5450 case AArch64::LDRQpre:
5451 return true;
5452 }
5453}
5454
5455bool AArch64InstrInfo::isPreSt(const MachineInstr &MI) {
5456 switch (MI.getOpcode()) {
5457 default:
5458 return false;
5459 case AArch64::STRWpre:
5460 case AArch64::STRXpre:
5461 case AArch64::STRSpre:
5462 case AArch64::STRDpre:
5463 case AArch64::STRQpre:
5464 return true;
5465 }
5466}
5467
5468bool AArch64InstrInfo::isPreLdSt(const MachineInstr &MI) {
5469 return isPreLd(MI) || isPreSt(MI);
5470}
5471
5472bool AArch64InstrInfo::isZExtLoad(const MachineInstr &MI) {
5473 switch (MI.getOpcode()) {
5474 default:
5475 return false;
5476 case AArch64::LDURBBi:
5477 case AArch64::LDURHHi:
5478 case AArch64::LDURWi:
5479 case AArch64::LDRBBui:
5480 case AArch64::LDRHHui:
5481 case AArch64::LDRWui:
5482 case AArch64::LDRBBroX:
5483 case AArch64::LDRHHroX:
5484 case AArch64::LDRWroX:
5485 case AArch64::LDRBBroW:
5486 case AArch64::LDRHHroW:
5487 case AArch64::LDRWroW:
5488 return true;
5489 }
5490}
5491
5492bool AArch64InstrInfo::isSExtLoad(const MachineInstr &MI) {
5493 switch (MI.getOpcode()) {
5494 default:
5495 return false;
5496 case AArch64::LDURSBWi:
5497 case AArch64::LDURSHWi:
5498 case AArch64::LDURSBXi:
5499 case AArch64::LDURSHXi:
5500 case AArch64::LDURSWi:
5501 case AArch64::LDRSBWui:
5502 case AArch64::LDRSHWui:
5503 case AArch64::LDRSBXui:
5504 case AArch64::LDRSHXui:
5505 case AArch64::LDRSWui:
5506 case AArch64::LDRSBWroX:
5507 case AArch64::LDRSHWroX:
5508 case AArch64::LDRSBXroX:
5509 case AArch64::LDRSHXroX:
5510 case AArch64::LDRSWroX:
5511 case AArch64::LDRSBWroW:
5512 case AArch64::LDRSHWroW:
5513 case AArch64::LDRSBXroW:
5514 case AArch64::LDRSHXroW:
5515 case AArch64::LDRSWroW:
5516 return true;
5517 }
5518}
5519
5520bool AArch64InstrInfo::isPairedLdSt(const MachineInstr &MI) {
5521 switch (MI.getOpcode()) {
5522 default:
5523 return false;
5524 case AArch64::LDPSi:
5525 case AArch64::LDPSWi:
5526 case AArch64::LDPDi:
5527 case AArch64::LDPQi:
5528 case AArch64::LDPWi:
5529 case AArch64::LDPXi:
5530 case AArch64::STPSi:
5531 case AArch64::STPDi:
5532 case AArch64::STPQi:
5533 case AArch64::STPWi:
5534 case AArch64::STPXi:
5535 case AArch64::STGPi:
5536 return true;
5537 }
5538}
5539
5540const MachineOperand &AArch64InstrInfo::getLdStBaseOp(const MachineInstr &MI) {
5541 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5542 unsigned Idx =
5543 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 2
5544 : 1;
5545 return MI.getOperand(i: Idx);
5546}
5547
5548const MachineOperand &
5549AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
5550 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5551 unsigned Idx =
5552 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 3
5553 : 2;
5554 return MI.getOperand(i: Idx);
5555}
5556
5557const MachineOperand &
5558AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
5559 switch (MI.getOpcode()) {
5560 default:
5561 llvm_unreachable("Unexpected opcode");
5562 case AArch64::LDRBroX:
5563 case AArch64::LDRBBroX:
5564 case AArch64::LDRSBXroX:
5565 case AArch64::LDRSBWroX:
5566 case AArch64::LDRHroX:
5567 case AArch64::LDRHHroX:
5568 case AArch64::LDRSHXroX:
5569 case AArch64::LDRSHWroX:
5570 case AArch64::LDRWroX:
5571 case AArch64::LDRSroX:
5572 case AArch64::LDRSWroX:
5573 case AArch64::LDRDroX:
5574 case AArch64::LDRXroX:
5575 case AArch64::LDRQroX:
5576 return MI.getOperand(i: 4);
5577 }
5578}
5579
5580static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
5581 Register Reg) {
5582 if (MI.getParent() == nullptr)
5583 return nullptr;
5584 const MachineFunction *MF = MI.getParent()->getParent();
5585 return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
5586}
5587
5588bool AArch64InstrInfo::isHForm(const MachineInstr &MI) {
5589 auto IsHFPR = [&](const MachineOperand &Op) {
5590 if (!Op.isReg())
5591 return false;
5592 auto Reg = Op.getReg();
5593 if (Reg.isPhysical())
5594 return AArch64::FPR16RegClass.contains(Reg);
5595 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5596 return TRC == &AArch64::FPR16RegClass ||
5597 TRC == &AArch64::FPR16_loRegClass;
5598 };
5599 return llvm::any_of(Range: MI.operands(), P: IsHFPR);
5600}
5601
5602bool AArch64InstrInfo::isQForm(const MachineInstr &MI) {
5603 auto IsQFPR = [&](const MachineOperand &Op) {
5604 if (!Op.isReg())
5605 return false;
5606 auto Reg = Op.getReg();
5607 if (Reg.isPhysical())
5608 return AArch64::FPR128RegClass.contains(Reg);
5609 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5610 return TRC == &AArch64::FPR128RegClass ||
5611 TRC == &AArch64::FPR128_loRegClass;
5612 };
5613 return llvm::any_of(Range: MI.operands(), P: IsQFPR);
5614}
5615
5616bool AArch64InstrInfo::hasBTISemantics(const MachineInstr &MI) {
5617 switch (MI.getOpcode()) {
5618 case AArch64::BRK:
5619 case AArch64::HLT:
5620 case AArch64::PACIASP:
5621 case AArch64::PACIBSP:
5622 case AArch64::PACIASPPC:
5623 case AArch64::PACIBSPPC:
5624 // Implicit BTI behavior.
5625 return true;
5626 case AArch64::PAUTH_PROLOGUE:
5627 // PAUTH_PROLOGUE expands to PACI(A|B)SP.
5628 return true;
5629 case AArch64::HINT: {
5630 unsigned Imm = MI.getOperand(i: 0).getImm();
5631 // Explicit BTI instruction.
5632 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
5633 return true;
5634 // PACI(A|B)SP instructions.
5635 if (Imm == 25 || Imm == 27)
5636 return true;
5637 return false;
5638 }
5639 default:
5640 return false;
5641 }
5642}
5643
5644bool AArch64InstrInfo::isFpOrNEON(Register Reg) {
5645 if (Reg == 0)
5646 return false;
5647 assert(Reg.isPhysical() && "Expected physical register in isFpOrNEON");
5648 return AArch64::FPR128RegClass.contains(Reg) ||
5649 AArch64::FPR64RegClass.contains(Reg) ||
5650 AArch64::FPR32RegClass.contains(Reg) ||
5651 AArch64::FPR16RegClass.contains(Reg) ||
5652 AArch64::FPR8RegClass.contains(Reg);
5653}
5654
5655bool AArch64InstrInfo::isFpOrNEON(const MachineInstr &MI) {
5656 auto IsFPR = [&](const MachineOperand &Op) {
5657 if (!Op.isReg())
5658 return false;
5659 auto Reg = Op.getReg();
5660 if (Reg.isPhysical())
5661 return isFpOrNEON(Reg);
5662
5663 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5664 return TRC == &AArch64::FPR128RegClass ||
5665 TRC == &AArch64::FPR128_loRegClass ||
5666 TRC == &AArch64::FPR64RegClass ||
5667 TRC == &AArch64::FPR64_loRegClass ||
5668 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
5669 TRC == &AArch64::FPR8RegClass;
5670 };
5671 return llvm::any_of(Range: MI.operands(), P: IsFPR);
5672}
5673
5674// Scale the unscaled offsets. Returns false if the unscaled offset can't be
5675// scaled.
5676static bool scaleOffset(unsigned Opc, int64_t &Offset) {
5677 int Scale = AArch64InstrInfo::getMemScale(Opc);
5678
5679 // If the byte-offset isn't a multiple of the stride, we can't scale this
5680 // offset.
5681 if (Offset % Scale != 0)
5682 return false;
5683
5684 // Convert the byte-offset used by unscaled into an "element" offset used
5685 // by the scaled pair load/store instructions.
5686 Offset /= Scale;
5687 return true;
5688}
5689
5690static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
5691 if (FirstOpc == SecondOpc)
5692 return true;
5693 // We can also pair sign-ext and zero-ext instructions.
5694 switch (FirstOpc) {
5695 default:
5696 return false;
5697 case AArch64::STRSui:
5698 case AArch64::STURSi:
5699 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
5700 case AArch64::STRDui:
5701 case AArch64::STURDi:
5702 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
5703 case AArch64::STRQui:
5704 case AArch64::STURQi:
5705 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
5706 case AArch64::STRWui:
5707 case AArch64::STURWi:
5708 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
5709 case AArch64::STRXui:
5710 case AArch64::STURXi:
5711 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
5712 case AArch64::LDRSui:
5713 case AArch64::LDURSi:
5714 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
5715 case AArch64::LDRDui:
5716 case AArch64::LDURDi:
5717 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
5718 case AArch64::LDRQui:
5719 case AArch64::LDURQi:
5720 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
5721 case AArch64::LDRWui:
5722 case AArch64::LDURWi:
5723 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
5724 case AArch64::LDRSWui:
5725 case AArch64::LDURSWi:
5726 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
5727 case AArch64::LDRXui:
5728 case AArch64::LDURXi:
5729 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
5730 }
5731 // These instructions can't be paired based on their opcodes.
5732 return false;
5733}
5734
5735static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
5736 int64_t Offset1, unsigned Opcode1, int FI2,
5737 int64_t Offset2, unsigned Opcode2) {
5738 // Accesses through fixed stack object frame indices may access a different
5739 // fixed stack slot. Check that the object offsets + offsets match.
5740 if (MFI.isFixedObjectIndex(ObjectIdx: FI1) && MFI.isFixedObjectIndex(ObjectIdx: FI2)) {
5741 int64_t ObjectOffset1 = MFI.getObjectOffset(ObjectIdx: FI1);
5742 int64_t ObjectOffset2 = MFI.getObjectOffset(ObjectIdx: FI2);
5743 assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
5744 // Convert to scaled object offsets.
5745 int Scale1 = AArch64InstrInfo::getMemScale(Opc: Opcode1);
5746 if (ObjectOffset1 % Scale1 != 0)
5747 return false;
5748 ObjectOffset1 /= Scale1;
5749 int Scale2 = AArch64InstrInfo::getMemScale(Opc: Opcode2);
5750 if (ObjectOffset2 % Scale2 != 0)
5751 return false;
5752 ObjectOffset2 /= Scale2;
5753 ObjectOffset1 += Offset1;
5754 ObjectOffset2 += Offset2;
5755 return ObjectOffset1 + 1 == ObjectOffset2;
5756 }
5757
5758 return FI1 == FI2;
5759}
5760
5761/// Detect opportunities for ldp/stp formation.
5762///
5763/// Only called for LdSt for which getMemOperandWithOffset returns true.
5764bool AArch64InstrInfo::shouldClusterMemOps(
5765 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
5766 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
5767 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
5768 unsigned NumBytes) const {
5769 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
5770 const MachineOperand &BaseOp1 = *BaseOps1.front();
5771 const MachineOperand &BaseOp2 = *BaseOps2.front();
5772 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
5773 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
5774 if (BaseOp1.getType() != BaseOp2.getType())
5775 return false;
5776
5777 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
5778 "Only base registers and frame indices are supported.");
5779
5780 // Check for both base regs and base FI.
5781 if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
5782 return false;
5783
5784 // Only cluster up to a single pair.
5785 if (ClusterSize > 2)
5786 return false;
5787
5788 if (!isPairableLdStInst(MI: FirstLdSt) || !isPairableLdStInst(MI: SecondLdSt))
5789 return false;
5790
5791 // Can we pair these instructions based on their opcodes?
5792 unsigned FirstOpc = FirstLdSt.getOpcode();
5793 unsigned SecondOpc = SecondLdSt.getOpcode();
5794 if (!canPairLdStOpc(FirstOpc, SecondOpc))
5795 return false;
5796
5797 // Can't merge volatiles or load/stores that have a hint to avoid pair
5798 // formation, for example.
5799 if (!isCandidateToMergeOrPair(MI: FirstLdSt) ||
5800 !isCandidateToMergeOrPair(MI: SecondLdSt))
5801 return false;
5802
5803 // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
5804 int64_t Offset1 = FirstLdSt.getOperand(i: 2).getImm();
5805 if (hasUnscaledLdStOffset(Opc: FirstOpc) && !scaleOffset(Opc: FirstOpc, Offset&: Offset1))
5806 return false;
5807
5808 int64_t Offset2 = SecondLdSt.getOperand(i: 2).getImm();
5809 if (hasUnscaledLdStOffset(Opc: SecondOpc) && !scaleOffset(Opc: SecondOpc, Offset&: Offset2))
5810 return false;
5811
5812 // Pairwise instructions have a 7-bit signed offset field.
5813 if (Offset1 > 63 || Offset1 < -64)
5814 return false;
5815
5816 // The caller should already have ordered First/SecondLdSt by offset.
5817 // Note: except for non-equal frame index bases
5818 if (BaseOp1.isFI()) {
5819 assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
5820 "Caller should have ordered offsets.");
5821
5822 const MachineFrameInfo &MFI =
5823 FirstLdSt.getParent()->getParent()->getFrameInfo();
5824 return shouldClusterFI(MFI, FI1: BaseOp1.getIndex(), Offset1, Opcode1: FirstOpc,
5825 FI2: BaseOp2.getIndex(), Offset2, Opcode2: SecondOpc);
5826 }
5827
5828 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
5829
5830 return Offset1 + 1 == Offset2;
5831}
5832
5833static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
5834 MCRegister Reg, unsigned SubIdx,
5835 RegState State,
5836 const TargetRegisterInfo *TRI) {
5837 if (!SubIdx)
5838 return MIB.addReg(RegNo: Reg, Flags: State);
5839
5840 if (Reg.isPhysical())
5841 return MIB.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx), Flags: State);
5842 return MIB.addReg(RegNo: Reg, Flags: State, SubReg: SubIdx);
5843}
5844
5845void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
5846 MachineBasicBlock::iterator I,
5847 const DebugLoc &DL, MCRegister DestReg,
5848 MCRegister SrcReg, bool KillSrc,
5849 ArrayRef<unsigned> Indices) const {
5850 assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
5851 const TargetRegisterInfo *TRI = &getRegisterInfo();
5852 uint16_t DestEncoding = TRI->getEncodingValue(Reg: DestReg);
5853 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
5854 unsigned NumRegs = Indices.size();
5855 MCRegister DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[0]);
5856 assert(!AArch64::PNRRegClass.contains(DestSubReg) &&
5857 "Unexpected predicate tuple copy");
5858 unsigned MaxRegs = AArch64::PPRRegClass.contains(Reg: DestSubReg) ? 15 : 31;
5859
5860 int SubReg = 0, End = NumRegs, Incr = 1;
5861 // Copy in reverse if a forward copy will clobber the tuple
5862 if (((DestEncoding - SrcEncoding) & MaxRegs) < NumRegs) {
5863 SubReg = NumRegs - 1;
5864 End = -1;
5865 Incr = -1;
5866 }
5867
5868 for (; SubReg != End; SubReg += Incr) {
5869 DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[SubReg]);
5870 MCRegister SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: Indices[SubReg]);
5871 copyPhysRegImpl(MBB, I, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc);
5872 }
5873}
5874
5875void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
5876 MachineBasicBlock::iterator I,
5877 const DebugLoc &DL, MCRegister DestReg,
5878 MCRegister SrcReg, bool KillSrc,
5879 unsigned Opcode, unsigned ZeroReg,
5880 llvm::ArrayRef<unsigned> Indices) const {
5881 const TargetRegisterInfo *TRI = &getRegisterInfo();
5882 unsigned NumRegs = Indices.size();
5883
5884#ifndef NDEBUG
5885 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
5886 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
5887 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5888 "GPR reg sequences should not be able to overlap");
5889#endif
5890
5891 for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
5892 const MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode));
5893 AddSubReg(MIB, Reg: DestReg, SubIdx: Indices[SubReg], State: RegState::Define, TRI);
5894 MIB.addReg(RegNo: ZeroReg);
5895 AddSubReg(MIB, Reg: SrcReg, SubIdx: Indices[SubReg], State: getKillRegState(B: KillSrc), TRI);
5896 MIB.addImm(Val: 0);
5897 }
5898}
5899
5900/// Returns true if the instruction at I is in a streaming call site region,
5901/// within a single basic block.
5902/// A "call site streaming region" starts after smstart and ends at smstop
5903/// around a call to a streaming function. This walks backward from I.
5904static bool isInStreamingCallSiteRegion(MachineBasicBlock &MBB,
5905 MachineBasicBlock::iterator I) {
5906 MachineFunction &MF = *MBB.getParent();
5907 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
5908 if (!AFI->hasStreamingModeChanges())
5909 return false;
5910 // Walk backwards to find smstart/smstop
5911 for (MachineInstr &MI : reverse(C: make_range(x: MBB.begin(), y: I))) {
5912 unsigned Opc = MI.getOpcode();
5913 if (Opc == AArch64::MSRpstatesvcrImm1 || Opc == AArch64::MSRpstatePseudo) {
5914 // Check if this is SM change (not ZA)
5915 int64_t PState = MI.getOperand(i: 0).getImm();
5916 if (PState == AArch64SVCR::SVCRSM || PState == AArch64SVCR::SVCRSMZA) {
5917 // Operand 1 is 1 for start, 0 for stop
5918 return MI.getOperand(i: 1).getImm() == 1;
5919 }
5920 }
5921 }
5922 return false;
5923}
5924
5925/// Returns true if in a streaming call site region without SME-FA64.
5926static bool mustAvoidNeonAtMBBI(const AArch64Subtarget &Subtarget,
5927 MachineBasicBlock &MBB,
5928 MachineBasicBlock::iterator I) {
5929 return !Subtarget.hasSMEFA64() && isInStreamingCallSiteRegion(MBB, I);
5930}
5931
5932void AArch64InstrInfo::copyPhysRegImpl(MachineBasicBlock &MBB,
5933 MachineBasicBlock::iterator I,
5934 const DebugLoc &DL, Register DestReg,
5935 Register SrcReg, bool KillSrc,
5936 bool RenamableDest,
5937 bool RenamableSrc) const {
5938 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) &&
5939 AArch64::GPR32spRegClass.contains(Reg: SrcReg)) {
5940 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5941 // If either operand is WSP, expand to ADD #0.
5942 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5943 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5944 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
5945 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5946 RC: &AArch64::GPR64spRegClass);
5947 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5948 RC: &AArch64::GPR64spRegClass);
5949 // This instruction is reading and writing X registers. This may upset
5950 // the register scavenger and machine verifier, so we need to indicate
5951 // that we are reading an undefined value from SrcRegX, but a proper
5952 // value from SrcReg.
5953 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: DestRegX)
5954 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5955 .addImm(Val: 0)
5956 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
5957 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5958 ++NumZCRegMoveInstrsGPR;
5959 } else {
5960 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDWri), DestReg)
5961 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5962 .addImm(Val: 0)
5963 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5964 if (Subtarget.hasZeroCycleRegMoveGPR32())
5965 ++NumZCRegMoveInstrsGPR;
5966 }
5967 } else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5968 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5969 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
5970 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5971 RC: &AArch64::GPR64spRegClass);
5972 assert(DestRegX.isValid() && "Destination super-reg not valid");
5973 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5974 RC: &AArch64::GPR64spRegClass);
5975 assert(SrcRegX.isValid() && "Source super-reg not valid");
5976 // This instruction is reading and writing X registers. This may upset
5977 // the register scavenger and machine verifier, so we need to indicate
5978 // that we are reading an undefined value from SrcRegX, but a proper
5979 // value from SrcReg.
5980 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg: DestRegX)
5981 .addReg(RegNo: AArch64::XZR)
5982 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5983 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5984 ++NumZCRegMoveInstrsGPR;
5985 } else {
5986 // Otherwise, expand to ORR WZR.
5987 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5988 .addReg(RegNo: AArch64::WZR)
5989 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5990 if (Subtarget.hasZeroCycleRegMoveGPR32())
5991 ++NumZCRegMoveInstrsGPR;
5992 }
5993 return;
5994 }
5995
5996 // GPR32 zeroing
5997 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::WZR) {
5998 if (Subtarget.hasZeroCycleZeroingGPR64() &&
5999 !Subtarget.hasZeroCycleZeroingGPR32()) {
6000 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
6001 RC: &AArch64::GPR64spRegClass);
6002 assert(DestRegX.isValid() && "Destination super-reg not valid");
6003 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: DestRegX)
6004 .addImm(Val: 0)
6005 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
6006 ++NumZCZeroingInstrsGPR;
6007 } else if (Subtarget.hasZeroCycleZeroingGPR32()) {
6008 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZWi), DestReg)
6009 .addImm(Val: 0)
6010 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
6011 ++NumZCZeroingInstrsGPR;
6012 } else {
6013 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
6014 .addReg(RegNo: AArch64::WZR)
6015 .addReg(RegNo: AArch64::WZR);
6016 }
6017 return;
6018 }
6019
6020 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) &&
6021 AArch64::GPR64spRegClass.contains(Reg: SrcReg)) {
6022 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
6023 // If either operand is SP, expand to ADD #0.
6024 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg)
6025 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6026 .addImm(Val: 0)
6027 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
6028 if (Subtarget.hasZeroCycleRegMoveGPR64())
6029 ++NumZCRegMoveInstrsGPR;
6030 } else {
6031 // Otherwise, expand to ORR XZR.
6032 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
6033 .addReg(RegNo: AArch64::XZR)
6034 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6035 if (Subtarget.hasZeroCycleRegMoveGPR64())
6036 ++NumZCRegMoveInstrsGPR;
6037 }
6038 return;
6039 }
6040
6041 // GPR64 zeroing
6042 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::XZR) {
6043 if (Subtarget.hasZeroCycleZeroingGPR64()) {
6044 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg)
6045 .addImm(Val: 0)
6046 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
6047 ++NumZCZeroingInstrsGPR;
6048 } else {
6049 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
6050 .addReg(RegNo: AArch64::XZR)
6051 .addReg(RegNo: AArch64::XZR);
6052 }
6053 return;
6054 }
6055
6056 // Copy a Predicate register by ORRing with itself.
6057 if (AArch64::PPRRegClass.contains(Reg: DestReg) &&
6058 AArch64::PPRRegClass.contains(Reg: SrcReg)) {
6059 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6060 "Unexpected SVE register.");
6061 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg)
6062 .addReg(RegNo: SrcReg) // Pg
6063 .addReg(RegNo: SrcReg)
6064 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6065 return;
6066 }
6067
6068 // Copy a predicate-as-counter register by ORRing with itself as if it
6069 // were a regular predicate (mask) register.
6070 bool DestIsPNR = AArch64::PNRRegClass.contains(Reg: DestReg);
6071 bool SrcIsPNR = AArch64::PNRRegClass.contains(Reg: SrcReg);
6072 if (DestIsPNR || SrcIsPNR) {
6073 auto ToPPR = [](MCRegister R) -> MCRegister {
6074 return (R - AArch64::PN0) + AArch64::P0;
6075 };
6076 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg.asMCReg();
6077 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg.asMCReg();
6078
6079 if (PPRSrcReg != PPRDestReg) {
6080 auto NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg: PPRDestReg)
6081 .addReg(RegNo: PPRSrcReg) // Pg
6082 .addReg(RegNo: PPRSrcReg)
6083 .addReg(RegNo: PPRSrcReg, Flags: getKillRegState(B: KillSrc));
6084 if (DestIsPNR)
6085 NewMI.addDef(RegNo: DestReg, Flags: RegState::Implicit);
6086 }
6087 return;
6088 }
6089
6090 // Copy a predicate register pair by copying the individual sub-registers.
6091 if (AArch64::PPR2RegClass.contains(Reg: DestReg) &&
6092 AArch64::PPR2RegClass.contains(Reg: SrcReg)) {
6093 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6094 "Unexpected SVE predicate register.");
6095 static const unsigned Indices[] = {AArch64::psub0, AArch64::psub1};
6096 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6097 return;
6098 }
6099
6100 // Copy a Z register by ORRing with itself.
6101 if (AArch64::ZPRRegClass.contains(Reg: DestReg) &&
6102 AArch64::ZPRRegClass.contains(Reg: SrcReg)) {
6103 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6104 "Unexpected SVE register.");
6105 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ), DestReg)
6106 .addReg(RegNo: SrcReg)
6107 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6108 return;
6109 }
6110
6111 // Copy a Z register pair by copying the individual sub-registers.
6112 if ((AArch64::ZPR2RegClass.contains(Reg: DestReg) ||
6113 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6114 (AArch64::ZPR2RegClass.contains(Reg: SrcReg) ||
6115 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6116 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6117 "Unexpected SVE register.");
6118 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
6119 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6120 return;
6121 }
6122
6123 // Copy a Z register triple by copying the individual sub-registers.
6124 if (AArch64::ZPR3RegClass.contains(Reg: DestReg) &&
6125 AArch64::ZPR3RegClass.contains(Reg: SrcReg)) {
6126 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6127 "Unexpected SVE register.");
6128 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6129 AArch64::zsub2};
6130 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6131 return;
6132 }
6133
6134 // Copy a Z register quad by copying the individual sub-registers.
6135 if ((AArch64::ZPR4RegClass.contains(Reg: DestReg) ||
6136 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6137 (AArch64::ZPR4RegClass.contains(Reg: SrcReg) ||
6138 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6139 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6140 "Unexpected SVE register.");
6141 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6142 AArch64::zsub2, AArch64::zsub3};
6143 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6144 return;
6145 }
6146
6147 // Copy a DDDD register quad by copying the individual sub-registers.
6148 if (AArch64::DDDDRegClass.contains(Reg: DestReg) &&
6149 AArch64::DDDDRegClass.contains(Reg: SrcReg)) {
6150 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6151 AArch64::dsub2, AArch64::dsub3};
6152 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6153 return;
6154 }
6155
6156 // Copy a DDD register triple by copying the individual sub-registers.
6157 if (AArch64::DDDRegClass.contains(Reg: DestReg) &&
6158 AArch64::DDDRegClass.contains(Reg: SrcReg)) {
6159 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6160 AArch64::dsub2};
6161 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6162 return;
6163 }
6164
6165 // Copy a DD register pair by copying the individual sub-registers.
6166 if (AArch64::DDRegClass.contains(Reg: DestReg) &&
6167 AArch64::DDRegClass.contains(Reg: SrcReg)) {
6168 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
6169 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6170 return;
6171 }
6172
6173 // Copy a QQQQ register quad by copying the individual sub-registers.
6174 if (AArch64::QQQQRegClass.contains(Reg: DestReg) &&
6175 AArch64::QQQQRegClass.contains(Reg: SrcReg)) {
6176 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6177 AArch64::qsub2, AArch64::qsub3};
6178 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6179 return;
6180 }
6181
6182 // Copy a QQQ register triple by copying the individual sub-registers.
6183 if (AArch64::QQQRegClass.contains(Reg: DestReg) &&
6184 AArch64::QQQRegClass.contains(Reg: SrcReg)) {
6185 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6186 AArch64::qsub2};
6187 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6188 return;
6189 }
6190
6191 // Copy a QQ register pair by copying the individual sub-registers.
6192 if (AArch64::QQRegClass.contains(Reg: DestReg) &&
6193 AArch64::QQRegClass.contains(Reg: SrcReg)) {
6194 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
6195 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6196 return;
6197 }
6198
6199 if (AArch64::XSeqPairsClassRegClass.contains(Reg: DestReg) &&
6200 AArch64::XSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6201 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
6202 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRXrs,
6203 ZeroReg: AArch64::XZR, Indices);
6204 return;
6205 }
6206
6207 if (AArch64::WSeqPairsClassRegClass.contains(Reg: DestReg) &&
6208 AArch64::WSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6209 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
6210 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRWrs,
6211 ZeroReg: AArch64::WZR, Indices);
6212 return;
6213 }
6214
6215 if (AArch64::FPR128RegClass.contains(Reg: DestReg) &&
6216 AArch64::FPR128RegClass.contains(Reg: SrcReg)) {
6217 // In streaming regions, NEON is illegal but streaming-SVE is available.
6218 // Use SVE for copies if we're in a streaming region and SME is available.
6219 // With +sme-fa64, NEON is legal in streaming mode so we can use it.
6220 if ((Subtarget.isSVEorStreamingSVEAvailable() &&
6221 !Subtarget.isNeonAvailable()) ||
6222 mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6223 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ))
6224 .addReg(RegNo: AArch64::Z0 + (DestReg - AArch64::Q0), Flags: RegState::Define)
6225 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0))
6226 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0));
6227 } else if (Subtarget.isNeonAvailable()) {
6228 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg)
6229 .addReg(RegNo: SrcReg)
6230 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6231 if (Subtarget.hasZeroCycleRegMoveFPR128())
6232 ++NumZCRegMoveInstrsFPR;
6233 } else {
6234 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::STRQpre))
6235 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6236 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6237 .addReg(RegNo: AArch64::SP)
6238 .addImm(Val: -16);
6239 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::LDRQpost))
6240 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6241 .addReg(RegNo: DestReg, Flags: RegState::Define)
6242 .addReg(RegNo: AArch64::SP)
6243 .addImm(Val: 16);
6244 }
6245 return;
6246 }
6247
6248 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6249 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6250 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6251 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6252 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6253 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6254 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
6255 RC: &AArch64::FPR128RegClass);
6256 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::dsub,
6257 RC: &AArch64::FPR128RegClass);
6258 // This instruction is reading and writing Q registers. This may upset
6259 // the register scavenger and machine verifier, so we need to indicate
6260 // that we are reading an undefined value from SrcRegQ, but a proper
6261 // value from SrcReg.
6262 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6263 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6264 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6265 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6266 ++NumZCRegMoveInstrsFPR;
6267 } else {
6268 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg)
6269 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6270 if (Subtarget.hasZeroCycleRegMoveFPR64())
6271 ++NumZCRegMoveInstrsFPR;
6272 }
6273 return;
6274 }
6275
6276 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6277 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6278 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6279 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6280 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6281 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6282 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6283 RC: &AArch64::FPR128RegClass);
6284 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6285 RC: &AArch64::FPR128RegClass);
6286 // This instruction is reading and writing Q registers. This may upset
6287 // the register scavenger and machine verifier, so we need to indicate
6288 // that we are reading an undefined value from SrcRegQ, but a proper
6289 // value from SrcReg.
6290 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6291 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6292 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6293 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6294 ++NumZCRegMoveInstrsFPR;
6295 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6296 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6297 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6298 RC: &AArch64::FPR64RegClass);
6299 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6300 RC: &AArch64::FPR64RegClass);
6301 // This instruction is reading and writing D registers. This may upset
6302 // the register scavenger and machine verifier, so we need to indicate
6303 // that we are reading an undefined value from SrcRegD, but a proper
6304 // value from SrcReg.
6305 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6306 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6307 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6308 ++NumZCRegMoveInstrsFPR;
6309 } else {
6310 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6311 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6312 if (Subtarget.hasZeroCycleRegMoveFPR32())
6313 ++NumZCRegMoveInstrsFPR;
6314 }
6315 return;
6316 }
6317
6318 if (AArch64::FPR16RegClass.contains(Reg: DestReg) &&
6319 AArch64::FPR16RegClass.contains(Reg: SrcReg)) {
6320 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6321 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6322 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6323 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6324 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6325 RC: &AArch64::FPR128RegClass);
6326 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6327 RC: &AArch64::FPR128RegClass);
6328 // This instruction is reading and writing Q registers. This may upset
6329 // the register scavenger and machine verifier, so we need to indicate
6330 // that we are reading an undefined value from SrcRegQ, but a proper
6331 // value from SrcReg.
6332 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6333 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6334 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6335 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6336 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6337 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6338 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6339 RC: &AArch64::FPR64RegClass);
6340 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6341 RC: &AArch64::FPR64RegClass);
6342 // This instruction is reading and writing D registers. This may upset
6343 // the register scavenger and machine verifier, so we need to indicate
6344 // that we are reading an undefined value from SrcRegD, but a proper
6345 // value from SrcReg.
6346 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6347 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6348 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6349 } else {
6350 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6351 RC: &AArch64::FPR32RegClass);
6352 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6353 RC: &AArch64::FPR32RegClass);
6354 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6355 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6356 }
6357 return;
6358 }
6359
6360 if (AArch64::FPR8RegClass.contains(Reg: DestReg) &&
6361 AArch64::FPR8RegClass.contains(Reg: SrcReg)) {
6362 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6363 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6364 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6365 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6366 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6367 RC: &AArch64::FPR128RegClass);
6368 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6369 RC: &AArch64::FPR128RegClass);
6370 // This instruction is reading and writing Q registers. This may upset
6371 // the register scavenger and machine verifier, so we need to indicate
6372 // that we are reading an undefined value from SrcRegQ, but a proper
6373 // value from SrcReg.
6374 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6375 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6376 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6377 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6378 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6379 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6380 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6381 RC: &AArch64::FPR64RegClass);
6382 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6383 RC: &AArch64::FPR64RegClass);
6384 // This instruction is reading and writing D registers. This may upset
6385 // the register scavenger and machine verifier, so we need to indicate
6386 // that we are reading an undefined value from SrcRegD, but a proper
6387 // value from SrcReg.
6388 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6389 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6390 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6391 } else {
6392 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6393 RC: &AArch64::FPR32RegClass);
6394 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6395 RC: &AArch64::FPR32RegClass);
6396 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6397 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6398 }
6399 return;
6400 }
6401
6402 // Copies between GPR64 and FPR64.
6403 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6404 AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
6405 if (AArch64::XZR == SrcReg) {
6406 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg);
6407 } else {
6408 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVXDr), DestReg)
6409 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6410 }
6411 return;
6412 }
6413 if (AArch64::GPR64RegClass.contains(Reg: DestReg) &&
6414 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6415 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDXr), DestReg)
6416 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6417 return;
6418 }
6419 // Copies between GPR32 and FPR32.
6420 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6421 AArch64::GPR32RegClass.contains(Reg: SrcReg)) {
6422 if (AArch64::WZR == SrcReg) {
6423 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVS0), DestReg);
6424 } else {
6425 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVWSr), DestReg)
6426 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6427 }
6428 return;
6429 }
6430 if (AArch64::GPR32RegClass.contains(Reg: DestReg) &&
6431 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6432 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSWr), DestReg)
6433 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6434 return;
6435 }
6436
6437 if (DestReg == AArch64::NZCV) {
6438 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
6439 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MSR))
6440 .addImm(Val: AArch64SysReg::NZCV)
6441 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6442 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Define);
6443 return;
6444 }
6445
6446 if (SrcReg == AArch64::NZCV) {
6447 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
6448 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MRS), DestReg)
6449 .addImm(Val: AArch64SysReg::NZCV)
6450 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6451 return;
6452 }
6453
6454#ifndef NDEBUG
6455 errs() << RI.getRegAsmName(DestReg) << " = COPY " << RI.getRegAsmName(SrcReg)
6456 << "\n";
6457#endif
6458 llvm_unreachable("unimplemented reg-to-reg copy");
6459}
6460
6461void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
6462 MachineBasicBlock::iterator I,
6463 const DebugLoc &DL, Register DestReg,
6464 Register SrcReg, bool KillSrc,
6465 bool RenamableDest,
6466 bool RenamableSrc) const {
6467 ++NumCopyInstrs;
6468 copyPhysRegImpl(MBB, I, DL, DestReg, SrcReg, KillSrc, RenamableDest,
6469 RenamableSrc);
6470 return;
6471}
6472
6473static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
6474 MachineBasicBlock &MBB,
6475 MachineBasicBlock::iterator InsertBefore,
6476 const MCInstrDesc &MCID,
6477 Register SrcReg, bool IsKill,
6478 unsigned SubIdx0, unsigned SubIdx1, int FI,
6479 MachineMemOperand *MMO) {
6480 Register SrcReg0 = SrcReg;
6481 Register SrcReg1 = SrcReg;
6482 if (SrcReg.isPhysical()) {
6483 SrcReg0 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx0);
6484 SubIdx0 = 0;
6485 SrcReg1 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx1);
6486 SubIdx1 = 0;
6487 }
6488 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6489 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: IsKill), SubReg: SubIdx0)
6490 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: IsKill), SubReg: SubIdx1)
6491 .addFrameIndex(Idx: FI)
6492 .addImm(Val: 0)
6493 .addMemOperand(MMO);
6494}
6495
6496void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
6497 MachineBasicBlock::iterator MBBI,
6498 Register SrcReg, bool isKill, int FI,
6499 const TargetRegisterClass *RC,
6500 Register VReg,
6501 MachineInstr::MIFlag Flags) const {
6502 MachineFunction &MF = *MBB.getParent();
6503 MachineFrameInfo &MFI = MF.getFrameInfo();
6504
6505 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6506 MachineMemOperand *MMO =
6507 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
6508 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6509 unsigned Opc = 0;
6510 bool Offset = true;
6511 MCRegister PNRReg = MCRegister::NoRegister;
6512 unsigned StackID = TargetStackID::Default;
6513 switch (RI.getSpillSize(RC: *RC)) {
6514 case 1:
6515 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6516 Opc = AArch64::STRBui;
6517 break;
6518 case 2: {
6519 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6520 Opc = AArch64::STRHui;
6521 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
6522 AArch64::PPRRegClass.hasSubClassEq(RC)) {
6523 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6524 "Unexpected register store without SVE store instructions");
6525 Opc = AArch64::STR_PXI;
6526 StackID = TargetStackID::ScalablePredicateVector;
6527 }
6528 break;
6529 }
6530 case 4:
6531 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6532 Opc = AArch64::STRWui;
6533 if (SrcReg.isVirtual())
6534 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32RegClass);
6535 else
6536 assert(SrcReg != AArch64::WSP);
6537 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6538 Opc = AArch64::STRSui;
6539 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6540 Opc = AArch64::STR_PPXI;
6541 StackID = TargetStackID::ScalablePredicateVector;
6542 }
6543 break;
6544 case 8:
6545 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6546 Opc = AArch64::STRXui;
6547 if (SrcReg.isVirtual())
6548 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
6549 else
6550 assert(SrcReg != AArch64::SP);
6551 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6552 Opc = AArch64::STRDui;
6553 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6554 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6555 MCID: get(Opcode: AArch64::STPWi), SrcReg, IsKill: isKill,
6556 SubIdx0: AArch64::sube32, SubIdx1: AArch64::subo32, FI, MMO);
6557 return;
6558 }
6559 break;
6560 case 16:
6561 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6562 Opc = AArch64::STRQui;
6563 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6564 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6565 Opc = AArch64::ST1Twov1d;
6566 Offset = false;
6567 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6568 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6569 MCID: get(Opcode: AArch64::STPXi), SrcReg, IsKill: isKill,
6570 SubIdx0: AArch64::sube64, SubIdx1: AArch64::subo64, FI, MMO);
6571 return;
6572 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6573 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6574 "Unexpected register store without SVE store instructions");
6575 Opc = AArch64::STR_ZXI;
6576 StackID = TargetStackID::ScalableVector;
6577 }
6578 break;
6579 case 24:
6580 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6581 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6582 Opc = AArch64::ST1Threev1d;
6583 Offset = false;
6584 }
6585 break;
6586 case 32:
6587 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6588 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6589 Opc = AArch64::ST1Fourv1d;
6590 Offset = false;
6591 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6592 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6593 Opc = AArch64::ST1Twov2d;
6594 Offset = false;
6595 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6596 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6597 "Unexpected register store without SVE store instructions");
6598 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
6599 StackID = TargetStackID::ScalableVector;
6600 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6601 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6602 "Unexpected register store without SVE store instructions");
6603 Opc = AArch64::STR_ZZXI;
6604 StackID = TargetStackID::ScalableVector;
6605 }
6606 break;
6607 case 48:
6608 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6609 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6610 Opc = AArch64::ST1Threev2d;
6611 Offset = false;
6612 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6613 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6614 "Unexpected register store without SVE store instructions");
6615 Opc = AArch64::STR_ZZZXI;
6616 StackID = TargetStackID::ScalableVector;
6617 }
6618 break;
6619 case 64:
6620 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6621 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6622 Opc = AArch64::ST1Fourv2d;
6623 Offset = false;
6624 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6625 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6626 "Unexpected register store without SVE store instructions");
6627 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
6628 StackID = TargetStackID::ScalableVector;
6629 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6630 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6631 "Unexpected register store without SVE store instructions");
6632 Opc = AArch64::STR_ZZZZXI;
6633 StackID = TargetStackID::ScalableVector;
6634 }
6635 break;
6636 }
6637 assert(Opc && "Unknown register class");
6638 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6639
6640 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6641 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
6642 .addFrameIndex(Idx: FI);
6643
6644 if (Offset)
6645 MI.addImm(Val: 0);
6646 if (PNRReg.isValid())
6647 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6648 MI.addMemOperand(MMO);
6649}
6650
6651static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
6652 MachineBasicBlock &MBB,
6653 MachineBasicBlock::iterator InsertBefore,
6654 const MCInstrDesc &MCID,
6655 Register DestReg, unsigned SubIdx0,
6656 unsigned SubIdx1, int FI,
6657 MachineMemOperand *MMO) {
6658 Register DestReg0 = DestReg;
6659 Register DestReg1 = DestReg;
6660 bool IsUndef = true;
6661 if (DestReg.isPhysical()) {
6662 DestReg0 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx0);
6663 SubIdx0 = 0;
6664 DestReg1 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx1);
6665 SubIdx1 = 0;
6666 IsUndef = false;
6667 }
6668 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6669 .addReg(RegNo: DestReg0, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx0)
6670 .addReg(RegNo: DestReg1, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx1)
6671 .addFrameIndex(Idx: FI)
6672 .addImm(Val: 0)
6673 .addMemOperand(MMO);
6674}
6675
6676void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
6677 MachineBasicBlock::iterator MBBI,
6678 Register DestReg, int FI,
6679 const TargetRegisterClass *RC,
6680 Register VReg, unsigned SubReg,
6681 MachineInstr::MIFlag Flags) const {
6682 MachineFunction &MF = *MBB.getParent();
6683 MachineFrameInfo &MFI = MF.getFrameInfo();
6684 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6685 MachineMemOperand *MMO =
6686 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
6687 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6688
6689 unsigned Opc = 0;
6690 bool Offset = true;
6691 unsigned StackID = TargetStackID::Default;
6692 Register PNRReg;
6693 switch (TRI.getSpillSize(RC: *RC)) {
6694 case 1:
6695 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6696 Opc = AArch64::LDRBui;
6697 break;
6698 case 2: {
6699 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
6700 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6701 Opc = AArch64::LDRHui;
6702 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
6703 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6704 "Unexpected register load without SVE load instructions");
6705 if (IsPNR)
6706 PNRReg = DestReg;
6707 Opc = AArch64::LDR_PXI;
6708 StackID = TargetStackID::ScalablePredicateVector;
6709 }
6710 break;
6711 }
6712 case 4:
6713 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6714 Opc = AArch64::LDRWui;
6715 if (DestReg.isVirtual())
6716 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR32RegClass);
6717 else
6718 assert(DestReg != AArch64::WSP);
6719 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6720 Opc = AArch64::LDRSui;
6721 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6722 Opc = AArch64::LDR_PPXI;
6723 StackID = TargetStackID::ScalablePredicateVector;
6724 }
6725 break;
6726 case 8:
6727 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6728 Opc = AArch64::LDRXui;
6729 if (DestReg.isVirtual())
6730 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR64RegClass);
6731 else
6732 assert(DestReg != AArch64::SP);
6733 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6734 Opc = AArch64::LDRDui;
6735 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6736 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6737 MCID: get(Opcode: AArch64::LDPWi), DestReg, SubIdx0: AArch64::sube32,
6738 SubIdx1: AArch64::subo32, FI, MMO);
6739 return;
6740 }
6741 break;
6742 case 16:
6743 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6744 Opc = AArch64::LDRQui;
6745 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6746 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6747 Opc = AArch64::LD1Twov1d;
6748 Offset = false;
6749 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6750 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6751 MCID: get(Opcode: AArch64::LDPXi), DestReg, SubIdx0: AArch64::sube64,
6752 SubIdx1: AArch64::subo64, FI, MMO);
6753 return;
6754 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6755 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6756 "Unexpected register load without SVE load instructions");
6757 Opc = AArch64::LDR_ZXI;
6758 StackID = TargetStackID::ScalableVector;
6759 }
6760 break;
6761 case 24:
6762 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6763 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6764 Opc = AArch64::LD1Threev1d;
6765 Offset = false;
6766 }
6767 break;
6768 case 32:
6769 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6770 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6771 Opc = AArch64::LD1Fourv1d;
6772 Offset = false;
6773 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6774 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6775 Opc = AArch64::LD1Twov2d;
6776 Offset = false;
6777 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6778 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6779 "Unexpected register load without SVE load instructions");
6780 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
6781 StackID = TargetStackID::ScalableVector;
6782 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6783 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6784 "Unexpected register load without SVE load instructions");
6785 Opc = AArch64::LDR_ZZXI;
6786 StackID = TargetStackID::ScalableVector;
6787 }
6788 break;
6789 case 48:
6790 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6791 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6792 Opc = AArch64::LD1Threev2d;
6793 Offset = false;
6794 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6795 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6796 "Unexpected register load without SVE load instructions");
6797 Opc = AArch64::LDR_ZZZXI;
6798 StackID = TargetStackID::ScalableVector;
6799 }
6800 break;
6801 case 64:
6802 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6803 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6804 Opc = AArch64::LD1Fourv2d;
6805 Offset = false;
6806 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6807 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6808 "Unexpected register load without SVE load instructions");
6809 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
6810 StackID = TargetStackID::ScalableVector;
6811 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6812 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6813 "Unexpected register load without SVE load instructions");
6814 Opc = AArch64::LDR_ZZZZXI;
6815 StackID = TargetStackID::ScalableVector;
6816 }
6817 break;
6818 }
6819
6820 assert(Opc && "Unknown register class");
6821 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6822
6823 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6824 .addReg(RegNo: DestReg, Flags: getDefRegState(B: true))
6825 .addFrameIndex(Idx: FI);
6826 if (Offset)
6827 MI.addImm(Val: 0);
6828 if (PNRReg.isValid() && !PNRReg.isVirtual())
6829 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6830 MI.addMemOperand(MMO);
6831}
6832
6833bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
6834 const MachineInstr &UseMI,
6835 const TargetRegisterInfo *TRI) {
6836 return any_of(Range: instructionsWithoutDebug(It: std::next(x: DefMI.getIterator()),
6837 End: UseMI.getIterator()),
6838 P: [TRI](const MachineInstr &I) {
6839 return I.modifiesRegister(Reg: AArch64::NZCV, TRI) ||
6840 I.readsRegister(Reg: AArch64::NZCV, TRI);
6841 });
6842}
6843
6844void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6845 const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized) {
6846 // The smallest scalable element supported by scaled SVE addressing
6847 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6848 // byte offset must always be a multiple of 2.
6849 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6850
6851 // VGSized offsets are divided by '2', because the VG register is the
6852 // the number of 64bit granules as opposed to 128bit vector chunks,
6853 // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
6854 // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
6855 // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
6856 ByteSized = Offset.getFixed();
6857 VGSized = Offset.getScalable() / 2;
6858}
6859
6860/// Returns the offset in parts to which this frame offset can be
6861/// decomposed for the purpose of describing a frame offset.
6862/// For non-scalable offsets this is simply its byte size.
6863void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6864 const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors,
6865 int64_t &NumDataVectors) {
6866 // The smallest scalable element supported by scaled SVE addressing
6867 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6868 // byte offset must always be a multiple of 2.
6869 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6870
6871 NumBytes = Offset.getFixed();
6872 NumDataVectors = 0;
6873 NumPredicateVectors = Offset.getScalable() / 2;
6874 // This method is used to get the offsets to adjust the frame offset.
6875 // If the function requires ADDPL to be used and needs more than two ADDPL
6876 // instructions, part of the offset is folded into NumDataVectors so that it
6877 // uses ADDVL for part of it, reducing the number of ADDPL instructions.
6878 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
6879 NumPredicateVectors > 62) {
6880 NumDataVectors = NumPredicateVectors / 8;
6881 NumPredicateVectors -= NumDataVectors * 8;
6882 }
6883}
6884
6885// Convenience function to create a DWARF expression for: Constant `Operation`.
6886// This helper emits compact sequences for common cases. For example, for`-15
6887// DW_OP_plus`, this helper would create DW_OP_lit15 DW_OP_minus.
6888static void appendConstantExpr(SmallVectorImpl<char> &Expr, int64_t Constant,
6889 dwarf::LocationAtom Operation) {
6890 if (Operation == dwarf::DW_OP_plus && Constant < 0 && -Constant <= 31) {
6891 // -Constant (1 to 31)
6892 Expr.push_back(Elt: dwarf::DW_OP_lit0 - Constant);
6893 Operation = dwarf::DW_OP_minus;
6894 } else if (Constant >= 0 && Constant <= 31) {
6895 // Literal value 0 to 31
6896 Expr.push_back(Elt: dwarf::DW_OP_lit0 + Constant);
6897 } else {
6898 // Signed constant
6899 Expr.push_back(Elt: dwarf::DW_OP_consts);
6900 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: Constant);
6901 }
6902 return Expr.push_back(Elt: Operation);
6903}
6904
6905// Convenience function to create a DWARF expression for a register.
6906static void appendReadRegExpr(SmallVectorImpl<char> &Expr, unsigned RegNum) {
6907 Expr.push_back(Elt: (char)dwarf::DW_OP_bregx);
6908 appendLEB128<LEB128Sign::Unsigned>(Buffer&: Expr, Value: RegNum);
6909 Expr.push_back(Elt: 0);
6910}
6911
6912// Convenience function to create a DWARF expression for loading a register from
6913// a CFA offset.
6914static void appendLoadRegExpr(SmallVectorImpl<char> &Expr,
6915 int64_t OffsetFromDefCFA) {
6916 // This assumes the top of the DWARF stack contains the CFA.
6917 Expr.push_back(Elt: dwarf::DW_OP_dup);
6918 // Add the offset to the register.
6919 appendConstantExpr(Expr, Constant: OffsetFromDefCFA, Operation: dwarf::DW_OP_plus);
6920 // Dereference the address (loads a 64 bit value)..
6921 Expr.push_back(Elt: dwarf::DW_OP_deref);
6922}
6923
6924// Convenience function to create a comment for
6925// (+/-) NumBytes (* RegScale)?
6926static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
6927 StringRef RegScale = {}) {
6928 if (NumBytes) {
6929 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(x: NumBytes);
6930 if (!RegScale.empty())
6931 Comment << ' ' << RegScale;
6932 }
6933}
6934
6935// Creates an MCCFIInstruction:
6936// { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
6937static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
6938 unsigned Reg,
6939 const StackOffset &Offset) {
6940 int64_t NumBytes, NumVGScaledBytes;
6941 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(Offset, ByteSized&: NumBytes,
6942 VGSized&: NumVGScaledBytes);
6943 std::string CommentBuffer;
6944 llvm::raw_string_ostream Comment(CommentBuffer);
6945
6946 if (Reg == AArch64::SP)
6947 Comment << "sp";
6948 else if (Reg == AArch64::FP)
6949 Comment << "fp";
6950 else
6951 Comment << printReg(Reg, TRI: &TRI);
6952
6953 // Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
6954 SmallString<64> Expr;
6955 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6956 assert(DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
6957 // Reg + NumBytes
6958 Expr.push_back(Elt: dwarf::DW_OP_breg0 + DwarfReg);
6959 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: NumBytes);
6960 appendOffsetComment(NumBytes, Comment);
6961 if (NumVGScaledBytes) {
6962 // + VG * NumVGScaledBytes
6963 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: "* VG");
6964 appendReadRegExpr(Expr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6965 appendConstantExpr(Expr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6966 Expr.push_back(Elt: dwarf::DW_OP_plus);
6967 }
6968
6969 // Wrap this into DW_CFA_def_cfa.
6970 SmallString<64> DefCfaExpr;
6971 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
6972 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
6973 DefCfaExpr.append(RHS: Expr.str());
6974 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
6975 Comment: Comment.str());
6976}
6977
6978MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
6979 unsigned FrameReg, unsigned Reg,
6980 const StackOffset &Offset,
6981 bool LastAdjustmentWasScalable) {
6982 if (Offset.getScalable())
6983 return createDefCFAExpression(TRI, Reg, Offset);
6984
6985 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6986 return MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: int(Offset.getFixed()));
6987
6988 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6989 return MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfReg, Offset: (int)Offset.getFixed());
6990}
6991
6992MCCFIInstruction
6993llvm::createCFAOffset(const TargetRegisterInfo &TRI, unsigned Reg,
6994 const StackOffset &OffsetFromDefCFA,
6995 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6996 int64_t NumBytes, NumVGScaledBytes;
6997 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6998 Offset: OffsetFromDefCFA, ByteSized&: NumBytes, VGSized&: NumVGScaledBytes);
6999
7000 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
7001
7002 // Non-scalable offsets can use DW_CFA_offset directly.
7003 if (!NumVGScaledBytes)
7004 return MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: NumBytes);
7005
7006 std::string CommentBuffer;
7007 llvm::raw_string_ostream Comment(CommentBuffer);
7008 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
7009
7010 // Build up expression (CFA + VG * NumVGScaledBytes + NumBytes)
7011 assert(NumVGScaledBytes && "Expected scalable offset");
7012 SmallString<64> OffsetExpr;
7013 // + VG * NumVGScaledBytes
7014 StringRef VGRegScale;
7015 if (IncomingVGOffsetFromDefCFA) {
7016 appendLoadRegExpr(Expr&: OffsetExpr, OffsetFromDefCFA: *IncomingVGOffsetFromDefCFA);
7017 VGRegScale = "* IncomingVG";
7018 } else {
7019 appendReadRegExpr(Expr&: OffsetExpr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
7020 VGRegScale = "* VG";
7021 }
7022 appendConstantExpr(Expr&: OffsetExpr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
7023 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: VGRegScale);
7024 OffsetExpr.push_back(Elt: dwarf::DW_OP_plus);
7025 if (NumBytes) {
7026 // + NumBytes
7027 appendOffsetComment(NumBytes, Comment);
7028 appendConstantExpr(Expr&: OffsetExpr, Constant: NumBytes, Operation: dwarf::DW_OP_plus);
7029 }
7030
7031 // Wrap this into DW_CFA_expression
7032 SmallString<64> CfaExpr;
7033 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
7034 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: DwarfReg);
7035 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: OffsetExpr.size());
7036 CfaExpr.append(RHS: OffsetExpr.str());
7037
7038 return MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str(), Loc: SMLoc(),
7039 Comment: Comment.str());
7040}
7041
7042// Helper function to emit a frame offset adjustment from a given
7043// pointer (SrcReg), stored into DestReg. This function is explicit
7044// in that it requires the opcode.
7045static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
7046 MachineBasicBlock::iterator MBBI,
7047 const DebugLoc &DL, unsigned DestReg,
7048 unsigned SrcReg, int64_t Offset, unsigned Opc,
7049 const TargetInstrInfo *TII,
7050 MachineInstr::MIFlag Flag, bool NeedsWinCFI,
7051 bool *HasWinCFI, bool EmitCFAOffset,
7052 StackOffset CFAOffset, unsigned FrameReg) {
7053 int Sign = 1;
7054 unsigned MaxEncoding, ShiftSize;
7055 switch (Opc) {
7056 case AArch64::ADDXri:
7057 case AArch64::ADDSXri:
7058 case AArch64::SUBXri:
7059 case AArch64::SUBSXri:
7060 MaxEncoding = 0xfff;
7061 ShiftSize = 12;
7062 break;
7063 case AArch64::ADDVL_XXI:
7064 case AArch64::ADDPL_XXI:
7065 case AArch64::ADDSVL_XXI:
7066 case AArch64::ADDSPL_XXI:
7067 MaxEncoding = 31;
7068 ShiftSize = 0;
7069 if (Offset < 0) {
7070 MaxEncoding = 32;
7071 Sign = -1;
7072 Offset = -Offset;
7073 }
7074 break;
7075 default:
7076 llvm_unreachable("Unsupported opcode");
7077 }
7078
7079 // `Offset` can be in bytes or in "scalable bytes".
7080 int VScale = 1;
7081 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
7082 VScale = 16;
7083 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
7084 VScale = 2;
7085
7086 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
7087 // scratch register. If DestReg is a virtual register, use it as the
7088 // scratch register; otherwise, create a new virtual register (to be
7089 // replaced by the scavenger at the end of PEI). That case can be optimized
7090 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
7091 // register can be loaded with offset%8 and the add/sub can use an extending
7092 // instruction with LSL#3.
7093 // Currently the function handles any offsets but generates a poor sequence
7094 // of code.
7095 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
7096
7097 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
7098 Register TmpReg = DestReg;
7099 if (TmpReg == AArch64::XZR)
7100 TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
7101 RegClass: &AArch64::GPR64RegClass);
7102 do {
7103 uint64_t ThisVal = std::min<uint64_t>(a: Offset, b: MaxEncodableValue);
7104 unsigned LocalShiftSize = 0;
7105 if (ThisVal > MaxEncoding) {
7106 ThisVal = ThisVal >> ShiftSize;
7107 LocalShiftSize = ShiftSize;
7108 }
7109 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
7110 "Encoding cannot handle value that big");
7111
7112 Offset -= ThisVal << LocalShiftSize;
7113 if (Offset == 0)
7114 TmpReg = DestReg;
7115 auto MBI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: TmpReg)
7116 .addReg(RegNo: SrcReg)
7117 .addImm(Val: Sign * (int)ThisVal);
7118 if (ShiftSize)
7119 MBI = MBI.addImm(
7120 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LocalShiftSize));
7121 MBI = MBI.setMIFlag(Flag);
7122
7123 auto Change =
7124 VScale == 1
7125 ? StackOffset::getFixed(Fixed: ThisVal << LocalShiftSize)
7126 : StackOffset::getScalable(Scalable: VScale * (ThisVal << LocalShiftSize));
7127 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
7128 CFAOffset += Change;
7129 else
7130 CFAOffset -= Change;
7131 if (EmitCFAOffset && DestReg == TmpReg) {
7132 MachineFunction &MF = *MBB.getParent();
7133 const TargetSubtargetInfo &STI = MF.getSubtarget();
7134 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
7135
7136 unsigned CFIIndex = MF.addFrameInst(
7137 Inst: createDefCFA(TRI, FrameReg, Reg: DestReg, Offset: CFAOffset, LastAdjustmentWasScalable: VScale != 1));
7138 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
7139 .addCFIIndex(CFIIndex)
7140 .setMIFlags(Flag);
7141 }
7142
7143 if (NeedsWinCFI) {
7144 int Imm = (int)(ThisVal << LocalShiftSize);
7145 if (VScale != 1 && DestReg == AArch64::SP) {
7146 if (HasWinCFI)
7147 *HasWinCFI = true;
7148 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AllocZ))
7149 .addImm(Val: ThisVal)
7150 .setMIFlag(Flag);
7151 } else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
7152 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
7153 assert(VScale == 1 && "Expected non-scalable operation");
7154 if (HasWinCFI)
7155 *HasWinCFI = true;
7156 if (Imm == 0)
7157 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_SetFP)).setMIFlag(Flag);
7158 else
7159 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AddFP))
7160 .addImm(Val: Imm)
7161 .setMIFlag(Flag);
7162 assert(Offset == 0 && "Expected remaining offset to be zero to "
7163 "emit a single SEH directive");
7164 } else if (DestReg == AArch64::SP) {
7165 assert(VScale == 1 && "Expected non-scalable operation");
7166 if (HasWinCFI)
7167 *HasWinCFI = true;
7168 assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
7169 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_StackAlloc))
7170 .addImm(Val: Imm)
7171 .setMIFlag(Flag);
7172 }
7173 }
7174
7175 SrcReg = TmpReg;
7176 } while (Offset);
7177}
7178
7179void llvm::emitFrameOffset(MachineBasicBlock &MBB,
7180 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
7181 unsigned DestReg, unsigned SrcReg,
7182 StackOffset Offset, const TargetInstrInfo *TII,
7183 MachineInstr::MIFlag Flag, bool SetNZCV,
7184 bool NeedsWinCFI, bool *HasWinCFI,
7185 bool EmitCFAOffset, StackOffset CFAOffset,
7186 unsigned FrameReg) {
7187 // If a function is marked as arm_locally_streaming, then the runtime value of
7188 // vscale in the prologue/epilogue is different the runtime value of vscale
7189 // in the function's body. To avoid having to consider multiple vscales,
7190 // we can use `addsvl` to allocate any scalable stack-slots, which under
7191 // most circumstances will be only locals, not callee-save slots.
7192 const Function &F = MBB.getParent()->getFunction();
7193 bool UseSVL = F.hasFnAttribute(Kind: "aarch64_pstate_sm_body");
7194
7195 int64_t Bytes, NumPredicateVectors, NumDataVectors;
7196 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
7197 Offset, NumBytes&: Bytes, NumPredicateVectors, NumDataVectors);
7198
7199 // Insert ADDSXri for scalable offset at the end.
7200 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
7201 if (NeedsFinalDefNZCV)
7202 SetNZCV = false;
7203
7204 // First emit non-scalable frame offsets, or a simple 'mov'.
7205 if (Bytes || (!Offset && SrcReg != DestReg)) {
7206 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
7207 "SP increment/decrement not 8-byte aligned");
7208 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
7209 if (Bytes < 0) {
7210 Bytes = -Bytes;
7211 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
7212 }
7213 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: Bytes, Opc, TII, Flag,
7214 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7215 FrameReg);
7216 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
7217 ? StackOffset::getFixed(Fixed: -Bytes)
7218 : StackOffset::getFixed(Fixed: Bytes);
7219 SrcReg = DestReg;
7220 FrameReg = DestReg;
7221 }
7222
7223 assert(!(NeedsWinCFI && NumPredicateVectors) &&
7224 "WinCFI can't allocate fractions of an SVE data vector");
7225
7226 if (NumDataVectors) {
7227 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumDataVectors,
7228 Opc: UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI, TII,
7229 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7230 FrameReg);
7231 CFAOffset += StackOffset::getScalable(Scalable: -NumDataVectors * 16);
7232 SrcReg = DestReg;
7233 }
7234
7235 if (NumPredicateVectors) {
7236 assert(DestReg != AArch64::SP && "Unaligned access to SP");
7237 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumPredicateVectors,
7238 Opc: UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI, TII,
7239 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7240 FrameReg);
7241 }
7242
7243 if (NeedsFinalDefNZCV)
7244 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDSXri), DestReg)
7245 .addReg(RegNo: DestReg)
7246 .addImm(Val: 0)
7247 .addImm(Val: 0);
7248}
7249
7250MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
7251 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
7252 int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS,
7253 VirtRegMap *VRM) const {
7254 MachineBasicBlock::iterator InsertPt = MI;
7255 // This is a bit of a hack. Consider this instruction:
7256 //
7257 // %0 = COPY %sp; GPR64all:%0
7258 //
7259 // We explicitly chose GPR64all for the virtual register so such a copy might
7260 // be eliminated by RegisterCoalescer. However, that may not be possible, and
7261 // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
7262 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
7263 //
7264 // To prevent that, we are going to constrain the %0 register class here.
7265 if (MI.isFullCopy()) {
7266 Register DstReg = MI.getOperand(i: 0).getReg();
7267 Register SrcReg = MI.getOperand(i: 1).getReg();
7268 if (SrcReg == AArch64::SP && DstReg.isVirtual()) {
7269 MF.getRegInfo().constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass);
7270 return nullptr;
7271 }
7272 if (DstReg == AArch64::SP && SrcReg.isVirtual()) {
7273 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
7274 return nullptr;
7275 }
7276 // Nothing can folded with copy from/to NZCV.
7277 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
7278 return nullptr;
7279 }
7280
7281 // Handle the case where a copy is being spilled or filled but the source
7282 // and destination register class don't match. For example:
7283 //
7284 // %0 = COPY %xzr; GPR64common:%0
7285 //
7286 // In this case we can still safely fold away the COPY and generate the
7287 // following spill code:
7288 //
7289 // STRXui %xzr, %stack.0
7290 //
7291 // This also eliminates spilled cross register class COPYs (e.g. between x and
7292 // d regs) of the same size. For example:
7293 //
7294 // %0 = COPY %1; GPR64:%0, FPR64:%1
7295 //
7296 // will be filled as
7297 //
7298 // LDRDui %0, fi<#0>
7299 //
7300 // instead of
7301 //
7302 // LDRXui %Temp, fi<#0>
7303 // %0 = FMOV %Temp
7304 //
7305 if (MI.isCopy() && Ops.size() == 1 &&
7306 // Make sure we're only folding the explicit COPY defs/uses.
7307 (Ops[0] == 0 || Ops[0] == 1)) {
7308 bool IsSpill = Ops[0] == 0;
7309 bool IsFill = !IsSpill;
7310 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7311 const MachineRegisterInfo &MRI = MF.getRegInfo();
7312 MachineBasicBlock &MBB = *MI.getParent();
7313 const MachineOperand &DstMO = MI.getOperand(i: 0);
7314 const MachineOperand &SrcMO = MI.getOperand(i: 1);
7315 Register DstReg = DstMO.getReg();
7316 Register SrcReg = SrcMO.getReg();
7317 // This is slightly expensive to compute for physical regs since
7318 // getMinimalPhysRegClass is slow.
7319 auto getRegClass = [&](unsigned Reg) {
7320 return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
7321 : TRI.getMinimalPhysRegClass(Reg);
7322 };
7323
7324 if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
7325 assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
7326 TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
7327 "Mismatched register size in non subreg COPY");
7328 if (IsSpill)
7329 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg, isKill: SrcMO.isKill(), FI: FrameIndex,
7330 RC: getRegClass(SrcReg), VReg: Register());
7331 else
7332 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex,
7333 RC: getRegClass(DstReg), VReg: Register());
7334 return &*--InsertPt;
7335 }
7336
7337 // Handle cases like spilling def of:
7338 //
7339 // %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
7340 //
7341 // where the physical register source can be widened and stored to the full
7342 // virtual reg destination stack slot, in this case producing:
7343 //
7344 // STRXui %xzr, %stack.0
7345 //
7346 if (IsSpill && DstMO.isUndef() && SrcReg == AArch64::WZR &&
7347 TRI.getRegSizeInBits(RC: *getRegClass(DstReg)) == 64) {
7348 assert(SrcMO.getSubReg() == 0 &&
7349 "Unexpected subreg on physical register");
7350 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg: AArch64::XZR, isKill: SrcMO.isKill(),
7351 FI: FrameIndex, RC: &AArch64::GPR64RegClass, VReg: Register());
7352 return &*--InsertPt;
7353 }
7354
7355 // Handle cases like filling use of:
7356 //
7357 // %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
7358 //
7359 // where we can load the full virtual reg source stack slot, into the subreg
7360 // destination, in this case producing:
7361 //
7362 // LDRWui %0:sub_32<def,read-undef>, %stack.0
7363 //
7364 if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
7365 const TargetRegisterClass *FillRC = nullptr;
7366 switch (DstMO.getSubReg()) {
7367 default:
7368 break;
7369 case AArch64::sub_32:
7370 if (AArch64::GPR64RegClass.hasSubClassEq(RC: getRegClass(DstReg)))
7371 FillRC = &AArch64::GPR32RegClass;
7372 break;
7373 case AArch64::ssub:
7374 FillRC = &AArch64::FPR32RegClass;
7375 break;
7376 case AArch64::dsub:
7377 FillRC = &AArch64::FPR64RegClass;
7378 break;
7379 }
7380
7381 if (FillRC) {
7382 assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
7383 TRI.getRegSizeInBits(*FillRC) &&
7384 "Mismatched regclass size on folded subreg COPY");
7385 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex, RC: FillRC,
7386 VReg: Register());
7387 MachineInstr &LoadMI = *--InsertPt;
7388 MachineOperand &LoadDst = LoadMI.getOperand(i: 0);
7389 assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
7390 LoadDst.setSubReg(DstMO.getSubReg());
7391 LoadDst.setIsUndef();
7392 return &LoadMI;
7393 }
7394 }
7395 }
7396
7397 // Cannot fold.
7398 return nullptr;
7399}
7400
7401int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
7402 StackOffset &SOffset,
7403 bool *OutUseUnscaledOp,
7404 unsigned *OutUnscaledOp,
7405 int64_t *EmittableOffset) {
7406 // Set output values in case of early exit.
7407 if (EmittableOffset)
7408 *EmittableOffset = 0;
7409 if (OutUseUnscaledOp)
7410 *OutUseUnscaledOp = false;
7411 if (OutUnscaledOp)
7412 *OutUnscaledOp = 0;
7413
7414 // Exit early for structured vector spills/fills as they can't take an
7415 // immediate offset.
7416 switch (MI.getOpcode()) {
7417 default:
7418 break;
7419 case AArch64::LD1Rv1d:
7420 case AArch64::LD1Rv2s:
7421 case AArch64::LD1Rv2d:
7422 case AArch64::LD1Rv4h:
7423 case AArch64::LD1Rv4s:
7424 case AArch64::LD1Rv8b:
7425 case AArch64::LD1Rv8h:
7426 case AArch64::LD1Rv16b:
7427 case AArch64::LD1Twov2d:
7428 case AArch64::LD1Threev2d:
7429 case AArch64::LD1Fourv2d:
7430 case AArch64::LD1Twov1d:
7431 case AArch64::LD1Threev1d:
7432 case AArch64::LD1Fourv1d:
7433 case AArch64::ST1Twov2d:
7434 case AArch64::ST1Threev2d:
7435 case AArch64::ST1Fourv2d:
7436 case AArch64::ST1Twov1d:
7437 case AArch64::ST1Threev1d:
7438 case AArch64::ST1Fourv1d:
7439 case AArch64::ST1i8:
7440 case AArch64::ST1i16:
7441 case AArch64::ST1i32:
7442 case AArch64::ST1i64:
7443 case AArch64::IRG:
7444 case AArch64::IRGstack:
7445 case AArch64::STGloop:
7446 case AArch64::STZGloop:
7447 return AArch64FrameOffsetCannotUpdate;
7448 }
7449
7450 // Get the min/max offset and the scale.
7451 TypeSize ScaleValue(0U, false), Width(0U, false);
7452 int64_t MinOff, MaxOff;
7453 if (!AArch64InstrInfo::getMemOpInfo(Opcode: MI.getOpcode(), Scale&: ScaleValue, Width, MinOffset&: MinOff,
7454 MaxOffset&: MaxOff))
7455 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7456
7457 // Construct the complete offset.
7458 bool IsMulVL = ScaleValue.isScalable();
7459 unsigned Scale = ScaleValue.getKnownMinValue();
7460 int64_t Offset = IsMulVL ? SOffset.getScalable() : SOffset.getFixed();
7461
7462 const MachineOperand &ImmOpnd =
7463 MI.getOperand(i: AArch64InstrInfo::getLoadStoreImmIdx(Opc: MI.getOpcode()));
7464 Offset += ImmOpnd.getImm() * Scale;
7465
7466 // If the offset doesn't match the scale, we rewrite the instruction to
7467 // use the unscaled instruction instead. Likewise, if we have a negative
7468 // offset and there is an unscaled op to use.
7469 std::optional<unsigned> UnscaledOp =
7470 AArch64InstrInfo::getUnscaledLdSt(Opc: MI.getOpcode());
7471 bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
7472 if (useUnscaledOp &&
7473 !AArch64InstrInfo::getMemOpInfo(Opcode: *UnscaledOp, Scale&: ScaleValue, Width, MinOffset&: MinOff,
7474 MaxOffset&: MaxOff))
7475 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7476
7477 Scale = ScaleValue.getKnownMinValue();
7478 assert(IsMulVL == ScaleValue.isScalable() &&
7479 "Unscaled opcode has different value for scalable");
7480
7481 int64_t Remainder = Offset % Scale;
7482 assert(!(Remainder && useUnscaledOp) &&
7483 "Cannot have remainder when using unscaled op");
7484
7485 assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
7486 int64_t NewOffset = Offset / Scale;
7487 if (MinOff <= NewOffset && NewOffset <= MaxOff)
7488 Offset = Remainder;
7489 else {
7490 // Try to minimise the number of instructions required to materialise the
7491 // offset calculation. Specifically, for fixed offsets, if masking out the
7492 // low 12 bits leaves a legal add immediate, we can realise the offset
7493 // calculation with a single add instruction. Whenever this is possible,
7494 // prefer this split.
7495 int64_t HighPart = Offset & ~0xFFF;
7496 int64_t LowPart = Offset & 0xFFF;
7497 int64_t LowScaled = LowPart / Scale;
7498 if (!IsMulVL && NewOffset >= 0 && LowPart % Scale == 0 &&
7499 MinOff <= LowScaled && LowScaled <= MaxOff &&
7500 AArch64_AM::isLegalArithImmed(C: HighPart)) {
7501 NewOffset = LowScaled;
7502 Offset = HighPart;
7503 } else {
7504 // Default to a greedy split: take the memop immediate to be maximum /
7505 // minimum expressible offset and materialise the remainder.
7506 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
7507 Offset = Offset - (NewOffset * Scale);
7508 }
7509 }
7510
7511 if (EmittableOffset)
7512 *EmittableOffset = NewOffset;
7513 if (OutUseUnscaledOp)
7514 *OutUseUnscaledOp = useUnscaledOp;
7515 if (OutUnscaledOp && UnscaledOp)
7516 *OutUnscaledOp = *UnscaledOp;
7517
7518 if (IsMulVL)
7519 SOffset = StackOffset::get(Fixed: SOffset.getFixed(), Scalable: Offset);
7520 else
7521 SOffset = StackOffset::get(Fixed: Offset, Scalable: SOffset.getScalable());
7522 return AArch64FrameOffsetCanUpdate |
7523 (SOffset ? 0 : AArch64FrameOffsetIsLegal);
7524}
7525
7526bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
7527 unsigned FrameReg, StackOffset &Offset,
7528 const AArch64InstrInfo *TII) {
7529 unsigned Opcode = MI.getOpcode();
7530 unsigned ImmIdx = FrameRegIdx + 1;
7531
7532 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
7533 Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: ImmIdx).getImm());
7534 emitFrameOffset(MBB&: *MI.getParent(), MBBI: MI, DL: MI.getDebugLoc(),
7535 DestReg: MI.getOperand(i: 0).getReg(), SrcReg: FrameReg, Offset, TII,
7536 Flag: MachineInstr::NoFlags, SetNZCV: (Opcode == AArch64::ADDSXri));
7537 MI.eraseFromParent();
7538 Offset = StackOffset();
7539 return true;
7540 }
7541
7542 int64_t NewOffset;
7543 unsigned UnscaledOp;
7544 bool UseUnscaledOp;
7545 int Status = isAArch64FrameOffsetLegal(MI, SOffset&: Offset, OutUseUnscaledOp: &UseUnscaledOp,
7546 OutUnscaledOp: &UnscaledOp, EmittableOffset: &NewOffset);
7547 if (Status & AArch64FrameOffsetCanUpdate) {
7548 if (Status & AArch64FrameOffsetIsLegal)
7549 // Replace the FrameIndex with FrameReg.
7550 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
7551 if (UseUnscaledOp)
7552 MI.setDesc(TII->get(Opcode: UnscaledOp));
7553
7554 MI.getOperand(i: ImmIdx).ChangeToImmediate(ImmVal: NewOffset);
7555 return !Offset;
7556 }
7557
7558 return false;
7559}
7560
7561void AArch64InstrInfo::insertNoop(MachineBasicBlock &MBB,
7562 MachineBasicBlock::iterator MI) const {
7563 DebugLoc DL;
7564 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::NOP));
7565}
7566
7567MCInst AArch64InstrInfo::getNop() const { return MCInstBuilder(AArch64::NOP); }
7568
7569// AArch64 supports MachineCombiner.
7570bool AArch64InstrInfo::useMachineCombiner() const { return true; }
7571
7572// True when Opc sets flag
7573static bool isCombineInstrSettingFlag(unsigned Opc) {
7574 switch (Opc) {
7575 case AArch64::ADDSWrr:
7576 case AArch64::ADDSWri:
7577 case AArch64::ADDSXrr:
7578 case AArch64::ADDSXri:
7579 case AArch64::SUBSWrr:
7580 case AArch64::SUBSXrr:
7581 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7582 case AArch64::SUBSWri:
7583 case AArch64::SUBSXri:
7584 return true;
7585 default:
7586 break;
7587 }
7588 return false;
7589}
7590
7591// 32b Opcodes that can be combined with a MUL
7592static bool isCombineInstrCandidate32(unsigned Opc) {
7593 switch (Opc) {
7594 case AArch64::ADDWrr:
7595 case AArch64::ADDWri:
7596 case AArch64::ADDWrs:
7597 case AArch64::SUBWrr:
7598 case AArch64::ADDSWrr:
7599 case AArch64::ADDSWri:
7600 case AArch64::SUBSWrr:
7601 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7602 case AArch64::SUBWri:
7603 case AArch64::SUBSWri:
7604 return true;
7605 default:
7606 break;
7607 }
7608 return false;
7609}
7610
7611// 64b Opcodes that can be combined with a MUL
7612static bool isCombineInstrCandidate64(unsigned Opc) {
7613 switch (Opc) {
7614 case AArch64::ADDXrr:
7615 case AArch64::ADDXri:
7616 case AArch64::ADDXrs:
7617 case AArch64::SUBXrr:
7618 case AArch64::ADDSXrr:
7619 case AArch64::ADDSXri:
7620 case AArch64::SUBSXrr:
7621 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7622 case AArch64::SUBXri:
7623 case AArch64::SUBSXri:
7624 case AArch64::ADDv8i8:
7625 case AArch64::ADDv16i8:
7626 case AArch64::ADDv4i16:
7627 case AArch64::ADDv8i16:
7628 case AArch64::ADDv2i32:
7629 case AArch64::ADDv4i32:
7630 case AArch64::SUBv8i8:
7631 case AArch64::SUBv16i8:
7632 case AArch64::SUBv4i16:
7633 case AArch64::SUBv8i16:
7634 case AArch64::SUBv2i32:
7635 case AArch64::SUBv4i32:
7636 return true;
7637 default:
7638 break;
7639 }
7640 return false;
7641}
7642
7643// FP Opcodes that can be combined with a FMUL.
7644static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
7645 switch (Inst.getOpcode()) {
7646 default:
7647 break;
7648 case AArch64::FADDHrr:
7649 case AArch64::FADDSrr:
7650 case AArch64::FADDDrr:
7651 case AArch64::FADDv4f16:
7652 case AArch64::FADDv8f16:
7653 case AArch64::FADDv2f32:
7654 case AArch64::FADDv2f64:
7655 case AArch64::FADDv4f32:
7656 case AArch64::FSUBHrr:
7657 case AArch64::FSUBSrr:
7658 case AArch64::FSUBDrr:
7659 case AArch64::FSUBv4f16:
7660 case AArch64::FSUBv8f16:
7661 case AArch64::FSUBv2f32:
7662 case AArch64::FSUBv2f64:
7663 case AArch64::FSUBv4f32:
7664 // We can fuse FADD/FSUB with FMUL, if FADD/FSUB has the contract fast-math
7665 // flag.
7666 return Inst.getFlag(Flag: MachineInstr::FmContract);
7667 }
7668 return false;
7669}
7670
7671// Opcodes that can be combined with a MUL
7672static bool isCombineInstrCandidate(unsigned Opc) {
7673 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
7674}
7675
7676//
7677// Utility routine that checks if \param MO is defined by an
7678// \param CombineOpc instruction in the basic block \param MBB
7679static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
7680 unsigned CombineOpc, unsigned ZeroReg = 0,
7681 bool CheckZeroReg = false) {
7682 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7683 MachineInstr *MI = nullptr;
7684
7685 if (MO.isReg() && MO.getReg().isVirtual())
7686 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
7687 // And it needs to be in the trace (otherwise, it won't have a depth).
7688 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
7689 return false;
7690 // Must only used by the user we combine with.
7691 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
7692 return false;
7693
7694 if (CheckZeroReg) {
7695 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
7696 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
7697 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
7698 // The third input reg must be zero.
7699 if (MI->getOperand(i: 3).getReg() != ZeroReg)
7700 return false;
7701 }
7702
7703 if (isCombineInstrSettingFlag(Opc: CombineOpc) &&
7704 MI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) == -1)
7705 return false;
7706
7707 return true;
7708}
7709
7710//
7711// Is \param MO defined by an integer multiply and can be combined?
7712static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7713 unsigned MulOpc, unsigned ZeroReg) {
7714 return canCombine(MBB, MO, CombineOpc: MulOpc, ZeroReg, CheckZeroReg: true);
7715}
7716
7717//
7718// Is \param MO defined by a floating-point multiply and can be combined?
7719static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7720 unsigned MulOpc) {
7721 return canCombine(MBB, MO, CombineOpc: MulOpc);
7722}
7723
7724// TODO: There are many more machine instruction opcodes to match:
7725// 1. Other data types (integer, vectors)
7726// 2. Other math / logic operations (xor, or)
7727// 3. Other forms of the same operation (intrinsics and other variants)
7728bool AArch64InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
7729 bool Invert) const {
7730 if (Invert)
7731 return false;
7732 switch (Inst.getOpcode()) {
7733 // == Floating-point types ==
7734 // -- Floating-point instructions --
7735 case AArch64::FADDHrr:
7736 case AArch64::FADDSrr:
7737 case AArch64::FADDDrr:
7738 case AArch64::FMULHrr:
7739 case AArch64::FMULSrr:
7740 case AArch64::FMULDrr:
7741 case AArch64::FMULX16:
7742 case AArch64::FMULX32:
7743 case AArch64::FMULX64:
7744 // -- Advanced SIMD instructions --
7745 case AArch64::FADDv4f16:
7746 case AArch64::FADDv8f16:
7747 case AArch64::FADDv2f32:
7748 case AArch64::FADDv4f32:
7749 case AArch64::FADDv2f64:
7750 case AArch64::FMULv4f16:
7751 case AArch64::FMULv8f16:
7752 case AArch64::FMULv2f32:
7753 case AArch64::FMULv4f32:
7754 case AArch64::FMULv2f64:
7755 case AArch64::FMULXv4f16:
7756 case AArch64::FMULXv8f16:
7757 case AArch64::FMULXv2f32:
7758 case AArch64::FMULXv4f32:
7759 case AArch64::FMULXv2f64:
7760 // -- SVE instructions --
7761 // Opcodes FMULX_ZZZ_? don't exist because there is no unpredicated FMULX
7762 // in the SVE instruction set (though there are predicated ones).
7763 case AArch64::FADD_ZZZ_H:
7764 case AArch64::FADD_ZZZ_S:
7765 case AArch64::FADD_ZZZ_D:
7766 case AArch64::FMUL_ZZZ_H:
7767 case AArch64::FMUL_ZZZ_S:
7768 case AArch64::FMUL_ZZZ_D:
7769 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
7770 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
7771
7772 // == Integer types ==
7773 // -- Base instructions --
7774 // Opcodes MULWrr and MULXrr don't exist because
7775 // `MUL <Wd>, <Wn>, <Wm>` and `MUL <Xd>, <Xn>, <Xm>` are aliases of
7776 // `MADD <Wd>, <Wn>, <Wm>, WZR` and `MADD <Xd>, <Xn>, <Xm>, XZR` respectively.
7777 // The machine-combiner does not support three-source-operands machine
7778 // instruction. So we cannot reassociate MULs.
7779 case AArch64::ADDWrr:
7780 case AArch64::ADDXrr:
7781 case AArch64::ANDWrr:
7782 case AArch64::ANDXrr:
7783 case AArch64::ORRWrr:
7784 case AArch64::ORRXrr:
7785 case AArch64::EORWrr:
7786 case AArch64::EORXrr:
7787 case AArch64::EONWrr:
7788 case AArch64::EONXrr:
7789 // -- Advanced SIMD instructions --
7790 // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL
7791 // in the Advanced SIMD instruction set.
7792 case AArch64::ADDv8i8:
7793 case AArch64::ADDv16i8:
7794 case AArch64::ADDv4i16:
7795 case AArch64::ADDv8i16:
7796 case AArch64::ADDv2i32:
7797 case AArch64::ADDv4i32:
7798 case AArch64::ADDv1i64:
7799 case AArch64::ADDv2i64:
7800 case AArch64::MULv8i8:
7801 case AArch64::MULv16i8:
7802 case AArch64::MULv4i16:
7803 case AArch64::MULv8i16:
7804 case AArch64::MULv2i32:
7805 case AArch64::MULv4i32:
7806 case AArch64::ANDv8i8:
7807 case AArch64::ANDv16i8:
7808 case AArch64::ORRv8i8:
7809 case AArch64::ORRv16i8:
7810 case AArch64::EORv8i8:
7811 case AArch64::EORv16i8:
7812 // -- SVE instructions --
7813 case AArch64::ADD_ZZZ_B:
7814 case AArch64::ADD_ZZZ_H:
7815 case AArch64::ADD_ZZZ_S:
7816 case AArch64::ADD_ZZZ_D:
7817 case AArch64::MUL_ZZZ_B:
7818 case AArch64::MUL_ZZZ_H:
7819 case AArch64::MUL_ZZZ_S:
7820 case AArch64::MUL_ZZZ_D:
7821 case AArch64::AND_ZZZ:
7822 case AArch64::ORR_ZZZ:
7823 case AArch64::EOR_ZZZ:
7824 return true;
7825
7826 default:
7827 return false;
7828 }
7829}
7830
7831/// Find instructions that can be turned into madd.
7832static bool getMaddPatterns(MachineInstr &Root,
7833 SmallVectorImpl<unsigned> &Patterns) {
7834 unsigned Opc = Root.getOpcode();
7835 MachineBasicBlock &MBB = *Root.getParent();
7836 bool Found = false;
7837
7838 if (!isCombineInstrCandidate(Opc))
7839 return false;
7840 if (isCombineInstrSettingFlag(Opc)) {
7841 int Cmp_NZCV =
7842 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
7843 // When NZCV is live bail out.
7844 if (Cmp_NZCV == -1)
7845 return false;
7846 unsigned NewOpc = convertToNonFlagSettingOpc(MI: Root);
7847 // When opcode can't change bail out.
7848 // CHECKME: do we miss any cases for opcode conversion?
7849 if (NewOpc == Opc)
7850 return false;
7851 Opc = NewOpc;
7852 }
7853
7854 auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
7855 unsigned Pattern) {
7856 if (canCombineWithMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode, ZeroReg)) {
7857 Patterns.push_back(Elt: Pattern);
7858 Found = true;
7859 }
7860 };
7861
7862 const AArch64Subtarget &STI =
7863 MBB.getParent()->getSubtarget<AArch64Subtarget>();
7864 auto setShfFound = [&](int Opcode, unsigned ZeroReg, unsigned Pattern) {
7865 // TODO: Currently its not possible to reject the combine for Apple targets
7866 // based on the scheduling model, because we have gaps in modeling and
7867 // analysis. Once we fill the gaps, remove the subtarget feature and the
7868 // associated check here.
7869 if (STI.hasShiftMaddSlow())
7870 return;
7871
7872 unsigned ShiftImm = Root.getOperand(i: 3).getImm();
7873 AArch64_AM::ShiftExtendType ShiftTy = AArch64_AM::getShiftType(Imm: ShiftImm);
7874
7875 if ((ShiftTy != AArch64_AM::LSL && ShiftTy != AArch64_AM::LSR &&
7876 ShiftTy != AArch64_AM::ASR) ||
7877 AArch64_AM::getShiftValue(Imm: ShiftImm) == 0)
7878 return;
7879
7880 if (!canCombineWithMUL(MBB, MO&: Root.getOperand(i: 1), MulOpc: Opcode, ZeroReg))
7881 return;
7882
7883 Patterns.push_back(Elt: Pattern);
7884 Found = true;
7885 };
7886
7887 auto setVFound = [&](int Opcode, int Operand, unsigned Pattern) {
7888 if (canCombine(MBB, MO&: Root.getOperand(i: Operand), CombineOpc: Opcode)) {
7889 Patterns.push_back(Elt: Pattern);
7890 Found = true;
7891 }
7892 };
7893
7894 typedef AArch64MachineCombinerPattern MCP;
7895
7896 switch (Opc) {
7897 default:
7898 break;
7899 case AArch64::ADDWrr:
7900 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
7901 "ADDWrr does not have register operands");
7902 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
7903 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
7904 break;
7905 case AArch64::ADDXrr:
7906 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
7907 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
7908 break;
7909 case AArch64::SUBWrr:
7910 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
7911 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
7912 break;
7913 case AArch64::SUBXrr:
7914 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
7915 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
7916 break;
7917 case AArch64::ADDWri:
7918 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
7919 break;
7920 case AArch64::ADDXri:
7921 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
7922 break;
7923 case AArch64::SUBWri:
7924 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
7925 break;
7926 case AArch64::SUBXri:
7927 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
7928 break;
7929 case AArch64::ADDWrs:
7930 setShfFound(AArch64::MADDWrrr, AArch64::WZR, MCP::MULADDWS_OP1);
7931 break;
7932 case AArch64::ADDXrs:
7933 setShfFound(AArch64::MADDXrrr, AArch64::XZR, MCP::MULADDXS_OP1);
7934 break;
7935 case AArch64::ADDv8i8:
7936 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
7937 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
7938 break;
7939 case AArch64::ADDv16i8:
7940 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
7941 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
7942 break;
7943 case AArch64::ADDv4i16:
7944 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
7945 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
7946 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
7947 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
7948 break;
7949 case AArch64::ADDv8i16:
7950 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
7951 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
7952 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
7953 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
7954 break;
7955 case AArch64::ADDv2i32:
7956 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
7957 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
7958 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
7959 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
7960 break;
7961 case AArch64::ADDv4i32:
7962 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
7963 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
7964 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
7965 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
7966 break;
7967 case AArch64::SUBv8i8:
7968 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
7969 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
7970 break;
7971 case AArch64::SUBv16i8:
7972 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
7973 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
7974 break;
7975 case AArch64::SUBv4i16:
7976 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
7977 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7978 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7979 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7980 break;
7981 case AArch64::SUBv8i16:
7982 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7983 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7984 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7985 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7986 break;
7987 case AArch64::SUBv2i32:
7988 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7989 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7990 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7991 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7992 break;
7993 case AArch64::SUBv4i32:
7994 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7995 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7996 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7997 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7998 break;
7999 }
8000 return Found;
8001}
8002
8003bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
8004 switch (Opcode) {
8005 default:
8006 break;
8007 case AArch64::UABALB_ZZZ_D:
8008 case AArch64::UABALB_ZZZ_H:
8009 case AArch64::UABALB_ZZZ_S:
8010 case AArch64::UABALT_ZZZ_D:
8011 case AArch64::UABALT_ZZZ_H:
8012 case AArch64::UABALT_ZZZ_S:
8013 case AArch64::SABALB_ZZZ_D:
8014 case AArch64::SABALB_ZZZ_S:
8015 case AArch64::SABALB_ZZZ_H:
8016 case AArch64::SABALT_ZZZ_D:
8017 case AArch64::SABALT_ZZZ_S:
8018 case AArch64::SABALT_ZZZ_H:
8019 case AArch64::UABALv16i8_v8i16:
8020 case AArch64::UABALv2i32_v2i64:
8021 case AArch64::UABALv4i16_v4i32:
8022 case AArch64::UABALv4i32_v2i64:
8023 case AArch64::UABALv8i16_v4i32:
8024 case AArch64::UABALv8i8_v8i16:
8025 case AArch64::UABAv16i8:
8026 case AArch64::UABAv2i32:
8027 case AArch64::UABAv4i16:
8028 case AArch64::UABAv4i32:
8029 case AArch64::UABAv8i16:
8030 case AArch64::UABAv8i8:
8031 case AArch64::SABALv16i8_v8i16:
8032 case AArch64::SABALv2i32_v2i64:
8033 case AArch64::SABALv4i16_v4i32:
8034 case AArch64::SABALv4i32_v2i64:
8035 case AArch64::SABALv8i16_v4i32:
8036 case AArch64::SABALv8i8_v8i16:
8037 case AArch64::SABAv16i8:
8038 case AArch64::SABAv2i32:
8039 case AArch64::SABAv4i16:
8040 case AArch64::SABAv4i32:
8041 case AArch64::SABAv8i16:
8042 case AArch64::SABAv8i8:
8043 return true;
8044 }
8045
8046 return false;
8047}
8048
8049unsigned AArch64InstrInfo::getAccumulationStartOpcode(
8050 unsigned AccumulationOpcode) const {
8051 switch (AccumulationOpcode) {
8052 default:
8053 llvm_unreachable("Unsupported accumulation Opcode!");
8054 case AArch64::UABALB_ZZZ_D:
8055 return AArch64::UABDLB_ZZZ_D;
8056 case AArch64::UABALB_ZZZ_H:
8057 return AArch64::UABDLB_ZZZ_H;
8058 case AArch64::UABALB_ZZZ_S:
8059 return AArch64::UABDLB_ZZZ_S;
8060 case AArch64::UABALT_ZZZ_D:
8061 return AArch64::UABDLT_ZZZ_D;
8062 case AArch64::UABALT_ZZZ_H:
8063 return AArch64::UABDLT_ZZZ_H;
8064 case AArch64::UABALT_ZZZ_S:
8065 return AArch64::UABDLT_ZZZ_S;
8066 case AArch64::UABALv16i8_v8i16:
8067 return AArch64::UABDLv16i8_v8i16;
8068 case AArch64::UABALv2i32_v2i64:
8069 return AArch64::UABDLv2i32_v2i64;
8070 case AArch64::UABALv4i16_v4i32:
8071 return AArch64::UABDLv4i16_v4i32;
8072 case AArch64::UABALv4i32_v2i64:
8073 return AArch64::UABDLv4i32_v2i64;
8074 case AArch64::UABALv8i16_v4i32:
8075 return AArch64::UABDLv8i16_v4i32;
8076 case AArch64::UABALv8i8_v8i16:
8077 return AArch64::UABDLv8i8_v8i16;
8078 case AArch64::UABAv16i8:
8079 return AArch64::UABDv16i8;
8080 case AArch64::UABAv2i32:
8081 return AArch64::UABDv2i32;
8082 case AArch64::UABAv4i16:
8083 return AArch64::UABDv4i16;
8084 case AArch64::UABAv4i32:
8085 return AArch64::UABDv4i32;
8086 case AArch64::UABAv8i16:
8087 return AArch64::UABDv8i16;
8088 case AArch64::UABAv8i8:
8089 return AArch64::UABDv8i8;
8090 case AArch64::SABALB_ZZZ_D:
8091 return AArch64::SABDLB_ZZZ_D;
8092 case AArch64::SABALB_ZZZ_S:
8093 return AArch64::SABDLB_ZZZ_S;
8094 case AArch64::SABALB_ZZZ_H:
8095 return AArch64::SABDLB_ZZZ_H;
8096 case AArch64::SABALT_ZZZ_D:
8097 return AArch64::SABDLT_ZZZ_D;
8098 case AArch64::SABALT_ZZZ_S:
8099 return AArch64::SABDLT_ZZZ_S;
8100 case AArch64::SABALT_ZZZ_H:
8101 return AArch64::SABDLT_ZZZ_H;
8102 case AArch64::SABALv16i8_v8i16:
8103 return AArch64::SABDLv16i8_v8i16;
8104 case AArch64::SABALv2i32_v2i64:
8105 return AArch64::SABDLv2i32_v2i64;
8106 case AArch64::SABALv4i16_v4i32:
8107 return AArch64::SABDLv4i16_v4i32;
8108 case AArch64::SABALv4i32_v2i64:
8109 return AArch64::SABDLv4i32_v2i64;
8110 case AArch64::SABALv8i16_v4i32:
8111 return AArch64::SABDLv8i16_v4i32;
8112 case AArch64::SABALv8i8_v8i16:
8113 return AArch64::SABDLv8i8_v8i16;
8114 case AArch64::SABAv16i8:
8115 return AArch64::SABDv16i8;
8116 case AArch64::SABAv2i32:
8117 return AArch64::SABDv2i32;
8118 case AArch64::SABAv4i16:
8119 return AArch64::SABDv4i16;
8120 case AArch64::SABAv4i32:
8121 return AArch64::SABDv4i32;
8122 case AArch64::SABAv8i16:
8123 return AArch64::SABDv8i16;
8124 case AArch64::SABAv8i8:
8125 return AArch64::SABDv8i8;
8126 }
8127}
8128
8129/// Floating-Point Support
8130
8131/// Find instructions that can be turned into madd.
8132static bool getFMAPatterns(MachineInstr &Root,
8133 SmallVectorImpl<unsigned> &Patterns) {
8134
8135 if (!isCombineInstrCandidateFP(Inst: Root))
8136 return false;
8137
8138 MachineBasicBlock &MBB = *Root.getParent();
8139 bool Found = false;
8140
8141 auto Match = [&](int Opcode, int Operand, unsigned Pattern) -> bool {
8142 if (canCombineWithFMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode)) {
8143 Patterns.push_back(Elt: Pattern);
8144 return true;
8145 }
8146 return false;
8147 };
8148
8149 typedef AArch64MachineCombinerPattern MCP;
8150
8151 switch (Root.getOpcode()) {
8152 default:
8153 assert(false && "Unsupported FP instruction in combiner\n");
8154 break;
8155 case AArch64::FADDHrr:
8156 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8157 "FADDHrr does not have register operands");
8158
8159 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
8160 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
8161 break;
8162 case AArch64::FADDSrr:
8163 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8164 "FADDSrr does not have register operands");
8165
8166 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
8167 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
8168
8169 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
8170 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
8171 break;
8172 case AArch64::FADDDrr:
8173 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
8174 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
8175
8176 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
8177 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
8178 break;
8179 case AArch64::FADDv4f16:
8180 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
8181 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
8182
8183 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
8184 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
8185 break;
8186 case AArch64::FADDv8f16:
8187 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
8188 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
8189
8190 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
8191 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
8192 break;
8193 case AArch64::FADDv2f32:
8194 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
8195 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
8196
8197 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
8198 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
8199 break;
8200 case AArch64::FADDv2f64:
8201 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
8202 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
8203
8204 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
8205 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
8206 break;
8207 case AArch64::FADDv4f32:
8208 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
8209 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
8210
8211 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
8212 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
8213 break;
8214 case AArch64::FSUBHrr:
8215 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
8216 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
8217 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
8218 break;
8219 case AArch64::FSUBSrr:
8220 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
8221
8222 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
8223 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
8224
8225 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
8226 break;
8227 case AArch64::FSUBDrr:
8228 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
8229
8230 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
8231 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
8232
8233 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
8234 break;
8235 case AArch64::FSUBv4f16:
8236 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
8237 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
8238
8239 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
8240 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
8241 break;
8242 case AArch64::FSUBv8f16:
8243 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
8244 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
8245
8246 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
8247 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
8248 break;
8249 case AArch64::FSUBv2f32:
8250 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
8251 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
8252
8253 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
8254 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
8255 break;
8256 case AArch64::FSUBv2f64:
8257 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
8258 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
8259
8260 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
8261 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
8262 break;
8263 case AArch64::FSUBv4f32:
8264 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
8265 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
8266
8267 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
8268 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
8269 break;
8270 }
8271 return Found;
8272}
8273
8274static bool getFMULPatterns(MachineInstr &Root,
8275 SmallVectorImpl<unsigned> &Patterns) {
8276 MachineBasicBlock &MBB = *Root.getParent();
8277 bool Found = false;
8278
8279 auto Match = [&](unsigned Opcode, int Operand, unsigned Pattern) -> bool {
8280 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8281 MachineOperand &MO = Root.getOperand(i: Operand);
8282 MachineInstr *MI = nullptr;
8283 if (MO.isReg() && MO.getReg().isVirtual())
8284 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8285 // Ignore No-op COPYs in FMUL(COPY(DUP(..)))
8286 if (MI && MI->getOpcode() == TargetOpcode::COPY &&
8287 MI->getOperand(i: 1).getReg().isVirtual())
8288 MI = MRI.getUniqueVRegDef(Reg: MI->getOperand(i: 1).getReg());
8289 if (MI && MI->getOpcode() == Opcode) {
8290 Patterns.push_back(Elt: Pattern);
8291 return true;
8292 }
8293 return false;
8294 };
8295
8296 typedef AArch64MachineCombinerPattern MCP;
8297
8298 switch (Root.getOpcode()) {
8299 default:
8300 return false;
8301 case AArch64::FMULv2f32:
8302 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
8303 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
8304 break;
8305 case AArch64::FMULv2f64:
8306 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
8307 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
8308 break;
8309 case AArch64::FMULv4f16:
8310 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
8311 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
8312 break;
8313 case AArch64::FMULv4f32:
8314 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
8315 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
8316 break;
8317 case AArch64::FMULv8f16:
8318 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
8319 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
8320 break;
8321 }
8322
8323 return Found;
8324}
8325
8326static bool getFNEGPatterns(MachineInstr &Root,
8327 SmallVectorImpl<unsigned> &Patterns) {
8328 unsigned Opc = Root.getOpcode();
8329 MachineBasicBlock &MBB = *Root.getParent();
8330 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8331
8332 auto Match = [&](unsigned Opcode, unsigned Pattern) -> bool {
8333 MachineOperand &MO = Root.getOperand(i: 1);
8334 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8335 if (MI != nullptr && (MI->getOpcode() == Opcode) &&
8336 MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()) &&
8337 Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8338 Root.getFlag(Flag: MachineInstr::MIFlag::FmNsz) &&
8339 MI->getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8340 MI->getFlag(Flag: MachineInstr::MIFlag::FmNsz)) {
8341 Patterns.push_back(Elt: Pattern);
8342 return true;
8343 }
8344 return false;
8345 };
8346
8347 switch (Opc) {
8348 default:
8349 break;
8350 case AArch64::FNEGDr:
8351 return Match(AArch64::FMADDDrrr, AArch64MachineCombinerPattern::FNMADD);
8352 case AArch64::FNEGSr:
8353 return Match(AArch64::FMADDSrrr, AArch64MachineCombinerPattern::FNMADD);
8354 }
8355
8356 return false;
8357}
8358
8359/// Return true when a code sequence can improve throughput. It
8360/// should be called only for instructions in loops.
8361/// \param Pattern - combiner pattern
8362bool AArch64InstrInfo::isThroughputPattern(unsigned Pattern) const {
8363 switch (Pattern) {
8364 default:
8365 break;
8366 case AArch64MachineCombinerPattern::FMULADDH_OP1:
8367 case AArch64MachineCombinerPattern::FMULADDH_OP2:
8368 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
8369 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
8370 case AArch64MachineCombinerPattern::FMULADDS_OP1:
8371 case AArch64MachineCombinerPattern::FMULADDS_OP2:
8372 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
8373 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
8374 case AArch64MachineCombinerPattern::FMULADDD_OP1:
8375 case AArch64MachineCombinerPattern::FMULADDD_OP2:
8376 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
8377 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
8378 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
8379 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
8380 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
8381 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
8382 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
8383 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
8384 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
8385 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
8386 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
8387 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
8388 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
8389 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
8390 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
8391 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
8392 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
8393 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
8394 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
8395 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
8396 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
8397 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
8398 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
8399 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
8400 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
8401 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
8402 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
8403 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
8404 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
8405 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1:
8406 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
8407 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1:
8408 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
8409 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
8410 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
8411 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
8412 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
8413 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
8414 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
8415 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
8416 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
8417 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
8418 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
8419 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
8420 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
8421 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
8422 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2:
8423 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
8424 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2:
8425 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
8426 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2:
8427 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
8428 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2:
8429 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
8430 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2:
8431 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
8432 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
8433 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
8434 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
8435 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
8436 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
8437 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
8438 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
8439 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
8440 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
8441 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
8442 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
8443 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
8444 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
8445 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
8446 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
8447 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
8448 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
8449 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
8450 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
8451 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
8452 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
8453 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
8454 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
8455 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
8456 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
8457 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
8458 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
8459 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
8460 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
8461 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
8462 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
8463 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
8464 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
8465 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
8466 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
8467 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
8468 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
8469 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
8470 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
8471 return true;
8472 } // end switch (Pattern)
8473 return false;
8474}
8475
8476/// Find other MI combine patterns.
8477static bool getMiscPatterns(MachineInstr &Root,
8478 SmallVectorImpl<unsigned> &Patterns) {
8479 // A - (B + C) ==> (A - B) - C or (A - C) - B
8480 unsigned Opc = Root.getOpcode();
8481 MachineBasicBlock &MBB = *Root.getParent();
8482
8483 switch (Opc) {
8484 case AArch64::SUBWrr:
8485 case AArch64::SUBSWrr:
8486 case AArch64::SUBXrr:
8487 case AArch64::SUBSXrr:
8488 // Found candidate root.
8489 break;
8490 default:
8491 return false;
8492 }
8493
8494 if (isCombineInstrSettingFlag(Opc) &&
8495 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) ==
8496 -1)
8497 return false;
8498
8499 if (canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDWrr) ||
8500 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSWrr) ||
8501 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDXrr) ||
8502 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSXrr)) {
8503 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP1);
8504 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP2);
8505 return true;
8506 }
8507
8508 return false;
8509}
8510
8511/// Check if the given instruction forms a gather load pattern that can be
8512/// optimized for better Memory-Level Parallelism (MLP). This function
8513/// identifies chains of NEON lane load instructions that load data from
8514/// different memory addresses into individual lanes of a 128-bit vector
8515/// register, then attempts to split the pattern into parallel loads to break
8516/// the serial dependency between instructions.
8517///
8518/// Pattern Matched:
8519/// Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) ->
8520/// LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)
8521///
8522/// Transformed Into:
8523/// Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64
8524/// to combine the results, enabling better memory-level parallelism.
8525///
8526/// Supported Element Types:
8527/// - 32-bit elements (LD1i32, 4 lanes total)
8528/// - 16-bit elements (LD1i16, 8 lanes total)
8529/// - 8-bit elements (LD1i8, 16 lanes total)
8530static bool getGatherLanePattern(MachineInstr &Root,
8531 SmallVectorImpl<unsigned> &Patterns,
8532 unsigned LoadLaneOpCode, unsigned NumLanes) {
8533 const MachineFunction *MF = Root.getMF();
8534
8535 // Early exit if optimizing for size.
8536 if (MF->getFunction().hasMinSize())
8537 return false;
8538
8539 const MachineRegisterInfo &MRI = MF->getRegInfo();
8540 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
8541
8542 // The root of the pattern must load into the last lane of the vector.
8543 if (Root.getOperand(i: 2).getImm() != NumLanes - 1)
8544 return false;
8545
8546 // Check that we have load into all lanes except lane 0.
8547 // For each load we also want to check that:
8548 // 1. It has a single non-debug use (since we will be replacing the virtual
8549 // register)
8550 // 2. That the addressing mode only uses a single pointer operand
8551 auto *CurrInstr = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8552 auto Range = llvm::seq<unsigned>(Begin: 1, End: NumLanes - 1);
8553 SmallSet<unsigned, 16> RemainingLanes(Range.begin(), Range.end());
8554 SmallVector<const MachineInstr *, 16> LoadInstrs;
8555 while (!RemainingLanes.empty() && CurrInstr &&
8556 CurrInstr->getOpcode() == LoadLaneOpCode &&
8557 MRI.hasOneNonDBGUse(RegNo: CurrInstr->getOperand(i: 0).getReg()) &&
8558 CurrInstr->getNumOperands() == 4) {
8559 RemainingLanes.erase(V: CurrInstr->getOperand(i: 2).getImm());
8560 LoadInstrs.push_back(Elt: CurrInstr);
8561 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8562 }
8563
8564 // Check that we have found a match for lanes N-1.. 1.
8565 if (!RemainingLanes.empty())
8566 return false;
8567
8568 // Match the SUBREG_TO_REG sequence.
8569 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
8570 return false;
8571
8572 // Verify that the subreg to reg loads an integer into the first lane.
8573 auto Lane0LoadReg = CurrInstr->getOperand(i: 1).getReg();
8574 unsigned SingleLaneSizeInBits = 128 / NumLanes;
8575 if (TRI->getRegSizeInBits(Reg: Lane0LoadReg, MRI) != SingleLaneSizeInBits)
8576 return false;
8577
8578 // Verify that it also has a single non debug use.
8579 if (!MRI.hasOneNonDBGUse(RegNo: Lane0LoadReg))
8580 return false;
8581
8582 LoadInstrs.push_back(Elt: MRI.getUniqueVRegDef(Reg: Lane0LoadReg));
8583
8584 // If there is any chance of aliasing, do not apply the pattern.
8585 // Walk backward through the MBB starting from Root.
8586 // Exit early if we've encountered all load instructions or hit the search
8587 // limit.
8588 auto MBBItr = Root.getIterator();
8589 unsigned RemainingSteps =
8590 MF->getSubtarget<AArch64Subtarget>().getCLOpts().search_limit;
8591 SmallPtrSet<const MachineInstr *, 16> RemainingLoadInstrs;
8592 RemainingLoadInstrs.insert(I: LoadInstrs.begin(), E: LoadInstrs.end());
8593 const MachineBasicBlock *MBB = Root.getParent();
8594
8595 for (; MBBItr != MBB->begin() && RemainingSteps > 0 &&
8596 !RemainingLoadInstrs.empty();
8597 --MBBItr, --RemainingSteps) {
8598 const MachineInstr &CurrInstr = *MBBItr;
8599
8600 // Remove this instruction from remaining loads if it's one we're tracking.
8601 RemainingLoadInstrs.erase(Ptr: &CurrInstr);
8602
8603 // Check for potential aliasing with any of the load instructions to
8604 // optimize.
8605 if (CurrInstr.isLoadFoldBarrier())
8606 return false;
8607 }
8608
8609 // If we hit the search limit without finding all load instructions,
8610 // don't match the pattern.
8611 if (RemainingSteps == 0 && !RemainingLoadInstrs.empty())
8612 return false;
8613
8614 switch (NumLanes) {
8615 case 4:
8616 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i32);
8617 break;
8618 case 8:
8619 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i16);
8620 break;
8621 case 16:
8622 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i8);
8623 break;
8624 default:
8625 llvm_unreachable("Got bad number of lanes for gather pattern.");
8626 }
8627
8628 return true;
8629}
8630
8631/// Search for patterns of LD instructions we can optimize.
8632static bool getLoadPatterns(MachineInstr &Root,
8633 SmallVectorImpl<unsigned> &Patterns) {
8634
8635 // The pattern searches for loads into single lanes.
8636 switch (Root.getOpcode()) {
8637 case AArch64::LD1i32:
8638 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 4);
8639 case AArch64::LD1i16:
8640 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 8);
8641 case AArch64::LD1i8:
8642 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 16);
8643 default:
8644 return false;
8645 }
8646}
8647
8648/// Generate optimized instruction sequence for gather load patterns to improve
8649/// Memory-Level Parallelism (MLP). This function transforms a chain of
8650/// sequential NEON lane loads into parallel vector loads that can execute
8651/// concurrently.
8652static void
8653generateGatherLanePattern(MachineInstr &Root,
8654 SmallVectorImpl<MachineInstr *> &InsInstrs,
8655 SmallVectorImpl<MachineInstr *> &DelInstrs,
8656 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8657 unsigned Pattern, unsigned NumLanes) {
8658 MachineFunction &MF = *Root.getParent()->getParent();
8659 MachineRegisterInfo &MRI = MF.getRegInfo();
8660 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8661
8662 // Gather the initial load instructions to build the pattern.
8663 SmallVector<MachineInstr *, 16> LoadToLaneInstrs;
8664 MachineInstr *CurrInstr = &Root;
8665 for (unsigned i = 0; i < NumLanes - 1; ++i) {
8666 LoadToLaneInstrs.push_back(Elt: CurrInstr);
8667 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8668 }
8669
8670 // Sort the load instructions according to the lane.
8671 llvm::sort(C&: LoadToLaneInstrs,
8672 Comp: [](const MachineInstr *A, const MachineInstr *B) {
8673 return A->getOperand(i: 2).getImm() > B->getOperand(i: 2).getImm();
8674 });
8675
8676 MachineInstr *SubregToReg = CurrInstr;
8677 LoadToLaneInstrs.push_back(
8678 Elt: MRI.getUniqueVRegDef(Reg: SubregToReg->getOperand(i: 1).getReg()));
8679 auto LoadToLaneInstrsAscending = llvm::reverse(C&: LoadToLaneInstrs);
8680
8681 const TargetRegisterClass *FPR128RegClass =
8682 MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg());
8683
8684 // Helper lambda to create a LD1 instruction.
8685 auto CreateLD1Instruction = [&](MachineInstr *OriginalInstr,
8686 Register SrcRegister, unsigned Lane,
8687 Register OffsetRegister) {
8688 auto NewRegister = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8689 MachineInstrBuilder LoadIndexIntoRegister =
8690 BuildMI(MF, MIMD: MIMetadata(*OriginalInstr), MCID: TII->get(Opcode: Root.getOpcode()),
8691 DestReg: NewRegister)
8692 .addReg(RegNo: SrcRegister)
8693 .addImm(Val: Lane)
8694 .addReg(RegNo: OffsetRegister)
8695 .setMemRefs(OriginalInstr->memoperands());
8696 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewRegister, y: InsInstrs.size()));
8697 InsInstrs.push_back(Elt: LoadIndexIntoRegister);
8698 return NewRegister;
8699 };
8700
8701 // Helper to create load instruction based on the NumLanes in the NEON
8702 // register we are rewriting.
8703 auto CreateLDRInstruction =
8704 [&](unsigned NumLanes, Register DestReg, Register OffsetReg,
8705 ArrayRef<MachineMemOperand *> MMOs) -> MachineInstrBuilder {
8706 unsigned Opcode;
8707 switch (NumLanes) {
8708 case 4:
8709 Opcode = AArch64::LDRSui;
8710 break;
8711 case 8:
8712 Opcode = AArch64::LDRHui;
8713 break;
8714 case 16:
8715 Opcode = AArch64::LDRBui;
8716 break;
8717 default:
8718 llvm_unreachable(
8719 "Got unsupported number of lanes in machine-combiner gather pattern");
8720 }
8721 // Immediate offset load
8722 return BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg)
8723 .addReg(RegNo: OffsetReg)
8724 .addImm(Val: 0)
8725 .setMemRefs(MMOs);
8726 };
8727
8728 // Load the remaining lanes into register 0.
8729 auto LanesToLoadToReg0 =
8730 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + 1,
8731 y: LoadToLaneInstrsAscending.begin() + NumLanes / 2);
8732 Register PrevReg = SubregToReg->getOperand(i: 0).getReg();
8733 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg0)) {
8734 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8735 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8736 OffsetRegOperand.getReg());
8737 DelInstrs.push_back(Elt: LoadInstr);
8738 }
8739 Register LastLoadReg0 = PrevReg;
8740
8741 // First load into register 1. Perform an integer load to zero out the upper
8742 // lanes in a single instruction.
8743 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
8744 MachineInstr *OriginalSplitLoad =
8745 *std::next(x: LoadToLaneInstrsAscending.begin(), n: NumLanes / 2);
8746 Register DestRegForMiddleIndex = MRI.createVirtualRegister(
8747 RegClass: MRI.getRegClass(Reg: Lane0Load->getOperand(i: 0).getReg()));
8748
8749 const MachineOperand &OriginalSplitToLoadOffsetOperand =
8750 OriginalSplitLoad->getOperand(i: 3);
8751 MachineInstrBuilder MiddleIndexLoadInstr =
8752 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
8753 OriginalSplitToLoadOffsetOperand.getReg(),
8754 OriginalSplitLoad->memoperands());
8755
8756 InstrIdxForVirtReg.insert(
8757 KV: std::make_pair(x&: DestRegForMiddleIndex, y: InsInstrs.size()));
8758 InsInstrs.push_back(Elt: MiddleIndexLoadInstr);
8759 DelInstrs.push_back(Elt: OriginalSplitLoad);
8760
8761 // Subreg To Reg instruction for register 1.
8762 Register DestRegForSubregToReg = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8763 unsigned SubregType;
8764 switch (NumLanes) {
8765 case 4:
8766 SubregType = AArch64::ssub;
8767 break;
8768 case 8:
8769 SubregType = AArch64::hsub;
8770 break;
8771 case 16:
8772 SubregType = AArch64::bsub;
8773 break;
8774 default:
8775 llvm_unreachable(
8776 "Got invalid NumLanes for machine-combiner gather pattern");
8777 }
8778
8779 auto SubRegToRegInstr =
8780 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubregToReg->getOpcode()),
8781 DestReg: DestRegForSubregToReg)
8782 .addReg(RegNo: DestRegForMiddleIndex)
8783 .addImm(Val: SubregType);
8784 InstrIdxForVirtReg.insert(
8785 KV: std::make_pair(x&: DestRegForSubregToReg, y: InsInstrs.size()));
8786 InsInstrs.push_back(Elt: SubRegToRegInstr);
8787
8788 // Load remaining lanes into register 1.
8789 auto LanesToLoadToReg1 =
8790 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + NumLanes / 2 + 1,
8791 y: LoadToLaneInstrsAscending.end());
8792 PrevReg = SubRegToRegInstr->getOperand(i: 0).getReg();
8793 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg1)) {
8794 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8795 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8796 OffsetRegOperand.getReg());
8797
8798 // Do not add the last reg to DelInstrs - it will be removed later.
8799 if (Index == NumLanes / 2 - 2) {
8800 break;
8801 }
8802 DelInstrs.push_back(Elt: LoadInstr);
8803 }
8804 Register LastLoadReg1 = PrevReg;
8805
8806 // Create the final zip instruction to combine the results.
8807 MachineInstrBuilder ZipInstr =
8808 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::ZIP1v2i64),
8809 DestReg: Root.getOperand(i: 0).getReg())
8810 .addReg(RegNo: LastLoadReg0)
8811 .addReg(RegNo: LastLoadReg1);
8812 InsInstrs.push_back(Elt: ZipInstr);
8813}
8814
8815CombinerObjective
8816AArch64InstrInfo::getCombinerObjective(unsigned Pattern) const {
8817 switch (Pattern) {
8818 case AArch64MachineCombinerPattern::SUBADD_OP1:
8819 case AArch64MachineCombinerPattern::SUBADD_OP2:
8820 case AArch64MachineCombinerPattern::GATHER_LANE_i32:
8821 case AArch64MachineCombinerPattern::GATHER_LANE_i16:
8822 case AArch64MachineCombinerPattern::GATHER_LANE_i8:
8823 return CombinerObjective::MustReduceDepth;
8824 default:
8825 return TargetInstrInfo::getCombinerObjective(Pattern);
8826 }
8827}
8828
8829/// Return true when there is potentially a faster code sequence for an
8830/// instruction chain ending in \p Root. All potential patterns are listed in
8831/// the \p Pattern vector. Pattern should be sorted in priority order since the
8832/// pattern evaluator stops checking as soon as it finds a faster sequence.
8833
8834bool AArch64InstrInfo::getMachineCombinerPatterns(
8835 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
8836 bool DoRegPressureReduce) const {
8837 // Integer patterns
8838 if (getMaddPatterns(Root, Patterns))
8839 return true;
8840 // Floating point patterns
8841 if (getFMULPatterns(Root, Patterns))
8842 return true;
8843 if (getFMAPatterns(Root, Patterns))
8844 return true;
8845 if (getFNEGPatterns(Root, Patterns))
8846 return true;
8847
8848 // Other patterns
8849 if (getMiscPatterns(Root, Patterns))
8850 return true;
8851
8852 // Load patterns
8853 if (getLoadPatterns(Root, Patterns))
8854 return true;
8855
8856 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
8857 DoRegPressureReduce);
8858}
8859
8860enum class FMAInstKind { Default, Indexed, Accumulator };
8861/// genFusedMultiply - Generate fused multiply instructions.
8862/// This function supports both integer and floating point instructions.
8863/// A typical example:
8864/// F|MUL I=A,B,0
8865/// F|ADD R,I,C
8866/// ==> F|MADD R,A,B,C
8867/// \param MF Containing MachineFunction
8868/// \param MRI Register information
8869/// \param TII Target information
8870/// \param Root is the F|ADD instruction
8871/// \param [out] InsInstrs is a vector of machine instructions and will
8872/// contain the generated madd instruction
8873/// \param IdxMulOpd is index of operand in Root that is the result of
8874/// the F|MUL. In the example above IdxMulOpd is 1.
8875/// \param MaddOpc the opcode fo the f|madd instruction
8876/// \param RC Register class of operands
8877/// \param kind of fma instruction (addressing mode) to be generated
8878/// \param ReplacedAddend is the result register from the instruction
8879/// replacing the non-combined operand, if any.
8880static MachineInstr *
8881genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
8882 const TargetInstrInfo *TII, MachineInstr &Root,
8883 SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
8884 unsigned MaddOpc, const TargetRegisterClass *RC,
8885 FMAInstKind kind = FMAInstKind::Default,
8886 const Register *ReplacedAddend = nullptr) {
8887 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8888
8889 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
8890 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
8891 Register ResultReg = Root.getOperand(i: 0).getReg();
8892 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8893 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8894
8895 Register SrcReg2 =
8896 ReplacedAddend ? *ReplacedAddend : Root.getOperand(i: IdxOtherOpd).getReg();
8897
8898 if (ResultReg.isVirtual())
8899 MRI.constrainRegClass(Reg: ResultReg, RC);
8900 if (SrcReg0.isVirtual())
8901 MRI.constrainRegClass(Reg: SrcReg0, RC);
8902 if (SrcReg1.isVirtual())
8903 MRI.constrainRegClass(Reg: SrcReg1, RC);
8904 if (SrcReg2.isVirtual())
8905 MRI.constrainRegClass(Reg: SrcReg2, RC);
8906
8907 MachineInstrBuilder MIB;
8908 if (kind == FMAInstKind::Default)
8909 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8910 .addReg(RegNo: SrcReg0)
8911 .addReg(RegNo: SrcReg1)
8912 .addReg(RegNo: SrcReg2);
8913 else if (kind == FMAInstKind::Indexed)
8914 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8915 .addReg(RegNo: SrcReg2)
8916 .addReg(RegNo: SrcReg0)
8917 .addReg(RegNo: SrcReg1)
8918 .addImm(Val: MUL->getOperand(i: 3).getImm());
8919 else if (kind == FMAInstKind::Accumulator)
8920 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8921 .addReg(RegNo: SrcReg2)
8922 .addReg(RegNo: SrcReg0)
8923 .addReg(RegNo: SrcReg1);
8924 else
8925 assert(false && "Invalid FMA instruction kind \n");
8926 // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
8927 InsInstrs.push_back(Elt: MIB);
8928 return MUL;
8929}
8930
8931/// MUL I=A,B,0
8932/// ADDrs R,I,C,shift
8933/// ==>
8934/// SBFM/UBFM J=C,shift
8935/// MADD R,A,B,J
8936static void genMaddShf(MachineFunction &MF, MachineRegisterInfo &MRI,
8937 const TargetInstrInfo *TII, MachineInstr &Root,
8938 SmallVectorImpl<MachineInstr *> &InsInstrs,
8939 SmallVectorImpl<MachineInstr *> &DelInstrs,
8940 unsigned MaddOpc, bool Is64Bit,
8941 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
8942 const TargetRegisterClass *RC =
8943 Is64Bit ? &AArch64::GPR64RegClass : &AArch64::GPR32RegClass;
8944 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8945 Register ResultReg = Root.getOperand(i: 0).getReg();
8946 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8947 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8948 Register ShiftedReg = Root.getOperand(i: 2).getReg();
8949
8950 if (ResultReg.isVirtual())
8951 MRI.constrainRegClass(Reg: ResultReg, RC);
8952 if (SrcReg0.isVirtual())
8953 MRI.constrainRegClass(Reg: SrcReg0, RC);
8954 if (SrcReg1.isVirtual())
8955 MRI.constrainRegClass(Reg: SrcReg1, RC);
8956 if (ShiftedReg.isVirtual())
8957 MRI.constrainRegClass(Reg: ShiftedReg, RC);
8958
8959 unsigned ShiftImm = Root.getOperand(i: 3).getImm();
8960 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm: ShiftImm);
8961 unsigned RegSize = Is64Bit ? 64 : 32;
8962 unsigned BFMOpc;
8963 unsigned ImmR, ImmS;
8964 switch (AArch64_AM::getShiftType(Imm: ShiftImm)) {
8965 case AArch64_AM::ASR:
8966 BFMOpc = Is64Bit ? AArch64::SBFMXri : AArch64::SBFMWri;
8967 ImmR = ShiftVal;
8968 ImmS = RegSize - 1;
8969 break;
8970 case AArch64_AM::LSR:
8971 BFMOpc = Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri;
8972 ImmR = ShiftVal;
8973 ImmS = RegSize - 1;
8974 break;
8975 case AArch64_AM::LSL:
8976 BFMOpc = Is64Bit ? AArch64::UBFMXri : AArch64::UBFMWri;
8977 ImmR = (RegSize - ShiftVal) % RegSize;
8978 ImmS = RegSize - 1 - ShiftVal;
8979 break;
8980 default:
8981 llvm_unreachable("Unexpected shift type");
8982 }
8983
8984 Register ShiftReg = MRI.createVirtualRegister(RegClass: RC);
8985 MachineInstrBuilder MIB1 =
8986 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: BFMOpc), DestReg: ShiftReg)
8987 .addReg(RegNo: ShiftedReg)
8988 .addImm(Val: ImmR)
8989 .addImm(Val: ImmS);
8990 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: ShiftReg, y: InsInstrs.size()));
8991 InsInstrs.push_back(Elt: MIB1);
8992
8993 uint32_t Flags = Root.getFlags() & MUL->getFlags() &
8994 (MachineInstr::NoSWrap | MachineInstr::NoUWrap);
8995
8996 MachineInstrBuilder MIB2 =
8997 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8998 .addReg(RegNo: SrcReg0)
8999 .addReg(RegNo: SrcReg1)
9000 .addReg(RegNo: ShiftReg)
9001 .setMIFlags(Flags);
9002
9003 InsInstrs.push_back(Elt: MIB2);
9004 DelInstrs.push_back(Elt: MUL);
9005 DelInstrs.push_back(Elt: &Root);
9006}
9007
9008static MachineInstr *
9009genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI,
9010 const TargetInstrInfo *TII, MachineInstr &Root,
9011 SmallVectorImpl<MachineInstr *> &InsInstrs) {
9012 MachineInstr *MAD = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
9013
9014 unsigned Opc = 0;
9015 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MAD->getOperand(i: 0).getReg());
9016 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
9017 Opc = AArch64::FNMADDSrrr;
9018 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
9019 Opc = AArch64::FNMADDDrrr;
9020 else
9021 return nullptr;
9022
9023 Register ResultReg = Root.getOperand(i: 0).getReg();
9024 Register SrcReg0 = MAD->getOperand(i: 1).getReg();
9025 Register SrcReg1 = MAD->getOperand(i: 2).getReg();
9026 Register SrcReg2 = MAD->getOperand(i: 3).getReg();
9027 if (ResultReg.isVirtual())
9028 MRI.constrainRegClass(Reg: ResultReg, RC);
9029 if (SrcReg0.isVirtual())
9030 MRI.constrainRegClass(Reg: SrcReg0, RC);
9031 if (SrcReg1.isVirtual())
9032 MRI.constrainRegClass(Reg: SrcReg1, RC);
9033 if (SrcReg2.isVirtual())
9034 MRI.constrainRegClass(Reg: SrcReg2, RC);
9035
9036 MachineInstrBuilder MIB =
9037 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Opc), DestReg: ResultReg)
9038 .addReg(RegNo: SrcReg0)
9039 .addReg(RegNo: SrcReg1)
9040 .addReg(RegNo: SrcReg2);
9041 InsInstrs.push_back(Elt: MIB);
9042
9043 return MAD;
9044}
9045
9046/// Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
9047static MachineInstr *
9048genIndexedMultiply(MachineInstr &Root,
9049 SmallVectorImpl<MachineInstr *> &InsInstrs,
9050 unsigned IdxDupOp, unsigned MulOpc,
9051 const TargetRegisterClass *RC, MachineRegisterInfo &MRI) {
9052 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
9053 "Invalid index of FMUL operand");
9054
9055 MachineFunction &MF = *Root.getMF();
9056 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
9057
9058 MachineInstr *Dup =
9059 MF.getRegInfo().getUniqueVRegDef(Reg: Root.getOperand(i: IdxDupOp).getReg());
9060
9061 if (Dup->getOpcode() == TargetOpcode::COPY)
9062 Dup = MRI.getUniqueVRegDef(Reg: Dup->getOperand(i: 1).getReg());
9063
9064 Register DupSrcReg = Dup->getOperand(i: 1).getReg();
9065 MRI.constrainRegClass(Reg: DupSrcReg, RC);
9066
9067 unsigned DupSrcLane = Dup->getOperand(i: 2).getImm();
9068
9069 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
9070 MachineOperand &MulOp = Root.getOperand(i: IdxMulOp);
9071
9072 Register ResultReg = Root.getOperand(i: 0).getReg();
9073
9074 MachineInstrBuilder MIB;
9075 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MulOpc), DestReg: ResultReg)
9076 .add(MO: MulOp)
9077 .addReg(RegNo: DupSrcReg)
9078 .addImm(Val: DupSrcLane);
9079
9080 InsInstrs.push_back(Elt: MIB);
9081 return &Root;
9082}
9083
9084/// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
9085/// instructions.
9086///
9087/// \see genFusedMultiply
9088static MachineInstr *genFusedMultiplyAcc(
9089 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
9090 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
9091 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
9092 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
9093 kind: FMAInstKind::Accumulator);
9094}
9095
9096/// genNeg - Helper to generate an intermediate negation of the second operand
9097/// of Root
9098static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
9099 const TargetInstrInfo *TII, MachineInstr &Root,
9100 SmallVectorImpl<MachineInstr *> &InsInstrs,
9101 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
9102 unsigned MnegOpc, const TargetRegisterClass *RC) {
9103 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9104 MachineInstrBuilder MIB =
9105 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MnegOpc), DestReg: NewVR)
9106 .add(MO: Root.getOperand(i: 2));
9107 InsInstrs.push_back(Elt: MIB);
9108
9109 assert(InstrIdxForVirtReg.empty());
9110 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9111
9112 return NewVR;
9113}
9114
9115/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
9116/// instructions with an additional negation of the accumulator
9117static MachineInstr *genFusedMultiplyAccNeg(
9118 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
9119 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
9120 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
9121 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
9122 assert(IdxMulOpd == 1);
9123
9124 Register NewVR =
9125 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
9126 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
9127 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9128}
9129
9130/// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
9131/// instructions.
9132///
9133/// \see genFusedMultiply
9134static MachineInstr *genFusedMultiplyIdx(
9135 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
9136 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
9137 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
9138 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
9139 kind: FMAInstKind::Indexed);
9140}
9141
9142/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
9143/// instructions with an additional negation of the accumulator
9144static MachineInstr *genFusedMultiplyIdxNeg(
9145 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
9146 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
9147 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
9148 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
9149 assert(IdxMulOpd == 1);
9150
9151 Register NewVR =
9152 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
9153
9154 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
9155 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9156}
9157
9158/// genMaddR - Generate madd instruction and combine mul and add using
9159/// an extra virtual register
9160/// Example - an ADD intermediate needs to be stored in a register:
9161/// MUL I=A,B,0
9162/// ADD R,I,Imm
9163/// ==> ORR V, ZR, Imm
9164/// ==> MADD R,A,B,V
9165/// \param MF Containing MachineFunction
9166/// \param MRI Register information
9167/// \param TII Target information
9168/// \param Root is the ADD instruction
9169/// \param [out] InsInstrs is a vector of machine instructions and will
9170/// contain the generated madd instruction
9171/// \param IdxMulOpd is index of operand in Root that is the result of
9172/// the MUL. In the example above IdxMulOpd is 1.
9173/// \param MaddOpc the opcode fo the madd instruction
9174/// \param VR is a virtual register that holds the value of an ADD operand
9175/// (V in the example above).
9176/// \param RC Register class of operands
9177static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
9178 const TargetInstrInfo *TII, MachineInstr &Root,
9179 SmallVectorImpl<MachineInstr *> &InsInstrs,
9180 unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
9181 const TargetRegisterClass *RC) {
9182 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
9183
9184 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
9185 Register ResultReg = Root.getOperand(i: 0).getReg();
9186 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
9187 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
9188
9189 if (ResultReg.isVirtual())
9190 MRI.constrainRegClass(Reg: ResultReg, RC);
9191 if (SrcReg0.isVirtual())
9192 MRI.constrainRegClass(Reg: SrcReg0, RC);
9193 if (SrcReg1.isVirtual())
9194 MRI.constrainRegClass(Reg: SrcReg1, RC);
9195 if (Register::isVirtualRegister(Reg: VR))
9196 MRI.constrainRegClass(Reg: VR, RC);
9197
9198 MachineInstrBuilder MIB =
9199 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
9200 .addReg(RegNo: SrcReg0)
9201 .addReg(RegNo: SrcReg1)
9202 .addReg(RegNo: VR);
9203 // Insert the MADD
9204 InsInstrs.push_back(Elt: MIB);
9205 return MUL;
9206}
9207
9208/// Do the following transformation
9209/// A - (B + C) ==> (A - B) - C
9210/// A - (B + C) ==> (A - C) - B
9211static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI,
9212 const TargetInstrInfo *TII, MachineInstr &Root,
9213 SmallVectorImpl<MachineInstr *> &InsInstrs,
9214 SmallVectorImpl<MachineInstr *> &DelInstrs,
9215 unsigned IdxOpd1,
9216 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
9217 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
9218 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
9219 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
9220
9221 Register ResultReg = Root.getOperand(i: 0).getReg();
9222 Register RegA = Root.getOperand(i: 1).getReg();
9223 Register RegB = AddMI->getOperand(i: IdxOpd1).getReg();
9224 Register RegC = AddMI->getOperand(i: IdxOtherOpd).getReg();
9225 Register NewVR =
9226 MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 2).getReg()));
9227
9228 unsigned Opcode = Root.getOpcode();
9229 if (Opcode == AArch64::SUBSWrr)
9230 Opcode = AArch64::SUBWrr;
9231 else if (Opcode == AArch64::SUBSXrr)
9232 Opcode = AArch64::SUBXrr;
9233 else
9234 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
9235 "Unexpected instruction opcode.");
9236
9237 uint32_t Flags = Root.mergeFlagsWith(Other: *AddMI);
9238 Flags &= ~MachineInstr::NoSWrap;
9239 Flags &= ~MachineInstr::NoUWrap;
9240
9241 MachineInstrBuilder MIB1 =
9242 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: NewVR)
9243 .addReg(RegNo: RegA)
9244 .addReg(RegNo: RegB)
9245 .setMIFlags(Flags);
9246 MachineInstrBuilder MIB2 =
9247 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: ResultReg)
9248 .addReg(RegNo: NewVR)
9249 .addReg(RegNo: RegC)
9250 .setMIFlags(Flags);
9251
9252 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9253 InsInstrs.push_back(Elt: MIB1);
9254 InsInstrs.push_back(Elt: MIB2);
9255 DelInstrs.push_back(Elt: AddMI);
9256 DelInstrs.push_back(Elt: &Root);
9257}
9258
9259unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
9260 unsigned int AccumulatorOpCode) const {
9261 switch (AccumulatorOpCode) {
9262 case AArch64::UABALB_ZZZ_D:
9263 case AArch64::SABALB_ZZZ_D:
9264 case AArch64::UABALT_ZZZ_D:
9265 case AArch64::SABALT_ZZZ_D:
9266 return AArch64::ADD_ZZZ_D;
9267 case AArch64::UABALB_ZZZ_H:
9268 case AArch64::SABALB_ZZZ_H:
9269 case AArch64::UABALT_ZZZ_H:
9270 case AArch64::SABALT_ZZZ_H:
9271 return AArch64::ADD_ZZZ_H;
9272 case AArch64::UABALB_ZZZ_S:
9273 case AArch64::SABALB_ZZZ_S:
9274 case AArch64::UABALT_ZZZ_S:
9275 case AArch64::SABALT_ZZZ_S:
9276 return AArch64::ADD_ZZZ_S;
9277 case AArch64::UABALv16i8_v8i16:
9278 case AArch64::SABALv8i8_v8i16:
9279 case AArch64::SABAv8i16:
9280 case AArch64::UABAv8i16:
9281 return AArch64::ADDv8i16;
9282 case AArch64::SABALv2i32_v2i64:
9283 case AArch64::UABALv2i32_v2i64:
9284 case AArch64::SABALv4i32_v2i64:
9285 return AArch64::ADDv2i64;
9286 case AArch64::UABALv4i16_v4i32:
9287 case AArch64::SABALv4i16_v4i32:
9288 case AArch64::SABALv8i16_v4i32:
9289 case AArch64::SABAv4i32:
9290 case AArch64::UABAv4i32:
9291 return AArch64::ADDv4i32;
9292 case AArch64::UABALv4i32_v2i64:
9293 return AArch64::ADDv2i64;
9294 case AArch64::UABALv8i16_v4i32:
9295 return AArch64::ADDv4i32;
9296 case AArch64::UABALv8i8_v8i16:
9297 case AArch64::SABALv16i8_v8i16:
9298 return AArch64::ADDv8i16;
9299 case AArch64::UABAv16i8:
9300 case AArch64::SABAv16i8:
9301 return AArch64::ADDv16i8;
9302 case AArch64::UABAv4i16:
9303 case AArch64::SABAv4i16:
9304 return AArch64::ADDv4i16;
9305 case AArch64::UABAv2i32:
9306 case AArch64::SABAv2i32:
9307 return AArch64::ADDv2i32;
9308 case AArch64::UABAv8i8:
9309 case AArch64::SABAv8i8:
9310 return AArch64::ADDv8i8;
9311 default:
9312 llvm_unreachable("Unknown accumulator opcode");
9313 }
9314}
9315
9316/// When getMachineCombinerPatterns() finds potential patterns,
9317/// this function generates the instructions that could replace the
9318/// original code sequence
9319void AArch64InstrInfo::genAlternativeCodeSequence(
9320 MachineInstr &Root, unsigned Pattern,
9321 SmallVectorImpl<MachineInstr *> &InsInstrs,
9322 SmallVectorImpl<MachineInstr *> &DelInstrs,
9323 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
9324 MachineBasicBlock &MBB = *Root.getParent();
9325 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9326 MachineFunction &MF = *MBB.getParent();
9327 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
9328
9329 MachineInstr *MUL = nullptr;
9330 const TargetRegisterClass *RC;
9331 unsigned Opc;
9332 switch (Pattern) {
9333 default:
9334 // Reassociate instructions.
9335 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
9336 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
9337 return;
9338 case AArch64MachineCombinerPattern::SUBADD_OP1:
9339 // A - (B + C)
9340 // ==> (A - B) - C
9341 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 1,
9342 InstrIdxForVirtReg);
9343 return;
9344 case AArch64MachineCombinerPattern::SUBADD_OP2:
9345 // A - (B + C)
9346 // ==> (A - C) - B
9347 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 2,
9348 InstrIdxForVirtReg);
9349 return;
9350 case AArch64MachineCombinerPattern::MULADDW_OP1:
9351 case AArch64MachineCombinerPattern::MULADDX_OP1:
9352 // MUL I=A,B,0
9353 // ADD R,I,C
9354 // ==> MADD R,A,B,C
9355 // --- Create(MADD);
9356 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP1) {
9357 Opc = AArch64::MADDWrrr;
9358 RC = &AArch64::GPR32RegClass;
9359 } else {
9360 Opc = AArch64::MADDXrrr;
9361 RC = &AArch64::GPR64RegClass;
9362 }
9363 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9364 break;
9365 case AArch64MachineCombinerPattern::MULADDW_OP2:
9366 case AArch64MachineCombinerPattern::MULADDX_OP2:
9367 // MUL I=A,B,0
9368 // ADD R,C,I
9369 // ==> MADD R,A,B,C
9370 // --- Create(MADD);
9371 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP2) {
9372 Opc = AArch64::MADDWrrr;
9373 RC = &AArch64::GPR32RegClass;
9374 } else {
9375 Opc = AArch64::MADDXrrr;
9376 RC = &AArch64::GPR64RegClass;
9377 }
9378 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9379 break;
9380 case AArch64MachineCombinerPattern::MULADDWI_OP1:
9381 case AArch64MachineCombinerPattern::MULADDXI_OP1:
9382 case AArch64MachineCombinerPattern::MULSUBWI_OP1:
9383 case AArch64MachineCombinerPattern::MULSUBXI_OP1: {
9384 // MUL I=A,B,0
9385 // ADD/SUB R,I,Imm
9386 // ==> MOV V, Imm/-Imm
9387 // ==> MADD R,A,B,V
9388 // --- Create(MADD);
9389 const TargetRegisterClass *RC;
9390 unsigned BitSize, MovImm;
9391 if (Pattern == AArch64MachineCombinerPattern::MULADDWI_OP1 ||
9392 Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1) {
9393 MovImm = AArch64::MOVi32imm;
9394 RC = &AArch64::GPR32spRegClass;
9395 BitSize = 32;
9396 Opc = AArch64::MADDWrrr;
9397 RC = &AArch64::GPR32RegClass;
9398 } else {
9399 MovImm = AArch64::MOVi64imm;
9400 RC = &AArch64::GPR64spRegClass;
9401 BitSize = 64;
9402 Opc = AArch64::MADDXrrr;
9403 RC = &AArch64::GPR64RegClass;
9404 }
9405 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9406 uint64_t Imm = Root.getOperand(i: 2).getImm();
9407
9408 if (Root.getOperand(i: 3).isImm()) {
9409 unsigned Val = Root.getOperand(i: 3).getImm();
9410 Imm = Imm << Val;
9411 }
9412 bool IsSub = Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1 ||
9413 Pattern == AArch64MachineCombinerPattern::MULSUBXI_OP1;
9414 uint64_t UImm = SignExtend64(X: IsSub ? -Imm : Imm, B: BitSize);
9415 // Check that the immediate can be composed via a single instruction.
9416 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
9417 AArch64_IMM::expandMOVImm(Imm: UImm, BitSize, Insn);
9418 if (Insn.size() != 1)
9419 return;
9420 MachineInstrBuilder MIB1 =
9421 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MovImm), DestReg: NewVR)
9422 .addImm(Val: IsSub ? -Imm : Imm);
9423 InsInstrs.push_back(Elt: MIB1);
9424 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9425 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9426 break;
9427 }
9428 case AArch64MachineCombinerPattern::MULSUBW_OP1:
9429 case AArch64MachineCombinerPattern::MULSUBX_OP1: {
9430 // MUL I=A,B,0
9431 // SUB R,I, C
9432 // ==> SUB V, 0, C
9433 // ==> MADD R,A,B,V // = -C + A*B
9434 // --- Create(MADD);
9435 const TargetRegisterClass *SubRC;
9436 unsigned SubOpc, ZeroReg;
9437 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP1) {
9438 SubOpc = AArch64::SUBWrr;
9439 SubRC = &AArch64::GPR32spRegClass;
9440 ZeroReg = AArch64::WZR;
9441 Opc = AArch64::MADDWrrr;
9442 RC = &AArch64::GPR32RegClass;
9443 } else {
9444 SubOpc = AArch64::SUBXrr;
9445 SubRC = &AArch64::GPR64spRegClass;
9446 ZeroReg = AArch64::XZR;
9447 Opc = AArch64::MADDXrrr;
9448 RC = &AArch64::GPR64RegClass;
9449 }
9450 Register NewVR = MRI.createVirtualRegister(RegClass: SubRC);
9451 // SUB NewVR, 0, C
9452 MachineInstrBuilder MIB1 =
9453 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubOpc), DestReg: NewVR)
9454 .addReg(RegNo: ZeroReg)
9455 .add(MO: Root.getOperand(i: 2));
9456 InsInstrs.push_back(Elt: MIB1);
9457 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9458 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9459 break;
9460 }
9461 case AArch64MachineCombinerPattern::MULSUBW_OP2:
9462 case AArch64MachineCombinerPattern::MULSUBX_OP2:
9463 // MUL I=A,B,0
9464 // SUB R,C,I
9465 // ==> MSUB R,A,B,C (computes C - A*B)
9466 // --- Create(MSUB);
9467 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP2) {
9468 Opc = AArch64::MSUBWrrr;
9469 RC = &AArch64::GPR32RegClass;
9470 } else {
9471 Opc = AArch64::MSUBXrrr;
9472 RC = &AArch64::GPR64RegClass;
9473 }
9474 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9475 break;
9476 case AArch64MachineCombinerPattern::MULADDWS_OP1:
9477 genMaddShf(MF, MRI, TII, Root, InsInstrs, DelInstrs, MaddOpc: AArch64::MADDWrrr,
9478 Is64Bit: false, InstrIdxForVirtReg);
9479 return;
9480 case AArch64MachineCombinerPattern::MULADDXS_OP1:
9481 genMaddShf(MF, MRI, TII, Root, InsInstrs, DelInstrs, MaddOpc: AArch64::MADDXrrr,
9482 Is64Bit: true, InstrIdxForVirtReg);
9483 return;
9484 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
9485 Opc = AArch64::MLAv8i8;
9486 RC = &AArch64::FPR64RegClass;
9487 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9488 break;
9489 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
9490 Opc = AArch64::MLAv8i8;
9491 RC = &AArch64::FPR64RegClass;
9492 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9493 break;
9494 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
9495 Opc = AArch64::MLAv16i8;
9496 RC = &AArch64::FPR128RegClass;
9497 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9498 break;
9499 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
9500 Opc = AArch64::MLAv16i8;
9501 RC = &AArch64::FPR128RegClass;
9502 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9503 break;
9504 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
9505 Opc = AArch64::MLAv4i16;
9506 RC = &AArch64::FPR64RegClass;
9507 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9508 break;
9509 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
9510 Opc = AArch64::MLAv4i16;
9511 RC = &AArch64::FPR64RegClass;
9512 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9513 break;
9514 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
9515 Opc = AArch64::MLAv8i16;
9516 RC = &AArch64::FPR128RegClass;
9517 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9518 break;
9519 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
9520 Opc = AArch64::MLAv8i16;
9521 RC = &AArch64::FPR128RegClass;
9522 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9523 break;
9524 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
9525 Opc = AArch64::MLAv2i32;
9526 RC = &AArch64::FPR64RegClass;
9527 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9528 break;
9529 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
9530 Opc = AArch64::MLAv2i32;
9531 RC = &AArch64::FPR64RegClass;
9532 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9533 break;
9534 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
9535 Opc = AArch64::MLAv4i32;
9536 RC = &AArch64::FPR128RegClass;
9537 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9538 break;
9539 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
9540 Opc = AArch64::MLAv4i32;
9541 RC = &AArch64::FPR128RegClass;
9542 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9543 break;
9544
9545 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
9546 Opc = AArch64::MLAv8i8;
9547 RC = &AArch64::FPR64RegClass;
9548 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9549 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i8,
9550 RC);
9551 break;
9552 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
9553 Opc = AArch64::MLSv8i8;
9554 RC = &AArch64::FPR64RegClass;
9555 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9556 break;
9557 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
9558 Opc = AArch64::MLAv16i8;
9559 RC = &AArch64::FPR128RegClass;
9560 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9561 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv16i8,
9562 RC);
9563 break;
9564 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
9565 Opc = AArch64::MLSv16i8;
9566 RC = &AArch64::FPR128RegClass;
9567 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9568 break;
9569 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
9570 Opc = AArch64::MLAv4i16;
9571 RC = &AArch64::FPR64RegClass;
9572 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9573 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9574 RC);
9575 break;
9576 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
9577 Opc = AArch64::MLSv4i16;
9578 RC = &AArch64::FPR64RegClass;
9579 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9580 break;
9581 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
9582 Opc = AArch64::MLAv8i16;
9583 RC = &AArch64::FPR128RegClass;
9584 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9585 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9586 RC);
9587 break;
9588 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
9589 Opc = AArch64::MLSv8i16;
9590 RC = &AArch64::FPR128RegClass;
9591 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9592 break;
9593 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
9594 Opc = AArch64::MLAv2i32;
9595 RC = &AArch64::FPR64RegClass;
9596 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9597 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9598 RC);
9599 break;
9600 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
9601 Opc = AArch64::MLSv2i32;
9602 RC = &AArch64::FPR64RegClass;
9603 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9604 break;
9605 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
9606 Opc = AArch64::MLAv4i32;
9607 RC = &AArch64::FPR128RegClass;
9608 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9609 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9610 RC);
9611 break;
9612 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
9613 Opc = AArch64::MLSv4i32;
9614 RC = &AArch64::FPR128RegClass;
9615 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9616 break;
9617
9618 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
9619 Opc = AArch64::MLAv4i16_indexed;
9620 RC = &AArch64::FPR64RegClass;
9621 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9622 break;
9623 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
9624 Opc = AArch64::MLAv4i16_indexed;
9625 RC = &AArch64::FPR64RegClass;
9626 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9627 break;
9628 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
9629 Opc = AArch64::MLAv8i16_indexed;
9630 RC = &AArch64::FPR128RegClass;
9631 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9632 break;
9633 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
9634 Opc = AArch64::MLAv8i16_indexed;
9635 RC = &AArch64::FPR128RegClass;
9636 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9637 break;
9638 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
9639 Opc = AArch64::MLAv2i32_indexed;
9640 RC = &AArch64::FPR64RegClass;
9641 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9642 break;
9643 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
9644 Opc = AArch64::MLAv2i32_indexed;
9645 RC = &AArch64::FPR64RegClass;
9646 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9647 break;
9648 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
9649 Opc = AArch64::MLAv4i32_indexed;
9650 RC = &AArch64::FPR128RegClass;
9651 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9652 break;
9653 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
9654 Opc = AArch64::MLAv4i32_indexed;
9655 RC = &AArch64::FPR128RegClass;
9656 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9657 break;
9658
9659 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
9660 Opc = AArch64::MLAv4i16_indexed;
9661 RC = &AArch64::FPR64RegClass;
9662 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9663 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9664 RC);
9665 break;
9666 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
9667 Opc = AArch64::MLSv4i16_indexed;
9668 RC = &AArch64::FPR64RegClass;
9669 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9670 break;
9671 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
9672 Opc = AArch64::MLAv8i16_indexed;
9673 RC = &AArch64::FPR128RegClass;
9674 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9675 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9676 RC);
9677 break;
9678 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
9679 Opc = AArch64::MLSv8i16_indexed;
9680 RC = &AArch64::FPR128RegClass;
9681 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9682 break;
9683 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
9684 Opc = AArch64::MLAv2i32_indexed;
9685 RC = &AArch64::FPR64RegClass;
9686 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9687 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9688 RC);
9689 break;
9690 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
9691 Opc = AArch64::MLSv2i32_indexed;
9692 RC = &AArch64::FPR64RegClass;
9693 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9694 break;
9695 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
9696 Opc = AArch64::MLAv4i32_indexed;
9697 RC = &AArch64::FPR128RegClass;
9698 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9699 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9700 RC);
9701 break;
9702 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
9703 Opc = AArch64::MLSv4i32_indexed;
9704 RC = &AArch64::FPR128RegClass;
9705 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9706 break;
9707
9708 // Floating Point Support
9709 case AArch64MachineCombinerPattern::FMULADDH_OP1:
9710 Opc = AArch64::FMADDHrrr;
9711 RC = &AArch64::FPR16RegClass;
9712 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9713 break;
9714 case AArch64MachineCombinerPattern::FMULADDS_OP1:
9715 Opc = AArch64::FMADDSrrr;
9716 RC = &AArch64::FPR32RegClass;
9717 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9718 break;
9719 case AArch64MachineCombinerPattern::FMULADDD_OP1:
9720 Opc = AArch64::FMADDDrrr;
9721 RC = &AArch64::FPR64RegClass;
9722 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9723 break;
9724
9725 case AArch64MachineCombinerPattern::FMULADDH_OP2:
9726 Opc = AArch64::FMADDHrrr;
9727 RC = &AArch64::FPR16RegClass;
9728 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9729 break;
9730 case AArch64MachineCombinerPattern::FMULADDS_OP2:
9731 Opc = AArch64::FMADDSrrr;
9732 RC = &AArch64::FPR32RegClass;
9733 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9734 break;
9735 case AArch64MachineCombinerPattern::FMULADDD_OP2:
9736 Opc = AArch64::FMADDDrrr;
9737 RC = &AArch64::FPR64RegClass;
9738 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9739 break;
9740
9741 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
9742 Opc = AArch64::FMLAv1i32_indexed;
9743 RC = &AArch64::FPR32RegClass;
9744 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9745 kind: FMAInstKind::Indexed);
9746 break;
9747 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
9748 Opc = AArch64::FMLAv1i32_indexed;
9749 RC = &AArch64::FPR32RegClass;
9750 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9751 kind: FMAInstKind::Indexed);
9752 break;
9753
9754 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
9755 Opc = AArch64::FMLAv1i64_indexed;
9756 RC = &AArch64::FPR64RegClass;
9757 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9758 kind: FMAInstKind::Indexed);
9759 break;
9760 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
9761 Opc = AArch64::FMLAv1i64_indexed;
9762 RC = &AArch64::FPR64RegClass;
9763 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9764 kind: FMAInstKind::Indexed);
9765 break;
9766
9767 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
9768 RC = &AArch64::FPR64RegClass;
9769 Opc = AArch64::FMLAv4i16_indexed;
9770 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9771 kind: FMAInstKind::Indexed);
9772 break;
9773 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
9774 RC = &AArch64::FPR64RegClass;
9775 Opc = AArch64::FMLAv4f16;
9776 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9777 kind: FMAInstKind::Accumulator);
9778 break;
9779 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
9780 RC = &AArch64::FPR64RegClass;
9781 Opc = AArch64::FMLAv4i16_indexed;
9782 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9783 kind: FMAInstKind::Indexed);
9784 break;
9785 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
9786 RC = &AArch64::FPR64RegClass;
9787 Opc = AArch64::FMLAv4f16;
9788 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9789 kind: FMAInstKind::Accumulator);
9790 break;
9791
9792 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
9793 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
9794 RC = &AArch64::FPR64RegClass;
9795 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
9796 Opc = AArch64::FMLAv2i32_indexed;
9797 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9798 kind: FMAInstKind::Indexed);
9799 } else {
9800 Opc = AArch64::FMLAv2f32;
9801 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9802 kind: FMAInstKind::Accumulator);
9803 }
9804 break;
9805 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
9806 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
9807 RC = &AArch64::FPR64RegClass;
9808 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
9809 Opc = AArch64::FMLAv2i32_indexed;
9810 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9811 kind: FMAInstKind::Indexed);
9812 } else {
9813 Opc = AArch64::FMLAv2f32;
9814 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9815 kind: FMAInstKind::Accumulator);
9816 }
9817 break;
9818
9819 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
9820 RC = &AArch64::FPR128RegClass;
9821 Opc = AArch64::FMLAv8i16_indexed;
9822 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9823 kind: FMAInstKind::Indexed);
9824 break;
9825 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
9826 RC = &AArch64::FPR128RegClass;
9827 Opc = AArch64::FMLAv8f16;
9828 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9829 kind: FMAInstKind::Accumulator);
9830 break;
9831 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
9832 RC = &AArch64::FPR128RegClass;
9833 Opc = AArch64::FMLAv8i16_indexed;
9834 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9835 kind: FMAInstKind::Indexed);
9836 break;
9837 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
9838 RC = &AArch64::FPR128RegClass;
9839 Opc = AArch64::FMLAv8f16;
9840 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9841 kind: FMAInstKind::Accumulator);
9842 break;
9843
9844 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
9845 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
9846 RC = &AArch64::FPR128RegClass;
9847 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
9848 Opc = AArch64::FMLAv2i64_indexed;
9849 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9850 kind: FMAInstKind::Indexed);
9851 } else {
9852 Opc = AArch64::FMLAv2f64;
9853 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9854 kind: FMAInstKind::Accumulator);
9855 }
9856 break;
9857 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
9858 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
9859 RC = &AArch64::FPR128RegClass;
9860 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
9861 Opc = AArch64::FMLAv2i64_indexed;
9862 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9863 kind: FMAInstKind::Indexed);
9864 } else {
9865 Opc = AArch64::FMLAv2f64;
9866 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9867 kind: FMAInstKind::Accumulator);
9868 }
9869 break;
9870
9871 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
9872 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
9873 RC = &AArch64::FPR128RegClass;
9874 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
9875 Opc = AArch64::FMLAv4i32_indexed;
9876 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9877 kind: FMAInstKind::Indexed);
9878 } else {
9879 Opc = AArch64::FMLAv4f32;
9880 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9881 kind: FMAInstKind::Accumulator);
9882 }
9883 break;
9884
9885 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
9886 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
9887 RC = &AArch64::FPR128RegClass;
9888 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
9889 Opc = AArch64::FMLAv4i32_indexed;
9890 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9891 kind: FMAInstKind::Indexed);
9892 } else {
9893 Opc = AArch64::FMLAv4f32;
9894 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9895 kind: FMAInstKind::Accumulator);
9896 }
9897 break;
9898
9899 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
9900 Opc = AArch64::FNMSUBHrrr;
9901 RC = &AArch64::FPR16RegClass;
9902 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9903 break;
9904 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
9905 Opc = AArch64::FNMSUBSrrr;
9906 RC = &AArch64::FPR32RegClass;
9907 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9908 break;
9909 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
9910 Opc = AArch64::FNMSUBDrrr;
9911 RC = &AArch64::FPR64RegClass;
9912 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9913 break;
9914
9915 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
9916 Opc = AArch64::FNMADDHrrr;
9917 RC = &AArch64::FPR16RegClass;
9918 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9919 break;
9920 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
9921 Opc = AArch64::FNMADDSrrr;
9922 RC = &AArch64::FPR32RegClass;
9923 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9924 break;
9925 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
9926 Opc = AArch64::FNMADDDrrr;
9927 RC = &AArch64::FPR64RegClass;
9928 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9929 break;
9930
9931 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
9932 Opc = AArch64::FMSUBHrrr;
9933 RC = &AArch64::FPR16RegClass;
9934 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9935 break;
9936 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
9937 Opc = AArch64::FMSUBSrrr;
9938 RC = &AArch64::FPR32RegClass;
9939 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9940 break;
9941 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
9942 Opc = AArch64::FMSUBDrrr;
9943 RC = &AArch64::FPR64RegClass;
9944 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9945 break;
9946
9947 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
9948 Opc = AArch64::FMLSv1i32_indexed;
9949 RC = &AArch64::FPR32RegClass;
9950 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9951 kind: FMAInstKind::Indexed);
9952 break;
9953
9954 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
9955 Opc = AArch64::FMLSv1i64_indexed;
9956 RC = &AArch64::FPR64RegClass;
9957 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9958 kind: FMAInstKind::Indexed);
9959 break;
9960
9961 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
9962 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
9963 RC = &AArch64::FPR64RegClass;
9964 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9965 MachineInstrBuilder MIB1 =
9966 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f16), DestReg: NewVR)
9967 .add(MO: Root.getOperand(i: 2));
9968 InsInstrs.push_back(Elt: MIB1);
9969 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9970 if (Pattern == AArch64MachineCombinerPattern::FMLSv4f16_OP1) {
9971 Opc = AArch64::FMLAv4f16;
9972 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9973 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9974 } else {
9975 Opc = AArch64::FMLAv4i16_indexed;
9976 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9977 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9978 }
9979 break;
9980 }
9981 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
9982 RC = &AArch64::FPR64RegClass;
9983 Opc = AArch64::FMLSv4f16;
9984 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9985 kind: FMAInstKind::Accumulator);
9986 break;
9987 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
9988 RC = &AArch64::FPR64RegClass;
9989 Opc = AArch64::FMLSv4i16_indexed;
9990 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9991 kind: FMAInstKind::Indexed);
9992 break;
9993
9994 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
9995 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
9996 RC = &AArch64::FPR64RegClass;
9997 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
9998 Opc = AArch64::FMLSv2i32_indexed;
9999 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
10000 kind: FMAInstKind::Indexed);
10001 } else {
10002 Opc = AArch64::FMLSv2f32;
10003 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
10004 kind: FMAInstKind::Accumulator);
10005 }
10006 break;
10007
10008 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
10009 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
10010 RC = &AArch64::FPR128RegClass;
10011 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
10012 MachineInstrBuilder MIB1 =
10013 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv8f16), DestReg: NewVR)
10014 .add(MO: Root.getOperand(i: 2));
10015 InsInstrs.push_back(Elt: MIB1);
10016 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
10017 if (Pattern == AArch64MachineCombinerPattern::FMLSv8f16_OP1) {
10018 Opc = AArch64::FMLAv8f16;
10019 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
10020 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
10021 } else {
10022 Opc = AArch64::FMLAv8i16_indexed;
10023 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
10024 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
10025 }
10026 break;
10027 }
10028 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
10029 RC = &AArch64::FPR128RegClass;
10030 Opc = AArch64::FMLSv8f16;
10031 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
10032 kind: FMAInstKind::Accumulator);
10033 break;
10034 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
10035 RC = &AArch64::FPR128RegClass;
10036 Opc = AArch64::FMLSv8i16_indexed;
10037 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
10038 kind: FMAInstKind::Indexed);
10039 break;
10040
10041 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
10042 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
10043 RC = &AArch64::FPR128RegClass;
10044 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
10045 Opc = AArch64::FMLSv2i64_indexed;
10046 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
10047 kind: FMAInstKind::Indexed);
10048 } else {
10049 Opc = AArch64::FMLSv2f64;
10050 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
10051 kind: FMAInstKind::Accumulator);
10052 }
10053 break;
10054
10055 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
10056 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
10057 RC = &AArch64::FPR128RegClass;
10058 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
10059 Opc = AArch64::FMLSv4i32_indexed;
10060 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
10061 kind: FMAInstKind::Indexed);
10062 } else {
10063 Opc = AArch64::FMLSv4f32;
10064 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
10065 kind: FMAInstKind::Accumulator);
10066 }
10067 break;
10068 case AArch64MachineCombinerPattern::FMLSv2f32_OP1:
10069 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
10070 RC = &AArch64::FPR64RegClass;
10071 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
10072 MachineInstrBuilder MIB1 =
10073 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f32), DestReg: NewVR)
10074 .add(MO: Root.getOperand(i: 2));
10075 InsInstrs.push_back(Elt: MIB1);
10076 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
10077 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
10078 Opc = AArch64::FMLAv2i32_indexed;
10079 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
10080 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
10081 } else {
10082 Opc = AArch64::FMLAv2f32;
10083 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
10084 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
10085 }
10086 break;
10087 }
10088 case AArch64MachineCombinerPattern::FMLSv4f32_OP1:
10089 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
10090 RC = &AArch64::FPR128RegClass;
10091 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
10092 MachineInstrBuilder MIB1 =
10093 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f32), DestReg: NewVR)
10094 .add(MO: Root.getOperand(i: 2));
10095 InsInstrs.push_back(Elt: MIB1);
10096 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
10097 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
10098 Opc = AArch64::FMLAv4i32_indexed;
10099 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
10100 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
10101 } else {
10102 Opc = AArch64::FMLAv4f32;
10103 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
10104 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
10105 }
10106 break;
10107 }
10108 case AArch64MachineCombinerPattern::FMLSv2f64_OP1:
10109 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
10110 RC = &AArch64::FPR128RegClass;
10111 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
10112 MachineInstrBuilder MIB1 =
10113 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f64), DestReg: NewVR)
10114 .add(MO: Root.getOperand(i: 2));
10115 InsInstrs.push_back(Elt: MIB1);
10116 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
10117 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
10118 Opc = AArch64::FMLAv2i64_indexed;
10119 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
10120 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
10121 } else {
10122 Opc = AArch64::FMLAv2f64;
10123 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
10124 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
10125 }
10126 break;
10127 }
10128 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
10129 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2: {
10130 unsigned IdxDupOp =
10131 (Pattern == AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1) ? 1
10132 : 2;
10133 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i32_indexed,
10134 RC: &AArch64::FPR128RegClass, MRI);
10135 break;
10136 }
10137 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
10138 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2: {
10139 unsigned IdxDupOp =
10140 (Pattern == AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1) ? 1
10141 : 2;
10142 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i64_indexed,
10143 RC: &AArch64::FPR128RegClass, MRI);
10144 break;
10145 }
10146 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
10147 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2: {
10148 unsigned IdxDupOp =
10149 (Pattern == AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1) ? 1
10150 : 2;
10151 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i16_indexed,
10152 RC: &AArch64::FPR128_loRegClass, MRI);
10153 break;
10154 }
10155 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
10156 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2: {
10157 unsigned IdxDupOp =
10158 (Pattern == AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1) ? 1
10159 : 2;
10160 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i32_indexed,
10161 RC: &AArch64::FPR128RegClass, MRI);
10162 break;
10163 }
10164 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
10165 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2: {
10166 unsigned IdxDupOp =
10167 (Pattern == AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1) ? 1
10168 : 2;
10169 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv8i16_indexed,
10170 RC: &AArch64::FPR128_loRegClass, MRI);
10171 break;
10172 }
10173 case AArch64MachineCombinerPattern::FNMADD: {
10174 MUL = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs);
10175 break;
10176 }
10177 case AArch64MachineCombinerPattern::GATHER_LANE_i32: {
10178 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10179 Pattern, NumLanes: 4);
10180 break;
10181 }
10182 case AArch64MachineCombinerPattern::GATHER_LANE_i16: {
10183 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10184 Pattern, NumLanes: 8);
10185 break;
10186 }
10187 case AArch64MachineCombinerPattern::GATHER_LANE_i8: {
10188 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10189 Pattern, NumLanes: 16);
10190 break;
10191 }
10192
10193 } // end switch (Pattern)
10194 // Record MUL and ADD/SUB for deletion
10195 if (MUL)
10196 DelInstrs.push_back(Elt: MUL);
10197 DelInstrs.push_back(Elt: &Root);
10198
10199 // Set the flags on the inserted instructions to be the merged flags of the
10200 // instructions that we have combined.
10201 uint32_t Flags = Root.getFlags();
10202 if (MUL)
10203 Flags = Root.mergeFlagsWith(Other: *MUL);
10204 for (auto *MI : InsInstrs)
10205 MI->setFlags(Flags);
10206}
10207
10208/// Replace csincr-branch sequence by simple conditional branch
10209///
10210/// Examples:
10211/// 1. \code
10212/// csinc w9, wzr, wzr, <condition code>
10213/// tbnz w9, #0, 0x44
10214/// \endcode
10215/// to
10216/// \code
10217/// b.<inverted condition code>
10218/// \endcode
10219///
10220/// 2. \code
10221/// csinc w9, wzr, wzr, <condition code>
10222/// tbz w9, #0, 0x44
10223/// \endcode
10224/// to
10225/// \code
10226/// b.<condition code>
10227/// \endcode
10228///
10229/// Replace compare and branch sequence by TBZ/TBNZ instruction when the
10230/// compare's constant operand is power of 2.
10231///
10232/// Examples:
10233/// \code
10234/// and w8, w8, #0x400
10235/// cbnz w8, L1
10236/// \endcode
10237/// to
10238/// \code
10239/// tbnz w8, #10, L1
10240/// \endcode
10241///
10242/// \param MI Conditional Branch
10243/// \return True when the simple conditional branch is generated
10244///
10245bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
10246 bool IsNegativeBranch = false;
10247 bool IsTestAndBranch = false;
10248 unsigned TargetBBInMI = 0;
10249 switch (MI.getOpcode()) {
10250 default:
10251 llvm_unreachable("Unknown branch instruction?");
10252 case AArch64::Bcc:
10253 case AArch64::CBWPri:
10254 case AArch64::CBXPri:
10255 case AArch64::CBBAssertExt:
10256 case AArch64::CBHAssertExt:
10257 case AArch64::CBWPrr:
10258 case AArch64::CBXPrr:
10259 return false;
10260 case AArch64::CBZW:
10261 case AArch64::CBZX:
10262 TargetBBInMI = 1;
10263 break;
10264 case AArch64::CBNZW:
10265 case AArch64::CBNZX:
10266 TargetBBInMI = 1;
10267 IsNegativeBranch = true;
10268 break;
10269 case AArch64::TBZW:
10270 case AArch64::TBZX:
10271 TargetBBInMI = 2;
10272 IsTestAndBranch = true;
10273 break;
10274 case AArch64::TBNZW:
10275 case AArch64::TBNZX:
10276 TargetBBInMI = 2;
10277 IsNegativeBranch = true;
10278 IsTestAndBranch = true;
10279 break;
10280 }
10281 // So we increment a zero register and test for bits other
10282 // than bit 0? Conservatively bail out in case the verifier
10283 // missed this case.
10284 if (IsTestAndBranch && MI.getOperand(i: 1).getImm())
10285 return false;
10286
10287 // Find Definition.
10288 assert(MI.getParent() && "Incomplete machine instruction\n");
10289 MachineBasicBlock *MBB = MI.getParent();
10290 MachineFunction *MF = MBB->getParent();
10291 MachineRegisterInfo *MRI = &MF->getRegInfo();
10292 Register VReg = MI.getOperand(i: 0).getReg();
10293 if (!VReg.isVirtual())
10294 return false;
10295
10296 MachineInstr *DefMI = MRI->getVRegDef(Reg: VReg);
10297 if (!DefMI)
10298 return false;
10299
10300 // Look through COPY instructions to find definition.
10301 while (DefMI->isCopy()) {
10302 Register CopyVReg = DefMI->getOperand(i: 1).getReg();
10303 if (!CopyVReg.isVirtual())
10304 return false;
10305 if (!MRI->hasOneNonDBGUse(RegNo: CopyVReg))
10306 return false;
10307 DefMI = MRI->getVRegDef(Reg: CopyVReg);
10308 if (!DefMI)
10309 return false;
10310 }
10311
10312 switch (DefMI->getOpcode()) {
10313 default:
10314 return false;
10315 // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
10316 case AArch64::ANDWri:
10317 case AArch64::ANDXri: {
10318 if (IsTestAndBranch)
10319 return false;
10320 if (DefMI->getParent() != MBB)
10321 return false;
10322 if (!MRI->hasOneNonDBGUse(RegNo: VReg))
10323 return false;
10324
10325 bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
10326 uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
10327 val: DefMI->getOperand(i: 2).getImm(), regSize: Is32Bit ? 32 : 64);
10328 if (!isPowerOf2_64(Value: Mask))
10329 return false;
10330
10331 MachineOperand &MO = DefMI->getOperand(i: 1);
10332 Register NewReg = MO.getReg();
10333 if (!NewReg.isVirtual())
10334 return false;
10335
10336 if (!MRI->getVRegDef(Reg: NewReg))
10337 return false;
10338
10339 MachineBasicBlock &RefToMBB = *MBB;
10340 MachineBasicBlock *TBB = MI.getOperand(i: 1).getMBB();
10341 DebugLoc DL = MI.getDebugLoc();
10342 unsigned Imm = Log2_64(Value: Mask);
10343 unsigned Opc = (Imm < 32)
10344 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
10345 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
10346 MachineInstr *NewMI = BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: Opc))
10347 .addReg(RegNo: NewReg)
10348 .addImm(Val: Imm)
10349 .addMBB(MBB: TBB);
10350 // Register lives on to the CBZ now.
10351 MO.setIsKill(false);
10352
10353 // For immediate smaller than 32, we need to use the 32-bit
10354 // variant (W) in all cases. Indeed the 64-bit variant does not
10355 // allow to encode them.
10356 // Therefore, if the input register is 64-bit, we need to take the
10357 // 32-bit sub-part.
10358 if (!Is32Bit && Imm < 32)
10359 NewMI->getOperand(i: 0).setSubReg(AArch64::sub_32);
10360 MI.eraseFromParent();
10361 return true;
10362 }
10363 // Look for CSINC
10364 case AArch64::CSINCWr:
10365 case AArch64::CSINCXr: {
10366 if (!(DefMI->getOperand(i: 1).getReg() == AArch64::WZR &&
10367 DefMI->getOperand(i: 2).getReg() == AArch64::WZR) &&
10368 !(DefMI->getOperand(i: 1).getReg() == AArch64::XZR &&
10369 DefMI->getOperand(i: 2).getReg() == AArch64::XZR))
10370 return false;
10371
10372 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
10373 isDead: true) != -1)
10374 return false;
10375
10376 AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(i: 3).getImm();
10377 // Convert only when the condition code is not modified between
10378 // the CSINC and the branch. The CC may be used by other
10379 // instructions in between.
10380 if (areCFlagsAccessedBetweenInstrs(From: DefMI, To: MI, TRI: &getRegisterInfo(), AccessToCheck: AK_Write))
10381 return false;
10382 MachineBasicBlock &RefToMBB = *MBB;
10383 MachineBasicBlock *TBB = MI.getOperand(i: TargetBBInMI).getMBB();
10384 DebugLoc DL = MI.getDebugLoc();
10385 if (IsNegativeBranch)
10386 CC = AArch64CC::getInvertedCondCode(Code: CC);
10387 BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: CC).addMBB(MBB: TBB);
10388 MI.eraseFromParent();
10389 return true;
10390 }
10391 }
10392}
10393
10394std::pair<unsigned, unsigned>
10395AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10396 const unsigned Mask = AArch64II::MO_FRAGMENT;
10397 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
10398}
10399
10400ArrayRef<std::pair<unsigned, const char *>>
10401AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10402 using namespace AArch64II;
10403
10404 static const std::pair<unsigned, const char *> TargetFlags[] = {
10405 {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
10406 {MO_G3, "aarch64-g3"}, {MO_G2, "aarch64-g2"},
10407 {MO_G1, "aarch64-g1"}, {MO_G0, "aarch64-g0"},
10408 {MO_HI12, "aarch64-hi12"}};
10409 return ArrayRef(TargetFlags);
10410}
10411
10412ArrayRef<std::pair<unsigned, const char *>>
10413AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
10414 using namespace AArch64II;
10415
10416 static const std::pair<unsigned, const char *> TargetFlags[] = {
10417 {MO_COFFSTUB, "aarch64-coffstub"},
10418 {MO_GOT, "aarch64-got"},
10419 {MO_NC, "aarch64-nc"},
10420 {MO_S, "aarch64-s"},
10421 {MO_TLS, "aarch64-tls"},
10422 {MO_DLLIMPORT, "aarch64-dllimport"},
10423 {MO_PREL, "aarch64-prel"},
10424 {MO_TAGGED, "aarch64-tagged"},
10425 {MO_ARM64EC_CALLMANGLE, "aarch64-arm64ec-callmangle"},
10426 };
10427 return ArrayRef(TargetFlags);
10428}
10429
10430ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10431AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10432 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10433 {{MOSuppressPair, "aarch64-suppress-pair"},
10434 {MOStridedAccess, "aarch64-strided-access"}};
10435 return ArrayRef(TargetFlags);
10436}
10437
10438/// Constants defining how certain sequences should be outlined.
10439/// This encompasses how an outlined function should be called, and what kind of
10440/// frame should be emitted for that outlined function.
10441///
10442/// \p MachineOutlinerDefault implies that the function should be called with
10443/// a save and restore of LR to the stack.
10444///
10445/// That is,
10446///
10447/// I1 Save LR OUTLINED_FUNCTION:
10448/// I2 --> BL OUTLINED_FUNCTION I1
10449/// I3 Restore LR I2
10450/// I3
10451/// RET
10452///
10453/// * Call construction overhead: 3 (save + BL + restore)
10454/// * Frame construction overhead: 1 (ret)
10455/// * Requires stack fixups? Yes
10456///
10457/// \p MachineOutlinerTailCall implies that the function is being created from
10458/// a sequence of instructions ending in a return.
10459///
10460/// That is,
10461///
10462/// I1 OUTLINED_FUNCTION:
10463/// I2 --> B OUTLINED_FUNCTION I1
10464/// RET I2
10465/// RET
10466///
10467/// * Call construction overhead: 1 (B)
10468/// * Frame construction overhead: 0 (Return included in sequence)
10469/// * Requires stack fixups? No
10470///
10471/// \p MachineOutlinerNoLRSave implies that the function should be called using
10472/// a BL instruction, but doesn't require LR to be saved and restored. This
10473/// happens when LR is known to be dead.
10474///
10475/// That is,
10476///
10477/// I1 OUTLINED_FUNCTION:
10478/// I2 --> BL OUTLINED_FUNCTION I1
10479/// I3 I2
10480/// I3
10481/// RET
10482///
10483/// * Call construction overhead: 1 (BL)
10484/// * Frame construction overhead: 1 (RET)
10485/// * Requires stack fixups? No
10486///
10487/// \p MachineOutlinerThunk implies that the function is being created from
10488/// a sequence of instructions ending in a call. The outlined function is
10489/// called with a BL instruction, and the outlined function tail-calls the
10490/// original call destination.
10491///
10492/// That is,
10493///
10494/// I1 OUTLINED_FUNCTION:
10495/// I2 --> BL OUTLINED_FUNCTION I1
10496/// BL f I2
10497/// B f
10498/// * Call construction overhead: 1 (BL)
10499/// * Frame construction overhead: 0
10500/// * Requires stack fixups? No
10501///
10502/// \p MachineOutlinerRegSave implies that the function should be called with a
10503/// save and restore of LR to an available register. This allows us to avoid
10504/// stack fixups. Note that this outlining variant is compatible with the
10505/// NoLRSave case.
10506///
10507/// That is,
10508///
10509/// I1 Save LR OUTLINED_FUNCTION:
10510/// I2 --> BL OUTLINED_FUNCTION I1
10511/// I3 Restore LR I2
10512/// I3
10513/// RET
10514///
10515/// * Call construction overhead: 3 (save + BL + restore)
10516/// * Frame construction overhead: 1 (ret)
10517/// * Requires stack fixups? No
10518///
10519/// When Pointer Authentication (PAC) is enabled:
10520/// - MachineOutlinerDefault, MachineOutlinerRegSave, MachineOutlinerThunk,
10521/// and MachineOutlinerNoLRSave: Return address is signed in the prologue and
10522/// authenticated in the epilogue if required by the signing scope.
10523/// - MachineOutlinerTailCall: Rejected if return address signing is required
10524/// to avoid corrupting the caller's LR.
10525
10526enum MachineOutlinerClass {
10527 MachineOutlinerDefault, /// Emit a save, restore, call, and return.
10528 MachineOutlinerTailCall, /// Only emit a branch.
10529 MachineOutlinerNoLRSave, /// Emit a call and return.
10530 MachineOutlinerThunk, /// Emit a call and tail-call.
10531 MachineOutlinerRegSave /// Same as default, but save to a register.
10532};
10533
10534enum MachineOutlinerMBBFlags {
10535 LRUnavailableSomewhere = 0x2,
10536 HasCalls = 0x4,
10537 UnsafeRegsDead = 0x8
10538};
10539
10540/// Return true if the frame-record form of the outlined prologue is enabled for
10541/// the target of \p MF.
10542///
10543/// A non-leaf outlined function must save LR. On MachO, saving LR alone
10544/// (str x30) has no compact unwind encoding, so we get a large DWARF FDE
10545/// instead. Saving FP and LR as a frame record (stp x29, x30 ; mov x29, sp)
10546/// gets the small FRAME encoding, and costs one extra instruction.
10547static bool isCompactUnwindFrameRecordEnabled(const MachineFunction &MF) {
10548 return MF.getSubtarget<AArch64Subtarget>()
10549 .getCLOpts()
10550 .outliner_compact_unwind_frame &&
10551 MF.getFunction().getParent()->getTargetTriple().isOSBinFormatMachO();
10552}
10553
10554/// Return true if the outlined function in \p MBB should save FP and LR as a
10555/// frame record instead of saving LR alone.
10556static bool shouldUseCompactUnwindFrameRecordForOutlinedFunction(
10557 const MachineBasicBlock &MBB) {
10558 const MachineFunction &MF = *MBB.getParent();
10559
10560 // Only worth it if the function has unwind info to shrink.
10561 if (!isCompactUnwindFrameRecordEnabled(MF) ||
10562 !MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF))
10563 return false;
10564
10565 // Only safe if the outlined code never touches FP, since we overwrite it.
10566 LiveRegUnits LRU(*MF.getSubtarget().getRegisterInfo());
10567 for (const MachineInstr &MI : MBB.instrs())
10568 LRU.accumulate(MI);
10569 return LRU.available(Reg: AArch64::FP);
10570}
10571
10572/// Predict what the above will answer, for use while costing candidates. The
10573/// outlined function does not exist yet, so answer from \p RepeatedSequenceLocs
10574/// instead. This is only an estimate; buildOutlinedFrame() makes the call.
10575static bool predictCompactUnwindFrameRecordForOutlinedFunction(
10576 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10577 const TargetRegisterInfo &TRI) {
10578 if (!isCompactUnwindFrameRecordEnabled(MF: *RepeatedSequenceLocs.front().getMF()))
10579 return false;
10580
10581 // The outlined function is nounwind only if every candidate is, so it has
10582 // unwind info if any candidate does.
10583 if (llvm::none_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10584 const MachineFunction &MF = *C.getMF();
10585 return MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF);
10586 }))
10587 return false;
10588
10589 // FP is free in the outlined function only if it is free in every candidate.
10590 return llvm::all_of(Range&: RepeatedSequenceLocs, P: [&TRI](outliner::Candidate &C) {
10591 return C.isAvailableInsideSeq(Reg: AArch64::FP, TRI);
10592 });
10593}
10594
10595Register
10596AArch64InstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
10597 MachineFunction *MF = C.getMF();
10598 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
10599 const AArch64RegisterInfo *ARI =
10600 static_cast<const AArch64RegisterInfo *>(&TRI);
10601 // Check if there is an available register across the sequence that we can
10602 // use.
10603 for (unsigned Reg : AArch64::GPR64RegClass) {
10604 if (!ARI->isReservedReg(MF: *MF, Reg) &&
10605 Reg != AArch64::LR && // LR is not reserved, but don't use it.
10606 Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
10607 Reg != AArch64::X17 && // Ditto for X17.
10608 C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
10609 C.isAvailableInsideSeq(Reg, TRI))
10610 return Reg;
10611 }
10612 return Register();
10613}
10614
10615static bool
10616outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
10617 const outliner::Candidate &b) {
10618 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10619 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10620
10621 return MFIa->getSignReturnAddressCondition() ==
10622 MFIb->getSignReturnAddressCondition();
10623}
10624
10625static bool
10626outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
10627 const outliner::Candidate &b) {
10628 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10629 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10630
10631 return MFIa->shouldSignWithBKey() == MFIb->shouldSignWithBKey();
10632}
10633
10634static bool outliningCandidatesPAuthLRConsensus(const outliner::Candidate &a,
10635 const outliner::Candidate &b) {
10636 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10637 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10638
10639 const AArch64Subtarget &SubtargetA =
10640 a.getMF()->getSubtarget<AArch64Subtarget>();
10641 const AArch64Subtarget &SubtargetB =
10642 b.getMF()->getSubtarget<AArch64Subtarget>();
10643
10644 // Return true only if branch-protection-pauth-lr attributes match
10645 // and +pauth-lr feature is enabled for both.
10646 if (MFIa->branchProtectionPAuthLR() != MFIb->branchProtectionPAuthLR())
10647 return false;
10648
10649 return !MFIa->branchProtectionPAuthLR() ||
10650 SubtargetA.hasPAuthLR() == SubtargetB.hasPAuthLR();
10651}
10652
10653static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
10654 const outliner::Candidate &b) {
10655 const AArch64Subtarget &SubtargetA =
10656 a.getMF()->getSubtarget<AArch64Subtarget>();
10657 const AArch64Subtarget &SubtargetB =
10658 b.getMF()->getSubtarget<AArch64Subtarget>();
10659 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
10660}
10661
10662namespace {
10663struct SPAddSubImmEncoding {
10664 unsigned Opcode;
10665 int64_t Imm;
10666 int64_t Shift;
10667};
10668} // namespace
10669
10670/// Returns true if \p MI is an ADD/SUBXri with an immediate offset that reads
10671/// SP but does not write SP (e.g. "add x0, sp, #48").
10672static bool isSPAddSubImm(const MachineInstr &MI) {
10673 if (MI.getOpcode() != AArch64::ADDXri && MI.getOpcode() != AArch64::SUBXri)
10674 return false;
10675
10676 assert(MI.getNumExplicitOperands() == 4 && "Wrong number of operands");
10677 assert(MI.getOperand(0).isReg() && MI.getOperand(1).isReg() &&
10678 MI.getOperand(3).isImm() && "Unexpected add/sub immediate operands");
10679
10680 if (MI.getOperand(i: 0).getReg() == AArch64::SP)
10681 return false;
10682 if (MI.getOperand(i: 1).getReg() != AArch64::SP)
10683 return false;
10684
10685 return MI.getOperand(i: 2).isImm();
10686}
10687
10688/// Adjust the signed byte offset of \p MI, which must satisfy isSPAddSubImm,
10689/// by \p Delta and re-encode the result. Returns std::nullopt if the adjusted
10690/// offset is not encodable.
10691static std::optional<SPAddSubImmEncoding>
10692adjustSPAddSubImm(const MachineInstr &MI, int64_t Delta) {
10693 assert(isSPAddSubImm(MI) && "Expected an SP-based add/sub immediate");
10694 bool IsSub = MI.getOpcode() == AArch64::SUBXri;
10695
10696 // ADD/SUBXri encode a 12-bit unsigned immediate with an optional left shift
10697 // which can only be 12.
10698 int64_t ShiftImm = MI.getOperand(i: 3).getImm();
10699 assert(AArch64_AM::getShiftType(ShiftImm) == AArch64_AM::LSL &&
10700 "Unexpected add/sub immediate shift type");
10701 unsigned Shift = AArch64_AM::getShiftValue(Imm: ShiftImm);
10702 assert((Shift == 0 || Shift == 12) && "Unexpected add/sub immediate shift");
10703
10704 int64_t Offset = MI.getOperand(i: 2).getImm() << Shift;
10705 if (IsSub)
10706 Offset = -Offset;
10707
10708 // Apply the adjustment and try to re-encode.
10709 int64_t NewOffset = Offset + Delta;
10710 int64_t AbsOffset = std::abs(i: NewOffset);
10711
10712 SPAddSubImmEncoding Enc;
10713 Enc.Opcode = NewOffset < 0 ? AArch64::SUBXri : AArch64::ADDXri;
10714
10715 // Try to encode as a 12-bit immediate with shift=0 first.
10716 if (AbsOffset <= 0xfff) {
10717 Enc.Imm = AbsOffset;
10718 Enc.Shift = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0);
10719 } else if ((AbsOffset & 0xfff) == 0 && (AbsOffset >> 12) <= 0xfff) {
10720 Enc.Imm = AbsOffset >> 12;
10721 Enc.Shift = AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 12);
10722 } else {
10723 return std::nullopt;
10724 }
10725
10726 return Enc;
10727}
10728
10729std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10730AArch64InstrInfo::getOutliningCandidateInfo(
10731 const MachineModuleInfo &MMI,
10732 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10733 unsigned MinRepeats) const {
10734 unsigned SequenceSize = 0;
10735 for (auto &MI : RepeatedSequenceLocs[0])
10736 SequenceSize += getInstSizeInBytes(MI);
10737
10738 unsigned NumBytesToCreateFrame = 0;
10739
10740 // Avoid splitting ADRP ADD/LDR pair into outlined functions.
10741 // These instructions are fused together by the scheduler.
10742 // Any candidate where ADRP is the last instruction should be rejected
10743 // as that will lead to splitting ADRP pair.
10744 MachineInstr &LastMI = RepeatedSequenceLocs[0].back();
10745 MachineInstr &FirstMI = RepeatedSequenceLocs[0].front();
10746 if (LastMI.getOpcode() == AArch64::ADRP &&
10747 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_PAGE) != 0 &&
10748 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10749 return std::nullopt;
10750 }
10751
10752 // Similarly any candidate where the first instruction is ADD/LDR with a
10753 // page offset should be rejected to avoid ADRP splitting.
10754 if ((FirstMI.getOpcode() == AArch64::ADDXri ||
10755 FirstMI.getOpcode() == AArch64::LDRXui) &&
10756 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_PAGEOFF) != 0 &&
10757 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10758 return std::nullopt;
10759 }
10760
10761 // We only allow outlining for functions having exactly matching return
10762 // address signing attributes, i.e., all share the same value for the
10763 // attribute "sign-return-address", all share the same type of key they
10764 // are signed with & all share the same pauth-lr attributes.
10765 // Additionally we require all functions to simultaneously either support
10766 // v8.3a features or not. Otherwise an outlined function could get signed
10767 // using dedicated v8.3 instructions and a call from a function that doesn't
10768 // support v8.3 instructions would therefore be invalid.
10769 if (std::adjacent_find(
10770 first: RepeatedSequenceLocs.begin(), last: RepeatedSequenceLocs.end(),
10771 binary_pred: [](const outliner::Candidate &a, const outliner::Candidate &b) {
10772 // Return true if a and b are non-equal w.r.t. return address
10773 // signing, support of v8.3a features or support of pauth-lr
10774 if (outliningCandidatesSigningScopeConsensus(a, b) &&
10775 outliningCandidatesSigningKeyConsensus(a, b) &&
10776 outliningCandidatesV8_3OpsConsensus(a, b) &&
10777 outliningCandidatesPAuthLRConsensus(a, b)) {
10778 return false;
10779 }
10780 return true;
10781 }) != RepeatedSequenceLocs.end()) {
10782 return std::nullopt;
10783 }
10784
10785 // Since at this point all candidates agree on their return address signing
10786 // picking just one is fine. If the candidate functions potentially sign their
10787 // return addresses, the outlined function should do the same. Note that in
10788 // the case of "sign-return-address"="non-leaf" this is an assumption: It is
10789 // not certainly true that the outlined function will have to sign its return
10790 // address but this decision is made later, when the decision to outline
10791 // has already been made.
10792 // The same holds for the number of additional instructions we need: On
10793 // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
10794 // necessary. However, at this point we don't know if the outlined function
10795 // will have a RET instruction so we assume the worst.
10796 const TargetRegisterInfo &TRI = getRegisterInfo();
10797 // Performing a tail call may require extra checks when PAuth is enabled.
10798 // If PAuth is disabled, set it to zero for uniformity.
10799 unsigned NumBytesToCheckLRInTCEpilogue = 0;
10800 const auto RASignCondition = RepeatedSequenceLocs[0]
10801 .getMF()
10802 ->getInfo<AArch64FunctionInfo>()
10803 ->getSignReturnAddressCondition();
10804 if (RASignCondition != SignReturnAddress::None) {
10805 // Candidates that have Return Address Authentication Hardening enabled are
10806 // discarded.
10807 //
10808 // In its current form, the machine outliner does not preserve X16/X17
10809 // across outlined function calls, even though it should as they are
10810 // caller-saved registers. And since the hardening based on load from the
10811 // return address may clobber one of these registers, if they are alive
10812 // across a call their value would be lost due to the hardening mechanism.
10813 llvm::erase_if(C&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10814 return C.getMF()
10815 ->getInfo<AArch64FunctionInfo>()
10816 ->shouldHardenSignReturnAddress();
10817 });
10818 // If the sequence doesn't have enough candidates left, then we're done.
10819 if (RepeatedSequenceLocs.size() < MinRepeats)
10820 return std::nullopt;
10821
10822 // PAuth is enabled - set extra tail call cost, if any.
10823 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
10824 MF: *RepeatedSequenceLocs[0].getMF());
10825 NumBytesToCheckLRInTCEpilogue =
10826 AArch64PAuth::getCheckerSizeInBytes(Method: LRCheckMethod);
10827 // Checking the authenticated LR value may significantly impact
10828 // SequenceSize, so account for it for more precise results.
10829 if (isTailCallReturnInst(MI: RepeatedSequenceLocs[0].back()))
10830 SequenceSize += NumBytesToCheckLRInTCEpilogue;
10831 }
10832
10833 // Properties about candidate MBBs that hold for all of them.
10834 unsigned FlagsSetInAll = 0xF;
10835
10836 // Compute liveness information for each candidate, and set FlagsSetInAll.
10837 for (outliner::Candidate &C : RepeatedSequenceLocs)
10838 FlagsSetInAll &= C.Flags;
10839
10840 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
10841
10842 // Helper lambda which sets call information for every candidate.
10843 auto SetCandidateCallInfo =
10844 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
10845 for (outliner::Candidate &C : RepeatedSequenceLocs)
10846 C.setCallInfo(CID: CallID, CO: NumBytesForCall);
10847 };
10848
10849 unsigned FrameID = MachineOutlinerDefault;
10850 NumBytesToCreateFrame += 4;
10851
10852 bool HasBTI = any_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10853 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
10854 });
10855
10856 // We check to see if CFI Instructions are present, and if they are
10857 // we find the number of CFI Instructions in the candidates.
10858 unsigned CFICount = 0;
10859 for (auto &I : RepeatedSequenceLocs[0]) {
10860 if (I.isCFIInstruction())
10861 CFICount++;
10862 }
10863
10864 // We compare the number of found CFI Instructions to the number of CFI
10865 // instructions in the parent function for each candidate. We must check this
10866 // since if we outline one of the CFI instructions in a function, we have to
10867 // outline them all for correctness. If we do not, the address offsets will be
10868 // incorrect between the two sections of the program.
10869 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10870 std::vector<MCCFIInstruction> CFIInstructions =
10871 C.getMF()->getFrameInstructions();
10872
10873 if (CFICount > 0 && CFICount != CFIInstructions.size())
10874 return std::nullopt;
10875 }
10876
10877 // Returns true if an instructions is safe to fix up, false otherwise.
10878 auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
10879 if (MI.isCall())
10880 return true;
10881
10882 if (!MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI) &&
10883 !MI.readsRegister(Reg: AArch64::SP, TRI: &TRI))
10884 return true;
10885
10886 // Any modification of SP will break our code to save/restore LR.
10887 // FIXME: We could handle some instructions which add a constant
10888 // offset to SP, with a bit more work.
10889 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI))
10890 return false;
10891
10892 // At this point, we have a stack instruction that we might need to
10893 // fix up. We'll handle it if it's a load or store.
10894 if (MI.mayLoadOrStore()) {
10895 const MachineOperand *Base; // Filled with the base operand of MI.
10896 int64_t Offset; // Filled with the offset of MI.
10897 bool OffsetIsScalable;
10898
10899 // Does it allow us to offset the base operand and is the base the
10900 // register SP?
10901 if (!getMemOperandWithOffset(MI, BaseOp&: Base, Offset, OffsetIsScalable) ||
10902 !Base->isReg() || Base->getReg() != AArch64::SP)
10903 return false;
10904
10905 // Fixe-up code below assumes bytes.
10906 if (OffsetIsScalable)
10907 return false;
10908
10909 // Find the minimum/maximum offset for this instruction and check
10910 // if fixing it up would be in range.
10911 int64_t MinOffset,
10912 MaxOffset; // Unscaled offsets for the instruction.
10913 // The scale to multiply the offsets by.
10914 TypeSize Scale(0U, false), DummyWidth(0U, false);
10915 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width&: DummyWidth, MinOffset, MaxOffset);
10916
10917 Offset += 16; // Update the offset to what it would be if we outlined.
10918 if (Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
10919 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
10920 return false;
10921
10922 // It's in range, so we can outline it.
10923 return true;
10924 }
10925
10926 if (isSPAddSubImm(MI))
10927 return adjustSPAddSubImm(MI, Delta: 16).has_value();
10928
10929 // FIXME: Add handling for other SP-based address calculations that can be
10930 // adjusted after outlining.
10931 return false;
10932 };
10933
10934 // True if it's possible to fix up each stack instruction in this sequence.
10935 // Important for frames/call variants that modify the stack.
10936 bool AllStackInstrsSafe =
10937 llvm::all_of(Range&: RepeatedSequenceLocs[0], P: IsSafeToFixup);
10938
10939 // If the last instruction in any candidate is a terminator, then we should
10940 // tail call all of the candidates.
10941 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10942 // Reject tail calls if they would require return address signing, as we
10943 // cannot safely sign them without corrupting the caller's inherited LR.
10944 if (AArch64FunctionInfo::shouldSignReturnAddress(
10945 Condition: RASignCondition,
10946 IsLRSpilled: any_of(Range&: RepeatedSequenceLocs[0], P: [](const MachineInstr &MI) {
10947 return MI.isCall() && !MI.isReturn();
10948 })))
10949 return std::nullopt;
10950
10951 FrameID = MachineOutlinerTailCall;
10952 NumBytesToCreateFrame = 0;
10953 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
10954 SetCandidateCallInfo(MachineOutlinerTailCall, NumBytesForCall);
10955 }
10956
10957 else if (LastInstrOpcode == AArch64::BL ||
10958 ((LastInstrOpcode == AArch64::BLR ||
10959 LastInstrOpcode == AArch64::BLRNoIP) &&
10960 !HasBTI)) {
10961 // FIXME: Do we need to check if the code after this uses the value of LR?
10962 FrameID = MachineOutlinerThunk;
10963 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
10964 SetCandidateCallInfo(MachineOutlinerThunk, 4);
10965 }
10966
10967 else {
10968 // We need to decide how to emit calls + frames. We can always emit the same
10969 // frame if we don't need to save to the stack. If we have to save to the
10970 // stack, then we need a different frame.
10971 unsigned NumBytesNoStackCalls = 0;
10972 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
10973
10974 // Check if we have to save LR.
10975 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10976 bool LRAvailable =
10977 (C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
10978 ? C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI)
10979 : true;
10980 // If we have a noreturn caller, then we're going to be conservative and
10981 // say that we have to save LR. If we don't have a ret at the end of the
10982 // block, then we can't reason about liveness accurately.
10983 //
10984 // FIXME: We can probably do better than always disabling this in
10985 // noreturn functions by fixing up the liveness info.
10986 bool IsNoReturn =
10987 C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoReturn);
10988
10989 // Is LR available? If so, we don't need a save.
10990 if (LRAvailable && !IsNoReturn) {
10991 NumBytesNoStackCalls += 4;
10992 C.setCallInfo(CID: MachineOutlinerNoLRSave, CO: 4);
10993 CandidatesWithoutStackFixups.push_back(x: C);
10994 }
10995
10996 // Is an unused register available? If so, we won't modify the stack, so
10997 // we can outline with the same frame type as those that don't save LR.
10998 else if (findRegisterToSaveLRTo(C)) {
10999 NumBytesNoStackCalls += 12;
11000 C.setCallInfo(CID: MachineOutlinerRegSave, CO: 12);
11001 CandidatesWithoutStackFixups.push_back(x: C);
11002 }
11003
11004 // Is SP used in the sequence at all? If not, we don't have to modify
11005 // the stack, so we are guaranteed to get the same frame.
11006 else if (C.isAvailableInsideSeq(Reg: AArch64::SP, TRI)) {
11007 NumBytesNoStackCalls += 12;
11008 C.setCallInfo(CID: MachineOutlinerDefault, CO: 12);
11009 CandidatesWithoutStackFixups.push_back(x: C);
11010 }
11011
11012 // If we outline this, we need to modify the stack. Pretend we don't
11013 // outline this by saving all of its bytes.
11014 else {
11015 NumBytesNoStackCalls += SequenceSize;
11016 }
11017 }
11018
11019 // If there are no places where we have to save LR, then note that we
11020 // don't have to update the stack. Otherwise, give every candidate the
11021 // default call type, as long as it's safe to do so.
11022 if (!AllStackInstrsSafe ||
11023 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
11024 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
11025 FrameID = MachineOutlinerNoLRSave;
11026 if (RepeatedSequenceLocs.size() < MinRepeats)
11027 return std::nullopt;
11028 } else {
11029 SetCandidateCallInfo(MachineOutlinerDefault, 12);
11030
11031 // Bugzilla ID: 46767
11032 // TODO: Check if fixing up the stack more than once is safe so we can
11033 // outline these.
11034 //
11035 // An outline resulting in a caller that requires stack fixups at the
11036 // callsite to a callee that also requires stack fixups can happen when
11037 // there are no available registers at the candidate callsite for a
11038 // candidate that itself also has calls.
11039 //
11040 // In other words if function_containing_sequence in the following pseudo
11041 // assembly requires that we save LR at the point of the call, but there
11042 // are no available registers: in this case we save using SP and as a
11043 // result the SP offsets requires stack fixups by multiples of 16.
11044 //
11045 // function_containing_sequence:
11046 // ...
11047 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
11048 // call OUTLINED_FUNCTION_N
11049 // restore LR from SP
11050 // ...
11051 //
11052 // OUTLINED_FUNCTION_N:
11053 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
11054 // ...
11055 // bl foo
11056 // restore LR from SP
11057 // ret
11058 //
11059 // Because the code to handle more than one stack fixup does not
11060 // currently have the proper checks for legality, these cases will assert
11061 // in the AArch64 MachineOutliner. This is because the code to do this
11062 // needs more hardening, testing, better checks that generated code is
11063 // legal, etc and because it is only verified to handle a single pass of
11064 // stack fixup.
11065 //
11066 // The assert happens in AArch64InstrInfo::buildOutlinedFrame to catch
11067 // these cases until they are known to be handled. Bugzilla 46767 is
11068 // referenced in comments at the assert site.
11069 //
11070 // To avoid asserting (or generating non-legal code on noassert builds)
11071 // we remove all candidates which would need more than one stack fixup by
11072 // pruning the cases where the candidate has calls while also having no
11073 // available LR and having no available general purpose registers to copy
11074 // LR to (ie one extra stack save/restore).
11075 //
11076 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
11077 erase_if(C&: RepeatedSequenceLocs, P: [this, &TRI](outliner::Candidate &C) {
11078 auto IsCall = [](const MachineInstr &MI) { return MI.isCall(); };
11079 return (llvm::any_of(Range&: C, P: IsCall)) &&
11080 (!C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI) ||
11081 !findRegisterToSaveLRTo(C));
11082 });
11083 }
11084 }
11085
11086 // If we dropped all of the candidates, bail out here.
11087 if (RepeatedSequenceLocs.size() < MinRepeats)
11088 return std::nullopt;
11089 }
11090
11091 // Does every candidate's MBB contain a call? If so, then we might have a call
11092 // in the range.
11093 bool ModStackToSaveLR = false;
11094 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
11095 // Check if the range contains a call. These require a save + restore of the
11096 // link register.
11097 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
11098 if (any_of(Range: drop_end(RangeOrContainer&: FirstCand),
11099 P: [](const MachineInstr &MI) { return MI.isCall(); }))
11100 ModStackToSaveLR = true;
11101
11102 // Handle the last instruction separately. If this is a tail call, then the
11103 // last instruction is a call. We don't want to save + restore in this case.
11104 // However, it could be possible that the last instruction is a call without
11105 // it being valid to tail call this sequence. We should consider this as
11106 // well.
11107 else if (FrameID != MachineOutlinerThunk &&
11108 FrameID != MachineOutlinerTailCall && FirstCand.back().isCall())
11109 ModStackToSaveLR = true;
11110
11111 if (ModStackToSaveLR) {
11112 // We can't fix up the stack. Bail out.
11113 if (!AllStackInstrsSafe)
11114 return std::nullopt;
11115
11116 // Save + restore LR.
11117 NumBytesToCreateFrame += 8;
11118
11119 // Add the extra mov if we will save a frame record instead of just LR.
11120 if (predictCompactUnwindFrameRecordForOutlinedFunction(
11121 RepeatedSequenceLocs, TRI))
11122 NumBytesToCreateFrame += 4;
11123 }
11124 }
11125
11126 // If we have CFI instructions, we can only outline if the outlined section
11127 // can be a tail call
11128 if (FrameID != MachineOutlinerTailCall && CFICount > 0)
11129 return std::nullopt;
11130
11131 if (FrameID != MachineOutlinerTailCall &&
11132 AArch64FunctionInfo::shouldSignReturnAddress(Condition: RASignCondition,
11133 IsLRSpilled: ModStackToSaveLR))
11134 NumBytesToCreateFrame += 8;
11135 if (RASignCondition == SignReturnAddress::All) {
11136 auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
11137 int SPValue = 0;
11138 for (const MachineInstr &MI : C) {
11139 if (!MI.isCall() && MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI)) {
11140 switch (MI.getOpcode()) {
11141 case AArch64::ADDXri:
11142 case AArch64::ADDWri:
11143 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
11144 assert(MI.getOperand(2).isImm() && "Expected immediate");
11145 assert(MI.getOperand(1).isReg() && "Expected register");
11146 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
11147 SPValue += MI.getOperand(i: 2).getImm();
11148 else
11149 return true;
11150 break;
11151 case AArch64::SUBXri:
11152 case AArch64::SUBWri:
11153 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
11154 assert(MI.getOperand(2).isImm() && "Expected immediate");
11155 assert(MI.getOperand(1).isReg() && "Expected register");
11156 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
11157 SPValue -= MI.getOperand(i: 2).getImm();
11158 else
11159 return true;
11160 break;
11161 default:
11162 return true;
11163 }
11164 }
11165 }
11166 return SPValue != 0;
11167 };
11168
11169 if (hasIllegalSPModification(RepeatedSequenceLocs[0]))
11170 return std::nullopt;
11171 }
11172
11173 return std::make_unique<outliner::OutlinedFunction>(
11174 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: NumBytesToCreateFrame, args&: FrameID);
11175}
11176
11177void AArch64InstrInfo::mergeOutliningCandidateAttributes(
11178 Function &F, std::vector<outliner::Candidate> &Candidates) const {
11179 // If a bunch of candidates reach this point they must agree on their return
11180 // address signing. It is therefore enough to just consider the signing
11181 // behaviour of one of them
11182 const auto &CFn = Candidates.front().getMF()->getFunction();
11183
11184 if (CFn.hasFnAttribute(Kind: "ptrauth-returns"))
11185 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-returns"));
11186 if (CFn.hasFnAttribute(Kind: "ptrauth-auth-traps"))
11187 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-auth-traps"));
11188 if (CFn.hasFnAttribute(Kind: "branch-protection-pauth-lr"))
11189 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "branch-protection-pauth-lr"));
11190 // Since all candidates belong to the same module, just copy the
11191 // function-level attributes of an arbitrary function.
11192 if (CFn.hasFnAttribute(Kind: "sign-return-address"))
11193 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address"));
11194 if (CFn.hasFnAttribute(Kind: "sign-return-address-key"))
11195 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address-key"));
11196 // The "sign-return-address-harden" attribute is not included because no
11197 // candidate is supposed to have hardening enabled.
11198 assert(llvm::none_of(Candidates, [](const outliner::Candidate &C) {
11199 return C.getMF()
11200 ->getInfo<AArch64FunctionInfo>()
11201 ->shouldHardenSignReturnAddress();
11202 }));
11203
11204 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
11205}
11206
11207bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
11208 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
11209 if (!TargetInstrInfo::isFunctionSafeToOutlineFrom(MF,
11210 OutlineFromLinkOnceODRs))
11211 return false;
11212
11213 // Outlining from functions with redzones is unsafe since the outliner may
11214 // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
11215 // outline from it.
11216 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
11217 if (!AFI || AFI->hasRedZone().value_or(u: true))
11218 return false;
11219
11220 // FIXME: Determine whether it is safe to outline from functions which contain
11221 // streaming-mode changes. We may need to ensure any smstart/smstop pairs are
11222 // outlined together and ensure it is safe to outline with async unwind info,
11223 // required for saving & restoring VG around calls.
11224 if (AFI->hasStreamingModeChanges())
11225 return false;
11226
11227 // FIXME: Teach the outliner to generate/handle Windows unwind info.
11228 if (MF.usesWindowsUnwindInfo())
11229 return false;
11230
11231 // It's safe to outline from MF.
11232 return true;
11233}
11234
11235SmallVector<std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
11236AArch64InstrInfo::getOutlinableRanges(MachineBasicBlock &MBB,
11237 unsigned &Flags) const {
11238 assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
11239 "Must track liveness!");
11240 SmallVector<
11241 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
11242 Ranges;
11243 // According to the AArch64 Procedure Call Standard, the following are
11244 // undefined on entry/exit from a function call:
11245 //
11246 // * Registers x16, x17, (and thus w16, w17)
11247 // * Condition codes (and thus the NZCV register)
11248 //
11249 // If any of these registers are used inside or live across an outlined
11250 // function, then they may be modified later, either by the compiler or
11251 // some other tool (like the linker).
11252 //
11253 // To avoid outlining in these situations, partition each block into ranges
11254 // where these registers are dead. We will only outline from those ranges.
11255 LiveRegUnits LRU(getRegisterInfo());
11256 auto AreAllUnsafeRegsDead = [&LRU]() {
11257 return LRU.available(Reg: AArch64::W16) && LRU.available(Reg: AArch64::W17) &&
11258 LRU.available(Reg: AArch64::NZCV);
11259 };
11260
11261 // We need to know if LR is live across an outlining boundary later on in
11262 // order to decide how we'll create the outlined call, frame, etc.
11263 //
11264 // It's pretty expensive to check this for *every candidate* within a block.
11265 // That's some potentially n^2 behaviour, since in the worst case, we'd need
11266 // to compute liveness from the end of the block for O(n) candidates within
11267 // the block.
11268 //
11269 // So, to improve the average case, let's keep track of liveness from the end
11270 // of the block to the beginning of *every outlinable range*. If we know that
11271 // LR is available in every range we could outline from, then we know that
11272 // we don't need to check liveness for any candidate within that range.
11273 bool LRAvailableEverywhere = true;
11274 // Compute liveness bottom-up.
11275 LRU.addLiveOuts(MBB);
11276 // Update flags that require info about the entire MBB.
11277 auto UpdateWholeMBBFlags = [&Flags](const MachineInstr &MI) {
11278 if (MI.isCall() && !MI.isTerminator())
11279 Flags |= MachineOutlinerMBBFlags::HasCalls;
11280 };
11281 // Range: [RangeBegin, RangeEnd)
11282 MachineBasicBlock::instr_iterator RangeBegin, RangeEnd;
11283 unsigned RangeLen;
11284 auto CreateNewRangeStartingAt =
11285 [&RangeBegin, &RangeEnd,
11286 &RangeLen](MachineBasicBlock::instr_iterator NewBegin) {
11287 RangeBegin = NewBegin;
11288 RangeEnd = std::next(x: RangeBegin);
11289 RangeLen = 0;
11290 };
11291 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
11292 // At least one unsafe register is not dead. We do not want to outline at
11293 // this point. If it is long enough to outline from and does not cross a
11294 // bundle boundary, save the range [RangeBegin, RangeEnd).
11295 if (RangeLen <= 1)
11296 return;
11297 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
11298 return;
11299 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
11300 return;
11301 Ranges.emplace_back(Args&: RangeBegin, Args&: RangeEnd);
11302 };
11303 // Find the first point where all unsafe registers are dead.
11304 // FIND: <safe instr> <-- end of first potential range
11305 // SKIP: <unsafe def>
11306 // SKIP: ... everything between ...
11307 // SKIP: <unsafe use>
11308 auto FirstPossibleEndPt = MBB.instr_rbegin();
11309 for (; FirstPossibleEndPt != MBB.instr_rend(); ++FirstPossibleEndPt) {
11310 if (!FirstPossibleEndPt->isDebugInstr())
11311 LRU.stepBackward(MI: *FirstPossibleEndPt);
11312 // Update flags that impact how we outline across the entire block,
11313 // regardless of safety.
11314 UpdateWholeMBBFlags(*FirstPossibleEndPt);
11315 if (AreAllUnsafeRegsDead())
11316 break;
11317 }
11318 // If we exhausted the entire block, we have no safe ranges to outline.
11319 if (FirstPossibleEndPt == MBB.instr_rend())
11320 return Ranges;
11321 // Current range.
11322 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
11323 // StartPt points to the first place where all unsafe registers
11324 // are dead (if there is any such point). Begin partitioning the MBB into
11325 // ranges.
11326 // Pseudo probes are ordinary instructions to the mapper, so only skip debug
11327 // instructions here.
11328 for (auto &MI : instructionsWithoutDebug(It: FirstPossibleEndPt, End: MBB.instr_rend(),
11329 /*SkipPseudoOp=*/false)) {
11330 LRU.stepBackward(MI);
11331 UpdateWholeMBBFlags(MI);
11332 if (!AreAllUnsafeRegsDead()) {
11333 SaveRangeIfNonEmpty();
11334 CreateNewRangeStartingAt(MI.getIterator());
11335 continue;
11336 }
11337 LRAvailableEverywhere &= LRU.available(Reg: AArch64::LR);
11338 // Pseudo probes count because the mapper treats them as ordinary
11339 // instructions.
11340 RangeBegin = MI.getIterator();
11341 ++RangeLen;
11342 }
11343 // Above loop misses the last (or only) range. If we are still safe, then
11344 // let's save the range.
11345 if (AreAllUnsafeRegsDead())
11346 SaveRangeIfNonEmpty();
11347 if (Ranges.empty())
11348 return Ranges;
11349 // We found the ranges bottom-up. Mapping expects the top-down. Reverse
11350 // the order.
11351 std::reverse(first: Ranges.begin(), last: Ranges.end());
11352 // If there is at least one outlinable range where LR is unavailable
11353 // somewhere, remember that.
11354 if (!LRAvailableEverywhere)
11355 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
11356 return Ranges;
11357}
11358
11359outliner::InstrType
11360AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
11361 MachineBasicBlock::iterator &MIT,
11362 unsigned Flags) const {
11363 MachineInstr &MI = *MIT;
11364
11365 // Don't outline anything used for return address signing. The outlined
11366 // function will get signed later if needed
11367 switch (MI.getOpcode()) {
11368 case AArch64::PACM:
11369 case AArch64::PACIASP:
11370 case AArch64::PACIBSP:
11371 case AArch64::PACIASPPC:
11372 case AArch64::PACIBSPPC:
11373 case AArch64::PACNBIASPPC:
11374 case AArch64::PACNBIBSPPC:
11375 case AArch64::AUTIASP:
11376 case AArch64::AUTIBSP:
11377 case AArch64::AUTIASPPCi:
11378 case AArch64::AUTIASPPCr:
11379 case AArch64::AUTIBSPPCi:
11380 case AArch64::AUTIBSPPCr:
11381 case AArch64::RETAA:
11382 case AArch64::RETAB:
11383 case AArch64::RETAASPPCi:
11384 case AArch64::RETAASPPCr:
11385 case AArch64::RETABSPPCi:
11386 case AArch64::RETABSPPCr:
11387 case AArch64::EMITBKEY:
11388 case AArch64::PAUTH_PROLOGUE:
11389 case AArch64::PAUTH_EPILOGUE:
11390 case AArch64::PAUTH_EPILOGUE_ENTRY_SP:
11391 return outliner::InstrType::Illegal;
11392 }
11393
11394 // We can only outline these if we will tail call the outlined function, or
11395 // fix up the CFI offsets. Currently, CFI instructions are outlined only if
11396 // in a tail call.
11397 //
11398 // FIXME: If the proper fixups for the offset are implemented, this should be
11399 // possible.
11400 if (MI.isCFIInstruction())
11401 return outliner::InstrType::Legal;
11402
11403 // Is this a terminator for a basic block?
11404 if (MI.isTerminator())
11405 // TargetInstrInfo::getOutliningType has already filtered out anything
11406 // that would break this, so we can allow it here.
11407 return outliner::InstrType::Legal;
11408
11409 // Make sure none of the operands are un-outlinable.
11410 for (const MachineOperand &MOP : MI.operands()) {
11411 // A check preventing CFI indices was here before, but only CFI
11412 // instructions should have those.
11413 assert(!MOP.isCFIIndex());
11414
11415 // If it uses LR or W30 explicitly, then don't touch it.
11416 if (MOP.isReg() && !MOP.isImplicit() &&
11417 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
11418 return outliner::InstrType::Illegal;
11419 }
11420
11421 // Special cases for instructions that can always be outlined, but will fail
11422 // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
11423 // be outlined because they don't require a *specific* value to be in LR.
11424 if (MI.getOpcode() == AArch64::ADRP)
11425 return outliner::InstrType::Legal;
11426
11427 // If MI is a call we might be able to outline it. We don't want to outline
11428 // any calls that rely on the position of items on the stack. When we outline
11429 // something containing a call, we have to emit a save and restore of LR in
11430 // the outlined function. Currently, this always happens by saving LR to the
11431 // stack. Thus, if we outline, say, half the parameters for a function call
11432 // plus the call, then we'll break the callee's expectations for the layout
11433 // of the stack.
11434 //
11435 // FIXME: Allow calls to functions which construct a stack frame, as long
11436 // as they don't access arguments on the stack.
11437 // FIXME: Figure out some way to analyze functions defined in other modules.
11438 // We should be able to compute the memory usage based on the IR calling
11439 // convention, even if we can't see the definition.
11440 if (MI.isCall()) {
11441 // Get the function associated with the call. Look at each operand and find
11442 // the one that represents the callee and get its name.
11443 const Function *Callee = nullptr;
11444 for (const MachineOperand &MOP : MI.operands()) {
11445 if (MOP.isGlobal()) {
11446 Callee = dyn_cast<Function>(Val: MOP.getGlobal());
11447 break;
11448 }
11449 }
11450
11451 // Never outline calls to mcount. There isn't any rule that would require
11452 // this, but the Linux kernel's "ftrace" feature depends on it.
11453 if (Callee && Callee->getName() == "\01_mcount")
11454 return outliner::InstrType::Illegal;
11455
11456 // If we don't know anything about the callee, assume it depends on the
11457 // stack layout of the caller. In that case, it's only legal to outline
11458 // as a tail-call. Explicitly list the call instructions we know about so we
11459 // don't get unexpected results with call pseudo-instructions.
11460 auto UnknownCallOutlineType = outliner::InstrType::Illegal;
11461 if (MI.getOpcode() == AArch64::BLR ||
11462 MI.getOpcode() == AArch64::BLRNoIP || MI.getOpcode() == AArch64::BL)
11463 UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
11464
11465 if (!Callee)
11466 return UnknownCallOutlineType;
11467
11468 // We have a function we have information about. Check it if it's something
11469 // can safely outline.
11470 MachineFunction *CalleeMF = MMI.getMachineFunction(F: *Callee);
11471
11472 // We don't know what's going on with the callee at all. Don't touch it.
11473 if (!CalleeMF)
11474 return UnknownCallOutlineType;
11475
11476 // Check if we know anything about the callee saves on the function. If we
11477 // don't, then don't touch it, since that implies that we haven't
11478 // computed anything about its stack frame yet.
11479 MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
11480 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
11481 MFI.getNumObjects() > 0)
11482 return UnknownCallOutlineType;
11483
11484 // At this point, we can say that CalleeMF ought to not pass anything on the
11485 // stack. Therefore, we can outline it.
11486 return outliner::InstrType::Legal;
11487 }
11488
11489 // Don't touch the link register or W30.
11490 if (MI.readsRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()) ||
11491 MI.modifiesRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()))
11492 return outliner::InstrType::Illegal;
11493
11494 // Don't outline BTI instructions, because that will prevent the outlining
11495 // site from being indirectly callable.
11496 if (hasBTISemantics(MI))
11497 return outliner::InstrType::Illegal;
11498
11499 return outliner::InstrType::Legal;
11500}
11501
11502void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
11503 for (MachineInstr &MI : MBB) {
11504 const MachineOperand *Base;
11505 TypeSize Width(0, false);
11506 int64_t Offset;
11507 bool OffsetIsScalable;
11508
11509 // Is this a load or store with an immediate offset with SP as the base?
11510 if (MI.mayLoadOrStore() &&
11511 getMemOperandWithOffsetWidth(LdSt: MI, BaseOp&: Base, Offset, OffsetIsScalable,
11512 Width) &&
11513 Base->isReg() && Base->getReg() == AArch64::SP) {
11514 TypeSize Scale(0U, false);
11515 int64_t Dummy1, Dummy2;
11516
11517 MachineOperand &StackOffsetOperand =
11518 getMemOpBaseRegImmOfsOffsetOperand(LdSt&: MI);
11519 assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
11520 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2);
11521 assert(Scale != 0 && "Unexpected opcode!");
11522 assert(!OffsetIsScalable && "Expected offset to be a byte offset");
11523
11524 // We've pushed the return address to the stack, so add 16 to the offset.
11525 // This is safe, since we already checked if it would overflow when we
11526 // checked if this instruction was legal to outline.
11527 int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedValue();
11528 StackOffsetOperand.setImm(NewImm);
11529 continue;
11530 }
11531
11532 if (isSPAddSubImm(MI)) {
11533 auto Enc = adjustSPAddSubImm(MI, Delta: 16);
11534 assert(Enc && "Unencodable SP add/sub should not have been outlined");
11535 MI.setDesc(get(Opcode: Enc->Opcode));
11536 MI.getOperand(i: 2).setImm(Enc->Imm);
11537 MI.getOperand(i: 3).setImm(Enc->Shift);
11538 }
11539 }
11540}
11541
11542static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
11543 const AArch64InstrInfo *TII,
11544 bool ShouldSignReturnAddr) {
11545 if (!ShouldSignReturnAddr)
11546 return;
11547
11548 // Thunks do not naturally have LR in their live-ins, but PAC signing
11549 // requires it. Add it to prevent machine verifier crashes.
11550 if (!MBB.isLiveIn(Reg: AArch64::LR))
11551 MBB.addLiveIn(PhysReg: AArch64::LR);
11552
11553 BuildMI(BB&: MBB, I: MBB.begin(), MIMD: DebugLoc(), MCID: TII->get(Opcode: AArch64::PAUTH_PROLOGUE))
11554 .setMIFlag(MachineInstr::FrameSetup);
11555 TII->createPauthEpilogueInstr(MBB, DL: DebugLoc());
11556}
11557
11558void AArch64InstrInfo::buildOutlinedFrame(
11559 MachineBasicBlock &MBB, MachineFunction &MF,
11560 const outliner::OutlinedFunction &OF) const {
11561
11562 AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
11563
11564 if (OF.FrameConstructionID == MachineOutlinerTailCall)
11565 FI->setOutliningStyle("Tail Call");
11566 else if (OF.FrameConstructionID == MachineOutlinerThunk) {
11567 // For thunk outlining, rewrite the last instruction from a call to a
11568 // tail-call.
11569 MachineInstr *Call = &*--MBB.instr_end();
11570 unsigned TailOpcode;
11571 if (Call->getOpcode() == AArch64::BL) {
11572 TailOpcode = AArch64::TCRETURNdi;
11573 } else {
11574 assert(Call->getOpcode() == AArch64::BLR ||
11575 Call->getOpcode() == AArch64::BLRNoIP);
11576 TailOpcode = AArch64::TCRETURNriALL;
11577 }
11578 MachineInstr *TC = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpcode))
11579 .add(MO: Call->getOperand(i: 0))
11580 .addImm(Val: 0);
11581 MBB.insert(I: MBB.end(), MI: TC);
11582 Call->eraseFromParent();
11583
11584 FI->setOutliningStyle("Thunk");
11585 }
11586
11587 bool IsLeafFunction = true;
11588
11589 // Is there a call in the outlined range?
11590 auto IsNonTailCall = [](const MachineInstr &MI) {
11591 return MI.isCall() && !MI.isReturn();
11592 };
11593
11594 if (llvm::any_of(Range: MBB.instrs(), P: IsNonTailCall)) {
11595 // Fix up the instructions in the range, since we're going to modify the
11596 // stack.
11597
11598 // Bugzilla ID: 46767
11599 // TODO: Check if fixing up twice is safe so we can outline these.
11600 assert(OF.FrameConstructionID != MachineOutlinerDefault &&
11601 "Can only fix up stack references once");
11602 fixupPostOutline(MBB);
11603
11604 IsLeafFunction = false;
11605
11606 // LR has to be a live in so that we can save it.
11607 if (!MBB.isLiveIn(Reg: AArch64::LR))
11608 MBB.addLiveIn(PhysReg: AArch64::LR);
11609
11610 MachineBasicBlock::iterator It = MBB.begin();
11611 MachineBasicBlock::iterator Et = MBB.end();
11612
11613 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11614 OF.FrameConstructionID == MachineOutlinerThunk)
11615 Et = std::prev(x: MBB.end());
11616
11617 // There is a call in the range, so we must save LR. Save it as part of a
11618 // frame record when that gives us a smaller compact unwind encoding.
11619 if (shouldUseCompactUnwindFrameRecordForOutlinedFunction(MBB)) {
11620 // FP is saved here, so it must be live-in.
11621 if (!MBB.isLiveIn(Reg: AArch64::FP))
11622 MBB.addLiveIn(PhysReg: AArch64::FP);
11623
11624 // stp x29, x30, [sp, #-16]! (the pre-index imm is scaled by 8: -2 * 8)
11625 MachineInstr *STPXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STPXpre))
11626 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11627 .addReg(RegNo: AArch64::FP)
11628 .addReg(RegNo: AArch64::LR)
11629 .addReg(RegNo: AArch64::SP)
11630 .addImm(Val: -2);
11631 It = MBB.insert(I: It, MI: STPXpre);
11632
11633 // mov x29, sp (add x29, sp, #0), so x29 points at the frame record.
11634 MachineInstr *SetFP = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ADDXri))
11635 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11636 .addReg(RegNo: AArch64::SP)
11637 .addImm(Val: 0)
11638 .addImm(Val: 0);
11639 MBB.insertAfter(I: It, MI: SetFP);
11640
11641 // Describe the frame record with FP as the CFA. The encoder needs all
11642 // three to pick FRAME. No need to check for unwind info here: we only
11643 // get here if the function has it.
11644 CFIInstBuilder CFIBuilder(MBB, std::next(x: SetFP->getIterator()),
11645 MachineInstr::FrameSetup);
11646 CFIBuilder.buildDefCFA(Reg: AArch64::FP, Offset: 16);
11647 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -8);
11648 CFIBuilder.buildOffset(Reg: AArch64::FP, Offset: -16);
11649
11650 // ldp x29, x30, [sp], #16
11651 MachineInstr *LDPXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDPXpost))
11652 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11653 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11654 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11655 .addReg(RegNo: AArch64::SP)
11656 .addImm(Val: 2);
11657 Et = MBB.insert(I: Et, MI: LDPXpost);
11658 } else {
11659 // Insert a save before the outlined region
11660 MachineInstr *STRXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11661 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11662 .addReg(RegNo: AArch64::LR)
11663 .addReg(RegNo: AArch64::SP)
11664 .addImm(Val: -16);
11665 It = MBB.insert(I: It, MI: STRXpre);
11666
11667 if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
11668 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
11669
11670 // Add a CFI saying the stack was moved 16 B down.
11671 CFIBuilder.buildDefCFAOffset(Offset: 16);
11672
11673 // Add a CFI saying that the LR that we want to find is now 16 B higher
11674 // than before.
11675 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -16);
11676 }
11677
11678 // Insert a restore before the terminator for the function.
11679 MachineInstr *LDRXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11680 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11681 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11682 .addReg(RegNo: AArch64::SP)
11683 .addImm(Val: 16);
11684 Et = MBB.insert(I: Et, MI: LDRXpost);
11685 }
11686 }
11687
11688 auto RASignCondition = FI->getSignReturnAddressCondition();
11689 bool ShouldSignReturnAddr = AArch64FunctionInfo::shouldSignReturnAddress(
11690 Condition: RASignCondition, IsLRSpilled: !IsLeafFunction);
11691
11692 // If this is a tail call outlined function, then there's already a return.
11693 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11694 OF.FrameConstructionID == MachineOutlinerThunk) {
11695 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11696 return;
11697 }
11698
11699 // It's not a tail call, so we have to insert the return ourselves.
11700
11701 // LR has to be a live in so that we can return to it.
11702 if (!MBB.isLiveIn(Reg: AArch64::LR))
11703 MBB.addLiveIn(PhysReg: AArch64::LR);
11704
11705 MachineInstr *ret = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::RET))
11706 .addReg(RegNo: AArch64::LR);
11707 MBB.insert(I: MBB.end(), MI: ret);
11708
11709 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11710
11711 FI->setOutliningStyle("Function");
11712
11713 // Did we have to modify the stack by saving the link register?
11714 if (OF.FrameConstructionID != MachineOutlinerDefault)
11715 return;
11716
11717 // We modified the stack.
11718 // Walk over the basic block and fix up all the stack accesses.
11719 fixupPostOutline(MBB);
11720}
11721
11722MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
11723 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
11724 MachineFunction &MF, outliner::Candidate &C) const {
11725
11726 // Are we tail calling?
11727 if (C.CallConstructionID == MachineOutlinerTailCall) {
11728 // If yes, then we can just branch to the label.
11729 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::TCRETURNdi))
11730 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()))
11731 .addImm(Val: 0));
11732 return It;
11733 }
11734
11735 // Are we saving the link register?
11736 if (C.CallConstructionID == MachineOutlinerNoLRSave ||
11737 C.CallConstructionID == MachineOutlinerThunk) {
11738 // No, so just insert the call.
11739 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11740 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11741 return It;
11742 }
11743
11744 // We want to return the spot where we inserted the call.
11745 MachineBasicBlock::iterator CallPt;
11746
11747 // Instructions for saving and restoring LR around the call instruction we're
11748 // going to insert.
11749 MachineInstr *Save;
11750 MachineInstr *Restore;
11751 // Can we save to a register?
11752 if (C.CallConstructionID == MachineOutlinerRegSave) {
11753 // FIXME: This logic should be sunk into a target-specific interface so that
11754 // we don't have to recompute the register.
11755 Register Reg = findRegisterToSaveLRTo(C);
11756 assert(Reg && "No callee-saved register available?");
11757
11758 // LR has to be a live in so that we can save it.
11759 if (!MBB.isLiveIn(Reg: AArch64::LR))
11760 MBB.addLiveIn(PhysReg: AArch64::LR);
11761
11762 // Save and restore LR from Reg.
11763 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: Reg)
11764 .addReg(RegNo: AArch64::XZR)
11765 .addReg(RegNo: AArch64::LR)
11766 .addImm(Val: 0);
11767 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: AArch64::LR)
11768 .addReg(RegNo: AArch64::XZR)
11769 .addReg(RegNo: Reg)
11770 .addImm(Val: 0);
11771 } else {
11772 // We have the default case. Save and restore from SP.
11773 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11774 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11775 .addReg(RegNo: AArch64::LR)
11776 .addReg(RegNo: AArch64::SP)
11777 .addImm(Val: -16);
11778 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11779 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11780 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11781 .addReg(RegNo: AArch64::SP)
11782 .addImm(Val: 16);
11783 }
11784
11785 It = MBB.insert(I: It, MI: Save);
11786 It++;
11787
11788 // Insert the call.
11789 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11790 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11791 CallPt = It;
11792 It++;
11793
11794 It = MBB.insert(I: It, MI: Restore);
11795 return CallPt;
11796}
11797
11798bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
11799 MachineFunction &MF) const {
11800 return MF.getFunction().hasMinSize();
11801}
11802
11803void AArch64InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
11804 MachineBasicBlock::iterator Iter,
11805 DebugLoc &DL,
11806 bool AllowSideEffects) const {
11807 const MachineFunction &MF = *MBB.getParent();
11808 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
11809 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
11810
11811 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
11812 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg).addImm(Val: 0).addImm(Val: 0);
11813 } else if (STI.isSVEorStreamingSVEAvailable()) {
11814 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::DUP_ZI_D), DestReg: Reg)
11815 .addImm(Val: 0)
11816 .addImm(Val: 0);
11817 } else if (STI.isNeonAvailable()) {
11818 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVIv2d_ns), DestReg: Reg)
11819 .addImm(Val: 0);
11820 } else {
11821 // No Advanced SIMD (streaming-compatible without SVE, or +nosimd), so use
11822 // `fmov d...` instead of `movi v...`; writing `d` also clears the upper
11823 // 64 bits.
11824 assert(STI.hasFPARMv8() && "Expected FP to be available.");
11825 Register Reg64 = TRI.getSubReg(Reg, Idx: AArch64::dsub);
11826 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg: Reg64);
11827 }
11828}
11829
11830std::optional<DestSourcePair>
11831AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
11832
11833 // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
11834 // and zero immediate operands used as an alias for mov instruction.
11835 if ((MI.getOpcode() == AArch64::ORRWrs &&
11836 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11837 MI.getOperand(i: 3).getImm() == 0x0) ||
11838 (MI.getOpcode() == AArch64::ORRWrr &&
11839 MI.getOperand(i: 1).getReg() == AArch64::WZR)) {
11840 // Check that the w->w move is not a zero-extending w->x mov.
11841 if ((MI.getOperand(i: 0).getReg().isPhysical() &&
11842 MI.findRegisterDefOperandIdx(
11843 Reg: getXRegFromWReg(Reg: MI.getOperand(i: 0).getReg()),
11844 /*TRI=*/nullptr) == -1) ||
11845 (MI.getOperand(i: 0).getReg().isVirtual() &&
11846 !MI.getOperand(i: 0).getSubReg()))
11847 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11848 }
11849
11850 if (MI.getOpcode() == AArch64::ORRXrs &&
11851 MI.getOperand(i: 1).getReg() == AArch64::XZR &&
11852 MI.getOperand(i: 3).getImm() == 0x0)
11853 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11854
11855 return std::nullopt;
11856}
11857
11858std::optional<DestSourcePair>
11859AArch64InstrInfo::isCopyLikeInstrImpl(const MachineInstr &MI) const {
11860 if ((MI.getOpcode() == AArch64::ORRWrs &&
11861 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11862 MI.getOperand(i: 3).getImm() == 0x0) ||
11863 (MI.getOpcode() == AArch64::ORRWrr &&
11864 MI.getOperand(i: 1).getReg() == AArch64::WZR))
11865 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11866 return std::nullopt;
11867}
11868
11869std::optional<RegImmPair>
11870AArch64InstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
11871 int Sign = 1;
11872 int64_t Offset = 0;
11873
11874 // TODO: Handle cases where Reg is a super- or sub-register of the
11875 // destination register.
11876 const MachineOperand &Op0 = MI.getOperand(i: 0);
11877 if (!Op0.isReg() || Reg != Op0.getReg())
11878 return std::nullopt;
11879
11880 switch (MI.getOpcode()) {
11881 default:
11882 return std::nullopt;
11883 case AArch64::SUBWri:
11884 case AArch64::SUBXri:
11885 case AArch64::SUBSWri:
11886 case AArch64::SUBSXri:
11887 Sign *= -1;
11888 [[fallthrough]];
11889 case AArch64::ADDSWri:
11890 case AArch64::ADDSXri:
11891 case AArch64::ADDWri:
11892 case AArch64::ADDXri: {
11893 // TODO: Third operand can be global address (usually some string).
11894 if (!MI.getOperand(i: 0).isReg() || !MI.getOperand(i: 1).isReg() ||
11895 !MI.getOperand(i: 2).isImm())
11896 return std::nullopt;
11897 int Shift = MI.getOperand(i: 3).getImm();
11898 assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
11899 Offset = Sign * (MI.getOperand(i: 2).getImm() << Shift);
11900 }
11901 }
11902 return RegImmPair{MI.getOperand(i: 1).getReg(), Offset};
11903}
11904
11905/// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
11906/// the destination register then, if possible, describe the value in terms of
11907/// the source register.
11908static std::optional<ParamLoadedValue>
11909describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
11910 const TargetInstrInfo *TII,
11911 const TargetRegisterInfo *TRI) {
11912 auto DestSrc = TII->isCopyLikeInstr(MI);
11913 if (!DestSrc)
11914 return std::nullopt;
11915
11916 Register DestReg = DestSrc->Destination->getReg();
11917 Register SrcReg = DestSrc->Source->getReg();
11918
11919 if (!DestReg.isValid() || !SrcReg.isValid())
11920 return std::nullopt;
11921
11922 auto Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: {});
11923
11924 // If the described register is the destination, just return the source.
11925 if (DestReg == DescribedReg)
11926 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11927
11928 // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
11929 if (MI.getOpcode() == AArch64::ORRWrs &&
11930 TRI->isSuperRegister(RegA: DestReg, RegB: DescribedReg))
11931 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11932
11933 // We may need to describe the lower part of a ORRXrs move.
11934 if (MI.getOpcode() == AArch64::ORRXrs &&
11935 TRI->isSubRegister(RegA: DestReg, RegB: DescribedReg)) {
11936 Register SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: AArch64::sub_32);
11937 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcSubReg, isDef: false), Expr);
11938 }
11939
11940 assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
11941 "Unhandled ORR[XW]rs copy case");
11942
11943 return std::nullopt;
11944}
11945
11946bool AArch64InstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
11947 // Functions cannot be split to different sections on AArch64 if they have
11948 // a red zone. This is because relaxing a cross-section branch may require
11949 // incrementing the stack pointer to spill a register, which would overwrite
11950 // the red zone.
11951 if (MF.getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(u: true))
11952 return false;
11953
11954 return TargetInstrInfo::isFunctionSafeToSplit(MF);
11955}
11956
11957bool AArch64InstrInfo::isMBBSafeToSplitToCold(
11958 const MachineBasicBlock &MBB) const {
11959 // Asm Goto blocks can contain conditional branches to goto labels, which can
11960 // get moved out of range of the branch instruction.
11961 auto isAsmGoto = [](const MachineInstr &MI) {
11962 return MI.getOpcode() == AArch64::INLINEASM_BR;
11963 };
11964 if (llvm::any_of(Range: MBB, P: isAsmGoto) || MBB.isInlineAsmBrIndirectTarget())
11965 return false;
11966
11967 // Because jump tables are label-relative instead of table-relative, they all
11968 // must be in the same section or relocation fixup handling will fail.
11969
11970 // Check if MBB is a jump table target
11971 const MachineJumpTableInfo *MJTI = MBB.getParent()->getJumpTableInfo();
11972 auto containsMBB = [&MBB](const MachineJumpTableEntry &JTE) {
11973 return llvm::is_contained(Range: JTE.MBBs, Element: &MBB);
11974 };
11975 if (MJTI != nullptr && llvm::any_of(Range: MJTI->getJumpTables(), P: containsMBB))
11976 return false;
11977
11978 // Check if MBB contains a jump table lookup
11979 for (const MachineInstr &MI : MBB) {
11980 switch (MI.getOpcode()) {
11981 case TargetOpcode::G_BRJT:
11982 case AArch64::JumpTableDest32:
11983 case AArch64::JumpTableDest16:
11984 case AArch64::JumpTableDest8:
11985 return false;
11986 default:
11987 continue;
11988 }
11989 }
11990
11991 // MBB isn't a special case, so it's safe to be split to the cold section.
11992 return true;
11993}
11994
11995std::optional<ParamLoadedValue>
11996AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
11997 Register Reg) const {
11998 const MachineFunction *MF = MI.getMF();
11999 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
12000 switch (MI.getOpcode()) {
12001 case AArch64::MOVZWi:
12002 case AArch64::MOVZXi: {
12003 // MOVZWi may be used for producing zero-extended 32-bit immediates in
12004 // 64-bit parameters, so we need to consider super-registers.
12005 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
12006 return std::nullopt;
12007
12008 if (!MI.getOperand(i: 1).isImm())
12009 return std::nullopt;
12010 int64_t Immediate = MI.getOperand(i: 1).getImm();
12011 int Shift = MI.getOperand(i: 2).getImm();
12012 return ParamLoadedValue(MachineOperand::CreateImm(Val: Immediate << Shift),
12013 nullptr);
12014 }
12015 case AArch64::ORRWrs:
12016 case AArch64::ORRXrs:
12017 return describeORRLoadedValue(MI, DescribedReg: Reg, TII: this, TRI);
12018 }
12019
12020 return TargetInstrInfo::describeLoadedValue(MI, Reg);
12021}
12022
12023bool AArch64InstrInfo::isExtendLikelyToBeFolded(
12024 MachineInstr &ExtMI, MachineRegisterInfo &MRI) const {
12025 assert(ExtMI.getOpcode() == TargetOpcode::G_SEXT ||
12026 ExtMI.getOpcode() == TargetOpcode::G_ZEXT ||
12027 ExtMI.getOpcode() == TargetOpcode::G_ANYEXT);
12028
12029 // Anyexts are nops.
12030 if (ExtMI.getOpcode() == TargetOpcode::G_ANYEXT)
12031 return true;
12032
12033 Register DefReg = ExtMI.getOperand(i: 0).getReg();
12034 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
12035 return false;
12036
12037 // It's likely that a sext/zext as a G_PTR_ADD offset will be folded into an
12038 // addressing mode.
12039 auto *UserMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
12040 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
12041}
12042
12043uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
12044 return get(Opcode: Opc).TSFlags & AArch64::ElementSizeMask;
12045}
12046
12047bool AArch64InstrInfo::isPTestLikeOpcode(unsigned Opc) const {
12048 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsPTestLike;
12049}
12050
12051bool AArch64InstrInfo::isWhileOpcode(unsigned Opc) const {
12052 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsWhile;
12053}
12054
12055unsigned int
12056AArch64InstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
12057 return OptLevel >= CodeGenOptLevel::Aggressive ? 6 : 2;
12058}
12059
12060bool AArch64InstrInfo::isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
12061 unsigned Scale) const {
12062 if (Offset && Scale)
12063 return false;
12064
12065 // Check Reg + Imm
12066 if (!Scale) {
12067 // 9-bit signed offset
12068 if (isInt<9>(x: Offset))
12069 return true;
12070
12071 // 12-bit unsigned offset
12072 unsigned Shift = Log2_64(Value: NumBytes);
12073 if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
12074 // Must be a multiple of NumBytes (NumBytes is a power of 2)
12075 (Offset >> Shift) << Shift == Offset)
12076 return true;
12077 return false;
12078 }
12079
12080 // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
12081 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
12082}
12083
12084unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
12085 if (MF.getSubtarget<AArch64Subtarget>().hardenSlsBlr())
12086 return AArch64::BLRNoIP;
12087 else
12088 return AArch64::BLR;
12089}
12090
12091void AArch64InstrInfo::createPauthEpilogueInstr(MachineBasicBlock &MBB,
12092 DebugLoc DL) const {
12093 MachineBasicBlock::iterator InsertPt = MBB.getFirstTerminator();
12094 auto Builder = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: get(Opcode: AArch64::PAUTH_EPILOGUE))
12095 .setMIFlag(MachineInstr::FrameDestroy);
12096
12097 MachineFunction &MF = *MBB.getParent();
12098 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
12099 auto &AFL = *static_cast<const AArch64FrameLowering *>(
12100 MF.getSubtarget().getFrameLowering());
12101 if (AFL.getArgumentStackToRestore(MF, MBB)) {
12102 Builder.addReg(RegNo: AArch64::X17, Flags: RegState::ImplicitDefine);
12103 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
12104 if (AFI->branchProtectionPAuthLR())
12105 Builder.addReg(RegNo: AArch64::X15, Flags: RegState::ImplicitDefine);
12106 return;
12107 }
12108
12109 if (AFI->branchProtectionPAuthLR() && !Subtarget.hasPAuthLR())
12110 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
12111}
12112
12113MachineBasicBlock::iterator
12114AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI,
12115 Register TargetReg, bool FrameSetup) const {
12116 assert(TargetReg != AArch64::SP && "New top of stack cannot already be in SP");
12117
12118 MachineBasicBlock &MBB = *MBBI->getParent();
12119 MachineFunction &MF = *MBB.getParent();
12120 const AArch64InstrInfo *TII =
12121 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
12122 int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
12123 DebugLoc DL = MBB.findDebugLoc(MBBI);
12124
12125 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
12126 MachineBasicBlock *LoopTestMBB =
12127 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
12128 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
12129 MachineBasicBlock *LoopBodyMBB =
12130 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
12131 MF.insert(MBBI: MBBInsertPoint, MBB: LoopBodyMBB);
12132 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
12133 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
12134 MachineInstr::MIFlag Flags =
12135 FrameSetup ? MachineInstr::FrameSetup : MachineInstr::NoFlags;
12136
12137 // LoopTest:
12138 // SUB SP, SP, #ProbeSize
12139 emitFrameOffset(MBB&: *LoopTestMBB, MBBI: LoopTestMBB->end(), DL, DestReg: AArch64::SP,
12140 SrcReg: AArch64::SP, Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII, Flag: Flags);
12141
12142 // CMP SP, TargetReg
12143 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrx64),
12144 DestReg: AArch64::XZR)
12145 .addReg(RegNo: AArch64::SP)
12146 .addReg(RegNo: TargetReg)
12147 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0))
12148 .setMIFlags(Flags);
12149
12150 // B.<Cond> LoopExit
12151 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
12152 .addImm(Val: AArch64CC::LE)
12153 .addMBB(MBB: ExitMBB)
12154 .setMIFlags(Flags);
12155
12156 // LDR XZR, [SP]
12157 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
12158 .addDef(RegNo: AArch64::XZR)
12159 .addReg(RegNo: AArch64::SP)
12160 .addImm(Val: 0)
12161 .addMemOperand(MMO: MF.getMachineMemOperand(
12162 PtrInfo: MachinePointerInfo::getUnknownStack(MF),
12163 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 8,
12164 BaseAlignment: Align(8)))
12165 .setMIFlags(Flags);
12166
12167 // B loop
12168 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
12169 .addMBB(MBB: LoopTestMBB)
12170 .setMIFlags(Flags);
12171
12172 // LoopExit:
12173 // MOV SP, TargetReg
12174 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: AArch64::SP)
12175 .addReg(RegNo: TargetReg)
12176 .addImm(Val: 0)
12177 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
12178 .setMIFlags(Flags);
12179
12180 // LDR XZR, [SP]
12181 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
12182 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define)
12183 .addReg(RegNo: AArch64::SP)
12184 .addImm(Val: 0)
12185 .setMIFlags(Flags);
12186
12187 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
12188 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
12189
12190 LoopTestMBB->addSuccessor(Succ: ExitMBB);
12191 LoopTestMBB->addSuccessor(Succ: LoopBodyMBB);
12192 LoopBodyMBB->addSuccessor(Succ: LoopTestMBB);
12193 MBB.addSuccessor(Succ: LoopTestMBB);
12194
12195 // Update liveins.
12196 if (MF.getRegInfo().reservedRegsFrozen())
12197 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopBodyMBB, LoopTestMBB});
12198
12199 return ExitMBB->begin();
12200}
12201
12202namespace {
12203class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
12204 MachineFunction *MF;
12205 const TargetInstrInfo *TII;
12206 const TargetRegisterInfo *TRI;
12207 MachineRegisterInfo &MRI;
12208
12209 /// The block of the loop
12210 MachineBasicBlock *LoopBB;
12211 /// The conditional branch of the loop
12212 MachineInstr *CondBranch;
12213 /// The compare instruction for loop control
12214 MachineInstr *Comp;
12215 /// The number of the operand of the loop counter value in Comp
12216 unsigned CompCounterOprNum;
12217 /// The instruction that updates the loop counter value
12218 MachineInstr *Update;
12219 /// The number of the operand of the loop counter value in Update
12220 unsigned UpdateCounterOprNum;
12221 /// The initial value of the loop counter
12222 Register Init;
12223 /// True iff Update is a predecessor of Comp
12224 bool IsUpdatePriorComp;
12225
12226 /// The normalized condition used by createTripCountGreaterCondition()
12227 SmallVector<MachineOperand, 4> Cond;
12228
12229public:
12230 AArch64PipelinerLoopInfo(MachineBasicBlock *LoopBB, MachineInstr *CondBranch,
12231 MachineInstr *Comp, unsigned CompCounterOprNum,
12232 MachineInstr *Update, unsigned UpdateCounterOprNum,
12233 Register Init, bool IsUpdatePriorComp,
12234 const SmallVectorImpl<MachineOperand> &Cond)
12235 : MF(Comp->getParent()->getParent()),
12236 TII(MF->getSubtarget().getInstrInfo()),
12237 TRI(MF->getSubtarget().getRegisterInfo()), MRI(MF->getRegInfo()),
12238 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
12239 CompCounterOprNum(CompCounterOprNum), Update(Update),
12240 UpdateCounterOprNum(UpdateCounterOprNum), Init(Init),
12241 IsUpdatePriorComp(IsUpdatePriorComp), Cond(Cond.begin(), Cond.end()) {}
12242
12243 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
12244 // Make the instructions for loop control be placed in stage 0.
12245 // The predecessors of Comp are considered by the caller.
12246 return MI == Comp;
12247 }
12248
12249 std::optional<bool> createTripCountGreaterCondition(
12250 int TC, MachineBasicBlock &MBB,
12251 SmallVectorImpl<MachineOperand> &CondParam) override {
12252 // A branch instruction will be inserted as "if (Cond) goto epilogue".
12253 // Cond is normalized for such use.
12254 // The predecessors of the branch are assumed to have already been inserted.
12255 CondParam = Cond;
12256 return {};
12257 }
12258
12259 void createRemainingIterationsGreaterCondition(
12260 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
12261 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) override;
12262
12263 void setPreheader(MachineBasicBlock *NewPreheader) override {}
12264
12265 void adjustTripCount(int TripCountAdjust) override {}
12266
12267 bool isMVEExpanderSupported() override { return true; }
12268};
12269} // namespace
12270
12271/// Clone an instruction from MI. The register of ReplaceOprNum-th operand
12272/// is replaced by ReplaceReg. The output register is newly created.
12273/// The other operands are unchanged from MI.
12274static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum,
12275 Register ReplaceReg, MachineBasicBlock &MBB,
12276 MachineBasicBlock::iterator InsertTo) {
12277 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
12278 const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
12279 MachineInstr *NewMI = MBB.getParent()->CloneMachineInstr(Orig: MI);
12280 Register Result = 0;
12281 for (unsigned I = 0; I < NewMI->getNumOperands(); ++I) {
12282 if (I == 0 && NewMI->getOperand(i: 0).getReg().isVirtual()) {
12283 Result = MRI.createVirtualRegister(
12284 RegClass: MRI.getRegClass(Reg: NewMI->getOperand(i: 0).getReg()));
12285 NewMI->getOperand(i: I).setReg(Result);
12286 } else if (I == ReplaceOprNum) {
12287 MRI.constrainRegClass(Reg: ReplaceReg, RC: TII->getRegClass(MCID: NewMI->getDesc(), OpNum: I));
12288 NewMI->getOperand(i: I).setReg(ReplaceReg);
12289 }
12290 }
12291 MBB.insert(I: InsertTo, MI: NewMI);
12292 return Result;
12293}
12294
12295void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
12296 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
12297 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) {
12298 // Create and accumulate conditions for next TC iterations.
12299 // Example:
12300 // SUBSXrr N, counter, implicit-def $nzcv # compare instruction for the last
12301 // # iteration of the kernel
12302 //
12303 // # insert the following instructions
12304 // cond = CSINCXr 0, 0, C, implicit $nzcv
12305 // counter = ADDXri counter, 1 # clone from this->Update
12306 // SUBSXrr n, counter, implicit-def $nzcv # clone from this->Comp
12307 // cond = CSINCXr cond, cond, C, implicit $nzcv
12308 // ... (repeat TC times)
12309 // SUBSXri cond, 0, implicit-def $nzcv
12310
12311 assert(CondBranch->getOpcode() == AArch64::Bcc);
12312 // CondCode to exit the loop
12313 AArch64CC::CondCode CC =
12314 (AArch64CC::CondCode)CondBranch->getOperand(i: 0).getImm();
12315 if (CondBranch->getOperand(i: 1).getMBB() == LoopBB)
12316 CC = AArch64CC::getInvertedCondCode(Code: CC);
12317
12318 // Accumulate conditions to exit the loop
12319 Register AccCond = AArch64::XZR;
12320
12321 // If CC holds, CurCond+1 is returned; otherwise CurCond is returned.
12322 auto AccumulateCond = [&](Register CurCond,
12323 AArch64CC::CondCode CC) -> Register {
12324 Register NewCond = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
12325 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::CSINCXr))
12326 .addReg(RegNo: NewCond, Flags: RegState::Define)
12327 .addReg(RegNo: CurCond)
12328 .addReg(RegNo: CurCond)
12329 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
12330 return NewCond;
12331 };
12332
12333 if (!LastStage0Insts.empty() && LastStage0Insts[Comp]->getParent() == &MBB) {
12334 // Update and Comp for I==0 are already exists in MBB
12335 // (MBB is an unrolled kernel)
12336 Register Counter;
12337 for (int I = 0; I <= TC; ++I) {
12338 Register NextCounter;
12339 if (I != 0)
12340 NextCounter =
12341 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12342
12343 AccCond = AccumulateCond(AccCond, CC);
12344
12345 if (I != TC) {
12346 if (I == 0) {
12347 if (Update != Comp && IsUpdatePriorComp) {
12348 Counter =
12349 LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12350 NextCounter = cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB,
12351 InsertTo: MBB.end());
12352 } else {
12353 // can use already calculated value
12354 NextCounter = LastStage0Insts[Update]->getOperand(i: 0).getReg();
12355 }
12356 } else if (Update != Comp) {
12357 NextCounter =
12358 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12359 }
12360 }
12361 Counter = NextCounter;
12362 }
12363 } else {
12364 Register Counter;
12365 if (LastStage0Insts.empty()) {
12366 // use initial counter value (testing if the trip count is sufficient to
12367 // be executed by pipelined code)
12368 Counter = Init;
12369 if (IsUpdatePriorComp)
12370 Counter =
12371 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12372 } else {
12373 // MBB is an epilogue block. LastStage0Insts[Comp] is in the kernel block.
12374 Counter = LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12375 }
12376
12377 for (int I = 0; I <= TC; ++I) {
12378 Register NextCounter;
12379 NextCounter =
12380 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12381 AccCond = AccumulateCond(AccCond, CC);
12382 if (I != TC && Update != Comp)
12383 NextCounter =
12384 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12385 Counter = NextCounter;
12386 }
12387 }
12388
12389 // If AccCond == 0, the remainder is greater than TC.
12390 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::SUBSXri))
12391 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define | RegState::Dead)
12392 .addReg(RegNo: AccCond)
12393 .addImm(Val: 0)
12394 .addImm(Val: 0);
12395 Cond.clear();
12396 Cond.push_back(Elt: MachineOperand::CreateImm(Val: AArch64CC::EQ));
12397}
12398
12399static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB,
12400 Register &RegMBB, Register &RegOther) {
12401 assert(Phi.getNumOperands() == 5);
12402 if (Phi.getOperand(i: 2).getMBB() == MBB) {
12403 RegMBB = Phi.getOperand(i: 1).getReg();
12404 RegOther = Phi.getOperand(i: 3).getReg();
12405 } else {
12406 assert(Phi.getOperand(4).getMBB() == MBB);
12407 RegMBB = Phi.getOperand(i: 3).getReg();
12408 RegOther = Phi.getOperand(i: 1).getReg();
12409 }
12410}
12411
12412static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB) {
12413 if (!Reg.isVirtual())
12414 return false;
12415 const MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
12416 return MRI.getDefBlock(Reg) != BB;
12417}
12418
12419/// If Reg is an induction variable, return true and set some parameters
12420static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
12421 MachineInstr *&UpdateInst,
12422 unsigned &UpdateCounterOprNum, Register &InitReg,
12423 bool &IsUpdatePriorComp) {
12424 // Example:
12425 //
12426 // Preheader:
12427 // InitReg = ...
12428 // LoopBB:
12429 // Reg0 = PHI (InitReg, Preheader), (Reg1, LoopBB)
12430 // Reg = COPY Reg0 ; COPY is ignored.
12431 // Reg1 = ADD Reg, #1; UpdateInst. Incremented by a loop invariant value.
12432 // ; Reg is the value calculated in the previous
12433 // ; iteration, so IsUpdatePriorComp == false.
12434
12435 if (LoopBB->pred_size() != 2)
12436 return false;
12437 if (!Reg.isVirtual())
12438 return false;
12439 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
12440 UpdateInst = nullptr;
12441 UpdateCounterOprNum = 0;
12442 InitReg = 0;
12443 IsUpdatePriorComp = true;
12444 Register CurReg = Reg;
12445 while (true) {
12446 MachineInstr *Def = MRI.getVRegDef(Reg: CurReg);
12447 if (Def->getParent() != LoopBB)
12448 return false;
12449 if (Def->isCopy()) {
12450 // Ignore copy instructions unless they contain subregisters
12451 if (Def->getOperand(i: 0).getSubReg() || Def->getOperand(i: 1).getSubReg())
12452 return false;
12453 CurReg = Def->getOperand(i: 1).getReg();
12454 } else if (Def->isPHI()) {
12455 if (InitReg != 0)
12456 return false;
12457 if (!UpdateInst)
12458 IsUpdatePriorComp = false;
12459 extractPhiReg(Phi: *Def, MBB: LoopBB, RegMBB&: CurReg, RegOther&: InitReg);
12460 } else {
12461 if (UpdateInst)
12462 return false;
12463 switch (Def->getOpcode()) {
12464 case AArch64::ADDSXri:
12465 case AArch64::ADDSWri:
12466 case AArch64::SUBSXri:
12467 case AArch64::SUBSWri:
12468 case AArch64::ADDXri:
12469 case AArch64::ADDWri:
12470 case AArch64::SUBXri:
12471 case AArch64::SUBWri:
12472 UpdateInst = Def;
12473 UpdateCounterOprNum = 1;
12474 break;
12475 case AArch64::ADDSXrr:
12476 case AArch64::ADDSWrr:
12477 case AArch64::SUBSXrr:
12478 case AArch64::SUBSWrr:
12479 case AArch64::ADDXrr:
12480 case AArch64::ADDWrr:
12481 case AArch64::SUBXrr:
12482 case AArch64::SUBWrr:
12483 UpdateInst = Def;
12484 if (isDefinedOutside(Reg: Def->getOperand(i: 2).getReg(), BB: LoopBB))
12485 UpdateCounterOprNum = 1;
12486 else if (isDefinedOutside(Reg: Def->getOperand(i: 1).getReg(), BB: LoopBB))
12487 UpdateCounterOprNum = 2;
12488 else
12489 return false;
12490 break;
12491 default:
12492 return false;
12493 }
12494 CurReg = Def->getOperand(i: UpdateCounterOprNum).getReg();
12495 }
12496
12497 if (!CurReg.isVirtual())
12498 return false;
12499 if (Reg == CurReg)
12500 break;
12501 }
12502
12503 if (!UpdateInst)
12504 return false;
12505
12506 return true;
12507}
12508
12509std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
12510AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
12511 // Accept loops that meet the following conditions
12512 // * The conditional branch is BCC
12513 // * The compare instruction is ADDS/SUBS/WHILEXX
12514 // * One operand of the compare is an induction variable and the other is a
12515 // loop invariant value
12516 // * The induction variable is incremented/decremented by a single instruction
12517 // * Does not contain CALL or instructions which have unmodeled side effects
12518
12519 for (MachineInstr &MI : *LoopBB)
12520 if (MI.isCall() || MI.hasUnmodeledSideEffects())
12521 // This instruction may use NZCV, which interferes with the instruction to
12522 // be inserted for loop control.
12523 return nullptr;
12524
12525 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
12526 SmallVector<MachineOperand, 4> Cond;
12527 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond))
12528 return nullptr;
12529
12530 // Infinite loops are not supported
12531 if (TBB == LoopBB && FBB == LoopBB)
12532 return nullptr;
12533
12534 // Must be conditional branch
12535 if (TBB != LoopBB && FBB == nullptr)
12536 return nullptr;
12537
12538 assert((TBB == LoopBB || FBB == LoopBB) &&
12539 "The Loop must be a single-basic-block loop");
12540
12541 MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
12542 const TargetRegisterInfo &TRI = getRegisterInfo();
12543
12544 if (CondBranch->getOpcode() != AArch64::Bcc)
12545 return nullptr;
12546
12547 // Normalization for createTripCountGreaterCondition()
12548 if (TBB == LoopBB)
12549 reverseBranchCondition(Cond);
12550
12551 MachineInstr *Comp = nullptr;
12552 unsigned CompCounterOprNum = 0;
12553 for (MachineInstr &MI : reverse(C&: *LoopBB)) {
12554 if (MI.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
12555 // Guarantee that the compare is SUBS/ADDS/WHILEXX and that one of the
12556 // operands is a loop invariant value
12557
12558 switch (MI.getOpcode()) {
12559 case AArch64::SUBSXri:
12560 case AArch64::SUBSWri:
12561 case AArch64::ADDSXri:
12562 case AArch64::ADDSWri:
12563 Comp = &MI;
12564 CompCounterOprNum = 1;
12565 break;
12566 case AArch64::ADDSWrr:
12567 case AArch64::ADDSXrr:
12568 case AArch64::SUBSWrr:
12569 case AArch64::SUBSXrr:
12570 Comp = &MI;
12571 break;
12572 default:
12573 if (isWhileOpcode(Opc: MI.getOpcode())) {
12574 Comp = &MI;
12575 break;
12576 }
12577 return nullptr;
12578 }
12579
12580 if (CompCounterOprNum == 0) {
12581 if (isDefinedOutside(Reg: Comp->getOperand(i: 1).getReg(), BB: LoopBB))
12582 CompCounterOprNum = 2;
12583 else if (isDefinedOutside(Reg: Comp->getOperand(i: 2).getReg(), BB: LoopBB))
12584 CompCounterOprNum = 1;
12585 else
12586 return nullptr;
12587 }
12588 break;
12589 }
12590 }
12591 if (!Comp)
12592 return nullptr;
12593
12594 MachineInstr *Update = nullptr;
12595 Register Init;
12596 bool IsUpdatePriorComp;
12597 unsigned UpdateCounterOprNum;
12598 if (!getIndVarInfo(Reg: Comp->getOperand(i: CompCounterOprNum).getReg(), LoopBB,
12599 UpdateInst&: Update, UpdateCounterOprNum, InitReg&: Init, IsUpdatePriorComp))
12600 return nullptr;
12601
12602 return std::make_unique<AArch64PipelinerLoopInfo>(
12603 args&: LoopBB, args&: CondBranch, args&: Comp, args&: CompCounterOprNum, args&: Update, args&: UpdateCounterOprNum,
12604 args&: Init, args&: IsUpdatePriorComp, args&: Cond);
12605}
12606
12607/// verifyInstruction - Perform target specific instruction verification.
12608bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI,
12609 StringRef &ErrInfo) const {
12610 // Verify that immediate offsets on load/store instructions are within range.
12611 // Stack objects with an FI operand are excluded as they can be fixed up
12612 // during PEI.
12613 TypeSize Scale(0U, false), Width(0U, false);
12614 int64_t MinOffset, MaxOffset;
12615 if (getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
12616 unsigned ImmIdx = getLoadStoreImmIdx(Opc: MI.getOpcode());
12617 if (MI.getOperand(i: ImmIdx).isImm() && !MI.getOperand(i: ImmIdx - 1).isFI()) {
12618 int64_t Imm = MI.getOperand(i: ImmIdx).getImm();
12619 if (Imm < MinOffset || Imm > MaxOffset) {
12620 ErrInfo = "Unexpected immediate on load/store instruction";
12621 return false;
12622 }
12623 }
12624 }
12625
12626 const MCInstrDesc &MCID = MI.getDesc();
12627 for (unsigned Op = 0; Op < MCID.getNumOperands(); Op++) {
12628 const MachineOperand &MO = MI.getOperand(i: Op);
12629 switch (MCID.operands()[Op].OperandType) {
12630 case AArch64::OPERAND_IMPLICIT_IMM_0:
12631 if (!MO.isImm() || MO.getImm() != 0) {
12632 ErrInfo = "OPERAND_IMPLICIT_IMM_0 should be 0";
12633 return false;
12634 }
12635 break;
12636 case AArch64::OPERAND_SHIFT_MSL:
12637 if (!MO.isImm() ||
12638 AArch64_AM::getShiftType(Imm: MO.getImm()) != AArch64_AM::MSL ||
12639 (AArch64_AM::getShiftValue(Imm: MO.getImm()) != 8 &&
12640 AArch64_AM::getShiftValue(Imm: MO.getImm()) != 16)) {
12641 ErrInfo = "OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
12642 return false;
12643 }
12644 break;
12645 case AArch64::OPERAND_IMM_UINT1:
12646 if (!MO.isImm() || (MO.getImm() != 0 && MO.getImm() != 1)) {
12647 ErrInfo = "OPERAND_IMM_UINT1 should be 0 or 1";
12648 return false;
12649 }
12650 break;
12651 case AArch64::OPERAND_IMM_UINT4plus1:
12652 if (!MO.isImm() || MO.getImm() <= 0 || MO.getImm() > 16) {
12653 ErrInfo = "OPERAND_IMM_UINT4plus1 should be in the range 1 to 16";
12654 return false;
12655 }
12656 break;
12657 case AArch64::OPERAND_IMM_UINT5:
12658 if (!MO.isImm() || !isUInt<5>(x: MO.getImm())) {
12659 ErrInfo = "OPERAND_IMM_UINT5 should be in the range 0 to 31";
12660 return false;
12661 }
12662 break;
12663 case AArch64::OPERAND_IMM_UINT8:
12664 if (!MO.isImm() || !isUInt<8>(x: MO.getImm())) {
12665 ErrInfo = "OPERAND_IMM_UINT8 should be in the range 0 to 255";
12666 return false;
12667 }
12668 break;
12669 default:
12670 break;
12671 }
12672 }
12673 return true;
12674}
12675
12676#define GET_INSTRINFO_HELPERS
12677#define GET_INSTRMAP_INFO
12678#include "AArch64GenInstrInfo.inc"
12679