1//===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64PointerAuth.h"
17#include "AArch64Subtarget.h"
18#include "MCTargetDesc/AArch64AddressingModes.h"
19#include "MCTargetDesc/AArch64MCLFIRewriter.h"
20#include "MCTargetDesc/AArch64MCTargetDesc.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/CodeGen/CFIInstBuilder.h"
28#include "llvm/CodeGen/LivePhysRegs.h"
29#include "llvm/CodeGen/LiveRegUnits.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineCombinerPattern.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstr.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/RegisterScavenging.h"
41#include "llvm/CodeGen/StackMaps.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/CodeGen/TargetSubtargetInfo.h"
44#include "llvm/IR/DebugInfoMetadata.h"
45#include "llvm/IR/DebugLoc.h"
46#include "llvm/IR/GlobalValue.h"
47#include "llvm/IR/Module.h"
48#include "llvm/MC/MCAsmInfo.h"
49#include "llvm/MC/MCInst.h"
50#include "llvm/MC/MCInstBuilder.h"
51#include "llvm/MC/MCInstrDesc.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CodeGen.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/LEB128.h"
56#include "llvm/Support/MathExtras.h"
57#include "llvm/Target/TargetMachine.h"
58#include "llvm/Target/TargetOptions.h"
59#include <cassert>
60#include <cstdint>
61#include <iterator>
62#include <utility>
63
64using namespace llvm;
65
66#define GET_INSTRINFO_CTOR_DTOR
67#include "AArch64GenInstrInfo.inc"
68
69#define DEBUG_TYPE "AArch64InstrInfo"
70
71STATISTIC(NumCopyInstrs, "Number of COPY instructions expanded");
72STATISTIC(NumZCRegMoveInstrsGPR, "Number of zero-cycle GPR register move "
73 "instructions expanded from canonical COPY");
74STATISTIC(NumZCRegMoveInstrsFPR, "Number of zero-cycle FPR register move "
75 "instructions expanded from canonical COPY");
76STATISTIC(NumZCZeroingInstrsGPR, "Number of zero-cycle GPR zeroing "
77 "instructions expanded from canonical COPY");
78// NumZCZeroingInstrsFPR is counted at AArch64AsmPrinter
79
80AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
81 : AArch64GenInstrInfo(STI, RI, AArch64::ADJCALLSTACKDOWN,
82 AArch64::ADJCALLSTACKUP, AArch64::CATCHRET),
83 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
84
85/// Return the maximum number of bytes of code the specified instruction may be
86/// after LFI rewriting. If the instruction is not rewritten, std::nullopt is
87/// returned (use default sizing).
88///
89/// NOTE: the size estimates here must be kept in sync with the rewrites in
90/// AArch64MCLFIRewriter.cpp. Sizes may be overestimates of the rewritten
91/// instruction sequences.
92static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
93 switch (MI.getOpcode()) {
94 case AArch64::SVC:
95 // SVC expands to 4 instructions.
96 return 16;
97 case AArch64::BR:
98 case AArch64::BLR:
99 // Indirect branches/calls expand to 2 instructions (guard + br/blr).
100 return 8;
101 case AArch64::RET:
102 // RET through another register expands to 2 instructions (guard + ret).
103 // RET through LR may also expand to 2 instructions if a deferred LR guard
104 // is flushed before the return.
105 return 8;
106 case AArch64::RETAA:
107 case AArch64::RETAB:
108 // Authenticated returns expand to 3 instructions (authenticate + guard +
109 // ret).
110 return 12;
111 case AArch64::BRAA:
112 case AArch64::BRAAZ:
113 case AArch64::BRAB:
114 case AArch64::BRABZ:
115 case AArch64::BLRAA:
116 case AArch64::BLRAAZ:
117 case AArch64::BLRAB:
118 case AArch64::BLRABZ:
119 // Authenticated branches/calls expand to 3 instructions (authenticate +
120 // guard + branch).
121 return 12;
122 case AArch64::AUTIASP:
123 case AArch64::AUTIBSP:
124 case AArch64::AUTIAZ:
125 case AArch64::AUTIBZ:
126 case AArch64::XPACLRI:
127 // Authenticating LR expands to the instruction plus a deferred LR guard.
128 return 8;
129 case AArch64::SYSxt:
130 // VA-based DC/IC ops (op1=3, Cn=7, op2=1) expand to 2 instructions.
131 if (MI.getOperand(i: 0).getImm() == 3 && MI.getOperand(i: 1).getImm() == 7 &&
132 MI.getOperand(i: 3).getImm() == 1)
133 return 8;
134 return std::nullopt;
135 default:
136 break;
137 }
138
139 // Detect instructions that explicitly define SP or LR.
140 bool ModifiesLR = false;
141 bool ModifiesSP = false;
142 for (const MachineOperand &MO : MI.defs()) {
143 if (!MO.isReg())
144 continue;
145 if (MO.getReg() == AArch64::LR)
146 ModifiesLR = true;
147 else if (MO.getReg() == AArch64::SP)
148 ModifiesSP = true;
149 }
150
151 // Memory accesses expand to a base-register guard plus the rewritten access
152 // (8 bytes), with an extra base-register update for pre/post-index forms (12
153 // bytes total). If the access also defines LR, an LR mask is appended (+4
154 // bytes). Depending on additional optimizations that the rewriter performs,
155 // this may be an overestimate.
156 if (MI.mayLoadOrStore()) {
157 unsigned Size = isLFIPrePostMemAccess(Opcode: MI.getOpcode()) ? 12 : 8;
158 if (ModifiesLR)
159 Size += 4;
160 return Size;
161 }
162
163 // Non memory operations that modify LR or SP expand to 2 instructions.
164 if (ModifiesSP || ModifiesLR)
165 return 8;
166
167 // Default case: instructions that don't cause expansion.
168 // - TP accesses in LFI are a single load/store, so no expansion.
169 // - All remaining instructions are not rewritten.
170 return std::nullopt;
171}
172
173/// GetInstSize - Return the number of bytes of code the specified
174/// instruction may be. This returns the maximum number of bytes.
175unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
176 const MCInstrDesc &Desc = MI.getDesc();
177 if (!Desc.isPseudo() && !Subtarget.isLFI()) {
178 assert(Desc.getSize() == 4 && "Unexpected instruction size");
179 return 4;
180 }
181
182 const MachineBasicBlock &MBB = *MI.getParent();
183 const MachineFunction *MF = MBB.getParent();
184 const Function &F = MF->getFunction();
185 const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
186
187 {
188 auto Op = MI.getOpcode();
189 if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
190 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(), MAI);
191 }
192
193 // Meta-instructions emit no code.
194 if (MI.isMetaInstruction())
195 return 0;
196
197 // FIXME: We currently only handle pseudoinstructions that don't get expanded
198 // before the assembly printer.
199 unsigned NumBytes = 0;
200
201 // LFI rewriter expansions that supersede normal sizing.
202 const auto &STI = MF->getSubtarget<AArch64Subtarget>();
203 if (STI.isLFI())
204 if (auto Size = getLFIInstSizeInBytes(MI))
205 return *Size;
206
207 if (!MI.isBundle() && isTailCallReturnInst(MI)) {
208 NumBytes = Desc.getSize() ? Desc.getSize() : 4;
209
210 const auto *MFI = MF->getInfo<AArch64FunctionInfo>();
211 if (!MFI->shouldSignReturnAddress(MF: *MF))
212 return NumBytes;
213
214 auto Method = STI.getAuthenticatedLRCheckMethod(MF: *MF);
215 NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
216 return NumBytes;
217 }
218
219 // Size should be preferably set in
220 // llvm/lib/Target/AArch64/AArch64InstrInfo.td (default case).
221 // Specific cases handle instructions of variable sizes
222 switch (Desc.getOpcode()) {
223 default:
224 if (Desc.getSize())
225 return Desc.getSize();
226
227 // Anything not explicitly designated otherwise (i.e. pseudo-instructions
228 // with fixed constant size but not specified in .td file) is a normal
229 // 4-byte insn.
230 NumBytes = 4;
231 break;
232 case TargetOpcode::STACKMAP:
233 // The upper bound for a stackmap intrinsic is the full length of its shadow
234 NumBytes = StackMapOpers(&MI).getNumPatchBytes();
235 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
236 break;
237 case TargetOpcode::PATCHPOINT:
238 // The size of the patchpoint intrinsic is the number of bytes requested
239 NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
240 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
241 break;
242 case TargetOpcode::STATEPOINT:
243 NumBytes = StatepointOpers(&MI).getNumPatchBytes();
244 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
245 // No patch bytes means a normal call inst is emitted
246 if (NumBytes == 0)
247 NumBytes = 4;
248 break;
249 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
250 // If `patchable-function-entry` is set, PATCHABLE_FUNCTION_ENTER
251 // instructions are expanded to the specified number of NOPs. Otherwise,
252 // they are expanded to 36-byte XRay sleds.
253 NumBytes =
254 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry", Default: 9) * 4;
255 break;
256 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
257 case TargetOpcode::PATCHABLE_TAIL_CALL:
258 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
259 // An XRay sled can be 4 bytes of alignment plus a 32-byte block.
260 NumBytes = 36;
261 break;
262 case TargetOpcode::PATCHABLE_EVENT_CALL:
263 // EVENT_CALL XRay sleds are exactly 6 instructions long (no alignment).
264 NumBytes = 24;
265 break;
266
267 case AArch64::SPACE:
268 NumBytes = MI.getOperand(i: 1).getImm();
269 break;
270 case AArch64::MOVaddr:
271 case AArch64::MOVaddrJT:
272 case AArch64::MOVaddrCP:
273 case AArch64::MOVaddrBA:
274 case AArch64::MOVaddrTLS:
275 case AArch64::MOVaddrEXT: {
276 // Use the same logic as the pseudo expansion to count instructions.
277 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
278 AArch64_IMM::expandMOVAddr(Opcode: Desc.getOpcode(),
279 TargetFlags: MI.getOperand(i: 1).getTargetFlags(),
280 IsTargetMachO: Subtarget.isTargetMachO(), Insn);
281 NumBytes = Insn.size() * 4;
282 break;
283 }
284
285 case AArch64::MOVi32imm:
286 case AArch64::MOVi64imm: {
287 // Use the same logic as the pseudo expansion to count instructions.
288 unsigned BitSize = Desc.getOpcode() == AArch64::MOVi32imm ? 32 : 64;
289 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
290 AArch64_IMM::expandMOVImm(Imm: MI.getOperand(i: 1).getImm(), BitSize, Insn);
291 NumBytes = Insn.size() * 4;
292 break;
293 }
294
295 case TargetOpcode::BUNDLE:
296 NumBytes = getInstBundleSize(MI);
297 break;
298 }
299
300 return NumBytes;
301}
302
303static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
304 SmallVectorImpl<MachineOperand> &Cond) {
305 // Block ends with fall-through condbranch.
306 switch (LastInst->getOpcode()) {
307 default:
308 llvm_unreachable("Unknown branch instruction?");
309 case AArch64::Bcc:
310 Target = LastInst->getOperand(i: 1).getMBB();
311 Cond.push_back(Elt: LastInst->getOperand(i: 0));
312 break;
313 case AArch64::CBZW:
314 case AArch64::CBZX:
315 case AArch64::CBNZW:
316 case AArch64::CBNZX:
317 Target = LastInst->getOperand(i: 1).getMBB();
318 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
319 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
320 Cond.push_back(Elt: LastInst->getOperand(i: 0));
321 break;
322 case AArch64::TBZW:
323 case AArch64::TBZX:
324 case AArch64::TBNZW:
325 case AArch64::TBNZX:
326 Target = LastInst->getOperand(i: 2).getMBB();
327 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
328 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
329 Cond.push_back(Elt: LastInst->getOperand(i: 0));
330 Cond.push_back(Elt: LastInst->getOperand(i: 1));
331 break;
332 case AArch64::CBWPri:
333 case AArch64::CBXPri:
334 case AArch64::CBWPrr:
335 case AArch64::CBXPrr:
336 Target = LastInst->getOperand(i: 3).getMBB();
337 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
338 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
339 Cond.push_back(Elt: LastInst->getOperand(i: 0));
340 Cond.push_back(Elt: LastInst->getOperand(i: 1));
341 Cond.push_back(Elt: LastInst->getOperand(i: 2));
342 break;
343 case AArch64::CBBAssertExt:
344 case AArch64::CBHAssertExt:
345 Target = LastInst->getOperand(i: 3).getMBB();
346 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1)); // -1
347 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode())); // Opc
348 Cond.push_back(Elt: LastInst->getOperand(i: 0)); // Cond
349 Cond.push_back(Elt: LastInst->getOperand(i: 1)); // Op0
350 Cond.push_back(Elt: LastInst->getOperand(i: 2)); // Op1
351 Cond.push_back(Elt: LastInst->getOperand(i: 4)); // Ext0
352 Cond.push_back(Elt: LastInst->getOperand(i: 5)); // Ext1
353 break;
354 }
355}
356
357static unsigned getBranchDisplacementBits(const AArch64Options &CLOpts,
358 unsigned Opc) {
359 switch (Opc) {
360 default:
361 llvm_unreachable("unexpected opcode!");
362 case AArch64::B:
363 return CLOpts.b_offset_bits;
364 case AArch64::TBNZW:
365 case AArch64::TBZW:
366 case AArch64::TBNZX:
367 case AArch64::TBZX:
368 return CLOpts.tbz_offset_bits;
369 case AArch64::CBNZW:
370 case AArch64::CBZW:
371 case AArch64::CBNZX:
372 case AArch64::CBZX:
373 return CLOpts.cbz_offset_bits;
374 case AArch64::Bcc:
375 return CLOpts.bcc_offset_bits;
376 case AArch64::CBWPri:
377 case AArch64::CBXPri:
378 case AArch64::CBBAssertExt:
379 case AArch64::CBHAssertExt:
380 case AArch64::CBWPrr:
381 case AArch64::CBXPrr:
382 return CLOpts.cb_offset_bits;
383 }
384}
385
386bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
387 int64_t BrOffset) const {
388 unsigned Bits = getBranchDisplacementBits(CLOpts: Subtarget.getCLOpts(), Opc: BranchOp);
389 assert(Bits >= 3 && "max branch displacement must be enough to jump"
390 "over conditional branch expansion");
391 return isIntN(N: Bits, x: BrOffset / 4);
392}
393
394MachineBasicBlock *
395AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
396 switch (MI.getOpcode()) {
397 default:
398 llvm_unreachable("unexpected opcode!");
399 case AArch64::B:
400 return MI.getOperand(i: 0).getMBB();
401 case AArch64::TBZW:
402 case AArch64::TBNZW:
403 case AArch64::TBZX:
404 case AArch64::TBNZX:
405 return MI.getOperand(i: 2).getMBB();
406 case AArch64::CBZW:
407 case AArch64::CBNZW:
408 case AArch64::CBZX:
409 case AArch64::CBNZX:
410 case AArch64::Bcc:
411 return MI.getOperand(i: 1).getMBB();
412 case AArch64::CBWPri:
413 case AArch64::CBXPri:
414 case AArch64::CBBAssertExt:
415 case AArch64::CBHAssertExt:
416 case AArch64::CBWPrr:
417 case AArch64::CBXPrr:
418 return MI.getOperand(i: 3).getMBB();
419 }
420}
421
422void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
423 MachineBasicBlock &NewDestBB,
424 MachineBasicBlock &RestoreBB,
425 const DebugLoc &DL,
426 int64_t BrOffset,
427 RegScavenger *RS) const {
428 assert(RS && "RegScavenger required for long branching");
429 assert(MBB.empty() &&
430 "new block should be inserted for expanding unconditional branch");
431 assert(MBB.pred_size() == 1);
432 assert(RestoreBB.empty() &&
433 "restore block should be inserted for restoring clobbered registers");
434
435 auto buildIndirectBranch = [&](Register Reg, MachineBasicBlock &DestBB) {
436 // Offsets outside of the signed 33-bit range are not supported for ADRP +
437 // ADD.
438 if (!isInt<33>(x: BrOffset))
439 report_fatal_error(
440 reason: "Branch offsets outside of the signed 33-bit range not supported");
441
442 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
443 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGE);
444 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: Reg)
445 .addReg(RegNo: Reg)
446 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
447 .addImm(Val: 0);
448 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::BR)).addReg(RegNo: Reg);
449 };
450
451 RS->enterBasicBlockEnd(MBB);
452 // If X16 is unused, we can rely on the linker to insert a range extension
453 // thunk if NewDestBB is out of range of a single B instruction.
454 constexpr Register Reg = AArch64::X16;
455 if (!RS->isRegUsed(Reg)) {
456 insertUnconditionalBranch(MBB, DestBB: &NewDestBB, DL);
457 RS->setRegUsed(Reg);
458 return;
459 }
460
461 // In a cold block without BTI, insert the indirect branch if a register is
462 // free. Skip this if BTI is enabled to avoid inserting a BTI at the target,
463 // prioritizing a dynamic cost in cold code over a static cost in hot code.
464 AArch64FunctionInfo *AFI = MBB.getParent()->getInfo<AArch64FunctionInfo>();
465 bool HasBTI = AFI && AFI->branchTargetEnforcement();
466 if (MBB.getSectionID() == MBBSectionID::ColdSectionID && !HasBTI) {
467 Register Scavenged = RS->FindUnusedReg(RC: &AArch64::GPR64RegClass);
468 if (Scavenged.isValid()) {
469 buildIndirectBranch(Scavenged, NewDestBB);
470 RS->setRegUsed(Reg: Scavenged);
471 return;
472 }
473 }
474
475 // Note: Spilling X16 briefly moves the stack pointer, making it incompatible
476 // with red zones.
477 if (!AFI || AFI->hasRedZone().value_or(u: true))
478 report_fatal_error(
479 reason: "Unable to insert indirect branch inside function that has red zone");
480
481 // Otherwise, spill X16 and defer range extension to the linker.
482 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::STRXpre))
483 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
484 .addReg(RegNo: Reg)
485 .addReg(RegNo: AArch64::SP)
486 .addImm(Val: -16);
487
488 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: &RestoreBB);
489
490 BuildMI(BB&: RestoreBB, I: RestoreBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::LDRXpost))
491 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
492 .addReg(RegNo: Reg, Flags: RegState::Define)
493 .addReg(RegNo: AArch64::SP)
494 .addImm(Val: 16);
495}
496
497// Branch analysis.
498bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
499 MachineBasicBlock *&TBB,
500 MachineBasicBlock *&FBB,
501 SmallVectorImpl<MachineOperand> &Cond,
502 bool AllowModify) const {
503 // If the block has no terminators, it just falls into the block after it.
504 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
505 if (I == MBB.end())
506 return false;
507
508 // Skip over SpeculationBarrierEndBB terminators
509 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
510 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
511 --I;
512 }
513
514 if (!isUnpredicatedTerminator(MI: *I))
515 return false;
516
517 // Get the last instruction in the block.
518 MachineInstr *LastInst = &*I;
519
520 // If there is only one terminator instruction, process it.
521 unsigned LastOpc = LastInst->getOpcode();
522 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
523 if (isUncondBranchOpcode(Opc: LastOpc)) {
524 TBB = LastInst->getOperand(i: 0).getMBB();
525 return false;
526 }
527 if (isCondBranchOpcode(Opc: LastOpc)) {
528 // Block ends with fall-through condbranch.
529 parseCondBranch(LastInst, Target&: TBB, Cond);
530 return false;
531 }
532 return true; // Can't handle indirect branch.
533 }
534
535 // Get the instruction before it if it is a terminator.
536 MachineInstr *SecondLastInst = &*I;
537 unsigned SecondLastOpc = SecondLastInst->getOpcode();
538
539 // If AllowModify is true and the block ends with two or more unconditional
540 // branches, delete all but the first unconditional branch.
541 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc)) {
542 while (isUncondBranchOpcode(Opc: SecondLastOpc)) {
543 LastInst->eraseFromParent();
544 LastInst = SecondLastInst;
545 LastOpc = LastInst->getOpcode();
546 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
547 // Return now the only terminator is an unconditional branch.
548 TBB = LastInst->getOperand(i: 0).getMBB();
549 return false;
550 }
551 SecondLastInst = &*I;
552 SecondLastOpc = SecondLastInst->getOpcode();
553 }
554 }
555
556 // If we're allowed to modify and the block ends in a unconditional branch
557 // which could simply fallthrough, remove the branch. (Note: This case only
558 // matters when we can't understand the whole sequence, otherwise it's also
559 // handled by BranchFolding.cpp.)
560 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc) &&
561 MBB.isLayoutSuccessor(MBB: getBranchDestBlock(MI: *LastInst))) {
562 LastInst->eraseFromParent();
563 LastInst = SecondLastInst;
564 LastOpc = LastInst->getOpcode();
565 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
566 assert(!isUncondBranchOpcode(LastOpc) &&
567 "unreachable unconditional branches removed above");
568
569 if (isCondBranchOpcode(Opc: LastOpc)) {
570 // Block ends with fall-through condbranch.
571 parseCondBranch(LastInst, Target&: TBB, Cond);
572 return false;
573 }
574 return true; // Can't handle indirect branch.
575 }
576 SecondLastInst = &*I;
577 SecondLastOpc = SecondLastInst->getOpcode();
578 }
579
580 // If there are three terminators, we don't know what sort of block this is.
581 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(MI: *--I))
582 return true;
583
584 // If the block ends with a B and a Bcc, handle it.
585 if (isCondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
586 parseCondBranch(LastInst: SecondLastInst, Target&: TBB, Cond);
587 FBB = LastInst->getOperand(i: 0).getMBB();
588 return false;
589 }
590
591 // If the block ends with two unconditional branches, handle it. The second
592 // one is not executed, so remove it.
593 if (isUncondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
594 TBB = SecondLastInst->getOperand(i: 0).getMBB();
595 I = LastInst;
596 if (AllowModify)
597 I->eraseFromParent();
598 return false;
599 }
600
601 // ...likewise if it ends with an indirect branch followed by an unconditional
602 // branch.
603 if (isIndirectBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
604 I = LastInst;
605 if (AllowModify)
606 I->eraseFromParent();
607 return true;
608 }
609
610 // Otherwise, can't handle this.
611 return true;
612}
613
614bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
615 MachineBranchPredicate &MBP,
616 bool AllowModify) const {
617 // Use analyzeBranch to validate the branch pattern.
618 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
619 SmallVector<MachineOperand, 4> Cond;
620 if (analyzeBranch(MBB, TBB, FBB, Cond, AllowModify))
621 return true;
622
623 // analyzeBranch returns success with empty Cond for unconditional branches.
624 if (Cond.empty())
625 return true;
626
627 MBP.TrueDest = TBB;
628 assert(MBP.TrueDest && "expected!");
629 MBP.FalseDest = FBB ? FBB : MBB.getNextNode();
630
631 MBP.ConditionDef = nullptr;
632 MBP.SingleUseCondition = false;
633
634 // Find the conditional branch. After analyzeBranch succeeds with non-empty
635 // Cond, there's exactly one conditional branch - either last (fallthrough)
636 // or second-to-last (followed by unconditional B).
637 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
638 if (I == MBB.end())
639 return true;
640
641 if (isUncondBranchOpcode(Opc: I->getOpcode())) {
642 if (I == MBB.begin())
643 return true;
644 --I;
645 }
646
647 MachineInstr *CondBranch = &*I;
648 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
649
650 switch (CondBranch->getOpcode()) {
651 default:
652 return true;
653
654 case AArch64::Bcc:
655 // Bcc takes the NZCV flag as the operand to branch on, walk up the
656 // instruction stream to find the last instruction to define NZCV.
657 for (MachineInstr &MI : llvm::drop_begin(RangeOrContainer: llvm::reverse(C&: MBB))) {
658 if (MI.modifiesRegister(Reg: AArch64::NZCV, /*TRI=*/nullptr)) {
659 MBP.ConditionDef = &MI;
660 break;
661 }
662 }
663 return false;
664
665 case AArch64::CBZW:
666 case AArch64::CBZX:
667 case AArch64::CBNZW:
668 case AArch64::CBNZX: {
669 MBP.LHS = CondBranch->getOperand(i: 0);
670 MBP.RHS = MachineOperand::CreateImm(Val: 0);
671 unsigned Opc = CondBranch->getOpcode();
672 MBP.Predicate = (Opc == AArch64::CBNZX || Opc == AArch64::CBNZW)
673 ? MachineBranchPredicate::PRED_NE
674 : MachineBranchPredicate::PRED_EQ;
675 Register CondReg = MBP.LHS.getReg();
676 if (CondReg.isVirtual())
677 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
678 return false;
679 }
680
681 case AArch64::TBZW:
682 case AArch64::TBZX:
683 case AArch64::TBNZW:
684 case AArch64::TBNZX: {
685 Register CondReg = CondBranch->getOperand(i: 0).getReg();
686 if (CondReg.isVirtual())
687 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
688 return false;
689 }
690 }
691}
692
693bool AArch64InstrInfo::reverseBranchCondition(
694 SmallVectorImpl<MachineOperand> &Cond) const {
695 if (Cond[0].getImm() != -1) {
696 // Regular Bcc
697 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
698 Cond[0].setImm(AArch64CC::getInvertedCondCode(Code: CC));
699 } else {
700 // Folded compare-and-branch
701 switch (Cond[1].getImm()) {
702 default:
703 llvm_unreachable("Unknown conditional branch!");
704 case AArch64::CBZW:
705 Cond[1].setImm(AArch64::CBNZW);
706 break;
707 case AArch64::CBNZW:
708 Cond[1].setImm(AArch64::CBZW);
709 break;
710 case AArch64::CBZX:
711 Cond[1].setImm(AArch64::CBNZX);
712 break;
713 case AArch64::CBNZX:
714 Cond[1].setImm(AArch64::CBZX);
715 break;
716 case AArch64::TBZW:
717 Cond[1].setImm(AArch64::TBNZW);
718 break;
719 case AArch64::TBNZW:
720 Cond[1].setImm(AArch64::TBZW);
721 break;
722 case AArch64::TBZX:
723 Cond[1].setImm(AArch64::TBNZX);
724 break;
725 case AArch64::TBNZX:
726 Cond[1].setImm(AArch64::TBZX);
727 break;
728
729 // Cond is { -1, Opcode, CC, Op0, Op1, ... }
730 case AArch64::CBWPri:
731 case AArch64::CBXPri:
732 case AArch64::CBBAssertExt:
733 case AArch64::CBHAssertExt:
734 case AArch64::CBWPrr:
735 case AArch64::CBXPrr: {
736 // Pseudos using standard 4bit Arm condition codes
737 AArch64CC::CondCode CC =
738 static_cast<AArch64CC::CondCode>(Cond[2].getImm());
739 Cond[2].setImm(AArch64CC::getInvertedCondCode(Code: CC));
740 }
741 }
742 }
743
744 return false;
745}
746
747unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
748 int *BytesRemoved) const {
749 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
750 if (I == MBB.end())
751 return 0;
752
753 if (!isUncondBranchOpcode(Opc: I->getOpcode()) &&
754 !isCondBranchOpcode(Opc: I->getOpcode()))
755 return 0;
756
757 // Remove the branch.
758 I->eraseFromParent();
759
760 I = MBB.end();
761
762 if (I == MBB.begin()) {
763 if (BytesRemoved)
764 *BytesRemoved = 4;
765 return 1;
766 }
767 --I;
768 if (!isCondBranchOpcode(Opc: I->getOpcode())) {
769 if (BytesRemoved)
770 *BytesRemoved = 4;
771 return 1;
772 }
773
774 // Remove the branch.
775 I->eraseFromParent();
776 if (BytesRemoved)
777 *BytesRemoved = 8;
778
779 return 2;
780}
781
782void AArch64InstrInfo::instantiateCondBranch(
783 MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
784 ArrayRef<MachineOperand> Cond) const {
785 if (Cond[0].getImm() != -1) {
786 // Regular Bcc
787 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: Cond[0].getImm()).addMBB(MBB: TBB);
788 } else {
789 // Folded compare-and-branch
790 // Note that we use addOperand instead of addReg to keep the flags.
791
792 // cbz, cbnz
793 const MachineInstrBuilder MIB =
794 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[1].getImm())).add(MO: Cond[2]);
795
796 // tbz/tbnz
797 if (Cond.size() > 3)
798 MIB.add(MO: Cond[3]);
799
800 // cb
801 if (Cond.size() > 4)
802 MIB.add(MO: Cond[4]);
803
804 MIB.addMBB(MBB: TBB);
805
806 // cb[b,h]
807 if (Cond.size() > 5) {
808 MIB.addImm(Val: Cond[5].getImm());
809 MIB.addImm(Val: Cond[6].getImm());
810 }
811 }
812}
813
814unsigned AArch64InstrInfo::insertBranch(
815 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
816 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
817 // Shouldn't be a fall through.
818 assert(TBB && "insertBranch must not be told to insert a fallthrough");
819
820 if (!FBB) {
821 if (Cond.empty()) // Unconditional branch?
822 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: TBB);
823 else
824 instantiateCondBranch(MBB, DL, TBB, Cond);
825
826 if (BytesAdded)
827 *BytesAdded = 4;
828
829 return 1;
830 }
831
832 // Two-way conditional branch.
833 instantiateCondBranch(MBB, DL, TBB, Cond);
834 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: FBB);
835
836 if (BytesAdded)
837 *BytesAdded = 8;
838
839 return 2;
840}
841
842#ifndef NDEBUG
843static bool isValidCBExtend(int64_t Opc, AArch64_AM::ShiftExtendType Ext) {
844 switch (Ext) {
845 default:
846 return false;
847 case AArch64_AM::UXTB:
848 case AArch64_AM::SXTB:
849 return Opc == AArch64::CBBAssertExt;
850 case AArch64_AM::UXTH:
851 case AArch64_AM::SXTH:
852 return Opc == AArch64::CBHAssertExt;
853 }
854}
855#endif
856
857AArch64CC::CondCode AArch64InstrInfo::insertCmpForCondBr(
858 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL,
859 ArrayRef<MachineOperand> Cond) const {
860 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
861
862 // Parse the condition code, see parseCondBranch() above.
863 AArch64CC::CondCode CC;
864 switch (Cond.size()) {
865 default:
866 llvm_unreachable("Unknown condition opcode in Cond");
867 case 1: // b.cc
868 CC = AArch64CC::CondCode(Cond[0].getImm());
869 break;
870 case 3: { // cbz/cbnz
871 // We must insert a compare against 0.
872 bool Is64Bit;
873 switch (Cond[1].getImm()) {
874 default:
875 llvm_unreachable("Unknown branch opcode in Cond");
876 case AArch64::CBZW:
877 Is64Bit = false;
878 CC = AArch64CC::EQ;
879 break;
880 case AArch64::CBZX:
881 Is64Bit = true;
882 CC = AArch64CC::EQ;
883 break;
884 case AArch64::CBNZW:
885 Is64Bit = false;
886 CC = AArch64CC::NE;
887 break;
888 case AArch64::CBNZX:
889 Is64Bit = true;
890 CC = AArch64CC::NE;
891 break;
892 }
893 Register SrcReg = Cond[2].getReg();
894 if (Is64Bit) {
895 // cmp reg, #0 is actually subs xzr, reg, #0.
896 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64spRegClass);
897 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSXri), DestReg: AArch64::XZR)
898 .addReg(RegNo: SrcReg)
899 .addImm(Val: 0)
900 .addImm(Val: 0);
901 } else {
902 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32spRegClass);
903 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWri), DestReg: AArch64::WZR)
904 .addReg(RegNo: SrcReg)
905 .addImm(Val: 0)
906 .addImm(Val: 0);
907 }
908 } break;
909 case 4: { // tbz/tbnz
910 // We must insert a tst instruction.
911 switch (Cond[1].getImm()) {
912 default:
913 llvm_unreachable("Unknown branch opcode in Cond");
914 case AArch64::TBZW:
915 case AArch64::TBZX:
916 CC = AArch64CC::EQ;
917 break;
918 case AArch64::TBNZW:
919 case AArch64::TBNZX:
920 CC = AArch64CC::NE;
921 break;
922 }
923 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
924 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
925 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::ANDSWri), DestReg: AArch64::WZR)
926 .addReg(RegNo: Cond[2].getReg())
927 .addImm(
928 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 32));
929 else
930 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::ANDSXri), DestReg: AArch64::XZR)
931 .addReg(RegNo: Cond[2].getReg())
932 .addImm(
933 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 64));
934 } break;
935 case 5: { // cb
936 // We must insert a cmp, that is a subs
937 // 0 1 2 3 4
938 // Cond is { -1, Opcode, CC, Op0, Op1 }
939 unsigned SubsOpc, SubsDestReg;
940 bool IsImm = false;
941 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
942 switch (Cond[1].getImm()) {
943 default:
944 llvm_unreachable("Unknown branch opcode in Cond");
945 case AArch64::CBWPri:
946 SubsOpc = AArch64::SUBSWri;
947 SubsDestReg = AArch64::WZR;
948 IsImm = true;
949 break;
950 case AArch64::CBXPri:
951 SubsOpc = AArch64::SUBSXri;
952 SubsDestReg = AArch64::XZR;
953 IsImm = true;
954 break;
955 case AArch64::CBWPrr:
956 SubsOpc = AArch64::SUBSWrr;
957 SubsDestReg = AArch64::WZR;
958 IsImm = false;
959 break;
960 case AArch64::CBXPrr:
961 SubsOpc = AArch64::SUBSXrr;
962 SubsDestReg = AArch64::XZR;
963 IsImm = false;
964 break;
965 }
966
967 if (IsImm) {
968 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 1));
969 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
970 .addReg(RegNo: Cond[3].getReg())
971 .addImm(Val: Cond[4].getImm())
972 .addImm(Val: 0);
973 } else {
974 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 1));
975 MRI.constrainRegClass(Reg: Cond[4].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 2));
976 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
977 .addReg(RegNo: Cond[3].getReg())
978 .addReg(RegNo: Cond[4].getReg());
979 }
980 } break;
981 case 7: { // cb[b,h]
982 // We must insert a cmp, that is a subs, but also zero- or sign-extensions
983 // that have been folded. For the first operand we codegen an explicit
984 // extension, for the second operand we fold the extension into cmp.
985 // 0 1 2 3 4 5 6
986 // Cond is { -1, Opcode, CC, Op0, Op1, Ext0, Ext1 }
987
988 // We need a new register for the now explicitly extended register
989 Register Reg = Cond[3].getReg();
990 if (Cond[5].getImm() != AArch64_AM::InvalidShiftExtend) {
991 unsigned ExtOpc;
992 unsigned ExtBits;
993 AArch64_AM::ShiftExtendType ExtendType =
994 AArch64_AM::getExtendType(Imm: Cond[5].getImm());
995 assert(isValidCBExtend(Cond[1].getImm(), ExtendType) &&
996 "Unexpected compare-and-branch instruction for extend type");
997 switch (ExtendType) {
998 default:
999 llvm_unreachable("Unknown shift-extend for CB instruction");
1000 case AArch64_AM::SXTB:
1001 ExtOpc = AArch64::SBFMWri;
1002 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1003 break;
1004 case AArch64_AM::SXTH:
1005 ExtOpc = AArch64::SBFMWri;
1006 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1007 break;
1008 case AArch64_AM::UXTB:
1009 ExtOpc = AArch64::ANDWri;
1010 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1011 break;
1012 case AArch64_AM::UXTH:
1013 ExtOpc = AArch64::ANDWri;
1014 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1015 break;
1016 }
1017
1018 // Build the explicit extension of the first operand
1019 Reg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32commonRegClass);
1020 MachineInstrBuilder MBBI =
1021 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ExtOpc), DestReg: Reg).addReg(RegNo: Cond[3].getReg());
1022 if (ExtOpc != AArch64::ANDWri)
1023 MBBI.addImm(Val: 0);
1024 MBBI.addImm(Val: ExtBits);
1025 }
1026
1027 // Now, subs with an extended second operand
1028 if (Cond[6].getImm() != AArch64_AM::InvalidShiftExtend) {
1029 MRI.constrainRegClass(Reg, RC: &AArch64::GPR32commonRegClass);
1030 AArch64_AM::ShiftExtendType ExtendType =
1031 AArch64_AM::getExtendType(Imm: Cond[6].getImm());
1032 assert(isValidCBExtend(Cond[1].getImm(), ExtendType) &&
1033 "Unexpected compare-and-branch instruction for extend type");
1034 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrx), DestReg: AArch64::WZR)
1035 .addReg(RegNo: Reg)
1036 .addReg(RegNo: Cond[4].getReg())
1037 .addImm(Val: AArch64_AM::getArithExtendImm(ET: ExtendType, Imm: 0));
1038 } // If no extension is needed, just a regular subs
1039 else {
1040 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrr), DestReg: AArch64::WZR)
1041 .addReg(RegNo: Reg)
1042 .addReg(RegNo: Cond[4].getReg());
1043 }
1044
1045 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
1046 } break;
1047 }
1048 return CC;
1049}
1050
1051bool llvm::optimizeTerminators(MachineBasicBlock *MBB,
1052 const TargetInstrInfo &TII) {
1053 for (MachineInstr &MI : MBB->terminators()) {
1054 unsigned Opc = MI.getOpcode();
1055 switch (Opc) {
1056 case AArch64::CBZW:
1057 case AArch64::CBZX:
1058 case AArch64::TBZW:
1059 case AArch64::TBZX:
1060 // CBZ/TBZ with WZR/XZR -> unconditional B
1061 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
1062 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
1063 DEBUG_WITH_TYPE("optimizeTerminators",
1064 dbgs() << "Removing always taken branch: " << MI);
1065 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
1066 SmallVector<MachineBasicBlock *> Succs(MBB->successors());
1067 for (auto *S : Succs)
1068 if (S != Target)
1069 MBB->removeSuccessor(Succ: S);
1070 DebugLoc DL = MI.getDebugLoc();
1071 while (MBB->rbegin() != &MI)
1072 MBB->rbegin()->eraseFromParent();
1073 MI.eraseFromParent();
1074 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: AArch64::B)).addMBB(MBB: Target);
1075 return true;
1076 }
1077 break;
1078 case AArch64::CBNZW:
1079 case AArch64::CBNZX:
1080 case AArch64::TBNZW:
1081 case AArch64::TBNZX:
1082 // CBNZ/TBNZ with WZR/XZR -> never taken, remove branch and successor
1083 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
1084 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
1085 DEBUG_WITH_TYPE("optimizeTerminators",
1086 dbgs() << "Removing never taken branch: " << MI);
1087 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
1088 MI.getParent()->removeSuccessor(Succ: Target);
1089 MI.eraseFromParent();
1090 return true;
1091 }
1092 break;
1093 }
1094 }
1095 return false;
1096}
1097
1098// Find the original register that VReg is copied from.
1099static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
1100 while (Register::isVirtualRegister(Reg: VReg)) {
1101 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1102 if (!DefMI || !DefMI->isFullCopy())
1103 return VReg;
1104 VReg = DefMI->getOperand(i: 1).getReg();
1105 }
1106 return VReg;
1107}
1108
1109// Determine if VReg is defined by an instruction that can be folded into a
1110// csel instruction. If so, return the folded opcode, and the replacement
1111// register.
1112static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
1113 unsigned *NewReg = nullptr) {
1114 VReg = removeCopies(MRI, VReg);
1115 if (!Register::isVirtualRegister(Reg: VReg))
1116 return 0;
1117
1118 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(RC: MRI.getRegClass(Reg: VReg));
1119 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1120 if (!DefMI)
1121 return 0;
1122 unsigned Opc = 0;
1123 unsigned SrcReg = 0;
1124 switch (DefMI->getOpcode()) {
1125 case AArch64::SUBREG_TO_REG:
1126 // Check for the following way to define an 64-bit immediate:
1127 // %0:gpr32 = MOVi32imm 1
1128 // %1:gpr64 = SUBREG_TO_REG %0:gpr32, %subreg.sub_32
1129 if (!DefMI->getOperand(i: 1).isReg())
1130 return 0;
1131 if (!DefMI->getOperand(i: 2).isImm() ||
1132 DefMI->getOperand(i: 2).getImm() != AArch64::sub_32)
1133 return 0;
1134 DefMI = MRI.getVRegDef(Reg: DefMI->getOperand(i: 1).getReg());
1135 if (DefMI->getOpcode() != AArch64::MOVi32imm)
1136 return 0;
1137 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
1138 return 0;
1139 assert(Is64Bit);
1140 SrcReg = AArch64::XZR;
1141 Opc = AArch64::CSINCXr;
1142 break;
1143
1144 case AArch64::MOVi32imm:
1145 case AArch64::MOVi64imm:
1146 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
1147 return 0;
1148 SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1149 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
1150 break;
1151
1152 case AArch64::ADDSXri:
1153 case AArch64::ADDSWri:
1154 // if NZCV is used, do not fold.
1155 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1156 isDead: true) == -1)
1157 return 0;
1158 // fall-through to ADDXri and ADDWri.
1159 [[fallthrough]];
1160 case AArch64::ADDXri:
1161 case AArch64::ADDWri:
1162 // add x, 1 -> csinc.
1163 if (!DefMI->getOperand(i: 2).isImm() || DefMI->getOperand(i: 2).getImm() != 1 ||
1164 DefMI->getOperand(i: 3).getImm() != 0)
1165 return 0;
1166 SrcReg = DefMI->getOperand(i: 1).getReg();
1167 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
1168 break;
1169
1170 case AArch64::ORNXrr:
1171 case AArch64::ORNWrr: {
1172 // not x -> csinv, represented as orn dst, xzr, src.
1173 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1174 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1175 return 0;
1176 SrcReg = DefMI->getOperand(i: 2).getReg();
1177 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
1178 break;
1179 }
1180
1181 case AArch64::SUBSXrr:
1182 case AArch64::SUBSWrr:
1183 // if NZCV is used, do not fold.
1184 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1185 isDead: true) == -1)
1186 return 0;
1187 // fall-through to SUBXrr and SUBWrr.
1188 [[fallthrough]];
1189 case AArch64::SUBXrr:
1190 case AArch64::SUBWrr: {
1191 // neg x -> csneg, represented as sub dst, xzr, src.
1192 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1193 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1194 return 0;
1195 SrcReg = DefMI->getOperand(i: 2).getReg();
1196 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
1197 break;
1198 }
1199 default:
1200 return 0;
1201 }
1202 assert(Opc && SrcReg && "Missing parameters");
1203
1204 if (NewReg)
1205 *NewReg = SrcReg;
1206 return Opc;
1207}
1208
1209bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1210 ArrayRef<MachineOperand> Cond,
1211 Register DstReg, Register TrueReg,
1212 Register FalseReg,
1213 SelectExpansion &Exp) const {
1214 // Check register classes.
1215 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1216 const TargetRegisterClass *RC =
1217 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1218 if (!RC)
1219 return false;
1220
1221 // Also need to check the dest regclass, in case we're trying to optimize
1222 // something like:
1223 // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
1224 if (!RI.getCommonSubClass(A: RC, B: MRI.getRegClass(Reg: DstReg)))
1225 return false;
1226
1227 // Expanding cbz/tbz requires a flag setting instruction in addition to the
1228 // select, adding an extra cycle of latency on the condition.
1229 unsigned ExtraCondLat = Cond.size() != 1;
1230 Exp.NumInsts = 1 + ExtraCondLat;
1231 if (ExtraCondLat)
1232 Exp.ClobberedPhysRegs.push_back(Elt: AArch64::NZCV);
1233
1234 // GPRs are handled by csel.
1235 // FIXME: Fold in x+1, -x, and ~x when applicable.
1236 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
1237 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1238 // Single-cycle csel, csinc, csinv, and csneg.
1239 Exp.CondCycles = 1 + ExtraCondLat;
1240 Exp.TrueCycles = Exp.FalseCycles = 1;
1241 if (canFoldIntoCSel(MRI, VReg: TrueReg))
1242 Exp.TrueCycles = 0;
1243 else if (canFoldIntoCSel(MRI, VReg: FalseReg))
1244 Exp.FalseCycles = 0;
1245 return true;
1246 }
1247
1248 // Scalar floating point is handled by fcsel.
1249 // FIXME: Form fabs, fmin, and fmax when applicable.
1250 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
1251 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
1252 Exp.CondCycles = 5 + ExtraCondLat;
1253 Exp.TrueCycles = Exp.FalseCycles = 2;
1254 return true;
1255 }
1256
1257 // No single conditional move for a 128-bit vector, but we can emit a sequence
1258 // of csetm (~1), dup (~5, cross domain), bsl (~2).
1259 if (AArch64::FPR128RegClass.hasSubClassEq(RC) &&
1260 Subtarget.isNeonAvailable() &&
1261 !MBB.getParent()->getFunction().hasMinSize()) {
1262 Exp.CondCycles = 8 + ExtraCondLat;
1263 Exp.TrueCycles = Exp.FalseCycles = 2;
1264 Exp.NumInsts = 3 + ExtraCondLat;
1265 return true;
1266 }
1267
1268 return false;
1269}
1270
1271void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
1272 MachineBasicBlock::iterator I,
1273 const DebugLoc &DL, Register DstReg,
1274 ArrayRef<MachineOperand> Cond,
1275 Register TrueReg, Register FalseReg) const {
1276
1277 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1278 AArch64CC::CondCode CC = insertCmpForCondBr(MBB, MI: I, DL, Cond);
1279
1280 // A 128-bit vector has no conditional move so blend the operands with a mask
1281 // built from the flags.
1282 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR128RegClass)) {
1283 assert(Subtarget.isNeonAvailable() && "Expected NEON for a vector select");
1284 MRI.constrainRegClass(Reg: TrueReg, RC: &AArch64::FPR128RegClass);
1285 MRI.constrainRegClass(Reg: FalseReg, RC: &AArch64::FPR128RegClass);
1286 Register CondSet = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1287 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::CSINVXr), DestReg: CondSet)
1288 .addReg(RegNo: AArch64::XZR)
1289 .addReg(RegNo: AArch64::XZR)
1290 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
1291 Register Mask = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
1292 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::DUPv2i64gpr), DestReg: Mask).addReg(RegNo: CondSet);
1293 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::BSPv16i8), DestReg: DstReg)
1294 .addReg(RegNo: Mask)
1295 .addReg(RegNo: TrueReg)
1296 .addReg(RegNo: FalseReg);
1297 return;
1298 }
1299
1300 unsigned Opc = 0;
1301 const TargetRegisterClass *RC = nullptr;
1302 bool TryFold = false;
1303 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass)) {
1304 RC = &AArch64::GPR64RegClass;
1305 Opc = AArch64::CSELXr;
1306 TryFold = true;
1307 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR32RegClass)) {
1308 RC = &AArch64::GPR32RegClass;
1309 Opc = AArch64::CSELWr;
1310 TryFold = true;
1311 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR64RegClass)) {
1312 RC = &AArch64::FPR64RegClass;
1313 Opc = AArch64::FCSELDrrr;
1314 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR32RegClass)) {
1315 RC = &AArch64::FPR32RegClass;
1316 Opc = AArch64::FCSELSrrr;
1317 }
1318 assert(RC && "Unsupported regclass");
1319
1320 // Try folding simple instructions into the csel.
1321 if (TryFold) {
1322 unsigned NewReg = 0;
1323 unsigned FoldedOpc = canFoldIntoCSel(MRI, VReg: TrueReg, NewReg: &NewReg);
1324 if (FoldedOpc) {
1325 // The folded opcodes csinc, csinc and csneg apply the operation to
1326 // FalseReg, so we need to invert the condition.
1327 CC = AArch64CC::getInvertedCondCode(Code: CC);
1328 TrueReg = FalseReg;
1329 } else
1330 FoldedOpc = canFoldIntoCSel(MRI, VReg: FalseReg, NewReg: &NewReg);
1331
1332 // Fold the operation. Leave any dead instructions for DCE to clean up.
1333 if (FoldedOpc) {
1334 FalseReg = NewReg;
1335 Opc = FoldedOpc;
1336 // Extend the live range of NewReg.
1337 MRI.clearKillFlags(Reg: NewReg);
1338 }
1339 }
1340
1341 // Pull all virtual register into the appropriate class.
1342 MRI.constrainRegClass(Reg: TrueReg, RC);
1343 // FalseReg might be WZR or XZR if the folded operand is a literal 1.
1344 assert(
1345 (FalseReg.isVirtual() || FalseReg == AArch64::WZR ||
1346 FalseReg == AArch64::XZR) &&
1347 "FalseReg was folded into a non-virtual register other than WZR or XZR");
1348 if (FalseReg.isVirtual())
1349 MRI.constrainRegClass(Reg: FalseReg, RC);
1350
1351 // Insert the csel.
1352 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
1353 .addReg(RegNo: TrueReg)
1354 .addReg(RegNo: FalseReg)
1355 .addImm(Val: CC);
1356}
1357
1358// Return true if Imm can be loaded into a register by a "cheap" sequence of
1359// instructions. For now, "cheap" means at most two instructions.
1360static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize) {
1361 if (BitSize == 32)
1362 return true;
1363
1364 assert(BitSize == 64 && "Only bit sizes of 32 or 64 allowed");
1365 uint64_t Imm = static_cast<uint64_t>(MI.getOperand(i: 1).getImm());
1366 SmallVector<AArch64_IMM::ImmInsnModel, 4> Is;
1367 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: Is);
1368
1369 return Is.size() <= 2;
1370}
1371
1372// Check if a COPY instruction is cheap.
1373static bool isCheapCopy(const MachineInstr &MI, const AArch64RegisterInfo &RI) {
1374 assert(MI.isCopy() && "Expected COPY instruction");
1375 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1376
1377 // Cross-bank copies (e.g., between GPR and FPR) are expensive on AArch64,
1378 // typically requiring an FMOV instruction with a 2-6 cycle latency.
1379 auto GetRegClass = [&](Register Reg) -> const TargetRegisterClass * {
1380 if (Reg.isVirtual())
1381 return MRI.getRegClass(Reg);
1382 if (Reg.isPhysical())
1383 return RI.getMinimalPhysRegClass(Reg);
1384 return nullptr;
1385 };
1386 const TargetRegisterClass *DstRC = GetRegClass(MI.getOperand(i: 0).getReg());
1387 const TargetRegisterClass *SrcRC = GetRegClass(MI.getOperand(i: 1).getReg());
1388 if (DstRC && SrcRC && !RI.getCommonSubClass(A: DstRC, B: SrcRC))
1389 return false;
1390
1391 return MI.isAsCheapAsAMove();
1392}
1393
1394// FIXME: this implementation should be micro-architecture dependent, so a
1395// micro-architecture target hook should be introduced here in future.
1396bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
1397 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1398 if (isExynosCheapAsMove(MI))
1399 return true;
1400 return MI.isAsCheapAsAMove();
1401 }
1402
1403 switch (MI.getOpcode()) {
1404 default:
1405 return MI.isAsCheapAsAMove();
1406
1407 case TargetOpcode::COPY:
1408 return isCheapCopy(MI, RI);
1409
1410 case AArch64::ADDWrs:
1411 case AArch64::ADDXrs:
1412 case AArch64::SUBWrs:
1413 case AArch64::SUBXrs:
1414 return Subtarget.hasALULSLFast() && MI.getOperand(i: 3).getImm() <= 4;
1415
1416 // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
1417 // ORRXri, it is as cheap as MOV.
1418 // Likewise if it can be expanded to MOVZ/MOVN/MOVK.
1419 case AArch64::MOVi32imm:
1420 return isCheapImmediate(MI, BitSize: 32);
1421 case AArch64::MOVi64imm:
1422 return isCheapImmediate(MI, BitSize: 64);
1423 }
1424}
1425
1426bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
1427 switch (MI.getOpcode()) {
1428 default:
1429 return false;
1430
1431 case AArch64::ADDWrs:
1432 case AArch64::ADDXrs:
1433 case AArch64::ADDSWrs:
1434 case AArch64::ADDSXrs: {
1435 unsigned Imm = MI.getOperand(i: 3).getImm();
1436 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1437 if (ShiftVal == 0)
1438 return true;
1439 return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
1440 }
1441
1442 case AArch64::ADDWrx:
1443 case AArch64::ADDXrx:
1444 case AArch64::ADDXrx64:
1445 case AArch64::ADDSWrx:
1446 case AArch64::ADDSXrx:
1447 case AArch64::ADDSXrx64: {
1448 unsigned Imm = MI.getOperand(i: 3).getImm();
1449 switch (AArch64_AM::getArithExtendType(Imm)) {
1450 default:
1451 return false;
1452 case AArch64_AM::UXTB:
1453 case AArch64_AM::UXTH:
1454 case AArch64_AM::UXTW:
1455 case AArch64_AM::UXTX:
1456 return AArch64_AM::getArithShiftValue(Imm) <= 4;
1457 }
1458 }
1459
1460 case AArch64::SUBWrs:
1461 case AArch64::SUBSWrs: {
1462 unsigned Imm = MI.getOperand(i: 3).getImm();
1463 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1464 return ShiftVal == 0 ||
1465 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
1466 }
1467
1468 case AArch64::SUBXrs:
1469 case AArch64::SUBSXrs: {
1470 unsigned Imm = MI.getOperand(i: 3).getImm();
1471 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1472 return ShiftVal == 0 ||
1473 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
1474 }
1475
1476 case AArch64::SUBWrx:
1477 case AArch64::SUBXrx:
1478 case AArch64::SUBXrx64:
1479 case AArch64::SUBSWrx:
1480 case AArch64::SUBSXrx:
1481 case AArch64::SUBSXrx64: {
1482 unsigned Imm = MI.getOperand(i: 3).getImm();
1483 switch (AArch64_AM::getArithExtendType(Imm)) {
1484 default:
1485 return false;
1486 case AArch64_AM::UXTB:
1487 case AArch64_AM::UXTH:
1488 case AArch64_AM::UXTW:
1489 case AArch64_AM::UXTX:
1490 return AArch64_AM::getArithShiftValue(Imm) == 0;
1491 }
1492 }
1493
1494 case AArch64::LDRBBroW:
1495 case AArch64::LDRBBroX:
1496 case AArch64::LDRBroW:
1497 case AArch64::LDRBroX:
1498 case AArch64::LDRDroW:
1499 case AArch64::LDRDroX:
1500 case AArch64::LDRHHroW:
1501 case AArch64::LDRHHroX:
1502 case AArch64::LDRHroW:
1503 case AArch64::LDRHroX:
1504 case AArch64::LDRQroW:
1505 case AArch64::LDRQroX:
1506 case AArch64::LDRSBWroW:
1507 case AArch64::LDRSBWroX:
1508 case AArch64::LDRSBXroW:
1509 case AArch64::LDRSBXroX:
1510 case AArch64::LDRSHWroW:
1511 case AArch64::LDRSHWroX:
1512 case AArch64::LDRSHXroW:
1513 case AArch64::LDRSHXroX:
1514 case AArch64::LDRSWroW:
1515 case AArch64::LDRSWroX:
1516 case AArch64::LDRSroW:
1517 case AArch64::LDRSroX:
1518 case AArch64::LDRWroW:
1519 case AArch64::LDRWroX:
1520 case AArch64::LDRXroW:
1521 case AArch64::LDRXroX:
1522 case AArch64::PRFMroW:
1523 case AArch64::PRFMroX:
1524 case AArch64::STRBBroW:
1525 case AArch64::STRBBroX:
1526 case AArch64::STRBroW:
1527 case AArch64::STRBroX:
1528 case AArch64::STRDroW:
1529 case AArch64::STRDroX:
1530 case AArch64::STRHHroW:
1531 case AArch64::STRHHroX:
1532 case AArch64::STRHroW:
1533 case AArch64::STRHroX:
1534 case AArch64::STRQroW:
1535 case AArch64::STRQroX:
1536 case AArch64::STRSroW:
1537 case AArch64::STRSroX:
1538 case AArch64::STRWroW:
1539 case AArch64::STRWroX:
1540 case AArch64::STRXroW:
1541 case AArch64::STRXroX: {
1542 unsigned IsSigned = MI.getOperand(i: 3).getImm();
1543 return !IsSigned;
1544 }
1545 }
1546}
1547
1548bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
1549 unsigned Opc = MI.getOpcode();
1550 switch (Opc) {
1551 default:
1552 return false;
1553 case AArch64::SEH_StackAlloc:
1554 case AArch64::SEH_SaveFPLR:
1555 case AArch64::SEH_SaveFPLR_X:
1556 case AArch64::SEH_SaveReg:
1557 case AArch64::SEH_SaveReg_X:
1558 case AArch64::SEH_SaveRegP:
1559 case AArch64::SEH_SaveRegP_X:
1560 case AArch64::SEH_SaveFReg:
1561 case AArch64::SEH_SaveFReg_X:
1562 case AArch64::SEH_SaveFRegP:
1563 case AArch64::SEH_SaveFRegP_X:
1564 case AArch64::SEH_SetFP:
1565 case AArch64::SEH_AddFP:
1566 case AArch64::SEH_Nop:
1567 case AArch64::SEH_PrologEnd:
1568 case AArch64::SEH_EpilogStart:
1569 case AArch64::SEH_EpilogEnd:
1570 case AArch64::SEH_PACSignLR:
1571 case AArch64::SEH_SaveAnyRegI:
1572 case AArch64::SEH_SaveAnyRegIP:
1573 case AArch64::SEH_SaveAnyRegQP:
1574 case AArch64::SEH_SaveAnyRegQPX:
1575 case AArch64::SEH_AllocZ:
1576 case AArch64::SEH_SaveZReg:
1577 case AArch64::SEH_SavePReg:
1578 return true;
1579 }
1580}
1581
1582bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1583 Register &SrcReg, Register &DstReg,
1584 unsigned &SubIdx) const {
1585 switch (MI.getOpcode()) {
1586 default:
1587 return false;
1588 case AArch64::SBFMXri: // aka sxtw
1589 case AArch64::UBFMXri: // aka uxtw
1590 // Check for the 32 -> 64 bit extension case, these instructions can do
1591 // much more.
1592 if (MI.getOperand(i: 2).getImm() != 0 || MI.getOperand(i: 3).getImm() != 31)
1593 return false;
1594 // This is a signed or unsigned 32 -> 64 bit extension.
1595 SrcReg = MI.getOperand(i: 1).getReg();
1596 DstReg = MI.getOperand(i: 0).getReg();
1597 SubIdx = AArch64::sub_32;
1598 return true;
1599 }
1600}
1601
1602bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
1603 const MachineInstr &MIa, const MachineInstr &MIb) const {
1604 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1605 int64_t OffsetA = 0, OffsetB = 0;
1606 TypeSize WidthA(0, false), WidthB(0, false);
1607 bool OffsetAIsScalable = false, OffsetBIsScalable = false;
1608
1609 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1610 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1611
1612 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
1613 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1614 return false;
1615
1616 // Retrieve the base, offset from the base and width. Width
1617 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
1618 // base are identical, and the offset of a lower memory access +
1619 // the width doesn't overlap the offset of a higher memory access,
1620 // then the memory accesses are different.
1621 // If OffsetAIsScalable and OffsetBIsScalable are both true, they
1622 // are assumed to have the same scale (vscale).
1623 if (getMemOperandWithOffsetWidth(MI: MIa, BaseOp&: BaseOpA, Offset&: OffsetA, OffsetIsScalable&: OffsetAIsScalable,
1624 Width&: WidthA) &&
1625 getMemOperandWithOffsetWidth(MI: MIb, BaseOp&: BaseOpB, Offset&: OffsetB, OffsetIsScalable&: OffsetBIsScalable,
1626 Width&: WidthB)) {
1627 if (BaseOpA->isIdenticalTo(Other: *BaseOpB) &&
1628 OffsetAIsScalable == OffsetBIsScalable) {
1629 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1630 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1631 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1632 if (LowWidth.isScalable() == OffsetAIsScalable &&
1633 LowOffset + (int)LowWidth.getKnownMinValue() <= HighOffset)
1634 return true;
1635 }
1636 }
1637 return false;
1638}
1639
1640bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1641 const MachineBasicBlock *MBB,
1642 const MachineFunction &MF) const {
1643 if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
1644 return true;
1645
1646 // Do not move an instruction that can be recognized as a branch target.
1647 if (hasBTISemantics(MI))
1648 return true;
1649
1650 switch (MI.getOpcode()) {
1651 case AArch64::HINT:
1652 // CSDB hints are scheduling barriers.
1653 if (MI.getOperand(i: 0).getImm() == 0x14)
1654 return true;
1655 break;
1656 case AArch64::DSB:
1657 case AArch64::ISB:
1658 // DSB and ISB also are scheduling barriers.
1659 return true;
1660 case AArch64::MSRpstatesvcrImm1:
1661 // SMSTART and SMSTOP are also scheduling barriers.
1662 return true;
1663 default:;
1664 }
1665 if (isSEHInstruction(MI))
1666 return true;
1667 auto Next = std::next(x: MI.getIterator());
1668 return Next != MBB->end() && Next->isCFIInstruction();
1669}
1670
1671/// analyzeCompare - For a comparison instruction, return the source registers
1672/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1673/// Return true if the comparison instruction can be analyzed.
1674bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1675 Register &SrcReg2, int64_t &CmpMask,
1676 int64_t &CmpValue) const {
1677 // The first operand can be a frame index where we'd normally expect a
1678 // register.
1679 // FIXME: Pass subregisters out of analyzeCompare
1680 assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1681 if (!MI.getOperand(i: 1).isReg() || MI.getOperand(i: 1).getSubReg())
1682 return false;
1683
1684 switch (MI.getOpcode()) {
1685 default:
1686 break;
1687 case AArch64::PTEST_PP:
1688 case AArch64::PTEST_PP_ANY:
1689 case AArch64::PTEST_PP_FIRST:
1690 SrcReg = MI.getOperand(i: 0).getReg();
1691 SrcReg2 = MI.getOperand(i: 1).getReg();
1692 if (MI.getOperand(i: 2).getSubReg())
1693 return false;
1694
1695 // Not sure about the mask and value for now...
1696 CmpMask = ~0;
1697 CmpValue = 0;
1698 return true;
1699 case AArch64::SUBSWrr:
1700 case AArch64::SUBSWrs:
1701 case AArch64::SUBSWrx:
1702 case AArch64::SUBSXrr:
1703 case AArch64::SUBSXrs:
1704 case AArch64::SUBSXrx:
1705 case AArch64::ADDSWrr:
1706 case AArch64::ADDSWrs:
1707 case AArch64::ADDSWrx:
1708 case AArch64::ADDSXrr:
1709 case AArch64::ADDSXrs:
1710 case AArch64::ADDSXrx:
1711 // Replace SUBSWrr with SUBWrr if NZCV is not used.
1712 SrcReg = MI.getOperand(i: 1).getReg();
1713 SrcReg2 = MI.getOperand(i: 2).getReg();
1714
1715 // FIXME: Pass subregisters out of analyzeCompare
1716 if (MI.getOperand(i: 2).getSubReg())
1717 return false;
1718
1719 CmpMask = ~0;
1720 CmpValue = 0;
1721 return true;
1722 case AArch64::SUBSWri:
1723 case AArch64::ADDSWri:
1724 case AArch64::SUBSXri:
1725 case AArch64::ADDSXri:
1726 SrcReg = MI.getOperand(i: 1).getReg();
1727 SrcReg2 = 0;
1728 CmpMask = ~0;
1729 CmpValue = MI.getOperand(i: 2).getImm();
1730 return true;
1731 case AArch64::ANDSWri:
1732 case AArch64::ANDSXri:
1733 // ANDS does not use the same encoding scheme as the others xxxS
1734 // instructions.
1735 SrcReg = MI.getOperand(i: 1).getReg();
1736 SrcReg2 = 0;
1737 CmpMask = ~0;
1738 CmpValue = AArch64_AM::decodeLogicalImmediate(
1739 val: MI.getOperand(i: 2).getImm(),
1740 regSize: MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1741 return true;
1742 }
1743
1744 return false;
1745}
1746
1747static bool UpdateOperandRegClass(MachineInstr &Instr) {
1748 MachineBasicBlock *MBB = Instr.getParent();
1749 assert(MBB && "Can't get MachineBasicBlock here");
1750 MachineFunction *MF = MBB->getParent();
1751 assert(MF && "Can't get MachineFunction here");
1752 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1753 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1754 MachineRegisterInfo *MRI = &MF->getRegInfo();
1755
1756 for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1757 ++OpIdx) {
1758 MachineOperand &MO = Instr.getOperand(i: OpIdx);
1759 const TargetRegisterClass *OpRegCstraints =
1760 Instr.getRegClassConstraint(OpIdx, TII, TRI);
1761
1762 // If there's no constraint, there's nothing to do.
1763 if (!OpRegCstraints)
1764 continue;
1765 // If the operand is a frame index, there's nothing to do here.
1766 // A frame index operand will resolve correctly during PEI.
1767 if (MO.isFI())
1768 continue;
1769
1770 assert(MO.isReg() &&
1771 "Operand has register constraints without being a register!");
1772
1773 Register Reg = MO.getReg();
1774 if (Reg.isPhysical()) {
1775 if (!OpRegCstraints->contains(Reg))
1776 return false;
1777 } else if (!OpRegCstraints->hasSubClassEq(RC: MRI->getRegClass(Reg)) &&
1778 !MRI->constrainRegClass(Reg, RC: OpRegCstraints))
1779 return false;
1780 }
1781
1782 return true;
1783}
1784
1785/// Return the opcode that does not set flags when possible - otherwise
1786/// return the original opcode. The caller is responsible to do the actual
1787/// substitution and legality checking.
1788static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1789 // Don't convert all compare instructions, because for some the zero register
1790 // encoding becomes the sp register.
1791 bool MIDefinesZeroReg = false;
1792 if (MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
1793 MI.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
1794 MIDefinesZeroReg = true;
1795
1796 switch (MI.getOpcode()) {
1797 default:
1798 return MI.getOpcode();
1799 case AArch64::ADDSWrr:
1800 return AArch64::ADDWrr;
1801 case AArch64::ADDSWri:
1802 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1803 case AArch64::ADDSWrs:
1804 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1805 case AArch64::ADDSWrx:
1806 return AArch64::ADDWrx;
1807 case AArch64::ADDSXrr:
1808 return AArch64::ADDXrr;
1809 case AArch64::ADDSXri:
1810 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1811 case AArch64::ADDSXrs:
1812 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1813 case AArch64::ADDSXrx:
1814 return AArch64::ADDXrx;
1815 case AArch64::SUBSWrr:
1816 return AArch64::SUBWrr;
1817 case AArch64::SUBSWri:
1818 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1819 case AArch64::SUBSWrs:
1820 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1821 case AArch64::SUBSWrx:
1822 return AArch64::SUBWrx;
1823 case AArch64::SUBSXrr:
1824 return AArch64::SUBXrr;
1825 case AArch64::SUBSXri:
1826 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1827 case AArch64::SUBSXrs:
1828 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1829 case AArch64::SUBSXrx:
1830 return AArch64::SUBXrx;
1831 }
1832}
1833
1834enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1835
1836/// True when condition flags are accessed (either by writing or reading)
1837/// on the instruction trace starting at From and ending at To.
1838///
1839/// Note: If From and To are from different blocks it's assumed CC are accessed
1840/// on the path.
1841static bool areCFlagsAccessedBetweenInstrs(
1842 MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1843 const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1844 // Early exit if To is at the beginning of the BB.
1845 if (To == To->getParent()->begin())
1846 return true;
1847
1848 // Check whether the instructions are in the same basic block
1849 // If not, assume the condition flags might get modified somewhere.
1850 if (To->getParent() != From->getParent())
1851 return true;
1852
1853 // From must be above To.
1854 assert(std::any_of(
1855 ++To.getReverse(), To->getParent()->rend(),
1856 [From](MachineInstr &MI) { return MI.getIterator() == From; }));
1857
1858 // We iterate backward starting at \p To until we hit \p From.
1859 for (const MachineInstr &Instr :
1860 instructionsWithoutDebug(It: ++To.getReverse(), End: From.getReverse())) {
1861 if (((AccessToCheck & AK_Write) &&
1862 Instr.modifiesRegister(Reg: AArch64::NZCV, TRI)) ||
1863 ((AccessToCheck & AK_Read) && Instr.readsRegister(Reg: AArch64::NZCV, TRI)))
1864 return true;
1865 }
1866 return false;
1867}
1868
1869std::optional<unsigned>
1870AArch64InstrInfo::canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
1871 MachineInstr *Pred,
1872 const MachineRegisterInfo *MRI) const {
1873 unsigned MaskOpcode = Mask->getOpcode();
1874 unsigned PredOpcode = Pred->getOpcode();
1875 bool PredIsPTestLike = isPTestLikeOpcode(Opc: PredOpcode);
1876 bool PredIsWhileLike = isWhileOpcode(Opc: PredOpcode);
1877
1878 if (PredIsWhileLike) {
1879 // For PTEST(PG, PG), PTEST is redundant when PG is the result of a WHILEcc
1880 // instruction and the condition is "any" since WHILcc does an implicit
1881 // PTEST(ALL, PG) check and PG is always a subset of ALL.
1882 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1883 return PredOpcode;
1884
1885 // For PTEST(PTRUE_ALL, WHILE), if the element size matches, the PTEST is
1886 // redundant since WHILE performs an implicit PTEST with an all active
1887 // mask.
1888 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1889 getElementSizeForOpcode(Opc: MaskOpcode) ==
1890 getElementSizeForOpcode(Opc: PredOpcode))
1891 return PredOpcode;
1892
1893 // For PTEST_FIRST(PTRUE_ALL, WHILE), the PTEST_FIRST is redundant since
1894 // WHILEcc performs an implicit PTEST with an all active mask, setting
1895 // the N flag as the PTEST_FIRST would.
1896 if (PTest->getOpcode() == AArch64::PTEST_PP_FIRST &&
1897 isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31)
1898 return PredOpcode;
1899
1900 return {};
1901 }
1902
1903 if (PredIsPTestLike) {
1904 // For PTEST(PG, PG), PTEST is redundant when PG is the result of an
1905 // instruction that sets the flags as PTEST would and the condition is
1906 // "any" since PG is always a subset of the governing predicate of the
1907 // ptest-like instruction.
1908 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1909 return PredOpcode;
1910
1911 auto PTestLikeMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1912
1913 // If the PTEST like instruction's general predicate is not `Mask`, attempt
1914 // to look through a copy and try again. This is because some instructions
1915 // take a predicate whose register class is a subset of its result class.
1916 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1917 PTestLikeMask->getOperand(i: 1).getReg().isVirtual())
1918 PTestLikeMask =
1919 MRI->getUniqueVRegDef(Reg: PTestLikeMask->getOperand(i: 1).getReg());
1920
1921 // For PTEST(PTRUE_ALL, PTEST_LIKE), the PTEST is redundant if the
1922 // the element size matches and either the PTEST_LIKE instruction uses
1923 // the same all active mask or the condition is "any".
1924 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1925 getElementSizeForOpcode(Opc: MaskOpcode) ==
1926 getElementSizeForOpcode(Opc: PredOpcode)) {
1927 if (Mask == PTestLikeMask || PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1928 return PredOpcode;
1929 }
1930
1931 // For PTEST(PG, PTEST_LIKE(PG, ...)), the PTEST is redundant since the
1932 // flags are set based on the same mask 'PG', but PTEST_LIKE must operate
1933 // on 8-bit predicates like the PTEST. Otherwise, for instructions like
1934 // compare that also support 16/32/64-bit predicates, the implicit PTEST
1935 // performed by the compare could consider fewer lanes for these element
1936 // sizes.
1937 //
1938 // For example, consider
1939 //
1940 // ptrue p0.b ; P0=1111-1111-1111-1111
1941 // index z0.s, #0, #1 ; Z0=<0,1,2,3>
1942 // index z1.s, #1, #1 ; Z1=<1,2,3,4>
1943 // cmphi p1.s, p0/z, z1.s, z0.s ; P1=0001-0001-0001-0001
1944 // ; ^ last active
1945 // ptest p0, p1.b ; P1=0001-0001-0001-0001
1946 // ; ^ last active
1947 //
1948 // where the compare generates a canonical all active 32-bit predicate
1949 // (equivalent to 'ptrue p1.s, all'). The implicit PTEST sets the last
1950 // active flag, whereas the PTEST instruction with the same mask doesn't.
1951 // For PTEST_ANY this doesn't apply as the flags in this case would be
1952 // identical regardless of element size.
1953 uint64_t PredElementSize = getElementSizeForOpcode(Opc: PredOpcode);
1954 if (Mask == PTestLikeMask && (PredElementSize == AArch64::ElementSizeB ||
1955 PTest->getOpcode() == AArch64::PTEST_PP_ANY))
1956 return PredOpcode;
1957
1958 return {};
1959 }
1960
1961 // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the
1962 // opcode so the PTEST becomes redundant.
1963 switch (PredOpcode) {
1964 case AArch64::AND_PPzPP:
1965 case AArch64::BIC_PPzPP:
1966 case AArch64::EOR_PPzPP:
1967 case AArch64::NAND_PPzPP:
1968 case AArch64::NOR_PPzPP:
1969 case AArch64::ORN_PPzPP:
1970 case AArch64::ORR_PPzPP:
1971 case AArch64::BRKA_PPzP:
1972 case AArch64::BRKPA_PPzPP:
1973 case AArch64::BRKB_PPzP:
1974 case AArch64::BRKPB_PPzPP:
1975 case AArch64::RDFFR_PPz: {
1976 // Check to see if our mask is the same. If not the resulting flag bits
1977 // may be different and we can't remove the ptest.
1978 auto *PredMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1979 if (Mask != PredMask)
1980 return {};
1981 break;
1982 }
1983 case AArch64::BRKN_PPzP: {
1984 // BRKN uses an all active implicit mask to set flags unlike the other
1985 // flag-setting instructions.
1986 // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B).
1987 if ((MaskOpcode != AArch64::PTRUE_B) ||
1988 (Mask->getOperand(i: 1).getImm() != 31))
1989 return {};
1990 break;
1991 }
1992 case AArch64::PTRUE_B:
1993 // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A)
1994 break;
1995 default:
1996 // Bail out if we don't recognize the input
1997 return {};
1998 }
1999
2000 return convertToFlagSettingOpc(Opc: PredOpcode);
2001}
2002
2003/// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
2004/// operation which could set the flags in an identical manner
2005bool AArch64InstrInfo::optimizePTestInstr(
2006 MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
2007 const MachineRegisterInfo *MRI) const {
2008 auto *Mask = MRI->getUniqueVRegDef(Reg: MaskReg);
2009 auto *Pred = MRI->getUniqueVRegDef(Reg: PredReg);
2010
2011 if (Pred->isCopy() && PTest->getOpcode() == AArch64::PTEST_PP_FIRST) {
2012 // Instructions which return a multi-vector (e.g. WHILECC_x2) require copies
2013 // before the branch to extract each subregister.
2014 auto Op = Pred->getOperand(i: 1);
2015 if (Op.isReg() && Op.getReg().isVirtual() &&
2016 Op.getSubReg() == AArch64::psub0)
2017 Pred = MRI->getUniqueVRegDef(Reg: Op.getReg());
2018 }
2019
2020 unsigned PredOpcode = Pred->getOpcode();
2021 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred, MRI);
2022 if (!NewOp)
2023 return false;
2024
2025 const TargetRegisterInfo *TRI = &getRegisterInfo();
2026
2027 // If another instruction between Pred and PTest accesses flags, don't remove
2028 // the ptest or update the earlier instruction to modify them.
2029 if (areCFlagsAccessedBetweenInstrs(From: Pred, To: PTest, TRI))
2030 return false;
2031
2032 // If we pass all the checks, it's safe to remove the PTEST and use the flags
2033 // as they are prior to PTEST. Sometimes this requires the tested PTEST
2034 // operand to be replaced with an equivalent instruction that also sets the
2035 // flags.
2036 PTest->eraseFromParent();
2037 if (*NewOp != PredOpcode) {
2038 Pred->setDesc(get(Opcode: *NewOp));
2039 bool succeeded = UpdateOperandRegClass(Instr&: *Pred);
2040 (void)succeeded;
2041 assert(succeeded && "Operands have incompatible register classes!");
2042 Pred->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: TRI);
2043 }
2044
2045 // Ensure that the flags def is live.
2046 if (Pred->registerDefIsDead(Reg: AArch64::NZCV, TRI)) {
2047 unsigned i = 0, e = Pred->getNumOperands();
2048 for (; i != e; ++i) {
2049 MachineOperand &MO = Pred->getOperand(i);
2050 if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::NZCV) {
2051 MO.setIsDead(false);
2052 break;
2053 }
2054 }
2055 }
2056 return true;
2057}
2058
2059/// Try to optimize a compare instruction. A compare instruction is an
2060/// instruction which produces AArch64::NZCV. It can be truly compare
2061/// instruction
2062/// when there are no uses of its destination register.
2063///
2064/// The following steps are tried in order:
2065/// 1. Convert CmpInstr into an unconditional version.
2066/// 2. Remove CmpInstr if above there is an instruction producing a needed
2067/// condition code or an instruction which can be converted into such an
2068/// instruction.
2069/// Only comparison with zero is supported.
2070bool AArch64InstrInfo::optimizeCompareInstr(
2071 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2072 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2073 assert(CmpInstr.getParent());
2074 assert(MRI);
2075
2076 // Replace SUBSWrr with SUBWrr if NZCV is not used.
2077 int DeadNZCVIdx =
2078 CmpInstr.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
2079 if (DeadNZCVIdx != -1) {
2080 if (CmpInstr.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
2081 CmpInstr.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr)) {
2082 CmpInstr.eraseFromParent();
2083 return true;
2084 }
2085 unsigned Opc = CmpInstr.getOpcode();
2086 unsigned NewOpc = convertToNonFlagSettingOpc(MI: CmpInstr);
2087 if (NewOpc == Opc)
2088 return false;
2089 const MCInstrDesc &MCID = get(Opcode: NewOpc);
2090 CmpInstr.setDesc(MCID);
2091 CmpInstr.removeOperand(OpNo: DeadNZCVIdx);
2092 bool succeeded = UpdateOperandRegClass(Instr&: CmpInstr);
2093 (void)succeeded;
2094 assert(succeeded && "Some operands reg class are incompatible!");
2095 return true;
2096 }
2097
2098 if (CmpInstr.getOpcode() == AArch64::PTEST_PP ||
2099 CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY ||
2100 CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST)
2101 return optimizePTestInstr(PTest: &CmpInstr, MaskReg: SrcReg, PredReg: SrcReg2, MRI);
2102
2103 if (SrcReg2 != 0)
2104 return false;
2105
2106 // CmpInstr is a Compare instruction if destination register is not used.
2107 if (!MRI->use_nodbg_empty(RegNo: CmpInstr.getOperand(i: 0).getReg()))
2108 return false;
2109
2110 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, MRI: *MRI))
2111 return true;
2112 return (CmpValue == 0 || CmpValue == 1) &&
2113 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, MRI: *MRI);
2114}
2115
2116/// Get opcode of S version of Instr.
2117/// If Instr is S version its opcode is returned.
2118/// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
2119/// or we are not interested in it.
2120static unsigned sForm(MachineInstr &Instr) {
2121 switch (Instr.getOpcode()) {
2122 default:
2123 return AArch64::INSTRUCTION_LIST_END;
2124
2125 case AArch64::ADDSWrr:
2126 case AArch64::ADDSWri:
2127 case AArch64::ADDSXrr:
2128 case AArch64::ADDSXri:
2129 case AArch64::ADDSWrx:
2130 case AArch64::ADDSXrx:
2131 case AArch64::ADDSWrs:
2132 case AArch64::ADDSXrs:
2133 case AArch64::SUBSWrr:
2134 case AArch64::SUBSWri:
2135 case AArch64::SUBSWrx:
2136 case AArch64::SUBSWrs:
2137 case AArch64::SUBSXrr:
2138 case AArch64::SUBSXri:
2139 case AArch64::SUBSXrx:
2140 case AArch64::SUBSXrs:
2141 case AArch64::ANDSWri:
2142 case AArch64::ANDSWrr:
2143 case AArch64::ANDSWrs:
2144 case AArch64::ANDSXri:
2145 case AArch64::ANDSXrr:
2146 case AArch64::ANDSXrs:
2147 case AArch64::BICSWrr:
2148 case AArch64::BICSXrr:
2149 case AArch64::BICSWrs:
2150 case AArch64::BICSXrs:
2151 case AArch64::ADCSWr:
2152 case AArch64::ADCSXr:
2153 case AArch64::SBCSWr:
2154 case AArch64::SBCSXr:
2155 return Instr.getOpcode();
2156
2157 case AArch64::ADDWrr:
2158 return AArch64::ADDSWrr;
2159 case AArch64::ADDWri:
2160 return AArch64::ADDSWri;
2161 case AArch64::ADDXrr:
2162 return AArch64::ADDSXrr;
2163 case AArch64::ADDXri:
2164 return AArch64::ADDSXri;
2165 case AArch64::ADDWrx:
2166 return AArch64::ADDSWrx;
2167 case AArch64::ADDXrx:
2168 return AArch64::ADDSXrx;
2169 case AArch64::ADDWrs:
2170 return AArch64::ADDSWrs;
2171 case AArch64::ADDXrs:
2172 return AArch64::ADDSXrs;
2173 case AArch64::ADCWr:
2174 return AArch64::ADCSWr;
2175 case AArch64::ADCXr:
2176 return AArch64::ADCSXr;
2177 case AArch64::SUBWrr:
2178 return AArch64::SUBSWrr;
2179 case AArch64::SUBWri:
2180 return AArch64::SUBSWri;
2181 case AArch64::SUBXrr:
2182 return AArch64::SUBSXrr;
2183 case AArch64::SUBXri:
2184 return AArch64::SUBSXri;
2185 case AArch64::SUBWrx:
2186 return AArch64::SUBSWrx;
2187 case AArch64::SUBXrx:
2188 return AArch64::SUBSXrx;
2189 case AArch64::SUBWrs:
2190 return AArch64::SUBSWrs;
2191 case AArch64::SUBXrs:
2192 return AArch64::SUBSXrs;
2193 case AArch64::SBCWr:
2194 return AArch64::SBCSWr;
2195 case AArch64::SBCXr:
2196 return AArch64::SBCSXr;
2197 case AArch64::ANDWri:
2198 return AArch64::ANDSWri;
2199 case AArch64::ANDXri:
2200 return AArch64::ANDSXri;
2201 case AArch64::ANDWrr:
2202 return AArch64::ANDSWrr;
2203 case AArch64::ANDWrs:
2204 return AArch64::ANDSWrs;
2205 case AArch64::ANDXrr:
2206 return AArch64::ANDSXrr;
2207 case AArch64::ANDXrs:
2208 return AArch64::ANDSXrs;
2209 case AArch64::BICWrr:
2210 return AArch64::BICSWrr;
2211 case AArch64::BICXrr:
2212 return AArch64::BICSXrr;
2213 case AArch64::BICWrs:
2214 return AArch64::BICSWrs;
2215 case AArch64::BICXrs:
2216 return AArch64::BICSXrs;
2217 }
2218}
2219
2220/// Check if AArch64::NZCV should be alive in successors of MBB.
2221static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB) {
2222 for (auto *BB : MBB->successors())
2223 if (BB->isLiveIn(Reg: AArch64::NZCV))
2224 return true;
2225 return false;
2226}
2227
2228/// \returns The condition code operand index for \p Instr if it is a branch
2229/// or select and -1 otherwise.
2230int AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(
2231 const MachineInstr &Instr) {
2232 switch (Instr.getOpcode()) {
2233 default:
2234 return -1;
2235
2236 case AArch64::Bcc: {
2237 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2238 assert(Idx >= 2);
2239 return Idx - 2;
2240 }
2241
2242 case AArch64::CSINVWr:
2243 case AArch64::CSINVXr:
2244 case AArch64::CSINCWr:
2245 case AArch64::CSINCXr:
2246 case AArch64::CSELWr:
2247 case AArch64::CSELXr:
2248 case AArch64::CSNEGWr:
2249 case AArch64::CSNEGXr:
2250 case AArch64::FCSELSrrr:
2251 case AArch64::FCSELDrrr: {
2252 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2253 assert(Idx >= 1);
2254 return Idx - 1;
2255 }
2256 }
2257}
2258
2259/// Find a condition code used by the instruction.
2260/// Returns AArch64CC::Invalid if either the instruction does not use condition
2261/// codes or we don't optimize CmpInstr in the presence of such instructions.
2262static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
2263 int CCIdx =
2264 AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(Instr);
2265 return CCIdx >= 0 ? static_cast<AArch64CC::CondCode>(
2266 Instr.getOperand(i: CCIdx).getImm())
2267 : AArch64CC::Invalid;
2268}
2269
2270static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
2271 assert(CC != AArch64CC::Invalid);
2272 UsedNZCV UsedFlags;
2273 switch (CC) {
2274 default:
2275 break;
2276
2277 case AArch64CC::EQ: // Z set
2278 case AArch64CC::NE: // Z clear
2279 UsedFlags.Z = true;
2280 break;
2281
2282 case AArch64CC::HI: // Z clear and C set
2283 case AArch64CC::LS: // Z set or C clear
2284 UsedFlags.Z = true;
2285 [[fallthrough]];
2286 case AArch64CC::HS: // C set
2287 case AArch64CC::LO: // C clear
2288 UsedFlags.C = true;
2289 break;
2290
2291 case AArch64CC::MI: // N set
2292 case AArch64CC::PL: // N clear
2293 UsedFlags.N = true;
2294 break;
2295
2296 case AArch64CC::VS: // V set
2297 case AArch64CC::VC: // V clear
2298 UsedFlags.V = true;
2299 break;
2300
2301 case AArch64CC::GT: // Z clear, N and V the same
2302 case AArch64CC::LE: // Z set, N and V differ
2303 UsedFlags.Z = true;
2304 [[fallthrough]];
2305 case AArch64CC::GE: // N and V the same
2306 case AArch64CC::LT: // N and V differ
2307 UsedFlags.N = true;
2308 UsedFlags.V = true;
2309 break;
2310 }
2311 return UsedFlags;
2312}
2313
2314/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
2315/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
2316/// \returns std::nullopt otherwise.
2317///
2318/// Collect instructions using that flags in \p CCUseInstrs if provided.
2319std::optional<UsedNZCV>
2320llvm::examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
2321 const TargetRegisterInfo &TRI,
2322 SmallVectorImpl<MachineInstr *> *CCUseInstrs) {
2323 MachineBasicBlock *CmpParent = CmpInstr.getParent();
2324 if (MI.getParent() != CmpParent)
2325 return std::nullopt;
2326
2327 if (areCFlagsAliveInSuccessors(MBB: CmpParent))
2328 return std::nullopt;
2329
2330 UsedNZCV NZCVUsedAfterCmp;
2331 for (MachineInstr &Instr : instructionsWithoutDebug(
2332 It: std::next(x: CmpInstr.getIterator()), End: CmpParent->instr_end())) {
2333 if (Instr.readsRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
2334 AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
2335 if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
2336 return std::nullopt;
2337 NZCVUsedAfterCmp |= getUsedNZCV(CC);
2338 if (CCUseInstrs)
2339 CCUseInstrs->push_back(Elt: &Instr);
2340 }
2341 if (Instr.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI))
2342 break;
2343 }
2344 return NZCVUsedAfterCmp;
2345}
2346
2347static bool isADDSRegImm(unsigned Opcode) {
2348 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
2349}
2350
2351static bool isSUBSRegImm(unsigned Opcode) {
2352 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
2353}
2354
2355static bool isANDOpcode(MachineInstr &MI) {
2356 unsigned Opc = sForm(Instr&: MI);
2357 switch (Opc) {
2358 case AArch64::ANDSWri:
2359 case AArch64::ANDSWrr:
2360 case AArch64::ANDSWrs:
2361 case AArch64::ANDSXri:
2362 case AArch64::ANDSXrr:
2363 case AArch64::ANDSXrs:
2364 case AArch64::BICSWrr:
2365 case AArch64::BICSXrr:
2366 case AArch64::BICSWrs:
2367 case AArch64::BICSXrs:
2368 return true;
2369 default:
2370 return false;
2371 }
2372}
2373
2374/// Check if CmpInstr can be substituted by MI.
2375///
2376/// CmpInstr can be substituted:
2377/// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2378/// - and, MI and CmpInstr are from the same MachineBB
2379/// - and, condition flags are not alive in successors of the CmpInstr parent
2380/// - and, if MI opcode is the S form there must be no defs of flags between
2381/// MI and CmpInstr
2382/// or if MI opcode is not the S form there must be neither defs of flags
2383/// nor uses of flags between MI and CmpInstr.
2384/// - and, C is not used after CmpInstr; CmpInstr's C is from adds/subs #0 on
2385/// SrcReg and can differ from MI (e.g. carry out of ADCS/SBCS).
2386/// - and, V is not used after CmpInstr unless MI is AND/BIC (V cleared) or MI
2387/// has NoSWrap (overflow is poison and the fold is still safe).
2388static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr,
2389 const TargetRegisterInfo &TRI) {
2390 // MI is an opcode sForm maps (add/sub/adc/sbc/and/bic and their S forms).
2391 assert(sForm(MI) != AArch64::INSTRUCTION_LIST_END);
2392
2393 const unsigned CmpOpcode = CmpInstr.getOpcode();
2394 if (!isADDSRegImm(Opcode: CmpOpcode) && !isSUBSRegImm(Opcode: CmpOpcode))
2395 return false;
2396
2397 assert((CmpInstr.getOperand(2).isImm() &&
2398 CmpInstr.getOperand(2).getImm() == 0) &&
2399 "Caller guarantees that CmpInstr compares with constant 0");
2400
2401 std::optional<UsedNZCV> NZVCUsed = examineCFlagsUse(MI, CmpInstr, TRI);
2402 if (!NZVCUsed || NZVCUsed->C)
2403 return false;
2404
2405 // CmpInstr is ADDS/SUBS with immediate 0 on SrcReg (compare SrcReg to zero).
2406 // After the fold, users see NZCV from MI (or its S form), not from CmpInstr.
2407 // N/Z match CmpInstr for the value in SrcReg; C/V need not match in general
2408 // (e.g. ADCS vs adds #0), so we require C unused after CmpInstr and gate V
2409 // as below. NoSWrap makes signed overflow poison; AND/BIC clear V.
2410 if (NZVCUsed->V && !MI.getFlag(Flag: MachineInstr::NoSWrap) && !isANDOpcode(MI))
2411 return false;
2412
2413 AccessKind AccessToCheck = AK_Write;
2414 if (sForm(Instr&: MI) != MI.getOpcode())
2415 AccessToCheck = AK_All;
2416 return !areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck);
2417}
2418
2419/// Substitute an instruction comparing to zero with another instruction
2420/// which produces needed condition flags.
2421///
2422/// Return true on success.
2423bool AArch64InstrInfo::substituteCmpToZero(
2424 MachineInstr &CmpInstr, unsigned SrcReg,
2425 const MachineRegisterInfo &MRI) const {
2426 // Get the unique definition of SrcReg.
2427 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2428 if (!MI)
2429 return false;
2430
2431 const TargetRegisterInfo &TRI = getRegisterInfo();
2432
2433 unsigned NewOpc = sForm(Instr&: *MI);
2434 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
2435 return false;
2436
2437 if (!canInstrSubstituteCmpInstr(MI&: *MI, CmpInstr, TRI))
2438 return false;
2439
2440 // Update the instruction to set NZCV.
2441 MI->setDesc(get(Opcode: NewOpc));
2442 CmpInstr.eraseFromParent();
2443 bool succeeded = UpdateOperandRegClass(Instr&: *MI);
2444 (void)succeeded;
2445 assert(succeeded && "Some operands reg class are incompatible!");
2446 MI->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: &TRI);
2447 return true;
2448}
2449
2450/// \returns True if \p CmpInstr can be removed.
2451///
2452/// \p IsInvertCC is true if, after removing \p CmpInstr, condition
2453/// codes used in \p CCUseInstrs must be inverted.
2454static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr,
2455 int CmpValue, const TargetRegisterInfo &TRI,
2456 SmallVectorImpl<MachineInstr *> &CCUseInstrs,
2457 bool &IsInvertCC) {
2458 assert((CmpValue == 0 || CmpValue == 1) &&
2459 "Only comparisons to 0 or 1 considered for removal!");
2460
2461 // MI is 'CSINCWr %vreg, wzr, wzr, <cc>' or 'CSINCXr %vreg, xzr, xzr, <cc>'
2462 unsigned MIOpc = MI.getOpcode();
2463 if (MIOpc == AArch64::CSINCWr) {
2464 if (MI.getOperand(i: 1).getReg() != AArch64::WZR ||
2465 MI.getOperand(i: 2).getReg() != AArch64::WZR)
2466 return false;
2467 } else if (MIOpc == AArch64::CSINCXr) {
2468 if (MI.getOperand(i: 1).getReg() != AArch64::XZR ||
2469 MI.getOperand(i: 2).getReg() != AArch64::XZR)
2470 return false;
2471 } else {
2472 return false;
2473 }
2474 AArch64CC::CondCode MICC = findCondCodeUsedByInstr(Instr: MI);
2475 if (MICC == AArch64CC::Invalid)
2476 return false;
2477
2478 // NZCV needs to be defined
2479 if (MI.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) != -1)
2480 return false;
2481
2482 // CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0' or 'SUBS %vreg, 1'
2483 const unsigned CmpOpcode = CmpInstr.getOpcode();
2484 bool IsSubsRegImm = isSUBSRegImm(Opcode: CmpOpcode);
2485 if (CmpValue && !IsSubsRegImm)
2486 return false;
2487 if (!CmpValue && !IsSubsRegImm && !isADDSRegImm(Opcode: CmpOpcode))
2488 return false;
2489
2490 // MI conditions allowed: eq, ne, mi, pl
2491 UsedNZCV MIUsedNZCV = getUsedNZCV(CC: MICC);
2492 if (MIUsedNZCV.C || MIUsedNZCV.V)
2493 return false;
2494
2495 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2496 examineCFlagsUse(MI, CmpInstr, TRI, CCUseInstrs: &CCUseInstrs);
2497 // Condition flags are not used in CmpInstr basic block successors and only
2498 // Z or N flags allowed to be used after CmpInstr within its basic block
2499 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2500 return false;
2501 // Z or N flag used after CmpInstr must correspond to the flag used in MI
2502 if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) ||
2503 (MIUsedNZCV.N && NZCVUsedAfterCmp->Z))
2504 return false;
2505 // If CmpInstr is comparison to zero MI conditions are limited to eq, ne
2506 if (MIUsedNZCV.N && !CmpValue)
2507 return false;
2508
2509 // There must be no defs of flags between MI and CmpInstr
2510 if (areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck: AK_Write))
2511 return false;
2512
2513 // Condition code is inverted in the following cases:
2514 // 1. MI condition is ne; CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2515 // 2. MI condition is eq, pl; CmpInstr is 'SUBS %vreg, 1'
2516 IsInvertCC = (CmpValue && (MICC == AArch64CC::EQ || MICC == AArch64CC::PL)) ||
2517 (!CmpValue && MICC == AArch64CC::NE);
2518 return true;
2519}
2520
2521/// Remove comparison in csinc-cmp sequence
2522///
2523/// Examples:
2524/// 1. \code
2525/// csinc w9, wzr, wzr, ne
2526/// cmp w9, #0
2527/// b.eq
2528/// \endcode
2529/// to
2530/// \code
2531/// csinc w9, wzr, wzr, ne
2532/// b.ne
2533/// \endcode
2534///
2535/// 2. \code
2536/// csinc x2, xzr, xzr, mi
2537/// cmp x2, #1
2538/// b.pl
2539/// \endcode
2540/// to
2541/// \code
2542/// csinc x2, xzr, xzr, mi
2543/// b.pl
2544/// \endcode
2545///
2546/// \param CmpInstr comparison instruction
2547/// \return True when comparison removed
2548bool AArch64InstrInfo::removeCmpToZeroOrOne(
2549 MachineInstr &CmpInstr, unsigned SrcReg, int CmpValue,
2550 const MachineRegisterInfo &MRI) const {
2551 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2552 if (!MI)
2553 return false;
2554 const TargetRegisterInfo &TRI = getRegisterInfo();
2555 SmallVector<MachineInstr *, 4> CCUseInstrs;
2556 bool IsInvertCC = false;
2557 if (!canCmpInstrBeRemoved(MI&: *MI, CmpInstr, CmpValue, TRI, CCUseInstrs,
2558 IsInvertCC))
2559 return false;
2560 // Make transformation
2561 CmpInstr.eraseFromParent();
2562 if (IsInvertCC) {
2563 // Invert condition codes in CmpInstr CC users
2564 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2565 int Idx = findCondCodeUseOperandIdxForBranchOrSelect(Instr: *CCUseInstr);
2566 assert(Idx >= 0 && "Unexpected instruction using CC.");
2567 MachineOperand &CCOperand = CCUseInstr->getOperand(i: Idx);
2568 AArch64CC::CondCode CCUse = AArch64CC::getInvertedCondCode(
2569 Code: static_cast<AArch64CC::CondCode>(CCOperand.getImm()));
2570 CCOperand.setImm(CCUse);
2571 }
2572 }
2573 return true;
2574}
2575
2576bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2577 if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2578 MI.getOpcode() != AArch64::CATCHRET &&
2579 MI.getOpcode() != AArch64::STACK_GUARD_UNMIX)
2580 return false;
2581
2582 MachineBasicBlock &MBB = *MI.getParent();
2583 auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
2584 auto TRI = Subtarget.getRegisterInfo();
2585 DebugLoc DL = MI.getDebugLoc();
2586
2587 if (MI.getOpcode() == AArch64::STACK_GUARD_UNMIX) {
2588 // Expand STACK_GUARD_UNMIX to: sub Rd, fp, Rs
2589 // This computes FP - stored_mixed_value to unmix the cookie
2590 Register DstReg = MI.getOperand(i: 0).getReg();
2591 Register SrcReg = MI.getOperand(i: 1).getReg();
2592
2593 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: DstReg)
2594 .addReg(RegNo: AArch64::FP)
2595 .addReg(RegNo: SrcReg);
2596
2597 MBB.erase(I: MI);
2598 return true;
2599 }
2600
2601 if (MI.getOpcode() == AArch64::CATCHRET) {
2602 // Skip to the first instruction before the epilog.
2603 const TargetInstrInfo *TII =
2604 MBB.getParent()->getSubtarget().getInstrInfo();
2605 MachineBasicBlock *TargetMBB = MI.getOperand(i: 0).getMBB();
2606 auto MBBI = MachineBasicBlock::iterator(MI);
2607 MachineBasicBlock::iterator FirstEpilogSEH = std::prev(x: MBBI);
2608 while (FirstEpilogSEH->getFlag(Flag: MachineInstr::FrameDestroy) &&
2609 FirstEpilogSEH != MBB.begin())
2610 FirstEpilogSEH = std::prev(x: FirstEpilogSEH);
2611 if (FirstEpilogSEH != MBB.begin())
2612 FirstEpilogSEH = std::next(x: FirstEpilogSEH);
2613 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADRP))
2614 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2615 .addMBB(MBB: TargetMBB, TargetFlags: AArch64II::MO_PAGE);
2616 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri))
2617 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2618 .addReg(RegNo: AArch64::X0)
2619 .addMBB(MBB: TargetMBB, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
2620 .addImm(Val: 0);
2621 TargetMBB->setMachineBlockAddressTaken();
2622 return true;
2623 }
2624
2625 Register Reg = MI.getOperand(i: 0).getReg();
2626 Module &M = *MBB.getParent()->getFunction().getParent();
2627 if (M.getStackProtectorGuard() == "sysreg") {
2628 const AArch64SysReg::SysReg *SrcReg =
2629 AArch64SysReg::lookupSysRegByName(Name: M.getStackProtectorGuardReg());
2630 if (!SrcReg)
2631 report_fatal_error(reason: "Unknown SysReg for Stack Protector Guard Register");
2632
2633 // mrs xN, sysreg
2634 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MRS))
2635 .addDef(RegNo: Reg, Flags: RegState::Renamable)
2636 .addImm(Val: SrcReg->Encoding);
2637 int Offset = M.getStackProtectorGuardOffset();
2638 if (Offset >= 0 && Offset <= 32760 && Offset % 8 == 0) {
2639 // ldr xN, [xN, #offset]
2640 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2641 .addDef(RegNo: Reg)
2642 .addUse(RegNo: Reg, Flags: RegState::Kill)
2643 .addImm(Val: Offset / 8);
2644 } else if (Offset >= -256 && Offset <= 255) {
2645 // ldur xN, [xN, #offset]
2646 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDURXi))
2647 .addDef(RegNo: Reg)
2648 .addUse(RegNo: Reg, Flags: RegState::Kill)
2649 .addImm(Val: Offset);
2650 } else if (Offset >= -4095 && Offset <= 4095) {
2651 if (Offset > 0) {
2652 // add xN, xN, #offset
2653 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri))
2654 .addDef(RegNo: Reg)
2655 .addUse(RegNo: Reg, Flags: RegState::Kill)
2656 .addImm(Val: Offset)
2657 .addImm(Val: 0);
2658 } else {
2659 // sub xN, xN, #offset
2660 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXri))
2661 .addDef(RegNo: Reg)
2662 .addUse(RegNo: Reg, Flags: RegState::Kill)
2663 .addImm(Val: -Offset)
2664 .addImm(Val: 0);
2665 }
2666 // ldr xN, [xN]
2667 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2668 .addDef(RegNo: Reg)
2669 .addUse(RegNo: Reg, Flags: RegState::Kill)
2670 .addImm(Val: 0);
2671 } else {
2672 // Cases that are larger than +/- 4095 and not a multiple of 8, or larger
2673 // than 23760.
2674 // It might be nice to use AArch64::MOVi32imm here, which would get
2675 // expanded in PreSched2 after PostRA, but our lone scratch Reg already
2676 // contains the MRS result. findScratchNonCalleeSaveRegister() in
2677 // AArch64FrameLowering might help us find such a scratch register
2678 // though. If we failed to find a scratch register, we could emit a
2679 // stream of add instructions to build up the immediate. Or, we could try
2680 // to insert a AArch64::MOVi32imm before register allocation so that we
2681 // didn't need to scavenge for a scratch register.
2682 report_fatal_error(reason: "Unable to encode Stack Protector Guard Offset");
2683 }
2684 MBB.erase(I: MI);
2685 return true;
2686 }
2687
2688 const GlobalValue *GV =
2689 cast<GlobalValue>(Val: (*MI.memoperands_begin())->getValue());
2690 const TargetMachine &TM = MBB.getParent()->getTarget();
2691 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2692 const unsigned char MO_NC = AArch64II::MO_NC;
2693
2694 unsigned GuardWidth = M.getStackProtectorGuardValueWidth().value_or(
2695 u: Subtarget.isTargetILP32() ? 4 : 8);
2696 if (GuardWidth != 4 && GuardWidth != 8)
2697 report_fatal_error(reason: "Unsupported stack protector value width");
2698 if ((OpFlags & AArch64II::MO_GOT) != 0) {
2699 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LOADgot), DestReg: Reg)
2700 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags);
2701 if (GuardWidth == 4) {
2702 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2703 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2704 .addDef(RegNo: Reg32)
2705 .addUse(RegNo: Reg, Flags: RegState::Kill)
2706 .addImm(Val: 0)
2707 .addMemOperand(MMO: *MI.memoperands_begin())
2708 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2709 } else {
2710 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2711 .addReg(RegNo: Reg, Flags: RegState::Kill)
2712 .addImm(Val: 0)
2713 .addMemOperand(MMO: *MI.memoperands_begin());
2714 }
2715 } else if (TM.getCodeModel() == CodeModel::Large) {
2716 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg)
2717 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G0 | MO_NC)
2718 .addImm(Val: 0);
2719 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2720 .addReg(RegNo: Reg, Flags: RegState::Kill)
2721 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G1 | MO_NC)
2722 .addImm(Val: 16);
2723 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2724 .addReg(RegNo: Reg, Flags: RegState::Kill)
2725 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G2 | MO_NC)
2726 .addImm(Val: 32);
2727 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2728 .addReg(RegNo: Reg, Flags: RegState::Kill)
2729 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G3)
2730 .addImm(Val: 48);
2731 if (GuardWidth == 4) {
2732 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2733 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2734 .addDef(RegNo: Reg32)
2735 .addUse(RegNo: Reg, Flags: RegState::Kill)
2736 .addImm(Val: 0)
2737 .addMemOperand(MMO: *MI.memoperands_begin())
2738 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2739 } else {
2740 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2741 .addReg(RegNo: Reg, Flags: RegState::Kill)
2742 .addImm(Val: 0)
2743 .addMemOperand(MMO: *MI.memoperands_begin());
2744 }
2745 } else {
2746 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
2747 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags | AArch64II::MO_PAGE);
2748 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
2749 if (GuardWidth == 4) {
2750 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2751 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2752 .addDef(RegNo: Reg32)
2753 .addUse(RegNo: Reg, Flags: RegState::Kill)
2754 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2755 .addMemOperand(MMO: *MI.memoperands_begin())
2756 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2757 } else {
2758 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2759 .addReg(RegNo: Reg, Flags: RegState::Kill)
2760 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2761 .addMemOperand(MMO: *MI.memoperands_begin());
2762 }
2763 }
2764 // To match MSVC. Unlike x86_64 which uses xor instruction to mix the cookie,
2765 // we use sub instruction to mix the cookie on aarch64.
2766 // The mixing happens here in expandPostRAPseudo (after RA) to ensure we use
2767 // the final frame pointer value.
2768 if (Subtarget.getTargetTriple().isOSMSVCRT())
2769 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: Reg)
2770 .addReg(RegNo: AArch64::FP)
2771 .addReg(RegNo: Reg, Flags: RegState::Kill);
2772
2773 MBB.erase(I: MI);
2774
2775 return true;
2776}
2777
2778// Return true if this instruction simply sets its single destination register
2779// to zero. This is equivalent to a register rename of the zero-register.
2780bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
2781 switch (MI.getOpcode()) {
2782 default:
2783 break;
2784 case AArch64::MOVZWi:
2785 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
2786 if (MI.getOperand(i: 1).isImm() && MI.getOperand(i: 1).getImm() == 0) {
2787 assert(MI.getDesc().getNumOperands() == 3 &&
2788 MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
2789 return true;
2790 }
2791 break;
2792 case AArch64::ANDWri: // and Rd, Rzr, #imm
2793 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2794 case AArch64::ANDXri:
2795 return MI.getOperand(i: 1).getReg() == AArch64::XZR;
2796 case TargetOpcode::COPY:
2797 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2798 }
2799 return false;
2800}
2801
2802// Return true if this instruction simply renames a general register without
2803// modifying bits.
2804bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
2805 switch (MI.getOpcode()) {
2806 default:
2807 break;
2808 case TargetOpcode::COPY: {
2809 // GPR32 copies will by lowered to ORRXrs
2810 Register DstReg = MI.getOperand(i: 0).getReg();
2811 return (AArch64::GPR32RegClass.contains(Reg: DstReg) ||
2812 AArch64::GPR64RegClass.contains(Reg: DstReg));
2813 }
2814 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
2815 if (MI.getOperand(i: 1).getReg() == AArch64::XZR) {
2816 assert(MI.getDesc().getNumOperands() == 4 &&
2817 MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
2818 return true;
2819 }
2820 break;
2821 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
2822 if (MI.getOperand(i: 2).getImm() == 0) {
2823 assert(MI.getDesc().getNumOperands() == 4 &&
2824 MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
2825 return true;
2826 }
2827 break;
2828 }
2829 return false;
2830}
2831
2832// Return true if this instruction simply renames a general register without
2833// modifying bits.
2834bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
2835 switch (MI.getOpcode()) {
2836 default:
2837 break;
2838 case TargetOpcode::COPY: {
2839 Register DstReg = MI.getOperand(i: 0).getReg();
2840 return AArch64::FPR128RegClass.contains(Reg: DstReg);
2841 }
2842 case AArch64::ORRv16i8:
2843 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
2844 assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
2845 "invalid ORRv16i8 operands");
2846 return true;
2847 }
2848 break;
2849 }
2850 return false;
2851}
2852
2853static bool isFrameLoadOpcode(int Opcode) {
2854 switch (Opcode) {
2855 default:
2856 return false;
2857 case AArch64::LDRWui:
2858 case AArch64::LDRXui:
2859 case AArch64::LDRBui:
2860 case AArch64::LDRHui:
2861 case AArch64::LDRSui:
2862 case AArch64::LDRDui:
2863 case AArch64::LDRQui:
2864 case AArch64::LDR_PXI:
2865 return true;
2866 }
2867}
2868
2869Register AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
2870 int &FrameIndex) const {
2871 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2872 return Register();
2873
2874 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2875 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2876 FrameIndex = MI.getOperand(i: 1).getIndex();
2877 return MI.getOperand(i: 0).getReg();
2878 }
2879 return Register();
2880}
2881
2882static bool isFrameStoreOpcode(int Opcode) {
2883 switch (Opcode) {
2884 default:
2885 return false;
2886 case AArch64::STRWui:
2887 case AArch64::STRXui:
2888 case AArch64::STRBui:
2889 case AArch64::STRHui:
2890 case AArch64::STRSui:
2891 case AArch64::STRDui:
2892 case AArch64::STRQui:
2893 case AArch64::STR_PXI:
2894 return true;
2895 }
2896}
2897
2898Register AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
2899 int &FrameIndex) const {
2900 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2901 return Register();
2902
2903 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2904 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2905 FrameIndex = MI.getOperand(i: 1).getIndex();
2906 return MI.getOperand(i: 0).getReg();
2907 }
2908 return Register();
2909}
2910
2911Register AArch64InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
2912 int &FrameIndex) const {
2913 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2914 return Register();
2915
2916 if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
2917 return Reg;
2918
2919 SmallVector<const MachineMemOperand *, 1> Accesses;
2920 if (hasStoreToStackSlot(MI, Accesses)) {
2921 if (Accesses.size() > 1)
2922 return Register();
2923
2924 FrameIndex =
2925 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2926 ->getFrameIndex();
2927 return MI.getOperand(i: 0).getReg();
2928 }
2929 return Register();
2930}
2931
2932Register AArch64InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
2933 int &FrameIndex) const {
2934 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2935 return Register();
2936
2937 if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
2938 return Reg;
2939
2940 SmallVector<const MachineMemOperand *, 1> Accesses;
2941 if (hasLoadFromStackSlot(MI, Accesses)) {
2942 if (Accesses.size() > 1)
2943 return Register();
2944
2945 FrameIndex =
2946 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2947 ->getFrameIndex();
2948 return MI.getOperand(i: 0).getReg();
2949 }
2950 return Register();
2951}
2952
2953/// Check all MachineMemOperands for a hint to suppress pairing.
2954bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
2955 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2956 return MMO->getFlags() & MOSuppressPair;
2957 });
2958}
2959
2960/// Set a flag on the first MachineMemOperand to suppress pairing.
2961void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
2962 if (MI.memoperands_empty())
2963 return;
2964 (*MI.memoperands_begin())->setFlags(MOSuppressPair);
2965}
2966
2967/// Check all MachineMemOperands for a hint that the load/store is strided.
2968bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
2969 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2970 return MMO->getFlags() & MOStridedAccess;
2971 });
2972}
2973
2974bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
2975 switch (Opc) {
2976 default:
2977 return false;
2978 case AArch64::STURSi:
2979 case AArch64::STRSpre:
2980 case AArch64::STURDi:
2981 case AArch64::STRDpre:
2982 case AArch64::STURQi:
2983 case AArch64::STRQpre:
2984 case AArch64::STURBBi:
2985 case AArch64::STURHHi:
2986 case AArch64::STURWi:
2987 case AArch64::STRWpre:
2988 case AArch64::STURXi:
2989 case AArch64::STRXpre:
2990 case AArch64::LDURSi:
2991 case AArch64::LDRSpre:
2992 case AArch64::LDURDi:
2993 case AArch64::LDRDpre:
2994 case AArch64::LDURQi:
2995 case AArch64::LDRQpre:
2996 case AArch64::LDURWi:
2997 case AArch64::LDRWpre:
2998 case AArch64::LDURXi:
2999 case AArch64::LDRXpre:
3000 case AArch64::LDRSWpre:
3001 case AArch64::LDURSWi:
3002 case AArch64::LDURHHi:
3003 case AArch64::LDURBBi:
3004 case AArch64::LDURSBWi:
3005 case AArch64::LDURSHWi:
3006 return true;
3007 }
3008}
3009
3010std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
3011 switch (Opc) {
3012 default: return {};
3013 case AArch64::PRFMui: return AArch64::PRFUMi;
3014 case AArch64::LDRXui: return AArch64::LDURXi;
3015 case AArch64::LDRWui: return AArch64::LDURWi;
3016 case AArch64::LDRBui: return AArch64::LDURBi;
3017 case AArch64::LDRHui: return AArch64::LDURHi;
3018 case AArch64::LDRSui: return AArch64::LDURSi;
3019 case AArch64::LDRDui: return AArch64::LDURDi;
3020 case AArch64::LDRQui: return AArch64::LDURQi;
3021 case AArch64::LDRBBui: return AArch64::LDURBBi;
3022 case AArch64::LDRHHui: return AArch64::LDURHHi;
3023 case AArch64::LDRSBXui: return AArch64::LDURSBXi;
3024 case AArch64::LDRSBWui: return AArch64::LDURSBWi;
3025 case AArch64::LDRSHXui: return AArch64::LDURSHXi;
3026 case AArch64::LDRSHWui: return AArch64::LDURSHWi;
3027 case AArch64::LDRSWui: return AArch64::LDURSWi;
3028 case AArch64::STRXui: return AArch64::STURXi;
3029 case AArch64::STRWui: return AArch64::STURWi;
3030 case AArch64::STRBui: return AArch64::STURBi;
3031 case AArch64::STRHui: return AArch64::STURHi;
3032 case AArch64::STRSui: return AArch64::STURSi;
3033 case AArch64::STRDui: return AArch64::STURDi;
3034 case AArch64::STRQui: return AArch64::STURQi;
3035 case AArch64::STRBBui: return AArch64::STURBBi;
3036 case AArch64::STRHHui: return AArch64::STURHHi;
3037 }
3038}
3039
3040unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
3041 switch (Opc) {
3042 default:
3043 llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
3044 case AArch64::ADDG:
3045 case AArch64::LDAPURBi:
3046 case AArch64::LDAPURHi:
3047 case AArch64::LDAPURi:
3048 case AArch64::LDAPURSBWi:
3049 case AArch64::LDAPURSBXi:
3050 case AArch64::LDAPURSHWi:
3051 case AArch64::LDAPURSHXi:
3052 case AArch64::LDAPURSWi:
3053 case AArch64::LDAPURXi:
3054 case AArch64::LDR_PPXI:
3055 case AArch64::LDR_PXI:
3056 case AArch64::LDR_ZXI:
3057 case AArch64::LDR_ZZXI:
3058 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
3059 case AArch64::LDR_ZZZXI:
3060 case AArch64::LDR_ZZZZXI:
3061 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
3062 case AArch64::LDRBBui:
3063 case AArch64::LDRBui:
3064 case AArch64::LDRDui:
3065 case AArch64::LDRHHui:
3066 case AArch64::LDRHui:
3067 case AArch64::LDRQui:
3068 case AArch64::LDRSBWui:
3069 case AArch64::LDRSBXui:
3070 case AArch64::LDRSHWui:
3071 case AArch64::LDRSHXui:
3072 case AArch64::LDRSui:
3073 case AArch64::LDRSWui:
3074 case AArch64::LDRWui:
3075 case AArch64::LDRXui:
3076 case AArch64::LDURBBi:
3077 case AArch64::LDURBi:
3078 case AArch64::LDURDi:
3079 case AArch64::LDURHHi:
3080 case AArch64::LDURHi:
3081 case AArch64::LDURQi:
3082 case AArch64::LDURSBWi:
3083 case AArch64::LDURSBXi:
3084 case AArch64::LDURSHWi:
3085 case AArch64::LDURSHXi:
3086 case AArch64::LDURSi:
3087 case AArch64::LDURSWi:
3088 case AArch64::LDURWi:
3089 case AArch64::LDURXi:
3090 case AArch64::PRFMui:
3091 case AArch64::PRFUMi:
3092 case AArch64::ST2Gi:
3093 case AArch64::STGi:
3094 case AArch64::STLURBi:
3095 case AArch64::STLURHi:
3096 case AArch64::STLURWi:
3097 case AArch64::STLURXi:
3098 case AArch64::StoreSwiftAsyncContext:
3099 case AArch64::STR_PPXI:
3100 case AArch64::STR_PXI:
3101 case AArch64::STR_ZXI:
3102 case AArch64::STR_ZZXI:
3103 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
3104 case AArch64::STR_ZZZXI:
3105 case AArch64::STR_ZZZZXI:
3106 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
3107 case AArch64::STRBBui:
3108 case AArch64::STRBui:
3109 case AArch64::STRDui:
3110 case AArch64::STRHHui:
3111 case AArch64::STRHui:
3112 case AArch64::STRQui:
3113 case AArch64::STRSui:
3114 case AArch64::STRWui:
3115 case AArch64::STRXui:
3116 case AArch64::STURBBi:
3117 case AArch64::STURBi:
3118 case AArch64::STURDi:
3119 case AArch64::STURHHi:
3120 case AArch64::STURHi:
3121 case AArch64::STURQi:
3122 case AArch64::STURSi:
3123 case AArch64::STURWi:
3124 case AArch64::STURXi:
3125 case AArch64::STZ2Gi:
3126 case AArch64::STZGi:
3127 case AArch64::TAGPstack:
3128 case AArch64::ATOMIC_STORE_HINT_Bi:
3129 case AArch64::ATOMIC_STORE_HINT_Hi:
3130 case AArch64::ATOMIC_STORE_HINT_Wi:
3131 case AArch64::ATOMIC_STORE_HINT_Si:
3132 case AArch64::ATOMIC_STORE_HINT_Xi:
3133 case AArch64::ATOMIC_STORE_HINT_Di:
3134 case AArch64::ATOMIC_STORE_HINT_Bui:
3135 case AArch64::ATOMIC_STORE_HINT_Hui:
3136 case AArch64::ATOMIC_STORE_HINT_Wui:
3137 case AArch64::ATOMIC_STORE_HINT_Sui:
3138 case AArch64::ATOMIC_STORE_HINT_Xui:
3139 case AArch64::ATOMIC_STORE_HINT_Dui:
3140 return 2;
3141 case AArch64::LD1B_D_IMM:
3142 case AArch64::LD1B_H_IMM:
3143 case AArch64::LD1B_IMM:
3144 case AArch64::LD1B_S_IMM:
3145 case AArch64::LD1D_IMM:
3146 case AArch64::LD1H_D_IMM:
3147 case AArch64::LD1H_IMM:
3148 case AArch64::LD1H_S_IMM:
3149 case AArch64::LD1RB_D_IMM:
3150 case AArch64::LD1RB_H_IMM:
3151 case AArch64::LD1RB_IMM:
3152 case AArch64::LD1RB_S_IMM:
3153 case AArch64::LD1RD_IMM:
3154 case AArch64::LD1RH_D_IMM:
3155 case AArch64::LD1RH_IMM:
3156 case AArch64::LD1RH_S_IMM:
3157 case AArch64::LD1RSB_D_IMM:
3158 case AArch64::LD1RSB_H_IMM:
3159 case AArch64::LD1RSB_S_IMM:
3160 case AArch64::LD1RSH_D_IMM:
3161 case AArch64::LD1RSH_S_IMM:
3162 case AArch64::LD1RSW_IMM:
3163 case AArch64::LD1RW_D_IMM:
3164 case AArch64::LD1RW_IMM:
3165 case AArch64::LD1SB_D_IMM:
3166 case AArch64::LD1SB_H_IMM:
3167 case AArch64::LD1SB_S_IMM:
3168 case AArch64::LD1SH_D_IMM:
3169 case AArch64::LD1SH_S_IMM:
3170 case AArch64::LD1SW_D_IMM:
3171 case AArch64::LD1W_D_IMM:
3172 case AArch64::LD1W_IMM:
3173 case AArch64::LD2B_IMM:
3174 case AArch64::LD2D_IMM:
3175 case AArch64::LD2H_IMM:
3176 case AArch64::LD2W_IMM:
3177 case AArch64::LD3B_IMM:
3178 case AArch64::LD3D_IMM:
3179 case AArch64::LD3H_IMM:
3180 case AArch64::LD3W_IMM:
3181 case AArch64::LD4B_IMM:
3182 case AArch64::LD4D_IMM:
3183 case AArch64::LD4H_IMM:
3184 case AArch64::LD4W_IMM:
3185 case AArch64::LDG:
3186 case AArch64::LDNF1B_D_IMM:
3187 case AArch64::LDNF1B_H_IMM:
3188 case AArch64::LDNF1B_IMM:
3189 case AArch64::LDNF1B_S_IMM:
3190 case AArch64::LDNF1D_IMM:
3191 case AArch64::LDNF1H_D_IMM:
3192 case AArch64::LDNF1H_IMM:
3193 case AArch64::LDNF1H_S_IMM:
3194 case AArch64::LDNF1SB_D_IMM:
3195 case AArch64::LDNF1SB_H_IMM:
3196 case AArch64::LDNF1SB_S_IMM:
3197 case AArch64::LDNF1SH_D_IMM:
3198 case AArch64::LDNF1SH_S_IMM:
3199 case AArch64::LDNF1SW_D_IMM:
3200 case AArch64::LDNF1W_D_IMM:
3201 case AArch64::LDNF1W_IMM:
3202 case AArch64::LDNPDi:
3203 case AArch64::LDNPQi:
3204 case AArch64::LDNPSi:
3205 case AArch64::LDNPWi:
3206 case AArch64::LDNPXi:
3207 case AArch64::LDNT1B_ZRI:
3208 case AArch64::LDNT1D_ZRI:
3209 case AArch64::LDNT1H_ZRI:
3210 case AArch64::LDNT1W_ZRI:
3211 case AArch64::LDPDi:
3212 case AArch64::LDPQi:
3213 case AArch64::LDPSi:
3214 case AArch64::LDPWi:
3215 case AArch64::LDPXi:
3216 case AArch64::LDRBBpost:
3217 case AArch64::LDRBBpre:
3218 case AArch64::LDRBpost:
3219 case AArch64::LDRBpre:
3220 case AArch64::LDRDpost:
3221 case AArch64::LDRDpre:
3222 case AArch64::LDRHHpost:
3223 case AArch64::LDRHHpre:
3224 case AArch64::LDRHpost:
3225 case AArch64::LDRHpre:
3226 case AArch64::LDRQpost:
3227 case AArch64::LDRQpre:
3228 case AArch64::LDRSpost:
3229 case AArch64::LDRSpre:
3230 case AArch64::LDRWpost:
3231 case AArch64::LDRWpre:
3232 case AArch64::LDRXpost:
3233 case AArch64::LDRXpre:
3234 case AArch64::ST1B_D_IMM:
3235 case AArch64::ST1B_H_IMM:
3236 case AArch64::ST1B_IMM:
3237 case AArch64::ST1B_S_IMM:
3238 case AArch64::ST1D_IMM:
3239 case AArch64::ST1H_D_IMM:
3240 case AArch64::ST1H_IMM:
3241 case AArch64::ST1H_S_IMM:
3242 case AArch64::ST1W_D_IMM:
3243 case AArch64::ST1W_IMM:
3244 case AArch64::ST2B_IMM:
3245 case AArch64::ST2D_IMM:
3246 case AArch64::ST2H_IMM:
3247 case AArch64::ST2W_IMM:
3248 case AArch64::ST3B_IMM:
3249 case AArch64::ST3D_IMM:
3250 case AArch64::ST3H_IMM:
3251 case AArch64::ST3W_IMM:
3252 case AArch64::ST4B_IMM:
3253 case AArch64::ST4D_IMM:
3254 case AArch64::ST4H_IMM:
3255 case AArch64::ST4W_IMM:
3256 case AArch64::STGPi:
3257 case AArch64::STGPreIndex:
3258 case AArch64::STZGPreIndex:
3259 case AArch64::ST2GPreIndex:
3260 case AArch64::STZ2GPreIndex:
3261 case AArch64::STGPostIndex:
3262 case AArch64::STZGPostIndex:
3263 case AArch64::ST2GPostIndex:
3264 case AArch64::STZ2GPostIndex:
3265 case AArch64::STNPDi:
3266 case AArch64::STNPQi:
3267 case AArch64::STNPSi:
3268 case AArch64::STNPWi:
3269 case AArch64::STNPXi:
3270 case AArch64::STNT1B_ZRI:
3271 case AArch64::STNT1D_ZRI:
3272 case AArch64::STNT1H_ZRI:
3273 case AArch64::STNT1W_ZRI:
3274 case AArch64::STPDi:
3275 case AArch64::STPQi:
3276 case AArch64::STPSi:
3277 case AArch64::STPWi:
3278 case AArch64::STPXi:
3279 case AArch64::STRBBpost:
3280 case AArch64::STRBBpre:
3281 case AArch64::STRBpost:
3282 case AArch64::STRBpre:
3283 case AArch64::STRDpost:
3284 case AArch64::STRDpre:
3285 case AArch64::STRHHpost:
3286 case AArch64::STRHHpre:
3287 case AArch64::STRHpost:
3288 case AArch64::STRHpre:
3289 case AArch64::STRQpost:
3290 case AArch64::STRQpre:
3291 case AArch64::STRSpost:
3292 case AArch64::STRSpre:
3293 case AArch64::STRWpost:
3294 case AArch64::STRWpre:
3295 case AArch64::STRXpost:
3296 case AArch64::STRXpre:
3297 case AArch64::LD1B_2Z_IMM:
3298 case AArch64::LD1B_2Z_STRIDED_IMM:
3299 case AArch64::LD1H_2Z_IMM:
3300 case AArch64::LD1H_2Z_STRIDED_IMM:
3301 case AArch64::LD1W_2Z_IMM:
3302 case AArch64::LD1W_2Z_STRIDED_IMM:
3303 case AArch64::LD1D_2Z_IMM:
3304 case AArch64::LD1D_2Z_STRIDED_IMM:
3305 case AArch64::LD1B_4Z_IMM:
3306 case AArch64::LD1B_4Z_STRIDED_IMM:
3307 case AArch64::LD1H_4Z_IMM:
3308 case AArch64::LD1H_4Z_STRIDED_IMM:
3309 case AArch64::LD1W_4Z_IMM:
3310 case AArch64::LD1W_4Z_STRIDED_IMM:
3311 case AArch64::LD1D_4Z_IMM:
3312 case AArch64::LD1D_4Z_STRIDED_IMM:
3313 case AArch64::LD1B_2Z_IMM_PSEUDO:
3314 case AArch64::LD1H_2Z_IMM_PSEUDO:
3315 case AArch64::LD1W_2Z_IMM_PSEUDO:
3316 case AArch64::LD1D_2Z_IMM_PSEUDO:
3317 case AArch64::LD1B_4Z_IMM_PSEUDO:
3318 case AArch64::LD1H_4Z_IMM_PSEUDO:
3319 case AArch64::LD1W_4Z_IMM_PSEUDO:
3320 case AArch64::LD1D_4Z_IMM_PSEUDO:
3321 case AArch64::ST1B_2Z_IMM:
3322 case AArch64::ST1B_2Z_STRIDED_IMM:
3323 case AArch64::ST1H_2Z_IMM:
3324 case AArch64::ST1H_2Z_STRIDED_IMM:
3325 case AArch64::ST1W_2Z_IMM:
3326 case AArch64::ST1W_2Z_STRIDED_IMM:
3327 case AArch64::ST1D_2Z_IMM:
3328 case AArch64::ST1D_2Z_STRIDED_IMM:
3329 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
3330 case AArch64::LDNT1B_2Z_IMM:
3331 case AArch64::LDNT1B_2Z_STRIDED_IMM:
3332 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
3333 case AArch64::LDNT1H_2Z_IMM:
3334 case AArch64::LDNT1H_2Z_STRIDED_IMM:
3335 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
3336 case AArch64::LDNT1W_2Z_IMM:
3337 case AArch64::LDNT1W_2Z_STRIDED_IMM:
3338 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
3339 case AArch64::LDNT1D_2Z_IMM:
3340 case AArch64::LDNT1D_2Z_STRIDED_IMM:
3341 case AArch64::STNT1B_2Z_IMM:
3342 case AArch64::STNT1B_2Z_STRIDED_IMM:
3343 case AArch64::STNT1H_2Z_IMM:
3344 case AArch64::STNT1H_2Z_STRIDED_IMM:
3345 case AArch64::STNT1W_2Z_IMM:
3346 case AArch64::STNT1W_2Z_STRIDED_IMM:
3347 case AArch64::STNT1D_2Z_IMM:
3348 case AArch64::STNT1D_2Z_STRIDED_IMM:
3349 case AArch64::ST1B_2Z_IMM_PSEUDO:
3350 case AArch64::ST1H_2Z_IMM_PSEUDO:
3351 case AArch64::ST1W_2Z_IMM_PSEUDO:
3352 case AArch64::ST1D_2Z_IMM_PSEUDO:
3353 case AArch64::STNT1B_2Z_IMM_PSEUDO:
3354 case AArch64::STNT1H_2Z_IMM_PSEUDO:
3355 case AArch64::STNT1W_2Z_IMM_PSEUDO:
3356 case AArch64::STNT1D_2Z_IMM_PSEUDO:
3357 case AArch64::ST1B_4Z_IMM:
3358 case AArch64::ST1B_4Z_STRIDED_IMM:
3359 case AArch64::ST1H_4Z_IMM:
3360 case AArch64::ST1H_4Z_STRIDED_IMM:
3361 case AArch64::ST1W_4Z_IMM:
3362 case AArch64::ST1W_4Z_STRIDED_IMM:
3363 case AArch64::ST1D_4Z_IMM:
3364 case AArch64::ST1D_4Z_STRIDED_IMM:
3365 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
3366 case AArch64::LDNT1B_4Z_IMM:
3367 case AArch64::LDNT1B_4Z_STRIDED_IMM:
3368 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
3369 case AArch64::LDNT1H_4Z_IMM:
3370 case AArch64::LDNT1H_4Z_STRIDED_IMM:
3371 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
3372 case AArch64::LDNT1W_4Z_IMM:
3373 case AArch64::LDNT1W_4Z_STRIDED_IMM:
3374 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
3375 case AArch64::LDNT1D_4Z_IMM:
3376 case AArch64::LDNT1D_4Z_STRIDED_IMM:
3377 case AArch64::STNT1B_4Z_IMM:
3378 case AArch64::STNT1B_4Z_STRIDED_IMM:
3379 case AArch64::STNT1H_4Z_IMM:
3380 case AArch64::STNT1H_4Z_STRIDED_IMM:
3381 case AArch64::STNT1W_4Z_IMM:
3382 case AArch64::STNT1W_4Z_STRIDED_IMM:
3383 case AArch64::STNT1D_4Z_IMM:
3384 case AArch64::STNT1D_4Z_STRIDED_IMM:
3385 case AArch64::ST1B_4Z_IMM_PSEUDO:
3386 case AArch64::ST1H_4Z_IMM_PSEUDO:
3387 case AArch64::ST1W_4Z_IMM_PSEUDO:
3388 case AArch64::ST1D_4Z_IMM_PSEUDO:
3389 case AArch64::STNT1B_4Z_IMM_PSEUDO:
3390 case AArch64::STNT1H_4Z_IMM_PSEUDO:
3391 case AArch64::STNT1W_4Z_IMM_PSEUDO:
3392 case AArch64::STNT1D_4Z_IMM_PSEUDO:
3393 return 3;
3394 case AArch64::LDPDpost:
3395 case AArch64::LDPDpre:
3396 case AArch64::LDPQpost:
3397 case AArch64::LDPQpre:
3398 case AArch64::LDPSpost:
3399 case AArch64::LDPSpre:
3400 case AArch64::LDPWpost:
3401 case AArch64::LDPWpre:
3402 case AArch64::LDPXpost:
3403 case AArch64::LDPXpre:
3404 case AArch64::STGPpre:
3405 case AArch64::STGPpost:
3406 case AArch64::STPDpost:
3407 case AArch64::STPDpre:
3408 case AArch64::STPQpost:
3409 case AArch64::STPQpre:
3410 case AArch64::STPSpost:
3411 case AArch64::STPSpre:
3412 case AArch64::STPWpost:
3413 case AArch64::STPWpre:
3414 case AArch64::STPXpost:
3415 case AArch64::STPXpre:
3416 return 4;
3417 }
3418}
3419
3420bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
3421 switch (MI.getOpcode()) {
3422 default:
3423 return false;
3424 // Scaled instructions.
3425 case AArch64::STRSui:
3426 case AArch64::STRDui:
3427 case AArch64::STRQui:
3428 case AArch64::STRXui:
3429 case AArch64::STRWui:
3430 case AArch64::LDRSui:
3431 case AArch64::LDRDui:
3432 case AArch64::LDRQui:
3433 case AArch64::LDRXui:
3434 case AArch64::LDRWui:
3435 case AArch64::LDRSWui:
3436 // Unscaled instructions.
3437 case AArch64::STURSi:
3438 case AArch64::STRSpre:
3439 case AArch64::STURDi:
3440 case AArch64::STRDpre:
3441 case AArch64::STURQi:
3442 case AArch64::STRQpre:
3443 case AArch64::STURWi:
3444 case AArch64::STRWpre:
3445 case AArch64::STURXi:
3446 case AArch64::STRXpre:
3447 case AArch64::LDURSi:
3448 case AArch64::LDRSpre:
3449 case AArch64::LDURDi:
3450 case AArch64::LDRDpre:
3451 case AArch64::LDURQi:
3452 case AArch64::LDRQpre:
3453 case AArch64::LDURWi:
3454 case AArch64::LDRWpre:
3455 case AArch64::LDURXi:
3456 case AArch64::LDRXpre:
3457 case AArch64::LDURSWi:
3458 case AArch64::LDRSWpre:
3459 // SVE instructions.
3460 case AArch64::LDR_ZXI:
3461 case AArch64::STR_ZXI:
3462 return true;
3463 }
3464}
3465
3466bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
3467 switch (MI.getOpcode()) {
3468 default:
3469 assert((!MI.isCall() || !MI.isReturn()) &&
3470 "Unexpected instruction - was a new tail call opcode introduced?");
3471 return false;
3472 case AArch64::TCRETURNdi:
3473 case AArch64::TCRETURNri:
3474 case AArch64::TCRETURNrix16x17:
3475 case AArch64::TCRETURNrix17:
3476 case AArch64::TCRETURNrinotx16:
3477 case AArch64::TCRETURNriALL:
3478 case AArch64::AUTH_TCRETURN:
3479 case AArch64::AUTH_TCRETURN_BTI:
3480 return true;
3481 }
3482}
3483
3484unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) {
3485 switch (Opc) {
3486 default:
3487 llvm_unreachable("Opcode has no flag setting equivalent!");
3488 // 32-bit cases:
3489 case AArch64::ADDWri:
3490 return AArch64::ADDSWri;
3491 case AArch64::ADDWrr:
3492 return AArch64::ADDSWrr;
3493 case AArch64::ADDWrs:
3494 return AArch64::ADDSWrs;
3495 case AArch64::ADDWrx:
3496 return AArch64::ADDSWrx;
3497 case AArch64::ANDWri:
3498 return AArch64::ANDSWri;
3499 case AArch64::ANDWrr:
3500 return AArch64::ANDSWrr;
3501 case AArch64::ANDWrs:
3502 return AArch64::ANDSWrs;
3503 case AArch64::BICWrr:
3504 return AArch64::BICSWrr;
3505 case AArch64::BICWrs:
3506 return AArch64::BICSWrs;
3507 case AArch64::SUBWri:
3508 return AArch64::SUBSWri;
3509 case AArch64::SUBWrr:
3510 return AArch64::SUBSWrr;
3511 case AArch64::SUBWrs:
3512 return AArch64::SUBSWrs;
3513 case AArch64::SUBWrx:
3514 return AArch64::SUBSWrx;
3515 // 64-bit cases:
3516 case AArch64::ADDXri:
3517 return AArch64::ADDSXri;
3518 case AArch64::ADDXrr:
3519 return AArch64::ADDSXrr;
3520 case AArch64::ADDXrs:
3521 return AArch64::ADDSXrs;
3522 case AArch64::ADDXrx:
3523 return AArch64::ADDSXrx;
3524 case AArch64::ANDXri:
3525 return AArch64::ANDSXri;
3526 case AArch64::ANDXrr:
3527 return AArch64::ANDSXrr;
3528 case AArch64::ANDXrs:
3529 return AArch64::ANDSXrs;
3530 case AArch64::BICXrr:
3531 return AArch64::BICSXrr;
3532 case AArch64::BICXrs:
3533 return AArch64::BICSXrs;
3534 case AArch64::SUBXri:
3535 return AArch64::SUBSXri;
3536 case AArch64::SUBXrr:
3537 return AArch64::SUBSXrr;
3538 case AArch64::SUBXrs:
3539 return AArch64::SUBSXrs;
3540 case AArch64::SUBXrx:
3541 return AArch64::SUBSXrx;
3542 // SVE instructions:
3543 case AArch64::AND_PPzPP:
3544 return AArch64::ANDS_PPzPP;
3545 case AArch64::BIC_PPzPP:
3546 return AArch64::BICS_PPzPP;
3547 case AArch64::EOR_PPzPP:
3548 return AArch64::EORS_PPzPP;
3549 case AArch64::NAND_PPzPP:
3550 return AArch64::NANDS_PPzPP;
3551 case AArch64::NOR_PPzPP:
3552 return AArch64::NORS_PPzPP;
3553 case AArch64::ORN_PPzPP:
3554 return AArch64::ORNS_PPzPP;
3555 case AArch64::ORR_PPzPP:
3556 return AArch64::ORRS_PPzPP;
3557 case AArch64::BRKA_PPzP:
3558 return AArch64::BRKAS_PPzP;
3559 case AArch64::BRKPA_PPzPP:
3560 return AArch64::BRKPAS_PPzPP;
3561 case AArch64::BRKB_PPzP:
3562 return AArch64::BRKBS_PPzP;
3563 case AArch64::BRKPB_PPzPP:
3564 return AArch64::BRKPBS_PPzPP;
3565 case AArch64::BRKN_PPzP:
3566 return AArch64::BRKNS_PPzP;
3567 case AArch64::RDFFR_PPz:
3568 return AArch64::RDFFRS_PPz;
3569 case AArch64::PTRUE_B:
3570 return AArch64::PTRUES_B;
3571 }
3572}
3573
3574// Is this a candidate for ld/st merging or pairing? For example, we don't
3575// touch volatiles or load/stores that have a hint to avoid pair formation.
3576bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
3577
3578 bool IsPreLdSt = isPreLdSt(MI);
3579
3580 // If this is a volatile load/store, don't mess with it.
3581 if (MI.hasOrderedMemoryRef())
3582 return false;
3583
3584 // Make sure this is a reg/fi+imm (as opposed to an address reloc).
3585 // For Pre-inc LD/ST, the operand is shifted by one.
3586 assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
3587 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
3588 "Expected a reg or frame index operand.");
3589
3590 // For Pre-indexed addressing quadword instructions, the third operand is the
3591 // immediate value.
3592 bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(i: 3).isImm();
3593
3594 if (!MI.getOperand(i: 2).isImm() && !IsImmPreLdSt)
3595 return false;
3596
3597 // Can't merge/pair if the instruction modifies the base register.
3598 // e.g., ldr x0, [x0]
3599 // This case will never occur with an FI base.
3600 // However, if the instruction is an LDR<S,D,Q,W,X,SW>pre or
3601 // STR<S,D,Q,W,X>pre, it can be merged.
3602 // For example:
3603 // ldr q0, [x11, #32]!
3604 // ldr q1, [x11, #16]
3605 // to
3606 // ldp q0, q1, [x11, #32]!
3607 if (MI.getOperand(i: 1).isReg() && !IsPreLdSt) {
3608 Register BaseReg = MI.getOperand(i: 1).getReg();
3609 const TargetRegisterInfo *TRI = &getRegisterInfo();
3610 if (MI.modifiesRegister(Reg: BaseReg, TRI))
3611 return false;
3612 }
3613
3614 // Pairing SVE fills/spills is only valid for little-endian targets that
3615 // implement VLS 128.
3616 switch (MI.getOpcode()) {
3617 default:
3618 break;
3619 case AArch64::LDR_ZXI:
3620 case AArch64::STR_ZXI:
3621 if (!Subtarget.isLittleEndian() ||
3622 Subtarget.getSVEVectorSizeInBits() != 128)
3623 return false;
3624 }
3625
3626 // Check if this load/store has a hint to avoid pair formation.
3627 // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
3628 if (isLdStPairSuppressed(MI))
3629 return false;
3630
3631 // Do not pair any callee-save store/reload instructions in the
3632 // prologue/epilogue if the CFI information encoded the operations as separate
3633 // instructions, as that will cause the size of the actual prologue to mismatch
3634 // with the prologue size recorded in the Windows CFI.
3635 if ((MI.getFlag(Flag: MachineInstr::FrameSetup) ||
3636 MI.getFlag(Flag: MachineInstr::FrameDestroy)) &&
3637 MI.getMF()->needsWinCFI())
3638 return false;
3639
3640 // On some CPUs quad load/store pairs are slower than two single load/stores.
3641 if (Subtarget.isPaired128Slow()) {
3642 switch (MI.getOpcode()) {
3643 default:
3644 break;
3645 case AArch64::LDURQi:
3646 case AArch64::STURQi:
3647 case AArch64::LDRQui:
3648 case AArch64::STRQui:
3649 return false;
3650 }
3651 }
3652
3653 return true;
3654}
3655
3656bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
3657 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3658 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
3659 if (!LdSt.mayLoadOrStore())
3660 return false;
3661
3662 const MachineOperand *BaseOp;
3663 TypeSize WidthN(0, false);
3664 if (!getMemOperandWithOffsetWidth(MI: LdSt, BaseOp, Offset, OffsetIsScalable,
3665 Width&: WidthN))
3666 return false;
3667 // The maximum vscale is 16 under AArch64, return the maximal extent for the
3668 // vector.
3669 Width = LocationSize::precise(Value: WidthN);
3670 BaseOps.push_back(Elt: BaseOp);
3671 return true;
3672}
3673
3674std::optional<ExtAddrMode>
3675AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI) const {
3676 const MachineOperand *Base; // Filled with the base operand of MI.
3677 int64_t Offset; // Filled with the offset of MI.
3678 bool OffsetIsScalable;
3679 if (!getMemOperandWithOffset(MI: MemI, BaseOp&: Base, Offset, OffsetIsScalable))
3680 return std::nullopt;
3681
3682 if (!Base->isReg())
3683 return std::nullopt;
3684 ExtAddrMode AM;
3685 AM.BaseReg = Base->getReg();
3686 AM.Displacement = Offset;
3687 AM.ScaledReg = 0;
3688 AM.Scale = 0;
3689 return AM;
3690}
3691
3692bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
3693 Register Reg,
3694 const MachineInstr &AddrI,
3695 ExtAddrMode &AM) const {
3696 // Filter out instructions into which we cannot fold.
3697 unsigned NumBytes;
3698 int64_t OffsetScale = 1;
3699 switch (MemI.getOpcode()) {
3700 default:
3701 return false;
3702
3703 case AArch64::LDURQi:
3704 case AArch64::STURQi:
3705 NumBytes = 16;
3706 break;
3707
3708 case AArch64::LDURDi:
3709 case AArch64::STURDi:
3710 case AArch64::LDURXi:
3711 case AArch64::STURXi:
3712 NumBytes = 8;
3713 break;
3714
3715 case AArch64::LDURWi:
3716 case AArch64::LDURSWi:
3717 case AArch64::STURWi:
3718 NumBytes = 4;
3719 break;
3720
3721 case AArch64::LDURHi:
3722 case AArch64::STURHi:
3723 case AArch64::LDURHHi:
3724 case AArch64::STURHHi:
3725 case AArch64::LDURSHXi:
3726 case AArch64::LDURSHWi:
3727 NumBytes = 2;
3728 break;
3729
3730 case AArch64::LDRBroX:
3731 case AArch64::LDRBBroX:
3732 case AArch64::LDRSBXroX:
3733 case AArch64::LDRSBWroX:
3734 case AArch64::STRBroX:
3735 case AArch64::STRBBroX:
3736 case AArch64::LDURBi:
3737 case AArch64::LDURBBi:
3738 case AArch64::LDURSBXi:
3739 case AArch64::LDURSBWi:
3740 case AArch64::STURBi:
3741 case AArch64::STURBBi:
3742 case AArch64::LDRBui:
3743 case AArch64::LDRBBui:
3744 case AArch64::LDRSBXui:
3745 case AArch64::LDRSBWui:
3746 case AArch64::STRBui:
3747 case AArch64::STRBBui:
3748 NumBytes = 1;
3749 break;
3750
3751 case AArch64::LDRQroX:
3752 case AArch64::STRQroX:
3753 case AArch64::LDRQui:
3754 case AArch64::STRQui:
3755 NumBytes = 16;
3756 OffsetScale = 16;
3757 break;
3758
3759 case AArch64::LDRDroX:
3760 case AArch64::STRDroX:
3761 case AArch64::LDRXroX:
3762 case AArch64::STRXroX:
3763 case AArch64::LDRDui:
3764 case AArch64::STRDui:
3765 case AArch64::LDRXui:
3766 case AArch64::STRXui:
3767 NumBytes = 8;
3768 OffsetScale = 8;
3769 break;
3770
3771 case AArch64::LDRWroX:
3772 case AArch64::LDRSWroX:
3773 case AArch64::STRWroX:
3774 case AArch64::LDRWui:
3775 case AArch64::LDRSWui:
3776 case AArch64::STRWui:
3777 NumBytes = 4;
3778 OffsetScale = 4;
3779 break;
3780
3781 case AArch64::LDRHroX:
3782 case AArch64::STRHroX:
3783 case AArch64::LDRHHroX:
3784 case AArch64::STRHHroX:
3785 case AArch64::LDRSHXroX:
3786 case AArch64::LDRSHWroX:
3787 case AArch64::LDRHui:
3788 case AArch64::STRHui:
3789 case AArch64::LDRHHui:
3790 case AArch64::STRHHui:
3791 case AArch64::LDRSHXui:
3792 case AArch64::LDRSHWui:
3793 NumBytes = 2;
3794 OffsetScale = 2;
3795 break;
3796 }
3797
3798 // Check the fold operand is not the loaded/stored value.
3799 const MachineOperand &BaseRegOp = MemI.getOperand(i: 0);
3800 if (BaseRegOp.isReg() && BaseRegOp.getReg() == Reg)
3801 return false;
3802
3803 // Handle memory instructions with a [Reg, Reg] addressing mode.
3804 if (MemI.getOperand(i: 2).isReg()) {
3805 // Bail if the addressing mode already includes extension of the offset
3806 // register.
3807 if (MemI.getOperand(i: 3).getImm())
3808 return false;
3809
3810 // Check if we actually have a scaled offset.
3811 if (MemI.getOperand(i: 4).getImm() == 0)
3812 OffsetScale = 1;
3813
3814 // If the address instructions is folded into the base register, then the
3815 // addressing mode must not have a scale. Then we can swap the base and the
3816 // scaled registers.
3817 if (MemI.getOperand(i: 1).getReg() == Reg && OffsetScale != 1)
3818 return false;
3819
3820 switch (AddrI.getOpcode()) {
3821 default:
3822 return false;
3823
3824 case AArch64::SBFMXri:
3825 // sxtw Xa, Wm
3826 // ldr Xd, [Xn, Xa, lsl #N]
3827 // ->
3828 // ldr Xd, [Xn, Wm, sxtw #N]
3829 if (AddrI.getOperand(i: 2).getImm() != 0 ||
3830 AddrI.getOperand(i: 3).getImm() != 31)
3831 return false;
3832
3833 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3834 if (AM.BaseReg == Reg)
3835 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3836 AM.ScaledReg = AddrI.getOperand(i: 1).getReg();
3837 AM.Scale = OffsetScale;
3838 AM.Displacement = 0;
3839 AM.Form = ExtAddrMode::Formula::SExtScaledReg;
3840 return true;
3841
3842 case TargetOpcode::SUBREG_TO_REG: {
3843 // mov Wa, Wm
3844 // ldr Xd, [Xn, Xa, lsl #N]
3845 // ->
3846 // ldr Xd, [Xn, Wm, uxtw #N]
3847
3848 // Zero-extension looks like an ORRWrs followed by a SUBREG_TO_REG.
3849 if (AddrI.getOperand(i: 2).getImm() != AArch64::sub_32)
3850 return false;
3851
3852 const MachineRegisterInfo &MRI = AddrI.getMF()->getRegInfo();
3853 Register OffsetReg = AddrI.getOperand(i: 1).getReg();
3854 if (!OffsetReg.isVirtual() || !MRI.hasOneNonDBGUse(RegNo: OffsetReg))
3855 return false;
3856
3857 const MachineInstr &DefMI = *MRI.getVRegDef(Reg: OffsetReg);
3858 if (DefMI.getOpcode() != AArch64::ORRWrs ||
3859 DefMI.getOperand(i: 1).getReg() != AArch64::WZR ||
3860 DefMI.getOperand(i: 3).getImm() != 0)
3861 return false;
3862
3863 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3864 if (AM.BaseReg == Reg)
3865 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3866 AM.ScaledReg = DefMI.getOperand(i: 2).getReg();
3867 AM.Scale = OffsetScale;
3868 AM.Displacement = 0;
3869 AM.Form = ExtAddrMode::Formula::ZExtScaledReg;
3870 return true;
3871 }
3872 }
3873 }
3874
3875 // Handle memory instructions with a [Reg, #Imm] addressing mode.
3876
3877 // Check we are not breaking a potential conversion to an LDP.
3878 auto validateOffsetForLDP = [](unsigned NumBytes, int64_t OldOffset,
3879 int64_t NewOffset) -> bool {
3880 int64_t MinOffset, MaxOffset;
3881 switch (NumBytes) {
3882 default:
3883 return true;
3884 case 4:
3885 MinOffset = -256;
3886 MaxOffset = 252;
3887 break;
3888 case 8:
3889 MinOffset = -512;
3890 MaxOffset = 504;
3891 break;
3892 case 16:
3893 MinOffset = -1024;
3894 MaxOffset = 1008;
3895 break;
3896 }
3897 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3898 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3899 };
3900 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) -> bool {
3901 int64_t OldOffset = MemI.getOperand(i: 2).getImm() * OffsetScale;
3902 int64_t NewOffset = OldOffset + Disp;
3903 if (!isLegalAddressingMode(NumBytes, Offset: NewOffset, /* Scale */ 0))
3904 return false;
3905 // If the old offset would fit into an LDP, but the new offset wouldn't,
3906 // bail out.
3907 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3908 return false;
3909 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3910 AM.ScaledReg = 0;
3911 AM.Scale = 0;
3912 AM.Displacement = NewOffset;
3913 AM.Form = ExtAddrMode::Formula::Basic;
3914 return true;
3915 };
3916
3917 auto canFoldAddRegIntoAddrMode =
3918 [&](int64_t Scale,
3919 ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool {
3920 if (MemI.getOperand(i: 2).getImm() != 0)
3921 return false;
3922 if ((unsigned)Scale != Scale)
3923 return false;
3924 if (!isLegalAddressingMode(NumBytes, /* Offset */ 0, Scale))
3925 return false;
3926 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3927 AM.ScaledReg = AddrI.getOperand(i: 2).getReg();
3928 AM.Scale = Scale;
3929 AM.Displacement = 0;
3930 AM.Form = Form;
3931 return true;
3932 };
3933
3934 auto avoidSlowSTRQ = [&](const MachineInstr &MemI) {
3935 unsigned Opcode = MemI.getOpcode();
3936 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3937 Subtarget.isSTRQroSlow();
3938 };
3939
3940 int64_t Disp = 0;
3941 const bool OptSize = MemI.getMF()->getFunction().hasOptSize();
3942 switch (AddrI.getOpcode()) {
3943 default:
3944 return false;
3945
3946 case AArch64::ADDXri:
3947 // add Xa, Xn, #N
3948 // ldr Xd, [Xa, #M]
3949 // ->
3950 // ldr Xd, [Xn, #N'+M]
3951 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3952 return canFoldAddSubImmIntoAddrMode(Disp);
3953
3954 case AArch64::SUBXri:
3955 // sub Xa, Xn, #N
3956 // ldr Xd, [Xa, #M]
3957 // ->
3958 // ldr Xd, [Xn, #N'+M]
3959 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3960 return canFoldAddSubImmIntoAddrMode(-Disp);
3961
3962 case AArch64::ADDXrs: {
3963 // add Xa, Xn, Xm, lsl #N
3964 // ldr Xd, [Xa]
3965 // ->
3966 // ldr Xd, [Xn, Xm, lsl #N]
3967
3968 // Don't fold the add if the result would be slower, unless optimising for
3969 // size.
3970 unsigned Shift = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
3971 if (AArch64_AM::getShiftType(Imm: Shift) != AArch64_AM::ShiftExtendType::LSL)
3972 return false;
3973 Shift = AArch64_AM::getShiftValue(Imm: Shift);
3974 if (!OptSize) {
3975 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3976 return false;
3977 if (avoidSlowSTRQ(MemI))
3978 return false;
3979 }
3980 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3981 }
3982
3983 case AArch64::ADDXrr:
3984 // add Xa, Xn, Xm
3985 // ldr Xd, [Xa]
3986 // ->
3987 // ldr Xd, [Xn, Xm, lsl #0]
3988
3989 // Don't fold the add if the result would be slower, unless optimising for
3990 // size.
3991 if (!OptSize && avoidSlowSTRQ(MemI))
3992 return false;
3993 return canFoldAddRegIntoAddrMode(1);
3994
3995 case AArch64::ADDXrx:
3996 // add Xa, Xn, Wm, {s,u}xtw #N
3997 // ldr Xd, [Xa]
3998 // ->
3999 // ldr Xd, [Xn, Wm, {s,u}xtw #N]
4000
4001 // Don't fold the add if the result would be slower, unless optimising for
4002 // size.
4003 if (!OptSize && avoidSlowSTRQ(MemI))
4004 return false;
4005
4006 // Can fold only sign-/zero-extend of a word.
4007 unsigned Imm = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
4008 AArch64_AM::ShiftExtendType Extend = AArch64_AM::getArithExtendType(Imm);
4009 if (Extend != AArch64_AM::UXTW && Extend != AArch64_AM::SXTW)
4010 return false;
4011
4012 return canFoldAddRegIntoAddrMode(
4013 1ULL << AArch64_AM::getArithShiftValue(Imm),
4014 (Extend == AArch64_AM::SXTW) ? ExtAddrMode::Formula::SExtScaledReg
4015 : ExtAddrMode::Formula::ZExtScaledReg);
4016 }
4017}
4018
4019// Given an opcode for an instruction with a [Reg, #Imm] addressing mode,
4020// return the opcode of an instruction performing the same operation, but using
4021// the [Reg, Reg] addressing mode.
4022static unsigned regOffsetOpcode(unsigned Opcode) {
4023 switch (Opcode) {
4024 default:
4025 llvm_unreachable("Address folding not implemented for instruction");
4026
4027 case AArch64::LDURQi:
4028 case AArch64::LDRQui:
4029 return AArch64::LDRQroX;
4030 case AArch64::STURQi:
4031 case AArch64::STRQui:
4032 return AArch64::STRQroX;
4033 case AArch64::LDURDi:
4034 case AArch64::LDRDui:
4035 return AArch64::LDRDroX;
4036 case AArch64::STURDi:
4037 case AArch64::STRDui:
4038 return AArch64::STRDroX;
4039 case AArch64::LDURXi:
4040 case AArch64::LDRXui:
4041 return AArch64::LDRXroX;
4042 case AArch64::STURXi:
4043 case AArch64::STRXui:
4044 return AArch64::STRXroX;
4045 case AArch64::LDURWi:
4046 case AArch64::LDRWui:
4047 return AArch64::LDRWroX;
4048 case AArch64::LDURSWi:
4049 case AArch64::LDRSWui:
4050 return AArch64::LDRSWroX;
4051 case AArch64::STURWi:
4052 case AArch64::STRWui:
4053 return AArch64::STRWroX;
4054 case AArch64::LDURHi:
4055 case AArch64::LDRHui:
4056 return AArch64::LDRHroX;
4057 case AArch64::STURHi:
4058 case AArch64::STRHui:
4059 return AArch64::STRHroX;
4060 case AArch64::LDURHHi:
4061 case AArch64::LDRHHui:
4062 return AArch64::LDRHHroX;
4063 case AArch64::STURHHi:
4064 case AArch64::STRHHui:
4065 return AArch64::STRHHroX;
4066 case AArch64::LDURSHXi:
4067 case AArch64::LDRSHXui:
4068 return AArch64::LDRSHXroX;
4069 case AArch64::LDURSHWi:
4070 case AArch64::LDRSHWui:
4071 return AArch64::LDRSHWroX;
4072 case AArch64::LDURBi:
4073 case AArch64::LDRBui:
4074 return AArch64::LDRBroX;
4075 case AArch64::LDURBBi:
4076 case AArch64::LDRBBui:
4077 return AArch64::LDRBBroX;
4078 case AArch64::LDURSBXi:
4079 case AArch64::LDRSBXui:
4080 return AArch64::LDRSBXroX;
4081 case AArch64::LDURSBWi:
4082 case AArch64::LDRSBWui:
4083 return AArch64::LDRSBWroX;
4084 case AArch64::STURBi:
4085 case AArch64::STRBui:
4086 return AArch64::STRBroX;
4087 case AArch64::STURBBi:
4088 case AArch64::STRBBui:
4089 return AArch64::STRBBroX;
4090 }
4091}
4092
4093// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4094// the opcode of an instruction performing the same operation, but using the
4095// [Reg, #Imm] addressing mode with scaled offset.
4096unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale) {
4097 switch (Opcode) {
4098 default:
4099 llvm_unreachable("Address folding not implemented for instruction");
4100
4101 case AArch64::LDURQi:
4102 Scale = 16;
4103 return AArch64::LDRQui;
4104 case AArch64::STURQi:
4105 Scale = 16;
4106 return AArch64::STRQui;
4107 case AArch64::LDURDi:
4108 Scale = 8;
4109 return AArch64::LDRDui;
4110 case AArch64::STURDi:
4111 Scale = 8;
4112 return AArch64::STRDui;
4113 case AArch64::LDURXi:
4114 Scale = 8;
4115 return AArch64::LDRXui;
4116 case AArch64::STURXi:
4117 Scale = 8;
4118 return AArch64::STRXui;
4119 case AArch64::LDURWi:
4120 Scale = 4;
4121 return AArch64::LDRWui;
4122 case AArch64::LDURSWi:
4123 Scale = 4;
4124 return AArch64::LDRSWui;
4125 case AArch64::STURWi:
4126 Scale = 4;
4127 return AArch64::STRWui;
4128 case AArch64::LDURHi:
4129 Scale = 2;
4130 return AArch64::LDRHui;
4131 case AArch64::STURHi:
4132 Scale = 2;
4133 return AArch64::STRHui;
4134 case AArch64::LDURHHi:
4135 Scale = 2;
4136 return AArch64::LDRHHui;
4137 case AArch64::STURHHi:
4138 Scale = 2;
4139 return AArch64::STRHHui;
4140 case AArch64::LDURSHXi:
4141 Scale = 2;
4142 return AArch64::LDRSHXui;
4143 case AArch64::LDURSHWi:
4144 Scale = 2;
4145 return AArch64::LDRSHWui;
4146 case AArch64::LDURBi:
4147 Scale = 1;
4148 return AArch64::LDRBui;
4149 case AArch64::LDURBBi:
4150 Scale = 1;
4151 return AArch64::LDRBBui;
4152 case AArch64::LDURSBXi:
4153 Scale = 1;
4154 return AArch64::LDRSBXui;
4155 case AArch64::LDURSBWi:
4156 Scale = 1;
4157 return AArch64::LDRSBWui;
4158 case AArch64::STURBi:
4159 Scale = 1;
4160 return AArch64::STRBui;
4161 case AArch64::STURBBi:
4162 Scale = 1;
4163 return AArch64::STRBBui;
4164 case AArch64::LDRQui:
4165 case AArch64::STRQui:
4166 Scale = 16;
4167 return Opcode;
4168 case AArch64::LDRDui:
4169 case AArch64::STRDui:
4170 case AArch64::LDRXui:
4171 case AArch64::STRXui:
4172 Scale = 8;
4173 return Opcode;
4174 case AArch64::LDRWui:
4175 case AArch64::LDRSWui:
4176 case AArch64::STRWui:
4177 Scale = 4;
4178 return Opcode;
4179 case AArch64::LDRHui:
4180 case AArch64::STRHui:
4181 case AArch64::LDRHHui:
4182 case AArch64::STRHHui:
4183 case AArch64::LDRSHXui:
4184 case AArch64::LDRSHWui:
4185 Scale = 2;
4186 return Opcode;
4187 case AArch64::LDRBui:
4188 case AArch64::LDRBBui:
4189 case AArch64::LDRSBXui:
4190 case AArch64::LDRSBWui:
4191 case AArch64::STRBui:
4192 case AArch64::STRBBui:
4193 Scale = 1;
4194 return Opcode;
4195 }
4196}
4197
4198// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4199// the opcode of an instruction performing the same operation, but using the
4200// [Reg, #Imm] addressing mode with unscaled offset.
4201unsigned unscaledOffsetOpcode(unsigned Opcode) {
4202 switch (Opcode) {
4203 default:
4204 llvm_unreachable("Address folding not implemented for instruction");
4205
4206 case AArch64::LDURQi:
4207 case AArch64::STURQi:
4208 case AArch64::LDURDi:
4209 case AArch64::STURDi:
4210 case AArch64::LDURXi:
4211 case AArch64::STURXi:
4212 case AArch64::LDURWi:
4213 case AArch64::LDURSWi:
4214 case AArch64::STURWi:
4215 case AArch64::LDURHi:
4216 case AArch64::STURHi:
4217 case AArch64::LDURHHi:
4218 case AArch64::STURHHi:
4219 case AArch64::LDURSHXi:
4220 case AArch64::LDURSHWi:
4221 case AArch64::LDURBi:
4222 case AArch64::STURBi:
4223 case AArch64::LDURBBi:
4224 case AArch64::STURBBi:
4225 case AArch64::LDURSBWi:
4226 case AArch64::LDURSBXi:
4227 return Opcode;
4228 case AArch64::LDRQui:
4229 return AArch64::LDURQi;
4230 case AArch64::STRQui:
4231 return AArch64::STURQi;
4232 case AArch64::LDRDui:
4233 return AArch64::LDURDi;
4234 case AArch64::STRDui:
4235 return AArch64::STURDi;
4236 case AArch64::LDRXui:
4237 return AArch64::LDURXi;
4238 case AArch64::STRXui:
4239 return AArch64::STURXi;
4240 case AArch64::LDRWui:
4241 return AArch64::LDURWi;
4242 case AArch64::LDRSWui:
4243 return AArch64::LDURSWi;
4244 case AArch64::STRWui:
4245 return AArch64::STURWi;
4246 case AArch64::LDRHui:
4247 return AArch64::LDURHi;
4248 case AArch64::STRHui:
4249 return AArch64::STURHi;
4250 case AArch64::LDRHHui:
4251 return AArch64::LDURHHi;
4252 case AArch64::STRHHui:
4253 return AArch64::STURHHi;
4254 case AArch64::LDRSHXui:
4255 return AArch64::LDURSHXi;
4256 case AArch64::LDRSHWui:
4257 return AArch64::LDURSHWi;
4258 case AArch64::LDRBBui:
4259 return AArch64::LDURBBi;
4260 case AArch64::LDRBui:
4261 return AArch64::LDURBi;
4262 case AArch64::STRBBui:
4263 return AArch64::STURBBi;
4264 case AArch64::STRBui:
4265 return AArch64::STURBi;
4266 case AArch64::LDRSBWui:
4267 return AArch64::LDURSBWi;
4268 case AArch64::LDRSBXui:
4269 return AArch64::LDURSBXi;
4270 }
4271}
4272
4273// Given the opcode of a memory load/store instruction, return the opcode of an
4274// instruction performing the same operation, but using
4275// the [Reg, Reg, {s,u}xtw #N] addressing mode with sign-/zero-extend of the
4276// offset register.
4277static unsigned offsetExtendOpcode(unsigned Opcode) {
4278 switch (Opcode) {
4279 default:
4280 llvm_unreachable("Address folding not implemented for instruction");
4281
4282 case AArch64::LDRQroX:
4283 case AArch64::LDURQi:
4284 case AArch64::LDRQui:
4285 return AArch64::LDRQroW;
4286 case AArch64::STRQroX:
4287 case AArch64::STURQi:
4288 case AArch64::STRQui:
4289 return AArch64::STRQroW;
4290 case AArch64::LDRDroX:
4291 case AArch64::LDURDi:
4292 case AArch64::LDRDui:
4293 return AArch64::LDRDroW;
4294 case AArch64::STRDroX:
4295 case AArch64::STURDi:
4296 case AArch64::STRDui:
4297 return AArch64::STRDroW;
4298 case AArch64::LDRXroX:
4299 case AArch64::LDURXi:
4300 case AArch64::LDRXui:
4301 return AArch64::LDRXroW;
4302 case AArch64::STRXroX:
4303 case AArch64::STURXi:
4304 case AArch64::STRXui:
4305 return AArch64::STRXroW;
4306 case AArch64::LDRWroX:
4307 case AArch64::LDURWi:
4308 case AArch64::LDRWui:
4309 return AArch64::LDRWroW;
4310 case AArch64::LDRSWroX:
4311 case AArch64::LDURSWi:
4312 case AArch64::LDRSWui:
4313 return AArch64::LDRSWroW;
4314 case AArch64::STRWroX:
4315 case AArch64::STURWi:
4316 case AArch64::STRWui:
4317 return AArch64::STRWroW;
4318 case AArch64::LDRHroX:
4319 case AArch64::LDURHi:
4320 case AArch64::LDRHui:
4321 return AArch64::LDRHroW;
4322 case AArch64::STRHroX:
4323 case AArch64::STURHi:
4324 case AArch64::STRHui:
4325 return AArch64::STRHroW;
4326 case AArch64::LDRHHroX:
4327 case AArch64::LDURHHi:
4328 case AArch64::LDRHHui:
4329 return AArch64::LDRHHroW;
4330 case AArch64::STRHHroX:
4331 case AArch64::STURHHi:
4332 case AArch64::STRHHui:
4333 return AArch64::STRHHroW;
4334 case AArch64::LDRSHXroX:
4335 case AArch64::LDURSHXi:
4336 case AArch64::LDRSHXui:
4337 return AArch64::LDRSHXroW;
4338 case AArch64::LDRSHWroX:
4339 case AArch64::LDURSHWi:
4340 case AArch64::LDRSHWui:
4341 return AArch64::LDRSHWroW;
4342 case AArch64::LDRBroX:
4343 case AArch64::LDURBi:
4344 case AArch64::LDRBui:
4345 return AArch64::LDRBroW;
4346 case AArch64::LDRBBroX:
4347 case AArch64::LDURBBi:
4348 case AArch64::LDRBBui:
4349 return AArch64::LDRBBroW;
4350 case AArch64::LDRSBXroX:
4351 case AArch64::LDURSBXi:
4352 case AArch64::LDRSBXui:
4353 return AArch64::LDRSBXroW;
4354 case AArch64::LDRSBWroX:
4355 case AArch64::LDURSBWi:
4356 case AArch64::LDRSBWui:
4357 return AArch64::LDRSBWroW;
4358 case AArch64::STRBroX:
4359 case AArch64::STURBi:
4360 case AArch64::STRBui:
4361 return AArch64::STRBroW;
4362 case AArch64::STRBBroX:
4363 case AArch64::STURBBi:
4364 case AArch64::STRBBui:
4365 return AArch64::STRBBroW;
4366 }
4367}
4368
4369MachineInstr *AArch64InstrInfo::emitLdStWithAddr(MachineInstr &MemI,
4370 const ExtAddrMode &AM) const {
4371
4372 const DebugLoc &DL = MemI.getDebugLoc();
4373 MachineBasicBlock &MBB = *MemI.getParent();
4374 MachineRegisterInfo &MRI = MemI.getMF()->getRegInfo();
4375
4376 if (AM.Form == ExtAddrMode::Formula::Basic) {
4377 if (AM.ScaledReg) {
4378 // The new instruction will be in the form `ldr Rt, [Xn, Xm, lsl #imm]`.
4379 unsigned Opcode = regOffsetOpcode(Opcode: MemI.getOpcode());
4380 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4381 auto B = BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4382 .addReg(RegNo: MemI.getOperand(i: 0).getReg(),
4383 Flags: getDefRegState(B: MemI.mayLoad()))
4384 .addReg(RegNo: AM.BaseReg)
4385 .addReg(RegNo: AM.ScaledReg)
4386 .addImm(Val: 0)
4387 .addImm(Val: AM.Scale > 1)
4388 .setMemRefs(MemI.memoperands())
4389 .setMIFlags(MemI.getFlags());
4390 return B.getInstr();
4391 }
4392
4393 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
4394 "Addressing mode not supported for folding");
4395
4396 // The new instruction will be in the form `ld[u]r Rt, [Xn, #imm]`.
4397 unsigned Scale = 1;
4398 unsigned Opcode = MemI.getOpcode();
4399 if (isInt<9>(x: AM.Displacement))
4400 Opcode = unscaledOffsetOpcode(Opcode);
4401 else
4402 Opcode = scaledOffsetOpcode(Opcode, Scale);
4403
4404 auto B =
4405 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4406 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4407 .addReg(RegNo: AM.BaseReg)
4408 .addImm(Val: AM.Displacement / Scale)
4409 .setMemRefs(MemI.memoperands())
4410 .setMIFlags(MemI.getFlags());
4411 return B.getInstr();
4412 }
4413
4414 if (AM.Form == ExtAddrMode::Formula::SExtScaledReg ||
4415 AM.Form == ExtAddrMode::Formula::ZExtScaledReg) {
4416 // The new instruction will be in the form `ldr Rt, [Xn, Wm, {s,u}xtw #N]`.
4417 assert(AM.ScaledReg && !AM.Displacement &&
4418 "Address offset can be a register or an immediate, but not both");
4419 unsigned Opcode = offsetExtendOpcode(Opcode: MemI.getOpcode());
4420 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4421 // Make sure the offset register is in the correct register class.
4422 Register OffsetReg = AM.ScaledReg;
4423 const TargetRegisterClass *RC = MRI.getRegClass(Reg: OffsetReg);
4424 if (RC->hasSuperClassEq(RC: &AArch64::GPR64RegClass)) {
4425 OffsetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4426 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: OffsetReg)
4427 .addReg(RegNo: AM.ScaledReg, Flags: {}, SubReg: AArch64::sub_32);
4428 }
4429 auto B =
4430 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4431 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4432 .addReg(RegNo: AM.BaseReg)
4433 .addReg(RegNo: OffsetReg)
4434 .addImm(Val: AM.Form == ExtAddrMode::Formula::SExtScaledReg)
4435 .addImm(Val: AM.Scale != 1)
4436 .setMemRefs(MemI.memoperands())
4437 .setMIFlags(MemI.getFlags());
4438
4439 return B.getInstr();
4440 }
4441
4442 llvm_unreachable(
4443 "Function must not be called with an addressing mode it can't handle");
4444}
4445
4446/// Return true if the opcode is a post-index ld/st instruction, which really
4447/// loads from base+0.
4448static bool isPostIndexLdStOpcode(unsigned Opcode) {
4449 switch (Opcode) {
4450 default:
4451 return false;
4452 case AArch64::LD1Fourv16b_POST:
4453 case AArch64::LD1Fourv1d_POST:
4454 case AArch64::LD1Fourv2d_POST:
4455 case AArch64::LD1Fourv2s_POST:
4456 case AArch64::LD1Fourv4h_POST:
4457 case AArch64::LD1Fourv4s_POST:
4458 case AArch64::LD1Fourv8b_POST:
4459 case AArch64::LD1Fourv8h_POST:
4460 case AArch64::LD1Onev16b_POST:
4461 case AArch64::LD1Onev1d_POST:
4462 case AArch64::LD1Onev2d_POST:
4463 case AArch64::LD1Onev2s_POST:
4464 case AArch64::LD1Onev4h_POST:
4465 case AArch64::LD1Onev4s_POST:
4466 case AArch64::LD1Onev8b_POST:
4467 case AArch64::LD1Onev8h_POST:
4468 case AArch64::LD1Rv16b_POST:
4469 case AArch64::LD1Rv1d_POST:
4470 case AArch64::LD1Rv2d_POST:
4471 case AArch64::LD1Rv2s_POST:
4472 case AArch64::LD1Rv4h_POST:
4473 case AArch64::LD1Rv4s_POST:
4474 case AArch64::LD1Rv8b_POST:
4475 case AArch64::LD1Rv8h_POST:
4476 case AArch64::LD1Threev16b_POST:
4477 case AArch64::LD1Threev1d_POST:
4478 case AArch64::LD1Threev2d_POST:
4479 case AArch64::LD1Threev2s_POST:
4480 case AArch64::LD1Threev4h_POST:
4481 case AArch64::LD1Threev4s_POST:
4482 case AArch64::LD1Threev8b_POST:
4483 case AArch64::LD1Threev8h_POST:
4484 case AArch64::LD1Twov16b_POST:
4485 case AArch64::LD1Twov1d_POST:
4486 case AArch64::LD1Twov2d_POST:
4487 case AArch64::LD1Twov2s_POST:
4488 case AArch64::LD1Twov4h_POST:
4489 case AArch64::LD1Twov4s_POST:
4490 case AArch64::LD1Twov8b_POST:
4491 case AArch64::LD1Twov8h_POST:
4492 case AArch64::LD1i16_POST:
4493 case AArch64::LD1i32_POST:
4494 case AArch64::LD1i64_POST:
4495 case AArch64::LD1i8_POST:
4496 case AArch64::LD2Rv16b_POST:
4497 case AArch64::LD2Rv1d_POST:
4498 case AArch64::LD2Rv2d_POST:
4499 case AArch64::LD2Rv2s_POST:
4500 case AArch64::LD2Rv4h_POST:
4501 case AArch64::LD2Rv4s_POST:
4502 case AArch64::LD2Rv8b_POST:
4503 case AArch64::LD2Rv8h_POST:
4504 case AArch64::LD2Twov16b_POST:
4505 case AArch64::LD2Twov2d_POST:
4506 case AArch64::LD2Twov2s_POST:
4507 case AArch64::LD2Twov4h_POST:
4508 case AArch64::LD2Twov4s_POST:
4509 case AArch64::LD2Twov8b_POST:
4510 case AArch64::LD2Twov8h_POST:
4511 case AArch64::LD2i16_POST:
4512 case AArch64::LD2i32_POST:
4513 case AArch64::LD2i64_POST:
4514 case AArch64::LD2i8_POST:
4515 case AArch64::LD3Rv16b_POST:
4516 case AArch64::LD3Rv1d_POST:
4517 case AArch64::LD3Rv2d_POST:
4518 case AArch64::LD3Rv2s_POST:
4519 case AArch64::LD3Rv4h_POST:
4520 case AArch64::LD3Rv4s_POST:
4521 case AArch64::LD3Rv8b_POST:
4522 case AArch64::LD3Rv8h_POST:
4523 case AArch64::LD3Threev16b_POST:
4524 case AArch64::LD3Threev2d_POST:
4525 case AArch64::LD3Threev2s_POST:
4526 case AArch64::LD3Threev4h_POST:
4527 case AArch64::LD3Threev4s_POST:
4528 case AArch64::LD3Threev8b_POST:
4529 case AArch64::LD3Threev8h_POST:
4530 case AArch64::LD3i16_POST:
4531 case AArch64::LD3i32_POST:
4532 case AArch64::LD3i64_POST:
4533 case AArch64::LD3i8_POST:
4534 case AArch64::LD4Fourv16b_POST:
4535 case AArch64::LD4Fourv2d_POST:
4536 case AArch64::LD4Fourv2s_POST:
4537 case AArch64::LD4Fourv4h_POST:
4538 case AArch64::LD4Fourv4s_POST:
4539 case AArch64::LD4Fourv8b_POST:
4540 case AArch64::LD4Fourv8h_POST:
4541 case AArch64::LD4Rv16b_POST:
4542 case AArch64::LD4Rv1d_POST:
4543 case AArch64::LD4Rv2d_POST:
4544 case AArch64::LD4Rv2s_POST:
4545 case AArch64::LD4Rv4h_POST:
4546 case AArch64::LD4Rv4s_POST:
4547 case AArch64::LD4Rv8b_POST:
4548 case AArch64::LD4Rv8h_POST:
4549 case AArch64::LD4i16_POST:
4550 case AArch64::LD4i32_POST:
4551 case AArch64::LD4i64_POST:
4552 case AArch64::LD4i8_POST:
4553 case AArch64::LDAPRWpost:
4554 case AArch64::LDAPRXpost:
4555 case AArch64::LDIAPPWpost:
4556 case AArch64::LDIAPPXpost:
4557 case AArch64::LDPDpost:
4558 case AArch64::LDPQpost:
4559 case AArch64::LDPSWpost:
4560 case AArch64::LDPSpost:
4561 case AArch64::LDPWpost:
4562 case AArch64::LDPXpost:
4563 case AArch64::LDRBBpost:
4564 case AArch64::LDRBpost:
4565 case AArch64::LDRDpost:
4566 case AArch64::LDRHHpost:
4567 case AArch64::LDRHpost:
4568 case AArch64::LDRQpost:
4569 case AArch64::LDRSBWpost:
4570 case AArch64::LDRSBXpost:
4571 case AArch64::LDRSHWpost:
4572 case AArch64::LDRSHXpost:
4573 case AArch64::LDRSWpost:
4574 case AArch64::LDRSpost:
4575 case AArch64::LDRWpost:
4576 case AArch64::LDRXpost:
4577 case AArch64::ST1Fourv16b_POST:
4578 case AArch64::ST1Fourv1d_POST:
4579 case AArch64::ST1Fourv2d_POST:
4580 case AArch64::ST1Fourv2s_POST:
4581 case AArch64::ST1Fourv4h_POST:
4582 case AArch64::ST1Fourv4s_POST:
4583 case AArch64::ST1Fourv8b_POST:
4584 case AArch64::ST1Fourv8h_POST:
4585 case AArch64::ST1Onev16b_POST:
4586 case AArch64::ST1Onev1d_POST:
4587 case AArch64::ST1Onev2d_POST:
4588 case AArch64::ST1Onev2s_POST:
4589 case AArch64::ST1Onev4h_POST:
4590 case AArch64::ST1Onev4s_POST:
4591 case AArch64::ST1Onev8b_POST:
4592 case AArch64::ST1Onev8h_POST:
4593 case AArch64::ST1Threev16b_POST:
4594 case AArch64::ST1Threev1d_POST:
4595 case AArch64::ST1Threev2d_POST:
4596 case AArch64::ST1Threev2s_POST:
4597 case AArch64::ST1Threev4h_POST:
4598 case AArch64::ST1Threev4s_POST:
4599 case AArch64::ST1Threev8b_POST:
4600 case AArch64::ST1Threev8h_POST:
4601 case AArch64::ST1Twov16b_POST:
4602 case AArch64::ST1Twov1d_POST:
4603 case AArch64::ST1Twov2d_POST:
4604 case AArch64::ST1Twov2s_POST:
4605 case AArch64::ST1Twov4h_POST:
4606 case AArch64::ST1Twov4s_POST:
4607 case AArch64::ST1Twov8b_POST:
4608 case AArch64::ST1Twov8h_POST:
4609 case AArch64::ST1i16_POST:
4610 case AArch64::ST1i32_POST:
4611 case AArch64::ST1i64_POST:
4612 case AArch64::ST1i8_POST:
4613 case AArch64::ST2GPostIndex:
4614 case AArch64::ST2Twov16b_POST:
4615 case AArch64::ST2Twov2d_POST:
4616 case AArch64::ST2Twov2s_POST:
4617 case AArch64::ST2Twov4h_POST:
4618 case AArch64::ST2Twov4s_POST:
4619 case AArch64::ST2Twov8b_POST:
4620 case AArch64::ST2Twov8h_POST:
4621 case AArch64::ST2i16_POST:
4622 case AArch64::ST2i32_POST:
4623 case AArch64::ST2i64_POST:
4624 case AArch64::ST2i8_POST:
4625 case AArch64::ST3Threev16b_POST:
4626 case AArch64::ST3Threev2d_POST:
4627 case AArch64::ST3Threev2s_POST:
4628 case AArch64::ST3Threev4h_POST:
4629 case AArch64::ST3Threev4s_POST:
4630 case AArch64::ST3Threev8b_POST:
4631 case AArch64::ST3Threev8h_POST:
4632 case AArch64::ST3i16_POST:
4633 case AArch64::ST3i32_POST:
4634 case AArch64::ST3i64_POST:
4635 case AArch64::ST3i8_POST:
4636 case AArch64::ST4Fourv16b_POST:
4637 case AArch64::ST4Fourv2d_POST:
4638 case AArch64::ST4Fourv2s_POST:
4639 case AArch64::ST4Fourv4h_POST:
4640 case AArch64::ST4Fourv4s_POST:
4641 case AArch64::ST4Fourv8b_POST:
4642 case AArch64::ST4Fourv8h_POST:
4643 case AArch64::ST4i16_POST:
4644 case AArch64::ST4i32_POST:
4645 case AArch64::ST4i64_POST:
4646 case AArch64::ST4i8_POST:
4647 case AArch64::STGPostIndex:
4648 case AArch64::STGPpost:
4649 case AArch64::STPDpost:
4650 case AArch64::STPQpost:
4651 case AArch64::STPSpost:
4652 case AArch64::STPWpost:
4653 case AArch64::STPXpost:
4654 case AArch64::STRBBpost:
4655 case AArch64::STRBpost:
4656 case AArch64::STRDpost:
4657 case AArch64::STRHHpost:
4658 case AArch64::STRHpost:
4659 case AArch64::STRQpost:
4660 case AArch64::STRSpost:
4661 case AArch64::STRWpost:
4662 case AArch64::STRXpost:
4663 case AArch64::STZ2GPostIndex:
4664 case AArch64::STZGPostIndex:
4665 return true;
4666 }
4667}
4668
4669bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
4670 const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
4671 bool &OffsetIsScalable, TypeSize &Width) const {
4672 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4673 // Handle only loads/stores with base register followed by immediate offset.
4674 if (LdSt.getNumExplicitOperands() == 3) {
4675 // Non-paired instruction (e.g., ldr x1, [x0, #8]).
4676 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
4677 !LdSt.getOperand(i: 2).isImm())
4678 return false;
4679 } else if (LdSt.getNumExplicitOperands() == 4) {
4680 // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
4681 if (!LdSt.getOperand(i: 1).isReg() ||
4682 (!LdSt.getOperand(i: 2).isReg() && !LdSt.getOperand(i: 2).isFI()) ||
4683 !LdSt.getOperand(i: 3).isImm())
4684 return false;
4685 } else
4686 return false;
4687
4688 // Get the scaling factor for the instruction and set the width for the
4689 // instruction.
4690 TypeSize Scale(0U, false);
4691 int64_t Dummy1, Dummy2;
4692
4693 // If this returns false, then it's an instruction we don't want to handle.
4694 if (!getMemOpInfo(Opcode: LdSt.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2))
4695 return false;
4696
4697 // Compute the offset. Offset is calculated as the immediate operand
4698 // multiplied by the scaling factor. Unscaled instructions have scaling factor
4699 // set to 1. Postindex are a special case which have an offset of 0.
4700 if (isPostIndexLdStOpcode(Opcode: LdSt.getOpcode())) {
4701 BaseOp = &LdSt.getOperand(i: 2);
4702 Offset = 0;
4703 } else if (LdSt.getNumExplicitOperands() == 3) {
4704 BaseOp = &LdSt.getOperand(i: 1);
4705 Offset = LdSt.getOperand(i: 2).getImm() * Scale.getKnownMinValue();
4706 } else {
4707 assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
4708 BaseOp = &LdSt.getOperand(i: 2);
4709 Offset = LdSt.getOperand(i: 3).getImm() * Scale.getKnownMinValue();
4710 }
4711 OffsetIsScalable = Scale.isScalable();
4712
4713 return BaseOp->isReg() || BaseOp->isFI();
4714}
4715
4716MachineOperand &
4717AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
4718 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4719 MachineOperand &OfsOp = LdSt.getOperand(i: LdSt.getNumExplicitOperands() - 1);
4720 assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
4721 return OfsOp;
4722}
4723
4724bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
4725 TypeSize &Width, int64_t &MinOffset,
4726 int64_t &MaxOffset) {
4727 switch (Opcode) {
4728 // Not a memory operation or something we want to handle.
4729 default:
4730 Scale = Width = TypeSize::getFixed(ExactSize: 0);
4731 MinOffset = MaxOffset = 0;
4732 return false;
4733 // LDR / STR
4734 case AArch64::LDRQui:
4735 case AArch64::STRQui:
4736 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4737 MinOffset = 0;
4738 MaxOffset = 4095;
4739 break;
4740 case AArch64::LDRXui:
4741 case AArch64::LDRDui:
4742 case AArch64::STRXui:
4743 case AArch64::STRDui:
4744 case AArch64::PRFMui:
4745 case AArch64::ATOMIC_STORE_HINT_Xui:
4746 case AArch64::ATOMIC_STORE_HINT_Dui:
4747 Scale = Width = TypeSize::getFixed(ExactSize: 8);
4748 MinOffset = 0;
4749 MaxOffset = 4095;
4750 break;
4751 case AArch64::LDRWui:
4752 case AArch64::LDRSui:
4753 case AArch64::LDRSWui:
4754 case AArch64::STRWui:
4755 case AArch64::STRSui:
4756 case AArch64::ATOMIC_STORE_HINT_Wui:
4757 case AArch64::ATOMIC_STORE_HINT_Sui:
4758 Scale = Width = TypeSize::getFixed(ExactSize: 4);
4759 MinOffset = 0;
4760 MaxOffset = 4095;
4761 break;
4762 case AArch64::LDRHui:
4763 case AArch64::LDRHHui:
4764 case AArch64::LDRSHWui:
4765 case AArch64::LDRSHXui:
4766 case AArch64::STRHui:
4767 case AArch64::STRHHui:
4768 case AArch64::ATOMIC_STORE_HINT_Hui:
4769 Scale = Width = TypeSize::getFixed(ExactSize: 2);
4770 MinOffset = 0;
4771 MaxOffset = 4095;
4772 break;
4773 case AArch64::LDRBui:
4774 case AArch64::LDRBBui:
4775 case AArch64::LDRSBWui:
4776 case AArch64::LDRSBXui:
4777 case AArch64::STRBui:
4778 case AArch64::STRBBui:
4779 case AArch64::ATOMIC_STORE_HINT_Bui:
4780 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4781 MinOffset = 0;
4782 MaxOffset = 4095;
4783 break;
4784 // post/pre inc
4785 case AArch64::STRQpre:
4786 case AArch64::LDRQpost:
4787 Scale = TypeSize::getFixed(ExactSize: 1);
4788 Width = TypeSize::getFixed(ExactSize: 16);
4789 MinOffset = -256;
4790 MaxOffset = 255;
4791 break;
4792 case AArch64::LDRDpost:
4793 case AArch64::LDRDpre:
4794 case AArch64::LDRXpost:
4795 case AArch64::LDRXpre:
4796 case AArch64::STRDpost:
4797 case AArch64::STRDpre:
4798 case AArch64::STRXpost:
4799 case AArch64::STRXpre:
4800 Scale = TypeSize::getFixed(ExactSize: 1);
4801 Width = TypeSize::getFixed(ExactSize: 8);
4802 MinOffset = -256;
4803 MaxOffset = 255;
4804 break;
4805 case AArch64::STRWpost:
4806 case AArch64::STRWpre:
4807 case AArch64::LDRWpost:
4808 case AArch64::LDRWpre:
4809 case AArch64::STRSpost:
4810 case AArch64::STRSpre:
4811 case AArch64::LDRSpost:
4812 case AArch64::LDRSpre:
4813 Scale = TypeSize::getFixed(ExactSize: 1);
4814 Width = TypeSize::getFixed(ExactSize: 4);
4815 MinOffset = -256;
4816 MaxOffset = 255;
4817 break;
4818 case AArch64::LDRHpost:
4819 case AArch64::LDRHpre:
4820 case AArch64::STRHpost:
4821 case AArch64::STRHpre:
4822 case AArch64::LDRHHpost:
4823 case AArch64::LDRHHpre:
4824 case AArch64::STRHHpost:
4825 case AArch64::STRHHpre:
4826 Scale = TypeSize::getFixed(ExactSize: 1);
4827 Width = TypeSize::getFixed(ExactSize: 2);
4828 MinOffset = -256;
4829 MaxOffset = 255;
4830 break;
4831 case AArch64::LDRBpost:
4832 case AArch64::LDRBpre:
4833 case AArch64::STRBpost:
4834 case AArch64::STRBpre:
4835 case AArch64::LDRBBpost:
4836 case AArch64::LDRBBpre:
4837 case AArch64::STRBBpost:
4838 case AArch64::STRBBpre:
4839 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4840 MinOffset = -256;
4841 MaxOffset = 255;
4842 break;
4843 // Unscaled
4844 case AArch64::LDURQi:
4845 case AArch64::STURQi:
4846 Scale = TypeSize::getFixed(ExactSize: 1);
4847 Width = TypeSize::getFixed(ExactSize: 16);
4848 MinOffset = -256;
4849 MaxOffset = 255;
4850 break;
4851 case AArch64::LDURXi:
4852 case AArch64::LDURDi:
4853 case AArch64::LDAPURXi:
4854 case AArch64::STURXi:
4855 case AArch64::STURDi:
4856 case AArch64::STLURXi:
4857 case AArch64::PRFUMi:
4858 case AArch64::ATOMIC_STORE_HINT_Xi:
4859 case AArch64::ATOMIC_STORE_HINT_Di:
4860 Scale = TypeSize::getFixed(ExactSize: 1);
4861 Width = TypeSize::getFixed(ExactSize: 8);
4862 MinOffset = -256;
4863 MaxOffset = 255;
4864 break;
4865 case AArch64::LDURWi:
4866 case AArch64::LDURSi:
4867 case AArch64::LDURSWi:
4868 case AArch64::LDAPURi:
4869 case AArch64::LDAPURSWi:
4870 case AArch64::STURWi:
4871 case AArch64::STURSi:
4872 case AArch64::STLURWi:
4873 case AArch64::ATOMIC_STORE_HINT_Wi:
4874 case AArch64::ATOMIC_STORE_HINT_Si:
4875 Scale = TypeSize::getFixed(ExactSize: 1);
4876 Width = TypeSize::getFixed(ExactSize: 4);
4877 MinOffset = -256;
4878 MaxOffset = 255;
4879 break;
4880 case AArch64::LDURHi:
4881 case AArch64::LDURHHi:
4882 case AArch64::LDURSHXi:
4883 case AArch64::LDURSHWi:
4884 case AArch64::LDAPURHi:
4885 case AArch64::LDAPURSHWi:
4886 case AArch64::LDAPURSHXi:
4887 case AArch64::STURHi:
4888 case AArch64::STURHHi:
4889 case AArch64::STLURHi:
4890 case AArch64::ATOMIC_STORE_HINT_Hi:
4891 Scale = TypeSize::getFixed(ExactSize: 1);
4892 Width = TypeSize::getFixed(ExactSize: 2);
4893 MinOffset = -256;
4894 MaxOffset = 255;
4895 break;
4896 case AArch64::LDURBi:
4897 case AArch64::LDURBBi:
4898 case AArch64::LDURSBXi:
4899 case AArch64::LDURSBWi:
4900 case AArch64::LDAPURBi:
4901 case AArch64::LDAPURSBWi:
4902 case AArch64::LDAPURSBXi:
4903 case AArch64::STURBi:
4904 case AArch64::STURBBi:
4905 case AArch64::STLURBi:
4906 case AArch64::ATOMIC_STORE_HINT_Bi:
4907 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4908 MinOffset = -256;
4909 MaxOffset = 255;
4910 break;
4911 // LDP / STP (including pre/post inc)
4912 case AArch64::LDPQi:
4913 case AArch64::LDNPQi:
4914 case AArch64::STPQi:
4915 case AArch64::STNPQi:
4916 case AArch64::LDPQpost:
4917 case AArch64::LDPQpre:
4918 case AArch64::STPQpost:
4919 case AArch64::STPQpre:
4920 Scale = TypeSize::getFixed(ExactSize: 16);
4921 Width = TypeSize::getFixed(ExactSize: 16 * 2);
4922 MinOffset = -64;
4923 MaxOffset = 63;
4924 break;
4925 case AArch64::LDPXi:
4926 case AArch64::LDPDi:
4927 case AArch64::LDNPXi:
4928 case AArch64::LDNPDi:
4929 case AArch64::STPXi:
4930 case AArch64::STPDi:
4931 case AArch64::STNPXi:
4932 case AArch64::STNPDi:
4933 case AArch64::LDPDpost:
4934 case AArch64::LDPDpre:
4935 case AArch64::LDPXpost:
4936 case AArch64::LDPXpre:
4937 case AArch64::STPDpost:
4938 case AArch64::STPDpre:
4939 case AArch64::STPXpost:
4940 case AArch64::STPXpre:
4941 Scale = TypeSize::getFixed(ExactSize: 8);
4942 Width = TypeSize::getFixed(ExactSize: 8 * 2);
4943 MinOffset = -64;
4944 MaxOffset = 63;
4945 break;
4946 case AArch64::LDPWi:
4947 case AArch64::LDPSi:
4948 case AArch64::LDNPWi:
4949 case AArch64::LDNPSi:
4950 case AArch64::STPWi:
4951 case AArch64::STPSi:
4952 case AArch64::STNPWi:
4953 case AArch64::STNPSi:
4954 case AArch64::LDPSpost:
4955 case AArch64::LDPSpre:
4956 case AArch64::LDPWpost:
4957 case AArch64::LDPWpre:
4958 case AArch64::STPSpost:
4959 case AArch64::STPSpre:
4960 case AArch64::STPWpost:
4961 case AArch64::STPWpre:
4962 Scale = TypeSize::getFixed(ExactSize: 4);
4963 Width = TypeSize::getFixed(ExactSize: 4 * 2);
4964 MinOffset = -64;
4965 MaxOffset = 63;
4966 break;
4967 case AArch64::StoreSwiftAsyncContext:
4968 // Store is an STRXui, but there might be an ADDXri in the expansion too.
4969 Scale = TypeSize::getFixed(ExactSize: 1);
4970 Width = TypeSize::getFixed(ExactSize: 8);
4971 MinOffset = 0;
4972 MaxOffset = 4095;
4973 break;
4974 case AArch64::ADDG:
4975 Scale = TypeSize::getFixed(ExactSize: 16);
4976 Width = TypeSize::getFixed(ExactSize: 0);
4977 MinOffset = 0;
4978 MaxOffset = 63;
4979 break;
4980 case AArch64::TAGPstack:
4981 Scale = TypeSize::getFixed(ExactSize: 16);
4982 Width = TypeSize::getFixed(ExactSize: 0);
4983 // TAGP with a negative offset turns into SUBP, which has a maximum offset
4984 // of 63 (not 64!).
4985 MinOffset = -63;
4986 MaxOffset = 63;
4987 break;
4988 case AArch64::LDG:
4989 case AArch64::STGi:
4990 case AArch64::STGPreIndex:
4991 case AArch64::STGPostIndex:
4992 case AArch64::STZGi:
4993 case AArch64::STZGPreIndex:
4994 case AArch64::STZGPostIndex:
4995 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4996 MinOffset = -256;
4997 MaxOffset = 255;
4998 break;
4999 // SVE
5000 case AArch64::STR_ZZZZXI:
5001 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
5002 case AArch64::LDR_ZZZZXI:
5003 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
5004 Scale = TypeSize::getScalable(MinimumSize: 16);
5005 Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5006 MinOffset = -256;
5007 MaxOffset = 252;
5008 break;
5009 case AArch64::STR_ZZZXI:
5010 case AArch64::LDR_ZZZXI:
5011 Scale = TypeSize::getScalable(MinimumSize: 16);
5012 Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5013 MinOffset = -256;
5014 MaxOffset = 253;
5015 break;
5016 case AArch64::STR_ZZXI:
5017 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
5018 case AArch64::LDR_ZZXI:
5019 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
5020 Scale = TypeSize::getScalable(MinimumSize: 16);
5021 Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5022 MinOffset = -256;
5023 MaxOffset = 254;
5024 break;
5025 case AArch64::LDR_PXI:
5026 case AArch64::STR_PXI:
5027 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5028 MinOffset = -256;
5029 MaxOffset = 255;
5030 break;
5031 case AArch64::LDR_PPXI:
5032 case AArch64::STR_PPXI:
5033 Scale = TypeSize::getScalable(MinimumSize: 2);
5034 Width = TypeSize::getScalable(MinimumSize: 2 * 2);
5035 MinOffset = -256;
5036 MaxOffset = 254;
5037 break;
5038 case AArch64::LDR_ZXI:
5039 case AArch64::STR_ZXI:
5040 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5041 MinOffset = -256;
5042 MaxOffset = 255;
5043 break;
5044 case AArch64::LD1B_IMM:
5045 case AArch64::LD1H_IMM:
5046 case AArch64::LD1W_IMM:
5047 case AArch64::LD1D_IMM:
5048 case AArch64::LDNT1B_ZRI:
5049 case AArch64::LDNT1H_ZRI:
5050 case AArch64::LDNT1W_ZRI:
5051 case AArch64::LDNT1D_ZRI:
5052 case AArch64::ST1B_IMM:
5053 case AArch64::ST1H_IMM:
5054 case AArch64::ST1W_IMM:
5055 case AArch64::ST1D_IMM:
5056 case AArch64::STNT1B_ZRI:
5057 case AArch64::STNT1H_ZRI:
5058 case AArch64::STNT1W_ZRI:
5059 case AArch64::STNT1D_ZRI:
5060 case AArch64::LDNF1B_IMM:
5061 case AArch64::LDNF1H_IMM:
5062 case AArch64::LDNF1W_IMM:
5063 case AArch64::LDNF1D_IMM:
5064 // A full vectors worth of data
5065 // Width = mbytes * elements
5066 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5067 MinOffset = -8;
5068 MaxOffset = 7;
5069 break;
5070 case AArch64::LD2B_IMM:
5071 case AArch64::LD2H_IMM:
5072 case AArch64::LD2W_IMM:
5073 case AArch64::LD2D_IMM:
5074 case AArch64::ST2B_IMM:
5075 case AArch64::ST2H_IMM:
5076 case AArch64::ST2W_IMM:
5077 case AArch64::ST2D_IMM:
5078 case AArch64::LD1B_2Z_IMM:
5079 case AArch64::LD1B_2Z_STRIDED_IMM:
5080 case AArch64::LD1H_2Z_IMM:
5081 case AArch64::LD1H_2Z_STRIDED_IMM:
5082 case AArch64::LD1W_2Z_IMM:
5083 case AArch64::LD1W_2Z_STRIDED_IMM:
5084 case AArch64::LD1D_2Z_IMM:
5085 case AArch64::LD1D_2Z_STRIDED_IMM:
5086 case AArch64::LD1B_2Z_IMM_PSEUDO:
5087 case AArch64::LD1H_2Z_IMM_PSEUDO:
5088 case AArch64::LD1W_2Z_IMM_PSEUDO:
5089 case AArch64::LD1D_2Z_IMM_PSEUDO:
5090 case AArch64::ST1B_2Z_IMM:
5091 case AArch64::ST1B_2Z_STRIDED_IMM:
5092 case AArch64::ST1H_2Z_IMM:
5093 case AArch64::ST1H_2Z_STRIDED_IMM:
5094 case AArch64::ST1W_2Z_IMM:
5095 case AArch64::ST1W_2Z_STRIDED_IMM:
5096 case AArch64::ST1D_2Z_IMM:
5097 case AArch64::ST1D_2Z_STRIDED_IMM:
5098 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
5099 case AArch64::LDNT1B_2Z_IMM:
5100 case AArch64::LDNT1B_2Z_STRIDED_IMM:
5101 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
5102 case AArch64::LDNT1H_2Z_IMM:
5103 case AArch64::LDNT1H_2Z_STRIDED_IMM:
5104 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
5105 case AArch64::LDNT1W_2Z_IMM:
5106 case AArch64::LDNT1W_2Z_STRIDED_IMM:
5107 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
5108 case AArch64::LDNT1D_2Z_IMM:
5109 case AArch64::LDNT1D_2Z_STRIDED_IMM:
5110 case AArch64::STNT1B_2Z_IMM:
5111 case AArch64::STNT1B_2Z_STRIDED_IMM:
5112 case AArch64::STNT1H_2Z_IMM:
5113 case AArch64::STNT1H_2Z_STRIDED_IMM:
5114 case AArch64::STNT1W_2Z_IMM:
5115 case AArch64::STNT1W_2Z_STRIDED_IMM:
5116 case AArch64::STNT1D_2Z_IMM:
5117 case AArch64::STNT1D_2Z_STRIDED_IMM:
5118 case AArch64::ST1B_2Z_IMM_PSEUDO:
5119 case AArch64::ST1H_2Z_IMM_PSEUDO:
5120 case AArch64::ST1W_2Z_IMM_PSEUDO:
5121 case AArch64::ST1D_2Z_IMM_PSEUDO:
5122 case AArch64::STNT1B_2Z_IMM_PSEUDO:
5123 case AArch64::STNT1H_2Z_IMM_PSEUDO:
5124 case AArch64::STNT1W_2Z_IMM_PSEUDO:
5125 case AArch64::STNT1D_2Z_IMM_PSEUDO:
5126 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5127 MinOffset = -8;
5128 MaxOffset = 7;
5129 break;
5130 case AArch64::LD3B_IMM:
5131 case AArch64::LD3H_IMM:
5132 case AArch64::LD3W_IMM:
5133 case AArch64::LD3D_IMM:
5134 case AArch64::ST3B_IMM:
5135 case AArch64::ST3H_IMM:
5136 case AArch64::ST3W_IMM:
5137 case AArch64::ST3D_IMM:
5138 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5139 MinOffset = -8;
5140 MaxOffset = 7;
5141 break;
5142 case AArch64::LD4B_IMM:
5143 case AArch64::LD4H_IMM:
5144 case AArch64::LD4W_IMM:
5145 case AArch64::LD4D_IMM:
5146 case AArch64::ST4B_IMM:
5147 case AArch64::ST4H_IMM:
5148 case AArch64::ST4W_IMM:
5149 case AArch64::ST4D_IMM:
5150 case AArch64::LD1B_4Z_IMM:
5151 case AArch64::LD1B_4Z_STRIDED_IMM:
5152 case AArch64::LD1H_4Z_IMM:
5153 case AArch64::LD1H_4Z_STRIDED_IMM:
5154 case AArch64::LD1W_4Z_IMM:
5155 case AArch64::LD1W_4Z_STRIDED_IMM:
5156 case AArch64::LD1D_4Z_IMM:
5157 case AArch64::LD1D_4Z_STRIDED_IMM:
5158 case AArch64::LD1B_4Z_IMM_PSEUDO:
5159 case AArch64::LD1H_4Z_IMM_PSEUDO:
5160 case AArch64::LD1W_4Z_IMM_PSEUDO:
5161 case AArch64::LD1D_4Z_IMM_PSEUDO:
5162 case AArch64::ST1B_4Z_IMM:
5163 case AArch64::ST1B_4Z_STRIDED_IMM:
5164 case AArch64::ST1H_4Z_IMM:
5165 case AArch64::ST1H_4Z_STRIDED_IMM:
5166 case AArch64::ST1W_4Z_IMM:
5167 case AArch64::ST1W_4Z_STRIDED_IMM:
5168 case AArch64::ST1D_4Z_IMM:
5169 case AArch64::ST1D_4Z_STRIDED_IMM:
5170 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
5171 case AArch64::LDNT1B_4Z_IMM:
5172 case AArch64::LDNT1B_4Z_STRIDED_IMM:
5173 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
5174 case AArch64::LDNT1H_4Z_IMM:
5175 case AArch64::LDNT1H_4Z_STRIDED_IMM:
5176 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
5177 case AArch64::LDNT1W_4Z_IMM:
5178 case AArch64::LDNT1W_4Z_STRIDED_IMM:
5179 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
5180 case AArch64::LDNT1D_4Z_IMM:
5181 case AArch64::LDNT1D_4Z_STRIDED_IMM:
5182 case AArch64::STNT1B_4Z_IMM:
5183 case AArch64::STNT1B_4Z_STRIDED_IMM:
5184 case AArch64::STNT1H_4Z_IMM:
5185 case AArch64::STNT1H_4Z_STRIDED_IMM:
5186 case AArch64::STNT1W_4Z_IMM:
5187 case AArch64::STNT1W_4Z_STRIDED_IMM:
5188 case AArch64::STNT1D_4Z_IMM:
5189 case AArch64::STNT1D_4Z_STRIDED_IMM:
5190 case AArch64::ST1B_4Z_IMM_PSEUDO:
5191 case AArch64::ST1H_4Z_IMM_PSEUDO:
5192 case AArch64::ST1W_4Z_IMM_PSEUDO:
5193 case AArch64::ST1D_4Z_IMM_PSEUDO:
5194 case AArch64::STNT1B_4Z_IMM_PSEUDO:
5195 case AArch64::STNT1H_4Z_IMM_PSEUDO:
5196 case AArch64::STNT1W_4Z_IMM_PSEUDO:
5197 case AArch64::STNT1D_4Z_IMM_PSEUDO:
5198 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5199 MinOffset = -8;
5200 MaxOffset = 7;
5201 break;
5202 case AArch64::LD1B_H_IMM:
5203 case AArch64::LD1SB_H_IMM:
5204 case AArch64::LD1H_S_IMM:
5205 case AArch64::LD1SH_S_IMM:
5206 case AArch64::LD1W_D_IMM:
5207 case AArch64::LD1SW_D_IMM:
5208 case AArch64::ST1B_H_IMM:
5209 case AArch64::ST1H_S_IMM:
5210 case AArch64::ST1W_D_IMM:
5211 case AArch64::LDNF1B_H_IMM:
5212 case AArch64::LDNF1SB_H_IMM:
5213 case AArch64::LDNF1H_S_IMM:
5214 case AArch64::LDNF1SH_S_IMM:
5215 case AArch64::LDNF1W_D_IMM:
5216 case AArch64::LDNF1SW_D_IMM:
5217 // A half vector worth of data
5218 // Width = mbytes * elements
5219 Scale = Width = TypeSize::getScalable(MinimumSize: 8);
5220 MinOffset = -8;
5221 MaxOffset = 7;
5222 break;
5223 case AArch64::LD1B_S_IMM:
5224 case AArch64::LD1SB_S_IMM:
5225 case AArch64::LD1H_D_IMM:
5226 case AArch64::LD1SH_D_IMM:
5227 case AArch64::ST1B_S_IMM:
5228 case AArch64::ST1H_D_IMM:
5229 case AArch64::LDNF1B_S_IMM:
5230 case AArch64::LDNF1SB_S_IMM:
5231 case AArch64::LDNF1H_D_IMM:
5232 case AArch64::LDNF1SH_D_IMM:
5233 // A quarter vector worth of data
5234 // Width = mbytes * elements
5235 Scale = Width = TypeSize::getScalable(MinimumSize: 4);
5236 MinOffset = -8;
5237 MaxOffset = 7;
5238 break;
5239 case AArch64::LD1B_D_IMM:
5240 case AArch64::LD1SB_D_IMM:
5241 case AArch64::ST1B_D_IMM:
5242 case AArch64::LDNF1B_D_IMM:
5243 case AArch64::LDNF1SB_D_IMM:
5244 // A eighth vector worth of data
5245 // Width = mbytes * elements
5246 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5247 MinOffset = -8;
5248 MaxOffset = 7;
5249 break;
5250 case AArch64::ST2Gi:
5251 case AArch64::ST2GPreIndex:
5252 case AArch64::ST2GPostIndex:
5253 case AArch64::STZ2Gi:
5254 case AArch64::STZ2GPreIndex:
5255 case AArch64::STZ2GPostIndex:
5256 Scale = TypeSize::getFixed(ExactSize: 16);
5257 Width = TypeSize::getFixed(ExactSize: 32);
5258 MinOffset = -256;
5259 MaxOffset = 255;
5260 break;
5261 case AArch64::STGPi:
5262 case AArch64::STGPpost:
5263 case AArch64::STGPpre:
5264 Scale = Width = TypeSize::getFixed(ExactSize: 16);
5265 MinOffset = -64;
5266 MaxOffset = 63;
5267 break;
5268 case AArch64::LD1RB_IMM:
5269 case AArch64::LD1RB_H_IMM:
5270 case AArch64::LD1RB_S_IMM:
5271 case AArch64::LD1RB_D_IMM:
5272 case AArch64::LD1RSB_H_IMM:
5273 case AArch64::LD1RSB_S_IMM:
5274 case AArch64::LD1RSB_D_IMM:
5275 Scale = Width = TypeSize::getFixed(ExactSize: 1);
5276 MinOffset = 0;
5277 MaxOffset = 63;
5278 break;
5279 case AArch64::LD1RH_IMM:
5280 case AArch64::LD1RH_S_IMM:
5281 case AArch64::LD1RH_D_IMM:
5282 case AArch64::LD1RSH_S_IMM:
5283 case AArch64::LD1RSH_D_IMM:
5284 Scale = Width = TypeSize::getFixed(ExactSize: 2);
5285 MinOffset = 0;
5286 MaxOffset = 63;
5287 break;
5288 case AArch64::LD1RW_IMM:
5289 case AArch64::LD1RW_D_IMM:
5290 case AArch64::LD1RSW_IMM:
5291 Scale = Width = TypeSize::getFixed(ExactSize: 4);
5292 MinOffset = 0;
5293 MaxOffset = 63;
5294 break;
5295 case AArch64::LD1RD_IMM:
5296 Scale = Width = TypeSize::getFixed(ExactSize: 8);
5297 MinOffset = 0;
5298 MaxOffset = 63;
5299 break;
5300 }
5301
5302 return true;
5303}
5304
5305// Scaling factor for unscaled load or store.
5306int AArch64InstrInfo::getMemScale(unsigned Opc) {
5307 switch (Opc) {
5308 default:
5309 llvm_unreachable("Opcode has unknown scale!");
5310 case AArch64::LDRBui:
5311 case AArch64::LDRBBui:
5312 case AArch64::LDURBBi:
5313 case AArch64::LDRSBWui:
5314 case AArch64::LDURSBWi:
5315 case AArch64::STRBui:
5316 case AArch64::STRBBui:
5317 case AArch64::STURBBi:
5318 return 1;
5319 case AArch64::LDRHui:
5320 case AArch64::LDRHHui:
5321 case AArch64::LDURHHi:
5322 case AArch64::LDRSHWui:
5323 case AArch64::LDURSHWi:
5324 case AArch64::STRHui:
5325 case AArch64::STRHHui:
5326 case AArch64::STURHHi:
5327 return 2;
5328 case AArch64::LDRSui:
5329 case AArch64::LDURSi:
5330 case AArch64::LDRSpre:
5331 case AArch64::LDRSWui:
5332 case AArch64::LDURSWi:
5333 case AArch64::LDRSWpre:
5334 case AArch64::LDRWpre:
5335 case AArch64::LDRWui:
5336 case AArch64::LDURWi:
5337 case AArch64::STRSui:
5338 case AArch64::STURSi:
5339 case AArch64::STRSpre:
5340 case AArch64::STRWui:
5341 case AArch64::STURWi:
5342 case AArch64::STRWpre:
5343 case AArch64::LDPSi:
5344 case AArch64::LDPSWi:
5345 case AArch64::LDPWi:
5346 case AArch64::STPSi:
5347 case AArch64::STPWi:
5348 return 4;
5349 case AArch64::LDRDui:
5350 case AArch64::LDURDi:
5351 case AArch64::LDRDpre:
5352 case AArch64::LDRXui:
5353 case AArch64::LDURXi:
5354 case AArch64::LDRXpre:
5355 case AArch64::STRDui:
5356 case AArch64::STURDi:
5357 case AArch64::STRDpre:
5358 case AArch64::STRXui:
5359 case AArch64::STURXi:
5360 case AArch64::STRXpre:
5361 case AArch64::LDPDi:
5362 case AArch64::LDPXi:
5363 case AArch64::STPDi:
5364 case AArch64::STPXi:
5365 return 8;
5366 case AArch64::LDRQui:
5367 case AArch64::LDURQi:
5368 case AArch64::STRQui:
5369 case AArch64::STURQi:
5370 case AArch64::STRQpre:
5371 case AArch64::LDPQi:
5372 case AArch64::LDRQpre:
5373 case AArch64::STPQi:
5374 case AArch64::STGi:
5375 case AArch64::STZGi:
5376 case AArch64::ST2Gi:
5377 case AArch64::STZ2Gi:
5378 case AArch64::STGPi:
5379 return 16;
5380 }
5381}
5382
5383bool AArch64InstrInfo::isPreLd(const MachineInstr &MI) {
5384 switch (MI.getOpcode()) {
5385 default:
5386 return false;
5387 case AArch64::LDRWpre:
5388 case AArch64::LDRXpre:
5389 case AArch64::LDRSWpre:
5390 case AArch64::LDRSpre:
5391 case AArch64::LDRDpre:
5392 case AArch64::LDRQpre:
5393 return true;
5394 }
5395}
5396
5397bool AArch64InstrInfo::isPreSt(const MachineInstr &MI) {
5398 switch (MI.getOpcode()) {
5399 default:
5400 return false;
5401 case AArch64::STRWpre:
5402 case AArch64::STRXpre:
5403 case AArch64::STRSpre:
5404 case AArch64::STRDpre:
5405 case AArch64::STRQpre:
5406 return true;
5407 }
5408}
5409
5410bool AArch64InstrInfo::isPreLdSt(const MachineInstr &MI) {
5411 return isPreLd(MI) || isPreSt(MI);
5412}
5413
5414bool AArch64InstrInfo::isZExtLoad(const MachineInstr &MI) {
5415 switch (MI.getOpcode()) {
5416 default:
5417 return false;
5418 case AArch64::LDURBBi:
5419 case AArch64::LDURHHi:
5420 case AArch64::LDURWi:
5421 case AArch64::LDRBBui:
5422 case AArch64::LDRHHui:
5423 case AArch64::LDRWui:
5424 case AArch64::LDRBBroX:
5425 case AArch64::LDRHHroX:
5426 case AArch64::LDRWroX:
5427 case AArch64::LDRBBroW:
5428 case AArch64::LDRHHroW:
5429 case AArch64::LDRWroW:
5430 return true;
5431 }
5432}
5433
5434bool AArch64InstrInfo::isSExtLoad(const MachineInstr &MI) {
5435 switch (MI.getOpcode()) {
5436 default:
5437 return false;
5438 case AArch64::LDURSBWi:
5439 case AArch64::LDURSHWi:
5440 case AArch64::LDURSBXi:
5441 case AArch64::LDURSHXi:
5442 case AArch64::LDURSWi:
5443 case AArch64::LDRSBWui:
5444 case AArch64::LDRSHWui:
5445 case AArch64::LDRSBXui:
5446 case AArch64::LDRSHXui:
5447 case AArch64::LDRSWui:
5448 case AArch64::LDRSBWroX:
5449 case AArch64::LDRSHWroX:
5450 case AArch64::LDRSBXroX:
5451 case AArch64::LDRSHXroX:
5452 case AArch64::LDRSWroX:
5453 case AArch64::LDRSBWroW:
5454 case AArch64::LDRSHWroW:
5455 case AArch64::LDRSBXroW:
5456 case AArch64::LDRSHXroW:
5457 case AArch64::LDRSWroW:
5458 return true;
5459 }
5460}
5461
5462bool AArch64InstrInfo::isPairedLdSt(const MachineInstr &MI) {
5463 switch (MI.getOpcode()) {
5464 default:
5465 return false;
5466 case AArch64::LDPSi:
5467 case AArch64::LDPSWi:
5468 case AArch64::LDPDi:
5469 case AArch64::LDPQi:
5470 case AArch64::LDPWi:
5471 case AArch64::LDPXi:
5472 case AArch64::STPSi:
5473 case AArch64::STPDi:
5474 case AArch64::STPQi:
5475 case AArch64::STPWi:
5476 case AArch64::STPXi:
5477 case AArch64::STGPi:
5478 return true;
5479 }
5480}
5481
5482const MachineOperand &AArch64InstrInfo::getLdStBaseOp(const MachineInstr &MI) {
5483 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5484 unsigned Idx =
5485 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 2
5486 : 1;
5487 return MI.getOperand(i: Idx);
5488}
5489
5490const MachineOperand &
5491AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
5492 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5493 unsigned Idx =
5494 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 3
5495 : 2;
5496 return MI.getOperand(i: Idx);
5497}
5498
5499const MachineOperand &
5500AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
5501 switch (MI.getOpcode()) {
5502 default:
5503 llvm_unreachable("Unexpected opcode");
5504 case AArch64::LDRBroX:
5505 case AArch64::LDRBBroX:
5506 case AArch64::LDRSBXroX:
5507 case AArch64::LDRSBWroX:
5508 case AArch64::LDRHroX:
5509 case AArch64::LDRHHroX:
5510 case AArch64::LDRSHXroX:
5511 case AArch64::LDRSHWroX:
5512 case AArch64::LDRWroX:
5513 case AArch64::LDRSroX:
5514 case AArch64::LDRSWroX:
5515 case AArch64::LDRDroX:
5516 case AArch64::LDRXroX:
5517 case AArch64::LDRQroX:
5518 return MI.getOperand(i: 4);
5519 }
5520}
5521
5522static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
5523 Register Reg) {
5524 if (MI.getParent() == nullptr)
5525 return nullptr;
5526 const MachineFunction *MF = MI.getParent()->getParent();
5527 return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
5528}
5529
5530bool AArch64InstrInfo::isHForm(const MachineInstr &MI) {
5531 auto IsHFPR = [&](const MachineOperand &Op) {
5532 if (!Op.isReg())
5533 return false;
5534 auto Reg = Op.getReg();
5535 if (Reg.isPhysical())
5536 return AArch64::FPR16RegClass.contains(Reg);
5537 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5538 return TRC == &AArch64::FPR16RegClass ||
5539 TRC == &AArch64::FPR16_loRegClass;
5540 };
5541 return llvm::any_of(Range: MI.operands(), P: IsHFPR);
5542}
5543
5544bool AArch64InstrInfo::isQForm(const MachineInstr &MI) {
5545 auto IsQFPR = [&](const MachineOperand &Op) {
5546 if (!Op.isReg())
5547 return false;
5548 auto Reg = Op.getReg();
5549 if (Reg.isPhysical())
5550 return AArch64::FPR128RegClass.contains(Reg);
5551 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5552 return TRC == &AArch64::FPR128RegClass ||
5553 TRC == &AArch64::FPR128_loRegClass;
5554 };
5555 return llvm::any_of(Range: MI.operands(), P: IsQFPR);
5556}
5557
5558bool AArch64InstrInfo::hasBTISemantics(const MachineInstr &MI) {
5559 switch (MI.getOpcode()) {
5560 case AArch64::BRK:
5561 case AArch64::HLT:
5562 case AArch64::PACIASP:
5563 case AArch64::PACIBSP:
5564 // Implicit BTI behavior.
5565 return true;
5566 case AArch64::PAUTH_PROLOGUE:
5567 // PAUTH_PROLOGUE expands to PACI(A|B)SP.
5568 return true;
5569 case AArch64::HINT: {
5570 unsigned Imm = MI.getOperand(i: 0).getImm();
5571 // Explicit BTI instruction.
5572 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
5573 return true;
5574 // PACI(A|B)SP instructions.
5575 if (Imm == 25 || Imm == 27)
5576 return true;
5577 return false;
5578 }
5579 default:
5580 return false;
5581 }
5582}
5583
5584bool AArch64InstrInfo::isFpOrNEON(Register Reg) {
5585 if (Reg == 0)
5586 return false;
5587 assert(Reg.isPhysical() && "Expected physical register in isFpOrNEON");
5588 return AArch64::FPR128RegClass.contains(Reg) ||
5589 AArch64::FPR64RegClass.contains(Reg) ||
5590 AArch64::FPR32RegClass.contains(Reg) ||
5591 AArch64::FPR16RegClass.contains(Reg) ||
5592 AArch64::FPR8RegClass.contains(Reg);
5593}
5594
5595bool AArch64InstrInfo::isFpOrNEON(const MachineInstr &MI) {
5596 auto IsFPR = [&](const MachineOperand &Op) {
5597 if (!Op.isReg())
5598 return false;
5599 auto Reg = Op.getReg();
5600 if (Reg.isPhysical())
5601 return isFpOrNEON(Reg);
5602
5603 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5604 return TRC == &AArch64::FPR128RegClass ||
5605 TRC == &AArch64::FPR128_loRegClass ||
5606 TRC == &AArch64::FPR64RegClass ||
5607 TRC == &AArch64::FPR64_loRegClass ||
5608 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
5609 TRC == &AArch64::FPR8RegClass;
5610 };
5611 return llvm::any_of(Range: MI.operands(), P: IsFPR);
5612}
5613
5614// Scale the unscaled offsets. Returns false if the unscaled offset can't be
5615// scaled.
5616static bool scaleOffset(unsigned Opc, int64_t &Offset) {
5617 int Scale = AArch64InstrInfo::getMemScale(Opc);
5618
5619 // If the byte-offset isn't a multiple of the stride, we can't scale this
5620 // offset.
5621 if (Offset % Scale != 0)
5622 return false;
5623
5624 // Convert the byte-offset used by unscaled into an "element" offset used
5625 // by the scaled pair load/store instructions.
5626 Offset /= Scale;
5627 return true;
5628}
5629
5630static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
5631 if (FirstOpc == SecondOpc)
5632 return true;
5633 // We can also pair sign-ext and zero-ext instructions.
5634 switch (FirstOpc) {
5635 default:
5636 return false;
5637 case AArch64::STRSui:
5638 case AArch64::STURSi:
5639 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
5640 case AArch64::STRDui:
5641 case AArch64::STURDi:
5642 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
5643 case AArch64::STRQui:
5644 case AArch64::STURQi:
5645 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
5646 case AArch64::STRWui:
5647 case AArch64::STURWi:
5648 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
5649 case AArch64::STRXui:
5650 case AArch64::STURXi:
5651 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
5652 case AArch64::LDRSui:
5653 case AArch64::LDURSi:
5654 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
5655 case AArch64::LDRDui:
5656 case AArch64::LDURDi:
5657 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
5658 case AArch64::LDRQui:
5659 case AArch64::LDURQi:
5660 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
5661 case AArch64::LDRWui:
5662 case AArch64::LDURWi:
5663 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
5664 case AArch64::LDRSWui:
5665 case AArch64::LDURSWi:
5666 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
5667 case AArch64::LDRXui:
5668 case AArch64::LDURXi:
5669 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
5670 }
5671 // These instructions can't be paired based on their opcodes.
5672 return false;
5673}
5674
5675static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
5676 int64_t Offset1, unsigned Opcode1, int FI2,
5677 int64_t Offset2, unsigned Opcode2) {
5678 // Accesses through fixed stack object frame indices may access a different
5679 // fixed stack slot. Check that the object offsets + offsets match.
5680 if (MFI.isFixedObjectIndex(ObjectIdx: FI1) && MFI.isFixedObjectIndex(ObjectIdx: FI2)) {
5681 int64_t ObjectOffset1 = MFI.getObjectOffset(ObjectIdx: FI1);
5682 int64_t ObjectOffset2 = MFI.getObjectOffset(ObjectIdx: FI2);
5683 assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
5684 // Convert to scaled object offsets.
5685 int Scale1 = AArch64InstrInfo::getMemScale(Opc: Opcode1);
5686 if (ObjectOffset1 % Scale1 != 0)
5687 return false;
5688 ObjectOffset1 /= Scale1;
5689 int Scale2 = AArch64InstrInfo::getMemScale(Opc: Opcode2);
5690 if (ObjectOffset2 % Scale2 != 0)
5691 return false;
5692 ObjectOffset2 /= Scale2;
5693 ObjectOffset1 += Offset1;
5694 ObjectOffset2 += Offset2;
5695 return ObjectOffset1 + 1 == ObjectOffset2;
5696 }
5697
5698 return FI1 == FI2;
5699}
5700
5701/// Detect opportunities for ldp/stp formation.
5702///
5703/// Only called for LdSt for which getMemOperandWithOffset returns true.
5704bool AArch64InstrInfo::shouldClusterMemOps(
5705 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
5706 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
5707 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
5708 unsigned NumBytes) const {
5709 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
5710 const MachineOperand &BaseOp1 = *BaseOps1.front();
5711 const MachineOperand &BaseOp2 = *BaseOps2.front();
5712 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
5713 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
5714 if (BaseOp1.getType() != BaseOp2.getType())
5715 return false;
5716
5717 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
5718 "Only base registers and frame indices are supported.");
5719
5720 // Check for both base regs and base FI.
5721 if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
5722 return false;
5723
5724 // Only cluster up to a single pair.
5725 if (ClusterSize > 2)
5726 return false;
5727
5728 if (!isPairableLdStInst(MI: FirstLdSt) || !isPairableLdStInst(MI: SecondLdSt))
5729 return false;
5730
5731 // Can we pair these instructions based on their opcodes?
5732 unsigned FirstOpc = FirstLdSt.getOpcode();
5733 unsigned SecondOpc = SecondLdSt.getOpcode();
5734 if (!canPairLdStOpc(FirstOpc, SecondOpc))
5735 return false;
5736
5737 // Can't merge volatiles or load/stores that have a hint to avoid pair
5738 // formation, for example.
5739 if (!isCandidateToMergeOrPair(MI: FirstLdSt) ||
5740 !isCandidateToMergeOrPair(MI: SecondLdSt))
5741 return false;
5742
5743 // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
5744 int64_t Offset1 = FirstLdSt.getOperand(i: 2).getImm();
5745 if (hasUnscaledLdStOffset(Opc: FirstOpc) && !scaleOffset(Opc: FirstOpc, Offset&: Offset1))
5746 return false;
5747
5748 int64_t Offset2 = SecondLdSt.getOperand(i: 2).getImm();
5749 if (hasUnscaledLdStOffset(Opc: SecondOpc) && !scaleOffset(Opc: SecondOpc, Offset&: Offset2))
5750 return false;
5751
5752 // Pairwise instructions have a 7-bit signed offset field.
5753 if (Offset1 > 63 || Offset1 < -64)
5754 return false;
5755
5756 // The caller should already have ordered First/SecondLdSt by offset.
5757 // Note: except for non-equal frame index bases
5758 if (BaseOp1.isFI()) {
5759 assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
5760 "Caller should have ordered offsets.");
5761
5762 const MachineFrameInfo &MFI =
5763 FirstLdSt.getParent()->getParent()->getFrameInfo();
5764 return shouldClusterFI(MFI, FI1: BaseOp1.getIndex(), Offset1, Opcode1: FirstOpc,
5765 FI2: BaseOp2.getIndex(), Offset2, Opcode2: SecondOpc);
5766 }
5767
5768 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
5769
5770 return Offset1 + 1 == Offset2;
5771}
5772
5773static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
5774 MCRegister Reg, unsigned SubIdx,
5775 RegState State,
5776 const TargetRegisterInfo *TRI) {
5777 if (!SubIdx)
5778 return MIB.addReg(RegNo: Reg, Flags: State);
5779
5780 if (Reg.isPhysical())
5781 return MIB.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx), Flags: State);
5782 return MIB.addReg(RegNo: Reg, Flags: State, SubReg: SubIdx);
5783}
5784
5785void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
5786 MachineBasicBlock::iterator I,
5787 const DebugLoc &DL, MCRegister DestReg,
5788 MCRegister SrcReg, bool KillSrc,
5789 ArrayRef<unsigned> Indices) const {
5790 assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
5791 const TargetRegisterInfo *TRI = &getRegisterInfo();
5792 uint16_t DestEncoding = TRI->getEncodingValue(Reg: DestReg);
5793 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
5794 unsigned NumRegs = Indices.size();
5795 MCRegister DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[0]);
5796 assert(!AArch64::PNRRegClass.contains(DestSubReg) &&
5797 "Unexpected predicate tuple copy");
5798 unsigned MaxRegs = AArch64::PPRRegClass.contains(Reg: DestSubReg) ? 15 : 31;
5799
5800 int SubReg = 0, End = NumRegs, Incr = 1;
5801 // Copy in reverse if a forward copy will clobber the tuple
5802 if (((DestEncoding - SrcEncoding) & MaxRegs) < NumRegs) {
5803 SubReg = NumRegs - 1;
5804 End = -1;
5805 Incr = -1;
5806 }
5807
5808 for (; SubReg != End; SubReg += Incr) {
5809 DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[SubReg]);
5810 MCRegister SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: Indices[SubReg]);
5811 copyPhysRegImpl(MBB, I, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc);
5812 }
5813}
5814
5815void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
5816 MachineBasicBlock::iterator I,
5817 const DebugLoc &DL, MCRegister DestReg,
5818 MCRegister SrcReg, bool KillSrc,
5819 unsigned Opcode, unsigned ZeroReg,
5820 llvm::ArrayRef<unsigned> Indices) const {
5821 const TargetRegisterInfo *TRI = &getRegisterInfo();
5822 unsigned NumRegs = Indices.size();
5823
5824#ifndef NDEBUG
5825 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
5826 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
5827 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5828 "GPR reg sequences should not be able to overlap");
5829#endif
5830
5831 for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
5832 const MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode));
5833 AddSubReg(MIB, Reg: DestReg, SubIdx: Indices[SubReg], State: RegState::Define, TRI);
5834 MIB.addReg(RegNo: ZeroReg);
5835 AddSubReg(MIB, Reg: SrcReg, SubIdx: Indices[SubReg], State: getKillRegState(B: KillSrc), TRI);
5836 MIB.addImm(Val: 0);
5837 }
5838}
5839
5840/// Returns true if the instruction at I is in a streaming call site region,
5841/// within a single basic block.
5842/// A "call site streaming region" starts after smstart and ends at smstop
5843/// around a call to a streaming function. This walks backward from I.
5844static bool isInStreamingCallSiteRegion(MachineBasicBlock &MBB,
5845 MachineBasicBlock::iterator I) {
5846 MachineFunction &MF = *MBB.getParent();
5847 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
5848 if (!AFI->hasStreamingModeChanges())
5849 return false;
5850 // Walk backwards to find smstart/smstop
5851 for (MachineInstr &MI : reverse(C: make_range(x: MBB.begin(), y: I))) {
5852 unsigned Opc = MI.getOpcode();
5853 if (Opc == AArch64::MSRpstatesvcrImm1 || Opc == AArch64::MSRpstatePseudo) {
5854 // Check if this is SM change (not ZA)
5855 int64_t PState = MI.getOperand(i: 0).getImm();
5856 if (PState == AArch64SVCR::SVCRSM || PState == AArch64SVCR::SVCRSMZA) {
5857 // Operand 1 is 1 for start, 0 for stop
5858 return MI.getOperand(i: 1).getImm() == 1;
5859 }
5860 }
5861 }
5862 return false;
5863}
5864
5865/// Returns true if in a streaming call site region without SME-FA64.
5866static bool mustAvoidNeonAtMBBI(const AArch64Subtarget &Subtarget,
5867 MachineBasicBlock &MBB,
5868 MachineBasicBlock::iterator I) {
5869 return !Subtarget.hasSMEFA64() && isInStreamingCallSiteRegion(MBB, I);
5870}
5871
5872void AArch64InstrInfo::copyPhysRegImpl(MachineBasicBlock &MBB,
5873 MachineBasicBlock::iterator I,
5874 const DebugLoc &DL, Register DestReg,
5875 Register SrcReg, bool KillSrc,
5876 bool RenamableDest,
5877 bool RenamableSrc) const {
5878 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) &&
5879 AArch64::GPR32spRegClass.contains(Reg: SrcReg)) {
5880 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5881 // If either operand is WSP, expand to ADD #0.
5882 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5883 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5884 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
5885 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5886 RC: &AArch64::GPR64spRegClass);
5887 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5888 RC: &AArch64::GPR64spRegClass);
5889 // This instruction is reading and writing X registers. This may upset
5890 // the register scavenger and machine verifier, so we need to indicate
5891 // that we are reading an undefined value from SrcRegX, but a proper
5892 // value from SrcReg.
5893 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: DestRegX)
5894 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5895 .addImm(Val: 0)
5896 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
5897 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5898 ++NumZCRegMoveInstrsGPR;
5899 } else {
5900 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDWri), DestReg)
5901 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5902 .addImm(Val: 0)
5903 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5904 if (Subtarget.hasZeroCycleRegMoveGPR32())
5905 ++NumZCRegMoveInstrsGPR;
5906 }
5907 } else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5908 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5909 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
5910 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5911 RC: &AArch64::GPR64spRegClass);
5912 assert(DestRegX.isValid() && "Destination super-reg not valid");
5913 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5914 RC: &AArch64::GPR64spRegClass);
5915 assert(SrcRegX.isValid() && "Source super-reg not valid");
5916 // This instruction is reading and writing X registers. This may upset
5917 // the register scavenger and machine verifier, so we need to indicate
5918 // that we are reading an undefined value from SrcRegX, but a proper
5919 // value from SrcReg.
5920 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg: DestRegX)
5921 .addReg(RegNo: AArch64::XZR)
5922 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5923 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5924 ++NumZCRegMoveInstrsGPR;
5925 } else {
5926 // Otherwise, expand to ORR WZR.
5927 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5928 .addReg(RegNo: AArch64::WZR)
5929 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5930 if (Subtarget.hasZeroCycleRegMoveGPR32())
5931 ++NumZCRegMoveInstrsGPR;
5932 }
5933 return;
5934 }
5935
5936 // GPR32 zeroing
5937 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::WZR) {
5938 if (Subtarget.hasZeroCycleZeroingGPR64() &&
5939 !Subtarget.hasZeroCycleZeroingGPR32()) {
5940 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5941 RC: &AArch64::GPR64spRegClass);
5942 assert(DestRegX.isValid() && "Destination super-reg not valid");
5943 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: DestRegX)
5944 .addImm(Val: 0)
5945 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5946 ++NumZCZeroingInstrsGPR;
5947 } else if (Subtarget.hasZeroCycleZeroingGPR32()) {
5948 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZWi), DestReg)
5949 .addImm(Val: 0)
5950 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5951 ++NumZCZeroingInstrsGPR;
5952 } else {
5953 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5954 .addReg(RegNo: AArch64::WZR)
5955 .addReg(RegNo: AArch64::WZR);
5956 }
5957 return;
5958 }
5959
5960 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) &&
5961 AArch64::GPR64spRegClass.contains(Reg: SrcReg)) {
5962 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5963 // If either operand is SP, expand to ADD #0.
5964 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg)
5965 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5966 .addImm(Val: 0)
5967 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5968 if (Subtarget.hasZeroCycleRegMoveGPR64())
5969 ++NumZCRegMoveInstrsGPR;
5970 } else {
5971 // Otherwise, expand to ORR XZR.
5972 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5973 .addReg(RegNo: AArch64::XZR)
5974 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5975 if (Subtarget.hasZeroCycleRegMoveGPR64())
5976 ++NumZCRegMoveInstrsGPR;
5977 }
5978 return;
5979 }
5980
5981 // GPR64 zeroing
5982 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::XZR) {
5983 if (Subtarget.hasZeroCycleZeroingGPR64()) {
5984 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg)
5985 .addImm(Val: 0)
5986 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5987 ++NumZCZeroingInstrsGPR;
5988 } else {
5989 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5990 .addReg(RegNo: AArch64::XZR)
5991 .addReg(RegNo: AArch64::XZR);
5992 }
5993 return;
5994 }
5995
5996 // Copy a Predicate register by ORRing with itself.
5997 if (AArch64::PPRRegClass.contains(Reg: DestReg) &&
5998 AArch64::PPRRegClass.contains(Reg: SrcReg)) {
5999 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6000 "Unexpected SVE register.");
6001 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg)
6002 .addReg(RegNo: SrcReg) // Pg
6003 .addReg(RegNo: SrcReg)
6004 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6005 return;
6006 }
6007
6008 // Copy a predicate-as-counter register by ORRing with itself as if it
6009 // were a regular predicate (mask) register.
6010 bool DestIsPNR = AArch64::PNRRegClass.contains(Reg: DestReg);
6011 bool SrcIsPNR = AArch64::PNRRegClass.contains(Reg: SrcReg);
6012 if (DestIsPNR || SrcIsPNR) {
6013 auto ToPPR = [](MCRegister R) -> MCRegister {
6014 return (R - AArch64::PN0) + AArch64::P0;
6015 };
6016 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg.asMCReg();
6017 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg.asMCReg();
6018
6019 if (PPRSrcReg != PPRDestReg) {
6020 auto NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg: PPRDestReg)
6021 .addReg(RegNo: PPRSrcReg) // Pg
6022 .addReg(RegNo: PPRSrcReg)
6023 .addReg(RegNo: PPRSrcReg, Flags: getKillRegState(B: KillSrc));
6024 if (DestIsPNR)
6025 NewMI.addDef(RegNo: DestReg, Flags: RegState::Implicit);
6026 }
6027 return;
6028 }
6029
6030 // Copy a predicate register pair by copying the individual sub-registers.
6031 if (AArch64::PPR2RegClass.contains(Reg: DestReg) &&
6032 AArch64::PPR2RegClass.contains(Reg: SrcReg)) {
6033 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6034 "Unexpected SVE predicate register.");
6035 static const unsigned Indices[] = {AArch64::psub0, AArch64::psub1};
6036 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6037 return;
6038 }
6039
6040 // Copy a Z register by ORRing with itself.
6041 if (AArch64::ZPRRegClass.contains(Reg: DestReg) &&
6042 AArch64::ZPRRegClass.contains(Reg: SrcReg)) {
6043 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6044 "Unexpected SVE register.");
6045 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ), DestReg)
6046 .addReg(RegNo: SrcReg)
6047 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6048 return;
6049 }
6050
6051 // Copy a Z register pair by copying the individual sub-registers.
6052 if ((AArch64::ZPR2RegClass.contains(Reg: DestReg) ||
6053 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6054 (AArch64::ZPR2RegClass.contains(Reg: SrcReg) ||
6055 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6056 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6057 "Unexpected SVE register.");
6058 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
6059 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6060 return;
6061 }
6062
6063 // Copy a Z register triple by copying the individual sub-registers.
6064 if (AArch64::ZPR3RegClass.contains(Reg: DestReg) &&
6065 AArch64::ZPR3RegClass.contains(Reg: SrcReg)) {
6066 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6067 "Unexpected SVE register.");
6068 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6069 AArch64::zsub2};
6070 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6071 return;
6072 }
6073
6074 // Copy a Z register quad by copying the individual sub-registers.
6075 if ((AArch64::ZPR4RegClass.contains(Reg: DestReg) ||
6076 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6077 (AArch64::ZPR4RegClass.contains(Reg: SrcReg) ||
6078 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6079 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6080 "Unexpected SVE register.");
6081 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6082 AArch64::zsub2, AArch64::zsub3};
6083 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6084 return;
6085 }
6086
6087 // Copy a DDDD register quad by copying the individual sub-registers.
6088 if (AArch64::DDDDRegClass.contains(Reg: DestReg) &&
6089 AArch64::DDDDRegClass.contains(Reg: SrcReg)) {
6090 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6091 AArch64::dsub2, AArch64::dsub3};
6092 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6093 return;
6094 }
6095
6096 // Copy a DDD register triple by copying the individual sub-registers.
6097 if (AArch64::DDDRegClass.contains(Reg: DestReg) &&
6098 AArch64::DDDRegClass.contains(Reg: SrcReg)) {
6099 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6100 AArch64::dsub2};
6101 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6102 return;
6103 }
6104
6105 // Copy a DD register pair by copying the individual sub-registers.
6106 if (AArch64::DDRegClass.contains(Reg: DestReg) &&
6107 AArch64::DDRegClass.contains(Reg: SrcReg)) {
6108 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
6109 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6110 return;
6111 }
6112
6113 // Copy a QQQQ register quad by copying the individual sub-registers.
6114 if (AArch64::QQQQRegClass.contains(Reg: DestReg) &&
6115 AArch64::QQQQRegClass.contains(Reg: SrcReg)) {
6116 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6117 AArch64::qsub2, AArch64::qsub3};
6118 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6119 return;
6120 }
6121
6122 // Copy a QQQ register triple by copying the individual sub-registers.
6123 if (AArch64::QQQRegClass.contains(Reg: DestReg) &&
6124 AArch64::QQQRegClass.contains(Reg: SrcReg)) {
6125 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6126 AArch64::qsub2};
6127 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6128 return;
6129 }
6130
6131 // Copy a QQ register pair by copying the individual sub-registers.
6132 if (AArch64::QQRegClass.contains(Reg: DestReg) &&
6133 AArch64::QQRegClass.contains(Reg: SrcReg)) {
6134 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
6135 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6136 return;
6137 }
6138
6139 if (AArch64::XSeqPairsClassRegClass.contains(Reg: DestReg) &&
6140 AArch64::XSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6141 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
6142 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRXrs,
6143 ZeroReg: AArch64::XZR, Indices);
6144 return;
6145 }
6146
6147 if (AArch64::WSeqPairsClassRegClass.contains(Reg: DestReg) &&
6148 AArch64::WSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6149 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
6150 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRWrs,
6151 ZeroReg: AArch64::WZR, Indices);
6152 return;
6153 }
6154
6155 if (AArch64::FPR128RegClass.contains(Reg: DestReg) &&
6156 AArch64::FPR128RegClass.contains(Reg: SrcReg)) {
6157 // In streaming regions, NEON is illegal but streaming-SVE is available.
6158 // Use SVE for copies if we're in a streaming region and SME is available.
6159 // With +sme-fa64, NEON is legal in streaming mode so we can use it.
6160 if ((Subtarget.isSVEorStreamingSVEAvailable() &&
6161 !Subtarget.isNeonAvailable()) ||
6162 mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6163 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ))
6164 .addReg(RegNo: AArch64::Z0 + (DestReg - AArch64::Q0), Flags: RegState::Define)
6165 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0))
6166 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0));
6167 } else if (Subtarget.isNeonAvailable()) {
6168 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg)
6169 .addReg(RegNo: SrcReg)
6170 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6171 if (Subtarget.hasZeroCycleRegMoveFPR128())
6172 ++NumZCRegMoveInstrsFPR;
6173 } else {
6174 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::STRQpre))
6175 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6176 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6177 .addReg(RegNo: AArch64::SP)
6178 .addImm(Val: -16);
6179 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::LDRQpost))
6180 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6181 .addReg(RegNo: DestReg, Flags: RegState::Define)
6182 .addReg(RegNo: AArch64::SP)
6183 .addImm(Val: 16);
6184 }
6185 return;
6186 }
6187
6188 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6189 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6190 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6191 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6192 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6193 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6194 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
6195 RC: &AArch64::FPR128RegClass);
6196 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::dsub,
6197 RC: &AArch64::FPR128RegClass);
6198 // This instruction is reading and writing Q registers. This may upset
6199 // the register scavenger and machine verifier, so we need to indicate
6200 // that we are reading an undefined value from SrcRegQ, but a proper
6201 // value from SrcReg.
6202 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6203 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6204 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6205 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6206 ++NumZCRegMoveInstrsFPR;
6207 } else {
6208 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg)
6209 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6210 if (Subtarget.hasZeroCycleRegMoveFPR64())
6211 ++NumZCRegMoveInstrsFPR;
6212 }
6213 return;
6214 }
6215
6216 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6217 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6218 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6219 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6220 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6221 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6222 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6223 RC: &AArch64::FPR128RegClass);
6224 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6225 RC: &AArch64::FPR128RegClass);
6226 // This instruction is reading and writing Q registers. This may upset
6227 // the register scavenger and machine verifier, so we need to indicate
6228 // that we are reading an undefined value from SrcRegQ, but a proper
6229 // value from SrcReg.
6230 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6231 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6232 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6233 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6234 ++NumZCRegMoveInstrsFPR;
6235 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6236 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6237 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6238 RC: &AArch64::FPR64RegClass);
6239 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6240 RC: &AArch64::FPR64RegClass);
6241 // This instruction is reading and writing D registers. This may upset
6242 // the register scavenger and machine verifier, so we need to indicate
6243 // that we are reading an undefined value from SrcRegD, but a proper
6244 // value from SrcReg.
6245 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6246 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6247 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6248 ++NumZCRegMoveInstrsFPR;
6249 } else {
6250 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6251 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6252 if (Subtarget.hasZeroCycleRegMoveFPR32())
6253 ++NumZCRegMoveInstrsFPR;
6254 }
6255 return;
6256 }
6257
6258 if (AArch64::FPR16RegClass.contains(Reg: DestReg) &&
6259 AArch64::FPR16RegClass.contains(Reg: SrcReg)) {
6260 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6261 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6262 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6263 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6264 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6265 RC: &AArch64::FPR128RegClass);
6266 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6267 RC: &AArch64::FPR128RegClass);
6268 // This instruction is reading and writing Q registers. This may upset
6269 // the register scavenger and machine verifier, so we need to indicate
6270 // that we are reading an undefined value from SrcRegQ, but a proper
6271 // value from SrcReg.
6272 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6273 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6274 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6275 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6276 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6277 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6278 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6279 RC: &AArch64::FPR64RegClass);
6280 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6281 RC: &AArch64::FPR64RegClass);
6282 // This instruction is reading and writing D registers. This may upset
6283 // the register scavenger and machine verifier, so we need to indicate
6284 // that we are reading an undefined value from SrcRegD, but a proper
6285 // value from SrcReg.
6286 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6287 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6288 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6289 } else {
6290 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6291 RC: &AArch64::FPR32RegClass);
6292 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6293 RC: &AArch64::FPR32RegClass);
6294 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6295 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6296 }
6297 return;
6298 }
6299
6300 if (AArch64::FPR8RegClass.contains(Reg: DestReg) &&
6301 AArch64::FPR8RegClass.contains(Reg: SrcReg)) {
6302 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6303 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6304 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6305 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6306 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6307 RC: &AArch64::FPR128RegClass);
6308 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6309 RC: &AArch64::FPR128RegClass);
6310 // This instruction is reading and writing Q registers. This may upset
6311 // the register scavenger and machine verifier, so we need to indicate
6312 // that we are reading an undefined value from SrcRegQ, but a proper
6313 // value from SrcReg.
6314 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6315 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6316 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6317 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6318 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6319 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6320 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6321 RC: &AArch64::FPR64RegClass);
6322 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6323 RC: &AArch64::FPR64RegClass);
6324 // This instruction is reading and writing D registers. This may upset
6325 // the register scavenger and machine verifier, so we need to indicate
6326 // that we are reading an undefined value from SrcRegD, but a proper
6327 // value from SrcReg.
6328 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6329 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6330 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6331 } else {
6332 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6333 RC: &AArch64::FPR32RegClass);
6334 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6335 RC: &AArch64::FPR32RegClass);
6336 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6337 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6338 }
6339 return;
6340 }
6341
6342 // Copies between GPR64 and FPR64.
6343 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6344 AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
6345 if (AArch64::XZR == SrcReg) {
6346 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg);
6347 } else {
6348 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVXDr), DestReg)
6349 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6350 }
6351 return;
6352 }
6353 if (AArch64::GPR64RegClass.contains(Reg: DestReg) &&
6354 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6355 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDXr), DestReg)
6356 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6357 return;
6358 }
6359 // Copies between GPR32 and FPR32.
6360 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6361 AArch64::GPR32RegClass.contains(Reg: SrcReg)) {
6362 if (AArch64::WZR == SrcReg) {
6363 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVS0), DestReg);
6364 } else {
6365 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVWSr), DestReg)
6366 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6367 }
6368 return;
6369 }
6370 if (AArch64::GPR32RegClass.contains(Reg: DestReg) &&
6371 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6372 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSWr), DestReg)
6373 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6374 return;
6375 }
6376
6377 if (DestReg == AArch64::NZCV) {
6378 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
6379 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MSR))
6380 .addImm(Val: AArch64SysReg::NZCV)
6381 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6382 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Define);
6383 return;
6384 }
6385
6386 if (SrcReg == AArch64::NZCV) {
6387 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
6388 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MRS), DestReg)
6389 .addImm(Val: AArch64SysReg::NZCV)
6390 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6391 return;
6392 }
6393
6394#ifndef NDEBUG
6395 errs() << RI.getRegAsmName(DestReg) << " = COPY " << RI.getRegAsmName(SrcReg)
6396 << "\n";
6397#endif
6398 llvm_unreachable("unimplemented reg-to-reg copy");
6399}
6400
6401void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
6402 MachineBasicBlock::iterator I,
6403 const DebugLoc &DL, Register DestReg,
6404 Register SrcReg, bool KillSrc,
6405 bool RenamableDest,
6406 bool RenamableSrc) const {
6407 ++NumCopyInstrs;
6408 copyPhysRegImpl(MBB, I, DL, DestReg, SrcReg, KillSrc, RenamableDest,
6409 RenamableSrc);
6410 return;
6411}
6412
6413static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
6414 MachineBasicBlock &MBB,
6415 MachineBasicBlock::iterator InsertBefore,
6416 const MCInstrDesc &MCID,
6417 Register SrcReg, bool IsKill,
6418 unsigned SubIdx0, unsigned SubIdx1, int FI,
6419 MachineMemOperand *MMO) {
6420 Register SrcReg0 = SrcReg;
6421 Register SrcReg1 = SrcReg;
6422 if (SrcReg.isPhysical()) {
6423 SrcReg0 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx0);
6424 SubIdx0 = 0;
6425 SrcReg1 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx1);
6426 SubIdx1 = 0;
6427 }
6428 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6429 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: IsKill), SubReg: SubIdx0)
6430 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: IsKill), SubReg: SubIdx1)
6431 .addFrameIndex(Idx: FI)
6432 .addImm(Val: 0)
6433 .addMemOperand(MMO);
6434}
6435
6436void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
6437 MachineBasicBlock::iterator MBBI,
6438 Register SrcReg, bool isKill, int FI,
6439 const TargetRegisterClass *RC,
6440 Register VReg,
6441 MachineInstr::MIFlag Flags) const {
6442 MachineFunction &MF = *MBB.getParent();
6443 MachineFrameInfo &MFI = MF.getFrameInfo();
6444
6445 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6446 MachineMemOperand *MMO =
6447 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
6448 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6449 unsigned Opc = 0;
6450 bool Offset = true;
6451 MCRegister PNRReg = MCRegister::NoRegister;
6452 unsigned StackID = TargetStackID::Default;
6453 switch (RI.getSpillSize(RC: *RC)) {
6454 case 1:
6455 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6456 Opc = AArch64::STRBui;
6457 break;
6458 case 2: {
6459 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6460 Opc = AArch64::STRHui;
6461 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
6462 AArch64::PPRRegClass.hasSubClassEq(RC)) {
6463 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6464 "Unexpected register store without SVE store instructions");
6465 Opc = AArch64::STR_PXI;
6466 StackID = TargetStackID::ScalablePredicateVector;
6467 }
6468 break;
6469 }
6470 case 4:
6471 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6472 Opc = AArch64::STRWui;
6473 if (SrcReg.isVirtual())
6474 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32RegClass);
6475 else
6476 assert(SrcReg != AArch64::WSP);
6477 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6478 Opc = AArch64::STRSui;
6479 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6480 Opc = AArch64::STR_PPXI;
6481 StackID = TargetStackID::ScalablePredicateVector;
6482 }
6483 break;
6484 case 8:
6485 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6486 Opc = AArch64::STRXui;
6487 if (SrcReg.isVirtual())
6488 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
6489 else
6490 assert(SrcReg != AArch64::SP);
6491 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6492 Opc = AArch64::STRDui;
6493 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6494 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6495 MCID: get(Opcode: AArch64::STPWi), SrcReg, IsKill: isKill,
6496 SubIdx0: AArch64::sube32, SubIdx1: AArch64::subo32, FI, MMO);
6497 return;
6498 }
6499 break;
6500 case 16:
6501 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6502 Opc = AArch64::STRQui;
6503 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6504 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6505 Opc = AArch64::ST1Twov1d;
6506 Offset = false;
6507 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6508 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6509 MCID: get(Opcode: AArch64::STPXi), SrcReg, IsKill: isKill,
6510 SubIdx0: AArch64::sube64, SubIdx1: AArch64::subo64, FI, MMO);
6511 return;
6512 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6513 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6514 "Unexpected register store without SVE store instructions");
6515 Opc = AArch64::STR_ZXI;
6516 StackID = TargetStackID::ScalableVector;
6517 }
6518 break;
6519 case 24:
6520 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6521 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6522 Opc = AArch64::ST1Threev1d;
6523 Offset = false;
6524 }
6525 break;
6526 case 32:
6527 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6528 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6529 Opc = AArch64::ST1Fourv1d;
6530 Offset = false;
6531 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6532 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6533 Opc = AArch64::ST1Twov2d;
6534 Offset = false;
6535 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6536 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6537 "Unexpected register store without SVE store instructions");
6538 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
6539 StackID = TargetStackID::ScalableVector;
6540 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6541 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6542 "Unexpected register store without SVE store instructions");
6543 Opc = AArch64::STR_ZZXI;
6544 StackID = TargetStackID::ScalableVector;
6545 }
6546 break;
6547 case 48:
6548 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6549 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6550 Opc = AArch64::ST1Threev2d;
6551 Offset = false;
6552 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6553 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6554 "Unexpected register store without SVE store instructions");
6555 Opc = AArch64::STR_ZZZXI;
6556 StackID = TargetStackID::ScalableVector;
6557 }
6558 break;
6559 case 64:
6560 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6561 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6562 Opc = AArch64::ST1Fourv2d;
6563 Offset = false;
6564 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6565 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6566 "Unexpected register store without SVE store instructions");
6567 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
6568 StackID = TargetStackID::ScalableVector;
6569 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6570 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6571 "Unexpected register store without SVE store instructions");
6572 Opc = AArch64::STR_ZZZZXI;
6573 StackID = TargetStackID::ScalableVector;
6574 }
6575 break;
6576 }
6577 assert(Opc && "Unknown register class");
6578 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6579
6580 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6581 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
6582 .addFrameIndex(Idx: FI);
6583
6584 if (Offset)
6585 MI.addImm(Val: 0);
6586 if (PNRReg.isValid())
6587 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6588 MI.addMemOperand(MMO);
6589}
6590
6591static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
6592 MachineBasicBlock &MBB,
6593 MachineBasicBlock::iterator InsertBefore,
6594 const MCInstrDesc &MCID,
6595 Register DestReg, unsigned SubIdx0,
6596 unsigned SubIdx1, int FI,
6597 MachineMemOperand *MMO) {
6598 Register DestReg0 = DestReg;
6599 Register DestReg1 = DestReg;
6600 bool IsUndef = true;
6601 if (DestReg.isPhysical()) {
6602 DestReg0 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx0);
6603 SubIdx0 = 0;
6604 DestReg1 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx1);
6605 SubIdx1 = 0;
6606 IsUndef = false;
6607 }
6608 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6609 .addReg(RegNo: DestReg0, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx0)
6610 .addReg(RegNo: DestReg1, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx1)
6611 .addFrameIndex(Idx: FI)
6612 .addImm(Val: 0)
6613 .addMemOperand(MMO);
6614}
6615
6616void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
6617 MachineBasicBlock::iterator MBBI,
6618 Register DestReg, int FI,
6619 const TargetRegisterClass *RC,
6620 Register VReg, unsigned SubReg,
6621 MachineInstr::MIFlag Flags) const {
6622 MachineFunction &MF = *MBB.getParent();
6623 MachineFrameInfo &MFI = MF.getFrameInfo();
6624 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6625 MachineMemOperand *MMO =
6626 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
6627 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6628
6629 unsigned Opc = 0;
6630 bool Offset = true;
6631 unsigned StackID = TargetStackID::Default;
6632 Register PNRReg;
6633 switch (TRI.getSpillSize(RC: *RC)) {
6634 case 1:
6635 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6636 Opc = AArch64::LDRBui;
6637 break;
6638 case 2: {
6639 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
6640 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6641 Opc = AArch64::LDRHui;
6642 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
6643 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6644 "Unexpected register load without SVE load instructions");
6645 if (IsPNR)
6646 PNRReg = DestReg;
6647 Opc = AArch64::LDR_PXI;
6648 StackID = TargetStackID::ScalablePredicateVector;
6649 }
6650 break;
6651 }
6652 case 4:
6653 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6654 Opc = AArch64::LDRWui;
6655 if (DestReg.isVirtual())
6656 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR32RegClass);
6657 else
6658 assert(DestReg != AArch64::WSP);
6659 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6660 Opc = AArch64::LDRSui;
6661 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6662 Opc = AArch64::LDR_PPXI;
6663 StackID = TargetStackID::ScalablePredicateVector;
6664 }
6665 break;
6666 case 8:
6667 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6668 Opc = AArch64::LDRXui;
6669 if (DestReg.isVirtual())
6670 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR64RegClass);
6671 else
6672 assert(DestReg != AArch64::SP);
6673 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6674 Opc = AArch64::LDRDui;
6675 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6676 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6677 MCID: get(Opcode: AArch64::LDPWi), DestReg, SubIdx0: AArch64::sube32,
6678 SubIdx1: AArch64::subo32, FI, MMO);
6679 return;
6680 }
6681 break;
6682 case 16:
6683 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6684 Opc = AArch64::LDRQui;
6685 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6686 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6687 Opc = AArch64::LD1Twov1d;
6688 Offset = false;
6689 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6690 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6691 MCID: get(Opcode: AArch64::LDPXi), DestReg, SubIdx0: AArch64::sube64,
6692 SubIdx1: AArch64::subo64, FI, MMO);
6693 return;
6694 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6695 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6696 "Unexpected register load without SVE load instructions");
6697 Opc = AArch64::LDR_ZXI;
6698 StackID = TargetStackID::ScalableVector;
6699 }
6700 break;
6701 case 24:
6702 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6703 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6704 Opc = AArch64::LD1Threev1d;
6705 Offset = false;
6706 }
6707 break;
6708 case 32:
6709 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6710 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6711 Opc = AArch64::LD1Fourv1d;
6712 Offset = false;
6713 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6714 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6715 Opc = AArch64::LD1Twov2d;
6716 Offset = false;
6717 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6718 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6719 "Unexpected register load without SVE load instructions");
6720 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
6721 StackID = TargetStackID::ScalableVector;
6722 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6723 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6724 "Unexpected register load without SVE load instructions");
6725 Opc = AArch64::LDR_ZZXI;
6726 StackID = TargetStackID::ScalableVector;
6727 }
6728 break;
6729 case 48:
6730 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6731 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6732 Opc = AArch64::LD1Threev2d;
6733 Offset = false;
6734 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6735 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6736 "Unexpected register load without SVE load instructions");
6737 Opc = AArch64::LDR_ZZZXI;
6738 StackID = TargetStackID::ScalableVector;
6739 }
6740 break;
6741 case 64:
6742 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6743 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6744 Opc = AArch64::LD1Fourv2d;
6745 Offset = false;
6746 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6747 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6748 "Unexpected register load without SVE load instructions");
6749 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
6750 StackID = TargetStackID::ScalableVector;
6751 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6752 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6753 "Unexpected register load without SVE load instructions");
6754 Opc = AArch64::LDR_ZZZZXI;
6755 StackID = TargetStackID::ScalableVector;
6756 }
6757 break;
6758 }
6759
6760 assert(Opc && "Unknown register class");
6761 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6762
6763 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6764 .addReg(RegNo: DestReg, Flags: getDefRegState(B: true))
6765 .addFrameIndex(Idx: FI);
6766 if (Offset)
6767 MI.addImm(Val: 0);
6768 if (PNRReg.isValid() && !PNRReg.isVirtual())
6769 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6770 MI.addMemOperand(MMO);
6771}
6772
6773bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
6774 const MachineInstr &UseMI,
6775 const TargetRegisterInfo *TRI) {
6776 return any_of(Range: instructionsWithoutDebug(It: std::next(x: DefMI.getIterator()),
6777 End: UseMI.getIterator()),
6778 P: [TRI](const MachineInstr &I) {
6779 return I.modifiesRegister(Reg: AArch64::NZCV, TRI) ||
6780 I.readsRegister(Reg: AArch64::NZCV, TRI);
6781 });
6782}
6783
6784void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6785 const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized) {
6786 // The smallest scalable element supported by scaled SVE addressing
6787 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6788 // byte offset must always be a multiple of 2.
6789 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6790
6791 // VGSized offsets are divided by '2', because the VG register is the
6792 // the number of 64bit granules as opposed to 128bit vector chunks,
6793 // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
6794 // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
6795 // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
6796 ByteSized = Offset.getFixed();
6797 VGSized = Offset.getScalable() / 2;
6798}
6799
6800/// Returns the offset in parts to which this frame offset can be
6801/// decomposed for the purpose of describing a frame offset.
6802/// For non-scalable offsets this is simply its byte size.
6803void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6804 const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors,
6805 int64_t &NumDataVectors) {
6806 // The smallest scalable element supported by scaled SVE addressing
6807 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6808 // byte offset must always be a multiple of 2.
6809 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6810
6811 NumBytes = Offset.getFixed();
6812 NumDataVectors = 0;
6813 NumPredicateVectors = Offset.getScalable() / 2;
6814 // This method is used to get the offsets to adjust the frame offset.
6815 // If the function requires ADDPL to be used and needs more than two ADDPL
6816 // instructions, part of the offset is folded into NumDataVectors so that it
6817 // uses ADDVL for part of it, reducing the number of ADDPL instructions.
6818 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
6819 NumPredicateVectors > 62) {
6820 NumDataVectors = NumPredicateVectors / 8;
6821 NumPredicateVectors -= NumDataVectors * 8;
6822 }
6823}
6824
6825// Convenience function to create a DWARF expression for: Constant `Operation`.
6826// This helper emits compact sequences for common cases. For example, for`-15
6827// DW_OP_plus`, this helper would create DW_OP_lit15 DW_OP_minus.
6828static void appendConstantExpr(SmallVectorImpl<char> &Expr, int64_t Constant,
6829 dwarf::LocationAtom Operation) {
6830 if (Operation == dwarf::DW_OP_plus && Constant < 0 && -Constant <= 31) {
6831 // -Constant (1 to 31)
6832 Expr.push_back(Elt: dwarf::DW_OP_lit0 - Constant);
6833 Operation = dwarf::DW_OP_minus;
6834 } else if (Constant >= 0 && Constant <= 31) {
6835 // Literal value 0 to 31
6836 Expr.push_back(Elt: dwarf::DW_OP_lit0 + Constant);
6837 } else {
6838 // Signed constant
6839 Expr.push_back(Elt: dwarf::DW_OP_consts);
6840 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: Constant);
6841 }
6842 return Expr.push_back(Elt: Operation);
6843}
6844
6845// Convenience function to create a DWARF expression for a register.
6846static void appendReadRegExpr(SmallVectorImpl<char> &Expr, unsigned RegNum) {
6847 Expr.push_back(Elt: (char)dwarf::DW_OP_bregx);
6848 appendLEB128<LEB128Sign::Unsigned>(Buffer&: Expr, Value: RegNum);
6849 Expr.push_back(Elt: 0);
6850}
6851
6852// Convenience function to create a DWARF expression for loading a register from
6853// a CFA offset.
6854static void appendLoadRegExpr(SmallVectorImpl<char> &Expr,
6855 int64_t OffsetFromDefCFA) {
6856 // This assumes the top of the DWARF stack contains the CFA.
6857 Expr.push_back(Elt: dwarf::DW_OP_dup);
6858 // Add the offset to the register.
6859 appendConstantExpr(Expr, Constant: OffsetFromDefCFA, Operation: dwarf::DW_OP_plus);
6860 // Dereference the address (loads a 64 bit value)..
6861 Expr.push_back(Elt: dwarf::DW_OP_deref);
6862}
6863
6864// Convenience function to create a comment for
6865// (+/-) NumBytes (* RegScale)?
6866static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
6867 StringRef RegScale = {}) {
6868 if (NumBytes) {
6869 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(x: NumBytes);
6870 if (!RegScale.empty())
6871 Comment << ' ' << RegScale;
6872 }
6873}
6874
6875// Creates an MCCFIInstruction:
6876// { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
6877static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
6878 unsigned Reg,
6879 const StackOffset &Offset) {
6880 int64_t NumBytes, NumVGScaledBytes;
6881 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(Offset, ByteSized&: NumBytes,
6882 VGSized&: NumVGScaledBytes);
6883 std::string CommentBuffer;
6884 llvm::raw_string_ostream Comment(CommentBuffer);
6885
6886 if (Reg == AArch64::SP)
6887 Comment << "sp";
6888 else if (Reg == AArch64::FP)
6889 Comment << "fp";
6890 else
6891 Comment << printReg(Reg, TRI: &TRI);
6892
6893 // Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
6894 SmallString<64> Expr;
6895 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6896 assert(DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
6897 // Reg + NumBytes
6898 Expr.push_back(Elt: dwarf::DW_OP_breg0 + DwarfReg);
6899 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: NumBytes);
6900 appendOffsetComment(NumBytes, Comment);
6901 if (NumVGScaledBytes) {
6902 // + VG * NumVGScaledBytes
6903 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: "* VG");
6904 appendReadRegExpr(Expr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6905 appendConstantExpr(Expr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6906 Expr.push_back(Elt: dwarf::DW_OP_plus);
6907 }
6908
6909 // Wrap this into DW_CFA_def_cfa.
6910 SmallString<64> DefCfaExpr;
6911 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
6912 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
6913 DefCfaExpr.append(RHS: Expr.str());
6914 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
6915 Comment: Comment.str());
6916}
6917
6918MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
6919 unsigned FrameReg, unsigned Reg,
6920 const StackOffset &Offset,
6921 bool LastAdjustmentWasScalable) {
6922 if (Offset.getScalable())
6923 return createDefCFAExpression(TRI, Reg, Offset);
6924
6925 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6926 return MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: int(Offset.getFixed()));
6927
6928 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6929 return MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfReg, Offset: (int)Offset.getFixed());
6930}
6931
6932MCCFIInstruction
6933llvm::createCFAOffset(const TargetRegisterInfo &TRI, unsigned Reg,
6934 const StackOffset &OffsetFromDefCFA,
6935 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6936 int64_t NumBytes, NumVGScaledBytes;
6937 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6938 Offset: OffsetFromDefCFA, ByteSized&: NumBytes, VGSized&: NumVGScaledBytes);
6939
6940 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6941
6942 // Non-scalable offsets can use DW_CFA_offset directly.
6943 if (!NumVGScaledBytes)
6944 return MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: NumBytes);
6945
6946 std::string CommentBuffer;
6947 llvm::raw_string_ostream Comment(CommentBuffer);
6948 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
6949
6950 // Build up expression (CFA + VG * NumVGScaledBytes + NumBytes)
6951 assert(NumVGScaledBytes && "Expected scalable offset");
6952 SmallString<64> OffsetExpr;
6953 // + VG * NumVGScaledBytes
6954 StringRef VGRegScale;
6955 if (IncomingVGOffsetFromDefCFA) {
6956 appendLoadRegExpr(Expr&: OffsetExpr, OffsetFromDefCFA: *IncomingVGOffsetFromDefCFA);
6957 VGRegScale = "* IncomingVG";
6958 } else {
6959 appendReadRegExpr(Expr&: OffsetExpr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6960 VGRegScale = "* VG";
6961 }
6962 appendConstantExpr(Expr&: OffsetExpr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6963 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: VGRegScale);
6964 OffsetExpr.push_back(Elt: dwarf::DW_OP_plus);
6965 if (NumBytes) {
6966 // + NumBytes
6967 appendOffsetComment(NumBytes, Comment);
6968 appendConstantExpr(Expr&: OffsetExpr, Constant: NumBytes, Operation: dwarf::DW_OP_plus);
6969 }
6970
6971 // Wrap this into DW_CFA_expression
6972 SmallString<64> CfaExpr;
6973 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
6974 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: DwarfReg);
6975 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: OffsetExpr.size());
6976 CfaExpr.append(RHS: OffsetExpr.str());
6977
6978 return MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str(), Loc: SMLoc(),
6979 Comment: Comment.str());
6980}
6981
6982// Helper function to emit a frame offset adjustment from a given
6983// pointer (SrcReg), stored into DestReg. This function is explicit
6984// in that it requires the opcode.
6985static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
6986 MachineBasicBlock::iterator MBBI,
6987 const DebugLoc &DL, unsigned DestReg,
6988 unsigned SrcReg, int64_t Offset, unsigned Opc,
6989 const TargetInstrInfo *TII,
6990 MachineInstr::MIFlag Flag, bool NeedsWinCFI,
6991 bool *HasWinCFI, bool EmitCFAOffset,
6992 StackOffset CFAOffset, unsigned FrameReg) {
6993 int Sign = 1;
6994 unsigned MaxEncoding, ShiftSize;
6995 switch (Opc) {
6996 case AArch64::ADDXri:
6997 case AArch64::ADDSXri:
6998 case AArch64::SUBXri:
6999 case AArch64::SUBSXri:
7000 MaxEncoding = 0xfff;
7001 ShiftSize = 12;
7002 break;
7003 case AArch64::ADDVL_XXI:
7004 case AArch64::ADDPL_XXI:
7005 case AArch64::ADDSVL_XXI:
7006 case AArch64::ADDSPL_XXI:
7007 MaxEncoding = 31;
7008 ShiftSize = 0;
7009 if (Offset < 0) {
7010 MaxEncoding = 32;
7011 Sign = -1;
7012 Offset = -Offset;
7013 }
7014 break;
7015 default:
7016 llvm_unreachable("Unsupported opcode");
7017 }
7018
7019 // `Offset` can be in bytes or in "scalable bytes".
7020 int VScale = 1;
7021 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
7022 VScale = 16;
7023 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
7024 VScale = 2;
7025
7026 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
7027 // scratch register. If DestReg is a virtual register, use it as the
7028 // scratch register; otherwise, create a new virtual register (to be
7029 // replaced by the scavenger at the end of PEI). That case can be optimized
7030 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
7031 // register can be loaded with offset%8 and the add/sub can use an extending
7032 // instruction with LSL#3.
7033 // Currently the function handles any offsets but generates a poor sequence
7034 // of code.
7035 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
7036
7037 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
7038 Register TmpReg = DestReg;
7039 if (TmpReg == AArch64::XZR)
7040 TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
7041 RegClass: &AArch64::GPR64RegClass);
7042 do {
7043 uint64_t ThisVal = std::min<uint64_t>(a: Offset, b: MaxEncodableValue);
7044 unsigned LocalShiftSize = 0;
7045 if (ThisVal > MaxEncoding) {
7046 ThisVal = ThisVal >> ShiftSize;
7047 LocalShiftSize = ShiftSize;
7048 }
7049 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
7050 "Encoding cannot handle value that big");
7051
7052 Offset -= ThisVal << LocalShiftSize;
7053 if (Offset == 0)
7054 TmpReg = DestReg;
7055 auto MBI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: TmpReg)
7056 .addReg(RegNo: SrcReg)
7057 .addImm(Val: Sign * (int)ThisVal);
7058 if (ShiftSize)
7059 MBI = MBI.addImm(
7060 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LocalShiftSize));
7061 MBI = MBI.setMIFlag(Flag);
7062
7063 auto Change =
7064 VScale == 1
7065 ? StackOffset::getFixed(Fixed: ThisVal << LocalShiftSize)
7066 : StackOffset::getScalable(Scalable: VScale * (ThisVal << LocalShiftSize));
7067 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
7068 CFAOffset += Change;
7069 else
7070 CFAOffset -= Change;
7071 if (EmitCFAOffset && DestReg == TmpReg) {
7072 MachineFunction &MF = *MBB.getParent();
7073 const TargetSubtargetInfo &STI = MF.getSubtarget();
7074 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
7075
7076 unsigned CFIIndex = MF.addFrameInst(
7077 Inst: createDefCFA(TRI, FrameReg, Reg: DestReg, Offset: CFAOffset, LastAdjustmentWasScalable: VScale != 1));
7078 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
7079 .addCFIIndex(CFIIndex)
7080 .setMIFlags(Flag);
7081 }
7082
7083 if (NeedsWinCFI) {
7084 int Imm = (int)(ThisVal << LocalShiftSize);
7085 if (VScale != 1 && DestReg == AArch64::SP) {
7086 if (HasWinCFI)
7087 *HasWinCFI = true;
7088 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AllocZ))
7089 .addImm(Val: ThisVal)
7090 .setMIFlag(Flag);
7091 } else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
7092 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
7093 assert(VScale == 1 && "Expected non-scalable operation");
7094 if (HasWinCFI)
7095 *HasWinCFI = true;
7096 if (Imm == 0)
7097 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_SetFP)).setMIFlag(Flag);
7098 else
7099 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AddFP))
7100 .addImm(Val: Imm)
7101 .setMIFlag(Flag);
7102 assert(Offset == 0 && "Expected remaining offset to be zero to "
7103 "emit a single SEH directive");
7104 } else if (DestReg == AArch64::SP) {
7105 assert(VScale == 1 && "Expected non-scalable operation");
7106 if (HasWinCFI)
7107 *HasWinCFI = true;
7108 assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
7109 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_StackAlloc))
7110 .addImm(Val: Imm)
7111 .setMIFlag(Flag);
7112 }
7113 }
7114
7115 SrcReg = TmpReg;
7116 } while (Offset);
7117}
7118
7119void llvm::emitFrameOffset(MachineBasicBlock &MBB,
7120 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
7121 unsigned DestReg, unsigned SrcReg,
7122 StackOffset Offset, const TargetInstrInfo *TII,
7123 MachineInstr::MIFlag Flag, bool SetNZCV,
7124 bool NeedsWinCFI, bool *HasWinCFI,
7125 bool EmitCFAOffset, StackOffset CFAOffset,
7126 unsigned FrameReg) {
7127 // If a function is marked as arm_locally_streaming, then the runtime value of
7128 // vscale in the prologue/epilogue is different the runtime value of vscale
7129 // in the function's body. To avoid having to consider multiple vscales,
7130 // we can use `addsvl` to allocate any scalable stack-slots, which under
7131 // most circumstances will be only locals, not callee-save slots.
7132 const Function &F = MBB.getParent()->getFunction();
7133 bool UseSVL = F.hasFnAttribute(Kind: "aarch64_pstate_sm_body");
7134
7135 int64_t Bytes, NumPredicateVectors, NumDataVectors;
7136 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
7137 Offset, NumBytes&: Bytes, NumPredicateVectors, NumDataVectors);
7138
7139 // Insert ADDSXri for scalable offset at the end.
7140 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
7141 if (NeedsFinalDefNZCV)
7142 SetNZCV = false;
7143
7144 // First emit non-scalable frame offsets, or a simple 'mov'.
7145 if (Bytes || (!Offset && SrcReg != DestReg)) {
7146 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
7147 "SP increment/decrement not 8-byte aligned");
7148 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
7149 if (Bytes < 0) {
7150 Bytes = -Bytes;
7151 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
7152 }
7153 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: Bytes, Opc, TII, Flag,
7154 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7155 FrameReg);
7156 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
7157 ? StackOffset::getFixed(Fixed: -Bytes)
7158 : StackOffset::getFixed(Fixed: Bytes);
7159 SrcReg = DestReg;
7160 FrameReg = DestReg;
7161 }
7162
7163 assert(!(NeedsWinCFI && NumPredicateVectors) &&
7164 "WinCFI can't allocate fractions of an SVE data vector");
7165
7166 if (NumDataVectors) {
7167 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumDataVectors,
7168 Opc: UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI, TII,
7169 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7170 FrameReg);
7171 CFAOffset += StackOffset::getScalable(Scalable: -NumDataVectors * 16);
7172 SrcReg = DestReg;
7173 }
7174
7175 if (NumPredicateVectors) {
7176 assert(DestReg != AArch64::SP && "Unaligned access to SP");
7177 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumPredicateVectors,
7178 Opc: UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI, TII,
7179 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7180 FrameReg);
7181 }
7182
7183 if (NeedsFinalDefNZCV)
7184 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDSXri), DestReg)
7185 .addReg(RegNo: DestReg)
7186 .addImm(Val: 0)
7187 .addImm(Val: 0);
7188}
7189
7190MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
7191 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
7192 int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS,
7193 VirtRegMap *VRM) const {
7194 MachineBasicBlock::iterator InsertPt = MI;
7195 // This is a bit of a hack. Consider this instruction:
7196 //
7197 // %0 = COPY %sp; GPR64all:%0
7198 //
7199 // We explicitly chose GPR64all for the virtual register so such a copy might
7200 // be eliminated by RegisterCoalescer. However, that may not be possible, and
7201 // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
7202 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
7203 //
7204 // To prevent that, we are going to constrain the %0 register class here.
7205 if (MI.isFullCopy()) {
7206 Register DstReg = MI.getOperand(i: 0).getReg();
7207 Register SrcReg = MI.getOperand(i: 1).getReg();
7208 if (SrcReg == AArch64::SP && DstReg.isVirtual()) {
7209 MF.getRegInfo().constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass);
7210 return nullptr;
7211 }
7212 if (DstReg == AArch64::SP && SrcReg.isVirtual()) {
7213 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
7214 return nullptr;
7215 }
7216 // Nothing can folded with copy from/to NZCV.
7217 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
7218 return nullptr;
7219 }
7220
7221 // Handle the case where a copy is being spilled or filled but the source
7222 // and destination register class don't match. For example:
7223 //
7224 // %0 = COPY %xzr; GPR64common:%0
7225 //
7226 // In this case we can still safely fold away the COPY and generate the
7227 // following spill code:
7228 //
7229 // STRXui %xzr, %stack.0
7230 //
7231 // This also eliminates spilled cross register class COPYs (e.g. between x and
7232 // d regs) of the same size. For example:
7233 //
7234 // %0 = COPY %1; GPR64:%0, FPR64:%1
7235 //
7236 // will be filled as
7237 //
7238 // LDRDui %0, fi<#0>
7239 //
7240 // instead of
7241 //
7242 // LDRXui %Temp, fi<#0>
7243 // %0 = FMOV %Temp
7244 //
7245 if (MI.isCopy() && Ops.size() == 1 &&
7246 // Make sure we're only folding the explicit COPY defs/uses.
7247 (Ops[0] == 0 || Ops[0] == 1)) {
7248 bool IsSpill = Ops[0] == 0;
7249 bool IsFill = !IsSpill;
7250 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7251 const MachineRegisterInfo &MRI = MF.getRegInfo();
7252 MachineBasicBlock &MBB = *MI.getParent();
7253 const MachineOperand &DstMO = MI.getOperand(i: 0);
7254 const MachineOperand &SrcMO = MI.getOperand(i: 1);
7255 Register DstReg = DstMO.getReg();
7256 Register SrcReg = SrcMO.getReg();
7257 // This is slightly expensive to compute for physical regs since
7258 // getMinimalPhysRegClass is slow.
7259 auto getRegClass = [&](unsigned Reg) {
7260 return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
7261 : TRI.getMinimalPhysRegClass(Reg);
7262 };
7263
7264 if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
7265 assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
7266 TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
7267 "Mismatched register size in non subreg COPY");
7268 if (IsSpill)
7269 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg, isKill: SrcMO.isKill(), FI: FrameIndex,
7270 RC: getRegClass(SrcReg), VReg: Register());
7271 else
7272 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex,
7273 RC: getRegClass(DstReg), VReg: Register());
7274 return &*--InsertPt;
7275 }
7276
7277 // Handle cases like spilling def of:
7278 //
7279 // %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
7280 //
7281 // where the physical register source can be widened and stored to the full
7282 // virtual reg destination stack slot, in this case producing:
7283 //
7284 // STRXui %xzr, %stack.0
7285 //
7286 if (IsSpill && DstMO.isUndef() && SrcReg == AArch64::WZR &&
7287 TRI.getRegSizeInBits(RC: *getRegClass(DstReg)) == 64) {
7288 assert(SrcMO.getSubReg() == 0 &&
7289 "Unexpected subreg on physical register");
7290 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg: AArch64::XZR, isKill: SrcMO.isKill(),
7291 FI: FrameIndex, RC: &AArch64::GPR64RegClass, VReg: Register());
7292 return &*--InsertPt;
7293 }
7294
7295 // Handle cases like filling use of:
7296 //
7297 // %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
7298 //
7299 // where we can load the full virtual reg source stack slot, into the subreg
7300 // destination, in this case producing:
7301 //
7302 // LDRWui %0:sub_32<def,read-undef>, %stack.0
7303 //
7304 if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
7305 const TargetRegisterClass *FillRC = nullptr;
7306 switch (DstMO.getSubReg()) {
7307 default:
7308 break;
7309 case AArch64::sub_32:
7310 if (AArch64::GPR64RegClass.hasSubClassEq(RC: getRegClass(DstReg)))
7311 FillRC = &AArch64::GPR32RegClass;
7312 break;
7313 case AArch64::ssub:
7314 FillRC = &AArch64::FPR32RegClass;
7315 break;
7316 case AArch64::dsub:
7317 FillRC = &AArch64::FPR64RegClass;
7318 break;
7319 }
7320
7321 if (FillRC) {
7322 assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
7323 TRI.getRegSizeInBits(*FillRC) &&
7324 "Mismatched regclass size on folded subreg COPY");
7325 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex, RC: FillRC,
7326 VReg: Register());
7327 MachineInstr &LoadMI = *--InsertPt;
7328 MachineOperand &LoadDst = LoadMI.getOperand(i: 0);
7329 assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
7330 LoadDst.setSubReg(DstMO.getSubReg());
7331 LoadDst.setIsUndef();
7332 return &LoadMI;
7333 }
7334 }
7335 }
7336
7337 // Cannot fold.
7338 return nullptr;
7339}
7340
7341int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
7342 StackOffset &SOffset,
7343 bool *OutUseUnscaledOp,
7344 unsigned *OutUnscaledOp,
7345 int64_t *EmittableOffset) {
7346 // Set output values in case of early exit.
7347 if (EmittableOffset)
7348 *EmittableOffset = 0;
7349 if (OutUseUnscaledOp)
7350 *OutUseUnscaledOp = false;
7351 if (OutUnscaledOp)
7352 *OutUnscaledOp = 0;
7353
7354 // Exit early for structured vector spills/fills as they can't take an
7355 // immediate offset.
7356 switch (MI.getOpcode()) {
7357 default:
7358 break;
7359 case AArch64::LD1Rv1d:
7360 case AArch64::LD1Rv2s:
7361 case AArch64::LD1Rv2d:
7362 case AArch64::LD1Rv4h:
7363 case AArch64::LD1Rv4s:
7364 case AArch64::LD1Rv8b:
7365 case AArch64::LD1Rv8h:
7366 case AArch64::LD1Rv16b:
7367 case AArch64::LD1Twov2d:
7368 case AArch64::LD1Threev2d:
7369 case AArch64::LD1Fourv2d:
7370 case AArch64::LD1Twov1d:
7371 case AArch64::LD1Threev1d:
7372 case AArch64::LD1Fourv1d:
7373 case AArch64::ST1Twov2d:
7374 case AArch64::ST1Threev2d:
7375 case AArch64::ST1Fourv2d:
7376 case AArch64::ST1Twov1d:
7377 case AArch64::ST1Threev1d:
7378 case AArch64::ST1Fourv1d:
7379 case AArch64::ST1i8:
7380 case AArch64::ST1i16:
7381 case AArch64::ST1i32:
7382 case AArch64::ST1i64:
7383 case AArch64::IRG:
7384 case AArch64::IRGstack:
7385 case AArch64::STGloop:
7386 case AArch64::STZGloop:
7387 return AArch64FrameOffsetCannotUpdate;
7388 }
7389
7390 // Get the min/max offset and the scale.
7391 TypeSize ScaleValue(0U, false), Width(0U, false);
7392 int64_t MinOff, MaxOff;
7393 if (!AArch64InstrInfo::getMemOpInfo(Opcode: MI.getOpcode(), Scale&: ScaleValue, Width, MinOffset&: MinOff,
7394 MaxOffset&: MaxOff))
7395 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7396
7397 // Construct the complete offset.
7398 bool IsMulVL = ScaleValue.isScalable();
7399 unsigned Scale = ScaleValue.getKnownMinValue();
7400 int64_t Offset = IsMulVL ? SOffset.getScalable() : SOffset.getFixed();
7401
7402 const MachineOperand &ImmOpnd =
7403 MI.getOperand(i: AArch64InstrInfo::getLoadStoreImmIdx(Opc: MI.getOpcode()));
7404 Offset += ImmOpnd.getImm() * Scale;
7405
7406 // If the offset doesn't match the scale, we rewrite the instruction to
7407 // use the unscaled instruction instead. Likewise, if we have a negative
7408 // offset and there is an unscaled op to use.
7409 std::optional<unsigned> UnscaledOp =
7410 AArch64InstrInfo::getUnscaledLdSt(Opc: MI.getOpcode());
7411 bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
7412 if (useUnscaledOp &&
7413 !AArch64InstrInfo::getMemOpInfo(Opcode: *UnscaledOp, Scale&: ScaleValue, Width, MinOffset&: MinOff,
7414 MaxOffset&: MaxOff))
7415 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7416
7417 Scale = ScaleValue.getKnownMinValue();
7418 assert(IsMulVL == ScaleValue.isScalable() &&
7419 "Unscaled opcode has different value for scalable");
7420
7421 int64_t Remainder = Offset % Scale;
7422 assert(!(Remainder && useUnscaledOp) &&
7423 "Cannot have remainder when using unscaled op");
7424
7425 assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
7426 int64_t NewOffset = Offset / Scale;
7427 if (MinOff <= NewOffset && NewOffset <= MaxOff)
7428 Offset = Remainder;
7429 else {
7430 // Try to minimise the number of instructions required to materialise the
7431 // offset calculation. Specifically, for fixed offsets, if masking out the
7432 // low 12 bits leaves a legal add immediate, we can realise the offset
7433 // calculation with a single add instruction. Whenever this is possible,
7434 // prefer this split.
7435 int64_t HighPart = Offset & ~0xFFF;
7436 int64_t LowPart = Offset & 0xFFF;
7437 int64_t LowScaled = LowPart / Scale;
7438 if (!IsMulVL && NewOffset >= 0 && LowPart % Scale == 0 &&
7439 MinOff <= LowScaled && LowScaled <= MaxOff &&
7440 AArch64_AM::isLegalArithImmed(C: HighPart)) {
7441 NewOffset = LowScaled;
7442 Offset = HighPart;
7443 } else {
7444 // Default to a greedy split: take the memop immediate to be maximum /
7445 // minimum expressible offset and materialise the remainder.
7446 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
7447 Offset = Offset - (NewOffset * Scale);
7448 }
7449 }
7450
7451 if (EmittableOffset)
7452 *EmittableOffset = NewOffset;
7453 if (OutUseUnscaledOp)
7454 *OutUseUnscaledOp = useUnscaledOp;
7455 if (OutUnscaledOp && UnscaledOp)
7456 *OutUnscaledOp = *UnscaledOp;
7457
7458 if (IsMulVL)
7459 SOffset = StackOffset::get(Fixed: SOffset.getFixed(), Scalable: Offset);
7460 else
7461 SOffset = StackOffset::get(Fixed: Offset, Scalable: SOffset.getScalable());
7462 return AArch64FrameOffsetCanUpdate |
7463 (SOffset ? 0 : AArch64FrameOffsetIsLegal);
7464}
7465
7466bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
7467 unsigned FrameReg, StackOffset &Offset,
7468 const AArch64InstrInfo *TII) {
7469 unsigned Opcode = MI.getOpcode();
7470 unsigned ImmIdx = FrameRegIdx + 1;
7471
7472 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
7473 Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: ImmIdx).getImm());
7474 emitFrameOffset(MBB&: *MI.getParent(), MBBI: MI, DL: MI.getDebugLoc(),
7475 DestReg: MI.getOperand(i: 0).getReg(), SrcReg: FrameReg, Offset, TII,
7476 Flag: MachineInstr::NoFlags, SetNZCV: (Opcode == AArch64::ADDSXri));
7477 MI.eraseFromParent();
7478 Offset = StackOffset();
7479 return true;
7480 }
7481
7482 int64_t NewOffset;
7483 unsigned UnscaledOp;
7484 bool UseUnscaledOp;
7485 int Status = isAArch64FrameOffsetLegal(MI, SOffset&: Offset, OutUseUnscaledOp: &UseUnscaledOp,
7486 OutUnscaledOp: &UnscaledOp, EmittableOffset: &NewOffset);
7487 if (Status & AArch64FrameOffsetCanUpdate) {
7488 if (Status & AArch64FrameOffsetIsLegal)
7489 // Replace the FrameIndex with FrameReg.
7490 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
7491 if (UseUnscaledOp)
7492 MI.setDesc(TII->get(Opcode: UnscaledOp));
7493
7494 MI.getOperand(i: ImmIdx).ChangeToImmediate(ImmVal: NewOffset);
7495 return !Offset;
7496 }
7497
7498 return false;
7499}
7500
7501void AArch64InstrInfo::insertNoop(MachineBasicBlock &MBB,
7502 MachineBasicBlock::iterator MI) const {
7503 DebugLoc DL;
7504 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::NOP));
7505}
7506
7507MCInst AArch64InstrInfo::getNop() const { return MCInstBuilder(AArch64::NOP); }
7508
7509// AArch64 supports MachineCombiner.
7510bool AArch64InstrInfo::useMachineCombiner() const { return true; }
7511
7512// True when Opc sets flag
7513static bool isCombineInstrSettingFlag(unsigned Opc) {
7514 switch (Opc) {
7515 case AArch64::ADDSWrr:
7516 case AArch64::ADDSWri:
7517 case AArch64::ADDSXrr:
7518 case AArch64::ADDSXri:
7519 case AArch64::SUBSWrr:
7520 case AArch64::SUBSXrr:
7521 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7522 case AArch64::SUBSWri:
7523 case AArch64::SUBSXri:
7524 return true;
7525 default:
7526 break;
7527 }
7528 return false;
7529}
7530
7531// 32b Opcodes that can be combined with a MUL
7532static bool isCombineInstrCandidate32(unsigned Opc) {
7533 switch (Opc) {
7534 case AArch64::ADDWrr:
7535 case AArch64::ADDWri:
7536 case AArch64::SUBWrr:
7537 case AArch64::ADDSWrr:
7538 case AArch64::ADDSWri:
7539 case AArch64::SUBSWrr:
7540 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7541 case AArch64::SUBWri:
7542 case AArch64::SUBSWri:
7543 return true;
7544 default:
7545 break;
7546 }
7547 return false;
7548}
7549
7550// 64b Opcodes that can be combined with a MUL
7551static bool isCombineInstrCandidate64(unsigned Opc) {
7552 switch (Opc) {
7553 case AArch64::ADDXrr:
7554 case AArch64::ADDXri:
7555 case AArch64::SUBXrr:
7556 case AArch64::ADDSXrr:
7557 case AArch64::ADDSXri:
7558 case AArch64::SUBSXrr:
7559 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7560 case AArch64::SUBXri:
7561 case AArch64::SUBSXri:
7562 case AArch64::ADDv8i8:
7563 case AArch64::ADDv16i8:
7564 case AArch64::ADDv4i16:
7565 case AArch64::ADDv8i16:
7566 case AArch64::ADDv2i32:
7567 case AArch64::ADDv4i32:
7568 case AArch64::SUBv8i8:
7569 case AArch64::SUBv16i8:
7570 case AArch64::SUBv4i16:
7571 case AArch64::SUBv8i16:
7572 case AArch64::SUBv2i32:
7573 case AArch64::SUBv4i32:
7574 return true;
7575 default:
7576 break;
7577 }
7578 return false;
7579}
7580
7581// FP Opcodes that can be combined with a FMUL.
7582static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
7583 switch (Inst.getOpcode()) {
7584 default:
7585 break;
7586 case AArch64::FADDHrr:
7587 case AArch64::FADDSrr:
7588 case AArch64::FADDDrr:
7589 case AArch64::FADDv4f16:
7590 case AArch64::FADDv8f16:
7591 case AArch64::FADDv2f32:
7592 case AArch64::FADDv2f64:
7593 case AArch64::FADDv4f32:
7594 case AArch64::FSUBHrr:
7595 case AArch64::FSUBSrr:
7596 case AArch64::FSUBDrr:
7597 case AArch64::FSUBv4f16:
7598 case AArch64::FSUBv8f16:
7599 case AArch64::FSUBv2f32:
7600 case AArch64::FSUBv2f64:
7601 case AArch64::FSUBv4f32:
7602 // We can fuse FADD/FSUB with FMUL, if FADD/FSUB has the contract fast-math
7603 // flag.
7604 return Inst.getFlag(Flag: MachineInstr::FmContract);
7605 }
7606 return false;
7607}
7608
7609// Opcodes that can be combined with a MUL
7610static bool isCombineInstrCandidate(unsigned Opc) {
7611 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
7612}
7613
7614//
7615// Utility routine that checks if \param MO is defined by an
7616// \param CombineOpc instruction in the basic block \param MBB
7617static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
7618 unsigned CombineOpc, unsigned ZeroReg = 0,
7619 bool CheckZeroReg = false) {
7620 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7621 MachineInstr *MI = nullptr;
7622
7623 if (MO.isReg() && MO.getReg().isVirtual())
7624 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
7625 // And it needs to be in the trace (otherwise, it won't have a depth).
7626 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
7627 return false;
7628 // Must only used by the user we combine with.
7629 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
7630 return false;
7631
7632 if (CheckZeroReg) {
7633 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
7634 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
7635 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
7636 // The third input reg must be zero.
7637 if (MI->getOperand(i: 3).getReg() != ZeroReg)
7638 return false;
7639 }
7640
7641 if (isCombineInstrSettingFlag(Opc: CombineOpc) &&
7642 MI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) == -1)
7643 return false;
7644
7645 return true;
7646}
7647
7648//
7649// Is \param MO defined by an integer multiply and can be combined?
7650static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7651 unsigned MulOpc, unsigned ZeroReg) {
7652 return canCombine(MBB, MO, CombineOpc: MulOpc, ZeroReg, CheckZeroReg: true);
7653}
7654
7655//
7656// Is \param MO defined by a floating-point multiply and can be combined?
7657static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7658 unsigned MulOpc) {
7659 return canCombine(MBB, MO, CombineOpc: MulOpc);
7660}
7661
7662// TODO: There are many more machine instruction opcodes to match:
7663// 1. Other data types (integer, vectors)
7664// 2. Other math / logic operations (xor, or)
7665// 3. Other forms of the same operation (intrinsics and other variants)
7666bool AArch64InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
7667 bool Invert) const {
7668 if (Invert)
7669 return false;
7670 switch (Inst.getOpcode()) {
7671 // == Floating-point types ==
7672 // -- Floating-point instructions --
7673 case AArch64::FADDHrr:
7674 case AArch64::FADDSrr:
7675 case AArch64::FADDDrr:
7676 case AArch64::FMULHrr:
7677 case AArch64::FMULSrr:
7678 case AArch64::FMULDrr:
7679 case AArch64::FMULX16:
7680 case AArch64::FMULX32:
7681 case AArch64::FMULX64:
7682 // -- Advanced SIMD instructions --
7683 case AArch64::FADDv4f16:
7684 case AArch64::FADDv8f16:
7685 case AArch64::FADDv2f32:
7686 case AArch64::FADDv4f32:
7687 case AArch64::FADDv2f64:
7688 case AArch64::FMULv4f16:
7689 case AArch64::FMULv8f16:
7690 case AArch64::FMULv2f32:
7691 case AArch64::FMULv4f32:
7692 case AArch64::FMULv2f64:
7693 case AArch64::FMULXv4f16:
7694 case AArch64::FMULXv8f16:
7695 case AArch64::FMULXv2f32:
7696 case AArch64::FMULXv4f32:
7697 case AArch64::FMULXv2f64:
7698 // -- SVE instructions --
7699 // Opcodes FMULX_ZZZ_? don't exist because there is no unpredicated FMULX
7700 // in the SVE instruction set (though there are predicated ones).
7701 case AArch64::FADD_ZZZ_H:
7702 case AArch64::FADD_ZZZ_S:
7703 case AArch64::FADD_ZZZ_D:
7704 case AArch64::FMUL_ZZZ_H:
7705 case AArch64::FMUL_ZZZ_S:
7706 case AArch64::FMUL_ZZZ_D:
7707 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
7708 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
7709
7710 // == Integer types ==
7711 // -- Base instructions --
7712 // Opcodes MULWrr and MULXrr don't exist because
7713 // `MUL <Wd>, <Wn>, <Wm>` and `MUL <Xd>, <Xn>, <Xm>` are aliases of
7714 // `MADD <Wd>, <Wn>, <Wm>, WZR` and `MADD <Xd>, <Xn>, <Xm>, XZR` respectively.
7715 // The machine-combiner does not support three-source-operands machine
7716 // instruction. So we cannot reassociate MULs.
7717 case AArch64::ADDWrr:
7718 case AArch64::ADDXrr:
7719 case AArch64::ANDWrr:
7720 case AArch64::ANDXrr:
7721 case AArch64::ORRWrr:
7722 case AArch64::ORRXrr:
7723 case AArch64::EORWrr:
7724 case AArch64::EORXrr:
7725 case AArch64::EONWrr:
7726 case AArch64::EONXrr:
7727 // -- Advanced SIMD instructions --
7728 // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL
7729 // in the Advanced SIMD instruction set.
7730 case AArch64::ADDv8i8:
7731 case AArch64::ADDv16i8:
7732 case AArch64::ADDv4i16:
7733 case AArch64::ADDv8i16:
7734 case AArch64::ADDv2i32:
7735 case AArch64::ADDv4i32:
7736 case AArch64::ADDv1i64:
7737 case AArch64::ADDv2i64:
7738 case AArch64::MULv8i8:
7739 case AArch64::MULv16i8:
7740 case AArch64::MULv4i16:
7741 case AArch64::MULv8i16:
7742 case AArch64::MULv2i32:
7743 case AArch64::MULv4i32:
7744 case AArch64::ANDv8i8:
7745 case AArch64::ANDv16i8:
7746 case AArch64::ORRv8i8:
7747 case AArch64::ORRv16i8:
7748 case AArch64::EORv8i8:
7749 case AArch64::EORv16i8:
7750 // -- SVE instructions --
7751 case AArch64::ADD_ZZZ_B:
7752 case AArch64::ADD_ZZZ_H:
7753 case AArch64::ADD_ZZZ_S:
7754 case AArch64::ADD_ZZZ_D:
7755 case AArch64::MUL_ZZZ_B:
7756 case AArch64::MUL_ZZZ_H:
7757 case AArch64::MUL_ZZZ_S:
7758 case AArch64::MUL_ZZZ_D:
7759 case AArch64::AND_ZZZ:
7760 case AArch64::ORR_ZZZ:
7761 case AArch64::EOR_ZZZ:
7762 return true;
7763
7764 default:
7765 return false;
7766 }
7767}
7768
7769/// Find instructions that can be turned into madd.
7770static bool getMaddPatterns(MachineInstr &Root,
7771 SmallVectorImpl<unsigned> &Patterns) {
7772 unsigned Opc = Root.getOpcode();
7773 MachineBasicBlock &MBB = *Root.getParent();
7774 bool Found = false;
7775
7776 if (!isCombineInstrCandidate(Opc))
7777 return false;
7778 if (isCombineInstrSettingFlag(Opc)) {
7779 int Cmp_NZCV =
7780 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
7781 // When NZCV is live bail out.
7782 if (Cmp_NZCV == -1)
7783 return false;
7784 unsigned NewOpc = convertToNonFlagSettingOpc(MI: Root);
7785 // When opcode can't change bail out.
7786 // CHECKME: do we miss any cases for opcode conversion?
7787 if (NewOpc == Opc)
7788 return false;
7789 Opc = NewOpc;
7790 }
7791
7792 auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
7793 unsigned Pattern) {
7794 if (canCombineWithMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode, ZeroReg)) {
7795 Patterns.push_back(Elt: Pattern);
7796 Found = true;
7797 }
7798 };
7799
7800 auto setVFound = [&](int Opcode, int Operand, unsigned Pattern) {
7801 if (canCombine(MBB, MO&: Root.getOperand(i: Operand), CombineOpc: Opcode)) {
7802 Patterns.push_back(Elt: Pattern);
7803 Found = true;
7804 }
7805 };
7806
7807 typedef AArch64MachineCombinerPattern MCP;
7808
7809 switch (Opc) {
7810 default:
7811 break;
7812 case AArch64::ADDWrr:
7813 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
7814 "ADDWrr does not have register operands");
7815 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
7816 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
7817 break;
7818 case AArch64::ADDXrr:
7819 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
7820 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
7821 break;
7822 case AArch64::SUBWrr:
7823 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
7824 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
7825 break;
7826 case AArch64::SUBXrr:
7827 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
7828 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
7829 break;
7830 case AArch64::ADDWri:
7831 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
7832 break;
7833 case AArch64::ADDXri:
7834 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
7835 break;
7836 case AArch64::SUBWri:
7837 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
7838 break;
7839 case AArch64::SUBXri:
7840 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
7841 break;
7842 case AArch64::ADDv8i8:
7843 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
7844 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
7845 break;
7846 case AArch64::ADDv16i8:
7847 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
7848 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
7849 break;
7850 case AArch64::ADDv4i16:
7851 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
7852 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
7853 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
7854 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
7855 break;
7856 case AArch64::ADDv8i16:
7857 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
7858 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
7859 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
7860 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
7861 break;
7862 case AArch64::ADDv2i32:
7863 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
7864 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
7865 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
7866 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
7867 break;
7868 case AArch64::ADDv4i32:
7869 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
7870 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
7871 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
7872 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
7873 break;
7874 case AArch64::SUBv8i8:
7875 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
7876 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
7877 break;
7878 case AArch64::SUBv16i8:
7879 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
7880 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
7881 break;
7882 case AArch64::SUBv4i16:
7883 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
7884 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7885 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7886 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7887 break;
7888 case AArch64::SUBv8i16:
7889 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7890 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7891 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7892 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7893 break;
7894 case AArch64::SUBv2i32:
7895 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7896 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7897 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7898 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7899 break;
7900 case AArch64::SUBv4i32:
7901 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7902 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7903 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7904 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7905 break;
7906 }
7907 return Found;
7908}
7909
7910bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
7911 switch (Opcode) {
7912 default:
7913 break;
7914 case AArch64::UABALB_ZZZ_D:
7915 case AArch64::UABALB_ZZZ_H:
7916 case AArch64::UABALB_ZZZ_S:
7917 case AArch64::UABALT_ZZZ_D:
7918 case AArch64::UABALT_ZZZ_H:
7919 case AArch64::UABALT_ZZZ_S:
7920 case AArch64::SABALB_ZZZ_D:
7921 case AArch64::SABALB_ZZZ_S:
7922 case AArch64::SABALB_ZZZ_H:
7923 case AArch64::SABALT_ZZZ_D:
7924 case AArch64::SABALT_ZZZ_S:
7925 case AArch64::SABALT_ZZZ_H:
7926 case AArch64::UABALv16i8_v8i16:
7927 case AArch64::UABALv2i32_v2i64:
7928 case AArch64::UABALv4i16_v4i32:
7929 case AArch64::UABALv4i32_v2i64:
7930 case AArch64::UABALv8i16_v4i32:
7931 case AArch64::UABALv8i8_v8i16:
7932 case AArch64::UABAv16i8:
7933 case AArch64::UABAv2i32:
7934 case AArch64::UABAv4i16:
7935 case AArch64::UABAv4i32:
7936 case AArch64::UABAv8i16:
7937 case AArch64::UABAv8i8:
7938 case AArch64::SABALv16i8_v8i16:
7939 case AArch64::SABALv2i32_v2i64:
7940 case AArch64::SABALv4i16_v4i32:
7941 case AArch64::SABALv4i32_v2i64:
7942 case AArch64::SABALv8i16_v4i32:
7943 case AArch64::SABALv8i8_v8i16:
7944 case AArch64::SABAv16i8:
7945 case AArch64::SABAv2i32:
7946 case AArch64::SABAv4i16:
7947 case AArch64::SABAv4i32:
7948 case AArch64::SABAv8i16:
7949 case AArch64::SABAv8i8:
7950 return true;
7951 }
7952
7953 return false;
7954}
7955
7956unsigned AArch64InstrInfo::getAccumulationStartOpcode(
7957 unsigned AccumulationOpcode) const {
7958 switch (AccumulationOpcode) {
7959 default:
7960 llvm_unreachable("Unsupported accumulation Opcode!");
7961 case AArch64::UABALB_ZZZ_D:
7962 return AArch64::UABDLB_ZZZ_D;
7963 case AArch64::UABALB_ZZZ_H:
7964 return AArch64::UABDLB_ZZZ_H;
7965 case AArch64::UABALB_ZZZ_S:
7966 return AArch64::UABDLB_ZZZ_S;
7967 case AArch64::UABALT_ZZZ_D:
7968 return AArch64::UABDLT_ZZZ_D;
7969 case AArch64::UABALT_ZZZ_H:
7970 return AArch64::UABDLT_ZZZ_H;
7971 case AArch64::UABALT_ZZZ_S:
7972 return AArch64::UABDLT_ZZZ_S;
7973 case AArch64::UABALv16i8_v8i16:
7974 return AArch64::UABDLv16i8_v8i16;
7975 case AArch64::UABALv2i32_v2i64:
7976 return AArch64::UABDLv2i32_v2i64;
7977 case AArch64::UABALv4i16_v4i32:
7978 return AArch64::UABDLv4i16_v4i32;
7979 case AArch64::UABALv4i32_v2i64:
7980 return AArch64::UABDLv4i32_v2i64;
7981 case AArch64::UABALv8i16_v4i32:
7982 return AArch64::UABDLv8i16_v4i32;
7983 case AArch64::UABALv8i8_v8i16:
7984 return AArch64::UABDLv8i8_v8i16;
7985 case AArch64::UABAv16i8:
7986 return AArch64::UABDv16i8;
7987 case AArch64::UABAv2i32:
7988 return AArch64::UABDv2i32;
7989 case AArch64::UABAv4i16:
7990 return AArch64::UABDv4i16;
7991 case AArch64::UABAv4i32:
7992 return AArch64::UABDv4i32;
7993 case AArch64::UABAv8i16:
7994 return AArch64::UABDv8i16;
7995 case AArch64::UABAv8i8:
7996 return AArch64::UABDv8i8;
7997 case AArch64::SABALB_ZZZ_D:
7998 return AArch64::SABDLB_ZZZ_D;
7999 case AArch64::SABALB_ZZZ_S:
8000 return AArch64::SABDLB_ZZZ_S;
8001 case AArch64::SABALB_ZZZ_H:
8002 return AArch64::SABDLB_ZZZ_H;
8003 case AArch64::SABALT_ZZZ_D:
8004 return AArch64::SABDLT_ZZZ_D;
8005 case AArch64::SABALT_ZZZ_S:
8006 return AArch64::SABDLT_ZZZ_S;
8007 case AArch64::SABALT_ZZZ_H:
8008 return AArch64::SABDLT_ZZZ_H;
8009 case AArch64::SABALv16i8_v8i16:
8010 return AArch64::SABDLv16i8_v8i16;
8011 case AArch64::SABALv2i32_v2i64:
8012 return AArch64::SABDLv2i32_v2i64;
8013 case AArch64::SABALv4i16_v4i32:
8014 return AArch64::SABDLv4i16_v4i32;
8015 case AArch64::SABALv4i32_v2i64:
8016 return AArch64::SABDLv4i32_v2i64;
8017 case AArch64::SABALv8i16_v4i32:
8018 return AArch64::SABDLv8i16_v4i32;
8019 case AArch64::SABALv8i8_v8i16:
8020 return AArch64::SABDLv8i8_v8i16;
8021 case AArch64::SABAv16i8:
8022 return AArch64::SABDv16i8;
8023 case AArch64::SABAv2i32:
8024 return AArch64::SABDv2i32;
8025 case AArch64::SABAv4i16:
8026 return AArch64::SABDv4i16;
8027 case AArch64::SABAv4i32:
8028 return AArch64::SABDv4i32;
8029 case AArch64::SABAv8i16:
8030 return AArch64::SABDv8i16;
8031 case AArch64::SABAv8i8:
8032 return AArch64::SABDv8i8;
8033 }
8034}
8035
8036/// Floating-Point Support
8037
8038/// Find instructions that can be turned into madd.
8039static bool getFMAPatterns(MachineInstr &Root,
8040 SmallVectorImpl<unsigned> &Patterns) {
8041
8042 if (!isCombineInstrCandidateFP(Inst: Root))
8043 return false;
8044
8045 MachineBasicBlock &MBB = *Root.getParent();
8046 bool Found = false;
8047
8048 auto Match = [&](int Opcode, int Operand, unsigned Pattern) -> bool {
8049 if (canCombineWithFMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode)) {
8050 Patterns.push_back(Elt: Pattern);
8051 return true;
8052 }
8053 return false;
8054 };
8055
8056 typedef AArch64MachineCombinerPattern MCP;
8057
8058 switch (Root.getOpcode()) {
8059 default:
8060 assert(false && "Unsupported FP instruction in combiner\n");
8061 break;
8062 case AArch64::FADDHrr:
8063 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8064 "FADDHrr does not have register operands");
8065
8066 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
8067 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
8068 break;
8069 case AArch64::FADDSrr:
8070 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8071 "FADDSrr does not have register operands");
8072
8073 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
8074 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
8075
8076 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
8077 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
8078 break;
8079 case AArch64::FADDDrr:
8080 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
8081 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
8082
8083 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
8084 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
8085 break;
8086 case AArch64::FADDv4f16:
8087 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
8088 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
8089
8090 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
8091 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
8092 break;
8093 case AArch64::FADDv8f16:
8094 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
8095 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
8096
8097 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
8098 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
8099 break;
8100 case AArch64::FADDv2f32:
8101 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
8102 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
8103
8104 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
8105 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
8106 break;
8107 case AArch64::FADDv2f64:
8108 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
8109 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
8110
8111 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
8112 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
8113 break;
8114 case AArch64::FADDv4f32:
8115 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
8116 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
8117
8118 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
8119 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
8120 break;
8121 case AArch64::FSUBHrr:
8122 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
8123 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
8124 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
8125 break;
8126 case AArch64::FSUBSrr:
8127 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
8128
8129 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
8130 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
8131
8132 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
8133 break;
8134 case AArch64::FSUBDrr:
8135 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
8136
8137 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
8138 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
8139
8140 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
8141 break;
8142 case AArch64::FSUBv4f16:
8143 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
8144 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
8145
8146 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
8147 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
8148 break;
8149 case AArch64::FSUBv8f16:
8150 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
8151 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
8152
8153 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
8154 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
8155 break;
8156 case AArch64::FSUBv2f32:
8157 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
8158 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
8159
8160 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
8161 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
8162 break;
8163 case AArch64::FSUBv2f64:
8164 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
8165 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
8166
8167 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
8168 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
8169 break;
8170 case AArch64::FSUBv4f32:
8171 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
8172 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
8173
8174 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
8175 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
8176 break;
8177 }
8178 return Found;
8179}
8180
8181static bool getFMULPatterns(MachineInstr &Root,
8182 SmallVectorImpl<unsigned> &Patterns) {
8183 MachineBasicBlock &MBB = *Root.getParent();
8184 bool Found = false;
8185
8186 auto Match = [&](unsigned Opcode, int Operand, unsigned Pattern) -> bool {
8187 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8188 MachineOperand &MO = Root.getOperand(i: Operand);
8189 MachineInstr *MI = nullptr;
8190 if (MO.isReg() && MO.getReg().isVirtual())
8191 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8192 // Ignore No-op COPYs in FMUL(COPY(DUP(..)))
8193 if (MI && MI->getOpcode() == TargetOpcode::COPY &&
8194 MI->getOperand(i: 1).getReg().isVirtual())
8195 MI = MRI.getUniqueVRegDef(Reg: MI->getOperand(i: 1).getReg());
8196 if (MI && MI->getOpcode() == Opcode) {
8197 Patterns.push_back(Elt: Pattern);
8198 return true;
8199 }
8200 return false;
8201 };
8202
8203 typedef AArch64MachineCombinerPattern MCP;
8204
8205 switch (Root.getOpcode()) {
8206 default:
8207 return false;
8208 case AArch64::FMULv2f32:
8209 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
8210 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
8211 break;
8212 case AArch64::FMULv2f64:
8213 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
8214 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
8215 break;
8216 case AArch64::FMULv4f16:
8217 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
8218 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
8219 break;
8220 case AArch64::FMULv4f32:
8221 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
8222 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
8223 break;
8224 case AArch64::FMULv8f16:
8225 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
8226 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
8227 break;
8228 }
8229
8230 return Found;
8231}
8232
8233static bool getFNEGPatterns(MachineInstr &Root,
8234 SmallVectorImpl<unsigned> &Patterns) {
8235 unsigned Opc = Root.getOpcode();
8236 MachineBasicBlock &MBB = *Root.getParent();
8237 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8238
8239 auto Match = [&](unsigned Opcode, unsigned Pattern) -> bool {
8240 MachineOperand &MO = Root.getOperand(i: 1);
8241 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8242 if (MI != nullptr && (MI->getOpcode() == Opcode) &&
8243 MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()) &&
8244 Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8245 Root.getFlag(Flag: MachineInstr::MIFlag::FmNsz) &&
8246 MI->getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8247 MI->getFlag(Flag: MachineInstr::MIFlag::FmNsz)) {
8248 Patterns.push_back(Elt: Pattern);
8249 return true;
8250 }
8251 return false;
8252 };
8253
8254 switch (Opc) {
8255 default:
8256 break;
8257 case AArch64::FNEGDr:
8258 return Match(AArch64::FMADDDrrr, AArch64MachineCombinerPattern::FNMADD);
8259 case AArch64::FNEGSr:
8260 return Match(AArch64::FMADDSrrr, AArch64MachineCombinerPattern::FNMADD);
8261 }
8262
8263 return false;
8264}
8265
8266/// Return true when a code sequence can improve throughput. It
8267/// should be called only for instructions in loops.
8268/// \param Pattern - combiner pattern
8269bool AArch64InstrInfo::isThroughputPattern(unsigned Pattern) const {
8270 switch (Pattern) {
8271 default:
8272 break;
8273 case AArch64MachineCombinerPattern::FMULADDH_OP1:
8274 case AArch64MachineCombinerPattern::FMULADDH_OP2:
8275 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
8276 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
8277 case AArch64MachineCombinerPattern::FMULADDS_OP1:
8278 case AArch64MachineCombinerPattern::FMULADDS_OP2:
8279 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
8280 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
8281 case AArch64MachineCombinerPattern::FMULADDD_OP1:
8282 case AArch64MachineCombinerPattern::FMULADDD_OP2:
8283 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
8284 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
8285 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
8286 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
8287 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
8288 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
8289 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
8290 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
8291 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
8292 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
8293 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
8294 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
8295 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
8296 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
8297 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
8298 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
8299 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
8300 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
8301 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
8302 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
8303 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
8304 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
8305 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
8306 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
8307 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
8308 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
8309 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
8310 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
8311 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
8312 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1:
8313 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
8314 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1:
8315 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
8316 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
8317 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
8318 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
8319 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
8320 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
8321 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
8322 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
8323 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
8324 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
8325 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
8326 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
8327 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
8328 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
8329 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2:
8330 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
8331 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2:
8332 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
8333 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2:
8334 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
8335 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2:
8336 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
8337 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2:
8338 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
8339 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
8340 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
8341 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
8342 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
8343 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
8344 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
8345 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
8346 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
8347 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
8348 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
8349 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
8350 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
8351 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
8352 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
8353 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
8354 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
8355 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
8356 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
8357 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
8358 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
8359 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
8360 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
8361 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
8362 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
8363 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
8364 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
8365 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
8366 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
8367 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
8368 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
8369 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
8370 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
8371 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
8372 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
8373 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
8374 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
8375 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
8376 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
8377 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
8378 return true;
8379 } // end switch (Pattern)
8380 return false;
8381}
8382
8383/// Find other MI combine patterns.
8384static bool getMiscPatterns(MachineInstr &Root,
8385 SmallVectorImpl<unsigned> &Patterns) {
8386 // A - (B + C) ==> (A - B) - C or (A - C) - B
8387 unsigned Opc = Root.getOpcode();
8388 MachineBasicBlock &MBB = *Root.getParent();
8389
8390 switch (Opc) {
8391 case AArch64::SUBWrr:
8392 case AArch64::SUBSWrr:
8393 case AArch64::SUBXrr:
8394 case AArch64::SUBSXrr:
8395 // Found candidate root.
8396 break;
8397 default:
8398 return false;
8399 }
8400
8401 if (isCombineInstrSettingFlag(Opc) &&
8402 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) ==
8403 -1)
8404 return false;
8405
8406 if (canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDWrr) ||
8407 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSWrr) ||
8408 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDXrr) ||
8409 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSXrr)) {
8410 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP1);
8411 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP2);
8412 return true;
8413 }
8414
8415 return false;
8416}
8417
8418/// Check if the given instruction forms a gather load pattern that can be
8419/// optimized for better Memory-Level Parallelism (MLP). This function
8420/// identifies chains of NEON lane load instructions that load data from
8421/// different memory addresses into individual lanes of a 128-bit vector
8422/// register, then attempts to split the pattern into parallel loads to break
8423/// the serial dependency between instructions.
8424///
8425/// Pattern Matched:
8426/// Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) ->
8427/// LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)
8428///
8429/// Transformed Into:
8430/// Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64
8431/// to combine the results, enabling better memory-level parallelism.
8432///
8433/// Supported Element Types:
8434/// - 32-bit elements (LD1i32, 4 lanes total)
8435/// - 16-bit elements (LD1i16, 8 lanes total)
8436/// - 8-bit elements (LD1i8, 16 lanes total)
8437static bool getGatherLanePattern(MachineInstr &Root,
8438 SmallVectorImpl<unsigned> &Patterns,
8439 unsigned LoadLaneOpCode, unsigned NumLanes) {
8440 const MachineFunction *MF = Root.getMF();
8441
8442 // Early exit if optimizing for size.
8443 if (MF->getFunction().hasMinSize())
8444 return false;
8445
8446 const MachineRegisterInfo &MRI = MF->getRegInfo();
8447 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
8448
8449 // The root of the pattern must load into the last lane of the vector.
8450 if (Root.getOperand(i: 2).getImm() != NumLanes - 1)
8451 return false;
8452
8453 // Check that we have load into all lanes except lane 0.
8454 // For each load we also want to check that:
8455 // 1. It has a single non-debug use (since we will be replacing the virtual
8456 // register)
8457 // 2. That the addressing mode only uses a single pointer operand
8458 auto *CurrInstr = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8459 auto Range = llvm::seq<unsigned>(Begin: 1, End: NumLanes - 1);
8460 SmallSet<unsigned, 16> RemainingLanes(Range.begin(), Range.end());
8461 SmallVector<const MachineInstr *, 16> LoadInstrs;
8462 while (!RemainingLanes.empty() && CurrInstr &&
8463 CurrInstr->getOpcode() == LoadLaneOpCode &&
8464 MRI.hasOneNonDBGUse(RegNo: CurrInstr->getOperand(i: 0).getReg()) &&
8465 CurrInstr->getNumOperands() == 4) {
8466 RemainingLanes.erase(V: CurrInstr->getOperand(i: 2).getImm());
8467 LoadInstrs.push_back(Elt: CurrInstr);
8468 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8469 }
8470
8471 // Check that we have found a match for lanes N-1.. 1.
8472 if (!RemainingLanes.empty())
8473 return false;
8474
8475 // Match the SUBREG_TO_REG sequence.
8476 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
8477 return false;
8478
8479 // Verify that the subreg to reg loads an integer into the first lane.
8480 auto Lane0LoadReg = CurrInstr->getOperand(i: 1).getReg();
8481 unsigned SingleLaneSizeInBits = 128 / NumLanes;
8482 if (TRI->getRegSizeInBits(Reg: Lane0LoadReg, MRI) != SingleLaneSizeInBits)
8483 return false;
8484
8485 // Verify that it also has a single non debug use.
8486 if (!MRI.hasOneNonDBGUse(RegNo: Lane0LoadReg))
8487 return false;
8488
8489 LoadInstrs.push_back(Elt: MRI.getUniqueVRegDef(Reg: Lane0LoadReg));
8490
8491 // If there is any chance of aliasing, do not apply the pattern.
8492 // Walk backward through the MBB starting from Root.
8493 // Exit early if we've encountered all load instructions or hit the search
8494 // limit.
8495 auto MBBItr = Root.getIterator();
8496 unsigned RemainingSteps =
8497 MF->getSubtarget<AArch64Subtarget>().getCLOpts().search_limit;
8498 SmallPtrSet<const MachineInstr *, 16> RemainingLoadInstrs;
8499 RemainingLoadInstrs.insert(I: LoadInstrs.begin(), E: LoadInstrs.end());
8500 const MachineBasicBlock *MBB = Root.getParent();
8501
8502 for (; MBBItr != MBB->begin() && RemainingSteps > 0 &&
8503 !RemainingLoadInstrs.empty();
8504 --MBBItr, --RemainingSteps) {
8505 const MachineInstr &CurrInstr = *MBBItr;
8506
8507 // Remove this instruction from remaining loads if it's one we're tracking.
8508 RemainingLoadInstrs.erase(Ptr: &CurrInstr);
8509
8510 // Check for potential aliasing with any of the load instructions to
8511 // optimize.
8512 if (CurrInstr.isLoadFoldBarrier())
8513 return false;
8514 }
8515
8516 // If we hit the search limit without finding all load instructions,
8517 // don't match the pattern.
8518 if (RemainingSteps == 0 && !RemainingLoadInstrs.empty())
8519 return false;
8520
8521 switch (NumLanes) {
8522 case 4:
8523 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i32);
8524 break;
8525 case 8:
8526 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i16);
8527 break;
8528 case 16:
8529 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i8);
8530 break;
8531 default:
8532 llvm_unreachable("Got bad number of lanes for gather pattern.");
8533 }
8534
8535 return true;
8536}
8537
8538/// Search for patterns of LD instructions we can optimize.
8539static bool getLoadPatterns(MachineInstr &Root,
8540 SmallVectorImpl<unsigned> &Patterns) {
8541
8542 // The pattern searches for loads into single lanes.
8543 switch (Root.getOpcode()) {
8544 case AArch64::LD1i32:
8545 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 4);
8546 case AArch64::LD1i16:
8547 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 8);
8548 case AArch64::LD1i8:
8549 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 16);
8550 default:
8551 return false;
8552 }
8553}
8554
8555/// Generate optimized instruction sequence for gather load patterns to improve
8556/// Memory-Level Parallelism (MLP). This function transforms a chain of
8557/// sequential NEON lane loads into parallel vector loads that can execute
8558/// concurrently.
8559static void
8560generateGatherLanePattern(MachineInstr &Root,
8561 SmallVectorImpl<MachineInstr *> &InsInstrs,
8562 SmallVectorImpl<MachineInstr *> &DelInstrs,
8563 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8564 unsigned Pattern, unsigned NumLanes) {
8565 MachineFunction &MF = *Root.getParent()->getParent();
8566 MachineRegisterInfo &MRI = MF.getRegInfo();
8567 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8568
8569 // Gather the initial load instructions to build the pattern.
8570 SmallVector<MachineInstr *, 16> LoadToLaneInstrs;
8571 MachineInstr *CurrInstr = &Root;
8572 for (unsigned i = 0; i < NumLanes - 1; ++i) {
8573 LoadToLaneInstrs.push_back(Elt: CurrInstr);
8574 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8575 }
8576
8577 // Sort the load instructions according to the lane.
8578 llvm::sort(C&: LoadToLaneInstrs,
8579 Comp: [](const MachineInstr *A, const MachineInstr *B) {
8580 return A->getOperand(i: 2).getImm() > B->getOperand(i: 2).getImm();
8581 });
8582
8583 MachineInstr *SubregToReg = CurrInstr;
8584 LoadToLaneInstrs.push_back(
8585 Elt: MRI.getUniqueVRegDef(Reg: SubregToReg->getOperand(i: 1).getReg()));
8586 auto LoadToLaneInstrsAscending = llvm::reverse(C&: LoadToLaneInstrs);
8587
8588 const TargetRegisterClass *FPR128RegClass =
8589 MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg());
8590
8591 // Helper lambda to create a LD1 instruction.
8592 auto CreateLD1Instruction = [&](MachineInstr *OriginalInstr,
8593 Register SrcRegister, unsigned Lane,
8594 Register OffsetRegister,
8595 bool OffsetRegisterKillState) {
8596 auto NewRegister = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8597 MachineInstrBuilder LoadIndexIntoRegister =
8598 BuildMI(MF, MIMD: MIMetadata(*OriginalInstr), MCID: TII->get(Opcode: Root.getOpcode()),
8599 DestReg: NewRegister)
8600 .addReg(RegNo: SrcRegister)
8601 .addImm(Val: Lane)
8602 .addReg(RegNo: OffsetRegister, Flags: getKillRegState(B: OffsetRegisterKillState))
8603 .setMemRefs(OriginalInstr->memoperands());
8604 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewRegister, y: InsInstrs.size()));
8605 InsInstrs.push_back(Elt: LoadIndexIntoRegister);
8606 return NewRegister;
8607 };
8608
8609 // Helper to create load instruction based on the NumLanes in the NEON
8610 // register we are rewriting.
8611 auto CreateLDRInstruction =
8612 [&](unsigned NumLanes, Register DestReg, Register OffsetReg,
8613 ArrayRef<MachineMemOperand *> MMOs) -> MachineInstrBuilder {
8614 unsigned Opcode;
8615 switch (NumLanes) {
8616 case 4:
8617 Opcode = AArch64::LDRSui;
8618 break;
8619 case 8:
8620 Opcode = AArch64::LDRHui;
8621 break;
8622 case 16:
8623 Opcode = AArch64::LDRBui;
8624 break;
8625 default:
8626 llvm_unreachable(
8627 "Got unsupported number of lanes in machine-combiner gather pattern");
8628 }
8629 // Immediate offset load
8630 return BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg)
8631 .addReg(RegNo: OffsetReg)
8632 .addImm(Val: 0)
8633 .setMemRefs(MMOs);
8634 };
8635
8636 // Load the remaining lanes into register 0.
8637 auto LanesToLoadToReg0 =
8638 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + 1,
8639 y: LoadToLaneInstrsAscending.begin() + NumLanes / 2);
8640 Register PrevReg = SubregToReg->getOperand(i: 0).getReg();
8641 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg0)) {
8642 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8643 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8644 OffsetRegOperand.getReg(),
8645 OffsetRegOperand.isKill());
8646 DelInstrs.push_back(Elt: LoadInstr);
8647 }
8648 Register LastLoadReg0 = PrevReg;
8649
8650 // First load into register 1. Perform an integer load to zero out the upper
8651 // lanes in a single instruction.
8652 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
8653 MachineInstr *OriginalSplitLoad =
8654 *std::next(x: LoadToLaneInstrsAscending.begin(), n: NumLanes / 2);
8655 Register DestRegForMiddleIndex = MRI.createVirtualRegister(
8656 RegClass: MRI.getRegClass(Reg: Lane0Load->getOperand(i: 0).getReg()));
8657
8658 const MachineOperand &OriginalSplitToLoadOffsetOperand =
8659 OriginalSplitLoad->getOperand(i: 3);
8660 MachineInstrBuilder MiddleIndexLoadInstr =
8661 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
8662 OriginalSplitToLoadOffsetOperand.getReg(),
8663 OriginalSplitLoad->memoperands());
8664
8665 InstrIdxForVirtReg.insert(
8666 KV: std::make_pair(x&: DestRegForMiddleIndex, y: InsInstrs.size()));
8667 InsInstrs.push_back(Elt: MiddleIndexLoadInstr);
8668 DelInstrs.push_back(Elt: OriginalSplitLoad);
8669
8670 // Subreg To Reg instruction for register 1.
8671 Register DestRegForSubregToReg = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8672 unsigned SubregType;
8673 switch (NumLanes) {
8674 case 4:
8675 SubregType = AArch64::ssub;
8676 break;
8677 case 8:
8678 SubregType = AArch64::hsub;
8679 break;
8680 case 16:
8681 SubregType = AArch64::bsub;
8682 break;
8683 default:
8684 llvm_unreachable(
8685 "Got invalid NumLanes for machine-combiner gather pattern");
8686 }
8687
8688 auto SubRegToRegInstr =
8689 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubregToReg->getOpcode()),
8690 DestReg: DestRegForSubregToReg)
8691 .addReg(RegNo: DestRegForMiddleIndex, Flags: getKillRegState(B: true))
8692 .addImm(Val: SubregType);
8693 InstrIdxForVirtReg.insert(
8694 KV: std::make_pair(x&: DestRegForSubregToReg, y: InsInstrs.size()));
8695 InsInstrs.push_back(Elt: SubRegToRegInstr);
8696
8697 // Load remaining lanes into register 1.
8698 auto LanesToLoadToReg1 =
8699 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + NumLanes / 2 + 1,
8700 y: LoadToLaneInstrsAscending.end());
8701 PrevReg = SubRegToRegInstr->getOperand(i: 0).getReg();
8702 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg1)) {
8703 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8704 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8705 OffsetRegOperand.getReg(),
8706 OffsetRegOperand.isKill());
8707
8708 // Do not add the last reg to DelInstrs - it will be removed later.
8709 if (Index == NumLanes / 2 - 2) {
8710 break;
8711 }
8712 DelInstrs.push_back(Elt: LoadInstr);
8713 }
8714 Register LastLoadReg1 = PrevReg;
8715
8716 // Create the final zip instruction to combine the results.
8717 MachineInstrBuilder ZipInstr =
8718 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::ZIP1v2i64),
8719 DestReg: Root.getOperand(i: 0).getReg())
8720 .addReg(RegNo: LastLoadReg0)
8721 .addReg(RegNo: LastLoadReg1);
8722 InsInstrs.push_back(Elt: ZipInstr);
8723}
8724
8725CombinerObjective
8726AArch64InstrInfo::getCombinerObjective(unsigned Pattern) const {
8727 switch (Pattern) {
8728 case AArch64MachineCombinerPattern::SUBADD_OP1:
8729 case AArch64MachineCombinerPattern::SUBADD_OP2:
8730 case AArch64MachineCombinerPattern::GATHER_LANE_i32:
8731 case AArch64MachineCombinerPattern::GATHER_LANE_i16:
8732 case AArch64MachineCombinerPattern::GATHER_LANE_i8:
8733 return CombinerObjective::MustReduceDepth;
8734 default:
8735 return TargetInstrInfo::getCombinerObjective(Pattern);
8736 }
8737}
8738
8739/// Return true when there is potentially a faster code sequence for an
8740/// instruction chain ending in \p Root. All potential patterns are listed in
8741/// the \p Pattern vector. Pattern should be sorted in priority order since the
8742/// pattern evaluator stops checking as soon as it finds a faster sequence.
8743
8744bool AArch64InstrInfo::getMachineCombinerPatterns(
8745 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
8746 bool DoRegPressureReduce) const {
8747 // Integer patterns
8748 if (getMaddPatterns(Root, Patterns))
8749 return true;
8750 // Floating point patterns
8751 if (getFMULPatterns(Root, Patterns))
8752 return true;
8753 if (getFMAPatterns(Root, Patterns))
8754 return true;
8755 if (getFNEGPatterns(Root, Patterns))
8756 return true;
8757
8758 // Other patterns
8759 if (getMiscPatterns(Root, Patterns))
8760 return true;
8761
8762 // Load patterns
8763 if (getLoadPatterns(Root, Patterns))
8764 return true;
8765
8766 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
8767 DoRegPressureReduce);
8768}
8769
8770enum class FMAInstKind { Default, Indexed, Accumulator };
8771/// genFusedMultiply - Generate fused multiply instructions.
8772/// This function supports both integer and floating point instructions.
8773/// A typical example:
8774/// F|MUL I=A,B,0
8775/// F|ADD R,I,C
8776/// ==> F|MADD R,A,B,C
8777/// \param MF Containing MachineFunction
8778/// \param MRI Register information
8779/// \param TII Target information
8780/// \param Root is the F|ADD instruction
8781/// \param [out] InsInstrs is a vector of machine instructions and will
8782/// contain the generated madd instruction
8783/// \param IdxMulOpd is index of operand in Root that is the result of
8784/// the F|MUL. In the example above IdxMulOpd is 1.
8785/// \param MaddOpc the opcode fo the f|madd instruction
8786/// \param RC Register class of operands
8787/// \param kind of fma instruction (addressing mode) to be generated
8788/// \param ReplacedAddend is the result register from the instruction
8789/// replacing the non-combined operand, if any.
8790static MachineInstr *
8791genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
8792 const TargetInstrInfo *TII, MachineInstr &Root,
8793 SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
8794 unsigned MaddOpc, const TargetRegisterClass *RC,
8795 FMAInstKind kind = FMAInstKind::Default,
8796 const Register *ReplacedAddend = nullptr) {
8797 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8798
8799 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
8800 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
8801 Register ResultReg = Root.getOperand(i: 0).getReg();
8802 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8803 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
8804 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8805 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
8806
8807 Register SrcReg2;
8808 bool Src2IsKill;
8809 if (ReplacedAddend) {
8810 // If we just generated a new addend, we must be it's only use.
8811 SrcReg2 = *ReplacedAddend;
8812 Src2IsKill = true;
8813 } else {
8814 SrcReg2 = Root.getOperand(i: IdxOtherOpd).getReg();
8815 Src2IsKill = Root.getOperand(i: IdxOtherOpd).isKill();
8816 }
8817
8818 if (ResultReg.isVirtual())
8819 MRI.constrainRegClass(Reg: ResultReg, RC);
8820 if (SrcReg0.isVirtual())
8821 MRI.constrainRegClass(Reg: SrcReg0, RC);
8822 if (SrcReg1.isVirtual())
8823 MRI.constrainRegClass(Reg: SrcReg1, RC);
8824 if (SrcReg2.isVirtual())
8825 MRI.constrainRegClass(Reg: SrcReg2, RC);
8826
8827 MachineInstrBuilder MIB;
8828 if (kind == FMAInstKind::Default)
8829 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8830 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8831 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8832 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8833 else if (kind == FMAInstKind::Indexed)
8834 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8835 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8836 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8837 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8838 .addImm(Val: MUL->getOperand(i: 3).getImm());
8839 else if (kind == FMAInstKind::Accumulator)
8840 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8841 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8842 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8843 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill));
8844 else
8845 assert(false && "Invalid FMA instruction kind \n");
8846 // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
8847 InsInstrs.push_back(Elt: MIB);
8848 return MUL;
8849}
8850
8851static MachineInstr *
8852genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI,
8853 const TargetInstrInfo *TII, MachineInstr &Root,
8854 SmallVectorImpl<MachineInstr *> &InsInstrs) {
8855 MachineInstr *MAD = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8856
8857 unsigned Opc = 0;
8858 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MAD->getOperand(i: 0).getReg());
8859 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
8860 Opc = AArch64::FNMADDSrrr;
8861 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
8862 Opc = AArch64::FNMADDDrrr;
8863 else
8864 return nullptr;
8865
8866 Register ResultReg = Root.getOperand(i: 0).getReg();
8867 Register SrcReg0 = MAD->getOperand(i: 1).getReg();
8868 Register SrcReg1 = MAD->getOperand(i: 2).getReg();
8869 Register SrcReg2 = MAD->getOperand(i: 3).getReg();
8870 bool Src0IsKill = MAD->getOperand(i: 1).isKill();
8871 bool Src1IsKill = MAD->getOperand(i: 2).isKill();
8872 bool Src2IsKill = MAD->getOperand(i: 3).isKill();
8873 if (ResultReg.isVirtual())
8874 MRI.constrainRegClass(Reg: ResultReg, RC);
8875 if (SrcReg0.isVirtual())
8876 MRI.constrainRegClass(Reg: SrcReg0, RC);
8877 if (SrcReg1.isVirtual())
8878 MRI.constrainRegClass(Reg: SrcReg1, RC);
8879 if (SrcReg2.isVirtual())
8880 MRI.constrainRegClass(Reg: SrcReg2, RC);
8881
8882 MachineInstrBuilder MIB =
8883 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Opc), DestReg: ResultReg)
8884 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8885 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8886 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8887 InsInstrs.push_back(Elt: MIB);
8888
8889 return MAD;
8890}
8891
8892/// Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
8893static MachineInstr *
8894genIndexedMultiply(MachineInstr &Root,
8895 SmallVectorImpl<MachineInstr *> &InsInstrs,
8896 unsigned IdxDupOp, unsigned MulOpc,
8897 const TargetRegisterClass *RC, MachineRegisterInfo &MRI) {
8898 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
8899 "Invalid index of FMUL operand");
8900
8901 MachineFunction &MF = *Root.getMF();
8902 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8903
8904 MachineInstr *Dup =
8905 MF.getRegInfo().getUniqueVRegDef(Reg: Root.getOperand(i: IdxDupOp).getReg());
8906
8907 if (Dup->getOpcode() == TargetOpcode::COPY)
8908 Dup = MRI.getUniqueVRegDef(Reg: Dup->getOperand(i: 1).getReg());
8909
8910 Register DupSrcReg = Dup->getOperand(i: 1).getReg();
8911 MRI.clearKillFlags(Reg: DupSrcReg);
8912 MRI.constrainRegClass(Reg: DupSrcReg, RC);
8913
8914 unsigned DupSrcLane = Dup->getOperand(i: 2).getImm();
8915
8916 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
8917 MachineOperand &MulOp = Root.getOperand(i: IdxMulOp);
8918
8919 Register ResultReg = Root.getOperand(i: 0).getReg();
8920
8921 MachineInstrBuilder MIB;
8922 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MulOpc), DestReg: ResultReg)
8923 .add(MO: MulOp)
8924 .addReg(RegNo: DupSrcReg)
8925 .addImm(Val: DupSrcLane);
8926
8927 InsInstrs.push_back(Elt: MIB);
8928 return &Root;
8929}
8930
8931/// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
8932/// instructions.
8933///
8934/// \see genFusedMultiply
8935static MachineInstr *genFusedMultiplyAcc(
8936 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8937 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8938 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8939 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8940 kind: FMAInstKind::Accumulator);
8941}
8942
8943/// genNeg - Helper to generate an intermediate negation of the second operand
8944/// of Root
8945static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
8946 const TargetInstrInfo *TII, MachineInstr &Root,
8947 SmallVectorImpl<MachineInstr *> &InsInstrs,
8948 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8949 unsigned MnegOpc, const TargetRegisterClass *RC) {
8950 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
8951 MachineInstrBuilder MIB =
8952 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MnegOpc), DestReg: NewVR)
8953 .add(MO: Root.getOperand(i: 2));
8954 InsInstrs.push_back(Elt: MIB);
8955
8956 assert(InstrIdxForVirtReg.empty());
8957 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
8958
8959 return NewVR;
8960}
8961
8962/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8963/// instructions with an additional negation of the accumulator
8964static MachineInstr *genFusedMultiplyAccNeg(
8965 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8966 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8967 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8968 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8969 assert(IdxMulOpd == 1);
8970
8971 Register NewVR =
8972 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8973 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8974 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
8975}
8976
8977/// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
8978/// instructions.
8979///
8980/// \see genFusedMultiply
8981static MachineInstr *genFusedMultiplyIdx(
8982 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8983 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8984 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8985 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8986 kind: FMAInstKind::Indexed);
8987}
8988
8989/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8990/// instructions with an additional negation of the accumulator
8991static MachineInstr *genFusedMultiplyIdxNeg(
8992 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8993 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8994 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8995 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8996 assert(IdxMulOpd == 1);
8997
8998 Register NewVR =
8999 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
9000
9001 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
9002 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9003}
9004
9005/// genMaddR - Generate madd instruction and combine mul and add using
9006/// an extra virtual register
9007/// Example - an ADD intermediate needs to be stored in a register:
9008/// MUL I=A,B,0
9009/// ADD R,I,Imm
9010/// ==> ORR V, ZR, Imm
9011/// ==> MADD R,A,B,V
9012/// \param MF Containing MachineFunction
9013/// \param MRI Register information
9014/// \param TII Target information
9015/// \param Root is the ADD instruction
9016/// \param [out] InsInstrs is a vector of machine instructions and will
9017/// contain the generated madd instruction
9018/// \param IdxMulOpd is index of operand in Root that is the result of
9019/// the MUL. In the example above IdxMulOpd is 1.
9020/// \param MaddOpc the opcode fo the madd instruction
9021/// \param VR is a virtual register that holds the value of an ADD operand
9022/// (V in the example above).
9023/// \param RC Register class of operands
9024static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
9025 const TargetInstrInfo *TII, MachineInstr &Root,
9026 SmallVectorImpl<MachineInstr *> &InsInstrs,
9027 unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
9028 const TargetRegisterClass *RC) {
9029 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
9030
9031 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
9032 Register ResultReg = Root.getOperand(i: 0).getReg();
9033 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
9034 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
9035 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
9036 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
9037
9038 if (ResultReg.isVirtual())
9039 MRI.constrainRegClass(Reg: ResultReg, RC);
9040 if (SrcReg0.isVirtual())
9041 MRI.constrainRegClass(Reg: SrcReg0, RC);
9042 if (SrcReg1.isVirtual())
9043 MRI.constrainRegClass(Reg: SrcReg1, RC);
9044 if (Register::isVirtualRegister(Reg: VR))
9045 MRI.constrainRegClass(Reg: VR, RC);
9046
9047 MachineInstrBuilder MIB =
9048 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
9049 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
9050 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
9051 .addReg(RegNo: VR);
9052 // Insert the MADD
9053 InsInstrs.push_back(Elt: MIB);
9054 return MUL;
9055}
9056
9057/// Do the following transformation
9058/// A - (B + C) ==> (A - B) - C
9059/// A - (B + C) ==> (A - C) - B
9060static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI,
9061 const TargetInstrInfo *TII, MachineInstr &Root,
9062 SmallVectorImpl<MachineInstr *> &InsInstrs,
9063 SmallVectorImpl<MachineInstr *> &DelInstrs,
9064 unsigned IdxOpd1,
9065 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
9066 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
9067 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
9068 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
9069
9070 Register ResultReg = Root.getOperand(i: 0).getReg();
9071 Register RegA = Root.getOperand(i: 1).getReg();
9072 bool RegAIsKill = Root.getOperand(i: 1).isKill();
9073 Register RegB = AddMI->getOperand(i: IdxOpd1).getReg();
9074 bool RegBIsKill = AddMI->getOperand(i: IdxOpd1).isKill();
9075 Register RegC = AddMI->getOperand(i: IdxOtherOpd).getReg();
9076 bool RegCIsKill = AddMI->getOperand(i: IdxOtherOpd).isKill();
9077 Register NewVR =
9078 MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 2).getReg()));
9079
9080 unsigned Opcode = Root.getOpcode();
9081 if (Opcode == AArch64::SUBSWrr)
9082 Opcode = AArch64::SUBWrr;
9083 else if (Opcode == AArch64::SUBSXrr)
9084 Opcode = AArch64::SUBXrr;
9085 else
9086 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
9087 "Unexpected instruction opcode.");
9088
9089 uint32_t Flags = Root.mergeFlagsWith(Other: *AddMI);
9090 Flags &= ~MachineInstr::NoSWrap;
9091 Flags &= ~MachineInstr::NoUWrap;
9092
9093 MachineInstrBuilder MIB1 =
9094 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: NewVR)
9095 .addReg(RegNo: RegA, Flags: getKillRegState(B: RegAIsKill))
9096 .addReg(RegNo: RegB, Flags: getKillRegState(B: RegBIsKill))
9097 .setMIFlags(Flags);
9098 MachineInstrBuilder MIB2 =
9099 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: ResultReg)
9100 .addReg(RegNo: NewVR, Flags: getKillRegState(B: true))
9101 .addReg(RegNo: RegC, Flags: getKillRegState(B: RegCIsKill))
9102 .setMIFlags(Flags);
9103
9104 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9105 InsInstrs.push_back(Elt: MIB1);
9106 InsInstrs.push_back(Elt: MIB2);
9107 DelInstrs.push_back(Elt: AddMI);
9108 DelInstrs.push_back(Elt: &Root);
9109}
9110
9111unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
9112 unsigned int AccumulatorOpCode) const {
9113 switch (AccumulatorOpCode) {
9114 case AArch64::UABALB_ZZZ_D:
9115 case AArch64::SABALB_ZZZ_D:
9116 case AArch64::UABALT_ZZZ_D:
9117 case AArch64::SABALT_ZZZ_D:
9118 return AArch64::ADD_ZZZ_D;
9119 case AArch64::UABALB_ZZZ_H:
9120 case AArch64::SABALB_ZZZ_H:
9121 case AArch64::UABALT_ZZZ_H:
9122 case AArch64::SABALT_ZZZ_H:
9123 return AArch64::ADD_ZZZ_H;
9124 case AArch64::UABALB_ZZZ_S:
9125 case AArch64::SABALB_ZZZ_S:
9126 case AArch64::UABALT_ZZZ_S:
9127 case AArch64::SABALT_ZZZ_S:
9128 return AArch64::ADD_ZZZ_S;
9129 case AArch64::UABALv16i8_v8i16:
9130 case AArch64::SABALv8i8_v8i16:
9131 case AArch64::SABAv8i16:
9132 case AArch64::UABAv8i16:
9133 return AArch64::ADDv8i16;
9134 case AArch64::SABALv2i32_v2i64:
9135 case AArch64::UABALv2i32_v2i64:
9136 case AArch64::SABALv4i32_v2i64:
9137 return AArch64::ADDv2i64;
9138 case AArch64::UABALv4i16_v4i32:
9139 case AArch64::SABALv4i16_v4i32:
9140 case AArch64::SABALv8i16_v4i32:
9141 case AArch64::SABAv4i32:
9142 case AArch64::UABAv4i32:
9143 return AArch64::ADDv4i32;
9144 case AArch64::UABALv4i32_v2i64:
9145 return AArch64::ADDv2i64;
9146 case AArch64::UABALv8i16_v4i32:
9147 return AArch64::ADDv4i32;
9148 case AArch64::UABALv8i8_v8i16:
9149 case AArch64::SABALv16i8_v8i16:
9150 return AArch64::ADDv8i16;
9151 case AArch64::UABAv16i8:
9152 case AArch64::SABAv16i8:
9153 return AArch64::ADDv16i8;
9154 case AArch64::UABAv4i16:
9155 case AArch64::SABAv4i16:
9156 return AArch64::ADDv4i16;
9157 case AArch64::UABAv2i32:
9158 case AArch64::SABAv2i32:
9159 return AArch64::ADDv2i32;
9160 case AArch64::UABAv8i8:
9161 case AArch64::SABAv8i8:
9162 return AArch64::ADDv8i8;
9163 default:
9164 llvm_unreachable("Unknown accumulator opcode");
9165 }
9166}
9167
9168/// When getMachineCombinerPatterns() finds potential patterns,
9169/// this function generates the instructions that could replace the
9170/// original code sequence
9171void AArch64InstrInfo::genAlternativeCodeSequence(
9172 MachineInstr &Root, unsigned Pattern,
9173 SmallVectorImpl<MachineInstr *> &InsInstrs,
9174 SmallVectorImpl<MachineInstr *> &DelInstrs,
9175 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
9176 MachineBasicBlock &MBB = *Root.getParent();
9177 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9178 MachineFunction &MF = *MBB.getParent();
9179 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
9180
9181 MachineInstr *MUL = nullptr;
9182 const TargetRegisterClass *RC;
9183 unsigned Opc;
9184 switch (Pattern) {
9185 default:
9186 // Reassociate instructions.
9187 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
9188 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
9189 return;
9190 case AArch64MachineCombinerPattern::SUBADD_OP1:
9191 // A - (B + C)
9192 // ==> (A - B) - C
9193 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 1,
9194 InstrIdxForVirtReg);
9195 return;
9196 case AArch64MachineCombinerPattern::SUBADD_OP2:
9197 // A - (B + C)
9198 // ==> (A - C) - B
9199 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 2,
9200 InstrIdxForVirtReg);
9201 return;
9202 case AArch64MachineCombinerPattern::MULADDW_OP1:
9203 case AArch64MachineCombinerPattern::MULADDX_OP1:
9204 // MUL I=A,B,0
9205 // ADD R,I,C
9206 // ==> MADD R,A,B,C
9207 // --- Create(MADD);
9208 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP1) {
9209 Opc = AArch64::MADDWrrr;
9210 RC = &AArch64::GPR32RegClass;
9211 } else {
9212 Opc = AArch64::MADDXrrr;
9213 RC = &AArch64::GPR64RegClass;
9214 }
9215 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9216 break;
9217 case AArch64MachineCombinerPattern::MULADDW_OP2:
9218 case AArch64MachineCombinerPattern::MULADDX_OP2:
9219 // MUL I=A,B,0
9220 // ADD R,C,I
9221 // ==> MADD R,A,B,C
9222 // --- Create(MADD);
9223 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP2) {
9224 Opc = AArch64::MADDWrrr;
9225 RC = &AArch64::GPR32RegClass;
9226 } else {
9227 Opc = AArch64::MADDXrrr;
9228 RC = &AArch64::GPR64RegClass;
9229 }
9230 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9231 break;
9232 case AArch64MachineCombinerPattern::MULADDWI_OP1:
9233 case AArch64MachineCombinerPattern::MULADDXI_OP1:
9234 case AArch64MachineCombinerPattern::MULSUBWI_OP1:
9235 case AArch64MachineCombinerPattern::MULSUBXI_OP1: {
9236 // MUL I=A,B,0
9237 // ADD/SUB R,I,Imm
9238 // ==> MOV V, Imm/-Imm
9239 // ==> MADD R,A,B,V
9240 // --- Create(MADD);
9241 const TargetRegisterClass *RC;
9242 unsigned BitSize, MovImm;
9243 if (Pattern == AArch64MachineCombinerPattern::MULADDWI_OP1 ||
9244 Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1) {
9245 MovImm = AArch64::MOVi32imm;
9246 RC = &AArch64::GPR32spRegClass;
9247 BitSize = 32;
9248 Opc = AArch64::MADDWrrr;
9249 RC = &AArch64::GPR32RegClass;
9250 } else {
9251 MovImm = AArch64::MOVi64imm;
9252 RC = &AArch64::GPR64spRegClass;
9253 BitSize = 64;
9254 Opc = AArch64::MADDXrrr;
9255 RC = &AArch64::GPR64RegClass;
9256 }
9257 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9258 uint64_t Imm = Root.getOperand(i: 2).getImm();
9259
9260 if (Root.getOperand(i: 3).isImm()) {
9261 unsigned Val = Root.getOperand(i: 3).getImm();
9262 Imm = Imm << Val;
9263 }
9264 bool IsSub = Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1 ||
9265 Pattern == AArch64MachineCombinerPattern::MULSUBXI_OP1;
9266 uint64_t UImm = SignExtend64(X: IsSub ? -Imm : Imm, B: BitSize);
9267 // Check that the immediate can be composed via a single instruction.
9268 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
9269 AArch64_IMM::expandMOVImm(Imm: UImm, BitSize, Insn);
9270 if (Insn.size() != 1)
9271 return;
9272 MachineInstrBuilder MIB1 =
9273 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MovImm), DestReg: NewVR)
9274 .addImm(Val: IsSub ? -Imm : Imm);
9275 InsInstrs.push_back(Elt: MIB1);
9276 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9277 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9278 break;
9279 }
9280 case AArch64MachineCombinerPattern::MULSUBW_OP1:
9281 case AArch64MachineCombinerPattern::MULSUBX_OP1: {
9282 // MUL I=A,B,0
9283 // SUB R,I, C
9284 // ==> SUB V, 0, C
9285 // ==> MADD R,A,B,V // = -C + A*B
9286 // --- Create(MADD);
9287 const TargetRegisterClass *SubRC;
9288 unsigned SubOpc, ZeroReg;
9289 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP1) {
9290 SubOpc = AArch64::SUBWrr;
9291 SubRC = &AArch64::GPR32spRegClass;
9292 ZeroReg = AArch64::WZR;
9293 Opc = AArch64::MADDWrrr;
9294 RC = &AArch64::GPR32RegClass;
9295 } else {
9296 SubOpc = AArch64::SUBXrr;
9297 SubRC = &AArch64::GPR64spRegClass;
9298 ZeroReg = AArch64::XZR;
9299 Opc = AArch64::MADDXrrr;
9300 RC = &AArch64::GPR64RegClass;
9301 }
9302 Register NewVR = MRI.createVirtualRegister(RegClass: SubRC);
9303 // SUB NewVR, 0, C
9304 MachineInstrBuilder MIB1 =
9305 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubOpc), DestReg: NewVR)
9306 .addReg(RegNo: ZeroReg)
9307 .add(MO: Root.getOperand(i: 2));
9308 InsInstrs.push_back(Elt: MIB1);
9309 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9310 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9311 break;
9312 }
9313 case AArch64MachineCombinerPattern::MULSUBW_OP2:
9314 case AArch64MachineCombinerPattern::MULSUBX_OP2:
9315 // MUL I=A,B,0
9316 // SUB R,C,I
9317 // ==> MSUB R,A,B,C (computes C - A*B)
9318 // --- Create(MSUB);
9319 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP2) {
9320 Opc = AArch64::MSUBWrrr;
9321 RC = &AArch64::GPR32RegClass;
9322 } else {
9323 Opc = AArch64::MSUBXrrr;
9324 RC = &AArch64::GPR64RegClass;
9325 }
9326 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9327 break;
9328 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
9329 Opc = AArch64::MLAv8i8;
9330 RC = &AArch64::FPR64RegClass;
9331 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9332 break;
9333 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
9334 Opc = AArch64::MLAv8i8;
9335 RC = &AArch64::FPR64RegClass;
9336 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9337 break;
9338 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
9339 Opc = AArch64::MLAv16i8;
9340 RC = &AArch64::FPR128RegClass;
9341 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9342 break;
9343 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
9344 Opc = AArch64::MLAv16i8;
9345 RC = &AArch64::FPR128RegClass;
9346 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9347 break;
9348 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
9349 Opc = AArch64::MLAv4i16;
9350 RC = &AArch64::FPR64RegClass;
9351 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9352 break;
9353 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
9354 Opc = AArch64::MLAv4i16;
9355 RC = &AArch64::FPR64RegClass;
9356 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9357 break;
9358 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
9359 Opc = AArch64::MLAv8i16;
9360 RC = &AArch64::FPR128RegClass;
9361 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9362 break;
9363 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
9364 Opc = AArch64::MLAv8i16;
9365 RC = &AArch64::FPR128RegClass;
9366 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9367 break;
9368 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
9369 Opc = AArch64::MLAv2i32;
9370 RC = &AArch64::FPR64RegClass;
9371 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9372 break;
9373 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
9374 Opc = AArch64::MLAv2i32;
9375 RC = &AArch64::FPR64RegClass;
9376 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9377 break;
9378 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
9379 Opc = AArch64::MLAv4i32;
9380 RC = &AArch64::FPR128RegClass;
9381 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9382 break;
9383 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
9384 Opc = AArch64::MLAv4i32;
9385 RC = &AArch64::FPR128RegClass;
9386 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9387 break;
9388
9389 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
9390 Opc = AArch64::MLAv8i8;
9391 RC = &AArch64::FPR64RegClass;
9392 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9393 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i8,
9394 RC);
9395 break;
9396 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
9397 Opc = AArch64::MLSv8i8;
9398 RC = &AArch64::FPR64RegClass;
9399 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9400 break;
9401 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
9402 Opc = AArch64::MLAv16i8;
9403 RC = &AArch64::FPR128RegClass;
9404 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9405 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv16i8,
9406 RC);
9407 break;
9408 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
9409 Opc = AArch64::MLSv16i8;
9410 RC = &AArch64::FPR128RegClass;
9411 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9412 break;
9413 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
9414 Opc = AArch64::MLAv4i16;
9415 RC = &AArch64::FPR64RegClass;
9416 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9417 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9418 RC);
9419 break;
9420 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
9421 Opc = AArch64::MLSv4i16;
9422 RC = &AArch64::FPR64RegClass;
9423 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9424 break;
9425 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
9426 Opc = AArch64::MLAv8i16;
9427 RC = &AArch64::FPR128RegClass;
9428 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9429 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9430 RC);
9431 break;
9432 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
9433 Opc = AArch64::MLSv8i16;
9434 RC = &AArch64::FPR128RegClass;
9435 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9436 break;
9437 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
9438 Opc = AArch64::MLAv2i32;
9439 RC = &AArch64::FPR64RegClass;
9440 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9441 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9442 RC);
9443 break;
9444 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
9445 Opc = AArch64::MLSv2i32;
9446 RC = &AArch64::FPR64RegClass;
9447 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9448 break;
9449 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
9450 Opc = AArch64::MLAv4i32;
9451 RC = &AArch64::FPR128RegClass;
9452 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9453 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9454 RC);
9455 break;
9456 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
9457 Opc = AArch64::MLSv4i32;
9458 RC = &AArch64::FPR128RegClass;
9459 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9460 break;
9461
9462 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
9463 Opc = AArch64::MLAv4i16_indexed;
9464 RC = &AArch64::FPR64RegClass;
9465 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9466 break;
9467 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
9468 Opc = AArch64::MLAv4i16_indexed;
9469 RC = &AArch64::FPR64RegClass;
9470 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9471 break;
9472 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
9473 Opc = AArch64::MLAv8i16_indexed;
9474 RC = &AArch64::FPR128RegClass;
9475 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9476 break;
9477 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
9478 Opc = AArch64::MLAv8i16_indexed;
9479 RC = &AArch64::FPR128RegClass;
9480 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9481 break;
9482 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
9483 Opc = AArch64::MLAv2i32_indexed;
9484 RC = &AArch64::FPR64RegClass;
9485 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9486 break;
9487 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
9488 Opc = AArch64::MLAv2i32_indexed;
9489 RC = &AArch64::FPR64RegClass;
9490 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9491 break;
9492 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
9493 Opc = AArch64::MLAv4i32_indexed;
9494 RC = &AArch64::FPR128RegClass;
9495 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9496 break;
9497 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
9498 Opc = AArch64::MLAv4i32_indexed;
9499 RC = &AArch64::FPR128RegClass;
9500 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9501 break;
9502
9503 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
9504 Opc = AArch64::MLAv4i16_indexed;
9505 RC = &AArch64::FPR64RegClass;
9506 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9507 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9508 RC);
9509 break;
9510 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
9511 Opc = AArch64::MLSv4i16_indexed;
9512 RC = &AArch64::FPR64RegClass;
9513 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9514 break;
9515 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
9516 Opc = AArch64::MLAv8i16_indexed;
9517 RC = &AArch64::FPR128RegClass;
9518 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9519 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9520 RC);
9521 break;
9522 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
9523 Opc = AArch64::MLSv8i16_indexed;
9524 RC = &AArch64::FPR128RegClass;
9525 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9526 break;
9527 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
9528 Opc = AArch64::MLAv2i32_indexed;
9529 RC = &AArch64::FPR64RegClass;
9530 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9531 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9532 RC);
9533 break;
9534 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
9535 Opc = AArch64::MLSv2i32_indexed;
9536 RC = &AArch64::FPR64RegClass;
9537 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9538 break;
9539 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
9540 Opc = AArch64::MLAv4i32_indexed;
9541 RC = &AArch64::FPR128RegClass;
9542 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9543 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9544 RC);
9545 break;
9546 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
9547 Opc = AArch64::MLSv4i32_indexed;
9548 RC = &AArch64::FPR128RegClass;
9549 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9550 break;
9551
9552 // Floating Point Support
9553 case AArch64MachineCombinerPattern::FMULADDH_OP1:
9554 Opc = AArch64::FMADDHrrr;
9555 RC = &AArch64::FPR16RegClass;
9556 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9557 break;
9558 case AArch64MachineCombinerPattern::FMULADDS_OP1:
9559 Opc = AArch64::FMADDSrrr;
9560 RC = &AArch64::FPR32RegClass;
9561 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9562 break;
9563 case AArch64MachineCombinerPattern::FMULADDD_OP1:
9564 Opc = AArch64::FMADDDrrr;
9565 RC = &AArch64::FPR64RegClass;
9566 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9567 break;
9568
9569 case AArch64MachineCombinerPattern::FMULADDH_OP2:
9570 Opc = AArch64::FMADDHrrr;
9571 RC = &AArch64::FPR16RegClass;
9572 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9573 break;
9574 case AArch64MachineCombinerPattern::FMULADDS_OP2:
9575 Opc = AArch64::FMADDSrrr;
9576 RC = &AArch64::FPR32RegClass;
9577 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9578 break;
9579 case AArch64MachineCombinerPattern::FMULADDD_OP2:
9580 Opc = AArch64::FMADDDrrr;
9581 RC = &AArch64::FPR64RegClass;
9582 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9583 break;
9584
9585 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
9586 Opc = AArch64::FMLAv1i32_indexed;
9587 RC = &AArch64::FPR32RegClass;
9588 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9589 kind: FMAInstKind::Indexed);
9590 break;
9591 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
9592 Opc = AArch64::FMLAv1i32_indexed;
9593 RC = &AArch64::FPR32RegClass;
9594 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9595 kind: FMAInstKind::Indexed);
9596 break;
9597
9598 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
9599 Opc = AArch64::FMLAv1i64_indexed;
9600 RC = &AArch64::FPR64RegClass;
9601 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9602 kind: FMAInstKind::Indexed);
9603 break;
9604 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
9605 Opc = AArch64::FMLAv1i64_indexed;
9606 RC = &AArch64::FPR64RegClass;
9607 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9608 kind: FMAInstKind::Indexed);
9609 break;
9610
9611 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
9612 RC = &AArch64::FPR64RegClass;
9613 Opc = AArch64::FMLAv4i16_indexed;
9614 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9615 kind: FMAInstKind::Indexed);
9616 break;
9617 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
9618 RC = &AArch64::FPR64RegClass;
9619 Opc = AArch64::FMLAv4f16;
9620 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9621 kind: FMAInstKind::Accumulator);
9622 break;
9623 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
9624 RC = &AArch64::FPR64RegClass;
9625 Opc = AArch64::FMLAv4i16_indexed;
9626 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9627 kind: FMAInstKind::Indexed);
9628 break;
9629 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
9630 RC = &AArch64::FPR64RegClass;
9631 Opc = AArch64::FMLAv4f16;
9632 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9633 kind: FMAInstKind::Accumulator);
9634 break;
9635
9636 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
9637 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
9638 RC = &AArch64::FPR64RegClass;
9639 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
9640 Opc = AArch64::FMLAv2i32_indexed;
9641 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9642 kind: FMAInstKind::Indexed);
9643 } else {
9644 Opc = AArch64::FMLAv2f32;
9645 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9646 kind: FMAInstKind::Accumulator);
9647 }
9648 break;
9649 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
9650 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
9651 RC = &AArch64::FPR64RegClass;
9652 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
9653 Opc = AArch64::FMLAv2i32_indexed;
9654 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9655 kind: FMAInstKind::Indexed);
9656 } else {
9657 Opc = AArch64::FMLAv2f32;
9658 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9659 kind: FMAInstKind::Accumulator);
9660 }
9661 break;
9662
9663 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
9664 RC = &AArch64::FPR128RegClass;
9665 Opc = AArch64::FMLAv8i16_indexed;
9666 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9667 kind: FMAInstKind::Indexed);
9668 break;
9669 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
9670 RC = &AArch64::FPR128RegClass;
9671 Opc = AArch64::FMLAv8f16;
9672 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9673 kind: FMAInstKind::Accumulator);
9674 break;
9675 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
9676 RC = &AArch64::FPR128RegClass;
9677 Opc = AArch64::FMLAv8i16_indexed;
9678 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9679 kind: FMAInstKind::Indexed);
9680 break;
9681 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
9682 RC = &AArch64::FPR128RegClass;
9683 Opc = AArch64::FMLAv8f16;
9684 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9685 kind: FMAInstKind::Accumulator);
9686 break;
9687
9688 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
9689 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
9690 RC = &AArch64::FPR128RegClass;
9691 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
9692 Opc = AArch64::FMLAv2i64_indexed;
9693 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9694 kind: FMAInstKind::Indexed);
9695 } else {
9696 Opc = AArch64::FMLAv2f64;
9697 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9698 kind: FMAInstKind::Accumulator);
9699 }
9700 break;
9701 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
9702 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
9703 RC = &AArch64::FPR128RegClass;
9704 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
9705 Opc = AArch64::FMLAv2i64_indexed;
9706 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9707 kind: FMAInstKind::Indexed);
9708 } else {
9709 Opc = AArch64::FMLAv2f64;
9710 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9711 kind: FMAInstKind::Accumulator);
9712 }
9713 break;
9714
9715 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
9716 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
9717 RC = &AArch64::FPR128RegClass;
9718 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
9719 Opc = AArch64::FMLAv4i32_indexed;
9720 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9721 kind: FMAInstKind::Indexed);
9722 } else {
9723 Opc = AArch64::FMLAv4f32;
9724 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9725 kind: FMAInstKind::Accumulator);
9726 }
9727 break;
9728
9729 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
9730 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
9731 RC = &AArch64::FPR128RegClass;
9732 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
9733 Opc = AArch64::FMLAv4i32_indexed;
9734 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9735 kind: FMAInstKind::Indexed);
9736 } else {
9737 Opc = AArch64::FMLAv4f32;
9738 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9739 kind: FMAInstKind::Accumulator);
9740 }
9741 break;
9742
9743 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
9744 Opc = AArch64::FNMSUBHrrr;
9745 RC = &AArch64::FPR16RegClass;
9746 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9747 break;
9748 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
9749 Opc = AArch64::FNMSUBSrrr;
9750 RC = &AArch64::FPR32RegClass;
9751 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9752 break;
9753 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
9754 Opc = AArch64::FNMSUBDrrr;
9755 RC = &AArch64::FPR64RegClass;
9756 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9757 break;
9758
9759 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
9760 Opc = AArch64::FNMADDHrrr;
9761 RC = &AArch64::FPR16RegClass;
9762 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9763 break;
9764 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
9765 Opc = AArch64::FNMADDSrrr;
9766 RC = &AArch64::FPR32RegClass;
9767 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9768 break;
9769 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
9770 Opc = AArch64::FNMADDDrrr;
9771 RC = &AArch64::FPR64RegClass;
9772 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9773 break;
9774
9775 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
9776 Opc = AArch64::FMSUBHrrr;
9777 RC = &AArch64::FPR16RegClass;
9778 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9779 break;
9780 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
9781 Opc = AArch64::FMSUBSrrr;
9782 RC = &AArch64::FPR32RegClass;
9783 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9784 break;
9785 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
9786 Opc = AArch64::FMSUBDrrr;
9787 RC = &AArch64::FPR64RegClass;
9788 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9789 break;
9790
9791 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
9792 Opc = AArch64::FMLSv1i32_indexed;
9793 RC = &AArch64::FPR32RegClass;
9794 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9795 kind: FMAInstKind::Indexed);
9796 break;
9797
9798 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
9799 Opc = AArch64::FMLSv1i64_indexed;
9800 RC = &AArch64::FPR64RegClass;
9801 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9802 kind: FMAInstKind::Indexed);
9803 break;
9804
9805 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
9806 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
9807 RC = &AArch64::FPR64RegClass;
9808 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9809 MachineInstrBuilder MIB1 =
9810 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f16), DestReg: NewVR)
9811 .add(MO: Root.getOperand(i: 2));
9812 InsInstrs.push_back(Elt: MIB1);
9813 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9814 if (Pattern == AArch64MachineCombinerPattern::FMLSv4f16_OP1) {
9815 Opc = AArch64::FMLAv4f16;
9816 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9817 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9818 } else {
9819 Opc = AArch64::FMLAv4i16_indexed;
9820 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9821 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9822 }
9823 break;
9824 }
9825 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
9826 RC = &AArch64::FPR64RegClass;
9827 Opc = AArch64::FMLSv4f16;
9828 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9829 kind: FMAInstKind::Accumulator);
9830 break;
9831 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
9832 RC = &AArch64::FPR64RegClass;
9833 Opc = AArch64::FMLSv4i16_indexed;
9834 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9835 kind: FMAInstKind::Indexed);
9836 break;
9837
9838 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
9839 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
9840 RC = &AArch64::FPR64RegClass;
9841 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
9842 Opc = AArch64::FMLSv2i32_indexed;
9843 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9844 kind: FMAInstKind::Indexed);
9845 } else {
9846 Opc = AArch64::FMLSv2f32;
9847 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9848 kind: FMAInstKind::Accumulator);
9849 }
9850 break;
9851
9852 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
9853 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
9854 RC = &AArch64::FPR128RegClass;
9855 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9856 MachineInstrBuilder MIB1 =
9857 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv8f16), DestReg: NewVR)
9858 .add(MO: Root.getOperand(i: 2));
9859 InsInstrs.push_back(Elt: MIB1);
9860 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9861 if (Pattern == AArch64MachineCombinerPattern::FMLSv8f16_OP1) {
9862 Opc = AArch64::FMLAv8f16;
9863 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9864 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9865 } else {
9866 Opc = AArch64::FMLAv8i16_indexed;
9867 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9868 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9869 }
9870 break;
9871 }
9872 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
9873 RC = &AArch64::FPR128RegClass;
9874 Opc = AArch64::FMLSv8f16;
9875 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9876 kind: FMAInstKind::Accumulator);
9877 break;
9878 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
9879 RC = &AArch64::FPR128RegClass;
9880 Opc = AArch64::FMLSv8i16_indexed;
9881 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9882 kind: FMAInstKind::Indexed);
9883 break;
9884
9885 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
9886 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
9887 RC = &AArch64::FPR128RegClass;
9888 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
9889 Opc = AArch64::FMLSv2i64_indexed;
9890 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9891 kind: FMAInstKind::Indexed);
9892 } else {
9893 Opc = AArch64::FMLSv2f64;
9894 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9895 kind: FMAInstKind::Accumulator);
9896 }
9897 break;
9898
9899 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
9900 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
9901 RC = &AArch64::FPR128RegClass;
9902 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
9903 Opc = AArch64::FMLSv4i32_indexed;
9904 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9905 kind: FMAInstKind::Indexed);
9906 } else {
9907 Opc = AArch64::FMLSv4f32;
9908 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9909 kind: FMAInstKind::Accumulator);
9910 }
9911 break;
9912 case AArch64MachineCombinerPattern::FMLSv2f32_OP1:
9913 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
9914 RC = &AArch64::FPR64RegClass;
9915 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9916 MachineInstrBuilder MIB1 =
9917 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f32), DestReg: NewVR)
9918 .add(MO: Root.getOperand(i: 2));
9919 InsInstrs.push_back(Elt: MIB1);
9920 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9921 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
9922 Opc = AArch64::FMLAv2i32_indexed;
9923 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9924 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9925 } else {
9926 Opc = AArch64::FMLAv2f32;
9927 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9928 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9929 }
9930 break;
9931 }
9932 case AArch64MachineCombinerPattern::FMLSv4f32_OP1:
9933 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
9934 RC = &AArch64::FPR128RegClass;
9935 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9936 MachineInstrBuilder MIB1 =
9937 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f32), DestReg: NewVR)
9938 .add(MO: Root.getOperand(i: 2));
9939 InsInstrs.push_back(Elt: MIB1);
9940 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9941 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
9942 Opc = AArch64::FMLAv4i32_indexed;
9943 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9944 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9945 } else {
9946 Opc = AArch64::FMLAv4f32;
9947 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9948 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9949 }
9950 break;
9951 }
9952 case AArch64MachineCombinerPattern::FMLSv2f64_OP1:
9953 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
9954 RC = &AArch64::FPR128RegClass;
9955 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9956 MachineInstrBuilder MIB1 =
9957 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f64), DestReg: NewVR)
9958 .add(MO: Root.getOperand(i: 2));
9959 InsInstrs.push_back(Elt: MIB1);
9960 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9961 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
9962 Opc = AArch64::FMLAv2i64_indexed;
9963 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9964 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9965 } else {
9966 Opc = AArch64::FMLAv2f64;
9967 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9968 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9969 }
9970 break;
9971 }
9972 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
9973 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2: {
9974 unsigned IdxDupOp =
9975 (Pattern == AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1) ? 1
9976 : 2;
9977 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i32_indexed,
9978 RC: &AArch64::FPR128RegClass, MRI);
9979 break;
9980 }
9981 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
9982 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2: {
9983 unsigned IdxDupOp =
9984 (Pattern == AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1) ? 1
9985 : 2;
9986 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i64_indexed,
9987 RC: &AArch64::FPR128RegClass, MRI);
9988 break;
9989 }
9990 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
9991 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2: {
9992 unsigned IdxDupOp =
9993 (Pattern == AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1) ? 1
9994 : 2;
9995 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i16_indexed,
9996 RC: &AArch64::FPR128_loRegClass, MRI);
9997 break;
9998 }
9999 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
10000 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2: {
10001 unsigned IdxDupOp =
10002 (Pattern == AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1) ? 1
10003 : 2;
10004 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i32_indexed,
10005 RC: &AArch64::FPR128RegClass, MRI);
10006 break;
10007 }
10008 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
10009 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2: {
10010 unsigned IdxDupOp =
10011 (Pattern == AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1) ? 1
10012 : 2;
10013 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv8i16_indexed,
10014 RC: &AArch64::FPR128_loRegClass, MRI);
10015 break;
10016 }
10017 case AArch64MachineCombinerPattern::FNMADD: {
10018 MUL = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs);
10019 break;
10020 }
10021 case AArch64MachineCombinerPattern::GATHER_LANE_i32: {
10022 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10023 Pattern, NumLanes: 4);
10024 break;
10025 }
10026 case AArch64MachineCombinerPattern::GATHER_LANE_i16: {
10027 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10028 Pattern, NumLanes: 8);
10029 break;
10030 }
10031 case AArch64MachineCombinerPattern::GATHER_LANE_i8: {
10032 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10033 Pattern, NumLanes: 16);
10034 break;
10035 }
10036
10037 } // end switch (Pattern)
10038 // Record MUL and ADD/SUB for deletion
10039 if (MUL)
10040 DelInstrs.push_back(Elt: MUL);
10041 DelInstrs.push_back(Elt: &Root);
10042
10043 // Set the flags on the inserted instructions to be the merged flags of the
10044 // instructions that we have combined.
10045 uint32_t Flags = Root.getFlags();
10046 if (MUL)
10047 Flags = Root.mergeFlagsWith(Other: *MUL);
10048 for (auto *MI : InsInstrs)
10049 MI->setFlags(Flags);
10050}
10051
10052/// Replace csincr-branch sequence by simple conditional branch
10053///
10054/// Examples:
10055/// 1. \code
10056/// csinc w9, wzr, wzr, <condition code>
10057/// tbnz w9, #0, 0x44
10058/// \endcode
10059/// to
10060/// \code
10061/// b.<inverted condition code>
10062/// \endcode
10063///
10064/// 2. \code
10065/// csinc w9, wzr, wzr, <condition code>
10066/// tbz w9, #0, 0x44
10067/// \endcode
10068/// to
10069/// \code
10070/// b.<condition code>
10071/// \endcode
10072///
10073/// Replace compare and branch sequence by TBZ/TBNZ instruction when the
10074/// compare's constant operand is power of 2.
10075///
10076/// Examples:
10077/// \code
10078/// and w8, w8, #0x400
10079/// cbnz w8, L1
10080/// \endcode
10081/// to
10082/// \code
10083/// tbnz w8, #10, L1
10084/// \endcode
10085///
10086/// \param MI Conditional Branch
10087/// \return True when the simple conditional branch is generated
10088///
10089bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
10090 bool IsNegativeBranch = false;
10091 bool IsTestAndBranch = false;
10092 unsigned TargetBBInMI = 0;
10093 switch (MI.getOpcode()) {
10094 default:
10095 llvm_unreachable("Unknown branch instruction?");
10096 case AArch64::Bcc:
10097 case AArch64::CBWPri:
10098 case AArch64::CBXPri:
10099 case AArch64::CBBAssertExt:
10100 case AArch64::CBHAssertExt:
10101 case AArch64::CBWPrr:
10102 case AArch64::CBXPrr:
10103 return false;
10104 case AArch64::CBZW:
10105 case AArch64::CBZX:
10106 TargetBBInMI = 1;
10107 break;
10108 case AArch64::CBNZW:
10109 case AArch64::CBNZX:
10110 TargetBBInMI = 1;
10111 IsNegativeBranch = true;
10112 break;
10113 case AArch64::TBZW:
10114 case AArch64::TBZX:
10115 TargetBBInMI = 2;
10116 IsTestAndBranch = true;
10117 break;
10118 case AArch64::TBNZW:
10119 case AArch64::TBNZX:
10120 TargetBBInMI = 2;
10121 IsNegativeBranch = true;
10122 IsTestAndBranch = true;
10123 break;
10124 }
10125 // So we increment a zero register and test for bits other
10126 // than bit 0? Conservatively bail out in case the verifier
10127 // missed this case.
10128 if (IsTestAndBranch && MI.getOperand(i: 1).getImm())
10129 return false;
10130
10131 // Find Definition.
10132 assert(MI.getParent() && "Incomplete machine instruction\n");
10133 MachineBasicBlock *MBB = MI.getParent();
10134 MachineFunction *MF = MBB->getParent();
10135 MachineRegisterInfo *MRI = &MF->getRegInfo();
10136 Register VReg = MI.getOperand(i: 0).getReg();
10137 if (!VReg.isVirtual())
10138 return false;
10139
10140 MachineInstr *DefMI = MRI->getVRegDef(Reg: VReg);
10141 if (!DefMI)
10142 return false;
10143
10144 // Look through COPY instructions to find definition.
10145 while (DefMI->isCopy()) {
10146 Register CopyVReg = DefMI->getOperand(i: 1).getReg();
10147 if (!CopyVReg.isVirtual())
10148 return false;
10149 if (!MRI->hasOneNonDBGUse(RegNo: CopyVReg))
10150 return false;
10151 DefMI = MRI->getVRegDef(Reg: CopyVReg);
10152 if (!DefMI)
10153 return false;
10154 }
10155
10156 switch (DefMI->getOpcode()) {
10157 default:
10158 return false;
10159 // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
10160 case AArch64::ANDWri:
10161 case AArch64::ANDXri: {
10162 if (IsTestAndBranch)
10163 return false;
10164 if (DefMI->getParent() != MBB)
10165 return false;
10166 if (!MRI->hasOneNonDBGUse(RegNo: VReg))
10167 return false;
10168
10169 bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
10170 uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
10171 val: DefMI->getOperand(i: 2).getImm(), regSize: Is32Bit ? 32 : 64);
10172 if (!isPowerOf2_64(Value: Mask))
10173 return false;
10174
10175 MachineOperand &MO = DefMI->getOperand(i: 1);
10176 Register NewReg = MO.getReg();
10177 if (!NewReg.isVirtual())
10178 return false;
10179
10180 if (!MRI->getVRegDef(Reg: NewReg))
10181 return false;
10182
10183 MachineBasicBlock &RefToMBB = *MBB;
10184 MachineBasicBlock *TBB = MI.getOperand(i: 1).getMBB();
10185 DebugLoc DL = MI.getDebugLoc();
10186 unsigned Imm = Log2_64(Value: Mask);
10187 unsigned Opc = (Imm < 32)
10188 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
10189 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
10190 MachineInstr *NewMI = BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: Opc))
10191 .addReg(RegNo: NewReg)
10192 .addImm(Val: Imm)
10193 .addMBB(MBB: TBB);
10194 // Register lives on to the CBZ now.
10195 MO.setIsKill(false);
10196
10197 // For immediate smaller than 32, we need to use the 32-bit
10198 // variant (W) in all cases. Indeed the 64-bit variant does not
10199 // allow to encode them.
10200 // Therefore, if the input register is 64-bit, we need to take the
10201 // 32-bit sub-part.
10202 if (!Is32Bit && Imm < 32)
10203 NewMI->getOperand(i: 0).setSubReg(AArch64::sub_32);
10204 MI.eraseFromParent();
10205 return true;
10206 }
10207 // Look for CSINC
10208 case AArch64::CSINCWr:
10209 case AArch64::CSINCXr: {
10210 if (!(DefMI->getOperand(i: 1).getReg() == AArch64::WZR &&
10211 DefMI->getOperand(i: 2).getReg() == AArch64::WZR) &&
10212 !(DefMI->getOperand(i: 1).getReg() == AArch64::XZR &&
10213 DefMI->getOperand(i: 2).getReg() == AArch64::XZR))
10214 return false;
10215
10216 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
10217 isDead: true) != -1)
10218 return false;
10219
10220 AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(i: 3).getImm();
10221 // Convert only when the condition code is not modified between
10222 // the CSINC and the branch. The CC may be used by other
10223 // instructions in between.
10224 if (areCFlagsAccessedBetweenInstrs(From: DefMI, To: MI, TRI: &getRegisterInfo(), AccessToCheck: AK_Write))
10225 return false;
10226 MachineBasicBlock &RefToMBB = *MBB;
10227 MachineBasicBlock *TBB = MI.getOperand(i: TargetBBInMI).getMBB();
10228 DebugLoc DL = MI.getDebugLoc();
10229 if (IsNegativeBranch)
10230 CC = AArch64CC::getInvertedCondCode(Code: CC);
10231 BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: CC).addMBB(MBB: TBB);
10232 MI.eraseFromParent();
10233 return true;
10234 }
10235 }
10236}
10237
10238std::pair<unsigned, unsigned>
10239AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10240 const unsigned Mask = AArch64II::MO_FRAGMENT;
10241 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
10242}
10243
10244ArrayRef<std::pair<unsigned, const char *>>
10245AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10246 using namespace AArch64II;
10247
10248 static const std::pair<unsigned, const char *> TargetFlags[] = {
10249 {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
10250 {MO_G3, "aarch64-g3"}, {MO_G2, "aarch64-g2"},
10251 {MO_G1, "aarch64-g1"}, {MO_G0, "aarch64-g0"},
10252 {MO_HI12, "aarch64-hi12"}};
10253 return ArrayRef(TargetFlags);
10254}
10255
10256ArrayRef<std::pair<unsigned, const char *>>
10257AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
10258 using namespace AArch64II;
10259
10260 static const std::pair<unsigned, const char *> TargetFlags[] = {
10261 {MO_COFFSTUB, "aarch64-coffstub"},
10262 {MO_GOT, "aarch64-got"},
10263 {MO_NC, "aarch64-nc"},
10264 {MO_S, "aarch64-s"},
10265 {MO_TLS, "aarch64-tls"},
10266 {MO_DLLIMPORT, "aarch64-dllimport"},
10267 {MO_PREL, "aarch64-prel"},
10268 {MO_TAGGED, "aarch64-tagged"},
10269 {MO_ARM64EC_CALLMANGLE, "aarch64-arm64ec-callmangle"},
10270 };
10271 return ArrayRef(TargetFlags);
10272}
10273
10274ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10275AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10276 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10277 {{MOSuppressPair, "aarch64-suppress-pair"},
10278 {MOStridedAccess, "aarch64-strided-access"}};
10279 return ArrayRef(TargetFlags);
10280}
10281
10282/// Constants defining how certain sequences should be outlined.
10283/// This encompasses how an outlined function should be called, and what kind of
10284/// frame should be emitted for that outlined function.
10285///
10286/// \p MachineOutlinerDefault implies that the function should be called with
10287/// a save and restore of LR to the stack.
10288///
10289/// That is,
10290///
10291/// I1 Save LR OUTLINED_FUNCTION:
10292/// I2 --> BL OUTLINED_FUNCTION I1
10293/// I3 Restore LR I2
10294/// I3
10295/// RET
10296///
10297/// * Call construction overhead: 3 (save + BL + restore)
10298/// * Frame construction overhead: 1 (ret)
10299/// * Requires stack fixups? Yes
10300///
10301/// \p MachineOutlinerTailCall implies that the function is being created from
10302/// a sequence of instructions ending in a return.
10303///
10304/// That is,
10305///
10306/// I1 OUTLINED_FUNCTION:
10307/// I2 --> B OUTLINED_FUNCTION I1
10308/// RET I2
10309/// RET
10310///
10311/// * Call construction overhead: 1 (B)
10312/// * Frame construction overhead: 0 (Return included in sequence)
10313/// * Requires stack fixups? No
10314///
10315/// \p MachineOutlinerNoLRSave implies that the function should be called using
10316/// a BL instruction, but doesn't require LR to be saved and restored. This
10317/// happens when LR is known to be dead.
10318///
10319/// That is,
10320///
10321/// I1 OUTLINED_FUNCTION:
10322/// I2 --> BL OUTLINED_FUNCTION I1
10323/// I3 I2
10324/// I3
10325/// RET
10326///
10327/// * Call construction overhead: 1 (BL)
10328/// * Frame construction overhead: 1 (RET)
10329/// * Requires stack fixups? No
10330///
10331/// \p MachineOutlinerThunk implies that the function is being created from
10332/// a sequence of instructions ending in a call. The outlined function is
10333/// called with a BL instruction, and the outlined function tail-calls the
10334/// original call destination.
10335///
10336/// That is,
10337///
10338/// I1 OUTLINED_FUNCTION:
10339/// I2 --> BL OUTLINED_FUNCTION I1
10340/// BL f I2
10341/// B f
10342/// * Call construction overhead: 1 (BL)
10343/// * Frame construction overhead: 0
10344/// * Requires stack fixups? No
10345///
10346/// \p MachineOutlinerRegSave implies that the function should be called with a
10347/// save and restore of LR to an available register. This allows us to avoid
10348/// stack fixups. Note that this outlining variant is compatible with the
10349/// NoLRSave case.
10350///
10351/// That is,
10352///
10353/// I1 Save LR OUTLINED_FUNCTION:
10354/// I2 --> BL OUTLINED_FUNCTION I1
10355/// I3 Restore LR I2
10356/// I3
10357/// RET
10358///
10359/// * Call construction overhead: 3 (save + BL + restore)
10360/// * Frame construction overhead: 1 (ret)
10361/// * Requires stack fixups? No
10362enum MachineOutlinerClass {
10363 MachineOutlinerDefault, /// Emit a save, restore, call, and return.
10364 MachineOutlinerTailCall, /// Only emit a branch.
10365 MachineOutlinerNoLRSave, /// Emit a call and return.
10366 MachineOutlinerThunk, /// Emit a call and tail-call.
10367 MachineOutlinerRegSave /// Same as default, but save to a register.
10368};
10369
10370enum MachineOutlinerMBBFlags {
10371 LRUnavailableSomewhere = 0x2,
10372 HasCalls = 0x4,
10373 UnsafeRegsDead = 0x8
10374};
10375
10376/// Return true if the frame-record form of the outlined prologue is enabled for
10377/// the target of \p MF.
10378///
10379/// A non-leaf outlined function must save LR. On MachO, saving LR alone
10380/// (str x30) has no compact unwind encoding, so we get a large DWARF FDE
10381/// instead. Saving FP and LR as a frame record (stp x29, x30 ; mov x29, sp)
10382/// gets the small FRAME encoding, and costs one extra instruction.
10383static bool isCompactUnwindFrameRecordEnabled(const MachineFunction &MF) {
10384 return MF.getSubtarget<AArch64Subtarget>()
10385 .getCLOpts()
10386 .outliner_compact_unwind_frame &&
10387 MF.getFunction().getParent()->getTargetTriple().isOSBinFormatMachO();
10388}
10389
10390/// Return true if the outlined function in \p MBB should save FP and LR as a
10391/// frame record instead of saving LR alone.
10392static bool shouldUseCompactUnwindFrameRecordForOutlinedFunction(
10393 const MachineBasicBlock &MBB) {
10394 const MachineFunction &MF = *MBB.getParent();
10395
10396 // Only worth it if the function has unwind info to shrink.
10397 if (!isCompactUnwindFrameRecordEnabled(MF) ||
10398 !MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF))
10399 return false;
10400
10401 // Only safe if the outlined code never touches FP, since we overwrite it.
10402 LiveRegUnits LRU(*MF.getSubtarget().getRegisterInfo());
10403 for (const MachineInstr &MI : MBB.instrs())
10404 LRU.accumulate(MI);
10405 return LRU.available(Reg: AArch64::FP);
10406}
10407
10408/// Predict what the above will answer, for use while costing candidates. The
10409/// outlined function does not exist yet, so answer from \p RepeatedSequenceLocs
10410/// instead. This is only an estimate; buildOutlinedFrame() makes the call.
10411static bool predictCompactUnwindFrameRecordForOutlinedFunction(
10412 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10413 const TargetRegisterInfo &TRI) {
10414 if (!isCompactUnwindFrameRecordEnabled(MF: *RepeatedSequenceLocs.front().getMF()))
10415 return false;
10416
10417 // The outlined function is nounwind only if every candidate is, so it has
10418 // unwind info if any candidate does.
10419 if (llvm::none_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10420 const MachineFunction &MF = *C.getMF();
10421 return MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF);
10422 }))
10423 return false;
10424
10425 // FP is free in the outlined function only if it is free in every candidate.
10426 return llvm::all_of(Range&: RepeatedSequenceLocs, P: [&TRI](outliner::Candidate &C) {
10427 return C.isAvailableInsideSeq(Reg: AArch64::FP, TRI);
10428 });
10429}
10430
10431Register
10432AArch64InstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
10433 MachineFunction *MF = C.getMF();
10434 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
10435 const AArch64RegisterInfo *ARI =
10436 static_cast<const AArch64RegisterInfo *>(&TRI);
10437 // Check if there is an available register across the sequence that we can
10438 // use.
10439 for (unsigned Reg : AArch64::GPR64RegClass) {
10440 if (!ARI->isReservedReg(MF: *MF, Reg) &&
10441 Reg != AArch64::LR && // LR is not reserved, but don't use it.
10442 Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
10443 Reg != AArch64::X17 && // Ditto for X17.
10444 C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
10445 C.isAvailableInsideSeq(Reg, TRI))
10446 return Reg;
10447 }
10448 return Register();
10449}
10450
10451static bool
10452outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
10453 const outliner::Candidate &b) {
10454 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10455 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10456
10457 return MFIa->getSignReturnAddressCondition() ==
10458 MFIb->getSignReturnAddressCondition();
10459}
10460
10461static bool
10462outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
10463 const outliner::Candidate &b) {
10464 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10465 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10466
10467 return MFIa->shouldSignWithBKey() == MFIb->shouldSignWithBKey();
10468}
10469
10470static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
10471 const outliner::Candidate &b) {
10472 const AArch64Subtarget &SubtargetA =
10473 a.getMF()->getSubtarget<AArch64Subtarget>();
10474 const AArch64Subtarget &SubtargetB =
10475 b.getMF()->getSubtarget<AArch64Subtarget>();
10476 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
10477}
10478
10479std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10480AArch64InstrInfo::getOutliningCandidateInfo(
10481 const MachineModuleInfo &MMI,
10482 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10483 unsigned MinRepeats) const {
10484 unsigned SequenceSize = 0;
10485 for (auto &MI : RepeatedSequenceLocs[0])
10486 SequenceSize += getInstSizeInBytes(MI);
10487
10488 unsigned NumBytesToCreateFrame = 0;
10489
10490 // Avoid splitting ADRP ADD/LDR pair into outlined functions.
10491 // These instructions are fused together by the scheduler.
10492 // Any candidate where ADRP is the last instruction should be rejected
10493 // as that will lead to splitting ADRP pair.
10494 MachineInstr &LastMI = RepeatedSequenceLocs[0].back();
10495 MachineInstr &FirstMI = RepeatedSequenceLocs[0].front();
10496 if (LastMI.getOpcode() == AArch64::ADRP &&
10497 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_PAGE) != 0 &&
10498 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10499 return std::nullopt;
10500 }
10501
10502 // Similarly any candidate where the first instruction is ADD/LDR with a
10503 // page offset should be rejected to avoid ADRP splitting.
10504 if ((FirstMI.getOpcode() == AArch64::ADDXri ||
10505 FirstMI.getOpcode() == AArch64::LDRXui) &&
10506 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_PAGEOFF) != 0 &&
10507 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10508 return std::nullopt;
10509 }
10510
10511 // We only allow outlining for functions having exactly matching return
10512 // address signing attributes, i.e., all share the same value for the
10513 // attribute "sign-return-address" and all share the same type of key they
10514 // are signed with.
10515 // Additionally we require all functions to simultaneously either support
10516 // v8.3a features or not. Otherwise an outlined function could get signed
10517 // using dedicated v8.3 instructions and a call from a function that doesn't
10518 // support v8.3 instructions would therefore be invalid.
10519 if (std::adjacent_find(
10520 first: RepeatedSequenceLocs.begin(), last: RepeatedSequenceLocs.end(),
10521 binary_pred: [](const outliner::Candidate &a, const outliner::Candidate &b) {
10522 // Return true if a and b are non-equal w.r.t. return address
10523 // signing or support of v8.3a features
10524 if (outliningCandidatesSigningScopeConsensus(a, b) &&
10525 outliningCandidatesSigningKeyConsensus(a, b) &&
10526 outliningCandidatesV8_3OpsConsensus(a, b)) {
10527 return false;
10528 }
10529 return true;
10530 }) != RepeatedSequenceLocs.end()) {
10531 return std::nullopt;
10532 }
10533
10534 // Since at this point all candidates agree on their return address signing
10535 // picking just one is fine. If the candidate functions potentially sign their
10536 // return addresses, the outlined function should do the same. Note that in
10537 // the case of "sign-return-address"="non-leaf" this is an assumption: It is
10538 // not certainly true that the outlined function will have to sign its return
10539 // address but this decision is made later, when the decision to outline
10540 // has already been made.
10541 // The same holds for the number of additional instructions we need: On
10542 // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
10543 // necessary. However, at this point we don't know if the outlined function
10544 // will have a RET instruction so we assume the worst.
10545 const TargetRegisterInfo &TRI = getRegisterInfo();
10546 // Performing a tail call may require extra checks when PAuth is enabled.
10547 // If PAuth is disabled, set it to zero for uniformity.
10548 unsigned NumBytesToCheckLRInTCEpilogue = 0;
10549 const auto RASignCondition = RepeatedSequenceLocs[0]
10550 .getMF()
10551 ->getInfo<AArch64FunctionInfo>()
10552 ->getSignReturnAddressCondition();
10553 if (RASignCondition != SignReturnAddress::None) {
10554 // Candidates that have Return Address Authentication Hardening enabled are
10555 // discarded.
10556 //
10557 // In its current form, the machine outliner does not preserve X16/X17
10558 // across outlined function calls, even though it should as they are
10559 // caller-saved registers. And since the hardening based on load from the
10560 // return address may clobber one of these registers, if they are alive
10561 // across a call their value would be lost due to the hardening mechanism.
10562 llvm::erase_if(C&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10563 return C.getMF()
10564 ->getInfo<AArch64FunctionInfo>()
10565 ->shouldHardenSignReturnAddress();
10566 });
10567 // If the sequence doesn't have enough candidates left, then we're done.
10568 if (RepeatedSequenceLocs.size() < MinRepeats)
10569 return std::nullopt;
10570
10571 // One PAC and one AUT instructions
10572 NumBytesToCreateFrame += 8;
10573
10574 // PAuth is enabled - set extra tail call cost, if any.
10575 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
10576 MF: *RepeatedSequenceLocs[0].getMF());
10577 NumBytesToCheckLRInTCEpilogue =
10578 AArch64PAuth::getCheckerSizeInBytes(Method: LRCheckMethod);
10579 // Checking the authenticated LR value may significantly impact
10580 // SequenceSize, so account for it for more precise results.
10581 if (isTailCallReturnInst(MI: RepeatedSequenceLocs[0].back()))
10582 SequenceSize += NumBytesToCheckLRInTCEpilogue;
10583
10584 // We have to check if sp modifying instructions would get outlined.
10585 // If so we only allow outlining if sp is unchanged overall, so matching
10586 // sub and add instructions are okay to outline, all other sp modifications
10587 // are not
10588 auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
10589 int SPValue = 0;
10590 for (auto &MI : C) {
10591 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI)) {
10592 switch (MI.getOpcode()) {
10593 case AArch64::ADDXri:
10594 case AArch64::ADDWri:
10595 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10596 assert(MI.getOperand(2).isImm() &&
10597 "Expected operand to be immediate");
10598 assert(MI.getOperand(1).isReg() &&
10599 "Expected operand to be a register");
10600 // Check if the add just increments sp. If so, we search for
10601 // matching sub instructions that decrement sp. If not, the
10602 // modification is illegal
10603 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10604 SPValue += MI.getOperand(i: 2).getImm();
10605 else
10606 return true;
10607 break;
10608 case AArch64::SUBXri:
10609 case AArch64::SUBWri:
10610 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10611 assert(MI.getOperand(2).isImm() &&
10612 "Expected operand to be immediate");
10613 assert(MI.getOperand(1).isReg() &&
10614 "Expected operand to be a register");
10615 // Check if the sub just decrements sp. If so, we search for
10616 // matching add instructions that increment sp. If not, the
10617 // modification is illegal
10618 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10619 SPValue -= MI.getOperand(i: 2).getImm();
10620 else
10621 return true;
10622 break;
10623 default:
10624 return true;
10625 }
10626 }
10627 }
10628 if (SPValue)
10629 return true;
10630 return false;
10631 };
10632 // Remove candidates with illegal stack modifying instructions
10633 llvm::erase_if(C&: RepeatedSequenceLocs, P: hasIllegalSPModification);
10634
10635 // If the sequence doesn't have enough candidates left, then we're done.
10636 if (RepeatedSequenceLocs.size() < MinRepeats)
10637 return std::nullopt;
10638 }
10639
10640 // Properties about candidate MBBs that hold for all of them.
10641 unsigned FlagsSetInAll = 0xF;
10642
10643 // Compute liveness information for each candidate, and set FlagsSetInAll.
10644 for (outliner::Candidate &C : RepeatedSequenceLocs)
10645 FlagsSetInAll &= C.Flags;
10646
10647 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
10648
10649 // Helper lambda which sets call information for every candidate.
10650 auto SetCandidateCallInfo =
10651 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
10652 for (outliner::Candidate &C : RepeatedSequenceLocs)
10653 C.setCallInfo(CID: CallID, CO: NumBytesForCall);
10654 };
10655
10656 unsigned FrameID = MachineOutlinerDefault;
10657 NumBytesToCreateFrame += 4;
10658
10659 bool HasBTI = any_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10660 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
10661 });
10662
10663 // We check to see if CFI Instructions are present, and if they are
10664 // we find the number of CFI Instructions in the candidates.
10665 unsigned CFICount = 0;
10666 for (auto &I : RepeatedSequenceLocs[0]) {
10667 if (I.isCFIInstruction())
10668 CFICount++;
10669 }
10670
10671 // We compare the number of found CFI Instructions to the number of CFI
10672 // instructions in the parent function for each candidate. We must check this
10673 // since if we outline one of the CFI instructions in a function, we have to
10674 // outline them all for correctness. If we do not, the address offsets will be
10675 // incorrect between the two sections of the program.
10676 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10677 std::vector<MCCFIInstruction> CFIInstructions =
10678 C.getMF()->getFrameInstructions();
10679
10680 if (CFICount > 0 && CFICount != CFIInstructions.size())
10681 return std::nullopt;
10682 }
10683
10684 // Returns true if an instructions is safe to fix up, false otherwise.
10685 auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
10686 if (MI.isCall())
10687 return true;
10688
10689 if (!MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI) &&
10690 !MI.readsRegister(Reg: AArch64::SP, TRI: &TRI))
10691 return true;
10692
10693 // Any modification of SP will break our code to save/restore LR.
10694 // FIXME: We could handle some instructions which add a constant
10695 // offset to SP, with a bit more work.
10696 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI))
10697 return false;
10698
10699 // At this point, we have a stack instruction that we might need to
10700 // fix up. We'll handle it if it's a load or store.
10701 if (MI.mayLoadOrStore()) {
10702 const MachineOperand *Base; // Filled with the base operand of MI.
10703 int64_t Offset; // Filled with the offset of MI.
10704 bool OffsetIsScalable;
10705
10706 // Does it allow us to offset the base operand and is the base the
10707 // register SP?
10708 if (!getMemOperandWithOffset(MI, BaseOp&: Base, Offset, OffsetIsScalable) ||
10709 !Base->isReg() || Base->getReg() != AArch64::SP)
10710 return false;
10711
10712 // Fixe-up code below assumes bytes.
10713 if (OffsetIsScalable)
10714 return false;
10715
10716 // Find the minimum/maximum offset for this instruction and check
10717 // if fixing it up would be in range.
10718 int64_t MinOffset,
10719 MaxOffset; // Unscaled offsets for the instruction.
10720 // The scale to multiply the offsets by.
10721 TypeSize Scale(0U, false), DummyWidth(0U, false);
10722 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width&: DummyWidth, MinOffset, MaxOffset);
10723
10724 Offset += 16; // Update the offset to what it would be if we outlined.
10725 if (Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
10726 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
10727 return false;
10728
10729 // It's in range, so we can outline it.
10730 return true;
10731 }
10732
10733 // FIXME: Add handling for instructions like "add x0, sp, #8".
10734
10735 // We can't fix it up, so don't outline it.
10736 return false;
10737 };
10738
10739 // True if it's possible to fix up each stack instruction in this sequence.
10740 // Important for frames/call variants that modify the stack.
10741 bool AllStackInstrsSafe =
10742 llvm::all_of(Range&: RepeatedSequenceLocs[0], P: IsSafeToFixup);
10743
10744 // If the last instruction in any candidate is a terminator, then we should
10745 // tail call all of the candidates.
10746 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10747 FrameID = MachineOutlinerTailCall;
10748 NumBytesToCreateFrame = 0;
10749 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
10750 SetCandidateCallInfo(MachineOutlinerTailCall, NumBytesForCall);
10751 }
10752
10753 else if (LastInstrOpcode == AArch64::BL ||
10754 ((LastInstrOpcode == AArch64::BLR ||
10755 LastInstrOpcode == AArch64::BLRNoIP) &&
10756 !HasBTI)) {
10757 // FIXME: Do we need to check if the code after this uses the value of LR?
10758 FrameID = MachineOutlinerThunk;
10759 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
10760 SetCandidateCallInfo(MachineOutlinerThunk, 4);
10761 }
10762
10763 else {
10764 // We need to decide how to emit calls + frames. We can always emit the same
10765 // frame if we don't need to save to the stack. If we have to save to the
10766 // stack, then we need a different frame.
10767 unsigned NumBytesNoStackCalls = 0;
10768 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
10769
10770 // Check if we have to save LR.
10771 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10772 bool LRAvailable =
10773 (C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
10774 ? C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI)
10775 : true;
10776 // If we have a noreturn caller, then we're going to be conservative and
10777 // say that we have to save LR. If we don't have a ret at the end of the
10778 // block, then we can't reason about liveness accurately.
10779 //
10780 // FIXME: We can probably do better than always disabling this in
10781 // noreturn functions by fixing up the liveness info.
10782 bool IsNoReturn =
10783 C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoReturn);
10784
10785 // Is LR available? If so, we don't need a save.
10786 if (LRAvailable && !IsNoReturn) {
10787 NumBytesNoStackCalls += 4;
10788 C.setCallInfo(CID: MachineOutlinerNoLRSave, CO: 4);
10789 CandidatesWithoutStackFixups.push_back(x: C);
10790 }
10791
10792 // Is an unused register available? If so, we won't modify the stack, so
10793 // we can outline with the same frame type as those that don't save LR.
10794 else if (findRegisterToSaveLRTo(C)) {
10795 NumBytesNoStackCalls += 12;
10796 C.setCallInfo(CID: MachineOutlinerRegSave, CO: 12);
10797 CandidatesWithoutStackFixups.push_back(x: C);
10798 }
10799
10800 // Is SP used in the sequence at all? If not, we don't have to modify
10801 // the stack, so we are guaranteed to get the same frame.
10802 else if (C.isAvailableInsideSeq(Reg: AArch64::SP, TRI)) {
10803 NumBytesNoStackCalls += 12;
10804 C.setCallInfo(CID: MachineOutlinerDefault, CO: 12);
10805 CandidatesWithoutStackFixups.push_back(x: C);
10806 }
10807
10808 // If we outline this, we need to modify the stack. Pretend we don't
10809 // outline this by saving all of its bytes.
10810 else {
10811 NumBytesNoStackCalls += SequenceSize;
10812 }
10813 }
10814
10815 // If there are no places where we have to save LR, then note that we
10816 // don't have to update the stack. Otherwise, give every candidate the
10817 // default call type, as long as it's safe to do so.
10818 if (!AllStackInstrsSafe ||
10819 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
10820 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
10821 FrameID = MachineOutlinerNoLRSave;
10822 if (RepeatedSequenceLocs.size() < MinRepeats)
10823 return std::nullopt;
10824 } else {
10825 SetCandidateCallInfo(MachineOutlinerDefault, 12);
10826
10827 // Bugzilla ID: 46767
10828 // TODO: Check if fixing up the stack more than once is safe so we can
10829 // outline these.
10830 //
10831 // An outline resulting in a caller that requires stack fixups at the
10832 // callsite to a callee that also requires stack fixups can happen when
10833 // there are no available registers at the candidate callsite for a
10834 // candidate that itself also has calls.
10835 //
10836 // In other words if function_containing_sequence in the following pseudo
10837 // assembly requires that we save LR at the point of the call, but there
10838 // are no available registers: in this case we save using SP and as a
10839 // result the SP offsets requires stack fixups by multiples of 16.
10840 //
10841 // function_containing_sequence:
10842 // ...
10843 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10844 // call OUTLINED_FUNCTION_N
10845 // restore LR from SP
10846 // ...
10847 //
10848 // OUTLINED_FUNCTION_N:
10849 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10850 // ...
10851 // bl foo
10852 // restore LR from SP
10853 // ret
10854 //
10855 // Because the code to handle more than one stack fixup does not
10856 // currently have the proper checks for legality, these cases will assert
10857 // in the AArch64 MachineOutliner. This is because the code to do this
10858 // needs more hardening, testing, better checks that generated code is
10859 // legal, etc and because it is only verified to handle a single pass of
10860 // stack fixup.
10861 //
10862 // The assert happens in AArch64InstrInfo::buildOutlinedFrame to catch
10863 // these cases until they are known to be handled. Bugzilla 46767 is
10864 // referenced in comments at the assert site.
10865 //
10866 // To avoid asserting (or generating non-legal code on noassert builds)
10867 // we remove all candidates which would need more than one stack fixup by
10868 // pruning the cases where the candidate has calls while also having no
10869 // available LR and having no available general purpose registers to copy
10870 // LR to (ie one extra stack save/restore).
10871 //
10872 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10873 erase_if(C&: RepeatedSequenceLocs, P: [this, &TRI](outliner::Candidate &C) {
10874 auto IsCall = [](const MachineInstr &MI) { return MI.isCall(); };
10875 return (llvm::any_of(Range&: C, P: IsCall)) &&
10876 (!C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI) ||
10877 !findRegisterToSaveLRTo(C));
10878 });
10879 }
10880 }
10881
10882 // If we dropped all of the candidates, bail out here.
10883 if (RepeatedSequenceLocs.size() < MinRepeats)
10884 return std::nullopt;
10885 }
10886
10887 // Does every candidate's MBB contain a call? If so, then we might have a call
10888 // in the range.
10889 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10890 // Check if the range contains a call. These require a save + restore of the
10891 // link register.
10892 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
10893 bool ModStackToSaveLR = false;
10894 if (any_of(Range: drop_end(RangeOrContainer&: FirstCand),
10895 P: [](const MachineInstr &MI) { return MI.isCall(); }))
10896 ModStackToSaveLR = true;
10897
10898 // Handle the last instruction separately. If this is a tail call, then the
10899 // last instruction is a call. We don't want to save + restore in this case.
10900 // However, it could be possible that the last instruction is a call without
10901 // it being valid to tail call this sequence. We should consider this as
10902 // well.
10903 else if (FrameID != MachineOutlinerThunk &&
10904 FrameID != MachineOutlinerTailCall && FirstCand.back().isCall())
10905 ModStackToSaveLR = true;
10906
10907 if (ModStackToSaveLR) {
10908 // We can't fix up the stack. Bail out.
10909 if (!AllStackInstrsSafe)
10910 return std::nullopt;
10911
10912 // Save + restore LR.
10913 NumBytesToCreateFrame += 8;
10914
10915 // Add the extra mov if we will save a frame record instead of just LR.
10916 if (predictCompactUnwindFrameRecordForOutlinedFunction(
10917 RepeatedSequenceLocs, TRI))
10918 NumBytesToCreateFrame += 4;
10919 }
10920 }
10921
10922 // If we have CFI instructions, we can only outline if the outlined section
10923 // can be a tail call
10924 if (FrameID != MachineOutlinerTailCall && CFICount > 0)
10925 return std::nullopt;
10926
10927 return std::make_unique<outliner::OutlinedFunction>(
10928 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: NumBytesToCreateFrame, args&: FrameID);
10929}
10930
10931void AArch64InstrInfo::mergeOutliningCandidateAttributes(
10932 Function &F, std::vector<outliner::Candidate> &Candidates) const {
10933 // If a bunch of candidates reach this point they must agree on their return
10934 // address signing. It is therefore enough to just consider the signing
10935 // behaviour of one of them
10936 const auto &CFn = Candidates.front().getMF()->getFunction();
10937
10938 if (CFn.hasFnAttribute(Kind: "ptrauth-returns"))
10939 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-returns"));
10940 if (CFn.hasFnAttribute(Kind: "ptrauth-auth-traps"))
10941 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-auth-traps"));
10942 // Since all candidates belong to the same module, just copy the
10943 // function-level attributes of an arbitrary function.
10944 if (CFn.hasFnAttribute(Kind: "sign-return-address"))
10945 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address"));
10946 if (CFn.hasFnAttribute(Kind: "sign-return-address-key"))
10947 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address-key"));
10948 // The "sign-return-address-harden" attribute is not included because no
10949 // candidate is supposed to have hardening enabled.
10950 assert(llvm::none_of(Candidates, [](const outliner::Candidate &C) {
10951 return C.getMF()
10952 ->getInfo<AArch64FunctionInfo>()
10953 ->shouldHardenSignReturnAddress();
10954 }));
10955
10956 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
10957}
10958
10959bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
10960 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
10961 if (!TargetInstrInfo::isFunctionSafeToOutlineFrom(MF,
10962 OutlineFromLinkOnceODRs))
10963 return false;
10964
10965 // Outlining from functions with redzones is unsafe since the outliner may
10966 // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
10967 // outline from it.
10968 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
10969 if (!AFI || AFI->hasRedZone().value_or(u: true))
10970 return false;
10971
10972 // FIXME: Determine whether it is safe to outline from functions which contain
10973 // streaming-mode changes. We may need to ensure any smstart/smstop pairs are
10974 // outlined together and ensure it is safe to outline with async unwind info,
10975 // required for saving & restoring VG around calls.
10976 if (AFI->hasStreamingModeChanges())
10977 return false;
10978
10979 // FIXME: Teach the outliner to generate/handle Windows unwind info.
10980 if (MF.usesWindowsUnwindInfo())
10981 return false;
10982
10983 // It's safe to outline from MF.
10984 return true;
10985}
10986
10987SmallVector<std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10988AArch64InstrInfo::getOutlinableRanges(MachineBasicBlock &MBB,
10989 unsigned &Flags) const {
10990 assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
10991 "Must track liveness!");
10992 SmallVector<
10993 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10994 Ranges;
10995 // According to the AArch64 Procedure Call Standard, the following are
10996 // undefined on entry/exit from a function call:
10997 //
10998 // * Registers x16, x17, (and thus w16, w17)
10999 // * Condition codes (and thus the NZCV register)
11000 //
11001 // If any of these registers are used inside or live across an outlined
11002 // function, then they may be modified later, either by the compiler or
11003 // some other tool (like the linker).
11004 //
11005 // To avoid outlining in these situations, partition each block into ranges
11006 // where these registers are dead. We will only outline from those ranges.
11007 LiveRegUnits LRU(getRegisterInfo());
11008 auto AreAllUnsafeRegsDead = [&LRU]() {
11009 return LRU.available(Reg: AArch64::W16) && LRU.available(Reg: AArch64::W17) &&
11010 LRU.available(Reg: AArch64::NZCV);
11011 };
11012
11013 // We need to know if LR is live across an outlining boundary later on in
11014 // order to decide how we'll create the outlined call, frame, etc.
11015 //
11016 // It's pretty expensive to check this for *every candidate* within a block.
11017 // That's some potentially n^2 behaviour, since in the worst case, we'd need
11018 // to compute liveness from the end of the block for O(n) candidates within
11019 // the block.
11020 //
11021 // So, to improve the average case, let's keep track of liveness from the end
11022 // of the block to the beginning of *every outlinable range*. If we know that
11023 // LR is available in every range we could outline from, then we know that
11024 // we don't need to check liveness for any candidate within that range.
11025 bool LRAvailableEverywhere = true;
11026 // Compute liveness bottom-up.
11027 LRU.addLiveOuts(MBB);
11028 // Update flags that require info about the entire MBB.
11029 auto UpdateWholeMBBFlags = [&Flags](const MachineInstr &MI) {
11030 if (MI.isCall() && !MI.isTerminator())
11031 Flags |= MachineOutlinerMBBFlags::HasCalls;
11032 };
11033 // Range: [RangeBegin, RangeEnd)
11034 MachineBasicBlock::instr_iterator RangeBegin, RangeEnd;
11035 unsigned RangeLen;
11036 auto CreateNewRangeStartingAt =
11037 [&RangeBegin, &RangeEnd,
11038 &RangeLen](MachineBasicBlock::instr_iterator NewBegin) {
11039 RangeBegin = NewBegin;
11040 RangeEnd = std::next(x: RangeBegin);
11041 RangeLen = 0;
11042 };
11043 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
11044 // At least one unsafe register is not dead. We do not want to outline at
11045 // this point. If it is long enough to outline from and does not cross a
11046 // bundle boundary, save the range [RangeBegin, RangeEnd).
11047 if (RangeLen <= 1)
11048 return;
11049 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
11050 return;
11051 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
11052 return;
11053 Ranges.emplace_back(Args&: RangeBegin, Args&: RangeEnd);
11054 };
11055 // Find the first point where all unsafe registers are dead.
11056 // FIND: <safe instr> <-- end of first potential range
11057 // SKIP: <unsafe def>
11058 // SKIP: ... everything between ...
11059 // SKIP: <unsafe use>
11060 auto FirstPossibleEndPt = MBB.instr_rbegin();
11061 for (; FirstPossibleEndPt != MBB.instr_rend(); ++FirstPossibleEndPt) {
11062 if (!FirstPossibleEndPt->isDebugInstr())
11063 LRU.stepBackward(MI: *FirstPossibleEndPt);
11064 // Update flags that impact how we outline across the entire block,
11065 // regardless of safety.
11066 UpdateWholeMBBFlags(*FirstPossibleEndPt);
11067 if (AreAllUnsafeRegsDead())
11068 break;
11069 }
11070 // If we exhausted the entire block, we have no safe ranges to outline.
11071 if (FirstPossibleEndPt == MBB.instr_rend())
11072 return Ranges;
11073 // Current range.
11074 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
11075 // StartPt points to the first place where all unsafe registers
11076 // are dead (if there is any such point). Begin partitioning the MBB into
11077 // ranges.
11078 // Pseudo probes are ordinary instructions to the mapper, so only skip debug
11079 // instructions here.
11080 for (auto &MI : instructionsWithoutDebug(It: FirstPossibleEndPt, End: MBB.instr_rend(),
11081 /*SkipPseudoOp=*/false)) {
11082 LRU.stepBackward(MI);
11083 UpdateWholeMBBFlags(MI);
11084 if (!AreAllUnsafeRegsDead()) {
11085 SaveRangeIfNonEmpty();
11086 CreateNewRangeStartingAt(MI.getIterator());
11087 continue;
11088 }
11089 LRAvailableEverywhere &= LRU.available(Reg: AArch64::LR);
11090 // Pseudo probes count because the mapper treats them as ordinary
11091 // instructions.
11092 RangeBegin = MI.getIterator();
11093 ++RangeLen;
11094 }
11095 // Above loop misses the last (or only) range. If we are still safe, then
11096 // let's save the range.
11097 if (AreAllUnsafeRegsDead())
11098 SaveRangeIfNonEmpty();
11099 if (Ranges.empty())
11100 return Ranges;
11101 // We found the ranges bottom-up. Mapping expects the top-down. Reverse
11102 // the order.
11103 std::reverse(first: Ranges.begin(), last: Ranges.end());
11104 // If there is at least one outlinable range where LR is unavailable
11105 // somewhere, remember that.
11106 if (!LRAvailableEverywhere)
11107 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
11108 return Ranges;
11109}
11110
11111outliner::InstrType
11112AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
11113 MachineBasicBlock::iterator &MIT,
11114 unsigned Flags) const {
11115 MachineInstr &MI = *MIT;
11116
11117 // Don't outline anything used for return address signing. The outlined
11118 // function will get signed later if needed
11119 switch (MI.getOpcode()) {
11120 case AArch64::PACM:
11121 case AArch64::PACIASP:
11122 case AArch64::PACIBSP:
11123 case AArch64::PACIASPPC:
11124 case AArch64::PACIBSPPC:
11125 case AArch64::AUTIASP:
11126 case AArch64::AUTIBSP:
11127 case AArch64::AUTIASPPCi:
11128 case AArch64::AUTIASPPCr:
11129 case AArch64::AUTIBSPPCi:
11130 case AArch64::AUTIBSPPCr:
11131 case AArch64::RETAA:
11132 case AArch64::RETAB:
11133 case AArch64::RETAASPPCi:
11134 case AArch64::RETAASPPCr:
11135 case AArch64::RETABSPPCi:
11136 case AArch64::RETABSPPCr:
11137 case AArch64::EMITBKEY:
11138 case AArch64::PAUTH_PROLOGUE:
11139 case AArch64::PAUTH_EPILOGUE:
11140 return outliner::InstrType::Illegal;
11141 }
11142
11143 // We can only outline these if we will tail call the outlined function, or
11144 // fix up the CFI offsets. Currently, CFI instructions are outlined only if
11145 // in a tail call.
11146 //
11147 // FIXME: If the proper fixups for the offset are implemented, this should be
11148 // possible.
11149 if (MI.isCFIInstruction())
11150 return outliner::InstrType::Legal;
11151
11152 // Is this a terminator for a basic block?
11153 if (MI.isTerminator())
11154 // TargetInstrInfo::getOutliningType has already filtered out anything
11155 // that would break this, so we can allow it here.
11156 return outliner::InstrType::Legal;
11157
11158 // Make sure none of the operands are un-outlinable.
11159 for (const MachineOperand &MOP : MI.operands()) {
11160 // A check preventing CFI indices was here before, but only CFI
11161 // instructions should have those.
11162 assert(!MOP.isCFIIndex());
11163
11164 // If it uses LR or W30 explicitly, then don't touch it.
11165 if (MOP.isReg() && !MOP.isImplicit() &&
11166 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
11167 return outliner::InstrType::Illegal;
11168 }
11169
11170 // Special cases for instructions that can always be outlined, but will fail
11171 // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
11172 // be outlined because they don't require a *specific* value to be in LR.
11173 if (MI.getOpcode() == AArch64::ADRP)
11174 return outliner::InstrType::Legal;
11175
11176 // If MI is a call we might be able to outline it. We don't want to outline
11177 // any calls that rely on the position of items on the stack. When we outline
11178 // something containing a call, we have to emit a save and restore of LR in
11179 // the outlined function. Currently, this always happens by saving LR to the
11180 // stack. Thus, if we outline, say, half the parameters for a function call
11181 // plus the call, then we'll break the callee's expectations for the layout
11182 // of the stack.
11183 //
11184 // FIXME: Allow calls to functions which construct a stack frame, as long
11185 // as they don't access arguments on the stack.
11186 // FIXME: Figure out some way to analyze functions defined in other modules.
11187 // We should be able to compute the memory usage based on the IR calling
11188 // convention, even if we can't see the definition.
11189 if (MI.isCall()) {
11190 // Get the function associated with the call. Look at each operand and find
11191 // the one that represents the callee and get its name.
11192 const Function *Callee = nullptr;
11193 for (const MachineOperand &MOP : MI.operands()) {
11194 if (MOP.isGlobal()) {
11195 Callee = dyn_cast<Function>(Val: MOP.getGlobal());
11196 break;
11197 }
11198 }
11199
11200 // Never outline calls to mcount. There isn't any rule that would require
11201 // this, but the Linux kernel's "ftrace" feature depends on it.
11202 if (Callee && Callee->getName() == "\01_mcount")
11203 return outliner::InstrType::Illegal;
11204
11205 // If we don't know anything about the callee, assume it depends on the
11206 // stack layout of the caller. In that case, it's only legal to outline
11207 // as a tail-call. Explicitly list the call instructions we know about so we
11208 // don't get unexpected results with call pseudo-instructions.
11209 auto UnknownCallOutlineType = outliner::InstrType::Illegal;
11210 if (MI.getOpcode() == AArch64::BLR ||
11211 MI.getOpcode() == AArch64::BLRNoIP || MI.getOpcode() == AArch64::BL)
11212 UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
11213
11214 if (!Callee)
11215 return UnknownCallOutlineType;
11216
11217 // We have a function we have information about. Check it if it's something
11218 // can safely outline.
11219 MachineFunction *CalleeMF = MMI.getMachineFunction(F: *Callee);
11220
11221 // We don't know what's going on with the callee at all. Don't touch it.
11222 if (!CalleeMF)
11223 return UnknownCallOutlineType;
11224
11225 // Check if we know anything about the callee saves on the function. If we
11226 // don't, then don't touch it, since that implies that we haven't
11227 // computed anything about its stack frame yet.
11228 MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
11229 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
11230 MFI.getNumObjects() > 0)
11231 return UnknownCallOutlineType;
11232
11233 // At this point, we can say that CalleeMF ought to not pass anything on the
11234 // stack. Therefore, we can outline it.
11235 return outliner::InstrType::Legal;
11236 }
11237
11238 // Don't touch the link register or W30.
11239 if (MI.readsRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()) ||
11240 MI.modifiesRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()))
11241 return outliner::InstrType::Illegal;
11242
11243 // Don't outline BTI instructions, because that will prevent the outlining
11244 // site from being indirectly callable.
11245 if (hasBTISemantics(MI))
11246 return outliner::InstrType::Illegal;
11247
11248 return outliner::InstrType::Legal;
11249}
11250
11251void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
11252 for (MachineInstr &MI : MBB) {
11253 const MachineOperand *Base;
11254 TypeSize Width(0, false);
11255 int64_t Offset;
11256 bool OffsetIsScalable;
11257
11258 // Is this a load or store with an immediate offset with SP as the base?
11259 if (!MI.mayLoadOrStore() ||
11260 !getMemOperandWithOffsetWidth(LdSt: MI, BaseOp&: Base, Offset, OffsetIsScalable,
11261 Width) ||
11262 (Base->isReg() && Base->getReg() != AArch64::SP))
11263 continue;
11264
11265 // It is, so we have to fix it up.
11266 TypeSize Scale(0U, false);
11267 int64_t Dummy1, Dummy2;
11268
11269 MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(LdSt&: MI);
11270 assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
11271 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2);
11272 assert(Scale != 0 && "Unexpected opcode!");
11273 assert(!OffsetIsScalable && "Expected offset to be a byte offset");
11274
11275 // We've pushed the return address to the stack, so add 16 to the offset.
11276 // This is safe, since we already checked if it would overflow when we
11277 // checked if this instruction was legal to outline.
11278 int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedValue();
11279 StackOffsetOperand.setImm(NewImm);
11280 }
11281}
11282
11283static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
11284 const AArch64InstrInfo *TII,
11285 bool ShouldSignReturnAddr) {
11286 if (!ShouldSignReturnAddr)
11287 return;
11288
11289 BuildMI(BB&: MBB, I: MBB.begin(), MIMD: DebugLoc(), MCID: TII->get(Opcode: AArch64::PAUTH_PROLOGUE))
11290 .setMIFlag(MachineInstr::FrameSetup);
11291 TII->createPauthEpilogueInstr(MBB, DL: DebugLoc());
11292}
11293
11294void AArch64InstrInfo::buildOutlinedFrame(
11295 MachineBasicBlock &MBB, MachineFunction &MF,
11296 const outliner::OutlinedFunction &OF) const {
11297
11298 AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
11299
11300 if (OF.FrameConstructionID == MachineOutlinerTailCall)
11301 FI->setOutliningStyle("Tail Call");
11302 else if (OF.FrameConstructionID == MachineOutlinerThunk) {
11303 // For thunk outlining, rewrite the last instruction from a call to a
11304 // tail-call.
11305 MachineInstr *Call = &*--MBB.instr_end();
11306 unsigned TailOpcode;
11307 if (Call->getOpcode() == AArch64::BL) {
11308 TailOpcode = AArch64::TCRETURNdi;
11309 } else {
11310 assert(Call->getOpcode() == AArch64::BLR ||
11311 Call->getOpcode() == AArch64::BLRNoIP);
11312 TailOpcode = AArch64::TCRETURNriALL;
11313 }
11314 MachineInstr *TC = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpcode))
11315 .add(MO: Call->getOperand(i: 0))
11316 .addImm(Val: 0);
11317 MBB.insert(I: MBB.end(), MI: TC);
11318 Call->eraseFromParent();
11319
11320 FI->setOutliningStyle("Thunk");
11321 }
11322
11323 bool IsLeafFunction = true;
11324
11325 // Is there a call in the outlined range?
11326 auto IsNonTailCall = [](const MachineInstr &MI) {
11327 return MI.isCall() && !MI.isReturn();
11328 };
11329
11330 if (llvm::any_of(Range: MBB.instrs(), P: IsNonTailCall)) {
11331 // Fix up the instructions in the range, since we're going to modify the
11332 // stack.
11333
11334 // Bugzilla ID: 46767
11335 // TODO: Check if fixing up twice is safe so we can outline these.
11336 assert(OF.FrameConstructionID != MachineOutlinerDefault &&
11337 "Can only fix up stack references once");
11338 fixupPostOutline(MBB);
11339
11340 IsLeafFunction = false;
11341
11342 // LR has to be a live in so that we can save it.
11343 if (!MBB.isLiveIn(Reg: AArch64::LR))
11344 MBB.addLiveIn(PhysReg: AArch64::LR);
11345
11346 MachineBasicBlock::iterator It = MBB.begin();
11347 MachineBasicBlock::iterator Et = MBB.end();
11348
11349 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11350 OF.FrameConstructionID == MachineOutlinerThunk)
11351 Et = std::prev(x: MBB.end());
11352
11353 // There is a call in the range, so we must save LR. Save it as part of a
11354 // frame record when that gives us a smaller compact unwind encoding.
11355 if (shouldUseCompactUnwindFrameRecordForOutlinedFunction(MBB)) {
11356 // FP is saved here, so it must be live-in.
11357 if (!MBB.isLiveIn(Reg: AArch64::FP))
11358 MBB.addLiveIn(PhysReg: AArch64::FP);
11359
11360 // stp x29, x30, [sp, #-16]! (the pre-index imm is scaled by 8: -2 * 8)
11361 MachineInstr *STPXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STPXpre))
11362 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11363 .addReg(RegNo: AArch64::FP)
11364 .addReg(RegNo: AArch64::LR)
11365 .addReg(RegNo: AArch64::SP)
11366 .addImm(Val: -2);
11367 It = MBB.insert(I: It, MI: STPXpre);
11368
11369 // mov x29, sp (add x29, sp, #0), so x29 points at the frame record.
11370 MachineInstr *SetFP = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ADDXri))
11371 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11372 .addReg(RegNo: AArch64::SP)
11373 .addImm(Val: 0)
11374 .addImm(Val: 0);
11375 MBB.insertAfter(I: It, MI: SetFP);
11376
11377 // Describe the frame record with FP as the CFA. The encoder needs all
11378 // three to pick FRAME. No need to check for unwind info here: we only
11379 // get here if the function has it.
11380 CFIInstBuilder CFIBuilder(MBB, std::next(x: SetFP->getIterator()),
11381 MachineInstr::FrameSetup);
11382 CFIBuilder.buildDefCFA(Reg: AArch64::FP, Offset: 16);
11383 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -8);
11384 CFIBuilder.buildOffset(Reg: AArch64::FP, Offset: -16);
11385
11386 // ldp x29, x30, [sp], #16
11387 MachineInstr *LDPXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDPXpost))
11388 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11389 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11390 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11391 .addReg(RegNo: AArch64::SP)
11392 .addImm(Val: 2);
11393 Et = MBB.insert(I: Et, MI: LDPXpost);
11394 } else {
11395 // Insert a save before the outlined region
11396 MachineInstr *STRXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11397 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11398 .addReg(RegNo: AArch64::LR)
11399 .addReg(RegNo: AArch64::SP)
11400 .addImm(Val: -16);
11401 It = MBB.insert(I: It, MI: STRXpre);
11402
11403 if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
11404 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
11405
11406 // Add a CFI saying the stack was moved 16 B down.
11407 CFIBuilder.buildDefCFAOffset(Offset: 16);
11408
11409 // Add a CFI saying that the LR that we want to find is now 16 B higher
11410 // than before.
11411 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -16);
11412 }
11413
11414 // Insert a restore before the terminator for the function.
11415 MachineInstr *LDRXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11416 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11417 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11418 .addReg(RegNo: AArch64::SP)
11419 .addImm(Val: 16);
11420 Et = MBB.insert(I: Et, MI: LDRXpost);
11421 }
11422 }
11423
11424 auto RASignCondition = FI->getSignReturnAddressCondition();
11425 bool ShouldSignReturnAddr = AArch64FunctionInfo::shouldSignReturnAddress(
11426 Condition: RASignCondition, IsLRSpilled: !IsLeafFunction);
11427
11428 // If this is a tail call outlined function, then there's already a return.
11429 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11430 OF.FrameConstructionID == MachineOutlinerThunk) {
11431 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11432 return;
11433 }
11434
11435 // It's not a tail call, so we have to insert the return ourselves.
11436
11437 // LR has to be a live in so that we can return to it.
11438 if (!MBB.isLiveIn(Reg: AArch64::LR))
11439 MBB.addLiveIn(PhysReg: AArch64::LR);
11440
11441 MachineInstr *ret = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::RET))
11442 .addReg(RegNo: AArch64::LR);
11443 MBB.insert(I: MBB.end(), MI: ret);
11444
11445 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11446
11447 FI->setOutliningStyle("Function");
11448
11449 // Did we have to modify the stack by saving the link register?
11450 if (OF.FrameConstructionID != MachineOutlinerDefault)
11451 return;
11452
11453 // We modified the stack.
11454 // Walk over the basic block and fix up all the stack accesses.
11455 fixupPostOutline(MBB);
11456}
11457
11458MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
11459 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
11460 MachineFunction &MF, outliner::Candidate &C) const {
11461
11462 // Are we tail calling?
11463 if (C.CallConstructionID == MachineOutlinerTailCall) {
11464 // If yes, then we can just branch to the label.
11465 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::TCRETURNdi))
11466 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()))
11467 .addImm(Val: 0));
11468 return It;
11469 }
11470
11471 // Are we saving the link register?
11472 if (C.CallConstructionID == MachineOutlinerNoLRSave ||
11473 C.CallConstructionID == MachineOutlinerThunk) {
11474 // No, so just insert the call.
11475 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11476 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11477 return It;
11478 }
11479
11480 // We want to return the spot where we inserted the call.
11481 MachineBasicBlock::iterator CallPt;
11482
11483 // Instructions for saving and restoring LR around the call instruction we're
11484 // going to insert.
11485 MachineInstr *Save;
11486 MachineInstr *Restore;
11487 // Can we save to a register?
11488 if (C.CallConstructionID == MachineOutlinerRegSave) {
11489 // FIXME: This logic should be sunk into a target-specific interface so that
11490 // we don't have to recompute the register.
11491 Register Reg = findRegisterToSaveLRTo(C);
11492 assert(Reg && "No callee-saved register available?");
11493
11494 // LR has to be a live in so that we can save it.
11495 if (!MBB.isLiveIn(Reg: AArch64::LR))
11496 MBB.addLiveIn(PhysReg: AArch64::LR);
11497
11498 // Save and restore LR from Reg.
11499 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: Reg)
11500 .addReg(RegNo: AArch64::XZR)
11501 .addReg(RegNo: AArch64::LR)
11502 .addImm(Val: 0);
11503 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: AArch64::LR)
11504 .addReg(RegNo: AArch64::XZR)
11505 .addReg(RegNo: Reg)
11506 .addImm(Val: 0);
11507 } else {
11508 // We have the default case. Save and restore from SP.
11509 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11510 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11511 .addReg(RegNo: AArch64::LR)
11512 .addReg(RegNo: AArch64::SP)
11513 .addImm(Val: -16);
11514 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11515 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11516 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11517 .addReg(RegNo: AArch64::SP)
11518 .addImm(Val: 16);
11519 }
11520
11521 It = MBB.insert(I: It, MI: Save);
11522 It++;
11523
11524 // Insert the call.
11525 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11526 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11527 CallPt = It;
11528 It++;
11529
11530 It = MBB.insert(I: It, MI: Restore);
11531 return CallPt;
11532}
11533
11534bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
11535 MachineFunction &MF) const {
11536 return MF.getFunction().hasMinSize();
11537}
11538
11539void AArch64InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
11540 MachineBasicBlock::iterator Iter,
11541 DebugLoc &DL,
11542 bool AllowSideEffects) const {
11543 const MachineFunction &MF = *MBB.getParent();
11544 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
11545 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
11546
11547 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
11548 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg).addImm(Val: 0).addImm(Val: 0);
11549 } else if (STI.isSVEorStreamingSVEAvailable()) {
11550 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::DUP_ZI_D), DestReg: Reg)
11551 .addImm(Val: 0)
11552 .addImm(Val: 0);
11553 } else if (STI.isNeonAvailable()) {
11554 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVIv2d_ns), DestReg: Reg)
11555 .addImm(Val: 0);
11556 } else {
11557 // No Advanced SIMD (streaming-compatible without SVE, or +nosimd), so use
11558 // `fmov d...` instead of `movi v...`; writing `d` also clears the upper
11559 // 64 bits.
11560 assert(STI.hasFPARMv8() && "Expected FP to be available.");
11561 Register Reg64 = TRI.getSubReg(Reg, Idx: AArch64::dsub);
11562 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg: Reg64);
11563 }
11564}
11565
11566std::optional<DestSourcePair>
11567AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
11568
11569 // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
11570 // and zero immediate operands used as an alias for mov instruction.
11571 if ((MI.getOpcode() == AArch64::ORRWrs &&
11572 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11573 MI.getOperand(i: 3).getImm() == 0x0) ||
11574 (MI.getOpcode() == AArch64::ORRWrr &&
11575 MI.getOperand(i: 1).getReg() == AArch64::WZR)) {
11576 // Check that the w->w move is not a zero-extending w->x mov.
11577 if ((MI.getOperand(i: 0).getReg().isPhysical() &&
11578 MI.findRegisterDefOperandIdx(
11579 Reg: getXRegFromWReg(Reg: MI.getOperand(i: 0).getReg()),
11580 /*TRI=*/nullptr) == -1) ||
11581 (MI.getOperand(i: 0).getReg().isVirtual() &&
11582 !MI.getOperand(i: 0).getSubReg()))
11583 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11584 }
11585
11586 if (MI.getOpcode() == AArch64::ORRXrs &&
11587 MI.getOperand(i: 1).getReg() == AArch64::XZR &&
11588 MI.getOperand(i: 3).getImm() == 0x0)
11589 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11590
11591 return std::nullopt;
11592}
11593
11594std::optional<DestSourcePair>
11595AArch64InstrInfo::isCopyLikeInstrImpl(const MachineInstr &MI) const {
11596 if ((MI.getOpcode() == AArch64::ORRWrs &&
11597 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11598 MI.getOperand(i: 3).getImm() == 0x0) ||
11599 (MI.getOpcode() == AArch64::ORRWrr &&
11600 MI.getOperand(i: 1).getReg() == AArch64::WZR))
11601 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11602 return std::nullopt;
11603}
11604
11605std::optional<RegImmPair>
11606AArch64InstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
11607 int Sign = 1;
11608 int64_t Offset = 0;
11609
11610 // TODO: Handle cases where Reg is a super- or sub-register of the
11611 // destination register.
11612 const MachineOperand &Op0 = MI.getOperand(i: 0);
11613 if (!Op0.isReg() || Reg != Op0.getReg())
11614 return std::nullopt;
11615
11616 switch (MI.getOpcode()) {
11617 default:
11618 return std::nullopt;
11619 case AArch64::SUBWri:
11620 case AArch64::SUBXri:
11621 case AArch64::SUBSWri:
11622 case AArch64::SUBSXri:
11623 Sign *= -1;
11624 [[fallthrough]];
11625 case AArch64::ADDSWri:
11626 case AArch64::ADDSXri:
11627 case AArch64::ADDWri:
11628 case AArch64::ADDXri: {
11629 // TODO: Third operand can be global address (usually some string).
11630 if (!MI.getOperand(i: 0).isReg() || !MI.getOperand(i: 1).isReg() ||
11631 !MI.getOperand(i: 2).isImm())
11632 return std::nullopt;
11633 int Shift = MI.getOperand(i: 3).getImm();
11634 assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
11635 Offset = Sign * (MI.getOperand(i: 2).getImm() << Shift);
11636 }
11637 }
11638 return RegImmPair{MI.getOperand(i: 1).getReg(), Offset};
11639}
11640
11641/// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
11642/// the destination register then, if possible, describe the value in terms of
11643/// the source register.
11644static std::optional<ParamLoadedValue>
11645describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
11646 const TargetInstrInfo *TII,
11647 const TargetRegisterInfo *TRI) {
11648 auto DestSrc = TII->isCopyLikeInstr(MI);
11649 if (!DestSrc)
11650 return std::nullopt;
11651
11652 Register DestReg = DestSrc->Destination->getReg();
11653 Register SrcReg = DestSrc->Source->getReg();
11654
11655 if (!DestReg.isValid() || !SrcReg.isValid())
11656 return std::nullopt;
11657
11658 auto Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: {});
11659
11660 // If the described register is the destination, just return the source.
11661 if (DestReg == DescribedReg)
11662 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11663
11664 // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
11665 if (MI.getOpcode() == AArch64::ORRWrs &&
11666 TRI->isSuperRegister(RegA: DestReg, RegB: DescribedReg))
11667 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11668
11669 // We may need to describe the lower part of a ORRXrs move.
11670 if (MI.getOpcode() == AArch64::ORRXrs &&
11671 TRI->isSubRegister(RegA: DestReg, RegB: DescribedReg)) {
11672 Register SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: AArch64::sub_32);
11673 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcSubReg, isDef: false), Expr);
11674 }
11675
11676 assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
11677 "Unhandled ORR[XW]rs copy case");
11678
11679 return std::nullopt;
11680}
11681
11682bool AArch64InstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
11683 // Functions cannot be split to different sections on AArch64 if they have
11684 // a red zone. This is because relaxing a cross-section branch may require
11685 // incrementing the stack pointer to spill a register, which would overwrite
11686 // the red zone.
11687 if (MF.getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(u: true))
11688 return false;
11689
11690 return TargetInstrInfo::isFunctionSafeToSplit(MF);
11691}
11692
11693bool AArch64InstrInfo::isMBBSafeToSplitToCold(
11694 const MachineBasicBlock &MBB) const {
11695 // Asm Goto blocks can contain conditional branches to goto labels, which can
11696 // get moved out of range of the branch instruction.
11697 auto isAsmGoto = [](const MachineInstr &MI) {
11698 return MI.getOpcode() == AArch64::INLINEASM_BR;
11699 };
11700 if (llvm::any_of(Range: MBB, P: isAsmGoto) || MBB.isInlineAsmBrIndirectTarget())
11701 return false;
11702
11703 // Because jump tables are label-relative instead of table-relative, they all
11704 // must be in the same section or relocation fixup handling will fail.
11705
11706 // Check if MBB is a jump table target
11707 const MachineJumpTableInfo *MJTI = MBB.getParent()->getJumpTableInfo();
11708 auto containsMBB = [&MBB](const MachineJumpTableEntry &JTE) {
11709 return llvm::is_contained(Range: JTE.MBBs, Element: &MBB);
11710 };
11711 if (MJTI != nullptr && llvm::any_of(Range: MJTI->getJumpTables(), P: containsMBB))
11712 return false;
11713
11714 // Check if MBB contains a jump table lookup
11715 for (const MachineInstr &MI : MBB) {
11716 switch (MI.getOpcode()) {
11717 case TargetOpcode::G_BRJT:
11718 case AArch64::JumpTableDest32:
11719 case AArch64::JumpTableDest16:
11720 case AArch64::JumpTableDest8:
11721 return false;
11722 default:
11723 continue;
11724 }
11725 }
11726
11727 // MBB isn't a special case, so it's safe to be split to the cold section.
11728 return true;
11729}
11730
11731std::optional<ParamLoadedValue>
11732AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
11733 Register Reg) const {
11734 const MachineFunction *MF = MI.getMF();
11735 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
11736 switch (MI.getOpcode()) {
11737 case AArch64::MOVZWi:
11738 case AArch64::MOVZXi: {
11739 // MOVZWi may be used for producing zero-extended 32-bit immediates in
11740 // 64-bit parameters, so we need to consider super-registers.
11741 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
11742 return std::nullopt;
11743
11744 if (!MI.getOperand(i: 1).isImm())
11745 return std::nullopt;
11746 int64_t Immediate = MI.getOperand(i: 1).getImm();
11747 int Shift = MI.getOperand(i: 2).getImm();
11748 return ParamLoadedValue(MachineOperand::CreateImm(Val: Immediate << Shift),
11749 nullptr);
11750 }
11751 case AArch64::ORRWrs:
11752 case AArch64::ORRXrs:
11753 return describeORRLoadedValue(MI, DescribedReg: Reg, TII: this, TRI);
11754 }
11755
11756 return TargetInstrInfo::describeLoadedValue(MI, Reg);
11757}
11758
11759bool AArch64InstrInfo::isExtendLikelyToBeFolded(
11760 MachineInstr &ExtMI, MachineRegisterInfo &MRI) const {
11761 assert(ExtMI.getOpcode() == TargetOpcode::G_SEXT ||
11762 ExtMI.getOpcode() == TargetOpcode::G_ZEXT ||
11763 ExtMI.getOpcode() == TargetOpcode::G_ANYEXT);
11764
11765 // Anyexts are nops.
11766 if (ExtMI.getOpcode() == TargetOpcode::G_ANYEXT)
11767 return true;
11768
11769 Register DefReg = ExtMI.getOperand(i: 0).getReg();
11770 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
11771 return false;
11772
11773 // It's likely that a sext/zext as a G_PTR_ADD offset will be folded into an
11774 // addressing mode.
11775 auto *UserMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
11776 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
11777}
11778
11779uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
11780 return get(Opcode: Opc).TSFlags & AArch64::ElementSizeMask;
11781}
11782
11783bool AArch64InstrInfo::isPTestLikeOpcode(unsigned Opc) const {
11784 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsPTestLike;
11785}
11786
11787bool AArch64InstrInfo::isWhileOpcode(unsigned Opc) const {
11788 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsWhile;
11789}
11790
11791unsigned int
11792AArch64InstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
11793 return OptLevel >= CodeGenOptLevel::Aggressive ? 6 : 2;
11794}
11795
11796bool AArch64InstrInfo::isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
11797 unsigned Scale) const {
11798 if (Offset && Scale)
11799 return false;
11800
11801 // Check Reg + Imm
11802 if (!Scale) {
11803 // 9-bit signed offset
11804 if (isInt<9>(x: Offset))
11805 return true;
11806
11807 // 12-bit unsigned offset
11808 unsigned Shift = Log2_64(Value: NumBytes);
11809 if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
11810 // Must be a multiple of NumBytes (NumBytes is a power of 2)
11811 (Offset >> Shift) << Shift == Offset)
11812 return true;
11813 return false;
11814 }
11815
11816 // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
11817 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
11818}
11819
11820unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
11821 if (MF.getSubtarget<AArch64Subtarget>().hardenSlsBlr())
11822 return AArch64::BLRNoIP;
11823 else
11824 return AArch64::BLR;
11825}
11826
11827void AArch64InstrInfo::createPauthEpilogueInstr(MachineBasicBlock &MBB,
11828 DebugLoc DL) const {
11829 MachineBasicBlock::iterator InsertPt = MBB.getFirstTerminator();
11830 auto Builder = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: get(Opcode: AArch64::PAUTH_EPILOGUE))
11831 .setMIFlag(MachineInstr::FrameDestroy);
11832
11833 MachineFunction &MF = *MBB.getParent();
11834 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
11835 auto &AFL = *static_cast<const AArch64FrameLowering *>(
11836 MF.getSubtarget().getFrameLowering());
11837 if (AFL.getArgumentStackToRestore(MF, MBB)) {
11838 Builder.addReg(RegNo: AArch64::X17, Flags: RegState::ImplicitDefine);
11839 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11840 if (AFI->branchProtectionPAuthLR())
11841 Builder.addReg(RegNo: AArch64::X15, Flags: RegState::ImplicitDefine);
11842 return;
11843 }
11844
11845 if (AFI->branchProtectionPAuthLR() && !Subtarget.hasPAuthLR())
11846 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11847}
11848
11849MachineBasicBlock::iterator
11850AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI,
11851 Register TargetReg, bool FrameSetup) const {
11852 assert(TargetReg != AArch64::SP && "New top of stack cannot already be in SP");
11853
11854 MachineBasicBlock &MBB = *MBBI->getParent();
11855 MachineFunction &MF = *MBB.getParent();
11856 const AArch64InstrInfo *TII =
11857 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
11858 int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
11859 DebugLoc DL = MBB.findDebugLoc(MBBI);
11860
11861 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
11862 MachineBasicBlock *LoopTestMBB =
11863 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11864 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
11865 MachineBasicBlock *LoopBodyMBB =
11866 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11867 MF.insert(MBBI: MBBInsertPoint, MBB: LoopBodyMBB);
11868 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11869 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
11870 MachineInstr::MIFlag Flags =
11871 FrameSetup ? MachineInstr::FrameSetup : MachineInstr::NoFlags;
11872
11873 // LoopTest:
11874 // SUB SP, SP, #ProbeSize
11875 emitFrameOffset(MBB&: *LoopTestMBB, MBBI: LoopTestMBB->end(), DL, DestReg: AArch64::SP,
11876 SrcReg: AArch64::SP, Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII, Flag: Flags);
11877
11878 // CMP SP, TargetReg
11879 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrx64),
11880 DestReg: AArch64::XZR)
11881 .addReg(RegNo: AArch64::SP)
11882 .addReg(RegNo: TargetReg)
11883 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0))
11884 .setMIFlags(Flags);
11885
11886 // B.<Cond> LoopExit
11887 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
11888 .addImm(Val: AArch64CC::LE)
11889 .addMBB(MBB: ExitMBB)
11890 .setMIFlags(Flags);
11891
11892 // LDR XZR, [SP]
11893 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11894 .addDef(RegNo: AArch64::XZR)
11895 .addReg(RegNo: AArch64::SP)
11896 .addImm(Val: 0)
11897 .addMemOperand(MMO: MF.getMachineMemOperand(
11898 PtrInfo: MachinePointerInfo::getUnknownStack(MF),
11899 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 8,
11900 BaseAlignment: Align(8)))
11901 .setMIFlags(Flags);
11902
11903 // B loop
11904 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
11905 .addMBB(MBB: LoopTestMBB)
11906 .setMIFlags(Flags);
11907
11908 // LoopExit:
11909 // MOV SP, TargetReg
11910 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: AArch64::SP)
11911 .addReg(RegNo: TargetReg)
11912 .addImm(Val: 0)
11913 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
11914 .setMIFlags(Flags);
11915
11916 // LDR XZR, [SP]
11917 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11918 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define)
11919 .addReg(RegNo: AArch64::SP)
11920 .addImm(Val: 0)
11921 .setMIFlags(Flags);
11922
11923 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
11924 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
11925
11926 LoopTestMBB->addSuccessor(Succ: ExitMBB);
11927 LoopTestMBB->addSuccessor(Succ: LoopBodyMBB);
11928 LoopBodyMBB->addSuccessor(Succ: LoopTestMBB);
11929 MBB.addSuccessor(Succ: LoopTestMBB);
11930
11931 // Update liveins.
11932 if (MF.getRegInfo().reservedRegsFrozen())
11933 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopBodyMBB, LoopTestMBB});
11934
11935 return ExitMBB->begin();
11936}
11937
11938namespace {
11939class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
11940 MachineFunction *MF;
11941 const TargetInstrInfo *TII;
11942 const TargetRegisterInfo *TRI;
11943 MachineRegisterInfo &MRI;
11944
11945 /// The block of the loop
11946 MachineBasicBlock *LoopBB;
11947 /// The conditional branch of the loop
11948 MachineInstr *CondBranch;
11949 /// The compare instruction for loop control
11950 MachineInstr *Comp;
11951 /// The number of the operand of the loop counter value in Comp
11952 unsigned CompCounterOprNum;
11953 /// The instruction that updates the loop counter value
11954 MachineInstr *Update;
11955 /// The number of the operand of the loop counter value in Update
11956 unsigned UpdateCounterOprNum;
11957 /// The initial value of the loop counter
11958 Register Init;
11959 /// True iff Update is a predecessor of Comp
11960 bool IsUpdatePriorComp;
11961
11962 /// The normalized condition used by createTripCountGreaterCondition()
11963 SmallVector<MachineOperand, 4> Cond;
11964
11965public:
11966 AArch64PipelinerLoopInfo(MachineBasicBlock *LoopBB, MachineInstr *CondBranch,
11967 MachineInstr *Comp, unsigned CompCounterOprNum,
11968 MachineInstr *Update, unsigned UpdateCounterOprNum,
11969 Register Init, bool IsUpdatePriorComp,
11970 const SmallVectorImpl<MachineOperand> &Cond)
11971 : MF(Comp->getParent()->getParent()),
11972 TII(MF->getSubtarget().getInstrInfo()),
11973 TRI(MF->getSubtarget().getRegisterInfo()), MRI(MF->getRegInfo()),
11974 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
11975 CompCounterOprNum(CompCounterOprNum), Update(Update),
11976 UpdateCounterOprNum(UpdateCounterOprNum), Init(Init),
11977 IsUpdatePriorComp(IsUpdatePriorComp), Cond(Cond.begin(), Cond.end()) {}
11978
11979 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
11980 // Make the instructions for loop control be placed in stage 0.
11981 // The predecessors of Comp are considered by the caller.
11982 return MI == Comp;
11983 }
11984
11985 std::optional<bool> createTripCountGreaterCondition(
11986 int TC, MachineBasicBlock &MBB,
11987 SmallVectorImpl<MachineOperand> &CondParam) override {
11988 // A branch instruction will be inserted as "if (Cond) goto epilogue".
11989 // Cond is normalized for such use.
11990 // The predecessors of the branch are assumed to have already been inserted.
11991 CondParam = Cond;
11992 return {};
11993 }
11994
11995 void createRemainingIterationsGreaterCondition(
11996 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
11997 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) override;
11998
11999 void setPreheader(MachineBasicBlock *NewPreheader) override {}
12000
12001 void adjustTripCount(int TripCountAdjust) override {}
12002
12003 bool isMVEExpanderSupported() override { return true; }
12004};
12005} // namespace
12006
12007/// Clone an instruction from MI. The register of ReplaceOprNum-th operand
12008/// is replaced by ReplaceReg. The output register is newly created.
12009/// The other operands are unchanged from MI.
12010static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum,
12011 Register ReplaceReg, MachineBasicBlock &MBB,
12012 MachineBasicBlock::iterator InsertTo) {
12013 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
12014 const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
12015 MachineInstr *NewMI = MBB.getParent()->CloneMachineInstr(Orig: MI);
12016 Register Result = 0;
12017 for (unsigned I = 0; I < NewMI->getNumOperands(); ++I) {
12018 if (I == 0 && NewMI->getOperand(i: 0).getReg().isVirtual()) {
12019 Result = MRI.createVirtualRegister(
12020 RegClass: MRI.getRegClass(Reg: NewMI->getOperand(i: 0).getReg()));
12021 NewMI->getOperand(i: I).setReg(Result);
12022 } else if (I == ReplaceOprNum) {
12023 MRI.constrainRegClass(Reg: ReplaceReg, RC: TII->getRegClass(MCID: NewMI->getDesc(), OpNum: I));
12024 NewMI->getOperand(i: I).setReg(ReplaceReg);
12025 }
12026 }
12027 MBB.insert(I: InsertTo, MI: NewMI);
12028 return Result;
12029}
12030
12031void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
12032 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
12033 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) {
12034 // Create and accumulate conditions for next TC iterations.
12035 // Example:
12036 // SUBSXrr N, counter, implicit-def $nzcv # compare instruction for the last
12037 // # iteration of the kernel
12038 //
12039 // # insert the following instructions
12040 // cond = CSINCXr 0, 0, C, implicit $nzcv
12041 // counter = ADDXri counter, 1 # clone from this->Update
12042 // SUBSXrr n, counter, implicit-def $nzcv # clone from this->Comp
12043 // cond = CSINCXr cond, cond, C, implicit $nzcv
12044 // ... (repeat TC times)
12045 // SUBSXri cond, 0, implicit-def $nzcv
12046
12047 assert(CondBranch->getOpcode() == AArch64::Bcc);
12048 // CondCode to exit the loop
12049 AArch64CC::CondCode CC =
12050 (AArch64CC::CondCode)CondBranch->getOperand(i: 0).getImm();
12051 if (CondBranch->getOperand(i: 1).getMBB() == LoopBB)
12052 CC = AArch64CC::getInvertedCondCode(Code: CC);
12053
12054 // Accumulate conditions to exit the loop
12055 Register AccCond = AArch64::XZR;
12056
12057 // If CC holds, CurCond+1 is returned; otherwise CurCond is returned.
12058 auto AccumulateCond = [&](Register CurCond,
12059 AArch64CC::CondCode CC) -> Register {
12060 Register NewCond = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
12061 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::CSINCXr))
12062 .addReg(RegNo: NewCond, Flags: RegState::Define)
12063 .addReg(RegNo: CurCond)
12064 .addReg(RegNo: CurCond)
12065 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
12066 return NewCond;
12067 };
12068
12069 if (!LastStage0Insts.empty() && LastStage0Insts[Comp]->getParent() == &MBB) {
12070 // Update and Comp for I==0 are already exists in MBB
12071 // (MBB is an unrolled kernel)
12072 Register Counter;
12073 for (int I = 0; I <= TC; ++I) {
12074 Register NextCounter;
12075 if (I != 0)
12076 NextCounter =
12077 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12078
12079 AccCond = AccumulateCond(AccCond, CC);
12080
12081 if (I != TC) {
12082 if (I == 0) {
12083 if (Update != Comp && IsUpdatePriorComp) {
12084 Counter =
12085 LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12086 NextCounter = cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB,
12087 InsertTo: MBB.end());
12088 } else {
12089 // can use already calculated value
12090 NextCounter = LastStage0Insts[Update]->getOperand(i: 0).getReg();
12091 }
12092 } else if (Update != Comp) {
12093 NextCounter =
12094 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12095 }
12096 }
12097 Counter = NextCounter;
12098 }
12099 } else {
12100 Register Counter;
12101 if (LastStage0Insts.empty()) {
12102 // use initial counter value (testing if the trip count is sufficient to
12103 // be executed by pipelined code)
12104 Counter = Init;
12105 if (IsUpdatePriorComp)
12106 Counter =
12107 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12108 } else {
12109 // MBB is an epilogue block. LastStage0Insts[Comp] is in the kernel block.
12110 Counter = LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12111 }
12112
12113 for (int I = 0; I <= TC; ++I) {
12114 Register NextCounter;
12115 NextCounter =
12116 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12117 AccCond = AccumulateCond(AccCond, CC);
12118 if (I != TC && Update != Comp)
12119 NextCounter =
12120 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12121 Counter = NextCounter;
12122 }
12123 }
12124
12125 // If AccCond == 0, the remainder is greater than TC.
12126 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::SUBSXri))
12127 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define | RegState::Dead)
12128 .addReg(RegNo: AccCond)
12129 .addImm(Val: 0)
12130 .addImm(Val: 0);
12131 Cond.clear();
12132 Cond.push_back(Elt: MachineOperand::CreateImm(Val: AArch64CC::EQ));
12133}
12134
12135static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB,
12136 Register &RegMBB, Register &RegOther) {
12137 assert(Phi.getNumOperands() == 5);
12138 if (Phi.getOperand(i: 2).getMBB() == MBB) {
12139 RegMBB = Phi.getOperand(i: 1).getReg();
12140 RegOther = Phi.getOperand(i: 3).getReg();
12141 } else {
12142 assert(Phi.getOperand(4).getMBB() == MBB);
12143 RegMBB = Phi.getOperand(i: 3).getReg();
12144 RegOther = Phi.getOperand(i: 1).getReg();
12145 }
12146}
12147
12148static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB) {
12149 if (!Reg.isVirtual())
12150 return false;
12151 const MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
12152 return MRI.getDefBlock(Reg) != BB;
12153}
12154
12155/// If Reg is an induction variable, return true and set some parameters
12156static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
12157 MachineInstr *&UpdateInst,
12158 unsigned &UpdateCounterOprNum, Register &InitReg,
12159 bool &IsUpdatePriorComp) {
12160 // Example:
12161 //
12162 // Preheader:
12163 // InitReg = ...
12164 // LoopBB:
12165 // Reg0 = PHI (InitReg, Preheader), (Reg1, LoopBB)
12166 // Reg = COPY Reg0 ; COPY is ignored.
12167 // Reg1 = ADD Reg, #1; UpdateInst. Incremented by a loop invariant value.
12168 // ; Reg is the value calculated in the previous
12169 // ; iteration, so IsUpdatePriorComp == false.
12170
12171 if (LoopBB->pred_size() != 2)
12172 return false;
12173 if (!Reg.isVirtual())
12174 return false;
12175 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
12176 UpdateInst = nullptr;
12177 UpdateCounterOprNum = 0;
12178 InitReg = 0;
12179 IsUpdatePriorComp = true;
12180 Register CurReg = Reg;
12181 while (true) {
12182 MachineInstr *Def = MRI.getVRegDef(Reg: CurReg);
12183 if (Def->getParent() != LoopBB)
12184 return false;
12185 if (Def->isCopy()) {
12186 // Ignore copy instructions unless they contain subregisters
12187 if (Def->getOperand(i: 0).getSubReg() || Def->getOperand(i: 1).getSubReg())
12188 return false;
12189 CurReg = Def->getOperand(i: 1).getReg();
12190 } else if (Def->isPHI()) {
12191 if (InitReg != 0)
12192 return false;
12193 if (!UpdateInst)
12194 IsUpdatePriorComp = false;
12195 extractPhiReg(Phi: *Def, MBB: LoopBB, RegMBB&: CurReg, RegOther&: InitReg);
12196 } else {
12197 if (UpdateInst)
12198 return false;
12199 switch (Def->getOpcode()) {
12200 case AArch64::ADDSXri:
12201 case AArch64::ADDSWri:
12202 case AArch64::SUBSXri:
12203 case AArch64::SUBSWri:
12204 case AArch64::ADDXri:
12205 case AArch64::ADDWri:
12206 case AArch64::SUBXri:
12207 case AArch64::SUBWri:
12208 UpdateInst = Def;
12209 UpdateCounterOprNum = 1;
12210 break;
12211 case AArch64::ADDSXrr:
12212 case AArch64::ADDSWrr:
12213 case AArch64::SUBSXrr:
12214 case AArch64::SUBSWrr:
12215 case AArch64::ADDXrr:
12216 case AArch64::ADDWrr:
12217 case AArch64::SUBXrr:
12218 case AArch64::SUBWrr:
12219 UpdateInst = Def;
12220 if (isDefinedOutside(Reg: Def->getOperand(i: 2).getReg(), BB: LoopBB))
12221 UpdateCounterOprNum = 1;
12222 else if (isDefinedOutside(Reg: Def->getOperand(i: 1).getReg(), BB: LoopBB))
12223 UpdateCounterOprNum = 2;
12224 else
12225 return false;
12226 break;
12227 default:
12228 return false;
12229 }
12230 CurReg = Def->getOperand(i: UpdateCounterOprNum).getReg();
12231 }
12232
12233 if (!CurReg.isVirtual())
12234 return false;
12235 if (Reg == CurReg)
12236 break;
12237 }
12238
12239 if (!UpdateInst)
12240 return false;
12241
12242 return true;
12243}
12244
12245std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
12246AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
12247 // Accept loops that meet the following conditions
12248 // * The conditional branch is BCC
12249 // * The compare instruction is ADDS/SUBS/WHILEXX
12250 // * One operand of the compare is an induction variable and the other is a
12251 // loop invariant value
12252 // * The induction variable is incremented/decremented by a single instruction
12253 // * Does not contain CALL or instructions which have unmodeled side effects
12254
12255 for (MachineInstr &MI : *LoopBB)
12256 if (MI.isCall() || MI.hasUnmodeledSideEffects())
12257 // This instruction may use NZCV, which interferes with the instruction to
12258 // be inserted for loop control.
12259 return nullptr;
12260
12261 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
12262 SmallVector<MachineOperand, 4> Cond;
12263 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond))
12264 return nullptr;
12265
12266 // Infinite loops are not supported
12267 if (TBB == LoopBB && FBB == LoopBB)
12268 return nullptr;
12269
12270 // Must be conditional branch
12271 if (TBB != LoopBB && FBB == nullptr)
12272 return nullptr;
12273
12274 assert((TBB == LoopBB || FBB == LoopBB) &&
12275 "The Loop must be a single-basic-block loop");
12276
12277 MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
12278 const TargetRegisterInfo &TRI = getRegisterInfo();
12279
12280 if (CondBranch->getOpcode() != AArch64::Bcc)
12281 return nullptr;
12282
12283 // Normalization for createTripCountGreaterCondition()
12284 if (TBB == LoopBB)
12285 reverseBranchCondition(Cond);
12286
12287 MachineInstr *Comp = nullptr;
12288 unsigned CompCounterOprNum = 0;
12289 for (MachineInstr &MI : reverse(C&: *LoopBB)) {
12290 if (MI.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
12291 // Guarantee that the compare is SUBS/ADDS/WHILEXX and that one of the
12292 // operands is a loop invariant value
12293
12294 switch (MI.getOpcode()) {
12295 case AArch64::SUBSXri:
12296 case AArch64::SUBSWri:
12297 case AArch64::ADDSXri:
12298 case AArch64::ADDSWri:
12299 Comp = &MI;
12300 CompCounterOprNum = 1;
12301 break;
12302 case AArch64::ADDSWrr:
12303 case AArch64::ADDSXrr:
12304 case AArch64::SUBSWrr:
12305 case AArch64::SUBSXrr:
12306 Comp = &MI;
12307 break;
12308 default:
12309 if (isWhileOpcode(Opc: MI.getOpcode())) {
12310 Comp = &MI;
12311 break;
12312 }
12313 return nullptr;
12314 }
12315
12316 if (CompCounterOprNum == 0) {
12317 if (isDefinedOutside(Reg: Comp->getOperand(i: 1).getReg(), BB: LoopBB))
12318 CompCounterOprNum = 2;
12319 else if (isDefinedOutside(Reg: Comp->getOperand(i: 2).getReg(), BB: LoopBB))
12320 CompCounterOprNum = 1;
12321 else
12322 return nullptr;
12323 }
12324 break;
12325 }
12326 }
12327 if (!Comp)
12328 return nullptr;
12329
12330 MachineInstr *Update = nullptr;
12331 Register Init;
12332 bool IsUpdatePriorComp;
12333 unsigned UpdateCounterOprNum;
12334 if (!getIndVarInfo(Reg: Comp->getOperand(i: CompCounterOprNum).getReg(), LoopBB,
12335 UpdateInst&: Update, UpdateCounterOprNum, InitReg&: Init, IsUpdatePriorComp))
12336 return nullptr;
12337
12338 return std::make_unique<AArch64PipelinerLoopInfo>(
12339 args&: LoopBB, args&: CondBranch, args&: Comp, args&: CompCounterOprNum, args&: Update, args&: UpdateCounterOprNum,
12340 args&: Init, args&: IsUpdatePriorComp, args&: Cond);
12341}
12342
12343/// verifyInstruction - Perform target specific instruction verification.
12344bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI,
12345 StringRef &ErrInfo) const {
12346 // Verify that immediate offsets on load/store instructions are within range.
12347 // Stack objects with an FI operand are excluded as they can be fixed up
12348 // during PEI.
12349 TypeSize Scale(0U, false), Width(0U, false);
12350 int64_t MinOffset, MaxOffset;
12351 if (getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
12352 unsigned ImmIdx = getLoadStoreImmIdx(Opc: MI.getOpcode());
12353 if (MI.getOperand(i: ImmIdx).isImm() && !MI.getOperand(i: ImmIdx - 1).isFI()) {
12354 int64_t Imm = MI.getOperand(i: ImmIdx).getImm();
12355 if (Imm < MinOffset || Imm > MaxOffset) {
12356 ErrInfo = "Unexpected immediate on load/store instruction";
12357 return false;
12358 }
12359 }
12360 }
12361
12362 const MCInstrDesc &MCID = MI.getDesc();
12363 for (unsigned Op = 0; Op < MCID.getNumOperands(); Op++) {
12364 const MachineOperand &MO = MI.getOperand(i: Op);
12365 switch (MCID.operands()[Op].OperandType) {
12366 case AArch64::OPERAND_IMPLICIT_IMM_0:
12367 if (!MO.isImm() || MO.getImm() != 0) {
12368 ErrInfo = "OPERAND_IMPLICIT_IMM_0 should be 0";
12369 return false;
12370 }
12371 break;
12372 case AArch64::OPERAND_SHIFT_MSL:
12373 if (!MO.isImm() ||
12374 AArch64_AM::getShiftType(Imm: MO.getImm()) != AArch64_AM::MSL ||
12375 (AArch64_AM::getShiftValue(Imm: MO.getImm()) != 8 &&
12376 AArch64_AM::getShiftValue(Imm: MO.getImm()) != 16)) {
12377 ErrInfo = "OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
12378 return false;
12379 }
12380 break;
12381 case AArch64::OPERAND_IMM_UINT1:
12382 if (!MO.isImm() || (MO.getImm() != 0 && MO.getImm() != 1)) {
12383 ErrInfo = "OPERAND_IMM_UINT1 should be 0 or 1";
12384 return false;
12385 }
12386 break;
12387 case AArch64::OPERAND_IMM_UINT4plus1:
12388 if (!MO.isImm() || MO.getImm() <= 0 || MO.getImm() > 16) {
12389 ErrInfo = "OPERAND_IMM_UINT4plus1 should be in the range 1 to 16";
12390 return false;
12391 }
12392 break;
12393 case AArch64::OPERAND_IMM_UINT5:
12394 if (!MO.isImm() || !isUInt<5>(x: MO.getImm())) {
12395 ErrInfo = "OPERAND_IMM_UINT5 should be in the range 0 to 31";
12396 return false;
12397 }
12398 break;
12399 case AArch64::OPERAND_IMM_UINT8:
12400 if (!MO.isImm() || !isUInt<8>(x: MO.getImm())) {
12401 ErrInfo = "OPERAND_IMM_UINT8 should be in the range 0 to 255";
12402 return false;
12403 }
12404 break;
12405 default:
12406 break;
12407 }
12408 }
12409 return true;
12410}
12411
12412#define GET_INSTRINFO_HELPERS
12413#define GET_INSTRMAP_INFO
12414#include "AArch64GenInstrInfo.inc"
12415