1//===- AArch64InstrInfo.cpp - AArch64 Instruction Information -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the AArch64 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64InstrInfo.h"
14#include "AArch64ExpandImm.h"
15#include "AArch64MachineFunctionInfo.h"
16#include "AArch64PointerAuth.h"
17#include "AArch64Subtarget.h"
18#include "MCTargetDesc/AArch64AddressingModes.h"
19#include "MCTargetDesc/AArch64MCLFIRewriter.h"
20#include "MCTargetDesc/AArch64MCTargetDesc.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/SmallVector.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/CodeGen/CFIInstBuilder.h"
28#include "llvm/CodeGen/LivePhysRegs.h"
29#include "llvm/CodeGen/LiveRegUnits.h"
30#include "llvm/CodeGen/MachineBasicBlock.h"
31#include "llvm/CodeGen/MachineCombinerPattern.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstr.h"
35#include "llvm/CodeGen/MachineInstrBuilder.h"
36#include "llvm/CodeGen/MachineMemOperand.h"
37#include "llvm/CodeGen/MachineModuleInfo.h"
38#include "llvm/CodeGen/MachineOperand.h"
39#include "llvm/CodeGen/MachineRegisterInfo.h"
40#include "llvm/CodeGen/RegisterScavenging.h"
41#include "llvm/CodeGen/StackMaps.h"
42#include "llvm/CodeGen/TargetRegisterInfo.h"
43#include "llvm/CodeGen/TargetSubtargetInfo.h"
44#include "llvm/IR/DebugInfoMetadata.h"
45#include "llvm/IR/DebugLoc.h"
46#include "llvm/IR/GlobalValue.h"
47#include "llvm/IR/Module.h"
48#include "llvm/MC/MCAsmInfo.h"
49#include "llvm/MC/MCInst.h"
50#include "llvm/MC/MCInstBuilder.h"
51#include "llvm/MC/MCInstrDesc.h"
52#include "llvm/Support/Casting.h"
53#include "llvm/Support/CodeGen.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/LEB128.h"
56#include "llvm/Support/MathExtras.h"
57#include "llvm/Target/TargetMachine.h"
58#include "llvm/Target/TargetOptions.h"
59#include <cassert>
60#include <cstdint>
61#include <iterator>
62#include <utility>
63
64using namespace llvm;
65
66#define GET_INSTRINFO_CTOR_DTOR
67#include "AArch64GenInstrInfo.inc"
68
69#define DEBUG_TYPE "AArch64InstrInfo"
70
71STATISTIC(NumCopyInstrs, "Number of COPY instructions expanded");
72STATISTIC(NumZCRegMoveInstrsGPR, "Number of zero-cycle GPR register move "
73 "instructions expanded from canonical COPY");
74STATISTIC(NumZCRegMoveInstrsFPR, "Number of zero-cycle FPR register move "
75 "instructions expanded from canonical COPY");
76STATISTIC(NumZCZeroingInstrsGPR, "Number of zero-cycle GPR zeroing "
77 "instructions expanded from canonical COPY");
78// NumZCZeroingInstrsFPR is counted at AArch64AsmPrinter
79
80AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
81 : AArch64GenInstrInfo(STI, RI, AArch64::ADJCALLSTACKDOWN,
82 AArch64::ADJCALLSTACKUP, AArch64::CATCHRET),
83 RI(STI.getTargetTriple(), STI.getHwMode()), Subtarget(STI) {}
84
85/// Return the maximum number of bytes of code the specified instruction may be
86/// after LFI rewriting. If the instruction is not rewritten, std::nullopt is
87/// returned (use default sizing).
88///
89/// NOTE: the size estimates here must be kept in sync with the rewrites in
90/// AArch64MCLFIRewriter.cpp. Sizes may be overestimates of the rewritten
91/// instruction sequences.
92static std::optional<unsigned> getLFIInstSizeInBytes(const MachineInstr &MI) {
93 switch (MI.getOpcode()) {
94 case AArch64::SVC:
95 // SVC expands to 4 instructions.
96 return 16;
97 case AArch64::BR:
98 case AArch64::BLR:
99 // Indirect branches/calls expand to 2 instructions (guard + br/blr).
100 return 8;
101 case AArch64::RET:
102 // RET through another register expands to 2 instructions (guard + ret).
103 // RET through LR may also expand to 2 instructions if a deferred LR guard
104 // is flushed before the return.
105 return 8;
106 case AArch64::RETAA:
107 case AArch64::RETAB:
108 // Authenticated returns expand to 3 instructions (authenticate + guard +
109 // ret).
110 return 12;
111 case AArch64::BRAA:
112 case AArch64::BRAAZ:
113 case AArch64::BRAB:
114 case AArch64::BRABZ:
115 case AArch64::BLRAA:
116 case AArch64::BLRAAZ:
117 case AArch64::BLRAB:
118 case AArch64::BLRABZ:
119 // Authenticated branches/calls expand to 3 instructions (authenticate +
120 // guard + branch).
121 return 12;
122 case AArch64::AUTIASP:
123 case AArch64::AUTIBSP:
124 case AArch64::AUTIAZ:
125 case AArch64::AUTIBZ:
126 case AArch64::XPACLRI:
127 // Authenticating LR expands to the instruction plus a deferred LR guard.
128 return 8;
129 case AArch64::SYSxt:
130 // VA-based DC/IC ops (op1=3, Cn=7, op2=1) expand to 2 instructions.
131 if (MI.getOperand(i: 0).getImm() == 3 && MI.getOperand(i: 1).getImm() == 7 &&
132 MI.getOperand(i: 3).getImm() == 1)
133 return 8;
134 return std::nullopt;
135 default:
136 break;
137 }
138
139 // Detect instructions that explicitly define SP or LR.
140 bool ModifiesLR = false;
141 bool ModifiesSP = false;
142 for (const MachineOperand &MO : MI.defs()) {
143 if (!MO.isReg())
144 continue;
145 if (MO.getReg() == AArch64::LR)
146 ModifiesLR = true;
147 else if (MO.getReg() == AArch64::SP)
148 ModifiesSP = true;
149 }
150
151 // Memory accesses expand to a base-register guard plus the rewritten access
152 // (8 bytes), with an extra base-register update for pre/post-index forms (12
153 // bytes total). If the access also defines LR, an LR mask is appended (+4
154 // bytes). Depending on additional optimizations that the rewriter performs,
155 // this may be an overestimate.
156 if (MI.mayLoadOrStore()) {
157 unsigned Size = isLFIPrePostMemAccess(Opcode: MI.getOpcode()) ? 12 : 8;
158 if (ModifiesLR)
159 Size += 4;
160 return Size;
161 }
162
163 // Non memory operations that modify LR or SP expand to 2 instructions.
164 if (ModifiesSP || ModifiesLR)
165 return 8;
166
167 // Default case: instructions that don't cause expansion.
168 // - TP accesses in LFI are a single load/store, so no expansion.
169 // - All remaining instructions are not rewritten.
170 return std::nullopt;
171}
172
173/// GetInstSize - Return the number of bytes of code the specified
174/// instruction may be. This returns the maximum number of bytes.
175unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
176 const MCInstrDesc &Desc = MI.getDesc();
177 if (!Desc.isPseudo() && !Subtarget.isLFI()) {
178 assert(Desc.getSize() == 4 && "Unexpected instruction size");
179 return 4;
180 }
181
182 const MachineBasicBlock &MBB = *MI.getParent();
183 const MachineFunction *MF = MBB.getParent();
184 const Function &F = MF->getFunction();
185 const MCAsmInfo &MAI = MF->getTarget().getMCAsmInfo();
186
187 {
188 auto Op = MI.getOpcode();
189 if (Op == AArch64::INLINEASM || Op == AArch64::INLINEASM_BR)
190 return getInlineAsmLength(Str: MI.getOperand(i: 0).getSymbolName(), MAI);
191 }
192
193 // Meta-instructions emit no code.
194 if (MI.isMetaInstruction())
195 return 0;
196
197 // FIXME: We currently only handle pseudoinstructions that don't get expanded
198 // before the assembly printer.
199 unsigned NumBytes = 0;
200
201 // LFI rewriter expansions that supersede normal sizing.
202 const auto &STI = MF->getSubtarget<AArch64Subtarget>();
203 if (STI.isLFI())
204 if (auto Size = getLFIInstSizeInBytes(MI))
205 return *Size;
206
207 if (!MI.isBundle() && isTailCallReturnInst(MI)) {
208 NumBytes = Desc.getSize() ? Desc.getSize() : 4;
209
210 const auto *MFI = MF->getInfo<AArch64FunctionInfo>();
211 if (!MFI->shouldSignReturnAddress(MF: *MF))
212 return NumBytes;
213
214 auto Method = STI.getAuthenticatedLRCheckMethod(MF: *MF);
215 NumBytes += AArch64PAuth::getCheckerSizeInBytes(Method);
216 return NumBytes;
217 }
218
219 // Size should be preferably set in
220 // llvm/lib/Target/AArch64/AArch64InstrInfo.td (default case).
221 // Specific cases handle instructions of variable sizes
222 switch (Desc.getOpcode()) {
223 default:
224 if (Desc.getSize())
225 return Desc.getSize();
226
227 // Anything not explicitly designated otherwise (i.e. pseudo-instructions
228 // with fixed constant size but not specified in .td file) is a normal
229 // 4-byte insn.
230 NumBytes = 4;
231 break;
232 case TargetOpcode::STACKMAP:
233 // The upper bound for a stackmap intrinsic is the full length of its shadow
234 NumBytes = StackMapOpers(&MI).getNumPatchBytes();
235 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
236 break;
237 case TargetOpcode::PATCHPOINT:
238 // The size of the patchpoint intrinsic is the number of bytes requested
239 NumBytes = PatchPointOpers(&MI).getNumPatchBytes();
240 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
241 break;
242 case TargetOpcode::STATEPOINT:
243 NumBytes = StatepointOpers(&MI).getNumPatchBytes();
244 assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
245 // No patch bytes means a normal call inst is emitted
246 if (NumBytes == 0)
247 NumBytes = 4;
248 break;
249 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
250 // If `patchable-function-entry` is set, PATCHABLE_FUNCTION_ENTER
251 // instructions are expanded to the specified number of NOPs. Otherwise,
252 // they are expanded to 36-byte XRay sleds.
253 NumBytes =
254 F.getFnAttributeAsParsedInteger(Kind: "patchable-function-entry", Default: 9) * 4;
255 break;
256 case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
257 case TargetOpcode::PATCHABLE_TAIL_CALL:
258 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
259 // An XRay sled can be 4 bytes of alignment plus a 32-byte block.
260 NumBytes = 36;
261 break;
262 case TargetOpcode::PATCHABLE_EVENT_CALL:
263 // EVENT_CALL XRay sleds are exactly 6 instructions long (no alignment).
264 NumBytes = 24;
265 break;
266
267 case AArch64::SPACE:
268 NumBytes = MI.getOperand(i: 1).getImm();
269 break;
270 case AArch64::MOVaddr:
271 case AArch64::MOVaddrJT:
272 case AArch64::MOVaddrCP:
273 case AArch64::MOVaddrBA:
274 case AArch64::MOVaddrTLS:
275 case AArch64::MOVaddrEXT: {
276 // Use the same logic as the pseudo expansion to count instructions.
277 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
278 AArch64_IMM::expandMOVAddr(Opcode: Desc.getOpcode(),
279 TargetFlags: MI.getOperand(i: 1).getTargetFlags(),
280 IsTargetMachO: Subtarget.isTargetMachO(), Insn);
281 NumBytes = Insn.size() * 4;
282 break;
283 }
284
285 case AArch64::MOVi32imm:
286 case AArch64::MOVi64imm: {
287 // Use the same logic as the pseudo expansion to count instructions.
288 unsigned BitSize = Desc.getOpcode() == AArch64::MOVi32imm ? 32 : 64;
289 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
290 AArch64_IMM::expandMOVImm(Imm: MI.getOperand(i: 1).getImm(), BitSize, Insn);
291 NumBytes = Insn.size() * 4;
292 break;
293 }
294
295 case TargetOpcode::BUNDLE:
296 NumBytes = getInstBundleSize(MI);
297 break;
298 }
299
300 return NumBytes;
301}
302
303static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target,
304 SmallVectorImpl<MachineOperand> &Cond) {
305 // Block ends with fall-through condbranch.
306 switch (LastInst->getOpcode()) {
307 default:
308 llvm_unreachable("Unknown branch instruction?");
309 case AArch64::Bcc:
310 Target = LastInst->getOperand(i: 1).getMBB();
311 Cond.push_back(Elt: LastInst->getOperand(i: 0));
312 break;
313 case AArch64::CBZW:
314 case AArch64::CBZX:
315 case AArch64::CBNZW:
316 case AArch64::CBNZX:
317 Target = LastInst->getOperand(i: 1).getMBB();
318 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
319 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
320 Cond.push_back(Elt: LastInst->getOperand(i: 0));
321 break;
322 case AArch64::TBZW:
323 case AArch64::TBZX:
324 case AArch64::TBNZW:
325 case AArch64::TBNZX:
326 Target = LastInst->getOperand(i: 2).getMBB();
327 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
328 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
329 Cond.push_back(Elt: LastInst->getOperand(i: 0));
330 Cond.push_back(Elt: LastInst->getOperand(i: 1));
331 break;
332 case AArch64::CBWPri:
333 case AArch64::CBXPri:
334 case AArch64::CBWPrr:
335 case AArch64::CBXPrr:
336 Target = LastInst->getOperand(i: 3).getMBB();
337 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1));
338 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode()));
339 Cond.push_back(Elt: LastInst->getOperand(i: 0));
340 Cond.push_back(Elt: LastInst->getOperand(i: 1));
341 Cond.push_back(Elt: LastInst->getOperand(i: 2));
342 break;
343 case AArch64::CBBAssertExt:
344 case AArch64::CBHAssertExt:
345 Target = LastInst->getOperand(i: 3).getMBB();
346 Cond.push_back(Elt: MachineOperand::CreateImm(Val: -1)); // -1
347 Cond.push_back(Elt: MachineOperand::CreateImm(Val: LastInst->getOpcode())); // Opc
348 Cond.push_back(Elt: LastInst->getOperand(i: 0)); // Cond
349 Cond.push_back(Elt: LastInst->getOperand(i: 1)); // Op0
350 Cond.push_back(Elt: LastInst->getOperand(i: 2)); // Op1
351 Cond.push_back(Elt: LastInst->getOperand(i: 4)); // Ext0
352 Cond.push_back(Elt: LastInst->getOperand(i: 5)); // Ext1
353 break;
354 }
355}
356
357static unsigned getBranchDisplacementBits(const AArch64Options &CLOpts,
358 unsigned Opc) {
359 switch (Opc) {
360 default:
361 llvm_unreachable("unexpected opcode!");
362 case AArch64::B:
363 return CLOpts.b_offset_bits;
364 case AArch64::TBNZW:
365 case AArch64::TBZW:
366 case AArch64::TBNZX:
367 case AArch64::TBZX:
368 return CLOpts.tbz_offset_bits;
369 case AArch64::CBNZW:
370 case AArch64::CBZW:
371 case AArch64::CBNZX:
372 case AArch64::CBZX:
373 return CLOpts.cbz_offset_bits;
374 case AArch64::Bcc:
375 return CLOpts.bcc_offset_bits;
376 case AArch64::CBWPri:
377 case AArch64::CBXPri:
378 case AArch64::CBBAssertExt:
379 case AArch64::CBHAssertExt:
380 case AArch64::CBWPrr:
381 case AArch64::CBXPrr:
382 return CLOpts.cb_offset_bits;
383 }
384}
385
386bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp,
387 int64_t BrOffset) const {
388 unsigned Bits = getBranchDisplacementBits(CLOpts: Subtarget.getCLOpts(), Opc: BranchOp);
389 assert(Bits >= 3 && "max branch displacement must be enough to jump"
390 "over conditional branch expansion");
391 return isIntN(N: Bits, x: BrOffset / 4);
392}
393
394MachineBasicBlock *
395AArch64InstrInfo::getBranchDestBlock(const MachineInstr &MI) const {
396 switch (MI.getOpcode()) {
397 default:
398 llvm_unreachable("unexpected opcode!");
399 case AArch64::B:
400 return MI.getOperand(i: 0).getMBB();
401 case AArch64::TBZW:
402 case AArch64::TBNZW:
403 case AArch64::TBZX:
404 case AArch64::TBNZX:
405 return MI.getOperand(i: 2).getMBB();
406 case AArch64::CBZW:
407 case AArch64::CBNZW:
408 case AArch64::CBZX:
409 case AArch64::CBNZX:
410 case AArch64::Bcc:
411 return MI.getOperand(i: 1).getMBB();
412 case AArch64::CBWPri:
413 case AArch64::CBXPri:
414 case AArch64::CBBAssertExt:
415 case AArch64::CBHAssertExt:
416 case AArch64::CBWPrr:
417 case AArch64::CBXPrr:
418 return MI.getOperand(i: 3).getMBB();
419 }
420}
421
422void AArch64InstrInfo::insertIndirectBranch(MachineBasicBlock &MBB,
423 MachineBasicBlock &NewDestBB,
424 MachineBasicBlock &RestoreBB,
425 const DebugLoc &DL,
426 int64_t BrOffset,
427 RegScavenger *RS) const {
428 assert(RS && "RegScavenger required for long branching");
429 assert(MBB.empty() &&
430 "new block should be inserted for expanding unconditional branch");
431 assert(MBB.pred_size() == 1);
432 assert(RestoreBB.empty() &&
433 "restore block should be inserted for restoring clobbered registers");
434
435 auto buildIndirectBranch = [&](Register Reg, MachineBasicBlock &DestBB) {
436 // Offsets outside of the signed 33-bit range are not supported for ADRP +
437 // ADD.
438 if (!isInt<33>(x: BrOffset))
439 report_fatal_error(
440 reason: "Branch offsets outside of the signed 33-bit range not supported");
441
442 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
443 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGE);
444 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: Reg)
445 .addReg(RegNo: Reg)
446 .addSym(Sym: DestBB.getSymbol(), TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
447 .addImm(Val: 0);
448 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::BR)).addReg(RegNo: Reg);
449 };
450
451 RS->enterBasicBlockEnd(MBB);
452 // If X16 is unused, we can rely on the linker to insert a range extension
453 // thunk if NewDestBB is out of range of a single B instruction.
454 constexpr Register Reg = AArch64::X16;
455 if (!RS->isRegUsed(Reg)) {
456 insertUnconditionalBranch(MBB, DestBB: &NewDestBB, DL);
457 RS->setRegUsed(Reg);
458 return;
459 }
460
461 // In a cold block without BTI, insert the indirect branch if a register is
462 // free. Skip this if BTI is enabled to avoid inserting a BTI at the target,
463 // prioritizing a dynamic cost in cold code over a static cost in hot code.
464 AArch64FunctionInfo *AFI = MBB.getParent()->getInfo<AArch64FunctionInfo>();
465 bool HasBTI = AFI && AFI->branchTargetEnforcement();
466 if (MBB.getSectionID() == MBBSectionID::ColdSectionID && !HasBTI) {
467 Register Scavenged = RS->FindUnusedReg(RC: &AArch64::GPR64RegClass);
468 if (Scavenged.isValid()) {
469 buildIndirectBranch(Scavenged, NewDestBB);
470 RS->setRegUsed(Reg: Scavenged);
471 return;
472 }
473 }
474
475 // Note: Spilling X16 briefly moves the stack pointer, making it incompatible
476 // with red zones.
477 if (!AFI || AFI->hasRedZone().value_or(u: true))
478 report_fatal_error(
479 reason: "Unable to insert indirect branch inside function that has red zone");
480
481 // Otherwise, spill X16 and defer range extension to the linker.
482 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::STRXpre))
483 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
484 .addReg(RegNo: Reg)
485 .addReg(RegNo: AArch64::SP)
486 .addImm(Val: -16);
487
488 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: &RestoreBB);
489
490 BuildMI(BB&: RestoreBB, I: RestoreBB.end(), MIMD: DL, MCID: get(Opcode: AArch64::LDRXpost))
491 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
492 .addReg(RegNo: Reg, Flags: RegState::Define)
493 .addReg(RegNo: AArch64::SP)
494 .addImm(Val: 16);
495}
496
497// Branch analysis.
498bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB,
499 MachineBasicBlock *&TBB,
500 MachineBasicBlock *&FBB,
501 SmallVectorImpl<MachineOperand> &Cond,
502 bool AllowModify) const {
503 // If the block has no terminators, it just falls into the block after it.
504 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
505 if (I == MBB.end())
506 return false;
507
508 // Skip over SpeculationBarrierEndBB terminators
509 if (I->getOpcode() == AArch64::SpeculationBarrierISBDSBEndBB ||
510 I->getOpcode() == AArch64::SpeculationBarrierSBEndBB) {
511 --I;
512 }
513
514 if (!isUnpredicatedTerminator(MI: *I))
515 return false;
516
517 // Get the last instruction in the block.
518 MachineInstr *LastInst = &*I;
519
520 // If there is only one terminator instruction, process it.
521 unsigned LastOpc = LastInst->getOpcode();
522 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
523 if (isUncondBranchOpcode(Opc: LastOpc)) {
524 TBB = LastInst->getOperand(i: 0).getMBB();
525 return false;
526 }
527 if (isCondBranchOpcode(Opc: LastOpc)) {
528 // Block ends with fall-through condbranch.
529 parseCondBranch(LastInst, Target&: TBB, Cond);
530 return false;
531 }
532 return true; // Can't handle indirect branch.
533 }
534
535 // Get the instruction before it if it is a terminator.
536 MachineInstr *SecondLastInst = &*I;
537 unsigned SecondLastOpc = SecondLastInst->getOpcode();
538
539 // If AllowModify is true and the block ends with two or more unconditional
540 // branches, delete all but the first unconditional branch.
541 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc)) {
542 while (isUncondBranchOpcode(Opc: SecondLastOpc)) {
543 LastInst->eraseFromParent();
544 LastInst = SecondLastInst;
545 LastOpc = LastInst->getOpcode();
546 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
547 // Return now the only terminator is an unconditional branch.
548 TBB = LastInst->getOperand(i: 0).getMBB();
549 return false;
550 }
551 SecondLastInst = &*I;
552 SecondLastOpc = SecondLastInst->getOpcode();
553 }
554 }
555
556 // If we're allowed to modify and the block ends in a unconditional branch
557 // which could simply fallthrough, remove the branch. (Note: This case only
558 // matters when we can't understand the whole sequence, otherwise it's also
559 // handled by BranchFolding.cpp.)
560 if (AllowModify && isUncondBranchOpcode(Opc: LastOpc) &&
561 MBB.isLayoutSuccessor(MBB: getBranchDestBlock(MI: *LastInst))) {
562 LastInst->eraseFromParent();
563 LastInst = SecondLastInst;
564 LastOpc = LastInst->getOpcode();
565 if (I == MBB.begin() || !isUnpredicatedTerminator(MI: *--I)) {
566 assert(!isUncondBranchOpcode(LastOpc) &&
567 "unreachable unconditional branches removed above");
568
569 if (isCondBranchOpcode(Opc: LastOpc)) {
570 // Block ends with fall-through condbranch.
571 parseCondBranch(LastInst, Target&: TBB, Cond);
572 return false;
573 }
574 return true; // Can't handle indirect branch.
575 }
576 SecondLastInst = &*I;
577 SecondLastOpc = SecondLastInst->getOpcode();
578 }
579
580 // If there are three terminators, we don't know what sort of block this is.
581 if (SecondLastInst && I != MBB.begin() && isUnpredicatedTerminator(MI: *--I))
582 return true;
583
584 // If the block ends with a B and a Bcc, handle it.
585 if (isCondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
586 parseCondBranch(LastInst: SecondLastInst, Target&: TBB, Cond);
587 FBB = LastInst->getOperand(i: 0).getMBB();
588 return false;
589 }
590
591 // If the block ends with two unconditional branches, handle it. The second
592 // one is not executed, so remove it.
593 if (isUncondBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
594 TBB = SecondLastInst->getOperand(i: 0).getMBB();
595 I = LastInst;
596 if (AllowModify)
597 I->eraseFromParent();
598 return false;
599 }
600
601 // ...likewise if it ends with an indirect branch followed by an unconditional
602 // branch.
603 if (isIndirectBranchOpcode(Opc: SecondLastOpc) && isUncondBranchOpcode(Opc: LastOpc)) {
604 I = LastInst;
605 if (AllowModify)
606 I->eraseFromParent();
607 return true;
608 }
609
610 // Otherwise, can't handle this.
611 return true;
612}
613
614bool AArch64InstrInfo::analyzeBranchPredicate(MachineBasicBlock &MBB,
615 MachineBranchPredicate &MBP,
616 bool AllowModify) const {
617 // Use analyzeBranch to validate the branch pattern.
618 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
619 SmallVector<MachineOperand, 4> Cond;
620 if (analyzeBranch(MBB, TBB, FBB, Cond, AllowModify))
621 return true;
622
623 // analyzeBranch returns success with empty Cond for unconditional branches.
624 if (Cond.empty())
625 return true;
626
627 MBP.TrueDest = TBB;
628 assert(MBP.TrueDest && "expected!");
629 MBP.FalseDest = FBB ? FBB : MBB.getNextNode();
630
631 MBP.ConditionDef = nullptr;
632 MBP.SingleUseCondition = false;
633
634 // Find the conditional branch. After analyzeBranch succeeds with non-empty
635 // Cond, there's exactly one conditional branch - either last (fallthrough)
636 // or second-to-last (followed by unconditional B).
637 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
638 if (I == MBB.end())
639 return true;
640
641 if (isUncondBranchOpcode(Opc: I->getOpcode())) {
642 if (I == MBB.begin())
643 return true;
644 --I;
645 }
646
647 MachineInstr *CondBranch = &*I;
648 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
649
650 switch (CondBranch->getOpcode()) {
651 default:
652 return true;
653
654 case AArch64::Bcc:
655 // Bcc takes the NZCV flag as the operand to branch on, walk up the
656 // instruction stream to find the last instruction to define NZCV.
657 for (MachineInstr &MI : llvm::drop_begin(RangeOrContainer: llvm::reverse(C&: MBB))) {
658 if (MI.modifiesRegister(Reg: AArch64::NZCV, /*TRI=*/nullptr)) {
659 MBP.ConditionDef = &MI;
660 break;
661 }
662 }
663 return false;
664
665 case AArch64::CBZW:
666 case AArch64::CBZX:
667 case AArch64::CBNZW:
668 case AArch64::CBNZX: {
669 MBP.LHS = CondBranch->getOperand(i: 0);
670 MBP.RHS = MachineOperand::CreateImm(Val: 0);
671 unsigned Opc = CondBranch->getOpcode();
672 MBP.Predicate = (Opc == AArch64::CBNZX || Opc == AArch64::CBNZW)
673 ? MachineBranchPredicate::PRED_NE
674 : MachineBranchPredicate::PRED_EQ;
675 Register CondReg = MBP.LHS.getReg();
676 if (CondReg.isVirtual())
677 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
678 return false;
679 }
680
681 case AArch64::TBZW:
682 case AArch64::TBZX:
683 case AArch64::TBNZW:
684 case AArch64::TBNZX: {
685 Register CondReg = CondBranch->getOperand(i: 0).getReg();
686 if (CondReg.isVirtual())
687 MBP.ConditionDef = MRI.getVRegDef(Reg: CondReg);
688 return false;
689 }
690 }
691}
692
693bool AArch64InstrInfo::reverseBranchCondition(
694 SmallVectorImpl<MachineOperand> &Cond) const {
695 if (Cond[0].getImm() != -1) {
696 // Regular Bcc
697 AArch64CC::CondCode CC = (AArch64CC::CondCode)(int)Cond[0].getImm();
698 Cond[0].setImm(AArch64CC::getInvertedCondCode(Code: CC));
699 } else {
700 // Folded compare-and-branch
701 switch (Cond[1].getImm()) {
702 default:
703 llvm_unreachable("Unknown conditional branch!");
704 case AArch64::CBZW:
705 Cond[1].setImm(AArch64::CBNZW);
706 break;
707 case AArch64::CBNZW:
708 Cond[1].setImm(AArch64::CBZW);
709 break;
710 case AArch64::CBZX:
711 Cond[1].setImm(AArch64::CBNZX);
712 break;
713 case AArch64::CBNZX:
714 Cond[1].setImm(AArch64::CBZX);
715 break;
716 case AArch64::TBZW:
717 Cond[1].setImm(AArch64::TBNZW);
718 break;
719 case AArch64::TBNZW:
720 Cond[1].setImm(AArch64::TBZW);
721 break;
722 case AArch64::TBZX:
723 Cond[1].setImm(AArch64::TBNZX);
724 break;
725 case AArch64::TBNZX:
726 Cond[1].setImm(AArch64::TBZX);
727 break;
728
729 // Cond is { -1, Opcode, CC, Op0, Op1, ... }
730 case AArch64::CBWPri:
731 case AArch64::CBXPri:
732 case AArch64::CBBAssertExt:
733 case AArch64::CBHAssertExt:
734 case AArch64::CBWPrr:
735 case AArch64::CBXPrr: {
736 // Pseudos using standard 4bit Arm condition codes
737 AArch64CC::CondCode CC =
738 static_cast<AArch64CC::CondCode>(Cond[2].getImm());
739 Cond[2].setImm(AArch64CC::getInvertedCondCode(Code: CC));
740 }
741 }
742 }
743
744 return false;
745}
746
747unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB,
748 int *BytesRemoved) const {
749 MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr();
750 if (I == MBB.end())
751 return 0;
752
753 if (!isUncondBranchOpcode(Opc: I->getOpcode()) &&
754 !isCondBranchOpcode(Opc: I->getOpcode()))
755 return 0;
756
757 // Remove the branch.
758 I->eraseFromParent();
759
760 I = MBB.end();
761
762 if (I == MBB.begin()) {
763 if (BytesRemoved)
764 *BytesRemoved = 4;
765 return 1;
766 }
767 --I;
768 if (!isCondBranchOpcode(Opc: I->getOpcode())) {
769 if (BytesRemoved)
770 *BytesRemoved = 4;
771 return 1;
772 }
773
774 // Remove the branch.
775 I->eraseFromParent();
776 if (BytesRemoved)
777 *BytesRemoved = 8;
778
779 return 2;
780}
781
782void AArch64InstrInfo::instantiateCondBranch(
783 MachineBasicBlock &MBB, const DebugLoc &DL, MachineBasicBlock *TBB,
784 ArrayRef<MachineOperand> Cond) const {
785 if (Cond[0].getImm() != -1) {
786 // Regular Bcc
787 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: Cond[0].getImm()).addMBB(MBB: TBB);
788 } else {
789 // Folded compare-and-branch
790 // Note that we use addOperand instead of addReg to keep the flags.
791
792 // cbz, cbnz
793 const MachineInstrBuilder MIB =
794 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: Cond[1].getImm())).add(MO: Cond[2]);
795
796 // tbz/tbnz
797 if (Cond.size() > 3)
798 MIB.add(MO: Cond[3]);
799
800 // cb
801 if (Cond.size() > 4)
802 MIB.add(MO: Cond[4]);
803
804 MIB.addMBB(MBB: TBB);
805
806 // cb[b,h]
807 if (Cond.size() > 5) {
808 MIB.addImm(Val: Cond[5].getImm());
809 MIB.addImm(Val: Cond[6].getImm());
810 }
811 }
812}
813
814unsigned AArch64InstrInfo::insertBranch(
815 MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB,
816 ArrayRef<MachineOperand> Cond, const DebugLoc &DL, int *BytesAdded) const {
817 // Shouldn't be a fall through.
818 assert(TBB && "insertBranch must not be told to insert a fallthrough");
819
820 if (!FBB) {
821 if (Cond.empty()) // Unconditional branch?
822 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: TBB);
823 else
824 instantiateCondBranch(MBB, DL, TBB, Cond);
825
826 if (BytesAdded)
827 *BytesAdded = 4;
828
829 return 1;
830 }
831
832 // Two-way conditional branch.
833 instantiateCondBranch(MBB, DL, TBB, Cond);
834 BuildMI(BB: &MBB, MIMD: DL, MCID: get(Opcode: AArch64::B)).addMBB(MBB: FBB);
835
836 if (BytesAdded)
837 *BytesAdded = 8;
838
839 return 2;
840}
841
842#ifndef NDEBUG
843static bool isValidCBExtend(int64_t Opc, AArch64_AM::ShiftExtendType Ext) {
844 switch (Ext) {
845 default:
846 return false;
847 case AArch64_AM::UXTB:
848 case AArch64_AM::SXTB:
849 return Opc == AArch64::CBBAssertExt;
850 case AArch64_AM::UXTH:
851 case AArch64_AM::SXTH:
852 return Opc == AArch64::CBHAssertExt;
853 }
854}
855#endif
856
857AArch64CC::CondCode AArch64InstrInfo::insertCmpForCondBr(
858 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, const DebugLoc &DL,
859 ArrayRef<MachineOperand> Cond) const {
860 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
861
862 // Parse the condition code, see parseCondBranch() above.
863 AArch64CC::CondCode CC;
864 switch (Cond.size()) {
865 default:
866 llvm_unreachable("Unknown condition opcode in Cond");
867 case 1: // b.cc
868 CC = AArch64CC::CondCode(Cond[0].getImm());
869 break;
870 case 3: { // cbz/cbnz
871 // We must insert a compare against 0.
872 bool Is64Bit;
873 switch (Cond[1].getImm()) {
874 default:
875 llvm_unreachable("Unknown branch opcode in Cond");
876 case AArch64::CBZW:
877 Is64Bit = false;
878 CC = AArch64CC::EQ;
879 break;
880 case AArch64::CBZX:
881 Is64Bit = true;
882 CC = AArch64CC::EQ;
883 break;
884 case AArch64::CBNZW:
885 Is64Bit = false;
886 CC = AArch64CC::NE;
887 break;
888 case AArch64::CBNZX:
889 Is64Bit = true;
890 CC = AArch64CC::NE;
891 break;
892 }
893 Register SrcReg = Cond[2].getReg();
894 if (Is64Bit) {
895 // cmp reg, #0 is actually subs xzr, reg, #0.
896 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64spRegClass);
897 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSXri), DestReg: AArch64::XZR)
898 .addReg(RegNo: SrcReg)
899 .addImm(Val: 0)
900 .addImm(Val: 0);
901 } else {
902 MRI.constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32spRegClass);
903 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWri), DestReg: AArch64::WZR)
904 .addReg(RegNo: SrcReg)
905 .addImm(Val: 0)
906 .addImm(Val: 0);
907 }
908 } break;
909 case 4: { // tbz/tbnz
910 // We must insert a tst instruction.
911 switch (Cond[1].getImm()) {
912 default:
913 llvm_unreachable("Unknown branch opcode in Cond");
914 case AArch64::TBZW:
915 case AArch64::TBZX:
916 CC = AArch64CC::EQ;
917 break;
918 case AArch64::TBNZW:
919 case AArch64::TBNZX:
920 CC = AArch64CC::NE;
921 break;
922 }
923 // cmp reg, #foo is actually ands xzr, reg, #1<<foo.
924 if (Cond[1].getImm() == AArch64::TBZW || Cond[1].getImm() == AArch64::TBNZW)
925 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::ANDSWri), DestReg: AArch64::WZR)
926 .addReg(RegNo: Cond[2].getReg())
927 .addImm(
928 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 32));
929 else
930 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::ANDSXri), DestReg: AArch64::XZR)
931 .addReg(RegNo: Cond[2].getReg())
932 .addImm(
933 Val: AArch64_AM::encodeLogicalImmediate(imm: 1ull << Cond[3].getImm(), regSize: 64));
934 } break;
935 case 5: { // cb
936 // We must insert a cmp, that is a subs
937 // 0 1 2 3 4
938 // Cond is { -1, Opcode, CC, Op0, Op1 }
939 unsigned SubsOpc, SubsDestReg;
940 bool IsImm = false;
941 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
942 switch (Cond[1].getImm()) {
943 default:
944 llvm_unreachable("Unknown branch opcode in Cond");
945 case AArch64::CBWPri:
946 SubsOpc = AArch64::SUBSWri;
947 SubsDestReg = AArch64::WZR;
948 IsImm = true;
949 break;
950 case AArch64::CBXPri:
951 SubsOpc = AArch64::SUBSXri;
952 SubsDestReg = AArch64::XZR;
953 IsImm = true;
954 break;
955 case AArch64::CBWPrr:
956 SubsOpc = AArch64::SUBSWrr;
957 SubsDestReg = AArch64::WZR;
958 IsImm = false;
959 break;
960 case AArch64::CBXPrr:
961 SubsOpc = AArch64::SUBSXrr;
962 SubsDestReg = AArch64::XZR;
963 IsImm = false;
964 break;
965 }
966
967 if (IsImm) {
968 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 1));
969 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
970 .addReg(RegNo: Cond[3].getReg())
971 .addImm(Val: Cond[4].getImm())
972 .addImm(Val: 0);
973 } else {
974 MRI.constrainRegClass(Reg: Cond[3].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 1));
975 MRI.constrainRegClass(Reg: Cond[4].getReg(), RC: getRegClass(MCID: get(Opcode: SubsOpc), OpNum: 2));
976 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: SubsOpc), DestReg: SubsDestReg)
977 .addReg(RegNo: Cond[3].getReg())
978 .addReg(RegNo: Cond[4].getReg());
979 }
980 } break;
981 case 7: { // cb[b,h]
982 // We must insert a cmp, that is a subs, but also zero- or sign-extensions
983 // that have been folded. For the first operand we codegen an explicit
984 // extension, for the second operand we fold the extension into cmp.
985 // 0 1 2 3 4 5 6
986 // Cond is { -1, Opcode, CC, Op0, Op1, Ext0, Ext1 }
987
988 // We need a new register for the now explicitly extended register
989 Register Reg = Cond[3].getReg();
990 if (Cond[5].getImm() != AArch64_AM::InvalidShiftExtend) {
991 unsigned ExtOpc;
992 unsigned ExtBits;
993 AArch64_AM::ShiftExtendType ExtendType =
994 AArch64_AM::getExtendType(Imm: Cond[5].getImm());
995 assert(isValidCBExtend(Cond[1].getImm(), ExtendType) &&
996 "Unexpected compare-and-branch instruction for extend type");
997 switch (ExtendType) {
998 default:
999 llvm_unreachable("Unknown shift-extend for CB instruction");
1000 case AArch64_AM::SXTB:
1001 ExtOpc = AArch64::SBFMWri;
1002 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1003 break;
1004 case AArch64_AM::SXTH:
1005 ExtOpc = AArch64::SBFMWri;
1006 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1007 break;
1008 case AArch64_AM::UXTB:
1009 ExtOpc = AArch64::ANDWri;
1010 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xff, regSize: 32);
1011 break;
1012 case AArch64_AM::UXTH:
1013 ExtOpc = AArch64::ANDWri;
1014 ExtBits = AArch64_AM::encodeLogicalImmediate(imm: 0xffff, regSize: 32);
1015 break;
1016 }
1017
1018 // Build the explicit extension of the first operand
1019 Reg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32commonRegClass);
1020 MachineInstrBuilder MBBI =
1021 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: ExtOpc), DestReg: Reg).addReg(RegNo: Cond[3].getReg());
1022 if (ExtOpc != AArch64::ANDWri)
1023 MBBI.addImm(Val: 0);
1024 MBBI.addImm(Val: ExtBits);
1025 }
1026
1027 // Now, subs with an extended second operand
1028 if (Cond[6].getImm() != AArch64_AM::InvalidShiftExtend) {
1029 MRI.constrainRegClass(Reg, RC: &AArch64::GPR32commonRegClass);
1030 AArch64_AM::ShiftExtendType ExtendType =
1031 AArch64_AM::getExtendType(Imm: Cond[6].getImm());
1032 assert(isValidCBExtend(Cond[1].getImm(), ExtendType) &&
1033 "Unexpected compare-and-branch instruction for extend type");
1034 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrx), DestReg: AArch64::WZR)
1035 .addReg(RegNo: Reg)
1036 .addReg(RegNo: Cond[4].getReg())
1037 .addImm(Val: AArch64_AM::getArithExtendImm(ET: ExtendType, Imm: 0));
1038 } // If no extension is needed, just a regular subs
1039 else {
1040 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBSWrr), DestReg: AArch64::WZR)
1041 .addReg(RegNo: Reg)
1042 .addReg(RegNo: Cond[4].getReg());
1043 }
1044
1045 CC = static_cast<AArch64CC::CondCode>(Cond[2].getImm());
1046 } break;
1047 }
1048 return CC;
1049}
1050
1051bool llvm::optimizeTerminators(MachineBasicBlock *MBB,
1052 const TargetInstrInfo &TII) {
1053 for (MachineInstr &MI : MBB->terminators()) {
1054 unsigned Opc = MI.getOpcode();
1055 switch (Opc) {
1056 case AArch64::CBZW:
1057 case AArch64::CBZX:
1058 case AArch64::TBZW:
1059 case AArch64::TBZX:
1060 // CBZ/TBZ with WZR/XZR -> unconditional B
1061 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
1062 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
1063 DEBUG_WITH_TYPE("optimizeTerminators",
1064 dbgs() << "Removing always taken branch: " << MI);
1065 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
1066 SmallVector<MachineBasicBlock *> Succs(MBB->successors());
1067 for (auto *S : Succs)
1068 if (S != Target)
1069 MBB->removeSuccessor(Succ: S);
1070 DebugLoc DL = MI.getDebugLoc();
1071 while (MBB->rbegin() != &MI)
1072 MBB->rbegin()->eraseFromParent();
1073 MI.eraseFromParent();
1074 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: AArch64::B)).addMBB(MBB: Target);
1075 return true;
1076 }
1077 break;
1078 case AArch64::CBNZW:
1079 case AArch64::CBNZX:
1080 case AArch64::TBNZW:
1081 case AArch64::TBNZX:
1082 // CBNZ/TBNZ with WZR/XZR -> never taken, remove branch and successor
1083 if (MI.getOperand(i: 0).getReg() == AArch64::WZR ||
1084 MI.getOperand(i: 0).getReg() == AArch64::XZR) {
1085 DEBUG_WITH_TYPE("optimizeTerminators",
1086 dbgs() << "Removing never taken branch: " << MI);
1087 MachineBasicBlock *Target = TII.getBranchDestBlock(MI);
1088 MI.getParent()->removeSuccessor(Succ: Target);
1089 MI.eraseFromParent();
1090 return true;
1091 }
1092 break;
1093 }
1094 }
1095 return false;
1096}
1097
1098// Find the original register that VReg is copied from.
1099static unsigned removeCopies(const MachineRegisterInfo &MRI, unsigned VReg) {
1100 while (Register::isVirtualRegister(Reg: VReg)) {
1101 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1102 if (!DefMI || !DefMI->isFullCopy())
1103 return VReg;
1104 VReg = DefMI->getOperand(i: 1).getReg();
1105 }
1106 return VReg;
1107}
1108
1109// Determine if VReg is defined by an instruction that can be folded into a
1110// csel instruction. If so, return the folded opcode, and the replacement
1111// register.
1112static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg,
1113 unsigned *NewReg = nullptr) {
1114 VReg = removeCopies(MRI, VReg);
1115 if (!Register::isVirtualRegister(Reg: VReg))
1116 return 0;
1117
1118 bool Is64Bit = AArch64::GPR64allRegClass.hasSubClassEq(RC: MRI.getRegClass(Reg: VReg));
1119 const MachineInstr *DefMI = MRI.getVRegDef(Reg: VReg);
1120 if (!DefMI)
1121 return 0;
1122 unsigned Opc = 0;
1123 unsigned SrcReg = 0;
1124 switch (DefMI->getOpcode()) {
1125 case AArch64::SUBREG_TO_REG:
1126 // Check for the following way to define an 64-bit immediate:
1127 // %0:gpr32 = MOVi32imm 1
1128 // %1:gpr64 = SUBREG_TO_REG %0:gpr32, %subreg.sub_32
1129 if (!DefMI->getOperand(i: 1).isReg())
1130 return 0;
1131 if (!DefMI->getOperand(i: 2).isImm() ||
1132 DefMI->getOperand(i: 2).getImm() != AArch64::sub_32)
1133 return 0;
1134 DefMI = MRI.getVRegDef(Reg: DefMI->getOperand(i: 1).getReg());
1135 if (DefMI->getOpcode() != AArch64::MOVi32imm)
1136 return 0;
1137 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
1138 return 0;
1139 assert(Is64Bit);
1140 SrcReg = AArch64::XZR;
1141 Opc = AArch64::CSINCXr;
1142 break;
1143
1144 case AArch64::MOVi32imm:
1145 case AArch64::MOVi64imm:
1146 if (!DefMI->getOperand(i: 1).isImm() || DefMI->getOperand(i: 1).getImm() != 1)
1147 return 0;
1148 SrcReg = Is64Bit ? AArch64::XZR : AArch64::WZR;
1149 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
1150 break;
1151
1152 case AArch64::ADDSXri:
1153 case AArch64::ADDSWri:
1154 // if NZCV is used, do not fold.
1155 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1156 isDead: true) == -1)
1157 return 0;
1158 // fall-through to ADDXri and ADDWri.
1159 [[fallthrough]];
1160 case AArch64::ADDXri:
1161 case AArch64::ADDWri:
1162 // add x, 1 -> csinc.
1163 if (!DefMI->getOperand(i: 2).isImm() || DefMI->getOperand(i: 2).getImm() != 1 ||
1164 DefMI->getOperand(i: 3).getImm() != 0)
1165 return 0;
1166 SrcReg = DefMI->getOperand(i: 1).getReg();
1167 Opc = Is64Bit ? AArch64::CSINCXr : AArch64::CSINCWr;
1168 break;
1169
1170 case AArch64::ORNXrr:
1171 case AArch64::ORNWrr: {
1172 // not x -> csinv, represented as orn dst, xzr, src.
1173 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1174 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1175 return 0;
1176 SrcReg = DefMI->getOperand(i: 2).getReg();
1177 Opc = Is64Bit ? AArch64::CSINVXr : AArch64::CSINVWr;
1178 break;
1179 }
1180
1181 case AArch64::SUBSXrr:
1182 case AArch64::SUBSWrr:
1183 // if NZCV is used, do not fold.
1184 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
1185 isDead: true) == -1)
1186 return 0;
1187 // fall-through to SUBXrr and SUBWrr.
1188 [[fallthrough]];
1189 case AArch64::SUBXrr:
1190 case AArch64::SUBWrr: {
1191 // neg x -> csneg, represented as sub dst, xzr, src.
1192 unsigned ZReg = removeCopies(MRI, VReg: DefMI->getOperand(i: 1).getReg());
1193 if (ZReg != AArch64::XZR && ZReg != AArch64::WZR)
1194 return 0;
1195 SrcReg = DefMI->getOperand(i: 2).getReg();
1196 Opc = Is64Bit ? AArch64::CSNEGXr : AArch64::CSNEGWr;
1197 break;
1198 }
1199 default:
1200 return 0;
1201 }
1202 assert(Opc && SrcReg && "Missing parameters");
1203
1204 if (NewReg)
1205 *NewReg = SrcReg;
1206 return Opc;
1207}
1208
1209bool AArch64InstrInfo::canInsertSelect(const MachineBasicBlock &MBB,
1210 ArrayRef<MachineOperand> Cond,
1211 Register DstReg, Register TrueReg,
1212 Register FalseReg, int &CondCycles,
1213 int &TrueCycles,
1214 int &FalseCycles) const {
1215 // Check register classes.
1216 const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1217 const TargetRegisterClass *RC =
1218 RI.getCommonSubClass(A: MRI.getRegClass(Reg: TrueReg), B: MRI.getRegClass(Reg: FalseReg));
1219 if (!RC)
1220 return false;
1221
1222 // Also need to check the dest regclass, in case we're trying to optimize
1223 // something like:
1224 // %1(gpr) = PHI %2(fpr), bb1, %(fpr), bb2
1225 if (!RI.getCommonSubClass(A: RC, B: MRI.getRegClass(Reg: DstReg)))
1226 return false;
1227
1228 // Expanding cbz/tbz requires an extra cycle of latency on the condition.
1229 unsigned ExtraCondLat = Cond.size() != 1;
1230
1231 // GPRs are handled by csel.
1232 // FIXME: Fold in x+1, -x, and ~x when applicable.
1233 if (AArch64::GPR64allRegClass.hasSubClassEq(RC) ||
1234 AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
1235 // Single-cycle csel, csinc, csinv, and csneg.
1236 CondCycles = 1 + ExtraCondLat;
1237 TrueCycles = FalseCycles = 1;
1238 if (canFoldIntoCSel(MRI, VReg: TrueReg))
1239 TrueCycles = 0;
1240 else if (canFoldIntoCSel(MRI, VReg: FalseReg))
1241 FalseCycles = 0;
1242 return true;
1243 }
1244
1245 // Scalar floating point is handled by fcsel.
1246 // FIXME: Form fabs, fmin, and fmax when applicable.
1247 if (AArch64::FPR64RegClass.hasSubClassEq(RC) ||
1248 AArch64::FPR32RegClass.hasSubClassEq(RC)) {
1249 CondCycles = 5 + ExtraCondLat;
1250 TrueCycles = FalseCycles = 2;
1251 return true;
1252 }
1253
1254 // No single conditional move for a 128-bit vector, but we can emit a sequence
1255 // of csetm (~1), dup (~5, cross domain), bsl (~2).
1256 if (AArch64::FPR128RegClass.hasSubClassEq(RC) &&
1257 Subtarget.isNeonAvailable() &&
1258 !MBB.getParent()->getFunction().hasMinSize()) {
1259 CondCycles = 8 + ExtraCondLat;
1260 TrueCycles = FalseCycles = 2;
1261 return true;
1262 }
1263
1264 return false;
1265}
1266
1267void AArch64InstrInfo::insertSelect(MachineBasicBlock &MBB,
1268 MachineBasicBlock::iterator I,
1269 const DebugLoc &DL, Register DstReg,
1270 ArrayRef<MachineOperand> Cond,
1271 Register TrueReg, Register FalseReg) const {
1272
1273 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
1274 AArch64CC::CondCode CC = insertCmpForCondBr(MBB, MI: I, DL, Cond);
1275
1276 // A 128-bit vector has no conditional move so blend the operands with a mask
1277 // built from the flags.
1278 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR128RegClass)) {
1279 assert(Subtarget.isNeonAvailable() && "Expected NEON for a vector select");
1280 MRI.constrainRegClass(Reg: TrueReg, RC: &AArch64::FPR128RegClass);
1281 MRI.constrainRegClass(Reg: FalseReg, RC: &AArch64::FPR128RegClass);
1282 Register CondSet = MRI.createVirtualRegister(RegClass: &AArch64::GPR64RegClass);
1283 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::CSINVXr), DestReg: CondSet)
1284 .addReg(RegNo: AArch64::XZR)
1285 .addReg(RegNo: AArch64::XZR)
1286 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
1287 Register Mask = MRI.createVirtualRegister(RegClass: &AArch64::FPR128RegClass);
1288 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::DUPv2i64gpr), DestReg: Mask).addReg(RegNo: CondSet);
1289 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::BSPv16i8), DestReg: DstReg)
1290 .addReg(RegNo: Mask)
1291 .addReg(RegNo: TrueReg)
1292 .addReg(RegNo: FalseReg);
1293 return;
1294 }
1295
1296 unsigned Opc = 0;
1297 const TargetRegisterClass *RC = nullptr;
1298 bool TryFold = false;
1299 if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass)) {
1300 RC = &AArch64::GPR64RegClass;
1301 Opc = AArch64::CSELXr;
1302 TryFold = true;
1303 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::GPR32RegClass)) {
1304 RC = &AArch64::GPR32RegClass;
1305 Opc = AArch64::CSELWr;
1306 TryFold = true;
1307 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR64RegClass)) {
1308 RC = &AArch64::FPR64RegClass;
1309 Opc = AArch64::FCSELDrrr;
1310 } else if (MRI.constrainRegClass(Reg: DstReg, RC: &AArch64::FPR32RegClass)) {
1311 RC = &AArch64::FPR32RegClass;
1312 Opc = AArch64::FCSELSrrr;
1313 }
1314 assert(RC && "Unsupported regclass");
1315
1316 // Try folding simple instructions into the csel.
1317 if (TryFold) {
1318 unsigned NewReg = 0;
1319 unsigned FoldedOpc = canFoldIntoCSel(MRI, VReg: TrueReg, NewReg: &NewReg);
1320 if (FoldedOpc) {
1321 // The folded opcodes csinc, csinc and csneg apply the operation to
1322 // FalseReg, so we need to invert the condition.
1323 CC = AArch64CC::getInvertedCondCode(Code: CC);
1324 TrueReg = FalseReg;
1325 } else
1326 FoldedOpc = canFoldIntoCSel(MRI, VReg: FalseReg, NewReg: &NewReg);
1327
1328 // Fold the operation. Leave any dead instructions for DCE to clean up.
1329 if (FoldedOpc) {
1330 FalseReg = NewReg;
1331 Opc = FoldedOpc;
1332 // Extend the live range of NewReg.
1333 MRI.clearKillFlags(Reg: NewReg);
1334 }
1335 }
1336
1337 // Pull all virtual register into the appropriate class.
1338 MRI.constrainRegClass(Reg: TrueReg, RC);
1339 // FalseReg might be WZR or XZR if the folded operand is a literal 1.
1340 assert(
1341 (FalseReg.isVirtual() || FalseReg == AArch64::WZR ||
1342 FalseReg == AArch64::XZR) &&
1343 "FalseReg was folded into a non-virtual register other than WZR or XZR");
1344 if (FalseReg.isVirtual())
1345 MRI.constrainRegClass(Reg: FalseReg, RC);
1346
1347 // Insert the csel.
1348 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: Opc), DestReg: DstReg)
1349 .addReg(RegNo: TrueReg)
1350 .addReg(RegNo: FalseReg)
1351 .addImm(Val: CC);
1352}
1353
1354// Return true if Imm can be loaded into a register by a "cheap" sequence of
1355// instructions. For now, "cheap" means at most two instructions.
1356static bool isCheapImmediate(const MachineInstr &MI, unsigned BitSize) {
1357 if (BitSize == 32)
1358 return true;
1359
1360 assert(BitSize == 64 && "Only bit sizes of 32 or 64 allowed");
1361 uint64_t Imm = static_cast<uint64_t>(MI.getOperand(i: 1).getImm());
1362 SmallVector<AArch64_IMM::ImmInsnModel, 4> Is;
1363 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn&: Is);
1364
1365 return Is.size() <= 2;
1366}
1367
1368// Check if a COPY instruction is cheap.
1369static bool isCheapCopy(const MachineInstr &MI, const AArch64RegisterInfo &RI) {
1370 assert(MI.isCopy() && "Expected COPY instruction");
1371 const MachineRegisterInfo &MRI = MI.getMF()->getRegInfo();
1372
1373 // Cross-bank copies (e.g., between GPR and FPR) are expensive on AArch64,
1374 // typically requiring an FMOV instruction with a 2-6 cycle latency.
1375 auto GetRegClass = [&](Register Reg) -> const TargetRegisterClass * {
1376 if (Reg.isVirtual())
1377 return MRI.getRegClass(Reg);
1378 if (Reg.isPhysical())
1379 return RI.getMinimalPhysRegClass(Reg);
1380 return nullptr;
1381 };
1382 const TargetRegisterClass *DstRC = GetRegClass(MI.getOperand(i: 0).getReg());
1383 const TargetRegisterClass *SrcRC = GetRegClass(MI.getOperand(i: 1).getReg());
1384 if (DstRC && SrcRC && !RI.getCommonSubClass(A: DstRC, B: SrcRC))
1385 return false;
1386
1387 return MI.isAsCheapAsAMove();
1388}
1389
1390// FIXME: this implementation should be micro-architecture dependent, so a
1391// micro-architecture target hook should be introduced here in future.
1392bool AArch64InstrInfo::isAsCheapAsAMove(const MachineInstr &MI) const {
1393 if (Subtarget.hasExynosCheapAsMoveHandling()) {
1394 if (isExynosCheapAsMove(MI))
1395 return true;
1396 return MI.isAsCheapAsAMove();
1397 }
1398
1399 switch (MI.getOpcode()) {
1400 default:
1401 return MI.isAsCheapAsAMove();
1402
1403 case TargetOpcode::COPY:
1404 return isCheapCopy(MI, RI);
1405
1406 case AArch64::ADDWrs:
1407 case AArch64::ADDXrs:
1408 case AArch64::SUBWrs:
1409 case AArch64::SUBXrs:
1410 return Subtarget.hasALULSLFast() && MI.getOperand(i: 3).getImm() <= 4;
1411
1412 // If MOVi32imm or MOVi64imm can be expanded into ORRWri or
1413 // ORRXri, it is as cheap as MOV.
1414 // Likewise if it can be expanded to MOVZ/MOVN/MOVK.
1415 case AArch64::MOVi32imm:
1416 return isCheapImmediate(MI, BitSize: 32);
1417 case AArch64::MOVi64imm:
1418 return isCheapImmediate(MI, BitSize: 64);
1419 }
1420}
1421
1422bool AArch64InstrInfo::isFalkorShiftExtFast(const MachineInstr &MI) {
1423 switch (MI.getOpcode()) {
1424 default:
1425 return false;
1426
1427 case AArch64::ADDWrs:
1428 case AArch64::ADDXrs:
1429 case AArch64::ADDSWrs:
1430 case AArch64::ADDSXrs: {
1431 unsigned Imm = MI.getOperand(i: 3).getImm();
1432 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1433 if (ShiftVal == 0)
1434 return true;
1435 return AArch64_AM::getShiftType(Imm) == AArch64_AM::LSL && ShiftVal <= 5;
1436 }
1437
1438 case AArch64::ADDWrx:
1439 case AArch64::ADDXrx:
1440 case AArch64::ADDXrx64:
1441 case AArch64::ADDSWrx:
1442 case AArch64::ADDSXrx:
1443 case AArch64::ADDSXrx64: {
1444 unsigned Imm = MI.getOperand(i: 3).getImm();
1445 switch (AArch64_AM::getArithExtendType(Imm)) {
1446 default:
1447 return false;
1448 case AArch64_AM::UXTB:
1449 case AArch64_AM::UXTH:
1450 case AArch64_AM::UXTW:
1451 case AArch64_AM::UXTX:
1452 return AArch64_AM::getArithShiftValue(Imm) <= 4;
1453 }
1454 }
1455
1456 case AArch64::SUBWrs:
1457 case AArch64::SUBSWrs: {
1458 unsigned Imm = MI.getOperand(i: 3).getImm();
1459 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1460 return ShiftVal == 0 ||
1461 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 31);
1462 }
1463
1464 case AArch64::SUBXrs:
1465 case AArch64::SUBSXrs: {
1466 unsigned Imm = MI.getOperand(i: 3).getImm();
1467 unsigned ShiftVal = AArch64_AM::getShiftValue(Imm);
1468 return ShiftVal == 0 ||
1469 (AArch64_AM::getShiftType(Imm) == AArch64_AM::ASR && ShiftVal == 63);
1470 }
1471
1472 case AArch64::SUBWrx:
1473 case AArch64::SUBXrx:
1474 case AArch64::SUBXrx64:
1475 case AArch64::SUBSWrx:
1476 case AArch64::SUBSXrx:
1477 case AArch64::SUBSXrx64: {
1478 unsigned Imm = MI.getOperand(i: 3).getImm();
1479 switch (AArch64_AM::getArithExtendType(Imm)) {
1480 default:
1481 return false;
1482 case AArch64_AM::UXTB:
1483 case AArch64_AM::UXTH:
1484 case AArch64_AM::UXTW:
1485 case AArch64_AM::UXTX:
1486 return AArch64_AM::getArithShiftValue(Imm) == 0;
1487 }
1488 }
1489
1490 case AArch64::LDRBBroW:
1491 case AArch64::LDRBBroX:
1492 case AArch64::LDRBroW:
1493 case AArch64::LDRBroX:
1494 case AArch64::LDRDroW:
1495 case AArch64::LDRDroX:
1496 case AArch64::LDRHHroW:
1497 case AArch64::LDRHHroX:
1498 case AArch64::LDRHroW:
1499 case AArch64::LDRHroX:
1500 case AArch64::LDRQroW:
1501 case AArch64::LDRQroX:
1502 case AArch64::LDRSBWroW:
1503 case AArch64::LDRSBWroX:
1504 case AArch64::LDRSBXroW:
1505 case AArch64::LDRSBXroX:
1506 case AArch64::LDRSHWroW:
1507 case AArch64::LDRSHWroX:
1508 case AArch64::LDRSHXroW:
1509 case AArch64::LDRSHXroX:
1510 case AArch64::LDRSWroW:
1511 case AArch64::LDRSWroX:
1512 case AArch64::LDRSroW:
1513 case AArch64::LDRSroX:
1514 case AArch64::LDRWroW:
1515 case AArch64::LDRWroX:
1516 case AArch64::LDRXroW:
1517 case AArch64::LDRXroX:
1518 case AArch64::PRFMroW:
1519 case AArch64::PRFMroX:
1520 case AArch64::STRBBroW:
1521 case AArch64::STRBBroX:
1522 case AArch64::STRBroW:
1523 case AArch64::STRBroX:
1524 case AArch64::STRDroW:
1525 case AArch64::STRDroX:
1526 case AArch64::STRHHroW:
1527 case AArch64::STRHHroX:
1528 case AArch64::STRHroW:
1529 case AArch64::STRHroX:
1530 case AArch64::STRQroW:
1531 case AArch64::STRQroX:
1532 case AArch64::STRSroW:
1533 case AArch64::STRSroX:
1534 case AArch64::STRWroW:
1535 case AArch64::STRWroX:
1536 case AArch64::STRXroW:
1537 case AArch64::STRXroX: {
1538 unsigned IsSigned = MI.getOperand(i: 3).getImm();
1539 return !IsSigned;
1540 }
1541 }
1542}
1543
1544bool AArch64InstrInfo::isSEHInstruction(const MachineInstr &MI) {
1545 unsigned Opc = MI.getOpcode();
1546 switch (Opc) {
1547 default:
1548 return false;
1549 case AArch64::SEH_StackAlloc:
1550 case AArch64::SEH_SaveFPLR:
1551 case AArch64::SEH_SaveFPLR_X:
1552 case AArch64::SEH_SaveReg:
1553 case AArch64::SEH_SaveReg_X:
1554 case AArch64::SEH_SaveRegP:
1555 case AArch64::SEH_SaveRegP_X:
1556 case AArch64::SEH_SaveFReg:
1557 case AArch64::SEH_SaveFReg_X:
1558 case AArch64::SEH_SaveFRegP:
1559 case AArch64::SEH_SaveFRegP_X:
1560 case AArch64::SEH_SetFP:
1561 case AArch64::SEH_AddFP:
1562 case AArch64::SEH_Nop:
1563 case AArch64::SEH_PrologEnd:
1564 case AArch64::SEH_EpilogStart:
1565 case AArch64::SEH_EpilogEnd:
1566 case AArch64::SEH_PACSignLR:
1567 case AArch64::SEH_SaveAnyRegI:
1568 case AArch64::SEH_SaveAnyRegIP:
1569 case AArch64::SEH_SaveAnyRegQP:
1570 case AArch64::SEH_SaveAnyRegQPX:
1571 case AArch64::SEH_AllocZ:
1572 case AArch64::SEH_SaveZReg:
1573 case AArch64::SEH_SavePReg:
1574 return true;
1575 }
1576}
1577
1578bool AArch64InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
1579 Register &SrcReg, Register &DstReg,
1580 unsigned &SubIdx) const {
1581 switch (MI.getOpcode()) {
1582 default:
1583 return false;
1584 case AArch64::SBFMXri: // aka sxtw
1585 case AArch64::UBFMXri: // aka uxtw
1586 // Check for the 32 -> 64 bit extension case, these instructions can do
1587 // much more.
1588 if (MI.getOperand(i: 2).getImm() != 0 || MI.getOperand(i: 3).getImm() != 31)
1589 return false;
1590 // This is a signed or unsigned 32 -> 64 bit extension.
1591 SrcReg = MI.getOperand(i: 1).getReg();
1592 DstReg = MI.getOperand(i: 0).getReg();
1593 SubIdx = AArch64::sub_32;
1594 return true;
1595 }
1596}
1597
1598bool AArch64InstrInfo::areMemAccessesTriviallyDisjoint(
1599 const MachineInstr &MIa, const MachineInstr &MIb) const {
1600 const MachineOperand *BaseOpA = nullptr, *BaseOpB = nullptr;
1601 int64_t OffsetA = 0, OffsetB = 0;
1602 TypeSize WidthA(0, false), WidthB(0, false);
1603 bool OffsetAIsScalable = false, OffsetBIsScalable = false;
1604
1605 assert(MIa.mayLoadOrStore() && "MIa must be a load or store.");
1606 assert(MIb.mayLoadOrStore() && "MIb must be a load or store.");
1607
1608 if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects() ||
1609 MIa.hasOrderedMemoryRef() || MIb.hasOrderedMemoryRef())
1610 return false;
1611
1612 // Retrieve the base, offset from the base and width. Width
1613 // is the size of memory that is being loaded/stored (e.g. 1, 2, 4, 8). If
1614 // base are identical, and the offset of a lower memory access +
1615 // the width doesn't overlap the offset of a higher memory access,
1616 // then the memory accesses are different.
1617 // If OffsetAIsScalable and OffsetBIsScalable are both true, they
1618 // are assumed to have the same scale (vscale).
1619 if (getMemOperandWithOffsetWidth(MI: MIa, BaseOp&: BaseOpA, Offset&: OffsetA, OffsetIsScalable&: OffsetAIsScalable,
1620 Width&: WidthA) &&
1621 getMemOperandWithOffsetWidth(MI: MIb, BaseOp&: BaseOpB, Offset&: OffsetB, OffsetIsScalable&: OffsetBIsScalable,
1622 Width&: WidthB)) {
1623 if (BaseOpA->isIdenticalTo(Other: *BaseOpB) &&
1624 OffsetAIsScalable == OffsetBIsScalable) {
1625 int LowOffset = OffsetA < OffsetB ? OffsetA : OffsetB;
1626 int HighOffset = OffsetA < OffsetB ? OffsetB : OffsetA;
1627 TypeSize LowWidth = (LowOffset == OffsetA) ? WidthA : WidthB;
1628 if (LowWidth.isScalable() == OffsetAIsScalable &&
1629 LowOffset + (int)LowWidth.getKnownMinValue() <= HighOffset)
1630 return true;
1631 }
1632 }
1633 return false;
1634}
1635
1636bool AArch64InstrInfo::isSchedulingBoundary(const MachineInstr &MI,
1637 const MachineBasicBlock *MBB,
1638 const MachineFunction &MF) const {
1639 if (TargetInstrInfo::isSchedulingBoundary(MI, MBB, MF))
1640 return true;
1641
1642 // Do not move an instruction that can be recognized as a branch target.
1643 if (hasBTISemantics(MI))
1644 return true;
1645
1646 switch (MI.getOpcode()) {
1647 case AArch64::HINT:
1648 // CSDB hints are scheduling barriers.
1649 if (MI.getOperand(i: 0).getImm() == 0x14)
1650 return true;
1651 break;
1652 case AArch64::DSB:
1653 case AArch64::ISB:
1654 // DSB and ISB also are scheduling barriers.
1655 return true;
1656 case AArch64::MSRpstatesvcrImm1:
1657 // SMSTART and SMSTOP are also scheduling barriers.
1658 return true;
1659 default:;
1660 }
1661 if (isSEHInstruction(MI))
1662 return true;
1663 auto Next = std::next(x: MI.getIterator());
1664 return Next != MBB->end() && Next->isCFIInstruction();
1665}
1666
1667/// analyzeCompare - For a comparison instruction, return the source registers
1668/// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
1669/// Return true if the comparison instruction can be analyzed.
1670bool AArch64InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
1671 Register &SrcReg2, int64_t &CmpMask,
1672 int64_t &CmpValue) const {
1673 // The first operand can be a frame index where we'd normally expect a
1674 // register.
1675 // FIXME: Pass subregisters out of analyzeCompare
1676 assert(MI.getNumOperands() >= 2 && "All AArch64 cmps should have 2 operands");
1677 if (!MI.getOperand(i: 1).isReg() || MI.getOperand(i: 1).getSubReg())
1678 return false;
1679
1680 switch (MI.getOpcode()) {
1681 default:
1682 break;
1683 case AArch64::PTEST_PP:
1684 case AArch64::PTEST_PP_ANY:
1685 case AArch64::PTEST_PP_FIRST:
1686 SrcReg = MI.getOperand(i: 0).getReg();
1687 SrcReg2 = MI.getOperand(i: 1).getReg();
1688 if (MI.getOperand(i: 2).getSubReg())
1689 return false;
1690
1691 // Not sure about the mask and value for now...
1692 CmpMask = ~0;
1693 CmpValue = 0;
1694 return true;
1695 case AArch64::SUBSWrr:
1696 case AArch64::SUBSWrs:
1697 case AArch64::SUBSWrx:
1698 case AArch64::SUBSXrr:
1699 case AArch64::SUBSXrs:
1700 case AArch64::SUBSXrx:
1701 case AArch64::ADDSWrr:
1702 case AArch64::ADDSWrs:
1703 case AArch64::ADDSWrx:
1704 case AArch64::ADDSXrr:
1705 case AArch64::ADDSXrs:
1706 case AArch64::ADDSXrx:
1707 // Replace SUBSWrr with SUBWrr if NZCV is not used.
1708 SrcReg = MI.getOperand(i: 1).getReg();
1709 SrcReg2 = MI.getOperand(i: 2).getReg();
1710
1711 // FIXME: Pass subregisters out of analyzeCompare
1712 if (MI.getOperand(i: 2).getSubReg())
1713 return false;
1714
1715 CmpMask = ~0;
1716 CmpValue = 0;
1717 return true;
1718 case AArch64::SUBSWri:
1719 case AArch64::ADDSWri:
1720 case AArch64::SUBSXri:
1721 case AArch64::ADDSXri:
1722 SrcReg = MI.getOperand(i: 1).getReg();
1723 SrcReg2 = 0;
1724 CmpMask = ~0;
1725 CmpValue = MI.getOperand(i: 2).getImm();
1726 return true;
1727 case AArch64::ANDSWri:
1728 case AArch64::ANDSXri:
1729 // ANDS does not use the same encoding scheme as the others xxxS
1730 // instructions.
1731 SrcReg = MI.getOperand(i: 1).getReg();
1732 SrcReg2 = 0;
1733 CmpMask = ~0;
1734 CmpValue = AArch64_AM::decodeLogicalImmediate(
1735 val: MI.getOperand(i: 2).getImm(),
1736 regSize: MI.getOpcode() == AArch64::ANDSWri ? 32 : 64);
1737 return true;
1738 }
1739
1740 return false;
1741}
1742
1743static bool UpdateOperandRegClass(MachineInstr &Instr) {
1744 MachineBasicBlock *MBB = Instr.getParent();
1745 assert(MBB && "Can't get MachineBasicBlock here");
1746 MachineFunction *MF = MBB->getParent();
1747 assert(MF && "Can't get MachineFunction here");
1748 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
1749 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
1750 MachineRegisterInfo *MRI = &MF->getRegInfo();
1751
1752 for (unsigned OpIdx = 0, EndIdx = Instr.getNumOperands(); OpIdx < EndIdx;
1753 ++OpIdx) {
1754 MachineOperand &MO = Instr.getOperand(i: OpIdx);
1755 const TargetRegisterClass *OpRegCstraints =
1756 Instr.getRegClassConstraint(OpIdx, TII, TRI);
1757
1758 // If there's no constraint, there's nothing to do.
1759 if (!OpRegCstraints)
1760 continue;
1761 // If the operand is a frame index, there's nothing to do here.
1762 // A frame index operand will resolve correctly during PEI.
1763 if (MO.isFI())
1764 continue;
1765
1766 assert(MO.isReg() &&
1767 "Operand has register constraints without being a register!");
1768
1769 Register Reg = MO.getReg();
1770 if (Reg.isPhysical()) {
1771 if (!OpRegCstraints->contains(Reg))
1772 return false;
1773 } else if (!OpRegCstraints->hasSubClassEq(RC: MRI->getRegClass(Reg)) &&
1774 !MRI->constrainRegClass(Reg, RC: OpRegCstraints))
1775 return false;
1776 }
1777
1778 return true;
1779}
1780
1781/// Return the opcode that does not set flags when possible - otherwise
1782/// return the original opcode. The caller is responsible to do the actual
1783/// substitution and legality checking.
1784static unsigned convertToNonFlagSettingOpc(const MachineInstr &MI) {
1785 // Don't convert all compare instructions, because for some the zero register
1786 // encoding becomes the sp register.
1787 bool MIDefinesZeroReg = false;
1788 if (MI.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
1789 MI.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr))
1790 MIDefinesZeroReg = true;
1791
1792 switch (MI.getOpcode()) {
1793 default:
1794 return MI.getOpcode();
1795 case AArch64::ADDSWrr:
1796 return AArch64::ADDWrr;
1797 case AArch64::ADDSWri:
1798 return MIDefinesZeroReg ? AArch64::ADDSWri : AArch64::ADDWri;
1799 case AArch64::ADDSWrs:
1800 return MIDefinesZeroReg ? AArch64::ADDSWrs : AArch64::ADDWrs;
1801 case AArch64::ADDSWrx:
1802 return AArch64::ADDWrx;
1803 case AArch64::ADDSXrr:
1804 return AArch64::ADDXrr;
1805 case AArch64::ADDSXri:
1806 return MIDefinesZeroReg ? AArch64::ADDSXri : AArch64::ADDXri;
1807 case AArch64::ADDSXrs:
1808 return MIDefinesZeroReg ? AArch64::ADDSXrs : AArch64::ADDXrs;
1809 case AArch64::ADDSXrx:
1810 return AArch64::ADDXrx;
1811 case AArch64::SUBSWrr:
1812 return AArch64::SUBWrr;
1813 case AArch64::SUBSWri:
1814 return MIDefinesZeroReg ? AArch64::SUBSWri : AArch64::SUBWri;
1815 case AArch64::SUBSWrs:
1816 return MIDefinesZeroReg ? AArch64::SUBSWrs : AArch64::SUBWrs;
1817 case AArch64::SUBSWrx:
1818 return AArch64::SUBWrx;
1819 case AArch64::SUBSXrr:
1820 return AArch64::SUBXrr;
1821 case AArch64::SUBSXri:
1822 return MIDefinesZeroReg ? AArch64::SUBSXri : AArch64::SUBXri;
1823 case AArch64::SUBSXrs:
1824 return MIDefinesZeroReg ? AArch64::SUBSXrs : AArch64::SUBXrs;
1825 case AArch64::SUBSXrx:
1826 return AArch64::SUBXrx;
1827 }
1828}
1829
1830enum AccessKind { AK_Write = 0x01, AK_Read = 0x10, AK_All = 0x11 };
1831
1832/// True when condition flags are accessed (either by writing or reading)
1833/// on the instruction trace starting at From and ending at To.
1834///
1835/// Note: If From and To are from different blocks it's assumed CC are accessed
1836/// on the path.
1837static bool areCFlagsAccessedBetweenInstrs(
1838 MachineBasicBlock::iterator From, MachineBasicBlock::iterator To,
1839 const TargetRegisterInfo *TRI, const AccessKind AccessToCheck = AK_All) {
1840 // Early exit if To is at the beginning of the BB.
1841 if (To == To->getParent()->begin())
1842 return true;
1843
1844 // Check whether the instructions are in the same basic block
1845 // If not, assume the condition flags might get modified somewhere.
1846 if (To->getParent() != From->getParent())
1847 return true;
1848
1849 // From must be above To.
1850 assert(std::any_of(
1851 ++To.getReverse(), To->getParent()->rend(),
1852 [From](MachineInstr &MI) { return MI.getIterator() == From; }));
1853
1854 // We iterate backward starting at \p To until we hit \p From.
1855 for (const MachineInstr &Instr :
1856 instructionsWithoutDebug(It: ++To.getReverse(), End: From.getReverse())) {
1857 if (((AccessToCheck & AK_Write) &&
1858 Instr.modifiesRegister(Reg: AArch64::NZCV, TRI)) ||
1859 ((AccessToCheck & AK_Read) && Instr.readsRegister(Reg: AArch64::NZCV, TRI)))
1860 return true;
1861 }
1862 return false;
1863}
1864
1865std::optional<unsigned>
1866AArch64InstrInfo::canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
1867 MachineInstr *Pred,
1868 const MachineRegisterInfo *MRI) const {
1869 unsigned MaskOpcode = Mask->getOpcode();
1870 unsigned PredOpcode = Pred->getOpcode();
1871 bool PredIsPTestLike = isPTestLikeOpcode(Opc: PredOpcode);
1872 bool PredIsWhileLike = isWhileOpcode(Opc: PredOpcode);
1873
1874 if (PredIsWhileLike) {
1875 // For PTEST(PG, PG), PTEST is redundant when PG is the result of a WHILEcc
1876 // instruction and the condition is "any" since WHILcc does an implicit
1877 // PTEST(ALL, PG) check and PG is always a subset of ALL.
1878 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1879 return PredOpcode;
1880
1881 // For PTEST(PTRUE_ALL, WHILE), if the element size matches, the PTEST is
1882 // redundant since WHILE performs an implicit PTEST with an all active
1883 // mask.
1884 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1885 getElementSizeForOpcode(Opc: MaskOpcode) ==
1886 getElementSizeForOpcode(Opc: PredOpcode))
1887 return PredOpcode;
1888
1889 // For PTEST_FIRST(PTRUE_ALL, WHILE), the PTEST_FIRST is redundant since
1890 // WHILEcc performs an implicit PTEST with an all active mask, setting
1891 // the N flag as the PTEST_FIRST would.
1892 if (PTest->getOpcode() == AArch64::PTEST_PP_FIRST &&
1893 isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31)
1894 return PredOpcode;
1895
1896 return {};
1897 }
1898
1899 if (PredIsPTestLike) {
1900 // For PTEST(PG, PG), PTEST is redundant when PG is the result of an
1901 // instruction that sets the flags as PTEST would and the condition is
1902 // "any" since PG is always a subset of the governing predicate of the
1903 // ptest-like instruction.
1904 if ((Mask == Pred) && PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1905 return PredOpcode;
1906
1907 auto PTestLikeMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1908
1909 // If the PTEST like instruction's general predicate is not `Mask`, attempt
1910 // to look through a copy and try again. This is because some instructions
1911 // take a predicate whose register class is a subset of its result class.
1912 if (Mask != PTestLikeMask && PTestLikeMask->isFullCopy() &&
1913 PTestLikeMask->getOperand(i: 1).getReg().isVirtual())
1914 PTestLikeMask =
1915 MRI->getUniqueVRegDef(Reg: PTestLikeMask->getOperand(i: 1).getReg());
1916
1917 // For PTEST(PTRUE_ALL, PTEST_LIKE), the PTEST is redundant if the
1918 // the element size matches and either the PTEST_LIKE instruction uses
1919 // the same all active mask or the condition is "any".
1920 if (isPTrueOpcode(Opc: MaskOpcode) && Mask->getOperand(i: 1).getImm() == 31 &&
1921 getElementSizeForOpcode(Opc: MaskOpcode) ==
1922 getElementSizeForOpcode(Opc: PredOpcode)) {
1923 if (Mask == PTestLikeMask || PTest->getOpcode() == AArch64::PTEST_PP_ANY)
1924 return PredOpcode;
1925 }
1926
1927 // For PTEST(PG, PTEST_LIKE(PG, ...)), the PTEST is redundant since the
1928 // flags are set based on the same mask 'PG', but PTEST_LIKE must operate
1929 // on 8-bit predicates like the PTEST. Otherwise, for instructions like
1930 // compare that also support 16/32/64-bit predicates, the implicit PTEST
1931 // performed by the compare could consider fewer lanes for these element
1932 // sizes.
1933 //
1934 // For example, consider
1935 //
1936 // ptrue p0.b ; P0=1111-1111-1111-1111
1937 // index z0.s, #0, #1 ; Z0=<0,1,2,3>
1938 // index z1.s, #1, #1 ; Z1=<1,2,3,4>
1939 // cmphi p1.s, p0/z, z1.s, z0.s ; P1=0001-0001-0001-0001
1940 // ; ^ last active
1941 // ptest p0, p1.b ; P1=0001-0001-0001-0001
1942 // ; ^ last active
1943 //
1944 // where the compare generates a canonical all active 32-bit predicate
1945 // (equivalent to 'ptrue p1.s, all'). The implicit PTEST sets the last
1946 // active flag, whereas the PTEST instruction with the same mask doesn't.
1947 // For PTEST_ANY this doesn't apply as the flags in this case would be
1948 // identical regardless of element size.
1949 uint64_t PredElementSize = getElementSizeForOpcode(Opc: PredOpcode);
1950 if (Mask == PTestLikeMask && (PredElementSize == AArch64::ElementSizeB ||
1951 PTest->getOpcode() == AArch64::PTEST_PP_ANY))
1952 return PredOpcode;
1953
1954 return {};
1955 }
1956
1957 // If OP in PTEST(PG, OP(PG, ...)) has a flag-setting variant change the
1958 // opcode so the PTEST becomes redundant.
1959 switch (PredOpcode) {
1960 case AArch64::AND_PPzPP:
1961 case AArch64::BIC_PPzPP:
1962 case AArch64::EOR_PPzPP:
1963 case AArch64::NAND_PPzPP:
1964 case AArch64::NOR_PPzPP:
1965 case AArch64::ORN_PPzPP:
1966 case AArch64::ORR_PPzPP:
1967 case AArch64::BRKA_PPzP:
1968 case AArch64::BRKPA_PPzPP:
1969 case AArch64::BRKB_PPzP:
1970 case AArch64::BRKPB_PPzPP:
1971 case AArch64::RDFFR_PPz: {
1972 // Check to see if our mask is the same. If not the resulting flag bits
1973 // may be different and we can't remove the ptest.
1974 auto *PredMask = MRI->getUniqueVRegDef(Reg: Pred->getOperand(i: 1).getReg());
1975 if (Mask != PredMask)
1976 return {};
1977 break;
1978 }
1979 case AArch64::BRKN_PPzP: {
1980 // BRKN uses an all active implicit mask to set flags unlike the other
1981 // flag-setting instructions.
1982 // PTEST(PTRUE_B(31), BRKN(PG, A, B)) -> BRKNS(PG, A, B).
1983 if ((MaskOpcode != AArch64::PTRUE_B) ||
1984 (Mask->getOperand(i: 1).getImm() != 31))
1985 return {};
1986 break;
1987 }
1988 case AArch64::PTRUE_B:
1989 // PTEST(OP=PTRUE_B(A), OP) -> PTRUES_B(A)
1990 break;
1991 default:
1992 // Bail out if we don't recognize the input
1993 return {};
1994 }
1995
1996 return convertToFlagSettingOpc(Opc: PredOpcode);
1997}
1998
1999/// optimizePTestInstr - Attempt to remove a ptest of a predicate-generating
2000/// operation which could set the flags in an identical manner
2001bool AArch64InstrInfo::optimizePTestInstr(
2002 MachineInstr *PTest, unsigned MaskReg, unsigned PredReg,
2003 const MachineRegisterInfo *MRI) const {
2004 auto *Mask = MRI->getUniqueVRegDef(Reg: MaskReg);
2005 auto *Pred = MRI->getUniqueVRegDef(Reg: PredReg);
2006
2007 if (Pred->isCopy() && PTest->getOpcode() == AArch64::PTEST_PP_FIRST) {
2008 // Instructions which return a multi-vector (e.g. WHILECC_x2) require copies
2009 // before the branch to extract each subregister.
2010 auto Op = Pred->getOperand(i: 1);
2011 if (Op.isReg() && Op.getReg().isVirtual() &&
2012 Op.getSubReg() == AArch64::psub0)
2013 Pred = MRI->getUniqueVRegDef(Reg: Op.getReg());
2014 }
2015
2016 unsigned PredOpcode = Pred->getOpcode();
2017 auto NewOp = canRemovePTestInstr(PTest, Mask, Pred, MRI);
2018 if (!NewOp)
2019 return false;
2020
2021 const TargetRegisterInfo *TRI = &getRegisterInfo();
2022
2023 // If another instruction between Pred and PTest accesses flags, don't remove
2024 // the ptest or update the earlier instruction to modify them.
2025 if (areCFlagsAccessedBetweenInstrs(From: Pred, To: PTest, TRI))
2026 return false;
2027
2028 // If we pass all the checks, it's safe to remove the PTEST and use the flags
2029 // as they are prior to PTEST. Sometimes this requires the tested PTEST
2030 // operand to be replaced with an equivalent instruction that also sets the
2031 // flags.
2032 PTest->eraseFromParent();
2033 if (*NewOp != PredOpcode) {
2034 Pred->setDesc(get(Opcode: *NewOp));
2035 bool succeeded = UpdateOperandRegClass(Instr&: *Pred);
2036 (void)succeeded;
2037 assert(succeeded && "Operands have incompatible register classes!");
2038 Pred->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: TRI);
2039 }
2040
2041 // Ensure that the flags def is live.
2042 if (Pred->registerDefIsDead(Reg: AArch64::NZCV, TRI)) {
2043 unsigned i = 0, e = Pred->getNumOperands();
2044 for (; i != e; ++i) {
2045 MachineOperand &MO = Pred->getOperand(i);
2046 if (MO.isReg() && MO.isDef() && MO.getReg() == AArch64::NZCV) {
2047 MO.setIsDead(false);
2048 break;
2049 }
2050 }
2051 }
2052 return true;
2053}
2054
2055/// Try to optimize a compare instruction. A compare instruction is an
2056/// instruction which produces AArch64::NZCV. It can be truly compare
2057/// instruction
2058/// when there are no uses of its destination register.
2059///
2060/// The following steps are tried in order:
2061/// 1. Convert CmpInstr into an unconditional version.
2062/// 2. Remove CmpInstr if above there is an instruction producing a needed
2063/// condition code or an instruction which can be converted into such an
2064/// instruction.
2065/// Only comparison with zero is supported.
2066bool AArch64InstrInfo::optimizeCompareInstr(
2067 MachineInstr &CmpInstr, Register SrcReg, Register SrcReg2, int64_t CmpMask,
2068 int64_t CmpValue, const MachineRegisterInfo *MRI) const {
2069 assert(CmpInstr.getParent());
2070 assert(MRI);
2071
2072 // Replace SUBSWrr with SUBWrr if NZCV is not used.
2073 int DeadNZCVIdx =
2074 CmpInstr.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
2075 if (DeadNZCVIdx != -1) {
2076 if (CmpInstr.definesRegister(Reg: AArch64::WZR, /*TRI=*/nullptr) ||
2077 CmpInstr.definesRegister(Reg: AArch64::XZR, /*TRI=*/nullptr)) {
2078 CmpInstr.eraseFromParent();
2079 return true;
2080 }
2081 unsigned Opc = CmpInstr.getOpcode();
2082 unsigned NewOpc = convertToNonFlagSettingOpc(MI: CmpInstr);
2083 if (NewOpc == Opc)
2084 return false;
2085 const MCInstrDesc &MCID = get(Opcode: NewOpc);
2086 CmpInstr.setDesc(MCID);
2087 CmpInstr.removeOperand(OpNo: DeadNZCVIdx);
2088 bool succeeded = UpdateOperandRegClass(Instr&: CmpInstr);
2089 (void)succeeded;
2090 assert(succeeded && "Some operands reg class are incompatible!");
2091 return true;
2092 }
2093
2094 if (CmpInstr.getOpcode() == AArch64::PTEST_PP ||
2095 CmpInstr.getOpcode() == AArch64::PTEST_PP_ANY ||
2096 CmpInstr.getOpcode() == AArch64::PTEST_PP_FIRST)
2097 return optimizePTestInstr(PTest: &CmpInstr, MaskReg: SrcReg, PredReg: SrcReg2, MRI);
2098
2099 if (SrcReg2 != 0)
2100 return false;
2101
2102 // CmpInstr is a Compare instruction if destination register is not used.
2103 if (!MRI->use_nodbg_empty(RegNo: CmpInstr.getOperand(i: 0).getReg()))
2104 return false;
2105
2106 if (CmpValue == 0 && substituteCmpToZero(CmpInstr, SrcReg, MRI: *MRI))
2107 return true;
2108 return (CmpValue == 0 || CmpValue == 1) &&
2109 removeCmpToZeroOrOne(CmpInstr, SrcReg, CmpValue, MRI: *MRI);
2110}
2111
2112/// Get opcode of S version of Instr.
2113/// If Instr is S version its opcode is returned.
2114/// AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version
2115/// or we are not interested in it.
2116static unsigned sForm(MachineInstr &Instr) {
2117 switch (Instr.getOpcode()) {
2118 default:
2119 return AArch64::INSTRUCTION_LIST_END;
2120
2121 case AArch64::ADDSWrr:
2122 case AArch64::ADDSWri:
2123 case AArch64::ADDSXrr:
2124 case AArch64::ADDSXri:
2125 case AArch64::ADDSWrx:
2126 case AArch64::ADDSXrx:
2127 case AArch64::ADDSWrs:
2128 case AArch64::ADDSXrs:
2129 case AArch64::SUBSWrr:
2130 case AArch64::SUBSWri:
2131 case AArch64::SUBSWrx:
2132 case AArch64::SUBSWrs:
2133 case AArch64::SUBSXrr:
2134 case AArch64::SUBSXri:
2135 case AArch64::SUBSXrx:
2136 case AArch64::SUBSXrs:
2137 case AArch64::ANDSWri:
2138 case AArch64::ANDSWrr:
2139 case AArch64::ANDSWrs:
2140 case AArch64::ANDSXri:
2141 case AArch64::ANDSXrr:
2142 case AArch64::ANDSXrs:
2143 case AArch64::BICSWrr:
2144 case AArch64::BICSXrr:
2145 case AArch64::BICSWrs:
2146 case AArch64::BICSXrs:
2147 case AArch64::ADCSWr:
2148 case AArch64::ADCSXr:
2149 case AArch64::SBCSWr:
2150 case AArch64::SBCSXr:
2151 return Instr.getOpcode();
2152
2153 case AArch64::ADDWrr:
2154 return AArch64::ADDSWrr;
2155 case AArch64::ADDWri:
2156 return AArch64::ADDSWri;
2157 case AArch64::ADDXrr:
2158 return AArch64::ADDSXrr;
2159 case AArch64::ADDXri:
2160 return AArch64::ADDSXri;
2161 case AArch64::ADDWrx:
2162 return AArch64::ADDSWrx;
2163 case AArch64::ADDXrx:
2164 return AArch64::ADDSXrx;
2165 case AArch64::ADDWrs:
2166 return AArch64::ADDSWrs;
2167 case AArch64::ADDXrs:
2168 return AArch64::ADDSXrs;
2169 case AArch64::ADCWr:
2170 return AArch64::ADCSWr;
2171 case AArch64::ADCXr:
2172 return AArch64::ADCSXr;
2173 case AArch64::SUBWrr:
2174 return AArch64::SUBSWrr;
2175 case AArch64::SUBWri:
2176 return AArch64::SUBSWri;
2177 case AArch64::SUBXrr:
2178 return AArch64::SUBSXrr;
2179 case AArch64::SUBXri:
2180 return AArch64::SUBSXri;
2181 case AArch64::SUBWrx:
2182 return AArch64::SUBSWrx;
2183 case AArch64::SUBXrx:
2184 return AArch64::SUBSXrx;
2185 case AArch64::SUBWrs:
2186 return AArch64::SUBSWrs;
2187 case AArch64::SUBXrs:
2188 return AArch64::SUBSXrs;
2189 case AArch64::SBCWr:
2190 return AArch64::SBCSWr;
2191 case AArch64::SBCXr:
2192 return AArch64::SBCSXr;
2193 case AArch64::ANDWri:
2194 return AArch64::ANDSWri;
2195 case AArch64::ANDXri:
2196 return AArch64::ANDSXri;
2197 case AArch64::ANDWrr:
2198 return AArch64::ANDSWrr;
2199 case AArch64::ANDWrs:
2200 return AArch64::ANDSWrs;
2201 case AArch64::ANDXrr:
2202 return AArch64::ANDSXrr;
2203 case AArch64::ANDXrs:
2204 return AArch64::ANDSXrs;
2205 case AArch64::BICWrr:
2206 return AArch64::BICSWrr;
2207 case AArch64::BICXrr:
2208 return AArch64::BICSXrr;
2209 case AArch64::BICWrs:
2210 return AArch64::BICSWrs;
2211 case AArch64::BICXrs:
2212 return AArch64::BICSXrs;
2213 }
2214}
2215
2216/// Check if AArch64::NZCV should be alive in successors of MBB.
2217static bool areCFlagsAliveInSuccessors(const MachineBasicBlock *MBB) {
2218 for (auto *BB : MBB->successors())
2219 if (BB->isLiveIn(Reg: AArch64::NZCV))
2220 return true;
2221 return false;
2222}
2223
2224/// \returns The condition code operand index for \p Instr if it is a branch
2225/// or select and -1 otherwise.
2226int AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(
2227 const MachineInstr &Instr) {
2228 switch (Instr.getOpcode()) {
2229 default:
2230 return -1;
2231
2232 case AArch64::Bcc: {
2233 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2234 assert(Idx >= 2);
2235 return Idx - 2;
2236 }
2237
2238 case AArch64::CSINVWr:
2239 case AArch64::CSINVXr:
2240 case AArch64::CSINCWr:
2241 case AArch64::CSINCXr:
2242 case AArch64::CSELWr:
2243 case AArch64::CSELXr:
2244 case AArch64::CSNEGWr:
2245 case AArch64::CSNEGXr:
2246 case AArch64::FCSELSrrr:
2247 case AArch64::FCSELDrrr: {
2248 int Idx = Instr.findRegisterUseOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr);
2249 assert(Idx >= 1);
2250 return Idx - 1;
2251 }
2252 }
2253}
2254
2255/// Find a condition code used by the instruction.
2256/// Returns AArch64CC::Invalid if either the instruction does not use condition
2257/// codes or we don't optimize CmpInstr in the presence of such instructions.
2258static AArch64CC::CondCode findCondCodeUsedByInstr(const MachineInstr &Instr) {
2259 int CCIdx =
2260 AArch64InstrInfo::findCondCodeUseOperandIdxForBranchOrSelect(Instr);
2261 return CCIdx >= 0 ? static_cast<AArch64CC::CondCode>(
2262 Instr.getOperand(i: CCIdx).getImm())
2263 : AArch64CC::Invalid;
2264}
2265
2266static UsedNZCV getUsedNZCV(AArch64CC::CondCode CC) {
2267 assert(CC != AArch64CC::Invalid);
2268 UsedNZCV UsedFlags;
2269 switch (CC) {
2270 default:
2271 break;
2272
2273 case AArch64CC::EQ: // Z set
2274 case AArch64CC::NE: // Z clear
2275 UsedFlags.Z = true;
2276 break;
2277
2278 case AArch64CC::HI: // Z clear and C set
2279 case AArch64CC::LS: // Z set or C clear
2280 UsedFlags.Z = true;
2281 [[fallthrough]];
2282 case AArch64CC::HS: // C set
2283 case AArch64CC::LO: // C clear
2284 UsedFlags.C = true;
2285 break;
2286
2287 case AArch64CC::MI: // N set
2288 case AArch64CC::PL: // N clear
2289 UsedFlags.N = true;
2290 break;
2291
2292 case AArch64CC::VS: // V set
2293 case AArch64CC::VC: // V clear
2294 UsedFlags.V = true;
2295 break;
2296
2297 case AArch64CC::GT: // Z clear, N and V the same
2298 case AArch64CC::LE: // Z set, N and V differ
2299 UsedFlags.Z = true;
2300 [[fallthrough]];
2301 case AArch64CC::GE: // N and V the same
2302 case AArch64CC::LT: // N and V differ
2303 UsedFlags.N = true;
2304 UsedFlags.V = true;
2305 break;
2306 }
2307 return UsedFlags;
2308}
2309
2310/// \returns Conditions flags used after \p CmpInstr in its MachineBB if NZCV
2311/// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
2312/// \returns std::nullopt otherwise.
2313///
2314/// Collect instructions using that flags in \p CCUseInstrs if provided.
2315std::optional<UsedNZCV>
2316llvm::examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
2317 const TargetRegisterInfo &TRI,
2318 SmallVectorImpl<MachineInstr *> *CCUseInstrs) {
2319 MachineBasicBlock *CmpParent = CmpInstr.getParent();
2320 if (MI.getParent() != CmpParent)
2321 return std::nullopt;
2322
2323 if (areCFlagsAliveInSuccessors(MBB: CmpParent))
2324 return std::nullopt;
2325
2326 UsedNZCV NZCVUsedAfterCmp;
2327 for (MachineInstr &Instr : instructionsWithoutDebug(
2328 It: std::next(x: CmpInstr.getIterator()), End: CmpParent->instr_end())) {
2329 if (Instr.readsRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
2330 AArch64CC::CondCode CC = findCondCodeUsedByInstr(Instr);
2331 if (CC == AArch64CC::Invalid) // Unsupported conditional instruction
2332 return std::nullopt;
2333 NZCVUsedAfterCmp |= getUsedNZCV(CC);
2334 if (CCUseInstrs)
2335 CCUseInstrs->push_back(Elt: &Instr);
2336 }
2337 if (Instr.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI))
2338 break;
2339 }
2340 return NZCVUsedAfterCmp;
2341}
2342
2343static bool isADDSRegImm(unsigned Opcode) {
2344 return Opcode == AArch64::ADDSWri || Opcode == AArch64::ADDSXri;
2345}
2346
2347static bool isSUBSRegImm(unsigned Opcode) {
2348 return Opcode == AArch64::SUBSWri || Opcode == AArch64::SUBSXri;
2349}
2350
2351static bool isANDOpcode(MachineInstr &MI) {
2352 unsigned Opc = sForm(Instr&: MI);
2353 switch (Opc) {
2354 case AArch64::ANDSWri:
2355 case AArch64::ANDSWrr:
2356 case AArch64::ANDSWrs:
2357 case AArch64::ANDSXri:
2358 case AArch64::ANDSXrr:
2359 case AArch64::ANDSXrs:
2360 case AArch64::BICSWrr:
2361 case AArch64::BICSXrr:
2362 case AArch64::BICSWrs:
2363 case AArch64::BICSXrs:
2364 return true;
2365 default:
2366 return false;
2367 }
2368}
2369
2370/// Check if CmpInstr can be substituted by MI.
2371///
2372/// CmpInstr can be substituted:
2373/// - CmpInstr is either 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2374/// - and, MI and CmpInstr are from the same MachineBB
2375/// - and, condition flags are not alive in successors of the CmpInstr parent
2376/// - and, if MI opcode is the S form there must be no defs of flags between
2377/// MI and CmpInstr
2378/// or if MI opcode is not the S form there must be neither defs of flags
2379/// nor uses of flags between MI and CmpInstr.
2380/// - and, C is not used after CmpInstr; CmpInstr's C is from adds/subs #0 on
2381/// SrcReg and can differ from MI (e.g. carry out of ADCS/SBCS).
2382/// - and, V is not used after CmpInstr unless MI is AND/BIC (V cleared) or MI
2383/// has NoSWrap (overflow is poison and the fold is still safe).
2384static bool canInstrSubstituteCmpInstr(MachineInstr &MI, MachineInstr &CmpInstr,
2385 const TargetRegisterInfo &TRI) {
2386 // MI is an opcode sForm maps (add/sub/adc/sbc/and/bic and their S forms).
2387 assert(sForm(MI) != AArch64::INSTRUCTION_LIST_END);
2388
2389 const unsigned CmpOpcode = CmpInstr.getOpcode();
2390 if (!isADDSRegImm(Opcode: CmpOpcode) && !isSUBSRegImm(Opcode: CmpOpcode))
2391 return false;
2392
2393 assert((CmpInstr.getOperand(2).isImm() &&
2394 CmpInstr.getOperand(2).getImm() == 0) &&
2395 "Caller guarantees that CmpInstr compares with constant 0");
2396
2397 std::optional<UsedNZCV> NZVCUsed = examineCFlagsUse(MI, CmpInstr, TRI);
2398 if (!NZVCUsed || NZVCUsed->C)
2399 return false;
2400
2401 // CmpInstr is ADDS/SUBS with immediate 0 on SrcReg (compare SrcReg to zero).
2402 // After the fold, users see NZCV from MI (or its S form), not from CmpInstr.
2403 // N/Z match CmpInstr for the value in SrcReg; C/V need not match in general
2404 // (e.g. ADCS vs adds #0), so we require C unused after CmpInstr and gate V
2405 // as below. NoSWrap makes signed overflow poison; AND/BIC clear V.
2406 if (NZVCUsed->V && !MI.getFlag(Flag: MachineInstr::NoSWrap) && !isANDOpcode(MI))
2407 return false;
2408
2409 AccessKind AccessToCheck = AK_Write;
2410 if (sForm(Instr&: MI) != MI.getOpcode())
2411 AccessToCheck = AK_All;
2412 return !areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck);
2413}
2414
2415/// Substitute an instruction comparing to zero with another instruction
2416/// which produces needed condition flags.
2417///
2418/// Return true on success.
2419bool AArch64InstrInfo::substituteCmpToZero(
2420 MachineInstr &CmpInstr, unsigned SrcReg,
2421 const MachineRegisterInfo &MRI) const {
2422 // Get the unique definition of SrcReg.
2423 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2424 if (!MI)
2425 return false;
2426
2427 const TargetRegisterInfo &TRI = getRegisterInfo();
2428
2429 unsigned NewOpc = sForm(Instr&: *MI);
2430 if (NewOpc == AArch64::INSTRUCTION_LIST_END)
2431 return false;
2432
2433 if (!canInstrSubstituteCmpInstr(MI&: *MI, CmpInstr, TRI))
2434 return false;
2435
2436 // Update the instruction to set NZCV.
2437 MI->setDesc(get(Opcode: NewOpc));
2438 CmpInstr.eraseFromParent();
2439 bool succeeded = UpdateOperandRegClass(Instr&: *MI);
2440 (void)succeeded;
2441 assert(succeeded && "Some operands reg class are incompatible!");
2442 MI->addRegisterDefined(Reg: AArch64::NZCV, RegInfo: &TRI);
2443 return true;
2444}
2445
2446/// \returns True if \p CmpInstr can be removed.
2447///
2448/// \p IsInvertCC is true if, after removing \p CmpInstr, condition
2449/// codes used in \p CCUseInstrs must be inverted.
2450static bool canCmpInstrBeRemoved(MachineInstr &MI, MachineInstr &CmpInstr,
2451 int CmpValue, const TargetRegisterInfo &TRI,
2452 SmallVectorImpl<MachineInstr *> &CCUseInstrs,
2453 bool &IsInvertCC) {
2454 assert((CmpValue == 0 || CmpValue == 1) &&
2455 "Only comparisons to 0 or 1 considered for removal!");
2456
2457 // MI is 'CSINCWr %vreg, wzr, wzr, <cc>' or 'CSINCXr %vreg, xzr, xzr, <cc>'
2458 unsigned MIOpc = MI.getOpcode();
2459 if (MIOpc == AArch64::CSINCWr) {
2460 if (MI.getOperand(i: 1).getReg() != AArch64::WZR ||
2461 MI.getOperand(i: 2).getReg() != AArch64::WZR)
2462 return false;
2463 } else if (MIOpc == AArch64::CSINCXr) {
2464 if (MI.getOperand(i: 1).getReg() != AArch64::XZR ||
2465 MI.getOperand(i: 2).getReg() != AArch64::XZR)
2466 return false;
2467 } else {
2468 return false;
2469 }
2470 AArch64CC::CondCode MICC = findCondCodeUsedByInstr(Instr: MI);
2471 if (MICC == AArch64CC::Invalid)
2472 return false;
2473
2474 // NZCV needs to be defined
2475 if (MI.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) != -1)
2476 return false;
2477
2478 // CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0' or 'SUBS %vreg, 1'
2479 const unsigned CmpOpcode = CmpInstr.getOpcode();
2480 bool IsSubsRegImm = isSUBSRegImm(Opcode: CmpOpcode);
2481 if (CmpValue && !IsSubsRegImm)
2482 return false;
2483 if (!CmpValue && !IsSubsRegImm && !isADDSRegImm(Opcode: CmpOpcode))
2484 return false;
2485
2486 // MI conditions allowed: eq, ne, mi, pl
2487 UsedNZCV MIUsedNZCV = getUsedNZCV(CC: MICC);
2488 if (MIUsedNZCV.C || MIUsedNZCV.V)
2489 return false;
2490
2491 std::optional<UsedNZCV> NZCVUsedAfterCmp =
2492 examineCFlagsUse(MI, CmpInstr, TRI, CCUseInstrs: &CCUseInstrs);
2493 // Condition flags are not used in CmpInstr basic block successors and only
2494 // Z or N flags allowed to be used after CmpInstr within its basic block
2495 if (!NZCVUsedAfterCmp || NZCVUsedAfterCmp->C || NZCVUsedAfterCmp->V)
2496 return false;
2497 // Z or N flag used after CmpInstr must correspond to the flag used in MI
2498 if ((MIUsedNZCV.Z && NZCVUsedAfterCmp->N) ||
2499 (MIUsedNZCV.N && NZCVUsedAfterCmp->Z))
2500 return false;
2501 // If CmpInstr is comparison to zero MI conditions are limited to eq, ne
2502 if (MIUsedNZCV.N && !CmpValue)
2503 return false;
2504
2505 // There must be no defs of flags between MI and CmpInstr
2506 if (areCFlagsAccessedBetweenInstrs(From: &MI, To: &CmpInstr, TRI: &TRI, AccessToCheck: AK_Write))
2507 return false;
2508
2509 // Condition code is inverted in the following cases:
2510 // 1. MI condition is ne; CmpInstr is 'ADDS %vreg, 0' or 'SUBS %vreg, 0'
2511 // 2. MI condition is eq, pl; CmpInstr is 'SUBS %vreg, 1'
2512 IsInvertCC = (CmpValue && (MICC == AArch64CC::EQ || MICC == AArch64CC::PL)) ||
2513 (!CmpValue && MICC == AArch64CC::NE);
2514 return true;
2515}
2516
2517/// Remove comparison in csinc-cmp sequence
2518///
2519/// Examples:
2520/// 1. \code
2521/// csinc w9, wzr, wzr, ne
2522/// cmp w9, #0
2523/// b.eq
2524/// \endcode
2525/// to
2526/// \code
2527/// csinc w9, wzr, wzr, ne
2528/// b.ne
2529/// \endcode
2530///
2531/// 2. \code
2532/// csinc x2, xzr, xzr, mi
2533/// cmp x2, #1
2534/// b.pl
2535/// \endcode
2536/// to
2537/// \code
2538/// csinc x2, xzr, xzr, mi
2539/// b.pl
2540/// \endcode
2541///
2542/// \param CmpInstr comparison instruction
2543/// \return True when comparison removed
2544bool AArch64InstrInfo::removeCmpToZeroOrOne(
2545 MachineInstr &CmpInstr, unsigned SrcReg, int CmpValue,
2546 const MachineRegisterInfo &MRI) const {
2547 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: SrcReg);
2548 if (!MI)
2549 return false;
2550 const TargetRegisterInfo &TRI = getRegisterInfo();
2551 SmallVector<MachineInstr *, 4> CCUseInstrs;
2552 bool IsInvertCC = false;
2553 if (!canCmpInstrBeRemoved(MI&: *MI, CmpInstr, CmpValue, TRI, CCUseInstrs,
2554 IsInvertCC))
2555 return false;
2556 // Make transformation
2557 CmpInstr.eraseFromParent();
2558 if (IsInvertCC) {
2559 // Invert condition codes in CmpInstr CC users
2560 for (MachineInstr *CCUseInstr : CCUseInstrs) {
2561 int Idx = findCondCodeUseOperandIdxForBranchOrSelect(Instr: *CCUseInstr);
2562 assert(Idx >= 0 && "Unexpected instruction using CC.");
2563 MachineOperand &CCOperand = CCUseInstr->getOperand(i: Idx);
2564 AArch64CC::CondCode CCUse = AArch64CC::getInvertedCondCode(
2565 Code: static_cast<AArch64CC::CondCode>(CCOperand.getImm()));
2566 CCOperand.setImm(CCUse);
2567 }
2568 }
2569 return true;
2570}
2571
2572bool AArch64InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
2573 if (MI.getOpcode() != TargetOpcode::LOAD_STACK_GUARD &&
2574 MI.getOpcode() != AArch64::CATCHRET &&
2575 MI.getOpcode() != AArch64::STACK_GUARD_UNMIX)
2576 return false;
2577
2578 MachineBasicBlock &MBB = *MI.getParent();
2579 auto &Subtarget = MBB.getParent()->getSubtarget<AArch64Subtarget>();
2580 auto TRI = Subtarget.getRegisterInfo();
2581 DebugLoc DL = MI.getDebugLoc();
2582
2583 if (MI.getOpcode() == AArch64::STACK_GUARD_UNMIX) {
2584 // Expand STACK_GUARD_UNMIX to: sub Rd, fp, Rs
2585 // This computes FP - stored_mixed_value to unmix the cookie
2586 Register DstReg = MI.getOperand(i: 0).getReg();
2587 Register SrcReg = MI.getOperand(i: 1).getReg();
2588
2589 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: DstReg)
2590 .addReg(RegNo: AArch64::FP)
2591 .addReg(RegNo: SrcReg);
2592
2593 MBB.erase(I: MI);
2594 return true;
2595 }
2596
2597 if (MI.getOpcode() == AArch64::CATCHRET) {
2598 // Skip to the first instruction before the epilog.
2599 const TargetInstrInfo *TII =
2600 MBB.getParent()->getSubtarget().getInstrInfo();
2601 MachineBasicBlock *TargetMBB = MI.getOperand(i: 0).getMBB();
2602 auto MBBI = MachineBasicBlock::iterator(MI);
2603 MachineBasicBlock::iterator FirstEpilogSEH = std::prev(x: MBBI);
2604 while (FirstEpilogSEH->getFlag(Flag: MachineInstr::FrameDestroy) &&
2605 FirstEpilogSEH != MBB.begin())
2606 FirstEpilogSEH = std::prev(x: FirstEpilogSEH);
2607 if (FirstEpilogSEH != MBB.begin())
2608 FirstEpilogSEH = std::next(x: FirstEpilogSEH);
2609 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADRP))
2610 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2611 .addMBB(MBB: TargetMBB, TargetFlags: AArch64II::MO_PAGE);
2612 BuildMI(BB&: MBB, I: FirstEpilogSEH, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri))
2613 .addReg(RegNo: AArch64::X0, Flags: RegState::Define)
2614 .addReg(RegNo: AArch64::X0)
2615 .addMBB(MBB: TargetMBB, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC)
2616 .addImm(Val: 0);
2617 TargetMBB->setMachineBlockAddressTaken();
2618 return true;
2619 }
2620
2621 Register Reg = MI.getOperand(i: 0).getReg();
2622 Module &M = *MBB.getParent()->getFunction().getParent();
2623 if (M.getStackProtectorGuard() == "sysreg") {
2624 const AArch64SysReg::SysReg *SrcReg =
2625 AArch64SysReg::lookupSysRegByName(Name: M.getStackProtectorGuardReg());
2626 if (!SrcReg)
2627 report_fatal_error(reason: "Unknown SysReg for Stack Protector Guard Register");
2628
2629 // mrs xN, sysreg
2630 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MRS))
2631 .addDef(RegNo: Reg, Flags: RegState::Renamable)
2632 .addImm(Val: SrcReg->Encoding);
2633 int Offset = M.getStackProtectorGuardOffset();
2634 if (Offset >= 0 && Offset <= 32760 && Offset % 8 == 0) {
2635 // ldr xN, [xN, #offset]
2636 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2637 .addDef(RegNo: Reg)
2638 .addUse(RegNo: Reg, Flags: RegState::Kill)
2639 .addImm(Val: Offset / 8);
2640 } else if (Offset >= -256 && Offset <= 255) {
2641 // ldur xN, [xN, #offset]
2642 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDURXi))
2643 .addDef(RegNo: Reg)
2644 .addUse(RegNo: Reg, Flags: RegState::Kill)
2645 .addImm(Val: Offset);
2646 } else if (Offset >= -4095 && Offset <= 4095) {
2647 if (Offset > 0) {
2648 // add xN, xN, #offset
2649 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri))
2650 .addDef(RegNo: Reg)
2651 .addUse(RegNo: Reg, Flags: RegState::Kill)
2652 .addImm(Val: Offset)
2653 .addImm(Val: 0);
2654 } else {
2655 // sub xN, xN, #offset
2656 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXri))
2657 .addDef(RegNo: Reg)
2658 .addUse(RegNo: Reg, Flags: RegState::Kill)
2659 .addImm(Val: -Offset)
2660 .addImm(Val: 0);
2661 }
2662 // ldr xN, [xN]
2663 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui))
2664 .addDef(RegNo: Reg)
2665 .addUse(RegNo: Reg, Flags: RegState::Kill)
2666 .addImm(Val: 0);
2667 } else {
2668 // Cases that are larger than +/- 4095 and not a multiple of 8, or larger
2669 // than 23760.
2670 // It might be nice to use AArch64::MOVi32imm here, which would get
2671 // expanded in PreSched2 after PostRA, but our lone scratch Reg already
2672 // contains the MRS result. findScratchNonCalleeSaveRegister() in
2673 // AArch64FrameLowering might help us find such a scratch register
2674 // though. If we failed to find a scratch register, we could emit a
2675 // stream of add instructions to build up the immediate. Or, we could try
2676 // to insert a AArch64::MOVi32imm before register allocation so that we
2677 // didn't need to scavenge for a scratch register.
2678 report_fatal_error(reason: "Unable to encode Stack Protector Guard Offset");
2679 }
2680 MBB.erase(I: MI);
2681 return true;
2682 }
2683
2684 const GlobalValue *GV =
2685 cast<GlobalValue>(Val: (*MI.memoperands_begin())->getValue());
2686 const TargetMachine &TM = MBB.getParent()->getTarget();
2687 unsigned OpFlags = Subtarget.ClassifyGlobalReference(GV, TM);
2688 const unsigned char MO_NC = AArch64II::MO_NC;
2689
2690 unsigned GuardWidth = M.getStackProtectorGuardValueWidth().value_or(
2691 u: Subtarget.isTargetILP32() ? 4 : 8);
2692 if (GuardWidth != 4 && GuardWidth != 8)
2693 report_fatal_error(reason: "Unsupported stack protector value width");
2694 if ((OpFlags & AArch64II::MO_GOT) != 0) {
2695 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LOADgot), DestReg: Reg)
2696 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags);
2697 if (GuardWidth == 4) {
2698 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2699 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2700 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2701 .addUse(RegNo: Reg, Flags: RegState::Kill)
2702 .addImm(Val: 0)
2703 .addMemOperand(MMO: *MI.memoperands_begin())
2704 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2705 } else {
2706 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2707 .addReg(RegNo: Reg, Flags: RegState::Kill)
2708 .addImm(Val: 0)
2709 .addMemOperand(MMO: *MI.memoperands_begin());
2710 }
2711 } else if (TM.getCodeModel() == CodeModel::Large) {
2712 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg)
2713 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G0 | MO_NC)
2714 .addImm(Val: 0);
2715 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2716 .addReg(RegNo: Reg, Flags: RegState::Kill)
2717 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G1 | MO_NC)
2718 .addImm(Val: 16);
2719 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2720 .addReg(RegNo: Reg, Flags: RegState::Kill)
2721 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G2 | MO_NC)
2722 .addImm(Val: 32);
2723 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::MOVKXi), DestReg: Reg)
2724 .addReg(RegNo: Reg, Flags: RegState::Kill)
2725 .addGlobalAddress(GV, Offset: 0, TargetFlags: AArch64II::MO_G3)
2726 .addImm(Val: 48);
2727 if (GuardWidth == 4) {
2728 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2729 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2730 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2731 .addUse(RegNo: Reg, Flags: RegState::Kill)
2732 .addImm(Val: 0)
2733 .addMemOperand(MMO: *MI.memoperands_begin())
2734 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2735 } else {
2736 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2737 .addReg(RegNo: Reg, Flags: RegState::Kill)
2738 .addImm(Val: 0)
2739 .addMemOperand(MMO: *MI.memoperands_begin());
2740 }
2741 } else {
2742 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::ADRP), DestReg: Reg)
2743 .addGlobalAddress(GV, Offset: 0, TargetFlags: OpFlags | AArch64II::MO_PAGE);
2744 unsigned char LoFlags = OpFlags | AArch64II::MO_PAGEOFF | MO_NC;
2745 if (GuardWidth == 4) {
2746 unsigned Reg32 = TRI->getSubReg(Reg, Idx: AArch64::sub_32);
2747 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRWui))
2748 .addDef(RegNo: Reg32, Flags: RegState::Dead)
2749 .addUse(RegNo: Reg, Flags: RegState::Kill)
2750 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2751 .addMemOperand(MMO: *MI.memoperands_begin())
2752 .addDef(RegNo: Reg, Flags: RegState::Implicit);
2753 } else {
2754 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::LDRXui), DestReg: Reg)
2755 .addReg(RegNo: Reg, Flags: RegState::Kill)
2756 .addGlobalAddress(GV, Offset: 0, TargetFlags: LoFlags)
2757 .addMemOperand(MMO: *MI.memoperands_begin());
2758 }
2759 }
2760 // To match MSVC. Unlike x86_64 which uses xor instruction to mix the cookie,
2761 // we use sub instruction to mix the cookie on aarch64.
2762 // The mixing happens here in expandPostRAPseudo (after RA) to ensure we use
2763 // the final frame pointer value.
2764 if (Subtarget.getTargetTriple().isOSMSVCRT())
2765 BuildMI(BB&: MBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::SUBXrr), DestReg: Reg)
2766 .addReg(RegNo: AArch64::FP)
2767 .addReg(RegNo: Reg, Flags: RegState::Kill);
2768
2769 MBB.erase(I: MI);
2770
2771 return true;
2772}
2773
2774// Return true if this instruction simply sets its single destination register
2775// to zero. This is equivalent to a register rename of the zero-register.
2776bool AArch64InstrInfo::isGPRZero(const MachineInstr &MI) {
2777 switch (MI.getOpcode()) {
2778 default:
2779 break;
2780 case AArch64::MOVZWi:
2781 case AArch64::MOVZXi: // movz Rd, #0 (LSL #0)
2782 if (MI.getOperand(i: 1).isImm() && MI.getOperand(i: 1).getImm() == 0) {
2783 assert(MI.getDesc().getNumOperands() == 3 &&
2784 MI.getOperand(2).getImm() == 0 && "invalid MOVZi operands");
2785 return true;
2786 }
2787 break;
2788 case AArch64::ANDWri: // and Rd, Rzr, #imm
2789 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2790 case AArch64::ANDXri:
2791 return MI.getOperand(i: 1).getReg() == AArch64::XZR;
2792 case TargetOpcode::COPY:
2793 return MI.getOperand(i: 1).getReg() == AArch64::WZR;
2794 }
2795 return false;
2796}
2797
2798// Return true if this instruction simply renames a general register without
2799// modifying bits.
2800bool AArch64InstrInfo::isGPRCopy(const MachineInstr &MI) {
2801 switch (MI.getOpcode()) {
2802 default:
2803 break;
2804 case TargetOpcode::COPY: {
2805 // GPR32 copies will by lowered to ORRXrs
2806 Register DstReg = MI.getOperand(i: 0).getReg();
2807 return (AArch64::GPR32RegClass.contains(Reg: DstReg) ||
2808 AArch64::GPR64RegClass.contains(Reg: DstReg));
2809 }
2810 case AArch64::ORRXrs: // orr Xd, Xzr, Xm (LSL #0)
2811 if (MI.getOperand(i: 1).getReg() == AArch64::XZR) {
2812 assert(MI.getDesc().getNumOperands() == 4 &&
2813 MI.getOperand(3).getImm() == 0 && "invalid ORRrs operands");
2814 return true;
2815 }
2816 break;
2817 case AArch64::ADDXri: // add Xd, Xn, #0 (LSL #0)
2818 if (MI.getOperand(i: 2).getImm() == 0) {
2819 assert(MI.getDesc().getNumOperands() == 4 &&
2820 MI.getOperand(3).getImm() == 0 && "invalid ADDXri operands");
2821 return true;
2822 }
2823 break;
2824 }
2825 return false;
2826}
2827
2828// Return true if this instruction simply renames a general register without
2829// modifying bits.
2830bool AArch64InstrInfo::isFPRCopy(const MachineInstr &MI) {
2831 switch (MI.getOpcode()) {
2832 default:
2833 break;
2834 case TargetOpcode::COPY: {
2835 Register DstReg = MI.getOperand(i: 0).getReg();
2836 return AArch64::FPR128RegClass.contains(Reg: DstReg);
2837 }
2838 case AArch64::ORRv16i8:
2839 if (MI.getOperand(i: 1).getReg() == MI.getOperand(i: 2).getReg()) {
2840 assert(MI.getDesc().getNumOperands() == 3 && MI.getOperand(0).isReg() &&
2841 "invalid ORRv16i8 operands");
2842 return true;
2843 }
2844 break;
2845 }
2846 return false;
2847}
2848
2849static bool isFrameLoadOpcode(int Opcode) {
2850 switch (Opcode) {
2851 default:
2852 return false;
2853 case AArch64::LDRWui:
2854 case AArch64::LDRXui:
2855 case AArch64::LDRBui:
2856 case AArch64::LDRHui:
2857 case AArch64::LDRSui:
2858 case AArch64::LDRDui:
2859 case AArch64::LDRQui:
2860 case AArch64::LDR_PXI:
2861 return true;
2862 }
2863}
2864
2865Register AArch64InstrInfo::isLoadFromStackSlot(const MachineInstr &MI,
2866 int &FrameIndex) const {
2867 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2868 return Register();
2869
2870 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2871 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2872 FrameIndex = MI.getOperand(i: 1).getIndex();
2873 return MI.getOperand(i: 0).getReg();
2874 }
2875 return Register();
2876}
2877
2878static bool isFrameStoreOpcode(int Opcode) {
2879 switch (Opcode) {
2880 default:
2881 return false;
2882 case AArch64::STRWui:
2883 case AArch64::STRXui:
2884 case AArch64::STRBui:
2885 case AArch64::STRHui:
2886 case AArch64::STRSui:
2887 case AArch64::STRDui:
2888 case AArch64::STRQui:
2889 case AArch64::STR_PXI:
2890 return true;
2891 }
2892}
2893
2894Register AArch64InstrInfo::isStoreToStackSlot(const MachineInstr &MI,
2895 int &FrameIndex) const {
2896 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2897 return Register();
2898
2899 if (MI.getOperand(i: 0).getSubReg() == 0 && MI.getOperand(i: 1).isFI() &&
2900 MI.getOperand(i: 2).isImm() && MI.getOperand(i: 2).getImm() == 0) {
2901 FrameIndex = MI.getOperand(i: 1).getIndex();
2902 return MI.getOperand(i: 0).getReg();
2903 }
2904 return Register();
2905}
2906
2907Register AArch64InstrInfo::isStoreToStackSlotPostFE(const MachineInstr &MI,
2908 int &FrameIndex) const {
2909 if (!isFrameStoreOpcode(Opcode: MI.getOpcode()))
2910 return Register();
2911
2912 if (Register Reg = isStoreToStackSlot(MI, FrameIndex))
2913 return Reg;
2914
2915 SmallVector<const MachineMemOperand *, 1> Accesses;
2916 if (hasStoreToStackSlot(MI, Accesses)) {
2917 if (Accesses.size() > 1)
2918 return Register();
2919
2920 FrameIndex =
2921 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2922 ->getFrameIndex();
2923 return MI.getOperand(i: 0).getReg();
2924 }
2925 return Register();
2926}
2927
2928Register AArch64InstrInfo::isLoadFromStackSlotPostFE(const MachineInstr &MI,
2929 int &FrameIndex) const {
2930 if (!isFrameLoadOpcode(Opcode: MI.getOpcode()))
2931 return Register();
2932
2933 if (Register Reg = isLoadFromStackSlot(MI, FrameIndex))
2934 return Reg;
2935
2936 SmallVector<const MachineMemOperand *, 1> Accesses;
2937 if (hasLoadFromStackSlot(MI, Accesses)) {
2938 if (Accesses.size() > 1)
2939 return Register();
2940
2941 FrameIndex =
2942 cast<FixedStackPseudoSourceValue>(Val: Accesses.front()->getPseudoValue())
2943 ->getFrameIndex();
2944 return MI.getOperand(i: 0).getReg();
2945 }
2946 return Register();
2947}
2948
2949/// Check all MachineMemOperands for a hint to suppress pairing.
2950bool AArch64InstrInfo::isLdStPairSuppressed(const MachineInstr &MI) {
2951 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2952 return MMO->getFlags() & MOSuppressPair;
2953 });
2954}
2955
2956/// Set a flag on the first MachineMemOperand to suppress pairing.
2957void AArch64InstrInfo::suppressLdStPair(MachineInstr &MI) {
2958 if (MI.memoperands_empty())
2959 return;
2960 (*MI.memoperands_begin())->setFlags(MOSuppressPair);
2961}
2962
2963/// Check all MachineMemOperands for a hint that the load/store is strided.
2964bool AArch64InstrInfo::isStridedAccess(const MachineInstr &MI) {
2965 return llvm::any_of(Range: MI.memoperands(), P: [](MachineMemOperand *MMO) {
2966 return MMO->getFlags() & MOStridedAccess;
2967 });
2968}
2969
2970bool AArch64InstrInfo::hasUnscaledLdStOffset(unsigned Opc) {
2971 switch (Opc) {
2972 default:
2973 return false;
2974 case AArch64::STURSi:
2975 case AArch64::STRSpre:
2976 case AArch64::STURDi:
2977 case AArch64::STRDpre:
2978 case AArch64::STURQi:
2979 case AArch64::STRQpre:
2980 case AArch64::STURBBi:
2981 case AArch64::STURHHi:
2982 case AArch64::STURWi:
2983 case AArch64::STRWpre:
2984 case AArch64::STURXi:
2985 case AArch64::STRXpre:
2986 case AArch64::LDURSi:
2987 case AArch64::LDRSpre:
2988 case AArch64::LDURDi:
2989 case AArch64::LDRDpre:
2990 case AArch64::LDURQi:
2991 case AArch64::LDRQpre:
2992 case AArch64::LDURWi:
2993 case AArch64::LDRWpre:
2994 case AArch64::LDURXi:
2995 case AArch64::LDRXpre:
2996 case AArch64::LDRSWpre:
2997 case AArch64::LDURSWi:
2998 case AArch64::LDURHHi:
2999 case AArch64::LDURBBi:
3000 case AArch64::LDURSBWi:
3001 case AArch64::LDURSHWi:
3002 return true;
3003 }
3004}
3005
3006std::optional<unsigned> AArch64InstrInfo::getUnscaledLdSt(unsigned Opc) {
3007 switch (Opc) {
3008 default: return {};
3009 case AArch64::PRFMui: return AArch64::PRFUMi;
3010 case AArch64::LDRXui: return AArch64::LDURXi;
3011 case AArch64::LDRWui: return AArch64::LDURWi;
3012 case AArch64::LDRBui: return AArch64::LDURBi;
3013 case AArch64::LDRHui: return AArch64::LDURHi;
3014 case AArch64::LDRSui: return AArch64::LDURSi;
3015 case AArch64::LDRDui: return AArch64::LDURDi;
3016 case AArch64::LDRQui: return AArch64::LDURQi;
3017 case AArch64::LDRBBui: return AArch64::LDURBBi;
3018 case AArch64::LDRHHui: return AArch64::LDURHHi;
3019 case AArch64::LDRSBXui: return AArch64::LDURSBXi;
3020 case AArch64::LDRSBWui: return AArch64::LDURSBWi;
3021 case AArch64::LDRSHXui: return AArch64::LDURSHXi;
3022 case AArch64::LDRSHWui: return AArch64::LDURSHWi;
3023 case AArch64::LDRSWui: return AArch64::LDURSWi;
3024 case AArch64::STRXui: return AArch64::STURXi;
3025 case AArch64::STRWui: return AArch64::STURWi;
3026 case AArch64::STRBui: return AArch64::STURBi;
3027 case AArch64::STRHui: return AArch64::STURHi;
3028 case AArch64::STRSui: return AArch64::STURSi;
3029 case AArch64::STRDui: return AArch64::STURDi;
3030 case AArch64::STRQui: return AArch64::STURQi;
3031 case AArch64::STRBBui: return AArch64::STURBBi;
3032 case AArch64::STRHHui: return AArch64::STURHHi;
3033 }
3034}
3035
3036unsigned AArch64InstrInfo::getLoadStoreImmIdx(unsigned Opc) {
3037 switch (Opc) {
3038 default:
3039 llvm_unreachable("Unhandled Opcode in getLoadStoreImmIdx");
3040 case AArch64::ADDG:
3041 case AArch64::LDAPURBi:
3042 case AArch64::LDAPURHi:
3043 case AArch64::LDAPURi:
3044 case AArch64::LDAPURSBWi:
3045 case AArch64::LDAPURSBXi:
3046 case AArch64::LDAPURSHWi:
3047 case AArch64::LDAPURSHXi:
3048 case AArch64::LDAPURSWi:
3049 case AArch64::LDAPURXi:
3050 case AArch64::LDR_PPXI:
3051 case AArch64::LDR_PXI:
3052 case AArch64::LDR_ZXI:
3053 case AArch64::LDR_ZZXI:
3054 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
3055 case AArch64::LDR_ZZZXI:
3056 case AArch64::LDR_ZZZZXI:
3057 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
3058 case AArch64::LDRBBui:
3059 case AArch64::LDRBui:
3060 case AArch64::LDRDui:
3061 case AArch64::LDRHHui:
3062 case AArch64::LDRHui:
3063 case AArch64::LDRQui:
3064 case AArch64::LDRSBWui:
3065 case AArch64::LDRSBXui:
3066 case AArch64::LDRSHWui:
3067 case AArch64::LDRSHXui:
3068 case AArch64::LDRSui:
3069 case AArch64::LDRSWui:
3070 case AArch64::LDRWui:
3071 case AArch64::LDRXui:
3072 case AArch64::LDURBBi:
3073 case AArch64::LDURBi:
3074 case AArch64::LDURDi:
3075 case AArch64::LDURHHi:
3076 case AArch64::LDURHi:
3077 case AArch64::LDURQi:
3078 case AArch64::LDURSBWi:
3079 case AArch64::LDURSBXi:
3080 case AArch64::LDURSHWi:
3081 case AArch64::LDURSHXi:
3082 case AArch64::LDURSi:
3083 case AArch64::LDURSWi:
3084 case AArch64::LDURWi:
3085 case AArch64::LDURXi:
3086 case AArch64::PRFMui:
3087 case AArch64::PRFUMi:
3088 case AArch64::ST2Gi:
3089 case AArch64::STGi:
3090 case AArch64::STLURBi:
3091 case AArch64::STLURHi:
3092 case AArch64::STLURWi:
3093 case AArch64::STLURXi:
3094 case AArch64::StoreSwiftAsyncContext:
3095 case AArch64::STR_PPXI:
3096 case AArch64::STR_PXI:
3097 case AArch64::STR_ZXI:
3098 case AArch64::STR_ZZXI:
3099 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
3100 case AArch64::STR_ZZZXI:
3101 case AArch64::STR_ZZZZXI:
3102 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
3103 case AArch64::STRBBui:
3104 case AArch64::STRBui:
3105 case AArch64::STRDui:
3106 case AArch64::STRHHui:
3107 case AArch64::STRHui:
3108 case AArch64::STRQui:
3109 case AArch64::STRSui:
3110 case AArch64::STRWui:
3111 case AArch64::STRXui:
3112 case AArch64::STURBBi:
3113 case AArch64::STURBi:
3114 case AArch64::STURDi:
3115 case AArch64::STURHHi:
3116 case AArch64::STURHi:
3117 case AArch64::STURQi:
3118 case AArch64::STURSi:
3119 case AArch64::STURWi:
3120 case AArch64::STURXi:
3121 case AArch64::STZ2Gi:
3122 case AArch64::STZGi:
3123 case AArch64::TAGPstack:
3124 case AArch64::ATOMIC_STORE_HINT_Bi:
3125 case AArch64::ATOMIC_STORE_HINT_Hi:
3126 case AArch64::ATOMIC_STORE_HINT_Wi:
3127 case AArch64::ATOMIC_STORE_HINT_Si:
3128 case AArch64::ATOMIC_STORE_HINT_Xi:
3129 case AArch64::ATOMIC_STORE_HINT_Di:
3130 case AArch64::ATOMIC_STORE_HINT_Bui:
3131 case AArch64::ATOMIC_STORE_HINT_Hui:
3132 case AArch64::ATOMIC_STORE_HINT_Wui:
3133 case AArch64::ATOMIC_STORE_HINT_Sui:
3134 case AArch64::ATOMIC_STORE_HINT_Xui:
3135 case AArch64::ATOMIC_STORE_HINT_Dui:
3136 return 2;
3137 case AArch64::LD1B_D_IMM:
3138 case AArch64::LD1B_H_IMM:
3139 case AArch64::LD1B_IMM:
3140 case AArch64::LD1B_S_IMM:
3141 case AArch64::LD1D_IMM:
3142 case AArch64::LD1H_D_IMM:
3143 case AArch64::LD1H_IMM:
3144 case AArch64::LD1H_S_IMM:
3145 case AArch64::LD1RB_D_IMM:
3146 case AArch64::LD1RB_H_IMM:
3147 case AArch64::LD1RB_IMM:
3148 case AArch64::LD1RB_S_IMM:
3149 case AArch64::LD1RD_IMM:
3150 case AArch64::LD1RH_D_IMM:
3151 case AArch64::LD1RH_IMM:
3152 case AArch64::LD1RH_S_IMM:
3153 case AArch64::LD1RSB_D_IMM:
3154 case AArch64::LD1RSB_H_IMM:
3155 case AArch64::LD1RSB_S_IMM:
3156 case AArch64::LD1RSH_D_IMM:
3157 case AArch64::LD1RSH_S_IMM:
3158 case AArch64::LD1RSW_IMM:
3159 case AArch64::LD1RW_D_IMM:
3160 case AArch64::LD1RW_IMM:
3161 case AArch64::LD1SB_D_IMM:
3162 case AArch64::LD1SB_H_IMM:
3163 case AArch64::LD1SB_S_IMM:
3164 case AArch64::LD1SH_D_IMM:
3165 case AArch64::LD1SH_S_IMM:
3166 case AArch64::LD1SW_D_IMM:
3167 case AArch64::LD1W_D_IMM:
3168 case AArch64::LD1W_IMM:
3169 case AArch64::LD2B_IMM:
3170 case AArch64::LD2D_IMM:
3171 case AArch64::LD2H_IMM:
3172 case AArch64::LD2W_IMM:
3173 case AArch64::LD3B_IMM:
3174 case AArch64::LD3D_IMM:
3175 case AArch64::LD3H_IMM:
3176 case AArch64::LD3W_IMM:
3177 case AArch64::LD4B_IMM:
3178 case AArch64::LD4D_IMM:
3179 case AArch64::LD4H_IMM:
3180 case AArch64::LD4W_IMM:
3181 case AArch64::LDG:
3182 case AArch64::LDNF1B_D_IMM:
3183 case AArch64::LDNF1B_H_IMM:
3184 case AArch64::LDNF1B_IMM:
3185 case AArch64::LDNF1B_S_IMM:
3186 case AArch64::LDNF1D_IMM:
3187 case AArch64::LDNF1H_D_IMM:
3188 case AArch64::LDNF1H_IMM:
3189 case AArch64::LDNF1H_S_IMM:
3190 case AArch64::LDNF1SB_D_IMM:
3191 case AArch64::LDNF1SB_H_IMM:
3192 case AArch64::LDNF1SB_S_IMM:
3193 case AArch64::LDNF1SH_D_IMM:
3194 case AArch64::LDNF1SH_S_IMM:
3195 case AArch64::LDNF1SW_D_IMM:
3196 case AArch64::LDNF1W_D_IMM:
3197 case AArch64::LDNF1W_IMM:
3198 case AArch64::LDNPDi:
3199 case AArch64::LDNPQi:
3200 case AArch64::LDNPSi:
3201 case AArch64::LDNPWi:
3202 case AArch64::LDNPXi:
3203 case AArch64::LDNT1B_ZRI:
3204 case AArch64::LDNT1D_ZRI:
3205 case AArch64::LDNT1H_ZRI:
3206 case AArch64::LDNT1W_ZRI:
3207 case AArch64::LDPDi:
3208 case AArch64::LDPQi:
3209 case AArch64::LDPSi:
3210 case AArch64::LDPWi:
3211 case AArch64::LDPXi:
3212 case AArch64::LDRBBpost:
3213 case AArch64::LDRBBpre:
3214 case AArch64::LDRBpost:
3215 case AArch64::LDRBpre:
3216 case AArch64::LDRDpost:
3217 case AArch64::LDRDpre:
3218 case AArch64::LDRHHpost:
3219 case AArch64::LDRHHpre:
3220 case AArch64::LDRHpost:
3221 case AArch64::LDRHpre:
3222 case AArch64::LDRQpost:
3223 case AArch64::LDRQpre:
3224 case AArch64::LDRSpost:
3225 case AArch64::LDRSpre:
3226 case AArch64::LDRWpost:
3227 case AArch64::LDRWpre:
3228 case AArch64::LDRXpost:
3229 case AArch64::LDRXpre:
3230 case AArch64::ST1B_D_IMM:
3231 case AArch64::ST1B_H_IMM:
3232 case AArch64::ST1B_IMM:
3233 case AArch64::ST1B_S_IMM:
3234 case AArch64::ST1D_IMM:
3235 case AArch64::ST1H_D_IMM:
3236 case AArch64::ST1H_IMM:
3237 case AArch64::ST1H_S_IMM:
3238 case AArch64::ST1W_D_IMM:
3239 case AArch64::ST1W_IMM:
3240 case AArch64::ST2B_IMM:
3241 case AArch64::ST2D_IMM:
3242 case AArch64::ST2H_IMM:
3243 case AArch64::ST2W_IMM:
3244 case AArch64::ST3B_IMM:
3245 case AArch64::ST3D_IMM:
3246 case AArch64::ST3H_IMM:
3247 case AArch64::ST3W_IMM:
3248 case AArch64::ST4B_IMM:
3249 case AArch64::ST4D_IMM:
3250 case AArch64::ST4H_IMM:
3251 case AArch64::ST4W_IMM:
3252 case AArch64::STGPi:
3253 case AArch64::STGPreIndex:
3254 case AArch64::STZGPreIndex:
3255 case AArch64::ST2GPreIndex:
3256 case AArch64::STZ2GPreIndex:
3257 case AArch64::STGPostIndex:
3258 case AArch64::STZGPostIndex:
3259 case AArch64::ST2GPostIndex:
3260 case AArch64::STZ2GPostIndex:
3261 case AArch64::STNPDi:
3262 case AArch64::STNPQi:
3263 case AArch64::STNPSi:
3264 case AArch64::STNPWi:
3265 case AArch64::STNPXi:
3266 case AArch64::STNT1B_ZRI:
3267 case AArch64::STNT1D_ZRI:
3268 case AArch64::STNT1H_ZRI:
3269 case AArch64::STNT1W_ZRI:
3270 case AArch64::STPDi:
3271 case AArch64::STPQi:
3272 case AArch64::STPSi:
3273 case AArch64::STPWi:
3274 case AArch64::STPXi:
3275 case AArch64::STRBBpost:
3276 case AArch64::STRBBpre:
3277 case AArch64::STRBpost:
3278 case AArch64::STRBpre:
3279 case AArch64::STRDpost:
3280 case AArch64::STRDpre:
3281 case AArch64::STRHHpost:
3282 case AArch64::STRHHpre:
3283 case AArch64::STRHpost:
3284 case AArch64::STRHpre:
3285 case AArch64::STRQpost:
3286 case AArch64::STRQpre:
3287 case AArch64::STRSpost:
3288 case AArch64::STRSpre:
3289 case AArch64::STRWpost:
3290 case AArch64::STRWpre:
3291 case AArch64::STRXpost:
3292 case AArch64::STRXpre:
3293 case AArch64::LD1B_2Z_IMM:
3294 case AArch64::LD1B_2Z_STRIDED_IMM:
3295 case AArch64::LD1H_2Z_IMM:
3296 case AArch64::LD1H_2Z_STRIDED_IMM:
3297 case AArch64::LD1W_2Z_IMM:
3298 case AArch64::LD1W_2Z_STRIDED_IMM:
3299 case AArch64::LD1D_2Z_IMM:
3300 case AArch64::LD1D_2Z_STRIDED_IMM:
3301 case AArch64::LD1B_4Z_IMM:
3302 case AArch64::LD1B_4Z_STRIDED_IMM:
3303 case AArch64::LD1H_4Z_IMM:
3304 case AArch64::LD1H_4Z_STRIDED_IMM:
3305 case AArch64::LD1W_4Z_IMM:
3306 case AArch64::LD1W_4Z_STRIDED_IMM:
3307 case AArch64::LD1D_4Z_IMM:
3308 case AArch64::LD1D_4Z_STRIDED_IMM:
3309 case AArch64::LD1B_2Z_IMM_PSEUDO:
3310 case AArch64::LD1H_2Z_IMM_PSEUDO:
3311 case AArch64::LD1W_2Z_IMM_PSEUDO:
3312 case AArch64::LD1D_2Z_IMM_PSEUDO:
3313 case AArch64::LD1B_4Z_IMM_PSEUDO:
3314 case AArch64::LD1H_4Z_IMM_PSEUDO:
3315 case AArch64::LD1W_4Z_IMM_PSEUDO:
3316 case AArch64::LD1D_4Z_IMM_PSEUDO:
3317 case AArch64::ST1B_2Z_IMM:
3318 case AArch64::ST1B_2Z_STRIDED_IMM:
3319 case AArch64::ST1H_2Z_IMM:
3320 case AArch64::ST1H_2Z_STRIDED_IMM:
3321 case AArch64::ST1W_2Z_IMM:
3322 case AArch64::ST1W_2Z_STRIDED_IMM:
3323 case AArch64::ST1D_2Z_IMM:
3324 case AArch64::ST1D_2Z_STRIDED_IMM:
3325 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
3326 case AArch64::LDNT1B_2Z_IMM:
3327 case AArch64::LDNT1B_2Z_STRIDED_IMM:
3328 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
3329 case AArch64::LDNT1H_2Z_IMM:
3330 case AArch64::LDNT1H_2Z_STRIDED_IMM:
3331 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
3332 case AArch64::LDNT1W_2Z_IMM:
3333 case AArch64::LDNT1W_2Z_STRIDED_IMM:
3334 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
3335 case AArch64::LDNT1D_2Z_IMM:
3336 case AArch64::LDNT1D_2Z_STRIDED_IMM:
3337 case AArch64::STNT1B_2Z_IMM:
3338 case AArch64::STNT1B_2Z_STRIDED_IMM:
3339 case AArch64::STNT1H_2Z_IMM:
3340 case AArch64::STNT1H_2Z_STRIDED_IMM:
3341 case AArch64::STNT1W_2Z_IMM:
3342 case AArch64::STNT1W_2Z_STRIDED_IMM:
3343 case AArch64::STNT1D_2Z_IMM:
3344 case AArch64::STNT1D_2Z_STRIDED_IMM:
3345 case AArch64::ST1B_2Z_IMM_PSEUDO:
3346 case AArch64::ST1H_2Z_IMM_PSEUDO:
3347 case AArch64::ST1W_2Z_IMM_PSEUDO:
3348 case AArch64::ST1D_2Z_IMM_PSEUDO:
3349 case AArch64::STNT1B_2Z_IMM_PSEUDO:
3350 case AArch64::STNT1H_2Z_IMM_PSEUDO:
3351 case AArch64::STNT1W_2Z_IMM_PSEUDO:
3352 case AArch64::STNT1D_2Z_IMM_PSEUDO:
3353 case AArch64::ST1B_4Z_IMM:
3354 case AArch64::ST1B_4Z_STRIDED_IMM:
3355 case AArch64::ST1H_4Z_IMM:
3356 case AArch64::ST1H_4Z_STRIDED_IMM:
3357 case AArch64::ST1W_4Z_IMM:
3358 case AArch64::ST1W_4Z_STRIDED_IMM:
3359 case AArch64::ST1D_4Z_IMM:
3360 case AArch64::ST1D_4Z_STRIDED_IMM:
3361 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
3362 case AArch64::LDNT1B_4Z_IMM:
3363 case AArch64::LDNT1B_4Z_STRIDED_IMM:
3364 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
3365 case AArch64::LDNT1H_4Z_IMM:
3366 case AArch64::LDNT1H_4Z_STRIDED_IMM:
3367 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
3368 case AArch64::LDNT1W_4Z_IMM:
3369 case AArch64::LDNT1W_4Z_STRIDED_IMM:
3370 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
3371 case AArch64::LDNT1D_4Z_IMM:
3372 case AArch64::LDNT1D_4Z_STRIDED_IMM:
3373 case AArch64::STNT1B_4Z_IMM:
3374 case AArch64::STNT1B_4Z_STRIDED_IMM:
3375 case AArch64::STNT1H_4Z_IMM:
3376 case AArch64::STNT1H_4Z_STRIDED_IMM:
3377 case AArch64::STNT1W_4Z_IMM:
3378 case AArch64::STNT1W_4Z_STRIDED_IMM:
3379 case AArch64::STNT1D_4Z_IMM:
3380 case AArch64::STNT1D_4Z_STRIDED_IMM:
3381 case AArch64::ST1B_4Z_IMM_PSEUDO:
3382 case AArch64::ST1H_4Z_IMM_PSEUDO:
3383 case AArch64::ST1W_4Z_IMM_PSEUDO:
3384 case AArch64::ST1D_4Z_IMM_PSEUDO:
3385 case AArch64::STNT1B_4Z_IMM_PSEUDO:
3386 case AArch64::STNT1H_4Z_IMM_PSEUDO:
3387 case AArch64::STNT1W_4Z_IMM_PSEUDO:
3388 case AArch64::STNT1D_4Z_IMM_PSEUDO:
3389 return 3;
3390 case AArch64::LDPDpost:
3391 case AArch64::LDPDpre:
3392 case AArch64::LDPQpost:
3393 case AArch64::LDPQpre:
3394 case AArch64::LDPSpost:
3395 case AArch64::LDPSpre:
3396 case AArch64::LDPWpost:
3397 case AArch64::LDPWpre:
3398 case AArch64::LDPXpost:
3399 case AArch64::LDPXpre:
3400 case AArch64::STGPpre:
3401 case AArch64::STGPpost:
3402 case AArch64::STPDpost:
3403 case AArch64::STPDpre:
3404 case AArch64::STPQpost:
3405 case AArch64::STPQpre:
3406 case AArch64::STPSpost:
3407 case AArch64::STPSpre:
3408 case AArch64::STPWpost:
3409 case AArch64::STPWpre:
3410 case AArch64::STPXpost:
3411 case AArch64::STPXpre:
3412 return 4;
3413 }
3414}
3415
3416bool AArch64InstrInfo::isPairableLdStInst(const MachineInstr &MI) {
3417 switch (MI.getOpcode()) {
3418 default:
3419 return false;
3420 // Scaled instructions.
3421 case AArch64::STRSui:
3422 case AArch64::STRDui:
3423 case AArch64::STRQui:
3424 case AArch64::STRXui:
3425 case AArch64::STRWui:
3426 case AArch64::LDRSui:
3427 case AArch64::LDRDui:
3428 case AArch64::LDRQui:
3429 case AArch64::LDRXui:
3430 case AArch64::LDRWui:
3431 case AArch64::LDRSWui:
3432 // Unscaled instructions.
3433 case AArch64::STURSi:
3434 case AArch64::STRSpre:
3435 case AArch64::STURDi:
3436 case AArch64::STRDpre:
3437 case AArch64::STURQi:
3438 case AArch64::STRQpre:
3439 case AArch64::STURWi:
3440 case AArch64::STRWpre:
3441 case AArch64::STURXi:
3442 case AArch64::STRXpre:
3443 case AArch64::LDURSi:
3444 case AArch64::LDRSpre:
3445 case AArch64::LDURDi:
3446 case AArch64::LDRDpre:
3447 case AArch64::LDURQi:
3448 case AArch64::LDRQpre:
3449 case AArch64::LDURWi:
3450 case AArch64::LDRWpre:
3451 case AArch64::LDURXi:
3452 case AArch64::LDRXpre:
3453 case AArch64::LDURSWi:
3454 case AArch64::LDRSWpre:
3455 // SVE instructions.
3456 case AArch64::LDR_ZXI:
3457 case AArch64::STR_ZXI:
3458 return true;
3459 }
3460}
3461
3462bool AArch64InstrInfo::isTailCallReturnInst(const MachineInstr &MI) {
3463 switch (MI.getOpcode()) {
3464 default:
3465 assert((!MI.isCall() || !MI.isReturn()) &&
3466 "Unexpected instruction - was a new tail call opcode introduced?");
3467 return false;
3468 case AArch64::TCRETURNdi:
3469 case AArch64::TCRETURNri:
3470 case AArch64::TCRETURNrix16x17:
3471 case AArch64::TCRETURNrix17:
3472 case AArch64::TCRETURNrinotx16:
3473 case AArch64::TCRETURNriALL:
3474 case AArch64::AUTH_TCRETURN:
3475 case AArch64::AUTH_TCRETURN_BTI:
3476 return true;
3477 }
3478}
3479
3480unsigned AArch64InstrInfo::convertToFlagSettingOpc(unsigned Opc) {
3481 switch (Opc) {
3482 default:
3483 llvm_unreachable("Opcode has no flag setting equivalent!");
3484 // 32-bit cases:
3485 case AArch64::ADDWri:
3486 return AArch64::ADDSWri;
3487 case AArch64::ADDWrr:
3488 return AArch64::ADDSWrr;
3489 case AArch64::ADDWrs:
3490 return AArch64::ADDSWrs;
3491 case AArch64::ADDWrx:
3492 return AArch64::ADDSWrx;
3493 case AArch64::ANDWri:
3494 return AArch64::ANDSWri;
3495 case AArch64::ANDWrr:
3496 return AArch64::ANDSWrr;
3497 case AArch64::ANDWrs:
3498 return AArch64::ANDSWrs;
3499 case AArch64::BICWrr:
3500 return AArch64::BICSWrr;
3501 case AArch64::BICWrs:
3502 return AArch64::BICSWrs;
3503 case AArch64::SUBWri:
3504 return AArch64::SUBSWri;
3505 case AArch64::SUBWrr:
3506 return AArch64::SUBSWrr;
3507 case AArch64::SUBWrs:
3508 return AArch64::SUBSWrs;
3509 case AArch64::SUBWrx:
3510 return AArch64::SUBSWrx;
3511 // 64-bit cases:
3512 case AArch64::ADDXri:
3513 return AArch64::ADDSXri;
3514 case AArch64::ADDXrr:
3515 return AArch64::ADDSXrr;
3516 case AArch64::ADDXrs:
3517 return AArch64::ADDSXrs;
3518 case AArch64::ADDXrx:
3519 return AArch64::ADDSXrx;
3520 case AArch64::ANDXri:
3521 return AArch64::ANDSXri;
3522 case AArch64::ANDXrr:
3523 return AArch64::ANDSXrr;
3524 case AArch64::ANDXrs:
3525 return AArch64::ANDSXrs;
3526 case AArch64::BICXrr:
3527 return AArch64::BICSXrr;
3528 case AArch64::BICXrs:
3529 return AArch64::BICSXrs;
3530 case AArch64::SUBXri:
3531 return AArch64::SUBSXri;
3532 case AArch64::SUBXrr:
3533 return AArch64::SUBSXrr;
3534 case AArch64::SUBXrs:
3535 return AArch64::SUBSXrs;
3536 case AArch64::SUBXrx:
3537 return AArch64::SUBSXrx;
3538 // SVE instructions:
3539 case AArch64::AND_PPzPP:
3540 return AArch64::ANDS_PPzPP;
3541 case AArch64::BIC_PPzPP:
3542 return AArch64::BICS_PPzPP;
3543 case AArch64::EOR_PPzPP:
3544 return AArch64::EORS_PPzPP;
3545 case AArch64::NAND_PPzPP:
3546 return AArch64::NANDS_PPzPP;
3547 case AArch64::NOR_PPzPP:
3548 return AArch64::NORS_PPzPP;
3549 case AArch64::ORN_PPzPP:
3550 return AArch64::ORNS_PPzPP;
3551 case AArch64::ORR_PPzPP:
3552 return AArch64::ORRS_PPzPP;
3553 case AArch64::BRKA_PPzP:
3554 return AArch64::BRKAS_PPzP;
3555 case AArch64::BRKPA_PPzPP:
3556 return AArch64::BRKPAS_PPzPP;
3557 case AArch64::BRKB_PPzP:
3558 return AArch64::BRKBS_PPzP;
3559 case AArch64::BRKPB_PPzPP:
3560 return AArch64::BRKPBS_PPzPP;
3561 case AArch64::BRKN_PPzP:
3562 return AArch64::BRKNS_PPzP;
3563 case AArch64::RDFFR_PPz:
3564 return AArch64::RDFFRS_PPz;
3565 case AArch64::PTRUE_B:
3566 return AArch64::PTRUES_B;
3567 }
3568}
3569
3570// Is this a candidate for ld/st merging or pairing? For example, we don't
3571// touch volatiles or load/stores that have a hint to avoid pair formation.
3572bool AArch64InstrInfo::isCandidateToMergeOrPair(const MachineInstr &MI) const {
3573
3574 bool IsPreLdSt = isPreLdSt(MI);
3575
3576 // If this is a volatile load/store, don't mess with it.
3577 if (MI.hasOrderedMemoryRef())
3578 return false;
3579
3580 // Make sure this is a reg/fi+imm (as opposed to an address reloc).
3581 // For Pre-inc LD/ST, the operand is shifted by one.
3582 assert((MI.getOperand(IsPreLdSt ? 2 : 1).isReg() ||
3583 MI.getOperand(IsPreLdSt ? 2 : 1).isFI()) &&
3584 "Expected a reg or frame index operand.");
3585
3586 // For Pre-indexed addressing quadword instructions, the third operand is the
3587 // immediate value.
3588 bool IsImmPreLdSt = IsPreLdSt && MI.getOperand(i: 3).isImm();
3589
3590 if (!MI.getOperand(i: 2).isImm() && !IsImmPreLdSt)
3591 return false;
3592
3593 // Can't merge/pair if the instruction modifies the base register.
3594 // e.g., ldr x0, [x0]
3595 // This case will never occur with an FI base.
3596 // However, if the instruction is an LDR<S,D,Q,W,X,SW>pre or
3597 // STR<S,D,Q,W,X>pre, it can be merged.
3598 // For example:
3599 // ldr q0, [x11, #32]!
3600 // ldr q1, [x11, #16]
3601 // to
3602 // ldp q0, q1, [x11, #32]!
3603 if (MI.getOperand(i: 1).isReg() && !IsPreLdSt) {
3604 Register BaseReg = MI.getOperand(i: 1).getReg();
3605 const TargetRegisterInfo *TRI = &getRegisterInfo();
3606 if (MI.modifiesRegister(Reg: BaseReg, TRI))
3607 return false;
3608 }
3609
3610 // Pairing SVE fills/spills is only valid for little-endian targets that
3611 // implement VLS 128.
3612 switch (MI.getOpcode()) {
3613 default:
3614 break;
3615 case AArch64::LDR_ZXI:
3616 case AArch64::STR_ZXI:
3617 if (!Subtarget.isLittleEndian() ||
3618 Subtarget.getSVEVectorSizeInBits() != 128)
3619 return false;
3620 }
3621
3622 // Check if this load/store has a hint to avoid pair formation.
3623 // MachineMemOperands hints are set by the AArch64StorePairSuppress pass.
3624 if (isLdStPairSuppressed(MI))
3625 return false;
3626
3627 // Do not pair any callee-save store/reload instructions in the
3628 // prologue/epilogue if the CFI information encoded the operations as separate
3629 // instructions, as that will cause the size of the actual prologue to mismatch
3630 // with the prologue size recorded in the Windows CFI.
3631 const MCAsmInfo &MAI = MI.getMF()->getTarget().getMCAsmInfo();
3632 bool NeedsWinCFI =
3633 MAI.usesWindowsCFI() && MI.getMF()->getFunction().needsUnwindTableEntry();
3634 if (NeedsWinCFI && (MI.getFlag(Flag: MachineInstr::FrameSetup) ||
3635 MI.getFlag(Flag: MachineInstr::FrameDestroy)))
3636 return false;
3637
3638 // On some CPUs quad load/store pairs are slower than two single load/stores.
3639 if (Subtarget.isPaired128Slow()) {
3640 switch (MI.getOpcode()) {
3641 default:
3642 break;
3643 case AArch64::LDURQi:
3644 case AArch64::STURQi:
3645 case AArch64::LDRQui:
3646 case AArch64::STRQui:
3647 return false;
3648 }
3649 }
3650
3651 return true;
3652}
3653
3654bool AArch64InstrInfo::getMemOperandsWithOffsetWidth(
3655 const MachineInstr &LdSt, SmallVectorImpl<const MachineOperand *> &BaseOps,
3656 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width) const {
3657 if (!LdSt.mayLoadOrStore())
3658 return false;
3659
3660 const MachineOperand *BaseOp;
3661 TypeSize WidthN(0, false);
3662 if (!getMemOperandWithOffsetWidth(MI: LdSt, BaseOp, Offset, OffsetIsScalable,
3663 Width&: WidthN))
3664 return false;
3665 // The maximum vscale is 16 under AArch64, return the maximal extent for the
3666 // vector.
3667 Width = LocationSize::precise(Value: WidthN);
3668 BaseOps.push_back(Elt: BaseOp);
3669 return true;
3670}
3671
3672std::optional<ExtAddrMode>
3673AArch64InstrInfo::getAddrModeFromMemoryOp(const MachineInstr &MemI) const {
3674 const MachineOperand *Base; // Filled with the base operand of MI.
3675 int64_t Offset; // Filled with the offset of MI.
3676 bool OffsetIsScalable;
3677 if (!getMemOperandWithOffset(MI: MemI, BaseOp&: Base, Offset, OffsetIsScalable))
3678 return std::nullopt;
3679
3680 if (!Base->isReg())
3681 return std::nullopt;
3682 ExtAddrMode AM;
3683 AM.BaseReg = Base->getReg();
3684 AM.Displacement = Offset;
3685 AM.ScaledReg = 0;
3686 AM.Scale = 0;
3687 return AM;
3688}
3689
3690bool AArch64InstrInfo::canFoldIntoAddrMode(const MachineInstr &MemI,
3691 Register Reg,
3692 const MachineInstr &AddrI,
3693 ExtAddrMode &AM) const {
3694 // Filter out instructions into which we cannot fold.
3695 unsigned NumBytes;
3696 int64_t OffsetScale = 1;
3697 switch (MemI.getOpcode()) {
3698 default:
3699 return false;
3700
3701 case AArch64::LDURQi:
3702 case AArch64::STURQi:
3703 NumBytes = 16;
3704 break;
3705
3706 case AArch64::LDURDi:
3707 case AArch64::STURDi:
3708 case AArch64::LDURXi:
3709 case AArch64::STURXi:
3710 NumBytes = 8;
3711 break;
3712
3713 case AArch64::LDURWi:
3714 case AArch64::LDURSWi:
3715 case AArch64::STURWi:
3716 NumBytes = 4;
3717 break;
3718
3719 case AArch64::LDURHi:
3720 case AArch64::STURHi:
3721 case AArch64::LDURHHi:
3722 case AArch64::STURHHi:
3723 case AArch64::LDURSHXi:
3724 case AArch64::LDURSHWi:
3725 NumBytes = 2;
3726 break;
3727
3728 case AArch64::LDRBroX:
3729 case AArch64::LDRBBroX:
3730 case AArch64::LDRSBXroX:
3731 case AArch64::LDRSBWroX:
3732 case AArch64::STRBroX:
3733 case AArch64::STRBBroX:
3734 case AArch64::LDURBi:
3735 case AArch64::LDURBBi:
3736 case AArch64::LDURSBXi:
3737 case AArch64::LDURSBWi:
3738 case AArch64::STURBi:
3739 case AArch64::STURBBi:
3740 case AArch64::LDRBui:
3741 case AArch64::LDRBBui:
3742 case AArch64::LDRSBXui:
3743 case AArch64::LDRSBWui:
3744 case AArch64::STRBui:
3745 case AArch64::STRBBui:
3746 NumBytes = 1;
3747 break;
3748
3749 case AArch64::LDRQroX:
3750 case AArch64::STRQroX:
3751 case AArch64::LDRQui:
3752 case AArch64::STRQui:
3753 NumBytes = 16;
3754 OffsetScale = 16;
3755 break;
3756
3757 case AArch64::LDRDroX:
3758 case AArch64::STRDroX:
3759 case AArch64::LDRXroX:
3760 case AArch64::STRXroX:
3761 case AArch64::LDRDui:
3762 case AArch64::STRDui:
3763 case AArch64::LDRXui:
3764 case AArch64::STRXui:
3765 NumBytes = 8;
3766 OffsetScale = 8;
3767 break;
3768
3769 case AArch64::LDRWroX:
3770 case AArch64::LDRSWroX:
3771 case AArch64::STRWroX:
3772 case AArch64::LDRWui:
3773 case AArch64::LDRSWui:
3774 case AArch64::STRWui:
3775 NumBytes = 4;
3776 OffsetScale = 4;
3777 break;
3778
3779 case AArch64::LDRHroX:
3780 case AArch64::STRHroX:
3781 case AArch64::LDRHHroX:
3782 case AArch64::STRHHroX:
3783 case AArch64::LDRSHXroX:
3784 case AArch64::LDRSHWroX:
3785 case AArch64::LDRHui:
3786 case AArch64::STRHui:
3787 case AArch64::LDRHHui:
3788 case AArch64::STRHHui:
3789 case AArch64::LDRSHXui:
3790 case AArch64::LDRSHWui:
3791 NumBytes = 2;
3792 OffsetScale = 2;
3793 break;
3794 }
3795
3796 // Check the fold operand is not the loaded/stored value.
3797 const MachineOperand &BaseRegOp = MemI.getOperand(i: 0);
3798 if (BaseRegOp.isReg() && BaseRegOp.getReg() == Reg)
3799 return false;
3800
3801 // Handle memory instructions with a [Reg, Reg] addressing mode.
3802 if (MemI.getOperand(i: 2).isReg()) {
3803 // Bail if the addressing mode already includes extension of the offset
3804 // register.
3805 if (MemI.getOperand(i: 3).getImm())
3806 return false;
3807
3808 // Check if we actually have a scaled offset.
3809 if (MemI.getOperand(i: 4).getImm() == 0)
3810 OffsetScale = 1;
3811
3812 // If the address instructions is folded into the base register, then the
3813 // addressing mode must not have a scale. Then we can swap the base and the
3814 // scaled registers.
3815 if (MemI.getOperand(i: 1).getReg() == Reg && OffsetScale != 1)
3816 return false;
3817
3818 switch (AddrI.getOpcode()) {
3819 default:
3820 return false;
3821
3822 case AArch64::SBFMXri:
3823 // sxtw Xa, Wm
3824 // ldr Xd, [Xn, Xa, lsl #N]
3825 // ->
3826 // ldr Xd, [Xn, Wm, sxtw #N]
3827 if (AddrI.getOperand(i: 2).getImm() != 0 ||
3828 AddrI.getOperand(i: 3).getImm() != 31)
3829 return false;
3830
3831 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3832 if (AM.BaseReg == Reg)
3833 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3834 AM.ScaledReg = AddrI.getOperand(i: 1).getReg();
3835 AM.Scale = OffsetScale;
3836 AM.Displacement = 0;
3837 AM.Form = ExtAddrMode::Formula::SExtScaledReg;
3838 return true;
3839
3840 case TargetOpcode::SUBREG_TO_REG: {
3841 // mov Wa, Wm
3842 // ldr Xd, [Xn, Xa, lsl #N]
3843 // ->
3844 // ldr Xd, [Xn, Wm, uxtw #N]
3845
3846 // Zero-extension looks like an ORRWrs followed by a SUBREG_TO_REG.
3847 if (AddrI.getOperand(i: 2).getImm() != AArch64::sub_32)
3848 return false;
3849
3850 const MachineRegisterInfo &MRI = AddrI.getMF()->getRegInfo();
3851 Register OffsetReg = AddrI.getOperand(i: 1).getReg();
3852 if (!OffsetReg.isVirtual() || !MRI.hasOneNonDBGUse(RegNo: OffsetReg))
3853 return false;
3854
3855 const MachineInstr &DefMI = *MRI.getVRegDef(Reg: OffsetReg);
3856 if (DefMI.getOpcode() != AArch64::ORRWrs ||
3857 DefMI.getOperand(i: 1).getReg() != AArch64::WZR ||
3858 DefMI.getOperand(i: 3).getImm() != 0)
3859 return false;
3860
3861 AM.BaseReg = MemI.getOperand(i: 1).getReg();
3862 if (AM.BaseReg == Reg)
3863 AM.BaseReg = MemI.getOperand(i: 2).getReg();
3864 AM.ScaledReg = DefMI.getOperand(i: 2).getReg();
3865 AM.Scale = OffsetScale;
3866 AM.Displacement = 0;
3867 AM.Form = ExtAddrMode::Formula::ZExtScaledReg;
3868 return true;
3869 }
3870 }
3871 }
3872
3873 // Handle memory instructions with a [Reg, #Imm] addressing mode.
3874
3875 // Check we are not breaking a potential conversion to an LDP.
3876 auto validateOffsetForLDP = [](unsigned NumBytes, int64_t OldOffset,
3877 int64_t NewOffset) -> bool {
3878 int64_t MinOffset, MaxOffset;
3879 switch (NumBytes) {
3880 default:
3881 return true;
3882 case 4:
3883 MinOffset = -256;
3884 MaxOffset = 252;
3885 break;
3886 case 8:
3887 MinOffset = -512;
3888 MaxOffset = 504;
3889 break;
3890 case 16:
3891 MinOffset = -1024;
3892 MaxOffset = 1008;
3893 break;
3894 }
3895 return OldOffset < MinOffset || OldOffset > MaxOffset ||
3896 (NewOffset >= MinOffset && NewOffset <= MaxOffset);
3897 };
3898 auto canFoldAddSubImmIntoAddrMode = [&](int64_t Disp) -> bool {
3899 int64_t OldOffset = MemI.getOperand(i: 2).getImm() * OffsetScale;
3900 int64_t NewOffset = OldOffset + Disp;
3901 if (!isLegalAddressingMode(NumBytes, Offset: NewOffset, /* Scale */ 0))
3902 return false;
3903 // If the old offset would fit into an LDP, but the new offset wouldn't,
3904 // bail out.
3905 if (!validateOffsetForLDP(NumBytes, OldOffset, NewOffset))
3906 return false;
3907 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3908 AM.ScaledReg = 0;
3909 AM.Scale = 0;
3910 AM.Displacement = NewOffset;
3911 AM.Form = ExtAddrMode::Formula::Basic;
3912 return true;
3913 };
3914
3915 auto canFoldAddRegIntoAddrMode =
3916 [&](int64_t Scale,
3917 ExtAddrMode::Formula Form = ExtAddrMode::Formula::Basic) -> bool {
3918 if (MemI.getOperand(i: 2).getImm() != 0)
3919 return false;
3920 if ((unsigned)Scale != Scale)
3921 return false;
3922 if (!isLegalAddressingMode(NumBytes, /* Offset */ 0, Scale))
3923 return false;
3924 AM.BaseReg = AddrI.getOperand(i: 1).getReg();
3925 AM.ScaledReg = AddrI.getOperand(i: 2).getReg();
3926 AM.Scale = Scale;
3927 AM.Displacement = 0;
3928 AM.Form = Form;
3929 return true;
3930 };
3931
3932 auto avoidSlowSTRQ = [&](const MachineInstr &MemI) {
3933 unsigned Opcode = MemI.getOpcode();
3934 return (Opcode == AArch64::STURQi || Opcode == AArch64::STRQui) &&
3935 Subtarget.isSTRQroSlow();
3936 };
3937
3938 int64_t Disp = 0;
3939 const bool OptSize = MemI.getMF()->getFunction().hasOptSize();
3940 switch (AddrI.getOpcode()) {
3941 default:
3942 return false;
3943
3944 case AArch64::ADDXri:
3945 // add Xa, Xn, #N
3946 // ldr Xd, [Xa, #M]
3947 // ->
3948 // ldr Xd, [Xn, #N'+M]
3949 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3950 return canFoldAddSubImmIntoAddrMode(Disp);
3951
3952 case AArch64::SUBXri:
3953 // sub Xa, Xn, #N
3954 // ldr Xd, [Xa, #M]
3955 // ->
3956 // ldr Xd, [Xn, #N'+M]
3957 Disp = AddrI.getOperand(i: 2).getImm() << AddrI.getOperand(i: 3).getImm();
3958 return canFoldAddSubImmIntoAddrMode(-Disp);
3959
3960 case AArch64::ADDXrs: {
3961 // add Xa, Xn, Xm, lsl #N
3962 // ldr Xd, [Xa]
3963 // ->
3964 // ldr Xd, [Xn, Xm, lsl #N]
3965
3966 // Don't fold the add if the result would be slower, unless optimising for
3967 // size.
3968 unsigned Shift = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
3969 if (AArch64_AM::getShiftType(Imm: Shift) != AArch64_AM::ShiftExtendType::LSL)
3970 return false;
3971 Shift = AArch64_AM::getShiftValue(Imm: Shift);
3972 if (!OptSize) {
3973 if (Shift != 2 && Shift != 3 && Subtarget.hasAddrLSLSlow14())
3974 return false;
3975 if (avoidSlowSTRQ(MemI))
3976 return false;
3977 }
3978 return canFoldAddRegIntoAddrMode(1ULL << Shift);
3979 }
3980
3981 case AArch64::ADDXrr:
3982 // add Xa, Xn, Xm
3983 // ldr Xd, [Xa]
3984 // ->
3985 // ldr Xd, [Xn, Xm, lsl #0]
3986
3987 // Don't fold the add if the result would be slower, unless optimising for
3988 // size.
3989 if (!OptSize && avoidSlowSTRQ(MemI))
3990 return false;
3991 return canFoldAddRegIntoAddrMode(1);
3992
3993 case AArch64::ADDXrx:
3994 // add Xa, Xn, Wm, {s,u}xtw #N
3995 // ldr Xd, [Xa]
3996 // ->
3997 // ldr Xd, [Xn, Wm, {s,u}xtw #N]
3998
3999 // Don't fold the add if the result would be slower, unless optimising for
4000 // size.
4001 if (!OptSize && avoidSlowSTRQ(MemI))
4002 return false;
4003
4004 // Can fold only sign-/zero-extend of a word.
4005 unsigned Imm = static_cast<unsigned>(AddrI.getOperand(i: 3).getImm());
4006 AArch64_AM::ShiftExtendType Extend = AArch64_AM::getArithExtendType(Imm);
4007 if (Extend != AArch64_AM::UXTW && Extend != AArch64_AM::SXTW)
4008 return false;
4009
4010 return canFoldAddRegIntoAddrMode(
4011 1ULL << AArch64_AM::getArithShiftValue(Imm),
4012 (Extend == AArch64_AM::SXTW) ? ExtAddrMode::Formula::SExtScaledReg
4013 : ExtAddrMode::Formula::ZExtScaledReg);
4014 }
4015}
4016
4017// Given an opcode for an instruction with a [Reg, #Imm] addressing mode,
4018// return the opcode of an instruction performing the same operation, but using
4019// the [Reg, Reg] addressing mode.
4020static unsigned regOffsetOpcode(unsigned Opcode) {
4021 switch (Opcode) {
4022 default:
4023 llvm_unreachable("Address folding not implemented for instruction");
4024
4025 case AArch64::LDURQi:
4026 case AArch64::LDRQui:
4027 return AArch64::LDRQroX;
4028 case AArch64::STURQi:
4029 case AArch64::STRQui:
4030 return AArch64::STRQroX;
4031 case AArch64::LDURDi:
4032 case AArch64::LDRDui:
4033 return AArch64::LDRDroX;
4034 case AArch64::STURDi:
4035 case AArch64::STRDui:
4036 return AArch64::STRDroX;
4037 case AArch64::LDURXi:
4038 case AArch64::LDRXui:
4039 return AArch64::LDRXroX;
4040 case AArch64::STURXi:
4041 case AArch64::STRXui:
4042 return AArch64::STRXroX;
4043 case AArch64::LDURWi:
4044 case AArch64::LDRWui:
4045 return AArch64::LDRWroX;
4046 case AArch64::LDURSWi:
4047 case AArch64::LDRSWui:
4048 return AArch64::LDRSWroX;
4049 case AArch64::STURWi:
4050 case AArch64::STRWui:
4051 return AArch64::STRWroX;
4052 case AArch64::LDURHi:
4053 case AArch64::LDRHui:
4054 return AArch64::LDRHroX;
4055 case AArch64::STURHi:
4056 case AArch64::STRHui:
4057 return AArch64::STRHroX;
4058 case AArch64::LDURHHi:
4059 case AArch64::LDRHHui:
4060 return AArch64::LDRHHroX;
4061 case AArch64::STURHHi:
4062 case AArch64::STRHHui:
4063 return AArch64::STRHHroX;
4064 case AArch64::LDURSHXi:
4065 case AArch64::LDRSHXui:
4066 return AArch64::LDRSHXroX;
4067 case AArch64::LDURSHWi:
4068 case AArch64::LDRSHWui:
4069 return AArch64::LDRSHWroX;
4070 case AArch64::LDURBi:
4071 case AArch64::LDRBui:
4072 return AArch64::LDRBroX;
4073 case AArch64::LDURBBi:
4074 case AArch64::LDRBBui:
4075 return AArch64::LDRBBroX;
4076 case AArch64::LDURSBXi:
4077 case AArch64::LDRSBXui:
4078 return AArch64::LDRSBXroX;
4079 case AArch64::LDURSBWi:
4080 case AArch64::LDRSBWui:
4081 return AArch64::LDRSBWroX;
4082 case AArch64::STURBi:
4083 case AArch64::STRBui:
4084 return AArch64::STRBroX;
4085 case AArch64::STURBBi:
4086 case AArch64::STRBBui:
4087 return AArch64::STRBBroX;
4088 }
4089}
4090
4091// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4092// the opcode of an instruction performing the same operation, but using the
4093// [Reg, #Imm] addressing mode with scaled offset.
4094unsigned scaledOffsetOpcode(unsigned Opcode, unsigned &Scale) {
4095 switch (Opcode) {
4096 default:
4097 llvm_unreachable("Address folding not implemented for instruction");
4098
4099 case AArch64::LDURQi:
4100 Scale = 16;
4101 return AArch64::LDRQui;
4102 case AArch64::STURQi:
4103 Scale = 16;
4104 return AArch64::STRQui;
4105 case AArch64::LDURDi:
4106 Scale = 8;
4107 return AArch64::LDRDui;
4108 case AArch64::STURDi:
4109 Scale = 8;
4110 return AArch64::STRDui;
4111 case AArch64::LDURXi:
4112 Scale = 8;
4113 return AArch64::LDRXui;
4114 case AArch64::STURXi:
4115 Scale = 8;
4116 return AArch64::STRXui;
4117 case AArch64::LDURWi:
4118 Scale = 4;
4119 return AArch64::LDRWui;
4120 case AArch64::LDURSWi:
4121 Scale = 4;
4122 return AArch64::LDRSWui;
4123 case AArch64::STURWi:
4124 Scale = 4;
4125 return AArch64::STRWui;
4126 case AArch64::LDURHi:
4127 Scale = 2;
4128 return AArch64::LDRHui;
4129 case AArch64::STURHi:
4130 Scale = 2;
4131 return AArch64::STRHui;
4132 case AArch64::LDURHHi:
4133 Scale = 2;
4134 return AArch64::LDRHHui;
4135 case AArch64::STURHHi:
4136 Scale = 2;
4137 return AArch64::STRHHui;
4138 case AArch64::LDURSHXi:
4139 Scale = 2;
4140 return AArch64::LDRSHXui;
4141 case AArch64::LDURSHWi:
4142 Scale = 2;
4143 return AArch64::LDRSHWui;
4144 case AArch64::LDURBi:
4145 Scale = 1;
4146 return AArch64::LDRBui;
4147 case AArch64::LDURBBi:
4148 Scale = 1;
4149 return AArch64::LDRBBui;
4150 case AArch64::LDURSBXi:
4151 Scale = 1;
4152 return AArch64::LDRSBXui;
4153 case AArch64::LDURSBWi:
4154 Scale = 1;
4155 return AArch64::LDRSBWui;
4156 case AArch64::STURBi:
4157 Scale = 1;
4158 return AArch64::STRBui;
4159 case AArch64::STURBBi:
4160 Scale = 1;
4161 return AArch64::STRBBui;
4162 case AArch64::LDRQui:
4163 case AArch64::STRQui:
4164 Scale = 16;
4165 return Opcode;
4166 case AArch64::LDRDui:
4167 case AArch64::STRDui:
4168 case AArch64::LDRXui:
4169 case AArch64::STRXui:
4170 Scale = 8;
4171 return Opcode;
4172 case AArch64::LDRWui:
4173 case AArch64::LDRSWui:
4174 case AArch64::STRWui:
4175 Scale = 4;
4176 return Opcode;
4177 case AArch64::LDRHui:
4178 case AArch64::STRHui:
4179 case AArch64::LDRHHui:
4180 case AArch64::STRHHui:
4181 case AArch64::LDRSHXui:
4182 case AArch64::LDRSHWui:
4183 Scale = 2;
4184 return Opcode;
4185 case AArch64::LDRBui:
4186 case AArch64::LDRBBui:
4187 case AArch64::LDRSBXui:
4188 case AArch64::LDRSBWui:
4189 case AArch64::STRBui:
4190 case AArch64::STRBBui:
4191 Scale = 1;
4192 return Opcode;
4193 }
4194}
4195
4196// Given an opcode for an instruction with a [Reg, #Imm] addressing mode, return
4197// the opcode of an instruction performing the same operation, but using the
4198// [Reg, #Imm] addressing mode with unscaled offset.
4199unsigned unscaledOffsetOpcode(unsigned Opcode) {
4200 switch (Opcode) {
4201 default:
4202 llvm_unreachable("Address folding not implemented for instruction");
4203
4204 case AArch64::LDURQi:
4205 case AArch64::STURQi:
4206 case AArch64::LDURDi:
4207 case AArch64::STURDi:
4208 case AArch64::LDURXi:
4209 case AArch64::STURXi:
4210 case AArch64::LDURWi:
4211 case AArch64::LDURSWi:
4212 case AArch64::STURWi:
4213 case AArch64::LDURHi:
4214 case AArch64::STURHi:
4215 case AArch64::LDURHHi:
4216 case AArch64::STURHHi:
4217 case AArch64::LDURSHXi:
4218 case AArch64::LDURSHWi:
4219 case AArch64::LDURBi:
4220 case AArch64::STURBi:
4221 case AArch64::LDURBBi:
4222 case AArch64::STURBBi:
4223 case AArch64::LDURSBWi:
4224 case AArch64::LDURSBXi:
4225 return Opcode;
4226 case AArch64::LDRQui:
4227 return AArch64::LDURQi;
4228 case AArch64::STRQui:
4229 return AArch64::STURQi;
4230 case AArch64::LDRDui:
4231 return AArch64::LDURDi;
4232 case AArch64::STRDui:
4233 return AArch64::STURDi;
4234 case AArch64::LDRXui:
4235 return AArch64::LDURXi;
4236 case AArch64::STRXui:
4237 return AArch64::STURXi;
4238 case AArch64::LDRWui:
4239 return AArch64::LDURWi;
4240 case AArch64::LDRSWui:
4241 return AArch64::LDURSWi;
4242 case AArch64::STRWui:
4243 return AArch64::STURWi;
4244 case AArch64::LDRHui:
4245 return AArch64::LDURHi;
4246 case AArch64::STRHui:
4247 return AArch64::STURHi;
4248 case AArch64::LDRHHui:
4249 return AArch64::LDURHHi;
4250 case AArch64::STRHHui:
4251 return AArch64::STURHHi;
4252 case AArch64::LDRSHXui:
4253 return AArch64::LDURSHXi;
4254 case AArch64::LDRSHWui:
4255 return AArch64::LDURSHWi;
4256 case AArch64::LDRBBui:
4257 return AArch64::LDURBBi;
4258 case AArch64::LDRBui:
4259 return AArch64::LDURBi;
4260 case AArch64::STRBBui:
4261 return AArch64::STURBBi;
4262 case AArch64::STRBui:
4263 return AArch64::STURBi;
4264 case AArch64::LDRSBWui:
4265 return AArch64::LDURSBWi;
4266 case AArch64::LDRSBXui:
4267 return AArch64::LDURSBXi;
4268 }
4269}
4270
4271// Given the opcode of a memory load/store instruction, return the opcode of an
4272// instruction performing the same operation, but using
4273// the [Reg, Reg, {s,u}xtw #N] addressing mode with sign-/zero-extend of the
4274// offset register.
4275static unsigned offsetExtendOpcode(unsigned Opcode) {
4276 switch (Opcode) {
4277 default:
4278 llvm_unreachable("Address folding not implemented for instruction");
4279
4280 case AArch64::LDRQroX:
4281 case AArch64::LDURQi:
4282 case AArch64::LDRQui:
4283 return AArch64::LDRQroW;
4284 case AArch64::STRQroX:
4285 case AArch64::STURQi:
4286 case AArch64::STRQui:
4287 return AArch64::STRQroW;
4288 case AArch64::LDRDroX:
4289 case AArch64::LDURDi:
4290 case AArch64::LDRDui:
4291 return AArch64::LDRDroW;
4292 case AArch64::STRDroX:
4293 case AArch64::STURDi:
4294 case AArch64::STRDui:
4295 return AArch64::STRDroW;
4296 case AArch64::LDRXroX:
4297 case AArch64::LDURXi:
4298 case AArch64::LDRXui:
4299 return AArch64::LDRXroW;
4300 case AArch64::STRXroX:
4301 case AArch64::STURXi:
4302 case AArch64::STRXui:
4303 return AArch64::STRXroW;
4304 case AArch64::LDRWroX:
4305 case AArch64::LDURWi:
4306 case AArch64::LDRWui:
4307 return AArch64::LDRWroW;
4308 case AArch64::LDRSWroX:
4309 case AArch64::LDURSWi:
4310 case AArch64::LDRSWui:
4311 return AArch64::LDRSWroW;
4312 case AArch64::STRWroX:
4313 case AArch64::STURWi:
4314 case AArch64::STRWui:
4315 return AArch64::STRWroW;
4316 case AArch64::LDRHroX:
4317 case AArch64::LDURHi:
4318 case AArch64::LDRHui:
4319 return AArch64::LDRHroW;
4320 case AArch64::STRHroX:
4321 case AArch64::STURHi:
4322 case AArch64::STRHui:
4323 return AArch64::STRHroW;
4324 case AArch64::LDRHHroX:
4325 case AArch64::LDURHHi:
4326 case AArch64::LDRHHui:
4327 return AArch64::LDRHHroW;
4328 case AArch64::STRHHroX:
4329 case AArch64::STURHHi:
4330 case AArch64::STRHHui:
4331 return AArch64::STRHHroW;
4332 case AArch64::LDRSHXroX:
4333 case AArch64::LDURSHXi:
4334 case AArch64::LDRSHXui:
4335 return AArch64::LDRSHXroW;
4336 case AArch64::LDRSHWroX:
4337 case AArch64::LDURSHWi:
4338 case AArch64::LDRSHWui:
4339 return AArch64::LDRSHWroW;
4340 case AArch64::LDRBroX:
4341 case AArch64::LDURBi:
4342 case AArch64::LDRBui:
4343 return AArch64::LDRBroW;
4344 case AArch64::LDRBBroX:
4345 case AArch64::LDURBBi:
4346 case AArch64::LDRBBui:
4347 return AArch64::LDRBBroW;
4348 case AArch64::LDRSBXroX:
4349 case AArch64::LDURSBXi:
4350 case AArch64::LDRSBXui:
4351 return AArch64::LDRSBXroW;
4352 case AArch64::LDRSBWroX:
4353 case AArch64::LDURSBWi:
4354 case AArch64::LDRSBWui:
4355 return AArch64::LDRSBWroW;
4356 case AArch64::STRBroX:
4357 case AArch64::STURBi:
4358 case AArch64::STRBui:
4359 return AArch64::STRBroW;
4360 case AArch64::STRBBroX:
4361 case AArch64::STURBBi:
4362 case AArch64::STRBBui:
4363 return AArch64::STRBBroW;
4364 }
4365}
4366
4367MachineInstr *AArch64InstrInfo::emitLdStWithAddr(MachineInstr &MemI,
4368 const ExtAddrMode &AM) const {
4369
4370 const DebugLoc &DL = MemI.getDebugLoc();
4371 MachineBasicBlock &MBB = *MemI.getParent();
4372 MachineRegisterInfo &MRI = MemI.getMF()->getRegInfo();
4373
4374 if (AM.Form == ExtAddrMode::Formula::Basic) {
4375 if (AM.ScaledReg) {
4376 // The new instruction will be in the form `ldr Rt, [Xn, Xm, lsl #imm]`.
4377 unsigned Opcode = regOffsetOpcode(Opcode: MemI.getOpcode());
4378 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4379 auto B = BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4380 .addReg(RegNo: MemI.getOperand(i: 0).getReg(),
4381 Flags: getDefRegState(B: MemI.mayLoad()))
4382 .addReg(RegNo: AM.BaseReg)
4383 .addReg(RegNo: AM.ScaledReg)
4384 .addImm(Val: 0)
4385 .addImm(Val: AM.Scale > 1)
4386 .setMemRefs(MemI.memoperands())
4387 .setMIFlags(MemI.getFlags());
4388 return B.getInstr();
4389 }
4390
4391 assert(AM.ScaledReg == 0 && AM.Scale == 0 &&
4392 "Addressing mode not supported for folding");
4393
4394 // The new instruction will be in the form `ld[u]r Rt, [Xn, #imm]`.
4395 unsigned Scale = 1;
4396 unsigned Opcode = MemI.getOpcode();
4397 if (isInt<9>(x: AM.Displacement))
4398 Opcode = unscaledOffsetOpcode(Opcode);
4399 else
4400 Opcode = scaledOffsetOpcode(Opcode, Scale);
4401
4402 auto B =
4403 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4404 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4405 .addReg(RegNo: AM.BaseReg)
4406 .addImm(Val: AM.Displacement / Scale)
4407 .setMemRefs(MemI.memoperands())
4408 .setMIFlags(MemI.getFlags());
4409 return B.getInstr();
4410 }
4411
4412 if (AM.Form == ExtAddrMode::Formula::SExtScaledReg ||
4413 AM.Form == ExtAddrMode::Formula::ZExtScaledReg) {
4414 // The new instruction will be in the form `ldr Rt, [Xn, Wm, {s,u}xtw #N]`.
4415 assert(AM.ScaledReg && !AM.Displacement &&
4416 "Address offset can be a register or an immediate, but not both");
4417 unsigned Opcode = offsetExtendOpcode(Opcode: MemI.getOpcode());
4418 MRI.constrainRegClass(Reg: AM.BaseReg, RC: &AArch64::GPR64spRegClass);
4419 // Make sure the offset register is in the correct register class.
4420 Register OffsetReg = AM.ScaledReg;
4421 const TargetRegisterClass *RC = MRI.getRegClass(Reg: OffsetReg);
4422 if (RC->hasSuperClassEq(RC: &AArch64::GPR64RegClass)) {
4423 OffsetReg = MRI.createVirtualRegister(RegClass: &AArch64::GPR32RegClass);
4424 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode: TargetOpcode::COPY), DestReg: OffsetReg)
4425 .addReg(RegNo: AM.ScaledReg, Flags: {}, SubReg: AArch64::sub_32);
4426 }
4427 auto B =
4428 BuildMI(BB&: MBB, I&: MemI, MIMD: DL, MCID: get(Opcode))
4429 .addReg(RegNo: MemI.getOperand(i: 0).getReg(), Flags: getDefRegState(B: MemI.mayLoad()))
4430 .addReg(RegNo: AM.BaseReg)
4431 .addReg(RegNo: OffsetReg)
4432 .addImm(Val: AM.Form == ExtAddrMode::Formula::SExtScaledReg)
4433 .addImm(Val: AM.Scale != 1)
4434 .setMemRefs(MemI.memoperands())
4435 .setMIFlags(MemI.getFlags());
4436
4437 return B.getInstr();
4438 }
4439
4440 llvm_unreachable(
4441 "Function must not be called with an addressing mode it can't handle");
4442}
4443
4444/// Return true if the opcode is a post-index ld/st instruction, which really
4445/// loads from base+0.
4446static bool isPostIndexLdStOpcode(unsigned Opcode) {
4447 switch (Opcode) {
4448 default:
4449 return false;
4450 case AArch64::LD1Fourv16b_POST:
4451 case AArch64::LD1Fourv1d_POST:
4452 case AArch64::LD1Fourv2d_POST:
4453 case AArch64::LD1Fourv2s_POST:
4454 case AArch64::LD1Fourv4h_POST:
4455 case AArch64::LD1Fourv4s_POST:
4456 case AArch64::LD1Fourv8b_POST:
4457 case AArch64::LD1Fourv8h_POST:
4458 case AArch64::LD1Onev16b_POST:
4459 case AArch64::LD1Onev1d_POST:
4460 case AArch64::LD1Onev2d_POST:
4461 case AArch64::LD1Onev2s_POST:
4462 case AArch64::LD1Onev4h_POST:
4463 case AArch64::LD1Onev4s_POST:
4464 case AArch64::LD1Onev8b_POST:
4465 case AArch64::LD1Onev8h_POST:
4466 case AArch64::LD1Rv16b_POST:
4467 case AArch64::LD1Rv1d_POST:
4468 case AArch64::LD1Rv2d_POST:
4469 case AArch64::LD1Rv2s_POST:
4470 case AArch64::LD1Rv4h_POST:
4471 case AArch64::LD1Rv4s_POST:
4472 case AArch64::LD1Rv8b_POST:
4473 case AArch64::LD1Rv8h_POST:
4474 case AArch64::LD1Threev16b_POST:
4475 case AArch64::LD1Threev1d_POST:
4476 case AArch64::LD1Threev2d_POST:
4477 case AArch64::LD1Threev2s_POST:
4478 case AArch64::LD1Threev4h_POST:
4479 case AArch64::LD1Threev4s_POST:
4480 case AArch64::LD1Threev8b_POST:
4481 case AArch64::LD1Threev8h_POST:
4482 case AArch64::LD1Twov16b_POST:
4483 case AArch64::LD1Twov1d_POST:
4484 case AArch64::LD1Twov2d_POST:
4485 case AArch64::LD1Twov2s_POST:
4486 case AArch64::LD1Twov4h_POST:
4487 case AArch64::LD1Twov4s_POST:
4488 case AArch64::LD1Twov8b_POST:
4489 case AArch64::LD1Twov8h_POST:
4490 case AArch64::LD1i16_POST:
4491 case AArch64::LD1i32_POST:
4492 case AArch64::LD1i64_POST:
4493 case AArch64::LD1i8_POST:
4494 case AArch64::LD2Rv16b_POST:
4495 case AArch64::LD2Rv1d_POST:
4496 case AArch64::LD2Rv2d_POST:
4497 case AArch64::LD2Rv2s_POST:
4498 case AArch64::LD2Rv4h_POST:
4499 case AArch64::LD2Rv4s_POST:
4500 case AArch64::LD2Rv8b_POST:
4501 case AArch64::LD2Rv8h_POST:
4502 case AArch64::LD2Twov16b_POST:
4503 case AArch64::LD2Twov2d_POST:
4504 case AArch64::LD2Twov2s_POST:
4505 case AArch64::LD2Twov4h_POST:
4506 case AArch64::LD2Twov4s_POST:
4507 case AArch64::LD2Twov8b_POST:
4508 case AArch64::LD2Twov8h_POST:
4509 case AArch64::LD2i16_POST:
4510 case AArch64::LD2i32_POST:
4511 case AArch64::LD2i64_POST:
4512 case AArch64::LD2i8_POST:
4513 case AArch64::LD3Rv16b_POST:
4514 case AArch64::LD3Rv1d_POST:
4515 case AArch64::LD3Rv2d_POST:
4516 case AArch64::LD3Rv2s_POST:
4517 case AArch64::LD3Rv4h_POST:
4518 case AArch64::LD3Rv4s_POST:
4519 case AArch64::LD3Rv8b_POST:
4520 case AArch64::LD3Rv8h_POST:
4521 case AArch64::LD3Threev16b_POST:
4522 case AArch64::LD3Threev2d_POST:
4523 case AArch64::LD3Threev2s_POST:
4524 case AArch64::LD3Threev4h_POST:
4525 case AArch64::LD3Threev4s_POST:
4526 case AArch64::LD3Threev8b_POST:
4527 case AArch64::LD3Threev8h_POST:
4528 case AArch64::LD3i16_POST:
4529 case AArch64::LD3i32_POST:
4530 case AArch64::LD3i64_POST:
4531 case AArch64::LD3i8_POST:
4532 case AArch64::LD4Fourv16b_POST:
4533 case AArch64::LD4Fourv2d_POST:
4534 case AArch64::LD4Fourv2s_POST:
4535 case AArch64::LD4Fourv4h_POST:
4536 case AArch64::LD4Fourv4s_POST:
4537 case AArch64::LD4Fourv8b_POST:
4538 case AArch64::LD4Fourv8h_POST:
4539 case AArch64::LD4Rv16b_POST:
4540 case AArch64::LD4Rv1d_POST:
4541 case AArch64::LD4Rv2d_POST:
4542 case AArch64::LD4Rv2s_POST:
4543 case AArch64::LD4Rv4h_POST:
4544 case AArch64::LD4Rv4s_POST:
4545 case AArch64::LD4Rv8b_POST:
4546 case AArch64::LD4Rv8h_POST:
4547 case AArch64::LD4i16_POST:
4548 case AArch64::LD4i32_POST:
4549 case AArch64::LD4i64_POST:
4550 case AArch64::LD4i8_POST:
4551 case AArch64::LDAPRWpost:
4552 case AArch64::LDAPRXpost:
4553 case AArch64::LDIAPPWpost:
4554 case AArch64::LDIAPPXpost:
4555 case AArch64::LDPDpost:
4556 case AArch64::LDPQpost:
4557 case AArch64::LDPSWpost:
4558 case AArch64::LDPSpost:
4559 case AArch64::LDPWpost:
4560 case AArch64::LDPXpost:
4561 case AArch64::LDRBBpost:
4562 case AArch64::LDRBpost:
4563 case AArch64::LDRDpost:
4564 case AArch64::LDRHHpost:
4565 case AArch64::LDRHpost:
4566 case AArch64::LDRQpost:
4567 case AArch64::LDRSBWpost:
4568 case AArch64::LDRSBXpost:
4569 case AArch64::LDRSHWpost:
4570 case AArch64::LDRSHXpost:
4571 case AArch64::LDRSWpost:
4572 case AArch64::LDRSpost:
4573 case AArch64::LDRWpost:
4574 case AArch64::LDRXpost:
4575 case AArch64::ST1Fourv16b_POST:
4576 case AArch64::ST1Fourv1d_POST:
4577 case AArch64::ST1Fourv2d_POST:
4578 case AArch64::ST1Fourv2s_POST:
4579 case AArch64::ST1Fourv4h_POST:
4580 case AArch64::ST1Fourv4s_POST:
4581 case AArch64::ST1Fourv8b_POST:
4582 case AArch64::ST1Fourv8h_POST:
4583 case AArch64::ST1Onev16b_POST:
4584 case AArch64::ST1Onev1d_POST:
4585 case AArch64::ST1Onev2d_POST:
4586 case AArch64::ST1Onev2s_POST:
4587 case AArch64::ST1Onev4h_POST:
4588 case AArch64::ST1Onev4s_POST:
4589 case AArch64::ST1Onev8b_POST:
4590 case AArch64::ST1Onev8h_POST:
4591 case AArch64::ST1Threev16b_POST:
4592 case AArch64::ST1Threev1d_POST:
4593 case AArch64::ST1Threev2d_POST:
4594 case AArch64::ST1Threev2s_POST:
4595 case AArch64::ST1Threev4h_POST:
4596 case AArch64::ST1Threev4s_POST:
4597 case AArch64::ST1Threev8b_POST:
4598 case AArch64::ST1Threev8h_POST:
4599 case AArch64::ST1Twov16b_POST:
4600 case AArch64::ST1Twov1d_POST:
4601 case AArch64::ST1Twov2d_POST:
4602 case AArch64::ST1Twov2s_POST:
4603 case AArch64::ST1Twov4h_POST:
4604 case AArch64::ST1Twov4s_POST:
4605 case AArch64::ST1Twov8b_POST:
4606 case AArch64::ST1Twov8h_POST:
4607 case AArch64::ST1i16_POST:
4608 case AArch64::ST1i32_POST:
4609 case AArch64::ST1i64_POST:
4610 case AArch64::ST1i8_POST:
4611 case AArch64::ST2GPostIndex:
4612 case AArch64::ST2Twov16b_POST:
4613 case AArch64::ST2Twov2d_POST:
4614 case AArch64::ST2Twov2s_POST:
4615 case AArch64::ST2Twov4h_POST:
4616 case AArch64::ST2Twov4s_POST:
4617 case AArch64::ST2Twov8b_POST:
4618 case AArch64::ST2Twov8h_POST:
4619 case AArch64::ST2i16_POST:
4620 case AArch64::ST2i32_POST:
4621 case AArch64::ST2i64_POST:
4622 case AArch64::ST2i8_POST:
4623 case AArch64::ST3Threev16b_POST:
4624 case AArch64::ST3Threev2d_POST:
4625 case AArch64::ST3Threev2s_POST:
4626 case AArch64::ST3Threev4h_POST:
4627 case AArch64::ST3Threev4s_POST:
4628 case AArch64::ST3Threev8b_POST:
4629 case AArch64::ST3Threev8h_POST:
4630 case AArch64::ST3i16_POST:
4631 case AArch64::ST3i32_POST:
4632 case AArch64::ST3i64_POST:
4633 case AArch64::ST3i8_POST:
4634 case AArch64::ST4Fourv16b_POST:
4635 case AArch64::ST4Fourv2d_POST:
4636 case AArch64::ST4Fourv2s_POST:
4637 case AArch64::ST4Fourv4h_POST:
4638 case AArch64::ST4Fourv4s_POST:
4639 case AArch64::ST4Fourv8b_POST:
4640 case AArch64::ST4Fourv8h_POST:
4641 case AArch64::ST4i16_POST:
4642 case AArch64::ST4i32_POST:
4643 case AArch64::ST4i64_POST:
4644 case AArch64::ST4i8_POST:
4645 case AArch64::STGPostIndex:
4646 case AArch64::STGPpost:
4647 case AArch64::STPDpost:
4648 case AArch64::STPQpost:
4649 case AArch64::STPSpost:
4650 case AArch64::STPWpost:
4651 case AArch64::STPXpost:
4652 case AArch64::STRBBpost:
4653 case AArch64::STRBpost:
4654 case AArch64::STRDpost:
4655 case AArch64::STRHHpost:
4656 case AArch64::STRHpost:
4657 case AArch64::STRQpost:
4658 case AArch64::STRSpost:
4659 case AArch64::STRWpost:
4660 case AArch64::STRXpost:
4661 case AArch64::STZ2GPostIndex:
4662 case AArch64::STZGPostIndex:
4663 return true;
4664 }
4665}
4666
4667bool AArch64InstrInfo::getMemOperandWithOffsetWidth(
4668 const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset,
4669 bool &OffsetIsScalable, TypeSize &Width) const {
4670 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4671 // Handle only loads/stores with base register followed by immediate offset.
4672 if (LdSt.getNumExplicitOperands() == 3) {
4673 // Non-paired instruction (e.g., ldr x1, [x0, #8]).
4674 if ((!LdSt.getOperand(i: 1).isReg() && !LdSt.getOperand(i: 1).isFI()) ||
4675 !LdSt.getOperand(i: 2).isImm())
4676 return false;
4677 } else if (LdSt.getNumExplicitOperands() == 4) {
4678 // Paired instruction (e.g., ldp x1, x2, [x0, #8]).
4679 if (!LdSt.getOperand(i: 1).isReg() ||
4680 (!LdSt.getOperand(i: 2).isReg() && !LdSt.getOperand(i: 2).isFI()) ||
4681 !LdSt.getOperand(i: 3).isImm())
4682 return false;
4683 } else
4684 return false;
4685
4686 // Get the scaling factor for the instruction and set the width for the
4687 // instruction.
4688 TypeSize Scale(0U, false);
4689 int64_t Dummy1, Dummy2;
4690
4691 // If this returns false, then it's an instruction we don't want to handle.
4692 if (!getMemOpInfo(Opcode: LdSt.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2))
4693 return false;
4694
4695 // Compute the offset. Offset is calculated as the immediate operand
4696 // multiplied by the scaling factor. Unscaled instructions have scaling factor
4697 // set to 1. Postindex are a special case which have an offset of 0.
4698 if (isPostIndexLdStOpcode(Opcode: LdSt.getOpcode())) {
4699 BaseOp = &LdSt.getOperand(i: 2);
4700 Offset = 0;
4701 } else if (LdSt.getNumExplicitOperands() == 3) {
4702 BaseOp = &LdSt.getOperand(i: 1);
4703 Offset = LdSt.getOperand(i: 2).getImm() * Scale.getKnownMinValue();
4704 } else {
4705 assert(LdSt.getNumExplicitOperands() == 4 && "invalid number of operands");
4706 BaseOp = &LdSt.getOperand(i: 2);
4707 Offset = LdSt.getOperand(i: 3).getImm() * Scale.getKnownMinValue();
4708 }
4709 OffsetIsScalable = Scale.isScalable();
4710
4711 return BaseOp->isReg() || BaseOp->isFI();
4712}
4713
4714MachineOperand &
4715AArch64InstrInfo::getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const {
4716 assert(LdSt.mayLoadOrStore() && "Expected a memory operation.");
4717 MachineOperand &OfsOp = LdSt.getOperand(i: LdSt.getNumExplicitOperands() - 1);
4718 assert(OfsOp.isImm() && "Offset operand wasn't immediate.");
4719 return OfsOp;
4720}
4721
4722bool AArch64InstrInfo::getMemOpInfo(unsigned Opcode, TypeSize &Scale,
4723 TypeSize &Width, int64_t &MinOffset,
4724 int64_t &MaxOffset) {
4725 switch (Opcode) {
4726 // Not a memory operation or something we want to handle.
4727 default:
4728 Scale = Width = TypeSize::getFixed(ExactSize: 0);
4729 MinOffset = MaxOffset = 0;
4730 return false;
4731 // LDR / STR
4732 case AArch64::LDRQui:
4733 case AArch64::STRQui:
4734 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4735 MinOffset = 0;
4736 MaxOffset = 4095;
4737 break;
4738 case AArch64::LDRXui:
4739 case AArch64::LDRDui:
4740 case AArch64::STRXui:
4741 case AArch64::STRDui:
4742 case AArch64::PRFMui:
4743 case AArch64::ATOMIC_STORE_HINT_Xui:
4744 case AArch64::ATOMIC_STORE_HINT_Dui:
4745 Scale = Width = TypeSize::getFixed(ExactSize: 8);
4746 MinOffset = 0;
4747 MaxOffset = 4095;
4748 break;
4749 case AArch64::LDRWui:
4750 case AArch64::LDRSui:
4751 case AArch64::LDRSWui:
4752 case AArch64::STRWui:
4753 case AArch64::STRSui:
4754 case AArch64::ATOMIC_STORE_HINT_Wui:
4755 case AArch64::ATOMIC_STORE_HINT_Sui:
4756 Scale = Width = TypeSize::getFixed(ExactSize: 4);
4757 MinOffset = 0;
4758 MaxOffset = 4095;
4759 break;
4760 case AArch64::LDRHui:
4761 case AArch64::LDRHHui:
4762 case AArch64::LDRSHWui:
4763 case AArch64::LDRSHXui:
4764 case AArch64::STRHui:
4765 case AArch64::STRHHui:
4766 case AArch64::ATOMIC_STORE_HINT_Hui:
4767 Scale = Width = TypeSize::getFixed(ExactSize: 2);
4768 MinOffset = 0;
4769 MaxOffset = 4095;
4770 break;
4771 case AArch64::LDRBui:
4772 case AArch64::LDRBBui:
4773 case AArch64::LDRSBWui:
4774 case AArch64::LDRSBXui:
4775 case AArch64::STRBui:
4776 case AArch64::STRBBui:
4777 case AArch64::ATOMIC_STORE_HINT_Bui:
4778 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4779 MinOffset = 0;
4780 MaxOffset = 4095;
4781 break;
4782 // post/pre inc
4783 case AArch64::STRQpre:
4784 case AArch64::LDRQpost:
4785 Scale = TypeSize::getFixed(ExactSize: 1);
4786 Width = TypeSize::getFixed(ExactSize: 16);
4787 MinOffset = -256;
4788 MaxOffset = 255;
4789 break;
4790 case AArch64::LDRDpost:
4791 case AArch64::LDRDpre:
4792 case AArch64::LDRXpost:
4793 case AArch64::LDRXpre:
4794 case AArch64::STRDpost:
4795 case AArch64::STRDpre:
4796 case AArch64::STRXpost:
4797 case AArch64::STRXpre:
4798 Scale = TypeSize::getFixed(ExactSize: 1);
4799 Width = TypeSize::getFixed(ExactSize: 8);
4800 MinOffset = -256;
4801 MaxOffset = 255;
4802 break;
4803 case AArch64::STRWpost:
4804 case AArch64::STRWpre:
4805 case AArch64::LDRWpost:
4806 case AArch64::LDRWpre:
4807 case AArch64::STRSpost:
4808 case AArch64::STRSpre:
4809 case AArch64::LDRSpost:
4810 case AArch64::LDRSpre:
4811 Scale = TypeSize::getFixed(ExactSize: 1);
4812 Width = TypeSize::getFixed(ExactSize: 4);
4813 MinOffset = -256;
4814 MaxOffset = 255;
4815 break;
4816 case AArch64::LDRHpost:
4817 case AArch64::LDRHpre:
4818 case AArch64::STRHpost:
4819 case AArch64::STRHpre:
4820 case AArch64::LDRHHpost:
4821 case AArch64::LDRHHpre:
4822 case AArch64::STRHHpost:
4823 case AArch64::STRHHpre:
4824 Scale = TypeSize::getFixed(ExactSize: 1);
4825 Width = TypeSize::getFixed(ExactSize: 2);
4826 MinOffset = -256;
4827 MaxOffset = 255;
4828 break;
4829 case AArch64::LDRBpost:
4830 case AArch64::LDRBpre:
4831 case AArch64::STRBpost:
4832 case AArch64::STRBpre:
4833 case AArch64::LDRBBpost:
4834 case AArch64::LDRBBpre:
4835 case AArch64::STRBBpost:
4836 case AArch64::STRBBpre:
4837 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4838 MinOffset = -256;
4839 MaxOffset = 255;
4840 break;
4841 // Unscaled
4842 case AArch64::LDURQi:
4843 case AArch64::STURQi:
4844 Scale = TypeSize::getFixed(ExactSize: 1);
4845 Width = TypeSize::getFixed(ExactSize: 16);
4846 MinOffset = -256;
4847 MaxOffset = 255;
4848 break;
4849 case AArch64::LDURXi:
4850 case AArch64::LDURDi:
4851 case AArch64::LDAPURXi:
4852 case AArch64::STURXi:
4853 case AArch64::STURDi:
4854 case AArch64::STLURXi:
4855 case AArch64::PRFUMi:
4856 case AArch64::ATOMIC_STORE_HINT_Xi:
4857 case AArch64::ATOMIC_STORE_HINT_Di:
4858 Scale = TypeSize::getFixed(ExactSize: 1);
4859 Width = TypeSize::getFixed(ExactSize: 8);
4860 MinOffset = -256;
4861 MaxOffset = 255;
4862 break;
4863 case AArch64::LDURWi:
4864 case AArch64::LDURSi:
4865 case AArch64::LDURSWi:
4866 case AArch64::LDAPURi:
4867 case AArch64::LDAPURSWi:
4868 case AArch64::STURWi:
4869 case AArch64::STURSi:
4870 case AArch64::STLURWi:
4871 case AArch64::ATOMIC_STORE_HINT_Wi:
4872 case AArch64::ATOMIC_STORE_HINT_Si:
4873 Scale = TypeSize::getFixed(ExactSize: 1);
4874 Width = TypeSize::getFixed(ExactSize: 4);
4875 MinOffset = -256;
4876 MaxOffset = 255;
4877 break;
4878 case AArch64::LDURHi:
4879 case AArch64::LDURHHi:
4880 case AArch64::LDURSHXi:
4881 case AArch64::LDURSHWi:
4882 case AArch64::LDAPURHi:
4883 case AArch64::LDAPURSHWi:
4884 case AArch64::LDAPURSHXi:
4885 case AArch64::STURHi:
4886 case AArch64::STURHHi:
4887 case AArch64::STLURHi:
4888 case AArch64::ATOMIC_STORE_HINT_Hi:
4889 Scale = TypeSize::getFixed(ExactSize: 1);
4890 Width = TypeSize::getFixed(ExactSize: 2);
4891 MinOffset = -256;
4892 MaxOffset = 255;
4893 break;
4894 case AArch64::LDURBi:
4895 case AArch64::LDURBBi:
4896 case AArch64::LDURSBXi:
4897 case AArch64::LDURSBWi:
4898 case AArch64::LDAPURBi:
4899 case AArch64::LDAPURSBWi:
4900 case AArch64::LDAPURSBXi:
4901 case AArch64::STURBi:
4902 case AArch64::STURBBi:
4903 case AArch64::STLURBi:
4904 case AArch64::ATOMIC_STORE_HINT_Bi:
4905 Scale = Width = TypeSize::getFixed(ExactSize: 1);
4906 MinOffset = -256;
4907 MaxOffset = 255;
4908 break;
4909 // LDP / STP (including pre/post inc)
4910 case AArch64::LDPQi:
4911 case AArch64::LDNPQi:
4912 case AArch64::STPQi:
4913 case AArch64::STNPQi:
4914 case AArch64::LDPQpost:
4915 case AArch64::LDPQpre:
4916 case AArch64::STPQpost:
4917 case AArch64::STPQpre:
4918 Scale = TypeSize::getFixed(ExactSize: 16);
4919 Width = TypeSize::getFixed(ExactSize: 16 * 2);
4920 MinOffset = -64;
4921 MaxOffset = 63;
4922 break;
4923 case AArch64::LDPXi:
4924 case AArch64::LDPDi:
4925 case AArch64::LDNPXi:
4926 case AArch64::LDNPDi:
4927 case AArch64::STPXi:
4928 case AArch64::STPDi:
4929 case AArch64::STNPXi:
4930 case AArch64::STNPDi:
4931 case AArch64::LDPDpost:
4932 case AArch64::LDPDpre:
4933 case AArch64::LDPXpost:
4934 case AArch64::LDPXpre:
4935 case AArch64::STPDpost:
4936 case AArch64::STPDpre:
4937 case AArch64::STPXpost:
4938 case AArch64::STPXpre:
4939 Scale = TypeSize::getFixed(ExactSize: 8);
4940 Width = TypeSize::getFixed(ExactSize: 8 * 2);
4941 MinOffset = -64;
4942 MaxOffset = 63;
4943 break;
4944 case AArch64::LDPWi:
4945 case AArch64::LDPSi:
4946 case AArch64::LDNPWi:
4947 case AArch64::LDNPSi:
4948 case AArch64::STPWi:
4949 case AArch64::STPSi:
4950 case AArch64::STNPWi:
4951 case AArch64::STNPSi:
4952 case AArch64::LDPSpost:
4953 case AArch64::LDPSpre:
4954 case AArch64::LDPWpost:
4955 case AArch64::LDPWpre:
4956 case AArch64::STPSpost:
4957 case AArch64::STPSpre:
4958 case AArch64::STPWpost:
4959 case AArch64::STPWpre:
4960 Scale = TypeSize::getFixed(ExactSize: 4);
4961 Width = TypeSize::getFixed(ExactSize: 4 * 2);
4962 MinOffset = -64;
4963 MaxOffset = 63;
4964 break;
4965 case AArch64::StoreSwiftAsyncContext:
4966 // Store is an STRXui, but there might be an ADDXri in the expansion too.
4967 Scale = TypeSize::getFixed(ExactSize: 1);
4968 Width = TypeSize::getFixed(ExactSize: 8);
4969 MinOffset = 0;
4970 MaxOffset = 4095;
4971 break;
4972 case AArch64::ADDG:
4973 Scale = TypeSize::getFixed(ExactSize: 16);
4974 Width = TypeSize::getFixed(ExactSize: 0);
4975 MinOffset = 0;
4976 MaxOffset = 63;
4977 break;
4978 case AArch64::TAGPstack:
4979 Scale = TypeSize::getFixed(ExactSize: 16);
4980 Width = TypeSize::getFixed(ExactSize: 0);
4981 // TAGP with a negative offset turns into SUBP, which has a maximum offset
4982 // of 63 (not 64!).
4983 MinOffset = -63;
4984 MaxOffset = 63;
4985 break;
4986 case AArch64::LDG:
4987 case AArch64::STGi:
4988 case AArch64::STGPreIndex:
4989 case AArch64::STGPostIndex:
4990 case AArch64::STZGi:
4991 case AArch64::STZGPreIndex:
4992 case AArch64::STZGPostIndex:
4993 Scale = Width = TypeSize::getFixed(ExactSize: 16);
4994 MinOffset = -256;
4995 MaxOffset = 255;
4996 break;
4997 // SVE
4998 case AArch64::STR_ZZZZXI:
4999 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
5000 case AArch64::LDR_ZZZZXI:
5001 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
5002 Scale = TypeSize::getScalable(MinimumSize: 16);
5003 Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5004 MinOffset = -256;
5005 MaxOffset = 252;
5006 break;
5007 case AArch64::STR_ZZZXI:
5008 case AArch64::LDR_ZZZXI:
5009 Scale = TypeSize::getScalable(MinimumSize: 16);
5010 Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5011 MinOffset = -256;
5012 MaxOffset = 253;
5013 break;
5014 case AArch64::STR_ZZXI:
5015 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
5016 case AArch64::LDR_ZZXI:
5017 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
5018 Scale = TypeSize::getScalable(MinimumSize: 16);
5019 Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5020 MinOffset = -256;
5021 MaxOffset = 254;
5022 break;
5023 case AArch64::LDR_PXI:
5024 case AArch64::STR_PXI:
5025 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5026 MinOffset = -256;
5027 MaxOffset = 255;
5028 break;
5029 case AArch64::LDR_PPXI:
5030 case AArch64::STR_PPXI:
5031 Scale = TypeSize::getScalable(MinimumSize: 2);
5032 Width = TypeSize::getScalable(MinimumSize: 2 * 2);
5033 MinOffset = -256;
5034 MaxOffset = 254;
5035 break;
5036 case AArch64::LDR_ZXI:
5037 case AArch64::STR_ZXI:
5038 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5039 MinOffset = -256;
5040 MaxOffset = 255;
5041 break;
5042 case AArch64::LD1B_IMM:
5043 case AArch64::LD1H_IMM:
5044 case AArch64::LD1W_IMM:
5045 case AArch64::LD1D_IMM:
5046 case AArch64::LDNT1B_ZRI:
5047 case AArch64::LDNT1H_ZRI:
5048 case AArch64::LDNT1W_ZRI:
5049 case AArch64::LDNT1D_ZRI:
5050 case AArch64::ST1B_IMM:
5051 case AArch64::ST1H_IMM:
5052 case AArch64::ST1W_IMM:
5053 case AArch64::ST1D_IMM:
5054 case AArch64::STNT1B_ZRI:
5055 case AArch64::STNT1H_ZRI:
5056 case AArch64::STNT1W_ZRI:
5057 case AArch64::STNT1D_ZRI:
5058 case AArch64::LDNF1B_IMM:
5059 case AArch64::LDNF1H_IMM:
5060 case AArch64::LDNF1W_IMM:
5061 case AArch64::LDNF1D_IMM:
5062 // A full vectors worth of data
5063 // Width = mbytes * elements
5064 Scale = Width = TypeSize::getScalable(MinimumSize: 16);
5065 MinOffset = -8;
5066 MaxOffset = 7;
5067 break;
5068 case AArch64::LD2B_IMM:
5069 case AArch64::LD2H_IMM:
5070 case AArch64::LD2W_IMM:
5071 case AArch64::LD2D_IMM:
5072 case AArch64::ST2B_IMM:
5073 case AArch64::ST2H_IMM:
5074 case AArch64::ST2W_IMM:
5075 case AArch64::ST2D_IMM:
5076 case AArch64::LD1B_2Z_IMM:
5077 case AArch64::LD1B_2Z_STRIDED_IMM:
5078 case AArch64::LD1H_2Z_IMM:
5079 case AArch64::LD1H_2Z_STRIDED_IMM:
5080 case AArch64::LD1W_2Z_IMM:
5081 case AArch64::LD1W_2Z_STRIDED_IMM:
5082 case AArch64::LD1D_2Z_IMM:
5083 case AArch64::LD1D_2Z_STRIDED_IMM:
5084 case AArch64::LD1B_2Z_IMM_PSEUDO:
5085 case AArch64::LD1H_2Z_IMM_PSEUDO:
5086 case AArch64::LD1W_2Z_IMM_PSEUDO:
5087 case AArch64::LD1D_2Z_IMM_PSEUDO:
5088 case AArch64::ST1B_2Z_IMM:
5089 case AArch64::ST1B_2Z_STRIDED_IMM:
5090 case AArch64::ST1H_2Z_IMM:
5091 case AArch64::ST1H_2Z_STRIDED_IMM:
5092 case AArch64::ST1W_2Z_IMM:
5093 case AArch64::ST1W_2Z_STRIDED_IMM:
5094 case AArch64::ST1D_2Z_IMM:
5095 case AArch64::ST1D_2Z_STRIDED_IMM:
5096 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
5097 case AArch64::LDNT1B_2Z_IMM:
5098 case AArch64::LDNT1B_2Z_STRIDED_IMM:
5099 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
5100 case AArch64::LDNT1H_2Z_IMM:
5101 case AArch64::LDNT1H_2Z_STRIDED_IMM:
5102 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
5103 case AArch64::LDNT1W_2Z_IMM:
5104 case AArch64::LDNT1W_2Z_STRIDED_IMM:
5105 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
5106 case AArch64::LDNT1D_2Z_IMM:
5107 case AArch64::LDNT1D_2Z_STRIDED_IMM:
5108 case AArch64::STNT1B_2Z_IMM:
5109 case AArch64::STNT1B_2Z_STRIDED_IMM:
5110 case AArch64::STNT1H_2Z_IMM:
5111 case AArch64::STNT1H_2Z_STRIDED_IMM:
5112 case AArch64::STNT1W_2Z_IMM:
5113 case AArch64::STNT1W_2Z_STRIDED_IMM:
5114 case AArch64::STNT1D_2Z_IMM:
5115 case AArch64::STNT1D_2Z_STRIDED_IMM:
5116 case AArch64::ST1B_2Z_IMM_PSEUDO:
5117 case AArch64::ST1H_2Z_IMM_PSEUDO:
5118 case AArch64::ST1W_2Z_IMM_PSEUDO:
5119 case AArch64::ST1D_2Z_IMM_PSEUDO:
5120 case AArch64::STNT1B_2Z_IMM_PSEUDO:
5121 case AArch64::STNT1H_2Z_IMM_PSEUDO:
5122 case AArch64::STNT1W_2Z_IMM_PSEUDO:
5123 case AArch64::STNT1D_2Z_IMM_PSEUDO:
5124 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 2);
5125 MinOffset = -8;
5126 MaxOffset = 7;
5127 break;
5128 case AArch64::LD3B_IMM:
5129 case AArch64::LD3H_IMM:
5130 case AArch64::LD3W_IMM:
5131 case AArch64::LD3D_IMM:
5132 case AArch64::ST3B_IMM:
5133 case AArch64::ST3H_IMM:
5134 case AArch64::ST3W_IMM:
5135 case AArch64::ST3D_IMM:
5136 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 3);
5137 MinOffset = -8;
5138 MaxOffset = 7;
5139 break;
5140 case AArch64::LD4B_IMM:
5141 case AArch64::LD4H_IMM:
5142 case AArch64::LD4W_IMM:
5143 case AArch64::LD4D_IMM:
5144 case AArch64::ST4B_IMM:
5145 case AArch64::ST4H_IMM:
5146 case AArch64::ST4W_IMM:
5147 case AArch64::ST4D_IMM:
5148 case AArch64::LD1B_4Z_IMM:
5149 case AArch64::LD1B_4Z_STRIDED_IMM:
5150 case AArch64::LD1H_4Z_IMM:
5151 case AArch64::LD1H_4Z_STRIDED_IMM:
5152 case AArch64::LD1W_4Z_IMM:
5153 case AArch64::LD1W_4Z_STRIDED_IMM:
5154 case AArch64::LD1D_4Z_IMM:
5155 case AArch64::LD1D_4Z_STRIDED_IMM:
5156 case AArch64::LD1B_4Z_IMM_PSEUDO:
5157 case AArch64::LD1H_4Z_IMM_PSEUDO:
5158 case AArch64::LD1W_4Z_IMM_PSEUDO:
5159 case AArch64::LD1D_4Z_IMM_PSEUDO:
5160 case AArch64::ST1B_4Z_IMM:
5161 case AArch64::ST1B_4Z_STRIDED_IMM:
5162 case AArch64::ST1H_4Z_IMM:
5163 case AArch64::ST1H_4Z_STRIDED_IMM:
5164 case AArch64::ST1W_4Z_IMM:
5165 case AArch64::ST1W_4Z_STRIDED_IMM:
5166 case AArch64::ST1D_4Z_IMM:
5167 case AArch64::ST1D_4Z_STRIDED_IMM:
5168 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
5169 case AArch64::LDNT1B_4Z_IMM:
5170 case AArch64::LDNT1B_4Z_STRIDED_IMM:
5171 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
5172 case AArch64::LDNT1H_4Z_IMM:
5173 case AArch64::LDNT1H_4Z_STRIDED_IMM:
5174 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
5175 case AArch64::LDNT1W_4Z_IMM:
5176 case AArch64::LDNT1W_4Z_STRIDED_IMM:
5177 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
5178 case AArch64::LDNT1D_4Z_IMM:
5179 case AArch64::LDNT1D_4Z_STRIDED_IMM:
5180 case AArch64::STNT1B_4Z_IMM:
5181 case AArch64::STNT1B_4Z_STRIDED_IMM:
5182 case AArch64::STNT1H_4Z_IMM:
5183 case AArch64::STNT1H_4Z_STRIDED_IMM:
5184 case AArch64::STNT1W_4Z_IMM:
5185 case AArch64::STNT1W_4Z_STRIDED_IMM:
5186 case AArch64::STNT1D_4Z_IMM:
5187 case AArch64::STNT1D_4Z_STRIDED_IMM:
5188 case AArch64::ST1B_4Z_IMM_PSEUDO:
5189 case AArch64::ST1H_4Z_IMM_PSEUDO:
5190 case AArch64::ST1W_4Z_IMM_PSEUDO:
5191 case AArch64::ST1D_4Z_IMM_PSEUDO:
5192 case AArch64::STNT1B_4Z_IMM_PSEUDO:
5193 case AArch64::STNT1H_4Z_IMM_PSEUDO:
5194 case AArch64::STNT1W_4Z_IMM_PSEUDO:
5195 case AArch64::STNT1D_4Z_IMM_PSEUDO:
5196 Scale = Width = TypeSize::getScalable(MinimumSize: 16 * 4);
5197 MinOffset = -8;
5198 MaxOffset = 7;
5199 break;
5200 case AArch64::LD1B_H_IMM:
5201 case AArch64::LD1SB_H_IMM:
5202 case AArch64::LD1H_S_IMM:
5203 case AArch64::LD1SH_S_IMM:
5204 case AArch64::LD1W_D_IMM:
5205 case AArch64::LD1SW_D_IMM:
5206 case AArch64::ST1B_H_IMM:
5207 case AArch64::ST1H_S_IMM:
5208 case AArch64::ST1W_D_IMM:
5209 case AArch64::LDNF1B_H_IMM:
5210 case AArch64::LDNF1SB_H_IMM:
5211 case AArch64::LDNF1H_S_IMM:
5212 case AArch64::LDNF1SH_S_IMM:
5213 case AArch64::LDNF1W_D_IMM:
5214 case AArch64::LDNF1SW_D_IMM:
5215 // A half vector worth of data
5216 // Width = mbytes * elements
5217 Scale = Width = TypeSize::getScalable(MinimumSize: 8);
5218 MinOffset = -8;
5219 MaxOffset = 7;
5220 break;
5221 case AArch64::LD1B_S_IMM:
5222 case AArch64::LD1SB_S_IMM:
5223 case AArch64::LD1H_D_IMM:
5224 case AArch64::LD1SH_D_IMM:
5225 case AArch64::ST1B_S_IMM:
5226 case AArch64::ST1H_D_IMM:
5227 case AArch64::LDNF1B_S_IMM:
5228 case AArch64::LDNF1SB_S_IMM:
5229 case AArch64::LDNF1H_D_IMM:
5230 case AArch64::LDNF1SH_D_IMM:
5231 // A quarter vector worth of data
5232 // Width = mbytes * elements
5233 Scale = Width = TypeSize::getScalable(MinimumSize: 4);
5234 MinOffset = -8;
5235 MaxOffset = 7;
5236 break;
5237 case AArch64::LD1B_D_IMM:
5238 case AArch64::LD1SB_D_IMM:
5239 case AArch64::ST1B_D_IMM:
5240 case AArch64::LDNF1B_D_IMM:
5241 case AArch64::LDNF1SB_D_IMM:
5242 // A eighth vector worth of data
5243 // Width = mbytes * elements
5244 Scale = Width = TypeSize::getScalable(MinimumSize: 2);
5245 MinOffset = -8;
5246 MaxOffset = 7;
5247 break;
5248 case AArch64::ST2Gi:
5249 case AArch64::ST2GPreIndex:
5250 case AArch64::ST2GPostIndex:
5251 case AArch64::STZ2Gi:
5252 case AArch64::STZ2GPreIndex:
5253 case AArch64::STZ2GPostIndex:
5254 Scale = TypeSize::getFixed(ExactSize: 16);
5255 Width = TypeSize::getFixed(ExactSize: 32);
5256 MinOffset = -256;
5257 MaxOffset = 255;
5258 break;
5259 case AArch64::STGPi:
5260 case AArch64::STGPpost:
5261 case AArch64::STGPpre:
5262 Scale = Width = TypeSize::getFixed(ExactSize: 16);
5263 MinOffset = -64;
5264 MaxOffset = 63;
5265 break;
5266 case AArch64::LD1RB_IMM:
5267 case AArch64::LD1RB_H_IMM:
5268 case AArch64::LD1RB_S_IMM:
5269 case AArch64::LD1RB_D_IMM:
5270 case AArch64::LD1RSB_H_IMM:
5271 case AArch64::LD1RSB_S_IMM:
5272 case AArch64::LD1RSB_D_IMM:
5273 Scale = Width = TypeSize::getFixed(ExactSize: 1);
5274 MinOffset = 0;
5275 MaxOffset = 63;
5276 break;
5277 case AArch64::LD1RH_IMM:
5278 case AArch64::LD1RH_S_IMM:
5279 case AArch64::LD1RH_D_IMM:
5280 case AArch64::LD1RSH_S_IMM:
5281 case AArch64::LD1RSH_D_IMM:
5282 Scale = Width = TypeSize::getFixed(ExactSize: 2);
5283 MinOffset = 0;
5284 MaxOffset = 63;
5285 break;
5286 case AArch64::LD1RW_IMM:
5287 case AArch64::LD1RW_D_IMM:
5288 case AArch64::LD1RSW_IMM:
5289 Scale = Width = TypeSize::getFixed(ExactSize: 4);
5290 MinOffset = 0;
5291 MaxOffset = 63;
5292 break;
5293 case AArch64::LD1RD_IMM:
5294 Scale = Width = TypeSize::getFixed(ExactSize: 8);
5295 MinOffset = 0;
5296 MaxOffset = 63;
5297 break;
5298 }
5299
5300 return true;
5301}
5302
5303// Scaling factor for unscaled load or store.
5304int AArch64InstrInfo::getMemScale(unsigned Opc) {
5305 switch (Opc) {
5306 default:
5307 llvm_unreachable("Opcode has unknown scale!");
5308 case AArch64::LDRBui:
5309 case AArch64::LDRBBui:
5310 case AArch64::LDURBBi:
5311 case AArch64::LDRSBWui:
5312 case AArch64::LDURSBWi:
5313 case AArch64::STRBui:
5314 case AArch64::STRBBui:
5315 case AArch64::STURBBi:
5316 return 1;
5317 case AArch64::LDRHui:
5318 case AArch64::LDRHHui:
5319 case AArch64::LDURHHi:
5320 case AArch64::LDRSHWui:
5321 case AArch64::LDURSHWi:
5322 case AArch64::STRHui:
5323 case AArch64::STRHHui:
5324 case AArch64::STURHHi:
5325 return 2;
5326 case AArch64::LDRSui:
5327 case AArch64::LDURSi:
5328 case AArch64::LDRSpre:
5329 case AArch64::LDRSWui:
5330 case AArch64::LDURSWi:
5331 case AArch64::LDRSWpre:
5332 case AArch64::LDRWpre:
5333 case AArch64::LDRWui:
5334 case AArch64::LDURWi:
5335 case AArch64::STRSui:
5336 case AArch64::STURSi:
5337 case AArch64::STRSpre:
5338 case AArch64::STRWui:
5339 case AArch64::STURWi:
5340 case AArch64::STRWpre:
5341 case AArch64::LDPSi:
5342 case AArch64::LDPSWi:
5343 case AArch64::LDPWi:
5344 case AArch64::STPSi:
5345 case AArch64::STPWi:
5346 return 4;
5347 case AArch64::LDRDui:
5348 case AArch64::LDURDi:
5349 case AArch64::LDRDpre:
5350 case AArch64::LDRXui:
5351 case AArch64::LDURXi:
5352 case AArch64::LDRXpre:
5353 case AArch64::STRDui:
5354 case AArch64::STURDi:
5355 case AArch64::STRDpre:
5356 case AArch64::STRXui:
5357 case AArch64::STURXi:
5358 case AArch64::STRXpre:
5359 case AArch64::LDPDi:
5360 case AArch64::LDPXi:
5361 case AArch64::STPDi:
5362 case AArch64::STPXi:
5363 return 8;
5364 case AArch64::LDRQui:
5365 case AArch64::LDURQi:
5366 case AArch64::STRQui:
5367 case AArch64::STURQi:
5368 case AArch64::STRQpre:
5369 case AArch64::LDPQi:
5370 case AArch64::LDRQpre:
5371 case AArch64::STPQi:
5372 case AArch64::STGi:
5373 case AArch64::STZGi:
5374 case AArch64::ST2Gi:
5375 case AArch64::STZ2Gi:
5376 case AArch64::STGPi:
5377 return 16;
5378 }
5379}
5380
5381bool AArch64InstrInfo::isPreLd(const MachineInstr &MI) {
5382 switch (MI.getOpcode()) {
5383 default:
5384 return false;
5385 case AArch64::LDRWpre:
5386 case AArch64::LDRXpre:
5387 case AArch64::LDRSWpre:
5388 case AArch64::LDRSpre:
5389 case AArch64::LDRDpre:
5390 case AArch64::LDRQpre:
5391 return true;
5392 }
5393}
5394
5395bool AArch64InstrInfo::isPreSt(const MachineInstr &MI) {
5396 switch (MI.getOpcode()) {
5397 default:
5398 return false;
5399 case AArch64::STRWpre:
5400 case AArch64::STRXpre:
5401 case AArch64::STRSpre:
5402 case AArch64::STRDpre:
5403 case AArch64::STRQpre:
5404 return true;
5405 }
5406}
5407
5408bool AArch64InstrInfo::isPreLdSt(const MachineInstr &MI) {
5409 return isPreLd(MI) || isPreSt(MI);
5410}
5411
5412bool AArch64InstrInfo::isZExtLoad(const MachineInstr &MI) {
5413 switch (MI.getOpcode()) {
5414 default:
5415 return false;
5416 case AArch64::LDURBBi:
5417 case AArch64::LDURHHi:
5418 case AArch64::LDURWi:
5419 case AArch64::LDRBBui:
5420 case AArch64::LDRHHui:
5421 case AArch64::LDRWui:
5422 case AArch64::LDRBBroX:
5423 case AArch64::LDRHHroX:
5424 case AArch64::LDRWroX:
5425 case AArch64::LDRBBroW:
5426 case AArch64::LDRHHroW:
5427 case AArch64::LDRWroW:
5428 return true;
5429 }
5430}
5431
5432bool AArch64InstrInfo::isSExtLoad(const MachineInstr &MI) {
5433 switch (MI.getOpcode()) {
5434 default:
5435 return false;
5436 case AArch64::LDURSBWi:
5437 case AArch64::LDURSHWi:
5438 case AArch64::LDURSBXi:
5439 case AArch64::LDURSHXi:
5440 case AArch64::LDURSWi:
5441 case AArch64::LDRSBWui:
5442 case AArch64::LDRSHWui:
5443 case AArch64::LDRSBXui:
5444 case AArch64::LDRSHXui:
5445 case AArch64::LDRSWui:
5446 case AArch64::LDRSBWroX:
5447 case AArch64::LDRSHWroX:
5448 case AArch64::LDRSBXroX:
5449 case AArch64::LDRSHXroX:
5450 case AArch64::LDRSWroX:
5451 case AArch64::LDRSBWroW:
5452 case AArch64::LDRSHWroW:
5453 case AArch64::LDRSBXroW:
5454 case AArch64::LDRSHXroW:
5455 case AArch64::LDRSWroW:
5456 return true;
5457 }
5458}
5459
5460bool AArch64InstrInfo::isPairedLdSt(const MachineInstr &MI) {
5461 switch (MI.getOpcode()) {
5462 default:
5463 return false;
5464 case AArch64::LDPSi:
5465 case AArch64::LDPSWi:
5466 case AArch64::LDPDi:
5467 case AArch64::LDPQi:
5468 case AArch64::LDPWi:
5469 case AArch64::LDPXi:
5470 case AArch64::STPSi:
5471 case AArch64::STPDi:
5472 case AArch64::STPQi:
5473 case AArch64::STPWi:
5474 case AArch64::STPXi:
5475 case AArch64::STGPi:
5476 return true;
5477 }
5478}
5479
5480const MachineOperand &AArch64InstrInfo::getLdStBaseOp(const MachineInstr &MI) {
5481 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5482 unsigned Idx =
5483 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 2
5484 : 1;
5485 return MI.getOperand(i: Idx);
5486}
5487
5488const MachineOperand &
5489AArch64InstrInfo::getLdStOffsetOp(const MachineInstr &MI) {
5490 assert(MI.mayLoadOrStore() && "Load or store instruction expected");
5491 unsigned Idx =
5492 AArch64InstrInfo::isPairedLdSt(MI) || AArch64InstrInfo::isPreLdSt(MI) ? 3
5493 : 2;
5494 return MI.getOperand(i: Idx);
5495}
5496
5497const MachineOperand &
5498AArch64InstrInfo::getLdStAmountOp(const MachineInstr &MI) {
5499 switch (MI.getOpcode()) {
5500 default:
5501 llvm_unreachable("Unexpected opcode");
5502 case AArch64::LDRBroX:
5503 case AArch64::LDRBBroX:
5504 case AArch64::LDRSBXroX:
5505 case AArch64::LDRSBWroX:
5506 case AArch64::LDRHroX:
5507 case AArch64::LDRHHroX:
5508 case AArch64::LDRSHXroX:
5509 case AArch64::LDRSHWroX:
5510 case AArch64::LDRWroX:
5511 case AArch64::LDRSroX:
5512 case AArch64::LDRSWroX:
5513 case AArch64::LDRDroX:
5514 case AArch64::LDRXroX:
5515 case AArch64::LDRQroX:
5516 return MI.getOperand(i: 4);
5517 }
5518}
5519
5520static const TargetRegisterClass *getRegClass(const MachineInstr &MI,
5521 Register Reg) {
5522 if (MI.getParent() == nullptr)
5523 return nullptr;
5524 const MachineFunction *MF = MI.getParent()->getParent();
5525 return MF ? MF->getRegInfo().getRegClassOrNull(Reg) : nullptr;
5526}
5527
5528bool AArch64InstrInfo::isHForm(const MachineInstr &MI) {
5529 auto IsHFPR = [&](const MachineOperand &Op) {
5530 if (!Op.isReg())
5531 return false;
5532 auto Reg = Op.getReg();
5533 if (Reg.isPhysical())
5534 return AArch64::FPR16RegClass.contains(Reg);
5535 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5536 return TRC == &AArch64::FPR16RegClass ||
5537 TRC == &AArch64::FPR16_loRegClass;
5538 };
5539 return llvm::any_of(Range: MI.operands(), P: IsHFPR);
5540}
5541
5542bool AArch64InstrInfo::isQForm(const MachineInstr &MI) {
5543 auto IsQFPR = [&](const MachineOperand &Op) {
5544 if (!Op.isReg())
5545 return false;
5546 auto Reg = Op.getReg();
5547 if (Reg.isPhysical())
5548 return AArch64::FPR128RegClass.contains(Reg);
5549 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5550 return TRC == &AArch64::FPR128RegClass ||
5551 TRC == &AArch64::FPR128_loRegClass;
5552 };
5553 return llvm::any_of(Range: MI.operands(), P: IsQFPR);
5554}
5555
5556bool AArch64InstrInfo::hasBTISemantics(const MachineInstr &MI) {
5557 switch (MI.getOpcode()) {
5558 case AArch64::BRK:
5559 case AArch64::HLT:
5560 case AArch64::PACIASP:
5561 case AArch64::PACIBSP:
5562 // Implicit BTI behavior.
5563 return true;
5564 case AArch64::PAUTH_PROLOGUE:
5565 // PAUTH_PROLOGUE expands to PACI(A|B)SP.
5566 return true;
5567 case AArch64::HINT: {
5568 unsigned Imm = MI.getOperand(i: 0).getImm();
5569 // Explicit BTI instruction.
5570 if (Imm == 32 || Imm == 34 || Imm == 36 || Imm == 38)
5571 return true;
5572 // PACI(A|B)SP instructions.
5573 if (Imm == 25 || Imm == 27)
5574 return true;
5575 return false;
5576 }
5577 default:
5578 return false;
5579 }
5580}
5581
5582bool AArch64InstrInfo::isFpOrNEON(Register Reg) {
5583 if (Reg == 0)
5584 return false;
5585 assert(Reg.isPhysical() && "Expected physical register in isFpOrNEON");
5586 return AArch64::FPR128RegClass.contains(Reg) ||
5587 AArch64::FPR64RegClass.contains(Reg) ||
5588 AArch64::FPR32RegClass.contains(Reg) ||
5589 AArch64::FPR16RegClass.contains(Reg) ||
5590 AArch64::FPR8RegClass.contains(Reg);
5591}
5592
5593bool AArch64InstrInfo::isFpOrNEON(const MachineInstr &MI) {
5594 auto IsFPR = [&](const MachineOperand &Op) {
5595 if (!Op.isReg())
5596 return false;
5597 auto Reg = Op.getReg();
5598 if (Reg.isPhysical())
5599 return isFpOrNEON(Reg);
5600
5601 const TargetRegisterClass *TRC = ::getRegClass(MI, Reg);
5602 return TRC == &AArch64::FPR128RegClass ||
5603 TRC == &AArch64::FPR128_loRegClass ||
5604 TRC == &AArch64::FPR64RegClass ||
5605 TRC == &AArch64::FPR64_loRegClass ||
5606 TRC == &AArch64::FPR32RegClass || TRC == &AArch64::FPR16RegClass ||
5607 TRC == &AArch64::FPR8RegClass;
5608 };
5609 return llvm::any_of(Range: MI.operands(), P: IsFPR);
5610}
5611
5612// Scale the unscaled offsets. Returns false if the unscaled offset can't be
5613// scaled.
5614static bool scaleOffset(unsigned Opc, int64_t &Offset) {
5615 int Scale = AArch64InstrInfo::getMemScale(Opc);
5616
5617 // If the byte-offset isn't a multiple of the stride, we can't scale this
5618 // offset.
5619 if (Offset % Scale != 0)
5620 return false;
5621
5622 // Convert the byte-offset used by unscaled into an "element" offset used
5623 // by the scaled pair load/store instructions.
5624 Offset /= Scale;
5625 return true;
5626}
5627
5628static bool canPairLdStOpc(unsigned FirstOpc, unsigned SecondOpc) {
5629 if (FirstOpc == SecondOpc)
5630 return true;
5631 // We can also pair sign-ext and zero-ext instructions.
5632 switch (FirstOpc) {
5633 default:
5634 return false;
5635 case AArch64::STRSui:
5636 case AArch64::STURSi:
5637 return SecondOpc == AArch64::STRSui || SecondOpc == AArch64::STURSi;
5638 case AArch64::STRDui:
5639 case AArch64::STURDi:
5640 return SecondOpc == AArch64::STRDui || SecondOpc == AArch64::STURDi;
5641 case AArch64::STRQui:
5642 case AArch64::STURQi:
5643 return SecondOpc == AArch64::STRQui || SecondOpc == AArch64::STURQi;
5644 case AArch64::STRWui:
5645 case AArch64::STURWi:
5646 return SecondOpc == AArch64::STRWui || SecondOpc == AArch64::STURWi;
5647 case AArch64::STRXui:
5648 case AArch64::STURXi:
5649 return SecondOpc == AArch64::STRXui || SecondOpc == AArch64::STURXi;
5650 case AArch64::LDRSui:
5651 case AArch64::LDURSi:
5652 return SecondOpc == AArch64::LDRSui || SecondOpc == AArch64::LDURSi;
5653 case AArch64::LDRDui:
5654 case AArch64::LDURDi:
5655 return SecondOpc == AArch64::LDRDui || SecondOpc == AArch64::LDURDi;
5656 case AArch64::LDRQui:
5657 case AArch64::LDURQi:
5658 return SecondOpc == AArch64::LDRQui || SecondOpc == AArch64::LDURQi;
5659 case AArch64::LDRWui:
5660 case AArch64::LDURWi:
5661 return SecondOpc == AArch64::LDRSWui || SecondOpc == AArch64::LDURSWi;
5662 case AArch64::LDRSWui:
5663 case AArch64::LDURSWi:
5664 return SecondOpc == AArch64::LDRWui || SecondOpc == AArch64::LDURWi;
5665 case AArch64::LDRXui:
5666 case AArch64::LDURXi:
5667 return SecondOpc == AArch64::LDRXui || SecondOpc == AArch64::LDURXi;
5668 }
5669 // These instructions can't be paired based on their opcodes.
5670 return false;
5671}
5672
5673static bool shouldClusterFI(const MachineFrameInfo &MFI, int FI1,
5674 int64_t Offset1, unsigned Opcode1, int FI2,
5675 int64_t Offset2, unsigned Opcode2) {
5676 // Accesses through fixed stack object frame indices may access a different
5677 // fixed stack slot. Check that the object offsets + offsets match.
5678 if (MFI.isFixedObjectIndex(ObjectIdx: FI1) && MFI.isFixedObjectIndex(ObjectIdx: FI2)) {
5679 int64_t ObjectOffset1 = MFI.getObjectOffset(ObjectIdx: FI1);
5680 int64_t ObjectOffset2 = MFI.getObjectOffset(ObjectIdx: FI2);
5681 assert(ObjectOffset1 <= ObjectOffset2 && "Object offsets are not ordered.");
5682 // Convert to scaled object offsets.
5683 int Scale1 = AArch64InstrInfo::getMemScale(Opc: Opcode1);
5684 if (ObjectOffset1 % Scale1 != 0)
5685 return false;
5686 ObjectOffset1 /= Scale1;
5687 int Scale2 = AArch64InstrInfo::getMemScale(Opc: Opcode2);
5688 if (ObjectOffset2 % Scale2 != 0)
5689 return false;
5690 ObjectOffset2 /= Scale2;
5691 ObjectOffset1 += Offset1;
5692 ObjectOffset2 += Offset2;
5693 return ObjectOffset1 + 1 == ObjectOffset2;
5694 }
5695
5696 return FI1 == FI2;
5697}
5698
5699/// Detect opportunities for ldp/stp formation.
5700///
5701/// Only called for LdSt for which getMemOperandWithOffset returns true.
5702bool AArch64InstrInfo::shouldClusterMemOps(
5703 ArrayRef<const MachineOperand *> BaseOps1, int64_t OpOffset1,
5704 bool OffsetIsScalable1, ArrayRef<const MachineOperand *> BaseOps2,
5705 int64_t OpOffset2, bool OffsetIsScalable2, unsigned ClusterSize,
5706 unsigned NumBytes) const {
5707 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
5708 const MachineOperand &BaseOp1 = *BaseOps1.front();
5709 const MachineOperand &BaseOp2 = *BaseOps2.front();
5710 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
5711 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
5712 if (BaseOp1.getType() != BaseOp2.getType())
5713 return false;
5714
5715 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
5716 "Only base registers and frame indices are supported.");
5717
5718 // Check for both base regs and base FI.
5719 if (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg())
5720 return false;
5721
5722 // Only cluster up to a single pair.
5723 if (ClusterSize > 2)
5724 return false;
5725
5726 if (!isPairableLdStInst(MI: FirstLdSt) || !isPairableLdStInst(MI: SecondLdSt))
5727 return false;
5728
5729 // Can we pair these instructions based on their opcodes?
5730 unsigned FirstOpc = FirstLdSt.getOpcode();
5731 unsigned SecondOpc = SecondLdSt.getOpcode();
5732 if (!canPairLdStOpc(FirstOpc, SecondOpc))
5733 return false;
5734
5735 // Can't merge volatiles or load/stores that have a hint to avoid pair
5736 // formation, for example.
5737 if (!isCandidateToMergeOrPair(MI: FirstLdSt) ||
5738 !isCandidateToMergeOrPair(MI: SecondLdSt))
5739 return false;
5740
5741 // isCandidateToMergeOrPair guarantees that operand 2 is an immediate.
5742 int64_t Offset1 = FirstLdSt.getOperand(i: 2).getImm();
5743 if (hasUnscaledLdStOffset(Opc: FirstOpc) && !scaleOffset(Opc: FirstOpc, Offset&: Offset1))
5744 return false;
5745
5746 int64_t Offset2 = SecondLdSt.getOperand(i: 2).getImm();
5747 if (hasUnscaledLdStOffset(Opc: SecondOpc) && !scaleOffset(Opc: SecondOpc, Offset&: Offset2))
5748 return false;
5749
5750 // Pairwise instructions have a 7-bit signed offset field.
5751 if (Offset1 > 63 || Offset1 < -64)
5752 return false;
5753
5754 // The caller should already have ordered First/SecondLdSt by offset.
5755 // Note: except for non-equal frame index bases
5756 if (BaseOp1.isFI()) {
5757 assert((!BaseOp1.isIdenticalTo(BaseOp2) || Offset1 <= Offset2) &&
5758 "Caller should have ordered offsets.");
5759
5760 const MachineFrameInfo &MFI =
5761 FirstLdSt.getParent()->getParent()->getFrameInfo();
5762 return shouldClusterFI(MFI, FI1: BaseOp1.getIndex(), Offset1, Opcode1: FirstOpc,
5763 FI2: BaseOp2.getIndex(), Offset2, Opcode2: SecondOpc);
5764 }
5765
5766 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
5767
5768 return Offset1 + 1 == Offset2;
5769}
5770
5771static const MachineInstrBuilder &AddSubReg(const MachineInstrBuilder &MIB,
5772 MCRegister Reg, unsigned SubIdx,
5773 RegState State,
5774 const TargetRegisterInfo *TRI) {
5775 if (!SubIdx)
5776 return MIB.addReg(RegNo: Reg, Flags: State);
5777
5778 if (Reg.isPhysical())
5779 return MIB.addReg(RegNo: TRI->getSubReg(Reg, Idx: SubIdx), Flags: State);
5780 return MIB.addReg(RegNo: Reg, Flags: State, SubReg: SubIdx);
5781}
5782
5783void AArch64InstrInfo::copyPhysRegTuple(MachineBasicBlock &MBB,
5784 MachineBasicBlock::iterator I,
5785 const DebugLoc &DL, MCRegister DestReg,
5786 MCRegister SrcReg, bool KillSrc,
5787 ArrayRef<unsigned> Indices) const {
5788 assert(Subtarget.hasNEON() && "Unexpected register copy without NEON");
5789 const TargetRegisterInfo *TRI = &getRegisterInfo();
5790 uint16_t DestEncoding = TRI->getEncodingValue(Reg: DestReg);
5791 uint16_t SrcEncoding = TRI->getEncodingValue(Reg: SrcReg);
5792 unsigned NumRegs = Indices.size();
5793 MCRegister DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[0]);
5794 assert(!AArch64::PNRRegClass.contains(DestSubReg) &&
5795 "Unexpected predicate tuple copy");
5796 unsigned MaxRegs = AArch64::PPRRegClass.contains(Reg: DestSubReg) ? 15 : 31;
5797
5798 int SubReg = 0, End = NumRegs, Incr = 1;
5799 // Copy in reverse if a forward copy will clobber the tuple
5800 if (((DestEncoding - SrcEncoding) & MaxRegs) < NumRegs) {
5801 SubReg = NumRegs - 1;
5802 End = -1;
5803 Incr = -1;
5804 }
5805
5806 for (; SubReg != End; SubReg += Incr) {
5807 DestSubReg = TRI->getSubReg(Reg: DestReg, Idx: Indices[SubReg]);
5808 MCRegister SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: Indices[SubReg]);
5809 copyPhysRegImpl(MBB, I, DL, DestReg: DestSubReg, SrcReg: SrcSubReg, KillSrc);
5810 }
5811}
5812
5813void AArch64InstrInfo::copyGPRRegTuple(MachineBasicBlock &MBB,
5814 MachineBasicBlock::iterator I,
5815 const DebugLoc &DL, MCRegister DestReg,
5816 MCRegister SrcReg, bool KillSrc,
5817 unsigned Opcode, unsigned ZeroReg,
5818 llvm::ArrayRef<unsigned> Indices) const {
5819 const TargetRegisterInfo *TRI = &getRegisterInfo();
5820 unsigned NumRegs = Indices.size();
5821
5822#ifndef NDEBUG
5823 uint16_t DestEncoding = TRI->getEncodingValue(DestReg);
5824 uint16_t SrcEncoding = TRI->getEncodingValue(SrcReg);
5825 assert(DestEncoding % NumRegs == 0 && SrcEncoding % NumRegs == 0 &&
5826 "GPR reg sequences should not be able to overlap");
5827#endif
5828
5829 for (unsigned SubReg = 0; SubReg != NumRegs; ++SubReg) {
5830 const MachineInstrBuilder MIB = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode));
5831 AddSubReg(MIB, Reg: DestReg, SubIdx: Indices[SubReg], State: RegState::Define, TRI);
5832 MIB.addReg(RegNo: ZeroReg);
5833 AddSubReg(MIB, Reg: SrcReg, SubIdx: Indices[SubReg], State: getKillRegState(B: KillSrc), TRI);
5834 MIB.addImm(Val: 0);
5835 }
5836}
5837
5838/// Returns true if the instruction at I is in a streaming call site region,
5839/// within a single basic block.
5840/// A "call site streaming region" starts after smstart and ends at smstop
5841/// around a call to a streaming function. This walks backward from I.
5842static bool isInStreamingCallSiteRegion(MachineBasicBlock &MBB,
5843 MachineBasicBlock::iterator I) {
5844 MachineFunction &MF = *MBB.getParent();
5845 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
5846 if (!AFI->hasStreamingModeChanges())
5847 return false;
5848 // Walk backwards to find smstart/smstop
5849 for (MachineInstr &MI : reverse(C: make_range(x: MBB.begin(), y: I))) {
5850 unsigned Opc = MI.getOpcode();
5851 if (Opc == AArch64::MSRpstatesvcrImm1 || Opc == AArch64::MSRpstatePseudo) {
5852 // Check if this is SM change (not ZA)
5853 int64_t PState = MI.getOperand(i: 0).getImm();
5854 if (PState == AArch64SVCR::SVCRSM || PState == AArch64SVCR::SVCRSMZA) {
5855 // Operand 1 is 1 for start, 0 for stop
5856 return MI.getOperand(i: 1).getImm() == 1;
5857 }
5858 }
5859 }
5860 return false;
5861}
5862
5863/// Returns true if in a streaming call site region without SME-FA64.
5864static bool mustAvoidNeonAtMBBI(const AArch64Subtarget &Subtarget,
5865 MachineBasicBlock &MBB,
5866 MachineBasicBlock::iterator I) {
5867 return !Subtarget.hasSMEFA64() && isInStreamingCallSiteRegion(MBB, I);
5868}
5869
5870void AArch64InstrInfo::copyPhysRegImpl(MachineBasicBlock &MBB,
5871 MachineBasicBlock::iterator I,
5872 const DebugLoc &DL, Register DestReg,
5873 Register SrcReg, bool KillSrc,
5874 bool RenamableDest,
5875 bool RenamableSrc) const {
5876 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) &&
5877 AArch64::GPR32spRegClass.contains(Reg: SrcReg)) {
5878 if (DestReg == AArch64::WSP || SrcReg == AArch64::WSP) {
5879 // If either operand is WSP, expand to ADD #0.
5880 if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5881 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5882 // Cyclone recognizes "ADD Xd, Xn, #0" as a zero-cycle register move.
5883 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5884 RC: &AArch64::GPR64spRegClass);
5885 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5886 RC: &AArch64::GPR64spRegClass);
5887 // This instruction is reading and writing X registers. This may upset
5888 // the register scavenger and machine verifier, so we need to indicate
5889 // that we are reading an undefined value from SrcRegX, but a proper
5890 // value from SrcReg.
5891 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg: DestRegX)
5892 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5893 .addImm(Val: 0)
5894 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
5895 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5896 ++NumZCRegMoveInstrsGPR;
5897 } else {
5898 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDWri), DestReg)
5899 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5900 .addImm(Val: 0)
5901 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5902 if (Subtarget.hasZeroCycleRegMoveGPR32())
5903 ++NumZCRegMoveInstrsGPR;
5904 }
5905 } else if (Subtarget.hasZeroCycleRegMoveGPR64() &&
5906 !Subtarget.hasZeroCycleRegMoveGPR32()) {
5907 // Cyclone recognizes "ORR Xd, XZR, Xm" as a zero-cycle register move.
5908 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5909 RC: &AArch64::GPR64spRegClass);
5910 assert(DestRegX.isValid() && "Destination super-reg not valid");
5911 MCRegister SrcRegX = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::sub_32,
5912 RC: &AArch64::GPR64spRegClass);
5913 assert(SrcRegX.isValid() && "Source super-reg not valid");
5914 // This instruction is reading and writing X registers. This may upset
5915 // the register scavenger and machine verifier, so we need to indicate
5916 // that we are reading an undefined value from SrcRegX, but a proper
5917 // value from SrcReg.
5918 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg: DestRegX)
5919 .addReg(RegNo: AArch64::XZR)
5920 .addReg(RegNo: SrcRegX, Flags: RegState::Undef)
5921 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
5922 ++NumZCRegMoveInstrsGPR;
5923 } else {
5924 // Otherwise, expand to ORR WZR.
5925 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5926 .addReg(RegNo: AArch64::WZR)
5927 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5928 if (Subtarget.hasZeroCycleRegMoveGPR32())
5929 ++NumZCRegMoveInstrsGPR;
5930 }
5931 return;
5932 }
5933
5934 // GPR32 zeroing
5935 if (AArch64::GPR32spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::WZR) {
5936 if (Subtarget.hasZeroCycleZeroingGPR64() &&
5937 !Subtarget.hasZeroCycleZeroingGPR32()) {
5938 MCRegister DestRegX = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::sub_32,
5939 RC: &AArch64::GPR64spRegClass);
5940 assert(DestRegX.isValid() && "Destination super-reg not valid");
5941 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: DestRegX)
5942 .addImm(Val: 0)
5943 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5944 ++NumZCZeroingInstrsGPR;
5945 } else if (Subtarget.hasZeroCycleZeroingGPR32()) {
5946 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZWi), DestReg)
5947 .addImm(Val: 0)
5948 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5949 ++NumZCZeroingInstrsGPR;
5950 } else {
5951 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRWrr), DestReg)
5952 .addReg(RegNo: AArch64::WZR)
5953 .addReg(RegNo: AArch64::WZR);
5954 }
5955 return;
5956 }
5957
5958 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) &&
5959 AArch64::GPR64spRegClass.contains(Reg: SrcReg)) {
5960 if (DestReg == AArch64::SP || SrcReg == AArch64::SP) {
5961 // If either operand is SP, expand to ADD #0.
5962 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ADDXri), DestReg)
5963 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
5964 .addImm(Val: 0)
5965 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5966 if (Subtarget.hasZeroCycleRegMoveGPR64())
5967 ++NumZCRegMoveInstrsGPR;
5968 } else {
5969 // Otherwise, expand to ORR XZR.
5970 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5971 .addReg(RegNo: AArch64::XZR)
5972 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
5973 if (Subtarget.hasZeroCycleRegMoveGPR64())
5974 ++NumZCRegMoveInstrsGPR;
5975 }
5976 return;
5977 }
5978
5979 // GPR64 zeroing
5980 if (AArch64::GPR64spRegClass.contains(Reg: DestReg) && SrcReg == AArch64::XZR) {
5981 if (Subtarget.hasZeroCycleZeroingGPR64()) {
5982 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg)
5983 .addImm(Val: 0)
5984 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
5985 ++NumZCZeroingInstrsGPR;
5986 } else {
5987 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRXrr), DestReg)
5988 .addReg(RegNo: AArch64::XZR)
5989 .addReg(RegNo: AArch64::XZR);
5990 }
5991 return;
5992 }
5993
5994 // Copy a Predicate register by ORRing with itself.
5995 if (AArch64::PPRRegClass.contains(Reg: DestReg) &&
5996 AArch64::PPRRegClass.contains(Reg: SrcReg)) {
5997 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
5998 "Unexpected SVE register.");
5999 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg)
6000 .addReg(RegNo: SrcReg) // Pg
6001 .addReg(RegNo: SrcReg)
6002 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6003 return;
6004 }
6005
6006 // Copy a predicate-as-counter register by ORRing with itself as if it
6007 // were a regular predicate (mask) register.
6008 bool DestIsPNR = AArch64::PNRRegClass.contains(Reg: DestReg);
6009 bool SrcIsPNR = AArch64::PNRRegClass.contains(Reg: SrcReg);
6010 if (DestIsPNR || SrcIsPNR) {
6011 auto ToPPR = [](MCRegister R) -> MCRegister {
6012 return (R - AArch64::PN0) + AArch64::P0;
6013 };
6014 MCRegister PPRSrcReg = SrcIsPNR ? ToPPR(SrcReg) : SrcReg.asMCReg();
6015 MCRegister PPRDestReg = DestIsPNR ? ToPPR(DestReg) : DestReg.asMCReg();
6016
6017 if (PPRSrcReg != PPRDestReg) {
6018 auto NewMI = BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_PPzPP), DestReg: PPRDestReg)
6019 .addReg(RegNo: PPRSrcReg) // Pg
6020 .addReg(RegNo: PPRSrcReg)
6021 .addReg(RegNo: PPRSrcReg, Flags: getKillRegState(B: KillSrc));
6022 if (DestIsPNR)
6023 NewMI.addDef(RegNo: DestReg, Flags: RegState::Implicit);
6024 }
6025 return;
6026 }
6027
6028 // Copy a predicate register pair by copying the individual sub-registers.
6029 if (AArch64::PPR2RegClass.contains(Reg: DestReg) &&
6030 AArch64::PPR2RegClass.contains(Reg: SrcReg)) {
6031 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6032 "Unexpected SVE predicate register.");
6033 static const unsigned Indices[] = {AArch64::psub0, AArch64::psub1};
6034 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6035 return;
6036 }
6037
6038 // Copy a Z register by ORRing with itself.
6039 if (AArch64::ZPRRegClass.contains(Reg: DestReg) &&
6040 AArch64::ZPRRegClass.contains(Reg: SrcReg)) {
6041 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6042 "Unexpected SVE register.");
6043 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ), DestReg)
6044 .addReg(RegNo: SrcReg)
6045 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6046 return;
6047 }
6048
6049 // Copy a Z register pair by copying the individual sub-registers.
6050 if ((AArch64::ZPR2RegClass.contains(Reg: DestReg) ||
6051 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6052 (AArch64::ZPR2RegClass.contains(Reg: SrcReg) ||
6053 AArch64::ZPR2StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6054 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6055 "Unexpected SVE register.");
6056 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
6057 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6058 return;
6059 }
6060
6061 // Copy a Z register triple by copying the individual sub-registers.
6062 if (AArch64::ZPR3RegClass.contains(Reg: DestReg) &&
6063 AArch64::ZPR3RegClass.contains(Reg: SrcReg)) {
6064 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6065 "Unexpected SVE register.");
6066 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6067 AArch64::zsub2};
6068 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6069 return;
6070 }
6071
6072 // Copy a Z register quad by copying the individual sub-registers.
6073 if ((AArch64::ZPR4RegClass.contains(Reg: DestReg) ||
6074 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: DestReg)) &&
6075 (AArch64::ZPR4RegClass.contains(Reg: SrcReg) ||
6076 AArch64::ZPR4StridedOrContiguousRegClass.contains(Reg: SrcReg))) {
6077 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6078 "Unexpected SVE register.");
6079 static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
6080 AArch64::zsub2, AArch64::zsub3};
6081 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6082 return;
6083 }
6084
6085 // Copy a DDDD register quad by copying the individual sub-registers.
6086 if (AArch64::DDDDRegClass.contains(Reg: DestReg) &&
6087 AArch64::DDDDRegClass.contains(Reg: SrcReg)) {
6088 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6089 AArch64::dsub2, AArch64::dsub3};
6090 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6091 return;
6092 }
6093
6094 // Copy a DDD register triple by copying the individual sub-registers.
6095 if (AArch64::DDDRegClass.contains(Reg: DestReg) &&
6096 AArch64::DDDRegClass.contains(Reg: SrcReg)) {
6097 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1,
6098 AArch64::dsub2};
6099 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6100 return;
6101 }
6102
6103 // Copy a DD register pair by copying the individual sub-registers.
6104 if (AArch64::DDRegClass.contains(Reg: DestReg) &&
6105 AArch64::DDRegClass.contains(Reg: SrcReg)) {
6106 static const unsigned Indices[] = {AArch64::dsub0, AArch64::dsub1};
6107 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6108 return;
6109 }
6110
6111 // Copy a QQQQ register quad by copying the individual sub-registers.
6112 if (AArch64::QQQQRegClass.contains(Reg: DestReg) &&
6113 AArch64::QQQQRegClass.contains(Reg: SrcReg)) {
6114 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6115 AArch64::qsub2, AArch64::qsub3};
6116 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6117 return;
6118 }
6119
6120 // Copy a QQQ register triple by copying the individual sub-registers.
6121 if (AArch64::QQQRegClass.contains(Reg: DestReg) &&
6122 AArch64::QQQRegClass.contains(Reg: SrcReg)) {
6123 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1,
6124 AArch64::qsub2};
6125 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6126 return;
6127 }
6128
6129 // Copy a QQ register pair by copying the individual sub-registers.
6130 if (AArch64::QQRegClass.contains(Reg: DestReg) &&
6131 AArch64::QQRegClass.contains(Reg: SrcReg)) {
6132 static const unsigned Indices[] = {AArch64::qsub0, AArch64::qsub1};
6133 copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Indices);
6134 return;
6135 }
6136
6137 if (AArch64::XSeqPairsClassRegClass.contains(Reg: DestReg) &&
6138 AArch64::XSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6139 static const unsigned Indices[] = {AArch64::sube64, AArch64::subo64};
6140 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRXrs,
6141 ZeroReg: AArch64::XZR, Indices);
6142 return;
6143 }
6144
6145 if (AArch64::WSeqPairsClassRegClass.contains(Reg: DestReg) &&
6146 AArch64::WSeqPairsClassRegClass.contains(Reg: SrcReg)) {
6147 static const unsigned Indices[] = {AArch64::sube32, AArch64::subo32};
6148 copyGPRRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, Opcode: AArch64::ORRWrs,
6149 ZeroReg: AArch64::WZR, Indices);
6150 return;
6151 }
6152
6153 if (AArch64::FPR128RegClass.contains(Reg: DestReg) &&
6154 AArch64::FPR128RegClass.contains(Reg: SrcReg)) {
6155 // In streaming regions, NEON is illegal but streaming-SVE is available.
6156 // Use SVE for copies if we're in a streaming region and SME is available.
6157 // With +sme-fa64, NEON is legal in streaming mode so we can use it.
6158 if ((Subtarget.isSVEorStreamingSVEAvailable() &&
6159 !Subtarget.isNeonAvailable()) ||
6160 mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6161 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORR_ZZZ))
6162 .addReg(RegNo: AArch64::Z0 + (DestReg - AArch64::Q0), Flags: RegState::Define)
6163 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0))
6164 .addReg(RegNo: AArch64::Z0 + (SrcReg - AArch64::Q0));
6165 } else if (Subtarget.isNeonAvailable()) {
6166 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg)
6167 .addReg(RegNo: SrcReg)
6168 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6169 if (Subtarget.hasZeroCycleRegMoveFPR128())
6170 ++NumZCRegMoveInstrsFPR;
6171 } else {
6172 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::STRQpre))
6173 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6174 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6175 .addReg(RegNo: AArch64::SP)
6176 .addImm(Val: -16);
6177 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::LDRQpost))
6178 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
6179 .addReg(RegNo: DestReg, Flags: RegState::Define)
6180 .addReg(RegNo: AArch64::SP)
6181 .addImm(Val: 16);
6182 }
6183 return;
6184 }
6185
6186 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6187 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6188 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6189 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6190 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6191 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6192 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::dsub,
6193 RC: &AArch64::FPR128RegClass);
6194 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::dsub,
6195 RC: &AArch64::FPR128RegClass);
6196 // This instruction is reading and writing Q registers. This may upset
6197 // the register scavenger and machine verifier, so we need to indicate
6198 // that we are reading an undefined value from SrcRegQ, but a proper
6199 // value from SrcReg.
6200 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6201 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6202 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6203 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6204 ++NumZCRegMoveInstrsFPR;
6205 } else {
6206 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg)
6207 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6208 if (Subtarget.hasZeroCycleRegMoveFPR64())
6209 ++NumZCRegMoveInstrsFPR;
6210 }
6211 return;
6212 }
6213
6214 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6215 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6216 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6217 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6218 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6219 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6220 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6221 RC: &AArch64::FPR128RegClass);
6222 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6223 RC: &AArch64::FPR128RegClass);
6224 // This instruction is reading and writing Q registers. This may upset
6225 // the register scavenger and machine verifier, so we need to indicate
6226 // that we are reading an undefined value from SrcRegQ, but a proper
6227 // value from SrcReg.
6228 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6229 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6230 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6231 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6232 ++NumZCRegMoveInstrsFPR;
6233 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6234 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6235 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::ssub,
6236 RC: &AArch64::FPR64RegClass);
6237 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::ssub,
6238 RC: &AArch64::FPR64RegClass);
6239 // This instruction is reading and writing D registers. This may upset
6240 // the register scavenger and machine verifier, so we need to indicate
6241 // that we are reading an undefined value from SrcRegD, but a proper
6242 // value from SrcReg.
6243 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6244 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6245 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6246 ++NumZCRegMoveInstrsFPR;
6247 } else {
6248 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6249 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6250 if (Subtarget.hasZeroCycleRegMoveFPR32())
6251 ++NumZCRegMoveInstrsFPR;
6252 }
6253 return;
6254 }
6255
6256 if (AArch64::FPR16RegClass.contains(Reg: DestReg) &&
6257 AArch64::FPR16RegClass.contains(Reg: SrcReg)) {
6258 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6259 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6260 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6261 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6262 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6263 RC: &AArch64::FPR128RegClass);
6264 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6265 RC: &AArch64::FPR128RegClass);
6266 // This instruction is reading and writing Q registers. This may upset
6267 // the register scavenger and machine verifier, so we need to indicate
6268 // that we are reading an undefined value from SrcRegQ, but a proper
6269 // value from SrcReg.
6270 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6271 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6272 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6273 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6274 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6275 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6276 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6277 RC: &AArch64::FPR64RegClass);
6278 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6279 RC: &AArch64::FPR64RegClass);
6280 // This instruction is reading and writing D registers. This may upset
6281 // the register scavenger and machine verifier, so we need to indicate
6282 // that we are reading an undefined value from SrcRegD, but a proper
6283 // value from SrcReg.
6284 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6285 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6286 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6287 } else {
6288 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::hsub,
6289 RC: &AArch64::FPR32RegClass);
6290 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::hsub,
6291 RC: &AArch64::FPR32RegClass);
6292 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6293 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6294 }
6295 return;
6296 }
6297
6298 if (AArch64::FPR8RegClass.contains(Reg: DestReg) &&
6299 AArch64::FPR8RegClass.contains(Reg: SrcReg)) {
6300 if (Subtarget.hasZeroCycleRegMoveFPR128() &&
6301 !Subtarget.hasZeroCycleRegMoveFPR64() &&
6302 !Subtarget.hasZeroCycleRegMoveFPR32() && Subtarget.isNeonAvailable() &&
6303 !mustAvoidNeonAtMBBI(Subtarget, MBB, I)) {
6304 MCRegister DestRegQ = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6305 RC: &AArch64::FPR128RegClass);
6306 MCRegister SrcRegQ = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6307 RC: &AArch64::FPR128RegClass);
6308 // This instruction is reading and writing Q registers. This may upset
6309 // the register scavenger and machine verifier, so we need to indicate
6310 // that we are reading an undefined value from SrcRegQ, but a proper
6311 // value from SrcReg.
6312 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::ORRv16i8), DestReg: DestRegQ)
6313 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6314 .addReg(RegNo: SrcRegQ, Flags: RegState::Undef)
6315 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6316 } else if (Subtarget.hasZeroCycleRegMoveFPR64() &&
6317 !Subtarget.hasZeroCycleRegMoveFPR32()) {
6318 MCRegister DestRegD = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6319 RC: &AArch64::FPR64RegClass);
6320 MCRegister SrcRegD = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6321 RC: &AArch64::FPR64RegClass);
6322 // This instruction is reading and writing D registers. This may upset
6323 // the register scavenger and machine verifier, so we need to indicate
6324 // that we are reading an undefined value from SrcRegD, but a proper
6325 // value from SrcReg.
6326 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDr), DestReg: DestRegD)
6327 .addReg(RegNo: SrcRegD, Flags: RegState::Undef)
6328 .addReg(RegNo: SrcReg, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6329 } else {
6330 DestReg = RI.getMatchingSuperReg(Reg: DestReg, SubIdx: AArch64::bsub,
6331 RC: &AArch64::FPR32RegClass);
6332 SrcReg = RI.getMatchingSuperReg(Reg: SrcReg, SubIdx: AArch64::bsub,
6333 RC: &AArch64::FPR32RegClass);
6334 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSr), DestReg)
6335 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6336 }
6337 return;
6338 }
6339
6340 // Copies between GPR64 and FPR64.
6341 if (AArch64::FPR64RegClass.contains(Reg: DestReg) &&
6342 AArch64::GPR64RegClass.contains(Reg: SrcReg)) {
6343 if (AArch64::XZR == SrcReg) {
6344 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg);
6345 } else {
6346 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVXDr), DestReg)
6347 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6348 }
6349 return;
6350 }
6351 if (AArch64::GPR64RegClass.contains(Reg: DestReg) &&
6352 AArch64::FPR64RegClass.contains(Reg: SrcReg)) {
6353 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVDXr), DestReg)
6354 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6355 return;
6356 }
6357 // Copies between GPR32 and FPR32.
6358 if (AArch64::FPR32RegClass.contains(Reg: DestReg) &&
6359 AArch64::GPR32RegClass.contains(Reg: SrcReg)) {
6360 if (AArch64::WZR == SrcReg) {
6361 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVS0), DestReg);
6362 } else {
6363 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVWSr), DestReg)
6364 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6365 }
6366 return;
6367 }
6368 if (AArch64::GPR32RegClass.contains(Reg: DestReg) &&
6369 AArch64::FPR32RegClass.contains(Reg: SrcReg)) {
6370 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::FMOVSWr), DestReg)
6371 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc));
6372 return;
6373 }
6374
6375 if (DestReg == AArch64::NZCV) {
6376 assert(AArch64::GPR64RegClass.contains(SrcReg) && "Invalid NZCV copy");
6377 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MSR))
6378 .addImm(Val: AArch64SysReg::NZCV)
6379 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: KillSrc))
6380 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Define);
6381 return;
6382 }
6383
6384 if (SrcReg == AArch64::NZCV) {
6385 assert(AArch64::GPR64RegClass.contains(DestReg) && "Invalid NZCV copy");
6386 BuildMI(BB&: MBB, I, MIMD: DL, MCID: get(Opcode: AArch64::MRS), DestReg)
6387 .addImm(Val: AArch64SysReg::NZCV)
6388 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | getKillRegState(B: KillSrc));
6389 return;
6390 }
6391
6392#ifndef NDEBUG
6393 errs() << RI.getRegAsmName(DestReg) << " = COPY " << RI.getRegAsmName(SrcReg)
6394 << "\n";
6395#endif
6396 llvm_unreachable("unimplemented reg-to-reg copy");
6397}
6398
6399void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
6400 MachineBasicBlock::iterator I,
6401 const DebugLoc &DL, Register DestReg,
6402 Register SrcReg, bool KillSrc,
6403 bool RenamableDest,
6404 bool RenamableSrc) const {
6405 ++NumCopyInstrs;
6406 copyPhysRegImpl(MBB, I, DL, DestReg, SrcReg, KillSrc, RenamableDest,
6407 RenamableSrc);
6408 return;
6409}
6410
6411static void storeRegPairToStackSlot(const TargetRegisterInfo &TRI,
6412 MachineBasicBlock &MBB,
6413 MachineBasicBlock::iterator InsertBefore,
6414 const MCInstrDesc &MCID,
6415 Register SrcReg, bool IsKill,
6416 unsigned SubIdx0, unsigned SubIdx1, int FI,
6417 MachineMemOperand *MMO) {
6418 Register SrcReg0 = SrcReg;
6419 Register SrcReg1 = SrcReg;
6420 if (SrcReg.isPhysical()) {
6421 SrcReg0 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx0);
6422 SubIdx0 = 0;
6423 SrcReg1 = TRI.getSubReg(Reg: SrcReg, Idx: SubIdx1);
6424 SubIdx1 = 0;
6425 }
6426 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6427 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: IsKill), SubReg: SubIdx0)
6428 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: IsKill), SubReg: SubIdx1)
6429 .addFrameIndex(Idx: FI)
6430 .addImm(Val: 0)
6431 .addMemOperand(MMO);
6432}
6433
6434void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
6435 MachineBasicBlock::iterator MBBI,
6436 Register SrcReg, bool isKill, int FI,
6437 const TargetRegisterClass *RC,
6438 Register VReg,
6439 MachineInstr::MIFlag Flags) const {
6440 MachineFunction &MF = *MBB.getParent();
6441 MachineFrameInfo &MFI = MF.getFrameInfo();
6442
6443 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6444 MachineMemOperand *MMO =
6445 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
6446 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6447 unsigned Opc = 0;
6448 bool Offset = true;
6449 MCRegister PNRReg = MCRegister::NoRegister;
6450 unsigned StackID = TargetStackID::Default;
6451 switch (RI.getSpillSize(RC: *RC)) {
6452 case 1:
6453 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6454 Opc = AArch64::STRBui;
6455 break;
6456 case 2: {
6457 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6458 Opc = AArch64::STRHui;
6459 else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
6460 AArch64::PPRRegClass.hasSubClassEq(RC)) {
6461 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6462 "Unexpected register store without SVE store instructions");
6463 Opc = AArch64::STR_PXI;
6464 StackID = TargetStackID::ScalablePredicateVector;
6465 }
6466 break;
6467 }
6468 case 4:
6469 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6470 Opc = AArch64::STRWui;
6471 if (SrcReg.isVirtual())
6472 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR32RegClass);
6473 else
6474 assert(SrcReg != AArch64::WSP);
6475 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6476 Opc = AArch64::STRSui;
6477 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6478 Opc = AArch64::STR_PPXI;
6479 StackID = TargetStackID::ScalablePredicateVector;
6480 }
6481 break;
6482 case 8:
6483 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6484 Opc = AArch64::STRXui;
6485 if (SrcReg.isVirtual())
6486 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
6487 else
6488 assert(SrcReg != AArch64::SP);
6489 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6490 Opc = AArch64::STRDui;
6491 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6492 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6493 MCID: get(Opcode: AArch64::STPWi), SrcReg, IsKill: isKill,
6494 SubIdx0: AArch64::sube32, SubIdx1: AArch64::subo32, FI, MMO);
6495 return;
6496 }
6497 break;
6498 case 16:
6499 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6500 Opc = AArch64::STRQui;
6501 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6502 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6503 Opc = AArch64::ST1Twov1d;
6504 Offset = false;
6505 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6506 storeRegPairToStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6507 MCID: get(Opcode: AArch64::STPXi), SrcReg, IsKill: isKill,
6508 SubIdx0: AArch64::sube64, SubIdx1: AArch64::subo64, FI, MMO);
6509 return;
6510 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6511 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6512 "Unexpected register store without SVE store instructions");
6513 Opc = AArch64::STR_ZXI;
6514 StackID = TargetStackID::ScalableVector;
6515 }
6516 break;
6517 case 24:
6518 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6519 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6520 Opc = AArch64::ST1Threev1d;
6521 Offset = false;
6522 }
6523 break;
6524 case 32:
6525 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6526 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6527 Opc = AArch64::ST1Fourv1d;
6528 Offset = false;
6529 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6530 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6531 Opc = AArch64::ST1Twov2d;
6532 Offset = false;
6533 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6534 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6535 "Unexpected register store without SVE store instructions");
6536 Opc = AArch64::STR_ZZXI_STRIDED_CONTIGUOUS;
6537 StackID = TargetStackID::ScalableVector;
6538 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6539 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6540 "Unexpected register store without SVE store instructions");
6541 Opc = AArch64::STR_ZZXI;
6542 StackID = TargetStackID::ScalableVector;
6543 }
6544 break;
6545 case 48:
6546 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6547 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6548 Opc = AArch64::ST1Threev2d;
6549 Offset = false;
6550 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6551 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6552 "Unexpected register store without SVE store instructions");
6553 Opc = AArch64::STR_ZZZXI;
6554 StackID = TargetStackID::ScalableVector;
6555 }
6556 break;
6557 case 64:
6558 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6559 assert(Subtarget.hasNEON() && "Unexpected register store without NEON");
6560 Opc = AArch64::ST1Fourv2d;
6561 Offset = false;
6562 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6563 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6564 "Unexpected register store without SVE store instructions");
6565 Opc = AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS;
6566 StackID = TargetStackID::ScalableVector;
6567 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6568 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6569 "Unexpected register store without SVE store instructions");
6570 Opc = AArch64::STR_ZZZZXI;
6571 StackID = TargetStackID::ScalableVector;
6572 }
6573 break;
6574 }
6575 assert(Opc && "Unknown register class");
6576 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6577
6578 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6579 .addReg(RegNo: SrcReg, Flags: getKillRegState(B: isKill))
6580 .addFrameIndex(Idx: FI);
6581
6582 if (Offset)
6583 MI.addImm(Val: 0);
6584 if (PNRReg.isValid())
6585 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6586 MI.addMemOperand(MMO);
6587}
6588
6589static void loadRegPairFromStackSlot(const TargetRegisterInfo &TRI,
6590 MachineBasicBlock &MBB,
6591 MachineBasicBlock::iterator InsertBefore,
6592 const MCInstrDesc &MCID,
6593 Register DestReg, unsigned SubIdx0,
6594 unsigned SubIdx1, int FI,
6595 MachineMemOperand *MMO) {
6596 Register DestReg0 = DestReg;
6597 Register DestReg1 = DestReg;
6598 bool IsUndef = true;
6599 if (DestReg.isPhysical()) {
6600 DestReg0 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx0);
6601 SubIdx0 = 0;
6602 DestReg1 = TRI.getSubReg(Reg: DestReg, Idx: SubIdx1);
6603 SubIdx1 = 0;
6604 IsUndef = false;
6605 }
6606 BuildMI(BB&: MBB, I: InsertBefore, MIMD: DebugLoc(), MCID)
6607 .addReg(RegNo: DestReg0, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx0)
6608 .addReg(RegNo: DestReg1, Flags: RegState::Define | getUndefRegState(B: IsUndef), SubReg: SubIdx1)
6609 .addFrameIndex(Idx: FI)
6610 .addImm(Val: 0)
6611 .addMemOperand(MMO);
6612}
6613
6614void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
6615 MachineBasicBlock::iterator MBBI,
6616 Register DestReg, int FI,
6617 const TargetRegisterClass *RC,
6618 Register VReg, unsigned SubReg,
6619 MachineInstr::MIFlag Flags) const {
6620 MachineFunction &MF = *MBB.getParent();
6621 MachineFrameInfo &MFI = MF.getFrameInfo();
6622 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
6623 MachineMemOperand *MMO =
6624 MF.getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOLoad,
6625 Size: MFI.getObjectSize(ObjectIdx: FI), BaseAlignment: MFI.getObjectAlign(ObjectIdx: FI));
6626
6627 unsigned Opc = 0;
6628 bool Offset = true;
6629 unsigned StackID = TargetStackID::Default;
6630 Register PNRReg;
6631 switch (TRI.getSpillSize(RC: *RC)) {
6632 case 1:
6633 if (AArch64::FPR8RegClass.hasSubClassEq(RC))
6634 Opc = AArch64::LDRBui;
6635 break;
6636 case 2: {
6637 bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
6638 if (AArch64::FPR16RegClass.hasSubClassEq(RC))
6639 Opc = AArch64::LDRHui;
6640 else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
6641 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6642 "Unexpected register load without SVE load instructions");
6643 if (IsPNR)
6644 PNRReg = DestReg;
6645 Opc = AArch64::LDR_PXI;
6646 StackID = TargetStackID::ScalablePredicateVector;
6647 }
6648 break;
6649 }
6650 case 4:
6651 if (AArch64::GPR32allRegClass.hasSubClassEq(RC)) {
6652 Opc = AArch64::LDRWui;
6653 if (DestReg.isVirtual())
6654 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR32RegClass);
6655 else
6656 assert(DestReg != AArch64::WSP);
6657 } else if (AArch64::FPR32RegClass.hasSubClassEq(RC))
6658 Opc = AArch64::LDRSui;
6659 else if (AArch64::PPR2RegClass.hasSubClassEq(RC)) {
6660 Opc = AArch64::LDR_PPXI;
6661 StackID = TargetStackID::ScalablePredicateVector;
6662 }
6663 break;
6664 case 8:
6665 if (AArch64::GPR64allRegClass.hasSubClassEq(RC)) {
6666 Opc = AArch64::LDRXui;
6667 if (DestReg.isVirtual())
6668 MF.getRegInfo().constrainRegClass(Reg: DestReg, RC: &AArch64::GPR64RegClass);
6669 else
6670 assert(DestReg != AArch64::SP);
6671 } else if (AArch64::FPR64RegClass.hasSubClassEq(RC)) {
6672 Opc = AArch64::LDRDui;
6673 } else if (AArch64::WSeqPairsClassRegClass.hasSubClassEq(RC)) {
6674 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6675 MCID: get(Opcode: AArch64::LDPWi), DestReg, SubIdx0: AArch64::sube32,
6676 SubIdx1: AArch64::subo32, FI, MMO);
6677 return;
6678 }
6679 break;
6680 case 16:
6681 if (AArch64::FPR128RegClass.hasSubClassEq(RC))
6682 Opc = AArch64::LDRQui;
6683 else if (AArch64::DDRegClass.hasSubClassEq(RC)) {
6684 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6685 Opc = AArch64::LD1Twov1d;
6686 Offset = false;
6687 } else if (AArch64::XSeqPairsClassRegClass.hasSubClassEq(RC)) {
6688 loadRegPairFromStackSlot(TRI: getRegisterInfo(), MBB, InsertBefore: MBBI,
6689 MCID: get(Opcode: AArch64::LDPXi), DestReg, SubIdx0: AArch64::sube64,
6690 SubIdx1: AArch64::subo64, FI, MMO);
6691 return;
6692 } else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
6693 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6694 "Unexpected register load without SVE load instructions");
6695 Opc = AArch64::LDR_ZXI;
6696 StackID = TargetStackID::ScalableVector;
6697 }
6698 break;
6699 case 24:
6700 if (AArch64::DDDRegClass.hasSubClassEq(RC)) {
6701 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6702 Opc = AArch64::LD1Threev1d;
6703 Offset = false;
6704 }
6705 break;
6706 case 32:
6707 if (AArch64::DDDDRegClass.hasSubClassEq(RC)) {
6708 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6709 Opc = AArch64::LD1Fourv1d;
6710 Offset = false;
6711 } else if (AArch64::QQRegClass.hasSubClassEq(RC)) {
6712 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6713 Opc = AArch64::LD1Twov2d;
6714 Offset = false;
6715 } else if (AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6716 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6717 "Unexpected register load without SVE load instructions");
6718 Opc = AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS;
6719 StackID = TargetStackID::ScalableVector;
6720 } else if (AArch64::ZPR2RegClass.hasSubClassEq(RC)) {
6721 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6722 "Unexpected register load without SVE load instructions");
6723 Opc = AArch64::LDR_ZZXI;
6724 StackID = TargetStackID::ScalableVector;
6725 }
6726 break;
6727 case 48:
6728 if (AArch64::QQQRegClass.hasSubClassEq(RC)) {
6729 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6730 Opc = AArch64::LD1Threev2d;
6731 Offset = false;
6732 } else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
6733 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6734 "Unexpected register load without SVE load instructions");
6735 Opc = AArch64::LDR_ZZZXI;
6736 StackID = TargetStackID::ScalableVector;
6737 }
6738 break;
6739 case 64:
6740 if (AArch64::QQQQRegClass.hasSubClassEq(RC)) {
6741 assert(Subtarget.hasNEON() && "Unexpected register load without NEON");
6742 Opc = AArch64::LD1Fourv2d;
6743 Offset = false;
6744 } else if (AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
6745 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6746 "Unexpected register load without SVE load instructions");
6747 Opc = AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS;
6748 StackID = TargetStackID::ScalableVector;
6749 } else if (AArch64::ZPR4RegClass.hasSubClassEq(RC)) {
6750 assert(Subtarget.isSVEorStreamingSVEAvailable() &&
6751 "Unexpected register load without SVE load instructions");
6752 Opc = AArch64::LDR_ZZZZXI;
6753 StackID = TargetStackID::ScalableVector;
6754 }
6755 break;
6756 }
6757
6758 assert(Opc && "Unknown register class");
6759 MFI.setStackID(ObjectIdx: FI, ID: StackID);
6760
6761 const MachineInstrBuilder MI = BuildMI(BB&: MBB, I: MBBI, MIMD: DebugLoc(), MCID: get(Opcode: Opc))
6762 .addReg(RegNo: DestReg, Flags: getDefRegState(B: true))
6763 .addFrameIndex(Idx: FI);
6764 if (Offset)
6765 MI.addImm(Val: 0);
6766 if (PNRReg.isValid() && !PNRReg.isVirtual())
6767 MI.addDef(RegNo: PNRReg, Flags: RegState::Implicit);
6768 MI.addMemOperand(MMO);
6769}
6770
6771bool llvm::isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
6772 const MachineInstr &UseMI,
6773 const TargetRegisterInfo *TRI) {
6774 return any_of(Range: instructionsWithoutDebug(It: std::next(x: DefMI.getIterator()),
6775 End: UseMI.getIterator()),
6776 P: [TRI](const MachineInstr &I) {
6777 return I.modifiesRegister(Reg: AArch64::NZCV, TRI) ||
6778 I.readsRegister(Reg: AArch64::NZCV, TRI);
6779 });
6780}
6781
6782void AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6783 const StackOffset &Offset, int64_t &ByteSized, int64_t &VGSized) {
6784 // The smallest scalable element supported by scaled SVE addressing
6785 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6786 // byte offset must always be a multiple of 2.
6787 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6788
6789 // VGSized offsets are divided by '2', because the VG register is the
6790 // the number of 64bit granules as opposed to 128bit vector chunks,
6791 // which is how the 'n' in e.g. MVT::nxv1i8 is modelled.
6792 // So, for a stack offset of 16 MVT::nxv1i8's, the size is n x 16 bytes.
6793 // VG = n * 2 and the dwarf offset must be VG * 8 bytes.
6794 ByteSized = Offset.getFixed();
6795 VGSized = Offset.getScalable() / 2;
6796}
6797
6798/// Returns the offset in parts to which this frame offset can be
6799/// decomposed for the purpose of describing a frame offset.
6800/// For non-scalable offsets this is simply its byte size.
6801void AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
6802 const StackOffset &Offset, int64_t &NumBytes, int64_t &NumPredicateVectors,
6803 int64_t &NumDataVectors) {
6804 // The smallest scalable element supported by scaled SVE addressing
6805 // modes are predicates, which are 2 scalable bytes in size. So the scalable
6806 // byte offset must always be a multiple of 2.
6807 assert(Offset.getScalable() % 2 == 0 && "Invalid frame offset");
6808
6809 NumBytes = Offset.getFixed();
6810 NumDataVectors = 0;
6811 NumPredicateVectors = Offset.getScalable() / 2;
6812 // This method is used to get the offsets to adjust the frame offset.
6813 // If the function requires ADDPL to be used and needs more than two ADDPL
6814 // instructions, part of the offset is folded into NumDataVectors so that it
6815 // uses ADDVL for part of it, reducing the number of ADDPL instructions.
6816 if (NumPredicateVectors % 8 == 0 || NumPredicateVectors < -64 ||
6817 NumPredicateVectors > 62) {
6818 NumDataVectors = NumPredicateVectors / 8;
6819 NumPredicateVectors -= NumDataVectors * 8;
6820 }
6821}
6822
6823// Convenience function to create a DWARF expression for: Constant `Operation`.
6824// This helper emits compact sequences for common cases. For example, for`-15
6825// DW_OP_plus`, this helper would create DW_OP_lit15 DW_OP_minus.
6826static void appendConstantExpr(SmallVectorImpl<char> &Expr, int64_t Constant,
6827 dwarf::LocationAtom Operation) {
6828 if (Operation == dwarf::DW_OP_plus && Constant < 0 && -Constant <= 31) {
6829 // -Constant (1 to 31)
6830 Expr.push_back(Elt: dwarf::DW_OP_lit0 - Constant);
6831 Operation = dwarf::DW_OP_minus;
6832 } else if (Constant >= 0 && Constant <= 31) {
6833 // Literal value 0 to 31
6834 Expr.push_back(Elt: dwarf::DW_OP_lit0 + Constant);
6835 } else {
6836 // Signed constant
6837 Expr.push_back(Elt: dwarf::DW_OP_consts);
6838 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: Constant);
6839 }
6840 return Expr.push_back(Elt: Operation);
6841}
6842
6843// Convenience function to create a DWARF expression for a register.
6844static void appendReadRegExpr(SmallVectorImpl<char> &Expr, unsigned RegNum) {
6845 Expr.push_back(Elt: (char)dwarf::DW_OP_bregx);
6846 appendLEB128<LEB128Sign::Unsigned>(Buffer&: Expr, Value: RegNum);
6847 Expr.push_back(Elt: 0);
6848}
6849
6850// Convenience function to create a DWARF expression for loading a register from
6851// a CFA offset.
6852static void appendLoadRegExpr(SmallVectorImpl<char> &Expr,
6853 int64_t OffsetFromDefCFA) {
6854 // This assumes the top of the DWARF stack contains the CFA.
6855 Expr.push_back(Elt: dwarf::DW_OP_dup);
6856 // Add the offset to the register.
6857 appendConstantExpr(Expr, Constant: OffsetFromDefCFA, Operation: dwarf::DW_OP_plus);
6858 // Dereference the address (loads a 64 bit value)..
6859 Expr.push_back(Elt: dwarf::DW_OP_deref);
6860}
6861
6862// Convenience function to create a comment for
6863// (+/-) NumBytes (* RegScale)?
6864static void appendOffsetComment(int NumBytes, llvm::raw_string_ostream &Comment,
6865 StringRef RegScale = {}) {
6866 if (NumBytes) {
6867 Comment << (NumBytes < 0 ? " - " : " + ") << std::abs(x: NumBytes);
6868 if (!RegScale.empty())
6869 Comment << ' ' << RegScale;
6870 }
6871}
6872
6873// Creates an MCCFIInstruction:
6874// { DW_CFA_def_cfa_expression, ULEB128 (sizeof expr), expr }
6875static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI,
6876 unsigned Reg,
6877 const StackOffset &Offset) {
6878 int64_t NumBytes, NumVGScaledBytes;
6879 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(Offset, ByteSized&: NumBytes,
6880 VGSized&: NumVGScaledBytes);
6881 std::string CommentBuffer;
6882 llvm::raw_string_ostream Comment(CommentBuffer);
6883
6884 if (Reg == AArch64::SP)
6885 Comment << "sp";
6886 else if (Reg == AArch64::FP)
6887 Comment << "fp";
6888 else
6889 Comment << printReg(Reg, TRI: &TRI);
6890
6891 // Build up the expression (Reg + NumBytes + VG * NumVGScaledBytes)
6892 SmallString<64> Expr;
6893 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6894 assert(DwarfReg <= 31 && "DwarfReg out of bounds (0..31)");
6895 // Reg + NumBytes
6896 Expr.push_back(Elt: dwarf::DW_OP_breg0 + DwarfReg);
6897 appendLEB128<LEB128Sign::Signed>(Buffer&: Expr, Value: NumBytes);
6898 appendOffsetComment(NumBytes, Comment);
6899 if (NumVGScaledBytes) {
6900 // + VG * NumVGScaledBytes
6901 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: "* VG");
6902 appendReadRegExpr(Expr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6903 appendConstantExpr(Expr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6904 Expr.push_back(Elt: dwarf::DW_OP_plus);
6905 }
6906
6907 // Wrap this into DW_CFA_def_cfa.
6908 SmallString<64> DefCfaExpr;
6909 DefCfaExpr.push_back(Elt: dwarf::DW_CFA_def_cfa_expression);
6910 appendLEB128<LEB128Sign::Unsigned>(Buffer&: DefCfaExpr, Value: Expr.size());
6911 DefCfaExpr.append(RHS: Expr.str());
6912 return MCCFIInstruction::createEscape(L: nullptr, Vals: DefCfaExpr.str(), Loc: SMLoc(),
6913 Comment: Comment.str());
6914}
6915
6916MCCFIInstruction llvm::createDefCFA(const TargetRegisterInfo &TRI,
6917 unsigned FrameReg, unsigned Reg,
6918 const StackOffset &Offset,
6919 bool LastAdjustmentWasScalable) {
6920 if (Offset.getScalable())
6921 return createDefCFAExpression(TRI, Reg, Offset);
6922
6923 if (FrameReg == Reg && !LastAdjustmentWasScalable)
6924 return MCCFIInstruction::cfiDefCfaOffset(L: nullptr, Offset: int(Offset.getFixed()));
6925
6926 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6927 return MCCFIInstruction::cfiDefCfa(L: nullptr, Register: DwarfReg, Offset: (int)Offset.getFixed());
6928}
6929
6930MCCFIInstruction
6931llvm::createCFAOffset(const TargetRegisterInfo &TRI, unsigned Reg,
6932 const StackOffset &OffsetFromDefCFA,
6933 std::optional<int64_t> IncomingVGOffsetFromDefCFA) {
6934 int64_t NumBytes, NumVGScaledBytes;
6935 AArch64InstrInfo::decomposeStackOffsetForDwarfOffsets(
6936 Offset: OffsetFromDefCFA, ByteSized&: NumBytes, VGSized&: NumVGScaledBytes);
6937
6938 unsigned DwarfReg = TRI.getDwarfRegNum(Reg, isEH: true);
6939
6940 // Non-scalable offsets can use DW_CFA_offset directly.
6941 if (!NumVGScaledBytes)
6942 return MCCFIInstruction::createOffset(L: nullptr, Register: DwarfReg, Offset: NumBytes);
6943
6944 std::string CommentBuffer;
6945 llvm::raw_string_ostream Comment(CommentBuffer);
6946 Comment << printReg(Reg, TRI: &TRI) << " @ cfa";
6947
6948 // Build up expression (CFA + VG * NumVGScaledBytes + NumBytes)
6949 assert(NumVGScaledBytes && "Expected scalable offset");
6950 SmallString<64> OffsetExpr;
6951 // + VG * NumVGScaledBytes
6952 StringRef VGRegScale;
6953 if (IncomingVGOffsetFromDefCFA) {
6954 appendLoadRegExpr(Expr&: OffsetExpr, OffsetFromDefCFA: *IncomingVGOffsetFromDefCFA);
6955 VGRegScale = "* IncomingVG";
6956 } else {
6957 appendReadRegExpr(Expr&: OffsetExpr, RegNum: TRI.getDwarfRegNum(Reg: AArch64::VG, isEH: true));
6958 VGRegScale = "* VG";
6959 }
6960 appendConstantExpr(Expr&: OffsetExpr, Constant: NumVGScaledBytes, Operation: dwarf::DW_OP_mul);
6961 appendOffsetComment(NumBytes: NumVGScaledBytes, Comment, RegScale: VGRegScale);
6962 OffsetExpr.push_back(Elt: dwarf::DW_OP_plus);
6963 if (NumBytes) {
6964 // + NumBytes
6965 appendOffsetComment(NumBytes, Comment);
6966 appendConstantExpr(Expr&: OffsetExpr, Constant: NumBytes, Operation: dwarf::DW_OP_plus);
6967 }
6968
6969 // Wrap this into DW_CFA_expression
6970 SmallString<64> CfaExpr;
6971 CfaExpr.push_back(Elt: dwarf::DW_CFA_expression);
6972 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: DwarfReg);
6973 appendLEB128<LEB128Sign::Unsigned>(Buffer&: CfaExpr, Value: OffsetExpr.size());
6974 CfaExpr.append(RHS: OffsetExpr.str());
6975
6976 return MCCFIInstruction::createEscape(L: nullptr, Vals: CfaExpr.str(), Loc: SMLoc(),
6977 Comment: Comment.str());
6978}
6979
6980// Helper function to emit a frame offset adjustment from a given
6981// pointer (SrcReg), stored into DestReg. This function is explicit
6982// in that it requires the opcode.
6983static void emitFrameOffsetAdj(MachineBasicBlock &MBB,
6984 MachineBasicBlock::iterator MBBI,
6985 const DebugLoc &DL, unsigned DestReg,
6986 unsigned SrcReg, int64_t Offset, unsigned Opc,
6987 const TargetInstrInfo *TII,
6988 MachineInstr::MIFlag Flag, bool NeedsWinCFI,
6989 bool *HasWinCFI, bool EmitCFAOffset,
6990 StackOffset CFAOffset, unsigned FrameReg) {
6991 int Sign = 1;
6992 unsigned MaxEncoding, ShiftSize;
6993 switch (Opc) {
6994 case AArch64::ADDXri:
6995 case AArch64::ADDSXri:
6996 case AArch64::SUBXri:
6997 case AArch64::SUBSXri:
6998 MaxEncoding = 0xfff;
6999 ShiftSize = 12;
7000 break;
7001 case AArch64::ADDVL_XXI:
7002 case AArch64::ADDPL_XXI:
7003 case AArch64::ADDSVL_XXI:
7004 case AArch64::ADDSPL_XXI:
7005 MaxEncoding = 31;
7006 ShiftSize = 0;
7007 if (Offset < 0) {
7008 MaxEncoding = 32;
7009 Sign = -1;
7010 Offset = -Offset;
7011 }
7012 break;
7013 default:
7014 llvm_unreachable("Unsupported opcode");
7015 }
7016
7017 // `Offset` can be in bytes or in "scalable bytes".
7018 int VScale = 1;
7019 if (Opc == AArch64::ADDVL_XXI || Opc == AArch64::ADDSVL_XXI)
7020 VScale = 16;
7021 else if (Opc == AArch64::ADDPL_XXI || Opc == AArch64::ADDSPL_XXI)
7022 VScale = 2;
7023
7024 // FIXME: If the offset won't fit in 24-bits, compute the offset into a
7025 // scratch register. If DestReg is a virtual register, use it as the
7026 // scratch register; otherwise, create a new virtual register (to be
7027 // replaced by the scavenger at the end of PEI). That case can be optimized
7028 // slightly if DestReg is SP which is always 16-byte aligned, so the scratch
7029 // register can be loaded with offset%8 and the add/sub can use an extending
7030 // instruction with LSL#3.
7031 // Currently the function handles any offsets but generates a poor sequence
7032 // of code.
7033 // assert(Offset < (1 << 24) && "unimplemented reg plus immediate");
7034
7035 const unsigned MaxEncodableValue = MaxEncoding << ShiftSize;
7036 Register TmpReg = DestReg;
7037 if (TmpReg == AArch64::XZR)
7038 TmpReg = MBB.getParent()->getRegInfo().createVirtualRegister(
7039 RegClass: &AArch64::GPR64RegClass);
7040 do {
7041 uint64_t ThisVal = std::min<uint64_t>(a: Offset, b: MaxEncodableValue);
7042 unsigned LocalShiftSize = 0;
7043 if (ThisVal > MaxEncoding) {
7044 ThisVal = ThisVal >> ShiftSize;
7045 LocalShiftSize = ShiftSize;
7046 }
7047 assert((ThisVal >> ShiftSize) <= MaxEncoding &&
7048 "Encoding cannot handle value that big");
7049
7050 Offset -= ThisVal << LocalShiftSize;
7051 if (Offset == 0)
7052 TmpReg = DestReg;
7053 auto MBI = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: TmpReg)
7054 .addReg(RegNo: SrcReg)
7055 .addImm(Val: Sign * (int)ThisVal);
7056 if (ShiftSize)
7057 MBI = MBI.addImm(
7058 Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: LocalShiftSize));
7059 MBI = MBI.setMIFlag(Flag);
7060
7061 auto Change =
7062 VScale == 1
7063 ? StackOffset::getFixed(Fixed: ThisVal << LocalShiftSize)
7064 : StackOffset::getScalable(Scalable: VScale * (ThisVal << LocalShiftSize));
7065 if (Sign == -1 || Opc == AArch64::SUBXri || Opc == AArch64::SUBSXri)
7066 CFAOffset += Change;
7067 else
7068 CFAOffset -= Change;
7069 if (EmitCFAOffset && DestReg == TmpReg) {
7070 MachineFunction &MF = *MBB.getParent();
7071 const TargetSubtargetInfo &STI = MF.getSubtarget();
7072 const TargetRegisterInfo &TRI = *STI.getRegisterInfo();
7073
7074 unsigned CFIIndex = MF.addFrameInst(
7075 Inst: createDefCFA(TRI, FrameReg, Reg: DestReg, Offset: CFAOffset, LastAdjustmentWasScalable: VScale != 1));
7076 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
7077 .addCFIIndex(CFIIndex)
7078 .setMIFlags(Flag);
7079 }
7080
7081 if (NeedsWinCFI) {
7082 int Imm = (int)(ThisVal << LocalShiftSize);
7083 if (VScale != 1 && DestReg == AArch64::SP) {
7084 if (HasWinCFI)
7085 *HasWinCFI = true;
7086 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AllocZ))
7087 .addImm(Val: ThisVal)
7088 .setMIFlag(Flag);
7089 } else if ((DestReg == AArch64::FP && SrcReg == AArch64::SP) ||
7090 (SrcReg == AArch64::FP && DestReg == AArch64::SP)) {
7091 assert(VScale == 1 && "Expected non-scalable operation");
7092 if (HasWinCFI)
7093 *HasWinCFI = true;
7094 if (Imm == 0)
7095 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_SetFP)).setMIFlag(Flag);
7096 else
7097 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_AddFP))
7098 .addImm(Val: Imm)
7099 .setMIFlag(Flag);
7100 assert(Offset == 0 && "Expected remaining offset to be zero to "
7101 "emit a single SEH directive");
7102 } else if (DestReg == AArch64::SP) {
7103 assert(VScale == 1 && "Expected non-scalable operation");
7104 if (HasWinCFI)
7105 *HasWinCFI = true;
7106 assert(SrcReg == AArch64::SP && "Unexpected SrcReg for SEH_StackAlloc");
7107 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::SEH_StackAlloc))
7108 .addImm(Val: Imm)
7109 .setMIFlag(Flag);
7110 }
7111 }
7112
7113 SrcReg = TmpReg;
7114 } while (Offset);
7115}
7116
7117void llvm::emitFrameOffset(MachineBasicBlock &MBB,
7118 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
7119 unsigned DestReg, unsigned SrcReg,
7120 StackOffset Offset, const TargetInstrInfo *TII,
7121 MachineInstr::MIFlag Flag, bool SetNZCV,
7122 bool NeedsWinCFI, bool *HasWinCFI,
7123 bool EmitCFAOffset, StackOffset CFAOffset,
7124 unsigned FrameReg) {
7125 // If a function is marked as arm_locally_streaming, then the runtime value of
7126 // vscale in the prologue/epilogue is different the runtime value of vscale
7127 // in the function's body. To avoid having to consider multiple vscales,
7128 // we can use `addsvl` to allocate any scalable stack-slots, which under
7129 // most circumstances will be only locals, not callee-save slots.
7130 const Function &F = MBB.getParent()->getFunction();
7131 bool UseSVL = F.hasFnAttribute(Kind: "aarch64_pstate_sm_body");
7132
7133 int64_t Bytes, NumPredicateVectors, NumDataVectors;
7134 AArch64InstrInfo::decomposeStackOffsetForFrameOffsets(
7135 Offset, NumBytes&: Bytes, NumPredicateVectors, NumDataVectors);
7136
7137 // Insert ADDSXri for scalable offset at the end.
7138 bool NeedsFinalDefNZCV = SetNZCV && (NumPredicateVectors || NumDataVectors);
7139 if (NeedsFinalDefNZCV)
7140 SetNZCV = false;
7141
7142 // First emit non-scalable frame offsets, or a simple 'mov'.
7143 if (Bytes || (!Offset && SrcReg != DestReg)) {
7144 assert((DestReg != AArch64::SP || Bytes % 8 == 0) &&
7145 "SP increment/decrement not 8-byte aligned");
7146 unsigned Opc = SetNZCV ? AArch64::ADDSXri : AArch64::ADDXri;
7147 if (Bytes < 0) {
7148 Bytes = -Bytes;
7149 Opc = SetNZCV ? AArch64::SUBSXri : AArch64::SUBXri;
7150 }
7151 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: Bytes, Opc, TII, Flag,
7152 NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7153 FrameReg);
7154 CFAOffset += (Opc == AArch64::ADDXri || Opc == AArch64::ADDSXri)
7155 ? StackOffset::getFixed(Fixed: -Bytes)
7156 : StackOffset::getFixed(Fixed: Bytes);
7157 SrcReg = DestReg;
7158 FrameReg = DestReg;
7159 }
7160
7161 assert(!(NeedsWinCFI && NumPredicateVectors) &&
7162 "WinCFI can't allocate fractions of an SVE data vector");
7163
7164 if (NumDataVectors) {
7165 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumDataVectors,
7166 Opc: UseSVL ? AArch64::ADDSVL_XXI : AArch64::ADDVL_XXI, TII,
7167 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7168 FrameReg);
7169 CFAOffset += StackOffset::getScalable(Scalable: -NumDataVectors * 16);
7170 SrcReg = DestReg;
7171 }
7172
7173 if (NumPredicateVectors) {
7174 assert(DestReg != AArch64::SP && "Unaligned access to SP");
7175 emitFrameOffsetAdj(MBB, MBBI, DL, DestReg, SrcReg, Offset: NumPredicateVectors,
7176 Opc: UseSVL ? AArch64::ADDSPL_XXI : AArch64::ADDPL_XXI, TII,
7177 Flag, NeedsWinCFI, HasWinCFI, EmitCFAOffset, CFAOffset,
7178 FrameReg);
7179 }
7180
7181 if (NeedsFinalDefNZCV)
7182 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDSXri), DestReg)
7183 .addReg(RegNo: DestReg)
7184 .addImm(Val: 0)
7185 .addImm(Val: 0);
7186}
7187
7188MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl(
7189 MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops,
7190 int FrameIndex, MachineInstr *&CopyMI, LiveIntervals *LIS,
7191 VirtRegMap *VRM) const {
7192 MachineBasicBlock::iterator InsertPt = MI;
7193 // This is a bit of a hack. Consider this instruction:
7194 //
7195 // %0 = COPY %sp; GPR64all:%0
7196 //
7197 // We explicitly chose GPR64all for the virtual register so such a copy might
7198 // be eliminated by RegisterCoalescer. However, that may not be possible, and
7199 // %0 may even spill. We can't spill %sp, and since it is in the GPR64all
7200 // register class, TargetInstrInfo::foldMemoryOperand() is going to try.
7201 //
7202 // To prevent that, we are going to constrain the %0 register class here.
7203 if (MI.isFullCopy()) {
7204 Register DstReg = MI.getOperand(i: 0).getReg();
7205 Register SrcReg = MI.getOperand(i: 1).getReg();
7206 if (SrcReg == AArch64::SP && DstReg.isVirtual()) {
7207 MF.getRegInfo().constrainRegClass(Reg: DstReg, RC: &AArch64::GPR64RegClass);
7208 return nullptr;
7209 }
7210 if (DstReg == AArch64::SP && SrcReg.isVirtual()) {
7211 MF.getRegInfo().constrainRegClass(Reg: SrcReg, RC: &AArch64::GPR64RegClass);
7212 return nullptr;
7213 }
7214 // Nothing can folded with copy from/to NZCV.
7215 if (SrcReg == AArch64::NZCV || DstReg == AArch64::NZCV)
7216 return nullptr;
7217 }
7218
7219 // Handle the case where a copy is being spilled or filled but the source
7220 // and destination register class don't match. For example:
7221 //
7222 // %0 = COPY %xzr; GPR64common:%0
7223 //
7224 // In this case we can still safely fold away the COPY and generate the
7225 // following spill code:
7226 //
7227 // STRXui %xzr, %stack.0
7228 //
7229 // This also eliminates spilled cross register class COPYs (e.g. between x and
7230 // d regs) of the same size. For example:
7231 //
7232 // %0 = COPY %1; GPR64:%0, FPR64:%1
7233 //
7234 // will be filled as
7235 //
7236 // LDRDui %0, fi<#0>
7237 //
7238 // instead of
7239 //
7240 // LDRXui %Temp, fi<#0>
7241 // %0 = FMOV %Temp
7242 //
7243 if (MI.isCopy() && Ops.size() == 1 &&
7244 // Make sure we're only folding the explicit COPY defs/uses.
7245 (Ops[0] == 0 || Ops[0] == 1)) {
7246 bool IsSpill = Ops[0] == 0;
7247 bool IsFill = !IsSpill;
7248 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7249 const MachineRegisterInfo &MRI = MF.getRegInfo();
7250 MachineBasicBlock &MBB = *MI.getParent();
7251 const MachineOperand &DstMO = MI.getOperand(i: 0);
7252 const MachineOperand &SrcMO = MI.getOperand(i: 1);
7253 Register DstReg = DstMO.getReg();
7254 Register SrcReg = SrcMO.getReg();
7255 // This is slightly expensive to compute for physical regs since
7256 // getMinimalPhysRegClass is slow.
7257 auto getRegClass = [&](unsigned Reg) {
7258 return Register::isVirtualRegister(Reg) ? MRI.getRegClass(Reg)
7259 : TRI.getMinimalPhysRegClass(Reg);
7260 };
7261
7262 if (DstMO.getSubReg() == 0 && SrcMO.getSubReg() == 0) {
7263 assert(TRI.getRegSizeInBits(*getRegClass(DstReg)) ==
7264 TRI.getRegSizeInBits(*getRegClass(SrcReg)) &&
7265 "Mismatched register size in non subreg COPY");
7266 if (IsSpill)
7267 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg, isKill: SrcMO.isKill(), FI: FrameIndex,
7268 RC: getRegClass(SrcReg), VReg: Register());
7269 else
7270 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex,
7271 RC: getRegClass(DstReg), VReg: Register());
7272 return &*--InsertPt;
7273 }
7274
7275 // Handle cases like spilling def of:
7276 //
7277 // %0:sub_32<def,read-undef> = COPY %wzr; GPR64common:%0
7278 //
7279 // where the physical register source can be widened and stored to the full
7280 // virtual reg destination stack slot, in this case producing:
7281 //
7282 // STRXui %xzr, %stack.0
7283 //
7284 if (IsSpill && DstMO.isUndef() && SrcReg == AArch64::WZR &&
7285 TRI.getRegSizeInBits(RC: *getRegClass(DstReg)) == 64) {
7286 assert(SrcMO.getSubReg() == 0 &&
7287 "Unexpected subreg on physical register");
7288 storeRegToStackSlot(MBB, MBBI: InsertPt, SrcReg: AArch64::XZR, isKill: SrcMO.isKill(),
7289 FI: FrameIndex, RC: &AArch64::GPR64RegClass, VReg: Register());
7290 return &*--InsertPt;
7291 }
7292
7293 // Handle cases like filling use of:
7294 //
7295 // %0:sub_32<def,read-undef> = COPY %1; GPR64:%0, GPR32:%1
7296 //
7297 // where we can load the full virtual reg source stack slot, into the subreg
7298 // destination, in this case producing:
7299 //
7300 // LDRWui %0:sub_32<def,read-undef>, %stack.0
7301 //
7302 if (IsFill && SrcMO.getSubReg() == 0 && DstMO.isUndef()) {
7303 const TargetRegisterClass *FillRC = nullptr;
7304 switch (DstMO.getSubReg()) {
7305 default:
7306 break;
7307 case AArch64::sub_32:
7308 if (AArch64::GPR64RegClass.hasSubClassEq(RC: getRegClass(DstReg)))
7309 FillRC = &AArch64::GPR32RegClass;
7310 break;
7311 case AArch64::ssub:
7312 FillRC = &AArch64::FPR32RegClass;
7313 break;
7314 case AArch64::dsub:
7315 FillRC = &AArch64::FPR64RegClass;
7316 break;
7317 }
7318
7319 if (FillRC) {
7320 assert(TRI.getRegSizeInBits(*getRegClass(SrcReg)) ==
7321 TRI.getRegSizeInBits(*FillRC) &&
7322 "Mismatched regclass size on folded subreg COPY");
7323 loadRegFromStackSlot(MBB, MBBI: InsertPt, DestReg: DstReg, FI: FrameIndex, RC: FillRC,
7324 VReg: Register());
7325 MachineInstr &LoadMI = *--InsertPt;
7326 MachineOperand &LoadDst = LoadMI.getOperand(i: 0);
7327 assert(LoadDst.getSubReg() == 0 && "unexpected subreg on fill load");
7328 LoadDst.setSubReg(DstMO.getSubReg());
7329 LoadDst.setIsUndef();
7330 return &LoadMI;
7331 }
7332 }
7333 }
7334
7335 // Cannot fold.
7336 return nullptr;
7337}
7338
7339int llvm::isAArch64FrameOffsetLegal(const MachineInstr &MI,
7340 StackOffset &SOffset,
7341 bool *OutUseUnscaledOp,
7342 unsigned *OutUnscaledOp,
7343 int64_t *EmittableOffset) {
7344 // Set output values in case of early exit.
7345 if (EmittableOffset)
7346 *EmittableOffset = 0;
7347 if (OutUseUnscaledOp)
7348 *OutUseUnscaledOp = false;
7349 if (OutUnscaledOp)
7350 *OutUnscaledOp = 0;
7351
7352 // Exit early for structured vector spills/fills as they can't take an
7353 // immediate offset.
7354 switch (MI.getOpcode()) {
7355 default:
7356 break;
7357 case AArch64::LD1Rv1d:
7358 case AArch64::LD1Rv2s:
7359 case AArch64::LD1Rv2d:
7360 case AArch64::LD1Rv4h:
7361 case AArch64::LD1Rv4s:
7362 case AArch64::LD1Rv8b:
7363 case AArch64::LD1Rv8h:
7364 case AArch64::LD1Rv16b:
7365 case AArch64::LD1Twov2d:
7366 case AArch64::LD1Threev2d:
7367 case AArch64::LD1Fourv2d:
7368 case AArch64::LD1Twov1d:
7369 case AArch64::LD1Threev1d:
7370 case AArch64::LD1Fourv1d:
7371 case AArch64::ST1Twov2d:
7372 case AArch64::ST1Threev2d:
7373 case AArch64::ST1Fourv2d:
7374 case AArch64::ST1Twov1d:
7375 case AArch64::ST1Threev1d:
7376 case AArch64::ST1Fourv1d:
7377 case AArch64::ST1i8:
7378 case AArch64::ST1i16:
7379 case AArch64::ST1i32:
7380 case AArch64::ST1i64:
7381 case AArch64::IRG:
7382 case AArch64::IRGstack:
7383 case AArch64::STGloop:
7384 case AArch64::STZGloop:
7385 return AArch64FrameOffsetCannotUpdate;
7386 }
7387
7388 // Get the min/max offset and the scale.
7389 TypeSize ScaleValue(0U, false), Width(0U, false);
7390 int64_t MinOff, MaxOff;
7391 if (!AArch64InstrInfo::getMemOpInfo(Opcode: MI.getOpcode(), Scale&: ScaleValue, Width, MinOffset&: MinOff,
7392 MaxOffset&: MaxOff))
7393 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7394
7395 // Construct the complete offset.
7396 bool IsMulVL = ScaleValue.isScalable();
7397 unsigned Scale = ScaleValue.getKnownMinValue();
7398 int64_t Offset = IsMulVL ? SOffset.getScalable() : SOffset.getFixed();
7399
7400 const MachineOperand &ImmOpnd =
7401 MI.getOperand(i: AArch64InstrInfo::getLoadStoreImmIdx(Opc: MI.getOpcode()));
7402 Offset += ImmOpnd.getImm() * Scale;
7403
7404 // If the offset doesn't match the scale, we rewrite the instruction to
7405 // use the unscaled instruction instead. Likewise, if we have a negative
7406 // offset and there is an unscaled op to use.
7407 std::optional<unsigned> UnscaledOp =
7408 AArch64InstrInfo::getUnscaledLdSt(Opc: MI.getOpcode());
7409 bool useUnscaledOp = UnscaledOp && (Offset % Scale || Offset < 0);
7410 if (useUnscaledOp &&
7411 !AArch64InstrInfo::getMemOpInfo(Opcode: *UnscaledOp, Scale&: ScaleValue, Width, MinOffset&: MinOff,
7412 MaxOffset&: MaxOff))
7413 llvm_unreachable("unhandled opcode in isAArch64FrameOffsetLegal");
7414
7415 Scale = ScaleValue.getKnownMinValue();
7416 assert(IsMulVL == ScaleValue.isScalable() &&
7417 "Unscaled opcode has different value for scalable");
7418
7419 int64_t Remainder = Offset % Scale;
7420 assert(!(Remainder && useUnscaledOp) &&
7421 "Cannot have remainder when using unscaled op");
7422
7423 assert(MinOff < MaxOff && "Unexpected Min/Max offsets");
7424 int64_t NewOffset = Offset / Scale;
7425 if (MinOff <= NewOffset && NewOffset <= MaxOff)
7426 Offset = Remainder;
7427 else {
7428 // Try to minimise the number of instructions required to materialise the
7429 // offset calculation. Specifically, for fixed offsets, if masking out the
7430 // low 12 bits leaves a legal add immediate, we can realise the offset
7431 // calculation with a single add instruction. Whenever this is possible,
7432 // prefer this split.
7433 int64_t HighPart = Offset & ~0xFFF;
7434 int64_t LowPart = Offset & 0xFFF;
7435 int64_t LowScaled = LowPart / Scale;
7436 if (!IsMulVL && NewOffset >= 0 && LowPart % Scale == 0 &&
7437 MinOff <= LowScaled && LowScaled <= MaxOff &&
7438 AArch64_AM::isLegalArithImmed(C: HighPart)) {
7439 NewOffset = LowScaled;
7440 Offset = HighPart;
7441 } else {
7442 // Default to a greedy split: take the memop immediate to be maximum /
7443 // minimum expressible offset and materialise the remainder.
7444 NewOffset = NewOffset < 0 ? MinOff : MaxOff;
7445 Offset = Offset - (NewOffset * Scale);
7446 }
7447 }
7448
7449 if (EmittableOffset)
7450 *EmittableOffset = NewOffset;
7451 if (OutUseUnscaledOp)
7452 *OutUseUnscaledOp = useUnscaledOp;
7453 if (OutUnscaledOp && UnscaledOp)
7454 *OutUnscaledOp = *UnscaledOp;
7455
7456 if (IsMulVL)
7457 SOffset = StackOffset::get(Fixed: SOffset.getFixed(), Scalable: Offset);
7458 else
7459 SOffset = StackOffset::get(Fixed: Offset, Scalable: SOffset.getScalable());
7460 return AArch64FrameOffsetCanUpdate |
7461 (SOffset ? 0 : AArch64FrameOffsetIsLegal);
7462}
7463
7464bool llvm::rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
7465 unsigned FrameReg, StackOffset &Offset,
7466 const AArch64InstrInfo *TII) {
7467 unsigned Opcode = MI.getOpcode();
7468 unsigned ImmIdx = FrameRegIdx + 1;
7469
7470 if (Opcode == AArch64::ADDSXri || Opcode == AArch64::ADDXri) {
7471 Offset += StackOffset::getFixed(Fixed: MI.getOperand(i: ImmIdx).getImm());
7472 emitFrameOffset(MBB&: *MI.getParent(), MBBI: MI, DL: MI.getDebugLoc(),
7473 DestReg: MI.getOperand(i: 0).getReg(), SrcReg: FrameReg, Offset, TII,
7474 Flag: MachineInstr::NoFlags, SetNZCV: (Opcode == AArch64::ADDSXri));
7475 MI.eraseFromParent();
7476 Offset = StackOffset();
7477 return true;
7478 }
7479
7480 int64_t NewOffset;
7481 unsigned UnscaledOp;
7482 bool UseUnscaledOp;
7483 int Status = isAArch64FrameOffsetLegal(MI, SOffset&: Offset, OutUseUnscaledOp: &UseUnscaledOp,
7484 OutUnscaledOp: &UnscaledOp, EmittableOffset: &NewOffset);
7485 if (Status & AArch64FrameOffsetCanUpdate) {
7486 if (Status & AArch64FrameOffsetIsLegal)
7487 // Replace the FrameIndex with FrameReg.
7488 MI.getOperand(i: FrameRegIdx).ChangeToRegister(Reg: FrameReg, isDef: false);
7489 if (UseUnscaledOp)
7490 MI.setDesc(TII->get(Opcode: UnscaledOp));
7491
7492 MI.getOperand(i: ImmIdx).ChangeToImmediate(ImmVal: NewOffset);
7493 return !Offset;
7494 }
7495
7496 return false;
7497}
7498
7499void AArch64InstrInfo::insertNoop(MachineBasicBlock &MBB,
7500 MachineBasicBlock::iterator MI) const {
7501 DebugLoc DL;
7502 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: get(Opcode: AArch64::NOP));
7503}
7504
7505MCInst AArch64InstrInfo::getNop() const { return MCInstBuilder(AArch64::NOP); }
7506
7507// AArch64 supports MachineCombiner.
7508bool AArch64InstrInfo::useMachineCombiner() const { return true; }
7509
7510// True when Opc sets flag
7511static bool isCombineInstrSettingFlag(unsigned Opc) {
7512 switch (Opc) {
7513 case AArch64::ADDSWrr:
7514 case AArch64::ADDSWri:
7515 case AArch64::ADDSXrr:
7516 case AArch64::ADDSXri:
7517 case AArch64::SUBSWrr:
7518 case AArch64::SUBSXrr:
7519 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7520 case AArch64::SUBSWri:
7521 case AArch64::SUBSXri:
7522 return true;
7523 default:
7524 break;
7525 }
7526 return false;
7527}
7528
7529// 32b Opcodes that can be combined with a MUL
7530static bool isCombineInstrCandidate32(unsigned Opc) {
7531 switch (Opc) {
7532 case AArch64::ADDWrr:
7533 case AArch64::ADDWri:
7534 case AArch64::SUBWrr:
7535 case AArch64::ADDSWrr:
7536 case AArch64::ADDSWri:
7537 case AArch64::SUBSWrr:
7538 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7539 case AArch64::SUBWri:
7540 case AArch64::SUBSWri:
7541 return true;
7542 default:
7543 break;
7544 }
7545 return false;
7546}
7547
7548// 64b Opcodes that can be combined with a MUL
7549static bool isCombineInstrCandidate64(unsigned Opc) {
7550 switch (Opc) {
7551 case AArch64::ADDXrr:
7552 case AArch64::ADDXri:
7553 case AArch64::SUBXrr:
7554 case AArch64::ADDSXrr:
7555 case AArch64::ADDSXri:
7556 case AArch64::SUBSXrr:
7557 // Note: MSUB Wd,Wn,Wm,Wi -> Wd = Wi - WnxWm, not Wd=WnxWm - Wi.
7558 case AArch64::SUBXri:
7559 case AArch64::SUBSXri:
7560 case AArch64::ADDv8i8:
7561 case AArch64::ADDv16i8:
7562 case AArch64::ADDv4i16:
7563 case AArch64::ADDv8i16:
7564 case AArch64::ADDv2i32:
7565 case AArch64::ADDv4i32:
7566 case AArch64::SUBv8i8:
7567 case AArch64::SUBv16i8:
7568 case AArch64::SUBv4i16:
7569 case AArch64::SUBv8i16:
7570 case AArch64::SUBv2i32:
7571 case AArch64::SUBv4i32:
7572 return true;
7573 default:
7574 break;
7575 }
7576 return false;
7577}
7578
7579// FP Opcodes that can be combined with a FMUL.
7580static bool isCombineInstrCandidateFP(const MachineInstr &Inst) {
7581 switch (Inst.getOpcode()) {
7582 default:
7583 break;
7584 case AArch64::FADDHrr:
7585 case AArch64::FADDSrr:
7586 case AArch64::FADDDrr:
7587 case AArch64::FADDv4f16:
7588 case AArch64::FADDv8f16:
7589 case AArch64::FADDv2f32:
7590 case AArch64::FADDv2f64:
7591 case AArch64::FADDv4f32:
7592 case AArch64::FSUBHrr:
7593 case AArch64::FSUBSrr:
7594 case AArch64::FSUBDrr:
7595 case AArch64::FSUBv4f16:
7596 case AArch64::FSUBv8f16:
7597 case AArch64::FSUBv2f32:
7598 case AArch64::FSUBv2f64:
7599 case AArch64::FSUBv4f32:
7600 // We can fuse FADD/FSUB with FMUL, if FADD/FSUB has the contract fast-math
7601 // flag.
7602 return Inst.getFlag(Flag: MachineInstr::FmContract);
7603 }
7604 return false;
7605}
7606
7607// Opcodes that can be combined with a MUL
7608static bool isCombineInstrCandidate(unsigned Opc) {
7609 return (isCombineInstrCandidate32(Opc) || isCombineInstrCandidate64(Opc));
7610}
7611
7612//
7613// Utility routine that checks if \param MO is defined by an
7614// \param CombineOpc instruction in the basic block \param MBB
7615static bool canCombine(MachineBasicBlock &MBB, MachineOperand &MO,
7616 unsigned CombineOpc, unsigned ZeroReg = 0,
7617 bool CheckZeroReg = false) {
7618 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
7619 MachineInstr *MI = nullptr;
7620
7621 if (MO.isReg() && MO.getReg().isVirtual())
7622 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
7623 // And it needs to be in the trace (otherwise, it won't have a depth).
7624 if (!MI || MI->getParent() != &MBB || MI->getOpcode() != CombineOpc)
7625 return false;
7626 // Must only used by the user we combine with.
7627 if (!MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()))
7628 return false;
7629
7630 if (CheckZeroReg) {
7631 assert(MI->getNumOperands() >= 4 && MI->getOperand(0).isReg() &&
7632 MI->getOperand(1).isReg() && MI->getOperand(2).isReg() &&
7633 MI->getOperand(3).isReg() && "MAdd/MSub must have a least 4 regs");
7634 // The third input reg must be zero.
7635 if (MI->getOperand(i: 3).getReg() != ZeroReg)
7636 return false;
7637 }
7638
7639 if (isCombineInstrSettingFlag(Opc: CombineOpc) &&
7640 MI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) == -1)
7641 return false;
7642
7643 return true;
7644}
7645
7646//
7647// Is \param MO defined by an integer multiply and can be combined?
7648static bool canCombineWithMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7649 unsigned MulOpc, unsigned ZeroReg) {
7650 return canCombine(MBB, MO, CombineOpc: MulOpc, ZeroReg, CheckZeroReg: true);
7651}
7652
7653//
7654// Is \param MO defined by a floating-point multiply and can be combined?
7655static bool canCombineWithFMUL(MachineBasicBlock &MBB, MachineOperand &MO,
7656 unsigned MulOpc) {
7657 return canCombine(MBB, MO, CombineOpc: MulOpc);
7658}
7659
7660// TODO: There are many more machine instruction opcodes to match:
7661// 1. Other data types (integer, vectors)
7662// 2. Other math / logic operations (xor, or)
7663// 3. Other forms of the same operation (intrinsics and other variants)
7664bool AArch64InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
7665 bool Invert) const {
7666 if (Invert)
7667 return false;
7668 switch (Inst.getOpcode()) {
7669 // == Floating-point types ==
7670 // -- Floating-point instructions --
7671 case AArch64::FADDHrr:
7672 case AArch64::FADDSrr:
7673 case AArch64::FADDDrr:
7674 case AArch64::FMULHrr:
7675 case AArch64::FMULSrr:
7676 case AArch64::FMULDrr:
7677 case AArch64::FMULX16:
7678 case AArch64::FMULX32:
7679 case AArch64::FMULX64:
7680 // -- Advanced SIMD instructions --
7681 case AArch64::FADDv4f16:
7682 case AArch64::FADDv8f16:
7683 case AArch64::FADDv2f32:
7684 case AArch64::FADDv4f32:
7685 case AArch64::FADDv2f64:
7686 case AArch64::FMULv4f16:
7687 case AArch64::FMULv8f16:
7688 case AArch64::FMULv2f32:
7689 case AArch64::FMULv4f32:
7690 case AArch64::FMULv2f64:
7691 case AArch64::FMULXv4f16:
7692 case AArch64::FMULXv8f16:
7693 case AArch64::FMULXv2f32:
7694 case AArch64::FMULXv4f32:
7695 case AArch64::FMULXv2f64:
7696 // -- SVE instructions --
7697 // Opcodes FMULX_ZZZ_? don't exist because there is no unpredicated FMULX
7698 // in the SVE instruction set (though there are predicated ones).
7699 case AArch64::FADD_ZZZ_H:
7700 case AArch64::FADD_ZZZ_S:
7701 case AArch64::FADD_ZZZ_D:
7702 case AArch64::FMUL_ZZZ_H:
7703 case AArch64::FMUL_ZZZ_S:
7704 case AArch64::FMUL_ZZZ_D:
7705 return Inst.getFlag(Flag: MachineInstr::MIFlag::FmReassoc) &&
7706 Inst.getFlag(Flag: MachineInstr::MIFlag::FmNsz);
7707
7708 // == Integer types ==
7709 // -- Base instructions --
7710 // Opcodes MULWrr and MULXrr don't exist because
7711 // `MUL <Wd>, <Wn>, <Wm>` and `MUL <Xd>, <Xn>, <Xm>` are aliases of
7712 // `MADD <Wd>, <Wn>, <Wm>, WZR` and `MADD <Xd>, <Xn>, <Xm>, XZR` respectively.
7713 // The machine-combiner does not support three-source-operands machine
7714 // instruction. So we cannot reassociate MULs.
7715 case AArch64::ADDWrr:
7716 case AArch64::ADDXrr:
7717 case AArch64::ANDWrr:
7718 case AArch64::ANDXrr:
7719 case AArch64::ORRWrr:
7720 case AArch64::ORRXrr:
7721 case AArch64::EORWrr:
7722 case AArch64::EORXrr:
7723 case AArch64::EONWrr:
7724 case AArch64::EONXrr:
7725 // -- Advanced SIMD instructions --
7726 // Opcodes MULv1i64 and MULv2i64 don't exist because there is no 64-bit MUL
7727 // in the Advanced SIMD instruction set.
7728 case AArch64::ADDv8i8:
7729 case AArch64::ADDv16i8:
7730 case AArch64::ADDv4i16:
7731 case AArch64::ADDv8i16:
7732 case AArch64::ADDv2i32:
7733 case AArch64::ADDv4i32:
7734 case AArch64::ADDv1i64:
7735 case AArch64::ADDv2i64:
7736 case AArch64::MULv8i8:
7737 case AArch64::MULv16i8:
7738 case AArch64::MULv4i16:
7739 case AArch64::MULv8i16:
7740 case AArch64::MULv2i32:
7741 case AArch64::MULv4i32:
7742 case AArch64::ANDv8i8:
7743 case AArch64::ANDv16i8:
7744 case AArch64::ORRv8i8:
7745 case AArch64::ORRv16i8:
7746 case AArch64::EORv8i8:
7747 case AArch64::EORv16i8:
7748 // -- SVE instructions --
7749 case AArch64::ADD_ZZZ_B:
7750 case AArch64::ADD_ZZZ_H:
7751 case AArch64::ADD_ZZZ_S:
7752 case AArch64::ADD_ZZZ_D:
7753 case AArch64::MUL_ZZZ_B:
7754 case AArch64::MUL_ZZZ_H:
7755 case AArch64::MUL_ZZZ_S:
7756 case AArch64::MUL_ZZZ_D:
7757 case AArch64::AND_ZZZ:
7758 case AArch64::ORR_ZZZ:
7759 case AArch64::EOR_ZZZ:
7760 return true;
7761
7762 default:
7763 return false;
7764 }
7765}
7766
7767/// Find instructions that can be turned into madd.
7768static bool getMaddPatterns(MachineInstr &Root,
7769 SmallVectorImpl<unsigned> &Patterns) {
7770 unsigned Opc = Root.getOpcode();
7771 MachineBasicBlock &MBB = *Root.getParent();
7772 bool Found = false;
7773
7774 if (!isCombineInstrCandidate(Opc))
7775 return false;
7776 if (isCombineInstrSettingFlag(Opc)) {
7777 int Cmp_NZCV =
7778 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true);
7779 // When NZCV is live bail out.
7780 if (Cmp_NZCV == -1)
7781 return false;
7782 unsigned NewOpc = convertToNonFlagSettingOpc(MI: Root);
7783 // When opcode can't change bail out.
7784 // CHECKME: do we miss any cases for opcode conversion?
7785 if (NewOpc == Opc)
7786 return false;
7787 Opc = NewOpc;
7788 }
7789
7790 auto setFound = [&](int Opcode, int Operand, unsigned ZeroReg,
7791 unsigned Pattern) {
7792 if (canCombineWithMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode, ZeroReg)) {
7793 Patterns.push_back(Elt: Pattern);
7794 Found = true;
7795 }
7796 };
7797
7798 auto setVFound = [&](int Opcode, int Operand, unsigned Pattern) {
7799 if (canCombine(MBB, MO&: Root.getOperand(i: Operand), CombineOpc: Opcode)) {
7800 Patterns.push_back(Elt: Pattern);
7801 Found = true;
7802 }
7803 };
7804
7805 typedef AArch64MachineCombinerPattern MCP;
7806
7807 switch (Opc) {
7808 default:
7809 break;
7810 case AArch64::ADDWrr:
7811 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
7812 "ADDWrr does not have register operands");
7813 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDW_OP1);
7814 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULADDW_OP2);
7815 break;
7816 case AArch64::ADDXrr:
7817 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDX_OP1);
7818 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULADDX_OP2);
7819 break;
7820 case AArch64::SUBWrr:
7821 setFound(AArch64::MADDWrrr, 2, AArch64::WZR, MCP::MULSUBW_OP2);
7822 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBW_OP1);
7823 break;
7824 case AArch64::SUBXrr:
7825 setFound(AArch64::MADDXrrr, 2, AArch64::XZR, MCP::MULSUBX_OP2);
7826 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBX_OP1);
7827 break;
7828 case AArch64::ADDWri:
7829 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULADDWI_OP1);
7830 break;
7831 case AArch64::ADDXri:
7832 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULADDXI_OP1);
7833 break;
7834 case AArch64::SUBWri:
7835 setFound(AArch64::MADDWrrr, 1, AArch64::WZR, MCP::MULSUBWI_OP1);
7836 break;
7837 case AArch64::SUBXri:
7838 setFound(AArch64::MADDXrrr, 1, AArch64::XZR, MCP::MULSUBXI_OP1);
7839 break;
7840 case AArch64::ADDv8i8:
7841 setVFound(AArch64::MULv8i8, 1, MCP::MULADDv8i8_OP1);
7842 setVFound(AArch64::MULv8i8, 2, MCP::MULADDv8i8_OP2);
7843 break;
7844 case AArch64::ADDv16i8:
7845 setVFound(AArch64::MULv16i8, 1, MCP::MULADDv16i8_OP1);
7846 setVFound(AArch64::MULv16i8, 2, MCP::MULADDv16i8_OP2);
7847 break;
7848 case AArch64::ADDv4i16:
7849 setVFound(AArch64::MULv4i16, 1, MCP::MULADDv4i16_OP1);
7850 setVFound(AArch64::MULv4i16, 2, MCP::MULADDv4i16_OP2);
7851 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULADDv4i16_indexed_OP1);
7852 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULADDv4i16_indexed_OP2);
7853 break;
7854 case AArch64::ADDv8i16:
7855 setVFound(AArch64::MULv8i16, 1, MCP::MULADDv8i16_OP1);
7856 setVFound(AArch64::MULv8i16, 2, MCP::MULADDv8i16_OP2);
7857 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULADDv8i16_indexed_OP1);
7858 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULADDv8i16_indexed_OP2);
7859 break;
7860 case AArch64::ADDv2i32:
7861 setVFound(AArch64::MULv2i32, 1, MCP::MULADDv2i32_OP1);
7862 setVFound(AArch64::MULv2i32, 2, MCP::MULADDv2i32_OP2);
7863 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULADDv2i32_indexed_OP1);
7864 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULADDv2i32_indexed_OP2);
7865 break;
7866 case AArch64::ADDv4i32:
7867 setVFound(AArch64::MULv4i32, 1, MCP::MULADDv4i32_OP1);
7868 setVFound(AArch64::MULv4i32, 2, MCP::MULADDv4i32_OP2);
7869 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULADDv4i32_indexed_OP1);
7870 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULADDv4i32_indexed_OP2);
7871 break;
7872 case AArch64::SUBv8i8:
7873 setVFound(AArch64::MULv8i8, 1, MCP::MULSUBv8i8_OP1);
7874 setVFound(AArch64::MULv8i8, 2, MCP::MULSUBv8i8_OP2);
7875 break;
7876 case AArch64::SUBv16i8:
7877 setVFound(AArch64::MULv16i8, 1, MCP::MULSUBv16i8_OP1);
7878 setVFound(AArch64::MULv16i8, 2, MCP::MULSUBv16i8_OP2);
7879 break;
7880 case AArch64::SUBv4i16:
7881 setVFound(AArch64::MULv4i16, 1, MCP::MULSUBv4i16_OP1);
7882 setVFound(AArch64::MULv4i16, 2, MCP::MULSUBv4i16_OP2);
7883 setVFound(AArch64::MULv4i16_indexed, 1, MCP::MULSUBv4i16_indexed_OP1);
7884 setVFound(AArch64::MULv4i16_indexed, 2, MCP::MULSUBv4i16_indexed_OP2);
7885 break;
7886 case AArch64::SUBv8i16:
7887 setVFound(AArch64::MULv8i16, 1, MCP::MULSUBv8i16_OP1);
7888 setVFound(AArch64::MULv8i16, 2, MCP::MULSUBv8i16_OP2);
7889 setVFound(AArch64::MULv8i16_indexed, 1, MCP::MULSUBv8i16_indexed_OP1);
7890 setVFound(AArch64::MULv8i16_indexed, 2, MCP::MULSUBv8i16_indexed_OP2);
7891 break;
7892 case AArch64::SUBv2i32:
7893 setVFound(AArch64::MULv2i32, 1, MCP::MULSUBv2i32_OP1);
7894 setVFound(AArch64::MULv2i32, 2, MCP::MULSUBv2i32_OP2);
7895 setVFound(AArch64::MULv2i32_indexed, 1, MCP::MULSUBv2i32_indexed_OP1);
7896 setVFound(AArch64::MULv2i32_indexed, 2, MCP::MULSUBv2i32_indexed_OP2);
7897 break;
7898 case AArch64::SUBv4i32:
7899 setVFound(AArch64::MULv4i32, 1, MCP::MULSUBv4i32_OP1);
7900 setVFound(AArch64::MULv4i32, 2, MCP::MULSUBv4i32_OP2);
7901 setVFound(AArch64::MULv4i32_indexed, 1, MCP::MULSUBv4i32_indexed_OP1);
7902 setVFound(AArch64::MULv4i32_indexed, 2, MCP::MULSUBv4i32_indexed_OP2);
7903 break;
7904 }
7905 return Found;
7906}
7907
7908bool AArch64InstrInfo::isAccumulationOpcode(unsigned Opcode) const {
7909 switch (Opcode) {
7910 default:
7911 break;
7912 case AArch64::UABALB_ZZZ_D:
7913 case AArch64::UABALB_ZZZ_H:
7914 case AArch64::UABALB_ZZZ_S:
7915 case AArch64::UABALT_ZZZ_D:
7916 case AArch64::UABALT_ZZZ_H:
7917 case AArch64::UABALT_ZZZ_S:
7918 case AArch64::SABALB_ZZZ_D:
7919 case AArch64::SABALB_ZZZ_S:
7920 case AArch64::SABALB_ZZZ_H:
7921 case AArch64::SABALT_ZZZ_D:
7922 case AArch64::SABALT_ZZZ_S:
7923 case AArch64::SABALT_ZZZ_H:
7924 case AArch64::UABALv16i8_v8i16:
7925 case AArch64::UABALv2i32_v2i64:
7926 case AArch64::UABALv4i16_v4i32:
7927 case AArch64::UABALv4i32_v2i64:
7928 case AArch64::UABALv8i16_v4i32:
7929 case AArch64::UABALv8i8_v8i16:
7930 case AArch64::UABAv16i8:
7931 case AArch64::UABAv2i32:
7932 case AArch64::UABAv4i16:
7933 case AArch64::UABAv4i32:
7934 case AArch64::UABAv8i16:
7935 case AArch64::UABAv8i8:
7936 case AArch64::SABALv16i8_v8i16:
7937 case AArch64::SABALv2i32_v2i64:
7938 case AArch64::SABALv4i16_v4i32:
7939 case AArch64::SABALv4i32_v2i64:
7940 case AArch64::SABALv8i16_v4i32:
7941 case AArch64::SABALv8i8_v8i16:
7942 case AArch64::SABAv16i8:
7943 case AArch64::SABAv2i32:
7944 case AArch64::SABAv4i16:
7945 case AArch64::SABAv4i32:
7946 case AArch64::SABAv8i16:
7947 case AArch64::SABAv8i8:
7948 return true;
7949 }
7950
7951 return false;
7952}
7953
7954unsigned AArch64InstrInfo::getAccumulationStartOpcode(
7955 unsigned AccumulationOpcode) const {
7956 switch (AccumulationOpcode) {
7957 default:
7958 llvm_unreachable("Unsupported accumulation Opcode!");
7959 case AArch64::UABALB_ZZZ_D:
7960 return AArch64::UABDLB_ZZZ_D;
7961 case AArch64::UABALB_ZZZ_H:
7962 return AArch64::UABDLB_ZZZ_H;
7963 case AArch64::UABALB_ZZZ_S:
7964 return AArch64::UABDLB_ZZZ_S;
7965 case AArch64::UABALT_ZZZ_D:
7966 return AArch64::UABDLT_ZZZ_D;
7967 case AArch64::UABALT_ZZZ_H:
7968 return AArch64::UABDLT_ZZZ_H;
7969 case AArch64::UABALT_ZZZ_S:
7970 return AArch64::UABDLT_ZZZ_S;
7971 case AArch64::UABALv16i8_v8i16:
7972 return AArch64::UABDLv16i8_v8i16;
7973 case AArch64::UABALv2i32_v2i64:
7974 return AArch64::UABDLv2i32_v2i64;
7975 case AArch64::UABALv4i16_v4i32:
7976 return AArch64::UABDLv4i16_v4i32;
7977 case AArch64::UABALv4i32_v2i64:
7978 return AArch64::UABDLv4i32_v2i64;
7979 case AArch64::UABALv8i16_v4i32:
7980 return AArch64::UABDLv8i16_v4i32;
7981 case AArch64::UABALv8i8_v8i16:
7982 return AArch64::UABDLv8i8_v8i16;
7983 case AArch64::UABAv16i8:
7984 return AArch64::UABDv16i8;
7985 case AArch64::UABAv2i32:
7986 return AArch64::UABDv2i32;
7987 case AArch64::UABAv4i16:
7988 return AArch64::UABDv4i16;
7989 case AArch64::UABAv4i32:
7990 return AArch64::UABDv4i32;
7991 case AArch64::UABAv8i16:
7992 return AArch64::UABDv8i16;
7993 case AArch64::UABAv8i8:
7994 return AArch64::UABDv8i8;
7995 case AArch64::SABALB_ZZZ_D:
7996 return AArch64::SABDLB_ZZZ_D;
7997 case AArch64::SABALB_ZZZ_S:
7998 return AArch64::SABDLB_ZZZ_S;
7999 case AArch64::SABALB_ZZZ_H:
8000 return AArch64::SABDLB_ZZZ_H;
8001 case AArch64::SABALT_ZZZ_D:
8002 return AArch64::SABDLT_ZZZ_D;
8003 case AArch64::SABALT_ZZZ_S:
8004 return AArch64::SABDLT_ZZZ_S;
8005 case AArch64::SABALT_ZZZ_H:
8006 return AArch64::SABDLT_ZZZ_H;
8007 case AArch64::SABALv16i8_v8i16:
8008 return AArch64::SABDLv16i8_v8i16;
8009 case AArch64::SABALv2i32_v2i64:
8010 return AArch64::SABDLv2i32_v2i64;
8011 case AArch64::SABALv4i16_v4i32:
8012 return AArch64::SABDLv4i16_v4i32;
8013 case AArch64::SABALv4i32_v2i64:
8014 return AArch64::SABDLv4i32_v2i64;
8015 case AArch64::SABALv8i16_v4i32:
8016 return AArch64::SABDLv8i16_v4i32;
8017 case AArch64::SABALv8i8_v8i16:
8018 return AArch64::SABDLv8i8_v8i16;
8019 case AArch64::SABAv16i8:
8020 return AArch64::SABDv16i8;
8021 case AArch64::SABAv2i32:
8022 return AArch64::SABDv2i32;
8023 case AArch64::SABAv4i16:
8024 return AArch64::SABDv4i16;
8025 case AArch64::SABAv4i32:
8026 return AArch64::SABDv4i32;
8027 case AArch64::SABAv8i16:
8028 return AArch64::SABDv8i16;
8029 case AArch64::SABAv8i8:
8030 return AArch64::SABDv8i8;
8031 }
8032}
8033
8034/// Floating-Point Support
8035
8036/// Find instructions that can be turned into madd.
8037static bool getFMAPatterns(MachineInstr &Root,
8038 SmallVectorImpl<unsigned> &Patterns) {
8039
8040 if (!isCombineInstrCandidateFP(Inst: Root))
8041 return false;
8042
8043 MachineBasicBlock &MBB = *Root.getParent();
8044 bool Found = false;
8045
8046 auto Match = [&](int Opcode, int Operand, unsigned Pattern) -> bool {
8047 if (canCombineWithFMUL(MBB, MO&: Root.getOperand(i: Operand), MulOpc: Opcode)) {
8048 Patterns.push_back(Elt: Pattern);
8049 return true;
8050 }
8051 return false;
8052 };
8053
8054 typedef AArch64MachineCombinerPattern MCP;
8055
8056 switch (Root.getOpcode()) {
8057 default:
8058 assert(false && "Unsupported FP instruction in combiner\n");
8059 break;
8060 case AArch64::FADDHrr:
8061 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8062 "FADDHrr does not have register operands");
8063
8064 Found = Match(AArch64::FMULHrr, 1, MCP::FMULADDH_OP1);
8065 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULADDH_OP2);
8066 break;
8067 case AArch64::FADDSrr:
8068 assert(Root.getOperand(1).isReg() && Root.getOperand(2).isReg() &&
8069 "FADDSrr does not have register operands");
8070
8071 Found |= Match(AArch64::FMULSrr, 1, MCP::FMULADDS_OP1) ||
8072 Match(AArch64::FMULv1i32_indexed, 1, MCP::FMLAv1i32_indexed_OP1);
8073
8074 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULADDS_OP2) ||
8075 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLAv1i32_indexed_OP2);
8076 break;
8077 case AArch64::FADDDrr:
8078 Found |= Match(AArch64::FMULDrr, 1, MCP::FMULADDD_OP1) ||
8079 Match(AArch64::FMULv1i64_indexed, 1, MCP::FMLAv1i64_indexed_OP1);
8080
8081 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULADDD_OP2) ||
8082 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLAv1i64_indexed_OP2);
8083 break;
8084 case AArch64::FADDv4f16:
8085 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLAv4i16_indexed_OP1) ||
8086 Match(AArch64::FMULv4f16, 1, MCP::FMLAv4f16_OP1);
8087
8088 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLAv4i16_indexed_OP2) ||
8089 Match(AArch64::FMULv4f16, 2, MCP::FMLAv4f16_OP2);
8090 break;
8091 case AArch64::FADDv8f16:
8092 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLAv8i16_indexed_OP1) ||
8093 Match(AArch64::FMULv8f16, 1, MCP::FMLAv8f16_OP1);
8094
8095 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLAv8i16_indexed_OP2) ||
8096 Match(AArch64::FMULv8f16, 2, MCP::FMLAv8f16_OP2);
8097 break;
8098 case AArch64::FADDv2f32:
8099 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLAv2i32_indexed_OP1) ||
8100 Match(AArch64::FMULv2f32, 1, MCP::FMLAv2f32_OP1);
8101
8102 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLAv2i32_indexed_OP2) ||
8103 Match(AArch64::FMULv2f32, 2, MCP::FMLAv2f32_OP2);
8104 break;
8105 case AArch64::FADDv2f64:
8106 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLAv2i64_indexed_OP1) ||
8107 Match(AArch64::FMULv2f64, 1, MCP::FMLAv2f64_OP1);
8108
8109 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLAv2i64_indexed_OP2) ||
8110 Match(AArch64::FMULv2f64, 2, MCP::FMLAv2f64_OP2);
8111 break;
8112 case AArch64::FADDv4f32:
8113 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLAv4i32_indexed_OP1) ||
8114 Match(AArch64::FMULv4f32, 1, MCP::FMLAv4f32_OP1);
8115
8116 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLAv4i32_indexed_OP2) ||
8117 Match(AArch64::FMULv4f32, 2, MCP::FMLAv4f32_OP2);
8118 break;
8119 case AArch64::FSUBHrr:
8120 Found = Match(AArch64::FMULHrr, 1, MCP::FMULSUBH_OP1);
8121 Found |= Match(AArch64::FMULHrr, 2, MCP::FMULSUBH_OP2);
8122 Found |= Match(AArch64::FNMULHrr, 1, MCP::FNMULSUBH_OP1);
8123 break;
8124 case AArch64::FSUBSrr:
8125 Found = Match(AArch64::FMULSrr, 1, MCP::FMULSUBS_OP1);
8126
8127 Found |= Match(AArch64::FMULSrr, 2, MCP::FMULSUBS_OP2) ||
8128 Match(AArch64::FMULv1i32_indexed, 2, MCP::FMLSv1i32_indexed_OP2);
8129
8130 Found |= Match(AArch64::FNMULSrr, 1, MCP::FNMULSUBS_OP1);
8131 break;
8132 case AArch64::FSUBDrr:
8133 Found = Match(AArch64::FMULDrr, 1, MCP::FMULSUBD_OP1);
8134
8135 Found |= Match(AArch64::FMULDrr, 2, MCP::FMULSUBD_OP2) ||
8136 Match(AArch64::FMULv1i64_indexed, 2, MCP::FMLSv1i64_indexed_OP2);
8137
8138 Found |= Match(AArch64::FNMULDrr, 1, MCP::FNMULSUBD_OP1);
8139 break;
8140 case AArch64::FSUBv4f16:
8141 Found |= Match(AArch64::FMULv4i16_indexed, 2, MCP::FMLSv4i16_indexed_OP2) ||
8142 Match(AArch64::FMULv4f16, 2, MCP::FMLSv4f16_OP2);
8143
8144 Found |= Match(AArch64::FMULv4i16_indexed, 1, MCP::FMLSv4i16_indexed_OP1) ||
8145 Match(AArch64::FMULv4f16, 1, MCP::FMLSv4f16_OP1);
8146 break;
8147 case AArch64::FSUBv8f16:
8148 Found |= Match(AArch64::FMULv8i16_indexed, 2, MCP::FMLSv8i16_indexed_OP2) ||
8149 Match(AArch64::FMULv8f16, 2, MCP::FMLSv8f16_OP2);
8150
8151 Found |= Match(AArch64::FMULv8i16_indexed, 1, MCP::FMLSv8i16_indexed_OP1) ||
8152 Match(AArch64::FMULv8f16, 1, MCP::FMLSv8f16_OP1);
8153 break;
8154 case AArch64::FSUBv2f32:
8155 Found |= Match(AArch64::FMULv2i32_indexed, 2, MCP::FMLSv2i32_indexed_OP2) ||
8156 Match(AArch64::FMULv2f32, 2, MCP::FMLSv2f32_OP2);
8157
8158 Found |= Match(AArch64::FMULv2i32_indexed, 1, MCP::FMLSv2i32_indexed_OP1) ||
8159 Match(AArch64::FMULv2f32, 1, MCP::FMLSv2f32_OP1);
8160 break;
8161 case AArch64::FSUBv2f64:
8162 Found |= Match(AArch64::FMULv2i64_indexed, 2, MCP::FMLSv2i64_indexed_OP2) ||
8163 Match(AArch64::FMULv2f64, 2, MCP::FMLSv2f64_OP2);
8164
8165 Found |= Match(AArch64::FMULv2i64_indexed, 1, MCP::FMLSv2i64_indexed_OP1) ||
8166 Match(AArch64::FMULv2f64, 1, MCP::FMLSv2f64_OP1);
8167 break;
8168 case AArch64::FSUBv4f32:
8169 Found |= Match(AArch64::FMULv4i32_indexed, 2, MCP::FMLSv4i32_indexed_OP2) ||
8170 Match(AArch64::FMULv4f32, 2, MCP::FMLSv4f32_OP2);
8171
8172 Found |= Match(AArch64::FMULv4i32_indexed, 1, MCP::FMLSv4i32_indexed_OP1) ||
8173 Match(AArch64::FMULv4f32, 1, MCP::FMLSv4f32_OP1);
8174 break;
8175 }
8176 return Found;
8177}
8178
8179static bool getFMULPatterns(MachineInstr &Root,
8180 SmallVectorImpl<unsigned> &Patterns) {
8181 MachineBasicBlock &MBB = *Root.getParent();
8182 bool Found = false;
8183
8184 auto Match = [&](unsigned Opcode, int Operand, unsigned Pattern) -> bool {
8185 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8186 MachineOperand &MO = Root.getOperand(i: Operand);
8187 MachineInstr *MI = nullptr;
8188 if (MO.isReg() && MO.getReg().isVirtual())
8189 MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8190 // Ignore No-op COPYs in FMUL(COPY(DUP(..)))
8191 if (MI && MI->getOpcode() == TargetOpcode::COPY &&
8192 MI->getOperand(i: 1).getReg().isVirtual())
8193 MI = MRI.getUniqueVRegDef(Reg: MI->getOperand(i: 1).getReg());
8194 if (MI && MI->getOpcode() == Opcode) {
8195 Patterns.push_back(Elt: Pattern);
8196 return true;
8197 }
8198 return false;
8199 };
8200
8201 typedef AArch64MachineCombinerPattern MCP;
8202
8203 switch (Root.getOpcode()) {
8204 default:
8205 return false;
8206 case AArch64::FMULv2f32:
8207 Found = Match(AArch64::DUPv2i32lane, 1, MCP::FMULv2i32_indexed_OP1);
8208 Found |= Match(AArch64::DUPv2i32lane, 2, MCP::FMULv2i32_indexed_OP2);
8209 break;
8210 case AArch64::FMULv2f64:
8211 Found = Match(AArch64::DUPv2i64lane, 1, MCP::FMULv2i64_indexed_OP1);
8212 Found |= Match(AArch64::DUPv2i64lane, 2, MCP::FMULv2i64_indexed_OP2);
8213 break;
8214 case AArch64::FMULv4f16:
8215 Found = Match(AArch64::DUPv4i16lane, 1, MCP::FMULv4i16_indexed_OP1);
8216 Found |= Match(AArch64::DUPv4i16lane, 2, MCP::FMULv4i16_indexed_OP2);
8217 break;
8218 case AArch64::FMULv4f32:
8219 Found = Match(AArch64::DUPv4i32lane, 1, MCP::FMULv4i32_indexed_OP1);
8220 Found |= Match(AArch64::DUPv4i32lane, 2, MCP::FMULv4i32_indexed_OP2);
8221 break;
8222 case AArch64::FMULv8f16:
8223 Found = Match(AArch64::DUPv8i16lane, 1, MCP::FMULv8i16_indexed_OP1);
8224 Found |= Match(AArch64::DUPv8i16lane, 2, MCP::FMULv8i16_indexed_OP2);
8225 break;
8226 }
8227
8228 return Found;
8229}
8230
8231static bool getFNEGPatterns(MachineInstr &Root,
8232 SmallVectorImpl<unsigned> &Patterns) {
8233 unsigned Opc = Root.getOpcode();
8234 MachineBasicBlock &MBB = *Root.getParent();
8235 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
8236
8237 auto Match = [&](unsigned Opcode, unsigned Pattern) -> bool {
8238 MachineOperand &MO = Root.getOperand(i: 1);
8239 MachineInstr *MI = MRI.getUniqueVRegDef(Reg: MO.getReg());
8240 if (MI != nullptr && (MI->getOpcode() == Opcode) &&
8241 MRI.hasOneNonDBGUse(RegNo: MI->getOperand(i: 0).getReg()) &&
8242 Root.getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8243 Root.getFlag(Flag: MachineInstr::MIFlag::FmNsz) &&
8244 MI->getFlag(Flag: MachineInstr::MIFlag::FmContract) &&
8245 MI->getFlag(Flag: MachineInstr::MIFlag::FmNsz)) {
8246 Patterns.push_back(Elt: Pattern);
8247 return true;
8248 }
8249 return false;
8250 };
8251
8252 switch (Opc) {
8253 default:
8254 break;
8255 case AArch64::FNEGDr:
8256 return Match(AArch64::FMADDDrrr, AArch64MachineCombinerPattern::FNMADD);
8257 case AArch64::FNEGSr:
8258 return Match(AArch64::FMADDSrrr, AArch64MachineCombinerPattern::FNMADD);
8259 }
8260
8261 return false;
8262}
8263
8264/// Return true when a code sequence can improve throughput. It
8265/// should be called only for instructions in loops.
8266/// \param Pattern - combiner pattern
8267bool AArch64InstrInfo::isThroughputPattern(unsigned Pattern) const {
8268 switch (Pattern) {
8269 default:
8270 break;
8271 case AArch64MachineCombinerPattern::FMULADDH_OP1:
8272 case AArch64MachineCombinerPattern::FMULADDH_OP2:
8273 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
8274 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
8275 case AArch64MachineCombinerPattern::FMULADDS_OP1:
8276 case AArch64MachineCombinerPattern::FMULADDS_OP2:
8277 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
8278 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
8279 case AArch64MachineCombinerPattern::FMULADDD_OP1:
8280 case AArch64MachineCombinerPattern::FMULADDD_OP2:
8281 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
8282 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
8283 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
8284 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
8285 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
8286 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
8287 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
8288 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
8289 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
8290 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
8291 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
8292 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
8293 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
8294 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
8295 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
8296 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
8297 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
8298 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
8299 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
8300 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
8301 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
8302 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
8303 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
8304 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
8305 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
8306 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
8307 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
8308 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
8309 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
8310 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1:
8311 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
8312 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1:
8313 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
8314 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
8315 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
8316 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
8317 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
8318 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
8319 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
8320 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
8321 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
8322 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
8323 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
8324 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
8325 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
8326 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
8327 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2:
8328 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
8329 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2:
8330 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
8331 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2:
8332 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
8333 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2:
8334 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
8335 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2:
8336 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
8337 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
8338 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
8339 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
8340 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
8341 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
8342 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
8343 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
8344 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
8345 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
8346 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
8347 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
8348 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
8349 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
8350 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
8351 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
8352 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
8353 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
8354 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
8355 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
8356 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
8357 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
8358 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
8359 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
8360 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
8361 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
8362 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
8363 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
8364 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
8365 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
8366 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
8367 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
8368 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
8369 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
8370 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
8371 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
8372 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
8373 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
8374 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
8375 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
8376 return true;
8377 } // end switch (Pattern)
8378 return false;
8379}
8380
8381/// Find other MI combine patterns.
8382static bool getMiscPatterns(MachineInstr &Root,
8383 SmallVectorImpl<unsigned> &Patterns) {
8384 // A - (B + C) ==> (A - B) - C or (A - C) - B
8385 unsigned Opc = Root.getOpcode();
8386 MachineBasicBlock &MBB = *Root.getParent();
8387
8388 switch (Opc) {
8389 case AArch64::SUBWrr:
8390 case AArch64::SUBSWrr:
8391 case AArch64::SUBXrr:
8392 case AArch64::SUBSXrr:
8393 // Found candidate root.
8394 break;
8395 default:
8396 return false;
8397 }
8398
8399 if (isCombineInstrSettingFlag(Opc) &&
8400 Root.findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr, isDead: true) ==
8401 -1)
8402 return false;
8403
8404 if (canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDWrr) ||
8405 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSWrr) ||
8406 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDXrr) ||
8407 canCombine(MBB, MO&: Root.getOperand(i: 2), CombineOpc: AArch64::ADDSXrr)) {
8408 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP1);
8409 Patterns.push_back(Elt: AArch64MachineCombinerPattern::SUBADD_OP2);
8410 return true;
8411 }
8412
8413 return false;
8414}
8415
8416/// Check if the given instruction forms a gather load pattern that can be
8417/// optimized for better Memory-Level Parallelism (MLP). This function
8418/// identifies chains of NEON lane load instructions that load data from
8419/// different memory addresses into individual lanes of a 128-bit vector
8420/// register, then attempts to split the pattern into parallel loads to break
8421/// the serial dependency between instructions.
8422///
8423/// Pattern Matched:
8424/// Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) ->
8425/// LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)
8426///
8427/// Transformed Into:
8428/// Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64
8429/// to combine the results, enabling better memory-level parallelism.
8430///
8431/// Supported Element Types:
8432/// - 32-bit elements (LD1i32, 4 lanes total)
8433/// - 16-bit elements (LD1i16, 8 lanes total)
8434/// - 8-bit elements (LD1i8, 16 lanes total)
8435static bool getGatherLanePattern(MachineInstr &Root,
8436 SmallVectorImpl<unsigned> &Patterns,
8437 unsigned LoadLaneOpCode, unsigned NumLanes) {
8438 const MachineFunction *MF = Root.getMF();
8439
8440 // Early exit if optimizing for size.
8441 if (MF->getFunction().hasMinSize())
8442 return false;
8443
8444 const MachineRegisterInfo &MRI = MF->getRegInfo();
8445 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
8446
8447 // The root of the pattern must load into the last lane of the vector.
8448 if (Root.getOperand(i: 2).getImm() != NumLanes - 1)
8449 return false;
8450
8451 // Check that we have load into all lanes except lane 0.
8452 // For each load we also want to check that:
8453 // 1. It has a single non-debug use (since we will be replacing the virtual
8454 // register)
8455 // 2. That the addressing mode only uses a single pointer operand
8456 auto *CurrInstr = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8457 auto Range = llvm::seq<unsigned>(Begin: 1, End: NumLanes - 1);
8458 SmallSet<unsigned, 16> RemainingLanes(Range.begin(), Range.end());
8459 SmallVector<const MachineInstr *, 16> LoadInstrs;
8460 while (!RemainingLanes.empty() && CurrInstr &&
8461 CurrInstr->getOpcode() == LoadLaneOpCode &&
8462 MRI.hasOneNonDBGUse(RegNo: CurrInstr->getOperand(i: 0).getReg()) &&
8463 CurrInstr->getNumOperands() == 4) {
8464 RemainingLanes.erase(V: CurrInstr->getOperand(i: 2).getImm());
8465 LoadInstrs.push_back(Elt: CurrInstr);
8466 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8467 }
8468
8469 // Check that we have found a match for lanes N-1.. 1.
8470 if (!RemainingLanes.empty())
8471 return false;
8472
8473 // Match the SUBREG_TO_REG sequence.
8474 if (CurrInstr->getOpcode() != TargetOpcode::SUBREG_TO_REG)
8475 return false;
8476
8477 // Verify that the subreg to reg loads an integer into the first lane.
8478 auto Lane0LoadReg = CurrInstr->getOperand(i: 1).getReg();
8479 unsigned SingleLaneSizeInBits = 128 / NumLanes;
8480 if (TRI->getRegSizeInBits(Reg: Lane0LoadReg, MRI) != SingleLaneSizeInBits)
8481 return false;
8482
8483 // Verify that it also has a single non debug use.
8484 if (!MRI.hasOneNonDBGUse(RegNo: Lane0LoadReg))
8485 return false;
8486
8487 LoadInstrs.push_back(Elt: MRI.getUniqueVRegDef(Reg: Lane0LoadReg));
8488
8489 // If there is any chance of aliasing, do not apply the pattern.
8490 // Walk backward through the MBB starting from Root.
8491 // Exit early if we've encountered all load instructions or hit the search
8492 // limit.
8493 auto MBBItr = Root.getIterator();
8494 unsigned RemainingSteps =
8495 MF->getSubtarget<AArch64Subtarget>().getCLOpts().search_limit;
8496 SmallPtrSet<const MachineInstr *, 16> RemainingLoadInstrs;
8497 RemainingLoadInstrs.insert(I: LoadInstrs.begin(), E: LoadInstrs.end());
8498 const MachineBasicBlock *MBB = Root.getParent();
8499
8500 for (; MBBItr != MBB->begin() && RemainingSteps > 0 &&
8501 !RemainingLoadInstrs.empty();
8502 --MBBItr, --RemainingSteps) {
8503 const MachineInstr &CurrInstr = *MBBItr;
8504
8505 // Remove this instruction from remaining loads if it's one we're tracking.
8506 RemainingLoadInstrs.erase(Ptr: &CurrInstr);
8507
8508 // Check for potential aliasing with any of the load instructions to
8509 // optimize.
8510 if (CurrInstr.isLoadFoldBarrier())
8511 return false;
8512 }
8513
8514 // If we hit the search limit without finding all load instructions,
8515 // don't match the pattern.
8516 if (RemainingSteps == 0 && !RemainingLoadInstrs.empty())
8517 return false;
8518
8519 switch (NumLanes) {
8520 case 4:
8521 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i32);
8522 break;
8523 case 8:
8524 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i16);
8525 break;
8526 case 16:
8527 Patterns.push_back(Elt: AArch64MachineCombinerPattern::GATHER_LANE_i8);
8528 break;
8529 default:
8530 llvm_unreachable("Got bad number of lanes for gather pattern.");
8531 }
8532
8533 return true;
8534}
8535
8536/// Search for patterns of LD instructions we can optimize.
8537static bool getLoadPatterns(MachineInstr &Root,
8538 SmallVectorImpl<unsigned> &Patterns) {
8539
8540 // The pattern searches for loads into single lanes.
8541 switch (Root.getOpcode()) {
8542 case AArch64::LD1i32:
8543 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 4);
8544 case AArch64::LD1i16:
8545 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 8);
8546 case AArch64::LD1i8:
8547 return getGatherLanePattern(Root, Patterns, LoadLaneOpCode: Root.getOpcode(), NumLanes: 16);
8548 default:
8549 return false;
8550 }
8551}
8552
8553/// Generate optimized instruction sequence for gather load patterns to improve
8554/// Memory-Level Parallelism (MLP). This function transforms a chain of
8555/// sequential NEON lane loads into parallel vector loads that can execute
8556/// concurrently.
8557static void
8558generateGatherLanePattern(MachineInstr &Root,
8559 SmallVectorImpl<MachineInstr *> &InsInstrs,
8560 SmallVectorImpl<MachineInstr *> &DelInstrs,
8561 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8562 unsigned Pattern, unsigned NumLanes) {
8563 MachineFunction &MF = *Root.getParent()->getParent();
8564 MachineRegisterInfo &MRI = MF.getRegInfo();
8565 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8566
8567 // Gather the initial load instructions to build the pattern.
8568 SmallVector<MachineInstr *, 16> LoadToLaneInstrs;
8569 MachineInstr *CurrInstr = &Root;
8570 for (unsigned i = 0; i < NumLanes - 1; ++i) {
8571 LoadToLaneInstrs.push_back(Elt: CurrInstr);
8572 CurrInstr = MRI.getUniqueVRegDef(Reg: CurrInstr->getOperand(i: 1).getReg());
8573 }
8574
8575 // Sort the load instructions according to the lane.
8576 llvm::sort(C&: LoadToLaneInstrs,
8577 Comp: [](const MachineInstr *A, const MachineInstr *B) {
8578 return A->getOperand(i: 2).getImm() > B->getOperand(i: 2).getImm();
8579 });
8580
8581 MachineInstr *SubregToReg = CurrInstr;
8582 LoadToLaneInstrs.push_back(
8583 Elt: MRI.getUniqueVRegDef(Reg: SubregToReg->getOperand(i: 1).getReg()));
8584 auto LoadToLaneInstrsAscending = llvm::reverse(C&: LoadToLaneInstrs);
8585
8586 const TargetRegisterClass *FPR128RegClass =
8587 MRI.getRegClass(Reg: Root.getOperand(i: 0).getReg());
8588
8589 // Helper lambda to create a LD1 instruction.
8590 auto CreateLD1Instruction = [&](MachineInstr *OriginalInstr,
8591 Register SrcRegister, unsigned Lane,
8592 Register OffsetRegister,
8593 bool OffsetRegisterKillState) {
8594 auto NewRegister = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8595 MachineInstrBuilder LoadIndexIntoRegister =
8596 BuildMI(MF, MIMD: MIMetadata(*OriginalInstr), MCID: TII->get(Opcode: Root.getOpcode()),
8597 DestReg: NewRegister)
8598 .addReg(RegNo: SrcRegister)
8599 .addImm(Val: Lane)
8600 .addReg(RegNo: OffsetRegister, Flags: getKillRegState(B: OffsetRegisterKillState))
8601 .setMemRefs(OriginalInstr->memoperands());
8602 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewRegister, y: InsInstrs.size()));
8603 InsInstrs.push_back(Elt: LoadIndexIntoRegister);
8604 return NewRegister;
8605 };
8606
8607 // Helper to create load instruction based on the NumLanes in the NEON
8608 // register we are rewriting.
8609 auto CreateLDRInstruction =
8610 [&](unsigned NumLanes, Register DestReg, Register OffsetReg,
8611 ArrayRef<MachineMemOperand *> MMOs) -> MachineInstrBuilder {
8612 unsigned Opcode;
8613 switch (NumLanes) {
8614 case 4:
8615 Opcode = AArch64::LDRSui;
8616 break;
8617 case 8:
8618 Opcode = AArch64::LDRHui;
8619 break;
8620 case 16:
8621 Opcode = AArch64::LDRBui;
8622 break;
8623 default:
8624 llvm_unreachable(
8625 "Got unsupported number of lanes in machine-combiner gather pattern");
8626 }
8627 // Immediate offset load
8628 return BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg)
8629 .addReg(RegNo: OffsetReg)
8630 .addImm(Val: 0)
8631 .setMemRefs(MMOs);
8632 };
8633
8634 // Load the remaining lanes into register 0.
8635 auto LanesToLoadToReg0 =
8636 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + 1,
8637 y: LoadToLaneInstrsAscending.begin() + NumLanes / 2);
8638 Register PrevReg = SubregToReg->getOperand(i: 0).getReg();
8639 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg0)) {
8640 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8641 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8642 OffsetRegOperand.getReg(),
8643 OffsetRegOperand.isKill());
8644 DelInstrs.push_back(Elt: LoadInstr);
8645 }
8646 Register LastLoadReg0 = PrevReg;
8647
8648 // First load into register 1. Perform an integer load to zero out the upper
8649 // lanes in a single instruction.
8650 MachineInstr *Lane0Load = *LoadToLaneInstrsAscending.begin();
8651 MachineInstr *OriginalSplitLoad =
8652 *std::next(x: LoadToLaneInstrsAscending.begin(), n: NumLanes / 2);
8653 Register DestRegForMiddleIndex = MRI.createVirtualRegister(
8654 RegClass: MRI.getRegClass(Reg: Lane0Load->getOperand(i: 0).getReg()));
8655
8656 const MachineOperand &OriginalSplitToLoadOffsetOperand =
8657 OriginalSplitLoad->getOperand(i: 3);
8658 MachineInstrBuilder MiddleIndexLoadInstr =
8659 CreateLDRInstruction(NumLanes, DestRegForMiddleIndex,
8660 OriginalSplitToLoadOffsetOperand.getReg(),
8661 OriginalSplitLoad->memoperands());
8662
8663 InstrIdxForVirtReg.insert(
8664 KV: std::make_pair(x&: DestRegForMiddleIndex, y: InsInstrs.size()));
8665 InsInstrs.push_back(Elt: MiddleIndexLoadInstr);
8666 DelInstrs.push_back(Elt: OriginalSplitLoad);
8667
8668 // Subreg To Reg instruction for register 1.
8669 Register DestRegForSubregToReg = MRI.createVirtualRegister(RegClass: FPR128RegClass);
8670 unsigned SubregType;
8671 switch (NumLanes) {
8672 case 4:
8673 SubregType = AArch64::ssub;
8674 break;
8675 case 8:
8676 SubregType = AArch64::hsub;
8677 break;
8678 case 16:
8679 SubregType = AArch64::bsub;
8680 break;
8681 default:
8682 llvm_unreachable(
8683 "Got invalid NumLanes for machine-combiner gather pattern");
8684 }
8685
8686 auto SubRegToRegInstr =
8687 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubregToReg->getOpcode()),
8688 DestReg: DestRegForSubregToReg)
8689 .addReg(RegNo: DestRegForMiddleIndex, Flags: getKillRegState(B: true))
8690 .addImm(Val: SubregType);
8691 InstrIdxForVirtReg.insert(
8692 KV: std::make_pair(x&: DestRegForSubregToReg, y: InsInstrs.size()));
8693 InsInstrs.push_back(Elt: SubRegToRegInstr);
8694
8695 // Load remaining lanes into register 1.
8696 auto LanesToLoadToReg1 =
8697 llvm::make_range(x: LoadToLaneInstrsAscending.begin() + NumLanes / 2 + 1,
8698 y: LoadToLaneInstrsAscending.end());
8699 PrevReg = SubRegToRegInstr->getOperand(i: 0).getReg();
8700 for (auto [Index, LoadInstr] : llvm::enumerate(First&: LanesToLoadToReg1)) {
8701 const MachineOperand &OffsetRegOperand = LoadInstr->getOperand(i: 3);
8702 PrevReg = CreateLD1Instruction(LoadInstr, PrevReg, Index + 1,
8703 OffsetRegOperand.getReg(),
8704 OffsetRegOperand.isKill());
8705
8706 // Do not add the last reg to DelInstrs - it will be removed later.
8707 if (Index == NumLanes / 2 - 2) {
8708 break;
8709 }
8710 DelInstrs.push_back(Elt: LoadInstr);
8711 }
8712 Register LastLoadReg1 = PrevReg;
8713
8714 // Create the final zip instruction to combine the results.
8715 MachineInstrBuilder ZipInstr =
8716 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::ZIP1v2i64),
8717 DestReg: Root.getOperand(i: 0).getReg())
8718 .addReg(RegNo: LastLoadReg0)
8719 .addReg(RegNo: LastLoadReg1);
8720 InsInstrs.push_back(Elt: ZipInstr);
8721}
8722
8723CombinerObjective
8724AArch64InstrInfo::getCombinerObjective(unsigned Pattern) const {
8725 switch (Pattern) {
8726 case AArch64MachineCombinerPattern::SUBADD_OP1:
8727 case AArch64MachineCombinerPattern::SUBADD_OP2:
8728 case AArch64MachineCombinerPattern::GATHER_LANE_i32:
8729 case AArch64MachineCombinerPattern::GATHER_LANE_i16:
8730 case AArch64MachineCombinerPattern::GATHER_LANE_i8:
8731 return CombinerObjective::MustReduceDepth;
8732 default:
8733 return TargetInstrInfo::getCombinerObjective(Pattern);
8734 }
8735}
8736
8737/// Return true when there is potentially a faster code sequence for an
8738/// instruction chain ending in \p Root. All potential patterns are listed in
8739/// the \p Pattern vector. Pattern should be sorted in priority order since the
8740/// pattern evaluator stops checking as soon as it finds a faster sequence.
8741
8742bool AArch64InstrInfo::getMachineCombinerPatterns(
8743 MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
8744 bool DoRegPressureReduce) const {
8745 // Integer patterns
8746 if (getMaddPatterns(Root, Patterns))
8747 return true;
8748 // Floating point patterns
8749 if (getFMULPatterns(Root, Patterns))
8750 return true;
8751 if (getFMAPatterns(Root, Patterns))
8752 return true;
8753 if (getFNEGPatterns(Root, Patterns))
8754 return true;
8755
8756 // Other patterns
8757 if (getMiscPatterns(Root, Patterns))
8758 return true;
8759
8760 // Load patterns
8761 if (getLoadPatterns(Root, Patterns))
8762 return true;
8763
8764 return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
8765 DoRegPressureReduce);
8766}
8767
8768enum class FMAInstKind { Default, Indexed, Accumulator };
8769/// genFusedMultiply - Generate fused multiply instructions.
8770/// This function supports both integer and floating point instructions.
8771/// A typical example:
8772/// F|MUL I=A,B,0
8773/// F|ADD R,I,C
8774/// ==> F|MADD R,A,B,C
8775/// \param MF Containing MachineFunction
8776/// \param MRI Register information
8777/// \param TII Target information
8778/// \param Root is the F|ADD instruction
8779/// \param [out] InsInstrs is a vector of machine instructions and will
8780/// contain the generated madd instruction
8781/// \param IdxMulOpd is index of operand in Root that is the result of
8782/// the F|MUL. In the example above IdxMulOpd is 1.
8783/// \param MaddOpc the opcode fo the f|madd instruction
8784/// \param RC Register class of operands
8785/// \param kind of fma instruction (addressing mode) to be generated
8786/// \param ReplacedAddend is the result register from the instruction
8787/// replacing the non-combined operand, if any.
8788static MachineInstr *
8789genFusedMultiply(MachineFunction &MF, MachineRegisterInfo &MRI,
8790 const TargetInstrInfo *TII, MachineInstr &Root,
8791 SmallVectorImpl<MachineInstr *> &InsInstrs, unsigned IdxMulOpd,
8792 unsigned MaddOpc, const TargetRegisterClass *RC,
8793 FMAInstKind kind = FMAInstKind::Default,
8794 const Register *ReplacedAddend = nullptr) {
8795 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
8796
8797 unsigned IdxOtherOpd = IdxMulOpd == 1 ? 2 : 1;
8798 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
8799 Register ResultReg = Root.getOperand(i: 0).getReg();
8800 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
8801 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
8802 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
8803 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
8804
8805 Register SrcReg2;
8806 bool Src2IsKill;
8807 if (ReplacedAddend) {
8808 // If we just generated a new addend, we must be it's only use.
8809 SrcReg2 = *ReplacedAddend;
8810 Src2IsKill = true;
8811 } else {
8812 SrcReg2 = Root.getOperand(i: IdxOtherOpd).getReg();
8813 Src2IsKill = Root.getOperand(i: IdxOtherOpd).isKill();
8814 }
8815
8816 if (ResultReg.isVirtual())
8817 MRI.constrainRegClass(Reg: ResultReg, RC);
8818 if (SrcReg0.isVirtual())
8819 MRI.constrainRegClass(Reg: SrcReg0, RC);
8820 if (SrcReg1.isVirtual())
8821 MRI.constrainRegClass(Reg: SrcReg1, RC);
8822 if (SrcReg2.isVirtual())
8823 MRI.constrainRegClass(Reg: SrcReg2, RC);
8824
8825 MachineInstrBuilder MIB;
8826 if (kind == FMAInstKind::Default)
8827 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8828 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8829 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8830 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8831 else if (kind == FMAInstKind::Indexed)
8832 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8833 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8834 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8835 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8836 .addImm(Val: MUL->getOperand(i: 3).getImm());
8837 else if (kind == FMAInstKind::Accumulator)
8838 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
8839 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill))
8840 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8841 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill));
8842 else
8843 assert(false && "Invalid FMA instruction kind \n");
8844 // Insert the MADD (MADD, FMA, FMS, FMLA, FMSL)
8845 InsInstrs.push_back(Elt: MIB);
8846 return MUL;
8847}
8848
8849static MachineInstr *
8850genFNegatedMAD(MachineFunction &MF, MachineRegisterInfo &MRI,
8851 const TargetInstrInfo *TII, MachineInstr &Root,
8852 SmallVectorImpl<MachineInstr *> &InsInstrs) {
8853 MachineInstr *MAD = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 1).getReg());
8854
8855 unsigned Opc = 0;
8856 const TargetRegisterClass *RC = MRI.getRegClass(Reg: MAD->getOperand(i: 0).getReg());
8857 if (AArch64::FPR32RegClass.hasSubClassEq(RC))
8858 Opc = AArch64::FNMADDSrrr;
8859 else if (AArch64::FPR64RegClass.hasSubClassEq(RC))
8860 Opc = AArch64::FNMADDDrrr;
8861 else
8862 return nullptr;
8863
8864 Register ResultReg = Root.getOperand(i: 0).getReg();
8865 Register SrcReg0 = MAD->getOperand(i: 1).getReg();
8866 Register SrcReg1 = MAD->getOperand(i: 2).getReg();
8867 Register SrcReg2 = MAD->getOperand(i: 3).getReg();
8868 bool Src0IsKill = MAD->getOperand(i: 1).isKill();
8869 bool Src1IsKill = MAD->getOperand(i: 2).isKill();
8870 bool Src2IsKill = MAD->getOperand(i: 3).isKill();
8871 if (ResultReg.isVirtual())
8872 MRI.constrainRegClass(Reg: ResultReg, RC);
8873 if (SrcReg0.isVirtual())
8874 MRI.constrainRegClass(Reg: SrcReg0, RC);
8875 if (SrcReg1.isVirtual())
8876 MRI.constrainRegClass(Reg: SrcReg1, RC);
8877 if (SrcReg2.isVirtual())
8878 MRI.constrainRegClass(Reg: SrcReg2, RC);
8879
8880 MachineInstrBuilder MIB =
8881 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: Opc), DestReg: ResultReg)
8882 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
8883 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
8884 .addReg(RegNo: SrcReg2, Flags: getKillRegState(B: Src2IsKill));
8885 InsInstrs.push_back(Elt: MIB);
8886
8887 return MAD;
8888}
8889
8890/// Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
8891static MachineInstr *
8892genIndexedMultiply(MachineInstr &Root,
8893 SmallVectorImpl<MachineInstr *> &InsInstrs,
8894 unsigned IdxDupOp, unsigned MulOpc,
8895 const TargetRegisterClass *RC, MachineRegisterInfo &MRI) {
8896 assert(((IdxDupOp == 1) || (IdxDupOp == 2)) &&
8897 "Invalid index of FMUL operand");
8898
8899 MachineFunction &MF = *Root.getMF();
8900 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
8901
8902 MachineInstr *Dup =
8903 MF.getRegInfo().getUniqueVRegDef(Reg: Root.getOperand(i: IdxDupOp).getReg());
8904
8905 if (Dup->getOpcode() == TargetOpcode::COPY)
8906 Dup = MRI.getUniqueVRegDef(Reg: Dup->getOperand(i: 1).getReg());
8907
8908 Register DupSrcReg = Dup->getOperand(i: 1).getReg();
8909 MRI.clearKillFlags(Reg: DupSrcReg);
8910 MRI.constrainRegClass(Reg: DupSrcReg, RC);
8911
8912 unsigned DupSrcLane = Dup->getOperand(i: 2).getImm();
8913
8914 unsigned IdxMulOp = IdxDupOp == 1 ? 2 : 1;
8915 MachineOperand &MulOp = Root.getOperand(i: IdxMulOp);
8916
8917 Register ResultReg = Root.getOperand(i: 0).getReg();
8918
8919 MachineInstrBuilder MIB;
8920 MIB = BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MulOpc), DestReg: ResultReg)
8921 .add(MO: MulOp)
8922 .addReg(RegNo: DupSrcReg)
8923 .addImm(Val: DupSrcLane);
8924
8925 InsInstrs.push_back(Elt: MIB);
8926 return &Root;
8927}
8928
8929/// genFusedMultiplyAcc - Helper to generate fused multiply accumulate
8930/// instructions.
8931///
8932/// \see genFusedMultiply
8933static MachineInstr *genFusedMultiplyAcc(
8934 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8935 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8936 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8937 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8938 kind: FMAInstKind::Accumulator);
8939}
8940
8941/// genNeg - Helper to generate an intermediate negation of the second operand
8942/// of Root
8943static Register genNeg(MachineFunction &MF, MachineRegisterInfo &MRI,
8944 const TargetInstrInfo *TII, MachineInstr &Root,
8945 SmallVectorImpl<MachineInstr *> &InsInstrs,
8946 DenseMap<Register, unsigned> &InstrIdxForVirtReg,
8947 unsigned MnegOpc, const TargetRegisterClass *RC) {
8948 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
8949 MachineInstrBuilder MIB =
8950 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MnegOpc), DestReg: NewVR)
8951 .add(MO: Root.getOperand(i: 2));
8952 InsInstrs.push_back(Elt: MIB);
8953
8954 assert(InstrIdxForVirtReg.empty());
8955 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
8956
8957 return NewVR;
8958}
8959
8960/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8961/// instructions with an additional negation of the accumulator
8962static MachineInstr *genFusedMultiplyAccNeg(
8963 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8964 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8965 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8966 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8967 assert(IdxMulOpd == 1);
8968
8969 Register NewVR =
8970 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8971 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8972 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
8973}
8974
8975/// genFusedMultiplyIdx - Helper to generate fused multiply accumulate
8976/// instructions.
8977///
8978/// \see genFusedMultiply
8979static MachineInstr *genFusedMultiplyIdx(
8980 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8981 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8982 unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC) {
8983 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
8984 kind: FMAInstKind::Indexed);
8985}
8986
8987/// genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate
8988/// instructions with an additional negation of the accumulator
8989static MachineInstr *genFusedMultiplyIdxNeg(
8990 MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII,
8991 MachineInstr &Root, SmallVectorImpl<MachineInstr *> &InsInstrs,
8992 DenseMap<Register, unsigned> &InstrIdxForVirtReg, unsigned IdxMulOpd,
8993 unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC) {
8994 assert(IdxMulOpd == 1);
8995
8996 Register NewVR =
8997 genNeg(MF, MRI, TII, Root, InsInstrs, InstrIdxForVirtReg, MnegOpc, RC);
8998
8999 return genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd, MaddOpc, RC,
9000 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9001}
9002
9003/// genMaddR - Generate madd instruction and combine mul and add using
9004/// an extra virtual register
9005/// Example - an ADD intermediate needs to be stored in a register:
9006/// MUL I=A,B,0
9007/// ADD R,I,Imm
9008/// ==> ORR V, ZR, Imm
9009/// ==> MADD R,A,B,V
9010/// \param MF Containing MachineFunction
9011/// \param MRI Register information
9012/// \param TII Target information
9013/// \param Root is the ADD instruction
9014/// \param [out] InsInstrs is a vector of machine instructions and will
9015/// contain the generated madd instruction
9016/// \param IdxMulOpd is index of operand in Root that is the result of
9017/// the MUL. In the example above IdxMulOpd is 1.
9018/// \param MaddOpc the opcode fo the madd instruction
9019/// \param VR is a virtual register that holds the value of an ADD operand
9020/// (V in the example above).
9021/// \param RC Register class of operands
9022static MachineInstr *genMaddR(MachineFunction &MF, MachineRegisterInfo &MRI,
9023 const TargetInstrInfo *TII, MachineInstr &Root,
9024 SmallVectorImpl<MachineInstr *> &InsInstrs,
9025 unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR,
9026 const TargetRegisterClass *RC) {
9027 assert(IdxMulOpd == 1 || IdxMulOpd == 2);
9028
9029 MachineInstr *MUL = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: IdxMulOpd).getReg());
9030 Register ResultReg = Root.getOperand(i: 0).getReg();
9031 Register SrcReg0 = MUL->getOperand(i: 1).getReg();
9032 bool Src0IsKill = MUL->getOperand(i: 1).isKill();
9033 Register SrcReg1 = MUL->getOperand(i: 2).getReg();
9034 bool Src1IsKill = MUL->getOperand(i: 2).isKill();
9035
9036 if (ResultReg.isVirtual())
9037 MRI.constrainRegClass(Reg: ResultReg, RC);
9038 if (SrcReg0.isVirtual())
9039 MRI.constrainRegClass(Reg: SrcReg0, RC);
9040 if (SrcReg1.isVirtual())
9041 MRI.constrainRegClass(Reg: SrcReg1, RC);
9042 if (Register::isVirtualRegister(Reg: VR))
9043 MRI.constrainRegClass(Reg: VR, RC);
9044
9045 MachineInstrBuilder MIB =
9046 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MaddOpc), DestReg: ResultReg)
9047 .addReg(RegNo: SrcReg0, Flags: getKillRegState(B: Src0IsKill))
9048 .addReg(RegNo: SrcReg1, Flags: getKillRegState(B: Src1IsKill))
9049 .addReg(RegNo: VR);
9050 // Insert the MADD
9051 InsInstrs.push_back(Elt: MIB);
9052 return MUL;
9053}
9054
9055/// Do the following transformation
9056/// A - (B + C) ==> (A - B) - C
9057/// A - (B + C) ==> (A - C) - B
9058static void genSubAdd2SubSub(MachineFunction &MF, MachineRegisterInfo &MRI,
9059 const TargetInstrInfo *TII, MachineInstr &Root,
9060 SmallVectorImpl<MachineInstr *> &InsInstrs,
9061 SmallVectorImpl<MachineInstr *> &DelInstrs,
9062 unsigned IdxOpd1,
9063 DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
9064 assert(IdxOpd1 == 1 || IdxOpd1 == 2);
9065 unsigned IdxOtherOpd = IdxOpd1 == 1 ? 2 : 1;
9066 MachineInstr *AddMI = MRI.getUniqueVRegDef(Reg: Root.getOperand(i: 2).getReg());
9067
9068 Register ResultReg = Root.getOperand(i: 0).getReg();
9069 Register RegA = Root.getOperand(i: 1).getReg();
9070 bool RegAIsKill = Root.getOperand(i: 1).isKill();
9071 Register RegB = AddMI->getOperand(i: IdxOpd1).getReg();
9072 bool RegBIsKill = AddMI->getOperand(i: IdxOpd1).isKill();
9073 Register RegC = AddMI->getOperand(i: IdxOtherOpd).getReg();
9074 bool RegCIsKill = AddMI->getOperand(i: IdxOtherOpd).isKill();
9075 Register NewVR =
9076 MRI.createVirtualRegister(RegClass: MRI.getRegClass(Reg: Root.getOperand(i: 2).getReg()));
9077
9078 unsigned Opcode = Root.getOpcode();
9079 if (Opcode == AArch64::SUBSWrr)
9080 Opcode = AArch64::SUBWrr;
9081 else if (Opcode == AArch64::SUBSXrr)
9082 Opcode = AArch64::SUBXrr;
9083 else
9084 assert((Opcode == AArch64::SUBWrr || Opcode == AArch64::SUBXrr) &&
9085 "Unexpected instruction opcode.");
9086
9087 uint32_t Flags = Root.mergeFlagsWith(Other: *AddMI);
9088 Flags &= ~MachineInstr::NoSWrap;
9089 Flags &= ~MachineInstr::NoUWrap;
9090
9091 MachineInstrBuilder MIB1 =
9092 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: NewVR)
9093 .addReg(RegNo: RegA, Flags: getKillRegState(B: RegAIsKill))
9094 .addReg(RegNo: RegB, Flags: getKillRegState(B: RegBIsKill))
9095 .setMIFlags(Flags);
9096 MachineInstrBuilder MIB2 =
9097 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode), DestReg: ResultReg)
9098 .addReg(RegNo: NewVR, Flags: getKillRegState(B: true))
9099 .addReg(RegNo: RegC, Flags: getKillRegState(B: RegCIsKill))
9100 .setMIFlags(Flags);
9101
9102 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9103 InsInstrs.push_back(Elt: MIB1);
9104 InsInstrs.push_back(Elt: MIB2);
9105 DelInstrs.push_back(Elt: AddMI);
9106 DelInstrs.push_back(Elt: &Root);
9107}
9108
9109unsigned AArch64InstrInfo::getReduceOpcodeForAccumulator(
9110 unsigned int AccumulatorOpCode) const {
9111 switch (AccumulatorOpCode) {
9112 case AArch64::UABALB_ZZZ_D:
9113 case AArch64::SABALB_ZZZ_D:
9114 case AArch64::UABALT_ZZZ_D:
9115 case AArch64::SABALT_ZZZ_D:
9116 return AArch64::ADD_ZZZ_D;
9117 case AArch64::UABALB_ZZZ_H:
9118 case AArch64::SABALB_ZZZ_H:
9119 case AArch64::UABALT_ZZZ_H:
9120 case AArch64::SABALT_ZZZ_H:
9121 return AArch64::ADD_ZZZ_H;
9122 case AArch64::UABALB_ZZZ_S:
9123 case AArch64::SABALB_ZZZ_S:
9124 case AArch64::UABALT_ZZZ_S:
9125 case AArch64::SABALT_ZZZ_S:
9126 return AArch64::ADD_ZZZ_S;
9127 case AArch64::UABALv16i8_v8i16:
9128 case AArch64::SABALv8i8_v8i16:
9129 case AArch64::SABAv8i16:
9130 case AArch64::UABAv8i16:
9131 return AArch64::ADDv8i16;
9132 case AArch64::SABALv2i32_v2i64:
9133 case AArch64::UABALv2i32_v2i64:
9134 case AArch64::SABALv4i32_v2i64:
9135 return AArch64::ADDv2i64;
9136 case AArch64::UABALv4i16_v4i32:
9137 case AArch64::SABALv4i16_v4i32:
9138 case AArch64::SABALv8i16_v4i32:
9139 case AArch64::SABAv4i32:
9140 case AArch64::UABAv4i32:
9141 return AArch64::ADDv4i32;
9142 case AArch64::UABALv4i32_v2i64:
9143 return AArch64::ADDv2i64;
9144 case AArch64::UABALv8i16_v4i32:
9145 return AArch64::ADDv4i32;
9146 case AArch64::UABALv8i8_v8i16:
9147 case AArch64::SABALv16i8_v8i16:
9148 return AArch64::ADDv8i16;
9149 case AArch64::UABAv16i8:
9150 case AArch64::SABAv16i8:
9151 return AArch64::ADDv16i8;
9152 case AArch64::UABAv4i16:
9153 case AArch64::SABAv4i16:
9154 return AArch64::ADDv4i16;
9155 case AArch64::UABAv2i32:
9156 case AArch64::SABAv2i32:
9157 return AArch64::ADDv2i32;
9158 case AArch64::UABAv8i8:
9159 case AArch64::SABAv8i8:
9160 return AArch64::ADDv8i8;
9161 default:
9162 llvm_unreachable("Unknown accumulator opcode");
9163 }
9164}
9165
9166/// When getMachineCombinerPatterns() finds potential patterns,
9167/// this function generates the instructions that could replace the
9168/// original code sequence
9169void AArch64InstrInfo::genAlternativeCodeSequence(
9170 MachineInstr &Root, unsigned Pattern,
9171 SmallVectorImpl<MachineInstr *> &InsInstrs,
9172 SmallVectorImpl<MachineInstr *> &DelInstrs,
9173 DenseMap<Register, unsigned> &InstrIdxForVirtReg) const {
9174 MachineBasicBlock &MBB = *Root.getParent();
9175 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
9176 MachineFunction &MF = *MBB.getParent();
9177 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
9178
9179 MachineInstr *MUL = nullptr;
9180 const TargetRegisterClass *RC;
9181 unsigned Opc;
9182 switch (Pattern) {
9183 default:
9184 // Reassociate instructions.
9185 TargetInstrInfo::genAlternativeCodeSequence(Root, Pattern, InsInstrs,
9186 DelInstrs, InstIdxForVirtReg&: InstrIdxForVirtReg);
9187 return;
9188 case AArch64MachineCombinerPattern::SUBADD_OP1:
9189 // A - (B + C)
9190 // ==> (A - B) - C
9191 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 1,
9192 InstrIdxForVirtReg);
9193 return;
9194 case AArch64MachineCombinerPattern::SUBADD_OP2:
9195 // A - (B + C)
9196 // ==> (A - C) - B
9197 genSubAdd2SubSub(MF, MRI, TII, Root, InsInstrs, DelInstrs, IdxOpd1: 2,
9198 InstrIdxForVirtReg);
9199 return;
9200 case AArch64MachineCombinerPattern::MULADDW_OP1:
9201 case AArch64MachineCombinerPattern::MULADDX_OP1:
9202 // MUL I=A,B,0
9203 // ADD R,I,C
9204 // ==> MADD R,A,B,C
9205 // --- Create(MADD);
9206 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP1) {
9207 Opc = AArch64::MADDWrrr;
9208 RC = &AArch64::GPR32RegClass;
9209 } else {
9210 Opc = AArch64::MADDXrrr;
9211 RC = &AArch64::GPR64RegClass;
9212 }
9213 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9214 break;
9215 case AArch64MachineCombinerPattern::MULADDW_OP2:
9216 case AArch64MachineCombinerPattern::MULADDX_OP2:
9217 // MUL I=A,B,0
9218 // ADD R,C,I
9219 // ==> MADD R,A,B,C
9220 // --- Create(MADD);
9221 if (Pattern == AArch64MachineCombinerPattern::MULADDW_OP2) {
9222 Opc = AArch64::MADDWrrr;
9223 RC = &AArch64::GPR32RegClass;
9224 } else {
9225 Opc = AArch64::MADDXrrr;
9226 RC = &AArch64::GPR64RegClass;
9227 }
9228 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9229 break;
9230 case AArch64MachineCombinerPattern::MULADDWI_OP1:
9231 case AArch64MachineCombinerPattern::MULADDXI_OP1:
9232 case AArch64MachineCombinerPattern::MULSUBWI_OP1:
9233 case AArch64MachineCombinerPattern::MULSUBXI_OP1: {
9234 // MUL I=A,B,0
9235 // ADD/SUB R,I,Imm
9236 // ==> MOV V, Imm/-Imm
9237 // ==> MADD R,A,B,V
9238 // --- Create(MADD);
9239 const TargetRegisterClass *RC;
9240 unsigned BitSize, MovImm;
9241 if (Pattern == AArch64MachineCombinerPattern::MULADDWI_OP1 ||
9242 Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1) {
9243 MovImm = AArch64::MOVi32imm;
9244 RC = &AArch64::GPR32spRegClass;
9245 BitSize = 32;
9246 Opc = AArch64::MADDWrrr;
9247 RC = &AArch64::GPR32RegClass;
9248 } else {
9249 MovImm = AArch64::MOVi64imm;
9250 RC = &AArch64::GPR64spRegClass;
9251 BitSize = 64;
9252 Opc = AArch64::MADDXrrr;
9253 RC = &AArch64::GPR64RegClass;
9254 }
9255 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9256 uint64_t Imm = Root.getOperand(i: 2).getImm();
9257
9258 if (Root.getOperand(i: 3).isImm()) {
9259 unsigned Val = Root.getOperand(i: 3).getImm();
9260 Imm = Imm << Val;
9261 }
9262 bool IsSub = Pattern == AArch64MachineCombinerPattern::MULSUBWI_OP1 ||
9263 Pattern == AArch64MachineCombinerPattern::MULSUBXI_OP1;
9264 uint64_t UImm = SignExtend64(X: IsSub ? -Imm : Imm, B: BitSize);
9265 // Check that the immediate can be composed via a single instruction.
9266 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
9267 AArch64_IMM::expandMOVImm(Imm: UImm, BitSize, Insn);
9268 if (Insn.size() != 1)
9269 return;
9270 MachineInstrBuilder MIB1 =
9271 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: MovImm), DestReg: NewVR)
9272 .addImm(Val: IsSub ? -Imm : Imm);
9273 InsInstrs.push_back(Elt: MIB1);
9274 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9275 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9276 break;
9277 }
9278 case AArch64MachineCombinerPattern::MULSUBW_OP1:
9279 case AArch64MachineCombinerPattern::MULSUBX_OP1: {
9280 // MUL I=A,B,0
9281 // SUB R,I, C
9282 // ==> SUB V, 0, C
9283 // ==> MADD R,A,B,V // = -C + A*B
9284 // --- Create(MADD);
9285 const TargetRegisterClass *SubRC;
9286 unsigned SubOpc, ZeroReg;
9287 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP1) {
9288 SubOpc = AArch64::SUBWrr;
9289 SubRC = &AArch64::GPR32spRegClass;
9290 ZeroReg = AArch64::WZR;
9291 Opc = AArch64::MADDWrrr;
9292 RC = &AArch64::GPR32RegClass;
9293 } else {
9294 SubOpc = AArch64::SUBXrr;
9295 SubRC = &AArch64::GPR64spRegClass;
9296 ZeroReg = AArch64::XZR;
9297 Opc = AArch64::MADDXrrr;
9298 RC = &AArch64::GPR64RegClass;
9299 }
9300 Register NewVR = MRI.createVirtualRegister(RegClass: SubRC);
9301 // SUB NewVR, 0, C
9302 MachineInstrBuilder MIB1 =
9303 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: SubOpc), DestReg: NewVR)
9304 .addReg(RegNo: ZeroReg)
9305 .add(MO: Root.getOperand(i: 2));
9306 InsInstrs.push_back(Elt: MIB1);
9307 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9308 MUL = genMaddR(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, VR: NewVR, RC);
9309 break;
9310 }
9311 case AArch64MachineCombinerPattern::MULSUBW_OP2:
9312 case AArch64MachineCombinerPattern::MULSUBX_OP2:
9313 // MUL I=A,B,0
9314 // SUB R,C,I
9315 // ==> MSUB R,A,B,C (computes C - A*B)
9316 // --- Create(MSUB);
9317 if (Pattern == AArch64MachineCombinerPattern::MULSUBW_OP2) {
9318 Opc = AArch64::MSUBWrrr;
9319 RC = &AArch64::GPR32RegClass;
9320 } else {
9321 Opc = AArch64::MSUBXrrr;
9322 RC = &AArch64::GPR64RegClass;
9323 }
9324 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9325 break;
9326 case AArch64MachineCombinerPattern::MULADDv8i8_OP1:
9327 Opc = AArch64::MLAv8i8;
9328 RC = &AArch64::FPR64RegClass;
9329 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9330 break;
9331 case AArch64MachineCombinerPattern::MULADDv8i8_OP2:
9332 Opc = AArch64::MLAv8i8;
9333 RC = &AArch64::FPR64RegClass;
9334 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9335 break;
9336 case AArch64MachineCombinerPattern::MULADDv16i8_OP1:
9337 Opc = AArch64::MLAv16i8;
9338 RC = &AArch64::FPR128RegClass;
9339 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9340 break;
9341 case AArch64MachineCombinerPattern::MULADDv16i8_OP2:
9342 Opc = AArch64::MLAv16i8;
9343 RC = &AArch64::FPR128RegClass;
9344 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9345 break;
9346 case AArch64MachineCombinerPattern::MULADDv4i16_OP1:
9347 Opc = AArch64::MLAv4i16;
9348 RC = &AArch64::FPR64RegClass;
9349 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9350 break;
9351 case AArch64MachineCombinerPattern::MULADDv4i16_OP2:
9352 Opc = AArch64::MLAv4i16;
9353 RC = &AArch64::FPR64RegClass;
9354 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9355 break;
9356 case AArch64MachineCombinerPattern::MULADDv8i16_OP1:
9357 Opc = AArch64::MLAv8i16;
9358 RC = &AArch64::FPR128RegClass;
9359 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9360 break;
9361 case AArch64MachineCombinerPattern::MULADDv8i16_OP2:
9362 Opc = AArch64::MLAv8i16;
9363 RC = &AArch64::FPR128RegClass;
9364 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9365 break;
9366 case AArch64MachineCombinerPattern::MULADDv2i32_OP1:
9367 Opc = AArch64::MLAv2i32;
9368 RC = &AArch64::FPR64RegClass;
9369 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9370 break;
9371 case AArch64MachineCombinerPattern::MULADDv2i32_OP2:
9372 Opc = AArch64::MLAv2i32;
9373 RC = &AArch64::FPR64RegClass;
9374 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9375 break;
9376 case AArch64MachineCombinerPattern::MULADDv4i32_OP1:
9377 Opc = AArch64::MLAv4i32;
9378 RC = &AArch64::FPR128RegClass;
9379 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9380 break;
9381 case AArch64MachineCombinerPattern::MULADDv4i32_OP2:
9382 Opc = AArch64::MLAv4i32;
9383 RC = &AArch64::FPR128RegClass;
9384 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9385 break;
9386
9387 case AArch64MachineCombinerPattern::MULSUBv8i8_OP1:
9388 Opc = AArch64::MLAv8i8;
9389 RC = &AArch64::FPR64RegClass;
9390 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9391 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i8,
9392 RC);
9393 break;
9394 case AArch64MachineCombinerPattern::MULSUBv8i8_OP2:
9395 Opc = AArch64::MLSv8i8;
9396 RC = &AArch64::FPR64RegClass;
9397 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9398 break;
9399 case AArch64MachineCombinerPattern::MULSUBv16i8_OP1:
9400 Opc = AArch64::MLAv16i8;
9401 RC = &AArch64::FPR128RegClass;
9402 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9403 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv16i8,
9404 RC);
9405 break;
9406 case AArch64MachineCombinerPattern::MULSUBv16i8_OP2:
9407 Opc = AArch64::MLSv16i8;
9408 RC = &AArch64::FPR128RegClass;
9409 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9410 break;
9411 case AArch64MachineCombinerPattern::MULSUBv4i16_OP1:
9412 Opc = AArch64::MLAv4i16;
9413 RC = &AArch64::FPR64RegClass;
9414 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9415 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9416 RC);
9417 break;
9418 case AArch64MachineCombinerPattern::MULSUBv4i16_OP2:
9419 Opc = AArch64::MLSv4i16;
9420 RC = &AArch64::FPR64RegClass;
9421 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9422 break;
9423 case AArch64MachineCombinerPattern::MULSUBv8i16_OP1:
9424 Opc = AArch64::MLAv8i16;
9425 RC = &AArch64::FPR128RegClass;
9426 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9427 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9428 RC);
9429 break;
9430 case AArch64MachineCombinerPattern::MULSUBv8i16_OP2:
9431 Opc = AArch64::MLSv8i16;
9432 RC = &AArch64::FPR128RegClass;
9433 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9434 break;
9435 case AArch64MachineCombinerPattern::MULSUBv2i32_OP1:
9436 Opc = AArch64::MLAv2i32;
9437 RC = &AArch64::FPR64RegClass;
9438 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9439 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9440 RC);
9441 break;
9442 case AArch64MachineCombinerPattern::MULSUBv2i32_OP2:
9443 Opc = AArch64::MLSv2i32;
9444 RC = &AArch64::FPR64RegClass;
9445 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9446 break;
9447 case AArch64MachineCombinerPattern::MULSUBv4i32_OP1:
9448 Opc = AArch64::MLAv4i32;
9449 RC = &AArch64::FPR128RegClass;
9450 MUL = genFusedMultiplyAccNeg(MF, MRI, TII, Root, InsInstrs,
9451 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9452 RC);
9453 break;
9454 case AArch64MachineCombinerPattern::MULSUBv4i32_OP2:
9455 Opc = AArch64::MLSv4i32;
9456 RC = &AArch64::FPR128RegClass;
9457 MUL = genFusedMultiplyAcc(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9458 break;
9459
9460 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP1:
9461 Opc = AArch64::MLAv4i16_indexed;
9462 RC = &AArch64::FPR64RegClass;
9463 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9464 break;
9465 case AArch64MachineCombinerPattern::MULADDv4i16_indexed_OP2:
9466 Opc = AArch64::MLAv4i16_indexed;
9467 RC = &AArch64::FPR64RegClass;
9468 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9469 break;
9470 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP1:
9471 Opc = AArch64::MLAv8i16_indexed;
9472 RC = &AArch64::FPR128RegClass;
9473 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9474 break;
9475 case AArch64MachineCombinerPattern::MULADDv8i16_indexed_OP2:
9476 Opc = AArch64::MLAv8i16_indexed;
9477 RC = &AArch64::FPR128RegClass;
9478 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9479 break;
9480 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP1:
9481 Opc = AArch64::MLAv2i32_indexed;
9482 RC = &AArch64::FPR64RegClass;
9483 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9484 break;
9485 case AArch64MachineCombinerPattern::MULADDv2i32_indexed_OP2:
9486 Opc = AArch64::MLAv2i32_indexed;
9487 RC = &AArch64::FPR64RegClass;
9488 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9489 break;
9490 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP1:
9491 Opc = AArch64::MLAv4i32_indexed;
9492 RC = &AArch64::FPR128RegClass;
9493 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9494 break;
9495 case AArch64MachineCombinerPattern::MULADDv4i32_indexed_OP2:
9496 Opc = AArch64::MLAv4i32_indexed;
9497 RC = &AArch64::FPR128RegClass;
9498 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9499 break;
9500
9501 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP1:
9502 Opc = AArch64::MLAv4i16_indexed;
9503 RC = &AArch64::FPR64RegClass;
9504 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9505 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i16,
9506 RC);
9507 break;
9508 case AArch64MachineCombinerPattern::MULSUBv4i16_indexed_OP2:
9509 Opc = AArch64::MLSv4i16_indexed;
9510 RC = &AArch64::FPR64RegClass;
9511 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9512 break;
9513 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP1:
9514 Opc = AArch64::MLAv8i16_indexed;
9515 RC = &AArch64::FPR128RegClass;
9516 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9517 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv8i16,
9518 RC);
9519 break;
9520 case AArch64MachineCombinerPattern::MULSUBv8i16_indexed_OP2:
9521 Opc = AArch64::MLSv8i16_indexed;
9522 RC = &AArch64::FPR128RegClass;
9523 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9524 break;
9525 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP1:
9526 Opc = AArch64::MLAv2i32_indexed;
9527 RC = &AArch64::FPR64RegClass;
9528 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9529 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv2i32,
9530 RC);
9531 break;
9532 case AArch64MachineCombinerPattern::MULSUBv2i32_indexed_OP2:
9533 Opc = AArch64::MLSv2i32_indexed;
9534 RC = &AArch64::FPR64RegClass;
9535 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9536 break;
9537 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP1:
9538 Opc = AArch64::MLAv4i32_indexed;
9539 RC = &AArch64::FPR128RegClass;
9540 MUL = genFusedMultiplyIdxNeg(MF, MRI, TII, Root, InsInstrs,
9541 InstrIdxForVirtReg, IdxMulOpd: 1, MaddOpc: Opc, MnegOpc: AArch64::NEGv4i32,
9542 RC);
9543 break;
9544 case AArch64MachineCombinerPattern::MULSUBv4i32_indexed_OP2:
9545 Opc = AArch64::MLSv4i32_indexed;
9546 RC = &AArch64::FPR128RegClass;
9547 MUL = genFusedMultiplyIdx(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9548 break;
9549
9550 // Floating Point Support
9551 case AArch64MachineCombinerPattern::FMULADDH_OP1:
9552 Opc = AArch64::FMADDHrrr;
9553 RC = &AArch64::FPR16RegClass;
9554 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9555 break;
9556 case AArch64MachineCombinerPattern::FMULADDS_OP1:
9557 Opc = AArch64::FMADDSrrr;
9558 RC = &AArch64::FPR32RegClass;
9559 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9560 break;
9561 case AArch64MachineCombinerPattern::FMULADDD_OP1:
9562 Opc = AArch64::FMADDDrrr;
9563 RC = &AArch64::FPR64RegClass;
9564 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9565 break;
9566
9567 case AArch64MachineCombinerPattern::FMULADDH_OP2:
9568 Opc = AArch64::FMADDHrrr;
9569 RC = &AArch64::FPR16RegClass;
9570 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9571 break;
9572 case AArch64MachineCombinerPattern::FMULADDS_OP2:
9573 Opc = AArch64::FMADDSrrr;
9574 RC = &AArch64::FPR32RegClass;
9575 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9576 break;
9577 case AArch64MachineCombinerPattern::FMULADDD_OP2:
9578 Opc = AArch64::FMADDDrrr;
9579 RC = &AArch64::FPR64RegClass;
9580 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9581 break;
9582
9583 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP1:
9584 Opc = AArch64::FMLAv1i32_indexed;
9585 RC = &AArch64::FPR32RegClass;
9586 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9587 kind: FMAInstKind::Indexed);
9588 break;
9589 case AArch64MachineCombinerPattern::FMLAv1i32_indexed_OP2:
9590 Opc = AArch64::FMLAv1i32_indexed;
9591 RC = &AArch64::FPR32RegClass;
9592 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9593 kind: FMAInstKind::Indexed);
9594 break;
9595
9596 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP1:
9597 Opc = AArch64::FMLAv1i64_indexed;
9598 RC = &AArch64::FPR64RegClass;
9599 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9600 kind: FMAInstKind::Indexed);
9601 break;
9602 case AArch64MachineCombinerPattern::FMLAv1i64_indexed_OP2:
9603 Opc = AArch64::FMLAv1i64_indexed;
9604 RC = &AArch64::FPR64RegClass;
9605 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9606 kind: FMAInstKind::Indexed);
9607 break;
9608
9609 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP1:
9610 RC = &AArch64::FPR64RegClass;
9611 Opc = AArch64::FMLAv4i16_indexed;
9612 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9613 kind: FMAInstKind::Indexed);
9614 break;
9615 case AArch64MachineCombinerPattern::FMLAv4f16_OP1:
9616 RC = &AArch64::FPR64RegClass;
9617 Opc = AArch64::FMLAv4f16;
9618 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9619 kind: FMAInstKind::Accumulator);
9620 break;
9621 case AArch64MachineCombinerPattern::FMLAv4i16_indexed_OP2:
9622 RC = &AArch64::FPR64RegClass;
9623 Opc = AArch64::FMLAv4i16_indexed;
9624 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9625 kind: FMAInstKind::Indexed);
9626 break;
9627 case AArch64MachineCombinerPattern::FMLAv4f16_OP2:
9628 RC = &AArch64::FPR64RegClass;
9629 Opc = AArch64::FMLAv4f16;
9630 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9631 kind: FMAInstKind::Accumulator);
9632 break;
9633
9634 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1:
9635 case AArch64MachineCombinerPattern::FMLAv2f32_OP1:
9636 RC = &AArch64::FPR64RegClass;
9637 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP1) {
9638 Opc = AArch64::FMLAv2i32_indexed;
9639 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9640 kind: FMAInstKind::Indexed);
9641 } else {
9642 Opc = AArch64::FMLAv2f32;
9643 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9644 kind: FMAInstKind::Accumulator);
9645 }
9646 break;
9647 case AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2:
9648 case AArch64MachineCombinerPattern::FMLAv2f32_OP2:
9649 RC = &AArch64::FPR64RegClass;
9650 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i32_indexed_OP2) {
9651 Opc = AArch64::FMLAv2i32_indexed;
9652 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9653 kind: FMAInstKind::Indexed);
9654 } else {
9655 Opc = AArch64::FMLAv2f32;
9656 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9657 kind: FMAInstKind::Accumulator);
9658 }
9659 break;
9660
9661 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP1:
9662 RC = &AArch64::FPR128RegClass;
9663 Opc = AArch64::FMLAv8i16_indexed;
9664 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9665 kind: FMAInstKind::Indexed);
9666 break;
9667 case AArch64MachineCombinerPattern::FMLAv8f16_OP1:
9668 RC = &AArch64::FPR128RegClass;
9669 Opc = AArch64::FMLAv8f16;
9670 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9671 kind: FMAInstKind::Accumulator);
9672 break;
9673 case AArch64MachineCombinerPattern::FMLAv8i16_indexed_OP2:
9674 RC = &AArch64::FPR128RegClass;
9675 Opc = AArch64::FMLAv8i16_indexed;
9676 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9677 kind: FMAInstKind::Indexed);
9678 break;
9679 case AArch64MachineCombinerPattern::FMLAv8f16_OP2:
9680 RC = &AArch64::FPR128RegClass;
9681 Opc = AArch64::FMLAv8f16;
9682 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9683 kind: FMAInstKind::Accumulator);
9684 break;
9685
9686 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1:
9687 case AArch64MachineCombinerPattern::FMLAv2f64_OP1:
9688 RC = &AArch64::FPR128RegClass;
9689 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP1) {
9690 Opc = AArch64::FMLAv2i64_indexed;
9691 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9692 kind: FMAInstKind::Indexed);
9693 } else {
9694 Opc = AArch64::FMLAv2f64;
9695 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9696 kind: FMAInstKind::Accumulator);
9697 }
9698 break;
9699 case AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2:
9700 case AArch64MachineCombinerPattern::FMLAv2f64_OP2:
9701 RC = &AArch64::FPR128RegClass;
9702 if (Pattern == AArch64MachineCombinerPattern::FMLAv2i64_indexed_OP2) {
9703 Opc = AArch64::FMLAv2i64_indexed;
9704 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9705 kind: FMAInstKind::Indexed);
9706 } else {
9707 Opc = AArch64::FMLAv2f64;
9708 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9709 kind: FMAInstKind::Accumulator);
9710 }
9711 break;
9712
9713 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1:
9714 case AArch64MachineCombinerPattern::FMLAv4f32_OP1:
9715 RC = &AArch64::FPR128RegClass;
9716 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP1) {
9717 Opc = AArch64::FMLAv4i32_indexed;
9718 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9719 kind: FMAInstKind::Indexed);
9720 } else {
9721 Opc = AArch64::FMLAv4f32;
9722 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9723 kind: FMAInstKind::Accumulator);
9724 }
9725 break;
9726
9727 case AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2:
9728 case AArch64MachineCombinerPattern::FMLAv4f32_OP2:
9729 RC = &AArch64::FPR128RegClass;
9730 if (Pattern == AArch64MachineCombinerPattern::FMLAv4i32_indexed_OP2) {
9731 Opc = AArch64::FMLAv4i32_indexed;
9732 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9733 kind: FMAInstKind::Indexed);
9734 } else {
9735 Opc = AArch64::FMLAv4f32;
9736 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9737 kind: FMAInstKind::Accumulator);
9738 }
9739 break;
9740
9741 case AArch64MachineCombinerPattern::FMULSUBH_OP1:
9742 Opc = AArch64::FNMSUBHrrr;
9743 RC = &AArch64::FPR16RegClass;
9744 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9745 break;
9746 case AArch64MachineCombinerPattern::FMULSUBS_OP1:
9747 Opc = AArch64::FNMSUBSrrr;
9748 RC = &AArch64::FPR32RegClass;
9749 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9750 break;
9751 case AArch64MachineCombinerPattern::FMULSUBD_OP1:
9752 Opc = AArch64::FNMSUBDrrr;
9753 RC = &AArch64::FPR64RegClass;
9754 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9755 break;
9756
9757 case AArch64MachineCombinerPattern::FNMULSUBH_OP1:
9758 Opc = AArch64::FNMADDHrrr;
9759 RC = &AArch64::FPR16RegClass;
9760 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9761 break;
9762 case AArch64MachineCombinerPattern::FNMULSUBS_OP1:
9763 Opc = AArch64::FNMADDSrrr;
9764 RC = &AArch64::FPR32RegClass;
9765 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9766 break;
9767 case AArch64MachineCombinerPattern::FNMULSUBD_OP1:
9768 Opc = AArch64::FNMADDDrrr;
9769 RC = &AArch64::FPR64RegClass;
9770 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC);
9771 break;
9772
9773 case AArch64MachineCombinerPattern::FMULSUBH_OP2:
9774 Opc = AArch64::FMSUBHrrr;
9775 RC = &AArch64::FPR16RegClass;
9776 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9777 break;
9778 case AArch64MachineCombinerPattern::FMULSUBS_OP2:
9779 Opc = AArch64::FMSUBSrrr;
9780 RC = &AArch64::FPR32RegClass;
9781 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9782 break;
9783 case AArch64MachineCombinerPattern::FMULSUBD_OP2:
9784 Opc = AArch64::FMSUBDrrr;
9785 RC = &AArch64::FPR64RegClass;
9786 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC);
9787 break;
9788
9789 case AArch64MachineCombinerPattern::FMLSv1i32_indexed_OP2:
9790 Opc = AArch64::FMLSv1i32_indexed;
9791 RC = &AArch64::FPR32RegClass;
9792 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9793 kind: FMAInstKind::Indexed);
9794 break;
9795
9796 case AArch64MachineCombinerPattern::FMLSv1i64_indexed_OP2:
9797 Opc = AArch64::FMLSv1i64_indexed;
9798 RC = &AArch64::FPR64RegClass;
9799 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9800 kind: FMAInstKind::Indexed);
9801 break;
9802
9803 case AArch64MachineCombinerPattern::FMLSv4f16_OP1:
9804 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP1: {
9805 RC = &AArch64::FPR64RegClass;
9806 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9807 MachineInstrBuilder MIB1 =
9808 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f16), DestReg: NewVR)
9809 .add(MO: Root.getOperand(i: 2));
9810 InsInstrs.push_back(Elt: MIB1);
9811 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9812 if (Pattern == AArch64MachineCombinerPattern::FMLSv4f16_OP1) {
9813 Opc = AArch64::FMLAv4f16;
9814 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9815 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9816 } else {
9817 Opc = AArch64::FMLAv4i16_indexed;
9818 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9819 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9820 }
9821 break;
9822 }
9823 case AArch64MachineCombinerPattern::FMLSv4f16_OP2:
9824 RC = &AArch64::FPR64RegClass;
9825 Opc = AArch64::FMLSv4f16;
9826 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9827 kind: FMAInstKind::Accumulator);
9828 break;
9829 case AArch64MachineCombinerPattern::FMLSv4i16_indexed_OP2:
9830 RC = &AArch64::FPR64RegClass;
9831 Opc = AArch64::FMLSv4i16_indexed;
9832 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9833 kind: FMAInstKind::Indexed);
9834 break;
9835
9836 case AArch64MachineCombinerPattern::FMLSv2f32_OP2:
9837 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2:
9838 RC = &AArch64::FPR64RegClass;
9839 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP2) {
9840 Opc = AArch64::FMLSv2i32_indexed;
9841 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9842 kind: FMAInstKind::Indexed);
9843 } else {
9844 Opc = AArch64::FMLSv2f32;
9845 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9846 kind: FMAInstKind::Accumulator);
9847 }
9848 break;
9849
9850 case AArch64MachineCombinerPattern::FMLSv8f16_OP1:
9851 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP1: {
9852 RC = &AArch64::FPR128RegClass;
9853 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9854 MachineInstrBuilder MIB1 =
9855 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv8f16), DestReg: NewVR)
9856 .add(MO: Root.getOperand(i: 2));
9857 InsInstrs.push_back(Elt: MIB1);
9858 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9859 if (Pattern == AArch64MachineCombinerPattern::FMLSv8f16_OP1) {
9860 Opc = AArch64::FMLAv8f16;
9861 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9862 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9863 } else {
9864 Opc = AArch64::FMLAv8i16_indexed;
9865 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9866 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9867 }
9868 break;
9869 }
9870 case AArch64MachineCombinerPattern::FMLSv8f16_OP2:
9871 RC = &AArch64::FPR128RegClass;
9872 Opc = AArch64::FMLSv8f16;
9873 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9874 kind: FMAInstKind::Accumulator);
9875 break;
9876 case AArch64MachineCombinerPattern::FMLSv8i16_indexed_OP2:
9877 RC = &AArch64::FPR128RegClass;
9878 Opc = AArch64::FMLSv8i16_indexed;
9879 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9880 kind: FMAInstKind::Indexed);
9881 break;
9882
9883 case AArch64MachineCombinerPattern::FMLSv2f64_OP2:
9884 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2:
9885 RC = &AArch64::FPR128RegClass;
9886 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP2) {
9887 Opc = AArch64::FMLSv2i64_indexed;
9888 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9889 kind: FMAInstKind::Indexed);
9890 } else {
9891 Opc = AArch64::FMLSv2f64;
9892 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9893 kind: FMAInstKind::Accumulator);
9894 }
9895 break;
9896
9897 case AArch64MachineCombinerPattern::FMLSv4f32_OP2:
9898 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2:
9899 RC = &AArch64::FPR128RegClass;
9900 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP2) {
9901 Opc = AArch64::FMLSv4i32_indexed;
9902 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9903 kind: FMAInstKind::Indexed);
9904 } else {
9905 Opc = AArch64::FMLSv4f32;
9906 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 2, MaddOpc: Opc, RC,
9907 kind: FMAInstKind::Accumulator);
9908 }
9909 break;
9910 case AArch64MachineCombinerPattern::FMLSv2f32_OP1:
9911 case AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1: {
9912 RC = &AArch64::FPR64RegClass;
9913 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9914 MachineInstrBuilder MIB1 =
9915 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f32), DestReg: NewVR)
9916 .add(MO: Root.getOperand(i: 2));
9917 InsInstrs.push_back(Elt: MIB1);
9918 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9919 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i32_indexed_OP1) {
9920 Opc = AArch64::FMLAv2i32_indexed;
9921 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9922 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9923 } else {
9924 Opc = AArch64::FMLAv2f32;
9925 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9926 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9927 }
9928 break;
9929 }
9930 case AArch64MachineCombinerPattern::FMLSv4f32_OP1:
9931 case AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1: {
9932 RC = &AArch64::FPR128RegClass;
9933 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9934 MachineInstrBuilder MIB1 =
9935 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv4f32), DestReg: NewVR)
9936 .add(MO: Root.getOperand(i: 2));
9937 InsInstrs.push_back(Elt: MIB1);
9938 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9939 if (Pattern == AArch64MachineCombinerPattern::FMLSv4i32_indexed_OP1) {
9940 Opc = AArch64::FMLAv4i32_indexed;
9941 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9942 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9943 } else {
9944 Opc = AArch64::FMLAv4f32;
9945 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9946 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9947 }
9948 break;
9949 }
9950 case AArch64MachineCombinerPattern::FMLSv2f64_OP1:
9951 case AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1: {
9952 RC = &AArch64::FPR128RegClass;
9953 Register NewVR = MRI.createVirtualRegister(RegClass: RC);
9954 MachineInstrBuilder MIB1 =
9955 BuildMI(MF, MIMD: MIMetadata(Root), MCID: TII->get(Opcode: AArch64::FNEGv2f64), DestReg: NewVR)
9956 .add(MO: Root.getOperand(i: 2));
9957 InsInstrs.push_back(Elt: MIB1);
9958 InstrIdxForVirtReg.insert(KV: std::make_pair(x&: NewVR, y: 0));
9959 if (Pattern == AArch64MachineCombinerPattern::FMLSv2i64_indexed_OP1) {
9960 Opc = AArch64::FMLAv2i64_indexed;
9961 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9962 kind: FMAInstKind::Indexed, ReplacedAddend: &NewVR);
9963 } else {
9964 Opc = AArch64::FMLAv2f64;
9965 MUL = genFusedMultiply(MF, MRI, TII, Root, InsInstrs, IdxMulOpd: 1, MaddOpc: Opc, RC,
9966 kind: FMAInstKind::Accumulator, ReplacedAddend: &NewVR);
9967 }
9968 break;
9969 }
9970 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1:
9971 case AArch64MachineCombinerPattern::FMULv2i32_indexed_OP2: {
9972 unsigned IdxDupOp =
9973 (Pattern == AArch64MachineCombinerPattern::FMULv2i32_indexed_OP1) ? 1
9974 : 2;
9975 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i32_indexed,
9976 RC: &AArch64::FPR128RegClass, MRI);
9977 break;
9978 }
9979 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1:
9980 case AArch64MachineCombinerPattern::FMULv2i64_indexed_OP2: {
9981 unsigned IdxDupOp =
9982 (Pattern == AArch64MachineCombinerPattern::FMULv2i64_indexed_OP1) ? 1
9983 : 2;
9984 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv2i64_indexed,
9985 RC: &AArch64::FPR128RegClass, MRI);
9986 break;
9987 }
9988 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1:
9989 case AArch64MachineCombinerPattern::FMULv4i16_indexed_OP2: {
9990 unsigned IdxDupOp =
9991 (Pattern == AArch64MachineCombinerPattern::FMULv4i16_indexed_OP1) ? 1
9992 : 2;
9993 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i16_indexed,
9994 RC: &AArch64::FPR128_loRegClass, MRI);
9995 break;
9996 }
9997 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1:
9998 case AArch64MachineCombinerPattern::FMULv4i32_indexed_OP2: {
9999 unsigned IdxDupOp =
10000 (Pattern == AArch64MachineCombinerPattern::FMULv4i32_indexed_OP1) ? 1
10001 : 2;
10002 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv4i32_indexed,
10003 RC: &AArch64::FPR128RegClass, MRI);
10004 break;
10005 }
10006 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1:
10007 case AArch64MachineCombinerPattern::FMULv8i16_indexed_OP2: {
10008 unsigned IdxDupOp =
10009 (Pattern == AArch64MachineCombinerPattern::FMULv8i16_indexed_OP1) ? 1
10010 : 2;
10011 genIndexedMultiply(Root, InsInstrs, IdxDupOp, MulOpc: AArch64::FMULv8i16_indexed,
10012 RC: &AArch64::FPR128_loRegClass, MRI);
10013 break;
10014 }
10015 case AArch64MachineCombinerPattern::FNMADD: {
10016 MUL = genFNegatedMAD(MF, MRI, TII, Root, InsInstrs);
10017 break;
10018 }
10019 case AArch64MachineCombinerPattern::GATHER_LANE_i32: {
10020 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10021 Pattern, NumLanes: 4);
10022 break;
10023 }
10024 case AArch64MachineCombinerPattern::GATHER_LANE_i16: {
10025 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10026 Pattern, NumLanes: 8);
10027 break;
10028 }
10029 case AArch64MachineCombinerPattern::GATHER_LANE_i8: {
10030 generateGatherLanePattern(Root, InsInstrs, DelInstrs, InstrIdxForVirtReg,
10031 Pattern, NumLanes: 16);
10032 break;
10033 }
10034
10035 } // end switch (Pattern)
10036 // Record MUL and ADD/SUB for deletion
10037 if (MUL)
10038 DelInstrs.push_back(Elt: MUL);
10039 DelInstrs.push_back(Elt: &Root);
10040
10041 // Set the flags on the inserted instructions to be the merged flags of the
10042 // instructions that we have combined.
10043 uint32_t Flags = Root.getFlags();
10044 if (MUL)
10045 Flags = Root.mergeFlagsWith(Other: *MUL);
10046 for (auto *MI : InsInstrs)
10047 MI->setFlags(Flags);
10048}
10049
10050/// Replace csincr-branch sequence by simple conditional branch
10051///
10052/// Examples:
10053/// 1. \code
10054/// csinc w9, wzr, wzr, <condition code>
10055/// tbnz w9, #0, 0x44
10056/// \endcode
10057/// to
10058/// \code
10059/// b.<inverted condition code>
10060/// \endcode
10061///
10062/// 2. \code
10063/// csinc w9, wzr, wzr, <condition code>
10064/// tbz w9, #0, 0x44
10065/// \endcode
10066/// to
10067/// \code
10068/// b.<condition code>
10069/// \endcode
10070///
10071/// Replace compare and branch sequence by TBZ/TBNZ instruction when the
10072/// compare's constant operand is power of 2.
10073///
10074/// Examples:
10075/// \code
10076/// and w8, w8, #0x400
10077/// cbnz w8, L1
10078/// \endcode
10079/// to
10080/// \code
10081/// tbnz w8, #10, L1
10082/// \endcode
10083///
10084/// \param MI Conditional Branch
10085/// \return True when the simple conditional branch is generated
10086///
10087bool AArch64InstrInfo::optimizeCondBranch(MachineInstr &MI) const {
10088 bool IsNegativeBranch = false;
10089 bool IsTestAndBranch = false;
10090 unsigned TargetBBInMI = 0;
10091 switch (MI.getOpcode()) {
10092 default:
10093 llvm_unreachable("Unknown branch instruction?");
10094 case AArch64::Bcc:
10095 case AArch64::CBWPri:
10096 case AArch64::CBXPri:
10097 case AArch64::CBBAssertExt:
10098 case AArch64::CBHAssertExt:
10099 case AArch64::CBWPrr:
10100 case AArch64::CBXPrr:
10101 return false;
10102 case AArch64::CBZW:
10103 case AArch64::CBZX:
10104 TargetBBInMI = 1;
10105 break;
10106 case AArch64::CBNZW:
10107 case AArch64::CBNZX:
10108 TargetBBInMI = 1;
10109 IsNegativeBranch = true;
10110 break;
10111 case AArch64::TBZW:
10112 case AArch64::TBZX:
10113 TargetBBInMI = 2;
10114 IsTestAndBranch = true;
10115 break;
10116 case AArch64::TBNZW:
10117 case AArch64::TBNZX:
10118 TargetBBInMI = 2;
10119 IsNegativeBranch = true;
10120 IsTestAndBranch = true;
10121 break;
10122 }
10123 // So we increment a zero register and test for bits other
10124 // than bit 0? Conservatively bail out in case the verifier
10125 // missed this case.
10126 if (IsTestAndBranch && MI.getOperand(i: 1).getImm())
10127 return false;
10128
10129 // Find Definition.
10130 assert(MI.getParent() && "Incomplete machine instruction\n");
10131 MachineBasicBlock *MBB = MI.getParent();
10132 MachineFunction *MF = MBB->getParent();
10133 MachineRegisterInfo *MRI = &MF->getRegInfo();
10134 Register VReg = MI.getOperand(i: 0).getReg();
10135 if (!VReg.isVirtual())
10136 return false;
10137
10138 MachineInstr *DefMI = MRI->getVRegDef(Reg: VReg);
10139 if (!DefMI)
10140 return false;
10141
10142 // Look through COPY instructions to find definition.
10143 while (DefMI->isCopy()) {
10144 Register CopyVReg = DefMI->getOperand(i: 1).getReg();
10145 if (!CopyVReg.isVirtual())
10146 return false;
10147 if (!MRI->hasOneNonDBGUse(RegNo: CopyVReg))
10148 return false;
10149 DefMI = MRI->getVRegDef(Reg: CopyVReg);
10150 if (!DefMI)
10151 return false;
10152 }
10153
10154 switch (DefMI->getOpcode()) {
10155 default:
10156 return false;
10157 // Fold AND into a TBZ/TBNZ if constant operand is power of 2.
10158 case AArch64::ANDWri:
10159 case AArch64::ANDXri: {
10160 if (IsTestAndBranch)
10161 return false;
10162 if (DefMI->getParent() != MBB)
10163 return false;
10164 if (!MRI->hasOneNonDBGUse(RegNo: VReg))
10165 return false;
10166
10167 bool Is32Bit = (DefMI->getOpcode() == AArch64::ANDWri);
10168 uint64_t Mask = AArch64_AM::decodeLogicalImmediate(
10169 val: DefMI->getOperand(i: 2).getImm(), regSize: Is32Bit ? 32 : 64);
10170 if (!isPowerOf2_64(Value: Mask))
10171 return false;
10172
10173 MachineOperand &MO = DefMI->getOperand(i: 1);
10174 Register NewReg = MO.getReg();
10175 if (!NewReg.isVirtual())
10176 return false;
10177
10178 if (!MRI->getVRegDef(Reg: NewReg))
10179 return false;
10180
10181 MachineBasicBlock &RefToMBB = *MBB;
10182 MachineBasicBlock *TBB = MI.getOperand(i: 1).getMBB();
10183 DebugLoc DL = MI.getDebugLoc();
10184 unsigned Imm = Log2_64(Value: Mask);
10185 unsigned Opc = (Imm < 32)
10186 ? (IsNegativeBranch ? AArch64::TBNZW : AArch64::TBZW)
10187 : (IsNegativeBranch ? AArch64::TBNZX : AArch64::TBZX);
10188 MachineInstr *NewMI = BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: Opc))
10189 .addReg(RegNo: NewReg)
10190 .addImm(Val: Imm)
10191 .addMBB(MBB: TBB);
10192 // Register lives on to the CBZ now.
10193 MO.setIsKill(false);
10194
10195 // For immediate smaller than 32, we need to use the 32-bit
10196 // variant (W) in all cases. Indeed the 64-bit variant does not
10197 // allow to encode them.
10198 // Therefore, if the input register is 64-bit, we need to take the
10199 // 32-bit sub-part.
10200 if (!Is32Bit && Imm < 32)
10201 NewMI->getOperand(i: 0).setSubReg(AArch64::sub_32);
10202 MI.eraseFromParent();
10203 return true;
10204 }
10205 // Look for CSINC
10206 case AArch64::CSINCWr:
10207 case AArch64::CSINCXr: {
10208 if (!(DefMI->getOperand(i: 1).getReg() == AArch64::WZR &&
10209 DefMI->getOperand(i: 2).getReg() == AArch64::WZR) &&
10210 !(DefMI->getOperand(i: 1).getReg() == AArch64::XZR &&
10211 DefMI->getOperand(i: 2).getReg() == AArch64::XZR))
10212 return false;
10213
10214 if (DefMI->findRegisterDefOperandIdx(Reg: AArch64::NZCV, /*TRI=*/nullptr,
10215 isDead: true) != -1)
10216 return false;
10217
10218 AArch64CC::CondCode CC = (AArch64CC::CondCode)DefMI->getOperand(i: 3).getImm();
10219 // Convert only when the condition code is not modified between
10220 // the CSINC and the branch. The CC may be used by other
10221 // instructions in between.
10222 if (areCFlagsAccessedBetweenInstrs(From: DefMI, To: MI, TRI: &getRegisterInfo(), AccessToCheck: AK_Write))
10223 return false;
10224 MachineBasicBlock &RefToMBB = *MBB;
10225 MachineBasicBlock *TBB = MI.getOperand(i: TargetBBInMI).getMBB();
10226 DebugLoc DL = MI.getDebugLoc();
10227 if (IsNegativeBranch)
10228 CC = AArch64CC::getInvertedCondCode(Code: CC);
10229 BuildMI(BB&: RefToMBB, I&: MI, MIMD: DL, MCID: get(Opcode: AArch64::Bcc)).addImm(Val: CC).addMBB(MBB: TBB);
10230 MI.eraseFromParent();
10231 return true;
10232 }
10233 }
10234}
10235
10236std::pair<unsigned, unsigned>
10237AArch64InstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const {
10238 const unsigned Mask = AArch64II::MO_FRAGMENT;
10239 return std::make_pair(x: TF & Mask, y: TF & ~Mask);
10240}
10241
10242ArrayRef<std::pair<unsigned, const char *>>
10243AArch64InstrInfo::getSerializableDirectMachineOperandTargetFlags() const {
10244 using namespace AArch64II;
10245
10246 static const std::pair<unsigned, const char *> TargetFlags[] = {
10247 {MO_PAGE, "aarch64-page"}, {MO_PAGEOFF, "aarch64-pageoff"},
10248 {MO_G3, "aarch64-g3"}, {MO_G2, "aarch64-g2"},
10249 {MO_G1, "aarch64-g1"}, {MO_G0, "aarch64-g0"},
10250 {MO_HI12, "aarch64-hi12"}};
10251 return ArrayRef(TargetFlags);
10252}
10253
10254ArrayRef<std::pair<unsigned, const char *>>
10255AArch64InstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const {
10256 using namespace AArch64II;
10257
10258 static const std::pair<unsigned, const char *> TargetFlags[] = {
10259 {MO_COFFSTUB, "aarch64-coffstub"},
10260 {MO_GOT, "aarch64-got"},
10261 {MO_NC, "aarch64-nc"},
10262 {MO_S, "aarch64-s"},
10263 {MO_TLS, "aarch64-tls"},
10264 {MO_DLLIMPORT, "aarch64-dllimport"},
10265 {MO_PREL, "aarch64-prel"},
10266 {MO_TAGGED, "aarch64-tagged"},
10267 {MO_ARM64EC_CALLMANGLE, "aarch64-arm64ec-callmangle"},
10268 };
10269 return ArrayRef(TargetFlags);
10270}
10271
10272ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
10273AArch64InstrInfo::getSerializableMachineMemOperandTargetFlags() const {
10274 static const std::pair<MachineMemOperand::Flags, const char *> TargetFlags[] =
10275 {{MOSuppressPair, "aarch64-suppress-pair"},
10276 {MOStridedAccess, "aarch64-strided-access"}};
10277 return ArrayRef(TargetFlags);
10278}
10279
10280/// Constants defining how certain sequences should be outlined.
10281/// This encompasses how an outlined function should be called, and what kind of
10282/// frame should be emitted for that outlined function.
10283///
10284/// \p MachineOutlinerDefault implies that the function should be called with
10285/// a save and restore of LR to the stack.
10286///
10287/// That is,
10288///
10289/// I1 Save LR OUTLINED_FUNCTION:
10290/// I2 --> BL OUTLINED_FUNCTION I1
10291/// I3 Restore LR I2
10292/// I3
10293/// RET
10294///
10295/// * Call construction overhead: 3 (save + BL + restore)
10296/// * Frame construction overhead: 1 (ret)
10297/// * Requires stack fixups? Yes
10298///
10299/// \p MachineOutlinerTailCall implies that the function is being created from
10300/// a sequence of instructions ending in a return.
10301///
10302/// That is,
10303///
10304/// I1 OUTLINED_FUNCTION:
10305/// I2 --> B OUTLINED_FUNCTION I1
10306/// RET I2
10307/// RET
10308///
10309/// * Call construction overhead: 1 (B)
10310/// * Frame construction overhead: 0 (Return included in sequence)
10311/// * Requires stack fixups? No
10312///
10313/// \p MachineOutlinerNoLRSave implies that the function should be called using
10314/// a BL instruction, but doesn't require LR to be saved and restored. This
10315/// happens when LR is known to be dead.
10316///
10317/// That is,
10318///
10319/// I1 OUTLINED_FUNCTION:
10320/// I2 --> BL OUTLINED_FUNCTION I1
10321/// I3 I2
10322/// I3
10323/// RET
10324///
10325/// * Call construction overhead: 1 (BL)
10326/// * Frame construction overhead: 1 (RET)
10327/// * Requires stack fixups? No
10328///
10329/// \p MachineOutlinerThunk implies that the function is being created from
10330/// a sequence of instructions ending in a call. The outlined function is
10331/// called with a BL instruction, and the outlined function tail-calls the
10332/// original call destination.
10333///
10334/// That is,
10335///
10336/// I1 OUTLINED_FUNCTION:
10337/// I2 --> BL OUTLINED_FUNCTION I1
10338/// BL f I2
10339/// B f
10340/// * Call construction overhead: 1 (BL)
10341/// * Frame construction overhead: 0
10342/// * Requires stack fixups? No
10343///
10344/// \p MachineOutlinerRegSave implies that the function should be called with a
10345/// save and restore of LR to an available register. This allows us to avoid
10346/// stack fixups. Note that this outlining variant is compatible with the
10347/// NoLRSave case.
10348///
10349/// That is,
10350///
10351/// I1 Save LR OUTLINED_FUNCTION:
10352/// I2 --> BL OUTLINED_FUNCTION I1
10353/// I3 Restore LR I2
10354/// I3
10355/// RET
10356///
10357/// * Call construction overhead: 3 (save + BL + restore)
10358/// * Frame construction overhead: 1 (ret)
10359/// * Requires stack fixups? No
10360enum MachineOutlinerClass {
10361 MachineOutlinerDefault, /// Emit a save, restore, call, and return.
10362 MachineOutlinerTailCall, /// Only emit a branch.
10363 MachineOutlinerNoLRSave, /// Emit a call and return.
10364 MachineOutlinerThunk, /// Emit a call and tail-call.
10365 MachineOutlinerRegSave /// Same as default, but save to a register.
10366};
10367
10368enum MachineOutlinerMBBFlags {
10369 LRUnavailableSomewhere = 0x2,
10370 HasCalls = 0x4,
10371 UnsafeRegsDead = 0x8
10372};
10373
10374/// Return true if the frame-record form of the outlined prologue is enabled for
10375/// the target of \p MF.
10376///
10377/// A non-leaf outlined function must save LR. On MachO, saving LR alone
10378/// (str x30) has no compact unwind encoding, so we get a large DWARF FDE
10379/// instead. Saving FP and LR as a frame record (stp x29, x30 ; mov x29, sp)
10380/// gets the small FRAME encoding, and costs one extra instruction.
10381static bool isCompactUnwindFrameRecordEnabled(const MachineFunction &MF) {
10382 return MF.getSubtarget<AArch64Subtarget>()
10383 .getCLOpts()
10384 .outliner_compact_unwind_frame &&
10385 MF.getFunction().getParent()->getTargetTriple().isOSBinFormatMachO();
10386}
10387
10388/// Return true if the outlined function in \p MBB should save FP and LR as a
10389/// frame record instead of saving LR alone.
10390static bool shouldUseCompactUnwindFrameRecordForOutlinedFunction(
10391 const MachineBasicBlock &MBB) {
10392 const MachineFunction &MF = *MBB.getParent();
10393
10394 // Only worth it if the function has unwind info to shrink.
10395 if (!isCompactUnwindFrameRecordEnabled(MF) ||
10396 !MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF))
10397 return false;
10398
10399 // Only safe if the outlined code never touches FP, since we overwrite it.
10400 LiveRegUnits LRU(*MF.getSubtarget().getRegisterInfo());
10401 for (const MachineInstr &MI : MBB.instrs())
10402 LRU.accumulate(MI);
10403 return LRU.available(Reg: AArch64::FP);
10404}
10405
10406/// Predict what the above will answer, for use while costing candidates. The
10407/// outlined function does not exist yet, so answer from \p RepeatedSequenceLocs
10408/// instead. This is only an estimate; buildOutlinedFrame() makes the call.
10409static bool predictCompactUnwindFrameRecordForOutlinedFunction(
10410 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10411 const TargetRegisterInfo &TRI) {
10412 if (!isCompactUnwindFrameRecordEnabled(MF: *RepeatedSequenceLocs.front().getMF()))
10413 return false;
10414
10415 // The outlined function is nounwind only if every candidate is, so it has
10416 // unwind info if any candidate does.
10417 if (llvm::none_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10418 const MachineFunction &MF = *C.getMF();
10419 return MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF);
10420 }))
10421 return false;
10422
10423 // FP is free in the outlined function only if it is free in every candidate.
10424 return llvm::all_of(Range&: RepeatedSequenceLocs, P: [&TRI](outliner::Candidate &C) {
10425 return C.isAvailableInsideSeq(Reg: AArch64::FP, TRI);
10426 });
10427}
10428
10429Register
10430AArch64InstrInfo::findRegisterToSaveLRTo(outliner::Candidate &C) const {
10431 MachineFunction *MF = C.getMF();
10432 const TargetRegisterInfo &TRI = *MF->getSubtarget().getRegisterInfo();
10433 const AArch64RegisterInfo *ARI =
10434 static_cast<const AArch64RegisterInfo *>(&TRI);
10435 // Check if there is an available register across the sequence that we can
10436 // use.
10437 for (unsigned Reg : AArch64::GPR64RegClass) {
10438 if (!ARI->isReservedReg(MF: *MF, Reg) &&
10439 Reg != AArch64::LR && // LR is not reserved, but don't use it.
10440 Reg != AArch64::X16 && // X16 is not guaranteed to be preserved.
10441 Reg != AArch64::X17 && // Ditto for X17.
10442 C.isAvailableAcrossAndOutOfSeq(Reg, TRI) &&
10443 C.isAvailableInsideSeq(Reg, TRI))
10444 return Reg;
10445 }
10446 return Register();
10447}
10448
10449static bool
10450outliningCandidatesSigningScopeConsensus(const outliner::Candidate &a,
10451 const outliner::Candidate &b) {
10452 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10453 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10454
10455 return MFIa->getSignReturnAddressCondition() ==
10456 MFIb->getSignReturnAddressCondition();
10457}
10458
10459static bool
10460outliningCandidatesSigningKeyConsensus(const outliner::Candidate &a,
10461 const outliner::Candidate &b) {
10462 const auto &MFIa = a.getMF()->getInfo<AArch64FunctionInfo>();
10463 const auto &MFIb = b.getMF()->getInfo<AArch64FunctionInfo>();
10464
10465 return MFIa->shouldSignWithBKey() == MFIb->shouldSignWithBKey();
10466}
10467
10468static bool outliningCandidatesV8_3OpsConsensus(const outliner::Candidate &a,
10469 const outliner::Candidate &b) {
10470 const AArch64Subtarget &SubtargetA =
10471 a.getMF()->getSubtarget<AArch64Subtarget>();
10472 const AArch64Subtarget &SubtargetB =
10473 b.getMF()->getSubtarget<AArch64Subtarget>();
10474 return SubtargetA.hasV8_3aOps() == SubtargetB.hasV8_3aOps();
10475}
10476
10477std::optional<std::unique_ptr<outliner::OutlinedFunction>>
10478AArch64InstrInfo::getOutliningCandidateInfo(
10479 const MachineModuleInfo &MMI,
10480 std::vector<outliner::Candidate> &RepeatedSequenceLocs,
10481 unsigned MinRepeats) const {
10482 unsigned SequenceSize = 0;
10483 for (auto &MI : RepeatedSequenceLocs[0])
10484 SequenceSize += getInstSizeInBytes(MI);
10485
10486 unsigned NumBytesToCreateFrame = 0;
10487
10488 // Avoid splitting ADRP ADD/LDR pair into outlined functions.
10489 // These instructions are fused together by the scheduler.
10490 // Any candidate where ADRP is the last instruction should be rejected
10491 // as that will lead to splitting ADRP pair.
10492 MachineInstr &LastMI = RepeatedSequenceLocs[0].back();
10493 MachineInstr &FirstMI = RepeatedSequenceLocs[0].front();
10494 if (LastMI.getOpcode() == AArch64::ADRP &&
10495 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_PAGE) != 0 &&
10496 (LastMI.getOperand(i: 1).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10497 return std::nullopt;
10498 }
10499
10500 // Similarly any candidate where the first instruction is ADD/LDR with a
10501 // page offset should be rejected to avoid ADRP splitting.
10502 if ((FirstMI.getOpcode() == AArch64::ADDXri ||
10503 FirstMI.getOpcode() == AArch64::LDRXui) &&
10504 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_PAGEOFF) != 0 &&
10505 (FirstMI.getOperand(i: 2).getTargetFlags() & AArch64II::MO_GOT) != 0) {
10506 return std::nullopt;
10507 }
10508
10509 // We only allow outlining for functions having exactly matching return
10510 // address signing attributes, i.e., all share the same value for the
10511 // attribute "sign-return-address" and all share the same type of key they
10512 // are signed with.
10513 // Additionally we require all functions to simultaneously either support
10514 // v8.3a features or not. Otherwise an outlined function could get signed
10515 // using dedicated v8.3 instructions and a call from a function that doesn't
10516 // support v8.3 instructions would therefore be invalid.
10517 if (std::adjacent_find(
10518 first: RepeatedSequenceLocs.begin(), last: RepeatedSequenceLocs.end(),
10519 binary_pred: [](const outliner::Candidate &a, const outliner::Candidate &b) {
10520 // Return true if a and b are non-equal w.r.t. return address
10521 // signing or support of v8.3a features
10522 if (outliningCandidatesSigningScopeConsensus(a, b) &&
10523 outliningCandidatesSigningKeyConsensus(a, b) &&
10524 outliningCandidatesV8_3OpsConsensus(a, b)) {
10525 return false;
10526 }
10527 return true;
10528 }) != RepeatedSequenceLocs.end()) {
10529 return std::nullopt;
10530 }
10531
10532 // Since at this point all candidates agree on their return address signing
10533 // picking just one is fine. If the candidate functions potentially sign their
10534 // return addresses, the outlined function should do the same. Note that in
10535 // the case of "sign-return-address"="non-leaf" this is an assumption: It is
10536 // not certainly true that the outlined function will have to sign its return
10537 // address but this decision is made later, when the decision to outline
10538 // has already been made.
10539 // The same holds for the number of additional instructions we need: On
10540 // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
10541 // necessary. However, at this point we don't know if the outlined function
10542 // will have a RET instruction so we assume the worst.
10543 const TargetRegisterInfo &TRI = getRegisterInfo();
10544 // Performing a tail call may require extra checks when PAuth is enabled.
10545 // If PAuth is disabled, set it to zero for uniformity.
10546 unsigned NumBytesToCheckLRInTCEpilogue = 0;
10547 const auto RASignCondition = RepeatedSequenceLocs[0]
10548 .getMF()
10549 ->getInfo<AArch64FunctionInfo>()
10550 ->getSignReturnAddressCondition();
10551 if (RASignCondition != SignReturnAddress::None) {
10552 // Candidates that have Return Address Authentication Hardening enabled are
10553 // discarded.
10554 //
10555 // In its current form, the machine outliner does not preserve X16/X17
10556 // across outlined function calls, even though it should as they are
10557 // caller-saved registers. And since the hardening based on load from the
10558 // return address may clobber one of these registers, if they are alive
10559 // across a call their value would be lost due to the hardening mechanism.
10560 llvm::erase_if(C&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10561 return C.getMF()
10562 ->getInfo<AArch64FunctionInfo>()
10563 ->shouldHardenSignReturnAddress();
10564 });
10565 // If the sequence doesn't have enough candidates left, then we're done.
10566 if (RepeatedSequenceLocs.size() < MinRepeats)
10567 return std::nullopt;
10568
10569 // One PAC and one AUT instructions
10570 NumBytesToCreateFrame += 8;
10571
10572 // PAuth is enabled - set extra tail call cost, if any.
10573 auto LRCheckMethod = Subtarget.getAuthenticatedLRCheckMethod(
10574 MF: *RepeatedSequenceLocs[0].getMF());
10575 NumBytesToCheckLRInTCEpilogue =
10576 AArch64PAuth::getCheckerSizeInBytes(Method: LRCheckMethod);
10577 // Checking the authenticated LR value may significantly impact
10578 // SequenceSize, so account for it for more precise results.
10579 if (isTailCallReturnInst(MI: RepeatedSequenceLocs[0].back()))
10580 SequenceSize += NumBytesToCheckLRInTCEpilogue;
10581
10582 // We have to check if sp modifying instructions would get outlined.
10583 // If so we only allow outlining if sp is unchanged overall, so matching
10584 // sub and add instructions are okay to outline, all other sp modifications
10585 // are not
10586 auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
10587 int SPValue = 0;
10588 for (auto &MI : C) {
10589 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI)) {
10590 switch (MI.getOpcode()) {
10591 case AArch64::ADDXri:
10592 case AArch64::ADDWri:
10593 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10594 assert(MI.getOperand(2).isImm() &&
10595 "Expected operand to be immediate");
10596 assert(MI.getOperand(1).isReg() &&
10597 "Expected operand to be a register");
10598 // Check if the add just increments sp. If so, we search for
10599 // matching sub instructions that decrement sp. If not, the
10600 // modification is illegal
10601 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10602 SPValue += MI.getOperand(i: 2).getImm();
10603 else
10604 return true;
10605 break;
10606 case AArch64::SUBXri:
10607 case AArch64::SUBWri:
10608 assert(MI.getNumOperands() == 4 && "Wrong number of operands");
10609 assert(MI.getOperand(2).isImm() &&
10610 "Expected operand to be immediate");
10611 assert(MI.getOperand(1).isReg() &&
10612 "Expected operand to be a register");
10613 // Check if the sub just decrements sp. If so, we search for
10614 // matching add instructions that increment sp. If not, the
10615 // modification is illegal
10616 if (MI.getOperand(i: 1).getReg() == AArch64::SP)
10617 SPValue -= MI.getOperand(i: 2).getImm();
10618 else
10619 return true;
10620 break;
10621 default:
10622 return true;
10623 }
10624 }
10625 }
10626 if (SPValue)
10627 return true;
10628 return false;
10629 };
10630 // Remove candidates with illegal stack modifying instructions
10631 llvm::erase_if(C&: RepeatedSequenceLocs, P: hasIllegalSPModification);
10632
10633 // If the sequence doesn't have enough candidates left, then we're done.
10634 if (RepeatedSequenceLocs.size() < MinRepeats)
10635 return std::nullopt;
10636 }
10637
10638 // Properties about candidate MBBs that hold for all of them.
10639 unsigned FlagsSetInAll = 0xF;
10640
10641 // Compute liveness information for each candidate, and set FlagsSetInAll.
10642 for (outliner::Candidate &C : RepeatedSequenceLocs)
10643 FlagsSetInAll &= C.Flags;
10644
10645 unsigned LastInstrOpcode = RepeatedSequenceLocs[0].back().getOpcode();
10646
10647 // Helper lambda which sets call information for every candidate.
10648 auto SetCandidateCallInfo =
10649 [&RepeatedSequenceLocs](unsigned CallID, unsigned NumBytesForCall) {
10650 for (outliner::Candidate &C : RepeatedSequenceLocs)
10651 C.setCallInfo(CID: CallID, CO: NumBytesForCall);
10652 };
10653
10654 unsigned FrameID = MachineOutlinerDefault;
10655 NumBytesToCreateFrame += 4;
10656
10657 bool HasBTI = any_of(Range&: RepeatedSequenceLocs, P: [](outliner::Candidate &C) {
10658 return C.getMF()->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement();
10659 });
10660
10661 // We check to see if CFI Instructions are present, and if they are
10662 // we find the number of CFI Instructions in the candidates.
10663 unsigned CFICount = 0;
10664 for (auto &I : RepeatedSequenceLocs[0]) {
10665 if (I.isCFIInstruction())
10666 CFICount++;
10667 }
10668
10669 // We compare the number of found CFI Instructions to the number of CFI
10670 // instructions in the parent function for each candidate. We must check this
10671 // since if we outline one of the CFI instructions in a function, we have to
10672 // outline them all for correctness. If we do not, the address offsets will be
10673 // incorrect between the two sections of the program.
10674 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10675 std::vector<MCCFIInstruction> CFIInstructions =
10676 C.getMF()->getFrameInstructions();
10677
10678 if (CFICount > 0 && CFICount != CFIInstructions.size())
10679 return std::nullopt;
10680 }
10681
10682 // Returns true if an instructions is safe to fix up, false otherwise.
10683 auto IsSafeToFixup = [this, &TRI](MachineInstr &MI) {
10684 if (MI.isCall())
10685 return true;
10686
10687 if (!MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI) &&
10688 !MI.readsRegister(Reg: AArch64::SP, TRI: &TRI))
10689 return true;
10690
10691 // Any modification of SP will break our code to save/restore LR.
10692 // FIXME: We could handle some instructions which add a constant
10693 // offset to SP, with a bit more work.
10694 if (MI.modifiesRegister(Reg: AArch64::SP, TRI: &TRI))
10695 return false;
10696
10697 // At this point, we have a stack instruction that we might need to
10698 // fix up. We'll handle it if it's a load or store.
10699 if (MI.mayLoadOrStore()) {
10700 const MachineOperand *Base; // Filled with the base operand of MI.
10701 int64_t Offset; // Filled with the offset of MI.
10702 bool OffsetIsScalable;
10703
10704 // Does it allow us to offset the base operand and is the base the
10705 // register SP?
10706 if (!getMemOperandWithOffset(MI, BaseOp&: Base, Offset, OffsetIsScalable) ||
10707 !Base->isReg() || Base->getReg() != AArch64::SP)
10708 return false;
10709
10710 // Fixe-up code below assumes bytes.
10711 if (OffsetIsScalable)
10712 return false;
10713
10714 // Find the minimum/maximum offset for this instruction and check
10715 // if fixing it up would be in range.
10716 int64_t MinOffset,
10717 MaxOffset; // Unscaled offsets for the instruction.
10718 // The scale to multiply the offsets by.
10719 TypeSize Scale(0U, false), DummyWidth(0U, false);
10720 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width&: DummyWidth, MinOffset, MaxOffset);
10721
10722 Offset += 16; // Update the offset to what it would be if we outlined.
10723 if (Offset < MinOffset * (int64_t)Scale.getFixedValue() ||
10724 Offset > MaxOffset * (int64_t)Scale.getFixedValue())
10725 return false;
10726
10727 // It's in range, so we can outline it.
10728 return true;
10729 }
10730
10731 // FIXME: Add handling for instructions like "add x0, sp, #8".
10732
10733 // We can't fix it up, so don't outline it.
10734 return false;
10735 };
10736
10737 // True if it's possible to fix up each stack instruction in this sequence.
10738 // Important for frames/call variants that modify the stack.
10739 bool AllStackInstrsSafe =
10740 llvm::all_of(Range&: RepeatedSequenceLocs[0], P: IsSafeToFixup);
10741
10742 // If the last instruction in any candidate is a terminator, then we should
10743 // tail call all of the candidates.
10744 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10745 FrameID = MachineOutlinerTailCall;
10746 NumBytesToCreateFrame = 0;
10747 unsigned NumBytesForCall = 4 + NumBytesToCheckLRInTCEpilogue;
10748 SetCandidateCallInfo(MachineOutlinerTailCall, NumBytesForCall);
10749 }
10750
10751 else if (LastInstrOpcode == AArch64::BL ||
10752 ((LastInstrOpcode == AArch64::BLR ||
10753 LastInstrOpcode == AArch64::BLRNoIP) &&
10754 !HasBTI)) {
10755 // FIXME: Do we need to check if the code after this uses the value of LR?
10756 FrameID = MachineOutlinerThunk;
10757 NumBytesToCreateFrame = NumBytesToCheckLRInTCEpilogue;
10758 SetCandidateCallInfo(MachineOutlinerThunk, 4);
10759 }
10760
10761 else {
10762 // We need to decide how to emit calls + frames. We can always emit the same
10763 // frame if we don't need to save to the stack. If we have to save to the
10764 // stack, then we need a different frame.
10765 unsigned NumBytesNoStackCalls = 0;
10766 std::vector<outliner::Candidate> CandidatesWithoutStackFixups;
10767
10768 // Check if we have to save LR.
10769 for (outliner::Candidate &C : RepeatedSequenceLocs) {
10770 bool LRAvailable =
10771 (C.Flags & MachineOutlinerMBBFlags::LRUnavailableSomewhere)
10772 ? C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI)
10773 : true;
10774 // If we have a noreturn caller, then we're going to be conservative and
10775 // say that we have to save LR. If we don't have a ret at the end of the
10776 // block, then we can't reason about liveness accurately.
10777 //
10778 // FIXME: We can probably do better than always disabling this in
10779 // noreturn functions by fixing up the liveness info.
10780 bool IsNoReturn =
10781 C.getMF()->getFunction().hasFnAttribute(Kind: Attribute::NoReturn);
10782
10783 // Is LR available? If so, we don't need a save.
10784 if (LRAvailable && !IsNoReturn) {
10785 NumBytesNoStackCalls += 4;
10786 C.setCallInfo(CID: MachineOutlinerNoLRSave, CO: 4);
10787 CandidatesWithoutStackFixups.push_back(x: C);
10788 }
10789
10790 // Is an unused register available? If so, we won't modify the stack, so
10791 // we can outline with the same frame type as those that don't save LR.
10792 else if (findRegisterToSaveLRTo(C)) {
10793 NumBytesNoStackCalls += 12;
10794 C.setCallInfo(CID: MachineOutlinerRegSave, CO: 12);
10795 CandidatesWithoutStackFixups.push_back(x: C);
10796 }
10797
10798 // Is SP used in the sequence at all? If not, we don't have to modify
10799 // the stack, so we are guaranteed to get the same frame.
10800 else if (C.isAvailableInsideSeq(Reg: AArch64::SP, TRI)) {
10801 NumBytesNoStackCalls += 12;
10802 C.setCallInfo(CID: MachineOutlinerDefault, CO: 12);
10803 CandidatesWithoutStackFixups.push_back(x: C);
10804 }
10805
10806 // If we outline this, we need to modify the stack. Pretend we don't
10807 // outline this by saving all of its bytes.
10808 else {
10809 NumBytesNoStackCalls += SequenceSize;
10810 }
10811 }
10812
10813 // If there are no places where we have to save LR, then note that we
10814 // don't have to update the stack. Otherwise, give every candidate the
10815 // default call type, as long as it's safe to do so.
10816 if (!AllStackInstrsSafe ||
10817 NumBytesNoStackCalls <= RepeatedSequenceLocs.size() * 12) {
10818 RepeatedSequenceLocs = CandidatesWithoutStackFixups;
10819 FrameID = MachineOutlinerNoLRSave;
10820 if (RepeatedSequenceLocs.size() < MinRepeats)
10821 return std::nullopt;
10822 } else {
10823 SetCandidateCallInfo(MachineOutlinerDefault, 12);
10824
10825 // Bugzilla ID: 46767
10826 // TODO: Check if fixing up the stack more than once is safe so we can
10827 // outline these.
10828 //
10829 // An outline resulting in a caller that requires stack fixups at the
10830 // callsite to a callee that also requires stack fixups can happen when
10831 // there are no available registers at the candidate callsite for a
10832 // candidate that itself also has calls.
10833 //
10834 // In other words if function_containing_sequence in the following pseudo
10835 // assembly requires that we save LR at the point of the call, but there
10836 // are no available registers: in this case we save using SP and as a
10837 // result the SP offsets requires stack fixups by multiples of 16.
10838 //
10839 // function_containing_sequence:
10840 // ...
10841 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10842 // call OUTLINED_FUNCTION_N
10843 // restore LR from SP
10844 // ...
10845 //
10846 // OUTLINED_FUNCTION_N:
10847 // save LR to SP <- Requires stack instr fixups in OUTLINED_FUNCTION_N
10848 // ...
10849 // bl foo
10850 // restore LR from SP
10851 // ret
10852 //
10853 // Because the code to handle more than one stack fixup does not
10854 // currently have the proper checks for legality, these cases will assert
10855 // in the AArch64 MachineOutliner. This is because the code to do this
10856 // needs more hardening, testing, better checks that generated code is
10857 // legal, etc and because it is only verified to handle a single pass of
10858 // stack fixup.
10859 //
10860 // The assert happens in AArch64InstrInfo::buildOutlinedFrame to catch
10861 // these cases until they are known to be handled. Bugzilla 46767 is
10862 // referenced in comments at the assert site.
10863 //
10864 // To avoid asserting (or generating non-legal code on noassert builds)
10865 // we remove all candidates which would need more than one stack fixup by
10866 // pruning the cases where the candidate has calls while also having no
10867 // available LR and having no available general purpose registers to copy
10868 // LR to (ie one extra stack save/restore).
10869 //
10870 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10871 erase_if(C&: RepeatedSequenceLocs, P: [this, &TRI](outliner::Candidate &C) {
10872 auto IsCall = [](const MachineInstr &MI) { return MI.isCall(); };
10873 return (llvm::any_of(Range&: C, P: IsCall)) &&
10874 (!C.isAvailableAcrossAndOutOfSeq(Reg: AArch64::LR, TRI) ||
10875 !findRegisterToSaveLRTo(C));
10876 });
10877 }
10878 }
10879
10880 // If we dropped all of the candidates, bail out here.
10881 if (RepeatedSequenceLocs.size() < MinRepeats)
10882 return std::nullopt;
10883 }
10884
10885 // Does every candidate's MBB contain a call? If so, then we might have a call
10886 // in the range.
10887 if (FlagsSetInAll & MachineOutlinerMBBFlags::HasCalls) {
10888 // Check if the range contains a call. These require a save + restore of the
10889 // link register.
10890 outliner::Candidate &FirstCand = RepeatedSequenceLocs[0];
10891 bool ModStackToSaveLR = false;
10892 if (any_of(Range: drop_end(RangeOrContainer&: FirstCand),
10893 P: [](const MachineInstr &MI) { return MI.isCall(); }))
10894 ModStackToSaveLR = true;
10895
10896 // Handle the last instruction separately. If this is a tail call, then the
10897 // last instruction is a call. We don't want to save + restore in this case.
10898 // However, it could be possible that the last instruction is a call without
10899 // it being valid to tail call this sequence. We should consider this as
10900 // well.
10901 else if (FrameID != MachineOutlinerThunk &&
10902 FrameID != MachineOutlinerTailCall && FirstCand.back().isCall())
10903 ModStackToSaveLR = true;
10904
10905 if (ModStackToSaveLR) {
10906 // We can't fix up the stack. Bail out.
10907 if (!AllStackInstrsSafe)
10908 return std::nullopt;
10909
10910 // Save + restore LR.
10911 NumBytesToCreateFrame += 8;
10912
10913 // Add the extra mov if we will save a frame record instead of just LR.
10914 if (predictCompactUnwindFrameRecordForOutlinedFunction(
10915 RepeatedSequenceLocs, TRI))
10916 NumBytesToCreateFrame += 4;
10917 }
10918 }
10919
10920 // If we have CFI instructions, we can only outline if the outlined section
10921 // can be a tail call
10922 if (FrameID != MachineOutlinerTailCall && CFICount > 0)
10923 return std::nullopt;
10924
10925 return std::make_unique<outliner::OutlinedFunction>(
10926 args&: RepeatedSequenceLocs, args&: SequenceSize, args&: NumBytesToCreateFrame, args&: FrameID);
10927}
10928
10929void AArch64InstrInfo::mergeOutliningCandidateAttributes(
10930 Function &F, std::vector<outliner::Candidate> &Candidates) const {
10931 // If a bunch of candidates reach this point they must agree on their return
10932 // address signing. It is therefore enough to just consider the signing
10933 // behaviour of one of them
10934 const auto &CFn = Candidates.front().getMF()->getFunction();
10935
10936 if (CFn.hasFnAttribute(Kind: "ptrauth-returns"))
10937 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-returns"));
10938 if (CFn.hasFnAttribute(Kind: "ptrauth-auth-traps"))
10939 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "ptrauth-auth-traps"));
10940 // Since all candidates belong to the same module, just copy the
10941 // function-level attributes of an arbitrary function.
10942 if (CFn.hasFnAttribute(Kind: "sign-return-address"))
10943 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address"));
10944 if (CFn.hasFnAttribute(Kind: "sign-return-address-key"))
10945 F.addFnAttr(Attr: CFn.getFnAttribute(Kind: "sign-return-address-key"));
10946 // The "sign-return-address-harden" attribute is not included because no
10947 // candidate is supposed to have hardening enabled.
10948 assert(llvm::none_of(Candidates, [](const outliner::Candidate &C) {
10949 return C.getMF()
10950 ->getInfo<AArch64FunctionInfo>()
10951 ->shouldHardenSignReturnAddress();
10952 }));
10953
10954 AArch64GenInstrInfo::mergeOutliningCandidateAttributes(F, Candidates);
10955}
10956
10957bool AArch64InstrInfo::isFunctionSafeToOutlineFrom(
10958 MachineFunction &MF, bool OutlineFromLinkOnceODRs) const {
10959 const Function &F = MF.getFunction();
10960
10961 // Can F be deduplicated by the linker? If it can, don't outline from it.
10962 if (!OutlineFromLinkOnceODRs && F.hasLinkOnceODRLinkage())
10963 return false;
10964
10965 // Don't outline from functions with section markings; the program could
10966 // expect that all the code is in the named section.
10967 // FIXME: Allow outlining from multiple functions with the same section
10968 // marking.
10969 if (F.hasSection())
10970 return false;
10971
10972 // Outlining from functions with redzones is unsafe since the outliner may
10973 // modify the stack. Check if hasRedZone is true or unknown; if yes, don't
10974 // outline from it.
10975 AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
10976 if (!AFI || AFI->hasRedZone().value_or(u: true))
10977 return false;
10978
10979 // FIXME: Determine whether it is safe to outline from functions which contain
10980 // streaming-mode changes. We may need to ensure any smstart/smstop pairs are
10981 // outlined together and ensure it is safe to outline with async unwind info,
10982 // required for saving & restoring VG around calls.
10983 if (AFI->hasStreamingModeChanges())
10984 return false;
10985
10986 // FIXME: Teach the outliner to generate/handle Windows unwind info.
10987 if (MF.getTarget().getMCAsmInfo().usesWindowsCFI())
10988 return false;
10989
10990 // It's safe to outline from MF.
10991 return true;
10992}
10993
10994SmallVector<std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
10995AArch64InstrInfo::getOutlinableRanges(MachineBasicBlock &MBB,
10996 unsigned &Flags) const {
10997 assert(MBB.getParent()->getRegInfo().tracksLiveness() &&
10998 "Must track liveness!");
10999 SmallVector<
11000 std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
11001 Ranges;
11002 // According to the AArch64 Procedure Call Standard, the following are
11003 // undefined on entry/exit from a function call:
11004 //
11005 // * Registers x16, x17, (and thus w16, w17)
11006 // * Condition codes (and thus the NZCV register)
11007 //
11008 // If any of these registers are used inside or live across an outlined
11009 // function, then they may be modified later, either by the compiler or
11010 // some other tool (like the linker).
11011 //
11012 // To avoid outlining in these situations, partition each block into ranges
11013 // where these registers are dead. We will only outline from those ranges.
11014 LiveRegUnits LRU(getRegisterInfo());
11015 auto AreAllUnsafeRegsDead = [&LRU]() {
11016 return LRU.available(Reg: AArch64::W16) && LRU.available(Reg: AArch64::W17) &&
11017 LRU.available(Reg: AArch64::NZCV);
11018 };
11019
11020 // We need to know if LR is live across an outlining boundary later on in
11021 // order to decide how we'll create the outlined call, frame, etc.
11022 //
11023 // It's pretty expensive to check this for *every candidate* within a block.
11024 // That's some potentially n^2 behaviour, since in the worst case, we'd need
11025 // to compute liveness from the end of the block for O(n) candidates within
11026 // the block.
11027 //
11028 // So, to improve the average case, let's keep track of liveness from the end
11029 // of the block to the beginning of *every outlinable range*. If we know that
11030 // LR is available in every range we could outline from, then we know that
11031 // we don't need to check liveness for any candidate within that range.
11032 bool LRAvailableEverywhere = true;
11033 // Compute liveness bottom-up.
11034 LRU.addLiveOuts(MBB);
11035 // Update flags that require info about the entire MBB.
11036 auto UpdateWholeMBBFlags = [&Flags](const MachineInstr &MI) {
11037 if (MI.isCall() && !MI.isTerminator())
11038 Flags |= MachineOutlinerMBBFlags::HasCalls;
11039 };
11040 // Range: [RangeBegin, RangeEnd)
11041 MachineBasicBlock::instr_iterator RangeBegin, RangeEnd;
11042 unsigned RangeLen;
11043 auto CreateNewRangeStartingAt =
11044 [&RangeBegin, &RangeEnd,
11045 &RangeLen](MachineBasicBlock::instr_iterator NewBegin) {
11046 RangeBegin = NewBegin;
11047 RangeEnd = std::next(x: RangeBegin);
11048 RangeLen = 0;
11049 };
11050 auto SaveRangeIfNonEmpty = [&RangeLen, &Ranges, &RangeBegin, &RangeEnd]() {
11051 // At least one unsafe register is not dead. We do not want to outline at
11052 // this point. If it is long enough to outline from and does not cross a
11053 // bundle boundary, save the range [RangeBegin, RangeEnd).
11054 if (RangeLen <= 1)
11055 return;
11056 if (!RangeBegin.isEnd() && RangeBegin->isBundledWithPred())
11057 return;
11058 if (!RangeEnd.isEnd() && RangeEnd->isBundledWithPred())
11059 return;
11060 Ranges.emplace_back(Args&: RangeBegin, Args&: RangeEnd);
11061 };
11062 // Find the first point where all unsafe registers are dead.
11063 // FIND: <safe instr> <-- end of first potential range
11064 // SKIP: <unsafe def>
11065 // SKIP: ... everything between ...
11066 // SKIP: <unsafe use>
11067 auto FirstPossibleEndPt = MBB.instr_rbegin();
11068 for (; FirstPossibleEndPt != MBB.instr_rend(); ++FirstPossibleEndPt) {
11069 if (!FirstPossibleEndPt->isDebugInstr())
11070 LRU.stepBackward(MI: *FirstPossibleEndPt);
11071 // Update flags that impact how we outline across the entire block,
11072 // regardless of safety.
11073 UpdateWholeMBBFlags(*FirstPossibleEndPt);
11074 if (AreAllUnsafeRegsDead())
11075 break;
11076 }
11077 // If we exhausted the entire block, we have no safe ranges to outline.
11078 if (FirstPossibleEndPt == MBB.instr_rend())
11079 return Ranges;
11080 // Current range.
11081 CreateNewRangeStartingAt(FirstPossibleEndPt->getIterator());
11082 // StartPt points to the first place where all unsafe registers
11083 // are dead (if there is any such point). Begin partitioning the MBB into
11084 // ranges.
11085 // Pseudo probes are ordinary instructions to the mapper, so only skip debug
11086 // instructions here.
11087 for (auto &MI : instructionsWithoutDebug(It: FirstPossibleEndPt, End: MBB.instr_rend(),
11088 /*SkipPseudoOp=*/false)) {
11089 LRU.stepBackward(MI);
11090 UpdateWholeMBBFlags(MI);
11091 if (!AreAllUnsafeRegsDead()) {
11092 SaveRangeIfNonEmpty();
11093 CreateNewRangeStartingAt(MI.getIterator());
11094 continue;
11095 }
11096 LRAvailableEverywhere &= LRU.available(Reg: AArch64::LR);
11097 // Pseudo probes count because the mapper treats them as ordinary
11098 // instructions.
11099 RangeBegin = MI.getIterator();
11100 ++RangeLen;
11101 }
11102 // Above loop misses the last (or only) range. If we are still safe, then
11103 // let's save the range.
11104 if (AreAllUnsafeRegsDead())
11105 SaveRangeIfNonEmpty();
11106 if (Ranges.empty())
11107 return Ranges;
11108 // We found the ranges bottom-up. Mapping expects the top-down. Reverse
11109 // the order.
11110 std::reverse(first: Ranges.begin(), last: Ranges.end());
11111 // If there is at least one outlinable range where LR is unavailable
11112 // somewhere, remember that.
11113 if (!LRAvailableEverywhere)
11114 Flags |= MachineOutlinerMBBFlags::LRUnavailableSomewhere;
11115 return Ranges;
11116}
11117
11118outliner::InstrType
11119AArch64InstrInfo::getOutliningTypeImpl(const MachineModuleInfo &MMI,
11120 MachineBasicBlock::iterator &MIT,
11121 unsigned Flags) const {
11122 MachineInstr &MI = *MIT;
11123
11124 // Don't outline anything used for return address signing. The outlined
11125 // function will get signed later if needed
11126 switch (MI.getOpcode()) {
11127 case AArch64::PACM:
11128 case AArch64::PACIASP:
11129 case AArch64::PACIBSP:
11130 case AArch64::PACIASPPC:
11131 case AArch64::PACIBSPPC:
11132 case AArch64::AUTIASP:
11133 case AArch64::AUTIBSP:
11134 case AArch64::AUTIASPPCi:
11135 case AArch64::AUTIASPPCr:
11136 case AArch64::AUTIBSPPCi:
11137 case AArch64::AUTIBSPPCr:
11138 case AArch64::RETAA:
11139 case AArch64::RETAB:
11140 case AArch64::RETAASPPCi:
11141 case AArch64::RETAASPPCr:
11142 case AArch64::RETABSPPCi:
11143 case AArch64::RETABSPPCr:
11144 case AArch64::EMITBKEY:
11145 case AArch64::PAUTH_PROLOGUE:
11146 case AArch64::PAUTH_EPILOGUE:
11147 return outliner::InstrType::Illegal;
11148 }
11149
11150 // We can only outline these if we will tail call the outlined function, or
11151 // fix up the CFI offsets. Currently, CFI instructions are outlined only if
11152 // in a tail call.
11153 //
11154 // FIXME: If the proper fixups for the offset are implemented, this should be
11155 // possible.
11156 if (MI.isCFIInstruction())
11157 return outliner::InstrType::Legal;
11158
11159 // Is this a terminator for a basic block?
11160 if (MI.isTerminator())
11161 // TargetInstrInfo::getOutliningType has already filtered out anything
11162 // that would break this, so we can allow it here.
11163 return outliner::InstrType::Legal;
11164
11165 // Make sure none of the operands are un-outlinable.
11166 for (const MachineOperand &MOP : MI.operands()) {
11167 // A check preventing CFI indices was here before, but only CFI
11168 // instructions should have those.
11169 assert(!MOP.isCFIIndex());
11170
11171 // If it uses LR or W30 explicitly, then don't touch it.
11172 if (MOP.isReg() && !MOP.isImplicit() &&
11173 (MOP.getReg() == AArch64::LR || MOP.getReg() == AArch64::W30))
11174 return outliner::InstrType::Illegal;
11175 }
11176
11177 // Special cases for instructions that can always be outlined, but will fail
11178 // the later tests. e.g, ADRPs, which are PC-relative use LR, but can always
11179 // be outlined because they don't require a *specific* value to be in LR.
11180 if (MI.getOpcode() == AArch64::ADRP)
11181 return outliner::InstrType::Legal;
11182
11183 // If MI is a call we might be able to outline it. We don't want to outline
11184 // any calls that rely on the position of items on the stack. When we outline
11185 // something containing a call, we have to emit a save and restore of LR in
11186 // the outlined function. Currently, this always happens by saving LR to the
11187 // stack. Thus, if we outline, say, half the parameters for a function call
11188 // plus the call, then we'll break the callee's expectations for the layout
11189 // of the stack.
11190 //
11191 // FIXME: Allow calls to functions which construct a stack frame, as long
11192 // as they don't access arguments on the stack.
11193 // FIXME: Figure out some way to analyze functions defined in other modules.
11194 // We should be able to compute the memory usage based on the IR calling
11195 // convention, even if we can't see the definition.
11196 if (MI.isCall()) {
11197 // Get the function associated with the call. Look at each operand and find
11198 // the one that represents the callee and get its name.
11199 const Function *Callee = nullptr;
11200 for (const MachineOperand &MOP : MI.operands()) {
11201 if (MOP.isGlobal()) {
11202 Callee = dyn_cast<Function>(Val: MOP.getGlobal());
11203 break;
11204 }
11205 }
11206
11207 // Never outline calls to mcount. There isn't any rule that would require
11208 // this, but the Linux kernel's "ftrace" feature depends on it.
11209 if (Callee && Callee->getName() == "\01_mcount")
11210 return outliner::InstrType::Illegal;
11211
11212 // If we don't know anything about the callee, assume it depends on the
11213 // stack layout of the caller. In that case, it's only legal to outline
11214 // as a tail-call. Explicitly list the call instructions we know about so we
11215 // don't get unexpected results with call pseudo-instructions.
11216 auto UnknownCallOutlineType = outliner::InstrType::Illegal;
11217 if (MI.getOpcode() == AArch64::BLR ||
11218 MI.getOpcode() == AArch64::BLRNoIP || MI.getOpcode() == AArch64::BL)
11219 UnknownCallOutlineType = outliner::InstrType::LegalTerminator;
11220
11221 if (!Callee)
11222 return UnknownCallOutlineType;
11223
11224 // We have a function we have information about. Check it if it's something
11225 // can safely outline.
11226 MachineFunction *CalleeMF = MMI.getMachineFunction(F: *Callee);
11227
11228 // We don't know what's going on with the callee at all. Don't touch it.
11229 if (!CalleeMF)
11230 return UnknownCallOutlineType;
11231
11232 // Check if we know anything about the callee saves on the function. If we
11233 // don't, then don't touch it, since that implies that we haven't
11234 // computed anything about its stack frame yet.
11235 MachineFrameInfo &MFI = CalleeMF->getFrameInfo();
11236 if (!MFI.isCalleeSavedInfoValid() || MFI.getStackSize() > 0 ||
11237 MFI.getNumObjects() > 0)
11238 return UnknownCallOutlineType;
11239
11240 // At this point, we can say that CalleeMF ought to not pass anything on the
11241 // stack. Therefore, we can outline it.
11242 return outliner::InstrType::Legal;
11243 }
11244
11245 // Don't touch the link register or W30.
11246 if (MI.readsRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()) ||
11247 MI.modifiesRegister(Reg: AArch64::W30, TRI: &getRegisterInfo()))
11248 return outliner::InstrType::Illegal;
11249
11250 // Don't outline BTI instructions, because that will prevent the outlining
11251 // site from being indirectly callable.
11252 if (hasBTISemantics(MI))
11253 return outliner::InstrType::Illegal;
11254
11255 return outliner::InstrType::Legal;
11256}
11257
11258void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
11259 for (MachineInstr &MI : MBB) {
11260 const MachineOperand *Base;
11261 TypeSize Width(0, false);
11262 int64_t Offset;
11263 bool OffsetIsScalable;
11264
11265 // Is this a load or store with an immediate offset with SP as the base?
11266 if (!MI.mayLoadOrStore() ||
11267 !getMemOperandWithOffsetWidth(LdSt: MI, BaseOp&: Base, Offset, OffsetIsScalable,
11268 Width) ||
11269 (Base->isReg() && Base->getReg() != AArch64::SP))
11270 continue;
11271
11272 // It is, so we have to fix it up.
11273 TypeSize Scale(0U, false);
11274 int64_t Dummy1, Dummy2;
11275
11276 MachineOperand &StackOffsetOperand = getMemOpBaseRegImmOfsOffsetOperand(LdSt&: MI);
11277 assert(StackOffsetOperand.isImm() && "Stack offset wasn't immediate!");
11278 getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset&: Dummy1, MaxOffset&: Dummy2);
11279 assert(Scale != 0 && "Unexpected opcode!");
11280 assert(!OffsetIsScalable && "Expected offset to be a byte offset");
11281
11282 // We've pushed the return address to the stack, so add 16 to the offset.
11283 // This is safe, since we already checked if it would overflow when we
11284 // checked if this instruction was legal to outline.
11285 int64_t NewImm = (Offset + 16) / (int64_t)Scale.getFixedValue();
11286 StackOffsetOperand.setImm(NewImm);
11287 }
11288}
11289
11290static void signOutlinedFunction(MachineFunction &MF, MachineBasicBlock &MBB,
11291 const AArch64InstrInfo *TII,
11292 bool ShouldSignReturnAddr) {
11293 if (!ShouldSignReturnAddr)
11294 return;
11295
11296 BuildMI(BB&: MBB, I: MBB.begin(), MIMD: DebugLoc(), MCID: TII->get(Opcode: AArch64::PAUTH_PROLOGUE))
11297 .setMIFlag(MachineInstr::FrameSetup);
11298 TII->createPauthEpilogueInstr(MBB, DL: DebugLoc());
11299}
11300
11301void AArch64InstrInfo::buildOutlinedFrame(
11302 MachineBasicBlock &MBB, MachineFunction &MF,
11303 const outliner::OutlinedFunction &OF) const {
11304
11305 AArch64FunctionInfo *FI = MF.getInfo<AArch64FunctionInfo>();
11306
11307 if (OF.FrameConstructionID == MachineOutlinerTailCall)
11308 FI->setOutliningStyle("Tail Call");
11309 else if (OF.FrameConstructionID == MachineOutlinerThunk) {
11310 // For thunk outlining, rewrite the last instruction from a call to a
11311 // tail-call.
11312 MachineInstr *Call = &*--MBB.instr_end();
11313 unsigned TailOpcode;
11314 if (Call->getOpcode() == AArch64::BL) {
11315 TailOpcode = AArch64::TCRETURNdi;
11316 } else {
11317 assert(Call->getOpcode() == AArch64::BLR ||
11318 Call->getOpcode() == AArch64::BLRNoIP);
11319 TailOpcode = AArch64::TCRETURNriALL;
11320 }
11321 MachineInstr *TC = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: TailOpcode))
11322 .add(MO: Call->getOperand(i: 0))
11323 .addImm(Val: 0);
11324 MBB.insert(I: MBB.end(), MI: TC);
11325 Call->eraseFromParent();
11326
11327 FI->setOutliningStyle("Thunk");
11328 }
11329
11330 bool IsLeafFunction = true;
11331
11332 // Is there a call in the outlined range?
11333 auto IsNonTailCall = [](const MachineInstr &MI) {
11334 return MI.isCall() && !MI.isReturn();
11335 };
11336
11337 if (llvm::any_of(Range: MBB.instrs(), P: IsNonTailCall)) {
11338 // Fix up the instructions in the range, since we're going to modify the
11339 // stack.
11340
11341 // Bugzilla ID: 46767
11342 // TODO: Check if fixing up twice is safe so we can outline these.
11343 assert(OF.FrameConstructionID != MachineOutlinerDefault &&
11344 "Can only fix up stack references once");
11345 fixupPostOutline(MBB);
11346
11347 IsLeafFunction = false;
11348
11349 // LR has to be a live in so that we can save it.
11350 if (!MBB.isLiveIn(Reg: AArch64::LR))
11351 MBB.addLiveIn(PhysReg: AArch64::LR);
11352
11353 MachineBasicBlock::iterator It = MBB.begin();
11354 MachineBasicBlock::iterator Et = MBB.end();
11355
11356 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11357 OF.FrameConstructionID == MachineOutlinerThunk)
11358 Et = std::prev(x: MBB.end());
11359
11360 // There is a call in the range, so we must save LR. Save it as part of a
11361 // frame record when that gives us a smaller compact unwind encoding.
11362 if (shouldUseCompactUnwindFrameRecordForOutlinedFunction(MBB)) {
11363 // FP is saved here, so it must be live-in.
11364 if (!MBB.isLiveIn(Reg: AArch64::FP))
11365 MBB.addLiveIn(PhysReg: AArch64::FP);
11366
11367 // stp x29, x30, [sp, #-16]! (the pre-index imm is scaled by 8: -2 * 8)
11368 MachineInstr *STPXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STPXpre))
11369 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11370 .addReg(RegNo: AArch64::FP)
11371 .addReg(RegNo: AArch64::LR)
11372 .addReg(RegNo: AArch64::SP)
11373 .addImm(Val: -2);
11374 It = MBB.insert(I: It, MI: STPXpre);
11375
11376 // mov x29, sp (add x29, sp, #0), so x29 points at the frame record.
11377 MachineInstr *SetFP = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ADDXri))
11378 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11379 .addReg(RegNo: AArch64::SP)
11380 .addImm(Val: 0)
11381 .addImm(Val: 0);
11382 MBB.insertAfter(I: It, MI: SetFP);
11383
11384 // Describe the frame record with FP as the CFA. The encoder needs all
11385 // three to pick FRAME. No need to check for unwind info here: we only
11386 // get here if the function has it.
11387 CFIInstBuilder CFIBuilder(MBB, std::next(x: SetFP->getIterator()),
11388 MachineInstr::FrameSetup);
11389 CFIBuilder.buildDefCFA(Reg: AArch64::FP, Offset: 16);
11390 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -8);
11391 CFIBuilder.buildOffset(Reg: AArch64::FP, Offset: -16);
11392
11393 // ldp x29, x30, [sp], #16
11394 MachineInstr *LDPXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDPXpost))
11395 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11396 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
11397 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11398 .addReg(RegNo: AArch64::SP)
11399 .addImm(Val: 2);
11400 Et = MBB.insert(I: Et, MI: LDPXpost);
11401 } else {
11402 // Insert a save before the outlined region
11403 MachineInstr *STRXpre = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11404 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11405 .addReg(RegNo: AArch64::LR)
11406 .addReg(RegNo: AArch64::SP)
11407 .addImm(Val: -16);
11408 It = MBB.insert(I: It, MI: STRXpre);
11409
11410 if (MF.getInfo<AArch64FunctionInfo>()->needsDwarfUnwindInfo(MF)) {
11411 CFIInstBuilder CFIBuilder(MBB, It, MachineInstr::FrameSetup);
11412
11413 // Add a CFI saying the stack was moved 16 B down.
11414 CFIBuilder.buildDefCFAOffset(Offset: 16);
11415
11416 // Add a CFI saying that the LR that we want to find is now 16 B higher
11417 // than before.
11418 CFIBuilder.buildOffset(Reg: AArch64::LR, Offset: -16);
11419 }
11420
11421 // Insert a restore before the terminator for the function.
11422 MachineInstr *LDRXpost = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11423 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11424 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11425 .addReg(RegNo: AArch64::SP)
11426 .addImm(Val: 16);
11427 Et = MBB.insert(I: Et, MI: LDRXpost);
11428 }
11429 }
11430
11431 auto RASignCondition = FI->getSignReturnAddressCondition();
11432 bool ShouldSignReturnAddr = AArch64FunctionInfo::shouldSignReturnAddress(
11433 Condition: RASignCondition, IsLRSpilled: !IsLeafFunction);
11434
11435 // If this is a tail call outlined function, then there's already a return.
11436 if (OF.FrameConstructionID == MachineOutlinerTailCall ||
11437 OF.FrameConstructionID == MachineOutlinerThunk) {
11438 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11439 return;
11440 }
11441
11442 // It's not a tail call, so we have to insert the return ourselves.
11443
11444 // LR has to be a live in so that we can return to it.
11445 if (!MBB.isLiveIn(Reg: AArch64::LR))
11446 MBB.addLiveIn(PhysReg: AArch64::LR);
11447
11448 MachineInstr *ret = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::RET))
11449 .addReg(RegNo: AArch64::LR);
11450 MBB.insert(I: MBB.end(), MI: ret);
11451
11452 signOutlinedFunction(MF, MBB, TII: this, ShouldSignReturnAddr);
11453
11454 FI->setOutliningStyle("Function");
11455
11456 // Did we have to modify the stack by saving the link register?
11457 if (OF.FrameConstructionID != MachineOutlinerDefault)
11458 return;
11459
11460 // We modified the stack.
11461 // Walk over the basic block and fix up all the stack accesses.
11462 fixupPostOutline(MBB);
11463}
11464
11465MachineBasicBlock::iterator AArch64InstrInfo::insertOutlinedCall(
11466 Module &M, MachineBasicBlock &MBB, MachineBasicBlock::iterator &It,
11467 MachineFunction &MF, outliner::Candidate &C) const {
11468
11469 // Are we tail calling?
11470 if (C.CallConstructionID == MachineOutlinerTailCall) {
11471 // If yes, then we can just branch to the label.
11472 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::TCRETURNdi))
11473 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName()))
11474 .addImm(Val: 0));
11475 return It;
11476 }
11477
11478 // Are we saving the link register?
11479 if (C.CallConstructionID == MachineOutlinerNoLRSave ||
11480 C.CallConstructionID == MachineOutlinerThunk) {
11481 // No, so just insert the call.
11482 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11483 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11484 return It;
11485 }
11486
11487 // We want to return the spot where we inserted the call.
11488 MachineBasicBlock::iterator CallPt;
11489
11490 // Instructions for saving and restoring LR around the call instruction we're
11491 // going to insert.
11492 MachineInstr *Save;
11493 MachineInstr *Restore;
11494 // Can we save to a register?
11495 if (C.CallConstructionID == MachineOutlinerRegSave) {
11496 // FIXME: This logic should be sunk into a target-specific interface so that
11497 // we don't have to recompute the register.
11498 Register Reg = findRegisterToSaveLRTo(C);
11499 assert(Reg && "No callee-saved register available?");
11500
11501 // LR has to be a live in so that we can save it.
11502 if (!MBB.isLiveIn(Reg: AArch64::LR))
11503 MBB.addLiveIn(PhysReg: AArch64::LR);
11504
11505 // Save and restore LR from Reg.
11506 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: Reg)
11507 .addReg(RegNo: AArch64::XZR)
11508 .addReg(RegNo: AArch64::LR)
11509 .addImm(Val: 0);
11510 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::ORRXrs), DestReg: AArch64::LR)
11511 .addReg(RegNo: AArch64::XZR)
11512 .addReg(RegNo: Reg)
11513 .addImm(Val: 0);
11514 } else {
11515 // We have the default case. Save and restore from SP.
11516 Save = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::STRXpre))
11517 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11518 .addReg(RegNo: AArch64::LR)
11519 .addReg(RegNo: AArch64::SP)
11520 .addImm(Val: -16);
11521 Restore = BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::LDRXpost))
11522 .addReg(RegNo: AArch64::SP, Flags: RegState::Define)
11523 .addReg(RegNo: AArch64::LR, Flags: RegState::Define)
11524 .addReg(RegNo: AArch64::SP)
11525 .addImm(Val: 16);
11526 }
11527
11528 It = MBB.insert(I: It, MI: Save);
11529 It++;
11530
11531 // Insert the call.
11532 It = MBB.insert(I: It, MI: BuildMI(MF, MIMD: DebugLoc(), MCID: get(Opcode: AArch64::BL))
11533 .addGlobalAddress(GV: M.getNamedValue(Name: MF.getName())));
11534 CallPt = It;
11535 It++;
11536
11537 It = MBB.insert(I: It, MI: Restore);
11538 return CallPt;
11539}
11540
11541bool AArch64InstrInfo::shouldOutlineFromFunctionByDefault(
11542 MachineFunction &MF) const {
11543 return MF.getFunction().hasMinSize();
11544}
11545
11546void AArch64InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
11547 MachineBasicBlock::iterator Iter,
11548 DebugLoc &DL,
11549 bool AllowSideEffects) const {
11550 const MachineFunction &MF = *MBB.getParent();
11551 const AArch64Subtarget &STI = MF.getSubtarget<AArch64Subtarget>();
11552 const AArch64RegisterInfo &TRI = *STI.getRegisterInfo();
11553
11554 if (TRI.isGeneralPurposeRegister(MF, Reg)) {
11555 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVZXi), DestReg: Reg).addImm(Val: 0).addImm(Val: 0);
11556 } else if (STI.isSVEorStreamingSVEAvailable()) {
11557 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::DUP_ZI_D), DestReg: Reg)
11558 .addImm(Val: 0)
11559 .addImm(Val: 0);
11560 } else if (STI.isNeonAvailable()) {
11561 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::MOVIv2d_ns), DestReg: Reg)
11562 .addImm(Val: 0);
11563 } else {
11564 // No Advanced SIMD (streaming-compatible without SVE, or +nosimd), so use
11565 // `fmov d...` instead of `movi v...`; writing `d` also clears the upper
11566 // 64 bits.
11567 assert(STI.hasFPARMv8() && "Expected FP to be available.");
11568 Register Reg64 = TRI.getSubReg(Reg, Idx: AArch64::dsub);
11569 BuildMI(BB&: MBB, I: Iter, MIMD: DL, MCID: get(Opcode: AArch64::FMOVD0), DestReg: Reg64);
11570 }
11571}
11572
11573std::optional<DestSourcePair>
11574AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
11575
11576 // AArch64::ORRWrs and AArch64::ORRXrs with WZR/XZR reg
11577 // and zero immediate operands used as an alias for mov instruction.
11578 if ((MI.getOpcode() == AArch64::ORRWrs &&
11579 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11580 MI.getOperand(i: 3).getImm() == 0x0) ||
11581 (MI.getOpcode() == AArch64::ORRWrr &&
11582 MI.getOperand(i: 1).getReg() == AArch64::WZR)) {
11583 // Check that the w->w move is not a zero-extending w->x mov.
11584 if ((MI.getOperand(i: 0).getReg().isPhysical() &&
11585 MI.findRegisterDefOperandIdx(
11586 Reg: getXRegFromWReg(Reg: MI.getOperand(i: 0).getReg()),
11587 /*TRI=*/nullptr) == -1) ||
11588 (MI.getOperand(i: 0).getReg().isVirtual() &&
11589 !MI.getOperand(i: 0).getSubReg()))
11590 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11591 }
11592
11593 if (MI.getOpcode() == AArch64::ORRXrs &&
11594 MI.getOperand(i: 1).getReg() == AArch64::XZR &&
11595 MI.getOperand(i: 3).getImm() == 0x0)
11596 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11597
11598 return std::nullopt;
11599}
11600
11601std::optional<DestSourcePair>
11602AArch64InstrInfo::isCopyLikeInstrImpl(const MachineInstr &MI) const {
11603 if ((MI.getOpcode() == AArch64::ORRWrs &&
11604 MI.getOperand(i: 1).getReg() == AArch64::WZR &&
11605 MI.getOperand(i: 3).getImm() == 0x0) ||
11606 (MI.getOpcode() == AArch64::ORRWrr &&
11607 MI.getOperand(i: 1).getReg() == AArch64::WZR))
11608 return DestSourcePair{MI.getOperand(i: 0), MI.getOperand(i: 2)};
11609 return std::nullopt;
11610}
11611
11612std::optional<RegImmPair>
11613AArch64InstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
11614 int Sign = 1;
11615 int64_t Offset = 0;
11616
11617 // TODO: Handle cases where Reg is a super- or sub-register of the
11618 // destination register.
11619 const MachineOperand &Op0 = MI.getOperand(i: 0);
11620 if (!Op0.isReg() || Reg != Op0.getReg())
11621 return std::nullopt;
11622
11623 switch (MI.getOpcode()) {
11624 default:
11625 return std::nullopt;
11626 case AArch64::SUBWri:
11627 case AArch64::SUBXri:
11628 case AArch64::SUBSWri:
11629 case AArch64::SUBSXri:
11630 Sign *= -1;
11631 [[fallthrough]];
11632 case AArch64::ADDSWri:
11633 case AArch64::ADDSXri:
11634 case AArch64::ADDWri:
11635 case AArch64::ADDXri: {
11636 // TODO: Third operand can be global address (usually some string).
11637 if (!MI.getOperand(i: 0).isReg() || !MI.getOperand(i: 1).isReg() ||
11638 !MI.getOperand(i: 2).isImm())
11639 return std::nullopt;
11640 int Shift = MI.getOperand(i: 3).getImm();
11641 assert((Shift == 0 || Shift == 12) && "Shift can be either 0 or 12");
11642 Offset = Sign * (MI.getOperand(i: 2).getImm() << Shift);
11643 }
11644 }
11645 return RegImmPair{MI.getOperand(i: 1).getReg(), Offset};
11646}
11647
11648/// If the given ORR instruction is a copy, and \p DescribedReg overlaps with
11649/// the destination register then, if possible, describe the value in terms of
11650/// the source register.
11651static std::optional<ParamLoadedValue>
11652describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
11653 const TargetInstrInfo *TII,
11654 const TargetRegisterInfo *TRI) {
11655 auto DestSrc = TII->isCopyLikeInstr(MI);
11656 if (!DestSrc)
11657 return std::nullopt;
11658
11659 Register DestReg = DestSrc->Destination->getReg();
11660 Register SrcReg = DestSrc->Source->getReg();
11661
11662 if (!DestReg.isValid() || !SrcReg.isValid())
11663 return std::nullopt;
11664
11665 auto Expr = DIExpression::get(Context&: MI.getMF()->getFunction().getContext(), Elements: {});
11666
11667 // If the described register is the destination, just return the source.
11668 if (DestReg == DescribedReg)
11669 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11670
11671 // ORRWrs zero-extends to 64-bits, so we need to consider such cases.
11672 if (MI.getOpcode() == AArch64::ORRWrs &&
11673 TRI->isSuperRegister(RegA: DestReg, RegB: DescribedReg))
11674 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcReg, isDef: false), Expr);
11675
11676 // We may need to describe the lower part of a ORRXrs move.
11677 if (MI.getOpcode() == AArch64::ORRXrs &&
11678 TRI->isSubRegister(RegA: DestReg, RegB: DescribedReg)) {
11679 Register SrcSubReg = TRI->getSubReg(Reg: SrcReg, Idx: AArch64::sub_32);
11680 return ParamLoadedValue(MachineOperand::CreateReg(Reg: SrcSubReg, isDef: false), Expr);
11681 }
11682
11683 assert(!TRI->isSuperOrSubRegisterEq(DestReg, DescribedReg) &&
11684 "Unhandled ORR[XW]rs copy case");
11685
11686 return std::nullopt;
11687}
11688
11689bool AArch64InstrInfo::isFunctionSafeToSplit(const MachineFunction &MF) const {
11690 // Functions cannot be split to different sections on AArch64 if they have
11691 // a red zone. This is because relaxing a cross-section branch may require
11692 // incrementing the stack pointer to spill a register, which would overwrite
11693 // the red zone.
11694 if (MF.getInfo<AArch64FunctionInfo>()->hasRedZone().value_or(u: true))
11695 return false;
11696
11697 return TargetInstrInfo::isFunctionSafeToSplit(MF);
11698}
11699
11700bool AArch64InstrInfo::isMBBSafeToSplitToCold(
11701 const MachineBasicBlock &MBB) const {
11702 // Asm Goto blocks can contain conditional branches to goto labels, which can
11703 // get moved out of range of the branch instruction.
11704 auto isAsmGoto = [](const MachineInstr &MI) {
11705 return MI.getOpcode() == AArch64::INLINEASM_BR;
11706 };
11707 if (llvm::any_of(Range: MBB, P: isAsmGoto) || MBB.isInlineAsmBrIndirectTarget())
11708 return false;
11709
11710 // Because jump tables are label-relative instead of table-relative, they all
11711 // must be in the same section or relocation fixup handling will fail.
11712
11713 // Check if MBB is a jump table target
11714 const MachineJumpTableInfo *MJTI = MBB.getParent()->getJumpTableInfo();
11715 auto containsMBB = [&MBB](const MachineJumpTableEntry &JTE) {
11716 return llvm::is_contained(Range: JTE.MBBs, Element: &MBB);
11717 };
11718 if (MJTI != nullptr && llvm::any_of(Range: MJTI->getJumpTables(), P: containsMBB))
11719 return false;
11720
11721 // Check if MBB contains a jump table lookup
11722 for (const MachineInstr &MI : MBB) {
11723 switch (MI.getOpcode()) {
11724 case TargetOpcode::G_BRJT:
11725 case AArch64::JumpTableDest32:
11726 case AArch64::JumpTableDest16:
11727 case AArch64::JumpTableDest8:
11728 return false;
11729 default:
11730 continue;
11731 }
11732 }
11733
11734 // MBB isn't a special case, so it's safe to be split to the cold section.
11735 return true;
11736}
11737
11738std::optional<ParamLoadedValue>
11739AArch64InstrInfo::describeLoadedValue(const MachineInstr &MI,
11740 Register Reg) const {
11741 const MachineFunction *MF = MI.getMF();
11742 const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
11743 switch (MI.getOpcode()) {
11744 case AArch64::MOVZWi:
11745 case AArch64::MOVZXi: {
11746 // MOVZWi may be used for producing zero-extended 32-bit immediates in
11747 // 64-bit parameters, so we need to consider super-registers.
11748 if (!TRI->isSuperRegisterEq(RegA: MI.getOperand(i: 0).getReg(), RegB: Reg))
11749 return std::nullopt;
11750
11751 if (!MI.getOperand(i: 1).isImm())
11752 return std::nullopt;
11753 int64_t Immediate = MI.getOperand(i: 1).getImm();
11754 int Shift = MI.getOperand(i: 2).getImm();
11755 return ParamLoadedValue(MachineOperand::CreateImm(Val: Immediate << Shift),
11756 nullptr);
11757 }
11758 case AArch64::ORRWrs:
11759 case AArch64::ORRXrs:
11760 return describeORRLoadedValue(MI, DescribedReg: Reg, TII: this, TRI);
11761 }
11762
11763 return TargetInstrInfo::describeLoadedValue(MI, Reg);
11764}
11765
11766bool AArch64InstrInfo::isExtendLikelyToBeFolded(
11767 MachineInstr &ExtMI, MachineRegisterInfo &MRI) const {
11768 assert(ExtMI.getOpcode() == TargetOpcode::G_SEXT ||
11769 ExtMI.getOpcode() == TargetOpcode::G_ZEXT ||
11770 ExtMI.getOpcode() == TargetOpcode::G_ANYEXT);
11771
11772 // Anyexts are nops.
11773 if (ExtMI.getOpcode() == TargetOpcode::G_ANYEXT)
11774 return true;
11775
11776 Register DefReg = ExtMI.getOperand(i: 0).getReg();
11777 if (!MRI.hasOneNonDBGUse(RegNo: DefReg))
11778 return false;
11779
11780 // It's likely that a sext/zext as a G_PTR_ADD offset will be folded into an
11781 // addressing mode.
11782 auto *UserMI = &*MRI.use_instr_nodbg_begin(RegNo: DefReg);
11783 return UserMI->getOpcode() == TargetOpcode::G_PTR_ADD;
11784}
11785
11786uint64_t AArch64InstrInfo::getElementSizeForOpcode(unsigned Opc) const {
11787 return get(Opcode: Opc).TSFlags & AArch64::ElementSizeMask;
11788}
11789
11790bool AArch64InstrInfo::isPTestLikeOpcode(unsigned Opc) const {
11791 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsPTestLike;
11792}
11793
11794bool AArch64InstrInfo::isWhileOpcode(unsigned Opc) const {
11795 return get(Opcode: Opc).TSFlags & AArch64::InstrFlagIsWhile;
11796}
11797
11798unsigned int
11799AArch64InstrInfo::getTailDuplicateSize(CodeGenOptLevel OptLevel) const {
11800 return OptLevel >= CodeGenOptLevel::Aggressive ? 6 : 2;
11801}
11802
11803bool AArch64InstrInfo::isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
11804 unsigned Scale) const {
11805 if (Offset && Scale)
11806 return false;
11807
11808 // Check Reg + Imm
11809 if (!Scale) {
11810 // 9-bit signed offset
11811 if (isInt<9>(x: Offset))
11812 return true;
11813
11814 // 12-bit unsigned offset
11815 unsigned Shift = Log2_64(Value: NumBytes);
11816 if (NumBytes && Offset > 0 && (Offset / NumBytes) <= (1LL << 12) - 1 &&
11817 // Must be a multiple of NumBytes (NumBytes is a power of 2)
11818 (Offset >> Shift) << Shift == Offset)
11819 return true;
11820 return false;
11821 }
11822
11823 // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2
11824 return Scale == 1 || (Scale > 0 && Scale == NumBytes);
11825}
11826
11827unsigned llvm::getBLRCallOpcode(const MachineFunction &MF) {
11828 if (MF.getSubtarget<AArch64Subtarget>().hardenSlsBlr())
11829 return AArch64::BLRNoIP;
11830 else
11831 return AArch64::BLR;
11832}
11833
11834void AArch64InstrInfo::createPauthEpilogueInstr(MachineBasicBlock &MBB,
11835 DebugLoc DL) const {
11836 MachineBasicBlock::iterator InsertPt = MBB.getFirstTerminator();
11837 auto Builder = BuildMI(BB&: MBB, I: InsertPt, MIMD: DL, MCID: get(Opcode: AArch64::PAUTH_EPILOGUE))
11838 .setMIFlag(MachineInstr::FrameDestroy);
11839
11840 MachineFunction &MF = *MBB.getParent();
11841 const auto *AFI = MF.getInfo<AArch64FunctionInfo>();
11842 auto &AFL = *static_cast<const AArch64FrameLowering *>(
11843 MF.getSubtarget().getFrameLowering());
11844 if (AFL.getArgumentStackToRestore(MF, MBB)) {
11845 Builder.addReg(RegNo: AArch64::X17, Flags: RegState::ImplicitDefine);
11846 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11847 if (AFI->branchProtectionPAuthLR())
11848 Builder.addReg(RegNo: AArch64::X15, Flags: RegState::ImplicitDefine);
11849 return;
11850 }
11851
11852 if (AFI->branchProtectionPAuthLR() && !Subtarget.hasPAuthLR())
11853 Builder.addReg(RegNo: AArch64::X16, Flags: RegState::ImplicitDefine);
11854}
11855
11856MachineBasicBlock::iterator
11857AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI,
11858 Register TargetReg, bool FrameSetup) const {
11859 assert(TargetReg != AArch64::SP && "New top of stack cannot already be in SP");
11860
11861 MachineBasicBlock &MBB = *MBBI->getParent();
11862 MachineFunction &MF = *MBB.getParent();
11863 const AArch64InstrInfo *TII =
11864 MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
11865 int64_t ProbeSize = MF.getInfo<AArch64FunctionInfo>()->getStackProbeSize();
11866 DebugLoc DL = MBB.findDebugLoc(MBBI);
11867
11868 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB.getIterator());
11869 MachineBasicBlock *LoopTestMBB =
11870 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11871 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
11872 MachineBasicBlock *LoopBodyMBB =
11873 MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11874 MF.insert(MBBI: MBBInsertPoint, MBB: LoopBodyMBB);
11875 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB.getBasicBlock());
11876 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
11877 MachineInstr::MIFlag Flags =
11878 FrameSetup ? MachineInstr::FrameSetup : MachineInstr::NoFlags;
11879
11880 // LoopTest:
11881 // SUB SP, SP, #ProbeSize
11882 emitFrameOffset(MBB&: *LoopTestMBB, MBBI: LoopTestMBB->end(), DL, DestReg: AArch64::SP,
11883 SrcReg: AArch64::SP, Offset: StackOffset::getFixed(Fixed: -ProbeSize), TII, Flag: Flags);
11884
11885 // CMP SP, TargetReg
11886 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXrx64),
11887 DestReg: AArch64::XZR)
11888 .addReg(RegNo: AArch64::SP)
11889 .addReg(RegNo: TargetReg)
11890 .addImm(Val: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTX, Imm: 0))
11891 .setMIFlags(Flags);
11892
11893 // B.<Cond> LoopExit
11894 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
11895 .addImm(Val: AArch64CC::LE)
11896 .addMBB(MBB: ExitMBB)
11897 .setMIFlags(Flags);
11898
11899 // LDR XZR, [SP]
11900 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11901 .addDef(RegNo: AArch64::XZR)
11902 .addReg(RegNo: AArch64::SP)
11903 .addImm(Val: 0)
11904 .addMemOperand(MMO: MF.getMachineMemOperand(
11905 PtrInfo: MachinePointerInfo::getUnknownStack(MF),
11906 F: MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile, Size: 8,
11907 BaseAlignment: Align(8)))
11908 .setMIFlags(Flags);
11909
11910 // B loop
11911 BuildMI(BB&: *LoopBodyMBB, I: LoopBodyMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
11912 .addMBB(MBB: LoopTestMBB)
11913 .setMIFlags(Flags);
11914
11915 // LoopExit:
11916 // MOV SP, TargetReg
11917 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri), DestReg: AArch64::SP)
11918 .addReg(RegNo: TargetReg)
11919 .addImm(Val: 0)
11920 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0))
11921 .setMIFlags(Flags);
11922
11923 // LDR XZR, [SP]
11924 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
11925 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define)
11926 .addReg(RegNo: AArch64::SP)
11927 .addImm(Val: 0)
11928 .setMIFlags(Flags);
11929
11930 ExitMBB->splice(Where: ExitMBB->end(), Other: &MBB, From: std::next(x: MBBI), To: MBB.end());
11931 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: &MBB);
11932
11933 LoopTestMBB->addSuccessor(Succ: ExitMBB);
11934 LoopTestMBB->addSuccessor(Succ: LoopBodyMBB);
11935 LoopBodyMBB->addSuccessor(Succ: LoopTestMBB);
11936 MBB.addSuccessor(Succ: LoopTestMBB);
11937
11938 // Update liveins.
11939 if (MF.getRegInfo().reservedRegsFrozen())
11940 fullyRecomputeLiveIns(MBBs: {ExitMBB, LoopBodyMBB, LoopTestMBB});
11941
11942 return ExitMBB->begin();
11943}
11944
11945namespace {
11946class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
11947 MachineFunction *MF;
11948 const TargetInstrInfo *TII;
11949 const TargetRegisterInfo *TRI;
11950 MachineRegisterInfo &MRI;
11951
11952 /// The block of the loop
11953 MachineBasicBlock *LoopBB;
11954 /// The conditional branch of the loop
11955 MachineInstr *CondBranch;
11956 /// The compare instruction for loop control
11957 MachineInstr *Comp;
11958 /// The number of the operand of the loop counter value in Comp
11959 unsigned CompCounterOprNum;
11960 /// The instruction that updates the loop counter value
11961 MachineInstr *Update;
11962 /// The number of the operand of the loop counter value in Update
11963 unsigned UpdateCounterOprNum;
11964 /// The initial value of the loop counter
11965 Register Init;
11966 /// True iff Update is a predecessor of Comp
11967 bool IsUpdatePriorComp;
11968
11969 /// The normalized condition used by createTripCountGreaterCondition()
11970 SmallVector<MachineOperand, 4> Cond;
11971
11972public:
11973 AArch64PipelinerLoopInfo(MachineBasicBlock *LoopBB, MachineInstr *CondBranch,
11974 MachineInstr *Comp, unsigned CompCounterOprNum,
11975 MachineInstr *Update, unsigned UpdateCounterOprNum,
11976 Register Init, bool IsUpdatePriorComp,
11977 const SmallVectorImpl<MachineOperand> &Cond)
11978 : MF(Comp->getParent()->getParent()),
11979 TII(MF->getSubtarget().getInstrInfo()),
11980 TRI(MF->getSubtarget().getRegisterInfo()), MRI(MF->getRegInfo()),
11981 LoopBB(LoopBB), CondBranch(CondBranch), Comp(Comp),
11982 CompCounterOprNum(CompCounterOprNum), Update(Update),
11983 UpdateCounterOprNum(UpdateCounterOprNum), Init(Init),
11984 IsUpdatePriorComp(IsUpdatePriorComp), Cond(Cond.begin(), Cond.end()) {}
11985
11986 bool shouldIgnoreForPipelining(const MachineInstr *MI) const override {
11987 // Make the instructions for loop control be placed in stage 0.
11988 // The predecessors of Comp are considered by the caller.
11989 return MI == Comp;
11990 }
11991
11992 std::optional<bool> createTripCountGreaterCondition(
11993 int TC, MachineBasicBlock &MBB,
11994 SmallVectorImpl<MachineOperand> &CondParam) override {
11995 // A branch instruction will be inserted as "if (Cond) goto epilogue".
11996 // Cond is normalized for such use.
11997 // The predecessors of the branch are assumed to have already been inserted.
11998 CondParam = Cond;
11999 return {};
12000 }
12001
12002 void createRemainingIterationsGreaterCondition(
12003 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
12004 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) override;
12005
12006 void setPreheader(MachineBasicBlock *NewPreheader) override {}
12007
12008 void adjustTripCount(int TripCountAdjust) override {}
12009
12010 bool isMVEExpanderSupported() override { return true; }
12011};
12012} // namespace
12013
12014/// Clone an instruction from MI. The register of ReplaceOprNum-th operand
12015/// is replaced by ReplaceReg. The output register is newly created.
12016/// The other operands are unchanged from MI.
12017static Register cloneInstr(const MachineInstr *MI, unsigned ReplaceOprNum,
12018 Register ReplaceReg, MachineBasicBlock &MBB,
12019 MachineBasicBlock::iterator InsertTo) {
12020 MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
12021 const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo();
12022 MachineInstr *NewMI = MBB.getParent()->CloneMachineInstr(Orig: MI);
12023 Register Result = 0;
12024 for (unsigned I = 0; I < NewMI->getNumOperands(); ++I) {
12025 if (I == 0 && NewMI->getOperand(i: 0).getReg().isVirtual()) {
12026 Result = MRI.createVirtualRegister(
12027 RegClass: MRI.getRegClass(Reg: NewMI->getOperand(i: 0).getReg()));
12028 NewMI->getOperand(i: I).setReg(Result);
12029 } else if (I == ReplaceOprNum) {
12030 MRI.constrainRegClass(Reg: ReplaceReg, RC: TII->getRegClass(MCID: NewMI->getDesc(), OpNum: I));
12031 NewMI->getOperand(i: I).setReg(ReplaceReg);
12032 }
12033 }
12034 MBB.insert(I: InsertTo, MI: NewMI);
12035 return Result;
12036}
12037
12038void AArch64PipelinerLoopInfo::createRemainingIterationsGreaterCondition(
12039 int TC, MachineBasicBlock &MBB, SmallVectorImpl<MachineOperand> &Cond,
12040 DenseMap<MachineInstr *, MachineInstr *> &LastStage0Insts) {
12041 // Create and accumulate conditions for next TC iterations.
12042 // Example:
12043 // SUBSXrr N, counter, implicit-def $nzcv # compare instruction for the last
12044 // # iteration of the kernel
12045 //
12046 // # insert the following instructions
12047 // cond = CSINCXr 0, 0, C, implicit $nzcv
12048 // counter = ADDXri counter, 1 # clone from this->Update
12049 // SUBSXrr n, counter, implicit-def $nzcv # clone from this->Comp
12050 // cond = CSINCXr cond, cond, C, implicit $nzcv
12051 // ... (repeat TC times)
12052 // SUBSXri cond, 0, implicit-def $nzcv
12053
12054 assert(CondBranch->getOpcode() == AArch64::Bcc);
12055 // CondCode to exit the loop
12056 AArch64CC::CondCode CC =
12057 (AArch64CC::CondCode)CondBranch->getOperand(i: 0).getImm();
12058 if (CondBranch->getOperand(i: 1).getMBB() == LoopBB)
12059 CC = AArch64CC::getInvertedCondCode(Code: CC);
12060
12061 // Accumulate conditions to exit the loop
12062 Register AccCond = AArch64::XZR;
12063
12064 // If CC holds, CurCond+1 is returned; otherwise CurCond is returned.
12065 auto AccumulateCond = [&](Register CurCond,
12066 AArch64CC::CondCode CC) -> Register {
12067 Register NewCond = MRI.createVirtualRegister(RegClass: &AArch64::GPR64commonRegClass);
12068 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::CSINCXr))
12069 .addReg(RegNo: NewCond, Flags: RegState::Define)
12070 .addReg(RegNo: CurCond)
12071 .addReg(RegNo: CurCond)
12072 .addImm(Val: AArch64CC::getInvertedCondCode(Code: CC));
12073 return NewCond;
12074 };
12075
12076 if (!LastStage0Insts.empty() && LastStage0Insts[Comp]->getParent() == &MBB) {
12077 // Update and Comp for I==0 are already exists in MBB
12078 // (MBB is an unrolled kernel)
12079 Register Counter;
12080 for (int I = 0; I <= TC; ++I) {
12081 Register NextCounter;
12082 if (I != 0)
12083 NextCounter =
12084 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12085
12086 AccCond = AccumulateCond(AccCond, CC);
12087
12088 if (I != TC) {
12089 if (I == 0) {
12090 if (Update != Comp && IsUpdatePriorComp) {
12091 Counter =
12092 LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12093 NextCounter = cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB,
12094 InsertTo: MBB.end());
12095 } else {
12096 // can use already calculated value
12097 NextCounter = LastStage0Insts[Update]->getOperand(i: 0).getReg();
12098 }
12099 } else if (Update != Comp) {
12100 NextCounter =
12101 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12102 }
12103 }
12104 Counter = NextCounter;
12105 }
12106 } else {
12107 Register Counter;
12108 if (LastStage0Insts.empty()) {
12109 // use initial counter value (testing if the trip count is sufficient to
12110 // be executed by pipelined code)
12111 Counter = Init;
12112 if (IsUpdatePriorComp)
12113 Counter =
12114 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12115 } else {
12116 // MBB is an epilogue block. LastStage0Insts[Comp] is in the kernel block.
12117 Counter = LastStage0Insts[Comp]->getOperand(i: CompCounterOprNum).getReg();
12118 }
12119
12120 for (int I = 0; I <= TC; ++I) {
12121 Register NextCounter;
12122 NextCounter =
12123 cloneInstr(MI: Comp, ReplaceOprNum: CompCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12124 AccCond = AccumulateCond(AccCond, CC);
12125 if (I != TC && Update != Comp)
12126 NextCounter =
12127 cloneInstr(MI: Update, ReplaceOprNum: UpdateCounterOprNum, ReplaceReg: Counter, MBB, InsertTo: MBB.end());
12128 Counter = NextCounter;
12129 }
12130 }
12131
12132 // If AccCond == 0, the remainder is greater than TC.
12133 BuildMI(BB&: MBB, I: MBB.end(), MIMD: Comp->getDebugLoc(), MCID: TII->get(Opcode: AArch64::SUBSXri))
12134 .addReg(RegNo: AArch64::XZR, Flags: RegState::Define | RegState::Dead)
12135 .addReg(RegNo: AccCond)
12136 .addImm(Val: 0)
12137 .addImm(Val: 0);
12138 Cond.clear();
12139 Cond.push_back(Elt: MachineOperand::CreateImm(Val: AArch64CC::EQ));
12140}
12141
12142static void extractPhiReg(const MachineInstr &Phi, const MachineBasicBlock *MBB,
12143 Register &RegMBB, Register &RegOther) {
12144 assert(Phi.getNumOperands() == 5);
12145 if (Phi.getOperand(i: 2).getMBB() == MBB) {
12146 RegMBB = Phi.getOperand(i: 1).getReg();
12147 RegOther = Phi.getOperand(i: 3).getReg();
12148 } else {
12149 assert(Phi.getOperand(4).getMBB() == MBB);
12150 RegMBB = Phi.getOperand(i: 3).getReg();
12151 RegOther = Phi.getOperand(i: 1).getReg();
12152 }
12153}
12154
12155static bool isDefinedOutside(Register Reg, const MachineBasicBlock *BB) {
12156 if (!Reg.isVirtual())
12157 return false;
12158 const MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
12159 return MRI.getDefBlock(Reg) != BB;
12160}
12161
12162/// If Reg is an induction variable, return true and set some parameters
12163static bool getIndVarInfo(Register Reg, const MachineBasicBlock *LoopBB,
12164 MachineInstr *&UpdateInst,
12165 unsigned &UpdateCounterOprNum, Register &InitReg,
12166 bool &IsUpdatePriorComp) {
12167 // Example:
12168 //
12169 // Preheader:
12170 // InitReg = ...
12171 // LoopBB:
12172 // Reg0 = PHI (InitReg, Preheader), (Reg1, LoopBB)
12173 // Reg = COPY Reg0 ; COPY is ignored.
12174 // Reg1 = ADD Reg, #1; UpdateInst. Incremented by a loop invariant value.
12175 // ; Reg is the value calculated in the previous
12176 // ; iteration, so IsUpdatePriorComp == false.
12177
12178 if (LoopBB->pred_size() != 2)
12179 return false;
12180 if (!Reg.isVirtual())
12181 return false;
12182 const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
12183 UpdateInst = nullptr;
12184 UpdateCounterOprNum = 0;
12185 InitReg = 0;
12186 IsUpdatePriorComp = true;
12187 Register CurReg = Reg;
12188 while (true) {
12189 MachineInstr *Def = MRI.getVRegDef(Reg: CurReg);
12190 if (Def->getParent() != LoopBB)
12191 return false;
12192 if (Def->isCopy()) {
12193 // Ignore copy instructions unless they contain subregisters
12194 if (Def->getOperand(i: 0).getSubReg() || Def->getOperand(i: 1).getSubReg())
12195 return false;
12196 CurReg = Def->getOperand(i: 1).getReg();
12197 } else if (Def->isPHI()) {
12198 if (InitReg != 0)
12199 return false;
12200 if (!UpdateInst)
12201 IsUpdatePriorComp = false;
12202 extractPhiReg(Phi: *Def, MBB: LoopBB, RegMBB&: CurReg, RegOther&: InitReg);
12203 } else {
12204 if (UpdateInst)
12205 return false;
12206 switch (Def->getOpcode()) {
12207 case AArch64::ADDSXri:
12208 case AArch64::ADDSWri:
12209 case AArch64::SUBSXri:
12210 case AArch64::SUBSWri:
12211 case AArch64::ADDXri:
12212 case AArch64::ADDWri:
12213 case AArch64::SUBXri:
12214 case AArch64::SUBWri:
12215 UpdateInst = Def;
12216 UpdateCounterOprNum = 1;
12217 break;
12218 case AArch64::ADDSXrr:
12219 case AArch64::ADDSWrr:
12220 case AArch64::SUBSXrr:
12221 case AArch64::SUBSWrr:
12222 case AArch64::ADDXrr:
12223 case AArch64::ADDWrr:
12224 case AArch64::SUBXrr:
12225 case AArch64::SUBWrr:
12226 UpdateInst = Def;
12227 if (isDefinedOutside(Reg: Def->getOperand(i: 2).getReg(), BB: LoopBB))
12228 UpdateCounterOprNum = 1;
12229 else if (isDefinedOutside(Reg: Def->getOperand(i: 1).getReg(), BB: LoopBB))
12230 UpdateCounterOprNum = 2;
12231 else
12232 return false;
12233 break;
12234 default:
12235 return false;
12236 }
12237 CurReg = Def->getOperand(i: UpdateCounterOprNum).getReg();
12238 }
12239
12240 if (!CurReg.isVirtual())
12241 return false;
12242 if (Reg == CurReg)
12243 break;
12244 }
12245
12246 if (!UpdateInst)
12247 return false;
12248
12249 return true;
12250}
12251
12252std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
12253AArch64InstrInfo::analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const {
12254 // Accept loops that meet the following conditions
12255 // * The conditional branch is BCC
12256 // * The compare instruction is ADDS/SUBS/WHILEXX
12257 // * One operand of the compare is an induction variable and the other is a
12258 // loop invariant value
12259 // * The induction variable is incremented/decremented by a single instruction
12260 // * Does not contain CALL or instructions which have unmodeled side effects
12261
12262 for (MachineInstr &MI : *LoopBB)
12263 if (MI.isCall() || MI.hasUnmodeledSideEffects())
12264 // This instruction may use NZCV, which interferes with the instruction to
12265 // be inserted for loop control.
12266 return nullptr;
12267
12268 MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
12269 SmallVector<MachineOperand, 4> Cond;
12270 if (analyzeBranch(MBB&: *LoopBB, TBB, FBB, Cond))
12271 return nullptr;
12272
12273 // Infinite loops are not supported
12274 if (TBB == LoopBB && FBB == LoopBB)
12275 return nullptr;
12276
12277 // Must be conditional branch
12278 if (TBB != LoopBB && FBB == nullptr)
12279 return nullptr;
12280
12281 assert((TBB == LoopBB || FBB == LoopBB) &&
12282 "The Loop must be a single-basic-block loop");
12283
12284 MachineInstr *CondBranch = &*LoopBB->getFirstTerminator();
12285 const TargetRegisterInfo &TRI = getRegisterInfo();
12286
12287 if (CondBranch->getOpcode() != AArch64::Bcc)
12288 return nullptr;
12289
12290 // Normalization for createTripCountGreaterCondition()
12291 if (TBB == LoopBB)
12292 reverseBranchCondition(Cond);
12293
12294 MachineInstr *Comp = nullptr;
12295 unsigned CompCounterOprNum = 0;
12296 for (MachineInstr &MI : reverse(C&: *LoopBB)) {
12297 if (MI.modifiesRegister(Reg: AArch64::NZCV, TRI: &TRI)) {
12298 // Guarantee that the compare is SUBS/ADDS/WHILEXX and that one of the
12299 // operands is a loop invariant value
12300
12301 switch (MI.getOpcode()) {
12302 case AArch64::SUBSXri:
12303 case AArch64::SUBSWri:
12304 case AArch64::ADDSXri:
12305 case AArch64::ADDSWri:
12306 Comp = &MI;
12307 CompCounterOprNum = 1;
12308 break;
12309 case AArch64::ADDSWrr:
12310 case AArch64::ADDSXrr:
12311 case AArch64::SUBSWrr:
12312 case AArch64::SUBSXrr:
12313 Comp = &MI;
12314 break;
12315 default:
12316 if (isWhileOpcode(Opc: MI.getOpcode())) {
12317 Comp = &MI;
12318 break;
12319 }
12320 return nullptr;
12321 }
12322
12323 if (CompCounterOprNum == 0) {
12324 if (isDefinedOutside(Reg: Comp->getOperand(i: 1).getReg(), BB: LoopBB))
12325 CompCounterOprNum = 2;
12326 else if (isDefinedOutside(Reg: Comp->getOperand(i: 2).getReg(), BB: LoopBB))
12327 CompCounterOprNum = 1;
12328 else
12329 return nullptr;
12330 }
12331 break;
12332 }
12333 }
12334 if (!Comp)
12335 return nullptr;
12336
12337 MachineInstr *Update = nullptr;
12338 Register Init;
12339 bool IsUpdatePriorComp;
12340 unsigned UpdateCounterOprNum;
12341 if (!getIndVarInfo(Reg: Comp->getOperand(i: CompCounterOprNum).getReg(), LoopBB,
12342 UpdateInst&: Update, UpdateCounterOprNum, InitReg&: Init, IsUpdatePriorComp))
12343 return nullptr;
12344
12345 return std::make_unique<AArch64PipelinerLoopInfo>(
12346 args&: LoopBB, args&: CondBranch, args&: Comp, args&: CompCounterOprNum, args&: Update, args&: UpdateCounterOprNum,
12347 args&: Init, args&: IsUpdatePriorComp, args&: Cond);
12348}
12349
12350/// verifyInstruction - Perform target specific instruction verification.
12351bool AArch64InstrInfo::verifyInstruction(const MachineInstr &MI,
12352 StringRef &ErrInfo) const {
12353 // Verify that immediate offsets on load/store instructions are within range.
12354 // Stack objects with an FI operand are excluded as they can be fixed up
12355 // during PEI.
12356 TypeSize Scale(0U, false), Width(0U, false);
12357 int64_t MinOffset, MaxOffset;
12358 if (getMemOpInfo(Opcode: MI.getOpcode(), Scale, Width, MinOffset, MaxOffset)) {
12359 unsigned ImmIdx = getLoadStoreImmIdx(Opc: MI.getOpcode());
12360 if (MI.getOperand(i: ImmIdx).isImm() && !MI.getOperand(i: ImmIdx - 1).isFI()) {
12361 int64_t Imm = MI.getOperand(i: ImmIdx).getImm();
12362 if (Imm < MinOffset || Imm > MaxOffset) {
12363 ErrInfo = "Unexpected immediate on load/store instruction";
12364 return false;
12365 }
12366 }
12367 }
12368
12369 const MCInstrDesc &MCID = MI.getDesc();
12370 for (unsigned Op = 0; Op < MCID.getNumOperands(); Op++) {
12371 const MachineOperand &MO = MI.getOperand(i: Op);
12372 switch (MCID.operands()[Op].OperandType) {
12373 case AArch64::OPERAND_IMPLICIT_IMM_0:
12374 if (!MO.isImm() || MO.getImm() != 0) {
12375 ErrInfo = "OPERAND_IMPLICIT_IMM_0 should be 0";
12376 return false;
12377 }
12378 break;
12379 case AArch64::OPERAND_SHIFT_MSL:
12380 if (!MO.isImm() ||
12381 AArch64_AM::getShiftType(Imm: MO.getImm()) != AArch64_AM::MSL ||
12382 (AArch64_AM::getShiftValue(Imm: MO.getImm()) != 8 &&
12383 AArch64_AM::getShiftValue(Imm: MO.getImm()) != 16)) {
12384 ErrInfo = "OPERAND_SHIFT_MSL should be msl shift of 8 or 16";
12385 return false;
12386 }
12387 break;
12388 case AArch64::OPERAND_IMM_UINT1:
12389 if (!MO.isImm() || (MO.getImm() != 0 && MO.getImm() != 1)) {
12390 ErrInfo = "OPERAND_IMM_UINT1 should be 0 or 1";
12391 return false;
12392 }
12393 break;
12394 case AArch64::OPERAND_IMM_UINT4plus1:
12395 if (!MO.isImm() || MO.getImm() <= 0 || MO.getImm() > 16) {
12396 ErrInfo = "OPERAND_IMM_UINT4plus1 should be in the range 1 to 16";
12397 return false;
12398 }
12399 break;
12400 case AArch64::OPERAND_IMM_UINT5:
12401 if (!MO.isImm() || !isUInt<5>(x: MO.getImm())) {
12402 ErrInfo = "OPERAND_IMM_UINT5 should be in the range 0 to 31";
12403 return false;
12404 }
12405 break;
12406 case AArch64::OPERAND_IMM_UINT8:
12407 if (!MO.isImm() || !isUInt<8>(x: MO.getImm())) {
12408 ErrInfo = "OPERAND_IMM_UINT8 should be in the range 0 to 255";
12409 return false;
12410 }
12411 break;
12412 default:
12413 break;
12414 }
12415 }
12416 return true;
12417}
12418
12419#define GET_INSTRINFO_HELPERS
12420#define GET_INSTRMAP_INFO
12421#include "AArch64GenInstrInfo.inc"
12422