1//===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling and other late optimizations. This
11// pass should be run after register allocation but before the post-regalloc
12// scheduling pass.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AArch64ExpandImm.h"
17#include "AArch64InstrInfo.h"
18#include "AArch64MachineFunctionInfo.h"
19#include "AArch64Subtarget.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/CodeGen/LivePhysRegs.h"
23#include "llvm/CodeGen/MachineBasicBlock.h"
24#include "llvm/CodeGen/MachineConstantPool.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include "llvm/CodeGen/MachineInstr.h"
28#include "llvm/CodeGen/MachineInstrBuilder.h"
29#include "llvm/CodeGen/MachineOperand.h"
30#include "llvm/CodeGen/TargetSubtargetInfo.h"
31#include "llvm/IR/DebugLoc.h"
32#include "llvm/MC/MCInstrDesc.h"
33#include "llvm/Pass.h"
34#include "llvm/Support/CodeGen.h"
35#include "llvm/Target/TargetMachine.h"
36#include "llvm/TargetParser/Triple.h"
37#include <cassert>
38#include <cstdint>
39#include <iterator>
40
41using namespace llvm;
42
43#define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
44
45namespace {
46
47class AArch64ExpandPseudoImpl {
48public:
49 const AArch64InstrInfo *TII;
50
51 bool run(MachineFunction &MF);
52
53private:
54 bool expandMBB(MachineBasicBlock &MBB);
55 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
56 MachineBasicBlock::iterator &NextMBBI);
57 bool expandMultiVecPseudo(MachineBasicBlock &MBB,
58 MachineBasicBlock::iterator MBBI,
59 const TargetRegisterClass &ContiguousClass,
60 const TargetRegisterClass &StridedClass,
61 unsigned ContiguousOpc, unsigned StridedOpc);
62 bool expandCopyIntoTuplePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
63 MachineBasicBlock::iterator MBBI);
64 bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
65 unsigned BitSize);
66
67 bool expand_DestructiveOp(MachineInstr &MI, MachineBasicBlock &MBB,
68 MachineBasicBlock::iterator MBBI);
69 bool expandSVEBitwisePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
70 MachineBasicBlock::iterator MBBI);
71 bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
72 unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
73 unsigned ExtendImm, unsigned ZeroReg,
74 MachineBasicBlock::iterator &NextMBBI);
75 bool expandCMP_SWAP_128(MachineBasicBlock &MBB,
76 MachineBasicBlock::iterator MBBI,
77 MachineBasicBlock::iterator &NextMBBI);
78 bool expandSetTagLoop(MachineBasicBlock &MBB,
79 MachineBasicBlock::iterator MBBI,
80 MachineBasicBlock::iterator &NextMBBI);
81 bool expandSVESpillFill(MachineBasicBlock &MBB,
82 MachineBasicBlock::iterator MBBI, unsigned Opc,
83 unsigned N);
84 bool expandCALL_RVMARKER(MachineBasicBlock &MBB,
85 MachineBasicBlock::iterator MBBI);
86 bool expandCALL_BTI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
87 bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,
88 MachineBasicBlock::iterator MBBI);
89 struct ConditionalBlocks {
90 MachineBasicBlock &CondBB;
91 MachineBasicBlock &EndBB;
92 };
93 ConditionalBlocks expandConditionalPseudo(MachineBasicBlock &MBB,
94 MachineBasicBlock::iterator MBBI,
95 DebugLoc DL,
96 MachineInstrBuilder &Branch);
97 MachineBasicBlock *expandRestoreZASave(MachineBasicBlock &MBB,
98 MachineBasicBlock::iterator MBBI);
99 MachineBasicBlock *expandCommitZASave(MachineBasicBlock &MBB,
100 MachineBasicBlock::iterator MBBI);
101 MachineBasicBlock *expandCondSMToggle(MachineBasicBlock &MBB,
102 MachineBasicBlock::iterator MBBI);
103};
104
105class AArch64ExpandPseudoLegacy : public MachineFunctionPass {
106public:
107 static char ID;
108
109 AArch64ExpandPseudoLegacy() : MachineFunctionPass(ID) {}
110
111 bool runOnMachineFunction(MachineFunction &MF) override;
112
113 void getAnalysisUsage(AnalysisUsage &AU) const override {
114 AU.addPreserved<MachineRegisterClassInfoWrapperPass>();
115 MachineFunctionPass::getAnalysisUsage(AU);
116 }
117
118 StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }
119};
120
121} // end anonymous namespace
122
123char AArch64ExpandPseudoLegacy::ID = 0;
124
125INITIALIZE_PASS(AArch64ExpandPseudoLegacy, "aarch64-expand-pseudo",
126 AARCH64_EXPAND_PSEUDO_NAME, false, false)
127
128/// Transfer implicit operands on the pseudo instruction to the
129/// instructions created from the expansion.
130static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,
131 MachineInstrBuilder &DefMI) {
132 const MCInstrDesc &Desc = OldMI.getDesc();
133 for (const MachineOperand &MO :
134 llvm::drop_begin(RangeOrContainer: OldMI.operands(), N: Desc.getNumOperands())) {
135 assert(MO.isReg() && MO.getReg());
136 if (MO.isUse())
137 UseMI.add(MO);
138 else
139 DefMI.add(MO);
140 }
141}
142
143/// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more
144/// real move-immediate instructions to synthesize the immediate.
145bool AArch64ExpandPseudoImpl::expandMOVImm(MachineBasicBlock &MBB,
146 MachineBasicBlock::iterator MBBI,
147 unsigned BitSize) {
148 MachineInstr &MI = *MBBI;
149 Register DstReg = MI.getOperand(i: 0).getReg();
150 RegState RenamableState =
151 getRenamableRegState(B: MI.getOperand(i: 0).isRenamable());
152 uint64_t Imm = MI.getOperand(i: 1).getImm();
153
154 if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
155 // Useless def, and we don't want to risk creating an invalid ORR (which
156 // would really write to sp).
157 MI.eraseFromParent();
158 return true;
159 }
160
161 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
162 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn);
163 assert(Insn.size() != 0);
164
165 SmallVector<MachineInstrBuilder, 4> MIBS;
166 for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {
167 bool LastItem = std::next(x: I) == E;
168 switch (I->Opcode)
169 {
170 default: llvm_unreachable("unhandled!"); break;
171
172 case AArch64::ORRWri:
173 case AArch64::ORRXri:
174 case AArch64::ANDXri:
175 case AArch64::EORXri:
176 if (I->Op1 == 0) {
177 MIBS.push_back(Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
178 .add(MO: MI.getOperand(i: 0))
179 .addReg(RegNo: BitSize == 32 ? AArch64::WZR : AArch64::XZR)
180 .addImm(Val: *I->Op2));
181 } else {
182 Register DstReg = MI.getOperand(i: 0).getReg();
183 bool DstIsDead = MI.getOperand(i: 0).isDead();
184 MIBS.push_back(
185 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
186 .addReg(RegNo: DstReg, Flags: RegState::Define |
187 getDeadRegState(B: DstIsDead && LastItem) |
188 RenamableState)
189 .addReg(RegNo: DstReg)
190 .addImm(Val: *I->Op2));
191 }
192 break;
193 case AArch64::EONXrs:
194 case AArch64::EORXrs:
195 case AArch64::ORRWrs:
196 case AArch64::ORRXrs: {
197 Register DstReg = MI.getOperand(i: 0).getReg();
198 bool DstIsDead = MI.getOperand(i: 0).isDead();
199 MIBS.push_back(
200 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
201 .addReg(RegNo: DstReg, Flags: RegState::Define |
202 getDeadRegState(B: DstIsDead && LastItem) |
203 RenamableState)
204 .addReg(RegNo: DstReg)
205 .addReg(RegNo: DstReg)
206 .addImm(Val: *I->Op2));
207 } break;
208 case AArch64::MOVNWi:
209 case AArch64::MOVNXi:
210 case AArch64::MOVZWi:
211 case AArch64::MOVZXi: {
212 bool DstIsDead = MI.getOperand(i: 0).isDead();
213 MIBS.push_back(
214 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
215 .addReg(RegNo: DstReg, Flags: RegState::Define |
216 getDeadRegState(B: DstIsDead && LastItem) |
217 RenamableState)
218 .addImm(Val: *I->Op1)
219 .addImm(Val: *I->Op2));
220 } break;
221 case AArch64::MOVKWi:
222 case AArch64::MOVKXi: {
223 Register DstReg = MI.getOperand(i: 0).getReg();
224 bool DstIsDead = MI.getOperand(i: 0).isDead();
225 MIBS.push_back(
226 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
227 .addReg(RegNo: DstReg, Flags: RegState::Define |
228 getDeadRegState(B: DstIsDead && LastItem) |
229 RenamableState)
230 .addReg(RegNo: DstReg)
231 .addImm(Val: *I->Op1)
232 .addImm(Val: *I->Op2));
233 } break;
234 }
235 }
236 transferImpOps(OldMI&: MI, UseMI&: MIBS.front(), DefMI&: MIBS.back());
237 MI.eraseFromParent();
238 return true;
239}
240
241bool AArch64ExpandPseudoImpl::expandCMP_SWAP(
242 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp,
243 unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
244 MachineBasicBlock::iterator &NextMBBI) {
245 MachineInstr &MI = *MBBI;
246 MIMetadata MIMD(MI);
247 const MachineOperand &Dest = MI.getOperand(i: 0);
248 Register StatusReg = MI.getOperand(i: 1).getReg();
249 bool StatusDead = MI.getOperand(i: 1).isDead();
250 // Duplicating undef operands into 2 instructions does not guarantee the same
251 // value on both; However undef should be replaced by xzr anyway.
252 assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
253 Register AddrReg = MI.getOperand(i: 2).getReg();
254 Register DesiredReg = MI.getOperand(i: 3).getReg();
255 Register NewReg = MI.getOperand(i: 4).getReg();
256
257 MachineFunction *MF = MBB.getParent();
258 auto LoadCmpBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
259 auto StoreBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
260 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
261
262 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoadCmpBB);
263 MF->insert(MBBI: ++LoadCmpBB->getIterator(), MBB: StoreBB);
264 MF->insert(MBBI: ++StoreBB->getIterator(), MBB: DoneBB);
265
266 // .Lloadcmp:
267 // mov wStatus, 0
268 // ldaxr xDest, [xAddr]
269 // cmp xDest, xDesired
270 // b.ne .Ldone
271 if (!StatusDead)
272 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::MOVZWi), DestReg: StatusReg)
273 .addImm(Val: 0).addImm(Val: 0);
274 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: LdarOp), DestReg: Dest.getReg())
275 .addReg(RegNo: AddrReg);
276 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: CmpOp), DestReg: ZeroReg)
277 .addReg(RegNo: Dest.getReg(), Flags: getKillRegState(B: Dest.isDead()))
278 .addReg(RegNo: DesiredReg)
279 .addImm(Val: ExtendImm);
280 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::Bcc))
281 .addImm(Val: AArch64CC::NE)
282 .addMBB(MBB: DoneBB)
283 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Kill);
284 LoadCmpBB->addSuccessor(Succ: DoneBB);
285 LoadCmpBB->addSuccessor(Succ: StoreBB);
286
287 // .Lstore:
288 // stlxr wStatus, xNew, [xAddr]
289 // cbnz wStatus, .Lloadcmp
290 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: StlrOp), DestReg: StatusReg)
291 .addReg(RegNo: NewReg)
292 .addReg(RegNo: AddrReg);
293 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
294 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
295 .addMBB(MBB: LoadCmpBB);
296 StoreBB->addSuccessor(Succ: LoadCmpBB);
297 StoreBB->addSuccessor(Succ: DoneBB);
298
299 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
300 DoneBB->transferSuccessors(FromMBB: &MBB);
301
302 MBB.addSuccessor(Succ: LoadCmpBB);
303
304 NextMBBI = MBB.end();
305 MI.eraseFromParent();
306
307 // Recompute livein lists.
308 LivePhysRegs LiveRegs;
309 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
310 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
311 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
312 // Do an extra pass around the loop to get loop carried registers right.
313 StoreBB->clearLiveIns();
314 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
315 LoadCmpBB->clearLiveIns();
316 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
317
318 return true;
319}
320
321bool AArch64ExpandPseudoImpl::expandCMP_SWAP_128(
322 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
323 MachineBasicBlock::iterator &NextMBBI) {
324 MachineInstr &MI = *MBBI;
325 MIMetadata MIMD(MI);
326 MachineOperand &DestLo = MI.getOperand(i: 0);
327 MachineOperand &DestHi = MI.getOperand(i: 1);
328 Register StatusReg = MI.getOperand(i: 2).getReg();
329 bool StatusDead = MI.getOperand(i: 2).isDead();
330 // Duplicating undef operands into 2 instructions does not guarantee the same
331 // value on both; However undef should be replaced by xzr anyway.
332 assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
333 Register AddrReg = MI.getOperand(i: 3).getReg();
334 Register DesiredLoReg = MI.getOperand(i: 4).getReg();
335 Register DesiredHiReg = MI.getOperand(i: 5).getReg();
336 Register NewLoReg = MI.getOperand(i: 6).getReg();
337 Register NewHiReg = MI.getOperand(i: 7).getReg();
338
339 auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
340 bool LittleEndian = STI.isLittleEndian();
341 MachineOperand &Dest0 = LittleEndian ? DestLo : DestHi;
342 MachineOperand &Dest1 = LittleEndian ? DestHi : DestLo;
343 Register New0Reg = LittleEndian ? NewLoReg : NewHiReg;
344 Register New1Reg = LittleEndian ? NewHiReg : NewLoReg;
345
346 unsigned LdxpOp, StxpOp;
347
348 switch (MI.getOpcode()) {
349 case AArch64::CMP_SWAP_128_MONOTONIC:
350 LdxpOp = AArch64::LDXPX;
351 StxpOp = AArch64::STXPX;
352 break;
353 case AArch64::CMP_SWAP_128_RELEASE:
354 LdxpOp = AArch64::LDXPX;
355 StxpOp = AArch64::STLXPX;
356 break;
357 case AArch64::CMP_SWAP_128_ACQUIRE:
358 LdxpOp = AArch64::LDAXPX;
359 StxpOp = AArch64::STXPX;
360 break;
361 case AArch64::CMP_SWAP_128:
362 LdxpOp = AArch64::LDAXPX;
363 StxpOp = AArch64::STLXPX;
364 break;
365 default:
366 llvm_unreachable("Unexpected opcode");
367 }
368
369 MachineFunction *MF = MBB.getParent();
370 auto LoadCmpBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
371 auto StoreBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
372 auto FailBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
373 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
374
375 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoadCmpBB);
376 MF->insert(MBBI: ++LoadCmpBB->getIterator(), MBB: StoreBB);
377 MF->insert(MBBI: ++StoreBB->getIterator(), MBB: FailBB);
378 MF->insert(MBBI: ++FailBB->getIterator(), MBB: DoneBB);
379
380 // .Lloadcmp:
381 // ldaxp xDestLo, xDestHi, [xAddr]
382 // cmp xDestLo, xDesiredLo
383 // sbcs xDestHi, xDesiredHi
384 // b.ne .Ldone
385 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: LdxpOp))
386 .addReg(RegNo: Dest0.getReg(), Flags: RegState::Define)
387 .addReg(RegNo: Dest1.getReg(), Flags: RegState::Define)
388 .addReg(RegNo: AddrReg);
389 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::SUBSXrs), DestReg: AArch64::XZR)
390 .addReg(RegNo: DestLo.getReg(), Flags: getKillRegState(B: DestLo.isDead()))
391 .addReg(RegNo: DesiredLoReg)
392 .addImm(Val: 0);
393 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CSINCWr), DestReg: StatusReg)
394 .addUse(RegNo: AArch64::WZR)
395 .addUse(RegNo: AArch64::WZR)
396 .addImm(Val: AArch64CC::EQ);
397 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::SUBSXrs), DestReg: AArch64::XZR)
398 .addReg(RegNo: DestHi.getReg(), Flags: getKillRegState(B: DestHi.isDead()))
399 .addReg(RegNo: DesiredHiReg)
400 .addImm(Val: 0);
401 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CSINCWr), DestReg: StatusReg)
402 .addUse(RegNo: StatusReg, Flags: RegState::Kill)
403 .addUse(RegNo: StatusReg, Flags: RegState::Kill)
404 .addImm(Val: AArch64CC::EQ);
405 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
406 .addUse(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
407 .addMBB(MBB: FailBB);
408 LoadCmpBB->addSuccessor(Succ: FailBB);
409 LoadCmpBB->addSuccessor(Succ: StoreBB);
410
411 // .Lstore:
412 // stlxp wStatus, xNewLo, xNewHi, [xAddr]
413 // cbnz wStatus, .Lloadcmp
414 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: StxpOp), DestReg: StatusReg)
415 .addReg(RegNo: New0Reg)
416 .addReg(RegNo: New1Reg)
417 .addReg(RegNo: AddrReg);
418 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
419 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
420 .addMBB(MBB: LoadCmpBB);
421 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::B)).addMBB(MBB: DoneBB);
422 StoreBB->addSuccessor(Succ: LoadCmpBB);
423 StoreBB->addSuccessor(Succ: DoneBB);
424
425 // .Lfail:
426 // stlxp wStatus, xDestLo, xDestHi, [xAddr]
427 // cbnz wStatus, .Lloadcmp
428 BuildMI(BB: FailBB, MIMD, MCID: TII->get(Opcode: StxpOp), DestReg: StatusReg)
429 .addReg(RegNo: Dest0.getReg())
430 .addReg(RegNo: Dest1.getReg())
431 .addReg(RegNo: AddrReg);
432 BuildMI(BB: FailBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
433 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
434 .addMBB(MBB: LoadCmpBB);
435 FailBB->addSuccessor(Succ: LoadCmpBB);
436 FailBB->addSuccessor(Succ: DoneBB);
437
438 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
439 DoneBB->transferSuccessors(FromMBB: &MBB);
440
441 MBB.addSuccessor(Succ: LoadCmpBB);
442
443 NextMBBI = MBB.end();
444 MI.eraseFromParent();
445
446 // Recompute liveness bottom up.
447 LivePhysRegs LiveRegs;
448 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
449 computeAndAddLiveIns(LiveRegs, MBB&: *FailBB);
450 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
451 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
452
453 // Do an extra pass in the loop to get the loop carried dependencies right.
454 FailBB->clearLiveIns();
455 computeAndAddLiveIns(LiveRegs, MBB&: *FailBB);
456 StoreBB->clearLiveIns();
457 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
458 LoadCmpBB->clearLiveIns();
459 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
460
461 return true;
462}
463
464/// \brief Expand Pseudos to Instructions with destructive operands.
465///
466/// This mechanism uses MOVPRFX instructions for zeroing the false lanes
467/// or for fixing relaxed register allocation conditions to comply with
468/// the instructions register constraints. The latter case may be cheaper
469/// than setting the register constraints in the register allocator,
470/// since that will insert regular MOV instructions rather than MOVPRFX.
471///
472/// Example (after register allocation):
473///
474/// FSUB_ZPZZ_ZERO_B Z0, Pg, Z1, Z0
475///
476/// * The Pseudo FSUB_ZPZZ_ZERO_B maps to FSUB_ZPmZ_B.
477/// * We cannot map directly to FSUB_ZPmZ_B because the register
478/// constraints of the instruction are not met.
479/// * Also the _ZERO specifies the false lanes need to be zeroed.
480///
481/// We first try to see if the destructive operand == result operand,
482/// if not, we try to swap the operands, e.g.
483///
484/// FSUB_ZPmZ_B Z0, Pg/m, Z0, Z1
485///
486/// But because FSUB_ZPmZ is not commutative, this is semantically
487/// different, so we need a reverse instruction:
488///
489/// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1
490///
491/// Then we implement the zeroing of the false lanes of Z0 by adding
492/// a zeroing MOVPRFX instruction:
493///
494/// MOVPRFX_ZPzZ_B Z0, Pg/z, Z0
495/// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1
496///
497/// Note that this can only be done for _ZERO or _UNDEF variants where
498/// we can guarantee the false lanes to be zeroed (by implementing this)
499/// or that they are undef (don't care / not used), otherwise the
500/// swapping of operands is illegal because the operation is not
501/// (or cannot be emulated to be) fully commutative.
502bool AArch64ExpandPseudoImpl::expand_DestructiveOp(
503 MachineInstr &MI, MachineBasicBlock &MBB,
504 MachineBasicBlock::iterator MBBI) {
505 unsigned Opcode = AArch64::getSVEPseudoMap(Opcode: MI.getOpcode());
506 uint64_t DType = TII->get(Opcode).TSFlags & AArch64::DestructiveInstTypeMask;
507 uint64_t FalseLanes = MI.getDesc().TSFlags & AArch64::FalseLanesMask;
508 bool FalseZero = FalseLanes == AArch64::FalseLanesZero;
509 Register DstReg = MI.getOperand(i: 0).getReg();
510 bool DstIsDead = MI.getOperand(i: 0).isDead();
511 bool UseRev = false;
512 unsigned PredIdx, DOPIdx, SrcIdx, Src2Idx;
513
514 switch (DType) {
515 case AArch64::DestructiveBinaryComm:
516 case AArch64::DestructiveBinaryCommWithRev:
517 if (DstReg == MI.getOperand(i: 3).getReg()) {
518 // FSUB Zd, Pg, Zs1, Zd ==> FSUBR Zd, Pg/m, Zd, Zs1
519 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 3, args: 2);
520 UseRev = true;
521 break;
522 }
523 [[fallthrough]];
524 case AArch64::DestructiveBinary:
525 case AArch64::DestructiveBinaryImm:
526 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 2, args: 3);
527 break;
528 case AArch64::DestructiveUnaryPassthru:
529 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 2, args: 3, args: 3);
530 break;
531 case AArch64::DestructiveTernaryCommWithRev:
532 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 2, args: 3, args: 4);
533 if (DstReg == MI.getOperand(i: 3).getReg()) {
534 // FMLA Zd, Pg, Za, Zd, Zm ==> FMAD Zdn, Pg, Zm, Za
535 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 3, args: 4, args: 2);
536 UseRev = true;
537 } else if (DstReg == MI.getOperand(i: 4).getReg()) {
538 // FMLA Zd, Pg, Za, Zm, Zd ==> FMAD Zdn, Pg, Zm, Za
539 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 4, args: 3, args: 2);
540 UseRev = true;
541 }
542 break;
543 case AArch64::Destructive2xRegImmUnpred:
544 // EXT_ZZI_CONSTRUCTIVE Zd, Zs, Imm
545 // ==> MOVPRFX Zd Zs; EXT_ZZI Zd, Zd, Zs, Imm
546 std::tie(args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 1, args: 2);
547 break;
548 case AArch64::DestructiveBinaryImmUnpred:
549 std::tie(args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 2);
550 break;
551 case AArch64::DestructiveBinaryShImmUnpred:
552 std::tie(args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 2, args: 3);
553 break;
554 default:
555 llvm_unreachable("Unsupported Destructive Operand type");
556 }
557
558 // MOVPRFX can only be used if the destination operand
559 // is the destructive operand, not as any other operand,
560 // so the Destructive Operand must be unique.
561 bool DOPRegIsUnique = false;
562 switch (DType) {
563 case AArch64::DestructiveBinary:
564 DOPRegIsUnique = DstReg != MI.getOperand(i: SrcIdx).getReg();
565 break;
566 case AArch64::DestructiveBinaryComm:
567 case AArch64::DestructiveBinaryCommWithRev:
568 DOPRegIsUnique =
569 DstReg != MI.getOperand(i: DOPIdx).getReg() ||
570 MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: SrcIdx).getReg();
571 break;
572 case AArch64::DestructiveUnaryPassthru:
573 case AArch64::DestructiveBinaryImm:
574 case AArch64::DestructiveBinaryImmUnpred:
575 case AArch64::DestructiveBinaryShImmUnpred:
576 case AArch64::Destructive2xRegImmUnpred:
577 DOPRegIsUnique = true;
578 break;
579 case AArch64::DestructiveTernaryCommWithRev:
580 DOPRegIsUnique =
581 DstReg != MI.getOperand(i: DOPIdx).getReg() ||
582 (MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: SrcIdx).getReg() &&
583 MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: Src2Idx).getReg());
584 break;
585 }
586
587 // Resolve the reverse opcode
588 if (UseRev) {
589 int NewOpcode;
590 // e.g. DIV -> DIVR
591 if ((NewOpcode = AArch64::getSVERevInstr(Opcode)) != -1)
592 Opcode = NewOpcode;
593 // e.g. DIVR -> DIV
594 else if ((NewOpcode = AArch64::getSVENonRevInstr(Opcode)) != -1)
595 Opcode = NewOpcode;
596 }
597
598 // Get the right MOVPRFX
599 uint64_t ElementSize = TII->getElementSizeForOpcode(Opc: Opcode);
600 unsigned MovPrfx, LSLZero, MovPrfxZero;
601 switch (ElementSize) {
602 case AArch64::ElementSizeNone:
603 case AArch64::ElementSizeB:
604 MovPrfx = AArch64::MOVPRFX_ZZ;
605 LSLZero = AArch64::LSL_ZPmI_B;
606 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_B;
607 break;
608 case AArch64::ElementSizeH:
609 MovPrfx = AArch64::MOVPRFX_ZZ;
610 LSLZero = AArch64::LSL_ZPmI_H;
611 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_H;
612 break;
613 case AArch64::ElementSizeS:
614 MovPrfx = AArch64::MOVPRFX_ZZ;
615 LSLZero = AArch64::LSL_ZPmI_S;
616 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_S;
617 break;
618 case AArch64::ElementSizeD:
619 MovPrfx = AArch64::MOVPRFX_ZZ;
620 LSLZero = AArch64::LSL_ZPmI_D;
621 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_D;
622 break;
623 default:
624 llvm_unreachable("Unsupported ElementSize");
625 }
626
627 // Preserve undef state until DOP's reg is defined.
628 RegState DOPRegState = getUndefRegState(B: MI.getOperand(i: DOPIdx).isUndef());
629
630 //
631 // Create the destructive operation (if required)
632 //
633 MachineInstrBuilder PRFX, DOP;
634 if (FalseZero) {
635 // If we cannot prefix the requested instruction we'll instead emit a
636 // prefixed_zeroing_mov for DestructiveBinary.
637 assert((DOPRegIsUnique || DType == AArch64::DestructiveBinary ||
638 DType == AArch64::DestructiveBinaryComm ||
639 DType == AArch64::DestructiveBinaryCommWithRev) &&
640 "The destructive operand should be unique");
641 assert(ElementSize != AArch64::ElementSizeNone &&
642 "This instruction is unpredicated");
643
644 // Merge source operand into destination register
645 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: MovPrfxZero))
646 .addReg(RegNo: DstReg, Flags: RegState::Define)
647 .addReg(RegNo: MI.getOperand(i: PredIdx).getReg())
648 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState);
649
650 // After the movprfx, the destructive operand is same as Dst
651 DOPIdx = 0;
652 DOPRegState = {};
653
654 // Create the additional LSL to zero the lanes when the DstReg is not
655 // unique. Zeros the lanes in z0 that aren't active in p0 with sequence
656 // movprfx z0.b, p0/z, z0.b; lsl z0.b, p0/m, z0.b, #0;
657 if ((DType == AArch64::DestructiveBinary ||
658 DType == AArch64::DestructiveBinaryComm ||
659 DType == AArch64::DestructiveBinaryCommWithRev) &&
660 !DOPRegIsUnique) {
661 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LSLZero))
662 .addReg(RegNo: DstReg, Flags: RegState::Define)
663 .add(MO: MI.getOperand(i: PredIdx))
664 .addReg(RegNo: DstReg)
665 .addImm(Val: 0);
666 }
667 } else if (DstReg != MI.getOperand(i: DOPIdx).getReg()) {
668 assert(DOPRegIsUnique && "The destructive operand should be unique");
669 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: MovPrfx))
670 .addReg(RegNo: DstReg, Flags: RegState::Define)
671 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState);
672 DOPIdx = 0;
673 DOPRegState = {};
674 }
675
676 //
677 // Create the destructive operation
678 //
679 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode))
680 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
681 DOPRegState = DOPRegState | RegState::Kill;
682
683 switch (DType) {
684 case AArch64::DestructiveUnaryPassthru:
685 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
686 .add(MO: MI.getOperand(i: PredIdx))
687 .add(MO: MI.getOperand(i: SrcIdx));
688 break;
689 case AArch64::DestructiveBinary:
690 case AArch64::DestructiveBinaryImm:
691 case AArch64::DestructiveBinaryComm:
692 case AArch64::DestructiveBinaryCommWithRev:
693 DOP.add(MO: MI.getOperand(i: PredIdx))
694 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
695 .add(MO: MI.getOperand(i: SrcIdx));
696 break;
697 case AArch64::DestructiveTernaryCommWithRev:
698 DOP.add(MO: MI.getOperand(i: PredIdx))
699 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
700 .add(MO: MI.getOperand(i: SrcIdx))
701 .add(MO: MI.getOperand(i: Src2Idx));
702 break;
703 case AArch64::DestructiveBinaryImmUnpred:
704 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
705 .add(MO: MI.getOperand(i: SrcIdx));
706 break;
707 case AArch64::DestructiveBinaryShImmUnpred:
708 case AArch64::Destructive2xRegImmUnpred:
709 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
710 .add(MO: MI.getOperand(i: SrcIdx))
711 .add(MO: MI.getOperand(i: Src2Idx));
712 break;
713 }
714
715 if (PRFX) {
716 transferImpOps(OldMI&: MI, UseMI&: PRFX, DefMI&: DOP);
717 finalizeBundle(MBB, FirstMI: PRFX->getIterator(), LastMI: MBBI->getIterator());
718 } else
719 transferImpOps(OldMI&: MI, UseMI&: DOP, DefMI&: DOP);
720
721 MI.eraseFromParent();
722 return true;
723}
724
725bool AArch64ExpandPseudoImpl::expandSVEBitwisePseudo(
726 MachineInstr &MI, MachineBasicBlock &MBB,
727 MachineBasicBlock::iterator MBBI) {
728 MachineInstrBuilder PRFX, DOP;
729 const unsigned Opcode = MI.getOpcode();
730 const MachineOperand &Op0 = MI.getOperand(i: 0);
731 const MachineOperand *Op1 = &MI.getOperand(i: 1);
732 const MachineOperand *Op2 = &MI.getOperand(i: 2);
733 const Register DOPReg = Op0.getReg();
734
735 if (DOPReg == Op2->getReg()) {
736 // Commute the operands to allow destroying the second source.
737 std::swap(a&: Op1, b&: Op2);
738 } else if (DOPReg != Op1->getReg()) {
739 // If not in destructive form, emit a MOVPRFX. The input should only be
740 // killed if unused by the subsequent instruction.
741 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MOVPRFX_ZZ))
742 .addDef(RegNo: DOPReg, Flags: getRenamableRegState(B: Op0.isRenamable()))
743 .addReg(RegNo: Op1->getReg(),
744 Flags: getRenamableRegState(B: Op1->isRenamable()) |
745 getUndefRegState(B: Op1->isUndef()) |
746 getKillRegState(B: Op1->isKill() &&
747 Opcode == AArch64::NAND_ZZZ));
748 }
749
750 assert((DOPReg == Op1->getReg() || PRFX) && "invalid expansion");
751
752 const RegState DOPRegState = getRenamableRegState(B: Op0.isRenamable()) |
753 getUndefRegState(B: !PRFX && Op1->isUndef()) |
754 RegState::Kill;
755
756 switch (Opcode) {
757 default:
758 llvm_unreachable("unhandled opcode");
759 case AArch64::EON_ZZZ:
760 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::BSL2N_ZZZZ))
761 .add(MO: Op0)
762 .addReg(RegNo: DOPReg, Flags: DOPRegState)
763 .add(MO: *Op1)
764 .add(MO: *Op2);
765 break;
766 case AArch64::NAND_ZZZ:
767 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::NBSL_ZZZZ))
768 .add(MO: Op0)
769 .addReg(RegNo: DOPReg, Flags: DOPRegState)
770 .add(MO: *Op2)
771 .add(MO: *Op2);
772 break;
773 case AArch64::NOR_ZZZ:
774 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::NBSL_ZZZZ))
775 .add(MO: Op0)
776 .addReg(RegNo: DOPReg, Flags: DOPRegState)
777 .add(MO: *Op2)
778 .add(MO: *Op1);
779 break;
780 }
781
782 if (PRFX) {
783 transferImpOps(OldMI&: MI, UseMI&: PRFX, DefMI&: DOP);
784 finalizeBundle(MBB, FirstMI: PRFX->getIterator(), LastMI: MBBI->getIterator());
785 } else {
786 transferImpOps(OldMI&: MI, UseMI&: DOP, DefMI&: DOP);
787 }
788
789 MI.eraseFromParent();
790 return true;
791}
792
793bool AArch64ExpandPseudoImpl::expandSetTagLoop(
794 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
795 MachineBasicBlock::iterator &NextMBBI) {
796 MachineInstr &MI = *MBBI;
797 DebugLoc DL = MI.getDebugLoc();
798 Register SizeReg = MI.getOperand(i: 0).getReg();
799 Register AddressReg = MI.getOperand(i: 1).getReg();
800
801 MachineFunction *MF = MBB.getParent();
802
803 bool ZeroData = MI.getOpcode() == AArch64::STZGloop_wback;
804 const unsigned OpCode1 =
805 ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex;
806 const unsigned OpCode2 =
807 ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
808
809 unsigned Size = MI.getOperand(i: 2).getImm();
810 assert(Size > 0 && Size % 16 == 0);
811 if (Size % (16 * 2) != 0) {
812 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: OpCode1), DestReg: AddressReg)
813 .addReg(RegNo: AddressReg)
814 .addReg(RegNo: AddressReg)
815 .addImm(Val: 1);
816 Size -= 16;
817 }
818 MachineBasicBlock::iterator I =
819 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVi64imm), DestReg: SizeReg)
820 .addImm(Val: Size);
821 expandMOVImm(MBB, MBBI: I, BitSize: 64);
822
823 auto LoopBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
824 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
825
826 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopBB);
827 MF->insert(MBBI: ++LoopBB->getIterator(), MBB: DoneBB);
828
829 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: OpCode2))
830 .addDef(RegNo: AddressReg)
831 .addReg(RegNo: AddressReg)
832 .addReg(RegNo: AddressReg)
833 .addImm(Val: 2)
834 .cloneMemRefs(OtherMI: MI)
835 .setMIFlags(MI.getFlags());
836 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXri))
837 .addDef(RegNo: SizeReg)
838 .addReg(RegNo: SizeReg)
839 .addImm(Val: 16 * 2)
840 .addImm(Val: 0);
841 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
842 .addImm(Val: AArch64CC::NE)
843 .addMBB(MBB: LoopBB)
844 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Kill);
845
846 LoopBB->addSuccessor(Succ: LoopBB);
847 LoopBB->addSuccessor(Succ: DoneBB);
848
849 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
850 DoneBB->transferSuccessors(FromMBB: &MBB);
851
852 MBB.addSuccessor(Succ: LoopBB);
853
854 NextMBBI = MBB.end();
855 MI.eraseFromParent();
856 // Recompute liveness bottom up.
857 LivePhysRegs LiveRegs;
858 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
859 computeAndAddLiveIns(LiveRegs, MBB&: *LoopBB);
860 // Do an extra pass in the loop to get the loop carried dependencies right.
861 // FIXME: is this necessary?
862 LoopBB->clearLiveIns();
863 computeAndAddLiveIns(LiveRegs, MBB&: *LoopBB);
864 DoneBB->clearLiveIns();
865 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
866
867 return true;
868}
869
870bool AArch64ExpandPseudoImpl::expandSVESpillFill(
871 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned Opc,
872 unsigned N) {
873 assert((Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI ||
874 Opc == AArch64::LDR_PXI || Opc == AArch64::STR_PXI) &&
875 "Unexpected opcode");
876 RegState RState =
877 getDefRegState(B: Opc == AArch64::LDR_ZXI || Opc == AArch64::LDR_PXI);
878 unsigned sub0 = (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI)
879 ? AArch64::zsub0
880 : AArch64::psub0;
881 const TargetRegisterInfo *TRI =
882 MBB.getParent()->getSubtarget().getRegisterInfo();
883 MachineInstr &MI = *MBBI;
884 for (unsigned Offset = 0; Offset < N; ++Offset) {
885 int ImmOffset = MI.getOperand(i: 2).getImm() + Offset;
886 bool Kill = (Offset + 1 == N) ? MI.getOperand(i: 1).isKill() : false;
887 assert(ImmOffset >= -256 && ImmOffset < 256 &&
888 "Immediate spill offset out of range");
889 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
890 .addReg(RegNo: TRI->getSubReg(Reg: MI.getOperand(i: 0).getReg(), Idx: sub0 + Offset),
891 Flags: RState)
892 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: getKillRegState(B: Kill))
893 .addImm(Val: ImmOffset);
894 }
895 MI.eraseFromParent();
896 return true;
897}
898
899// Create a call with the passed opcode and explicit operands, copying over all
900// the implicit operands from *MBBI, starting at the regmask.
901static MachineInstr *createCallWithOps(MachineBasicBlock &MBB,
902 MachineBasicBlock::iterator MBBI,
903 const AArch64InstrInfo *TII,
904 unsigned Opcode,
905 ArrayRef<MachineOperand> ExplicitOps,
906 unsigned RegMaskStartIdx) {
907 // Be careful not to duplicate the LR def which the original instruction
908 // already carries.
909 MachineFunction &MF = *MBB.getParent();
910 MachineInstr *Call =
911 MF.CreateMachineInstr(MCID: TII->get(Opcode), DL: MBBI->getDebugLoc(),
912 /*NoImplicit=*/true);
913 MBB.insert(I: MBBI, MI: Call);
914 MachineInstrBuilder(MF, Call).add(MOs: ExplicitOps);
915
916 // Register arguments are added during ISel, but cannot be added as explicit
917 // operands of the branch as it expects to be B <target> which is only one
918 // operand. Instead they are implicit operands used by the branch.
919 while (!MBBI->getOperand(i: RegMaskStartIdx).isRegMask()) {
920 const MachineOperand &MOP = MBBI->getOperand(i: RegMaskStartIdx);
921 assert(MOP.isReg() && "can only add register operands");
922 Call->addOperand(Op: MachineOperand::CreateReg(
923 Reg: MOP.getReg(), /*Def=*/isDef: false, /*Implicit=*/isImp: true, /*isKill=*/false,
924 /*isDead=*/false, /*isUndef=*/MOP.isUndef()));
925 RegMaskStartIdx++;
926 }
927 for (const MachineOperand &MO :
928 llvm::drop_begin(RangeOrContainer: MBBI->operands(), N: RegMaskStartIdx))
929 Call->addOperand(Op: MO);
930
931 return Call;
932}
933
934// Create a call to CallTarget, copying over all the operands from *MBBI,
935// starting at the regmask.
936static MachineInstr *createCall(MachineBasicBlock &MBB,
937 MachineBasicBlock::iterator MBBI,
938 const AArch64InstrInfo *TII,
939 MachineOperand &CallTarget,
940 unsigned RegMaskStartIdx) {
941 unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
942
943 assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
944 "invalid operand for regular call");
945 return createCallWithOps(MBB, MBBI, TII, Opcode: Opc, ExplicitOps: CallTarget, RegMaskStartIdx);
946}
947
948bool AArch64ExpandPseudoImpl::expandCALL_RVMARKER(
949 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
950 // Expand CALL_RVMARKER pseudo to:
951 // - a branch to the call target, followed by
952 // - the special `mov x29, x29` marker, if necessary, and
953 // - another branch, to the runtime function
954 // Mark the sequence as bundle, to avoid passes moving other code in between.
955 MachineInstr &MI = *MBBI;
956 MachineOperand &RVTarget = MI.getOperand(i: 0);
957 bool DoEmitMarker = MI.getOperand(i: 1).getImm();
958 assert(RVTarget.isGlobal() && "invalid operand for attached call");
959
960 MachineInstr *OriginalCall = nullptr;
961
962 if (MI.getOpcode() == AArch64::BLRA_RVMARKER) {
963 // ptrauth call.
964 const MachineOperand &CallTarget = MI.getOperand(i: 2);
965 const MachineOperand &Key = MI.getOperand(i: 3);
966 const MachineOperand &IntDisc = MI.getOperand(i: 4);
967 const MachineOperand &AddrDisc = MI.getOperand(i: 5);
968
969 assert((Key.getImm() == AArch64PACKey::IA ||
970 Key.getImm() == AArch64PACKey::IB) &&
971 "Invalid auth call key");
972
973 MachineOperand Ops[] = {CallTarget, Key, IntDisc, AddrDisc};
974
975 OriginalCall = createCallWithOps(MBB, MBBI, TII, Opcode: AArch64::BLRA, ExplicitOps: Ops,
976 /*RegMaskStartIdx=*/6);
977 } else {
978 assert(MI.getOpcode() == AArch64::BLR_RVMARKER && "unknown rvmarker MI");
979 OriginalCall = createCall(MBB, MBBI, TII, CallTarget&: MI.getOperand(i: 2),
980 // Regmask starts after the RV and call targets.
981 /*RegMaskStartIdx=*/3);
982 }
983
984 if (DoEmitMarker)
985 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ORRXrs))
986 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
987 .addReg(RegNo: AArch64::XZR)
988 .addReg(RegNo: AArch64::FP)
989 .addImm(Val: 0);
990
991 auto *RVCall = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::BL))
992 .add(MO: RVTarget)
993 .getInstr();
994
995 if (MI.shouldUpdateAdditionalCallInfo())
996 MBB.getParent()->moveAdditionalCallInfo(Old: &MI, New: OriginalCall);
997
998 MI.eraseFromParent();
999 finalizeBundle(MBB, FirstMI: OriginalCall->getIterator(),
1000 LastMI: std::next(x: RVCall->getIterator()));
1001 return true;
1002}
1003
1004bool AArch64ExpandPseudoImpl::expandCALL_BTI(MachineBasicBlock &MBB,
1005 MachineBasicBlock::iterator MBBI) {
1006 // Expand CALL_BTI pseudo to:
1007 // - a branch to the call target
1008 // - a BTI instruction
1009 // Mark the sequence as a bundle, to avoid passes moving other code in
1010 // between.
1011 MachineInstr &MI = *MBBI;
1012 MachineInstr *Call = createCall(MBB, MBBI, TII, CallTarget&: MI.getOperand(i: 0),
1013 // Regmask starts after the call target.
1014 /*RegMaskStartIdx=*/1);
1015
1016 Call->setCFIType(MF&: *MBB.getParent(), Type: MI.getCFIType());
1017
1018 MachineInstr *BTI =
1019 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::HINT))
1020 // BTI J so that setjmp can to BR to this.
1021 .addImm(Val: 36)
1022 .getInstr();
1023
1024 if (MI.shouldUpdateAdditionalCallInfo())
1025 MBB.getParent()->moveAdditionalCallInfo(Old: &MI, New: Call);
1026
1027 MI.eraseFromParent();
1028 finalizeBundle(MBB, FirstMI: Call->getIterator(), LastMI: std::next(x: BTI->getIterator()));
1029 return true;
1030}
1031
1032bool AArch64ExpandPseudoImpl::expandStoreSwiftAsyncContext(
1033 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
1034 Register CtxReg = MBBI->getOperand(i: 0).getReg();
1035 Register BaseReg = MBBI->getOperand(i: 1).getReg();
1036 int Offset = MBBI->getOperand(i: 2).getImm();
1037 DebugLoc DL(MBBI->getDebugLoc());
1038 auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1039
1040 if (STI.getTargetTriple().getArchName() != "arm64e") {
1041 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
1042 .addUse(RegNo: CtxReg)
1043 .addUse(RegNo: BaseReg)
1044 .addImm(Val: Offset / 8)
1045 .setMIFlag(MachineInstr::FrameSetup);
1046 MBBI->eraseFromParent();
1047 return true;
1048 }
1049
1050 // We need to sign the context in an address-discriminated way. 0xc31a is a
1051 // fixed random value, chosen as part of the ABI.
1052 // add x16, xBase, #Offset
1053 // movk x16, #0xc31a, lsl #48
1054 // mov x17, x22/xzr
1055 // pacdb x17, x16
1056 // str x17, [xBase, #Offset]
1057 unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
1058 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: AArch64::X16)
1059 .addUse(RegNo: BaseReg)
1060 .addImm(Val: abs(x: Offset))
1061 .addImm(Val: 0)
1062 .setMIFlag(MachineInstr::FrameSetup);
1063 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVKXi), DestReg: AArch64::X16)
1064 .addUse(RegNo: AArch64::X16)
1065 .addImm(Val: 0xc31a)
1066 .addImm(Val: 48)
1067 .setMIFlag(MachineInstr::FrameSetup);
1068 // We're not allowed to clobber X22 (and couldn't clobber XZR if we tried), so
1069 // move it somewhere before signing.
1070 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: AArch64::X17)
1071 .addUse(RegNo: AArch64::XZR)
1072 .addUse(RegNo: CtxReg)
1073 .addImm(Val: 0)
1074 .setMIFlag(MachineInstr::FrameSetup);
1075 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACDB), DestReg: AArch64::X17)
1076 .addUse(RegNo: AArch64::X17)
1077 .addUse(RegNo: AArch64::X16)
1078 .setMIFlag(MachineInstr::FrameSetup);
1079 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
1080 .addUse(RegNo: AArch64::X17)
1081 .addUse(RegNo: BaseReg)
1082 .addImm(Val: Offset / 8)
1083 .setMIFlag(MachineInstr::FrameSetup);
1084
1085 MBBI->eraseFromParent();
1086 return true;
1087}
1088
1089AArch64ExpandPseudoImpl::ConditionalBlocks
1090AArch64ExpandPseudoImpl::expandConditionalPseudo(
1091 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL,
1092 MachineInstrBuilder &Branch) {
1093 assert((std::next(MBBI) != MBB.end() ||
1094 MBB.successors().begin() != MBB.successors().end()) &&
1095 "Unexpected unreachable in block");
1096
1097 // Split MBB and create two new blocks:
1098 // - MBB now contains all instructions before the conditional pseudo.
1099 // - CondBB contains the conditional pseudo instruction only.
1100 // - EndBB contains all instructions after the conditional pseudo.
1101 MachineInstr &PrevMI = *std::prev(x: MBBI);
1102 MachineBasicBlock *CondBB = MBB.splitAt(SplitInst&: PrevMI, /*UpdateLiveIns*/ true);
1103 MachineBasicBlock *EndBB =
1104 std::next(x: MBBI) == CondBB->end()
1105 ? *CondBB->successors().begin()
1106 : CondBB->splitAt(SplitInst&: *MBBI, /*UpdateLiveIns*/ true);
1107
1108 // Add the SMBB label to the branch instruction & create a branch to EndBB.
1109 Branch.addMBB(MBB: CondBB);
1110 BuildMI(BB: &MBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
1111 .addMBB(MBB: EndBB);
1112 MBB.addSuccessor(Succ: EndBB);
1113
1114 // Create branch from CondBB to EndBB. Users of this helper should insert new
1115 // instructions at CondBB.back() -- i.e. before the branch.
1116 BuildMI(BB: CondBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::B)).addMBB(MBB: EndBB);
1117 return {.CondBB: *CondBB, .EndBB: *EndBB};
1118}
1119
1120MachineBasicBlock *
1121AArch64ExpandPseudoImpl::expandRestoreZASave(MachineBasicBlock &MBB,
1122 MachineBasicBlock::iterator MBBI) {
1123 MachineInstr &MI = *MBBI;
1124 DebugLoc DL = MI.getDebugLoc();
1125
1126 // Compare TPIDR2_EL0 against 0. Restore ZA if TPIDR2_EL0 is zero.
1127 MachineInstrBuilder Branch =
1128 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::CBZX)).add(MO: MI.getOperand(i: 0));
1129
1130 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);
1131 // Replace the pseudo with a call (BL).
1132 MachineInstrBuilder MIB =
1133 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::BL));
1134 // Copy operands (mainly the regmask) from the pseudo.
1135 for (unsigned I = 2; I < MI.getNumOperands(); ++I)
1136 MIB.add(MO: MI.getOperand(i: I));
1137 // Mark the TPIDR2 block pointer (X0) as an implicit use.
1138 MIB.addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState::Implicit);
1139
1140 MI.eraseFromParent();
1141 return &EndBB;
1142}
1143
1144static constexpr unsigned ZERO_ALL_ZA_MASK = 0b11111111;
1145
1146MachineBasicBlock *
1147AArch64ExpandPseudoImpl::expandCommitZASave(MachineBasicBlock &MBB,
1148 MachineBasicBlock::iterator MBBI) {
1149 MachineInstr &MI = *MBBI;
1150 DebugLoc DL = MI.getDebugLoc();
1151 [[maybe_unused]] auto *RI = MBB.getParent()->getSubtarget().getRegisterInfo();
1152
1153 // Compare TPIDR2_EL0 against 0. Commit ZA if TPIDR2_EL0 is non-zero.
1154 MachineInstrBuilder Branch =
1155 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::CBNZX)).add(MO: MI.getOperand(i: 0));
1156
1157 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);
1158 // Replace the pseudo with a call (BL).
1159 MachineInstrBuilder MIB =
1160 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::BL));
1161 // Copy operands (mainly the regmask) from the pseudo.
1162 for (unsigned I = 3; I < MI.getNumOperands(); ++I)
1163 MIB.add(MO: MI.getOperand(i: I));
1164 // Clear TPIDR2_EL0.
1165 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::MSR))
1166 .addImm(Val: AArch64SysReg::TPIDR2_EL0)
1167 .addReg(RegNo: AArch64::XZR);
1168 bool ZeroZA = MI.getOperand(i: 1).getImm() != 0;
1169 bool ZeroZT0 = MI.getOperand(i: 2).getImm() != 0;
1170 if (ZeroZA) {
1171 assert(MI.definesRegister(AArch64::ZAB0, RI) && "should define ZA!");
1172 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ZERO_M))
1173 .addImm(Val: ZERO_ALL_ZA_MASK)
1174 .addDef(RegNo: AArch64::ZAB0, Flags: RegState::ImplicitDefine);
1175 }
1176 if (ZeroZT0) {
1177 assert(MI.definesRegister(AArch64::ZT0, RI) && "should define ZT0!");
1178 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ZERO_T))
1179 .addDef(RegNo: AArch64::ZT0);
1180 }
1181
1182 MI.eraseFromParent();
1183 return &EndBB;
1184}
1185
1186MachineBasicBlock *
1187AArch64ExpandPseudoImpl::expandCondSMToggle(MachineBasicBlock &MBB,
1188 MachineBasicBlock::iterator MBBI) {
1189 MachineInstr &MI = *MBBI;
1190 // In the case of a smstart/smstop before a unreachable, just remove the pseudo.
1191 // Exception handling code generated by Clang may introduce unreachables and it
1192 // seems unnecessary to restore pstate.sm when that happens. Note that it is
1193 // not just an optimisation, the code below expects a successor instruction/block
1194 // in order to split the block at MBBI.
1195 if (std::next(x: MBBI) == MBB.end() &&
1196 MI.getParent()->successors().begin() ==
1197 MI.getParent()->successors().end()) {
1198 MI.eraseFromParent();
1199 return &MBB;
1200 }
1201
1202 // Expand the pseudo into smstart or smstop instruction. The pseudo has the
1203 // following operands:
1204 //
1205 // MSRpstatePseudo <za|sm|both>, <0|1>, condition[, pstate.sm], <regmask>
1206 //
1207 // The pseudo is expanded into a conditional smstart/smstop, with a
1208 // check if pstate.sm (register) equals the expected value, and if not,
1209 // invokes the smstart/smstop.
1210 //
1211 // As an example, the following block contains a normal call from a
1212 // streaming-compatible function:
1213 //
1214 // OrigBB:
1215 // MSRpstatePseudo 3, 0, IfCallerIsStreaming, %0, <regmask> <- Cond SMSTOP
1216 // bl @normal_callee
1217 // MSRpstatePseudo 3, 1, IfCallerIsStreaming, %0, <regmask> <- Cond SMSTART
1218 //
1219 // ...which will be transformed into:
1220 //
1221 // OrigBB:
1222 // TBNZx %0:gpr64, 0, SMBB
1223 // b EndBB
1224 //
1225 // SMBB:
1226 // MSRpstatesvcrImm1 3, 0, <regmask> <- SMSTOP
1227 //
1228 // EndBB:
1229 // bl @normal_callee
1230 // MSRcond_pstatesvcrImm1 3, 1, <regmask> <- SMSTART
1231 //
1232 DebugLoc DL = MI.getDebugLoc();
1233
1234 // Create the conditional branch based on the third operand of the
1235 // instruction, which tells us if we are wrapping a normal or streaming
1236 // function.
1237 // We test the live value of pstate.sm and toggle pstate.sm if this is not the
1238 // expected value for the callee (0 for a normal callee and 1 for a streaming
1239 // callee).
1240 unsigned Opc;
1241 switch (MI.getOperand(i: 2).getImm()) {
1242 case AArch64SME::Always:
1243 llvm_unreachable("Should have matched to instruction directly");
1244 case AArch64SME::IfCallerIsStreaming:
1245 Opc = AArch64::TBNZW;
1246 break;
1247 case AArch64SME::IfCallerIsNonStreaming:
1248 Opc = AArch64::TBZW;
1249 break;
1250 }
1251 auto PStateSM = MI.getOperand(i: 3).getReg();
1252 auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
1253 unsigned SMReg32 = TRI->getSubReg(Reg: PStateSM, Idx: AArch64::sub_32);
1254 MachineInstrBuilder Tbx =
1255 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc)).addReg(RegNo: SMReg32).addImm(Val: 0);
1256
1257 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch&: Tbx);
1258 // Create the SMSTART/SMSTOP (MSRpstatesvcrImm1) instruction in SMBB.
1259 MachineInstrBuilder MIB = BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: MI.getDebugLoc(),
1260 MCID: TII->get(Opcode: AArch64::MSRpstatesvcrImm1));
1261 // Copy all but the second and third operands of MSRcond_pstatesvcrImm1 (as
1262 // these contain the CopyFromReg for the first argument and the flag to
1263 // indicate whether the callee is streaming or normal).
1264 MIB.add(MO: MI.getOperand(i: 0));
1265 MIB.add(MO: MI.getOperand(i: 1));
1266 for (unsigned i = 4; i < MI.getNumOperands(); ++i)
1267 MIB.add(MO: MI.getOperand(i));
1268
1269 MI.eraseFromParent();
1270 return &EndBB;
1271}
1272
1273bool AArch64ExpandPseudoImpl::expandMultiVecPseudo(
1274 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1275 const TargetRegisterClass &ContiguousClass,
1276 const TargetRegisterClass &StridedClass, unsigned ContiguousOp,
1277 unsigned StridedOpc) {
1278 MachineInstr &MI = *MBBI;
1279 Register Tuple = MI.getOperand(i: 0).getReg();
1280
1281 auto ContiguousRange = ContiguousClass.getRegisters();
1282 auto StridedRange = StridedClass.getRegisters();
1283 unsigned Opc;
1284 if (llvm::is_contained(Range&: ContiguousRange, Element: Tuple.asMCReg())) {
1285 Opc = ContiguousOp;
1286 } else if (llvm::is_contained(Range&: StridedRange, Element: Tuple.asMCReg())) {
1287 Opc = StridedOpc;
1288 } else
1289 llvm_unreachable("Cannot expand Multi-Vector pseudo");
1290
1291 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
1292 .add(MO: MI.getOperand(i: 0))
1293 .add(MO: MI.getOperand(i: 1))
1294 .add(MO: MI.getOperand(i: 2))
1295 .add(MO: MI.getOperand(i: 3));
1296 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1297 MI.eraseFromParent();
1298 return true;
1299}
1300
1301bool AArch64ExpandPseudoImpl::expandCopyIntoTuplePseudo(
1302 MachineInstr &MI, MachineBasicBlock &MBB,
1303 MachineBasicBlock::iterator MBBI) {
1304 Register Src = MI.getOperand(i: 1).getReg();
1305 Register Dest = MI.getOperand(i: 0).getReg();
1306
1307 if (Src != Dest)
1308 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ORR_ZZZ))
1309 .addReg(RegNo: Dest, Flags: RegState::Define)
1310 .addReg(RegNo: Src)
1311 .addReg(RegNo: Src);
1312
1313 MI.eraseFromParent();
1314 return true;
1315}
1316
1317/// If MBBI references a pseudo instruction that should be expanded here,
1318/// do the expansion and return true. Otherwise return false.
1319bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
1320 MachineBasicBlock::iterator MBBI,
1321 MachineBasicBlock::iterator &NextMBBI) {
1322 MachineInstr &MI = *MBBI;
1323 unsigned Opcode = MI.getOpcode();
1324
1325 // Check if we can expand the destructive op
1326 int OrigInstr = AArch64::getSVEPseudoMap(Opcode: MI.getOpcode());
1327 if (OrigInstr != -1) {
1328 auto &Orig = TII->get(Opcode: OrigInstr);
1329 if ((Orig.TSFlags & AArch64::DestructiveInstTypeMask) !=
1330 AArch64::NotDestructive) {
1331 return expand_DestructiveOp(MI, MBB, MBBI);
1332 }
1333 }
1334
1335 switch (Opcode) {
1336 default:
1337 break;
1338
1339 case AArch64::BSPv8i8:
1340 case AArch64::BSPv16i8: {
1341 Register DstReg = MI.getOperand(i: 0).getReg();
1342 if (DstReg == MI.getOperand(i: 3).getReg()) {
1343 // Expand to BIT
1344 auto I = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1345 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BITv8i8
1346 : AArch64::BITv16i8))
1347 .add(MO: MI.getOperand(i: 0))
1348 .add(MO: MI.getOperand(i: 3))
1349 .add(MO: MI.getOperand(i: 2))
1350 .add(MO: MI.getOperand(i: 1));
1351 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1352 } else if (DstReg == MI.getOperand(i: 2).getReg()) {
1353 // Expand to BIF
1354 auto I = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1355 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BIFv8i8
1356 : AArch64::BIFv16i8))
1357 .add(MO: MI.getOperand(i: 0))
1358 .add(MO: MI.getOperand(i: 2))
1359 .add(MO: MI.getOperand(i: 3))
1360 .add(MO: MI.getOperand(i: 1));
1361 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1362 } else {
1363 // Expand to BSL, use additional move if required
1364 const auto &ST = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1365 if (DstReg == MI.getOperand(i: 1).getReg()) {
1366 auto I =
1367 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1368 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1369 : AArch64::BSLv16i8))
1370 .add(MO: MI.getOperand(i: 0))
1371 .add(MO: MI.getOperand(i: 1))
1372 .add(MO: MI.getOperand(i: 2))
1373 .add(MO: MI.getOperand(i: 3));
1374 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1375 } else if (Opcode == AArch64::BSPv16i8 &&
1376 ST.isSVEorStreamingSVEAvailable() &&
1377 (ST.hasSVE2() || ST.hasSME())) {
1378 // Expand BSPv16i8 with MOVPRFX + BSL. Don't expand BSPv8i8 as FPR64
1379 // defs are assumed to zero the upper bits.
1380 const TargetRegisterInfo &TRI = TII->getRegisterInfo();
1381 auto getZReg = [&TRI](const MachineOperand &MO) -> Register {
1382 return TRI.getMatchingSuperReg(Reg: MO.getReg(), SubIdx: AArch64::zsub,
1383 RC: &AArch64::ZPRRegClass);
1384 };
1385 const MachineOperand &Dst = MI.getOperand(i: 0);
1386 const MachineOperand &Mask = MI.getOperand(i: 1);
1387 const MachineOperand &TVal = MI.getOperand(i: 2);
1388 const MachineOperand &FVal = MI.getOperand(i: 3);
1389 Register DOPReg = getZReg(Dst);
1390 auto PRFX =
1391 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MOVPRFX_ZZ))
1392 .addDef(RegNo: DOPReg, Flags: getRenamableRegState(B: Dst.isRenamable()))
1393 .addReg(RegNo: getZReg(TVal),
1394 Flags: getRenamableRegState(B: TVal.isRenamable()) |
1395 getUndefRegState(B: TVal.isUndef()) |
1396 getKillRegState(B: TVal.isKill() &&
1397 TVal.getReg() != Mask.getReg() &&
1398 TVal.getReg() != FVal.getReg()));
1399 auto DOP =
1400 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::BSL_ZZZZ))
1401 .addDef(RegNo: DOPReg, Flags: getDeadRegState(B: Dst.isDead()) |
1402 getRenamableRegState(B: Dst.isRenamable()))
1403 .addReg(RegNo: DOPReg, Flags: RegState::Kill |
1404 getRenamableRegState(B: Dst.isRenamable()))
1405 .addReg(RegNo: getZReg(FVal),
1406 Flags: getRenamableRegState(B: FVal.isRenamable()) |
1407 getUndefRegState(B: FVal.isUndef()) |
1408 getKillRegState(B: FVal.isKill()))
1409 .addReg(RegNo: getZReg(Mask),
1410 Flags: getRenamableRegState(B: Mask.isRenamable()) |
1411 getUndefRegState(B: Mask.isUndef()) |
1412 getKillRegState(B: Mask.isKill()));
1413 transferImpOps(OldMI&: MI, UseMI&: PRFX, DefMI&: DOP);
1414 finalizeBundle(MBB, FirstMI: PRFX->getIterator(), LastMI: MBBI->getIterator());
1415 } else {
1416 RegState RegState =
1417 getRenamableRegState(B: MI.getOperand(i: 1).isRenamable()) |
1418 getUndefRegState(B: MI.getOperand(i: 1).isUndef()) |
1419 getKillRegState(
1420 B: MI.getOperand(i: 1).isKill() &&
1421 MI.getOperand(i: 1).getReg() != MI.getOperand(i: 2).getReg() &&
1422 MI.getOperand(i: 1).getReg() != MI.getOperand(i: 3).getReg());
1423 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1424 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i8
1425 : AArch64::ORRv16i8))
1426 .addReg(RegNo: DstReg,
1427 Flags: RegState::Define |
1428 getRenamableRegState(B: MI.getOperand(i: 0).isRenamable()))
1429 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState)
1430 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState);
1431 auto I2 =
1432 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1433 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1434 : AArch64::BSLv16i8))
1435 .add(MO: MI.getOperand(i: 0))
1436 .addReg(RegNo: DstReg,
1437 Flags: RegState::Kill | getRenamableRegState(
1438 B: MI.getOperand(i: 0).isRenamable()))
1439 .add(MO: MI.getOperand(i: 2))
1440 .add(MO: MI.getOperand(i: 3));
1441 transferImpOps(OldMI&: MI, UseMI&: I2, DefMI&: I2);
1442 }
1443 }
1444 MI.eraseFromParent();
1445 return true;
1446 }
1447
1448 case AArch64::ADDWrr:
1449 case AArch64::SUBWrr:
1450 case AArch64::ADDXrr:
1451 case AArch64::SUBXrr:
1452 case AArch64::ADDSWrr:
1453 case AArch64::SUBSWrr:
1454 case AArch64::ADDSXrr:
1455 case AArch64::SUBSXrr:
1456 case AArch64::ANDWrr:
1457 case AArch64::ANDXrr:
1458 case AArch64::BICWrr:
1459 case AArch64::BICXrr:
1460 case AArch64::ANDSWrr:
1461 case AArch64::ANDSXrr:
1462 case AArch64::BICSWrr:
1463 case AArch64::BICSXrr:
1464 case AArch64::EONWrr:
1465 case AArch64::EONXrr:
1466 case AArch64::EORWrr:
1467 case AArch64::EORXrr:
1468 case AArch64::ORNWrr:
1469 case AArch64::ORNXrr:
1470 case AArch64::ORRWrr:
1471 case AArch64::ORRXrr: {
1472 unsigned Opcode;
1473 switch (MI.getOpcode()) {
1474 default:
1475 return false;
1476 case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;
1477 case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;
1478 case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;
1479 case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;
1480 case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;
1481 case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;
1482 case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;
1483 case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;
1484 case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;
1485 case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;
1486 case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;
1487 case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;
1488 case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;
1489 case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;
1490 case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;
1491 case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;
1492 case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;
1493 case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;
1494 case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;
1495 case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;
1496 case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;
1497 case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;
1498 case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;
1499 case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;
1500 }
1501 MachineFunction &MF = *MBB.getParent();
1502 // Try to create new inst without implicit operands added.
1503 MachineInstr *NewMI = MF.CreateMachineInstr(
1504 MCID: TII->get(Opcode), DL: MI.getDebugLoc(), /*NoImplicit=*/true);
1505 MBB.insert(I: MBBI, MI: NewMI);
1506 MachineInstrBuilder MIB1(MF, NewMI);
1507 MIB1->setPCSections(MF, MD: MI.getPCSections());
1508 MIB1.addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Define)
1509 .add(MO: MI.getOperand(i: 1))
1510 .add(MO: MI.getOperand(i: 2))
1511 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
1512 transferImpOps(OldMI&: MI, UseMI&: MIB1, DefMI&: MIB1);
1513 if (auto DebugNumber = MI.peekDebugInstrNum())
1514 NewMI->setDebugInstrNum(DebugNumber);
1515 MI.eraseFromParent();
1516 return true;
1517 }
1518
1519 case AArch64::LOADgot: {
1520 MachineFunction *MF = MBB.getParent();
1521 Register DstReg = MI.getOperand(i: 0).getReg();
1522 const MachineOperand &MO1 = MI.getOperand(i: 1);
1523 unsigned Flags = MO1.getTargetFlags();
1524
1525 if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {
1526 // Tiny codemodel expand to LDR
1527 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1528 MCID: TII->get(Opcode: AArch64::LDRXl), DestReg: DstReg);
1529
1530 if (MO1.isGlobal()) {
1531 MIB.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0, TargetFlags: Flags);
1532 } else if (MO1.isSymbol()) {
1533 MIB.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags);
1534 } else {
1535 assert(MO1.isCPI() &&
1536 "Only expect globals, externalsymbols, or constant pools");
1537 MIB.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(), TargetFlags: Flags);
1538 }
1539 } else {
1540 // Small codemodel expand into ADRP + LDR.
1541 MachineFunction &MF = *MI.getParent()->getParent();
1542 DebugLoc DL = MI.getDebugLoc();
1543 MachineInstrBuilder MIB1 =
1544 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADRP), DestReg: DstReg);
1545
1546 MachineInstrBuilder MIB2;
1547 if (MF.getSubtarget<AArch64Subtarget>().isTargetILP32()) {
1548 auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
1549 unsigned Reg32 = TRI->getSubReg(Reg: DstReg, Idx: AArch64::sub_32);
1550 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::LDRWui))
1551 .addDef(RegNo: Reg32)
1552 .addReg(RegNo: DstReg, Flags: RegState::Kill)
1553 .addReg(RegNo: DstReg, Flags: RegState::Implicit);
1554 } else {
1555 Register DstReg = MI.getOperand(i: 0).getReg();
1556 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
1557 .add(MO: MI.getOperand(i: 0))
1558 .addUse(RegNo: DstReg, Flags: RegState::Kill);
1559 }
1560
1561 if (MO1.isGlobal()) {
1562 MIB1.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0, TargetFlags: Flags | AArch64II::MO_PAGE);
1563 MIB2.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0,
1564 TargetFlags: Flags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1565 } else if (MO1.isSymbol()) {
1566 MIB1.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags | AArch64II::MO_PAGE);
1567 MIB2.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags |
1568 AArch64II::MO_PAGEOFF |
1569 AArch64II::MO_NC);
1570 } else {
1571 assert(MO1.isCPI() &&
1572 "Only expect globals, externalsymbols, or constant pools");
1573 MIB1.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(),
1574 TargetFlags: Flags | AArch64II::MO_PAGE);
1575 MIB2.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(),
1576 TargetFlags: Flags | AArch64II::MO_PAGEOFF |
1577 AArch64II::MO_NC);
1578 }
1579
1580 // If the LOADgot instruction has a debug-instr-number, annotate the
1581 // LDRWui instruction that it is expanded to with the same
1582 // debug-instr-number to preserve debug information.
1583 if (MI.peekDebugInstrNum() != 0)
1584 MIB2->setDebugInstrNum(MI.peekDebugInstrNum());
1585 transferImpOps(OldMI&: MI, UseMI&: MIB1, DefMI&: MIB2);
1586 }
1587 MI.eraseFromParent();
1588 return true;
1589 }
1590 case AArch64::MOVaddrBA:
1591 case AArch64::MOVaddr:
1592 case AArch64::MOVaddrJT:
1593 case AArch64::MOVaddrCP:
1594 case AArch64::MOVaddrTLS:
1595 case AArch64::MOVaddrEXT: {
1596 MachineFunction &MF = *MI.getParent()->getParent();
1597 Register DstReg = MI.getOperand(i: 0).getReg();
1598 assert(DstReg != AArch64::XZR);
1599
1600 bool IsTargetMachO = MF.getSubtarget<AArch64Subtarget>().isTargetMachO();
1601 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
1602 AArch64_IMM::expandMOVAddr(
1603 Opcode: MI.getOpcode(), TargetFlags: MI.getOperand(i: 1).getTargetFlags(), IsTargetMachO, Insn);
1604
1605 // Compute the constant pool index, if any.
1606 std::optional<unsigned> CPIdx;
1607 if (Opcode == AArch64::MOVaddrBA && IsTargetMachO) {
1608 // blockaddress expressions have to come from a constant pool because the
1609 // largest addend (and hence offset within a function) allowed for ADRP is
1610 // only 8MB.
1611 const BlockAddress *BA = MI.getOperand(i: 1).getBlockAddress();
1612 assert(MI.getOperand(1).getOffset() == 0 && "unexpected offset");
1613 MachineConstantPool *MCP = MF.getConstantPool();
1614 CPIdx = MCP->getConstantPoolIndex(C: BA, Alignment: Align(8));
1615 }
1616
1617 MachineInstrBuilder FirstMIB;
1618 MachineInstrBuilder LastMIB;
1619 for (const auto &I : Insn) {
1620 MachineInstrBuilder MIB;
1621 switch (I.Opcode) {
1622 case AArch64::ADRP:
1623 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADRP),
1624 DestReg: DstReg);
1625 if (CPIdx)
1626 MIB.addConstantPoolIndex(Idx: *CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGE);
1627 else
1628 MIB.add(MO: MI.getOperand(i: 1));
1629 break;
1630 case AArch64::LDRXui:
1631 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::LDRXui),
1632 DestReg: DstReg)
1633 .addUse(RegNo: DstReg)
1634 .addConstantPoolIndex(
1635 Idx: *CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1636 break;
1637 case AArch64::MOVKXi: {
1638 // MO_TAGGED on the page indicates a tagged address. Set the tag now.
1639 // We do so by creating a MOVK that sets bits 48-63 of the register to
1640 // (global address + 0x100000000 - PC) >> 48. This assumes that we're in
1641 // the small code model so we can assume a binary size of <= 4GB, which
1642 // makes the untagged PC relative offset positive. The binary must also
1643 // be loaded into address range [0, 2^48). Both of these properties need
1644 // to be ensured at runtime when using tagged addresses.
1645 auto Tag = MI.getOperand(i: 1);
1646 Tag.setTargetFlags(AArch64II::MO_PREL | AArch64II::MO_G3);
1647 Tag.setOffset(0x100000000);
1648 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MOVKXi),
1649 DestReg: DstReg)
1650 .addReg(RegNo: DstReg)
1651 .add(MO: Tag)
1652 .addImm(Val: 48);
1653 break;
1654 }
1655 case AArch64::ADDXri:
1656 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
1657 .add(MO: MI.getOperand(i: 0))
1658 .addReg(RegNo: DstReg)
1659 .add(MO: MI.getOperand(i: 2))
1660 .addImm(Val: 0);
1661 break;
1662 default:
1663 llvm_unreachable("unexpected opcode in MOVaddr expansion");
1664 }
1665
1666 if (!FirstMIB.getInstr())
1667 FirstMIB = MIB;
1668 LastMIB = MIB;
1669 }
1670
1671 transferImpOps(OldMI&: MI, UseMI&: FirstMIB, DefMI&: LastMIB);
1672 MI.eraseFromParent();
1673 return true;
1674 }
1675 case AArch64::ADDlowTLS:
1676 // Produce a plain ADD
1677 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
1678 .add(MO: MI.getOperand(i: 0))
1679 .add(MO: MI.getOperand(i: 1))
1680 .add(MO: MI.getOperand(i: 2))
1681 .addImm(Val: 0);
1682 MI.eraseFromParent();
1683 return true;
1684
1685 case AArch64::MOVbaseTLS: {
1686 Register DstReg = MI.getOperand(i: 0).getReg();
1687 auto SysReg = AArch64SysReg::TPIDR_EL0;
1688 MachineFunction *MF = MBB.getParent();
1689 if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())
1690 SysReg = AArch64SysReg::TPIDR_EL3;
1691 else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())
1692 SysReg = AArch64SysReg::TPIDR_EL2;
1693 else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())
1694 SysReg = AArch64SysReg::TPIDR_EL1;
1695 else if (MF->getSubtarget<AArch64Subtarget>().useROEL0ForTP())
1696 SysReg = AArch64SysReg::TPIDRRO_EL0;
1697 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MRS), DestReg: DstReg)
1698 .addImm(Val: SysReg);
1699 MI.eraseFromParent();
1700 return true;
1701 }
1702
1703 case AArch64::MOVi32imm:
1704 return expandMOVImm(MBB, MBBI, BitSize: 32);
1705 case AArch64::MOVi64imm:
1706 return expandMOVImm(MBB, MBBI, BitSize: 64);
1707 case AArch64::RET_ReallyLR: {
1708 // Hiding the LR use with RET_ReallyLR may lead to extra kills in the
1709 // function and missing live-ins. We are fine in practice because callee
1710 // saved register handling ensures the register value is restored before
1711 // RET, but we need the undef flag here to appease the MachineVerifier
1712 // liveness checks.
1713 MachineInstrBuilder MIB =
1714 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::RET))
1715 .addReg(RegNo: AArch64::LR, Flags: RegState::Undef);
1716 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1717 MI.eraseFromParent();
1718 return true;
1719 }
1720 case AArch64::CMP_SWAP_8:
1721 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRB, StlrOp: AArch64::STLXRB,
1722 CmpOp: AArch64::SUBSWrx,
1723 ExtendImm: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTB, Imm: 0),
1724 ZeroReg: AArch64::WZR, NextMBBI);
1725 case AArch64::CMP_SWAP_16:
1726 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRH, StlrOp: AArch64::STLXRH,
1727 CmpOp: AArch64::SUBSWrx,
1728 ExtendImm: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTH, Imm: 0),
1729 ZeroReg: AArch64::WZR, NextMBBI);
1730 case AArch64::CMP_SWAP_32:
1731 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRW, StlrOp: AArch64::STLXRW,
1732 CmpOp: AArch64::SUBSWrs,
1733 ExtendImm: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0),
1734 ZeroReg: AArch64::WZR, NextMBBI);
1735 case AArch64::CMP_SWAP_64:
1736 return expandCMP_SWAP(MBB, MBBI,
1737 LdarOp: AArch64::LDAXRX, StlrOp: AArch64::STLXRX, CmpOp: AArch64::SUBSXrs,
1738 ExtendImm: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0),
1739 ZeroReg: AArch64::XZR, NextMBBI);
1740 case AArch64::CMP_SWAP_128:
1741 case AArch64::CMP_SWAP_128_RELEASE:
1742 case AArch64::CMP_SWAP_128_ACQUIRE:
1743 case AArch64::CMP_SWAP_128_MONOTONIC:
1744 return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
1745
1746 case AArch64::AESMCrrTied:
1747 case AArch64::AESIMCrrTied: {
1748 MachineInstrBuilder MIB =
1749 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1750 MCID: TII->get(Opcode: Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
1751 AArch64::AESIMCrr))
1752 .add(MO: MI.getOperand(i: 0))
1753 .add(MO: MI.getOperand(i: 1));
1754 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1755 MI.eraseFromParent();
1756 return true;
1757 }
1758 case AArch64::IRGstack: {
1759 MachineFunction &MF = *MBB.getParent();
1760 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1761 const AArch64FrameLowering *TFI =
1762 MF.getSubtarget<AArch64Subtarget>().getFrameLowering();
1763
1764 // IRG does not allow immediate offset. getTaggedBasePointerOffset should
1765 // almost always point to SP-after-prologue; if not, emit a longer
1766 // instruction sequence.
1767 int BaseOffset = -AFI->getTaggedBasePointerOffset();
1768 Register FrameReg;
1769 StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
1770 MF, ObjectOffset: BaseOffset, isFixed: false /*isFixed*/, StackID: TargetStackID::Default /*StackID*/,
1771 FrameReg,
1772 /*PreferFP=*/false,
1773 /*ForSimm=*/true);
1774 Register SrcReg = FrameReg;
1775 if (FrameRegOffset) {
1776 // Use output register as temporary.
1777 SrcReg = MI.getOperand(i: 0).getReg();
1778 emitFrameOffset(MBB, MBBI: &MI, DL: MI.getDebugLoc(), DestReg: SrcReg, SrcReg: FrameReg,
1779 Offset: FrameRegOffset, TII);
1780 }
1781 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::IRG))
1782 .add(MO: MI.getOperand(i: 0))
1783 .addUse(RegNo: SrcReg)
1784 .add(MO: MI.getOperand(i: 2));
1785 MI.eraseFromParent();
1786 return true;
1787 }
1788 case AArch64::TAGPstack: {
1789 int64_t Offset = MI.getOperand(i: 2).getImm();
1790 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1791 MCID: TII->get(Opcode: Offset >= 0 ? AArch64::ADDG : AArch64::SUBG))
1792 .add(MO: MI.getOperand(i: 0))
1793 .add(MO: MI.getOperand(i: 1))
1794 .addImm(Val: std::abs(i: Offset))
1795 .add(MO: MI.getOperand(i: 4));
1796 MI.eraseFromParent();
1797 return true;
1798 }
1799 case AArch64::STGloop_wback:
1800 case AArch64::STZGloop_wback:
1801 return expandSetTagLoop(MBB, MBBI, NextMBBI);
1802 case AArch64::STGloop:
1803 case AArch64::STZGloop:
1804 report_fatal_error(
1805 reason: "Non-writeback variants of STGloop / STZGloop should not "
1806 "survive past PrologEpilogInserter.");
1807 case AArch64::STR_ZZZZXI:
1808 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
1809 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 4);
1810 case AArch64::STR_ZZZXI:
1811 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 3);
1812 case AArch64::STR_ZZXI:
1813 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
1814 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 2);
1815 case AArch64::STR_PPXI:
1816 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_PXI, N: 2);
1817 case AArch64::LDR_ZZZZXI:
1818 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
1819 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 4);
1820 case AArch64::LDR_ZZZXI:
1821 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 3);
1822 case AArch64::LDR_ZZXI:
1823 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
1824 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 2);
1825 case AArch64::LDR_PPXI:
1826 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_PXI, N: 2);
1827 case AArch64::BLR_RVMARKER:
1828 case AArch64::BLRA_RVMARKER:
1829 return expandCALL_RVMARKER(MBB, MBBI);
1830 case AArch64::BLR_BTI:
1831 return expandCALL_BTI(MBB, MBBI);
1832 case AArch64::StoreSwiftAsyncContext:
1833 return expandStoreSwiftAsyncContext(MBB, MBBI);
1834 case AArch64::RestoreZAPseudo:
1835 case AArch64::CommitZASavePseudo:
1836 case AArch64::MSRpstatePseudo: {
1837 auto *NewMBB = [&] {
1838 switch (Opcode) {
1839 case AArch64::RestoreZAPseudo:
1840 return expandRestoreZASave(MBB, MBBI);
1841 case AArch64::CommitZASavePseudo:
1842 return expandCommitZASave(MBB, MBBI);
1843 case AArch64::MSRpstatePseudo:
1844 return expandCondSMToggle(MBB, MBBI);
1845 default:
1846 llvm_unreachable("Unexpected conditional pseudo!");
1847 }
1848 }();
1849 if (NewMBB != &MBB)
1850 NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated.
1851 return true;
1852 }
1853 case AArch64::InOutZAUsePseudo:
1854 case AArch64::RequiresZASavePseudo:
1855 case AArch64::RequiresZT0SavePseudo:
1856 case AArch64::SMEStateAllocPseudo:
1857 case AArch64::COALESCER_BARRIER_FPR16:
1858 case AArch64::COALESCER_BARRIER_FPR32:
1859 case AArch64::COALESCER_BARRIER_FPR64:
1860 case AArch64::COALESCER_BARRIER_FPR128:
1861 MI.eraseFromParent();
1862 return true;
1863 case AArch64::LD1B_2Z_IMM_PSEUDO:
1864 return expandMultiVecPseudo(
1865 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1866 ContiguousOp: AArch64::LD1B_2Z_IMM, StridedOpc: AArch64::LD1B_2Z_STRIDED_IMM);
1867 case AArch64::LD1H_2Z_IMM_PSEUDO:
1868 return expandMultiVecPseudo(
1869 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1870 ContiguousOp: AArch64::LD1H_2Z_IMM, StridedOpc: AArch64::LD1H_2Z_STRIDED_IMM);
1871 case AArch64::LD1W_2Z_IMM_PSEUDO:
1872 return expandMultiVecPseudo(
1873 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1874 ContiguousOp: AArch64::LD1W_2Z_IMM, StridedOpc: AArch64::LD1W_2Z_STRIDED_IMM);
1875 case AArch64::LD1D_2Z_IMM_PSEUDO:
1876 return expandMultiVecPseudo(
1877 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1878 ContiguousOp: AArch64::LD1D_2Z_IMM, StridedOpc: AArch64::LD1D_2Z_STRIDED_IMM);
1879 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
1880 return expandMultiVecPseudo(
1881 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1882 ContiguousOp: AArch64::LDNT1B_2Z_IMM, StridedOpc: AArch64::LDNT1B_2Z_STRIDED_IMM);
1883 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
1884 return expandMultiVecPseudo(
1885 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1886 ContiguousOp: AArch64::LDNT1H_2Z_IMM, StridedOpc: AArch64::LDNT1H_2Z_STRIDED_IMM);
1887 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
1888 return expandMultiVecPseudo(
1889 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1890 ContiguousOp: AArch64::LDNT1W_2Z_IMM, StridedOpc: AArch64::LDNT1W_2Z_STRIDED_IMM);
1891 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
1892 return expandMultiVecPseudo(
1893 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1894 ContiguousOp: AArch64::LDNT1D_2Z_IMM, StridedOpc: AArch64::LDNT1D_2Z_STRIDED_IMM);
1895 case AArch64::ST1B_2Z_IMM_PSEUDO:
1896 return expandMultiVecPseudo(
1897 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1898 ContiguousOp: AArch64::ST1B_2Z_IMM, StridedOpc: AArch64::ST1B_2Z_STRIDED_IMM);
1899 case AArch64::ST1H_2Z_IMM_PSEUDO:
1900 return expandMultiVecPseudo(
1901 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1902 ContiguousOp: AArch64::ST1H_2Z_IMM, StridedOpc: AArch64::ST1H_2Z_STRIDED_IMM);
1903 case AArch64::ST1W_2Z_IMM_PSEUDO:
1904 return expandMultiVecPseudo(
1905 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1906 ContiguousOp: AArch64::ST1W_2Z_IMM, StridedOpc: AArch64::ST1W_2Z_STRIDED_IMM);
1907 case AArch64::ST1D_2Z_IMM_PSEUDO:
1908 return expandMultiVecPseudo(
1909 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1910 ContiguousOp: AArch64::ST1D_2Z_IMM, StridedOpc: AArch64::ST1D_2Z_STRIDED_IMM);
1911 case AArch64::STNT1B_2Z_IMM_PSEUDO:
1912 return expandMultiVecPseudo(
1913 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1914 ContiguousOp: AArch64::STNT1B_2Z_IMM, StridedOpc: AArch64::STNT1B_2Z_STRIDED_IMM);
1915 case AArch64::STNT1H_2Z_IMM_PSEUDO:
1916 return expandMultiVecPseudo(
1917 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1918 ContiguousOp: AArch64::STNT1H_2Z_IMM, StridedOpc: AArch64::STNT1H_2Z_STRIDED_IMM);
1919 case AArch64::STNT1W_2Z_IMM_PSEUDO:
1920 return expandMultiVecPseudo(
1921 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1922 ContiguousOp: AArch64::STNT1W_2Z_IMM, StridedOpc: AArch64::STNT1W_2Z_STRIDED_IMM);
1923 case AArch64::STNT1D_2Z_IMM_PSEUDO:
1924 return expandMultiVecPseudo(
1925 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1926 ContiguousOp: AArch64::STNT1D_2Z_IMM, StridedOpc: AArch64::STNT1D_2Z_STRIDED_IMM);
1927 case AArch64::LD1B_2Z_PSEUDO:
1928 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1929 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1B_2Z,
1930 StridedOpc: AArch64::LD1B_2Z_STRIDED);
1931 case AArch64::LD1H_2Z_PSEUDO:
1932 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1933 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1H_2Z,
1934 StridedOpc: AArch64::LD1H_2Z_STRIDED);
1935 case AArch64::LD1W_2Z_PSEUDO:
1936 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1937 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1W_2Z,
1938 StridedOpc: AArch64::LD1W_2Z_STRIDED);
1939 case AArch64::LD1D_2Z_PSEUDO:
1940 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1941 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1D_2Z,
1942 StridedOpc: AArch64::LD1D_2Z_STRIDED);
1943 case AArch64::LDNT1B_2Z_PSEUDO:
1944 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1945 StridedClass: AArch64::ZPR2StridedRegClass,
1946 ContiguousOp: AArch64::LDNT1B_2Z, StridedOpc: AArch64::LDNT1B_2Z_STRIDED);
1947 case AArch64::LDNT1H_2Z_PSEUDO:
1948 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1949 StridedClass: AArch64::ZPR2StridedRegClass,
1950 ContiguousOp: AArch64::LDNT1H_2Z, StridedOpc: AArch64::LDNT1H_2Z_STRIDED);
1951 case AArch64::LDNT1W_2Z_PSEUDO:
1952 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1953 StridedClass: AArch64::ZPR2StridedRegClass,
1954 ContiguousOp: AArch64::LDNT1W_2Z, StridedOpc: AArch64::LDNT1W_2Z_STRIDED);
1955 case AArch64::LDNT1D_2Z_PSEUDO:
1956 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1957 StridedClass: AArch64::ZPR2StridedRegClass,
1958 ContiguousOp: AArch64::LDNT1D_2Z, StridedOpc: AArch64::LDNT1D_2Z_STRIDED);
1959 case AArch64::LD1B_4Z_IMM_PSEUDO:
1960 return expandMultiVecPseudo(
1961 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1962 ContiguousOp: AArch64::LD1B_4Z_IMM, StridedOpc: AArch64::LD1B_4Z_STRIDED_IMM);
1963 case AArch64::LD1H_4Z_IMM_PSEUDO:
1964 return expandMultiVecPseudo(
1965 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1966 ContiguousOp: AArch64::LD1H_4Z_IMM, StridedOpc: AArch64::LD1H_4Z_STRIDED_IMM);
1967 case AArch64::LD1W_4Z_IMM_PSEUDO:
1968 return expandMultiVecPseudo(
1969 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1970 ContiguousOp: AArch64::LD1W_4Z_IMM, StridedOpc: AArch64::LD1W_4Z_STRIDED_IMM);
1971 case AArch64::LD1D_4Z_IMM_PSEUDO:
1972 return expandMultiVecPseudo(
1973 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1974 ContiguousOp: AArch64::LD1D_4Z_IMM, StridedOpc: AArch64::LD1D_4Z_STRIDED_IMM);
1975 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
1976 return expandMultiVecPseudo(
1977 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1978 ContiguousOp: AArch64::LDNT1B_4Z_IMM, StridedOpc: AArch64::LDNT1B_4Z_STRIDED_IMM);
1979 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
1980 return expandMultiVecPseudo(
1981 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1982 ContiguousOp: AArch64::LDNT1H_4Z_IMM, StridedOpc: AArch64::LDNT1H_4Z_STRIDED_IMM);
1983 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
1984 return expandMultiVecPseudo(
1985 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1986 ContiguousOp: AArch64::LDNT1W_4Z_IMM, StridedOpc: AArch64::LDNT1W_4Z_STRIDED_IMM);
1987 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
1988 return expandMultiVecPseudo(
1989 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1990 ContiguousOp: AArch64::LDNT1D_4Z_IMM, StridedOpc: AArch64::LDNT1D_4Z_STRIDED_IMM);
1991 case AArch64::ST1B_4Z_IMM_PSEUDO:
1992 return expandMultiVecPseudo(
1993 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1994 ContiguousOp: AArch64::ST1B_4Z_IMM, StridedOpc: AArch64::ST1B_4Z_STRIDED_IMM);
1995 case AArch64::ST1H_4Z_IMM_PSEUDO:
1996 return expandMultiVecPseudo(
1997 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1998 ContiguousOp: AArch64::ST1H_4Z_IMM, StridedOpc: AArch64::ST1H_4Z_STRIDED_IMM);
1999 case AArch64::ST1W_4Z_IMM_PSEUDO:
2000 return expandMultiVecPseudo(
2001 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
2002 ContiguousOp: AArch64::ST1W_4Z_IMM, StridedOpc: AArch64::ST1W_4Z_STRIDED_IMM);
2003 case AArch64::ST1D_4Z_IMM_PSEUDO:
2004 return expandMultiVecPseudo(
2005 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
2006 ContiguousOp: AArch64::ST1D_4Z_IMM, StridedOpc: AArch64::ST1D_4Z_STRIDED_IMM);
2007 case AArch64::STNT1B_4Z_IMM_PSEUDO:
2008 return expandMultiVecPseudo(
2009 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
2010 ContiguousOp: AArch64::STNT1B_4Z_IMM, StridedOpc: AArch64::STNT1B_4Z_STRIDED_IMM);
2011 case AArch64::STNT1H_4Z_IMM_PSEUDO:
2012 return expandMultiVecPseudo(
2013 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
2014 ContiguousOp: AArch64::STNT1H_4Z_IMM, StridedOpc: AArch64::STNT1H_4Z_STRIDED_IMM);
2015 case AArch64::STNT1W_4Z_IMM_PSEUDO:
2016 return expandMultiVecPseudo(
2017 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
2018 ContiguousOp: AArch64::STNT1W_4Z_IMM, StridedOpc: AArch64::STNT1W_4Z_STRIDED_IMM);
2019 case AArch64::STNT1D_4Z_IMM_PSEUDO:
2020 return expandMultiVecPseudo(
2021 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
2022 ContiguousOp: AArch64::STNT1D_4Z_IMM, StridedOpc: AArch64::STNT1D_4Z_STRIDED_IMM);
2023 case AArch64::LD1B_4Z_PSEUDO:
2024 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2025 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1B_4Z,
2026 StridedOpc: AArch64::LD1B_4Z_STRIDED);
2027 case AArch64::LD1H_4Z_PSEUDO:
2028 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2029 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1H_4Z,
2030 StridedOpc: AArch64::LD1H_4Z_STRIDED);
2031 case AArch64::LD1W_4Z_PSEUDO:
2032 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2033 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1W_4Z,
2034 StridedOpc: AArch64::LD1W_4Z_STRIDED);
2035 case AArch64::LD1D_4Z_PSEUDO:
2036 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2037 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1D_4Z,
2038 StridedOpc: AArch64::LD1D_4Z_STRIDED);
2039 case AArch64::LDNT1B_4Z_PSEUDO:
2040 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2041 StridedClass: AArch64::ZPR4StridedRegClass,
2042 ContiguousOp: AArch64::LDNT1B_4Z, StridedOpc: AArch64::LDNT1B_4Z_STRIDED);
2043 case AArch64::LDNT1H_4Z_PSEUDO:
2044 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2045 StridedClass: AArch64::ZPR4StridedRegClass,
2046 ContiguousOp: AArch64::LDNT1H_4Z, StridedOpc: AArch64::LDNT1H_4Z_STRIDED);
2047 case AArch64::LDNT1W_4Z_PSEUDO:
2048 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2049 StridedClass: AArch64::ZPR4StridedRegClass,
2050 ContiguousOp: AArch64::LDNT1W_4Z, StridedOpc: AArch64::LDNT1W_4Z_STRIDED);
2051 case AArch64::LDNT1D_4Z_PSEUDO:
2052 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2053 StridedClass: AArch64::ZPR4StridedRegClass,
2054 ContiguousOp: AArch64::LDNT1D_4Z, StridedOpc: AArch64::LDNT1D_4Z_STRIDED);
2055 case AArch64::COPY_INTO_TRANSPOSED_TUPLE:
2056 return expandCopyIntoTuplePseudo(MI, MBB, MBBI);
2057 case AArch64::EON_ZZZ:
2058 case AArch64::NAND_ZZZ:
2059 case AArch64::NOR_ZZZ:
2060 return expandSVEBitwisePseudo(MI, MBB, MBBI);
2061 }
2062 return false;
2063}
2064
2065/// Iterate over the instructions in basic block MBB and expand any
2066/// pseudo instructions. Return true if anything was modified.
2067bool AArch64ExpandPseudoImpl::expandMBB(MachineBasicBlock &MBB) {
2068 bool Modified = false;
2069
2070 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
2071 while (MBBI != E) {
2072 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
2073 if (MBBI->isPseudo())
2074 Modified |= expandMI(MBB, MBBI, NextMBBI&: NMBBI);
2075 MBBI = NMBBI;
2076 }
2077
2078 return Modified;
2079}
2080
2081bool AArch64ExpandPseudoImpl::run(MachineFunction &MF) {
2082 TII = MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
2083
2084 bool Modified = false;
2085 for (auto &MBB : MF)
2086 Modified |= expandMBB(MBB);
2087 return Modified;
2088}
2089
2090bool AArch64ExpandPseudoLegacy::runOnMachineFunction(MachineFunction &MF) {
2091 return AArch64ExpandPseudoImpl().run(MF);
2092}
2093
2094/// Returns an instance of the pseudo instruction expansion pass.
2095FunctionPass *llvm::createAArch64ExpandPseudoLegacyPass() {
2096 return new AArch64ExpandPseudoLegacy();
2097}
2098
2099PreservedAnalyses
2100AArch64ExpandPseudoPass::run(MachineFunction &MF,
2101 MachineFunctionAnalysisManager &MFAM) {
2102 const bool Changed = AArch64ExpandPseudoImpl().run(MF);
2103 if (!Changed)
2104 return PreservedAnalyses::all();
2105 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
2106 PA.preserveSet<CFGAnalyses>();
2107 return PA;
2108}
2109