1//===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling and other late optimizations. This
11// pass should be run after register allocation but before the post-regalloc
12// scheduling pass.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AArch64ExpandImm.h"
17#include "AArch64InstrInfo.h"
18#include "AArch64MachineFunctionInfo.h"
19#include "AArch64Subtarget.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/CodeGen/LivePhysRegs.h"
23#include "llvm/CodeGen/MachineBasicBlock.h"
24#include "llvm/CodeGen/MachineConstantPool.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include "llvm/CodeGen/MachineInstr.h"
28#include "llvm/CodeGen/MachineInstrBuilder.h"
29#include "llvm/CodeGen/MachineOperand.h"
30#include "llvm/CodeGen/TargetSubtargetInfo.h"
31#include "llvm/IR/DebugLoc.h"
32#include "llvm/MC/MCInstrDesc.h"
33#include "llvm/Pass.h"
34#include "llvm/Support/CodeGen.h"
35#include "llvm/Target/TargetMachine.h"
36#include "llvm/TargetParser/Triple.h"
37#include <cassert>
38#include <cstdint>
39#include <iterator>
40
41using namespace llvm;
42
43#define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
44
45namespace {
46
47class AArch64ExpandPseudoImpl {
48public:
49 const AArch64InstrInfo *TII;
50
51 bool run(MachineFunction &MF);
52
53private:
54 bool expandMBB(MachineBasicBlock &MBB);
55 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
56 MachineBasicBlock::iterator &NextMBBI);
57 bool expandMultiVecPseudo(MachineBasicBlock &MBB,
58 MachineBasicBlock::iterator MBBI,
59 const TargetRegisterClass &ContiguousClass,
60 const TargetRegisterClass &StridedClass,
61 unsigned ContiguousOpc, unsigned StridedOpc);
62 bool expandCopyIntoTuplePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
63 MachineBasicBlock::iterator MBBI);
64 bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
65 unsigned BitSize);
66
67 bool expand_DestructiveOp(MachineInstr &MI, MachineBasicBlock &MBB,
68 MachineBasicBlock::iterator MBBI);
69 bool expandSVEBitwisePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
70 MachineBasicBlock::iterator MBBI);
71 bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
72 unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
73 unsigned ExtendImm, unsigned ZeroReg,
74 MachineBasicBlock::iterator &NextMBBI);
75 bool expandCMP_SWAP_128(MachineBasicBlock &MBB,
76 MachineBasicBlock::iterator MBBI,
77 MachineBasicBlock::iterator &NextMBBI);
78 bool expandSetTagLoop(MachineBasicBlock &MBB,
79 MachineBasicBlock::iterator MBBI,
80 MachineBasicBlock::iterator &NextMBBI);
81 bool expandSVESpillFill(MachineBasicBlock &MBB,
82 MachineBasicBlock::iterator MBBI, unsigned Opc,
83 unsigned N);
84 bool expandCALL_RVMARKER(MachineBasicBlock &MBB,
85 MachineBasicBlock::iterator MBBI);
86 bool expandCALL_BTI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
87 bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,
88 MachineBasicBlock::iterator MBBI);
89 struct ConditionalBlocks {
90 MachineBasicBlock &CondBB;
91 MachineBasicBlock &EndBB;
92 };
93 ConditionalBlocks expandConditionalPseudo(MachineBasicBlock &MBB,
94 MachineBasicBlock::iterator MBBI,
95 DebugLoc DL,
96 MachineInstrBuilder &Branch);
97 MachineBasicBlock *expandRestoreZASave(MachineBasicBlock &MBB,
98 MachineBasicBlock::iterator MBBI);
99 MachineBasicBlock *expandCommitZASave(MachineBasicBlock &MBB,
100 MachineBasicBlock::iterator MBBI);
101 MachineBasicBlock *expandCondSMToggle(MachineBasicBlock &MBB,
102 MachineBasicBlock::iterator MBBI);
103};
104
105class AArch64ExpandPseudoLegacy : public MachineFunctionPass {
106public:
107 static char ID;
108
109 AArch64ExpandPseudoLegacy() : MachineFunctionPass(ID) {}
110
111 bool runOnMachineFunction(MachineFunction &MF) override;
112
113 StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }
114};
115
116} // end anonymous namespace
117
118char AArch64ExpandPseudoLegacy::ID = 0;
119
120INITIALIZE_PASS(AArch64ExpandPseudoLegacy, "aarch64-expand-pseudo",
121 AARCH64_EXPAND_PSEUDO_NAME, false, false)
122
123/// Transfer implicit operands on the pseudo instruction to the
124/// instructions created from the expansion.
125static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,
126 MachineInstrBuilder &DefMI) {
127 const MCInstrDesc &Desc = OldMI.getDesc();
128 for (const MachineOperand &MO :
129 llvm::drop_begin(RangeOrContainer: OldMI.operands(), N: Desc.getNumOperands())) {
130 assert(MO.isReg() && MO.getReg());
131 if (MO.isUse())
132 UseMI.add(MO);
133 else
134 DefMI.add(MO);
135 }
136}
137
138/// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more
139/// real move-immediate instructions to synthesize the immediate.
140bool AArch64ExpandPseudoImpl::expandMOVImm(MachineBasicBlock &MBB,
141 MachineBasicBlock::iterator MBBI,
142 unsigned BitSize) {
143 MachineInstr &MI = *MBBI;
144 Register DstReg = MI.getOperand(i: 0).getReg();
145 RegState RenamableState =
146 getRenamableRegState(B: MI.getOperand(i: 0).isRenamable());
147 uint64_t Imm = MI.getOperand(i: 1).getImm();
148
149 if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
150 // Useless def, and we don't want to risk creating an invalid ORR (which
151 // would really write to sp).
152 MI.eraseFromParent();
153 return true;
154 }
155
156 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
157 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn);
158 assert(Insn.size() != 0);
159
160 SmallVector<MachineInstrBuilder, 4> MIBS;
161 for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {
162 bool LastItem = std::next(x: I) == E;
163 switch (I->Opcode)
164 {
165 default: llvm_unreachable("unhandled!"); break;
166
167 case AArch64::ORRWri:
168 case AArch64::ORRXri:
169 case AArch64::ANDXri:
170 case AArch64::EORXri:
171 if (I->Op1 == 0) {
172 MIBS.push_back(Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
173 .add(MO: MI.getOperand(i: 0))
174 .addReg(RegNo: BitSize == 32 ? AArch64::WZR : AArch64::XZR)
175 .addImm(Val: *I->Op2));
176 } else {
177 Register DstReg = MI.getOperand(i: 0).getReg();
178 bool DstIsDead = MI.getOperand(i: 0).isDead();
179 MIBS.push_back(
180 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
181 .addReg(RegNo: DstReg, Flags: RegState::Define |
182 getDeadRegState(B: DstIsDead && LastItem) |
183 RenamableState)
184 .addReg(RegNo: DstReg)
185 .addImm(Val: *I->Op2));
186 }
187 break;
188 case AArch64::EONXrs:
189 case AArch64::EORXrs:
190 case AArch64::ORRWrs:
191 case AArch64::ORRXrs: {
192 Register DstReg = MI.getOperand(i: 0).getReg();
193 bool DstIsDead = MI.getOperand(i: 0).isDead();
194 MIBS.push_back(
195 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
196 .addReg(RegNo: DstReg, Flags: RegState::Define |
197 getDeadRegState(B: DstIsDead && LastItem) |
198 RenamableState)
199 .addReg(RegNo: DstReg)
200 .addReg(RegNo: DstReg)
201 .addImm(Val: *I->Op2));
202 } break;
203 case AArch64::MOVNWi:
204 case AArch64::MOVNXi:
205 case AArch64::MOVZWi:
206 case AArch64::MOVZXi: {
207 bool DstIsDead = MI.getOperand(i: 0).isDead();
208 MIBS.push_back(
209 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
210 .addReg(RegNo: DstReg, Flags: RegState::Define |
211 getDeadRegState(B: DstIsDead && LastItem) |
212 RenamableState)
213 .addImm(Val: *I->Op1)
214 .addImm(Val: *I->Op2));
215 } break;
216 case AArch64::MOVKWi:
217 case AArch64::MOVKXi: {
218 Register DstReg = MI.getOperand(i: 0).getReg();
219 bool DstIsDead = MI.getOperand(i: 0).isDead();
220 MIBS.push_back(
221 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
222 .addReg(RegNo: DstReg, Flags: RegState::Define |
223 getDeadRegState(B: DstIsDead && LastItem) |
224 RenamableState)
225 .addReg(RegNo: DstReg)
226 .addImm(Val: *I->Op1)
227 .addImm(Val: *I->Op2));
228 } break;
229 }
230 }
231 transferImpOps(OldMI&: MI, UseMI&: MIBS.front(), DefMI&: MIBS.back());
232 MI.eraseFromParent();
233 return true;
234}
235
236bool AArch64ExpandPseudoImpl::expandCMP_SWAP(
237 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp,
238 unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
239 MachineBasicBlock::iterator &NextMBBI) {
240 MachineInstr &MI = *MBBI;
241 MIMetadata MIMD(MI);
242 const MachineOperand &Dest = MI.getOperand(i: 0);
243 Register StatusReg = MI.getOperand(i: 1).getReg();
244 bool StatusDead = MI.getOperand(i: 1).isDead();
245 // Duplicating undef operands into 2 instructions does not guarantee the same
246 // value on both; However undef should be replaced by xzr anyway.
247 assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
248 Register AddrReg = MI.getOperand(i: 2).getReg();
249 Register DesiredReg = MI.getOperand(i: 3).getReg();
250 Register NewReg = MI.getOperand(i: 4).getReg();
251
252 MachineFunction *MF = MBB.getParent();
253 auto LoadCmpBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
254 auto StoreBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
255 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
256
257 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoadCmpBB);
258 MF->insert(MBBI: ++LoadCmpBB->getIterator(), MBB: StoreBB);
259 MF->insert(MBBI: ++StoreBB->getIterator(), MBB: DoneBB);
260
261 // .Lloadcmp:
262 // mov wStatus, 0
263 // ldaxr xDest, [xAddr]
264 // cmp xDest, xDesired
265 // b.ne .Ldone
266 if (!StatusDead)
267 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::MOVZWi), DestReg: StatusReg)
268 .addImm(Val: 0).addImm(Val: 0);
269 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: LdarOp), DestReg: Dest.getReg())
270 .addReg(RegNo: AddrReg);
271 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: CmpOp), DestReg: ZeroReg)
272 .addReg(RegNo: Dest.getReg(), Flags: getKillRegState(B: Dest.isDead()))
273 .addReg(RegNo: DesiredReg)
274 .addImm(Val: ExtendImm);
275 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::Bcc))
276 .addImm(Val: AArch64CC::NE)
277 .addMBB(MBB: DoneBB)
278 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Kill);
279 LoadCmpBB->addSuccessor(Succ: DoneBB);
280 LoadCmpBB->addSuccessor(Succ: StoreBB);
281
282 // .Lstore:
283 // stlxr wStatus, xNew, [xAddr]
284 // cbnz wStatus, .Lloadcmp
285 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: StlrOp), DestReg: StatusReg)
286 .addReg(RegNo: NewReg)
287 .addReg(RegNo: AddrReg);
288 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
289 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
290 .addMBB(MBB: LoadCmpBB);
291 StoreBB->addSuccessor(Succ: LoadCmpBB);
292 StoreBB->addSuccessor(Succ: DoneBB);
293
294 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
295 DoneBB->transferSuccessors(FromMBB: &MBB);
296
297 MBB.addSuccessor(Succ: LoadCmpBB);
298
299 NextMBBI = MBB.end();
300 MI.eraseFromParent();
301
302 // Recompute livein lists.
303 LivePhysRegs LiveRegs;
304 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
305 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
306 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
307 // Do an extra pass around the loop to get loop carried registers right.
308 StoreBB->clearLiveIns();
309 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
310 LoadCmpBB->clearLiveIns();
311 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
312
313 return true;
314}
315
316bool AArch64ExpandPseudoImpl::expandCMP_SWAP_128(
317 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
318 MachineBasicBlock::iterator &NextMBBI) {
319 MachineInstr &MI = *MBBI;
320 MIMetadata MIMD(MI);
321 MachineOperand &DestLo = MI.getOperand(i: 0);
322 MachineOperand &DestHi = MI.getOperand(i: 1);
323 Register StatusReg = MI.getOperand(i: 2).getReg();
324 bool StatusDead = MI.getOperand(i: 2).isDead();
325 // Duplicating undef operands into 2 instructions does not guarantee the same
326 // value on both; However undef should be replaced by xzr anyway.
327 assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
328 Register AddrReg = MI.getOperand(i: 3).getReg();
329 Register DesiredLoReg = MI.getOperand(i: 4).getReg();
330 Register DesiredHiReg = MI.getOperand(i: 5).getReg();
331 Register NewLoReg = MI.getOperand(i: 6).getReg();
332 Register NewHiReg = MI.getOperand(i: 7).getReg();
333
334 auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
335 bool LittleEndian = STI.isLittleEndian();
336 MachineOperand &Dest0 = LittleEndian ? DestLo : DestHi;
337 MachineOperand &Dest1 = LittleEndian ? DestHi : DestLo;
338 Register New0Reg = LittleEndian ? NewLoReg : NewHiReg;
339 Register New1Reg = LittleEndian ? NewHiReg : NewLoReg;
340
341 unsigned LdxpOp, StxpOp;
342
343 switch (MI.getOpcode()) {
344 case AArch64::CMP_SWAP_128_MONOTONIC:
345 LdxpOp = AArch64::LDXPX;
346 StxpOp = AArch64::STXPX;
347 break;
348 case AArch64::CMP_SWAP_128_RELEASE:
349 LdxpOp = AArch64::LDXPX;
350 StxpOp = AArch64::STLXPX;
351 break;
352 case AArch64::CMP_SWAP_128_ACQUIRE:
353 LdxpOp = AArch64::LDAXPX;
354 StxpOp = AArch64::STXPX;
355 break;
356 case AArch64::CMP_SWAP_128:
357 LdxpOp = AArch64::LDAXPX;
358 StxpOp = AArch64::STLXPX;
359 break;
360 default:
361 llvm_unreachable("Unexpected opcode");
362 }
363
364 MachineFunction *MF = MBB.getParent();
365 auto LoadCmpBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
366 auto StoreBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
367 auto FailBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
368 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
369
370 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoadCmpBB);
371 MF->insert(MBBI: ++LoadCmpBB->getIterator(), MBB: StoreBB);
372 MF->insert(MBBI: ++StoreBB->getIterator(), MBB: FailBB);
373 MF->insert(MBBI: ++FailBB->getIterator(), MBB: DoneBB);
374
375 // .Lloadcmp:
376 // ldaxp xDestLo, xDestHi, [xAddr]
377 // cmp xDestLo, xDesiredLo
378 // sbcs xDestHi, xDesiredHi
379 // b.ne .Ldone
380 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: LdxpOp))
381 .addReg(RegNo: Dest0.getReg(), Flags: RegState::Define)
382 .addReg(RegNo: Dest1.getReg(), Flags: RegState::Define)
383 .addReg(RegNo: AddrReg);
384 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::SUBSXrs), DestReg: AArch64::XZR)
385 .addReg(RegNo: DestLo.getReg(), Flags: getKillRegState(B: DestLo.isDead()))
386 .addReg(RegNo: DesiredLoReg)
387 .addImm(Val: 0);
388 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CSINCWr), DestReg: StatusReg)
389 .addUse(RegNo: AArch64::WZR)
390 .addUse(RegNo: AArch64::WZR)
391 .addImm(Val: AArch64CC::EQ);
392 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::SUBSXrs), DestReg: AArch64::XZR)
393 .addReg(RegNo: DestHi.getReg(), Flags: getKillRegState(B: DestHi.isDead()))
394 .addReg(RegNo: DesiredHiReg)
395 .addImm(Val: 0);
396 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CSINCWr), DestReg: StatusReg)
397 .addUse(RegNo: StatusReg, Flags: RegState::Kill)
398 .addUse(RegNo: StatusReg, Flags: RegState::Kill)
399 .addImm(Val: AArch64CC::EQ);
400 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
401 .addUse(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
402 .addMBB(MBB: FailBB);
403 LoadCmpBB->addSuccessor(Succ: FailBB);
404 LoadCmpBB->addSuccessor(Succ: StoreBB);
405
406 // .Lstore:
407 // stlxp wStatus, xNewLo, xNewHi, [xAddr]
408 // cbnz wStatus, .Lloadcmp
409 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: StxpOp), DestReg: StatusReg)
410 .addReg(RegNo: New0Reg)
411 .addReg(RegNo: New1Reg)
412 .addReg(RegNo: AddrReg);
413 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
414 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
415 .addMBB(MBB: LoadCmpBB);
416 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::B)).addMBB(MBB: DoneBB);
417 StoreBB->addSuccessor(Succ: LoadCmpBB);
418 StoreBB->addSuccessor(Succ: DoneBB);
419
420 // .Lfail:
421 // stlxp wStatus, xDestLo, xDestHi, [xAddr]
422 // cbnz wStatus, .Lloadcmp
423 BuildMI(BB: FailBB, MIMD, MCID: TII->get(Opcode: StxpOp), DestReg: StatusReg)
424 .addReg(RegNo: Dest0.getReg())
425 .addReg(RegNo: Dest1.getReg())
426 .addReg(RegNo: AddrReg);
427 BuildMI(BB: FailBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
428 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
429 .addMBB(MBB: LoadCmpBB);
430 FailBB->addSuccessor(Succ: LoadCmpBB);
431 FailBB->addSuccessor(Succ: DoneBB);
432
433 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
434 DoneBB->transferSuccessors(FromMBB: &MBB);
435
436 MBB.addSuccessor(Succ: LoadCmpBB);
437
438 NextMBBI = MBB.end();
439 MI.eraseFromParent();
440
441 // Recompute liveness bottom up.
442 LivePhysRegs LiveRegs;
443 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
444 computeAndAddLiveIns(LiveRegs, MBB&: *FailBB);
445 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
446 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
447
448 // Do an extra pass in the loop to get the loop carried dependencies right.
449 FailBB->clearLiveIns();
450 computeAndAddLiveIns(LiveRegs, MBB&: *FailBB);
451 StoreBB->clearLiveIns();
452 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
453 LoadCmpBB->clearLiveIns();
454 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
455
456 return true;
457}
458
459/// \brief Expand Pseudos to Instructions with destructive operands.
460///
461/// This mechanism uses MOVPRFX instructions for zeroing the false lanes
462/// or for fixing relaxed register allocation conditions to comply with
463/// the instructions register constraints. The latter case may be cheaper
464/// than setting the register constraints in the register allocator,
465/// since that will insert regular MOV instructions rather than MOVPRFX.
466///
467/// Example (after register allocation):
468///
469/// FSUB_ZPZZ_ZERO_B Z0, Pg, Z1, Z0
470///
471/// * The Pseudo FSUB_ZPZZ_ZERO_B maps to FSUB_ZPmZ_B.
472/// * We cannot map directly to FSUB_ZPmZ_B because the register
473/// constraints of the instruction are not met.
474/// * Also the _ZERO specifies the false lanes need to be zeroed.
475///
476/// We first try to see if the destructive operand == result operand,
477/// if not, we try to swap the operands, e.g.
478///
479/// FSUB_ZPmZ_B Z0, Pg/m, Z0, Z1
480///
481/// But because FSUB_ZPmZ is not commutative, this is semantically
482/// different, so we need a reverse instruction:
483///
484/// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1
485///
486/// Then we implement the zeroing of the false lanes of Z0 by adding
487/// a zeroing MOVPRFX instruction:
488///
489/// MOVPRFX_ZPzZ_B Z0, Pg/z, Z0
490/// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1
491///
492/// Note that this can only be done for _ZERO or _UNDEF variants where
493/// we can guarantee the false lanes to be zeroed (by implementing this)
494/// or that they are undef (don't care / not used), otherwise the
495/// swapping of operands is illegal because the operation is not
496/// (or cannot be emulated to be) fully commutative.
497bool AArch64ExpandPseudoImpl::expand_DestructiveOp(
498 MachineInstr &MI, MachineBasicBlock &MBB,
499 MachineBasicBlock::iterator MBBI) {
500 unsigned Opcode = AArch64::getSVEPseudoMap(Opcode: MI.getOpcode());
501 uint64_t DType = TII->get(Opcode).TSFlags & AArch64::DestructiveInstTypeMask;
502 uint64_t FalseLanes = MI.getDesc().TSFlags & AArch64::FalseLanesMask;
503 bool FalseZero = FalseLanes == AArch64::FalseLanesZero;
504 Register DstReg = MI.getOperand(i: 0).getReg();
505 bool DstIsDead = MI.getOperand(i: 0).isDead();
506 bool UseRev = false;
507 unsigned PredIdx, DOPIdx, SrcIdx, Src2Idx;
508
509 switch (DType) {
510 case AArch64::DestructiveBinaryComm:
511 case AArch64::DestructiveBinaryCommWithRev:
512 if (DstReg == MI.getOperand(i: 3).getReg()) {
513 // FSUB Zd, Pg, Zs1, Zd ==> FSUBR Zd, Pg/m, Zd, Zs1
514 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 3, args: 2);
515 UseRev = true;
516 break;
517 }
518 [[fallthrough]];
519 case AArch64::DestructiveBinary:
520 case AArch64::DestructiveBinaryImm:
521 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 2, args: 3);
522 break;
523 case AArch64::DestructiveUnaryPassthru:
524 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 2, args: 3, args: 3);
525 break;
526 case AArch64::DestructiveTernaryCommWithRev:
527 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 2, args: 3, args: 4);
528 if (DstReg == MI.getOperand(i: 3).getReg()) {
529 // FMLA Zd, Pg, Za, Zd, Zm ==> FMAD Zdn, Pg, Zm, Za
530 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 3, args: 4, args: 2);
531 UseRev = true;
532 } else if (DstReg == MI.getOperand(i: 4).getReg()) {
533 // FMLA Zd, Pg, Za, Zm, Zd ==> FMAD Zdn, Pg, Zm, Za
534 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 4, args: 3, args: 2);
535 UseRev = true;
536 }
537 break;
538 case AArch64::Destructive2xRegImmUnpred:
539 // EXT_ZZI_CONSTRUCTIVE Zd, Zs, Imm
540 // ==> MOVPRFX Zd Zs; EXT_ZZI Zd, Zd, Zs, Imm
541 std::tie(args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 1, args: 2);
542 break;
543 case AArch64::DestructiveBinaryImmUnpred:
544 std::tie(args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 2);
545 break;
546 case AArch64::DestructiveBinaryShImmUnpred:
547 std::tie(args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 2, args: 3);
548 break;
549 default:
550 llvm_unreachable("Unsupported Destructive Operand type");
551 }
552
553 // MOVPRFX can only be used if the destination operand
554 // is the destructive operand, not as any other operand,
555 // so the Destructive Operand must be unique.
556 bool DOPRegIsUnique = false;
557 switch (DType) {
558 case AArch64::DestructiveBinary:
559 DOPRegIsUnique = DstReg != MI.getOperand(i: SrcIdx).getReg();
560 break;
561 case AArch64::DestructiveBinaryComm:
562 case AArch64::DestructiveBinaryCommWithRev:
563 DOPRegIsUnique =
564 DstReg != MI.getOperand(i: DOPIdx).getReg() ||
565 MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: SrcIdx).getReg();
566 break;
567 case AArch64::DestructiveUnaryPassthru:
568 case AArch64::DestructiveBinaryImm:
569 case AArch64::DestructiveBinaryImmUnpred:
570 case AArch64::DestructiveBinaryShImmUnpred:
571 case AArch64::Destructive2xRegImmUnpred:
572 DOPRegIsUnique = true;
573 break;
574 case AArch64::DestructiveTernaryCommWithRev:
575 DOPRegIsUnique =
576 DstReg != MI.getOperand(i: DOPIdx).getReg() ||
577 (MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: SrcIdx).getReg() &&
578 MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: Src2Idx).getReg());
579 break;
580 }
581
582 // Resolve the reverse opcode
583 if (UseRev) {
584 int NewOpcode;
585 // e.g. DIV -> DIVR
586 if ((NewOpcode = AArch64::getSVERevInstr(Opcode)) != -1)
587 Opcode = NewOpcode;
588 // e.g. DIVR -> DIV
589 else if ((NewOpcode = AArch64::getSVENonRevInstr(Opcode)) != -1)
590 Opcode = NewOpcode;
591 }
592
593 // Get the right MOVPRFX
594 uint64_t ElementSize = TII->getElementSizeForOpcode(Opc: Opcode);
595 unsigned MovPrfx, LSLZero, MovPrfxZero;
596 switch (ElementSize) {
597 case AArch64::ElementSizeNone:
598 case AArch64::ElementSizeB:
599 MovPrfx = AArch64::MOVPRFX_ZZ;
600 LSLZero = AArch64::LSL_ZPmI_B;
601 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_B;
602 break;
603 case AArch64::ElementSizeH:
604 MovPrfx = AArch64::MOVPRFX_ZZ;
605 LSLZero = AArch64::LSL_ZPmI_H;
606 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_H;
607 break;
608 case AArch64::ElementSizeS:
609 MovPrfx = AArch64::MOVPRFX_ZZ;
610 LSLZero = AArch64::LSL_ZPmI_S;
611 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_S;
612 break;
613 case AArch64::ElementSizeD:
614 MovPrfx = AArch64::MOVPRFX_ZZ;
615 LSLZero = AArch64::LSL_ZPmI_D;
616 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_D;
617 break;
618 default:
619 llvm_unreachable("Unsupported ElementSize");
620 }
621
622 // Preserve undef state until DOP's reg is defined.
623 RegState DOPRegState = getUndefRegState(B: MI.getOperand(i: DOPIdx).isUndef());
624
625 //
626 // Create the destructive operation (if required)
627 //
628 MachineInstrBuilder PRFX, DOP;
629 if (FalseZero) {
630 // If we cannot prefix the requested instruction we'll instead emit a
631 // prefixed_zeroing_mov for DestructiveBinary.
632 assert((DOPRegIsUnique || DType == AArch64::DestructiveBinary ||
633 DType == AArch64::DestructiveBinaryComm ||
634 DType == AArch64::DestructiveBinaryCommWithRev) &&
635 "The destructive operand should be unique");
636 assert(ElementSize != AArch64::ElementSizeNone &&
637 "This instruction is unpredicated");
638
639 // Merge source operand into destination register
640 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: MovPrfxZero))
641 .addReg(RegNo: DstReg, Flags: RegState::Define)
642 .addReg(RegNo: MI.getOperand(i: PredIdx).getReg())
643 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState);
644
645 // After the movprfx, the destructive operand is same as Dst
646 DOPIdx = 0;
647 DOPRegState = {};
648
649 // Create the additional LSL to zero the lanes when the DstReg is not
650 // unique. Zeros the lanes in z0 that aren't active in p0 with sequence
651 // movprfx z0.b, p0/z, z0.b; lsl z0.b, p0/m, z0.b, #0;
652 if ((DType == AArch64::DestructiveBinary ||
653 DType == AArch64::DestructiveBinaryComm ||
654 DType == AArch64::DestructiveBinaryCommWithRev) &&
655 !DOPRegIsUnique) {
656 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LSLZero))
657 .addReg(RegNo: DstReg, Flags: RegState::Define)
658 .add(MO: MI.getOperand(i: PredIdx))
659 .addReg(RegNo: DstReg)
660 .addImm(Val: 0);
661 }
662 } else if (DstReg != MI.getOperand(i: DOPIdx).getReg()) {
663 assert(DOPRegIsUnique && "The destructive operand should be unique");
664 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: MovPrfx))
665 .addReg(RegNo: DstReg, Flags: RegState::Define)
666 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState);
667 DOPIdx = 0;
668 DOPRegState = {};
669 }
670
671 //
672 // Create the destructive operation
673 //
674 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode))
675 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
676 DOPRegState = DOPRegState | RegState::Kill;
677
678 switch (DType) {
679 case AArch64::DestructiveUnaryPassthru:
680 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
681 .add(MO: MI.getOperand(i: PredIdx))
682 .add(MO: MI.getOperand(i: SrcIdx));
683 break;
684 case AArch64::DestructiveBinary:
685 case AArch64::DestructiveBinaryImm:
686 case AArch64::DestructiveBinaryComm:
687 case AArch64::DestructiveBinaryCommWithRev:
688 DOP.add(MO: MI.getOperand(i: PredIdx))
689 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
690 .add(MO: MI.getOperand(i: SrcIdx));
691 break;
692 case AArch64::DestructiveTernaryCommWithRev:
693 DOP.add(MO: MI.getOperand(i: PredIdx))
694 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
695 .add(MO: MI.getOperand(i: SrcIdx))
696 .add(MO: MI.getOperand(i: Src2Idx));
697 break;
698 case AArch64::DestructiveBinaryImmUnpred:
699 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
700 .add(MO: MI.getOperand(i: SrcIdx));
701 break;
702 case AArch64::DestructiveBinaryShImmUnpred:
703 case AArch64::Destructive2xRegImmUnpred:
704 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
705 .add(MO: MI.getOperand(i: SrcIdx))
706 .add(MO: MI.getOperand(i: Src2Idx));
707 break;
708 }
709
710 if (PRFX) {
711 transferImpOps(OldMI&: MI, UseMI&: PRFX, DefMI&: DOP);
712 finalizeBundle(MBB, FirstMI: PRFX->getIterator(), LastMI: MBBI->getIterator());
713 } else
714 transferImpOps(OldMI&: MI, UseMI&: DOP, DefMI&: DOP);
715
716 MI.eraseFromParent();
717 return true;
718}
719
720bool AArch64ExpandPseudoImpl::expandSVEBitwisePseudo(
721 MachineInstr &MI, MachineBasicBlock &MBB,
722 MachineBasicBlock::iterator MBBI) {
723 MachineInstrBuilder PRFX, DOP;
724 const unsigned Opcode = MI.getOpcode();
725 const MachineOperand &Op0 = MI.getOperand(i: 0);
726 const MachineOperand *Op1 = &MI.getOperand(i: 1);
727 const MachineOperand *Op2 = &MI.getOperand(i: 2);
728 const Register DOPReg = Op0.getReg();
729
730 if (DOPReg == Op2->getReg()) {
731 // Commute the operands to allow destroying the second source.
732 std::swap(a&: Op1, b&: Op2);
733 } else if (DOPReg != Op1->getReg()) {
734 // If not in destructive form, emit a MOVPRFX. The input should only be
735 // killed if unused by the subsequent instruction.
736 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MOVPRFX_ZZ))
737 .addDef(RegNo: DOPReg, Flags: getRenamableRegState(B: Op0.isRenamable()))
738 .addReg(RegNo: Op1->getReg(),
739 Flags: getRenamableRegState(B: Op1->isRenamable()) |
740 getUndefRegState(B: Op1->isUndef()) |
741 getKillRegState(B: Op1->isKill() &&
742 Opcode == AArch64::NAND_ZZZ));
743 }
744
745 assert((DOPReg == Op1->getReg() || PRFX) && "invalid expansion");
746
747 const RegState DOPRegState = getRenamableRegState(B: Op0.isRenamable()) |
748 getUndefRegState(B: !PRFX && Op1->isUndef()) |
749 RegState::Kill;
750
751 switch (Opcode) {
752 default:
753 llvm_unreachable("unhandled opcode");
754 case AArch64::EON_ZZZ:
755 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::BSL2N_ZZZZ))
756 .add(MO: Op0)
757 .addReg(RegNo: DOPReg, Flags: DOPRegState)
758 .add(MO: *Op1)
759 .add(MO: *Op2);
760 break;
761 case AArch64::NAND_ZZZ:
762 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::NBSL_ZZZZ))
763 .add(MO: Op0)
764 .addReg(RegNo: DOPReg, Flags: DOPRegState)
765 .add(MO: *Op2)
766 .add(MO: *Op2);
767 break;
768 case AArch64::NOR_ZZZ:
769 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::NBSL_ZZZZ))
770 .add(MO: Op0)
771 .addReg(RegNo: DOPReg, Flags: DOPRegState)
772 .add(MO: *Op2)
773 .add(MO: *Op1);
774 break;
775 }
776
777 if (PRFX) {
778 transferImpOps(OldMI&: MI, UseMI&: PRFX, DefMI&: DOP);
779 finalizeBundle(MBB, FirstMI: PRFX->getIterator(), LastMI: MBBI->getIterator());
780 } else {
781 transferImpOps(OldMI&: MI, UseMI&: DOP, DefMI&: DOP);
782 }
783
784 MI.eraseFromParent();
785 return true;
786}
787
788bool AArch64ExpandPseudoImpl::expandSetTagLoop(
789 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
790 MachineBasicBlock::iterator &NextMBBI) {
791 MachineInstr &MI = *MBBI;
792 DebugLoc DL = MI.getDebugLoc();
793 Register SizeReg = MI.getOperand(i: 0).getReg();
794 Register AddressReg = MI.getOperand(i: 1).getReg();
795
796 MachineFunction *MF = MBB.getParent();
797
798 bool ZeroData = MI.getOpcode() == AArch64::STZGloop_wback;
799 const unsigned OpCode1 =
800 ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex;
801 const unsigned OpCode2 =
802 ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
803
804 unsigned Size = MI.getOperand(i: 2).getImm();
805 assert(Size > 0 && Size % 16 == 0);
806 if (Size % (16 * 2) != 0) {
807 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: OpCode1), DestReg: AddressReg)
808 .addReg(RegNo: AddressReg)
809 .addReg(RegNo: AddressReg)
810 .addImm(Val: 1);
811 Size -= 16;
812 }
813 MachineBasicBlock::iterator I =
814 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVi64imm), DestReg: SizeReg)
815 .addImm(Val: Size);
816 expandMOVImm(MBB, MBBI: I, BitSize: 64);
817
818 auto LoopBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
819 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
820
821 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopBB);
822 MF->insert(MBBI: ++LoopBB->getIterator(), MBB: DoneBB);
823
824 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: OpCode2))
825 .addDef(RegNo: AddressReg)
826 .addReg(RegNo: AddressReg)
827 .addReg(RegNo: AddressReg)
828 .addImm(Val: 2)
829 .cloneMemRefs(OtherMI: MI)
830 .setMIFlags(MI.getFlags());
831 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXri))
832 .addDef(RegNo: SizeReg)
833 .addReg(RegNo: SizeReg)
834 .addImm(Val: 16 * 2)
835 .addImm(Val: 0);
836 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
837 .addImm(Val: AArch64CC::NE)
838 .addMBB(MBB: LoopBB)
839 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Kill);
840
841 LoopBB->addSuccessor(Succ: LoopBB);
842 LoopBB->addSuccessor(Succ: DoneBB);
843
844 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
845 DoneBB->transferSuccessors(FromMBB: &MBB);
846
847 MBB.addSuccessor(Succ: LoopBB);
848
849 NextMBBI = MBB.end();
850 MI.eraseFromParent();
851 // Recompute liveness bottom up.
852 LivePhysRegs LiveRegs;
853 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
854 computeAndAddLiveIns(LiveRegs, MBB&: *LoopBB);
855 // Do an extra pass in the loop to get the loop carried dependencies right.
856 // FIXME: is this necessary?
857 LoopBB->clearLiveIns();
858 computeAndAddLiveIns(LiveRegs, MBB&: *LoopBB);
859 DoneBB->clearLiveIns();
860 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
861
862 return true;
863}
864
865bool AArch64ExpandPseudoImpl::expandSVESpillFill(
866 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned Opc,
867 unsigned N) {
868 assert((Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI ||
869 Opc == AArch64::LDR_PXI || Opc == AArch64::STR_PXI) &&
870 "Unexpected opcode");
871 RegState RState =
872 getDefRegState(B: Opc == AArch64::LDR_ZXI || Opc == AArch64::LDR_PXI);
873 unsigned sub0 = (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI)
874 ? AArch64::zsub0
875 : AArch64::psub0;
876 const TargetRegisterInfo *TRI =
877 MBB.getParent()->getSubtarget().getRegisterInfo();
878 MachineInstr &MI = *MBBI;
879 for (unsigned Offset = 0; Offset < N; ++Offset) {
880 int ImmOffset = MI.getOperand(i: 2).getImm() + Offset;
881 bool Kill = (Offset + 1 == N) ? MI.getOperand(i: 1).isKill() : false;
882 assert(ImmOffset >= -256 && ImmOffset < 256 &&
883 "Immediate spill offset out of range");
884 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
885 .addReg(RegNo: TRI->getSubReg(Reg: MI.getOperand(i: 0).getReg(), Idx: sub0 + Offset),
886 Flags: RState)
887 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: getKillRegState(B: Kill))
888 .addImm(Val: ImmOffset);
889 }
890 MI.eraseFromParent();
891 return true;
892}
893
894// Create a call with the passed opcode and explicit operands, copying over all
895// the implicit operands from *MBBI, starting at the regmask.
896static MachineInstr *createCallWithOps(MachineBasicBlock &MBB,
897 MachineBasicBlock::iterator MBBI,
898 const AArch64InstrInfo *TII,
899 unsigned Opcode,
900 ArrayRef<MachineOperand> ExplicitOps,
901 unsigned RegMaskStartIdx) {
902 // Build the MI, with explicit operands first (including the call target).
903 MachineInstr *Call = BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode))
904 .add(MOs: ExplicitOps)
905 .getInstr();
906
907 // Register arguments are added during ISel, but cannot be added as explicit
908 // operands of the branch as it expects to be B <target> which is only one
909 // operand. Instead they are implicit operands used by the branch.
910 while (!MBBI->getOperand(i: RegMaskStartIdx).isRegMask()) {
911 const MachineOperand &MOP = MBBI->getOperand(i: RegMaskStartIdx);
912 assert(MOP.isReg() && "can only add register operands");
913 Call->addOperand(Op: MachineOperand::CreateReg(
914 Reg: MOP.getReg(), /*Def=*/isDef: false, /*Implicit=*/isImp: true, /*isKill=*/false,
915 /*isDead=*/false, /*isUndef=*/MOP.isUndef()));
916 RegMaskStartIdx++;
917 }
918 for (const MachineOperand &MO :
919 llvm::drop_begin(RangeOrContainer: MBBI->operands(), N: RegMaskStartIdx))
920 Call->addOperand(Op: MO);
921
922 return Call;
923}
924
925// Create a call to CallTarget, copying over all the operands from *MBBI,
926// starting at the regmask.
927static MachineInstr *createCall(MachineBasicBlock &MBB,
928 MachineBasicBlock::iterator MBBI,
929 const AArch64InstrInfo *TII,
930 MachineOperand &CallTarget,
931 unsigned RegMaskStartIdx) {
932 unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
933
934 assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
935 "invalid operand for regular call");
936 return createCallWithOps(MBB, MBBI, TII, Opcode: Opc, ExplicitOps: CallTarget, RegMaskStartIdx);
937}
938
939bool AArch64ExpandPseudoImpl::expandCALL_RVMARKER(
940 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
941 // Expand CALL_RVMARKER pseudo to:
942 // - a branch to the call target, followed by
943 // - the special `mov x29, x29` marker, if necessary, and
944 // - another branch, to the runtime function
945 // Mark the sequence as bundle, to avoid passes moving other code in between.
946 MachineInstr &MI = *MBBI;
947 MachineOperand &RVTarget = MI.getOperand(i: 0);
948 bool DoEmitMarker = MI.getOperand(i: 1).getImm();
949 assert(RVTarget.isGlobal() && "invalid operand for attached call");
950
951 MachineInstr *OriginalCall = nullptr;
952
953 if (MI.getOpcode() == AArch64::BLRA_RVMARKER) {
954 // ptrauth call.
955 const MachineOperand &CallTarget = MI.getOperand(i: 2);
956 const MachineOperand &Key = MI.getOperand(i: 3);
957 const MachineOperand &IntDisc = MI.getOperand(i: 4);
958 const MachineOperand &AddrDisc = MI.getOperand(i: 5);
959
960 assert((Key.getImm() == AArch64PACKey::IA ||
961 Key.getImm() == AArch64PACKey::IB) &&
962 "Invalid auth call key");
963
964 MachineOperand Ops[] = {CallTarget, Key, IntDisc, AddrDisc};
965
966 OriginalCall = createCallWithOps(MBB, MBBI, TII, Opcode: AArch64::BLRA, ExplicitOps: Ops,
967 /*RegMaskStartIdx=*/6);
968 } else {
969 assert(MI.getOpcode() == AArch64::BLR_RVMARKER && "unknown rvmarker MI");
970 OriginalCall = createCall(MBB, MBBI, TII, CallTarget&: MI.getOperand(i: 2),
971 // Regmask starts after the RV and call targets.
972 /*RegMaskStartIdx=*/3);
973 }
974
975 if (DoEmitMarker)
976 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ORRXrs))
977 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
978 .addReg(RegNo: AArch64::XZR)
979 .addReg(RegNo: AArch64::FP)
980 .addImm(Val: 0);
981
982 auto *RVCall = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::BL))
983 .add(MO: RVTarget)
984 .getInstr();
985
986 if (MI.shouldUpdateAdditionalCallInfo())
987 MBB.getParent()->moveAdditionalCallInfo(Old: &MI, New: OriginalCall);
988
989 MI.eraseFromParent();
990 finalizeBundle(MBB, FirstMI: OriginalCall->getIterator(),
991 LastMI: std::next(x: RVCall->getIterator()));
992 return true;
993}
994
995bool AArch64ExpandPseudoImpl::expandCALL_BTI(MachineBasicBlock &MBB,
996 MachineBasicBlock::iterator MBBI) {
997 // Expand CALL_BTI pseudo to:
998 // - a branch to the call target
999 // - a BTI instruction
1000 // Mark the sequence as a bundle, to avoid passes moving other code in
1001 // between.
1002 MachineInstr &MI = *MBBI;
1003 MachineInstr *Call = createCall(MBB, MBBI, TII, CallTarget&: MI.getOperand(i: 0),
1004 // Regmask starts after the call target.
1005 /*RegMaskStartIdx=*/1);
1006
1007 Call->setCFIType(MF&: *MBB.getParent(), Type: MI.getCFIType());
1008
1009 MachineInstr *BTI =
1010 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::HINT))
1011 // BTI J so that setjmp can to BR to this.
1012 .addImm(Val: 36)
1013 .getInstr();
1014
1015 if (MI.shouldUpdateAdditionalCallInfo())
1016 MBB.getParent()->moveAdditionalCallInfo(Old: &MI, New: Call);
1017
1018 MI.eraseFromParent();
1019 finalizeBundle(MBB, FirstMI: Call->getIterator(), LastMI: std::next(x: BTI->getIterator()));
1020 return true;
1021}
1022
1023bool AArch64ExpandPseudoImpl::expandStoreSwiftAsyncContext(
1024 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
1025 Register CtxReg = MBBI->getOperand(i: 0).getReg();
1026 Register BaseReg = MBBI->getOperand(i: 1).getReg();
1027 int Offset = MBBI->getOperand(i: 2).getImm();
1028 DebugLoc DL(MBBI->getDebugLoc());
1029 auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1030
1031 if (STI.getTargetTriple().getArchName() != "arm64e") {
1032 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
1033 .addUse(RegNo: CtxReg)
1034 .addUse(RegNo: BaseReg)
1035 .addImm(Val: Offset / 8)
1036 .setMIFlag(MachineInstr::FrameSetup);
1037 MBBI->eraseFromParent();
1038 return true;
1039 }
1040
1041 // We need to sign the context in an address-discriminated way. 0xc31a is a
1042 // fixed random value, chosen as part of the ABI.
1043 // add x16, xBase, #Offset
1044 // movk x16, #0xc31a, lsl #48
1045 // mov x17, x22/xzr
1046 // pacdb x17, x16
1047 // str x17, [xBase, #Offset]
1048 unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
1049 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: AArch64::X16)
1050 .addUse(RegNo: BaseReg)
1051 .addImm(Val: abs(x: Offset))
1052 .addImm(Val: 0)
1053 .setMIFlag(MachineInstr::FrameSetup);
1054 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVKXi), DestReg: AArch64::X16)
1055 .addUse(RegNo: AArch64::X16)
1056 .addImm(Val: 0xc31a)
1057 .addImm(Val: 48)
1058 .setMIFlag(MachineInstr::FrameSetup);
1059 // We're not allowed to clobber X22 (and couldn't clobber XZR if we tried), so
1060 // move it somewhere before signing.
1061 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: AArch64::X17)
1062 .addUse(RegNo: AArch64::XZR)
1063 .addUse(RegNo: CtxReg)
1064 .addImm(Val: 0)
1065 .setMIFlag(MachineInstr::FrameSetup);
1066 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACDB), DestReg: AArch64::X17)
1067 .addUse(RegNo: AArch64::X17)
1068 .addUse(RegNo: AArch64::X16)
1069 .setMIFlag(MachineInstr::FrameSetup);
1070 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
1071 .addUse(RegNo: AArch64::X17)
1072 .addUse(RegNo: BaseReg)
1073 .addImm(Val: Offset / 8)
1074 .setMIFlag(MachineInstr::FrameSetup);
1075
1076 MBBI->eraseFromParent();
1077 return true;
1078}
1079
1080AArch64ExpandPseudoImpl::ConditionalBlocks
1081AArch64ExpandPseudoImpl::expandConditionalPseudo(
1082 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL,
1083 MachineInstrBuilder &Branch) {
1084 assert((std::next(MBBI) != MBB.end() ||
1085 MBB.successors().begin() != MBB.successors().end()) &&
1086 "Unexpected unreachable in block");
1087
1088 // Split MBB and create two new blocks:
1089 // - MBB now contains all instructions before the conditional pseudo.
1090 // - CondBB contains the conditional pseudo instruction only.
1091 // - EndBB contains all instructions after the conditional pseudo.
1092 MachineInstr &PrevMI = *std::prev(x: MBBI);
1093 MachineBasicBlock *CondBB = MBB.splitAt(SplitInst&: PrevMI, /*UpdateLiveIns*/ true);
1094 MachineBasicBlock *EndBB =
1095 std::next(x: MBBI) == CondBB->end()
1096 ? *CondBB->successors().begin()
1097 : CondBB->splitAt(SplitInst&: *MBBI, /*UpdateLiveIns*/ true);
1098
1099 // Add the SMBB label to the branch instruction & create a branch to EndBB.
1100 Branch.addMBB(MBB: CondBB);
1101 BuildMI(BB: &MBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
1102 .addMBB(MBB: EndBB);
1103 MBB.addSuccessor(Succ: EndBB);
1104
1105 // Create branch from CondBB to EndBB. Users of this helper should insert new
1106 // instructions at CondBB.back() -- i.e. before the branch.
1107 BuildMI(BB: CondBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::B)).addMBB(MBB: EndBB);
1108 return {.CondBB: *CondBB, .EndBB: *EndBB};
1109}
1110
1111MachineBasicBlock *
1112AArch64ExpandPseudoImpl::expandRestoreZASave(MachineBasicBlock &MBB,
1113 MachineBasicBlock::iterator MBBI) {
1114 MachineInstr &MI = *MBBI;
1115 DebugLoc DL = MI.getDebugLoc();
1116
1117 // Compare TPIDR2_EL0 against 0. Restore ZA if TPIDR2_EL0 is zero.
1118 MachineInstrBuilder Branch =
1119 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::CBZX)).add(MO: MI.getOperand(i: 0));
1120
1121 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);
1122 // Replace the pseudo with a call (BL).
1123 MachineInstrBuilder MIB =
1124 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::BL));
1125 // Copy operands (mainly the regmask) from the pseudo.
1126 for (unsigned I = 2; I < MI.getNumOperands(); ++I)
1127 MIB.add(MO: MI.getOperand(i: I));
1128 // Mark the TPIDR2 block pointer (X0) as an implicit use.
1129 MIB.addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState::Implicit);
1130
1131 MI.eraseFromParent();
1132 return &EndBB;
1133}
1134
1135static constexpr unsigned ZERO_ALL_ZA_MASK = 0b11111111;
1136
1137MachineBasicBlock *
1138AArch64ExpandPseudoImpl::expandCommitZASave(MachineBasicBlock &MBB,
1139 MachineBasicBlock::iterator MBBI) {
1140 MachineInstr &MI = *MBBI;
1141 DebugLoc DL = MI.getDebugLoc();
1142 [[maybe_unused]] auto *RI = MBB.getParent()->getSubtarget().getRegisterInfo();
1143
1144 // Compare TPIDR2_EL0 against 0. Commit ZA if TPIDR2_EL0 is non-zero.
1145 MachineInstrBuilder Branch =
1146 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::CBNZX)).add(MO: MI.getOperand(i: 0));
1147
1148 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);
1149 // Replace the pseudo with a call (BL).
1150 MachineInstrBuilder MIB =
1151 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::BL));
1152 // Copy operands (mainly the regmask) from the pseudo.
1153 for (unsigned I = 3; I < MI.getNumOperands(); ++I)
1154 MIB.add(MO: MI.getOperand(i: I));
1155 // Clear TPIDR2_EL0.
1156 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::MSR))
1157 .addImm(Val: AArch64SysReg::TPIDR2_EL0)
1158 .addReg(RegNo: AArch64::XZR);
1159 bool ZeroZA = MI.getOperand(i: 1).getImm() != 0;
1160 bool ZeroZT0 = MI.getOperand(i: 2).getImm() != 0;
1161 if (ZeroZA) {
1162 assert(MI.definesRegister(AArch64::ZAB0, RI) && "should define ZA!");
1163 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ZERO_M))
1164 .addImm(Val: ZERO_ALL_ZA_MASK)
1165 .addDef(RegNo: AArch64::ZAB0, Flags: RegState::ImplicitDefine);
1166 }
1167 if (ZeroZT0) {
1168 assert(MI.definesRegister(AArch64::ZT0, RI) && "should define ZT0!");
1169 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ZERO_T))
1170 .addDef(RegNo: AArch64::ZT0);
1171 }
1172
1173 MI.eraseFromParent();
1174 return &EndBB;
1175}
1176
1177MachineBasicBlock *
1178AArch64ExpandPseudoImpl::expandCondSMToggle(MachineBasicBlock &MBB,
1179 MachineBasicBlock::iterator MBBI) {
1180 MachineInstr &MI = *MBBI;
1181 // In the case of a smstart/smstop before a unreachable, just remove the pseudo.
1182 // Exception handling code generated by Clang may introduce unreachables and it
1183 // seems unnecessary to restore pstate.sm when that happens. Note that it is
1184 // not just an optimisation, the code below expects a successor instruction/block
1185 // in order to split the block at MBBI.
1186 if (std::next(x: MBBI) == MBB.end() &&
1187 MI.getParent()->successors().begin() ==
1188 MI.getParent()->successors().end()) {
1189 MI.eraseFromParent();
1190 return &MBB;
1191 }
1192
1193 // Expand the pseudo into smstart or smstop instruction. The pseudo has the
1194 // following operands:
1195 //
1196 // MSRpstatePseudo <za|sm|both>, <0|1>, condition[, pstate.sm], <regmask>
1197 //
1198 // The pseudo is expanded into a conditional smstart/smstop, with a
1199 // check if pstate.sm (register) equals the expected value, and if not,
1200 // invokes the smstart/smstop.
1201 //
1202 // As an example, the following block contains a normal call from a
1203 // streaming-compatible function:
1204 //
1205 // OrigBB:
1206 // MSRpstatePseudo 3, 0, IfCallerIsStreaming, %0, <regmask> <- Cond SMSTOP
1207 // bl @normal_callee
1208 // MSRpstatePseudo 3, 1, IfCallerIsStreaming, %0, <regmask> <- Cond SMSTART
1209 //
1210 // ...which will be transformed into:
1211 //
1212 // OrigBB:
1213 // TBNZx %0:gpr64, 0, SMBB
1214 // b EndBB
1215 //
1216 // SMBB:
1217 // MSRpstatesvcrImm1 3, 0, <regmask> <- SMSTOP
1218 //
1219 // EndBB:
1220 // bl @normal_callee
1221 // MSRcond_pstatesvcrImm1 3, 1, <regmask> <- SMSTART
1222 //
1223 DebugLoc DL = MI.getDebugLoc();
1224
1225 // Create the conditional branch based on the third operand of the
1226 // instruction, which tells us if we are wrapping a normal or streaming
1227 // function.
1228 // We test the live value of pstate.sm and toggle pstate.sm if this is not the
1229 // expected value for the callee (0 for a normal callee and 1 for a streaming
1230 // callee).
1231 unsigned Opc;
1232 switch (MI.getOperand(i: 2).getImm()) {
1233 case AArch64SME::Always:
1234 llvm_unreachable("Should have matched to instruction directly");
1235 case AArch64SME::IfCallerIsStreaming:
1236 Opc = AArch64::TBNZW;
1237 break;
1238 case AArch64SME::IfCallerIsNonStreaming:
1239 Opc = AArch64::TBZW;
1240 break;
1241 }
1242 auto PStateSM = MI.getOperand(i: 3).getReg();
1243 auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
1244 unsigned SMReg32 = TRI->getSubReg(Reg: PStateSM, Idx: AArch64::sub_32);
1245 MachineInstrBuilder Tbx =
1246 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc)).addReg(RegNo: SMReg32).addImm(Val: 0);
1247
1248 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch&: Tbx);
1249 // Create the SMSTART/SMSTOP (MSRpstatesvcrImm1) instruction in SMBB.
1250 MachineInstrBuilder MIB = BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: MI.getDebugLoc(),
1251 MCID: TII->get(Opcode: AArch64::MSRpstatesvcrImm1));
1252 // Copy all but the second and third operands of MSRcond_pstatesvcrImm1 (as
1253 // these contain the CopyFromReg for the first argument and the flag to
1254 // indicate whether the callee is streaming or normal).
1255 MIB.add(MO: MI.getOperand(i: 0));
1256 MIB.add(MO: MI.getOperand(i: 1));
1257 for (unsigned i = 4; i < MI.getNumOperands(); ++i)
1258 MIB.add(MO: MI.getOperand(i));
1259
1260 MI.eraseFromParent();
1261 return &EndBB;
1262}
1263
1264bool AArch64ExpandPseudoImpl::expandMultiVecPseudo(
1265 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1266 const TargetRegisterClass &ContiguousClass,
1267 const TargetRegisterClass &StridedClass, unsigned ContiguousOp,
1268 unsigned StridedOpc) {
1269 MachineInstr &MI = *MBBI;
1270 Register Tuple = MI.getOperand(i: 0).getReg();
1271
1272 auto ContiguousRange = ContiguousClass.getRegisters();
1273 auto StridedRange = StridedClass.getRegisters();
1274 unsigned Opc;
1275 if (llvm::is_contained(Range&: ContiguousRange, Element: Tuple.asMCReg())) {
1276 Opc = ContiguousOp;
1277 } else if (llvm::is_contained(Range&: StridedRange, Element: Tuple.asMCReg())) {
1278 Opc = StridedOpc;
1279 } else
1280 llvm_unreachable("Cannot expand Multi-Vector pseudo");
1281
1282 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
1283 .add(MO: MI.getOperand(i: 0))
1284 .add(MO: MI.getOperand(i: 1))
1285 .add(MO: MI.getOperand(i: 2))
1286 .add(MO: MI.getOperand(i: 3));
1287 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1288 MI.eraseFromParent();
1289 return true;
1290}
1291
1292bool AArch64ExpandPseudoImpl::expandCopyIntoTuplePseudo(
1293 MachineInstr &MI, MachineBasicBlock &MBB,
1294 MachineBasicBlock::iterator MBBI) {
1295 Register Src = MI.getOperand(i: 1).getReg();
1296 Register Dest = MI.getOperand(i: 0).getReg();
1297
1298 if (Src != Dest)
1299 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ORR_ZZZ))
1300 .addReg(RegNo: Dest, Flags: RegState::Define)
1301 .addReg(RegNo: Src)
1302 .addReg(RegNo: Src);
1303
1304 MI.eraseFromParent();
1305 return true;
1306}
1307
1308/// If MBBI references a pseudo instruction that should be expanded here,
1309/// do the expansion and return true. Otherwise return false.
1310bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
1311 MachineBasicBlock::iterator MBBI,
1312 MachineBasicBlock::iterator &NextMBBI) {
1313 MachineInstr &MI = *MBBI;
1314 unsigned Opcode = MI.getOpcode();
1315
1316 // Check if we can expand the destructive op
1317 int OrigInstr = AArch64::getSVEPseudoMap(Opcode: MI.getOpcode());
1318 if (OrigInstr != -1) {
1319 auto &Orig = TII->get(Opcode: OrigInstr);
1320 if ((Orig.TSFlags & AArch64::DestructiveInstTypeMask) !=
1321 AArch64::NotDestructive) {
1322 return expand_DestructiveOp(MI, MBB, MBBI);
1323 }
1324 }
1325
1326 switch (Opcode) {
1327 default:
1328 break;
1329
1330 case AArch64::BSPv8i8:
1331 case AArch64::BSPv16i8: {
1332 Register DstReg = MI.getOperand(i: 0).getReg();
1333 if (DstReg == MI.getOperand(i: 3).getReg()) {
1334 // Expand to BIT
1335 auto I = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1336 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BITv8i8
1337 : AArch64::BITv16i8))
1338 .add(MO: MI.getOperand(i: 0))
1339 .add(MO: MI.getOperand(i: 3))
1340 .add(MO: MI.getOperand(i: 2))
1341 .add(MO: MI.getOperand(i: 1));
1342 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1343 } else if (DstReg == MI.getOperand(i: 2).getReg()) {
1344 // Expand to BIF
1345 auto I = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1346 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BIFv8i8
1347 : AArch64::BIFv16i8))
1348 .add(MO: MI.getOperand(i: 0))
1349 .add(MO: MI.getOperand(i: 2))
1350 .add(MO: MI.getOperand(i: 3))
1351 .add(MO: MI.getOperand(i: 1));
1352 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1353 } else {
1354 // Expand to BSL, use additional move if required
1355 if (DstReg == MI.getOperand(i: 1).getReg()) {
1356 auto I =
1357 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1358 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1359 : AArch64::BSLv16i8))
1360 .add(MO: MI.getOperand(i: 0))
1361 .add(MO: MI.getOperand(i: 1))
1362 .add(MO: MI.getOperand(i: 2))
1363 .add(MO: MI.getOperand(i: 3));
1364 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1365 } else {
1366 RegState RegState =
1367 getRenamableRegState(B: MI.getOperand(i: 1).isRenamable()) |
1368 getKillRegState(
1369 B: MI.getOperand(i: 1).isKill() &&
1370 MI.getOperand(i: 1).getReg() != MI.getOperand(i: 2).getReg() &&
1371 MI.getOperand(i: 1).getReg() != MI.getOperand(i: 3).getReg());
1372 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1373 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i8
1374 : AArch64::ORRv16i8))
1375 .addReg(RegNo: DstReg,
1376 Flags: RegState::Define |
1377 getRenamableRegState(B: MI.getOperand(i: 0).isRenamable()))
1378 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState)
1379 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState);
1380 auto I2 =
1381 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1382 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1383 : AArch64::BSLv16i8))
1384 .add(MO: MI.getOperand(i: 0))
1385 .addReg(RegNo: DstReg,
1386 Flags: RegState::Kill | getRenamableRegState(
1387 B: MI.getOperand(i: 0).isRenamable()))
1388 .add(MO: MI.getOperand(i: 2))
1389 .add(MO: MI.getOperand(i: 3));
1390 transferImpOps(OldMI&: MI, UseMI&: I2, DefMI&: I2);
1391 }
1392 }
1393 MI.eraseFromParent();
1394 return true;
1395 }
1396
1397 case AArch64::ADDWrr:
1398 case AArch64::SUBWrr:
1399 case AArch64::ADDXrr:
1400 case AArch64::SUBXrr:
1401 case AArch64::ADDSWrr:
1402 case AArch64::SUBSWrr:
1403 case AArch64::ADDSXrr:
1404 case AArch64::SUBSXrr:
1405 case AArch64::ANDWrr:
1406 case AArch64::ANDXrr:
1407 case AArch64::BICWrr:
1408 case AArch64::BICXrr:
1409 case AArch64::ANDSWrr:
1410 case AArch64::ANDSXrr:
1411 case AArch64::BICSWrr:
1412 case AArch64::BICSXrr:
1413 case AArch64::EONWrr:
1414 case AArch64::EONXrr:
1415 case AArch64::EORWrr:
1416 case AArch64::EORXrr:
1417 case AArch64::ORNWrr:
1418 case AArch64::ORNXrr:
1419 case AArch64::ORRWrr:
1420 case AArch64::ORRXrr: {
1421 unsigned Opcode;
1422 switch (MI.getOpcode()) {
1423 default:
1424 return false;
1425 case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;
1426 case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;
1427 case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;
1428 case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;
1429 case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;
1430 case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;
1431 case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;
1432 case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;
1433 case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;
1434 case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;
1435 case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;
1436 case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;
1437 case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;
1438 case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;
1439 case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;
1440 case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;
1441 case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;
1442 case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;
1443 case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;
1444 case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;
1445 case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;
1446 case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;
1447 case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;
1448 case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;
1449 }
1450 MachineFunction &MF = *MBB.getParent();
1451 // Try to create new inst without implicit operands added.
1452 MachineInstr *NewMI = MF.CreateMachineInstr(
1453 MCID: TII->get(Opcode), DL: MI.getDebugLoc(), /*NoImplicit=*/true);
1454 MBB.insert(I: MBBI, MI: NewMI);
1455 MachineInstrBuilder MIB1(MF, NewMI);
1456 MIB1->setPCSections(MF, MD: MI.getPCSections());
1457 MIB1.addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Define)
1458 .add(MO: MI.getOperand(i: 1))
1459 .add(MO: MI.getOperand(i: 2))
1460 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
1461 transferImpOps(OldMI&: MI, UseMI&: MIB1, DefMI&: MIB1);
1462 if (auto DebugNumber = MI.peekDebugInstrNum())
1463 NewMI->setDebugInstrNum(DebugNumber);
1464 MI.eraseFromParent();
1465 return true;
1466 }
1467
1468 case AArch64::LOADgot: {
1469 MachineFunction *MF = MBB.getParent();
1470 Register DstReg = MI.getOperand(i: 0).getReg();
1471 const MachineOperand &MO1 = MI.getOperand(i: 1);
1472 unsigned Flags = MO1.getTargetFlags();
1473
1474 if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {
1475 // Tiny codemodel expand to LDR
1476 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1477 MCID: TII->get(Opcode: AArch64::LDRXl), DestReg: DstReg);
1478
1479 if (MO1.isGlobal()) {
1480 MIB.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0, TargetFlags: Flags);
1481 } else if (MO1.isSymbol()) {
1482 MIB.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags);
1483 } else {
1484 assert(MO1.isCPI() &&
1485 "Only expect globals, externalsymbols, or constant pools");
1486 MIB.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(), TargetFlags: Flags);
1487 }
1488 } else {
1489 // Small codemodel expand into ADRP + LDR.
1490 MachineFunction &MF = *MI.getParent()->getParent();
1491 DebugLoc DL = MI.getDebugLoc();
1492 MachineInstrBuilder MIB1 =
1493 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADRP), DestReg: DstReg);
1494
1495 MachineInstrBuilder MIB2;
1496 if (MF.getSubtarget<AArch64Subtarget>().isTargetILP32()) {
1497 auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
1498 unsigned Reg32 = TRI->getSubReg(Reg: DstReg, Idx: AArch64::sub_32);
1499 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::LDRWui))
1500 .addDef(RegNo: Reg32)
1501 .addReg(RegNo: DstReg, Flags: RegState::Kill)
1502 .addReg(RegNo: DstReg, Flags: RegState::Implicit);
1503 } else {
1504 Register DstReg = MI.getOperand(i: 0).getReg();
1505 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
1506 .add(MO: MI.getOperand(i: 0))
1507 .addUse(RegNo: DstReg, Flags: RegState::Kill);
1508 }
1509
1510 if (MO1.isGlobal()) {
1511 MIB1.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0, TargetFlags: Flags | AArch64II::MO_PAGE);
1512 MIB2.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0,
1513 TargetFlags: Flags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1514 } else if (MO1.isSymbol()) {
1515 MIB1.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags | AArch64II::MO_PAGE);
1516 MIB2.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags |
1517 AArch64II::MO_PAGEOFF |
1518 AArch64II::MO_NC);
1519 } else {
1520 assert(MO1.isCPI() &&
1521 "Only expect globals, externalsymbols, or constant pools");
1522 MIB1.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(),
1523 TargetFlags: Flags | AArch64II::MO_PAGE);
1524 MIB2.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(),
1525 TargetFlags: Flags | AArch64II::MO_PAGEOFF |
1526 AArch64II::MO_NC);
1527 }
1528
1529 // If the LOADgot instruction has a debug-instr-number, annotate the
1530 // LDRWui instruction that it is expanded to with the same
1531 // debug-instr-number to preserve debug information.
1532 if (MI.peekDebugInstrNum() != 0)
1533 MIB2->setDebugInstrNum(MI.peekDebugInstrNum());
1534 transferImpOps(OldMI&: MI, UseMI&: MIB1, DefMI&: MIB2);
1535 }
1536 MI.eraseFromParent();
1537 return true;
1538 }
1539 case AArch64::MOVaddrBA:
1540 case AArch64::MOVaddr:
1541 case AArch64::MOVaddrJT:
1542 case AArch64::MOVaddrCP:
1543 case AArch64::MOVaddrTLS:
1544 case AArch64::MOVaddrEXT: {
1545 MachineFunction &MF = *MI.getParent()->getParent();
1546 Register DstReg = MI.getOperand(i: 0).getReg();
1547 assert(DstReg != AArch64::XZR);
1548
1549 bool IsTargetMachO = MF.getSubtarget<AArch64Subtarget>().isTargetMachO();
1550 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
1551 AArch64_IMM::expandMOVAddr(
1552 Opcode: MI.getOpcode(), TargetFlags: MI.getOperand(i: 1).getTargetFlags(), IsTargetMachO, Insn);
1553
1554 // Compute the constant pool index, if any.
1555 std::optional<unsigned> CPIdx;
1556 if (Opcode == AArch64::MOVaddrBA && IsTargetMachO) {
1557 // blockaddress expressions have to come from a constant pool because the
1558 // largest addend (and hence offset within a function) allowed for ADRP is
1559 // only 8MB.
1560 const BlockAddress *BA = MI.getOperand(i: 1).getBlockAddress();
1561 assert(MI.getOperand(1).getOffset() == 0 && "unexpected offset");
1562 MachineConstantPool *MCP = MF.getConstantPool();
1563 CPIdx = MCP->getConstantPoolIndex(C: BA, Alignment: Align(8));
1564 }
1565
1566 MachineInstrBuilder FirstMIB;
1567 MachineInstrBuilder LastMIB;
1568 for (const auto &I : Insn) {
1569 MachineInstrBuilder MIB;
1570 switch (I.Opcode) {
1571 case AArch64::ADRP:
1572 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADRP),
1573 DestReg: DstReg);
1574 if (CPIdx)
1575 MIB.addConstantPoolIndex(Idx: *CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGE);
1576 else
1577 MIB.add(MO: MI.getOperand(i: 1));
1578 break;
1579 case AArch64::LDRXui:
1580 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::LDRXui),
1581 DestReg: DstReg)
1582 .addUse(RegNo: DstReg)
1583 .addConstantPoolIndex(
1584 Idx: *CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1585 break;
1586 case AArch64::MOVKXi: {
1587 // MO_TAGGED on the page indicates a tagged address. Set the tag now.
1588 // We do so by creating a MOVK that sets bits 48-63 of the register to
1589 // (global address + 0x100000000 - PC) >> 48. This assumes that we're in
1590 // the small code model so we can assume a binary size of <= 4GB, which
1591 // makes the untagged PC relative offset positive. The binary must also
1592 // be loaded into address range [0, 2^48). Both of these properties need
1593 // to be ensured at runtime when using tagged addresses.
1594 auto Tag = MI.getOperand(i: 1);
1595 Tag.setTargetFlags(AArch64II::MO_PREL | AArch64II::MO_G3);
1596 Tag.setOffset(0x100000000);
1597 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MOVKXi),
1598 DestReg: DstReg)
1599 .addReg(RegNo: DstReg)
1600 .add(MO: Tag)
1601 .addImm(Val: 48);
1602 break;
1603 }
1604 case AArch64::ADDXri:
1605 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
1606 .add(MO: MI.getOperand(i: 0))
1607 .addReg(RegNo: DstReg)
1608 .add(MO: MI.getOperand(i: 2))
1609 .addImm(Val: 0);
1610 break;
1611 default:
1612 llvm_unreachable("unexpected opcode in MOVaddr expansion");
1613 }
1614
1615 if (!FirstMIB.getInstr())
1616 FirstMIB = MIB;
1617 LastMIB = MIB;
1618 }
1619
1620 transferImpOps(OldMI&: MI, UseMI&: FirstMIB, DefMI&: LastMIB);
1621 MI.eraseFromParent();
1622 return true;
1623 }
1624 case AArch64::ADDlowTLS:
1625 // Produce a plain ADD
1626 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
1627 .add(MO: MI.getOperand(i: 0))
1628 .add(MO: MI.getOperand(i: 1))
1629 .add(MO: MI.getOperand(i: 2))
1630 .addImm(Val: 0);
1631 MI.eraseFromParent();
1632 return true;
1633
1634 case AArch64::MOVbaseTLS: {
1635 Register DstReg = MI.getOperand(i: 0).getReg();
1636 auto SysReg = AArch64SysReg::TPIDR_EL0;
1637 MachineFunction *MF = MBB.getParent();
1638 if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())
1639 SysReg = AArch64SysReg::TPIDR_EL3;
1640 else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())
1641 SysReg = AArch64SysReg::TPIDR_EL2;
1642 else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())
1643 SysReg = AArch64SysReg::TPIDR_EL1;
1644 else if (MF->getSubtarget<AArch64Subtarget>().useROEL0ForTP())
1645 SysReg = AArch64SysReg::TPIDRRO_EL0;
1646 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MRS), DestReg: DstReg)
1647 .addImm(Val: SysReg);
1648 MI.eraseFromParent();
1649 return true;
1650 }
1651
1652 case AArch64::MOVi32imm:
1653 return expandMOVImm(MBB, MBBI, BitSize: 32);
1654 case AArch64::MOVi64imm:
1655 return expandMOVImm(MBB, MBBI, BitSize: 64);
1656 case AArch64::RET_ReallyLR: {
1657 // Hiding the LR use with RET_ReallyLR may lead to extra kills in the
1658 // function and missing live-ins. We are fine in practice because callee
1659 // saved register handling ensures the register value is restored before
1660 // RET, but we need the undef flag here to appease the MachineVerifier
1661 // liveness checks.
1662 MachineInstrBuilder MIB =
1663 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::RET))
1664 .addReg(RegNo: AArch64::LR, Flags: RegState::Undef);
1665 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1666 MI.eraseFromParent();
1667 return true;
1668 }
1669 case AArch64::CMP_SWAP_8:
1670 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRB, StlrOp: AArch64::STLXRB,
1671 CmpOp: AArch64::SUBSWrx,
1672 ExtendImm: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTB, Imm: 0),
1673 ZeroReg: AArch64::WZR, NextMBBI);
1674 case AArch64::CMP_SWAP_16:
1675 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRH, StlrOp: AArch64::STLXRH,
1676 CmpOp: AArch64::SUBSWrx,
1677 ExtendImm: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTH, Imm: 0),
1678 ZeroReg: AArch64::WZR, NextMBBI);
1679 case AArch64::CMP_SWAP_32:
1680 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRW, StlrOp: AArch64::STLXRW,
1681 CmpOp: AArch64::SUBSWrs,
1682 ExtendImm: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0),
1683 ZeroReg: AArch64::WZR, NextMBBI);
1684 case AArch64::CMP_SWAP_64:
1685 return expandCMP_SWAP(MBB, MBBI,
1686 LdarOp: AArch64::LDAXRX, StlrOp: AArch64::STLXRX, CmpOp: AArch64::SUBSXrs,
1687 ExtendImm: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0),
1688 ZeroReg: AArch64::XZR, NextMBBI);
1689 case AArch64::CMP_SWAP_128:
1690 case AArch64::CMP_SWAP_128_RELEASE:
1691 case AArch64::CMP_SWAP_128_ACQUIRE:
1692 case AArch64::CMP_SWAP_128_MONOTONIC:
1693 return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
1694
1695 case AArch64::AESMCrrTied:
1696 case AArch64::AESIMCrrTied: {
1697 MachineInstrBuilder MIB =
1698 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1699 MCID: TII->get(Opcode: Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
1700 AArch64::AESIMCrr))
1701 .add(MO: MI.getOperand(i: 0))
1702 .add(MO: MI.getOperand(i: 1));
1703 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1704 MI.eraseFromParent();
1705 return true;
1706 }
1707 case AArch64::IRGstack: {
1708 MachineFunction &MF = *MBB.getParent();
1709 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1710 const AArch64FrameLowering *TFI =
1711 MF.getSubtarget<AArch64Subtarget>().getFrameLowering();
1712
1713 // IRG does not allow immediate offset. getTaggedBasePointerOffset should
1714 // almost always point to SP-after-prologue; if not, emit a longer
1715 // instruction sequence.
1716 int BaseOffset = -AFI->getTaggedBasePointerOffset();
1717 Register FrameReg;
1718 StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
1719 MF, ObjectOffset: BaseOffset, isFixed: false /*isFixed*/, StackID: TargetStackID::Default /*StackID*/,
1720 FrameReg,
1721 /*PreferFP=*/false,
1722 /*ForSimm=*/true);
1723 Register SrcReg = FrameReg;
1724 if (FrameRegOffset) {
1725 // Use output register as temporary.
1726 SrcReg = MI.getOperand(i: 0).getReg();
1727 emitFrameOffset(MBB, MBBI: &MI, DL: MI.getDebugLoc(), DestReg: SrcReg, SrcReg: FrameReg,
1728 Offset: FrameRegOffset, TII);
1729 }
1730 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::IRG))
1731 .add(MO: MI.getOperand(i: 0))
1732 .addUse(RegNo: SrcReg)
1733 .add(MO: MI.getOperand(i: 2));
1734 MI.eraseFromParent();
1735 return true;
1736 }
1737 case AArch64::TAGPstack: {
1738 int64_t Offset = MI.getOperand(i: 2).getImm();
1739 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1740 MCID: TII->get(Opcode: Offset >= 0 ? AArch64::ADDG : AArch64::SUBG))
1741 .add(MO: MI.getOperand(i: 0))
1742 .add(MO: MI.getOperand(i: 1))
1743 .addImm(Val: std::abs(i: Offset))
1744 .add(MO: MI.getOperand(i: 4));
1745 MI.eraseFromParent();
1746 return true;
1747 }
1748 case AArch64::STGloop_wback:
1749 case AArch64::STZGloop_wback:
1750 return expandSetTagLoop(MBB, MBBI, NextMBBI);
1751 case AArch64::STGloop:
1752 case AArch64::STZGloop:
1753 report_fatal_error(
1754 reason: "Non-writeback variants of STGloop / STZGloop should not "
1755 "survive past PrologEpilogInserter.");
1756 case AArch64::STR_ZZZZXI:
1757 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
1758 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 4);
1759 case AArch64::STR_ZZZXI:
1760 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 3);
1761 case AArch64::STR_ZZXI:
1762 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
1763 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 2);
1764 case AArch64::STR_PPXI:
1765 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_PXI, N: 2);
1766 case AArch64::LDR_ZZZZXI:
1767 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
1768 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 4);
1769 case AArch64::LDR_ZZZXI:
1770 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 3);
1771 case AArch64::LDR_ZZXI:
1772 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
1773 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 2);
1774 case AArch64::LDR_PPXI:
1775 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_PXI, N: 2);
1776 case AArch64::BLR_RVMARKER:
1777 case AArch64::BLRA_RVMARKER:
1778 return expandCALL_RVMARKER(MBB, MBBI);
1779 case AArch64::BLR_BTI:
1780 return expandCALL_BTI(MBB, MBBI);
1781 case AArch64::StoreSwiftAsyncContext:
1782 return expandStoreSwiftAsyncContext(MBB, MBBI);
1783 case AArch64::RestoreZAPseudo:
1784 case AArch64::CommitZASavePseudo:
1785 case AArch64::MSRpstatePseudo: {
1786 auto *NewMBB = [&] {
1787 switch (Opcode) {
1788 case AArch64::RestoreZAPseudo:
1789 return expandRestoreZASave(MBB, MBBI);
1790 case AArch64::CommitZASavePseudo:
1791 return expandCommitZASave(MBB, MBBI);
1792 case AArch64::MSRpstatePseudo:
1793 return expandCondSMToggle(MBB, MBBI);
1794 default:
1795 llvm_unreachable("Unexpected conditional pseudo!");
1796 }
1797 }();
1798 if (NewMBB != &MBB)
1799 NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated.
1800 return true;
1801 }
1802 case AArch64::InOutZAUsePseudo:
1803 case AArch64::RequiresZASavePseudo:
1804 case AArch64::RequiresZT0SavePseudo:
1805 case AArch64::SMEStateAllocPseudo:
1806 case AArch64::COALESCER_BARRIER_FPR16:
1807 case AArch64::COALESCER_BARRIER_FPR32:
1808 case AArch64::COALESCER_BARRIER_FPR64:
1809 case AArch64::COALESCER_BARRIER_FPR128:
1810 MI.eraseFromParent();
1811 return true;
1812 case AArch64::LD1B_2Z_IMM_PSEUDO:
1813 return expandMultiVecPseudo(
1814 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1815 ContiguousOp: AArch64::LD1B_2Z_IMM, StridedOpc: AArch64::LD1B_2Z_STRIDED_IMM);
1816 case AArch64::LD1H_2Z_IMM_PSEUDO:
1817 return expandMultiVecPseudo(
1818 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1819 ContiguousOp: AArch64::LD1H_2Z_IMM, StridedOpc: AArch64::LD1H_2Z_STRIDED_IMM);
1820 case AArch64::LD1W_2Z_IMM_PSEUDO:
1821 return expandMultiVecPseudo(
1822 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1823 ContiguousOp: AArch64::LD1W_2Z_IMM, StridedOpc: AArch64::LD1W_2Z_STRIDED_IMM);
1824 case AArch64::LD1D_2Z_IMM_PSEUDO:
1825 return expandMultiVecPseudo(
1826 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1827 ContiguousOp: AArch64::LD1D_2Z_IMM, StridedOpc: AArch64::LD1D_2Z_STRIDED_IMM);
1828 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
1829 return expandMultiVecPseudo(
1830 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1831 ContiguousOp: AArch64::LDNT1B_2Z_IMM, StridedOpc: AArch64::LDNT1B_2Z_STRIDED_IMM);
1832 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
1833 return expandMultiVecPseudo(
1834 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1835 ContiguousOp: AArch64::LDNT1H_2Z_IMM, StridedOpc: AArch64::LDNT1H_2Z_STRIDED_IMM);
1836 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
1837 return expandMultiVecPseudo(
1838 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1839 ContiguousOp: AArch64::LDNT1W_2Z_IMM, StridedOpc: AArch64::LDNT1W_2Z_STRIDED_IMM);
1840 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
1841 return expandMultiVecPseudo(
1842 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1843 ContiguousOp: AArch64::LDNT1D_2Z_IMM, StridedOpc: AArch64::LDNT1D_2Z_STRIDED_IMM);
1844 case AArch64::ST1B_2Z_IMM_PSEUDO:
1845 return expandMultiVecPseudo(
1846 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1847 ContiguousOp: AArch64::ST1B_2Z_IMM, StridedOpc: AArch64::ST1B_2Z_STRIDED_IMM);
1848 case AArch64::ST1H_2Z_IMM_PSEUDO:
1849 return expandMultiVecPseudo(
1850 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1851 ContiguousOp: AArch64::ST1H_2Z_IMM, StridedOpc: AArch64::ST1H_2Z_STRIDED_IMM);
1852 case AArch64::ST1W_2Z_IMM_PSEUDO:
1853 return expandMultiVecPseudo(
1854 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1855 ContiguousOp: AArch64::ST1W_2Z_IMM, StridedOpc: AArch64::ST1W_2Z_STRIDED_IMM);
1856 case AArch64::ST1D_2Z_IMM_PSEUDO:
1857 return expandMultiVecPseudo(
1858 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1859 ContiguousOp: AArch64::ST1D_2Z_IMM, StridedOpc: AArch64::ST1D_2Z_STRIDED_IMM);
1860 case AArch64::STNT1B_2Z_IMM_PSEUDO:
1861 return expandMultiVecPseudo(
1862 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1863 ContiguousOp: AArch64::STNT1B_2Z_IMM, StridedOpc: AArch64::STNT1B_2Z_STRIDED_IMM);
1864 case AArch64::STNT1H_2Z_IMM_PSEUDO:
1865 return expandMultiVecPseudo(
1866 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1867 ContiguousOp: AArch64::STNT1H_2Z_IMM, StridedOpc: AArch64::STNT1H_2Z_STRIDED_IMM);
1868 case AArch64::STNT1W_2Z_IMM_PSEUDO:
1869 return expandMultiVecPseudo(
1870 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1871 ContiguousOp: AArch64::STNT1W_2Z_IMM, StridedOpc: AArch64::STNT1W_2Z_STRIDED_IMM);
1872 case AArch64::STNT1D_2Z_IMM_PSEUDO:
1873 return expandMultiVecPseudo(
1874 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1875 ContiguousOp: AArch64::STNT1D_2Z_IMM, StridedOpc: AArch64::STNT1D_2Z_STRIDED_IMM);
1876 case AArch64::LD1B_2Z_PSEUDO:
1877 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1878 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1B_2Z,
1879 StridedOpc: AArch64::LD1B_2Z_STRIDED);
1880 case AArch64::LD1H_2Z_PSEUDO:
1881 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1882 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1H_2Z,
1883 StridedOpc: AArch64::LD1H_2Z_STRIDED);
1884 case AArch64::LD1W_2Z_PSEUDO:
1885 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1886 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1W_2Z,
1887 StridedOpc: AArch64::LD1W_2Z_STRIDED);
1888 case AArch64::LD1D_2Z_PSEUDO:
1889 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1890 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1D_2Z,
1891 StridedOpc: AArch64::LD1D_2Z_STRIDED);
1892 case AArch64::LDNT1B_2Z_PSEUDO:
1893 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1894 StridedClass: AArch64::ZPR2StridedRegClass,
1895 ContiguousOp: AArch64::LDNT1B_2Z, StridedOpc: AArch64::LDNT1B_2Z_STRIDED);
1896 case AArch64::LDNT1H_2Z_PSEUDO:
1897 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1898 StridedClass: AArch64::ZPR2StridedRegClass,
1899 ContiguousOp: AArch64::LDNT1H_2Z, StridedOpc: AArch64::LDNT1H_2Z_STRIDED);
1900 case AArch64::LDNT1W_2Z_PSEUDO:
1901 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1902 StridedClass: AArch64::ZPR2StridedRegClass,
1903 ContiguousOp: AArch64::LDNT1W_2Z, StridedOpc: AArch64::LDNT1W_2Z_STRIDED);
1904 case AArch64::LDNT1D_2Z_PSEUDO:
1905 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1906 StridedClass: AArch64::ZPR2StridedRegClass,
1907 ContiguousOp: AArch64::LDNT1D_2Z, StridedOpc: AArch64::LDNT1D_2Z_STRIDED);
1908 case AArch64::LD1B_4Z_IMM_PSEUDO:
1909 return expandMultiVecPseudo(
1910 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1911 ContiguousOp: AArch64::LD1B_4Z_IMM, StridedOpc: AArch64::LD1B_4Z_STRIDED_IMM);
1912 case AArch64::LD1H_4Z_IMM_PSEUDO:
1913 return expandMultiVecPseudo(
1914 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1915 ContiguousOp: AArch64::LD1H_4Z_IMM, StridedOpc: AArch64::LD1H_4Z_STRIDED_IMM);
1916 case AArch64::LD1W_4Z_IMM_PSEUDO:
1917 return expandMultiVecPseudo(
1918 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1919 ContiguousOp: AArch64::LD1W_4Z_IMM, StridedOpc: AArch64::LD1W_4Z_STRIDED_IMM);
1920 case AArch64::LD1D_4Z_IMM_PSEUDO:
1921 return expandMultiVecPseudo(
1922 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1923 ContiguousOp: AArch64::LD1D_4Z_IMM, StridedOpc: AArch64::LD1D_4Z_STRIDED_IMM);
1924 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
1925 return expandMultiVecPseudo(
1926 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1927 ContiguousOp: AArch64::LDNT1B_4Z_IMM, StridedOpc: AArch64::LDNT1B_4Z_STRIDED_IMM);
1928 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
1929 return expandMultiVecPseudo(
1930 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1931 ContiguousOp: AArch64::LDNT1H_4Z_IMM, StridedOpc: AArch64::LDNT1H_4Z_STRIDED_IMM);
1932 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
1933 return expandMultiVecPseudo(
1934 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1935 ContiguousOp: AArch64::LDNT1W_4Z_IMM, StridedOpc: AArch64::LDNT1W_4Z_STRIDED_IMM);
1936 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
1937 return expandMultiVecPseudo(
1938 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1939 ContiguousOp: AArch64::LDNT1D_4Z_IMM, StridedOpc: AArch64::LDNT1D_4Z_STRIDED_IMM);
1940 case AArch64::ST1B_4Z_IMM_PSEUDO:
1941 return expandMultiVecPseudo(
1942 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1943 ContiguousOp: AArch64::ST1B_4Z_IMM, StridedOpc: AArch64::ST1B_4Z_STRIDED_IMM);
1944 case AArch64::ST1H_4Z_IMM_PSEUDO:
1945 return expandMultiVecPseudo(
1946 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1947 ContiguousOp: AArch64::ST1H_4Z_IMM, StridedOpc: AArch64::ST1H_4Z_STRIDED_IMM);
1948 case AArch64::ST1W_4Z_IMM_PSEUDO:
1949 return expandMultiVecPseudo(
1950 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1951 ContiguousOp: AArch64::ST1W_4Z_IMM, StridedOpc: AArch64::ST1W_4Z_STRIDED_IMM);
1952 case AArch64::ST1D_4Z_IMM_PSEUDO:
1953 return expandMultiVecPseudo(
1954 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1955 ContiguousOp: AArch64::ST1D_4Z_IMM, StridedOpc: AArch64::ST1D_4Z_STRIDED_IMM);
1956 case AArch64::STNT1B_4Z_IMM_PSEUDO:
1957 return expandMultiVecPseudo(
1958 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1959 ContiguousOp: AArch64::STNT1B_4Z_IMM, StridedOpc: AArch64::STNT1B_4Z_STRIDED_IMM);
1960 case AArch64::STNT1H_4Z_IMM_PSEUDO:
1961 return expandMultiVecPseudo(
1962 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1963 ContiguousOp: AArch64::STNT1H_4Z_IMM, StridedOpc: AArch64::STNT1H_4Z_STRIDED_IMM);
1964 case AArch64::STNT1W_4Z_IMM_PSEUDO:
1965 return expandMultiVecPseudo(
1966 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1967 ContiguousOp: AArch64::STNT1W_4Z_IMM, StridedOpc: AArch64::STNT1W_4Z_STRIDED_IMM);
1968 case AArch64::STNT1D_4Z_IMM_PSEUDO:
1969 return expandMultiVecPseudo(
1970 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1971 ContiguousOp: AArch64::STNT1D_4Z_IMM, StridedOpc: AArch64::STNT1D_4Z_STRIDED_IMM);
1972 case AArch64::LD1B_4Z_PSEUDO:
1973 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1974 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1B_4Z,
1975 StridedOpc: AArch64::LD1B_4Z_STRIDED);
1976 case AArch64::LD1H_4Z_PSEUDO:
1977 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1978 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1H_4Z,
1979 StridedOpc: AArch64::LD1H_4Z_STRIDED);
1980 case AArch64::LD1W_4Z_PSEUDO:
1981 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1982 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1W_4Z,
1983 StridedOpc: AArch64::LD1W_4Z_STRIDED);
1984 case AArch64::LD1D_4Z_PSEUDO:
1985 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1986 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1D_4Z,
1987 StridedOpc: AArch64::LD1D_4Z_STRIDED);
1988 case AArch64::LDNT1B_4Z_PSEUDO:
1989 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1990 StridedClass: AArch64::ZPR4StridedRegClass,
1991 ContiguousOp: AArch64::LDNT1B_4Z, StridedOpc: AArch64::LDNT1B_4Z_STRIDED);
1992 case AArch64::LDNT1H_4Z_PSEUDO:
1993 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1994 StridedClass: AArch64::ZPR4StridedRegClass,
1995 ContiguousOp: AArch64::LDNT1H_4Z, StridedOpc: AArch64::LDNT1H_4Z_STRIDED);
1996 case AArch64::LDNT1W_4Z_PSEUDO:
1997 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1998 StridedClass: AArch64::ZPR4StridedRegClass,
1999 ContiguousOp: AArch64::LDNT1W_4Z, StridedOpc: AArch64::LDNT1W_4Z_STRIDED);
2000 case AArch64::LDNT1D_4Z_PSEUDO:
2001 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
2002 StridedClass: AArch64::ZPR4StridedRegClass,
2003 ContiguousOp: AArch64::LDNT1D_4Z, StridedOpc: AArch64::LDNT1D_4Z_STRIDED);
2004 case AArch64::COPY_INTO_TRANSPOSED_TUPLE:
2005 return expandCopyIntoTuplePseudo(MI, MBB, MBBI);
2006 case AArch64::EON_ZZZ:
2007 case AArch64::NAND_ZZZ:
2008 case AArch64::NOR_ZZZ:
2009 return expandSVEBitwisePseudo(MI, MBB, MBBI);
2010 }
2011 return false;
2012}
2013
2014/// Iterate over the instructions in basic block MBB and expand any
2015/// pseudo instructions. Return true if anything was modified.
2016bool AArch64ExpandPseudoImpl::expandMBB(MachineBasicBlock &MBB) {
2017 bool Modified = false;
2018
2019 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
2020 while (MBBI != E) {
2021 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
2022 if (MBBI->isPseudo())
2023 Modified |= expandMI(MBB, MBBI, NextMBBI&: NMBBI);
2024 MBBI = NMBBI;
2025 }
2026
2027 return Modified;
2028}
2029
2030bool AArch64ExpandPseudoImpl::run(MachineFunction &MF) {
2031 TII = MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
2032
2033 bool Modified = false;
2034 for (auto &MBB : MF)
2035 Modified |= expandMBB(MBB);
2036 return Modified;
2037}
2038
2039bool AArch64ExpandPseudoLegacy::runOnMachineFunction(MachineFunction &MF) {
2040 return AArch64ExpandPseudoImpl().run(MF);
2041}
2042
2043/// Returns an instance of the pseudo instruction expansion pass.
2044FunctionPass *llvm::createAArch64ExpandPseudoLegacyPass() {
2045 return new AArch64ExpandPseudoLegacy();
2046}
2047
2048PreservedAnalyses
2049AArch64ExpandPseudoPass::run(MachineFunction &MF,
2050 MachineFunctionAnalysisManager &MFAM) {
2051 const bool Changed = AArch64ExpandPseudoImpl().run(MF);
2052 if (!Changed)
2053 return PreservedAnalyses::all();
2054 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
2055 PA.preserveSet<CFGAnalyses>();
2056 return PA;
2057}
2058