1//===- AArch64ExpandPseudoInsts.cpp - Expand pseudo instructions ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that expands pseudo instructions into target
10// instructions to allow proper scheduling and other late optimizations. This
11// pass should be run after register allocation but before the post-regalloc
12// scheduling pass.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AArch64ExpandImm.h"
17#include "AArch64InstrInfo.h"
18#include "AArch64MachineFunctionInfo.h"
19#include "AArch64Subtarget.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/CodeGen/LivePhysRegs.h"
23#include "llvm/CodeGen/MachineBasicBlock.h"
24#include "llvm/CodeGen/MachineConstantPool.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineFunctionPass.h"
27#include "llvm/CodeGen/MachineInstr.h"
28#include "llvm/CodeGen/MachineInstrBuilder.h"
29#include "llvm/CodeGen/MachineOperand.h"
30#include "llvm/CodeGen/TargetSubtargetInfo.h"
31#include "llvm/IR/DebugLoc.h"
32#include "llvm/MC/MCInstrDesc.h"
33#include "llvm/Pass.h"
34#include "llvm/Support/CodeGen.h"
35#include "llvm/Target/TargetMachine.h"
36#include "llvm/TargetParser/Triple.h"
37#include <cassert>
38#include <cstdint>
39#include <iterator>
40
41using namespace llvm;
42
43#define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass"
44
45namespace {
46
47class AArch64ExpandPseudoImpl {
48public:
49 const AArch64InstrInfo *TII;
50
51 bool run(MachineFunction &MF);
52
53private:
54 bool expandMBB(MachineBasicBlock &MBB);
55 bool expandMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
56 MachineBasicBlock::iterator &NextMBBI);
57 bool expandMultiVecPseudo(MachineBasicBlock &MBB,
58 MachineBasicBlock::iterator MBBI,
59 const TargetRegisterClass &ContiguousClass,
60 const TargetRegisterClass &StridedClass,
61 unsigned ContiguousOpc, unsigned StridedOpc);
62 bool expandCopyIntoTuplePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
63 MachineBasicBlock::iterator MBBI);
64 bool expandMOVImm(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
65 unsigned BitSize);
66
67 bool expand_DestructiveOp(MachineInstr &MI, MachineBasicBlock &MBB,
68 MachineBasicBlock::iterator MBBI);
69 bool expandSVEBitwisePseudo(MachineInstr &MI, MachineBasicBlock &MBB,
70 MachineBasicBlock::iterator MBBI);
71 bool expandCMP_SWAP(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
72 unsigned LdarOp, unsigned StlrOp, unsigned CmpOp,
73 unsigned ExtendImm, unsigned ZeroReg,
74 MachineBasicBlock::iterator &NextMBBI);
75 bool expandCMP_SWAP_128(MachineBasicBlock &MBB,
76 MachineBasicBlock::iterator MBBI,
77 MachineBasicBlock::iterator &NextMBBI);
78 bool expandSetTagLoop(MachineBasicBlock &MBB,
79 MachineBasicBlock::iterator MBBI,
80 MachineBasicBlock::iterator &NextMBBI);
81 bool expandSVESpillFill(MachineBasicBlock &MBB,
82 MachineBasicBlock::iterator MBBI, unsigned Opc,
83 unsigned N);
84 bool expandCALL_RVMARKER(MachineBasicBlock &MBB,
85 MachineBasicBlock::iterator MBBI);
86 bool expandCALL_BTI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI);
87 bool expandStoreSwiftAsyncContext(MachineBasicBlock &MBB,
88 MachineBasicBlock::iterator MBBI);
89 struct ConditionalBlocks {
90 MachineBasicBlock &CondBB;
91 MachineBasicBlock &EndBB;
92 };
93 ConditionalBlocks expandConditionalPseudo(MachineBasicBlock &MBB,
94 MachineBasicBlock::iterator MBBI,
95 DebugLoc DL,
96 MachineInstrBuilder &Branch);
97 MachineBasicBlock *expandRestoreZASave(MachineBasicBlock &MBB,
98 MachineBasicBlock::iterator MBBI);
99 MachineBasicBlock *expandCommitZASave(MachineBasicBlock &MBB,
100 MachineBasicBlock::iterator MBBI);
101 MachineBasicBlock *expandCondSMToggle(MachineBasicBlock &MBB,
102 MachineBasicBlock::iterator MBBI);
103};
104
105class AArch64ExpandPseudoLegacy : public MachineFunctionPass {
106public:
107 static char ID;
108
109 AArch64ExpandPseudoLegacy() : MachineFunctionPass(ID) {}
110
111 bool runOnMachineFunction(MachineFunction &MF) override;
112
113 StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; }
114};
115
116} // end anonymous namespace
117
118char AArch64ExpandPseudoLegacy::ID = 0;
119
120INITIALIZE_PASS(AArch64ExpandPseudoLegacy, "aarch64-expand-pseudo",
121 AARCH64_EXPAND_PSEUDO_NAME, false, false)
122
123/// Transfer implicit operands on the pseudo instruction to the
124/// instructions created from the expansion.
125static void transferImpOps(MachineInstr &OldMI, MachineInstrBuilder &UseMI,
126 MachineInstrBuilder &DefMI) {
127 const MCInstrDesc &Desc = OldMI.getDesc();
128 for (const MachineOperand &MO :
129 llvm::drop_begin(RangeOrContainer: OldMI.operands(), N: Desc.getNumOperands())) {
130 assert(MO.isReg() && MO.getReg());
131 if (MO.isUse())
132 UseMI.add(MO);
133 else
134 DefMI.add(MO);
135 }
136}
137
138/// Expand a MOVi32imm or MOVi64imm pseudo instruction to one or more
139/// real move-immediate instructions to synthesize the immediate.
140bool AArch64ExpandPseudoImpl::expandMOVImm(MachineBasicBlock &MBB,
141 MachineBasicBlock::iterator MBBI,
142 unsigned BitSize) {
143 MachineInstr &MI = *MBBI;
144 Register DstReg = MI.getOperand(i: 0).getReg();
145 RegState RenamableState =
146 getRenamableRegState(B: MI.getOperand(i: 0).isRenamable());
147 uint64_t Imm = MI.getOperand(i: 1).getImm();
148
149 if (DstReg == AArch64::XZR || DstReg == AArch64::WZR) {
150 // Useless def, and we don't want to risk creating an invalid ORR (which
151 // would really write to sp).
152 MI.eraseFromParent();
153 return true;
154 }
155
156 SmallVector<AArch64_IMM::ImmInsnModel, 4> Insn;
157 AArch64_IMM::expandMOVImm(Imm, BitSize, Insn);
158 assert(Insn.size() != 0);
159
160 SmallVector<MachineInstrBuilder, 4> MIBS;
161 for (auto I = Insn.begin(), E = Insn.end(); I != E; ++I) {
162 bool LastItem = std::next(x: I) == E;
163 switch (I->Opcode)
164 {
165 default: llvm_unreachable("unhandled!"); break;
166
167 case AArch64::ORRWri:
168 case AArch64::ORRXri:
169 case AArch64::ANDXri:
170 case AArch64::EORXri:
171 if (I->Op1 == 0) {
172 MIBS.push_back(Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
173 .add(MO: MI.getOperand(i: 0))
174 .addReg(RegNo: BitSize == 32 ? AArch64::WZR : AArch64::XZR)
175 .addImm(Val: I->Op2));
176 } else {
177 Register DstReg = MI.getOperand(i: 0).getReg();
178 bool DstIsDead = MI.getOperand(i: 0).isDead();
179 MIBS.push_back(
180 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
181 .addReg(RegNo: DstReg, Flags: RegState::Define |
182 getDeadRegState(B: DstIsDead && LastItem) |
183 RenamableState)
184 .addReg(RegNo: DstReg)
185 .addImm(Val: I->Op2));
186 }
187 break;
188 case AArch64::EONXrs:
189 case AArch64::EORXrs:
190 case AArch64::ORRWrs:
191 case AArch64::ORRXrs: {
192 Register DstReg = MI.getOperand(i: 0).getReg();
193 bool DstIsDead = MI.getOperand(i: 0).isDead();
194 MIBS.push_back(
195 Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
196 .addReg(RegNo: DstReg, Flags: RegState::Define |
197 getDeadRegState(B: DstIsDead && LastItem) |
198 RenamableState)
199 .addReg(RegNo: DstReg)
200 .addReg(RegNo: DstReg)
201 .addImm(Val: I->Op2));
202 } break;
203 case AArch64::MOVNWi:
204 case AArch64::MOVNXi:
205 case AArch64::MOVZWi:
206 case AArch64::MOVZXi: {
207 bool DstIsDead = MI.getOperand(i: 0).isDead();
208 MIBS.push_back(Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
209 .addReg(RegNo: DstReg, Flags: RegState::Define |
210 getDeadRegState(B: DstIsDead && LastItem) |
211 RenamableState)
212 .addImm(Val: I->Op1)
213 .addImm(Val: I->Op2));
214 } break;
215 case AArch64::MOVKWi:
216 case AArch64::MOVKXi: {
217 Register DstReg = MI.getOperand(i: 0).getReg();
218 bool DstIsDead = MI.getOperand(i: 0).isDead();
219 MIBS.push_back(Elt: BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: I->Opcode))
220 .addReg(RegNo: DstReg,
221 Flags: RegState::Define |
222 getDeadRegState(B: DstIsDead && LastItem) |
223 RenamableState)
224 .addReg(RegNo: DstReg)
225 .addImm(Val: I->Op1)
226 .addImm(Val: I->Op2));
227 } break;
228 }
229 }
230 transferImpOps(OldMI&: MI, UseMI&: MIBS.front(), DefMI&: MIBS.back());
231 MI.eraseFromParent();
232 return true;
233}
234
235bool AArch64ExpandPseudoImpl::expandCMP_SWAP(
236 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned LdarOp,
237 unsigned StlrOp, unsigned CmpOp, unsigned ExtendImm, unsigned ZeroReg,
238 MachineBasicBlock::iterator &NextMBBI) {
239 MachineInstr &MI = *MBBI;
240 MIMetadata MIMD(MI);
241 const MachineOperand &Dest = MI.getOperand(i: 0);
242 Register StatusReg = MI.getOperand(i: 1).getReg();
243 bool StatusDead = MI.getOperand(i: 1).isDead();
244 // Duplicating undef operands into 2 instructions does not guarantee the same
245 // value on both; However undef should be replaced by xzr anyway.
246 assert(!MI.getOperand(2).isUndef() && "cannot handle undef");
247 Register AddrReg = MI.getOperand(i: 2).getReg();
248 Register DesiredReg = MI.getOperand(i: 3).getReg();
249 Register NewReg = MI.getOperand(i: 4).getReg();
250
251 MachineFunction *MF = MBB.getParent();
252 auto LoadCmpBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
253 auto StoreBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
254 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
255
256 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoadCmpBB);
257 MF->insert(MBBI: ++LoadCmpBB->getIterator(), MBB: StoreBB);
258 MF->insert(MBBI: ++StoreBB->getIterator(), MBB: DoneBB);
259
260 // .Lloadcmp:
261 // mov wStatus, 0
262 // ldaxr xDest, [xAddr]
263 // cmp xDest, xDesired
264 // b.ne .Ldone
265 if (!StatusDead)
266 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::MOVZWi), DestReg: StatusReg)
267 .addImm(Val: 0).addImm(Val: 0);
268 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: LdarOp), DestReg: Dest.getReg())
269 .addReg(RegNo: AddrReg);
270 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: CmpOp), DestReg: ZeroReg)
271 .addReg(RegNo: Dest.getReg(), Flags: getKillRegState(B: Dest.isDead()))
272 .addReg(RegNo: DesiredReg)
273 .addImm(Val: ExtendImm);
274 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::Bcc))
275 .addImm(Val: AArch64CC::NE)
276 .addMBB(MBB: DoneBB)
277 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Kill);
278 LoadCmpBB->addSuccessor(Succ: DoneBB);
279 LoadCmpBB->addSuccessor(Succ: StoreBB);
280
281 // .Lstore:
282 // stlxr wStatus, xNew, [xAddr]
283 // cbnz wStatus, .Lloadcmp
284 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: StlrOp), DestReg: StatusReg)
285 .addReg(RegNo: NewReg)
286 .addReg(RegNo: AddrReg);
287 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
288 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
289 .addMBB(MBB: LoadCmpBB);
290 StoreBB->addSuccessor(Succ: LoadCmpBB);
291 StoreBB->addSuccessor(Succ: DoneBB);
292
293 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
294 DoneBB->transferSuccessors(FromMBB: &MBB);
295
296 MBB.addSuccessor(Succ: LoadCmpBB);
297
298 NextMBBI = MBB.end();
299 MI.eraseFromParent();
300
301 // Recompute livein lists.
302 LivePhysRegs LiveRegs;
303 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
304 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
305 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
306 // Do an extra pass around the loop to get loop carried registers right.
307 StoreBB->clearLiveIns();
308 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
309 LoadCmpBB->clearLiveIns();
310 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
311
312 return true;
313}
314
315bool AArch64ExpandPseudoImpl::expandCMP_SWAP_128(
316 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
317 MachineBasicBlock::iterator &NextMBBI) {
318 MachineInstr &MI = *MBBI;
319 MIMetadata MIMD(MI);
320 MachineOperand &DestLo = MI.getOperand(i: 0);
321 MachineOperand &DestHi = MI.getOperand(i: 1);
322 Register StatusReg = MI.getOperand(i: 2).getReg();
323 bool StatusDead = MI.getOperand(i: 2).isDead();
324 // Duplicating undef operands into 2 instructions does not guarantee the same
325 // value on both; However undef should be replaced by xzr anyway.
326 assert(!MI.getOperand(3).isUndef() && "cannot handle undef");
327 Register AddrReg = MI.getOperand(i: 3).getReg();
328 Register DesiredLoReg = MI.getOperand(i: 4).getReg();
329 Register DesiredHiReg = MI.getOperand(i: 5).getReg();
330 Register NewLoReg = MI.getOperand(i: 6).getReg();
331 Register NewHiReg = MI.getOperand(i: 7).getReg();
332
333 auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
334 bool LittleEndian = STI.isLittleEndian();
335 MachineOperand &Dest0 = LittleEndian ? DestLo : DestHi;
336 MachineOperand &Dest1 = LittleEndian ? DestHi : DestLo;
337 Register New0Reg = LittleEndian ? NewLoReg : NewHiReg;
338 Register New1Reg = LittleEndian ? NewHiReg : NewLoReg;
339
340 unsigned LdxpOp, StxpOp;
341
342 switch (MI.getOpcode()) {
343 case AArch64::CMP_SWAP_128_MONOTONIC:
344 LdxpOp = AArch64::LDXPX;
345 StxpOp = AArch64::STXPX;
346 break;
347 case AArch64::CMP_SWAP_128_RELEASE:
348 LdxpOp = AArch64::LDXPX;
349 StxpOp = AArch64::STLXPX;
350 break;
351 case AArch64::CMP_SWAP_128_ACQUIRE:
352 LdxpOp = AArch64::LDAXPX;
353 StxpOp = AArch64::STXPX;
354 break;
355 case AArch64::CMP_SWAP_128:
356 LdxpOp = AArch64::LDAXPX;
357 StxpOp = AArch64::STLXPX;
358 break;
359 default:
360 llvm_unreachable("Unexpected opcode");
361 }
362
363 MachineFunction *MF = MBB.getParent();
364 auto LoadCmpBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
365 auto StoreBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
366 auto FailBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
367 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
368
369 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoadCmpBB);
370 MF->insert(MBBI: ++LoadCmpBB->getIterator(), MBB: StoreBB);
371 MF->insert(MBBI: ++StoreBB->getIterator(), MBB: FailBB);
372 MF->insert(MBBI: ++FailBB->getIterator(), MBB: DoneBB);
373
374 // .Lloadcmp:
375 // ldaxp xDestLo, xDestHi, [xAddr]
376 // cmp xDestLo, xDesiredLo
377 // sbcs xDestHi, xDesiredHi
378 // b.ne .Ldone
379 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: LdxpOp))
380 .addReg(RegNo: Dest0.getReg(), Flags: RegState::Define)
381 .addReg(RegNo: Dest1.getReg(), Flags: RegState::Define)
382 .addReg(RegNo: AddrReg);
383 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::SUBSXrs), DestReg: AArch64::XZR)
384 .addReg(RegNo: DestLo.getReg(), Flags: getKillRegState(B: DestLo.isDead()))
385 .addReg(RegNo: DesiredLoReg)
386 .addImm(Val: 0);
387 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CSINCWr), DestReg: StatusReg)
388 .addUse(RegNo: AArch64::WZR)
389 .addUse(RegNo: AArch64::WZR)
390 .addImm(Val: AArch64CC::EQ);
391 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::SUBSXrs), DestReg: AArch64::XZR)
392 .addReg(RegNo: DestHi.getReg(), Flags: getKillRegState(B: DestHi.isDead()))
393 .addReg(RegNo: DesiredHiReg)
394 .addImm(Val: 0);
395 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CSINCWr), DestReg: StatusReg)
396 .addUse(RegNo: StatusReg, Flags: RegState::Kill)
397 .addUse(RegNo: StatusReg, Flags: RegState::Kill)
398 .addImm(Val: AArch64CC::EQ);
399 BuildMI(BB: LoadCmpBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
400 .addUse(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
401 .addMBB(MBB: FailBB);
402 LoadCmpBB->addSuccessor(Succ: FailBB);
403 LoadCmpBB->addSuccessor(Succ: StoreBB);
404
405 // .Lstore:
406 // stlxp wStatus, xNewLo, xNewHi, [xAddr]
407 // cbnz wStatus, .Lloadcmp
408 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: StxpOp), DestReg: StatusReg)
409 .addReg(RegNo: New0Reg)
410 .addReg(RegNo: New1Reg)
411 .addReg(RegNo: AddrReg);
412 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
413 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
414 .addMBB(MBB: LoadCmpBB);
415 BuildMI(BB: StoreBB, MIMD, MCID: TII->get(Opcode: AArch64::B)).addMBB(MBB: DoneBB);
416 StoreBB->addSuccessor(Succ: LoadCmpBB);
417 StoreBB->addSuccessor(Succ: DoneBB);
418
419 // .Lfail:
420 // stlxp wStatus, xDestLo, xDestHi, [xAddr]
421 // cbnz wStatus, .Lloadcmp
422 BuildMI(BB: FailBB, MIMD, MCID: TII->get(Opcode: StxpOp), DestReg: StatusReg)
423 .addReg(RegNo: Dest0.getReg())
424 .addReg(RegNo: Dest1.getReg())
425 .addReg(RegNo: AddrReg);
426 BuildMI(BB: FailBB, MIMD, MCID: TII->get(Opcode: AArch64::CBNZW))
427 .addReg(RegNo: StatusReg, Flags: getKillRegState(B: StatusDead))
428 .addMBB(MBB: LoadCmpBB);
429 FailBB->addSuccessor(Succ: LoadCmpBB);
430 FailBB->addSuccessor(Succ: DoneBB);
431
432 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
433 DoneBB->transferSuccessors(FromMBB: &MBB);
434
435 MBB.addSuccessor(Succ: LoadCmpBB);
436
437 NextMBBI = MBB.end();
438 MI.eraseFromParent();
439
440 // Recompute liveness bottom up.
441 LivePhysRegs LiveRegs;
442 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
443 computeAndAddLiveIns(LiveRegs, MBB&: *FailBB);
444 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
445 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
446
447 // Do an extra pass in the loop to get the loop carried dependencies right.
448 FailBB->clearLiveIns();
449 computeAndAddLiveIns(LiveRegs, MBB&: *FailBB);
450 StoreBB->clearLiveIns();
451 computeAndAddLiveIns(LiveRegs, MBB&: *StoreBB);
452 LoadCmpBB->clearLiveIns();
453 computeAndAddLiveIns(LiveRegs, MBB&: *LoadCmpBB);
454
455 return true;
456}
457
458/// \brief Expand Pseudos to Instructions with destructive operands.
459///
460/// This mechanism uses MOVPRFX instructions for zeroing the false lanes
461/// or for fixing relaxed register allocation conditions to comply with
462/// the instructions register constraints. The latter case may be cheaper
463/// than setting the register constraints in the register allocator,
464/// since that will insert regular MOV instructions rather than MOVPRFX.
465///
466/// Example (after register allocation):
467///
468/// FSUB_ZPZZ_ZERO_B Z0, Pg, Z1, Z0
469///
470/// * The Pseudo FSUB_ZPZZ_ZERO_B maps to FSUB_ZPmZ_B.
471/// * We cannot map directly to FSUB_ZPmZ_B because the register
472/// constraints of the instruction are not met.
473/// * Also the _ZERO specifies the false lanes need to be zeroed.
474///
475/// We first try to see if the destructive operand == result operand,
476/// if not, we try to swap the operands, e.g.
477///
478/// FSUB_ZPmZ_B Z0, Pg/m, Z0, Z1
479///
480/// But because FSUB_ZPmZ is not commutative, this is semantically
481/// different, so we need a reverse instruction:
482///
483/// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1
484///
485/// Then we implement the zeroing of the false lanes of Z0 by adding
486/// a zeroing MOVPRFX instruction:
487///
488/// MOVPRFX_ZPzZ_B Z0, Pg/z, Z0
489/// FSUBR_ZPmZ_B Z0, Pg/m, Z0, Z1
490///
491/// Note that this can only be done for _ZERO or _UNDEF variants where
492/// we can guarantee the false lanes to be zeroed (by implementing this)
493/// or that they are undef (don't care / not used), otherwise the
494/// swapping of operands is illegal because the operation is not
495/// (or cannot be emulated to be) fully commutative.
496bool AArch64ExpandPseudoImpl::expand_DestructiveOp(
497 MachineInstr &MI, MachineBasicBlock &MBB,
498 MachineBasicBlock::iterator MBBI) {
499 unsigned Opcode = AArch64::getSVEPseudoMap(Opcode: MI.getOpcode());
500 uint64_t DType = TII->get(Opcode).TSFlags & AArch64::DestructiveInstTypeMask;
501 uint64_t FalseLanes = MI.getDesc().TSFlags & AArch64::FalseLanesMask;
502 bool FalseZero = FalseLanes == AArch64::FalseLanesZero;
503 Register DstReg = MI.getOperand(i: 0).getReg();
504 bool DstIsDead = MI.getOperand(i: 0).isDead();
505 bool UseRev = false;
506 unsigned PredIdx, DOPIdx, SrcIdx, Src2Idx;
507
508 switch (DType) {
509 case AArch64::DestructiveBinaryComm:
510 case AArch64::DestructiveBinaryCommWithRev:
511 if (DstReg == MI.getOperand(i: 3).getReg()) {
512 // FSUB Zd, Pg, Zs1, Zd ==> FSUBR Zd, Pg/m, Zd, Zs1
513 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 3, args: 2);
514 UseRev = true;
515 break;
516 }
517 [[fallthrough]];
518 case AArch64::DestructiveBinary:
519 case AArch64::DestructiveBinaryImm:
520 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 2, args: 3);
521 break;
522 case AArch64::DestructiveUnaryPassthru:
523 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 2, args: 3, args: 3);
524 break;
525 case AArch64::DestructiveTernaryCommWithRev:
526 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 2, args: 3, args: 4);
527 if (DstReg == MI.getOperand(i: 3).getReg()) {
528 // FMLA Zd, Pg, Za, Zd, Zm ==> FMAD Zdn, Pg, Zm, Za
529 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 3, args: 4, args: 2);
530 UseRev = true;
531 } else if (DstReg == MI.getOperand(i: 4).getReg()) {
532 // FMLA Zd, Pg, Za, Zm, Zd ==> FMAD Zdn, Pg, Zm, Za
533 std::tie(args&: PredIdx, args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 4, args: 3, args: 2);
534 UseRev = true;
535 }
536 break;
537 case AArch64::Destructive2xRegImmUnpred:
538 // EXT_ZZI_CONSTRUCTIVE Zd, Zs, Imm
539 // ==> MOVPRFX Zd Zs; EXT_ZZI Zd, Zd, Zs, Imm
540 std::tie(args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 1, args: 2);
541 break;
542 case AArch64::DestructiveBinaryImmUnpred:
543 std::tie(args&: DOPIdx, args&: SrcIdx) = std::make_tuple(args: 1, args: 2);
544 break;
545 case AArch64::DestructiveBinaryShImmUnpred:
546 std::tie(args&: DOPIdx, args&: SrcIdx, args&: Src2Idx) = std::make_tuple(args: 1, args: 2, args: 3);
547 break;
548 default:
549 llvm_unreachable("Unsupported Destructive Operand type");
550 }
551
552 // MOVPRFX can only be used if the destination operand
553 // is the destructive operand, not as any other operand,
554 // so the Destructive Operand must be unique.
555 bool DOPRegIsUnique = false;
556 switch (DType) {
557 case AArch64::DestructiveBinary:
558 DOPRegIsUnique = DstReg != MI.getOperand(i: SrcIdx).getReg();
559 break;
560 case AArch64::DestructiveBinaryComm:
561 case AArch64::DestructiveBinaryCommWithRev:
562 DOPRegIsUnique =
563 DstReg != MI.getOperand(i: DOPIdx).getReg() ||
564 MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: SrcIdx).getReg();
565 break;
566 case AArch64::DestructiveUnaryPassthru:
567 case AArch64::DestructiveBinaryImm:
568 case AArch64::DestructiveBinaryImmUnpred:
569 case AArch64::DestructiveBinaryShImmUnpred:
570 case AArch64::Destructive2xRegImmUnpred:
571 DOPRegIsUnique = true;
572 break;
573 case AArch64::DestructiveTernaryCommWithRev:
574 DOPRegIsUnique =
575 DstReg != MI.getOperand(i: DOPIdx).getReg() ||
576 (MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: SrcIdx).getReg() &&
577 MI.getOperand(i: DOPIdx).getReg() != MI.getOperand(i: Src2Idx).getReg());
578 break;
579 }
580
581 // Resolve the reverse opcode
582 if (UseRev) {
583 int NewOpcode;
584 // e.g. DIV -> DIVR
585 if ((NewOpcode = AArch64::getSVERevInstr(Opcode)) != -1)
586 Opcode = NewOpcode;
587 // e.g. DIVR -> DIV
588 else if ((NewOpcode = AArch64::getSVENonRevInstr(Opcode)) != -1)
589 Opcode = NewOpcode;
590 }
591
592 // Get the right MOVPRFX
593 uint64_t ElementSize = TII->getElementSizeForOpcode(Opc: Opcode);
594 unsigned MovPrfx, LSLZero, MovPrfxZero;
595 switch (ElementSize) {
596 case AArch64::ElementSizeNone:
597 case AArch64::ElementSizeB:
598 MovPrfx = AArch64::MOVPRFX_ZZ;
599 LSLZero = AArch64::LSL_ZPmI_B;
600 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_B;
601 break;
602 case AArch64::ElementSizeH:
603 MovPrfx = AArch64::MOVPRFX_ZZ;
604 LSLZero = AArch64::LSL_ZPmI_H;
605 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_H;
606 break;
607 case AArch64::ElementSizeS:
608 MovPrfx = AArch64::MOVPRFX_ZZ;
609 LSLZero = AArch64::LSL_ZPmI_S;
610 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_S;
611 break;
612 case AArch64::ElementSizeD:
613 MovPrfx = AArch64::MOVPRFX_ZZ;
614 LSLZero = AArch64::LSL_ZPmI_D;
615 MovPrfxZero = AArch64::MOVPRFX_ZPzZ_D;
616 break;
617 default:
618 llvm_unreachable("Unsupported ElementSize");
619 }
620
621 // Preserve undef state until DOP's reg is defined.
622 RegState DOPRegState = getUndefRegState(B: MI.getOperand(i: DOPIdx).isUndef());
623
624 //
625 // Create the destructive operation (if required)
626 //
627 MachineInstrBuilder PRFX, DOP;
628 if (FalseZero) {
629 // If we cannot prefix the requested instruction we'll instead emit a
630 // prefixed_zeroing_mov for DestructiveBinary.
631 assert((DOPRegIsUnique || DType == AArch64::DestructiveBinary ||
632 DType == AArch64::DestructiveBinaryComm ||
633 DType == AArch64::DestructiveBinaryCommWithRev) &&
634 "The destructive operand should be unique");
635 assert(ElementSize != AArch64::ElementSizeNone &&
636 "This instruction is unpredicated");
637
638 // Merge source operand into destination register
639 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: MovPrfxZero))
640 .addReg(RegNo: DstReg, Flags: RegState::Define)
641 .addReg(RegNo: MI.getOperand(i: PredIdx).getReg())
642 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState);
643
644 // After the movprfx, the destructive operand is same as Dst
645 DOPIdx = 0;
646 DOPRegState = {};
647
648 // Create the additional LSL to zero the lanes when the DstReg is not
649 // unique. Zeros the lanes in z0 that aren't active in p0 with sequence
650 // movprfx z0.b, p0/z, z0.b; lsl z0.b, p0/m, z0.b, #0;
651 if ((DType == AArch64::DestructiveBinary ||
652 DType == AArch64::DestructiveBinaryComm ||
653 DType == AArch64::DestructiveBinaryCommWithRev) &&
654 !DOPRegIsUnique) {
655 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: LSLZero))
656 .addReg(RegNo: DstReg, Flags: RegState::Define)
657 .add(MO: MI.getOperand(i: PredIdx))
658 .addReg(RegNo: DstReg)
659 .addImm(Val: 0);
660 }
661 } else if (DstReg != MI.getOperand(i: DOPIdx).getReg()) {
662 assert(DOPRegIsUnique && "The destructive operand should be unique");
663 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: MovPrfx))
664 .addReg(RegNo: DstReg, Flags: RegState::Define)
665 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState);
666 DOPIdx = 0;
667 DOPRegState = {};
668 }
669
670 //
671 // Create the destructive operation
672 //
673 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode))
674 .addReg(RegNo: DstReg, Flags: RegState::Define | getDeadRegState(B: DstIsDead));
675 DOPRegState = DOPRegState | RegState::Kill;
676
677 switch (DType) {
678 case AArch64::DestructiveUnaryPassthru:
679 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
680 .add(MO: MI.getOperand(i: PredIdx))
681 .add(MO: MI.getOperand(i: SrcIdx));
682 break;
683 case AArch64::DestructiveBinary:
684 case AArch64::DestructiveBinaryImm:
685 case AArch64::DestructiveBinaryComm:
686 case AArch64::DestructiveBinaryCommWithRev:
687 DOP.add(MO: MI.getOperand(i: PredIdx))
688 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
689 .add(MO: MI.getOperand(i: SrcIdx));
690 break;
691 case AArch64::DestructiveTernaryCommWithRev:
692 DOP.add(MO: MI.getOperand(i: PredIdx))
693 .addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
694 .add(MO: MI.getOperand(i: SrcIdx))
695 .add(MO: MI.getOperand(i: Src2Idx));
696 break;
697 case AArch64::DestructiveBinaryImmUnpred:
698 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
699 .add(MO: MI.getOperand(i: SrcIdx));
700 break;
701 case AArch64::DestructiveBinaryShImmUnpred:
702 case AArch64::Destructive2xRegImmUnpred:
703 DOP.addReg(RegNo: MI.getOperand(i: DOPIdx).getReg(), Flags: DOPRegState)
704 .add(MO: MI.getOperand(i: SrcIdx))
705 .add(MO: MI.getOperand(i: Src2Idx));
706 break;
707 }
708
709 if (PRFX) {
710 transferImpOps(OldMI&: MI, UseMI&: PRFX, DefMI&: DOP);
711 finalizeBundle(MBB, FirstMI: PRFX->getIterator(), LastMI: MBBI->getIterator());
712 } else
713 transferImpOps(OldMI&: MI, UseMI&: DOP, DefMI&: DOP);
714
715 MI.eraseFromParent();
716 return true;
717}
718
719bool AArch64ExpandPseudoImpl::expandSVEBitwisePseudo(
720 MachineInstr &MI, MachineBasicBlock &MBB,
721 MachineBasicBlock::iterator MBBI) {
722 MachineInstrBuilder PRFX, DOP;
723 const unsigned Opcode = MI.getOpcode();
724 const MachineOperand &Op0 = MI.getOperand(i: 0);
725 const MachineOperand *Op1 = &MI.getOperand(i: 1);
726 const MachineOperand *Op2 = &MI.getOperand(i: 2);
727 const Register DOPReg = Op0.getReg();
728
729 if (DOPReg == Op2->getReg()) {
730 // Commute the operands to allow destroying the second source.
731 std::swap(a&: Op1, b&: Op2);
732 } else if (DOPReg != Op1->getReg()) {
733 // If not in destructive form, emit a MOVPRFX. The input should only be
734 // killed if unused by the subsequent instruction.
735 PRFX = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MOVPRFX_ZZ))
736 .addDef(RegNo: DOPReg, Flags: getRenamableRegState(B: Op0.isRenamable()))
737 .addReg(RegNo: Op1->getReg(),
738 Flags: getRenamableRegState(B: Op1->isRenamable()) |
739 getUndefRegState(B: Op1->isUndef()) |
740 getKillRegState(B: Op1->isKill() &&
741 Opcode == AArch64::NAND_ZZZ));
742 }
743
744 assert((DOPReg == Op1->getReg() || PRFX) && "invalid expansion");
745
746 const RegState DOPRegState = getRenamableRegState(B: Op0.isRenamable()) |
747 getUndefRegState(B: !PRFX && Op1->isUndef()) |
748 RegState::Kill;
749
750 switch (Opcode) {
751 default:
752 llvm_unreachable("unhandled opcode");
753 case AArch64::EON_ZZZ:
754 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::BSL2N_ZZZZ))
755 .add(MO: Op0)
756 .addReg(RegNo: DOPReg, Flags: DOPRegState)
757 .add(MO: *Op1)
758 .add(MO: *Op2);
759 break;
760 case AArch64::NAND_ZZZ:
761 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::NBSL_ZZZZ))
762 .add(MO: Op0)
763 .addReg(RegNo: DOPReg, Flags: DOPRegState)
764 .add(MO: *Op2)
765 .add(MO: *Op2);
766 break;
767 case AArch64::NOR_ZZZ:
768 DOP = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::NBSL_ZZZZ))
769 .add(MO: Op0)
770 .addReg(RegNo: DOPReg, Flags: DOPRegState)
771 .add(MO: *Op2)
772 .add(MO: *Op1);
773 break;
774 }
775
776 if (PRFX) {
777 transferImpOps(OldMI&: MI, UseMI&: PRFX, DefMI&: DOP);
778 finalizeBundle(MBB, FirstMI: PRFX->getIterator(), LastMI: MBBI->getIterator());
779 } else {
780 transferImpOps(OldMI&: MI, UseMI&: DOP, DefMI&: DOP);
781 }
782
783 MI.eraseFromParent();
784 return true;
785}
786
787bool AArch64ExpandPseudoImpl::expandSetTagLoop(
788 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
789 MachineBasicBlock::iterator &NextMBBI) {
790 MachineInstr &MI = *MBBI;
791 DebugLoc DL = MI.getDebugLoc();
792 Register SizeReg = MI.getOperand(i: 0).getReg();
793 Register AddressReg = MI.getOperand(i: 1).getReg();
794
795 MachineFunction *MF = MBB.getParent();
796
797 bool ZeroData = MI.getOpcode() == AArch64::STZGloop_wback;
798 const unsigned OpCode1 =
799 ZeroData ? AArch64::STZGPostIndex : AArch64::STGPostIndex;
800 const unsigned OpCode2 =
801 ZeroData ? AArch64::STZ2GPostIndex : AArch64::ST2GPostIndex;
802
803 unsigned Size = MI.getOperand(i: 2).getImm();
804 assert(Size > 0 && Size % 16 == 0);
805 if (Size % (16 * 2) != 0) {
806 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: OpCode1), DestReg: AddressReg)
807 .addReg(RegNo: AddressReg)
808 .addReg(RegNo: AddressReg)
809 .addImm(Val: 1);
810 Size -= 16;
811 }
812 MachineBasicBlock::iterator I =
813 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVi64imm), DestReg: SizeReg)
814 .addImm(Val: Size);
815 expandMOVImm(MBB, MBBI: I, BitSize: 64);
816
817 auto LoopBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
818 auto DoneBB = MF->CreateMachineBasicBlock(BB: MBB.getBasicBlock());
819
820 MF->insert(MBBI: ++MBB.getIterator(), MBB: LoopBB);
821 MF->insert(MBBI: ++LoopBB->getIterator(), MBB: DoneBB);
822
823 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: OpCode2))
824 .addDef(RegNo: AddressReg)
825 .addReg(RegNo: AddressReg)
826 .addReg(RegNo: AddressReg)
827 .addImm(Val: 2)
828 .cloneMemRefs(OtherMI: MI)
829 .setMIFlags(MI.getFlags());
830 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::SUBSXri))
831 .addDef(RegNo: SizeReg)
832 .addReg(RegNo: SizeReg)
833 .addImm(Val: 16 * 2)
834 .addImm(Val: 0);
835 BuildMI(BB: LoopBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::Bcc))
836 .addImm(Val: AArch64CC::NE)
837 .addMBB(MBB: LoopBB)
838 .addReg(RegNo: AArch64::NZCV, Flags: RegState::Implicit | RegState::Kill);
839
840 LoopBB->addSuccessor(Succ: LoopBB);
841 LoopBB->addSuccessor(Succ: DoneBB);
842
843 DoneBB->splice(Where: DoneBB->end(), Other: &MBB, From: MI, To: MBB.end());
844 DoneBB->transferSuccessors(FromMBB: &MBB);
845
846 MBB.addSuccessor(Succ: LoopBB);
847
848 NextMBBI = MBB.end();
849 MI.eraseFromParent();
850 // Recompute liveness bottom up.
851 LivePhysRegs LiveRegs;
852 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
853 computeAndAddLiveIns(LiveRegs, MBB&: *LoopBB);
854 // Do an extra pass in the loop to get the loop carried dependencies right.
855 // FIXME: is this necessary?
856 LoopBB->clearLiveIns();
857 computeAndAddLiveIns(LiveRegs, MBB&: *LoopBB);
858 DoneBB->clearLiveIns();
859 computeAndAddLiveIns(LiveRegs, MBB&: *DoneBB);
860
861 return true;
862}
863
864bool AArch64ExpandPseudoImpl::expandSVESpillFill(
865 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, unsigned Opc,
866 unsigned N) {
867 assert((Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI ||
868 Opc == AArch64::LDR_PXI || Opc == AArch64::STR_PXI) &&
869 "Unexpected opcode");
870 RegState RState =
871 getDefRegState(B: Opc == AArch64::LDR_ZXI || Opc == AArch64::LDR_PXI);
872 unsigned sub0 = (Opc == AArch64::LDR_ZXI || Opc == AArch64::STR_ZXI)
873 ? AArch64::zsub0
874 : AArch64::psub0;
875 const TargetRegisterInfo *TRI =
876 MBB.getParent()->getSubtarget().getRegisterInfo();
877 MachineInstr &MI = *MBBI;
878 for (unsigned Offset = 0; Offset < N; ++Offset) {
879 int ImmOffset = MI.getOperand(i: 2).getImm() + Offset;
880 bool Kill = (Offset + 1 == N) ? MI.getOperand(i: 1).isKill() : false;
881 assert(ImmOffset >= -256 && ImmOffset < 256 &&
882 "Immediate spill offset out of range");
883 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
884 .addReg(RegNo: TRI->getSubReg(Reg: MI.getOperand(i: 0).getReg(), Idx: sub0 + Offset),
885 Flags: RState)
886 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: getKillRegState(B: Kill))
887 .addImm(Val: ImmOffset);
888 }
889 MI.eraseFromParent();
890 return true;
891}
892
893// Create a call with the passed opcode and explicit operands, copying over all
894// the implicit operands from *MBBI, starting at the regmask.
895static MachineInstr *createCallWithOps(MachineBasicBlock &MBB,
896 MachineBasicBlock::iterator MBBI,
897 const AArch64InstrInfo *TII,
898 unsigned Opcode,
899 ArrayRef<MachineOperand> ExplicitOps,
900 unsigned RegMaskStartIdx) {
901 // Build the MI, with explicit operands first (including the call target).
902 MachineInstr *Call = BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode))
903 .add(MOs: ExplicitOps)
904 .getInstr();
905
906 // Register arguments are added during ISel, but cannot be added as explicit
907 // operands of the branch as it expects to be B <target> which is only one
908 // operand. Instead they are implicit operands used by the branch.
909 while (!MBBI->getOperand(i: RegMaskStartIdx).isRegMask()) {
910 const MachineOperand &MOP = MBBI->getOperand(i: RegMaskStartIdx);
911 assert(MOP.isReg() && "can only add register operands");
912 Call->addOperand(Op: MachineOperand::CreateReg(
913 Reg: MOP.getReg(), /*Def=*/isDef: false, /*Implicit=*/isImp: true, /*isKill=*/false,
914 /*isDead=*/false, /*isUndef=*/MOP.isUndef()));
915 RegMaskStartIdx++;
916 }
917 for (const MachineOperand &MO :
918 llvm::drop_begin(RangeOrContainer: MBBI->operands(), N: RegMaskStartIdx))
919 Call->addOperand(Op: MO);
920
921 return Call;
922}
923
924// Create a call to CallTarget, copying over all the operands from *MBBI,
925// starting at the regmask.
926static MachineInstr *createCall(MachineBasicBlock &MBB,
927 MachineBasicBlock::iterator MBBI,
928 const AArch64InstrInfo *TII,
929 MachineOperand &CallTarget,
930 unsigned RegMaskStartIdx) {
931 unsigned Opc = CallTarget.isGlobal() ? AArch64::BL : AArch64::BLR;
932
933 assert((CallTarget.isGlobal() || CallTarget.isReg()) &&
934 "invalid operand for regular call");
935 return createCallWithOps(MBB, MBBI, TII, Opcode: Opc, ExplicitOps: CallTarget, RegMaskStartIdx);
936}
937
938bool AArch64ExpandPseudoImpl::expandCALL_RVMARKER(
939 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
940 // Expand CALL_RVMARKER pseudo to:
941 // - a branch to the call target, followed by
942 // - the special `mov x29, x29` marker, if necessary, and
943 // - another branch, to the runtime function
944 // Mark the sequence as bundle, to avoid passes moving other code in between.
945 MachineInstr &MI = *MBBI;
946 MachineOperand &RVTarget = MI.getOperand(i: 0);
947 bool DoEmitMarker = MI.getOperand(i: 1).getImm();
948 assert(RVTarget.isGlobal() && "invalid operand for attached call");
949
950 MachineInstr *OriginalCall = nullptr;
951
952 if (MI.getOpcode() == AArch64::BLRA_RVMARKER) {
953 // ptrauth call.
954 const MachineOperand &CallTarget = MI.getOperand(i: 2);
955 const MachineOperand &Key = MI.getOperand(i: 3);
956 const MachineOperand &IntDisc = MI.getOperand(i: 4);
957 const MachineOperand &AddrDisc = MI.getOperand(i: 5);
958
959 assert((Key.getImm() == AArch64PACKey::IA ||
960 Key.getImm() == AArch64PACKey::IB) &&
961 "Invalid auth call key");
962
963 MachineOperand Ops[] = {CallTarget, Key, IntDisc, AddrDisc};
964
965 OriginalCall = createCallWithOps(MBB, MBBI, TII, Opcode: AArch64::BLRA, ExplicitOps: Ops,
966 /*RegMaskStartIdx=*/6);
967 } else {
968 assert(MI.getOpcode() == AArch64::BLR_RVMARKER && "unknown rvmarker MI");
969 OriginalCall = createCall(MBB, MBBI, TII, CallTarget&: MI.getOperand(i: 2),
970 // Regmask starts after the RV and call targets.
971 /*RegMaskStartIdx=*/3);
972 }
973
974 if (DoEmitMarker)
975 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ORRXrs))
976 .addReg(RegNo: AArch64::FP, Flags: RegState::Define)
977 .addReg(RegNo: AArch64::XZR)
978 .addReg(RegNo: AArch64::FP)
979 .addImm(Val: 0);
980
981 auto *RVCall = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::BL))
982 .add(MO: RVTarget)
983 .getInstr();
984
985 if (MI.shouldUpdateAdditionalCallInfo())
986 MBB.getParent()->moveAdditionalCallInfo(Old: &MI, New: OriginalCall);
987
988 MI.eraseFromParent();
989 finalizeBundle(MBB, FirstMI: OriginalCall->getIterator(),
990 LastMI: std::next(x: RVCall->getIterator()));
991 return true;
992}
993
994bool AArch64ExpandPseudoImpl::expandCALL_BTI(MachineBasicBlock &MBB,
995 MachineBasicBlock::iterator MBBI) {
996 // Expand CALL_BTI pseudo to:
997 // - a branch to the call target
998 // - a BTI instruction
999 // Mark the sequence as a bundle, to avoid passes moving other code in
1000 // between.
1001 MachineInstr &MI = *MBBI;
1002 MachineInstr *Call = createCall(MBB, MBBI, TII, CallTarget&: MI.getOperand(i: 0),
1003 // Regmask starts after the call target.
1004 /*RegMaskStartIdx=*/1);
1005
1006 Call->setCFIType(MF&: *MBB.getParent(), Type: MI.getCFIType());
1007
1008 MachineInstr *BTI =
1009 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::HINT))
1010 // BTI J so that setjmp can to BR to this.
1011 .addImm(Val: 36)
1012 .getInstr();
1013
1014 if (MI.shouldUpdateAdditionalCallInfo())
1015 MBB.getParent()->moveAdditionalCallInfo(Old: &MI, New: Call);
1016
1017 MI.eraseFromParent();
1018 finalizeBundle(MBB, FirstMI: Call->getIterator(), LastMI: std::next(x: BTI->getIterator()));
1019 return true;
1020}
1021
1022bool AArch64ExpandPseudoImpl::expandStoreSwiftAsyncContext(
1023 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) {
1024 Register CtxReg = MBBI->getOperand(i: 0).getReg();
1025 Register BaseReg = MBBI->getOperand(i: 1).getReg();
1026 int Offset = MBBI->getOperand(i: 2).getImm();
1027 DebugLoc DL(MBBI->getDebugLoc());
1028 auto &STI = MBB.getParent()->getSubtarget<AArch64Subtarget>();
1029
1030 if (STI.getTargetTriple().getArchName() != "arm64e") {
1031 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
1032 .addUse(RegNo: CtxReg)
1033 .addUse(RegNo: BaseReg)
1034 .addImm(Val: Offset / 8)
1035 .setMIFlag(MachineInstr::FrameSetup);
1036 MBBI->eraseFromParent();
1037 return true;
1038 }
1039
1040 // We need to sign the context in an address-discriminated way. 0xc31a is a
1041 // fixed random value, chosen as part of the ABI.
1042 // add x16, xBase, #Offset
1043 // movk x16, #0xc31a, lsl #48
1044 // mov x17, x22/xzr
1045 // pacdb x17, x16
1046 // str x17, [xBase, #Offset]
1047 unsigned Opc = Offset >= 0 ? AArch64::ADDXri : AArch64::SUBXri;
1048 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc), DestReg: AArch64::X16)
1049 .addUse(RegNo: BaseReg)
1050 .addImm(Val: abs(x: Offset))
1051 .addImm(Val: 0)
1052 .setMIFlag(MachineInstr::FrameSetup);
1053 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::MOVKXi), DestReg: AArch64::X16)
1054 .addUse(RegNo: AArch64::X16)
1055 .addImm(Val: 0xc31a)
1056 .addImm(Val: 48)
1057 .setMIFlag(MachineInstr::FrameSetup);
1058 // We're not allowed to clobber X22 (and couldn't clobber XZR if we tried), so
1059 // move it somewhere before signing.
1060 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ORRXrs), DestReg: AArch64::X17)
1061 .addUse(RegNo: AArch64::XZR)
1062 .addUse(RegNo: CtxReg)
1063 .addImm(Val: 0)
1064 .setMIFlag(MachineInstr::FrameSetup);
1065 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::PACDB), DestReg: AArch64::X17)
1066 .addUse(RegNo: AArch64::X17)
1067 .addUse(RegNo: AArch64::X16)
1068 .setMIFlag(MachineInstr::FrameSetup);
1069 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::STRXui))
1070 .addUse(RegNo: AArch64::X17)
1071 .addUse(RegNo: BaseReg)
1072 .addImm(Val: Offset / 8)
1073 .setMIFlag(MachineInstr::FrameSetup);
1074
1075 MBBI->eraseFromParent();
1076 return true;
1077}
1078
1079AArch64ExpandPseudoImpl::ConditionalBlocks
1080AArch64ExpandPseudoImpl::expandConditionalPseudo(
1081 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, DebugLoc DL,
1082 MachineInstrBuilder &Branch) {
1083 assert((std::next(MBBI) != MBB.end() ||
1084 MBB.successors().begin() != MBB.successors().end()) &&
1085 "Unexpected unreachable in block");
1086
1087 // Split MBB and create two new blocks:
1088 // - MBB now contains all instructions before the conditional pseudo.
1089 // - CondBB contains the conditional pseudo instruction only.
1090 // - EndBB contains all instructions after the conditional pseudo.
1091 MachineInstr &PrevMI = *std::prev(x: MBBI);
1092 MachineBasicBlock *CondBB = MBB.splitAt(SplitInst&: PrevMI, /*UpdateLiveIns*/ true);
1093 MachineBasicBlock *EndBB =
1094 std::next(x: MBBI) == CondBB->end()
1095 ? *CondBB->successors().begin()
1096 : CondBB->splitAt(SplitInst&: *MBBI, /*UpdateLiveIns*/ true);
1097
1098 // Add the SMBB label to the branch instruction & create a branch to EndBB.
1099 Branch.addMBB(MBB: CondBB);
1100 BuildMI(BB: &MBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::B))
1101 .addMBB(MBB: EndBB);
1102 MBB.addSuccessor(Succ: EndBB);
1103
1104 // Create branch from CondBB to EndBB. Users of this helper should insert new
1105 // instructions at CondBB.back() -- i.e. before the branch.
1106 BuildMI(BB: CondBB, MIMD: DL, MCID: TII->get(Opcode: AArch64::B)).addMBB(MBB: EndBB);
1107 return {.CondBB: *CondBB, .EndBB: *EndBB};
1108}
1109
1110MachineBasicBlock *
1111AArch64ExpandPseudoImpl::expandRestoreZASave(MachineBasicBlock &MBB,
1112 MachineBasicBlock::iterator MBBI) {
1113 MachineInstr &MI = *MBBI;
1114 DebugLoc DL = MI.getDebugLoc();
1115
1116 // Compare TPIDR2_EL0 against 0. Restore ZA if TPIDR2_EL0 is zero.
1117 MachineInstrBuilder Branch =
1118 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::CBZX)).add(MO: MI.getOperand(i: 0));
1119
1120 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);
1121 // Replace the pseudo with a call (BL).
1122 MachineInstrBuilder MIB =
1123 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::BL));
1124 // Copy operands (mainly the regmask) from the pseudo.
1125 for (unsigned I = 2; I < MI.getNumOperands(); ++I)
1126 MIB.add(MO: MI.getOperand(i: I));
1127 // Mark the TPIDR2 block pointer (X0) as an implicit use.
1128 MIB.addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState::Implicit);
1129
1130 MI.eraseFromParent();
1131 return &EndBB;
1132}
1133
1134static constexpr unsigned ZERO_ALL_ZA_MASK = 0b11111111;
1135
1136MachineBasicBlock *
1137AArch64ExpandPseudoImpl::expandCommitZASave(MachineBasicBlock &MBB,
1138 MachineBasicBlock::iterator MBBI) {
1139 MachineInstr &MI = *MBBI;
1140 DebugLoc DL = MI.getDebugLoc();
1141 [[maybe_unused]] auto *RI = MBB.getParent()->getSubtarget().getRegisterInfo();
1142
1143 // Compare TPIDR2_EL0 against 0. Commit ZA if TPIDR2_EL0 is non-zero.
1144 MachineInstrBuilder Branch =
1145 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::CBNZX)).add(MO: MI.getOperand(i: 0));
1146
1147 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch);
1148 // Replace the pseudo with a call (BL).
1149 MachineInstrBuilder MIB =
1150 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::BL));
1151 // Copy operands (mainly the regmask) from the pseudo.
1152 for (unsigned I = 3; I < MI.getNumOperands(); ++I)
1153 MIB.add(MO: MI.getOperand(i: I));
1154 // Clear TPIDR2_EL0.
1155 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::MSR))
1156 .addImm(Val: AArch64SysReg::TPIDR2_EL0)
1157 .addReg(RegNo: AArch64::XZR);
1158 bool ZeroZA = MI.getOperand(i: 1).getImm() != 0;
1159 bool ZeroZT0 = MI.getOperand(i: 2).getImm() != 0;
1160 if (ZeroZA) {
1161 assert(MI.definesRegister(AArch64::ZAB0, RI) && "should define ZA!");
1162 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ZERO_M))
1163 .addImm(Val: ZERO_ALL_ZA_MASK)
1164 .addDef(RegNo: AArch64::ZAB0, Flags: RegState::ImplicitDefine);
1165 }
1166 if (ZeroZT0) {
1167 assert(MI.definesRegister(AArch64::ZT0, RI) && "should define ZT0!");
1168 BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: DL, MCID: TII->get(Opcode: AArch64::ZERO_T))
1169 .addDef(RegNo: AArch64::ZT0);
1170 }
1171
1172 MI.eraseFromParent();
1173 return &EndBB;
1174}
1175
1176MachineBasicBlock *
1177AArch64ExpandPseudoImpl::expandCondSMToggle(MachineBasicBlock &MBB,
1178 MachineBasicBlock::iterator MBBI) {
1179 MachineInstr &MI = *MBBI;
1180 // In the case of a smstart/smstop before a unreachable, just remove the pseudo.
1181 // Exception handling code generated by Clang may introduce unreachables and it
1182 // seems unnecessary to restore pstate.sm when that happens. Note that it is
1183 // not just an optimisation, the code below expects a successor instruction/block
1184 // in order to split the block at MBBI.
1185 if (std::next(x: MBBI) == MBB.end() &&
1186 MI.getParent()->successors().begin() ==
1187 MI.getParent()->successors().end()) {
1188 MI.eraseFromParent();
1189 return &MBB;
1190 }
1191
1192 // Expand the pseudo into smstart or smstop instruction. The pseudo has the
1193 // following operands:
1194 //
1195 // MSRpstatePseudo <za|sm|both>, <0|1>, condition[, pstate.sm], <regmask>
1196 //
1197 // The pseudo is expanded into a conditional smstart/smstop, with a
1198 // check if pstate.sm (register) equals the expected value, and if not,
1199 // invokes the smstart/smstop.
1200 //
1201 // As an example, the following block contains a normal call from a
1202 // streaming-compatible function:
1203 //
1204 // OrigBB:
1205 // MSRpstatePseudo 3, 0, IfCallerIsStreaming, %0, <regmask> <- Cond SMSTOP
1206 // bl @normal_callee
1207 // MSRpstatePseudo 3, 1, IfCallerIsStreaming, %0, <regmask> <- Cond SMSTART
1208 //
1209 // ...which will be transformed into:
1210 //
1211 // OrigBB:
1212 // TBNZx %0:gpr64, 0, SMBB
1213 // b EndBB
1214 //
1215 // SMBB:
1216 // MSRpstatesvcrImm1 3, 0, <regmask> <- SMSTOP
1217 //
1218 // EndBB:
1219 // bl @normal_callee
1220 // MSRcond_pstatesvcrImm1 3, 1, <regmask> <- SMSTART
1221 //
1222 DebugLoc DL = MI.getDebugLoc();
1223
1224 // Create the conditional branch based on the third operand of the
1225 // instruction, which tells us if we are wrapping a normal or streaming
1226 // function.
1227 // We test the live value of pstate.sm and toggle pstate.sm if this is not the
1228 // expected value for the callee (0 for a normal callee and 1 for a streaming
1229 // callee).
1230 unsigned Opc;
1231 switch (MI.getOperand(i: 2).getImm()) {
1232 case AArch64SME::Always:
1233 llvm_unreachable("Should have matched to instruction directly");
1234 case AArch64SME::IfCallerIsStreaming:
1235 Opc = AArch64::TBNZW;
1236 break;
1237 case AArch64SME::IfCallerIsNonStreaming:
1238 Opc = AArch64::TBZW;
1239 break;
1240 }
1241 auto PStateSM = MI.getOperand(i: 3).getReg();
1242 auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
1243 unsigned SMReg32 = TRI->getSubReg(Reg: PStateSM, Idx: AArch64::sub_32);
1244 MachineInstrBuilder Tbx =
1245 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: Opc)).addReg(RegNo: SMReg32).addImm(Val: 0);
1246
1247 auto [CondBB, EndBB] = expandConditionalPseudo(MBB, MBBI, DL, Branch&: Tbx);
1248 // Create the SMSTART/SMSTOP (MSRpstatesvcrImm1) instruction in SMBB.
1249 MachineInstrBuilder MIB = BuildMI(BB&: CondBB, I&: CondBB.back(), MIMD: MI.getDebugLoc(),
1250 MCID: TII->get(Opcode: AArch64::MSRpstatesvcrImm1));
1251 // Copy all but the second and third operands of MSRcond_pstatesvcrImm1 (as
1252 // these contain the CopyFromReg for the first argument and the flag to
1253 // indicate whether the callee is streaming or normal).
1254 MIB.add(MO: MI.getOperand(i: 0));
1255 MIB.add(MO: MI.getOperand(i: 1));
1256 for (unsigned i = 4; i < MI.getNumOperands(); ++i)
1257 MIB.add(MO: MI.getOperand(i));
1258
1259 MI.eraseFromParent();
1260 return &EndBB;
1261}
1262
1263bool AArch64ExpandPseudoImpl::expandMultiVecPseudo(
1264 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
1265 const TargetRegisterClass &ContiguousClass,
1266 const TargetRegisterClass &StridedClass, unsigned ContiguousOp,
1267 unsigned StridedOpc) {
1268 MachineInstr &MI = *MBBI;
1269 Register Tuple = MI.getOperand(i: 0).getReg();
1270
1271 auto ContiguousRange = ContiguousClass.getRegisters();
1272 auto StridedRange = StridedClass.getRegisters();
1273 unsigned Opc;
1274 if (llvm::is_contained(Range&: ContiguousRange, Element: Tuple.asMCReg())) {
1275 Opc = ContiguousOp;
1276 } else if (llvm::is_contained(Range&: StridedRange, Element: Tuple.asMCReg())) {
1277 Opc = StridedOpc;
1278 } else
1279 llvm_unreachable("Cannot expand Multi-Vector pseudo");
1280
1281 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: Opc))
1282 .add(MO: MI.getOperand(i: 0))
1283 .add(MO: MI.getOperand(i: 1))
1284 .add(MO: MI.getOperand(i: 2))
1285 .add(MO: MI.getOperand(i: 3));
1286 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1287 MI.eraseFromParent();
1288 return true;
1289}
1290
1291bool AArch64ExpandPseudoImpl::expandCopyIntoTuplePseudo(
1292 MachineInstr &MI, MachineBasicBlock &MBB,
1293 MachineBasicBlock::iterator MBBI) {
1294 Register Src = MI.getOperand(i: 1).getReg();
1295 Register Dest = MI.getOperand(i: 0).getReg();
1296
1297 if (Src != Dest)
1298 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ORR_ZZZ))
1299 .addReg(RegNo: Dest, Flags: RegState::Define)
1300 .addReg(RegNo: Src)
1301 .addReg(RegNo: Src);
1302
1303 MI.eraseFromParent();
1304 return true;
1305}
1306
1307/// If MBBI references a pseudo instruction that should be expanded here,
1308/// do the expansion and return true. Otherwise return false.
1309bool AArch64ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
1310 MachineBasicBlock::iterator MBBI,
1311 MachineBasicBlock::iterator &NextMBBI) {
1312 MachineInstr &MI = *MBBI;
1313 unsigned Opcode = MI.getOpcode();
1314
1315 // Check if we can expand the destructive op
1316 int OrigInstr = AArch64::getSVEPseudoMap(Opcode: MI.getOpcode());
1317 if (OrigInstr != -1) {
1318 auto &Orig = TII->get(Opcode: OrigInstr);
1319 if ((Orig.TSFlags & AArch64::DestructiveInstTypeMask) !=
1320 AArch64::NotDestructive) {
1321 return expand_DestructiveOp(MI, MBB, MBBI);
1322 }
1323 }
1324
1325 switch (Opcode) {
1326 default:
1327 break;
1328
1329 case AArch64::BSPv8i8:
1330 case AArch64::BSPv16i8: {
1331 Register DstReg = MI.getOperand(i: 0).getReg();
1332 if (DstReg == MI.getOperand(i: 3).getReg()) {
1333 // Expand to BIT
1334 auto I = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1335 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BITv8i8
1336 : AArch64::BITv16i8))
1337 .add(MO: MI.getOperand(i: 0))
1338 .add(MO: MI.getOperand(i: 3))
1339 .add(MO: MI.getOperand(i: 2))
1340 .add(MO: MI.getOperand(i: 1));
1341 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1342 } else if (DstReg == MI.getOperand(i: 2).getReg()) {
1343 // Expand to BIF
1344 auto I = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1345 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BIFv8i8
1346 : AArch64::BIFv16i8))
1347 .add(MO: MI.getOperand(i: 0))
1348 .add(MO: MI.getOperand(i: 2))
1349 .add(MO: MI.getOperand(i: 3))
1350 .add(MO: MI.getOperand(i: 1));
1351 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1352 } else {
1353 // Expand to BSL, use additional move if required
1354 if (DstReg == MI.getOperand(i: 1).getReg()) {
1355 auto I =
1356 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1357 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1358 : AArch64::BSLv16i8))
1359 .add(MO: MI.getOperand(i: 0))
1360 .add(MO: MI.getOperand(i: 1))
1361 .add(MO: MI.getOperand(i: 2))
1362 .add(MO: MI.getOperand(i: 3));
1363 transferImpOps(OldMI&: MI, UseMI&: I, DefMI&: I);
1364 } else {
1365 RegState RegState =
1366 getRenamableRegState(B: MI.getOperand(i: 1).isRenamable()) |
1367 getKillRegState(
1368 B: MI.getOperand(i: 1).isKill() &&
1369 MI.getOperand(i: 1).getReg() != MI.getOperand(i: 2).getReg() &&
1370 MI.getOperand(i: 1).getReg() != MI.getOperand(i: 3).getReg());
1371 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1372 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::ORRv8i8
1373 : AArch64::ORRv16i8))
1374 .addReg(RegNo: DstReg,
1375 Flags: RegState::Define |
1376 getRenamableRegState(B: MI.getOperand(i: 0).isRenamable()))
1377 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState)
1378 .addReg(RegNo: MI.getOperand(i: 1).getReg(), Flags: RegState);
1379 auto I2 =
1380 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1381 MCID: TII->get(Opcode: Opcode == AArch64::BSPv8i8 ? AArch64::BSLv8i8
1382 : AArch64::BSLv16i8))
1383 .add(MO: MI.getOperand(i: 0))
1384 .addReg(RegNo: DstReg,
1385 Flags: RegState::Kill | getRenamableRegState(
1386 B: MI.getOperand(i: 0).isRenamable()))
1387 .add(MO: MI.getOperand(i: 2))
1388 .add(MO: MI.getOperand(i: 3));
1389 transferImpOps(OldMI&: MI, UseMI&: I2, DefMI&: I2);
1390 }
1391 }
1392 MI.eraseFromParent();
1393 return true;
1394 }
1395
1396 case AArch64::ADDWrr:
1397 case AArch64::SUBWrr:
1398 case AArch64::ADDXrr:
1399 case AArch64::SUBXrr:
1400 case AArch64::ADDSWrr:
1401 case AArch64::SUBSWrr:
1402 case AArch64::ADDSXrr:
1403 case AArch64::SUBSXrr:
1404 case AArch64::ANDWrr:
1405 case AArch64::ANDXrr:
1406 case AArch64::BICWrr:
1407 case AArch64::BICXrr:
1408 case AArch64::ANDSWrr:
1409 case AArch64::ANDSXrr:
1410 case AArch64::BICSWrr:
1411 case AArch64::BICSXrr:
1412 case AArch64::EONWrr:
1413 case AArch64::EONXrr:
1414 case AArch64::EORWrr:
1415 case AArch64::EORXrr:
1416 case AArch64::ORNWrr:
1417 case AArch64::ORNXrr:
1418 case AArch64::ORRWrr:
1419 case AArch64::ORRXrr: {
1420 unsigned Opcode;
1421 switch (MI.getOpcode()) {
1422 default:
1423 return false;
1424 case AArch64::ADDWrr: Opcode = AArch64::ADDWrs; break;
1425 case AArch64::SUBWrr: Opcode = AArch64::SUBWrs; break;
1426 case AArch64::ADDXrr: Opcode = AArch64::ADDXrs; break;
1427 case AArch64::SUBXrr: Opcode = AArch64::SUBXrs; break;
1428 case AArch64::ADDSWrr: Opcode = AArch64::ADDSWrs; break;
1429 case AArch64::SUBSWrr: Opcode = AArch64::SUBSWrs; break;
1430 case AArch64::ADDSXrr: Opcode = AArch64::ADDSXrs; break;
1431 case AArch64::SUBSXrr: Opcode = AArch64::SUBSXrs; break;
1432 case AArch64::ANDWrr: Opcode = AArch64::ANDWrs; break;
1433 case AArch64::ANDXrr: Opcode = AArch64::ANDXrs; break;
1434 case AArch64::BICWrr: Opcode = AArch64::BICWrs; break;
1435 case AArch64::BICXrr: Opcode = AArch64::BICXrs; break;
1436 case AArch64::ANDSWrr: Opcode = AArch64::ANDSWrs; break;
1437 case AArch64::ANDSXrr: Opcode = AArch64::ANDSXrs; break;
1438 case AArch64::BICSWrr: Opcode = AArch64::BICSWrs; break;
1439 case AArch64::BICSXrr: Opcode = AArch64::BICSXrs; break;
1440 case AArch64::EONWrr: Opcode = AArch64::EONWrs; break;
1441 case AArch64::EONXrr: Opcode = AArch64::EONXrs; break;
1442 case AArch64::EORWrr: Opcode = AArch64::EORWrs; break;
1443 case AArch64::EORXrr: Opcode = AArch64::EORXrs; break;
1444 case AArch64::ORNWrr: Opcode = AArch64::ORNWrs; break;
1445 case AArch64::ORNXrr: Opcode = AArch64::ORNXrs; break;
1446 case AArch64::ORRWrr: Opcode = AArch64::ORRWrs; break;
1447 case AArch64::ORRXrr: Opcode = AArch64::ORRXrs; break;
1448 }
1449 MachineFunction &MF = *MBB.getParent();
1450 // Try to create new inst without implicit operands added.
1451 MachineInstr *NewMI = MF.CreateMachineInstr(
1452 MCID: TII->get(Opcode), DL: MI.getDebugLoc(), /*NoImplicit=*/true);
1453 MBB.insert(I: MBBI, MI: NewMI);
1454 MachineInstrBuilder MIB1(MF, NewMI);
1455 MIB1->setPCSections(MF, MD: MI.getPCSections());
1456 MIB1.addReg(RegNo: MI.getOperand(i: 0).getReg(), Flags: RegState::Define)
1457 .add(MO: MI.getOperand(i: 1))
1458 .add(MO: MI.getOperand(i: 2))
1459 .addImm(Val: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0));
1460 transferImpOps(OldMI&: MI, UseMI&: MIB1, DefMI&: MIB1);
1461 if (auto DebugNumber = MI.peekDebugInstrNum())
1462 NewMI->setDebugInstrNum(DebugNumber);
1463 MI.eraseFromParent();
1464 return true;
1465 }
1466
1467 case AArch64::LOADgot: {
1468 MachineFunction *MF = MBB.getParent();
1469 Register DstReg = MI.getOperand(i: 0).getReg();
1470 const MachineOperand &MO1 = MI.getOperand(i: 1);
1471 unsigned Flags = MO1.getTargetFlags();
1472
1473 if (MF->getTarget().getCodeModel() == CodeModel::Tiny) {
1474 // Tiny codemodel expand to LDR
1475 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1476 MCID: TII->get(Opcode: AArch64::LDRXl), DestReg: DstReg);
1477
1478 if (MO1.isGlobal()) {
1479 MIB.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0, TargetFlags: Flags);
1480 } else if (MO1.isSymbol()) {
1481 MIB.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags);
1482 } else {
1483 assert(MO1.isCPI() &&
1484 "Only expect globals, externalsymbols, or constant pools");
1485 MIB.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(), TargetFlags: Flags);
1486 }
1487 } else {
1488 // Small codemodel expand into ADRP + LDR.
1489 MachineFunction &MF = *MI.getParent()->getParent();
1490 DebugLoc DL = MI.getDebugLoc();
1491 MachineInstrBuilder MIB1 =
1492 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADRP), DestReg: DstReg);
1493
1494 MachineInstrBuilder MIB2;
1495 if (MF.getSubtarget<AArch64Subtarget>().isTargetILP32()) {
1496 auto TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
1497 unsigned Reg32 = TRI->getSubReg(Reg: DstReg, Idx: AArch64::sub_32);
1498 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::LDRWui))
1499 .addDef(RegNo: Reg32)
1500 .addReg(RegNo: DstReg, Flags: RegState::Kill)
1501 .addReg(RegNo: DstReg, Flags: RegState::Implicit);
1502 } else {
1503 Register DstReg = MI.getOperand(i: 0).getReg();
1504 MIB2 = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::LDRXui))
1505 .add(MO: MI.getOperand(i: 0))
1506 .addUse(RegNo: DstReg, Flags: RegState::Kill);
1507 }
1508
1509 if (MO1.isGlobal()) {
1510 MIB1.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0, TargetFlags: Flags | AArch64II::MO_PAGE);
1511 MIB2.addGlobalAddress(GV: MO1.getGlobal(), Offset: 0,
1512 TargetFlags: Flags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1513 } else if (MO1.isSymbol()) {
1514 MIB1.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags | AArch64II::MO_PAGE);
1515 MIB2.addExternalSymbol(FnName: MO1.getSymbolName(), TargetFlags: Flags |
1516 AArch64II::MO_PAGEOFF |
1517 AArch64II::MO_NC);
1518 } else {
1519 assert(MO1.isCPI() &&
1520 "Only expect globals, externalsymbols, or constant pools");
1521 MIB1.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(),
1522 TargetFlags: Flags | AArch64II::MO_PAGE);
1523 MIB2.addConstantPoolIndex(Idx: MO1.getIndex(), Offset: MO1.getOffset(),
1524 TargetFlags: Flags | AArch64II::MO_PAGEOFF |
1525 AArch64II::MO_NC);
1526 }
1527
1528 // If the LOADgot instruction has a debug-instr-number, annotate the
1529 // LDRWui instruction that it is expanded to with the same
1530 // debug-instr-number to preserve debug information.
1531 if (MI.peekDebugInstrNum() != 0)
1532 MIB2->setDebugInstrNum(MI.peekDebugInstrNum());
1533 transferImpOps(OldMI&: MI, UseMI&: MIB1, DefMI&: MIB2);
1534 }
1535 MI.eraseFromParent();
1536 return true;
1537 }
1538 case AArch64::MOVaddrBA:
1539 case AArch64::MOVaddr:
1540 case AArch64::MOVaddrJT:
1541 case AArch64::MOVaddrCP:
1542 case AArch64::MOVaddrTLS:
1543 case AArch64::MOVaddrEXT: {
1544 MachineFunction &MF = *MI.getParent()->getParent();
1545 Register DstReg = MI.getOperand(i: 0).getReg();
1546 assert(DstReg != AArch64::XZR);
1547
1548 bool IsTargetMachO = MF.getSubtarget<AArch64Subtarget>().isTargetMachO();
1549 SmallVector<AArch64_IMM::AddrInsnModel, 3> Insn;
1550 AArch64_IMM::expandMOVAddr(
1551 Opcode: MI.getOpcode(), TargetFlags: MI.getOperand(i: 1).getTargetFlags(), IsTargetMachO, Insn);
1552
1553 // Compute the constant pool index, if any.
1554 std::optional<unsigned> CPIdx;
1555 if (Opcode == AArch64::MOVaddrBA && IsTargetMachO) {
1556 // blockaddress expressions have to come from a constant pool because the
1557 // largest addend (and hence offset within a function) allowed for ADRP is
1558 // only 8MB.
1559 const BlockAddress *BA = MI.getOperand(i: 1).getBlockAddress();
1560 assert(MI.getOperand(1).getOffset() == 0 && "unexpected offset");
1561 MachineConstantPool *MCP = MF.getConstantPool();
1562 CPIdx = MCP->getConstantPoolIndex(C: BA, Alignment: Align(8));
1563 }
1564
1565 MachineInstrBuilder FirstMIB;
1566 MachineInstrBuilder LastMIB;
1567 for (const auto &I : Insn) {
1568 MachineInstrBuilder MIB;
1569 switch (I.Opcode) {
1570 case AArch64::ADRP:
1571 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADRP),
1572 DestReg: DstReg);
1573 if (CPIdx)
1574 MIB.addConstantPoolIndex(Idx: *CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGE);
1575 else
1576 MIB.add(MO: MI.getOperand(i: 1));
1577 break;
1578 case AArch64::LDRXui:
1579 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::LDRXui),
1580 DestReg: DstReg)
1581 .addUse(RegNo: DstReg)
1582 .addConstantPoolIndex(
1583 Idx: *CPIdx, Offset: 0, TargetFlags: AArch64II::MO_PAGEOFF | AArch64II::MO_NC);
1584 break;
1585 case AArch64::MOVKXi: {
1586 // MO_TAGGED on the page indicates a tagged address. Set the tag now.
1587 // We do so by creating a MOVK that sets bits 48-63 of the register to
1588 // (global address + 0x100000000 - PC) >> 48. This assumes that we're in
1589 // the small code model so we can assume a binary size of <= 4GB, which
1590 // makes the untagged PC relative offset positive. The binary must also
1591 // be loaded into address range [0, 2^48). Both of these properties need
1592 // to be ensured at runtime when using tagged addresses.
1593 auto Tag = MI.getOperand(i: 1);
1594 Tag.setTargetFlags(AArch64II::MO_PREL | AArch64II::MO_G3);
1595 Tag.setOffset(0x100000000);
1596 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MOVKXi),
1597 DestReg: DstReg)
1598 .addReg(RegNo: DstReg)
1599 .add(MO: Tag)
1600 .addImm(Val: 48);
1601 break;
1602 }
1603 case AArch64::ADDXri:
1604 MIB = BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
1605 .add(MO: MI.getOperand(i: 0))
1606 .addReg(RegNo: DstReg)
1607 .add(MO: MI.getOperand(i: 2))
1608 .addImm(Val: 0);
1609 break;
1610 default:
1611 llvm_unreachable("unexpected opcode in MOVaddr expansion");
1612 }
1613
1614 if (!FirstMIB.getInstr())
1615 FirstMIB = MIB;
1616 LastMIB = MIB;
1617 }
1618
1619 transferImpOps(OldMI&: MI, UseMI&: FirstMIB, DefMI&: LastMIB);
1620 MI.eraseFromParent();
1621 return true;
1622 }
1623 case AArch64::ADDlowTLS:
1624 // Produce a plain ADD
1625 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::ADDXri))
1626 .add(MO: MI.getOperand(i: 0))
1627 .add(MO: MI.getOperand(i: 1))
1628 .add(MO: MI.getOperand(i: 2))
1629 .addImm(Val: 0);
1630 MI.eraseFromParent();
1631 return true;
1632
1633 case AArch64::MOVbaseTLS: {
1634 Register DstReg = MI.getOperand(i: 0).getReg();
1635 auto SysReg = AArch64SysReg::TPIDR_EL0;
1636 MachineFunction *MF = MBB.getParent();
1637 if (MF->getSubtarget<AArch64Subtarget>().useEL3ForTP())
1638 SysReg = AArch64SysReg::TPIDR_EL3;
1639 else if (MF->getSubtarget<AArch64Subtarget>().useEL2ForTP())
1640 SysReg = AArch64SysReg::TPIDR_EL2;
1641 else if (MF->getSubtarget<AArch64Subtarget>().useEL1ForTP())
1642 SysReg = AArch64SysReg::TPIDR_EL1;
1643 else if (MF->getSubtarget<AArch64Subtarget>().useROEL0ForTP())
1644 SysReg = AArch64SysReg::TPIDRRO_EL0;
1645 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::MRS), DestReg: DstReg)
1646 .addImm(Val: SysReg);
1647 MI.eraseFromParent();
1648 return true;
1649 }
1650
1651 case AArch64::MOVi32imm:
1652 return expandMOVImm(MBB, MBBI, BitSize: 32);
1653 case AArch64::MOVi64imm:
1654 return expandMOVImm(MBB, MBBI, BitSize: 64);
1655 case AArch64::RET_ReallyLR: {
1656 // Hiding the LR use with RET_ReallyLR may lead to extra kills in the
1657 // function and missing live-ins. We are fine in practice because callee
1658 // saved register handling ensures the register value is restored before
1659 // RET, but we need the undef flag here to appease the MachineVerifier
1660 // liveness checks.
1661 MachineInstrBuilder MIB =
1662 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::RET))
1663 .addReg(RegNo: AArch64::LR, Flags: RegState::Undef);
1664 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1665 MI.eraseFromParent();
1666 return true;
1667 }
1668 case AArch64::CMP_SWAP_8:
1669 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRB, StlrOp: AArch64::STLXRB,
1670 CmpOp: AArch64::SUBSWrx,
1671 ExtendImm: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTB, Imm: 0),
1672 ZeroReg: AArch64::WZR, NextMBBI);
1673 case AArch64::CMP_SWAP_16:
1674 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRH, StlrOp: AArch64::STLXRH,
1675 CmpOp: AArch64::SUBSWrx,
1676 ExtendImm: AArch64_AM::getArithExtendImm(ET: AArch64_AM::UXTH, Imm: 0),
1677 ZeroReg: AArch64::WZR, NextMBBI);
1678 case AArch64::CMP_SWAP_32:
1679 return expandCMP_SWAP(MBB, MBBI, LdarOp: AArch64::LDAXRW, StlrOp: AArch64::STLXRW,
1680 CmpOp: AArch64::SUBSWrs,
1681 ExtendImm: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0),
1682 ZeroReg: AArch64::WZR, NextMBBI);
1683 case AArch64::CMP_SWAP_64:
1684 return expandCMP_SWAP(MBB, MBBI,
1685 LdarOp: AArch64::LDAXRX, StlrOp: AArch64::STLXRX, CmpOp: AArch64::SUBSXrs,
1686 ExtendImm: AArch64_AM::getShifterImm(ST: AArch64_AM::LSL, Imm: 0),
1687 ZeroReg: AArch64::XZR, NextMBBI);
1688 case AArch64::CMP_SWAP_128:
1689 case AArch64::CMP_SWAP_128_RELEASE:
1690 case AArch64::CMP_SWAP_128_ACQUIRE:
1691 case AArch64::CMP_SWAP_128_MONOTONIC:
1692 return expandCMP_SWAP_128(MBB, MBBI, NextMBBI);
1693
1694 case AArch64::AESMCrrTied:
1695 case AArch64::AESIMCrrTied: {
1696 MachineInstrBuilder MIB =
1697 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1698 MCID: TII->get(Opcode: Opcode == AArch64::AESMCrrTied ? AArch64::AESMCrr :
1699 AArch64::AESIMCrr))
1700 .add(MO: MI.getOperand(i: 0))
1701 .add(MO: MI.getOperand(i: 1));
1702 transferImpOps(OldMI&: MI, UseMI&: MIB, DefMI&: MIB);
1703 MI.eraseFromParent();
1704 return true;
1705 }
1706 case AArch64::IRGstack: {
1707 MachineFunction &MF = *MBB.getParent();
1708 const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>();
1709 const AArch64FrameLowering *TFI =
1710 MF.getSubtarget<AArch64Subtarget>().getFrameLowering();
1711
1712 // IRG does not allow immediate offset. getTaggedBasePointerOffset should
1713 // almost always point to SP-after-prologue; if not, emit a longer
1714 // instruction sequence.
1715 int BaseOffset = -AFI->getTaggedBasePointerOffset();
1716 Register FrameReg;
1717 StackOffset FrameRegOffset = TFI->resolveFrameOffsetReference(
1718 MF, ObjectOffset: BaseOffset, isFixed: false /*isFixed*/, StackID: TargetStackID::Default /*StackID*/,
1719 FrameReg,
1720 /*PreferFP=*/false,
1721 /*ForSimm=*/true);
1722 Register SrcReg = FrameReg;
1723 if (FrameRegOffset) {
1724 // Use output register as temporary.
1725 SrcReg = MI.getOperand(i: 0).getReg();
1726 emitFrameOffset(MBB, MBBI: &MI, DL: MI.getDebugLoc(), DestReg: SrcReg, SrcReg: FrameReg,
1727 Offset: FrameRegOffset, TII);
1728 }
1729 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(), MCID: TII->get(Opcode: AArch64::IRG))
1730 .add(MO: MI.getOperand(i: 0))
1731 .addUse(RegNo: SrcReg)
1732 .add(MO: MI.getOperand(i: 2));
1733 MI.eraseFromParent();
1734 return true;
1735 }
1736 case AArch64::TAGPstack: {
1737 int64_t Offset = MI.getOperand(i: 2).getImm();
1738 BuildMI(BB&: MBB, I: MBBI, MIMD: MI.getDebugLoc(),
1739 MCID: TII->get(Opcode: Offset >= 0 ? AArch64::ADDG : AArch64::SUBG))
1740 .add(MO: MI.getOperand(i: 0))
1741 .add(MO: MI.getOperand(i: 1))
1742 .addImm(Val: std::abs(i: Offset))
1743 .add(MO: MI.getOperand(i: 4));
1744 MI.eraseFromParent();
1745 return true;
1746 }
1747 case AArch64::STGloop_wback:
1748 case AArch64::STZGloop_wback:
1749 return expandSetTagLoop(MBB, MBBI, NextMBBI);
1750 case AArch64::STGloop:
1751 case AArch64::STZGloop:
1752 report_fatal_error(
1753 reason: "Non-writeback variants of STGloop / STZGloop should not "
1754 "survive past PrologEpilogInserter.");
1755 case AArch64::STR_ZZZZXI:
1756 case AArch64::STR_ZZZZXI_STRIDED_CONTIGUOUS:
1757 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 4);
1758 case AArch64::STR_ZZZXI:
1759 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 3);
1760 case AArch64::STR_ZZXI:
1761 case AArch64::STR_ZZXI_STRIDED_CONTIGUOUS:
1762 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_ZXI, N: 2);
1763 case AArch64::STR_PPXI:
1764 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::STR_PXI, N: 2);
1765 case AArch64::LDR_ZZZZXI:
1766 case AArch64::LDR_ZZZZXI_STRIDED_CONTIGUOUS:
1767 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 4);
1768 case AArch64::LDR_ZZZXI:
1769 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 3);
1770 case AArch64::LDR_ZZXI:
1771 case AArch64::LDR_ZZXI_STRIDED_CONTIGUOUS:
1772 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_ZXI, N: 2);
1773 case AArch64::LDR_PPXI:
1774 return expandSVESpillFill(MBB, MBBI, Opc: AArch64::LDR_PXI, N: 2);
1775 case AArch64::BLR_RVMARKER:
1776 case AArch64::BLRA_RVMARKER:
1777 return expandCALL_RVMARKER(MBB, MBBI);
1778 case AArch64::BLR_BTI:
1779 return expandCALL_BTI(MBB, MBBI);
1780 case AArch64::StoreSwiftAsyncContext:
1781 return expandStoreSwiftAsyncContext(MBB, MBBI);
1782 case AArch64::RestoreZAPseudo:
1783 case AArch64::CommitZASavePseudo:
1784 case AArch64::MSRpstatePseudo: {
1785 auto *NewMBB = [&] {
1786 switch (Opcode) {
1787 case AArch64::RestoreZAPseudo:
1788 return expandRestoreZASave(MBB, MBBI);
1789 case AArch64::CommitZASavePseudo:
1790 return expandCommitZASave(MBB, MBBI);
1791 case AArch64::MSRpstatePseudo:
1792 return expandCondSMToggle(MBB, MBBI);
1793 default:
1794 llvm_unreachable("Unexpected conditional pseudo!");
1795 }
1796 }();
1797 if (NewMBB != &MBB)
1798 NextMBBI = MBB.end(); // The NextMBBI iterator is invalidated.
1799 return true;
1800 }
1801 case AArch64::InOutZAUsePseudo:
1802 case AArch64::RequiresZASavePseudo:
1803 case AArch64::RequiresZT0SavePseudo:
1804 case AArch64::SMEStateAllocPseudo:
1805 case AArch64::COALESCER_BARRIER_FPR16:
1806 case AArch64::COALESCER_BARRIER_FPR32:
1807 case AArch64::COALESCER_BARRIER_FPR64:
1808 case AArch64::COALESCER_BARRIER_FPR128:
1809 MI.eraseFromParent();
1810 return true;
1811 case AArch64::LD1B_2Z_IMM_PSEUDO:
1812 return expandMultiVecPseudo(
1813 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1814 ContiguousOp: AArch64::LD1B_2Z_IMM, StridedOpc: AArch64::LD1B_2Z_STRIDED_IMM);
1815 case AArch64::LD1H_2Z_IMM_PSEUDO:
1816 return expandMultiVecPseudo(
1817 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1818 ContiguousOp: AArch64::LD1H_2Z_IMM, StridedOpc: AArch64::LD1H_2Z_STRIDED_IMM);
1819 case AArch64::LD1W_2Z_IMM_PSEUDO:
1820 return expandMultiVecPseudo(
1821 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1822 ContiguousOp: AArch64::LD1W_2Z_IMM, StridedOpc: AArch64::LD1W_2Z_STRIDED_IMM);
1823 case AArch64::LD1D_2Z_IMM_PSEUDO:
1824 return expandMultiVecPseudo(
1825 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1826 ContiguousOp: AArch64::LD1D_2Z_IMM, StridedOpc: AArch64::LD1D_2Z_STRIDED_IMM);
1827 case AArch64::LDNT1B_2Z_IMM_PSEUDO:
1828 return expandMultiVecPseudo(
1829 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1830 ContiguousOp: AArch64::LDNT1B_2Z_IMM, StridedOpc: AArch64::LDNT1B_2Z_STRIDED_IMM);
1831 case AArch64::LDNT1H_2Z_IMM_PSEUDO:
1832 return expandMultiVecPseudo(
1833 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1834 ContiguousOp: AArch64::LDNT1H_2Z_IMM, StridedOpc: AArch64::LDNT1H_2Z_STRIDED_IMM);
1835 case AArch64::LDNT1W_2Z_IMM_PSEUDO:
1836 return expandMultiVecPseudo(
1837 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1838 ContiguousOp: AArch64::LDNT1W_2Z_IMM, StridedOpc: AArch64::LDNT1W_2Z_STRIDED_IMM);
1839 case AArch64::LDNT1D_2Z_IMM_PSEUDO:
1840 return expandMultiVecPseudo(
1841 MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass, StridedClass: AArch64::ZPR2StridedRegClass,
1842 ContiguousOp: AArch64::LDNT1D_2Z_IMM, StridedOpc: AArch64::LDNT1D_2Z_STRIDED_IMM);
1843 case AArch64::LD1B_2Z_PSEUDO:
1844 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1845 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1B_2Z,
1846 StridedOpc: AArch64::LD1B_2Z_STRIDED);
1847 case AArch64::LD1H_2Z_PSEUDO:
1848 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1849 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1H_2Z,
1850 StridedOpc: AArch64::LD1H_2Z_STRIDED);
1851 case AArch64::LD1W_2Z_PSEUDO:
1852 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1853 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1W_2Z,
1854 StridedOpc: AArch64::LD1W_2Z_STRIDED);
1855 case AArch64::LD1D_2Z_PSEUDO:
1856 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1857 StridedClass: AArch64::ZPR2StridedRegClass, ContiguousOp: AArch64::LD1D_2Z,
1858 StridedOpc: AArch64::LD1D_2Z_STRIDED);
1859 case AArch64::LDNT1B_2Z_PSEUDO:
1860 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1861 StridedClass: AArch64::ZPR2StridedRegClass,
1862 ContiguousOp: AArch64::LDNT1B_2Z, StridedOpc: AArch64::LDNT1B_2Z_STRIDED);
1863 case AArch64::LDNT1H_2Z_PSEUDO:
1864 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1865 StridedClass: AArch64::ZPR2StridedRegClass,
1866 ContiguousOp: AArch64::LDNT1H_2Z, StridedOpc: AArch64::LDNT1H_2Z_STRIDED);
1867 case AArch64::LDNT1W_2Z_PSEUDO:
1868 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1869 StridedClass: AArch64::ZPR2StridedRegClass,
1870 ContiguousOp: AArch64::LDNT1W_2Z, StridedOpc: AArch64::LDNT1W_2Z_STRIDED);
1871 case AArch64::LDNT1D_2Z_PSEUDO:
1872 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR2RegClass,
1873 StridedClass: AArch64::ZPR2StridedRegClass,
1874 ContiguousOp: AArch64::LDNT1D_2Z, StridedOpc: AArch64::LDNT1D_2Z_STRIDED);
1875 case AArch64::LD1B_4Z_IMM_PSEUDO:
1876 return expandMultiVecPseudo(
1877 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1878 ContiguousOp: AArch64::LD1B_4Z_IMM, StridedOpc: AArch64::LD1B_4Z_STRIDED_IMM);
1879 case AArch64::LD1H_4Z_IMM_PSEUDO:
1880 return expandMultiVecPseudo(
1881 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1882 ContiguousOp: AArch64::LD1H_4Z_IMM, StridedOpc: AArch64::LD1H_4Z_STRIDED_IMM);
1883 case AArch64::LD1W_4Z_IMM_PSEUDO:
1884 return expandMultiVecPseudo(
1885 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1886 ContiguousOp: AArch64::LD1W_4Z_IMM, StridedOpc: AArch64::LD1W_4Z_STRIDED_IMM);
1887 case AArch64::LD1D_4Z_IMM_PSEUDO:
1888 return expandMultiVecPseudo(
1889 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1890 ContiguousOp: AArch64::LD1D_4Z_IMM, StridedOpc: AArch64::LD1D_4Z_STRIDED_IMM);
1891 case AArch64::LDNT1B_4Z_IMM_PSEUDO:
1892 return expandMultiVecPseudo(
1893 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1894 ContiguousOp: AArch64::LDNT1B_4Z_IMM, StridedOpc: AArch64::LDNT1B_4Z_STRIDED_IMM);
1895 case AArch64::LDNT1H_4Z_IMM_PSEUDO:
1896 return expandMultiVecPseudo(
1897 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1898 ContiguousOp: AArch64::LDNT1H_4Z_IMM, StridedOpc: AArch64::LDNT1H_4Z_STRIDED_IMM);
1899 case AArch64::LDNT1W_4Z_IMM_PSEUDO:
1900 return expandMultiVecPseudo(
1901 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1902 ContiguousOp: AArch64::LDNT1W_4Z_IMM, StridedOpc: AArch64::LDNT1W_4Z_STRIDED_IMM);
1903 case AArch64::LDNT1D_4Z_IMM_PSEUDO:
1904 return expandMultiVecPseudo(
1905 MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass, StridedClass: AArch64::ZPR4StridedRegClass,
1906 ContiguousOp: AArch64::LDNT1D_4Z_IMM, StridedOpc: AArch64::LDNT1D_4Z_STRIDED_IMM);
1907 case AArch64::LD1B_4Z_PSEUDO:
1908 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1909 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1B_4Z,
1910 StridedOpc: AArch64::LD1B_4Z_STRIDED);
1911 case AArch64::LD1H_4Z_PSEUDO:
1912 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1913 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1H_4Z,
1914 StridedOpc: AArch64::LD1H_4Z_STRIDED);
1915 case AArch64::LD1W_4Z_PSEUDO:
1916 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1917 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1W_4Z,
1918 StridedOpc: AArch64::LD1W_4Z_STRIDED);
1919 case AArch64::LD1D_4Z_PSEUDO:
1920 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1921 StridedClass: AArch64::ZPR4StridedRegClass, ContiguousOp: AArch64::LD1D_4Z,
1922 StridedOpc: AArch64::LD1D_4Z_STRIDED);
1923 case AArch64::LDNT1B_4Z_PSEUDO:
1924 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1925 StridedClass: AArch64::ZPR4StridedRegClass,
1926 ContiguousOp: AArch64::LDNT1B_4Z, StridedOpc: AArch64::LDNT1B_4Z_STRIDED);
1927 case AArch64::LDNT1H_4Z_PSEUDO:
1928 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1929 StridedClass: AArch64::ZPR4StridedRegClass,
1930 ContiguousOp: AArch64::LDNT1H_4Z, StridedOpc: AArch64::LDNT1H_4Z_STRIDED);
1931 case AArch64::LDNT1W_4Z_PSEUDO:
1932 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1933 StridedClass: AArch64::ZPR4StridedRegClass,
1934 ContiguousOp: AArch64::LDNT1W_4Z, StridedOpc: AArch64::LDNT1W_4Z_STRIDED);
1935 case AArch64::LDNT1D_4Z_PSEUDO:
1936 return expandMultiVecPseudo(MBB, MBBI, ContiguousClass: AArch64::ZPR4RegClass,
1937 StridedClass: AArch64::ZPR4StridedRegClass,
1938 ContiguousOp: AArch64::LDNT1D_4Z, StridedOpc: AArch64::LDNT1D_4Z_STRIDED);
1939 case AArch64::COPY_INTO_TRANSPOSED_TUPLE:
1940 return expandCopyIntoTuplePseudo(MI, MBB, MBBI);
1941 case AArch64::EON_ZZZ:
1942 case AArch64::NAND_ZZZ:
1943 case AArch64::NOR_ZZZ:
1944 return expandSVEBitwisePseudo(MI, MBB, MBBI);
1945 }
1946 return false;
1947}
1948
1949/// Iterate over the instructions in basic block MBB and expand any
1950/// pseudo instructions. Return true if anything was modified.
1951bool AArch64ExpandPseudoImpl::expandMBB(MachineBasicBlock &MBB) {
1952 bool Modified = false;
1953
1954 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
1955 while (MBBI != E) {
1956 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
1957 if (MBBI->isPseudo())
1958 Modified |= expandMI(MBB, MBBI, NextMBBI&: NMBBI);
1959 MBBI = NMBBI;
1960 }
1961
1962 return Modified;
1963}
1964
1965bool AArch64ExpandPseudoImpl::run(MachineFunction &MF) {
1966 TII = MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
1967
1968 bool Modified = false;
1969 for (auto &MBB : MF)
1970 Modified |= expandMBB(MBB);
1971 return Modified;
1972}
1973
1974bool AArch64ExpandPseudoLegacy::runOnMachineFunction(MachineFunction &MF) {
1975 return AArch64ExpandPseudoImpl().run(MF);
1976}
1977
1978/// Returns an instance of the pseudo instruction expansion pass.
1979FunctionPass *llvm::createAArch64ExpandPseudoLegacyPass() {
1980 return new AArch64ExpandPseudoLegacy();
1981}
1982
1983PreservedAnalyses
1984AArch64ExpandPseudoPass::run(MachineFunction &MF,
1985 MachineFunctionAnalysisManager &MFAM) {
1986 const bool Changed = AArch64ExpandPseudoImpl().run(MF);
1987 if (!Changed)
1988 return PreservedAnalyses::all();
1989 PreservedAnalyses PA = getMachineFunctionPassPreservedAnalyses();
1990 PA.preserveSet<CFGAnalyses>();
1991 return PA;
1992}
1993