1//===- AArch64LowerHomogeneousPrologEpilog.cpp ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that lowers homogeneous prolog/epilog instructions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64.h"
14#include "AArch64InstrInfo.h"
15#include "AArch64Subtarget.h"
16#include "MCTargetDesc/AArch64InstPrinter.h"
17#include "llvm/CodeGen/MachineBasicBlock.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/MachineFunctionAnalysis.h"
20#include "llvm/CodeGen/MachineInstr.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/MachineModuleInfo.h"
23#include "llvm/CodeGen/MachineOperand.h"
24#include "llvm/CodeGen/TargetSubtargetInfo.h"
25#include "llvm/IR/DebugLoc.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/PassManager.h"
29#include "llvm/Pass.h"
30#include <optional>
31#include <sstream>
32
33using namespace llvm;
34
35#define AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME \
36 "AArch64 homogeneous prolog/epilog lowering pass"
37
38static cl::opt<int> FrameHelperSizeThreshold(
39 "frame-helper-size-threshold", cl::init(Val: 2), cl::Hidden,
40 cl::desc("The minimum number of instructions that are outlined in a frame "
41 "helper (default = 2)"));
42
43namespace {
44
45class AArch64LowerHomogeneousPrologEpilogImpl {
46public:
47 const AArch64InstrInfo *TII;
48
49 AArch64LowerHomogeneousPrologEpilogImpl(Module *M, MachineModuleInfo *MMI)
50 : M(M), MMI(MMI) {}
51
52 bool run();
53 bool runOnMachineFunction(MachineFunction &Fn);
54
55private:
56 Module *M;
57 MachineModuleInfo *MMI;
58
59 bool runOnMBB(MachineBasicBlock &MBB);
60 bool runOnMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
61 MachineBasicBlock::iterator &NextMBBI);
62
63 /// Lower a HOM_Prolog pseudo instruction into a helper call
64 /// or a sequence of homogeneous stores.
65 /// When a fp setup follows, it can be optimized.
66 bool lowerProlog(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
67 MachineBasicBlock::iterator &NextMBBI);
68 /// Lower a HOM_Epilog pseudo instruction into a helper call
69 /// or a sequence of homogeneous loads.
70 /// When a return follow, it can be optimized.
71 bool lowerEpilog(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
72 MachineBasicBlock::iterator &NextMBBI);
73};
74
75class AArch64LowerHomogeneousPrologEpilogLegacy : public ModulePass {
76public:
77 static char ID;
78
79 AArch64LowerHomogeneousPrologEpilogLegacy() : ModulePass(ID) {}
80 void getAnalysisUsage(AnalysisUsage &AU) const override {
81 AU.addRequired<MachineModuleInfoWrapperPass>();
82 AU.addPreserved<MachineModuleInfoWrapperPass>();
83 AU.setPreservesAll();
84 ModulePass::getAnalysisUsage(AU);
85 }
86 bool runOnModule(Module &M) override;
87
88 StringRef getPassName() const override {
89 return AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME;
90 }
91};
92
93} // end anonymous namespace
94
95char AArch64LowerHomogeneousPrologEpilogLegacy::ID = 0;
96
97INITIALIZE_PASS(AArch64LowerHomogeneousPrologEpilogLegacy,
98 "aarch64-lower-homogeneous-prolog-epilog",
99 AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME, false, false)
100
101bool AArch64LowerHomogeneousPrologEpilogLegacy::runOnModule(Module &M) {
102 if (skipModule(M))
103 return false;
104
105 MachineModuleInfo *MMI =
106 &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
107 return AArch64LowerHomogeneousPrologEpilogImpl(&M, MMI).run();
108}
109
110PreservedAnalyses
111AArch64LowerHomogeneousPrologEpilogPass::run(Module &M,
112 ModuleAnalysisManager &MAM) {
113 MachineModuleInfo *MMI = &MAM.getResult<MachineModuleAnalysis>(IR&: M).getMMI();
114 bool Changed = AArch64LowerHomogeneousPrologEpilogImpl(&M, MMI).run();
115 if (!Changed)
116 return PreservedAnalyses::all();
117 PreservedAnalyses PA;
118 PA.preserve<MachineModuleAnalysis>();
119 return PA;
120}
121
122bool AArch64LowerHomogeneousPrologEpilogImpl::run() {
123 bool Changed = false;
124 for (auto &F : *M) {
125 if (F.empty())
126 continue;
127
128 MachineFunction *MF = MMI->getMachineFunction(F);
129 if (!MF)
130 continue;
131 Changed |= runOnMachineFunction(Fn&: *MF);
132 }
133
134 return Changed;
135}
136enum FrameHelperType { Prolog, PrologFrame, Epilog, EpilogTail };
137
138/// Return a frame helper name with the given CSRs and the helper type.
139/// For instance, a prolog helper that saves x19 and x20 is named as
140/// OUTLINED_FUNCTION_PROLOG_x19x20.
141static std::string getFrameHelperName(SmallVectorImpl<unsigned> &Regs,
142 FrameHelperType Type, unsigned FpOffset) {
143 std::ostringstream RegStream;
144 switch (Type) {
145 case FrameHelperType::Prolog:
146 RegStream << "OUTLINED_FUNCTION_PROLOG_";
147 break;
148 case FrameHelperType::PrologFrame:
149 RegStream << "OUTLINED_FUNCTION_PROLOG_FRAME" << FpOffset << "_";
150 break;
151 case FrameHelperType::Epilog:
152 RegStream << "OUTLINED_FUNCTION_EPILOG_";
153 break;
154 case FrameHelperType::EpilogTail:
155 RegStream << "OUTLINED_FUNCTION_EPILOG_TAIL_";
156 break;
157 }
158
159 for (auto Reg : Regs) {
160 if (Reg == AArch64::NoRegister)
161 continue;
162 RegStream << AArch64InstPrinter::getRegisterName(Reg);
163 }
164
165 return RegStream.str();
166}
167
168/// Create a Function for the unique frame helper with the given name.
169/// Return a newly created MachineFunction with an empty MachineBasicBlock.
170static MachineFunction &createFrameHelperMachineFunction(Module *M,
171 MachineModuleInfo *MMI,
172 StringRef Name) {
173 LLVMContext &C = M->getContext();
174 Function *F = M->getFunction(Name);
175 assert(F == nullptr && "Function has been created before");
176 F = Function::Create(Ty: FunctionType::get(Result: Type::getVoidTy(C), isVarArg: false),
177 Linkage: Function::ExternalLinkage, N: Name, M);
178 assert(F && "Function was null!");
179
180 // Use ODR linkage to avoid duplication.
181 F->setLinkage(GlobalValue::LinkOnceODRLinkage);
182 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
183
184 // Set minsize, so we don't insert padding between outlined functions.
185 F->addFnAttr(Kind: Attribute::NoInline);
186 F->addFnAttr(Kind: Attribute::MinSize);
187 F->addFnAttr(Kind: Attribute::Naked);
188
189 MachineFunction &MF = MMI->getOrCreateMachineFunction(F&: *F);
190 // Remove unnecessary register liveness and set NoVRegs.
191 MF.getProperties()
192 .resetTracksLiveness()
193 .resetIsSSA()
194 .setNoVRegs()
195 .setNoPHIs();
196 MF.getRegInfo().freezeReservedRegs();
197
198 // Create entry block.
199 BasicBlock *EntryBB = BasicBlock::Create(Context&: C, Name: "entry", Parent: F);
200 IRBuilder<> Builder(EntryBB);
201 Builder.CreateRetVoid();
202
203 // Insert the new block into the function.
204 MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
205 MF.insert(MBBI: MF.begin(), MBB);
206
207 return MF;
208}
209
210/// Emit a store-pair instruction for frame-setup.
211/// If Reg2 is AArch64::NoRegister, emit STR instead.
212static void emitStore(MachineFunction &MF, MachineBasicBlock &MBB,
213 MachineBasicBlock::iterator Pos,
214 const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2,
215 int Offset, bool IsPreDec) {
216 assert(Reg1 != AArch64::NoRegister);
217 const bool IsPaired = Reg2 != AArch64::NoRegister;
218 bool IsFloat = AArch64::FPR64RegClass.contains(Reg: Reg1);
219 assert(!(IsFloat ^ AArch64::FPR64RegClass.contains(Reg2)));
220 unsigned Opc;
221 if (IsPreDec) {
222 if (IsFloat)
223 Opc = IsPaired ? AArch64::STPDpre : AArch64::STRDpre;
224 else
225 Opc = IsPaired ? AArch64::STPXpre : AArch64::STRXpre;
226 } else {
227 if (IsFloat)
228 Opc = IsPaired ? AArch64::STPDi : AArch64::STRDui;
229 else
230 Opc = IsPaired ? AArch64::STPXi : AArch64::STRXui;
231 }
232 // The implicit scale for Offset is 8.
233 TypeSize Scale(0U, false), Width(0U, false);
234 int64_t MinOffset, MaxOffset;
235 [[maybe_unused]] bool Success =
236 AArch64InstrInfo::getMemOpInfo(Opcode: Opc, Scale, Width, MinOffset, MaxOffset);
237 assert(Success && "Invalid Opcode");
238 Offset *= (8 / (int)Scale);
239
240 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: Pos, MIMD: DebugLoc(), MCID: TII.get(Opcode: Opc));
241 if (IsPreDec)
242 MIB.addDef(RegNo: AArch64::SP);
243 if (IsPaired)
244 MIB.addReg(RegNo: Reg2);
245 MIB.addReg(RegNo: Reg1)
246 .addReg(RegNo: AArch64::SP)
247 .addImm(Val: Offset)
248 .setMIFlag(MachineInstr::FrameSetup);
249}
250
251/// Emit a load-pair instruction for frame-destroy.
252/// If Reg2 is AArch64::NoRegister, emit LDR instead.
253static void emitLoad(MachineFunction &MF, MachineBasicBlock &MBB,
254 MachineBasicBlock::iterator Pos,
255 const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2,
256 int Offset, bool IsPostDec) {
257 assert(Reg1 != AArch64::NoRegister);
258 const bool IsPaired = Reg2 != AArch64::NoRegister;
259 bool IsFloat = AArch64::FPR64RegClass.contains(Reg: Reg1);
260 assert(!(IsFloat ^ AArch64::FPR64RegClass.contains(Reg2)));
261 unsigned Opc;
262 if (IsPostDec) {
263 if (IsFloat)
264 Opc = IsPaired ? AArch64::LDPDpost : AArch64::LDRDpost;
265 else
266 Opc = IsPaired ? AArch64::LDPXpost : AArch64::LDRXpost;
267 } else {
268 if (IsFloat)
269 Opc = IsPaired ? AArch64::LDPDi : AArch64::LDRDui;
270 else
271 Opc = IsPaired ? AArch64::LDPXi : AArch64::LDRXui;
272 }
273 // The implicit scale for Offset is 8.
274 TypeSize Scale(0U, false), Width(0U, false);
275 int64_t MinOffset, MaxOffset;
276 [[maybe_unused]] bool Success =
277 AArch64InstrInfo::getMemOpInfo(Opcode: Opc, Scale, Width, MinOffset, MaxOffset);
278 assert(Success && "Invalid Opcode");
279 Offset *= (8 / (int)Scale);
280
281 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: Pos, MIMD: DebugLoc(), MCID: TII.get(Opcode: Opc));
282 if (IsPostDec)
283 MIB.addDef(RegNo: AArch64::SP);
284 if (IsPaired)
285 MIB.addReg(RegNo: Reg2, Flags: getDefRegState(B: true));
286 MIB.addReg(RegNo: Reg1, Flags: getDefRegState(B: true))
287 .addReg(RegNo: AArch64::SP)
288 .addImm(Val: Offset)
289 .setMIFlag(MachineInstr::FrameDestroy);
290}
291
292/// Return a unique function if a helper can be formed with the given Regs
293/// and frame type.
294/// 1) _OUTLINED_FUNCTION_PROLOG_x30x29x19x20x21x22:
295/// stp x22, x21, [sp, #-32]! ; x29/x30 has been stored at the caller
296/// stp x20, x19, [sp, #16]
297/// ret
298///
299/// 2) _OUTLINED_FUNCTION_PROLOG_FRAME32_x30x29x19x20x21x22:
300/// stp x22, x21, [sp, #-32]! ; x29/x30 has been stored at the caller
301/// stp x20, x19, [sp, #16]
302/// add fp, sp, #32
303/// ret
304///
305/// 3) _OUTLINED_FUNCTION_EPILOG_x30x29x19x20x21x22:
306/// mov x16, x30
307/// ldp x29, x30, [sp, #32]
308/// ldp x20, x19, [sp, #16]
309/// ldp x22, x21, [sp], #48
310/// ret x16
311///
312/// 4) _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20x21x22:
313/// ldp x29, x30, [sp, #32]
314/// ldp x20, x19, [sp, #16]
315/// ldp x22, x21, [sp], #48
316/// ret
317/// @param M module
318/// @param MMI machine module info
319/// @param Regs callee save regs that the helper will handle
320/// @param Type frame helper type
321/// @return a helper function
322static Function *getOrCreateFrameHelper(Module *M, MachineModuleInfo *MMI,
323 SmallVectorImpl<unsigned> &Regs,
324 FrameHelperType Type,
325 unsigned FpOffset = 0) {
326 assert(Regs.size() >= 2);
327 auto Name = getFrameHelperName(Regs, Type, FpOffset);
328 auto *F = M->getFunction(Name);
329 if (F)
330 return F;
331
332 auto &MF = createFrameHelperMachineFunction(M, MMI, Name);
333 MachineBasicBlock &MBB = *MF.begin();
334 const TargetSubtargetInfo &STI = MF.getSubtarget();
335 const TargetInstrInfo &TII = *STI.getInstrInfo();
336
337 int Size = (int)Regs.size();
338 switch (Type) {
339 case FrameHelperType::Prolog:
340 case FrameHelperType::PrologFrame: {
341 // Compute the remaining SP adjust beyond FP/LR.
342 auto LRIdx = std::distance(first: Regs.begin(), last: llvm::find(Range&: Regs, Val: AArch64::LR));
343
344 // If the register stored to the lowest address is not LR, we must subtract
345 // more from SP here.
346 if (LRIdx != Size - 2) {
347 assert(Regs[Size - 2] != AArch64::LR);
348 emitStore(MF, MBB, Pos: MBB.end(), TII, Reg1: Regs[Size - 2], Reg2: Regs[Size - 1],
349 Offset: LRIdx - Size + 2, IsPreDec: true);
350 }
351
352 // Store CSRs in the reverse order.
353 for (int I = Size - 3; I >= 0; I -= 2) {
354 // FP/LR has been stored at call-site.
355 if (Regs[I - 1] == AArch64::LR)
356 continue;
357 emitStore(MF, MBB, Pos: MBB.end(), TII, Reg1: Regs[I - 1], Reg2: Regs[I], Offset: Size - I - 1,
358 IsPreDec: false);
359 }
360 if (Type == FrameHelperType::PrologFrame)
361 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AArch64::ADDXri))
362 .addDef(RegNo: AArch64::FP)
363 .addUse(RegNo: AArch64::SP)
364 .addImm(Val: FpOffset)
365 .addImm(Val: 0)
366 .setMIFlag(MachineInstr::FrameSetup);
367
368 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AArch64::RET))
369 .addReg(RegNo: AArch64::LR);
370 break;
371 }
372 case FrameHelperType::Epilog:
373 case FrameHelperType::EpilogTail:
374 if (Type == FrameHelperType::Epilog)
375 // Stash LR to X16
376 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AArch64::ORRXrs))
377 .addDef(RegNo: AArch64::X16)
378 .addReg(RegNo: AArch64::XZR)
379 .addUse(RegNo: AArch64::LR)
380 .addImm(Val: 0);
381
382 for (int I = 0; I < Size - 2; I += 2)
383 emitLoad(MF, MBB, Pos: MBB.end(), TII, Reg1: Regs[I], Reg2: Regs[I + 1], Offset: Size - I - 2,
384 IsPostDec: false);
385 // Restore the last CSR with post-increment of SP.
386 emitLoad(MF, MBB, Pos: MBB.end(), TII, Reg1: Regs[Size - 2], Reg2: Regs[Size - 1], Offset: Size,
387 IsPostDec: true);
388
389 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AArch64::RET))
390 .addReg(RegNo: Type == FrameHelperType::Epilog ? AArch64::X16 : AArch64::LR);
391 break;
392 }
393
394 return M->getFunction(Name);
395}
396
397/// This function checks if a frame helper should be used for
398/// HOM_Prolog/HOM_Epilog pseudo instruction expansion.
399/// @param MBB machine basic block
400/// @param NextMBBI next instruction following HOM_Prolog/HOM_Epilog
401/// @param Regs callee save registers that are saved or restored.
402/// @param Type frame helper type
403/// @return True if a use of helper is qualified.
404static bool shouldUseFrameHelper(MachineBasicBlock &MBB,
405 MachineBasicBlock::iterator &NextMBBI,
406 SmallVectorImpl<unsigned> &Regs,
407 FrameHelperType Type) {
408 const auto *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
409 auto RegCount = Regs.size();
410 assert(RegCount > 0 && (RegCount % 2 == 0));
411 // # of instructions that will be outlined.
412 int InstCount = RegCount / 2;
413
414 // Do not use a helper call when not saving LR.
415 if (!llvm::is_contained(Range&: Regs, Element: AArch64::LR))
416 return false;
417
418 switch (Type) {
419 case FrameHelperType::Prolog:
420 // Prolog helper cannot save FP/LR.
421 InstCount--;
422 break;
423 case FrameHelperType::PrologFrame: {
424 // Effectively no change in InstCount since FpAdjustment is included.
425 break;
426 }
427 case FrameHelperType::Epilog:
428 // Bail-out if X16 is live across the epilog helper because it is used in
429 // the helper to handle X30.
430 for (auto NextMI = NextMBBI; NextMI != MBB.end(); NextMI++) {
431 if (NextMI->readsRegister(Reg: AArch64::W16, TRI))
432 return false;
433 }
434 // Epilog may not be in the last block. Check the liveness in successors.
435 for (const MachineBasicBlock *SuccMBB : MBB.successors()) {
436 if (SuccMBB->isLiveIn(Reg: AArch64::W16) || SuccMBB->isLiveIn(Reg: AArch64::X16))
437 return false;
438 }
439 // No change in InstCount for the regular epilog case.
440 break;
441 case FrameHelperType::EpilogTail: {
442 // EpilogTail helper includes the caller's return.
443 if (NextMBBI == MBB.end())
444 return false;
445 if (NextMBBI->getOpcode() != AArch64::RET_ReallyLR)
446 return false;
447 InstCount++;
448 break;
449 }
450 }
451
452 return InstCount >= FrameHelperSizeThreshold;
453}
454
455/// Lower a HOM_Epilog pseudo instruction into a helper call while
456/// creating the helper on demand. Or emit a sequence of loads in place when not
457/// using a helper call.
458///
459/// 1. With a helper including ret
460/// HOM_Epilog x30, x29, x19, x20, x21, x22 ; MBBI
461/// ret ; NextMBBI
462/// =>
463/// b _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20x21x22
464/// ... ; NextMBBI
465///
466/// 2. With a helper
467/// HOM_Epilog x30, x29, x19, x20, x21, x22
468/// =>
469/// bl _OUTLINED_FUNCTION_EPILOG_x30x29x19x20x21x22
470///
471/// 3. Without a helper
472/// HOM_Epilog x30, x29, x19, x20, x21, x22
473/// =>
474/// ldp x29, x30, [sp, #32]
475/// ldp x20, x19, [sp, #16]
476/// ldp x22, x21, [sp], #48
477bool AArch64LowerHomogeneousPrologEpilogImpl::lowerEpilog(
478 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
479 MachineBasicBlock::iterator &NextMBBI) {
480 auto &MF = *MBB.getParent();
481 MachineInstr &MI = *MBBI;
482
483 DebugLoc DL = MI.getDebugLoc();
484 SmallVector<unsigned, 8> Regs;
485 bool HasUnpairedReg = false;
486 for (auto &MO : MI.operands())
487 if (MO.isReg()) {
488 if (!MO.getReg().isValid()) {
489 // For now we are only expecting unpaired GP registers which should
490 // occur exactly once.
491 assert(!HasUnpairedReg);
492 HasUnpairedReg = true;
493 }
494 Regs.push_back(Elt: MO.getReg());
495 }
496 (void)HasUnpairedReg;
497 int Size = (int)Regs.size();
498 if (Size == 0)
499 return false;
500 // Registers are in pair.
501 assert(Size % 2 == 0);
502 assert(MI.getOpcode() == AArch64::HOM_Epilog);
503
504 auto Return = NextMBBI;
505 MachineInstr *HelperCall = nullptr;
506 if (shouldUseFrameHelper(MBB, NextMBBI, Regs, Type: FrameHelperType::EpilogTail)) {
507 // When MBB ends with a return, emit a tail-call to the epilog helper
508 auto *EpilogTailHelper =
509 getOrCreateFrameHelper(M, MMI, Regs, Type: FrameHelperType::EpilogTail);
510 HelperCall = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::TCRETURNdi))
511 .addGlobalAddress(GV: EpilogTailHelper)
512 .addImm(Val: 0)
513 .setMIFlag(MachineInstr::FrameDestroy)
514 .copyImplicitOps(OtherMI: MI)
515 .copyImplicitOps(OtherMI: *Return);
516 NextMBBI = std::next(x: Return);
517 Return->removeFromParent();
518 } else if (shouldUseFrameHelper(MBB, NextMBBI, Regs,
519 Type: FrameHelperType::Epilog)) {
520 // The default epilog helper case.
521 auto *EpilogHelper =
522 getOrCreateFrameHelper(M, MMI, Regs, Type: FrameHelperType::Epilog);
523 HelperCall = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
524 .addGlobalAddress(GV: EpilogHelper)
525 .setMIFlag(MachineInstr::FrameDestroy)
526 .copyImplicitOps(OtherMI: MI);
527 } else {
528 // Fall back to no-helper.
529 for (int I = 0; I < Size - 2; I += 2)
530 emitLoad(MF, MBB, Pos: MBBI, TII: *TII, Reg1: Regs[I], Reg2: Regs[I + 1], Offset: Size - I - 2, IsPostDec: false);
531 // Restore the last CSR with post-increment of SP.
532 emitLoad(MF, MBB, Pos: MBBI, TII: *TII, Reg1: Regs[Size - 2], Reg2: Regs[Size - 1], Offset: Size, IsPostDec: true);
533 }
534
535 // Make sure all explicit definitions are preserved in the helper call;
536 // implicit ones are already handled by copyImplicitOps.
537 if (HelperCall)
538 for (auto &Def : MBBI->defs())
539 HelperCall->addRegisterDefined(Reg: Def.getReg(),
540 RegInfo: MF.getRegInfo().getTargetRegisterInfo());
541 MBBI->removeFromParent();
542 return true;
543}
544
545/// Lower a HOM_Prolog pseudo instruction into a helper call while
546/// creating the helper on demand. Or emit a sequence of stores in place when
547/// not using a helper call.
548///
549/// 1. With a helper including frame-setup
550/// HOM_Prolog x30, x29, x19, x20, x21, x22, 32
551/// =>
552/// stp x29, x30, [sp, #-16]!
553/// bl _OUTLINED_FUNCTION_PROLOG_FRAME32_x30x29x19x20x21x22
554///
555/// 2. With a helper
556/// HOM_Prolog x30, x29, x19, x20, x21, x22
557/// =>
558/// stp x29, x30, [sp, #-16]!
559/// bl _OUTLINED_FUNCTION_PROLOG_x30x29x19x20x21x22
560///
561/// 3. Without a helper
562/// HOM_Prolog x30, x29, x19, x20, x21, x22
563/// =>
564/// stp x22, x21, [sp, #-48]!
565/// stp x20, x19, [sp, #16]
566/// stp x29, x30, [sp, #32]
567bool AArch64LowerHomogeneousPrologEpilogImpl::lowerProlog(
568 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
569 MachineBasicBlock::iterator &NextMBBI) {
570 auto &MF = *MBB.getParent();
571 MachineInstr &MI = *MBBI;
572
573 DebugLoc DL = MI.getDebugLoc();
574 SmallVector<unsigned, 8> Regs;
575 bool HasUnpairedReg = false;
576 int LRIdx = 0;
577 std::optional<int> FpOffset;
578 for (auto &MO : MI.operands()) {
579 if (MO.isReg()) {
580 if (MO.getReg().isValid()) {
581 if (MO.getReg() == AArch64::LR)
582 LRIdx = Regs.size();
583 } else {
584 // For now we are only expecting unpaired GP registers which should
585 // occur exactly once.
586 assert(!HasUnpairedReg);
587 HasUnpairedReg = true;
588 }
589 Regs.push_back(Elt: MO.getReg());
590 } else if (MO.isImm()) {
591 FpOffset = MO.getImm();
592 }
593 }
594 (void)HasUnpairedReg;
595 int Size = (int)Regs.size();
596 if (Size == 0)
597 return false;
598 // Allow compact unwind case only for oww.
599 assert(Size % 2 == 0);
600 assert(MI.getOpcode() == AArch64::HOM_Prolog);
601
602 if (FpOffset &&
603 shouldUseFrameHelper(MBB, NextMBBI, Regs, Type: FrameHelperType::PrologFrame)) {
604 // FP/LR is stored at the top of stack before the prolog helper call.
605 emitStore(MF, MBB, Pos: MBBI, TII: *TII, Reg1: AArch64::LR, Reg2: AArch64::FP, Offset: -LRIdx - 2, IsPreDec: true);
606 auto *PrologFrameHelper = getOrCreateFrameHelper(
607 M, MMI, Regs, Type: FrameHelperType::PrologFrame, FpOffset: *FpOffset);
608 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
609 .addGlobalAddress(GV: PrologFrameHelper)
610 .setMIFlag(MachineInstr::FrameSetup)
611 .copyImplicitOps(OtherMI: MI)
612 .addReg(RegNo: AArch64::FP, Flags: RegState::Implicit | RegState::Define)
613 .addReg(RegNo: AArch64::SP, Flags: RegState::Implicit);
614 } else if (!FpOffset && shouldUseFrameHelper(MBB, NextMBBI, Regs,
615 Type: FrameHelperType::Prolog)) {
616 // FP/LR is stored at the top of stack before the prolog helper call.
617 emitStore(MF, MBB, Pos: MBBI, TII: *TII, Reg1: AArch64::LR, Reg2: AArch64::FP, Offset: -LRIdx - 2, IsPreDec: true);
618 auto *PrologHelper =
619 getOrCreateFrameHelper(M, MMI, Regs, Type: FrameHelperType::Prolog);
620 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
621 .addGlobalAddress(GV: PrologHelper)
622 .setMIFlag(MachineInstr::FrameSetup)
623 .copyImplicitOps(OtherMI: MI);
624 } else {
625 // Fall back to no-helper.
626 emitStore(MF, MBB, Pos: MBBI, TII: *TII, Reg1: Regs[Size - 2], Reg2: Regs[Size - 1], Offset: -Size, IsPreDec: true);
627 for (int I = Size - 3; I >= 0; I -= 2)
628 emitStore(MF, MBB, Pos: MBBI, TII: *TII, Reg1: Regs[I - 1], Reg2: Regs[I], Offset: Size - I - 1, IsPreDec: false);
629 if (FpOffset) {
630 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri))
631 .addDef(RegNo: AArch64::FP)
632 .addUse(RegNo: AArch64::SP)
633 .addImm(Val: *FpOffset)
634 .addImm(Val: 0)
635 .setMIFlag(MachineInstr::FrameSetup);
636 }
637 }
638
639 MBBI->removeFromParent();
640 return true;
641}
642
643/// Process each machine instruction
644/// @param MBB machine basic block
645/// @param MBBI current instruction iterator
646/// @param NextMBBI next instruction iterator which can be updated
647/// @return True when IR is changed.
648bool AArch64LowerHomogeneousPrologEpilogImpl::runOnMI(
649 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
650 MachineBasicBlock::iterator &NextMBBI) {
651 MachineInstr &MI = *MBBI;
652 unsigned Opcode = MI.getOpcode();
653 switch (Opcode) {
654 default:
655 break;
656 case AArch64::HOM_Prolog:
657 return lowerProlog(MBB, MBBI, NextMBBI);
658 case AArch64::HOM_Epilog:
659 return lowerEpilog(MBB, MBBI, NextMBBI);
660 }
661 return false;
662}
663
664bool AArch64LowerHomogeneousPrologEpilogImpl::runOnMBB(MachineBasicBlock &MBB) {
665 bool Modified = false;
666
667 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
668 while (MBBI != E) {
669 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
670 Modified |= runOnMI(MBB, MBBI, NextMBBI&: NMBBI);
671 MBBI = NMBBI;
672 }
673
674 return Modified;
675}
676
677bool AArch64LowerHomogeneousPrologEpilogImpl::runOnMachineFunction(
678 MachineFunction &MF) {
679 TII = MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
680
681 bool Modified = false;
682 for (auto &MBB : MF)
683 Modified |= runOnMBB(MBB);
684 return Modified;
685}
686
687ModulePass *llvm::createAArch64LowerHomogeneousPrologEpilogPass() {
688 return new AArch64LowerHomogeneousPrologEpilogLegacy();
689}
690