1//===- AArch64LowerHomogeneousPrologEpilog.cpp ----------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains a pass that lowers homogeneous prolog/epilog instructions.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64.h"
14#include "AArch64InstrInfo.h"
15#include "AArch64Subtarget.h"
16#include "MCTargetDesc/AArch64InstPrinter.h"
17#include "llvm/CodeGen/MachineBasicBlock.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/MachineFunctionAnalysis.h"
20#include "llvm/CodeGen/MachineInstr.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/MachineModuleInfo.h"
23#include "llvm/CodeGen/MachineOperand.h"
24#include "llvm/CodeGen/TargetSubtargetInfo.h"
25#include "llvm/IR/DebugLoc.h"
26#include "llvm/IR/IRBuilder.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/PassManager.h"
29#include "llvm/Pass.h"
30#include <optional>
31#include <sstream>
32
33using namespace llvm;
34
35#define AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME \
36 "AArch64 homogeneous prolog/epilog lowering pass"
37
38namespace {
39
40class AArch64LowerHomogeneousPrologEpilogImpl {
41public:
42 const AArch64InstrInfo *TII;
43
44 AArch64LowerHomogeneousPrologEpilogImpl(Module *M, MachineModuleInfo *MMI)
45 : M(M), MMI(MMI) {}
46
47 bool run();
48 bool runOnMachineFunction(MachineFunction &Fn);
49
50private:
51 Module *M;
52 MachineModuleInfo *MMI;
53
54 bool runOnMBB(MachineBasicBlock &MBB);
55 bool runOnMI(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
56 MachineBasicBlock::iterator &NextMBBI);
57
58 /// Lower a HOM_Prolog pseudo instruction into a helper call
59 /// or a sequence of homogeneous stores.
60 /// When a fp setup follows, it can be optimized.
61 bool lowerProlog(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
62 MachineBasicBlock::iterator &NextMBBI);
63 /// Lower a HOM_Epilog pseudo instruction into a helper call
64 /// or a sequence of homogeneous loads.
65 /// When a return follow, it can be optimized.
66 bool lowerEpilog(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
67 MachineBasicBlock::iterator &NextMBBI);
68};
69
70class AArch64LowerHomogeneousPrologEpilogLegacy : public ModulePass {
71public:
72 static char ID;
73
74 AArch64LowerHomogeneousPrologEpilogLegacy() : ModulePass(ID) {}
75 void getAnalysisUsage(AnalysisUsage &AU) const override {
76 AU.addRequired<MachineModuleInfoWrapperPass>();
77 AU.addPreserved<MachineModuleInfoWrapperPass>();
78 AU.setPreservesAll();
79 ModulePass::getAnalysisUsage(AU);
80 }
81 bool runOnModule(Module &M) override;
82
83 StringRef getPassName() const override {
84 return AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME;
85 }
86};
87
88} // end anonymous namespace
89
90char AArch64LowerHomogeneousPrologEpilogLegacy::ID = 0;
91
92INITIALIZE_PASS(AArch64LowerHomogeneousPrologEpilogLegacy,
93 "aarch64-lower-homogeneous-prolog-epilog",
94 AARCH64_LOWER_HOMOGENEOUS_PROLOG_EPILOG_NAME, false, false)
95
96bool AArch64LowerHomogeneousPrologEpilogLegacy::runOnModule(Module &M) {
97 if (skipModule(M))
98 return false;
99
100 MachineModuleInfo *MMI =
101 &getAnalysis<MachineModuleInfoWrapperPass>().getMMI();
102 return AArch64LowerHomogeneousPrologEpilogImpl(&M, MMI).run();
103}
104
105PreservedAnalyses
106AArch64LowerHomogeneousPrologEpilogPass::run(Module &M,
107 ModuleAnalysisManager &MAM) {
108 MachineModuleInfo *MMI = &MAM.getResult<MachineModuleAnalysis>(IR&: M).getMMI();
109 bool Changed = AArch64LowerHomogeneousPrologEpilogImpl(&M, MMI).run();
110 if (!Changed)
111 return PreservedAnalyses::all();
112 PreservedAnalyses PA;
113 PA.preserve<MachineModuleAnalysis>();
114 return PA;
115}
116
117bool AArch64LowerHomogeneousPrologEpilogImpl::run() {
118 bool Changed = false;
119 for (auto &F : *M) {
120 if (F.empty())
121 continue;
122
123 MachineFunction *MF = MMI->getMachineFunction(F);
124 if (!MF)
125 continue;
126 Changed |= runOnMachineFunction(Fn&: *MF);
127 }
128
129 return Changed;
130}
131enum FrameHelperType { Prolog, PrologFrame, Epilog, EpilogTail };
132
133/// Return a frame helper name with the given CSRs and the helper type.
134/// For instance, a prolog helper that saves x19 and x20 is named as
135/// OUTLINED_FUNCTION_PROLOG_x19x20.
136static std::string getFrameHelperName(SmallVectorImpl<unsigned> &Regs,
137 FrameHelperType Type, unsigned FpOffset) {
138 std::ostringstream RegStream;
139 switch (Type) {
140 case FrameHelperType::Prolog:
141 RegStream << "OUTLINED_FUNCTION_PROLOG_";
142 break;
143 case FrameHelperType::PrologFrame:
144 RegStream << "OUTLINED_FUNCTION_PROLOG_FRAME" << FpOffset << "_";
145 break;
146 case FrameHelperType::Epilog:
147 RegStream << "OUTLINED_FUNCTION_EPILOG_";
148 break;
149 case FrameHelperType::EpilogTail:
150 RegStream << "OUTLINED_FUNCTION_EPILOG_TAIL_";
151 break;
152 }
153
154 for (auto Reg : Regs) {
155 if (Reg == AArch64::NoRegister)
156 continue;
157 RegStream << AArch64InstPrinter::getRegisterName(Reg);
158 }
159
160 return RegStream.str();
161}
162
163/// Create a Function for the unique frame helper with the given name.
164/// Return a newly created MachineFunction with an empty MachineBasicBlock.
165static MachineFunction &createFrameHelperMachineFunction(Module *M,
166 MachineModuleInfo *MMI,
167 StringRef Name) {
168 LLVMContext &C = M->getContext();
169 Function *F = M->getFunction(Name);
170 assert(F == nullptr && "Function has been created before");
171 F = Function::Create(Ty: FunctionType::get(Result: Type::getVoidTy(C), isVarArg: false),
172 Linkage: Function::ExternalLinkage, N: Name, M);
173 assert(F && "Function was null!");
174
175 // Use ODR linkage to avoid duplication.
176 F->setLinkage(GlobalValue::LinkOnceODRLinkage);
177 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
178
179 // Set minsize, so we don't insert padding between outlined functions.
180 F->addFnAttr(Kind: Attribute::NoInline);
181 F->addFnAttr(Kind: Attribute::MinSize);
182 F->addFnAttr(Kind: Attribute::Naked);
183
184 MachineFunction &MF = MMI->getOrCreateMachineFunction(F&: *F);
185 // Remove unnecessary register liveness and set NoVRegs.
186 MF.getProperties()
187 .resetTracksLiveness()
188 .resetIsSSA()
189 .setNoVRegs()
190 .setNoPHIs();
191 MF.getRegInfo().freezeReservedRegs();
192
193 // Create entry block.
194 BasicBlock *EntryBB = BasicBlock::Create(Context&: C, Name: "entry", Parent: F);
195 IRBuilder<> Builder(EntryBB);
196 Builder.CreateRetVoid();
197
198 // Insert the new block into the function.
199 MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
200 MF.insert(MBBI: MF.begin(), MBB);
201
202 return MF;
203}
204
205/// Emit a store-pair instruction for frame-setup.
206/// If Reg2 is AArch64::NoRegister, emit STR instead.
207static void emitStore(MachineFunction &MF, MachineBasicBlock &MBB,
208 MachineBasicBlock::iterator Pos,
209 const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2,
210 int Offset, bool IsPreDec) {
211 assert(Reg1 != AArch64::NoRegister);
212 const bool IsPaired = Reg2 != AArch64::NoRegister;
213 bool IsFloat = AArch64::FPR64RegClass.contains(Reg: Reg1);
214 assert(!(IsFloat ^ AArch64::FPR64RegClass.contains(Reg2)));
215 unsigned Opc;
216 if (IsPreDec) {
217 if (IsFloat)
218 Opc = IsPaired ? AArch64::STPDpre : AArch64::STRDpre;
219 else
220 Opc = IsPaired ? AArch64::STPXpre : AArch64::STRXpre;
221 } else {
222 if (IsFloat)
223 Opc = IsPaired ? AArch64::STPDi : AArch64::STRDui;
224 else
225 Opc = IsPaired ? AArch64::STPXi : AArch64::STRXui;
226 }
227 // The implicit scale for Offset is 8.
228 TypeSize Scale(0U, false), Width(0U, false);
229 int64_t MinOffset, MaxOffset;
230 [[maybe_unused]] bool Success =
231 AArch64InstrInfo::getMemOpInfo(Opcode: Opc, Scale, Width, MinOffset, MaxOffset);
232 assert(Success && "Invalid Opcode");
233 Offset *= (8 / (int)Scale);
234
235 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: Pos, MIMD: DebugLoc(), MCID: TII.get(Opcode: Opc));
236 if (IsPreDec)
237 MIB.addDef(RegNo: AArch64::SP);
238 if (IsPaired)
239 MIB.addReg(RegNo: Reg2);
240 MIB.addReg(RegNo: Reg1)
241 .addReg(RegNo: AArch64::SP)
242 .addImm(Val: Offset)
243 .setMIFlag(MachineInstr::FrameSetup);
244}
245
246/// Emit a load-pair instruction for frame-destroy.
247/// If Reg2 is AArch64::NoRegister, emit LDR instead.
248static void emitLoad(MachineFunction &MF, MachineBasicBlock &MBB,
249 MachineBasicBlock::iterator Pos,
250 const TargetInstrInfo &TII, unsigned Reg1, unsigned Reg2,
251 int Offset, bool IsPostDec) {
252 assert(Reg1 != AArch64::NoRegister);
253 const bool IsPaired = Reg2 != AArch64::NoRegister;
254 bool IsFloat = AArch64::FPR64RegClass.contains(Reg: Reg1);
255 assert(!(IsFloat ^ AArch64::FPR64RegClass.contains(Reg2)));
256 unsigned Opc;
257 if (IsPostDec) {
258 if (IsFloat)
259 Opc = IsPaired ? AArch64::LDPDpost : AArch64::LDRDpost;
260 else
261 Opc = IsPaired ? AArch64::LDPXpost : AArch64::LDRXpost;
262 } else {
263 if (IsFloat)
264 Opc = IsPaired ? AArch64::LDPDi : AArch64::LDRDui;
265 else
266 Opc = IsPaired ? AArch64::LDPXi : AArch64::LDRXui;
267 }
268 // The implicit scale for Offset is 8.
269 TypeSize Scale(0U, false), Width(0U, false);
270 int64_t MinOffset, MaxOffset;
271 [[maybe_unused]] bool Success =
272 AArch64InstrInfo::getMemOpInfo(Opcode: Opc, Scale, Width, MinOffset, MaxOffset);
273 assert(Success && "Invalid Opcode");
274 Offset *= (8 / (int)Scale);
275
276 MachineInstrBuilder MIB = BuildMI(BB&: MBB, I: Pos, MIMD: DebugLoc(), MCID: TII.get(Opcode: Opc));
277 if (IsPostDec)
278 MIB.addDef(RegNo: AArch64::SP);
279 if (IsPaired)
280 MIB.addReg(RegNo: Reg2, Flags: getDefRegState(B: true));
281 MIB.addReg(RegNo: Reg1, Flags: getDefRegState(B: true))
282 .addReg(RegNo: AArch64::SP)
283 .addImm(Val: Offset)
284 .setMIFlag(MachineInstr::FrameDestroy);
285}
286
287/// Return a unique function if a helper can be formed with the given Regs
288/// and frame type.
289/// 1) _OUTLINED_FUNCTION_PROLOG_x30x29x19x20x21x22:
290/// stp x22, x21, [sp, #-32]! ; x29/x30 has been stored at the caller
291/// stp x20, x19, [sp, #16]
292/// ret
293///
294/// 2) _OUTLINED_FUNCTION_PROLOG_FRAME32_x30x29x19x20x21x22:
295/// stp x22, x21, [sp, #-32]! ; x29/x30 has been stored at the caller
296/// stp x20, x19, [sp, #16]
297/// add fp, sp, #32
298/// ret
299///
300/// 3) _OUTLINED_FUNCTION_EPILOG_x30x29x19x20x21x22:
301/// mov x16, x30
302/// ldp x29, x30, [sp, #32]
303/// ldp x20, x19, [sp, #16]
304/// ldp x22, x21, [sp], #48
305/// ret x16
306///
307/// 4) _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20x21x22:
308/// ldp x29, x30, [sp, #32]
309/// ldp x20, x19, [sp, #16]
310/// ldp x22, x21, [sp], #48
311/// ret
312/// @param M module
313/// @param MMI machine module info
314/// @param Regs callee save regs that the helper will handle
315/// @param Type frame helper type
316/// @return a helper function
317static Function *getOrCreateFrameHelper(Module *M, MachineModuleInfo *MMI,
318 SmallVectorImpl<unsigned> &Regs,
319 FrameHelperType Type,
320 unsigned FpOffset = 0) {
321 assert(Regs.size() >= 2);
322 auto Name = getFrameHelperName(Regs, Type, FpOffset);
323 auto *F = M->getFunction(Name);
324 if (F)
325 return F;
326
327 auto &MF = createFrameHelperMachineFunction(M, MMI, Name);
328 MachineBasicBlock &MBB = *MF.begin();
329 const TargetSubtargetInfo &STI = MF.getSubtarget();
330 const TargetInstrInfo &TII = *STI.getInstrInfo();
331
332 int Size = (int)Regs.size();
333 switch (Type) {
334 case FrameHelperType::Prolog:
335 case FrameHelperType::PrologFrame: {
336 // Compute the remaining SP adjust beyond FP/LR.
337 auto LRIdx = std::distance(first: Regs.begin(), last: llvm::find(Range&: Regs, Val: AArch64::LR));
338
339 // If the register stored to the lowest address is not LR, we must subtract
340 // more from SP here.
341 if (LRIdx != Size - 2) {
342 assert(Regs[Size - 2] != AArch64::LR);
343 emitStore(MF, MBB, Pos: MBB.end(), TII, Reg1: Regs[Size - 2], Reg2: Regs[Size - 1],
344 Offset: LRIdx - Size + 2, IsPreDec: true);
345 }
346
347 // Store CSRs in the reverse order.
348 for (int I = Size - 3; I >= 0; I -= 2) {
349 // FP/LR has been stored at call-site.
350 if (Regs[I - 1] == AArch64::LR)
351 continue;
352 emitStore(MF, MBB, Pos: MBB.end(), TII, Reg1: Regs[I - 1], Reg2: Regs[I], Offset: Size - I - 1,
353 IsPreDec: false);
354 }
355 if (Type == FrameHelperType::PrologFrame)
356 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AArch64::ADDXri))
357 .addDef(RegNo: AArch64::FP)
358 .addUse(RegNo: AArch64::SP)
359 .addImm(Val: FpOffset)
360 .addImm(Val: 0)
361 .setMIFlag(MachineInstr::FrameSetup);
362
363 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AArch64::RET))
364 .addReg(RegNo: AArch64::LR);
365 break;
366 }
367 case FrameHelperType::Epilog:
368 case FrameHelperType::EpilogTail:
369 if (Type == FrameHelperType::Epilog)
370 // Stash LR to X16
371 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AArch64::ORRXrs))
372 .addDef(RegNo: AArch64::X16)
373 .addReg(RegNo: AArch64::XZR)
374 .addUse(RegNo: AArch64::LR)
375 .addImm(Val: 0);
376
377 for (int I = 0; I < Size - 2; I += 2)
378 emitLoad(MF, MBB, Pos: MBB.end(), TII, Reg1: Regs[I], Reg2: Regs[I + 1], Offset: Size - I - 2,
379 IsPostDec: false);
380 // Restore the last CSR with post-increment of SP.
381 emitLoad(MF, MBB, Pos: MBB.end(), TII, Reg1: Regs[Size - 2], Reg2: Regs[Size - 1], Offset: Size,
382 IsPostDec: true);
383
384 BuildMI(BB&: MBB, I: MBB.end(), MIMD: DebugLoc(), MCID: TII.get(Opcode: AArch64::RET))
385 .addReg(RegNo: Type == FrameHelperType::Epilog ? AArch64::X16 : AArch64::LR);
386 break;
387 }
388
389 return M->getFunction(Name);
390}
391
392/// This function checks if a frame helper should be used for
393/// HOM_Prolog/HOM_Epilog pseudo instruction expansion.
394/// @param MBB machine basic block
395/// @param NextMBBI next instruction following HOM_Prolog/HOM_Epilog
396/// @param Regs callee save registers that are saved or restored.
397/// @param Type frame helper type
398/// @return True if a use of helper is qualified.
399static bool shouldUseFrameHelper(MachineBasicBlock &MBB,
400 MachineBasicBlock::iterator &NextMBBI,
401 SmallVectorImpl<unsigned> &Regs,
402 FrameHelperType Type) {
403 const auto *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
404 auto RegCount = Regs.size();
405 assert(RegCount > 0 && (RegCount % 2 == 0));
406 // # of instructions that will be outlined.
407 int InstCount = RegCount / 2;
408
409 // Do not use a helper call when not saving LR.
410 if (!llvm::is_contained(Range&: Regs, Element: AArch64::LR))
411 return false;
412
413 switch (Type) {
414 case FrameHelperType::Prolog:
415 // Prolog helper cannot save FP/LR.
416 InstCount--;
417 break;
418 case FrameHelperType::PrologFrame: {
419 // Effectively no change in InstCount since FpAdjustment is included.
420 break;
421 }
422 case FrameHelperType::Epilog:
423 // Bail-out if X16 is live across the epilog helper because it is used in
424 // the helper to handle X30.
425 for (auto NextMI = NextMBBI; NextMI != MBB.end(); NextMI++) {
426 if (NextMI->readsRegister(Reg: AArch64::W16, TRI))
427 return false;
428 }
429 // Epilog may not be in the last block. Check the liveness in successors.
430 for (const MachineBasicBlock *SuccMBB : MBB.successors()) {
431 if (SuccMBB->isLiveIn(Reg: AArch64::W16) || SuccMBB->isLiveIn(Reg: AArch64::X16))
432 return false;
433 }
434 // No change in InstCount for the regular epilog case.
435 break;
436 case FrameHelperType::EpilogTail: {
437 // EpilogTail helper includes the caller's return.
438 if (NextMBBI == MBB.end())
439 return false;
440 if (NextMBBI->getOpcode() != AArch64::RET_ReallyLR)
441 return false;
442 InstCount++;
443 break;
444 }
445 }
446
447 return InstCount >= MBB.getParent()
448 ->getSubtarget<AArch64Subtarget>()
449 .getCLOpts()
450 .frame_helper_size_threshold;
451}
452
453/// Lower a HOM_Epilog pseudo instruction into a helper call while
454/// creating the helper on demand. Or emit a sequence of loads in place when not
455/// using a helper call.
456///
457/// 1. With a helper including ret
458/// HOM_Epilog x30, x29, x19, x20, x21, x22 ; MBBI
459/// ret ; NextMBBI
460/// =>
461/// b _OUTLINED_FUNCTION_EPILOG_TAIL_x30x29x19x20x21x22
462/// ... ; NextMBBI
463///
464/// 2. With a helper
465/// HOM_Epilog x30, x29, x19, x20, x21, x22
466/// =>
467/// bl _OUTLINED_FUNCTION_EPILOG_x30x29x19x20x21x22
468///
469/// 3. Without a helper
470/// HOM_Epilog x30, x29, x19, x20, x21, x22
471/// =>
472/// ldp x29, x30, [sp, #32]
473/// ldp x20, x19, [sp, #16]
474/// ldp x22, x21, [sp], #48
475bool AArch64LowerHomogeneousPrologEpilogImpl::lowerEpilog(
476 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
477 MachineBasicBlock::iterator &NextMBBI) {
478 auto &MF = *MBB.getParent();
479 MachineInstr &MI = *MBBI;
480
481 DebugLoc DL = MI.getDebugLoc();
482 SmallVector<unsigned, 8> Regs;
483 bool HasUnpairedReg = false;
484 for (auto &MO : MI.operands())
485 if (MO.isReg()) {
486 if (!MO.getReg().isValid()) {
487 // For now we are only expecting unpaired GP registers which should
488 // occur exactly once.
489 assert(!HasUnpairedReg);
490 HasUnpairedReg = true;
491 }
492 Regs.push_back(Elt: MO.getReg());
493 }
494 (void)HasUnpairedReg;
495 int Size = (int)Regs.size();
496 if (Size == 0)
497 return false;
498 // Registers are in pair.
499 assert(Size % 2 == 0);
500 assert(MI.getOpcode() == AArch64::HOM_Epilog);
501
502 auto Return = NextMBBI;
503 MachineInstr *HelperCall = nullptr;
504 if (shouldUseFrameHelper(MBB, NextMBBI, Regs, Type: FrameHelperType::EpilogTail)) {
505 // When MBB ends with a return, emit a tail-call to the epilog helper
506 auto *EpilogTailHelper =
507 getOrCreateFrameHelper(M, MMI, Regs, Type: FrameHelperType::EpilogTail);
508 HelperCall = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::TCRETURNdi))
509 .addGlobalAddress(GV: EpilogTailHelper)
510 .addImm(Val: 0)
511 .setMIFlag(MachineInstr::FrameDestroy)
512 .copyImplicitOps(OtherMI: MI)
513 .copyImplicitOps(OtherMI: *Return);
514 NextMBBI = std::next(x: Return);
515 Return->removeFromParent();
516 } else if (shouldUseFrameHelper(MBB, NextMBBI, Regs,
517 Type: FrameHelperType::Epilog)) {
518 // The default epilog helper case.
519 auto *EpilogHelper =
520 getOrCreateFrameHelper(M, MMI, Regs, Type: FrameHelperType::Epilog);
521 HelperCall = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
522 .addGlobalAddress(GV: EpilogHelper)
523 .setMIFlag(MachineInstr::FrameDestroy)
524 .copyImplicitOps(OtherMI: MI);
525 } else {
526 // Fall back to no-helper.
527 for (int I = 0; I < Size - 2; I += 2)
528 emitLoad(MF, MBB, Pos: MBBI, TII: *TII, Reg1: Regs[I], Reg2: Regs[I + 1], Offset: Size - I - 2, IsPostDec: false);
529 // Restore the last CSR with post-increment of SP.
530 emitLoad(MF, MBB, Pos: MBBI, TII: *TII, Reg1: Regs[Size - 2], Reg2: Regs[Size - 1], Offset: Size, IsPostDec: true);
531 }
532
533 // Make sure all explicit definitions are preserved in the helper call;
534 // implicit ones are already handled by copyImplicitOps.
535 if (HelperCall)
536 for (auto &Def : MBBI->defs())
537 HelperCall->addRegisterDefined(Reg: Def.getReg(),
538 RegInfo: MF.getRegInfo().getTargetRegisterInfo());
539 MBBI->removeFromParent();
540 return true;
541}
542
543/// Lower a HOM_Prolog pseudo instruction into a helper call while
544/// creating the helper on demand. Or emit a sequence of stores in place when
545/// not using a helper call.
546///
547/// 1. With a helper including frame-setup
548/// HOM_Prolog x30, x29, x19, x20, x21, x22, 32
549/// =>
550/// stp x29, x30, [sp, #-16]!
551/// bl _OUTLINED_FUNCTION_PROLOG_FRAME32_x30x29x19x20x21x22
552///
553/// 2. With a helper
554/// HOM_Prolog x30, x29, x19, x20, x21, x22
555/// =>
556/// stp x29, x30, [sp, #-16]!
557/// bl _OUTLINED_FUNCTION_PROLOG_x30x29x19x20x21x22
558///
559/// 3. Without a helper
560/// HOM_Prolog x30, x29, x19, x20, x21, x22
561/// =>
562/// stp x22, x21, [sp, #-48]!
563/// stp x20, x19, [sp, #16]
564/// stp x29, x30, [sp, #32]
565bool AArch64LowerHomogeneousPrologEpilogImpl::lowerProlog(
566 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
567 MachineBasicBlock::iterator &NextMBBI) {
568 auto &MF = *MBB.getParent();
569 MachineInstr &MI = *MBBI;
570
571 DebugLoc DL = MI.getDebugLoc();
572 SmallVector<unsigned, 8> Regs;
573 bool HasUnpairedReg = false;
574 int LRIdx = 0;
575 std::optional<int> FpOffset;
576 for (auto &MO : MI.operands()) {
577 if (MO.isReg()) {
578 if (MO.getReg().isValid()) {
579 if (MO.getReg() == AArch64::LR)
580 LRIdx = Regs.size();
581 } else {
582 // For now we are only expecting unpaired GP registers which should
583 // occur exactly once.
584 assert(!HasUnpairedReg);
585 HasUnpairedReg = true;
586 }
587 Regs.push_back(Elt: MO.getReg());
588 } else if (MO.isImm()) {
589 FpOffset = MO.getImm();
590 }
591 }
592 (void)HasUnpairedReg;
593 int Size = (int)Regs.size();
594 if (Size == 0)
595 return false;
596 // Allow compact unwind case only for oww.
597 assert(Size % 2 == 0);
598 assert(MI.getOpcode() == AArch64::HOM_Prolog);
599
600 if (FpOffset &&
601 shouldUseFrameHelper(MBB, NextMBBI, Regs, Type: FrameHelperType::PrologFrame)) {
602 // FP/LR is stored at the top of stack before the prolog helper call.
603 emitStore(MF, MBB, Pos: MBBI, TII: *TII, Reg1: AArch64::LR, Reg2: AArch64::FP, Offset: -LRIdx - 2, IsPreDec: true);
604 auto *PrologFrameHelper = getOrCreateFrameHelper(
605 M, MMI, Regs, Type: FrameHelperType::PrologFrame, FpOffset: *FpOffset);
606 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
607 .addGlobalAddress(GV: PrologFrameHelper)
608 .setMIFlag(MachineInstr::FrameSetup)
609 .copyImplicitOps(OtherMI: MI)
610 .addReg(RegNo: AArch64::FP, Flags: RegState::Implicit | RegState::Define)
611 .addReg(RegNo: AArch64::SP, Flags: RegState::Implicit);
612 } else if (!FpOffset && shouldUseFrameHelper(MBB, NextMBBI, Regs,
613 Type: FrameHelperType::Prolog)) {
614 // FP/LR is stored at the top of stack before the prolog helper call.
615 emitStore(MF, MBB, Pos: MBBI, TII: *TII, Reg1: AArch64::LR, Reg2: AArch64::FP, Offset: -LRIdx - 2, IsPreDec: true);
616 auto *PrologHelper =
617 getOrCreateFrameHelper(M, MMI, Regs, Type: FrameHelperType::Prolog);
618 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::BL))
619 .addGlobalAddress(GV: PrologHelper)
620 .setMIFlag(MachineInstr::FrameSetup)
621 .copyImplicitOps(OtherMI: MI);
622 } else {
623 // Fall back to no-helper.
624 emitStore(MF, MBB, Pos: MBBI, TII: *TII, Reg1: Regs[Size - 2], Reg2: Regs[Size - 1], Offset: -Size, IsPreDec: true);
625 for (int I = Size - 3; I >= 0; I -= 2)
626 emitStore(MF, MBB, Pos: MBBI, TII: *TII, Reg1: Regs[I - 1], Reg2: Regs[I], Offset: Size - I - 1, IsPreDec: false);
627 if (FpOffset) {
628 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AArch64::ADDXri))
629 .addDef(RegNo: AArch64::FP)
630 .addUse(RegNo: AArch64::SP)
631 .addImm(Val: *FpOffset)
632 .addImm(Val: 0)
633 .setMIFlag(MachineInstr::FrameSetup);
634 }
635 }
636
637 MBBI->removeFromParent();
638 return true;
639}
640
641/// Process each machine instruction
642/// @param MBB machine basic block
643/// @param MBBI current instruction iterator
644/// @param NextMBBI next instruction iterator which can be updated
645/// @return True when IR is changed.
646bool AArch64LowerHomogeneousPrologEpilogImpl::runOnMI(
647 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
648 MachineBasicBlock::iterator &NextMBBI) {
649 MachineInstr &MI = *MBBI;
650 unsigned Opcode = MI.getOpcode();
651 switch (Opcode) {
652 default:
653 break;
654 case AArch64::HOM_Prolog:
655 return lowerProlog(MBB, MBBI, NextMBBI);
656 case AArch64::HOM_Epilog:
657 return lowerEpilog(MBB, MBBI, NextMBBI);
658 }
659 return false;
660}
661
662bool AArch64LowerHomogeneousPrologEpilogImpl::runOnMBB(MachineBasicBlock &MBB) {
663 bool Modified = false;
664
665 MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
666 while (MBBI != E) {
667 MachineBasicBlock::iterator NMBBI = std::next(x: MBBI);
668 Modified |= runOnMI(MBB, MBBI, NextMBBI&: NMBBI);
669 MBBI = NMBBI;
670 }
671
672 return Modified;
673}
674
675bool AArch64LowerHomogeneousPrologEpilogImpl::runOnMachineFunction(
676 MachineFunction &MF) {
677 TII = MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
678
679 bool Modified = false;
680 for (auto &MBB : MF)
681 Modified |= runOnMBB(MBB);
682 return Modified;
683}
684
685ModulePass *llvm::createAArch64LowerHomogeneousPrologEpilogPass() {
686 return new AArch64LowerHomogeneousPrologEpilogLegacy();
687}
688