1//===- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass is responsible for finalizing the functions frame layout, saving
10// callee saved registers, and for emitting prolog & epilog code for the
11// function.
12//
13// This pass must be run after register allocation. After this pass is
14// executed, it is illegal to construct MO_FrameIndex operands.
15//
16//===----------------------------------------------------------------------===//
17
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SetVector.h"
22#include "llvm/ADT/SmallPtrSet.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/Analysis/OptimizationRemarkEmitter.h"
27#include "llvm/CodeGen/MachineBasicBlock.h"
28#include "llvm/CodeGen/MachineDominators.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineFunctionPass.h"
32#include "llvm/CodeGen/MachineInstr.h"
33#include "llvm/CodeGen/MachineLoopInfo.h"
34#include "llvm/CodeGen/MachineModuleInfo.h"
35#include "llvm/CodeGen/MachineOperand.h"
36#include "llvm/CodeGen/MachineOptimizationRemarkEmitter.h"
37#include "llvm/CodeGen/MachineRegisterInfo.h"
38#include "llvm/CodeGen/PEI.h"
39#include "llvm/CodeGen/RegisterScavenging.h"
40#include "llvm/CodeGen/TargetFrameLowering.h"
41#include "llvm/CodeGen/TargetInstrInfo.h"
42#include "llvm/CodeGen/TargetOpcodes.h"
43#include "llvm/CodeGen/TargetRegisterInfo.h"
44#include "llvm/CodeGen/TargetSubtargetInfo.h"
45#include "llvm/CodeGen/WinEHFuncInfo.h"
46#include "llvm/IR/Attributes.h"
47#include "llvm/IR/CallingConv.h"
48#include "llvm/IR/DebugInfoMetadata.h"
49#include "llvm/IR/DiagnosticInfo.h"
50#include "llvm/IR/Function.h"
51#include "llvm/IR/LLVMContext.h"
52#include "llvm/InitializePasses.h"
53#include "llvm/Pass.h"
54#include "llvm/Support/CodeGen.h"
55#include "llvm/Support/Debug.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Support/FormatVariadic.h"
58#include "llvm/Support/raw_ostream.h"
59#include "llvm/Target/TargetMachine.h"
60#include "llvm/Target/TargetOptions.h"
61#include <algorithm>
62#include <cassert>
63#include <cstdint>
64#include <limits>
65#include <utility>
66#include <vector>
67
68using namespace llvm;
69
70#define DEBUG_TYPE "prologepilog"
71
72using MBBVector = SmallVector<MachineBasicBlock *, 4>;
73
74STATISTIC(NumLeafFuncWithSpills, "Number of leaf functions with CSRs");
75STATISTIC(NumFuncSeen, "Number of functions seen in PEI");
76
77
78namespace {
79
80class PEIImpl {
81 RegScavenger *RS = nullptr;
82
83 // Save and Restore blocks of the current function. Typically there is a
84 // single save block, unless Windows EH funclets are involved.
85 MBBVector SaveBlocks;
86 MBBVector RestoreBlocks;
87
88 // Flag to control whether to use the register scavenger to resolve
89 // frame index materialization registers. Set according to
90 // TRI->requiresFrameIndexScavenging() for the current function.
91 bool FrameIndexVirtualScavenging = false;
92
93 // Flag to control whether the scavenger should be passed even though
94 // FrameIndexVirtualScavenging is used.
95 bool FrameIndexEliminationScavenging = false;
96
97 // Emit remarks.
98 MachineOptimizationRemarkEmitter *ORE = nullptr;
99
100 void calculateCallFrameInfo(MachineFunction &MF);
101 void calculateSaveRestoreBlocks(MachineFunction &MF);
102 void spillCalleeSavedRegs(MachineFunction &MF);
103
104 void calculateFrameObjectOffsets(MachineFunction &MF);
105 void replaceFrameIndices(MachineFunction &MF);
106 void replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
107 int &SPAdj);
108 // Frame indices in debug values are encoded in a target independent
109 // way with simply the frame index and offset rather than any
110 // target-specific addressing mode.
111 bool replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI,
112 unsigned OpIdx, int SPAdj = 0);
113 // Does same as replaceFrameIndices but using the backward MIR walk and
114 // backward register scavenger walk.
115 void replaceFrameIndicesBackward(MachineFunction &MF);
116 void replaceFrameIndicesBackward(MachineBasicBlock *BB, MachineFunction &MF,
117 int &SPAdj);
118
119 void insertPrologEpilogCode(MachineFunction &MF);
120 void insertZeroCallUsedRegs(MachineFunction &MF);
121
122public:
123 PEIImpl(MachineOptimizationRemarkEmitter *ORE) : ORE(ORE) {}
124 bool run(MachineFunction &MF);
125};
126
127class PEILegacy : public MachineFunctionPass {
128public:
129 static char ID;
130
131 PEILegacy() : MachineFunctionPass(ID) {}
132
133 void getAnalysisUsage(AnalysisUsage &AU) const override;
134
135 /// runOnMachineFunction - Insert prolog/epilog code and replace abstract
136 /// frame indexes with appropriate references.
137 bool runOnMachineFunction(MachineFunction &MF) override;
138};
139
140} // end anonymous namespace
141
142char PEILegacy::ID = 0;
143
144char &llvm::PrologEpilogCodeInserterID = PEILegacy::ID;
145
146INITIALIZE_PASS_BEGIN(PEILegacy, DEBUG_TYPE, "Prologue/Epilogue Insertion",
147 false, false)
148INITIALIZE_PASS_DEPENDENCY(MachineLoopInfoWrapperPass)
149INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
150INITIALIZE_PASS_DEPENDENCY(MachineOptimizationRemarkEmitterPass)
151INITIALIZE_PASS_END(PEILegacy, DEBUG_TYPE,
152 "Prologue/Epilogue Insertion & Frame Finalization", false,
153 false)
154
155MachineFunctionPass *llvm::createPrologEpilogInserterPass() {
156 return new PEILegacy();
157}
158
159STATISTIC(NumBytesStackSpace,
160 "Number of bytes used for stack in all functions");
161
162void PEILegacy::getAnalysisUsage(AnalysisUsage &AU) const {
163 AU.setPreservesCFG();
164 AU.addPreserved<MachineLoopInfoWrapperPass>();
165 AU.addPreserved<MachineDominatorTreeWrapperPass>();
166 AU.addRequired<MachineOptimizationRemarkEmitterPass>();
167 MachineFunctionPass::getAnalysisUsage(AU);
168}
169
170/// StackObjSet - A set of stack object indexes
171using StackObjSet = SmallSetVector<int, 8>;
172
173using SavedDbgValuesMap =
174 SmallDenseMap<MachineBasicBlock *, SmallVector<MachineInstr *, 4>, 4>;
175
176/// Stash DBG_VALUEs that describe parameters and which are placed at the start
177/// of the block. Later on, after the prologue code has been emitted, the
178/// stashed DBG_VALUEs will be reinserted at the start of the block.
179static void stashEntryDbgValues(MachineBasicBlock &MBB,
180 SavedDbgValuesMap &EntryDbgValues) {
181 SmallVector<const MachineInstr *, 4> FrameIndexValues;
182
183 for (auto &MI : MBB) {
184 if (!MI.isDebugInstr())
185 break;
186 if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter())
187 continue;
188 if (any_of(Range: MI.debug_operands(),
189 P: [](const MachineOperand &MO) { return MO.isFI(); })) {
190 // We can only emit valid locations for frame indices after the frame
191 // setup, so do not stash away them.
192 FrameIndexValues.push_back(Elt: &MI);
193 continue;
194 }
195 const DILocalVariable *Var = MI.getDebugVariable();
196 const DIExpression *Expr = MI.getDebugExpression();
197 auto Overlaps = [Var, Expr](const MachineInstr *DV) {
198 return Var == DV->getDebugVariable() &&
199 Expr->fragmentsOverlap(Other: DV->getDebugExpression());
200 };
201 // See if the debug value overlaps with any preceding debug value that will
202 // not be stashed. If that is the case, then we can't stash this value, as
203 // we would then reorder the values at reinsertion.
204 if (llvm::none_of(Range&: FrameIndexValues, P: Overlaps))
205 EntryDbgValues[&MBB].push_back(Elt: &MI);
206 }
207
208 // Remove stashed debug values from the block.
209 if (auto It = EntryDbgValues.find(Val: &MBB); It != EntryDbgValues.end())
210 for (auto *MI : It->second)
211 MI->removeFromParent();
212}
213
214bool PEIImpl::run(MachineFunction &MF) {
215 NumFuncSeen++;
216 const Function &F = MF.getFunction();
217 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
218 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
219
220 RS = TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr;
221 FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(MF);
222
223 // Spill frame pointer and/or base pointer registers if they are clobbered.
224 // It is placed before call frame instruction elimination so it will not mess
225 // with stack arguments.
226 TFI->spillFPBP(MF);
227
228 // Calculate the MaxCallFrameSize value for the function's frame
229 // information. Also eliminates call frame pseudo instructions.
230 calculateCallFrameInfo(MF);
231
232 // Determine placement of CSR spill/restore code and prolog/epilog code:
233 // place all spills in the entry block, all restores in return blocks.
234 calculateSaveRestoreBlocks(MF);
235
236 // Stash away DBG_VALUEs that should not be moved by insertion of prolog code.
237 SavedDbgValuesMap EntryDbgValues;
238 for (MachineBasicBlock *SaveBlock : SaveBlocks)
239 stashEntryDbgValues(MBB&: *SaveBlock, EntryDbgValues);
240
241 // Handle CSR spilling and restoring, for targets that need it.
242 if (MF.getTarget().usesPhysRegsForValues())
243 spillCalleeSavedRegs(MF);
244
245 // Allow the target machine to make final modifications to the function
246 // before the frame layout is finalized.
247 TFI->processFunctionBeforeFrameFinalized(MF, RS);
248
249 // Calculate actual frame offsets for all abstract stack objects...
250 calculateFrameObjectOffsets(MF);
251
252 // Add prolog and epilog code to the function. This function is required
253 // to align the stack frame as necessary for any stack variables or
254 // called functions. Because of this, calculateCalleeSavedRegisters()
255 // must be called before this function in order to set the AdjustsStack
256 // and MaxCallFrameSize variables.
257 if (!F.hasFnAttribute(Kind: Attribute::Naked))
258 insertPrologEpilogCode(MF);
259
260 // Reinsert stashed debug values at the start of the entry blocks.
261 for (auto &I : EntryDbgValues)
262 I.first->insert(I: I.first->begin(), S: I.second.begin(), E: I.second.end());
263
264 // Allow the target machine to make final modifications to the function
265 // before the frame layout is finalized.
266 TFI->processFunctionBeforeFrameIndicesReplaced(MF, RS);
267
268 // Replace all MO_FrameIndex operands with physical register references
269 // and actual offsets.
270 if (TFI->needsFrameIndexResolution(MF)) {
271 // Allow the target to determine this after knowing the frame size.
272 FrameIndexEliminationScavenging =
273 (RS && !FrameIndexVirtualScavenging) ||
274 TRI->requiresFrameIndexReplacementScavenging(MF);
275
276 if (TRI->eliminateFrameIndicesBackwards())
277 replaceFrameIndicesBackward(MF);
278 else
279 replaceFrameIndices(MF);
280 }
281
282 // If register scavenging is needed, as we've enabled doing it as a
283 // post-pass, scavenge the virtual registers that frame index elimination
284 // inserted.
285 if (TRI->requiresRegisterScavenging(MF) && FrameIndexVirtualScavenging)
286 scavengeFrameVirtualRegs(MF, RS&: *RS);
287
288 // Warn on stack size when we exceeds the given limit.
289 MachineFrameInfo &MFI = MF.getFrameInfo();
290 uint64_t StackSize = MFI.getStackSize();
291
292 uint64_t Threshold = TFI->getStackThreshold();
293 if (MF.getFunction().hasFnAttribute(Kind: "warn-stack-size")) {
294 bool Failed = MF.getFunction()
295 .getFnAttribute(Kind: "warn-stack-size")
296 .getValueAsString()
297 .getAsInteger(Radix: 10, Result&: Threshold);
298 // Verifier should have caught this.
299 assert(!Failed && "Invalid warn-stack-size fn attr value");
300 (void)Failed;
301 }
302 uint64_t UnsafeStackSize = MFI.getUnsafeStackSize();
303 if (MF.getFunction().hasFnAttribute(Kind: Attribute::SafeStack))
304 StackSize += UnsafeStackSize;
305
306 if (StackSize > Threshold) {
307 DiagnosticInfoStackSize DiagStackSize(F, StackSize, Threshold, DS_Warning);
308 F.getContext().diagnose(DI: DiagStackSize);
309 int64_t SpillSize = 0;
310 for (int Idx = MFI.getObjectIndexBegin(), End = MFI.getObjectIndexEnd();
311 Idx != End; ++Idx) {
312 if (MFI.isSpillSlotObjectIndex(ObjectIdx: Idx))
313 SpillSize += MFI.getObjectSize(ObjectIdx: Idx);
314 }
315
316 [[maybe_unused]] float SpillPct =
317 static_cast<float>(SpillSize) / static_cast<float>(StackSize);
318 LLVM_DEBUG(
319 dbgs() << formatv("{0}/{1} ({3:P}) spills, {2}/{1} ({4:P}) variables",
320 SpillSize, StackSize, StackSize - SpillSize, SpillPct,
321 1.0f - SpillPct));
322 if (UnsafeStackSize != 0) {
323 LLVM_DEBUG(dbgs() << formatv(", {0}/{2} ({1:P}) unsafe stack",
324 UnsafeStackSize,
325 static_cast<float>(UnsafeStackSize) /
326 static_cast<float>(StackSize),
327 StackSize));
328 }
329 LLVM_DEBUG(dbgs() << "\n");
330 }
331
332 ORE->emit(RemarkBuilder: [&]() {
333 return MachineOptimizationRemarkAnalysis(DEBUG_TYPE, "StackSize",
334 MF.getFunction().getSubprogram(),
335 &MF.front())
336 << ore::NV("NumStackBytes", StackSize)
337 << " stack bytes in function '"
338 << ore::NV("Function", MF.getFunction().getName()) << "'";
339 });
340
341 // Emit any remarks implemented for the target, based on final frame layout.
342 TFI->emitRemarks(MF, ORE);
343
344 delete RS;
345 SaveBlocks.clear();
346 RestoreBlocks.clear();
347 MFI.clearSavePoints();
348 MFI.clearRestorePoints();
349 return true;
350}
351
352/// runOnMachineFunction - Insert prolog/epilog code and replace abstract
353/// frame indexes with appropriate references.
354bool PEILegacy::runOnMachineFunction(MachineFunction &MF) {
355 MachineOptimizationRemarkEmitter *ORE =
356 &getAnalysis<MachineOptimizationRemarkEmitterPass>().getORE();
357 return PEIImpl(ORE).run(MF);
358}
359
360PreservedAnalyses
361PrologEpilogInserterPass::run(MachineFunction &MF,
362 MachineFunctionAnalysisManager &MFAM) {
363 MachineOptimizationRemarkEmitter &ORE =
364 MFAM.getResult<MachineOptimizationRemarkEmitterAnalysis>(IR&: MF);
365 if (!PEIImpl(&ORE).run(MF))
366 return PreservedAnalyses::all();
367
368 return getMachineFunctionPassPreservedAnalyses()
369 .preserveSet<CFGAnalyses>()
370 .preserve<MachineDominatorTreeAnalysis>()
371 .preserve<MachineLoopAnalysis>();
372}
373
374/// Calculate the MaxCallFrameSize variable for the function's frame
375/// information and eliminate call frame pseudo instructions.
376void PEIImpl::calculateCallFrameInfo(MachineFunction &MF) {
377 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
378 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
379 MachineFrameInfo &MFI = MF.getFrameInfo();
380
381 // Get the function call frame set-up and tear-down instruction opcode
382 unsigned FrameSetupOpcode = TII.getCallFrameSetupOpcode();
383 unsigned FrameDestroyOpcode = TII.getCallFrameDestroyOpcode();
384
385 // Early exit for targets which have no call frame setup/destroy pseudo
386 // instructions.
387 if (FrameSetupOpcode == ~0u && FrameDestroyOpcode == ~0u)
388 return;
389
390 // (Re-)Compute the MaxCallFrameSize.
391 [[maybe_unused]] uint64_t MaxCFSIn =
392 MFI.isMaxCallFrameSizeComputed() ? MFI.getMaxCallFrameSize() : UINT64_MAX;
393 std::vector<MachineBasicBlock::iterator> FrameSDOps;
394 MFI.computeMaxCallFrameSize(MF, FrameSDOps: &FrameSDOps);
395 assert(MFI.getMaxCallFrameSize() <= MaxCFSIn &&
396 "Recomputing MaxCFS gave a larger value.");
397 assert((FrameSDOps.empty() || MF.getFrameInfo().adjustsStack()) &&
398 "AdjustsStack not set in presence of a frame pseudo instruction.");
399
400 if (TFI->canSimplifyCallFramePseudos(MF)) {
401 // If call frames are not being included as part of the stack frame, and
402 // the target doesn't indicate otherwise, remove the call frame pseudos
403 // here. The sub/add sp instruction pairs are still inserted, but we don't
404 // need to track the SP adjustment for frame index elimination.
405 for (MachineBasicBlock::iterator I : FrameSDOps)
406 TFI->eliminateCallFramePseudoInstr(MF, MBB&: *I->getParent(), MI: I);
407
408 // We can't track the call frame size after call frame pseudos have been
409 // eliminated. Set it to zero everywhere to keep MachineVerifier happy.
410 for (MachineBasicBlock &MBB : MF)
411 MBB.setCallFrameSize(0);
412 }
413}
414
415/// Compute the sets of entry and return blocks for saving and restoring
416/// callee-saved registers, and placing prolog and epilog code.
417void PEIImpl::calculateSaveRestoreBlocks(MachineFunction &MF) {
418 const MachineFrameInfo &MFI = MF.getFrameInfo();
419 // Even when we do not change any CSR, we still want to insert the
420 // prologue and epilogue of the function.
421 // So set the save points for those.
422
423 // Use the points found by shrink-wrapping, if any.
424 if (!MFI.getSavePoints().empty()) {
425 assert(MFI.getSavePoints().size() == 1 &&
426 "Multiple save points are not yet supported!");
427 const auto &SavePoint = *MFI.getSavePoints().begin();
428 SaveBlocks.push_back(Elt: SavePoint.first);
429 assert(MFI.getRestorePoints().size() == 1 &&
430 "Multiple restore points are not yet supported!");
431 const auto &RestorePoint = *MFI.getRestorePoints().begin();
432 MachineBasicBlock *RestoreBlock = RestorePoint.first;
433 // If RestoreBlock does not have any successor and is not a return block
434 // then the end point is unreachable and we do not need to insert any
435 // epilogue.
436 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
437 RestoreBlocks.push_back(Elt: RestoreBlock);
438 return;
439 }
440
441 // Save refs to entry and return blocks.
442 SaveBlocks.push_back(Elt: &MF.front());
443 for (MachineBasicBlock &MBB : MF) {
444 if (MBB.isEHFuncletEntry())
445 SaveBlocks.push_back(Elt: &MBB);
446 if (MBB.isReturnBlock())
447 RestoreBlocks.push_back(Elt: &MBB);
448 }
449}
450
451static void assignCalleeSavedSpillSlots(MachineFunction &F,
452 const BitVector &SavedRegs) {
453 if (SavedRegs.empty())
454 return;
455
456 const TargetRegisterInfo *RegInfo = F.getSubtarget().getRegisterInfo();
457 const MCPhysReg *CSRegs = F.getRegInfo().getCalleeSavedRegs();
458 BitVector CSMask(SavedRegs.size());
459
460 for (unsigned i = 0; CSRegs[i]; ++i)
461 CSMask.set(CSRegs[i]);
462
463 std::vector<CalleeSavedInfo> CSI;
464 for (unsigned i = 0; CSRegs[i]; ++i) {
465 unsigned Reg = CSRegs[i];
466 if (SavedRegs.test(Idx: Reg)) {
467 bool SavedSuper = false;
468 for (const MCPhysReg &SuperReg : RegInfo->superregs(Reg)) {
469 // Some backends set all aliases for some registers as saved, such as
470 // Mips's $fp, so they appear in SavedRegs but not CSRegs.
471 if (SavedRegs.test(Idx: SuperReg) && CSMask.test(Idx: SuperReg)) {
472 SavedSuper = true;
473 break;
474 }
475 }
476
477 if (!SavedSuper)
478 CSI.push_back(x: CalleeSavedInfo(Reg));
479 }
480 }
481
482 const TargetFrameLowering *TFI = F.getSubtarget().getFrameLowering();
483 MachineFrameInfo &MFI = F.getFrameInfo();
484 if (!TFI->assignCalleeSavedSpillSlots(MF&: F, TRI: RegInfo, CSI)) {
485 // If target doesn't implement this, use generic code.
486
487 if (CSI.empty())
488 return; // Early exit if no callee saved registers are modified!
489
490 unsigned NumFixedSpillSlots;
491 const TargetFrameLowering::SpillSlot *FixedSpillSlots =
492 TFI->getCalleeSavedSpillSlots(NumEntries&: NumFixedSpillSlots);
493
494 // Now that we know which registers need to be saved and restored, allocate
495 // stack slots for them.
496 for (auto &CS : CSI) {
497 // If the target has spilled this register to another register or already
498 // handled it , we don't need to allocate a stack slot.
499 if (CS.isSpilledToReg())
500 continue;
501
502 MCRegister Reg = CS.getReg();
503 const TargetRegisterClass *RC = RegInfo->getMinimalPhysRegClass(Reg);
504
505 int FrameIdx;
506 if (RegInfo->hasReservedSpillSlot(MF: F, Reg, FrameIdx)) {
507 CS.setFrameIdx(FrameIdx);
508 continue;
509 }
510
511 // Check to see if this physreg must be spilled to a particular stack slot
512 // on this target.
513 const TargetFrameLowering::SpillSlot *FixedSlot = FixedSpillSlots;
514 while (FixedSlot != FixedSpillSlots + NumFixedSpillSlots &&
515 FixedSlot->Reg != Reg)
516 ++FixedSlot;
517
518 unsigned Size = RegInfo->getSpillSize(RC: *RC);
519 if (FixedSlot == FixedSpillSlots + NumFixedSpillSlots) {
520 // Nope, just spill it anywhere convenient.
521 Align Alignment = RegInfo->getSpillAlign(RC: *RC);
522 // We may not be able to satisfy the desired alignment specification of
523 // the TargetRegisterClass if the stack alignment is smaller. Use the
524 // min.
525 Alignment = std::min(a: Alignment, b: TFI->getStackAlign());
526 FrameIdx = MFI.CreateStackObject(Size, Alignment, isSpillSlot: true, Alloca: nullptr,
527 ID: RegInfo->getSpillStackID(RC: *RC));
528 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
529 } else {
530 // Spill it to the stack where we must.
531 FrameIdx = MFI.CreateFixedSpillStackObject(Size, SPOffset: FixedSlot->Offset);
532 }
533
534 CS.setFrameIdx(FrameIdx);
535 }
536 }
537
538 MFI.setCalleeSavedInfo(CSI);
539}
540
541/// Helper function to update the liveness information for the callee-saved
542/// registers.
543static void updateLiveness(MachineFunction &MF) {
544 MachineFrameInfo &MFI = MF.getFrameInfo();
545 // Visited will contain all the basic blocks that are in the region
546 // where the callee saved registers are alive:
547 // - Anything that is not Save or Restore -> LiveThrough.
548 // - Save -> LiveIn.
549 // - Restore -> LiveOut.
550 // The live-out is not attached to the block, so no need to keep
551 // Restore in this set.
552 SmallPtrSet<MachineBasicBlock *, 8> Visited;
553 SmallVector<MachineBasicBlock *, 8> WorkList;
554 MachineBasicBlock *Entry = &MF.front();
555
556 assert(MFI.getSavePoints().size() < 2 &&
557 "Multiple save points not yet supported!");
558 MachineBasicBlock *Save = MFI.getSavePoints().empty()
559 ? nullptr
560 : (*MFI.getSavePoints().begin()).first;
561
562 if (!Save)
563 Save = Entry;
564
565 if (Entry != Save) {
566 WorkList.push_back(Elt: Entry);
567 Visited.insert(Ptr: Entry);
568 }
569 Visited.insert(Ptr: Save);
570
571 assert(MFI.getRestorePoints().size() < 2 &&
572 "Multiple restore points not yet supported!");
573 MachineBasicBlock *Restore = MFI.getRestorePoints().empty()
574 ? nullptr
575 : (*MFI.getRestorePoints().begin()).first;
576 if (Restore)
577 // By construction Restore cannot be visited, otherwise it
578 // means there exists a path to Restore that does not go
579 // through Save.
580 WorkList.push_back(Elt: Restore);
581
582 while (!WorkList.empty()) {
583 const MachineBasicBlock *CurBB = WorkList.pop_back_val();
584 // By construction, the region that is after the save point is
585 // dominated by the Save and post-dominated by the Restore.
586 if (CurBB == Save && Save != Restore)
587 continue;
588 // Enqueue all the successors not already visited.
589 // Those are by construction either before Save or after Restore.
590 for (MachineBasicBlock *SuccBB : CurBB->successors())
591 if (Visited.insert(Ptr: SuccBB).second)
592 WorkList.push_back(Elt: SuccBB);
593 }
594
595 const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
596
597 MachineRegisterInfo &MRI = MF.getRegInfo();
598 for (const CalleeSavedInfo &I : CSI) {
599 for (MachineBasicBlock *MBB : Visited) {
600 MCRegister Reg = I.getReg();
601 // Add the callee-saved register as live-in.
602 // It's killed at the spill.
603 if (!MRI.isReserved(PhysReg: Reg) && !MBB->isLiveIn(Reg))
604 MBB->addLiveIn(PhysReg: Reg);
605 }
606 // If callee-saved register is spilled to another register rather than
607 // spilling to stack, the destination register has to be marked as live for
608 // each MBB between the prologue and epilogue so that it is not clobbered
609 // before it is reloaded in the epilogue. The Visited set contains all
610 // blocks outside of the region delimited by prologue/epilogue.
611 if (I.isSpilledToReg()) {
612 for (MachineBasicBlock &MBB : MF) {
613 if (Visited.count(Ptr: &MBB))
614 continue;
615 MCRegister DstReg = I.getDstReg();
616 if (!MBB.isLiveIn(Reg: DstReg))
617 MBB.addLiveIn(PhysReg: DstReg);
618 }
619 }
620 }
621}
622
623/// Insert spill code for the callee-saved registers used in the function.
624static void insertCSRSaves(MachineBasicBlock &SaveBlock,
625 ArrayRef<CalleeSavedInfo> CSI) {
626 MachineFunction &MF = *SaveBlock.getParent();
627 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
628 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
629 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
630
631 MachineBasicBlock::iterator I = SaveBlock.begin();
632 if (!TFI->spillCalleeSavedRegisters(MBB&: SaveBlock, MI: I, CSI, TRI)) {
633 for (const CalleeSavedInfo &CS : CSI) {
634 TFI->spillCalleeSavedRegister(SaveBlock, MI: I, CS, TII, TRI);
635 }
636 }
637}
638
639/// Insert restore code for the callee-saved registers used in the function.
640static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
641 std::vector<CalleeSavedInfo> &CSI) {
642 MachineFunction &MF = *RestoreBlock.getParent();
643 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
644 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
645 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
646
647 // Restore all registers immediately before the return and any
648 // terminators that precede it.
649 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();
650
651 if (!TFI->restoreCalleeSavedRegisters(MBB&: RestoreBlock, MI: I, CSI, TRI)) {
652 for (const CalleeSavedInfo &CI : reverse(C&: CSI)) {
653 TFI->restoreCalleeSavedRegister(MBB&: RestoreBlock, MI: I, CS: CI, TII, TRI);
654 }
655 }
656}
657
658void PEIImpl::spillCalleeSavedRegs(MachineFunction &MF) {
659 // We can't list this requirement in getRequiredProperties because some
660 // targets (WebAssembly) use virtual registers past this point, and the pass
661 // pipeline is set up without giving the passes a chance to look at the
662 // TargetMachine.
663 // FIXME: Find a way to express this in getRequiredProperties.
664 assert(MF.getProperties().hasNoVRegs());
665
666 const Function &F = MF.getFunction();
667 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
668 MachineFrameInfo &MFI = MF.getFrameInfo();
669
670 // Determine which of the registers in the callee save list should be saved.
671 BitVector SavedRegs;
672 TFI->determineCalleeSaves(MF, SavedRegs, RS);
673
674 // Assign stack slots for any callee-saved registers that must be spilled.
675 assignCalleeSavedSpillSlots(F&: MF, SavedRegs);
676
677 // Add the code to save and restore the callee saved registers.
678 if (!F.hasFnAttribute(Kind: Attribute::Naked)) {
679 MFI.setCalleeSavedInfoValid(true);
680
681 std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo();
682
683 // Fill SavePoints and RestorePoints with CalleeSavedRegisters
684 if (!MFI.getSavePoints().empty()) {
685 SaveRestorePoints SaveRestorePts;
686 for (const auto &SavePoint : MFI.getSavePoints())
687 SaveRestorePts.insert(KV: {SavePoint.first, CSI});
688 MFI.setSavePoints(std::move(SaveRestorePts));
689
690 SaveRestorePts.clear();
691 for (const auto &RestorePoint : MFI.getRestorePoints())
692 SaveRestorePts.insert(KV: {RestorePoint.first, CSI});
693 MFI.setRestorePoints(std::move(SaveRestorePts));
694 }
695
696 if (!CSI.empty()) {
697 if (!MFI.hasCalls())
698 NumLeafFuncWithSpills++;
699
700 for (MachineBasicBlock *SaveBlock : SaveBlocks)
701 insertCSRSaves(SaveBlock&: *SaveBlock, CSI);
702
703 // Update the live-in information of all the blocks up to the save point.
704 updateLiveness(MF);
705
706 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
707 insertCSRRestores(RestoreBlock&: *RestoreBlock, CSI);
708 }
709 }
710}
711
712/// AdjustStackOffset - Helper function used to adjust the stack frame offset.
713static inline void AdjustStackOffset(MachineFrameInfo &MFI, int FrameIdx,
714 bool StackGrowsDown, int64_t &Offset,
715 Align &MaxAlign) {
716 // If the stack grows down, add the object size to find the lowest address.
717 if (StackGrowsDown)
718 Offset += MFI.getObjectSize(ObjectIdx: FrameIdx);
719
720 Align Alignment = MFI.getObjectAlign(ObjectIdx: FrameIdx);
721
722 // If the alignment of this object is greater than that of the stack, then
723 // increase the stack alignment to match.
724 MaxAlign = std::max(a: MaxAlign, b: Alignment);
725
726 // Adjust to alignment boundary.
727 Offset = alignTo(Size: Offset, A: Alignment);
728
729 if (StackGrowsDown) {
730 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << -Offset
731 << "]\n");
732 MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: -Offset); // Set the computed offset
733 } else {
734 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") at SP[" << Offset
735 << "]\n");
736 MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: Offset);
737 Offset += MFI.getObjectSize(ObjectIdx: FrameIdx);
738 }
739}
740
741/// Compute which bytes of fixed and callee-save stack area are unused and keep
742/// track of them in StackBytesFree.
743static inline void computeFreeStackSlots(MachineFrameInfo &MFI,
744 bool StackGrowsDown,
745 int64_t FixedCSEnd,
746 BitVector &StackBytesFree) {
747 // Avoid undefined int64_t -> int conversion below in extreme case.
748 if (FixedCSEnd > std::numeric_limits<int>::max())
749 return;
750
751 StackBytesFree.resize(N: FixedCSEnd, t: true);
752
753 SmallVector<int, 16> AllocatedFrameSlots;
754 // Add fixed objects.
755 for (int i = MFI.getObjectIndexBegin(); i != 0; ++i)
756 // StackSlot scavenging is only implemented for the default stack.
757 if (MFI.getStackID(ObjectIdx: i) == TargetStackID::Default)
758 AllocatedFrameSlots.push_back(Elt: i);
759 // Add callee-save objects if there are any.
760 for (int i = MFI.getObjectIndexBegin(); i < MFI.getObjectIndexEnd(); i++)
761 if (MFI.isCalleeSavedObjectIndex(ObjectIdx: i) &&
762 MFI.getStackID(ObjectIdx: i) == TargetStackID::Default)
763 AllocatedFrameSlots.push_back(Elt: i);
764
765 for (int i : AllocatedFrameSlots) {
766 // These are converted from int64_t, but they should always fit in int
767 // because of the FixedCSEnd check above.
768 int ObjOffset = MFI.getObjectOffset(ObjectIdx: i);
769 int ObjSize = MFI.getObjectSize(ObjectIdx: i);
770 int ObjStart, ObjEnd;
771 if (StackGrowsDown) {
772 // ObjOffset is negative when StackGrowsDown is true.
773 ObjStart = -ObjOffset - ObjSize;
774 ObjEnd = -ObjOffset;
775 } else {
776 ObjStart = ObjOffset;
777 ObjEnd = ObjOffset + ObjSize;
778 }
779 // Ignore fixed holes that are in the previous stack frame.
780 if (ObjEnd > 0)
781 StackBytesFree.reset(I: ObjStart, E: ObjEnd);
782 }
783}
784
785/// Assign frame object to an unused portion of the stack in the fixed stack
786/// object range. Return true if the allocation was successful.
787static inline bool scavengeStackSlot(MachineFrameInfo &MFI, int FrameIdx,
788 bool StackGrowsDown, Align MaxAlign,
789 BitVector &StackBytesFree) {
790 if (MFI.isVariableSizedObjectIndex(ObjectIdx: FrameIdx))
791 return false;
792
793 if (StackBytesFree.none()) {
794 // clear it to speed up later scavengeStackSlot calls to
795 // StackBytesFree.none()
796 StackBytesFree.clear();
797 return false;
798 }
799
800 Align ObjAlign = MFI.getObjectAlign(ObjectIdx: FrameIdx);
801 if (ObjAlign > MaxAlign)
802 return false;
803
804 int64_t ObjSize = MFI.getObjectSize(ObjectIdx: FrameIdx);
805 int FreeStart;
806 for (FreeStart = StackBytesFree.find_first(); FreeStart != -1;
807 FreeStart = StackBytesFree.find_next(Prev: FreeStart)) {
808
809 // Check that free space has suitable alignment.
810 unsigned ObjStart = StackGrowsDown ? FreeStart + ObjSize : FreeStart;
811 if (alignTo(Size: ObjStart, A: ObjAlign) != ObjStart)
812 continue;
813
814 if (FreeStart + ObjSize > StackBytesFree.size())
815 return false;
816
817 bool AllBytesFree = true;
818 for (unsigned Byte = 0; Byte < ObjSize; ++Byte)
819 if (!StackBytesFree.test(Idx: FreeStart + Byte)) {
820 AllBytesFree = false;
821 break;
822 }
823 if (AllBytesFree)
824 break;
825 }
826
827 if (FreeStart == -1)
828 return false;
829
830 if (StackGrowsDown) {
831 int ObjStart = -(FreeStart + ObjSize);
832 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP["
833 << ObjStart << "]\n");
834 MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: ObjStart);
835 } else {
836 LLVM_DEBUG(dbgs() << "alloc FI(" << FrameIdx << ") scavenged at SP["
837 << FreeStart << "]\n");
838 MFI.setObjectOffset(ObjectIdx: FrameIdx, SPOffset: FreeStart);
839 }
840
841 StackBytesFree.reset(I: FreeStart, E: FreeStart + ObjSize);
842 return true;
843}
844
845/// AssignProtectedObjSet - Helper function to assign large stack objects (i.e.,
846/// those required to be close to the Stack Protector) to stack offsets.
847static void AssignProtectedObjSet(const StackObjSet &UnassignedObjs,
848 SmallSet<int, 16> &ProtectedObjs,
849 MachineFrameInfo &MFI, bool StackGrowsDown,
850 int64_t &Offset, Align &MaxAlign) {
851
852 for (int i : UnassignedObjs) {
853 AdjustStackOffset(MFI, FrameIdx: i, StackGrowsDown, Offset, MaxAlign);
854 ProtectedObjs.insert(V: i);
855 }
856}
857
858/// calculateFrameObjectOffsets - Calculate actual frame offsets for all of the
859/// abstract stack objects.
860void PEIImpl::calculateFrameObjectOffsets(MachineFunction &MF) {
861 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
862
863 bool StackGrowsDown =
864 TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
865
866 // Loop over all of the stack objects, assigning sequential addresses...
867 MachineFrameInfo &MFI = MF.getFrameInfo();
868
869 // Start at the beginning of the local area.
870 // The Offset is the distance from the stack top in the direction
871 // of stack growth -- so it's always nonnegative.
872 int LocalAreaOffset = TFI.getOffsetOfLocalArea();
873 if (StackGrowsDown)
874 LocalAreaOffset = -LocalAreaOffset;
875 assert(LocalAreaOffset >= 0
876 && "Local area offset should be in direction of stack growth");
877 int64_t Offset = LocalAreaOffset;
878
879#ifdef EXPENSIVE_CHECKS
880 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i)
881 if (!MFI.isDeadObjectIndex(i) &&
882 MFI.getStackID(i) == TargetStackID::Default)
883 assert(MFI.getObjectAlign(i) <= MFI.getMaxAlign() &&
884 "MaxAlignment is invalid");
885#endif
886
887 // If there are fixed sized objects that are preallocated in the local area,
888 // non-fixed objects can't be allocated right at the start of local area.
889 // Adjust 'Offset' to point to the end of last fixed sized preallocated
890 // object.
891 for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) {
892 // Only allocate objects on the default stack.
893 if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default)
894 continue;
895
896 int64_t FixedOff;
897 if (StackGrowsDown) {
898 // The maximum distance from the stack pointer is at lower address of
899 // the object -- which is given by offset. For down growing stack
900 // the offset is negative, so we negate the offset to get the distance.
901 FixedOff = -MFI.getObjectOffset(ObjectIdx: i);
902 } else {
903 // The maximum distance from the start pointer is at the upper
904 // address of the object.
905 FixedOff = MFI.getObjectOffset(ObjectIdx: i) + MFI.getObjectSize(ObjectIdx: i);
906 }
907 if (FixedOff > Offset) Offset = FixedOff;
908 }
909
910 Align MaxAlign = MFI.getMaxAlign();
911 // First assign frame offsets to stack objects that are used to spill
912 // callee saved registers.
913 auto AllFIs = seq(Begin: MFI.getObjectIndexBegin(), End: MFI.getObjectIndexEnd());
914 for (int FI : reverse_conditionally(C&: AllFIs, /*Reverse=*/ShouldReverse: !StackGrowsDown)) {
915 // Only allocate objects on the default stack.
916 if (!MFI.isCalleeSavedObjectIndex(ObjectIdx: FI) ||
917 MFI.getStackID(ObjectIdx: FI) != TargetStackID::Default)
918 continue;
919
920 // TODO: should this just be if (MFI.isDeadObjectIndex(FI))
921 if (!StackGrowsDown && MFI.isDeadObjectIndex(ObjectIdx: FI))
922 continue;
923
924 AdjustStackOffset(MFI, FrameIdx: FI, StackGrowsDown, Offset, MaxAlign);
925 }
926
927 assert(MaxAlign == MFI.getMaxAlign() &&
928 "MFI.getMaxAlign should already account for all callee-saved "
929 "registers without a fixed stack slot");
930
931 // FixedCSEnd is the stack offset to the end of the fixed and callee-save
932 // stack area.
933 int64_t FixedCSEnd = Offset;
934
935 // Make sure the special register scavenging spill slot is closest to the
936 // incoming stack pointer if a frame pointer is required and is closer
937 // to the incoming rather than the final stack pointer.
938 const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo();
939 bool EarlyScavengingSlots = TFI.allocateScavengingFrameIndexesNearIncomingSP(MF);
940 if (RS && EarlyScavengingSlots) {
941 SmallVector<int, 2> SFIs;
942 RS->getScavengingFrameIndices(A&: SFIs);
943 for (int SFI : SFIs)
944 AdjustStackOffset(MFI, FrameIdx: SFI, StackGrowsDown, Offset, MaxAlign);
945 }
946
947 // FIXME: Once this is working, then enable flag will change to a target
948 // check for whether the frame is large enough to want to use virtual
949 // frame index registers. Functions which don't want/need this optimization
950 // will continue to use the existing code path.
951 if (MFI.getUseLocalStackAllocationBlock()) {
952 Align Alignment = MFI.getLocalFrameMaxAlign();
953
954 // Adjust to alignment boundary.
955 Offset = alignTo(Size: Offset, A: Alignment);
956
957 LLVM_DEBUG(dbgs() << "Local frame base offset: " << Offset << "\n");
958
959 // Resolve offsets for objects in the local block.
960 for (unsigned i = 0, e = MFI.getLocalFrameObjectCount(); i != e; ++i) {
961 std::pair<int, int64_t> Entry = MFI.getLocalFrameObjectMap(i);
962 int64_t FIOffset = (StackGrowsDown ? -Offset : Offset) + Entry.second;
963 LLVM_DEBUG(dbgs() << "alloc FI(" << Entry.first << ") at SP[" << FIOffset
964 << "]\n");
965 MFI.setObjectOffset(ObjectIdx: Entry.first, SPOffset: FIOffset);
966 }
967 // Allocate the local block
968 Offset += MFI.getLocalFrameSize();
969
970 MaxAlign = std::max(a: Alignment, b: MaxAlign);
971 }
972
973 // Retrieve the Exception Handler registration node.
974 int EHRegNodeFrameIndex = std::numeric_limits<int>::max();
975 if (const WinEHFuncInfo *FuncInfo = MF.getWinEHFuncInfo())
976 EHRegNodeFrameIndex = FuncInfo->EHRegNodeFrameIndex;
977
978 // Make sure that the stack protector comes before the local variables on the
979 // stack.
980 SmallSet<int, 16> ProtectedObjs;
981 if (MFI.hasStackProtectorIndex()) {
982 int StackProtectorFI = MFI.getStackProtectorIndex();
983 StackObjSet LargeArrayObjs;
984 StackObjSet SmallArrayObjs;
985 StackObjSet AddrOfObjs;
986
987 // If we need a stack protector, we need to make sure that
988 // LocalStackSlotPass didn't already allocate a slot for it.
989 // If we are told to use the LocalStackAllocationBlock, the stack protector
990 // is expected to be already pre-allocated.
991 if (MFI.getStackID(ObjectIdx: StackProtectorFI) != TargetStackID::Default) {
992 // If the stack protector isn't on the default stack then it's up to the
993 // target to set the stack offset.
994 assert(MFI.getObjectOffset(StackProtectorFI) != 0 &&
995 "Offset of stack protector on non-default stack expected to be "
996 "already set.");
997 assert(!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex()) &&
998 "Stack protector on non-default stack expected to not be "
999 "pre-allocated by LocalStackSlotPass.");
1000 } else if (!MFI.getUseLocalStackAllocationBlock()) {
1001 AdjustStackOffset(MFI, FrameIdx: StackProtectorFI, StackGrowsDown, Offset,
1002 MaxAlign);
1003 } else if (!MFI.isObjectPreAllocated(ObjectIdx: MFI.getStackProtectorIndex())) {
1004 llvm_unreachable(
1005 "Stack protector not pre-allocated by LocalStackSlotPass.");
1006 }
1007
1008 // Assign large stack objects first.
1009 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
1010 if (MFI.isObjectPreAllocated(ObjectIdx: i) && MFI.getUseLocalStackAllocationBlock())
1011 continue;
1012 if (MFI.isCalleeSavedObjectIndex(ObjectIdx: i))
1013 continue;
1014 if (RS && RS->isScavengingFrameIndex(FI: (int)i))
1015 continue;
1016 if (MFI.isDeadObjectIndex(ObjectIdx: i))
1017 continue;
1018 if (StackProtectorFI == (int)i || EHRegNodeFrameIndex == (int)i)
1019 continue;
1020 // Only allocate objects on the default stack.
1021 if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default)
1022 continue;
1023
1024 switch (MFI.getObjectSSPLayout(ObjectIdx: i)) {
1025 case MachineFrameInfo::SSPLK_None:
1026 continue;
1027 case MachineFrameInfo::SSPLK_SmallArray:
1028 SmallArrayObjs.insert(X: i);
1029 continue;
1030 case MachineFrameInfo::SSPLK_AddrOf:
1031 AddrOfObjs.insert(X: i);
1032 continue;
1033 case MachineFrameInfo::SSPLK_LargeArray:
1034 LargeArrayObjs.insert(X: i);
1035 continue;
1036 }
1037 llvm_unreachable("Unexpected SSPLayoutKind.");
1038 }
1039
1040 // We expect **all** the protected stack objects to be pre-allocated by
1041 // LocalStackSlotPass. If it turns out that PEI still has to allocate some
1042 // of them, we may end up messing up the expected order of the objects.
1043 if (MFI.getUseLocalStackAllocationBlock() &&
1044 !(LargeArrayObjs.empty() && SmallArrayObjs.empty() &&
1045 AddrOfObjs.empty()))
1046 llvm_unreachable("Found protected stack objects not pre-allocated by "
1047 "LocalStackSlotPass.");
1048
1049 AssignProtectedObjSet(UnassignedObjs: LargeArrayObjs, ProtectedObjs, MFI, StackGrowsDown,
1050 Offset, MaxAlign);
1051 AssignProtectedObjSet(UnassignedObjs: SmallArrayObjs, ProtectedObjs, MFI, StackGrowsDown,
1052 Offset, MaxAlign);
1053 AssignProtectedObjSet(UnassignedObjs: AddrOfObjs, ProtectedObjs, MFI, StackGrowsDown,
1054 Offset, MaxAlign);
1055 }
1056
1057 SmallVector<int, 8> ObjectsToAllocate;
1058
1059 // Then prepare to assign frame offsets to stack objects that are not used to
1060 // spill callee saved registers.
1061 for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) {
1062 if (MFI.isObjectPreAllocated(ObjectIdx: i) && MFI.getUseLocalStackAllocationBlock())
1063 continue;
1064 if (MFI.isCalleeSavedObjectIndex(ObjectIdx: i))
1065 continue;
1066 if (RS && RS->isScavengingFrameIndex(FI: (int)i))
1067 continue;
1068 if (MFI.isDeadObjectIndex(ObjectIdx: i))
1069 continue;
1070 if (MFI.getStackProtectorIndex() == (int)i || EHRegNodeFrameIndex == (int)i)
1071 continue;
1072 if (ProtectedObjs.count(V: i))
1073 continue;
1074 // Only allocate objects on the default stack.
1075 if (MFI.getStackID(ObjectIdx: i) != TargetStackID::Default)
1076 continue;
1077
1078 // Add the objects that we need to allocate to our working set.
1079 ObjectsToAllocate.push_back(Elt: i);
1080 }
1081
1082 // Allocate the EH registration node first if one is present.
1083 if (EHRegNodeFrameIndex != std::numeric_limits<int>::max())
1084 AdjustStackOffset(MFI, FrameIdx: EHRegNodeFrameIndex, StackGrowsDown, Offset,
1085 MaxAlign);
1086
1087 // Give the targets a chance to order the objects the way they like it.
1088 if (MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
1089 MF.getTarget().Options.StackSymbolOrdering)
1090 TFI.orderFrameObjects(MF, objectsToAllocate&: ObjectsToAllocate);
1091
1092 // Keep track of which bytes in the fixed and callee-save range are used so we
1093 // can use the holes when allocating later stack objects. Only do this if
1094 // stack protector isn't being used and the target requests it and we're
1095 // optimizing.
1096 BitVector StackBytesFree;
1097 if (!ObjectsToAllocate.empty() &&
1098 MF.getTarget().getOptLevel() != CodeGenOptLevel::None &&
1099 MFI.getStackProtectorIndex() < 0 && TFI.enableStackSlotScavenging(MF))
1100 computeFreeStackSlots(MFI, StackGrowsDown, FixedCSEnd, StackBytesFree);
1101
1102 // Now walk the objects and actually assign base offsets to them.
1103 for (auto &Object : ObjectsToAllocate)
1104 if (!scavengeStackSlot(MFI, FrameIdx: Object, StackGrowsDown, MaxAlign,
1105 StackBytesFree))
1106 AdjustStackOffset(MFI, FrameIdx: Object, StackGrowsDown, Offset, MaxAlign);
1107
1108 // Make sure the special register scavenging spill slot is closest to the
1109 // stack pointer.
1110 if (RS && !EarlyScavengingSlots) {
1111 SmallVector<int, 2> SFIs;
1112 RS->getScavengingFrameIndices(A&: SFIs);
1113 for (int SFI : SFIs)
1114 AdjustStackOffset(MFI, FrameIdx: SFI, StackGrowsDown, Offset, MaxAlign);
1115 }
1116
1117 if (!TFI.targetHandlesStackFrameRounding()) {
1118 // If we have reserved argument space for call sites in the function
1119 // immediately on entry to the current function, count it as part of the
1120 // overall stack size.
1121 if (MFI.adjustsStack() && TFI.hasReservedCallFrame(MF))
1122 Offset += MFI.getMaxCallFrameSize();
1123
1124 // Round up the size to a multiple of the alignment. If the function has
1125 // any calls or alloca's, align to the target's StackAlignment value to
1126 // ensure that the callee's frame or the alloca data is suitably aligned;
1127 // otherwise, for leaf functions, align to the TransientStackAlignment
1128 // value.
1129 Align StackAlign;
1130 if (MFI.adjustsStack() || MFI.hasVarSizedObjects() ||
1131 (RegInfo->hasStackRealignment(MF) && MFI.getObjectIndexEnd() != 0))
1132 StackAlign = TFI.getStackAlign();
1133 else
1134 StackAlign = TFI.getTransientStackAlign();
1135
1136 // If the frame pointer is eliminated, all frame offsets will be relative to
1137 // SP not FP. Align to MaxAlign so this works.
1138 StackAlign = std::max(a: StackAlign, b: MaxAlign);
1139 int64_t OffsetBeforeAlignment = Offset;
1140 Offset = alignTo(Size: Offset, A: StackAlign);
1141
1142 // If we have increased the offset to fulfill the alignment constrants,
1143 // then the scavenging spill slots may become harder to reach from the
1144 // stack pointer, float them so they stay close.
1145 if (StackGrowsDown && OffsetBeforeAlignment != Offset && RS &&
1146 !EarlyScavengingSlots) {
1147 SmallVector<int, 2> SFIs;
1148 RS->getScavengingFrameIndices(A&: SFIs);
1149 LLVM_DEBUG(if (!SFIs.empty()) llvm::dbgs()
1150 << "Adjusting emergency spill slots!\n";);
1151 int64_t Delta = Offset - OffsetBeforeAlignment;
1152 for (int SFI : SFIs) {
1153 LLVM_DEBUG(llvm::dbgs()
1154 << "Adjusting offset of emergency spill slot #" << SFI
1155 << " from " << MFI.getObjectOffset(SFI););
1156 MFI.setObjectOffset(ObjectIdx: SFI, SPOffset: MFI.getObjectOffset(ObjectIdx: SFI) - Delta);
1157 LLVM_DEBUG(llvm::dbgs() << " to " << MFI.getObjectOffset(SFI) << "\n";);
1158 }
1159 }
1160 }
1161
1162 // Update frame info to pretend that this is part of the stack...
1163 int64_t StackSize = Offset - LocalAreaOffset;
1164 MFI.setStackSize(StackSize);
1165 NumBytesStackSpace += StackSize;
1166}
1167
1168/// insertPrologEpilogCode - Scan the function for modified callee saved
1169/// registers, insert spill code for these callee saved registers, then add
1170/// prolog and epilog code to the function.
1171void PEIImpl::insertPrologEpilogCode(MachineFunction &MF) {
1172 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1173
1174 // Add prologue to the function...
1175 for (MachineBasicBlock *SaveBlock : SaveBlocks)
1176 TFI.emitPrologue(MF, MBB&: *SaveBlock);
1177
1178 // Add epilogue to restore the callee-save registers in each exiting block.
1179 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
1180 TFI.emitEpilogue(MF, MBB&: *RestoreBlock);
1181
1182 // Zero call used registers before restoring callee-saved registers.
1183 insertZeroCallUsedRegs(MF);
1184
1185 for (MachineBasicBlock *SaveBlock : SaveBlocks)
1186 TFI.inlineStackProbe(MF, PrologueMBB&: *SaveBlock);
1187
1188 // Emit additional code that is required to support segmented stacks, if
1189 // we've been asked for it. This, when linked with a runtime with support
1190 // for segmented stacks (libgcc is one), will result in allocating stack
1191 // space in small chunks instead of one large contiguous block.
1192 if (MF.shouldSplitStack()) {
1193 for (MachineBasicBlock *SaveBlock : SaveBlocks)
1194 TFI.adjustForSegmentedStacks(MF, PrologueMBB&: *SaveBlock);
1195 }
1196
1197 // Emit additional code that is required to explicitly handle the stack in
1198 // HiPE native code (if needed) when loaded in the Erlang/OTP runtime. The
1199 // approach is rather similar to that of Segmented Stacks, but it uses a
1200 // different conditional check and another BIF for allocating more stack
1201 // space.
1202 if (MF.getFunction().getCallingConv() == CallingConv::HiPE)
1203 for (MachineBasicBlock *SaveBlock : SaveBlocks)
1204 TFI.adjustForHiPEPrologue(MF, PrologueMBB&: *SaveBlock);
1205}
1206
1207/// insertZeroCallUsedRegs - Zero out call used registers.
1208void PEIImpl::insertZeroCallUsedRegs(MachineFunction &MF) {
1209 const Function &F = MF.getFunction();
1210
1211 if (!F.hasFnAttribute(Kind: "zero-call-used-regs"))
1212 return;
1213
1214 using namespace ZeroCallUsedRegs;
1215
1216 ZeroCallUsedRegsKind ZeroRegsKind =
1217 StringSwitch<ZeroCallUsedRegsKind>(
1218 F.getFnAttribute(Kind: "zero-call-used-regs").getValueAsString())
1219 .Case(S: "skip", Value: ZeroCallUsedRegsKind::Skip)
1220 .Case(S: "used-gpr-arg", Value: ZeroCallUsedRegsKind::UsedGPRArg)
1221 .Case(S: "used-gpr", Value: ZeroCallUsedRegsKind::UsedGPR)
1222 .Case(S: "used-arg", Value: ZeroCallUsedRegsKind::UsedArg)
1223 .Case(S: "used", Value: ZeroCallUsedRegsKind::Used)
1224 .Case(S: "all-gpr-arg", Value: ZeroCallUsedRegsKind::AllGPRArg)
1225 .Case(S: "all-gpr", Value: ZeroCallUsedRegsKind::AllGPR)
1226 .Case(S: "all-arg", Value: ZeroCallUsedRegsKind::AllArg)
1227 .Case(S: "all", Value: ZeroCallUsedRegsKind::All);
1228
1229 if (ZeroRegsKind == ZeroCallUsedRegsKind::Skip)
1230 return;
1231
1232 const bool OnlyGPR = static_cast<unsigned>(ZeroRegsKind) & ONLY_GPR;
1233 const bool OnlyUsed = static_cast<unsigned>(ZeroRegsKind) & ONLY_USED;
1234 const bool OnlyArg = static_cast<unsigned>(ZeroRegsKind) & ONLY_ARG;
1235
1236 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1237 const BitVector AllocatableSet(TRI.getAllocatableSet(MF));
1238
1239 // Mark all used registers.
1240 BitVector UsedRegs(TRI.getNumRegs());
1241 if (OnlyUsed)
1242 for (const MachineBasicBlock &MBB : MF)
1243 for (const MachineInstr &MI : MBB) {
1244 // skip debug instructions
1245 if (MI.isDebugInstr())
1246 continue;
1247
1248 for (const MachineOperand &MO : MI.operands()) {
1249 if (!MO.isReg())
1250 continue;
1251
1252 MCRegister Reg = MO.getReg();
1253 if (AllocatableSet[Reg.id()] && !MO.isImplicit() &&
1254 (MO.isDef() || MO.isUse()))
1255 UsedRegs.set(Reg.id());
1256 }
1257 }
1258
1259 // Get a list of registers that are used.
1260 BitVector LiveIns(TRI.getNumRegs());
1261 for (const MachineBasicBlock::RegisterMaskPair &LI : MF.front().liveins())
1262 LiveIns.set(LI.PhysReg);
1263
1264 BitVector RegsToZero(TRI.getNumRegs());
1265 for (MCRegister Reg : AllocatableSet.set_bits()) {
1266 // Skip over fixed registers.
1267 if (TRI.isFixedRegister(MF, PhysReg: Reg))
1268 continue;
1269
1270 // Want only general purpose registers.
1271 if (OnlyGPR && !TRI.isGeneralPurposeRegister(MF, PhysReg: Reg))
1272 continue;
1273
1274 // Want only used registers.
1275 if (OnlyUsed && !UsedRegs[Reg.id()])
1276 continue;
1277
1278 // Want only registers used for arguments.
1279 if (OnlyArg) {
1280 if (OnlyUsed) {
1281 if (!LiveIns[Reg.id()])
1282 continue;
1283 } else if (!TRI.isArgumentRegister(MF, PhysReg: Reg)) {
1284 continue;
1285 }
1286 }
1287
1288 RegsToZero.set(Reg.id());
1289 }
1290
1291 // Don't clear registers that are live when leaving the function.
1292 for (const MachineBasicBlock &MBB : MF)
1293 for (const MachineInstr &MI : MBB.terminators()) {
1294 if (!MI.isReturn())
1295 continue;
1296
1297 for (const auto &MO : MI.operands()) {
1298 if (!MO.isReg())
1299 continue;
1300
1301 MCRegister Reg = MO.getReg();
1302 if (!Reg)
1303 continue;
1304
1305 // This picks up sibling registers (e.q. %al -> %ah).
1306 // FIXME: Mixing physical registers and register units is likely a bug.
1307 for (MCRegUnit Unit : TRI.regunits(Reg))
1308 RegsToZero.reset(Idx: static_cast<unsigned>(Unit));
1309
1310 for (MCPhysReg SReg : TRI.sub_and_superregs_inclusive(Reg))
1311 RegsToZero.reset(Idx: SReg);
1312 }
1313 }
1314
1315 // Don't need to clear registers that are used/clobbered by terminating
1316 // instructions.
1317 for (const MachineBasicBlock &MBB : MF) {
1318 if (!MBB.isReturnBlock())
1319 continue;
1320
1321 MachineBasicBlock::const_iterator MBBI = MBB.getFirstTerminator();
1322 for (MachineBasicBlock::const_iterator I = MBBI, E = MBB.end(); I != E;
1323 ++I) {
1324 for (const MachineOperand &MO : I->operands()) {
1325 if (!MO.isReg())
1326 continue;
1327
1328 MCRegister Reg = MO.getReg();
1329 if (!Reg)
1330 continue;
1331
1332 for (const MCPhysReg Reg : TRI.sub_and_superregs_inclusive(Reg))
1333 RegsToZero.reset(Idx: Reg);
1334 }
1335 }
1336 }
1337
1338 // Don't clear registers that must be preserved.
1339 for (const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(MF: &MF);
1340 MCPhysReg CSReg = *CSRegs; ++CSRegs)
1341 for (MCRegister Reg : TRI.sub_and_superregs_inclusive(Reg: CSReg))
1342 RegsToZero.reset(Idx: Reg.id());
1343
1344 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1345 for (MachineBasicBlock &MBB : MF)
1346 if (MBB.isReturnBlock())
1347 TFI.emitZeroCallUsedRegs(RegsToZero, MBB);
1348}
1349
1350/// Replace all FrameIndex operands with physical register references and actual
1351/// offsets.
1352void PEIImpl::replaceFrameIndicesBackward(MachineFunction &MF) {
1353 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1354
1355 for (auto &MBB : MF) {
1356 int SPAdj = 0;
1357 if (!MBB.succ_empty()) {
1358 // Get the SP adjustment for the end of MBB from the start of any of its
1359 // successors. They should all be the same.
1360 assert(all_of(MBB.successors(), [&MBB](const MachineBasicBlock *Succ) {
1361 return Succ->getCallFrameSize() ==
1362 (*MBB.succ_begin())->getCallFrameSize();
1363 }));
1364 const MachineBasicBlock &FirstSucc = **MBB.succ_begin();
1365 SPAdj = TFI.alignSPAdjust(SPAdj: FirstSucc.getCallFrameSize());
1366 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
1367 SPAdj = -SPAdj;
1368 }
1369
1370 replaceFrameIndicesBackward(BB: &MBB, MF, SPAdj);
1371
1372 // We can't track the call frame size after call frame pseudos have been
1373 // eliminated. Set it to zero everywhere to keep MachineVerifier happy.
1374 MBB.setCallFrameSize(0);
1375 }
1376}
1377
1378/// replaceFrameIndices - Replace all MO_FrameIndex operands with physical
1379/// register references and actual offsets.
1380void PEIImpl::replaceFrameIndices(MachineFunction &MF) {
1381 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1382
1383 for (auto &MBB : MF) {
1384 int SPAdj = TFI.alignSPAdjust(SPAdj: MBB.getCallFrameSize());
1385 if (TFI.getStackGrowthDirection() == TargetFrameLowering::StackGrowsUp)
1386 SPAdj = -SPAdj;
1387
1388 replaceFrameIndices(BB: &MBB, MF, SPAdj);
1389
1390 // We can't track the call frame size after call frame pseudos have been
1391 // eliminated. Set it to zero everywhere to keep MachineVerifier happy.
1392 MBB.setCallFrameSize(0);
1393 }
1394}
1395
1396bool PEIImpl::replaceFrameIndexDebugInstr(MachineFunction &MF, MachineInstr &MI,
1397 unsigned OpIdx, int SPAdj) {
1398 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
1399 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1400 if (MI.isDebugValue()) {
1401
1402 MachineOperand &Op = MI.getOperand(i: OpIdx);
1403 assert(MI.isDebugOperand(&Op) &&
1404 "Frame indices can only appear as a debug operand in a DBG_VALUE*"
1405 " machine instruction");
1406 Register Reg;
1407 unsigned FrameIdx = Op.getIndex();
1408 unsigned Size = MF.getFrameInfo().getObjectSize(ObjectIdx: FrameIdx);
1409
1410 StackOffset Offset = TFI->getFrameIndexReference(MF, FI: FrameIdx, FrameReg&: Reg);
1411 Op.ChangeToRegister(Reg, isDef: false /*isDef*/);
1412
1413 const DIExpression *DIExpr = MI.getDebugExpression();
1414
1415 // If we have a direct DBG_VALUE, and its location expression isn't
1416 // currently complex, then adding an offset will morph it into a
1417 // complex location that is interpreted as being a memory address.
1418 // This changes a pointer-valued variable to dereference that pointer,
1419 // which is incorrect. Fix by adding DW_OP_stack_value.
1420
1421 if (MI.isNonListDebugValue()) {
1422 unsigned PrependFlags = DIExpression::ApplyOffset;
1423 if (!MI.isIndirectDebugValue() && !DIExpr->isComplex())
1424 PrependFlags |= DIExpression::StackValue;
1425
1426 // If we have DBG_VALUE that is indirect and has a Implicit location
1427 // expression need to insert a deref before prepending a Memory
1428 // location expression. Also after doing this we change the DBG_VALUE
1429 // to be direct.
1430 if (MI.isIndirectDebugValue() && DIExpr->isImplicit()) {
1431 SmallVector<uint64_t, 2> Ops = {dwarf::DW_OP_deref_size, Size};
1432 bool WithStackValue = true;
1433 DIExpr = DIExpression::prependOpcodes(Expr: DIExpr, Ops, StackValue: WithStackValue);
1434 // Make the DBG_VALUE direct.
1435 MI.getDebugOffset().ChangeToRegister(Reg: 0, isDef: false);
1436 }
1437 DIExpr = TRI.prependOffsetExpression(Expr: DIExpr, PrependFlags, Offset);
1438 } else {
1439 // The debug operand at DebugOpIndex was a frame index at offset
1440 // `Offset`; now the operand has been replaced with the frame
1441 // register, we must add Offset with `register x, plus Offset`.
1442 unsigned DebugOpIndex = MI.getDebugOperandIndex(Op: &Op);
1443 SmallVector<uint64_t, 3> Ops;
1444 TRI.getOffsetOpcodes(Offset, Ops);
1445 DIExpr = DIExpression::appendOpsToArg(Expr: DIExpr, Ops, ArgNo: DebugOpIndex);
1446 }
1447 MI.getDebugExpressionOp().setMetadata(DIExpr);
1448 return true;
1449 }
1450
1451 if (MI.isDebugPHI()) {
1452 // Allow stack ref to continue onwards.
1453 return true;
1454 }
1455
1456 // TODO: This code should be commoned with the code for
1457 // PATCHPOINT. There's no good reason for the difference in
1458 // implementation other than historical accident. The only
1459 // remaining difference is the unconditional use of the stack
1460 // pointer as the base register.
1461 if (MI.getOpcode() == TargetOpcode::STATEPOINT) {
1462 assert((!MI.isDebugValue() || OpIdx == 0) &&
1463 "Frame indices can only appear as the first operand of a "
1464 "DBG_VALUE machine instruction");
1465 Register Reg;
1466 MachineOperand &Offset = MI.getOperand(i: OpIdx + 1);
1467 StackOffset refOffset = TFI->getFrameIndexReferencePreferSP(
1468 MF, FI: MI.getOperand(i: OpIdx).getIndex(), FrameReg&: Reg, /*IgnoreSPUpdates*/ false);
1469 assert(!refOffset.getScalable() &&
1470 "Frame offsets with a scalable component are not supported");
1471 Offset.setImm(Offset.getImm() + refOffset.getFixed() + SPAdj);
1472 MI.getOperand(i: OpIdx).ChangeToRegister(Reg, isDef: false /*isDef*/);
1473 return true;
1474 }
1475 return false;
1476}
1477
1478void PEIImpl::replaceFrameIndicesBackward(MachineBasicBlock *BB,
1479 MachineFunction &MF, int &SPAdj) {
1480 assert(MF.getSubtarget().getRegisterInfo() &&
1481 "getRegisterInfo() must be implemented!");
1482
1483 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1484 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1485 const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering();
1486
1487 RegScavenger *LocalRS = FrameIndexEliminationScavenging ? RS : nullptr;
1488 if (LocalRS)
1489 LocalRS->enterBasicBlockEnd(MBB&: *BB);
1490
1491 for (MachineBasicBlock::iterator I = BB->end(); I != BB->begin();) {
1492 MachineInstr &MI = *std::prev(x: I);
1493
1494 if (TII.isFrameInstr(I: MI)) {
1495 SPAdj -= TII.getSPAdjust(MI);
1496 TFI.eliminateCallFramePseudoInstr(MF, MBB&: *BB, MI: &MI);
1497 continue;
1498 }
1499
1500 // Step backwards to get the liveness state at (immedately after) MI.
1501 if (LocalRS)
1502 LocalRS->backward(I);
1503
1504 bool RemovedMI = false;
1505 for (const auto &[Idx, Op] : enumerate(First: MI.operands())) {
1506 if (!Op.isFI())
1507 continue;
1508
1509 if (replaceFrameIndexDebugInstr(MF, MI, OpIdx: Idx, SPAdj))
1510 continue;
1511
1512 // Eliminate this FrameIndex operand.
1513 RemovedMI = TRI.eliminateFrameIndex(MI, SPAdj, FIOperandNum: Idx, RS: LocalRS);
1514 if (RemovedMI)
1515 break;
1516 }
1517
1518 if (!RemovedMI)
1519 --I;
1520 }
1521}
1522
1523void PEIImpl::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF,
1524 int &SPAdj) {
1525 assert(MF.getSubtarget().getRegisterInfo() &&
1526 "getRegisterInfo() must be implemented!");
1527 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1528 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1529 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
1530
1531 bool InsideCallSequence = false;
1532
1533 for (MachineBasicBlock::iterator I = BB->begin(); I != BB->end(); ) {
1534 if (TII.isFrameInstr(I: *I)) {
1535 InsideCallSequence = TII.isFrameSetup(I: *I);
1536 SPAdj += TII.getSPAdjust(MI: *I);
1537 I = TFI->eliminateCallFramePseudoInstr(MF, MBB&: *BB, MI: I);
1538 continue;
1539 }
1540
1541 MachineInstr &MI = *I;
1542 bool DoIncr = true;
1543 bool DidFinishLoop = true;
1544 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
1545 if (!MI.getOperand(i).isFI())
1546 continue;
1547
1548 if (replaceFrameIndexDebugInstr(MF, MI, OpIdx: i, SPAdj))
1549 continue;
1550
1551 // Some instructions (e.g. inline asm instructions) can have
1552 // multiple frame indices and/or cause eliminateFrameIndex
1553 // to insert more than one instruction. We need the register
1554 // scavenger to go through all of these instructions so that
1555 // it can update its register information. We keep the
1556 // iterator at the point before insertion so that we can
1557 // revisit them in full.
1558 bool AtBeginning = (I == BB->begin());
1559 if (!AtBeginning) --I;
1560
1561 // If this instruction has a FrameIndex operand, we need to
1562 // use that target machine register info object to eliminate
1563 // it.
1564 TRI.eliminateFrameIndex(MI, SPAdj, FIOperandNum: i, RS);
1565
1566 // Reset the iterator if we were at the beginning of the BB.
1567 if (AtBeginning) {
1568 I = BB->begin();
1569 DoIncr = false;
1570 }
1571
1572 DidFinishLoop = false;
1573 break;
1574 }
1575
1576 // If we are looking at a call sequence, we need to keep track of
1577 // the SP adjustment made by each instruction in the sequence.
1578 // This includes both the frame setup/destroy pseudos (handled above),
1579 // as well as other instructions that have side effects w.r.t the SP.
1580 // Note that this must come after eliminateFrameIndex, because
1581 // if I itself referred to a frame index, we shouldn't count its own
1582 // adjustment.
1583 if (DidFinishLoop && InsideCallSequence)
1584 SPAdj += TII.getSPAdjust(MI);
1585
1586 if (DoIncr && I != BB->end())
1587 ++I;
1588 }
1589}
1590