1//===-- FixupStatepointCallerSaved.cpp - Fixup caller saved registers ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Statepoint instruction in deopt parameters contains values which are
11/// meaningful to the runtime and should be able to be read at the moment the
12/// call returns. So we can say that we need to encode the fact that these
13/// values are "late read" by runtime. If we could express this notion for
14/// register allocator it would produce the right form for us.
15/// The need to fixup (i.e this pass) is specifically handling the fact that
16/// we cannot describe such a late read for the register allocator.
17/// Register allocator may put the value on a register clobbered by the call.
18/// This pass forces the spill of such registers and replaces corresponding
19/// statepoint operands to added spill slots.
20///
21//===----------------------------------------------------------------------===//
22
23#include "llvm/CodeGen/FixupStatepointCallerSaved.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/MachineFunctionPass.h"
28#include "llvm/CodeGen/StackMaps.h"
29#include "llvm/CodeGen/TargetInstrInfo.h"
30#include "llvm/IR/Statepoint.h"
31#include "llvm/InitializePasses.h"
32#include "llvm/Support/Debug.h"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "fixup-statepoint-caller-saved"
37STATISTIC(NumSpilledRegisters, "Number of spilled register");
38STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated");
39STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended");
40
41static cl::opt<bool> FixupSCSExtendSlotSize(
42 "fixup-scs-extend-slot-size", cl::Hidden, cl::init(Val: false),
43 cl::desc("Allow spill in spill slot of greater size than register size"),
44 cl::Hidden);
45
46static cl::opt<bool> PassGCPtrInCSR(
47 "fixup-allow-gcptr-in-csr", cl::Hidden, cl::init(Val: false),
48 cl::desc("Allow passing GC Pointer arguments in callee saved registers"));
49
50static cl::opt<bool> EnableCopyProp(
51 "fixup-scs-enable-copy-propagation", cl::Hidden, cl::init(Val: true),
52 cl::desc("Enable simple copy propagation during register reloading"));
53
54// This is purely debugging option.
55// It may be handy for investigating statepoint spilling issues.
56static cl::opt<unsigned> MaxStatepointsWithRegs(
57 "fixup-max-csr-statepoints", cl::Hidden,
58 cl::desc("Max number of statepoints allowed to pass GC Ptrs in registers"));
59
60namespace {
61
62struct FixupStatepointCallerSavedImpl {
63 bool run(MachineFunction &MF);
64};
65
66class FixupStatepointCallerSavedLegacy : public MachineFunctionPass {
67public:
68 static char ID;
69
70 FixupStatepointCallerSavedLegacy() : MachineFunctionPass(ID) {}
71 void getAnalysisUsage(AnalysisUsage &AU) const override {
72 AU.setPreservesCFG();
73 MachineFunctionPass::getAnalysisUsage(AU);
74 }
75
76 StringRef getPassName() const override {
77 return "Fixup Statepoint Caller Saved";
78 }
79
80 bool runOnMachineFunction(MachineFunction &MF) override;
81};
82
83} // End anonymous namespace.
84
85char FixupStatepointCallerSavedLegacy::ID = 0;
86char &llvm::FixupStatepointCallerSavedID = FixupStatepointCallerSavedLegacy::ID;
87
88INITIALIZE_PASS_BEGIN(FixupStatepointCallerSavedLegacy, DEBUG_TYPE,
89 "Fixup Statepoint Caller Saved", false, false)
90INITIALIZE_PASS_END(FixupStatepointCallerSavedLegacy, DEBUG_TYPE,
91 "Fixup Statepoint Caller Saved", false, false)
92
93// Utility function to get size of the register.
94static unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg) {
95 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
96 return TRI.getSpillSize(RC: *RC);
97}
98
99// Try to eliminate redundant copy to register which we're going to
100// spill, i.e. try to change:
101// X = COPY Y
102// SPILL X
103// to
104// SPILL Y
105// If there are no uses of X between copy and STATEPOINT, that COPY
106// may be eliminated.
107// Reg - register we're about to spill
108// RI - On entry points to statepoint.
109// On successful copy propagation set to new spill point.
110// IsKill - set to true if COPY is Kill (there are no uses of Y)
111// Returns either found source copy register or original one.
112static Register performCopyPropagation(Register Reg,
113 MachineBasicBlock::iterator &RI,
114 bool &IsKill, const TargetInstrInfo &TII,
115 const TargetRegisterInfo &TRI) {
116 // First check if statepoint itself uses Reg in non-meta operands.
117 int Idx = RI->findRegisterUseOperandIdx(Reg, TRI: &TRI, isKill: false);
118 if (Idx >= 0 && (unsigned)Idx < StatepointOpers(&*RI).getNumDeoptArgsIdx()) {
119 IsKill = false;
120 return Reg;
121 }
122
123 if (!EnableCopyProp)
124 return Reg;
125
126 MachineBasicBlock *MBB = RI->getParent();
127 MachineBasicBlock::reverse_iterator E = MBB->rend();
128 MachineInstr *Def = nullptr, *Use = nullptr;
129 for (auto It = ++(RI.getReverse()); It != E; ++It) {
130 if (It->readsRegister(Reg, TRI: &TRI) && !Use)
131 Use = &*It;
132 if (It->modifiesRegister(Reg, TRI: &TRI)) {
133 Def = &*It;
134 break;
135 }
136 }
137
138 if (!Def)
139 return Reg;
140
141 auto DestSrc = TII.isCopyInstr(MI: *Def);
142 if (!DestSrc || DestSrc->Destination->getReg() != Reg)
143 return Reg;
144
145 Register SrcReg = DestSrc->Source->getReg();
146
147 if (getRegisterSize(TRI, Reg) != getRegisterSize(TRI, Reg: SrcReg))
148 return Reg;
149
150 LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation "
151 << printReg(Reg, &TRI) << " -> " << printReg(SrcReg, &TRI)
152 << "\n");
153
154 // Insert spill immediately after Def
155 RI = ++MachineBasicBlock::iterator(Def);
156 IsKill = DestSrc->Source->isKill();
157
158 if (!Use) {
159 // There are no uses of original register between COPY and STATEPOINT.
160 // There can't be any after STATEPOINT, so we can eliminate Def.
161 LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def);
162 Def->eraseFromParent();
163 } else if (IsKill) {
164 // COPY will remain in place, spill will be inserted *after* it, so it is
165 // not a kill of source anymore.
166 const_cast<MachineOperand *>(DestSrc->Source)->setIsKill(false);
167 }
168
169 return SrcReg;
170}
171
172namespace {
173// Pair {Register, FrameIndex}
174using RegSlotPair = std::pair<Register, int>;
175
176// Keeps track of what reloads were inserted in MBB.
177class RegReloadCache {
178 using ReloadSet = SmallSet<RegSlotPair, 8>;
179 DenseMap<const MachineBasicBlock *, ReloadSet> Reloads;
180
181public:
182 RegReloadCache() = default;
183
184 // Record reload of Reg from FI in block MBB if not present yet.
185 // Return true if the reload is successfully recorded.
186 bool tryRecordReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
187 RegSlotPair RSP(Reg, FI);
188 return Reloads[MBB].insert(V: RSP).second;
189 }
190};
191
192// Cache used frame indexes during statepoint re-write to re-use them in
193// processing next statepoint instruction.
194// Two strategies. One is to preserve the size of spill slot while another one
195// extends the size of spill slots to reduce the number of them, causing
196// the less total frame size. But unspill will have "implicit" any extend.
197class FrameIndexesCache {
198private:
199 struct FrameIndexesPerSize {
200 // List of used frame indexes during processing previous statepoints.
201 SmallVector<int, 8> Slots;
202 // Current index of un-used yet frame index.
203 unsigned Index = 0;
204 };
205 MachineFrameInfo &MFI;
206 const TargetRegisterInfo &TRI;
207 // Map size to list of frame indexes of this size. If the mode is
208 // FixupSCSExtendSlotSize then the key 0 is used to keep all frame indexes.
209 // If the size of required spill slot is greater than in a cache then the
210 // size will be increased.
211 DenseMap<unsigned, FrameIndexesPerSize> Cache;
212
213 // Keeps track of slots reserved for the shared landing pad processing.
214 // Initialized from GlobalIndices for the current EHPad.
215 SmallSet<int, 8> ReservedSlots;
216
217 // Landing pad can be destination of several statepoints. Every register
218 // defined by such statepoints must be spilled to the same stack slot.
219 // This map keeps that information.
220 DenseMap<const MachineBasicBlock *, SmallVector<RegSlotPair, 8>>
221 GlobalIndices;
222
223 FrameIndexesPerSize &getCacheBucket(unsigned Size) {
224 // In FixupSCSExtendSlotSize mode the bucket with 0 index is used
225 // for all sizes.
226 return Cache[FixupSCSExtendSlotSize ? 0 : Size];
227 }
228
229public:
230 FrameIndexesCache(MachineFrameInfo &MFI, const TargetRegisterInfo &TRI)
231 : MFI(MFI), TRI(TRI) {}
232 // Reset the current state of used frame indexes. After invocation of
233 // this function all frame indexes are available for allocation with
234 // the exception of slots reserved for landing pad processing (if any).
235 void reset(const MachineBasicBlock *EHPad) {
236 for (auto &It : Cache)
237 It.second.Index = 0;
238
239 ReservedSlots.clear();
240 if (EHPad)
241 if (auto It = GlobalIndices.find(Val: EHPad); It != GlobalIndices.end())
242 ReservedSlots.insert_range(R: llvm::make_second_range(c&: It->second));
243 }
244
245 // Get frame index to spill the register.
246 int getFrameIndex(Register Reg, MachineBasicBlock *EHPad) {
247 // Check if slot for Reg is already reserved at EHPad.
248 auto It = GlobalIndices.find(Val: EHPad);
249 if (It != GlobalIndices.end()) {
250 auto &Vec = It->second;
251 auto Idx = llvm::find_if(
252 Range&: Vec, P: [Reg](RegSlotPair &RSP) { return Reg == RSP.first; });
253 if (Idx != Vec.end()) {
254 int FI = Idx->second;
255 LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register "
256 << printReg(Reg, &TRI) << " at "
257 << printMBBReference(*EHPad) << "\n");
258 assert(ReservedSlots.count(FI) && "using unreserved slot");
259 return FI;
260 }
261 }
262
263 unsigned Size = getRegisterSize(TRI, Reg);
264 FrameIndexesPerSize &Line = getCacheBucket(Size);
265 while (Line.Index < Line.Slots.size()) {
266 int FI = Line.Slots[Line.Index++];
267 if (ReservedSlots.count(V: FI))
268 continue;
269 // If all sizes are kept together we probably need to extend the
270 // spill slot size.
271 if (MFI.getObjectSize(ObjectIdx: FI) < Size) {
272 MFI.setObjectSize(ObjectIdx: FI, Size);
273 MFI.setObjectAlignment(ObjectIdx: FI, Alignment: Align(Size));
274 NumSpillSlotsExtended++;
275 }
276 return FI;
277 }
278 int FI = MFI.CreateSpillStackObject(Size, Alignment: Align(Size));
279 NumSpillSlotsAllocated++;
280 Line.Slots.push_back(Elt: FI);
281 ++Line.Index;
282
283 // Remember assignment {Reg, FI} for EHPad
284 if (EHPad) {
285 GlobalIndices[EHPad].push_back(Elt: std::make_pair(x&: Reg, y&: FI));
286 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg "
287 << printReg(Reg, &TRI) << " at landing pad "
288 << printMBBReference(*EHPad) << "\n");
289 }
290
291 return FI;
292 }
293
294 // Sort all registers to spill in descendent order. In the
295 // FixupSCSExtendSlotSize mode it will minimize the total frame size.
296 // In non FixupSCSExtendSlotSize mode we can skip this step.
297 void sortRegisters(SmallVectorImpl<Register> &Regs) {
298 if (!FixupSCSExtendSlotSize)
299 return;
300 llvm::sort(C&: Regs, Comp: [&](Register &A, Register &B) {
301 return getRegisterSize(TRI, Reg: A) > getRegisterSize(TRI, Reg: B);
302 });
303 }
304};
305
306// Describes the state of the current processing statepoint instruction.
307class StatepointState {
308private:
309 // statepoint instruction.
310 MachineInstr &MI;
311 MachineFunction &MF;
312 // If non-null then statepoint is invoke, and this points to the landing pad.
313 MachineBasicBlock *EHPad;
314 const TargetRegisterInfo &TRI;
315 const TargetInstrInfo &TII;
316 MachineFrameInfo &MFI;
317 // Mask with callee saved registers.
318 const uint32_t *Mask;
319 // Cache of frame indexes used on previous instruction processing.
320 FrameIndexesCache &CacheFI;
321 bool AllowGCPtrInCSR;
322 // Operands with physical registers requiring spilling.
323 SmallVector<unsigned, 8> OpsToSpill;
324 // Set of register to spill.
325 SmallVector<Register, 8> RegsToSpill;
326 // Set of registers to reload after statepoint.
327 SmallVector<Register, 8> RegsToReload;
328 // Map Register to Frame Slot index.
329 DenseMap<Register, int> RegToSlotIdx;
330
331public:
332 StatepointState(MachineInstr &MI, const uint32_t *Mask,
333 FrameIndexesCache &CacheFI, bool AllowGCPtrInCSR)
334 : MI(MI), MF(*MI.getMF()), TRI(*MF.getSubtarget().getRegisterInfo()),
335 TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),
336 Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {
337
338 // Find statepoint's landing pad, if any.
339 EHPad = nullptr;
340 MachineBasicBlock *MBB = MI.getParent();
341 // Invoke statepoint must be last one in block.
342 bool Last = std::none_of(first: ++MI.getIterator(), last: MBB->end().getInstrIterator(),
343 pred: [](MachineInstr &I) {
344 return I.getOpcode() == TargetOpcode::STATEPOINT;
345 });
346
347 if (!Last)
348 return;
349
350 auto IsEHPad = [](MachineBasicBlock *B) { return B->isEHPad(); };
351
352 assert(llvm::count_if(MBB->successors(), IsEHPad) < 2 && "multiple EHPads");
353
354 auto It = llvm::find_if(Range: MBB->successors(), P: IsEHPad);
355 if (It != MBB->succ_end())
356 EHPad = *It;
357 }
358
359 MachineBasicBlock *getEHPad() const { return EHPad; }
360
361 // Return true if register is callee saved.
362 bool isCalleeSaved(Register Reg) {
363 return (Mask[Reg.id() / 32] >> (Reg.id() % 32)) & 1;
364 }
365
366 // Iterates over statepoint meta args to find caller saver registers.
367 // Also cache the size of found registers.
368 // Returns true if caller save registers found.
369 bool findRegistersToSpill() {
370 SmallSet<Register, 8> GCRegs;
371 // All GC pointer operands assigned to registers produce new value.
372 // Since they're tied to their defs, it is enough to collect def registers.
373 for (const auto &Def : MI.defs())
374 GCRegs.insert(V: Def.getReg());
375
376 SmallSet<Register, 8> VisitedRegs;
377 for (unsigned Idx = StatepointOpers(&MI).getVarIdx(),
378 EndIdx = MI.getNumOperands();
379 Idx < EndIdx; ++Idx) {
380 MachineOperand &MO = MI.getOperand(i: Idx);
381 if (!MO.isReg() || MO.isImplicit() || MO.isUndef())
382 continue;
383 Register Reg = MO.getReg();
384 assert(Reg.isPhysical() && "Only physical regs are expected");
385
386 if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !GCRegs.contains(V: Reg)))
387 continue;
388
389 LLVM_DEBUG(dbgs() << "Will spill " << printReg(Reg, &TRI) << " at index "
390 << Idx << "\n");
391
392 if (VisitedRegs.insert(V: Reg).second)
393 RegsToSpill.push_back(Elt: Reg);
394 OpsToSpill.push_back(Elt: Idx);
395 }
396 CacheFI.sortRegisters(Regs&: RegsToSpill);
397 return !RegsToSpill.empty();
398 }
399
400 // Spill all caller saved registers right before statepoint instruction.
401 // Remember frame index where register is spilled.
402 void spillRegisters() {
403 for (Register Reg : RegsToSpill) {
404 int FI = CacheFI.getFrameIndex(Reg, EHPad);
405
406 NumSpilledRegisters++;
407 RegToSlotIdx[Reg] = FI;
408
409 LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, &TRI) << " to FI " << FI
410 << "\n");
411
412 // Perform trivial copy propagation
413 bool IsKill = true;
414 MachineBasicBlock::iterator InsertBefore(MI);
415 Reg = performCopyPropagation(Reg, RI&: InsertBefore, IsKill, TII, TRI);
416 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
417
418 LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore);
419 TII.storeRegToStackSlot(MBB&: *MI.getParent(), MI: InsertBefore, SrcReg: Reg, isKill: IsKill, FrameIndex: FI,
420 RC, VReg: Register());
421 }
422 }
423
424 void insertReloadBefore(Register Reg, MachineBasicBlock::iterator It,
425 MachineBasicBlock *MBB) {
426 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
427 int FI = RegToSlotIdx[Reg];
428 if (It != MBB->end()) {
429 TII.loadRegFromStackSlot(MBB&: *MBB, MI: It, DestReg: Reg, FrameIndex: FI, RC, VReg: Register());
430 return;
431 }
432
433 // To insert reload at the end of MBB, insert it before last instruction
434 // and then swap them.
435 assert(!MBB->empty() && "Empty block");
436 --It;
437 TII.loadRegFromStackSlot(MBB&: *MBB, MI: It, DestReg: Reg, FrameIndex: FI, RC, VReg: Register());
438 MachineInstr *Reload = It->getPrevNode();
439 int Dummy = 0;
440 (void)Dummy;
441 assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg);
442 assert(Dummy == FI);
443 MBB->remove(I: Reload);
444 MBB->insertAfter(I: It, MI: Reload);
445 }
446
447 // Insert reloads of (relocated) registers spilled in statepoint.
448 void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) {
449 MachineBasicBlock *MBB = NewStatepoint->getParent();
450 auto InsertPoint = std::next(x: NewStatepoint->getIterator());
451
452 for (auto Reg : RegsToReload) {
453 insertReloadBefore(Reg, It: InsertPoint, MBB);
454 LLVM_DEBUG(dbgs() << "Reloading " << printReg(Reg, &TRI) << " from FI "
455 << RegToSlotIdx[Reg] << " after statepoint\n");
456
457 if (EHPad && RC.tryRecordReload(Reg, FI: RegToSlotIdx[Reg], MBB: EHPad)) {
458 auto EHPadInsertPoint =
459 EHPad->SkipPHIsLabelsAndDebug(I: EHPad->begin(), Reg);
460 insertReloadBefore(Reg, It: EHPadInsertPoint, MBB: EHPad);
461 LLVM_DEBUG(dbgs() << "...also reload at EHPad "
462 << printMBBReference(*EHPad) << "\n");
463 }
464 }
465 }
466
467 // Re-write statepoint machine instruction to replace caller saved operands
468 // with indirect memory location (frame index).
469 MachineInstr *rewriteStatepoint() {
470 MachineInstr *NewMI =
471 MF.CreateMachineInstr(MCID: TII.get(Opcode: MI.getOpcode()), DL: MI.getDebugLoc(), NoImplicit: true);
472 MachineInstrBuilder MIB(MF, NewMI);
473
474 unsigned NumOps = MI.getNumOperands();
475
476 // New indices for the remaining defs.
477 SmallVector<unsigned, 8> NewIndices;
478 unsigned NumDefs = MI.getNumDefs();
479 for (unsigned I = 0; I < NumDefs; ++I) {
480 MachineOperand &DefMO = MI.getOperand(i: I);
481 assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand");
482 Register Reg = DefMO.getReg();
483 assert(DefMO.isTied() && "Def is expected to be tied");
484 // We skipped undef uses and did not spill them, so we should not
485 // proceed with defs here.
486 if (MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: I)).isUndef()) {
487 if (AllowGCPtrInCSR) {
488 NewIndices.push_back(Elt: NewMI->getNumOperands());
489 MIB.addReg(RegNo: Reg, Flags: RegState::Define);
490 }
491 continue;
492 }
493 if (!AllowGCPtrInCSR) {
494 assert(is_contained(RegsToSpill, Reg));
495 RegsToReload.push_back(Elt: Reg);
496 } else {
497 if (isCalleeSaved(Reg)) {
498 NewIndices.push_back(Elt: NewMI->getNumOperands());
499 MIB.addReg(RegNo: Reg, Flags: RegState::Define);
500 } else {
501 NewIndices.push_back(Elt: NumOps);
502 RegsToReload.push_back(Elt: Reg);
503 }
504 }
505 }
506
507 // Add End marker.
508 OpsToSpill.push_back(Elt: MI.getNumOperands());
509 unsigned CurOpIdx = 0;
510
511 for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) {
512 MachineOperand &MO = MI.getOperand(i: I);
513 if (I == OpsToSpill[CurOpIdx]) {
514 int FI = RegToSlotIdx[MO.getReg()];
515 MIB.addImm(Val: StackMaps::IndirectMemRefOp);
516 MIB.addImm(Val: getRegisterSize(TRI, Reg: MO.getReg()));
517 assert(MO.isReg() && "Should be register");
518 assert(MO.getReg().isPhysical() && "Should be physical register");
519 MIB.addFrameIndex(Idx: FI);
520 MIB.addImm(Val: 0);
521 ++CurOpIdx;
522 } else {
523 MIB.add(MO);
524 unsigned OldDef;
525 if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(UseOpIdx: I, DefOpIdx: &OldDef)) {
526 assert(OldDef < NumDefs);
527 assert(NewIndices[OldDef] < NumOps);
528 MIB->tieOperands(DefIdx: NewIndices[OldDef], UseIdx: MIB->getNumOperands() - 1);
529 }
530 }
531 }
532 assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed");
533 // Add mem operands.
534 NewMI->setMemRefs(MF, MemRefs: MI.memoperands());
535 for (auto It : RegToSlotIdx) {
536 Register R = It.first;
537 int FrameIndex = It.second;
538 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
539 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
540 if (is_contained(Range&: RegsToReload, Element: R))
541 Flags |= MachineMemOperand::MOStore;
542 auto *MMO =
543 MF.getMachineMemOperand(PtrInfo, F: Flags, Size: getRegisterSize(TRI, Reg: R),
544 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIndex));
545 NewMI->addMemOperand(MF, MO: MMO);
546 }
547
548 // Insert new statepoint and erase old one.
549 MI.getParent()->insert(I: MI, MI: NewMI);
550
551 LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n");
552 MI.eraseFromParent();
553 return NewMI;
554 }
555};
556
557class StatepointProcessor {
558private:
559 MachineFunction &MF;
560 const TargetRegisterInfo &TRI;
561 FrameIndexesCache CacheFI;
562 RegReloadCache ReloadCache;
563
564public:
565 StatepointProcessor(MachineFunction &MF)
566 : MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()),
567 CacheFI(MF.getFrameInfo(), TRI) {}
568
569 bool process(MachineInstr &MI, bool AllowGCPtrInCSR) {
570 StatepointOpers SO(&MI);
571 uint64_t Flags = SO.getFlags();
572 // Do nothing for LiveIn, it supports all registers.
573 if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn)
574 return false;
575 LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " "
576 << MI.getParent()->getName() << " : process statepoint "
577 << MI);
578 CallingConv::ID CC = SO.getCallingConv();
579 const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC);
580 StatepointState SS(MI, Mask, CacheFI, AllowGCPtrInCSR);
581 CacheFI.reset(EHPad: SS.getEHPad());
582
583 if (!SS.findRegistersToSpill())
584 return false;
585
586 SS.spillRegisters();
587 auto *NewStatepoint = SS.rewriteStatepoint();
588 SS.insertReloads(NewStatepoint, RC&: ReloadCache);
589 return true;
590 }
591};
592} // namespace
593
594bool FixupStatepointCallerSavedImpl::run(MachineFunction &MF) {
595 const Function &F = MF.getFunction();
596 if (!F.hasGC())
597 return false;
598
599 SmallVector<MachineInstr *, 16> Statepoints;
600 for (MachineBasicBlock &BB : MF)
601 for (MachineInstr &I : BB)
602 if (I.getOpcode() == TargetOpcode::STATEPOINT)
603 Statepoints.push_back(Elt: &I);
604
605 if (Statepoints.empty())
606 return false;
607
608 bool Changed = false;
609 StatepointProcessor SPP(MF);
610 unsigned NumStatepoints = 0;
611 bool AllowGCPtrInCSR = PassGCPtrInCSR;
612 for (MachineInstr *I : Statepoints) {
613 ++NumStatepoints;
614 if (MaxStatepointsWithRegs.getNumOccurrences() &&
615 NumStatepoints >= MaxStatepointsWithRegs)
616 AllowGCPtrInCSR = false;
617 Changed |= SPP.process(MI&: *I, AllowGCPtrInCSR);
618 }
619 return Changed;
620}
621
622bool FixupStatepointCallerSavedLegacy::runOnMachineFunction(
623 MachineFunction &MF) {
624 if (skipFunction(F: MF.getFunction()))
625 return false;
626
627 return FixupStatepointCallerSavedImpl().run(MF);
628}
629
630PreservedAnalyses
631FixupStatepointCallerSavedPass::run(MachineFunction &MF,
632 MachineFunctionAnalysisManager &MFAM) {
633
634 if (!FixupStatepointCallerSavedImpl().run(MF))
635 return PreservedAnalyses::all();
636
637 auto PA = getMachineFunctionPassPreservedAnalyses();
638 PA.preserveSet<CFGAnalyses>();
639 return PA;
640}
641