1//===-- FixupStatepointCallerSaved.cpp - Fixup caller saved registers ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// Statepoint instruction in deopt parameters contains values which are
11/// meaningful to the runtime and should be able to be read at the moment the
12/// call returns. So we can say that we need to encode the fact that these
13/// values are "late read" by runtime. If we could express this notion for
14/// register allocator it would produce the right form for us.
15/// The need to fixup (i.e this pass) is specifically handling the fact that
16/// we cannot describe such a late read for the register allocator.
17/// Register allocator may put the value on a register clobbered by the call.
18/// This pass forces the spill of such registers and replaces corresponding
19/// statepoint operands to added spill slots.
20///
21//===----------------------------------------------------------------------===//
22
23#include "llvm/CodeGen/FixupStatepointCallerSaved.h"
24#include "CodeGenOptions.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/MachineFunctionPass.h"
29#include "llvm/CodeGen/StackMaps.h"
30#include "llvm/CodeGen/TargetInstrInfo.h"
31#include "llvm/IR/Statepoint.h"
32#include "llvm/InitializePasses.h"
33#include "llvm/Support/Debug.h"
34
35using namespace llvm;
36
37#define DEBUG_TYPE "fixup-statepoint-caller-saved"
38STATISTIC(NumSpilledRegisters, "Number of spilled register");
39STATISTIC(NumSpillSlotsAllocated, "Number of spill slots allocated");
40STATISTIC(NumSpillSlotsExtended, "Number of spill slots extended");
41
42namespace {
43
44struct FixupStatepointCallerSavedImpl {
45 bool run(MachineFunction &MF);
46};
47
48class FixupStatepointCallerSavedLegacy : public MachineFunctionPass {
49public:
50 static char ID;
51
52 FixupStatepointCallerSavedLegacy() : MachineFunctionPass(ID) {}
53 void getAnalysisUsage(AnalysisUsage &AU) const override {
54 AU.setPreservesCFG();
55 MachineFunctionPass::getAnalysisUsage(AU);
56 }
57
58 StringRef getPassName() const override {
59 return "Fixup Statepoint Caller Saved";
60 }
61
62 bool runOnMachineFunction(MachineFunction &MF) override;
63};
64
65} // End anonymous namespace.
66
67char FixupStatepointCallerSavedLegacy::ID = 0;
68char &llvm::FixupStatepointCallerSavedID = FixupStatepointCallerSavedLegacy::ID;
69
70INITIALIZE_PASS_BEGIN(FixupStatepointCallerSavedLegacy, DEBUG_TYPE,
71 "Fixup Statepoint Caller Saved", false, false)
72INITIALIZE_PASS_END(FixupStatepointCallerSavedLegacy, DEBUG_TYPE,
73 "Fixup Statepoint Caller Saved", false, false)
74
75// Utility function to get size of the register.
76static unsigned getRegisterSize(const TargetRegisterInfo &TRI, Register Reg) {
77 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
78 return TRI.getSpillSize(RC: *RC);
79}
80
81// Try to eliminate redundant copy to register which we're going to
82// spill, i.e. try to change:
83// X = COPY Y
84// SPILL X
85// to
86// SPILL Y
87// If there are no uses of X between copy and STATEPOINT, that COPY
88// may be eliminated.
89// Reg - register we're about to spill
90// RI - On entry points to statepoint.
91// On successful copy propagation set to new spill point.
92// IsKill - set to true if COPY is Kill (there are no uses of Y)
93// Returns either found source copy register or original one.
94static Register performCopyPropagation(const CodeGenOptions &Opts, Register Reg,
95 MachineBasicBlock::iterator &RI,
96 bool &IsKill, const TargetInstrInfo &TII,
97 const TargetRegisterInfo &TRI) {
98 // First check if statepoint itself uses Reg in non-meta operands.
99 int Idx = RI->findRegisterUseOperandIdx(Reg, TRI: &TRI, isKill: false);
100 if (Idx >= 0 && (unsigned)Idx < StatepointOpers(&*RI).getNumDeoptArgsIdx()) {
101 IsKill = false;
102 return Reg;
103 }
104
105 if (!Opts.fixup_scs_enable_copy_propagation)
106 return Reg;
107
108 MachineBasicBlock *MBB = RI->getParent();
109 MachineBasicBlock::reverse_iterator E = MBB->rend();
110 MachineInstr *Def = nullptr, *Use = nullptr;
111 for (auto It = ++(RI.getReverse()); It != E; ++It) {
112 if (It->readsRegister(Reg, TRI: &TRI) && !Use)
113 Use = &*It;
114 if (It->modifiesRegister(Reg, TRI: &TRI)) {
115 Def = &*It;
116 break;
117 }
118 }
119
120 if (!Def)
121 return Reg;
122
123 auto DestSrc = TII.isCopyInstr(MI: *Def);
124 if (!DestSrc || DestSrc->Destination->getReg() != Reg)
125 return Reg;
126
127 Register SrcReg = DestSrc->Source->getReg();
128
129 if (getRegisterSize(TRI, Reg) != getRegisterSize(TRI, Reg: SrcReg))
130 return Reg;
131
132 LLVM_DEBUG(dbgs() << "spillRegisters: perform copy propagation "
133 << printReg(Reg, &TRI) << " -> " << printReg(SrcReg, &TRI)
134 << "\n");
135
136 // Insert spill immediately after Def
137 RI = ++MachineBasicBlock::iterator(Def);
138 IsKill = DestSrc->Source->isKill();
139
140 if (!Use) {
141 // There are no uses of original register between COPY and STATEPOINT.
142 // There can't be any after STATEPOINT, so we can eliminate Def.
143 LLVM_DEBUG(dbgs() << "spillRegisters: removing dead copy " << *Def);
144 Def->eraseFromParent();
145 } else if (IsKill) {
146 // COPY will remain in place, spill will be inserted *after* it, so it is
147 // not a kill of source anymore.
148 const_cast<MachineOperand *>(DestSrc->Source)->setIsKill(false);
149 }
150
151 return SrcReg;
152}
153
154namespace {
155// Pair {Register, FrameIndex}
156using RegSlotPair = std::pair<Register, int>;
157
158// Keeps track of what reloads were inserted in MBB.
159class RegReloadCache {
160 using ReloadSet = SmallSet<RegSlotPair, 8>;
161 DenseMap<const MachineBasicBlock *, ReloadSet> Reloads;
162
163public:
164 RegReloadCache() = default;
165
166 // Record reload of Reg from FI in block MBB if not present yet.
167 // Return true if the reload is successfully recorded.
168 bool tryRecordReload(Register Reg, int FI, const MachineBasicBlock *MBB) {
169 RegSlotPair RSP(Reg, FI);
170 return Reloads[MBB].insert(V: RSP).second;
171 }
172};
173
174// Cache used frame indexes during statepoint re-write to re-use them in
175// processing next statepoint instruction.
176// Two strategies. One is to preserve the size of spill slot while another one
177// extends the size of spill slots to reduce the number of them, causing
178// the less total frame size. But unspill will have "implicit" any extend.
179class FrameIndexesCache {
180private:
181 struct FrameIndexesPerSize {
182 // List of used frame indexes during processing previous statepoints.
183 SmallVector<int, 8> Slots;
184 // Current index of un-used yet frame index.
185 unsigned Index = 0;
186 };
187 const CodeGenOptions &Opts;
188 MachineFrameInfo &MFI;
189 const TargetRegisterInfo &TRI;
190 // Map size to list of frame indexes of this size. If the mode is
191 // FixupSCSExtendSlotSize then the key 0 is used to keep all frame indexes.
192 // If the size of required spill slot is greater than in a cache then the
193 // size will be increased.
194 DenseMap<unsigned, FrameIndexesPerSize> Cache;
195
196 // Keeps track of slots reserved for the shared landing pad processing.
197 // Initialized from GlobalIndices for the current EHPad.
198 SmallSet<int, 8> ReservedSlots;
199
200 // Landing pad can be destination of several statepoints. Every register
201 // defined by such statepoints must be spilled to the same stack slot.
202 // This map keeps that information.
203 DenseMap<const MachineBasicBlock *, SmallVector<RegSlotPair, 8>>
204 GlobalIndices;
205
206 FrameIndexesPerSize &getCacheBucket(unsigned Size) {
207 // In FixupSCSExtendSlotSize mode the bucket with 0 index is used
208 // for all sizes.
209 return Cache[Opts.fixup_scs_extend_slot_size ? 0 : Size];
210 }
211
212public:
213 FrameIndexesCache(const CodeGenOptions &Opts, MachineFrameInfo &MFI,
214 const TargetRegisterInfo &TRI)
215 : Opts(Opts), MFI(MFI), TRI(TRI) {}
216 // Reset the current state of used frame indexes. After invocation of
217 // this function all frame indexes are available for allocation with
218 // the exception of slots reserved for landing pad processing (if any).
219 void reset(const MachineBasicBlock *EHPad) {
220 for (auto &It : Cache)
221 It.second.Index = 0;
222
223 ReservedSlots.clear();
224 if (EHPad)
225 if (auto It = GlobalIndices.find(Val: EHPad); It != GlobalIndices.end())
226 ReservedSlots.insert_range(R: llvm::make_second_range(c&: It->second));
227 }
228
229 // Get frame index to spill the register.
230 int getFrameIndex(Register Reg, MachineBasicBlock *EHPad) {
231 // Check if slot for Reg is already reserved at EHPad.
232 auto It = GlobalIndices.find(Val: EHPad);
233 if (It != GlobalIndices.end()) {
234 auto &Vec = It->second;
235 auto Idx = llvm::find_if(
236 Range&: Vec, P: [Reg](RegSlotPair &RSP) { return Reg == RSP.first; });
237 if (Idx != Vec.end()) {
238 int FI = Idx->second;
239 LLVM_DEBUG(dbgs() << "Found global FI " << FI << " for register "
240 << printReg(Reg, &TRI) << " at "
241 << printMBBReference(*EHPad) << "\n");
242 assert(ReservedSlots.count(FI) && "using unreserved slot");
243 return FI;
244 }
245 }
246
247 unsigned Size = getRegisterSize(TRI, Reg);
248 FrameIndexesPerSize &Line = getCacheBucket(Size);
249 while (Line.Index < Line.Slots.size()) {
250 int FI = Line.Slots[Line.Index++];
251 if (ReservedSlots.count(V: FI))
252 continue;
253 // If all sizes are kept together we probably need to extend the
254 // spill slot size.
255 if (MFI.getObjectSize(ObjectIdx: FI) < Size) {
256 MFI.setObjectSize(ObjectIdx: FI, Size);
257 MFI.setObjectAlignment(ObjectIdx: FI, Alignment: Align(Size));
258 NumSpillSlotsExtended++;
259 }
260 return FI;
261 }
262 int FI = MFI.CreateSpillStackObject(Size, Alignment: Align(Size));
263 NumSpillSlotsAllocated++;
264 Line.Slots.push_back(Elt: FI);
265 ++Line.Index;
266
267 // Remember assignment {Reg, FI} for EHPad
268 if (EHPad) {
269 GlobalIndices[EHPad].push_back(Elt: std::make_pair(x&: Reg, y&: FI));
270 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling reg "
271 << printReg(Reg, &TRI) << " at landing pad "
272 << printMBBReference(*EHPad) << "\n");
273 }
274
275 return FI;
276 }
277
278 // Sort all registers to spill in descendent order. In the
279 // FixupSCSExtendSlotSize mode it will minimize the total frame size.
280 // In non FixupSCSExtendSlotSize mode we can skip this step.
281 void sortRegisters(SmallVectorImpl<Register> &Regs) {
282 if (!Opts.fixup_scs_extend_slot_size)
283 return;
284 llvm::sort(C&: Regs, Comp: [&](Register &A, Register &B) {
285 return getRegisterSize(TRI, Reg: A) > getRegisterSize(TRI, Reg: B);
286 });
287 }
288};
289
290// Describes the state of the current processing statepoint instruction.
291class StatepointState {
292private:
293 const CodeGenOptions &Opts;
294 // statepoint instruction.
295 MachineInstr &MI;
296 MachineFunction &MF;
297 // If non-null then statepoint is invoke, and this points to the landing pad.
298 MachineBasicBlock *EHPad;
299 const TargetRegisterInfo &TRI;
300 const TargetInstrInfo &TII;
301 MachineFrameInfo &MFI;
302 // Mask with callee saved registers.
303 const uint32_t *Mask;
304 // Cache of frame indexes used on previous instruction processing.
305 FrameIndexesCache &CacheFI;
306 bool AllowGCPtrInCSR;
307 // Operands with physical registers requiring spilling.
308 SmallVector<unsigned, 8> OpsToSpill;
309 // Set of register to spill.
310 SmallVector<Register, 8> RegsToSpill;
311 // Set of registers to reload after statepoint.
312 SmallVector<Register, 8> RegsToReload;
313 // Map Register to Frame Slot index.
314 DenseMap<Register, int> RegToSlotIdx;
315
316public:
317 StatepointState(const CodeGenOptions &Opts, MachineInstr &MI,
318 const uint32_t *Mask, FrameIndexesCache &CacheFI,
319 bool AllowGCPtrInCSR)
320 : Opts(Opts), MI(MI), MF(*MI.getMF()),
321 TRI(*MF.getSubtarget().getRegisterInfo()),
322 TII(*MF.getSubtarget().getInstrInfo()), MFI(MF.getFrameInfo()),
323 Mask(Mask), CacheFI(CacheFI), AllowGCPtrInCSR(AllowGCPtrInCSR) {
324
325 // Find statepoint's landing pad, if any.
326 EHPad = nullptr;
327 MachineBasicBlock *MBB = MI.getParent();
328 // Invoke statepoint must be last one in block.
329 bool Last = std::none_of(first: ++MI.getIterator(), last: MBB->end().getInstrIterator(),
330 pred: [](MachineInstr &I) {
331 return I.getOpcode() == TargetOpcode::STATEPOINT;
332 });
333
334 if (!Last)
335 return;
336
337 auto IsEHPad = [](MachineBasicBlock *B) { return B->isEHPad(); };
338
339 assert(llvm::count_if(MBB->successors(), IsEHPad) < 2 && "multiple EHPads");
340
341 auto It = llvm::find_if(Range: MBB->successors(), P: IsEHPad);
342 if (It != MBB->succ_end())
343 EHPad = *It;
344 }
345
346 MachineBasicBlock *getEHPad() const { return EHPad; }
347
348 // Return true if register is callee saved.
349 bool isCalleeSaved(Register Reg) {
350 return (Mask[Reg.id() / 32] >> (Reg.id() % 32)) & 1;
351 }
352
353 // Iterates over statepoint meta args to find caller saver registers.
354 // Also cache the size of found registers.
355 // Returns true if caller save registers found.
356 bool findRegistersToSpill() {
357 SmallSet<Register, 8> GCRegs;
358 // All GC pointer operands assigned to registers produce new value.
359 // Since they're tied to their defs, it is enough to collect def registers.
360 for (const auto &Def : MI.defs())
361 GCRegs.insert(V: Def.getReg());
362
363 SmallSet<Register, 8> VisitedRegs;
364 for (unsigned Idx = StatepointOpers(&MI).getVarIdx(),
365 EndIdx = MI.getNumOperands();
366 Idx < EndIdx; ++Idx) {
367 MachineOperand &MO = MI.getOperand(i: Idx);
368 if (!MO.isReg() || MO.isImplicit() || MO.isUndef())
369 continue;
370 Register Reg = MO.getReg();
371 assert(Reg.isPhysical() && "Only physical regs are expected");
372
373 if (isCalleeSaved(Reg) && (AllowGCPtrInCSR || !GCRegs.contains(V: Reg)))
374 continue;
375
376 LLVM_DEBUG(dbgs() << "Will spill " << printReg(Reg, &TRI) << " at index "
377 << Idx << "\n");
378
379 if (VisitedRegs.insert(V: Reg).second)
380 RegsToSpill.push_back(Elt: Reg);
381 OpsToSpill.push_back(Elt: Idx);
382 }
383 CacheFI.sortRegisters(Regs&: RegsToSpill);
384 return !RegsToSpill.empty();
385 }
386
387 // Spill all caller saved registers right before statepoint instruction.
388 // Remember frame index where register is spilled.
389 void spillRegisters() {
390 for (Register Reg : RegsToSpill) {
391 int FI = CacheFI.getFrameIndex(Reg, EHPad);
392
393 NumSpilledRegisters++;
394 RegToSlotIdx[Reg] = FI;
395
396 LLVM_DEBUG(dbgs() << "Spilling " << printReg(Reg, &TRI) << " to FI " << FI
397 << "\n");
398
399 // Perform trivial copy propagation
400 bool IsKill = true;
401 MachineBasicBlock::iterator InsertBefore(MI);
402 Reg = performCopyPropagation(Opts, Reg, RI&: InsertBefore, IsKill, TII, TRI);
403 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
404
405 LLVM_DEBUG(dbgs() << "Insert spill before " << *InsertBefore);
406 TII.storeRegToStackSlot(MBB&: *MI.getParent(), MI: InsertBefore, SrcReg: Reg, isKill: IsKill, FrameIndex: FI,
407 RC, VReg: Register());
408 }
409 }
410
411 void insertReloadBefore(Register Reg, MachineBasicBlock::iterator It,
412 MachineBasicBlock *MBB) {
413 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
414 int FI = RegToSlotIdx[Reg];
415 if (It != MBB->end()) {
416 TII.loadRegFromStackSlot(MBB&: *MBB, MI: It, DestReg: Reg, FrameIndex: FI, RC, VReg: Register());
417 return;
418 }
419
420 // To insert reload at the end of MBB, insert it before last instruction
421 // and then swap them.
422 assert(!MBB->empty() && "Empty block");
423 --It;
424 TII.loadRegFromStackSlot(MBB&: *MBB, MI: It, DestReg: Reg, FrameIndex: FI, RC, VReg: Register());
425 MachineInstr *Reload = It->getPrevNode();
426 int Dummy = 0;
427 (void)Dummy;
428 assert(TII.isLoadFromStackSlot(*Reload, Dummy) == Reg);
429 assert(Dummy == FI);
430 MBB->remove(I: Reload);
431 MBB->insertAfter(I: It, MI: Reload);
432 }
433
434 // Insert reloads of (relocated) registers spilled in statepoint.
435 void insertReloads(MachineInstr *NewStatepoint, RegReloadCache &RC) {
436 MachineBasicBlock *MBB = NewStatepoint->getParent();
437 auto InsertPoint = std::next(x: NewStatepoint->getIterator());
438
439 for (auto Reg : RegsToReload) {
440 insertReloadBefore(Reg, It: InsertPoint, MBB);
441 LLVM_DEBUG(dbgs() << "Reloading " << printReg(Reg, &TRI) << " from FI "
442 << RegToSlotIdx[Reg] << " after statepoint\n");
443
444 if (EHPad && RC.tryRecordReload(Reg, FI: RegToSlotIdx[Reg], MBB: EHPad)) {
445 auto EHPadInsertPoint =
446 EHPad->SkipPHIsLabelsAndDebug(I: EHPad->begin(), Reg);
447 insertReloadBefore(Reg, It: EHPadInsertPoint, MBB: EHPad);
448 LLVM_DEBUG(dbgs() << "...also reload at EHPad "
449 << printMBBReference(*EHPad) << "\n");
450 }
451 }
452 }
453
454 // Re-write statepoint machine instruction to replace caller saved operands
455 // with indirect memory location (frame index).
456 MachineInstr *rewriteStatepoint() {
457 MachineInstr *NewMI =
458 MF.CreateMachineInstr(MCID: TII.get(Opcode: MI.getOpcode()), DL: MI.getDebugLoc(), NoImplicit: true);
459 MachineInstrBuilder MIB(MF, NewMI);
460
461 unsigned NumOps = MI.getNumOperands();
462
463 // New indices for the remaining defs.
464 SmallVector<unsigned, 8> NewIndices;
465 unsigned NumDefs = MI.getNumDefs();
466 for (unsigned I = 0; I < NumDefs; ++I) {
467 MachineOperand &DefMO = MI.getOperand(i: I);
468 assert(DefMO.isReg() && DefMO.isDef() && "Expected Reg Def operand");
469 Register Reg = DefMO.getReg();
470 assert(DefMO.isTied() && "Def is expected to be tied");
471 // We skipped undef uses and did not spill them, so we should not
472 // proceed with defs here.
473 if (MI.getOperand(i: MI.findTiedOperandIdx(OpIdx: I)).isUndef()) {
474 if (AllowGCPtrInCSR) {
475 NewIndices.push_back(Elt: NewMI->getNumOperands());
476 MIB.addReg(RegNo: Reg, Flags: RegState::Define);
477 }
478 continue;
479 }
480 if (!AllowGCPtrInCSR) {
481 assert(is_contained(RegsToSpill, Reg));
482 RegsToReload.push_back(Elt: Reg);
483 } else {
484 if (isCalleeSaved(Reg)) {
485 NewIndices.push_back(Elt: NewMI->getNumOperands());
486 MIB.addReg(RegNo: Reg, Flags: RegState::Define);
487 } else {
488 NewIndices.push_back(Elt: NumOps);
489 RegsToReload.push_back(Elt: Reg);
490 }
491 }
492 }
493
494 // Add End marker.
495 OpsToSpill.push_back(Elt: MI.getNumOperands());
496 unsigned CurOpIdx = 0;
497
498 for (unsigned I = NumDefs; I < MI.getNumOperands(); ++I) {
499 MachineOperand &MO = MI.getOperand(i: I);
500 if (I == OpsToSpill[CurOpIdx]) {
501 int FI = RegToSlotIdx[MO.getReg()];
502 MIB.addImm(Val: StackMaps::IndirectMemRefOp);
503 MIB.addImm(Val: getRegisterSize(TRI, Reg: MO.getReg()));
504 assert(MO.isReg() && "Should be register");
505 assert(MO.getReg().isPhysical() && "Should be physical register");
506 MIB.addFrameIndex(Idx: FI);
507 MIB.addImm(Val: 0);
508 ++CurOpIdx;
509 } else {
510 MIB.add(MO);
511 unsigned OldDef;
512 if (AllowGCPtrInCSR && MI.isRegTiedToDefOperand(UseOpIdx: I, DefOpIdx: &OldDef)) {
513 assert(OldDef < NumDefs);
514 assert(NewIndices[OldDef] < NumOps);
515 MIB->tieOperands(DefIdx: NewIndices[OldDef], UseIdx: MIB->getNumOperands() - 1);
516 }
517 }
518 }
519 assert(CurOpIdx == (OpsToSpill.size() - 1) && "Not all operands processed");
520 // Add mem operands.
521 NewMI->setMemRefs(MF, MemRefs: MI.memoperands());
522 for (auto It : RegToSlotIdx) {
523 Register R = It.first;
524 int FrameIndex = It.second;
525 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
526 MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad;
527 if (is_contained(Range&: RegsToReload, Element: R))
528 Flags |= MachineMemOperand::MOStore;
529 auto *MMO =
530 MF.getMachineMemOperand(PtrInfo, F: Flags, Size: getRegisterSize(TRI, Reg: R),
531 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIndex));
532 NewMI->addMemOperand(MF, MO: MMO);
533 }
534
535 // Insert new statepoint and erase old one.
536 MI.getParent()->insert(I: MI, MI: NewMI);
537
538 LLVM_DEBUG(dbgs() << "rewritten statepoint to : " << *NewMI << "\n");
539 MI.eraseFromParent();
540 return NewMI;
541 }
542};
543
544class StatepointProcessor {
545private:
546 const CodeGenOptions &Opts;
547 MachineFunction &MF;
548 const TargetRegisterInfo &TRI;
549 FrameIndexesCache CacheFI;
550 RegReloadCache ReloadCache;
551
552public:
553 StatepointProcessor(const CodeGenOptions &Opts, MachineFunction &MF)
554 : Opts(Opts), MF(MF), TRI(*MF.getSubtarget().getRegisterInfo()),
555 CacheFI(Opts, MF.getFrameInfo(), TRI) {}
556
557 bool process(MachineInstr &MI, bool AllowGCPtrInCSR) {
558 StatepointOpers SO(&MI);
559 uint64_t Flags = SO.getFlags();
560 // Do nothing for LiveIn, it supports all registers.
561 if (Flags & (uint64_t)StatepointFlags::DeoptLiveIn)
562 return false;
563 LLVM_DEBUG(dbgs() << "\nMBB " << MI.getParent()->getNumber() << " "
564 << MI.getParent()->getName() << " : process statepoint "
565 << MI);
566 CallingConv::ID CC = SO.getCallingConv();
567 const uint32_t *Mask = TRI.getCallPreservedMask(MF, CC);
568 StatepointState SS(Opts, MI, Mask, CacheFI, AllowGCPtrInCSR);
569 CacheFI.reset(EHPad: SS.getEHPad());
570
571 if (!SS.findRegistersToSpill())
572 return false;
573
574 SS.spillRegisters();
575 auto *NewStatepoint = SS.rewriteStatepoint();
576 SS.insertReloads(NewStatepoint, RC&: ReloadCache);
577 return true;
578 }
579};
580} // namespace
581
582bool FixupStatepointCallerSavedImpl::run(MachineFunction &MF) {
583 const Function &F = MF.getFunction();
584 if (!F.hasGC())
585 return false;
586
587 SmallVector<MachineInstr *, 16> Statepoints;
588 for (MachineBasicBlock &BB : MF)
589 for (MachineInstr &I : BB)
590 if (I.getOpcode() == TargetOpcode::STATEPOINT)
591 Statepoints.push_back(Elt: &I);
592
593 if (Statepoints.empty())
594 return false;
595
596 const CodeGenOptions &Opts = getCodeGenOptions(MF);
597 bool Changed = false;
598 StatepointProcessor SPP(Opts, MF);
599 unsigned NumStatepoints = 0;
600 bool AllowGCPtrInCSR = Opts.fixup_allow_gcptr_in_csr;
601 for (MachineInstr *I : Statepoints) {
602 ++NumStatepoints;
603 if (Opts.fixup_max_csr_statepoints &&
604 NumStatepoints >= *Opts.fixup_max_csr_statepoints)
605 AllowGCPtrInCSR = false;
606 Changed |= SPP.process(MI&: *I, AllowGCPtrInCSR);
607 }
608 return Changed;
609}
610
611bool FixupStatepointCallerSavedLegacy::runOnMachineFunction(
612 MachineFunction &MF) {
613 if (skipFunction(F: MF.getFunction()))
614 return false;
615
616 return FixupStatepointCallerSavedImpl().run(MF);
617}
618
619PreservedAnalyses
620FixupStatepointCallerSavedPass::run(MachineFunction &MF,
621 MachineFunctionAnalysisManager &MFAM) {
622
623 if (!FixupStatepointCallerSavedImpl().run(MF))
624 return PreservedAnalyses::all();
625
626 auto PA = getMachineFunctionPassPreservedAnalyses();
627 PA.preserveSet<CFGAnalyses>();
628 return PA;
629}
630