1//===-- SILowerSGPRSPills.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Handle SGPR spills. This pass takes the place of PrologEpilogInserter for all
10// SGPR spills, so must insert CSR SGPR spills as well as expand them.
11//
12// This pass must never create new SGPR virtual registers.
13//
14// FIXME: Must stop RegScavenger spills in later passes.
15//
16//===----------------------------------------------------------------------===//
17
18#include "SILowerSGPRSpills.h"
19#include "AMDGPU.h"
20#include "GCNSubtarget.h"
21#include "SIMachineFunctionInfo.h"
22#include "SIPreAllocateWWMRegs.h"
23#include "SISpillUtils.h"
24#include "llvm/CodeGen/LiveIntervals.h"
25#include "llvm/CodeGen/MachineCycleAnalysis.h"
26#include "llvm/CodeGen/MachineDominators.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/RegisterScavenging.h"
29#include "llvm/InitializePasses.h"
30
31using namespace llvm;
32
33#define DEBUG_TYPE "si-lower-sgpr-spills"
34
35using MBBVector = SmallVector<MachineBasicBlock *, 4>;
36
37namespace {
38
39/// Insertion point for IMPLICIT_DEF: iterator may be MBB::end() and can't be
40/// dereferenced so the parent block is stored explicitly.
41struct LaneVGPRInsertPt {
42 MachineBasicBlock *MBB;
43 MachineBasicBlock::iterator It;
44};
45
46static LaneVGPRInsertPt insertPt(MachineBasicBlock *MBB,
47 MachineBasicBlock::iterator It) {
48 return {.MBB: MBB, .It: It};
49}
50
51static cl::opt<unsigned> MaxNumVGPRsForWwmAllocation(
52 "amdgpu-num-vgprs-for-wwm-alloc",
53 cl::desc("Max num VGPRs for whole-wave register allocation."),
54 cl::ReallyHidden, cl::init(Val: 10));
55
56class SILowerSGPRSpills {
57private:
58 const SIRegisterInfo *TRI = nullptr;
59 const SIInstrInfo *TII = nullptr;
60 LiveIntervals *LIS = nullptr;
61 SlotIndexes *Indexes = nullptr;
62 MachineDominatorTree *MDT = nullptr;
63 MachineCycleInfo *MCI = nullptr;
64
65 // Save and Restore blocks of the current function. Typically there is a
66 // single save block, unless Windows EH funclets are involved.
67 MBBVector SaveBlocks;
68 MBBVector RestoreBlocks;
69
70 MachineBasicBlock *getCycleDomBB(CycleRef C);
71
72public:
73 SILowerSGPRSpills(LiveIntervals *LIS, SlotIndexes *Indexes,
74 MachineDominatorTree *MDT, MachineCycleInfo *MCI)
75 : LIS(LIS), Indexes(Indexes), MDT(MDT), MCI(MCI) {}
76 bool run(MachineFunction &MF);
77 void calculateSaveRestoreBlocks(MachineFunction &MF);
78 bool spillCalleeSavedRegs(MachineFunction &MF,
79 SmallVectorImpl<int> &CalleeSavedFIs);
80 void updateLaneVGPRDomInstr(
81 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
82 DenseMap<Register, LaneVGPRInsertPt> &LaneVGPRDomInstr);
83 SmallVector<MCRegister> determineRegsForWWMAllocation(MachineFunction &MF);
84 void assignWWMRegs(MachineFunction &MF, ArrayRef<MCRegister> WWMRegCandidates,
85 bool RequiresFullWWMPool);
86};
87
88class SILowerSGPRSpillsLegacy : public MachineFunctionPass {
89public:
90 static char ID;
91
92 SILowerSGPRSpillsLegacy() : MachineFunctionPass(ID) {}
93
94 bool runOnMachineFunction(MachineFunction &MF) override;
95
96 void getAnalysisUsage(AnalysisUsage &AU) const override {
97 AU.addRequired<MachineDominatorTreeWrapperPass>();
98 AU.addRequired<MachineCycleInfoWrapperPass>();
99 AU.setPreservesAll();
100 MachineFunctionPass::getAnalysisUsage(AU);
101 }
102
103 MachineFunctionProperties getClearedProperties() const override {
104 // SILowerSGPRSpills introduces new Virtual VGPRs for spilling SGPRs.
105 return MachineFunctionProperties().setIsSSA().setNoVRegs();
106 }
107};
108
109} // end anonymous namespace
110
111char SILowerSGPRSpillsLegacy::ID = 0;
112
113INITIALIZE_PASS_BEGIN(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
114 "SI lower SGPR spill instructions", false, false)
115INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
116INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
117INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
118INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass)
119INITIALIZE_PASS_END(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
120 "SI lower SGPR spill instructions", false, false)
121
122char &llvm::SILowerSGPRSpillsLegacyID = SILowerSGPRSpillsLegacy::ID;
123
124/// Insert spill code for the callee-saved registers used in the function.
125static void insertCSRSaves(const GCNSubtarget &ST, MachineBasicBlock &SaveBlock,
126 ArrayRef<CalleeSavedInfo> CSI, SlotIndexes *Indexes,
127 LiveIntervals *LIS) {
128 const TargetFrameLowering *TFI = ST.getFrameLowering();
129 const TargetRegisterInfo *TRI = ST.getRegisterInfo();
130 MachineBasicBlock::iterator I = SaveBlock.begin();
131 MachineInstrSpan MIS(I, &SaveBlock);
132 bool Success = TFI->spillCalleeSavedRegisters(MBB&: SaveBlock, MI: I, CSI, TRI);
133 assert(Success && "spillCalleeSavedRegisters should always succeed");
134 (void)Success;
135
136 // TFI doesn't update Indexes and LIS, so we have to do it separately.
137 if (Indexes)
138 Indexes->repairIndexesInRange(MBB: &SaveBlock, Begin: SaveBlock.begin(), End: I);
139
140 if (LIS)
141 for (const CalleeSavedInfo &CS : CSI)
142 LIS->removeAllRegUnitsForPhysReg(Reg: CS.getReg());
143}
144
145/// Insert restore code for the callee-saved registers used in the function.
146static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
147 MutableArrayRef<CalleeSavedInfo> CSI,
148 SlotIndexes *Indexes, LiveIntervals *LIS) {
149 MachineFunction &MF = *RestoreBlock.getParent();
150 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
151 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
152 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
153 // Restore all registers immediately before the return and any
154 // terminators that precede it.
155 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();
156 const MachineBasicBlock::iterator BeforeRestoresI =
157 I == RestoreBlock.begin() ? I : std::prev(x: I);
158
159 // FIXME: Just emit the readlane/writelane directly
160 if (!TFI->restoreCalleeSavedRegisters(MBB&: RestoreBlock, MI: I, CSI, TRI)) {
161 for (const CalleeSavedInfo &CI : reverse(C&: CSI)) {
162 // Insert in reverse order. loadRegFromStackSlot can insert
163 // multiple instructions.
164 TFI->restoreCalleeSavedRegister(MBB&: RestoreBlock, MI: I, CS: CI, TII: &TII, TRI);
165
166 if (Indexes) {
167 MachineInstr &Inst = *std::prev(x: I);
168 Indexes->insertMachineInstrInMaps(MI&: Inst);
169 }
170
171 if (LIS)
172 LIS->removeAllRegUnitsForPhysReg(Reg: CI.getReg());
173 }
174 } else {
175 // TFI doesn't update Indexes and LIS, so we have to do it separately.
176 if (Indexes)
177 Indexes->repairIndexesInRange(MBB: &RestoreBlock, Begin: BeforeRestoresI,
178 End: RestoreBlock.getFirstTerminator());
179
180 if (LIS)
181 for (const CalleeSavedInfo &CS : CSI)
182 LIS->removeAllRegUnitsForPhysReg(Reg: CS.getReg());
183 }
184}
185
186/// Compute the sets of entry and return blocks for saving and restoring
187/// callee-saved registers, and placing prolog and epilog code.
188void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {
189 const MachineFrameInfo &MFI = MF.getFrameInfo();
190
191 // Even when we do not change any CSR, we still want to insert the
192 // prologue and epilogue of the function.
193 // So set the save points for those.
194
195 // Use the points found by shrink-wrapping, if any.
196 if (!MFI.getSavePoints().empty()) {
197 assert(MFI.getSavePoints().size() == 1 &&
198 "Multiple save points not yet supported!");
199 const auto &SavePoint = *MFI.getSavePoints().begin();
200 SaveBlocks.push_back(Elt: SavePoint.first);
201 assert(MFI.getRestorePoints().size() == 1 &&
202 "Multiple restore points not yet supported!");
203 const auto &RestorePoint = *MFI.getRestorePoints().begin();
204 MachineBasicBlock *RestoreBlock = RestorePoint.first;
205 // If RestoreBlock does not have any successor and is not a return block
206 // then the end point is unreachable and we do not need to insert any
207 // epilogue.
208 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
209 RestoreBlocks.push_back(Elt: RestoreBlock);
210 return;
211 }
212
213 // Save refs to entry and return blocks.
214 SaveBlocks.push_back(Elt: &MF.front());
215 for (MachineBasicBlock &MBB : MF) {
216 if (MBB.isEHFuncletEntry())
217 SaveBlocks.push_back(Elt: &MBB);
218 if (MBB.isReturnBlock())
219 RestoreBlocks.push_back(Elt: &MBB);
220 }
221}
222
223// TODO: To support shrink wrapping, this would need to copy
224// PrologEpilogInserter's updateLiveness.
225static void updateLiveness(MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI) {
226 MachineBasicBlock &EntryBB = MF.front();
227
228 for (const CalleeSavedInfo &CSIReg : CSI)
229 EntryBB.addLiveIn(PhysReg: CSIReg.getReg());
230 EntryBB.sortUniqueLiveIns();
231}
232
233bool SILowerSGPRSpills::spillCalleeSavedRegs(
234 MachineFunction &MF, SmallVectorImpl<int> &CalleeSavedFIs) {
235 MachineRegisterInfo &MRI = MF.getRegInfo();
236 const Function &F = MF.getFunction();
237 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
238 const SIFrameLowering *TFI = ST.getFrameLowering();
239 MachineFrameInfo &MFI = MF.getFrameInfo();
240 RegScavenger *RS = nullptr;
241
242 // Determine which of the registers in the callee save list should be saved.
243 BitVector SavedRegs;
244 TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);
245
246 // Add the code to save and restore the callee saved registers.
247 if (!F.hasFnAttribute(Kind: Attribute::Naked)) {
248 // FIXME: This is a lie. The CalleeSavedInfo is incomplete, but this is
249 // necessary for verifier liveness checks.
250 MFI.setCalleeSavedInfoValid(true);
251
252 std::vector<CalleeSavedInfo> CSI;
253 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
254 MCRegister RetAddrReg = TRI->getReturnAddressReg(MF);
255 MCRegister RetAddrRegSub0 = TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub0);
256 MCRegister RetAddrRegSub1 = TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub1);
257 bool SpillRetAddrReg = false;
258
259 for (unsigned I = 0; CSRegs[I]; ++I) {
260 MCRegister Reg = CSRegs[I];
261
262 if (SavedRegs.test(Idx: Reg)) {
263 if (Reg == RetAddrRegSub0 || Reg == RetAddrRegSub1) {
264 SpillRetAddrReg = true;
265 continue;
266 }
267
268 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
269 int JunkFI = MFI.CreateStackObject(Size: TRI->getSpillSize(RC: *RC),
270 Alignment: TRI->getSpillAlign(RC: *RC), isSpillSlot: true,
271 Alloca: nullptr, ID: TRI->getSpillStackID(RC: *RC));
272
273 CSI.emplace_back(args&: Reg, args&: JunkFI);
274 CalleeSavedFIs.push_back(Elt: JunkFI);
275 }
276 }
277
278 // Return address uses a register pair. Add the super register to the
279 // CSI list so that it's easier to identify the entire spill and CFI
280 // can be emitted appropriately.
281 if (SpillRetAddrReg) {
282 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: RetAddrReg);
283 int JunkFI =
284 MFI.CreateStackObject(Size: TRI->getSpillSize(RC: *RC), Alignment: TRI->getSpillAlign(RC: *RC),
285 isSpillSlot: true, Alloca: nullptr, ID: TRI->getSpillStackID(RC: *RC));
286 CSI.push_back(x: CalleeSavedInfo(RetAddrReg, JunkFI));
287 CalleeSavedFIs.push_back(Elt: JunkFI);
288 }
289
290 if (!CSI.empty()) {
291 for (MachineBasicBlock *SaveBlock : SaveBlocks)
292 insertCSRSaves(ST, SaveBlock&: *SaveBlock, CSI, Indexes, LIS);
293
294 // Add live ins to save blocks.
295 assert(SaveBlocks.size() == 1 && "shrink wrapping not fully implemented");
296 updateLiveness(MF, CSI);
297
298 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
299 insertCSRRestores(RestoreBlock&: *RestoreBlock, CSI, Indexes, LIS);
300 return true;
301 }
302 }
303
304 return false;
305}
306
307MachineBasicBlock *SILowerSGPRSpills::getCycleDomBB(CycleRef C) {
308 // If the insertion point lands on a cycle entry, move it to a block that
309 // dominates all entries.
310 if (MCI->isReducible(C)) {
311 if (auto *IDom = MDT->getNode(BB: MCI->getHeader(C))->getIDom())
312 return IDom->getBlock();
313 llvm_unreachable("Expected cycle to have an IDom.");
314 return nullptr;
315 }
316
317 ArrayRef<MachineBasicBlock *> Entries = MCI->getEntries(C);
318 assert(!Entries.empty() && "Expected cycle to have at least one entry.");
319 MachineBasicBlock *EntryBB = Entries[0];
320 for (unsigned I = 1; I < Entries.size(); ++I)
321 EntryBB = MDT->findNearestCommonDominator(A: EntryBB, B: Entries[I]);
322 return EntryBB;
323}
324
325void SILowerSGPRSpills::updateLaneVGPRDomInstr(
326 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
327 DenseMap<Register, LaneVGPRInsertPt> &LaneVGPRDomInstr) {
328 // For the Def of a virtual LaneVGPR to dominate all its uses, we should
329 // insert an IMPLICIT_DEF before the dominating spill. Switching to a
330 // depth first order doesn't really help since the machine function can be in
331 // the unstructured control flow post-SSA. For each virtual register, hence
332 // finding the common dominator to get either the dominating spill or a block
333 // dominating all spills.
334 SIMachineFunctionInfo *FuncInfo =
335 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
336 ArrayRef<SIRegisterInfo::SpilledReg> VGPRSpills =
337 FuncInfo->getSGPRSpillToVirtualVGPRLanes(FrameIndex: FI);
338 Register PrevLaneVGPR;
339 for (auto &Spill : VGPRSpills) {
340 if (PrevLaneVGPR == Spill.VGPR)
341 continue;
342
343 PrevLaneVGPR = Spill.VGPR;
344 auto I = LaneVGPRDomInstr.find(Val: Spill.VGPR);
345 if (Spill.Lane == 0 && I == LaneVGPRDomInstr.end()) {
346 LaneVGPRDomInstr[Spill.VGPR] = insertPt(MBB, It: InsertPt);
347 } else {
348 assert(I != LaneVGPRDomInstr.end());
349 LaneVGPRInsertPt Prev = I->second;
350 MachineBasicBlock *PrevInsertMBB = Prev.MBB;
351 MachineBasicBlock::iterator PrevInsertPt = Prev.It;
352 MachineBasicBlock *DomMBB = PrevInsertMBB;
353 if (DomMBB == MBB) {
354 // The insertion point earlier selected in a predecessor block whose
355 // spills are currently being lowered. The earlier InsertPt would be
356 // the one just before the block terminator and it should be changed
357 // if we insert any new spill in it.
358 if (PrevInsertPt == MBB->end() ||
359 MDT->dominates(A: &*InsertPt, B: &*PrevInsertPt))
360 I->second = insertPt(MBB, It: InsertPt);
361
362 continue;
363 }
364
365 // Find the common dominator block between PrevInsertPt and the
366 // current spill.
367 DomMBB = MDT->findNearestCommonDominator(A: DomMBB, B: MBB);
368
369 if (DomMBB == MBB)
370 I->second = insertPt(MBB, It: InsertPt);
371 else if (DomMBB != PrevInsertMBB)
372 I->second = insertPt(MBB: DomMBB, It: DomMBB->getFirstTerminator());
373 }
374 }
375}
376
377SmallVector<MCRegister>
378SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF) {
379 SmallVector<MCRegister> WWMRegCandidates;
380 if (!MaxNumVGPRsForWwmAllocation)
381 return WWMRegCandidates;
382
383 MachineRegisterInfo &MRI = MF.getRegInfo();
384 BitVector ReservedRegs = TRI->getReservedRegs(MF);
385 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
386 unsigned MaxNumVGPRs = ST.getMaxNumVectorRegs(F: MF.getFunction()).first;
387
388 // Try to use the highest available registers for now. Later after
389 // vgpr-regalloc, they can be shifted to the lowest range.
390 for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;
391 WWMRegCandidates.size() < MaxNumVGPRsForWwmAllocation &&
392 Reg >= AMDGPU::VGPR0;
393 --Reg) {
394 if (!ReservedRegs.test(Idx: Reg) &&
395 !MRI.isPhysRegUsed(PhysReg: Reg, /*SkipRegMaskTest=*/true))
396 WWMRegCandidates.push_back(Elt: Reg);
397 }
398
399 return WWMRegCandidates;
400}
401
402void SILowerSGPRSpills::assignWWMRegs(MachineFunction &MF,
403 ArrayRef<MCRegister> WWMRegCandidates,
404 bool RequiresFullWWMPool) {
405 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
406 if (FuncInfo->getSGPRSpillVGPRs().empty())
407 return;
408
409 BitVector WwmRegMask(TRI->getNumRegs());
410
411 unsigned DesiredPoolSize =
412 std::min(a: static_cast<unsigned>(FuncInfo->getSGPRSpillVGPRs().size()),
413 b: static_cast<unsigned>(MaxNumVGPRsForWwmAllocation));
414 unsigned SelectedPoolSize =
415 std::min<unsigned>(a: DesiredPoolSize, b: WWMRegCandidates.size());
416 // WWM register candidates are ordered high-to-low, so take the highest
417 // available registers when the desired pool is smaller than the candidate
418 // list.
419 for (MCRegister Reg : WWMRegCandidates.take_front(N: SelectedPoolSize))
420 TRI->markSuperRegs(RegisterSet&: WwmRegMask, Reg);
421
422 if (RequiresFullWWMPool && SelectedPoolSize != DesiredPoolSize) {
423 // Reserve an arbitrary register and report the error.
424 TRI->markSuperRegs(RegisterSet&: WwmRegMask, Reg: AMDGPU::VGPR0);
425 MF.getFunction().getContext().emitError(
426 ErrorStr: "cannot find enough VGPRs for wwm-regalloc");
427 }
428
429 BitVector PerLaneVGPRMask(WwmRegMask);
430 PerLaneVGPRMask.flip().clearBitsNotInMask(Mask: TRI->getAllVGPRRegMask());
431
432 // The complement set will be the registers for per-lane VGPR allocation.
433 FuncInfo->updatePerLaneVGPRMask(RegMask&: PerLaneVGPRMask);
434}
435
436bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
437 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
438 LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
439 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
440 SlotIndexes *Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
441 MachineDominatorTree *MDT =
442 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
443 MachineCycleInfo *MCI =
444 &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo();
445 return SILowerSGPRSpills(LIS, Indexes, MDT, MCI).run(MF);
446}
447
448bool SILowerSGPRSpills::run(MachineFunction &MF) {
449 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
450 TII = ST.getInstrInfo();
451 TRI = &TII->getRegisterInfo();
452
453 assert(SaveBlocks.empty() && RestoreBlocks.empty());
454
455 // First, expose any CSR SGPR spills. This is mostly the same as what PEI
456 // does, but somewhat simpler.
457 calculateSaveRestoreBlocks(MF);
458 SmallVector<int> CalleeSavedFIs;
459 bool HasCSRs = spillCalleeSavedRegs(MF, CalleeSavedFIs);
460
461 MachineFrameInfo &MFI = MF.getFrameInfo();
462 MachineRegisterInfo &MRI = MF.getRegInfo();
463 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
464
465 if (!MFI.hasStackObjects() && !HasCSRs) {
466 SaveBlocks.clear();
467 RestoreBlocks.clear();
468 return false;
469 }
470
471 bool MadeChange = false;
472 bool SpilledToVirtVGPRLanes = false;
473
474 // TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be
475 // handled as SpilledToReg in regular PrologEpilogInserter.
476 const bool HasSGPRSpillToVGPR = TRI->spillSGPRToVGPR() &&
477 (HasCSRs || FuncInfo->hasSpilledSGPRs());
478 if (HasSGPRSpillToVGPR) {
479 // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
480 // are spilled to VGPRs, in which case we can eliminate the stack usage.
481 //
482 // This operates under the assumption that only other SGPR spills are users
483 // of the frame index.
484
485 // To track the spill frame indices handled in this pass.
486 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
487
488 // To track the IMPLICIT_DEF insertion point for the lane vgprs.
489 DenseMap<Register, LaneVGPRInsertPt> LaneVGPRDomInstr;
490
491 // Defer ordinary spills until physical CSR spills have reserved their
492 // lane VGPRs and the WWM allocation pool can be selected.
493 SmallVector<MachineInstr *> OrdinarySGPRSpills;
494 bool HasStrictWWMRegion = false;
495
496 for (MachineBasicBlock &MBB : MF) {
497 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
498 if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||
499 MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) {
500 HasStrictWWMRegion = true;
501 continue;
502 }
503
504 if (!TII->isSGPRSpill(MI))
505 continue;
506
507 if (MI.getOperand(i: 0).isUndef()) {
508 if (Indexes)
509 Indexes->removeMachineInstrFromMaps(MI);
510 MI.eraseFromParent();
511 continue;
512 }
513
514 int FI = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::addr)->getIndex();
515 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
516
517 bool IsCalleeSaveSGPRSpill = llvm::is_contained(Range&: CalleeSavedFIs, Element: FI);
518 if (IsCalleeSaveSGPRSpill) {
519 // Spill callee-saved SGPRs into physical VGPR lanes.
520
521 // TODO: This is to ensure the CFIs are static for efficient frame
522 // unwinding in the debugger. Spilling them into virtual VGPR lanes
523 // involve regalloc to allocate the physical VGPRs and that might
524 // cause intermediate spill/split of such liveranges for successful
525 // allocation. This would result in broken CFI encoding unless the
526 // regalloc aware CFI generation to insert new CFIs along with the
527 // intermediate spills is implemented. There is no such support
528 // currently exist in the LLVM compiler.
529 if (FuncInfo->allocateSGPRSpillToVGPRLane(
530 MF, FI, /*SpillToPhysVGPRLane=*/true)) {
531 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
532 MI, FI, RS: nullptr, Indexes, LIS, SpillToPhysVGPRLane: true);
533 if (!Spilled)
534 llvm_unreachable(
535 "failed to spill SGPR to physical VGPR lane when allocated");
536 }
537 } else
538 OrdinarySGPRSpills.push_back(Elt: &MI);
539 }
540 }
541
542 // Select candidates once, before ordinary lane lowering creates virtual
543 // VGPRs and changes the number of registers desired for the WWM pool.
544 SmallVector<MCRegister> WWMRegCandidates;
545 // These non-spillable WWM users retain the old all-or-nothing pool policy.
546 const bool RequiresFullWWMPool =
547 HasStrictWWMRegion || isPreallocateSGPRSpillVGPRsEnabled(MF);
548 if (!OrdinarySGPRSpills.empty())
549 WWMRegCandidates = determineRegsForWWMAllocation(MF);
550
551 const bool ShouldLowerOrdinarySpillsToVGPRLanes =
552 RequiresFullWWMPool || !WWMRegCandidates.empty();
553 if (!ShouldLowerOrdinarySpillsToVGPRLanes && !OrdinarySGPRSpills.empty())
554 FuncInfo->setNoWWMPoolSGPRSpillFallback();
555
556 if (ShouldLowerOrdinarySpillsToVGPRLanes) {
557 for (MachineInstr *MI : OrdinarySGPRSpills) {
558 int FI = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::addr)->getIndex();
559 if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
560 MachineBasicBlock *MBB = MI->getParent();
561 MachineInstrSpan MIS(MI, MBB);
562 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
563 MI: *MI, FI, RS: nullptr, Indexes, LIS);
564 if (!Spilled)
565 llvm_unreachable(
566 "failed to spill SGPR to virtual VGPR lane when allocated");
567 SpillFIs.set(FI);
568 updateLaneVGPRDomInstr(FI, MBB, InsertPt: MIS.begin(), LaneVGPRDomInstr);
569 SpilledToVirtVGPRLanes = true;
570 }
571 }
572 }
573
574 for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
575 LaneVGPRInsertPt IP = LaneVGPRDomInstr[Reg];
576 if (CycleRef C = MCI->getTopLevelParentCycle(Block: IP.MBB)) {
577 MachineBasicBlock *AdjMBB = getCycleDomBB(C);
578 IP = insertPt(MBB: AdjMBB, It: AdjMBB->getFirstTerminator());
579 }
580 // Insert the IMPLICIT_DEF at the identified points.
581 MachineBasicBlock &Block = *IP.MBB;
582 DebugLoc DL = Block.findDebugLoc(MBBI: IP.It);
583 auto MIB = BuildMI(BB&: Block, I: IP.It, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Reg);
584
585 // Add WWM flag to the virtual register.
586 FuncInfo->setFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
587
588 // Set SGPR_SPILL asm printer flag
589 MIB->setAsmPrinterFlag(AMDGPU::SGPR_SPILL);
590 if (LIS) {
591 LIS->InsertMachineInstrInMaps(MI&: *MIB);
592 LIS->createAndComputeVirtRegInterval(Reg);
593 }
594 }
595
596 // Assign the WWM pool from the pre-selected candidates and compute the
597 // complement mask for per-thread VGPR allocation.
598 assignWWMRegs(MF, WWMRegCandidates, RequiresFullWWMPool);
599
600 for (MachineBasicBlock &MBB : MF)
601 clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
602
603 // All those frame indices which are dead by now should be removed from the
604 // function frame. Otherwise, there is a side effect such as re-mapping of
605 // free frame index ids by the later pass(es) like "stack slot coloring"
606 // which in turn could mess-up with the book keeping of "frame index to VGPR
607 // lane".
608 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ false);
609
610 MadeChange = true;
611 }
612
613 if (SpilledToVirtVGPRLanes) {
614 const TargetRegisterClass *RC = TRI->getWaveMaskRegClass();
615 // Shift back the reserved SGPR for EXEC copy into the lowest range.
616 // This SGPR is reserved to handle the whole-wave spill/copy operations
617 // that might get inserted during vgpr regalloc.
618 Register UnusedLowSGPR = TRI->findUnusedRegister(MRI, RC, MF);
619 if (UnusedLowSGPR && TRI->getHWRegIndex(Reg: UnusedLowSGPR) <
620 TRI->getHWRegIndex(Reg: FuncInfo->getSGPRForEXECCopy()))
621 FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
622 } else {
623 // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
624 // spills/copies. Reset the SGPR reserved for EXEC copy.
625 FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
626 }
627
628 SaveBlocks.clear();
629 RestoreBlocks.clear();
630
631 return MadeChange;
632}
633
634PreservedAnalyses
635SILowerSGPRSpillsPass::run(MachineFunction &MF,
636 MachineFunctionAnalysisManager &MFAM) {
637 MFPropsModifier _(*this, MF);
638 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(IR&: MF);
639 auto *Indexes = MFAM.getCachedResult<SlotIndexesAnalysis>(IR&: MF);
640 MachineDominatorTree *MDT = &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
641 MachineCycleInfo &MCI = MFAM.getResult<MachineCycleAnalysis>(IR&: MF);
642 SILowerSGPRSpills(LIS, Indexes, MDT, &MCI).run(MF);
643 return PreservedAnalyses::all();
644}
645