1//===-- SILowerSGPRSPills.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Handle SGPR spills. This pass takes the place of PrologEpilogInserter for all
10// SGPR spills, so must insert CSR SGPR spills as well as expand them.
11//
12// This pass must never create new SGPR virtual registers.
13//
14// FIXME: Must stop RegScavenger spills in later passes.
15//
16//===----------------------------------------------------------------------===//
17
18#include "SILowerSGPRSpills.h"
19#include "AMDGPU.h"
20#include "GCNSubtarget.h"
21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22#include "SIMachineFunctionInfo.h"
23#include "SIPreAllocateWWMRegs.h"
24#include "SISpillUtils.h"
25#include "llvm/CodeGen/LiveIntervals.h"
26#include "llvm/CodeGen/MachineCycleAnalysis.h"
27#include "llvm/CodeGen/MachineDominators.h"
28#include "llvm/CodeGen/MachineFrameInfo.h"
29#include "llvm/CodeGen/RegisterScavenging.h"
30#include "llvm/InitializePasses.h"
31
32using namespace llvm;
33
34#define DEBUG_TYPE "si-lower-sgpr-spills"
35
36using MBBVector = SmallVector<MachineBasicBlock *, 4>;
37
38namespace {
39
40/// Insertion point for IMPLICIT_DEF: iterator may be MBB::end() and can't be
41/// dereferenced so the parent block is stored explicitly.
42struct LaneVGPRInsertPt {
43 MachineBasicBlock *MBB;
44 MachineBasicBlock::iterator It;
45};
46
47static LaneVGPRInsertPt insertPt(MachineBasicBlock *MBB,
48 MachineBasicBlock::iterator It) {
49 return {.MBB: MBB, .It: It};
50}
51
52static cl::opt<unsigned> MaxNumVGPRsForWwmAllocation(
53 "amdgpu-num-vgprs-for-wwm-alloc",
54 cl::desc("Max num VGPRs for whole-wave register allocation."),
55 cl::ReallyHidden, cl::init(Val: 10));
56
57class SILowerSGPRSpills {
58private:
59 const SIRegisterInfo *TRI = nullptr;
60 const SIInstrInfo *TII = nullptr;
61 LiveIntervals *LIS = nullptr;
62 SlotIndexes *Indexes = nullptr;
63 MachineDominatorTree *MDT = nullptr;
64 MachineCycleInfo *MCI = nullptr;
65
66 // Save and Restore blocks of the current function. Typically there is a
67 // single save block, unless Windows EH funclets are involved.
68 MBBVector SaveBlocks;
69 MBBVector RestoreBlocks;
70
71 MachineBasicBlock *getCycleDomBB(CycleRef C);
72
73public:
74 SILowerSGPRSpills(LiveIntervals *LIS, SlotIndexes *Indexes,
75 MachineDominatorTree *MDT, MachineCycleInfo *MCI)
76 : LIS(LIS), Indexes(Indexes), MDT(MDT), MCI(MCI) {}
77 bool run(MachineFunction &MF);
78 void calculateSaveRestoreBlocks(MachineFunction &MF);
79 bool spillCalleeSavedRegs(MachineFunction &MF,
80 SmallVectorImpl<int> &CalleeSavedFIs);
81 void updateLaneVGPRDomInstr(
82 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
83 DenseMap<Register, LaneVGPRInsertPt> &LaneVGPRDomInstr);
84 SmallVector<MCRegister> determineRegsForWWMAllocation(MachineFunction &MF);
85 void assignWWMRegs(MachineFunction &MF, ArrayRef<MCRegister> WWMRegCandidates,
86 bool RequiresFullWWMPool);
87};
88
89class SILowerSGPRSpillsLegacy : public MachineFunctionPass {
90public:
91 static char ID;
92
93 SILowerSGPRSpillsLegacy() : MachineFunctionPass(ID) {}
94
95 bool runOnMachineFunction(MachineFunction &MF) override;
96
97 void getAnalysisUsage(AnalysisUsage &AU) const override {
98 AU.addRequired<MachineDominatorTreeWrapperPass>();
99 AU.addRequired<MachineCycleInfoWrapperPass>();
100 AU.setPreservesAll();
101 MachineFunctionPass::getAnalysisUsage(AU);
102 }
103
104 MachineFunctionProperties getClearedProperties() const override {
105 // SILowerSGPRSpills introduces new Virtual VGPRs for spilling SGPRs.
106 return MachineFunctionProperties().setIsSSA().setNoVRegs();
107 }
108};
109
110} // end anonymous namespace
111
112char SILowerSGPRSpillsLegacy::ID = 0;
113
114INITIALIZE_PASS_BEGIN(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
115 "SI lower SGPR spill instructions", false, false)
116INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
117INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
118INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
119INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass)
120INITIALIZE_PASS_END(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
121 "SI lower SGPR spill instructions", false, false)
122
123char &llvm::SILowerSGPRSpillsLegacyID = SILowerSGPRSpillsLegacy::ID;
124
125/// Insert spill code for the callee-saved registers used in the function.
126static void insertCSRSaves(const GCNSubtarget &ST, MachineBasicBlock &SaveBlock,
127 ArrayRef<CalleeSavedInfo> CSI, SlotIndexes *Indexes,
128 LiveIntervals *LIS) {
129 const TargetFrameLowering *TFI = ST.getFrameLowering();
130 const TargetRegisterInfo *TRI = ST.getRegisterInfo();
131 MachineBasicBlock::iterator I = SaveBlock.begin();
132 MachineInstrSpan MIS(I, &SaveBlock);
133 bool Success = TFI->spillCalleeSavedRegisters(MBB&: SaveBlock, MI: I, CSI, TRI);
134 assert(Success && "spillCalleeSavedRegisters should always succeed");
135 (void)Success;
136
137 // TFI doesn't update Indexes and LIS, so we have to do it separately.
138 if (Indexes)
139 Indexes->repairIndexesInRange(MBB: &SaveBlock, Begin: SaveBlock.begin(), End: I);
140
141 if (LIS)
142 for (const CalleeSavedInfo &CS : CSI)
143 LIS->removeAllRegUnitsForPhysReg(Reg: CS.getReg());
144}
145
146/// Insert restore code for the callee-saved registers used in the function.
147static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
148 MutableArrayRef<CalleeSavedInfo> CSI,
149 SlotIndexes *Indexes, LiveIntervals *LIS) {
150 MachineFunction &MF = *RestoreBlock.getParent();
151 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
152 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
153 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
154 // Restore all registers immediately before the return and any
155 // terminators that precede it.
156 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();
157 const MachineBasicBlock::iterator BeforeRestoresI =
158 I == RestoreBlock.begin() ? I : std::prev(x: I);
159
160 // FIXME: Just emit the readlane/writelane directly
161 if (!TFI->restoreCalleeSavedRegisters(MBB&: RestoreBlock, MI: I, CSI, TRI)) {
162 for (const CalleeSavedInfo &CI : reverse(C&: CSI)) {
163 // Insert in reverse order. loadRegFromStackSlot can insert
164 // multiple instructions.
165 TFI->restoreCalleeSavedRegister(MBB&: RestoreBlock, MI: I, CS: CI, TII: &TII, TRI);
166
167 if (Indexes) {
168 MachineInstr &Inst = *std::prev(x: I);
169 Indexes->insertMachineInstrInMaps(MI&: Inst);
170 }
171
172 if (LIS)
173 LIS->removeAllRegUnitsForPhysReg(Reg: CI.getReg());
174 }
175 } else {
176 // TFI doesn't update Indexes and LIS, so we have to do it separately.
177 if (Indexes)
178 Indexes->repairIndexesInRange(MBB: &RestoreBlock, Begin: BeforeRestoresI,
179 End: RestoreBlock.getFirstTerminator());
180
181 if (LIS)
182 for (const CalleeSavedInfo &CS : CSI)
183 LIS->removeAllRegUnitsForPhysReg(Reg: CS.getReg());
184 }
185}
186
187/// Compute the sets of entry and return blocks for saving and restoring
188/// callee-saved registers, and placing prolog and epilog code.
189void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {
190 const MachineFrameInfo &MFI = MF.getFrameInfo();
191
192 // Even when we do not change any CSR, we still want to insert the
193 // prologue and epilogue of the function.
194 // So set the save points for those.
195
196 // Use the points found by shrink-wrapping, if any.
197 if (!MFI.getSavePoints().empty()) {
198 assert(MFI.getSavePoints().size() == 1 &&
199 "Multiple save points not yet supported!");
200 const auto &SavePoint = *MFI.getSavePoints().begin();
201 SaveBlocks.push_back(Elt: SavePoint.first);
202 assert(MFI.getRestorePoints().size() == 1 &&
203 "Multiple restore points not yet supported!");
204 const auto &RestorePoint = *MFI.getRestorePoints().begin();
205 MachineBasicBlock *RestoreBlock = RestorePoint.first;
206 // If RestoreBlock does not have any successor and is not a return block
207 // then the end point is unreachable and we do not need to insert any
208 // epilogue.
209 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
210 RestoreBlocks.push_back(Elt: RestoreBlock);
211 return;
212 }
213
214 // Save refs to entry and return blocks.
215 SaveBlocks.push_back(Elt: &MF.front());
216 for (MachineBasicBlock &MBB : MF) {
217 if (MBB.isEHFuncletEntry())
218 SaveBlocks.push_back(Elt: &MBB);
219 if (MBB.isReturnBlock())
220 RestoreBlocks.push_back(Elt: &MBB);
221 }
222}
223
224// TODO: To support shrink wrapping, this would need to copy
225// PrologEpilogInserter's updateLiveness.
226static void updateLiveness(MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI) {
227 MachineBasicBlock &EntryBB = MF.front();
228
229 for (const CalleeSavedInfo &CSIReg : CSI)
230 EntryBB.addLiveIn(PhysReg: CSIReg.getReg());
231 EntryBB.sortUniqueLiveIns();
232}
233
234bool SILowerSGPRSpills::spillCalleeSavedRegs(
235 MachineFunction &MF, SmallVectorImpl<int> &CalleeSavedFIs) {
236 MachineRegisterInfo &MRI = MF.getRegInfo();
237 const Function &F = MF.getFunction();
238 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
239 const SIFrameLowering *TFI = ST.getFrameLowering();
240 MachineFrameInfo &MFI = MF.getFrameInfo();
241 RegScavenger *RS = nullptr;
242
243 // Determine which of the registers in the callee save list should be saved.
244 BitVector SavedRegs;
245 TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);
246
247 // Add the code to save and restore the callee saved registers.
248 if (!F.hasFnAttribute(Kind: Attribute::Naked)) {
249 // FIXME: This is a lie. The CalleeSavedInfo is incomplete, but this is
250 // necessary for verifier liveness checks.
251 MFI.setCalleeSavedInfoValid(true);
252
253 std::vector<CalleeSavedInfo> CSI;
254 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
255 MCRegister RetAddrReg = TRI->getReturnAddressReg(MF);
256 MCRegister RetAddrRegSub0 = TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub0);
257 MCRegister RetAddrRegSub1 = TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub1);
258 bool SpillRetAddrReg = false;
259
260 for (unsigned I = 0; CSRegs[I]; ++I) {
261 MCRegister Reg = CSRegs[I];
262
263 if (SavedRegs.test(Idx: Reg)) {
264 if (Reg == RetAddrRegSub0 || Reg == RetAddrRegSub1) {
265 SpillRetAddrReg = true;
266 continue;
267 }
268
269 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
270 int JunkFI = MFI.CreateStackObject(Size: TRI->getSpillSize(RC: *RC),
271 Alignment: TRI->getSpillAlign(RC: *RC), isSpillSlot: true,
272 Alloca: nullptr, ID: TRI->getSpillStackID(RC: *RC));
273
274 CSI.emplace_back(args&: Reg, args&: JunkFI);
275 CalleeSavedFIs.push_back(Elt: JunkFI);
276 }
277 }
278
279 // Return address uses a register pair. Add the super register to the
280 // CSI list so that it's easier to identify the entire spill and CFI
281 // can be emitted appropriately.
282 if (SpillRetAddrReg) {
283 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: RetAddrReg);
284 int JunkFI =
285 MFI.CreateStackObject(Size: TRI->getSpillSize(RC: *RC), Alignment: TRI->getSpillAlign(RC: *RC),
286 isSpillSlot: true, Alloca: nullptr, ID: TRI->getSpillStackID(RC: *RC));
287 CSI.push_back(x: CalleeSavedInfo(RetAddrReg, JunkFI));
288 CalleeSavedFIs.push_back(Elt: JunkFI);
289 }
290
291 if (!CSI.empty()) {
292 for (MachineBasicBlock *SaveBlock : SaveBlocks)
293 insertCSRSaves(ST, SaveBlock&: *SaveBlock, CSI, Indexes, LIS);
294
295 // Add live ins to save blocks.
296 assert(SaveBlocks.size() == 1 && "shrink wrapping not fully implemented");
297 updateLiveness(MF, CSI);
298
299 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
300 insertCSRRestores(RestoreBlock&: *RestoreBlock, CSI, Indexes, LIS);
301 return true;
302 }
303 }
304
305 return false;
306}
307
308MachineBasicBlock *SILowerSGPRSpills::getCycleDomBB(CycleRef C) {
309 // If the insertion point lands on a cycle entry, move it to a block that
310 // dominates all entries.
311 if (MCI->isReducible(C)) {
312 if (auto *IDom = MDT->getNode(BB: MCI->getHeader(C))->getIDom())
313 return IDom->getBlock();
314 llvm_unreachable("Expected cycle to have an IDom.");
315 return nullptr;
316 }
317
318 ArrayRef<MachineBasicBlock *> Entries = MCI->getEntries(C);
319 assert(!Entries.empty() && "Expected cycle to have at least one entry.");
320 MachineBasicBlock *EntryBB = Entries[0];
321 for (unsigned I = 1; I < Entries.size(); ++I)
322 EntryBB = MDT->findNearestCommonDominator(A: EntryBB, B: Entries[I]);
323 return EntryBB;
324}
325
326void SILowerSGPRSpills::updateLaneVGPRDomInstr(
327 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
328 DenseMap<Register, LaneVGPRInsertPt> &LaneVGPRDomInstr) {
329 // For the Def of a virtual LaneVGPR to dominate all its uses, we should
330 // insert an IMPLICIT_DEF before the dominating spill. Switching to a
331 // depth first order doesn't really help since the machine function can be in
332 // the unstructured control flow post-SSA. For each virtual register, hence
333 // finding the common dominator to get either the dominating spill or a block
334 // dominating all spills.
335 SIMachineFunctionInfo *FuncInfo =
336 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
337 ArrayRef<SIRegisterInfo::SpilledReg> VGPRSpills =
338 FuncInfo->getSGPRSpillToVirtualVGPRLanes(FrameIndex: FI);
339 Register PrevLaneVGPR;
340 for (auto &Spill : VGPRSpills) {
341 if (PrevLaneVGPR == Spill.VGPR)
342 continue;
343
344 PrevLaneVGPR = Spill.VGPR;
345 auto I = LaneVGPRDomInstr.find(Val: Spill.VGPR);
346 if (Spill.Lane == 0 && I == LaneVGPRDomInstr.end()) {
347 LaneVGPRDomInstr[Spill.VGPR] = insertPt(MBB, It: InsertPt);
348 } else {
349 assert(I != LaneVGPRDomInstr.end());
350 LaneVGPRInsertPt Prev = I->second;
351 MachineBasicBlock *PrevInsertMBB = Prev.MBB;
352 MachineBasicBlock::iterator PrevInsertPt = Prev.It;
353 MachineBasicBlock *DomMBB = PrevInsertMBB;
354 if (DomMBB == MBB) {
355 // The insertion point earlier selected in a predecessor block whose
356 // spills are currently being lowered. The earlier InsertPt would be
357 // the one just before the block terminator and it should be changed
358 // if we insert any new spill in it.
359 if (PrevInsertPt == MBB->end() ||
360 MDT->dominates(A: &*InsertPt, B: &*PrevInsertPt))
361 I->second = insertPt(MBB, It: InsertPt);
362
363 continue;
364 }
365
366 // Find the common dominator block between PrevInsertPt and the
367 // current spill.
368 DomMBB = MDT->findNearestCommonDominator(A: DomMBB, B: MBB);
369
370 if (DomMBB == MBB)
371 I->second = insertPt(MBB, It: InsertPt);
372 else if (DomMBB != PrevInsertMBB)
373 I->second = insertPt(MBB: DomMBB, It: DomMBB->getFirstTerminator());
374 }
375 }
376}
377
378SmallVector<MCRegister>
379SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF) {
380 SmallVector<MCRegister> WWMRegCandidates;
381 if (!MaxNumVGPRsForWwmAllocation)
382 return WWMRegCandidates;
383
384 MachineRegisterInfo &MRI = MF.getRegInfo();
385 BitVector ReservedRegs = TRI->getReservedRegs(MF);
386 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
387 unsigned MaxNumVGPRs = ST.getMaxNumVectorRegs(F: MF.getFunction()).first;
388
389 // Try to use the highest available registers for now. Later after
390 // vgpr-regalloc, they can be shifted to the lowest range.
391 for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;
392 WWMRegCandidates.size() < MaxNumVGPRsForWwmAllocation &&
393 Reg >= AMDGPU::VGPR0;
394 --Reg) {
395 if (!ReservedRegs.test(Idx: Reg) &&
396 !MRI.isPhysRegUsed(PhysReg: Reg, /*SkipRegMaskTest=*/true))
397 WWMRegCandidates.push_back(Elt: Reg);
398 }
399
400 return WWMRegCandidates;
401}
402
403void SILowerSGPRSpills::assignWWMRegs(MachineFunction &MF,
404 ArrayRef<MCRegister> WWMRegCandidates,
405 bool RequiresFullWWMPool) {
406 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
407 if (FuncInfo->getSGPRSpillVGPRs().empty())
408 return;
409
410 BitVector WwmRegMask(TRI->getNumRegs());
411
412 unsigned DesiredPoolSize =
413 std::min(a: static_cast<unsigned>(FuncInfo->getSGPRSpillVGPRs().size()),
414 b: static_cast<unsigned>(MaxNumVGPRsForWwmAllocation));
415 unsigned SelectedPoolSize =
416 std::min<unsigned>(a: DesiredPoolSize, b: WWMRegCandidates.size());
417 // WWM register candidates are ordered high-to-low, so take the highest
418 // available registers when the desired pool is smaller than the candidate
419 // list.
420 for (MCRegister Reg : WWMRegCandidates.take_front(N: SelectedPoolSize))
421 TRI->markSuperRegs(RegisterSet&: WwmRegMask, Reg);
422
423 if (RequiresFullWWMPool && SelectedPoolSize != DesiredPoolSize) {
424 // Reserve an arbitrary register and report the error.
425 TRI->markSuperRegs(RegisterSet&: WwmRegMask, Reg: AMDGPU::VGPR0);
426 MF.getFunction().getContext().emitError(
427 ErrorStr: "cannot find enough VGPRs for wwm-regalloc");
428 }
429
430 BitVector PerLaneVGPRMask(WwmRegMask);
431 PerLaneVGPRMask.flip().clearBitsNotInMask(Mask: TRI->getAllVGPRRegMask());
432
433 // The complement set will be the registers for per-lane VGPR allocation.
434 FuncInfo->updatePerLaneVGPRMask(RegMask&: PerLaneVGPRMask);
435}
436
437bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
438 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
439 LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
440 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
441 SlotIndexes *Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
442 MachineDominatorTree *MDT =
443 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
444 MachineCycleInfo *MCI =
445 &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo();
446 return SILowerSGPRSpills(LIS, Indexes, MDT, MCI).run(MF);
447}
448
449bool SILowerSGPRSpills::run(MachineFunction &MF) {
450 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
451 TII = ST.getInstrInfo();
452 TRI = &TII->getRegisterInfo();
453
454 assert(SaveBlocks.empty() && RestoreBlocks.empty());
455
456 // First, expose any CSR SGPR spills. This is mostly the same as what PEI
457 // does, but somewhat simpler.
458 calculateSaveRestoreBlocks(MF);
459 SmallVector<int> CalleeSavedFIs;
460 bool HasCSRs = spillCalleeSavedRegs(MF, CalleeSavedFIs);
461
462 MachineFrameInfo &MFI = MF.getFrameInfo();
463 MachineRegisterInfo &MRI = MF.getRegInfo();
464 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
465
466 if (!MFI.hasStackObjects() && !HasCSRs) {
467 SaveBlocks.clear();
468 RestoreBlocks.clear();
469 return false;
470 }
471
472 bool MadeChange = false;
473 bool SpilledToVirtVGPRLanes = false;
474
475 // TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be
476 // handled as SpilledToReg in regular PrologEpilogInserter.
477 const bool HasSGPRSpillToVGPR = TRI->spillSGPRToVGPR() &&
478 (HasCSRs || FuncInfo->hasSpilledSGPRs());
479 if (HasSGPRSpillToVGPR) {
480 // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
481 // are spilled to VGPRs, in which case we can eliminate the stack usage.
482 //
483 // This operates under the assumption that only other SGPR spills are users
484 // of the frame index.
485
486 // To track the spill frame indices handled in this pass.
487 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
488
489 // To track the IMPLICIT_DEF insertion point for the lane vgprs.
490 DenseMap<Register, LaneVGPRInsertPt> LaneVGPRDomInstr;
491
492 // Defer ordinary spills until physical CSR spills have reserved their
493 // lane VGPRs and the WWM allocation pool can be selected.
494 SmallVector<MachineInstr *> OrdinarySGPRSpills;
495 bool HasStrictWWMRegion = false;
496
497 for (MachineBasicBlock &MBB : MF) {
498 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
499 if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||
500 MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) {
501 HasStrictWWMRegion = true;
502 continue;
503 }
504
505 if (!TII->isSGPRSpill(MI))
506 continue;
507
508 if (MI.getOperand(i: 0).isUndef()) {
509 if (Indexes)
510 Indexes->removeMachineInstrFromMaps(MI);
511 MI.eraseFromParent();
512 continue;
513 }
514
515 int FI = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::addr)->getIndex();
516 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
517
518 bool IsCalleeSaveSGPRSpill = llvm::is_contained(Range&: CalleeSavedFIs, Element: FI);
519 if (IsCalleeSaveSGPRSpill) {
520 // Spill callee-saved SGPRs into physical VGPR lanes.
521
522 // TODO: This is to ensure the CFIs are static for efficient frame
523 // unwinding in the debugger. Spilling them into virtual VGPR lanes
524 // involve regalloc to allocate the physical VGPRs and that might
525 // cause intermediate spill/split of such liveranges for successful
526 // allocation. This would result in broken CFI encoding unless the
527 // regalloc aware CFI generation to insert new CFIs along with the
528 // intermediate spills is implemented. There is no such support
529 // currently exist in the LLVM compiler.
530 if (FuncInfo->allocateSGPRSpillToVGPRLane(
531 MF, FI, /*SpillToPhysVGPRLane=*/true)) {
532 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
533 MI, FI, RS: nullptr, Indexes, LIS, SpillToPhysVGPRLane: true);
534 if (!Spilled)
535 llvm_unreachable(
536 "failed to spill SGPR to physical VGPR lane when allocated");
537 }
538 } else
539 OrdinarySGPRSpills.push_back(Elt: &MI);
540 }
541 }
542
543 // Select candidates once, before ordinary lane lowering creates virtual
544 // VGPRs and changes the number of registers desired for the WWM pool.
545 SmallVector<MCRegister> WWMRegCandidates;
546 // These non-spillable WWM users retain the old all-or-nothing pool policy.
547 const bool RequiresFullWWMPool =
548 HasStrictWWMRegion || isPreallocateSGPRSpillVGPRsEnabled(MF);
549 if (!OrdinarySGPRSpills.empty())
550 WWMRegCandidates = determineRegsForWWMAllocation(MF);
551
552 const bool ShouldLowerOrdinarySpillsToVGPRLanes =
553 RequiresFullWWMPool || !WWMRegCandidates.empty();
554 if (!ShouldLowerOrdinarySpillsToVGPRLanes && !OrdinarySGPRSpills.empty())
555 FuncInfo->setNoWWMPoolSGPRSpillFallback();
556
557 if (ShouldLowerOrdinarySpillsToVGPRLanes) {
558 for (MachineInstr *MI : OrdinarySGPRSpills) {
559 int FI = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::addr)->getIndex();
560 if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
561 MachineBasicBlock *MBB = MI->getParent();
562 MachineInstrSpan MIS(MI, MBB);
563 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
564 MI: *MI, FI, RS: nullptr, Indexes, LIS);
565 if (!Spilled)
566 llvm_unreachable(
567 "failed to spill SGPR to virtual VGPR lane when allocated");
568 SpillFIs.set(FI);
569 updateLaneVGPRDomInstr(FI, MBB, InsertPt: MIS.begin(), LaneVGPRDomInstr);
570 SpilledToVirtVGPRLanes = true;
571 }
572 }
573 }
574
575 for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
576 LaneVGPRInsertPt IP = LaneVGPRDomInstr[Reg];
577 if (CycleRef C = MCI->getTopLevelParentCycle(Block: IP.MBB)) {
578 MachineBasicBlock *AdjMBB = getCycleDomBB(C);
579 IP = insertPt(MBB: AdjMBB, It: AdjMBB->getFirstTerminator());
580 }
581 // Insert the IMPLICIT_DEF at the identified points.
582 MachineBasicBlock &Block = *IP.MBB;
583 DebugLoc DL = Block.findDebugLoc(MBBI: IP.It);
584 auto MIB = BuildMI(BB&: Block, I: IP.It, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Reg);
585
586 // Add WWM flag to the virtual register.
587 FuncInfo->setFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
588
589 // Set SGPR_SPILL asm printer flag
590 MIB->setAsmPrinterFlag(AMDGPU::SGPR_SPILL);
591 if (LIS) {
592 LIS->InsertMachineInstrInMaps(MI&: *MIB);
593 LIS->createAndComputeVirtRegInterval(Reg);
594 }
595 }
596
597 // Assign the WWM pool from the pre-selected candidates and compute the
598 // complement mask for per-thread VGPR allocation.
599 assignWWMRegs(MF, WWMRegCandidates, RequiresFullWWMPool);
600
601 for (MachineBasicBlock &MBB : MF)
602 clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
603
604 // All those frame indices which are dead by now should be removed from the
605 // function frame. Otherwise, there is a side effect such as re-mapping of
606 // free frame index ids by the later pass(es) like "stack slot coloring"
607 // which in turn could mess-up with the book keeping of "frame index to VGPR
608 // lane".
609 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ false);
610
611 MadeChange = true;
612 }
613
614 if (SpilledToVirtVGPRLanes) {
615 const TargetRegisterClass *RC = TRI->getWaveMaskRegClass();
616 // Shift back the reserved SGPR for EXEC copy into the lowest range.
617 // This SGPR is reserved to handle the whole-wave spill/copy operations
618 // that might get inserted during vgpr regalloc.
619 Register UnusedLowSGPR = TRI->findUnusedRegister(MRI, RC, MF);
620 if (UnusedLowSGPR && TRI->getHWRegIndex(Reg: UnusedLowSGPR) <
621 TRI->getHWRegIndex(Reg: FuncInfo->getSGPRForEXECCopy()))
622 FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
623 } else {
624 // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
625 // spills/copies. Reset the SGPR reserved for EXEC copy.
626 FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
627 }
628
629 SaveBlocks.clear();
630 RestoreBlocks.clear();
631
632 return MadeChange;
633}
634
635PreservedAnalyses
636SILowerSGPRSpillsPass::run(MachineFunction &MF,
637 MachineFunctionAnalysisManager &MFAM) {
638 MFPropsModifier _(*this, MF);
639 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(IR&: MF);
640 auto *Indexes = MFAM.getCachedResult<SlotIndexesAnalysis>(IR&: MF);
641 MachineDominatorTree *MDT = &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
642 MachineCycleInfo &MCI = MFAM.getResult<MachineCycleAnalysis>(IR&: MF);
643 SILowerSGPRSpills(LIS, Indexes, MDT, &MCI).run(MF);
644 return PreservedAnalyses::all();
645}
646