1//===-- SILowerSGPRSPills.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Handle SGPR spills. This pass takes the place of PrologEpilogInserter for all
10// SGPR spills, so must insert CSR SGPR spills as well as expand them.
11//
12// This pass must never create new SGPR virtual registers.
13//
14// FIXME: Must stop RegScavenger spills in later passes.
15//
16//===----------------------------------------------------------------------===//
17
18#include "SILowerSGPRSpills.h"
19#include "AMDGPU.h"
20#include "GCNSubtarget.h"
21#include "SIMachineFunctionInfo.h"
22#include "SIPreAllocateWWMRegs.h"
23#include "SISpillUtils.h"
24#include "llvm/CodeGen/LiveIntervals.h"
25#include "llvm/CodeGen/MachineCycleAnalysis.h"
26#include "llvm/CodeGen/MachineDominators.h"
27#include "llvm/CodeGen/MachineFrameInfo.h"
28#include "llvm/CodeGen/RegisterScavenging.h"
29#include "llvm/InitializePasses.h"
30
31using namespace llvm;
32
33#define DEBUG_TYPE "si-lower-sgpr-spills"
34
35using MBBVector = SmallVector<MachineBasicBlock *, 4>;
36
37namespace {
38
39/// Insertion point for IMPLICIT_DEF: iterator may be MBB::end() and can't be
40/// dereferenced so the parent block is stored explicitly.
41struct LaneVGPRInsertPt {
42 MachineBasicBlock *MBB;
43 MachineBasicBlock::iterator It;
44};
45
46static LaneVGPRInsertPt insertPt(MachineBasicBlock *MBB,
47 MachineBasicBlock::iterator It) {
48 return {.MBB: MBB, .It: It};
49}
50
51class SILowerSGPRSpills {
52private:
53 const SIRegisterInfo *TRI = nullptr;
54 const SIInstrInfo *TII = nullptr;
55 LiveIntervals *LIS = nullptr;
56 SlotIndexes *Indexes = nullptr;
57 MachineDominatorTree *MDT = nullptr;
58 MachineCycleInfo *MCI = nullptr;
59
60 // Save and Restore blocks of the current function. Typically there is a
61 // single save block, unless Windows EH funclets are involved.
62 MBBVector SaveBlocks;
63 MBBVector RestoreBlocks;
64
65 MachineBasicBlock *getCycleDomBB(CycleRef C);
66
67public:
68 SILowerSGPRSpills(LiveIntervals *LIS, SlotIndexes *Indexes,
69 MachineDominatorTree *MDT, MachineCycleInfo *MCI)
70 : LIS(LIS), Indexes(Indexes), MDT(MDT), MCI(MCI) {}
71 bool run(MachineFunction &MF);
72 void calculateSaveRestoreBlocks(MachineFunction &MF);
73 bool spillCalleeSavedRegs(MachineFunction &MF,
74 SmallVectorImpl<int> &CalleeSavedFIs);
75 void updateLaneVGPRDomInstr(
76 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
77 DenseMap<Register, LaneVGPRInsertPt> &LaneVGPRDomInstr);
78 SmallVector<MCRegister> determineRegsForWWMAllocation(MachineFunction &MF);
79 void assignWWMRegs(MachineFunction &MF, ArrayRef<MCRegister> WWMRegCandidates,
80 bool RequiresFullWWMPool);
81};
82
83class SILowerSGPRSpillsLegacy : public MachineFunctionPass {
84public:
85 static char ID;
86
87 SILowerSGPRSpillsLegacy() : MachineFunctionPass(ID) {}
88
89 bool runOnMachineFunction(MachineFunction &MF) override;
90
91 void getAnalysisUsage(AnalysisUsage &AU) const override {
92 AU.addRequired<MachineDominatorTreeWrapperPass>();
93 AU.addRequired<MachineCycleInfoWrapperPass>();
94 AU.setPreservesAll();
95 MachineFunctionPass::getAnalysisUsage(AU);
96 }
97
98 MachineFunctionProperties getClearedProperties() const override {
99 // SILowerSGPRSpills introduces new Virtual VGPRs for spilling SGPRs.
100 return MachineFunctionProperties().setIsSSA().setNoVRegs();
101 }
102};
103
104} // end anonymous namespace
105
106char SILowerSGPRSpillsLegacy::ID = 0;
107
108INITIALIZE_PASS_BEGIN(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
109 "SI lower SGPR spill instructions", false, false)
110INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
111INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
112INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
113INITIALIZE_PASS_DEPENDENCY(MachineCycleInfoWrapperPass)
114INITIALIZE_PASS_END(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
115 "SI lower SGPR spill instructions", false, false)
116
117char &llvm::SILowerSGPRSpillsLegacyID = SILowerSGPRSpillsLegacy::ID;
118
119/// Insert spill code for the callee-saved registers used in the function.
120static void insertCSRSaves(const GCNSubtarget &ST, MachineBasicBlock &SaveBlock,
121 ArrayRef<CalleeSavedInfo> CSI, SlotIndexes *Indexes,
122 LiveIntervals *LIS) {
123 const TargetFrameLowering *TFI = ST.getFrameLowering();
124 const TargetRegisterInfo *TRI = ST.getRegisterInfo();
125 MachineBasicBlock::iterator I = SaveBlock.begin();
126 MachineInstrSpan MIS(I, &SaveBlock);
127 bool Success = TFI->spillCalleeSavedRegisters(MBB&: SaveBlock, MI: I, CSI, TRI);
128 assert(Success && "spillCalleeSavedRegisters should always succeed");
129 (void)Success;
130
131 // TFI doesn't update Indexes and LIS, so we have to do it separately.
132 if (Indexes)
133 Indexes->repairIndexesInRange(MBB: &SaveBlock, Begin: SaveBlock.begin(), End: I);
134
135 if (LIS)
136 for (const CalleeSavedInfo &CS : CSI)
137 LIS->removeAllRegUnitsForPhysReg(Reg: CS.getReg());
138}
139
140/// Insert restore code for the callee-saved registers used in the function.
141static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
142 MutableArrayRef<CalleeSavedInfo> CSI,
143 SlotIndexes *Indexes, LiveIntervals *LIS) {
144 MachineFunction &MF = *RestoreBlock.getParent();
145 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
146 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
147 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
148 // Restore all registers immediately before the return and any
149 // terminators that precede it.
150 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();
151 const MachineBasicBlock::iterator BeforeRestoresI =
152 I == RestoreBlock.begin() ? I : std::prev(x: I);
153
154 // FIXME: Just emit the readlane/writelane directly
155 if (!TFI->restoreCalleeSavedRegisters(MBB&: RestoreBlock, MI: I, CSI, TRI)) {
156 for (const CalleeSavedInfo &CI : reverse(C&: CSI)) {
157 // Insert in reverse order. loadRegFromStackSlot can insert
158 // multiple instructions.
159 TFI->restoreCalleeSavedRegister(MBB&: RestoreBlock, MI: I, CS: CI, TII: &TII, TRI);
160
161 if (Indexes) {
162 MachineInstr &Inst = *std::prev(x: I);
163 Indexes->insertMachineInstrInMaps(MI&: Inst);
164 }
165
166 if (LIS)
167 LIS->removeAllRegUnitsForPhysReg(Reg: CI.getReg());
168 }
169 } else {
170 // TFI doesn't update Indexes and LIS, so we have to do it separately.
171 if (Indexes)
172 Indexes->repairIndexesInRange(MBB: &RestoreBlock, Begin: BeforeRestoresI,
173 End: RestoreBlock.getFirstTerminator());
174
175 if (LIS)
176 for (const CalleeSavedInfo &CS : CSI)
177 LIS->removeAllRegUnitsForPhysReg(Reg: CS.getReg());
178 }
179}
180
181/// Compute the sets of entry and return blocks for saving and restoring
182/// callee-saved registers, and placing prolog and epilog code.
183void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {
184 const MachineFrameInfo &MFI = MF.getFrameInfo();
185
186 // Even when we do not change any CSR, we still want to insert the
187 // prologue and epilogue of the function.
188 // So set the save points for those.
189
190 // Use the points found by shrink-wrapping, if any.
191 if (!MFI.getSavePoints().empty()) {
192 assert(MFI.getSavePoints().size() == 1 &&
193 "Multiple save points not yet supported!");
194 const auto &SavePoint = *MFI.getSavePoints().begin();
195 SaveBlocks.push_back(Elt: SavePoint.first);
196 assert(MFI.getRestorePoints().size() == 1 &&
197 "Multiple restore points not yet supported!");
198 const auto &RestorePoint = *MFI.getRestorePoints().begin();
199 MachineBasicBlock *RestoreBlock = RestorePoint.first;
200 // If RestoreBlock does not have any successor and is not a return block
201 // then the end point is unreachable and we do not need to insert any
202 // epilogue.
203 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
204 RestoreBlocks.push_back(Elt: RestoreBlock);
205 return;
206 }
207
208 // Save refs to entry and return blocks.
209 SaveBlocks.push_back(Elt: &MF.front());
210 for (MachineBasicBlock &MBB : MF) {
211 if (MBB.isEHFuncletEntry())
212 SaveBlocks.push_back(Elt: &MBB);
213 if (MBB.isReturnBlock())
214 RestoreBlocks.push_back(Elt: &MBB);
215 }
216}
217
218// TODO: To support shrink wrapping, this would need to copy
219// PrologEpilogInserter's updateLiveness.
220static void updateLiveness(MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI) {
221 MachineBasicBlock &EntryBB = MF.front();
222
223 for (const CalleeSavedInfo &CSIReg : CSI)
224 EntryBB.addLiveIn(PhysReg: CSIReg.getReg());
225 EntryBB.sortUniqueLiveIns();
226}
227
228bool SILowerSGPRSpills::spillCalleeSavedRegs(
229 MachineFunction &MF, SmallVectorImpl<int> &CalleeSavedFIs) {
230 MachineRegisterInfo &MRI = MF.getRegInfo();
231 const Function &F = MF.getFunction();
232 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
233 const SIFrameLowering *TFI = ST.getFrameLowering();
234 MachineFrameInfo &MFI = MF.getFrameInfo();
235 RegScavenger *RS = nullptr;
236
237 // Determine which of the registers in the callee save list should be saved.
238 BitVector SavedRegs;
239 TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);
240
241 // Add the code to save and restore the callee saved registers.
242 if (!F.hasFnAttribute(Kind: Attribute::Naked)) {
243 // FIXME: This is a lie. The CalleeSavedInfo is incomplete, but this is
244 // necessary for verifier liveness checks.
245 MFI.setCalleeSavedInfoValid(true);
246
247 std::vector<CalleeSavedInfo> CSI;
248 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
249 MCRegister RetAddrReg = TRI->getReturnAddressReg(MF);
250 MCRegister RetAddrRegSub0 = TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub0);
251 MCRegister RetAddrRegSub1 = TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub1);
252 bool SpillRetAddrReg = false;
253
254 for (unsigned I = 0; CSRegs[I]; ++I) {
255 MCRegister Reg = CSRegs[I];
256
257 if (SavedRegs.test(Idx: Reg)) {
258 if (Reg == RetAddrRegSub0 || Reg == RetAddrRegSub1) {
259 SpillRetAddrReg = true;
260 continue;
261 }
262
263 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
264 int JunkFI = MFI.CreateStackObject(Size: TRI->getSpillSize(RC: *RC),
265 Alignment: TRI->getSpillAlign(RC: *RC), isSpillSlot: true,
266 Alloca: nullptr, ID: TRI->getSpillStackID(RC: *RC));
267
268 CSI.emplace_back(args&: Reg, args&: JunkFI);
269 CalleeSavedFIs.push_back(Elt: JunkFI);
270 }
271 }
272
273 // Return address uses a register pair. Add the super register to the
274 // CSI list so that it's easier to identify the entire spill and CFI
275 // can be emitted appropriately.
276 if (SpillRetAddrReg) {
277 const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg: RetAddrReg);
278 int JunkFI =
279 MFI.CreateStackObject(Size: TRI->getSpillSize(RC: *RC), Alignment: TRI->getSpillAlign(RC: *RC),
280 isSpillSlot: true, Alloca: nullptr, ID: TRI->getSpillStackID(RC: *RC));
281 CSI.push_back(x: CalleeSavedInfo(RetAddrReg, JunkFI));
282 CalleeSavedFIs.push_back(Elt: JunkFI);
283 }
284
285 if (!CSI.empty()) {
286 for (MachineBasicBlock *SaveBlock : SaveBlocks)
287 insertCSRSaves(ST, SaveBlock&: *SaveBlock, CSI, Indexes, LIS);
288
289 // Add live ins to save blocks.
290 assert(SaveBlocks.size() == 1 && "shrink wrapping not fully implemented");
291 updateLiveness(MF, CSI);
292
293 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
294 insertCSRRestores(RestoreBlock&: *RestoreBlock, CSI, Indexes, LIS);
295 return true;
296 }
297 }
298
299 return false;
300}
301
302MachineBasicBlock *SILowerSGPRSpills::getCycleDomBB(CycleRef C) {
303 // If the insertion point lands on a cycle entry, move it to a block that
304 // dominates all entries.
305 if (MCI->isReducible(C)) {
306 if (auto *IDom = MDT->getNode(BB: MCI->getHeader(C))->getIDom())
307 return IDom->getBlock();
308 llvm_unreachable("Expected cycle to have an IDom.");
309 return nullptr;
310 }
311
312 ArrayRef<MachineBasicBlock *> Entries = MCI->getEntries(C);
313 assert(!Entries.empty() && "Expected cycle to have at least one entry.");
314 MachineBasicBlock *EntryBB = Entries[0];
315 for (unsigned I = 1; I < Entries.size(); ++I)
316 EntryBB = MDT->findNearestCommonDominator(A: EntryBB, B: Entries[I]);
317 return EntryBB;
318}
319
320void SILowerSGPRSpills::updateLaneVGPRDomInstr(
321 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
322 DenseMap<Register, LaneVGPRInsertPt> &LaneVGPRDomInstr) {
323 // For the Def of a virtual LaneVGPR to dominate all its uses, we should
324 // insert an IMPLICIT_DEF before the dominating spill. Switching to a
325 // depth first order doesn't really help since the machine function can be in
326 // the unstructured control flow post-SSA. For each virtual register, hence
327 // finding the common dominator to get either the dominating spill or a block
328 // dominating all spills.
329 SIMachineFunctionInfo *FuncInfo =
330 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
331 ArrayRef<SIRegisterInfo::SpilledReg> VGPRSpills =
332 FuncInfo->getSGPRSpillToVirtualVGPRLanes(FrameIndex: FI);
333 Register PrevLaneVGPR;
334 for (auto &Spill : VGPRSpills) {
335 if (PrevLaneVGPR == Spill.VGPR)
336 continue;
337
338 PrevLaneVGPR = Spill.VGPR;
339 auto I = LaneVGPRDomInstr.find(Val: Spill.VGPR);
340 if (Spill.Lane == 0 && I == LaneVGPRDomInstr.end()) {
341 LaneVGPRDomInstr[Spill.VGPR] = insertPt(MBB, It: InsertPt);
342 } else {
343 assert(I != LaneVGPRDomInstr.end());
344 LaneVGPRInsertPt Prev = I->second;
345 MachineBasicBlock *PrevInsertMBB = Prev.MBB;
346 MachineBasicBlock::iterator PrevInsertPt = Prev.It;
347 MachineBasicBlock *DomMBB = PrevInsertMBB;
348 if (DomMBB == MBB) {
349 // The insertion point earlier selected in a predecessor block whose
350 // spills are currently being lowered. The earlier InsertPt would be
351 // the one just before the block terminator and it should be changed
352 // if we insert any new spill in it.
353 if (PrevInsertPt == MBB->end() ||
354 MDT->dominates(A: &*InsertPt, B: &*PrevInsertPt))
355 I->second = insertPt(MBB, It: InsertPt);
356
357 continue;
358 }
359
360 // Find the common dominator block between PrevInsertPt and the
361 // current spill.
362 DomMBB = MDT->findNearestCommonDominator(A: DomMBB, B: MBB);
363
364 if (DomMBB == MBB)
365 I->second = insertPt(MBB, It: InsertPt);
366 else if (DomMBB != PrevInsertMBB)
367 I->second = insertPt(MBB: DomMBB, It: DomMBB->getFirstTerminator());
368 }
369 }
370}
371
372SmallVector<MCRegister>
373SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF) {
374 SmallVector<MCRegister> WWMRegCandidates;
375 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
376 if (!ST.getCLOpts().num_vgprs_for_wwm_alloc)
377 return WWMRegCandidates;
378
379 MachineRegisterInfo &MRI = MF.getRegInfo();
380 BitVector ReservedRegs = TRI->getReservedRegs(MF);
381 unsigned MaxNumVGPRs = ST.getMaxNumVectorRegs(F: MF.getFunction()).first;
382
383 // Try to use the highest available registers for now. Later after
384 // vgpr-regalloc, they can be shifted to the lowest range.
385 for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;
386 WWMRegCandidates.size() < ST.getCLOpts().num_vgprs_for_wwm_alloc &&
387 Reg >= AMDGPU::VGPR0;
388 --Reg) {
389 if (!ReservedRegs.test(Idx: Reg) &&
390 !MRI.isPhysRegUsed(PhysReg: Reg, /*SkipRegMaskTest=*/true))
391 WWMRegCandidates.push_back(Elt: Reg);
392 }
393
394 return WWMRegCandidates;
395}
396
397void SILowerSGPRSpills::assignWWMRegs(MachineFunction &MF,
398 ArrayRef<MCRegister> WWMRegCandidates,
399 bool RequiresFullWWMPool) {
400 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
401 if (FuncInfo->getSGPRSpillVGPRs().empty())
402 return;
403
404 BitVector WwmRegMask(TRI->getNumRegs());
405
406 unsigned DesiredPoolSize = std::min(
407 a: static_cast<unsigned>(FuncInfo->getSGPRSpillVGPRs().size()),
408 b: MF.getSubtarget<GCNSubtarget>().getCLOpts().num_vgprs_for_wwm_alloc);
409 unsigned SelectedPoolSize =
410 std::min<unsigned>(a: DesiredPoolSize, b: WWMRegCandidates.size());
411 // WWM register candidates are ordered high-to-low, so take the highest
412 // available registers when the desired pool is smaller than the candidate
413 // list.
414 for (MCRegister Reg : WWMRegCandidates.take_front(N: SelectedPoolSize))
415 TRI->markSuperRegs(RegisterSet&: WwmRegMask, Reg);
416
417 if (RequiresFullWWMPool && SelectedPoolSize != DesiredPoolSize) {
418 // Reserve an arbitrary register and report the error.
419 TRI->markSuperRegs(RegisterSet&: WwmRegMask, Reg: AMDGPU::VGPR0);
420 MF.getFunction().getContext().emitError(
421 ErrorStr: "cannot find enough VGPRs for wwm-regalloc");
422 }
423
424 BitVector PerLaneVGPRMask(WwmRegMask);
425 PerLaneVGPRMask.flip().clearBitsNotInMask(Mask: TRI->getAllVGPRRegMask());
426
427 // The complement set will be the registers for per-lane VGPR allocation.
428 FuncInfo->updatePerLaneVGPRMask(RegMask&: PerLaneVGPRMask);
429}
430
431bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
432 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
433 LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
434 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
435 SlotIndexes *Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
436 MachineDominatorTree *MDT =
437 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
438 MachineCycleInfo *MCI =
439 &getAnalysis<MachineCycleInfoWrapperPass>().getCycleInfo();
440 return SILowerSGPRSpills(LIS, Indexes, MDT, MCI).run(MF);
441}
442
443bool SILowerSGPRSpills::run(MachineFunction &MF) {
444 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
445 TII = ST.getInstrInfo();
446 TRI = &TII->getRegisterInfo();
447
448 assert(SaveBlocks.empty() && RestoreBlocks.empty());
449
450 // First, expose any CSR SGPR spills. This is mostly the same as what PEI
451 // does, but somewhat simpler.
452 calculateSaveRestoreBlocks(MF);
453 SmallVector<int> CalleeSavedFIs;
454 bool HasCSRs = spillCalleeSavedRegs(MF, CalleeSavedFIs);
455
456 MachineFrameInfo &MFI = MF.getFrameInfo();
457 MachineRegisterInfo &MRI = MF.getRegInfo();
458 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
459
460 if (!MFI.hasStackObjects() && !HasCSRs) {
461 SaveBlocks.clear();
462 RestoreBlocks.clear();
463 return false;
464 }
465
466 bool MadeChange = false;
467 bool SpilledToVirtVGPRLanes = false;
468
469 // TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be
470 // handled as SpilledToReg in regular PrologEpilogInserter.
471 const bool HasSGPRSpillToVGPR = TRI->spillSGPRToVGPR() &&
472 (HasCSRs || FuncInfo->hasSpilledSGPRs());
473 if (HasSGPRSpillToVGPR) {
474 // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
475 // are spilled to VGPRs, in which case we can eliminate the stack usage.
476 //
477 // This operates under the assumption that only other SGPR spills are users
478 // of the frame index.
479
480 // To track the spill frame indices handled in this pass.
481 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
482
483 // To track the IMPLICIT_DEF insertion point for the lane vgprs.
484 DenseMap<Register, LaneVGPRInsertPt> LaneVGPRDomInstr;
485
486 // Defer ordinary spills until physical CSR spills have reserved their
487 // lane VGPRs and the WWM allocation pool can be selected.
488 SmallVector<MachineInstr *> OrdinarySGPRSpills;
489 bool HasStrictWWMRegion = false;
490
491 for (MachineBasicBlock &MBB : MF) {
492 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
493 if (MI.getOpcode() == AMDGPU::ENTER_STRICT_WWM ||
494 MI.getOpcode() == AMDGPU::ENTER_STRICT_WQM) {
495 HasStrictWWMRegion = true;
496 continue;
497 }
498
499 if (!TII->isSGPRSpill(MI))
500 continue;
501
502 if (MI.getOperand(i: 0).isUndef()) {
503 if (Indexes)
504 Indexes->removeMachineInstrFromMaps(MI);
505 MI.eraseFromParent();
506 continue;
507 }
508
509 int FI = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::addr)->getIndex();
510 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
511
512 bool IsCalleeSaveSGPRSpill = llvm::is_contained(Range&: CalleeSavedFIs, Element: FI);
513 if (IsCalleeSaveSGPRSpill) {
514 // Spill callee-saved SGPRs into physical VGPR lanes.
515
516 // TODO: This is to ensure the CFIs are static for efficient frame
517 // unwinding in the debugger. Spilling them into virtual VGPR lanes
518 // involve regalloc to allocate the physical VGPRs and that might
519 // cause intermediate spill/split of such liveranges for successful
520 // allocation. This would result in broken CFI encoding unless the
521 // regalloc aware CFI generation to insert new CFIs along with the
522 // intermediate spills is implemented. There is no such support
523 // currently exist in the LLVM compiler.
524 if (FuncInfo->allocateSGPRSpillToVGPRLane(
525 MF, FI, /*SpillToPhysVGPRLane=*/true)) {
526 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
527 MI, FI, RS: nullptr, Indexes, LIS, SpillToPhysVGPRLane: true);
528 if (!Spilled)
529 llvm_unreachable(
530 "failed to spill SGPR to physical VGPR lane when allocated");
531 }
532 } else
533 OrdinarySGPRSpills.push_back(Elt: &MI);
534 }
535 }
536
537 // Select candidates once, before ordinary lane lowering creates virtual
538 // VGPRs and changes the number of registers desired for the WWM pool.
539 SmallVector<MCRegister> WWMRegCandidates;
540 // These non-spillable WWM users retain the old all-or-nothing pool policy.
541 const bool RequiresFullWWMPool =
542 HasStrictWWMRegion || isPreallocateSGPRSpillVGPRsEnabled(MF);
543 if (!OrdinarySGPRSpills.empty())
544 WWMRegCandidates = determineRegsForWWMAllocation(MF);
545
546 const bool ShouldLowerOrdinarySpillsToVGPRLanes =
547 RequiresFullWWMPool || !WWMRegCandidates.empty();
548 if (!ShouldLowerOrdinarySpillsToVGPRLanes && !OrdinarySGPRSpills.empty())
549 FuncInfo->setNoWWMPoolSGPRSpillFallback();
550
551 if (ShouldLowerOrdinarySpillsToVGPRLanes) {
552 for (MachineInstr *MI : OrdinarySGPRSpills) {
553 int FI = TII->getNamedOperand(MI&: *MI, OperandName: AMDGPU::OpName::addr)->getIndex();
554 if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
555 MachineBasicBlock *MBB = MI->getParent();
556 MachineInstrSpan MIS(MI, MBB);
557 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
558 MI: *MI, FI, RS: nullptr, Indexes, LIS);
559 if (!Spilled)
560 llvm_unreachable(
561 "failed to spill SGPR to virtual VGPR lane when allocated");
562 SpillFIs.set(FI);
563 updateLaneVGPRDomInstr(FI, MBB, InsertPt: MIS.begin(), LaneVGPRDomInstr);
564 SpilledToVirtVGPRLanes = true;
565 }
566 }
567 }
568
569 for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
570 LaneVGPRInsertPt IP = LaneVGPRDomInstr[Reg];
571 if (CycleRef C = MCI->getTopLevelParentCycle(Block: IP.MBB)) {
572 MachineBasicBlock *AdjMBB = getCycleDomBB(C);
573 IP = insertPt(MBB: AdjMBB, It: AdjMBB->getFirstTerminator());
574 }
575 // Insert the IMPLICIT_DEF at the identified points.
576 MachineBasicBlock &Block = *IP.MBB;
577 DebugLoc DL = Block.findDebugLoc(MBBI: IP.It);
578 auto MIB = BuildMI(BB&: Block, I: IP.It, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Reg);
579
580 // Add WWM flag to the virtual register.
581 FuncInfo->setFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
582
583 // Set SGPR_SPILL asm printer flag
584 MIB->setAsmPrinterFlag(AMDGPU::SGPR_SPILL);
585 if (LIS) {
586 LIS->InsertMachineInstrInMaps(MI&: *MIB);
587 LIS->createAndComputeVirtRegInterval(Reg);
588 }
589 }
590
591 // Assign the WWM pool from the pre-selected candidates and compute the
592 // complement mask for per-thread VGPR allocation.
593 assignWWMRegs(MF, WWMRegCandidates, RequiresFullWWMPool);
594
595 for (MachineBasicBlock &MBB : MF)
596 clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
597
598 // All those frame indices which are dead by now should be removed from the
599 // function frame. Otherwise, there is a side effect such as re-mapping of
600 // free frame index ids by the later pass(es) like "stack slot coloring"
601 // which in turn could mess-up with the book keeping of "frame index to VGPR
602 // lane".
603 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ false);
604
605 MadeChange = true;
606 }
607
608 if (SpilledToVirtVGPRLanes) {
609 const TargetRegisterClass *RC = TRI->getWaveMaskRegClass();
610 // Shift back the reserved SGPR for EXEC copy into the lowest range.
611 // This SGPR is reserved to handle the whole-wave spill/copy operations
612 // that might get inserted during vgpr regalloc.
613 Register UnusedLowSGPR = TRI->findUnusedRegister(MRI, RC, MF);
614 if (UnusedLowSGPR && TRI->getHWRegIndex(Reg: UnusedLowSGPR) <
615 TRI->getHWRegIndex(Reg: FuncInfo->getSGPRForEXECCopy()))
616 FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
617 } else if (!ST.needsAsyncBarrierArriveReplayWorkaround() ||
618 none_of(Range: MRI.def_instructions(Reg: AMDGPU::ASYNCcnt),
619 P: [](const MachineInstr &MI) {
620 return MI.getOpcode() ==
621 AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64;
622 })) {
623 // Neither SGPR spills to virtual VGPR lanes nor the barrier workaround
624 // needs the reserved register for EXEC copy.
625 FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
626 }
627
628 SaveBlocks.clear();
629 RestoreBlocks.clear();
630
631 return MadeChange;
632}
633
634PreservedAnalyses
635SILowerSGPRSpillsPass::run(MachineFunction &MF,
636 MachineFunctionAnalysisManager &MFAM) {
637 MFPropsModifier _(*this, MF);
638 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(IR&: MF);
639 auto *Indexes = MFAM.getCachedResult<SlotIndexesAnalysis>(IR&: MF);
640 MachineDominatorTree *MDT = &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
641 MachineCycleInfo &MCI = MFAM.getResult<MachineCycleAnalysis>(IR&: MF);
642 SILowerSGPRSpills(LIS, Indexes, MDT, &MCI).run(MF);
643 return PreservedAnalyses::all();
644}
645