1//===-- SILowerSGPRSPills.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Handle SGPR spills. This pass takes the place of PrologEpilogInserter for all
10// SGPR spills, so must insert CSR SGPR spills as well as expand them.
11//
12// This pass must never create new SGPR virtual registers.
13//
14// FIXME: Must stop RegScavenger spills in later passes.
15//
16//===----------------------------------------------------------------------===//
17
18#include "SILowerSGPRSpills.h"
19#include "AMDGPU.h"
20#include "GCNSubtarget.h"
21#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
22#include "SIMachineFunctionInfo.h"
23#include "SISpillUtils.h"
24#include "llvm/CodeGen/LiveIntervals.h"
25#include "llvm/CodeGen/MachineDominators.h"
26#include "llvm/CodeGen/MachineFrameInfo.h"
27#include "llvm/CodeGen/RegisterScavenging.h"
28#include "llvm/InitializePasses.h"
29
30using namespace llvm;
31
32#define DEBUG_TYPE "si-lower-sgpr-spills"
33
34using MBBVector = SmallVector<MachineBasicBlock *, 4>;
35
36namespace {
37
38static cl::opt<unsigned> MaxNumVGPRsForWwmAllocation(
39 "amdgpu-num-vgprs-for-wwm-alloc",
40 cl::desc("Max num VGPRs for whole-wave register allocation."),
41 cl::ReallyHidden, cl::init(Val: 10));
42
43class SILowerSGPRSpills {
44private:
45 const SIRegisterInfo *TRI = nullptr;
46 const SIInstrInfo *TII = nullptr;
47 LiveIntervals *LIS = nullptr;
48 SlotIndexes *Indexes = nullptr;
49 MachineDominatorTree *MDT = nullptr;
50
51 // Save and Restore blocks of the current function. Typically there is a
52 // single save block, unless Windows EH funclets are involved.
53 MBBVector SaveBlocks;
54 MBBVector RestoreBlocks;
55
56public:
57 SILowerSGPRSpills(LiveIntervals *LIS, SlotIndexes *Indexes,
58 MachineDominatorTree *MDT)
59 : LIS(LIS), Indexes(Indexes), MDT(MDT) {}
60 bool run(MachineFunction &MF);
61 void calculateSaveRestoreBlocks(MachineFunction &MF);
62 bool spillCalleeSavedRegs(MachineFunction &MF,
63 SmallVectorImpl<int> &CalleeSavedFIs);
64 void updateLaneVGPRDomInstr(
65 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
66 DenseMap<Register, MachineBasicBlock::iterator> &LaneVGPRDomInstr);
67 void determineRegsForWWMAllocation(MachineFunction &MF, BitVector &RegMask);
68};
69
70class SILowerSGPRSpillsLegacy : public MachineFunctionPass {
71public:
72 static char ID;
73
74 SILowerSGPRSpillsLegacy() : MachineFunctionPass(ID) {}
75
76 bool runOnMachineFunction(MachineFunction &MF) override;
77
78 void getAnalysisUsage(AnalysisUsage &AU) const override {
79 AU.addRequired<MachineDominatorTreeWrapperPass>();
80 AU.setPreservesAll();
81 MachineFunctionPass::getAnalysisUsage(AU);
82 }
83
84 MachineFunctionProperties getClearedProperties() const override {
85 // SILowerSGPRSpills introduces new Virtual VGPRs for spilling SGPRs.
86 return MachineFunctionProperties().setIsSSA().setNoVRegs();
87 }
88};
89
90} // end anonymous namespace
91
92char SILowerSGPRSpillsLegacy::ID = 0;
93
94INITIALIZE_PASS_BEGIN(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
95 "SI lower SGPR spill instructions", false, false)
96INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
97INITIALIZE_PASS_DEPENDENCY(MachineDominatorTreeWrapperPass)
98INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
99INITIALIZE_PASS_END(SILowerSGPRSpillsLegacy, DEBUG_TYPE,
100 "SI lower SGPR spill instructions", false, false)
101
102char &llvm::SILowerSGPRSpillsLegacyID = SILowerSGPRSpillsLegacy::ID;
103
104static bool isLiveIntoMBB(MCRegister Reg, MachineBasicBlock &MBB,
105 const TargetRegisterInfo *TRI) {
106 for (MCRegAliasIterator R(Reg, TRI, true); R.isValid(); ++R) {
107 if (MBB.isLiveIn(Reg: *R)) {
108 return true;
109 }
110 }
111 return false;
112}
113
114/// Insert spill code for the callee-saved registers used in the function.
115static void insertCSRSaves(MachineBasicBlock &SaveBlock,
116 ArrayRef<CalleeSavedInfo> CSI, SlotIndexes *Indexes,
117 LiveIntervals *LIS) {
118 MachineFunction &MF = *SaveBlock.getParent();
119 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
120 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
121 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
122 const SIRegisterInfo *RI = ST.getRegisterInfo();
123
124 MachineBasicBlock::iterator I = SaveBlock.begin();
125 if (!TFI->spillCalleeSavedRegisters(MBB&: SaveBlock, MI: I, CSI, TRI: RI)) {
126 for (const CalleeSavedInfo &CS : CSI) {
127 // Insert the spill to the stack frame.
128 MCRegister Reg = CS.getReg();
129
130 MachineInstrSpan MIS(I, &SaveBlock);
131 const TargetRegisterClass *RC = RI->getMinimalPhysRegClass(
132 Reg, VT: Reg == RI->getReturnAddressReg(MF) ? MVT::i64 : MVT::i32);
133
134 // If this value was already livein, we probably have a direct use of the
135 // incoming register value, so don't kill at the spill point. This happens
136 // since we pass some special inputs (workgroup IDs) in the callee saved
137 // range.
138 const bool IsLiveIn = isLiveIntoMBB(Reg, MBB&: SaveBlock, TRI: RI);
139 TII.storeRegToStackSlot(MBB&: SaveBlock, MI: I, SrcReg: Reg, isKill: !IsLiveIn, FrameIndex: CS.getFrameIdx(),
140 RC, VReg: Register());
141
142 if (Indexes) {
143 assert(std::distance(MIS.begin(), I) == 1);
144 MachineInstr &Inst = *std::prev(x: I);
145 Indexes->insertMachineInstrInMaps(MI&: Inst);
146 }
147
148 if (LIS)
149 LIS->removeAllRegUnitsForPhysReg(Reg);
150 }
151 } else {
152 // TFI doesn't update Indexes and LIS, so we have to do it separately.
153 if (Indexes)
154 Indexes->repairIndexesInRange(MBB: &SaveBlock, Begin: SaveBlock.begin(), End: I);
155
156 if (LIS)
157 for (const CalleeSavedInfo &CS : CSI)
158 LIS->removeAllRegUnitsForPhysReg(Reg: CS.getReg());
159 }
160}
161
162/// Insert restore code for the callee-saved registers used in the function.
163static void insertCSRRestores(MachineBasicBlock &RestoreBlock,
164 MutableArrayRef<CalleeSavedInfo> CSI,
165 SlotIndexes *Indexes, LiveIntervals *LIS) {
166 MachineFunction &MF = *RestoreBlock.getParent();
167 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
168 const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering();
169 const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
170 // Restore all registers immediately before the return and any
171 // terminators that precede it.
172 MachineBasicBlock::iterator I = RestoreBlock.getFirstTerminator();
173 const MachineBasicBlock::iterator BeforeRestoresI =
174 I == RestoreBlock.begin() ? I : std::prev(x: I);
175
176 // FIXME: Just emit the readlane/writelane directly
177 if (!TFI->restoreCalleeSavedRegisters(MBB&: RestoreBlock, MI: I, CSI, TRI)) {
178 for (const CalleeSavedInfo &CI : reverse(C&: CSI)) {
179 // Insert in reverse order. loadRegFromStackSlot can insert
180 // multiple instructions.
181 TFI->restoreCalleeSavedRegister(MBB&: RestoreBlock, MI: I, CS: CI, TII: &TII, TRI);
182
183 if (Indexes) {
184 MachineInstr &Inst = *std::prev(x: I);
185 Indexes->insertMachineInstrInMaps(MI&: Inst);
186 }
187
188 if (LIS)
189 LIS->removeAllRegUnitsForPhysReg(Reg: CI.getReg());
190 }
191 } else {
192 // TFI doesn't update Indexes and LIS, so we have to do it separately.
193 if (Indexes)
194 Indexes->repairIndexesInRange(MBB: &RestoreBlock, Begin: BeforeRestoresI,
195 End: RestoreBlock.getFirstTerminator());
196
197 if (LIS)
198 for (const CalleeSavedInfo &CS : CSI)
199 LIS->removeAllRegUnitsForPhysReg(Reg: CS.getReg());
200 }
201}
202
203/// Compute the sets of entry and return blocks for saving and restoring
204/// callee-saved registers, and placing prolog and epilog code.
205void SILowerSGPRSpills::calculateSaveRestoreBlocks(MachineFunction &MF) {
206 const MachineFrameInfo &MFI = MF.getFrameInfo();
207
208 // Even when we do not change any CSR, we still want to insert the
209 // prologue and epilogue of the function.
210 // So set the save points for those.
211
212 // Use the points found by shrink-wrapping, if any.
213 if (!MFI.getSavePoints().empty()) {
214 assert(MFI.getSavePoints().size() == 1 &&
215 "Multiple save points not yet supported!");
216 const auto &SavePoint = *MFI.getSavePoints().begin();
217 SaveBlocks.push_back(Elt: SavePoint.first);
218 assert(MFI.getRestorePoints().size() == 1 &&
219 "Multiple restore points not yet supported!");
220 const auto &RestorePoint = *MFI.getRestorePoints().begin();
221 MachineBasicBlock *RestoreBlock = RestorePoint.first;
222 // If RestoreBlock does not have any successor and is not a return block
223 // then the end point is unreachable and we do not need to insert any
224 // epilogue.
225 if (!RestoreBlock->succ_empty() || RestoreBlock->isReturnBlock())
226 RestoreBlocks.push_back(Elt: RestoreBlock);
227 return;
228 }
229
230 // Save refs to entry and return blocks.
231 SaveBlocks.push_back(Elt: &MF.front());
232 for (MachineBasicBlock &MBB : MF) {
233 if (MBB.isEHFuncletEntry())
234 SaveBlocks.push_back(Elt: &MBB);
235 if (MBB.isReturnBlock())
236 RestoreBlocks.push_back(Elt: &MBB);
237 }
238}
239
240// TODO: To support shrink wrapping, this would need to copy
241// PrologEpilogInserter's updateLiveness.
242static void updateLiveness(MachineFunction &MF, ArrayRef<CalleeSavedInfo> CSI) {
243 MachineBasicBlock &EntryBB = MF.front();
244
245 for (const CalleeSavedInfo &CSIReg : CSI)
246 EntryBB.addLiveIn(PhysReg: CSIReg.getReg());
247 EntryBB.sortUniqueLiveIns();
248}
249
250bool SILowerSGPRSpills::spillCalleeSavedRegs(
251 MachineFunction &MF, SmallVectorImpl<int> &CalleeSavedFIs) {
252 MachineRegisterInfo &MRI = MF.getRegInfo();
253 const Function &F = MF.getFunction();
254 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
255 const SIFrameLowering *TFI = ST.getFrameLowering();
256 MachineFrameInfo &MFI = MF.getFrameInfo();
257 RegScavenger *RS = nullptr;
258
259 // Determine which of the registers in the callee save list should be saved.
260 BitVector SavedRegs;
261 TFI->determineCalleeSavesSGPR(MF, SavedRegs, RS);
262
263 // Add the code to save and restore the callee saved registers.
264 if (!F.hasFnAttribute(Kind: Attribute::Naked)) {
265 // FIXME: This is a lie. The CalleeSavedInfo is incomplete, but this is
266 // necessary for verifier liveness checks.
267 MFI.setCalleeSavedInfoValid(true);
268
269 std::vector<CalleeSavedInfo> CSI;
270 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
271
272 for (unsigned I = 0; CSRegs[I]; ++I) {
273 MCRegister Reg = CSRegs[I];
274
275 if (SavedRegs.test(Idx: Reg)) {
276 const TargetRegisterClass *RC =
277 TRI->getMinimalPhysRegClass(Reg, VT: MVT::i32);
278 int JunkFI = MFI.CreateStackObject(Size: TRI->getSpillSize(RC: *RC),
279 Alignment: TRI->getSpillAlign(RC: *RC), isSpillSlot: true);
280
281 CSI.emplace_back(args&: Reg, args&: JunkFI);
282 CalleeSavedFIs.push_back(Elt: JunkFI);
283 }
284 }
285
286 if (!CSI.empty()) {
287 for (MachineBasicBlock *SaveBlock : SaveBlocks)
288 insertCSRSaves(SaveBlock&: *SaveBlock, CSI, Indexes, LIS);
289
290 // Add live ins to save blocks.
291 assert(SaveBlocks.size() == 1 && "shrink wrapping not fully implemented");
292 updateLiveness(MF, CSI);
293
294 for (MachineBasicBlock *RestoreBlock : RestoreBlocks)
295 insertCSRRestores(RestoreBlock&: *RestoreBlock, CSI, Indexes, LIS);
296 return true;
297 }
298 }
299
300 return false;
301}
302
303void SILowerSGPRSpills::updateLaneVGPRDomInstr(
304 int FI, MachineBasicBlock *MBB, MachineBasicBlock::iterator InsertPt,
305 DenseMap<Register, MachineBasicBlock::iterator> &LaneVGPRDomInstr) {
306 // For the Def of a virtual LaneVGPR to dominate all its uses, we should
307 // insert an IMPLICIT_DEF before the dominating spill. Switching to a
308 // depth first order doesn't really help since the machine function can be in
309 // the unstructured control flow post-SSA. For each virtual register, hence
310 // finding the common dominator to get either the dominating spill or a block
311 // dominating all spills.
312 SIMachineFunctionInfo *FuncInfo =
313 MBB->getParent()->getInfo<SIMachineFunctionInfo>();
314 ArrayRef<SIRegisterInfo::SpilledReg> VGPRSpills =
315 FuncInfo->getSGPRSpillToVirtualVGPRLanes(FrameIndex: FI);
316 Register PrevLaneVGPR;
317 for (auto &Spill : VGPRSpills) {
318 if (PrevLaneVGPR == Spill.VGPR)
319 continue;
320
321 PrevLaneVGPR = Spill.VGPR;
322 auto I = LaneVGPRDomInstr.find(Val: Spill.VGPR);
323 if (Spill.Lane == 0 && I == LaneVGPRDomInstr.end()) {
324 // Initially add the spill instruction itself for Insertion point.
325 LaneVGPRDomInstr[Spill.VGPR] = InsertPt;
326 } else {
327 assert(I != LaneVGPRDomInstr.end());
328 auto PrevInsertPt = I->second;
329 MachineBasicBlock *DomMBB = PrevInsertPt->getParent();
330 if (DomMBB == MBB) {
331 // The insertion point earlier selected in a predecessor block whose
332 // spills are currently being lowered. The earlier InsertPt would be
333 // the one just before the block terminator and it should be changed
334 // if we insert any new spill in it.
335 if (MDT->dominates(A: &*InsertPt, B: &*PrevInsertPt))
336 I->second = InsertPt;
337
338 continue;
339 }
340
341 // Find the common dominator block between PrevInsertPt and the
342 // current spill.
343 DomMBB = MDT->findNearestCommonDominator(A: DomMBB, B: MBB);
344 if (DomMBB == MBB)
345 I->second = InsertPt;
346 else if (DomMBB != PrevInsertPt->getParent())
347 I->second = &(*DomMBB->getFirstTerminator());
348 }
349 }
350}
351
352void SILowerSGPRSpills::determineRegsForWWMAllocation(MachineFunction &MF,
353 BitVector &RegMask) {
354 // Determine an optimal number of VGPRs for WWM allocation. The complement
355 // list will be available for allocating other VGPR virtual registers.
356 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
357 MachineRegisterInfo &MRI = MF.getRegInfo();
358 BitVector ReservedRegs = TRI->getReservedRegs(MF);
359 BitVector NonWwmAllocMask(TRI->getNumRegs());
360 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
361
362 // FIXME: MaxNumVGPRsForWwmAllocation might need to be adjusted in the future
363 // to have a balanced allocation between WWM values and per-thread vector
364 // register operands.
365 unsigned NumRegs = MaxNumVGPRsForWwmAllocation;
366 NumRegs =
367 std::min(a: static_cast<unsigned>(MFI->getSGPRSpillVGPRs().size()), b: NumRegs);
368
369 auto [MaxNumVGPRs, MaxNumAGPRs] = ST.getMaxNumVectorRegs(F: MF.getFunction());
370 // Try to use the highest available registers for now. Later after
371 // vgpr-regalloc, they can be shifted to the lowest range.
372 unsigned I = 0;
373 for (unsigned Reg = AMDGPU::VGPR0 + MaxNumVGPRs - 1;
374 (I < NumRegs) && (Reg >= AMDGPU::VGPR0); --Reg) {
375 if (!ReservedRegs.test(Idx: Reg) &&
376 !MRI.isPhysRegUsed(PhysReg: Reg, /*SkipRegMaskTest=*/true)) {
377 TRI->markSuperRegs(RegisterSet&: RegMask, Reg);
378 ++I;
379 }
380 }
381
382 if (I != NumRegs) {
383 // Reserve an arbitrary register and report the error.
384 TRI->markSuperRegs(RegisterSet&: RegMask, Reg: AMDGPU::VGPR0);
385 MF.getFunction().getContext().emitError(
386 ErrorStr: "cannot find enough VGPRs for wwm-regalloc");
387 }
388}
389
390bool SILowerSGPRSpillsLegacy::runOnMachineFunction(MachineFunction &MF) {
391 auto *LISWrapper = getAnalysisIfAvailable<LiveIntervalsWrapperPass>();
392 LiveIntervals *LIS = LISWrapper ? &LISWrapper->getLIS() : nullptr;
393 auto *SIWrapper = getAnalysisIfAvailable<SlotIndexesWrapperPass>();
394 SlotIndexes *Indexes = SIWrapper ? &SIWrapper->getSI() : nullptr;
395 MachineDominatorTree *MDT =
396 &getAnalysis<MachineDominatorTreeWrapperPass>().getDomTree();
397 return SILowerSGPRSpills(LIS, Indexes, MDT).run(MF);
398}
399
400bool SILowerSGPRSpills::run(MachineFunction &MF) {
401 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
402 TII = ST.getInstrInfo();
403 TRI = &TII->getRegisterInfo();
404
405 assert(SaveBlocks.empty() && RestoreBlocks.empty());
406
407 // First, expose any CSR SGPR spills. This is mostly the same as what PEI
408 // does, but somewhat simpler.
409 calculateSaveRestoreBlocks(MF);
410 SmallVector<int> CalleeSavedFIs;
411 bool HasCSRs = spillCalleeSavedRegs(MF, CalleeSavedFIs);
412
413 MachineFrameInfo &MFI = MF.getFrameInfo();
414 MachineRegisterInfo &MRI = MF.getRegInfo();
415 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
416
417 if (!MFI.hasStackObjects() && !HasCSRs) {
418 SaveBlocks.clear();
419 RestoreBlocks.clear();
420 return false;
421 }
422
423 bool MadeChange = false;
424 bool SpilledToVirtVGPRLanes = false;
425
426 // TODO: CSR VGPRs will never be spilled to AGPRs. These can probably be
427 // handled as SpilledToReg in regular PrologEpilogInserter.
428 const bool HasSGPRSpillToVGPR = TRI->spillSGPRToVGPR() &&
429 (HasCSRs || FuncInfo->hasSpilledSGPRs());
430 if (HasSGPRSpillToVGPR) {
431 // Process all SGPR spills before frame offsets are finalized. Ideally SGPRs
432 // are spilled to VGPRs, in which case we can eliminate the stack usage.
433 //
434 // This operates under the assumption that only other SGPR spills are users
435 // of the frame index.
436
437 // To track the spill frame indices handled in this pass.
438 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
439
440 // To track the IMPLICIT_DEF insertion point for the lane vgprs.
441 DenseMap<Register, MachineBasicBlock::iterator> LaneVGPRDomInstr;
442
443 for (MachineBasicBlock &MBB : MF) {
444 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
445 if (!TII->isSGPRSpill(MI))
446 continue;
447
448 if (MI.getOperand(i: 0).isUndef()) {
449 if (Indexes)
450 Indexes->removeMachineInstrFromMaps(MI);
451 MI.eraseFromParent();
452 continue;
453 }
454
455 int FI = TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::addr)->getIndex();
456 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
457
458 bool IsCalleeSaveSGPRSpill = llvm::is_contained(Range&: CalleeSavedFIs, Element: FI);
459 if (IsCalleeSaveSGPRSpill) {
460 // Spill callee-saved SGPRs into physical VGPR lanes.
461
462 // TODO: This is to ensure the CFIs are static for efficient frame
463 // unwinding in the debugger. Spilling them into virtual VGPR lanes
464 // involve regalloc to allocate the physical VGPRs and that might
465 // cause intermediate spill/split of such liveranges for successful
466 // allocation. This would result in broken CFI encoding unless the
467 // regalloc aware CFI generation to insert new CFIs along with the
468 // intermediate spills is implemented. There is no such support
469 // currently exist in the LLVM compiler.
470 if (FuncInfo->allocateSGPRSpillToVGPRLane(
471 MF, FI, /*SpillToPhysVGPRLane=*/true)) {
472 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
473 MI, FI, RS: nullptr, Indexes, LIS, SpillToPhysVGPRLane: true);
474 if (!Spilled)
475 llvm_unreachable(
476 "failed to spill SGPR to physical VGPR lane when allocated");
477 }
478 } else {
479 MachineInstrSpan MIS(&MI, &MBB);
480 if (FuncInfo->allocateSGPRSpillToVGPRLane(MF, FI)) {
481 bool Spilled = TRI->eliminateSGPRToVGPRSpillFrameIndex(
482 MI, FI, RS: nullptr, Indexes, LIS);
483 if (!Spilled)
484 llvm_unreachable(
485 "failed to spill SGPR to virtual VGPR lane when allocated");
486 SpillFIs.set(FI);
487 updateLaneVGPRDomInstr(FI, MBB: &MBB, InsertPt: MIS.begin(), LaneVGPRDomInstr);
488 SpilledToVirtVGPRLanes = true;
489 }
490 }
491 }
492 }
493
494 for (auto Reg : FuncInfo->getSGPRSpillVGPRs()) {
495 auto InsertPt = LaneVGPRDomInstr[Reg];
496 // Insert the IMPLICIT_DEF at the identified points.
497 MachineBasicBlock &Block = *InsertPt->getParent();
498 DebugLoc DL = Block.findDebugLoc(MBBI: InsertPt);
499 auto MIB =
500 BuildMI(BB&: Block, I&: *InsertPt, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::IMPLICIT_DEF), DestReg: Reg);
501
502 // Add WWM flag to the virtual register.
503 FuncInfo->setFlag(Reg, Flag: AMDGPU::VirtRegFlag::WWM_REG);
504
505 // Set SGPR_SPILL asm printer flag
506 MIB->setAsmPrinterFlag(AMDGPU::SGPR_SPILL);
507 if (LIS) {
508 LIS->InsertMachineInstrInMaps(MI&: *MIB);
509 LIS->createAndComputeVirtRegInterval(Reg);
510 }
511 }
512
513 // Determine the registers for WWM allocation and also compute the register
514 // mask for non-wwm VGPR allocation.
515 if (FuncInfo->getSGPRSpillVGPRs().size()) {
516 BitVector WwmRegMask(TRI->getNumRegs());
517
518 determineRegsForWWMAllocation(MF, RegMask&: WwmRegMask);
519
520 BitVector NonWwmRegMask(WwmRegMask);
521 NonWwmRegMask.flip().clearBitsNotInMask(Mask: TRI->getAllVGPRRegMask());
522
523 // The complement set will be the registers for non-wwm (per-thread) vgpr
524 // allocation.
525 FuncInfo->updateNonWWMRegMask(RegMask&: NonWwmRegMask);
526 }
527
528 for (MachineBasicBlock &MBB : MF)
529 clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
530
531 // All those frame indices which are dead by now should be removed from the
532 // function frame. Otherwise, there is a side effect such as re-mapping of
533 // free frame index ids by the later pass(es) like "stack slot coloring"
534 // which in turn could mess-up with the book keeping of "frame index to VGPR
535 // lane".
536 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ false);
537
538 MadeChange = true;
539 }
540
541 if (SpilledToVirtVGPRLanes) {
542 const TargetRegisterClass *RC = TRI->getWaveMaskRegClass();
543 // Shift back the reserved SGPR for EXEC copy into the lowest range.
544 // This SGPR is reserved to handle the whole-wave spill/copy operations
545 // that might get inserted during vgpr regalloc.
546 Register UnusedLowSGPR = TRI->findUnusedRegister(MRI, RC, MF);
547 if (UnusedLowSGPR && TRI->getHWRegIndex(Reg: UnusedLowSGPR) <
548 TRI->getHWRegIndex(Reg: FuncInfo->getSGPRForEXECCopy()))
549 FuncInfo->setSGPRForEXECCopy(UnusedLowSGPR);
550 } else {
551 // No SGPR spills to virtual VGPR lanes and hence there won't be any WWM
552 // spills/copies. Reset the SGPR reserved for EXEC copy.
553 FuncInfo->setSGPRForEXECCopy(AMDGPU::NoRegister);
554 }
555
556 SaveBlocks.clear();
557 RestoreBlocks.clear();
558
559 return MadeChange;
560}
561
562PreservedAnalyses
563SILowerSGPRSpillsPass::run(MachineFunction &MF,
564 MachineFunctionAnalysisManager &MFAM) {
565 MFPropsModifier _(*this, MF);
566 auto *LIS = MFAM.getCachedResult<LiveIntervalsAnalysis>(IR&: MF);
567 auto *Indexes = MFAM.getCachedResult<SlotIndexesAnalysis>(IR&: MF);
568 MachineDominatorTree *MDT = &MFAM.getResult<MachineDominatorTreeAnalysis>(IR&: MF);
569 SILowerSGPRSpills(LIS, Indexes, MDT).run(MF);
570 return PreservedAnalyses::all();
571}
572