1//===----------------------- SIFrameLowering.cpp --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8
9#include "SIFrameLowering.h"
10#include "AMDGPU.h"
11#include "AMDGPULaneMaskUtils.h"
12#include "GCNSubtarget.h"
13#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
14#include "SIMachineFunctionInfo.h"
15#include "SISpillUtils.h"
16#include "llvm/BinaryFormat/Dwarf.h"
17#include "llvm/CodeGen/LiveRegUnits.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineModuleInfo.h"
20#include "llvm/CodeGen/RegisterScavenging.h"
21#include "llvm/Support/LEB128.h"
22#include "llvm/Target/TargetMachine.h"
23
24using namespace llvm;
25
26#define DEBUG_TYPE "frame-info"
27
28static cl::opt<bool> EnableSpillVGPRToAGPR(
29 "amdgpu-spill-vgpr-to-agpr",
30 cl::desc("Enable spilling VGPRs to AGPRs"),
31 cl::ReallyHidden,
32 cl::init(Val: true));
33
34static constexpr unsigned SGPRBitSize = 32;
35static constexpr unsigned SGPRByteSize = SGPRBitSize / 8;
36static constexpr unsigned VGPRLaneBitSize = 32;
37
38// Find a register matching \p RC from \p LiveUnits which is unused and
39// available throughout the function. On failure, returns AMDGPU::NoRegister.
40// TODO: Rewrite the loop here to iterate over MCRegUnits instead of
41// MCRegisters. This should reduce the number of iterations and avoid redundant
42// checking.
43static MCRegister findUnusedRegister(MachineRegisterInfo &MRI,
44 const LiveRegUnits &LiveUnits,
45 const TargetRegisterClass &RC) {
46 for (MCRegister Reg : RC) {
47 if (!MRI.isPhysRegUsed(PhysReg: Reg) && LiveUnits.available(Reg) &&
48 !MRI.isReserved(PhysReg: Reg))
49 return Reg;
50 }
51 return MCRegister();
52}
53
54static void encodeDwarfRegisterLocation(int DwarfReg, raw_ostream &OS) {
55 assert(DwarfReg >= 0);
56 if (DwarfReg < 32) {
57 OS << uint8_t(dwarf::DW_OP_reg0 + DwarfReg);
58 } else {
59 OS << uint8_t(dwarf::DW_OP_regx);
60 encodeULEB128(Value: DwarfReg, OS);
61 }
62}
63
64static MCCFIInstruction createScaledCFAInPrivateWave(const GCNSubtarget &ST,
65 int64_t DwarfStackPtrReg) {
66 assert(ST.hasFlatScratchEnabled());
67
68 // When flat scratch is enabled, the stack pointer is an address in the
69 // private_lane DWARF address space (i.e. swizzled), but in order to
70 // accurately and efficiently describe things like masked spills of vector
71 // registers we want to define the CFA to be an address in the private_wave
72 // DWARF address space (i.e. unswizzled). To achieve this we scale the stack
73 // pointer by the wavefront size, implemented as (SP << wave_size_log2).
74 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
75 assert(WavefrontSizeLog2 < 32);
76
77 SmallString<20> Block;
78 raw_svector_ostream OSBlock(Block);
79 encodeDwarfRegisterLocation(DwarfReg: DwarfStackPtrReg, OS&: OSBlock);
80 OSBlock << uint8_t(dwarf::DW_OP_deref_size) << uint8_t(SGPRByteSize)
81 << uint8_t(dwarf::DW_OP_lit0 + WavefrontSizeLog2)
82 << uint8_t(dwarf::DW_OP_shl)
83 << uint8_t(dwarf::DW_OP_lit0 +
84 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave)
85 << uint8_t(dwarf::DW_OP_LLVM_user)
86 << uint8_t(dwarf::DW_OP_LLVM_form_aspace_address);
87
88 SmallString<20> CFIInst;
89 raw_svector_ostream OSCFIInst(CFIInst);
90 OSCFIInst << uint8_t(dwarf::DW_CFA_def_cfa_expression);
91 encodeULEB128(Value: Block.size(), OS&: OSCFIInst);
92 OSCFIInst << Block;
93
94 return MCCFIInstruction::createEscape(L: nullptr, Vals: OSCFIInst.str());
95}
96
97void SIFrameLowering::emitDefCFA(MachineBasicBlock &MBB,
98 MachineBasicBlock::iterator MBBI,
99 DebugLoc const &DL, MCRegister StackPtrReg,
100 bool AspaceAlreadyDefined,
101 MachineInstr::MIFlag Flags) const {
102 MachineFunction &MF = *MBB.getParent();
103 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
104 const SIRegisterInfo *TRI = ST.getRegisterInfo();
105
106 int64_t DwarfStackPtrReg = TRI->getDwarfRegNum(Reg: StackPtrReg, isEH: false);
107 MCCFIInstruction CFIInst =
108 ST.hasFlatScratchEnabled()
109 ? createScaledCFAInPrivateWave(ST, DwarfStackPtrReg)
110 : (AspaceAlreadyDefined
111 ? MCCFIInstruction::createLLVMDefAspaceCfa(
112 L: nullptr, Register: DwarfStackPtrReg, Offset: 0,
113 AddressSpace: dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave, Loc: SMLoc())
114 : MCCFIInstruction::createDefCfaRegister(L: nullptr,
115 Register: DwarfStackPtrReg));
116 buildCFI(MBB, MBBI, DL, CFIInst, flag: Flags);
117}
118
119// Find a scratch register that we can use in the prologue. We avoid using
120// callee-save registers since they may appear to be free when this is called
121// from canUseAsPrologue (during shrink wrapping), but then no longer be free
122// when this is called from emitPrologue.
123static MCRegister findScratchNonCalleeSaveRegister(
124 MachineRegisterInfo &MRI, LiveRegUnits &LiveUnits,
125 const TargetRegisterClass &RC, bool Unused = false) {
126 // Mark callee saved registers as used so we will not choose them.
127 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
128 for (unsigned i = 0; CSRegs[i]; ++i)
129 LiveUnits.addReg(Reg: CSRegs[i]);
130
131 // We are looking for a register that can be used throughout the entire
132 // function, so any use is unacceptable.
133 if (Unused)
134 return findUnusedRegister(MRI, LiveUnits, RC);
135
136 for (MCRegister Reg : RC) {
137 if (LiveUnits.available(Reg) && !MRI.isReserved(PhysReg: Reg))
138 return Reg;
139 }
140
141 return MCRegister();
142}
143
144/// Query target location for spilling SGPRs
145/// \p IncludeScratchCopy : Also look for free scratch SGPRs
146static void getVGPRSpillLaneOrTempRegister(
147 MachineFunction &MF, LiveRegUnits &LiveUnits, Register SGPR,
148 const TargetRegisterClass &RC = AMDGPU::SReg_32_XM0_XEXECRegClass,
149 bool IncludeScratchCopy = true) {
150 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
151 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
152
153 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
154 const SIRegisterInfo *TRI = ST.getRegisterInfo();
155 unsigned Size = TRI->getSpillSize(RC);
156 Align Alignment = TRI->getSpillAlign(RC);
157
158 // We need to save and restore the given SGPR.
159
160 Register ScratchSGPR;
161 // 1: Try to save the given register into an unused scratch SGPR. The
162 // LiveUnits should have all the callee saved registers marked as used. For
163 // certain cases we skip copy to scratch SGPR.
164 if (IncludeScratchCopy)
165 ScratchSGPR = findUnusedRegister(MRI&: MF.getRegInfo(), LiveUnits, RC);
166
167 if (!ScratchSGPR) {
168 int FI = FrameInfo.CreateStackObject(Size, Alignment, isSpillSlot: true, Alloca: nullptr,
169 ID: TargetStackID::SGPRSpill);
170
171 if (TRI->spillSGPRToVGPR() &&
172 MFI->allocateSGPRSpillToVGPRLane(MF, FI, /*SpillToPhysVGPRLane=*/true,
173 /*IsPrologEpilog=*/true)) {
174 // 2: There's no free lane to spill, and no free register to save the
175 // SGPR, so we're forced to take another VGPR to use for the spill.
176 MFI->addToPrologEpilogSGPRSpills(
177 Reg: SGPR, SI: PrologEpilogSGPRSaveRestoreInfo(
178 SGPRSaveKind::SPILL_TO_VGPR_LANE, FI));
179
180 LLVM_DEBUG(auto Spill = MFI->getSGPRSpillToPhysicalVGPRLanes(FI).front();
181 dbgs() << printReg(SGPR, TRI) << " requires fallback spill to "
182 << printReg(Spill.VGPR, TRI) << ':' << Spill.Lane
183 << '\n';);
184 } else {
185 // Remove dead <FI> index
186 MF.getFrameInfo().RemoveStackObject(ObjectIdx: FI);
187 // 3: If all else fails, spill the register to memory.
188 FI = FrameInfo.CreateSpillStackObject(Size, Alignment);
189 MFI->addToPrologEpilogSGPRSpills(
190 Reg: SGPR,
191 SI: PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind::SPILL_TO_MEM, FI));
192 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling "
193 << printReg(SGPR, TRI) << '\n');
194 }
195 } else {
196 MFI->addToPrologEpilogSGPRSpills(
197 Reg: SGPR, SI: PrologEpilogSGPRSaveRestoreInfo(
198 SGPRSaveKind::COPY_TO_SCRATCH_SGPR, ScratchSGPR));
199 LiveUnits.addReg(Reg: ScratchSGPR);
200 LLVM_DEBUG(dbgs() << "Saving " << printReg(SGPR, TRI) << " with copy to "
201 << printReg(ScratchSGPR, TRI) << '\n');
202 }
203}
204
205// We need to specially emit stack operations here because a different frame
206// register is used than in the rest of the function, as getFrameRegister would
207// use.
208static void buildPrologSpill(const GCNSubtarget &ST, const SIRegisterInfo &TRI,
209 const SIMachineFunctionInfo &FuncInfo,
210 LiveRegUnits &LiveUnits, MachineFunction &MF,
211 MachineBasicBlock &MBB,
212 MachineBasicBlock::iterator I, const DebugLoc &DL,
213 Register SpillReg, int FI, Register FrameReg,
214 int64_t DwordOff = 0) {
215 unsigned Opc = ST.hasFlatScratchEnabled() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
216 : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
217
218 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
219 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
220 MachineMemOperand *MMO = MF.getMachineMemOperand(
221 PtrInfo, F: MachineMemOperand::MOStore, Size: FrameInfo.getObjectSize(ObjectIdx: FI),
222 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FI));
223 LiveUnits.addReg(Reg: SpillReg);
224 bool IsKill = !MBB.isLiveIn(Reg: SpillReg);
225 TRI.buildSpillLoadStore(MBB, MI: I, DL, LoadStoreOp: Opc, Index: FI, ValueReg: SpillReg, ValueIsKill: IsKill, ScratchOffsetReg: FrameReg,
226 InstrOffset: DwordOff, MMO, RS: nullptr, LiveUnits: &LiveUnits);
227 if (IsKill)
228 LiveUnits.removeReg(Reg: SpillReg);
229}
230
231static void buildEpilogRestore(const GCNSubtarget &ST,
232 const SIRegisterInfo &TRI,
233 const SIMachineFunctionInfo &FuncInfo,
234 LiveRegUnits &LiveUnits, MachineFunction &MF,
235 MachineBasicBlock &MBB,
236 MachineBasicBlock::iterator I,
237 const DebugLoc &DL, Register SpillReg, int FI,
238 Register FrameReg, int64_t DwordOff = 0) {
239 unsigned Opc = ST.hasFlatScratchEnabled() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
240 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
241
242 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
243 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
244 MachineMemOperand *MMO = MF.getMachineMemOperand(
245 PtrInfo, F: MachineMemOperand::MOLoad, Size: FrameInfo.getObjectSize(ObjectIdx: FI),
246 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FI));
247 TRI.buildSpillLoadStore(MBB, MI: I, DL, LoadStoreOp: Opc, Index: FI, ValueReg: SpillReg, ValueIsKill: false, ScratchOffsetReg: FrameReg,
248 InstrOffset: DwordOff, MMO, RS: nullptr, LiveUnits: &LiveUnits);
249}
250
251static void buildGitPtr(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
252 const DebugLoc &DL, const SIInstrInfo *TII,
253 Register TargetReg) {
254 MachineFunction *MF = MBB.getParent();
255 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
256 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
257 const MCInstrDesc &SMovB32 = TII->get(Opcode: AMDGPU::S_MOV_B32);
258 Register TargetLo = TRI->getSubReg(Reg: TargetReg, Idx: AMDGPU::sub0);
259 Register TargetHi = TRI->getSubReg(Reg: TargetReg, Idx: AMDGPU::sub1);
260
261 if (MFI->getGITPtrHigh() != 0xffffffff) {
262 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: TargetHi)
263 .addImm(Val: MFI->getGITPtrHigh())
264 .addReg(RegNo: TargetReg, Flags: RegState::ImplicitDefine);
265 } else {
266 const MCInstrDesc &GetPC64 = TII->get(Opcode: AMDGPU::S_GETPC_B64_pseudo);
267 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GetPC64, DestReg: TargetReg);
268 }
269 Register GitPtrLo = MFI->getGITPtrLoReg(MF: *MF);
270 MF->getRegInfo().addLiveIn(Reg: GitPtrLo);
271 MBB.addLiveIn(PhysReg: GitPtrLo);
272 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: TargetLo)
273 .addReg(RegNo: GitPtrLo);
274}
275
276static void initLiveUnits(LiveRegUnits &LiveUnits, const SIRegisterInfo &TRI,
277 const SIMachineFunctionInfo *FuncInfo,
278 MachineFunction &MF, MachineBasicBlock &MBB,
279 MachineBasicBlock::iterator MBBI, bool IsProlog) {
280 if (LiveUnits.empty()) {
281 LiveUnits.init(TRI);
282 if (IsProlog) {
283 LiveUnits.addLiveIns(MBB);
284 } else {
285 // In epilog.
286 LiveUnits.addLiveOuts(MBB);
287 LiveUnits.stepBackward(MI: *MBBI);
288 }
289 }
290}
291
292namespace llvm {
293
294// SpillBuilder to save/restore special SGPR spills like the one needed for FP,
295// BP, etc. These spills are delayed until the current function's frame is
296// finalized. For a given register, the builder uses the
297// PrologEpilogSGPRSaveRestoreInfo to decide the spill method.
298class PrologEpilogSGPRSpillBuilder {
299 MachineBasicBlock::iterator MI;
300 MachineBasicBlock &MBB;
301 MachineFunction &MF;
302 const GCNSubtarget &ST;
303 MachineFrameInfo &MFI;
304 SIMachineFunctionInfo *FuncInfo;
305 const SIInstrInfo *TII;
306 const SIRegisterInfo &TRI;
307 const MCRegisterInfo *MCRI;
308 const SIFrameLowering *TFI;
309 Register SuperReg;
310 const PrologEpilogSGPRSaveRestoreInfo SI;
311 LiveRegUnits &LiveUnits;
312 const DebugLoc &DL;
313 Register FrameReg;
314 ArrayRef<int16_t> SplitParts;
315 unsigned NumSubRegs;
316 unsigned EltSize = 4;
317 bool IsFramePtrPrologSpill;
318 bool NeedsFrameMoves;
319
320 static bool isExec(Register Reg) {
321 return Reg == AMDGPU::EXEC_LO || Reg == AMDGPU::EXEC;
322 }
323
324 /// If this builder requires SuperReg-based CFI, which is emitted after all
325 /// SubRegs are actually spilled, return the Register which should be used
326 /// as input to getDwarfRegNum. Otherwise, CFI should be generated per-SubReg.
327 ///
328 /// Note: Most spills handled by this builder generate CFI after each
329 /// SubReg spill, as each SubReg maps directly to a CFI register via
330 /// getDwarfRegNum(SubReg, false). All other cases currently currently
331 /// correspond to the SuperReg directly.
332 MCRegister getCFISuperReg() const {
333 if (IsFramePtrPrologSpill)
334 return FuncInfo->getFrameOffsetReg();
335 // FIXME: CFI for EXEC needs a fix by accurately computing the spill
336 // offset for both the low and high components.
337 if (isExec(Reg: SuperReg))
338 return AMDGPU::EXEC;
339 return {};
340 }
341
342 void saveToMemory(const int FI) const {
343 MachineRegisterInfo &MRI = MF.getRegInfo();
344 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
345 assert(!MFI.isDeadObjectIndex(FI));
346
347 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI: MI, /*IsProlog*/ true);
348
349 MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
350 MRI, LiveUnits, RC: AMDGPU::VGPR_32RegClass);
351 if (!TmpVGPR)
352 report_fatal_error(reason: "failed to find free scratch register");
353
354 auto BuildCFI = [&](Register Reg) {
355 TFI->buildCFI(MBB, MBBI: MI, DL,
356 CFIInst: MCCFIInstruction::createOffset(
357 L: nullptr, Register: MCRI->getDwarfRegNum(Reg, isEH: false),
358 Offset: MFI.getObjectOffset(ObjectIdx: FI) * ST.getWavefrontSize()));
359 };
360 MCRegister CFISuperReg = getCFISuperReg();
361 for (unsigned I = 0, DwordOff = 0; I < NumSubRegs; ++I) {
362 Register SubReg = NumSubRegs == 1
363 ? SuperReg
364 : Register(TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]));
365 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: TmpVGPR)
366 .addReg(RegNo: SubReg);
367
368 buildPrologSpill(ST, TRI, FuncInfo: *FuncInfo, LiveUnits, MF, MBB, I: MI, DL, SpillReg: TmpVGPR,
369 FI, FrameReg, DwordOff);
370 if (NeedsFrameMoves && !CFISuperReg)
371 BuildCFI(SubReg);
372 DwordOff += 4;
373 }
374 if (NeedsFrameMoves && CFISuperReg)
375 BuildCFI(CFISuperReg);
376 }
377
378 void saveToVGPRLane(const int FI) const {
379 assert(!MFI.isDeadObjectIndex(FI));
380
381 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
382 ArrayRef<SIRegisterInfo::SpilledReg> Spill =
383 FuncInfo->getSGPRSpillToPhysicalVGPRLanes(FrameIndex: FI);
384 assert(Spill.size() == NumSubRegs);
385
386 MCRegister CFISuperReg = getCFISuperReg();
387 for (unsigned I = 0; I < NumSubRegs; ++I) {
388 Register SubReg = NumSubRegs == 1
389 ? SuperReg
390 : Register(TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]));
391 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_SPILL_S32_TO_VGPR),
392 DestReg: Spill[I].VGPR)
393 .addReg(RegNo: SubReg)
394 .addImm(Val: Spill[I].Lane)
395 .addReg(RegNo: Spill[I].VGPR, Flags: RegState::Undef);
396 if (NeedsFrameMoves && !CFISuperReg)
397 TFI->buildCFIForSGPRToVGPRSpill(MBB, MBBI: MI, DL, SGPR: SubReg, VGPR: Spill[I].VGPR,
398 Lane: Spill[I].Lane);
399 }
400 if (NeedsFrameMoves && CFISuperReg)
401 TFI->buildCFIForSGPRToVGPRSpill(MBB, MBBI: MI, DL, SGPR: CFISuperReg, VGPRSpills: Spill);
402 }
403
404 void copyToScratchSGPR(Register DstReg) const {
405 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
406 .addReg(RegNo: SuperReg)
407 .setMIFlag(MachineInstr::FrameSetup);
408 if (NeedsFrameMoves) {
409 const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg: DstReg);
410 ArrayRef<int16_t> DstSplitParts = TRI.getRegSplitParts(RC, EltSize);
411 assert(NumSubRegs == (DstSplitParts.empty() ? 1 : DstSplitParts.size()));
412 MCRegister CFISuperReg = getCFISuperReg();
413 if (!CFISuperReg)
414 CFISuperReg = SuperReg;
415 int64_t DwarfCFISuperReg = MCRI->getDwarfRegNum(Reg: CFISuperReg, isEH: false);
416 int64_t DwarfDstSuperReg = MCRI->getDwarfRegNum(Reg: DstReg, isEH: false);
417 if (DwarfCFISuperReg >= 0 && DwarfDstSuperReg >= 0) {
418 TFI->buildCFI(MBB, MBBI: MI, DL,
419 CFIInst: MCCFIInstruction::createRegister(
420 L: nullptr, Register1: DwarfCFISuperReg, Register2: DwarfDstSuperReg));
421 } else if (isExec(Reg: CFISuperReg)) {
422 assert(NumSubRegs == 2 && "EXEC larger than 64-bit");
423 TFI->buildCFIForRegToSGPRPairSpill(MBB, MBBI: MI, DL, Reg: CFISuperReg, SGPRPair: DstReg);
424 } else {
425 for (unsigned I = 0; I < NumSubRegs; ++I) {
426 MCRegister SrcSubReg = TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]);
427 MCRegister DstSubReg = TRI.getSubReg(Reg: DstReg, Idx: DstSplitParts[I]);
428 TFI->buildCFI(MBB, MBBI: MI, DL,
429 CFIInst: MCCFIInstruction::createRegister(
430 L: nullptr, Register1: MCRI->getDwarfRegNum(Reg: SrcSubReg, isEH: false),
431 Register2: MCRI->getDwarfRegNum(Reg: DstSubReg, isEH: false)));
432 }
433 }
434 }
435 }
436
437 void restoreFromMemory(const int FI) {
438 MachineRegisterInfo &MRI = MF.getRegInfo();
439 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
440
441 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI: MI, /*IsProlog*/ false);
442 MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
443 MRI, LiveUnits, RC: AMDGPU::VGPR_32RegClass);
444 if (!TmpVGPR)
445 report_fatal_error(reason: "failed to find free scratch register");
446
447 for (unsigned I = 0, DwordOff = 0; I < NumSubRegs; ++I) {
448 MCRegister SubReg = NumSubRegs == 1
449 ? SuperReg.asMCReg()
450 : TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]);
451
452 buildEpilogRestore(ST, TRI, FuncInfo: *FuncInfo, LiveUnits, MF, MBB, I: MI, DL,
453 SpillReg: TmpVGPR, FI, FrameReg, DwordOff);
454 assert(SubReg.isPhysical());
455
456 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: SubReg)
457 .addReg(RegNo: TmpVGPR, Flags: RegState::Kill);
458 DwordOff += 4;
459 }
460 }
461
462 void restoreFromVGPRLane(const int FI) {
463 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
464 ArrayRef<SIRegisterInfo::SpilledReg> Spill =
465 FuncInfo->getSGPRSpillToPhysicalVGPRLanes(FrameIndex: FI);
466 assert(Spill.size() == NumSubRegs);
467
468 for (unsigned I = 0; I < NumSubRegs; ++I) {
469 MCRegister SubReg = NumSubRegs == 1
470 ? SuperReg.asMCReg()
471 : TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]);
472 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_RESTORE_S32_FROM_VGPR), DestReg: SubReg)
473 .addReg(RegNo: Spill[I].VGPR)
474 .addImm(Val: Spill[I].Lane);
475 }
476 }
477
478 void copyFromScratchSGPR(Register SrcReg) const {
479 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SuperReg)
480 .addReg(RegNo: SrcReg)
481 .setMIFlag(MachineInstr::FrameDestroy);
482 }
483
484public:
485 PrologEpilogSGPRSpillBuilder(Register Reg,
486 const PrologEpilogSGPRSaveRestoreInfo SI,
487 MachineBasicBlock &MBB,
488 MachineBasicBlock::iterator MI,
489 const DebugLoc &DL, const SIInstrInfo *TII,
490 const SIRegisterInfo &TRI,
491 LiveRegUnits &LiveUnits, Register FrameReg,
492 bool IsFramePtrPrologSpill = false)
493 : MI(MI), MBB(MBB), MF(*MBB.getParent()),
494 ST(MF.getSubtarget<GCNSubtarget>()), MFI(MF.getFrameInfo()),
495 FuncInfo(MF.getInfo<SIMachineFunctionInfo>()), TII(TII), TRI(TRI),
496 MCRI(MF.getContext().getRegisterInfo()), TFI(ST.getFrameLowering()),
497 SuperReg(Reg), SI(SI), LiveUnits(LiveUnits), DL(DL), FrameReg(FrameReg),
498 IsFramePtrPrologSpill(IsFramePtrPrologSpill),
499 NeedsFrameMoves(MF.needsFrameMoves()) {
500 const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg: SuperReg);
501 SplitParts = TRI.getRegSplitParts(RC, EltSize);
502 NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
503
504 assert(SuperReg != AMDGPU::M0 && "m0 should never spill");
505 }
506
507 void save() {
508 switch (SI.getKind()) {
509 case SGPRSaveKind::SPILL_TO_MEM:
510 return saveToMemory(FI: SI.getIndex());
511 case SGPRSaveKind::SPILL_TO_VGPR_LANE:
512 return saveToVGPRLane(FI: SI.getIndex());
513 case SGPRSaveKind::COPY_TO_SCRATCH_SGPR:
514 return copyToScratchSGPR(DstReg: SI.getReg());
515 }
516 }
517
518 void restore() {
519 switch (SI.getKind()) {
520 case SGPRSaveKind::SPILL_TO_MEM:
521 return restoreFromMemory(FI: SI.getIndex());
522 case SGPRSaveKind::SPILL_TO_VGPR_LANE:
523 return restoreFromVGPRLane(FI: SI.getIndex());
524 case SGPRSaveKind::COPY_TO_SCRATCH_SGPR:
525 return copyFromScratchSGPR(SrcReg: SI.getReg());
526 }
527 }
528};
529
530} // namespace llvm
531
532// Emit flat scratch setup code, assuming `MFI->hasFlatScratchInit()`
533void SIFrameLowering::emitEntryFunctionFlatScratchInit(
534 MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
535 const DebugLoc &DL, Register ScratchWaveOffsetReg) const {
536 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
537 const SIInstrInfo *TII = ST.getInstrInfo();
538 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
539 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
540
541 // We don't need this if we only have spills since there is no user facing
542 // scratch.
543
544 // TODO: If we know we don't have flat instructions earlier, we can omit
545 // this from the input registers.
546 //
547 // TODO: We only need to know if we access scratch space through a flat
548 // pointer. Because we only detect if flat instructions are used at all,
549 // this will be used more often than necessary on VI.
550
551 Register FlatScrInitLo;
552 Register FlatScrInitHi;
553
554 if (ST.isAmdPalOS()) {
555 // Extract the scratch offset from the descriptor in the GIT
556 LiveRegUnits LiveUnits;
557 LiveUnits.init(TRI: *TRI);
558 LiveUnits.addLiveIns(MBB);
559
560 // Find unused reg to load flat scratch init into
561 MachineRegisterInfo &MRI = MF.getRegInfo();
562 Register FlatScrInit = AMDGPU::NoRegister;
563 ArrayRef<MCPhysReg> AllSGPR64s = TRI->getAllSGPR64(MF);
564 unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 1) / 2;
565 AllSGPR64s = AllSGPR64s.slice(
566 N: std::min(a: static_cast<unsigned>(AllSGPR64s.size()), b: NumPreloaded));
567 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
568 for (MCPhysReg Reg : AllSGPR64s) {
569 if (LiveUnits.available(Reg) && !MRI.isReserved(PhysReg: Reg) &&
570 MRI.isAllocatable(PhysReg: Reg) && !TRI->isSubRegisterEq(RegA: Reg, RegB: GITPtrLoReg)) {
571 FlatScrInit = Reg;
572 break;
573 }
574 }
575 assert(FlatScrInit && "Failed to find free register for scratch init");
576
577 FlatScrInitLo = TRI->getSubReg(Reg: FlatScrInit, Idx: AMDGPU::sub0);
578 FlatScrInitHi = TRI->getSubReg(Reg: FlatScrInit, Idx: AMDGPU::sub1);
579
580 buildGitPtr(MBB, I, DL, TII, TargetReg: FlatScrInit);
581
582 // We now have the GIT ptr - now get the scratch descriptor from the entry
583 // at offset 0 (or offset 16 for a compute shader).
584 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
585 const MCInstrDesc &LoadDwordX2 = TII->get(Opcode: AMDGPU::S_LOAD_DWORDX2_IMM);
586 auto *MMO = MF.getMachineMemOperand(
587 PtrInfo,
588 F: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
589 MachineMemOperand::MODereferenceable,
590 Size: 8, BaseAlignment: Align(4));
591 unsigned Offset =
592 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
593 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
594 unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(ST: Subtarget, ByteOffset: Offset);
595 BuildMI(BB&: MBB, I, MIMD: DL, MCID: LoadDwordX2, DestReg: FlatScrInit)
596 .addReg(RegNo: FlatScrInit)
597 .addImm(Val: EncodedOffset) // offset
598 .addImm(Val: 0) // cpol
599 .addMemOperand(MMO);
600
601 // Mask the offset in [47:0] of the descriptor
602 const MCInstrDesc &SAndB32 = TII->get(Opcode: AMDGPU::S_AND_B32);
603 auto And = BuildMI(BB&: MBB, I, MIMD: DL, MCID: SAndB32, DestReg: FlatScrInitHi)
604 .addReg(RegNo: FlatScrInitHi)
605 .addImm(Val: 0xffff);
606 And->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
607 } else {
608 Register FlatScratchInitReg =
609 MFI->getPreloadedReg(Value: AMDGPUFunctionArgInfo::FLAT_SCRATCH_INIT);
610 assert(FlatScratchInitReg);
611
612 MachineRegisterInfo &MRI = MF.getRegInfo();
613 MRI.addLiveIn(Reg: FlatScratchInitReg);
614 MBB.addLiveIn(PhysReg: FlatScratchInitReg);
615
616 FlatScrInitLo = TRI->getSubReg(Reg: FlatScratchInitReg, Idx: AMDGPU::sub0);
617 FlatScrInitHi = TRI->getSubReg(Reg: FlatScratchInitReg, Idx: AMDGPU::sub1);
618 }
619
620 // Do a 64-bit pointer add.
621 if (ST.flatScratchIsPointer()) {
622 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
623 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: FlatScrInitLo)
624 .addReg(RegNo: FlatScrInitLo)
625 .addReg(RegNo: ScratchWaveOffsetReg);
626 auto Addc = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADDC_U32),
627 DestReg: FlatScrInitHi)
628 .addReg(RegNo: FlatScrInitHi)
629 .addImm(Val: 0);
630 Addc->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
631
632 using namespace AMDGPU::Hwreg;
633 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_B32))
634 .addReg(RegNo: FlatScrInitLo)
635 .addImm(Val: int16_t(HwregEncoding::encode(Values: ID_FLAT_SCR_LO, Values: 0, Values: 32)));
636 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_B32))
637 .addReg(RegNo: FlatScrInitHi)
638 .addImm(Val: int16_t(HwregEncoding::encode(Values: ID_FLAT_SCR_HI, Values: 0, Values: 32)));
639 return;
640 }
641
642 // For GFX9.
643 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: AMDGPU::FLAT_SCR_LO)
644 .addReg(RegNo: FlatScrInitLo)
645 .addReg(RegNo: ScratchWaveOffsetReg);
646 auto Addc = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADDC_U32),
647 DestReg: AMDGPU::FLAT_SCR_HI)
648 .addReg(RegNo: FlatScrInitHi)
649 .addImm(Val: 0);
650 Addc->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
651
652 return;
653 }
654
655 assert(ST.getGeneration() < AMDGPUSubtarget::GFX9);
656
657 // Copy the size in bytes.
658 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::FLAT_SCR_LO)
659 .addReg(RegNo: FlatScrInitHi, Flags: RegState::Kill);
660
661 // Add wave offset in bytes to private base offset.
662 // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init.
663 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: FlatScrInitLo)
664 .addReg(RegNo: FlatScrInitLo)
665 .addReg(RegNo: ScratchWaveOffsetReg);
666
667 // Convert offset to 256-byte units.
668 auto LShr = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_LSHR_B32),
669 DestReg: AMDGPU::FLAT_SCR_HI)
670 .addReg(RegNo: FlatScrInitLo, Flags: RegState::Kill)
671 .addImm(Val: 8);
672 LShr->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
673}
674
675// Note SGPRSpill stack IDs should only be used for SGPR spilling to VGPRs, not
676// memory. They should have been removed by now.
677static bool allStackObjectsAreDead(const MachineFrameInfo &MFI) {
678 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
679 I != E; ++I) {
680 if (!MFI.isDeadObjectIndex(ObjectIdx: I))
681 return false;
682 }
683
684 return true;
685}
686
687// Shift down registers reserved for the scratch RSRC.
688Register SIFrameLowering::getEntryFunctionReservedScratchRsrcReg(
689 MachineFunction &MF) const {
690
691 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
692 const SIInstrInfo *TII = ST.getInstrInfo();
693 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
694 MachineRegisterInfo &MRI = MF.getRegInfo();
695 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
696
697 assert(MFI->isEntryFunction());
698
699 Register ScratchRsrcReg = MFI->getScratchRSrcReg();
700
701 if (!ScratchRsrcReg || (!MRI.isPhysRegUsed(PhysReg: ScratchRsrcReg) &&
702 allStackObjectsAreDead(MFI: MF.getFrameInfo())))
703 return Register();
704
705 if (ST.hasSGPRInitBug() ||
706 ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF))
707 return ScratchRsrcReg;
708
709 // We reserved the last registers for this. Shift it down to the end of those
710 // which were actually used.
711 //
712 // FIXME: It might be safer to use a pseudoregister before replacement.
713
714 // FIXME: We should be able to eliminate unused input registers. We only
715 // cannot do this for the resources required for scratch access. For now we
716 // skip over user SGPRs and may leave unused holes.
717
718 unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 3) / 4;
719 ArrayRef<MCPhysReg> AllSGPR128s = TRI->getAllSGPR128(MF);
720 AllSGPR128s = AllSGPR128s.slice(N: std::min(a: static_cast<unsigned>(AllSGPR128s.size()), b: NumPreloaded));
721
722 // Skip the last N reserved elements because they should have already been
723 // reserved for VCC etc.
724 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
725 for (MCPhysReg Reg : AllSGPR128s) {
726 // Pick the first unallocated one. Make sure we don't clobber the other
727 // reserved input we needed. Also for PAL, make sure we don't clobber
728 // the GIT pointer passed in SGPR0 or SGPR8.
729 if (!MRI.isPhysRegUsed(PhysReg: Reg) && MRI.isAllocatable(PhysReg: Reg) &&
730 (!GITPtrLoReg || !TRI->isSubRegisterEq(RegA: Reg, RegB: GITPtrLoReg))) {
731 MRI.replaceRegWith(FromReg: ScratchRsrcReg, ToReg: Reg);
732 MFI->setScratchRSrcReg(Reg);
733 MRI.reserveReg(PhysReg: Reg, TRI);
734 return Reg;
735 }
736 }
737
738 return ScratchRsrcReg;
739}
740
741static unsigned getScratchScaleFactor(const GCNSubtarget &ST) {
742 return ST.hasFlatScratchEnabled() ? 1 : ST.getWavefrontSize();
743}
744
745void SIFrameLowering::emitEntryFunctionPrologue(MachineFunction &MF,
746 MachineBasicBlock &MBB) const {
747 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
748
749 // FIXME: If we only have SGPR spills, we won't actually be using scratch
750 // memory since these spill to VGPRs. We should be cleaning up these unused
751 // SGPR spill frame indices somewhere.
752
753 // FIXME: We still have implicit uses on SGPR spill instructions in case they
754 // need to spill to vector memory. It's likely that will not happen, but at
755 // this point it appears we need the setup. This part of the prolog should be
756 // emitted after frame indices are eliminated.
757
758 // FIXME: Remove all of the isPhysRegUsed checks
759
760 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
761 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
762 const SIInstrInfo *TII = ST.getInstrInfo();
763 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
764 MachineRegisterInfo &MRI = MF.getRegInfo();
765 const Function &F = MF.getFunction();
766 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
767
768 assert(MFI->isEntryFunction());
769
770 // Debug location must be unknown since the first debug location is used to
771 // determine the end of the prologue.
772 DebugLoc DL;
773 MachineBasicBlock::iterator I = MBB.begin();
774
775 if (MF.needsFrameMoves()) {
776 // On entry the SP/FP are not set up, so we need to define the CFA in terms
777 // of a literal location expression.
778 static const char CFAEncodedInstUserOpsArr[] = {
779 dwarf::DW_CFA_def_cfa_expression,
780 4, // length
781 static_cast<char>(dwarf::DW_OP_lit0),
782 static_cast<char>(dwarf::DW_OP_lit0 +
783 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave),
784 static_cast<char>(dwarf::DW_OP_LLVM_user),
785 static_cast<char>(dwarf::DW_OP_LLVM_form_aspace_address)};
786 static StringRef CFAEncodedInstUserOps =
787 StringRef(CFAEncodedInstUserOpsArr, sizeof(CFAEncodedInstUserOpsArr));
788 buildCFI(MBB, MBBI: I, DL,
789 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: CFAEncodedInstUserOps,
790 Loc: SMLoc(),
791 Comment: "CFA is 0 in private_wave aspace"));
792 // Unwinding halts when the return address (PC) is undefined.
793 buildCFI(MBB, MBBI: I, DL,
794 CFIInst: MCCFIInstruction::createUndefined(
795 L: nullptr, Register: TRI->getDwarfRegNum(Reg: AMDGPU::PC_REG, isEH: false)));
796 }
797
798 Register PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
799 Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET);
800
801 // We need to do the replacement of the private segment buffer register even
802 // if there are no stack objects. There could be stores to undef or a
803 // constant without an associated object.
804 //
805 // This will return `Register()` in cases where there are no actual
806 // uses of the SRSRC.
807 Register ScratchRsrcReg;
808 if (!ST.hasFlatScratchEnabled())
809 ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
810
811 // Make the selected register live throughout the function.
812 if (ScratchRsrcReg) {
813 for (MachineBasicBlock &OtherBB : MF) {
814 if (&OtherBB != &MBB) {
815 OtherBB.addLiveIn(PhysReg: ScratchRsrcReg);
816 }
817 }
818 }
819
820 // Now that we have fixed the reserved SRSRC we need to locate the
821 // (potentially) preloaded SRSRC.
822 Register PreloadedScratchRsrcReg;
823 if (ST.isAmdHsaOrMesa(F)) {
824 PreloadedScratchRsrcReg =
825 MFI->getPreloadedReg(Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
826 if (ScratchRsrcReg && PreloadedScratchRsrcReg) {
827 // We added live-ins during argument lowering, but since they were not
828 // used they were deleted. We're adding the uses now, so add them back.
829 MRI.addLiveIn(Reg: PreloadedScratchRsrcReg);
830 MBB.addLiveIn(PhysReg: PreloadedScratchRsrcReg);
831 }
832 }
833
834 // We found the SRSRC first because it needs four registers and has an
835 // alignment requirement. If the SRSRC that we found is clobbering with
836 // the scratch wave offset, which may be in a fixed SGPR or a free SGPR
837 // chosen by SITargetLowering::allocateSystemSGPRs, COPY the scratch
838 // wave offset to a free SGPR.
839 Register ScratchWaveOffsetReg;
840 if (PreloadedScratchWaveOffsetReg &&
841 TRI->isSubRegisterEq(RegA: ScratchRsrcReg, RegB: PreloadedScratchWaveOffsetReg)) {
842 ArrayRef<MCPhysReg> AllSGPRs = TRI->getAllSGPR32(MF);
843 unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
844 AllSGPRs = AllSGPRs.slice(
845 N: std::min(a: static_cast<unsigned>(AllSGPRs.size()), b: NumPreloaded));
846 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
847 for (MCPhysReg Reg : AllSGPRs) {
848 if (!MRI.isPhysRegUsed(PhysReg: Reg) && MRI.isAllocatable(PhysReg: Reg) &&
849 !TRI->isSubRegisterEq(RegA: ScratchRsrcReg, RegB: Reg) && GITPtrLoReg != Reg) {
850 ScratchWaveOffsetReg = Reg;
851 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: ScratchWaveOffsetReg)
852 .addReg(RegNo: PreloadedScratchWaveOffsetReg, Flags: RegState::Kill);
853 break;
854 }
855 }
856
857 // FIXME: We can spill incoming arguments and restore at the end of the
858 // prolog.
859 if (!ScratchWaveOffsetReg)
860 report_fatal_error(
861 reason: "could not find temporary scratch offset register in prolog");
862 } else {
863 ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg;
864 }
865 assert(ScratchWaveOffsetReg || !PreloadedScratchWaveOffsetReg);
866
867 unsigned Offset = FrameInfo.getStackSize() * getScratchScaleFactor(ST);
868 if (!mayReserveScratchForCWSR(MF)) {
869 if (hasFP(MF)) {
870 Register FPReg = MFI->getFrameOffsetReg();
871 assert(FPReg != AMDGPU::FP_REG);
872 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: FPReg).addImm(Val: 0);
873 }
874
875 if (requiresStackPointerReference(MF)) {
876 Register SPReg = MFI->getStackPtrOffsetReg();
877 assert(SPReg != AMDGPU::SP_REG);
878 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: SPReg).addImm(Val: Offset);
879 }
880 } else {
881 // We need to check if we're on a compute queue - if we are, then the CWSR
882 // trap handler may need to store some VGPRs on the stack. The first VGPR
883 // block is saved separately, so we only need to allocate space for any
884 // additional VGPR blocks used. For now, we will make sure there's enough
885 // room for the theoretical maximum number of VGPRs that can be allocated.
886 // FIXME: Figure out if the shader uses fewer VGPRs in practice.
887 assert(hasFP(MF));
888 Register FPReg = MFI->getFrameOffsetReg();
889 assert(FPReg != AMDGPU::FP_REG);
890 unsigned VGPRSize = llvm::alignTo(
891 Size: (ST.getAddressableNumVGPRs(DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize()) -
892 AMDGPU::IsaInfo::getVGPRAllocGranule(STI: ST,
893 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize())) *
894 4,
895 A: FrameInfo.getMaxAlign());
896 MFI->setScratchReservedForDynamicVGPRs(VGPRSize);
897
898 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::GET_STACK_BASE), DestReg: FPReg);
899 if (requiresStackPointerReference(MF)) {
900 Register SPReg = MFI->getStackPtrOffsetReg();
901 assert(SPReg != AMDGPU::SP_REG);
902
903 // If at least one of the constants can be inlined, then we can use
904 // s_cselect. Otherwise, use a mov and cmovk.
905 if (AMDGPU::isInlinableLiteral32(Literal: Offset, HasInv2Pi: ST.hasInv2PiInlineImm()) ||
906 AMDGPU::isInlinableLiteral32(Literal: Offset + VGPRSize,
907 HasInv2Pi: ST.hasInv2PiInlineImm())) {
908 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: SPReg)
909 .addImm(Val: Offset + VGPRSize)
910 .addImm(Val: Offset);
911 } else {
912 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: SPReg).addImm(Val: Offset);
913 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMOVK_I32), DestReg: SPReg)
914 .addImm(Val: Offset + VGPRSize);
915 }
916 }
917 }
918
919 bool NeedsFlatScratchInit =
920 MFI->getUserSGPRInfo().hasFlatScratchInit() &&
921 (MRI.isPhysRegUsed(PhysReg: AMDGPU::FLAT_SCR) || FrameInfo.hasCalls() ||
922 (!allStackObjectsAreDead(MFI: FrameInfo) && ST.hasFlatScratchEnabled()));
923
924 if ((NeedsFlatScratchInit || ScratchRsrcReg) &&
925 PreloadedScratchWaveOffsetReg && !ST.hasArchitectedFlatScratch()) {
926 MRI.addLiveIn(Reg: PreloadedScratchWaveOffsetReg);
927 MBB.addLiveIn(PhysReg: PreloadedScratchWaveOffsetReg);
928 }
929
930 if (NeedsFlatScratchInit) {
931 emitEntryFunctionFlatScratchInit(MF, MBB, I, DL, ScratchWaveOffsetReg);
932 }
933
934 if (ScratchRsrcReg) {
935 emitEntryFunctionScratchRsrcRegSetup(MF, MBB, I, DL,
936 PreloadedPrivateBufferReg: PreloadedScratchRsrcReg,
937 ScratchRsrcReg, ScratchWaveOffsetReg);
938 }
939}
940
941// Emit scratch RSRC setup code, assuming `ScratchRsrcReg != AMDGPU::NoReg`
942void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
943 MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
944 const DebugLoc &DL, Register PreloadedScratchRsrcReg,
945 Register ScratchRsrcReg, Register ScratchWaveOffsetReg) const {
946
947 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
948 const SIInstrInfo *TII = ST.getInstrInfo();
949 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
950 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
951 const Function &Fn = MF.getFunction();
952
953 if (ST.isAmdPalOS()) {
954 // The pointer to the GIT is formed from the offset passed in and either
955 // the amdgpu-git-ptr-high function attribute or the top part of the PC
956 Register Rsrc01 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub0_sub1);
957 Register Rsrc03 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub3);
958
959 buildGitPtr(MBB, I, DL, TII, TargetReg: Rsrc01);
960
961 // We now have the GIT ptr - now get the scratch descriptor from the entry
962 // at offset 0 (or offset 16 for a compute shader).
963 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
964 const MCInstrDesc &LoadDwordX4 = TII->get(Opcode: AMDGPU::S_LOAD_DWORDX4_IMM);
965 auto *MMO = MF.getMachineMemOperand(
966 PtrInfo,
967 F: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
968 MachineMemOperand::MODereferenceable,
969 Size: 16, BaseAlignment: Align(4));
970 unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
971 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
972 unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(ST: Subtarget, ByteOffset: Offset);
973 BuildMI(BB&: MBB, I, MIMD: DL, MCID: LoadDwordX4, DestReg: ScratchRsrcReg)
974 .addReg(RegNo: Rsrc01)
975 .addImm(Val: EncodedOffset) // offset
976 .addImm(Val: 0) // cpol
977 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine)
978 .addMemOperand(MMO);
979
980 // The driver will always set the SRD for wave 64 (bits 118:117 of
981 // descriptor / bits 22:21 of third sub-reg will be 0b11)
982 // If the shader is actually wave32 we have to modify the const_index_stride
983 // field of the descriptor 3rd sub-reg (bits 22:21) to 0b10 (stride=32). The
984 // reason the driver does this is that there can be cases where it presents
985 // 2 shaders with different wave size (e.g. VsFs).
986 // TODO: convert to using SCRATCH instructions or multiple SRD buffers
987 if (ST.isWave32()) {
988 const MCInstrDesc &SBitsetB32 = TII->get(Opcode: AMDGPU::S_BITSET0_B32);
989 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SBitsetB32, DestReg: Rsrc03)
990 .addImm(Val: 21)
991 .addReg(RegNo: Rsrc03);
992 }
993 } else if (ST.isMesaGfxShader(F: Fn) || !PreloadedScratchRsrcReg) {
994 assert(!ST.isAmdHsaOrMesa(Fn));
995 const MCInstrDesc &SMovB32 = TII->get(Opcode: AMDGPU::S_MOV_B32);
996
997 Register Rsrc2 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub2);
998 Register Rsrc3 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub3);
999
1000 // Use relocations to get the pointer, and setup the other bits manually.
1001 uint64_t Rsrc23 = TII->getScratchRsrcWords23();
1002
1003 if (MFI->getUserSGPRInfo().hasImplicitBufferPtr()) {
1004 Register Rsrc01 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub0_sub1);
1005
1006 if (AMDGPU::isCompute(CC: MF.getFunction().getCallingConv())) {
1007 const MCInstrDesc &Mov64 = TII->get(Opcode: AMDGPU::S_MOV_B64);
1008
1009 BuildMI(BB&: MBB, I, MIMD: DL, MCID: Mov64, DestReg: Rsrc01)
1010 .addReg(RegNo: MFI->getImplicitBufferPtrUserSGPR())
1011 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1012 } else {
1013 const MCInstrDesc &LoadDwordX2 = TII->get(Opcode: AMDGPU::S_LOAD_DWORDX2_IMM);
1014
1015 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
1016 auto *MMO = MF.getMachineMemOperand(
1017 PtrInfo,
1018 F: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1019 MachineMemOperand::MODereferenceable,
1020 Size: 8, BaseAlignment: Align(4));
1021 BuildMI(BB&: MBB, I, MIMD: DL, MCID: LoadDwordX2, DestReg: Rsrc01)
1022 .addReg(RegNo: MFI->getImplicitBufferPtrUserSGPR())
1023 .addImm(Val: 0) // offset
1024 .addImm(Val: 0) // cpol
1025 .addMemOperand(MMO)
1026 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1027
1028 MF.getRegInfo().addLiveIn(Reg: MFI->getImplicitBufferPtrUserSGPR());
1029 MBB.addLiveIn(PhysReg: MFI->getImplicitBufferPtrUserSGPR());
1030 }
1031 } else {
1032 Register Rsrc0 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub0);
1033 Register Rsrc1 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub1);
1034
1035 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: Rsrc0)
1036 .addExternalSymbol(FnName: "SCRATCH_RSRC_DWORD0")
1037 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1038
1039 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: Rsrc1)
1040 .addExternalSymbol(FnName: "SCRATCH_RSRC_DWORD1")
1041 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1042 }
1043
1044 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: Rsrc2)
1045 .addImm(Val: Lo_32(Value: Rsrc23))
1046 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1047
1048 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: Rsrc3)
1049 .addImm(Val: Hi_32(Value: Rsrc23))
1050 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1051 } else if (ST.isAmdHsaOrMesa(F: Fn)) {
1052 assert(PreloadedScratchRsrcReg);
1053
1054 if (ScratchRsrcReg != PreloadedScratchRsrcReg) {
1055 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: ScratchRsrcReg)
1056 .addReg(RegNo: PreloadedScratchRsrcReg, Flags: RegState::Kill);
1057 }
1058 }
1059
1060 // Add the scratch wave offset into the scratch RSRC.
1061 //
1062 // We only want to update the first 48 bits, which is the base address
1063 // pointer, without touching the adjacent 16 bits of flags. We know this add
1064 // cannot carry-out from bit 47, otherwise the scratch allocation would be
1065 // impossible to fit in the 48-bit global address space.
1066 //
1067 // TODO: Evaluate if it is better to just construct an SRD using the flat
1068 // scratch init and some constants rather than update the one we are passed.
1069 Register ScratchRsrcSub0 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub0);
1070 Register ScratchRsrcSub1 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub1);
1071
1072 // We cannot Kill ScratchWaveOffsetReg here because we allow it to be used in
1073 // the kernel body via inreg arguments.
1074 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: ScratchRsrcSub0)
1075 .addReg(RegNo: ScratchRsrcSub0)
1076 .addReg(RegNo: ScratchWaveOffsetReg)
1077 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1078 auto Addc = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADDC_U32), DestReg: ScratchRsrcSub1)
1079 .addReg(RegNo: ScratchRsrcSub1)
1080 .addImm(Val: 0)
1081 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1082 Addc->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
1083}
1084
1085bool SIFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
1086 switch (ID) {
1087 case TargetStackID::Default:
1088 case TargetStackID::NoAlloc:
1089 case TargetStackID::SGPRSpill:
1090 return true;
1091 case TargetStackID::ScalableVector:
1092 case TargetStackID::WasmLocal:
1093 case TargetStackID::ScalablePredicateVector:
1094 case TargetStackID::AvrAlign:
1095 return false;
1096 }
1097 llvm_unreachable("Invalid TargetStackID::Value");
1098}
1099
1100void SIFrameLowering::emitPrologueEntryCFI(MachineBasicBlock &MBB,
1101 MachineBasicBlock::iterator MBBI,
1102 const DebugLoc &DL) const {
1103 const MachineFunction &MF = *MBB.getParent();
1104 const MachineRegisterInfo &MRI = MF.getRegInfo();
1105 const MCRegisterInfo *MCRI = MF.getContext().getRegisterInfo();
1106 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1107 const SIRegisterInfo &TRI = ST.getInstrInfo()->getRegisterInfo();
1108 MCRegister StackPtrReg =
1109 MF.getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg();
1110
1111 emitDefCFA(MBB, MBBI, DL, StackPtrReg, /*AspaceAlreadyDefined=*/true,
1112 Flags: MachineInstr::FrameSetup);
1113
1114 buildCFIForRegToSGPRPairSpill(MBB, MBBI, DL, Reg: AMDGPU::PC_REG,
1115 SGPRPair: TRI.getReturnAddressReg(MF));
1116
1117 BitVector IsCalleeSaved(TRI.getNumRegs());
1118 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
1119 for (unsigned I = 0; CSRegs[I]; ++I) {
1120 IsCalleeSaved.set(CSRegs[I]);
1121 }
1122 auto ProcessReg = [&](MCPhysReg Reg) {
1123 // VCC is not preserved across calls.
1124 if (Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI)
1125 return;
1126 if (IsCalleeSaved.test(Idx: Reg) || !MRI.isPhysRegModified(PhysReg: Reg))
1127 return;
1128 unsigned DwarfReg = MCRI->getDwarfRegNum(Reg, isEH: false);
1129 buildCFI(MBB, MBBI, DL,
1130 CFIInst: MCCFIInstruction::createUndefined(L: nullptr, Register: DwarfReg));
1131 };
1132
1133 // Emit CFI rules for caller saved Arch VGPRs which are clobbered
1134 unsigned NumArchVGPRs = ST.has1024AddressableVGPRs() ? 1024 : 256;
1135 for_each(Range: AMDGPU::VGPR_32RegClass.getRegisters().take_front(N: NumArchVGPRs),
1136 F: ProcessReg);
1137
1138 // Emit CFI rules for caller saved Accum VGPRs which are clobbered
1139 if (ST.hasMAIInsts()) {
1140 for_each(Range: AMDGPU::AGPR_32RegClass.getRegisters(), F: ProcessReg);
1141 }
1142
1143 // Emit CFI rules for caller saved SGPRs which are clobbered
1144 for_each(Range: AMDGPU::SGPR_32RegClass.getRegisters(), F: ProcessReg);
1145}
1146
1147// Activate only the inactive lanes when \p EnableInactiveLanes is true.
1148// Otherwise, activate all lanes. It returns the saved exec.
1149static Register buildScratchExecCopy(LiveRegUnits &LiveUnits,
1150 MachineFunction &MF,
1151 MachineBasicBlock &MBB,
1152 MachineBasicBlock::iterator MBBI,
1153 const DebugLoc &DL, bool IsProlog,
1154 bool EnableInactiveLanes) {
1155 Register ScratchExecCopy;
1156 MachineRegisterInfo &MRI = MF.getRegInfo();
1157 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1158 const SIInstrInfo *TII = ST.getInstrInfo();
1159 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1160 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1161
1162 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, IsProlog);
1163
1164 if (FuncInfo->isWholeWaveFunction()) {
1165 // Whole wave functions already have a copy of the original EXEC mask that
1166 // we can use.
1167 assert(IsProlog && "Epilog should look at return, not setup");
1168 ScratchExecCopy =
1169 TII->getWholeWaveFunctionSetup(MF)->getOperand(i: 0).getReg();
1170 assert(ScratchExecCopy && "Couldn't find copy of EXEC");
1171 } else {
1172 ScratchExecCopy = findScratchNonCalleeSaveRegister(
1173 MRI, LiveUnits, RC: *TRI.getWaveMaskRegClass());
1174 }
1175
1176 if (!ScratchExecCopy)
1177 report_fatal_error(reason: "failed to find free scratch register");
1178
1179 LiveUnits.addReg(Reg: ScratchExecCopy);
1180
1181 const unsigned SaveExecOpc =
1182 ST.isWave32() ? (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B32
1183 : AMDGPU::S_OR_SAVEEXEC_B32)
1184 : (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B64
1185 : AMDGPU::S_OR_SAVEEXEC_B64);
1186 auto SaveExec =
1187 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: SaveExecOpc), DestReg: ScratchExecCopy).addImm(Val: -1);
1188 SaveExec->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
1189
1190 return ScratchExecCopy;
1191}
1192
1193void SIFrameLowering::emitCSRSpillStores(
1194 MachineFunction &MF, MachineBasicBlock &MBB,
1195 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1196 LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy,
1197 const bool NeedsFrameMoves) const {
1198 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1199 MachineFrameInfo &MFI = MF.getFrameInfo();
1200 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1201 const SIInstrInfo *TII = ST.getInstrInfo();
1202 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1203 const MCRegisterInfo *MCRI = MF.getContext().getRegisterInfo();
1204 MachineRegisterInfo &MRI = MF.getRegInfo();
1205 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
1206
1207 // Spill Whole-Wave Mode VGPRs. Save only the inactive lanes of the scratch
1208 // registers. However, save all lanes of callee-saved VGPRs. Due to this, we
1209 // might end up flipping the EXEC bits twice.
1210 Register ScratchExecCopy;
1211 SmallVector<std::pair<Register, int>, 2> WWMCalleeSavedRegs, WWMScratchRegs;
1212 FuncInfo->splitWWMSpillRegisters(MF, CalleeSavedRegs&: WWMCalleeSavedRegs, ScratchRegs&: WWMScratchRegs);
1213 if (!WWMScratchRegs.empty())
1214 ScratchExecCopy =
1215 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1216 /*IsProlog*/ true, /*EnableInactiveLanes*/ true);
1217
1218 auto StoreWWMRegisters =
1219 [&](SmallVectorImpl<std::pair<Register, int>> &WWMRegs) {
1220 for (const auto &Reg : WWMRegs) {
1221 Register VGPR = Reg.first;
1222 int FI = Reg.second;
1223 buildPrologSpill(ST, TRI, FuncInfo: *FuncInfo, LiveUnits, MF, MBB, I: MBBI, DL,
1224 SpillReg: VGPR, FI, FrameReg);
1225 if (NeedsFrameMoves) {
1226 // We spill the entire VGPR, so we can get away with just cfi_offset
1227 buildCFI(MBB, MBBI, DL,
1228 CFIInst: MCCFIInstruction::createOffset(
1229 L: nullptr, Register: MCRI->getDwarfRegNum(Reg: VGPR, isEH: false),
1230 Offset: MFI.getObjectOffset(ObjectIdx: FI) * ST.getWavefrontSize()));
1231 }
1232 }
1233 };
1234
1235 for (const Register Reg : make_first_range(c&: WWMScratchRegs)) {
1236 if (!MRI.isReserved(PhysReg: Reg)) {
1237 MRI.addLiveIn(Reg);
1238 MBB.addLiveIn(PhysReg: Reg);
1239 }
1240 }
1241 StoreWWMRegisters(WWMScratchRegs);
1242
1243 auto EnableAllLanes = [&]() {
1244 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
1245 };
1246
1247 if (!WWMCalleeSavedRegs.empty()) {
1248 if (ScratchExecCopy) {
1249 EnableAllLanes();
1250 } else {
1251 ScratchExecCopy = buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1252 /*IsProlog*/ true,
1253 /*EnableInactiveLanes*/ false);
1254 }
1255 }
1256
1257 StoreWWMRegisters(WWMCalleeSavedRegs);
1258 if (FuncInfo->isWholeWaveFunction()) {
1259 // If we have already saved some WWM CSR registers, then the EXEC is already
1260 // -1 and we don't need to do anything else. Otherwise, save the original
1261 // EXEC into the setup register and set EXEC to -1 here.
1262 if (!ScratchExecCopy)
1263 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL, /*IsProlog*/ true,
1264 /*EnableInactiveLanes*/ false);
1265 else if (WWMCalleeSavedRegs.empty())
1266 EnableAllLanes();
1267 } else if (ScratchExecCopy) {
1268 // FIXME: Split block and make terminator.
1269 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
1270 .addReg(RegNo: ScratchExecCopy, Flags: RegState::Kill);
1271 LiveUnits.addReg(Reg: ScratchExecCopy);
1272 }
1273
1274 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1275
1276 for (const auto &Spill : FuncInfo->getPrologEpilogSGPRSpills()) {
1277 // Special handle FP spill:
1278 // Skip if FP is saved to a scratch SGPR, the save has already been emitted.
1279 // Otherwise, FP has been moved to a temporary register and spill it
1280 // instead.
1281 bool IsFramePtrPrologSpill = Spill.first == FramePtrReg;
1282 Register Reg = IsFramePtrPrologSpill ? FramePtrRegScratchCopy : Spill.first;
1283 if (!Reg)
1284 continue;
1285
1286 PrologEpilogSGPRSpillBuilder SB(Reg, Spill.second, MBB, MBBI, DL, TII, TRI,
1287 LiveUnits, FrameReg, IsFramePtrPrologSpill);
1288 SB.save();
1289 }
1290
1291 // If a copy to scratch SGPR has been chosen for any of the SGPR spills, make
1292 // such scratch registers live throughout the function.
1293 SmallVector<Register, 1> ScratchSGPRs;
1294 FuncInfo->getAllScratchSGPRCopyDstRegs(Regs&: ScratchSGPRs);
1295 if (!ScratchSGPRs.empty()) {
1296 for (MachineBasicBlock &MBB : MF) {
1297 for (MCPhysReg Reg : ScratchSGPRs)
1298 MBB.addLiveIn(PhysReg: Reg);
1299
1300 MBB.sortUniqueLiveIns();
1301 }
1302 if (!LiveUnits.empty()) {
1303 for (MCPhysReg Reg : ScratchSGPRs)
1304 LiveUnits.addReg(Reg);
1305 }
1306 }
1307
1308 // Remove the spill entry created for EXEC. It is needed only for CFISaves in
1309 // the prologue.
1310 if (TRI.isCFISavedRegsSpillEnabled())
1311 FuncInfo->removePrologEpilogSGPRSpillEntry(Reg: TRI.getExec());
1312}
1313
1314void SIFrameLowering::emitCSRSpillRestores(
1315 MachineFunction &MF, MachineBasicBlock &MBB,
1316 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1317 LiveRegUnits &LiveUnits, Register FrameReg,
1318 Register FramePtrRegScratchCopy) const {
1319 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1320 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1321 const SIInstrInfo *TII = ST.getInstrInfo();
1322 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1323 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
1324 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1325
1326 for (const auto &Spill : FuncInfo->getPrologEpilogSGPRSpills()) {
1327 // Special handle FP restore:
1328 // Skip if FP needs to be restored from the scratch SGPR. Otherwise, restore
1329 // the FP value to a temporary register. The frame pointer should be
1330 // overwritten only at the end when all other spills are restored from
1331 // current frame.
1332 Register Reg =
1333 Spill.first == FramePtrReg ? FramePtrRegScratchCopy : Spill.first;
1334 if (!Reg)
1335 continue;
1336
1337 PrologEpilogSGPRSpillBuilder SB(Reg, Spill.second, MBB, MBBI, DL, TII, TRI,
1338 LiveUnits, FrameReg);
1339 SB.restore();
1340 }
1341
1342 // Restore Whole-Wave Mode VGPRs. Restore only the inactive lanes of the
1343 // scratch registers. However, restore all lanes of callee-saved VGPRs. Due to
1344 // this, we might end up flipping the EXEC bits twice.
1345 Register ScratchExecCopy;
1346 SmallVector<std::pair<Register, int>, 2> WWMCalleeSavedRegs, WWMScratchRegs;
1347 FuncInfo->splitWWMSpillRegisters(MF, CalleeSavedRegs&: WWMCalleeSavedRegs, ScratchRegs&: WWMScratchRegs);
1348 auto RestoreWWMRegisters =
1349 [&](SmallVectorImpl<std::pair<Register, int>> &WWMRegs) {
1350 for (const auto &Reg : WWMRegs) {
1351 Register VGPR = Reg.first;
1352 int FI = Reg.second;
1353 buildEpilogRestore(ST, TRI, FuncInfo: *FuncInfo, LiveUnits, MF, MBB, I: MBBI, DL,
1354 SpillReg: VGPR, FI, FrameReg);
1355 }
1356 };
1357
1358 if (FuncInfo->isWholeWaveFunction()) {
1359 // For whole wave functions, the EXEC is already -1 at this point.
1360 // Therefore, we can restore the CSR WWM registers right away.
1361 RestoreWWMRegisters(WWMCalleeSavedRegs);
1362
1363 // The original EXEC is the first operand of the return instruction.
1364 MachineInstr &Return = MBB.instr_back();
1365 unsigned Opcode = Return.getOpcode();
1366 switch (Opcode) {
1367 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
1368 Opcode = AMDGPU::SI_RETURN;
1369 break;
1370 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
1371 Opcode = AMDGPU::SI_TCRETURN_GFX;
1372 break;
1373 default:
1374 llvm_unreachable("Unexpected return inst");
1375 }
1376 Register OrigExec = Return.getOperand(i: 0).getReg();
1377
1378 if (!WWMScratchRegs.empty()) {
1379 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.XorOpc), DestReg: LMC.ExecReg)
1380 .addReg(RegNo: OrigExec)
1381 .addImm(Val: -1);
1382 RestoreWWMRegisters(WWMScratchRegs);
1383 }
1384
1385 // Restore original EXEC.
1386 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addReg(RegNo: OrigExec);
1387
1388 // Drop the first operand and update the opcode.
1389 Return.removeOperand(OpNo: 0);
1390 Return.setDesc(TII->get(Opcode));
1391
1392 return;
1393 }
1394
1395 if (!WWMScratchRegs.empty()) {
1396 ScratchExecCopy =
1397 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1398 /*IsProlog=*/false, /*EnableInactiveLanes=*/true);
1399 }
1400 RestoreWWMRegisters(WWMScratchRegs);
1401 if (!WWMCalleeSavedRegs.empty()) {
1402 if (ScratchExecCopy) {
1403 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
1404 } else {
1405 ScratchExecCopy = buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1406 /*IsProlog*/ false,
1407 /*EnableInactiveLanes*/ false);
1408 }
1409 }
1410
1411 RestoreWWMRegisters(WWMCalleeSavedRegs);
1412 if (ScratchExecCopy) {
1413 // FIXME: Split block and make terminator.
1414 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
1415 .addReg(RegNo: ScratchExecCopy, Flags: RegState::Kill);
1416 }
1417}
1418
1419void SIFrameLowering::emitPrologue(MachineFunction &MF,
1420 MachineBasicBlock &MBB) const {
1421 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1422 if (FuncInfo->isEntryFunction()) {
1423 emitEntryFunctionPrologue(MF, MBB);
1424 return;
1425 }
1426
1427 MachineFrameInfo &MFI = MF.getFrameInfo();
1428 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1429 const SIInstrInfo *TII = ST.getInstrInfo();
1430 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1431 MachineRegisterInfo &MRI = MF.getRegInfo();
1432
1433 Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
1434 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1435 Register BasePtrReg =
1436 TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register();
1437 LiveRegUnits LiveUnits;
1438
1439 MachineBasicBlock::iterator MBBI = MBB.begin();
1440 // DebugLoc must be unknown since the first instruction with DebugLoc is used
1441 // to determine the end of the prologue.
1442 DebugLoc DL;
1443
1444 bool HasFP = false;
1445 bool HasBP = false;
1446 uint32_t NumBytes = MFI.getStackSize();
1447 uint32_t RoundedSize = NumBytes;
1448
1449 // Functions that never return don't need to save and restore the FP or BP.
1450 const Function &F = MF.getFunction();
1451 bool SavesStackRegs =
1452 !F.hasFnAttribute(Kind: Attribute::NoReturn) && !FuncInfo->isChainFunction();
1453
1454 const bool NeedsFrameMoves = MF.needsFrameMoves();
1455
1456 if (NeedsFrameMoves)
1457 emitPrologueEntryCFI(MBB, MBBI, DL);
1458
1459 if (TRI.hasStackRealignment(MF))
1460 HasFP = true;
1461
1462 Register FramePtrRegScratchCopy;
1463 if (!HasFP && !hasFP(MF)) {
1464 // Emit the CSR spill stores with SP base register.
1465 emitCSRSpillStores(MF, MBB, MBBI, DL, LiveUnits, FrameReg: StackPtrReg,
1466 FramePtrRegScratchCopy, NeedsFrameMoves);
1467 } else if (SavesStackRegs) {
1468 // CSR spill stores will use FP as base register.
1469 Register SGPRForFPSaveRestoreCopy =
1470 FuncInfo->getScratchSGPRCopyDstReg(Reg: FramePtrReg);
1471
1472 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ true);
1473 if (SGPRForFPSaveRestoreCopy) {
1474 // Copy FP to the scratch register now and emit the CFI entry. It avoids
1475 // the extra FP copy needed in the other two cases when FP is spilled to
1476 // memory or to a VGPR lane.
1477 PrologEpilogSGPRSpillBuilder SB(
1478 FramePtrReg,
1479 FuncInfo->getPrologEpilogSGPRSaveRestoreInfo(Reg: FramePtrReg), MBB, MBBI,
1480 DL, TII, TRI, LiveUnits, FramePtrReg,
1481 /*IsFramePtrPrologSpill*/ true);
1482 SB.save();
1483 LiveUnits.addReg(Reg: SGPRForFPSaveRestoreCopy);
1484 } else {
1485 // Copy FP into a new scratch register so that its previous value can be
1486 // spilled after setting up the new frame.
1487 FramePtrRegScratchCopy = findScratchNonCalleeSaveRegister(
1488 MRI, LiveUnits, RC: AMDGPU::SReg_32_XM0_XEXECRegClass);
1489 if (!FramePtrRegScratchCopy)
1490 report_fatal_error(reason: "failed to find free scratch register");
1491
1492 LiveUnits.addReg(Reg: FramePtrRegScratchCopy);
1493 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: FramePtrRegScratchCopy)
1494 .addReg(RegNo: FramePtrReg);
1495 }
1496 }
1497
1498 if (HasFP) {
1499 const unsigned Alignment = MFI.getMaxAlign().value();
1500
1501 RoundedSize += Alignment;
1502 if (LiveUnits.empty()) {
1503 LiveUnits.init(TRI);
1504 LiveUnits.addLiveIns(MBB);
1505 }
1506
1507 // s_add_i32 s33, s32, NumBytes
1508 // s_and_b32 s33, s33, 0b111...0000
1509 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: FramePtrReg)
1510 .addReg(RegNo: StackPtrReg)
1511 .addImm(Val: (Alignment - 1) * getScratchScaleFactor(ST))
1512 .setMIFlag(MachineInstr::FrameSetup);
1513 auto And = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_AND_B32), DestReg: FramePtrReg)
1514 .addReg(RegNo: FramePtrReg, Flags: RegState::Kill)
1515 .addImm(Val: -Alignment * getScratchScaleFactor(ST))
1516 .setMIFlag(MachineInstr::FrameSetup);
1517 And->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
1518 FuncInfo->setIsStackRealigned(true);
1519 } else if ((HasFP = hasFP(MF))) {
1520 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: FramePtrReg)
1521 .addReg(RegNo: StackPtrReg)
1522 .setMIFlag(MachineInstr::FrameSetup);
1523 }
1524
1525 // If FP is used, emit the CSR spills with FP base register.
1526 if (HasFP) {
1527 emitCSRSpillStores(MF, MBB, MBBI, DL, LiveUnits, FrameReg: FramePtrReg,
1528 FramePtrRegScratchCopy, NeedsFrameMoves);
1529 if (FramePtrRegScratchCopy)
1530 LiveUnits.removeReg(Reg: FramePtrRegScratchCopy);
1531 }
1532
1533 // If we need a base pointer, set it up here. It's whatever the value of
1534 // the stack pointer is at this point. Any variable size objects will be
1535 // allocated after this, so we can still use the base pointer to reference
1536 // the incoming arguments.
1537 if ((HasBP = TRI.hasBasePointer(MF))) {
1538 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: BasePtrReg)
1539 .addReg(RegNo: StackPtrReg)
1540 .setMIFlag(MachineInstr::FrameSetup);
1541 }
1542
1543 if (HasFP) {
1544 if (NeedsFrameMoves)
1545 emitDefCFA(MBB, MBBI, DL, StackPtrReg: FramePtrReg, /*AspaceAlreadyDefined=*/false,
1546 Flags: MachineInstr::FrameSetup);
1547 }
1548
1549 if (HasFP && RoundedSize != 0) {
1550 auto Add = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: StackPtrReg)
1551 .addReg(RegNo: StackPtrReg)
1552 .addImm(Val: RoundedSize * getScratchScaleFactor(ST))
1553 .setMIFlag(MachineInstr::FrameSetup);
1554 Add->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
1555 }
1556
1557 bool FPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(Reg: FramePtrReg);
1558 (void)FPSaved;
1559 assert((!HasFP || FPSaved || !SavesStackRegs) &&
1560 "Needed to save FP but didn't save it anywhere");
1561
1562 // If we allow spilling to AGPRs we may have saved FP but then spill
1563 // everything into AGPRs instead of the stack.
1564 assert((HasFP || !FPSaved || !SavesStackRegs || EnableSpillVGPRToAGPR) &&
1565 "Saved FP but didn't need it");
1566
1567 bool BPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(Reg: BasePtrReg);
1568 (void)BPSaved;
1569 assert((!HasBP || BPSaved || !SavesStackRegs) &&
1570 "Needed to save BP but didn't save it anywhere");
1571
1572 assert((HasBP || !BPSaved) && "Saved BP but didn't need it");
1573
1574 if (FuncInfo->isWholeWaveFunction()) {
1575 // SI_WHOLE_WAVE_FUNC_SETUP has outlived its purpose.
1576 TII->getWholeWaveFunctionSetup(MF)->eraseFromParent();
1577 }
1578}
1579
1580void SIFrameLowering::emitEpilogue(MachineFunction &MF,
1581 MachineBasicBlock &MBB) const {
1582 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1583 if (FuncInfo->isEntryFunction())
1584 return;
1585
1586 const MachineFrameInfo &MFI = MF.getFrameInfo();
1587 if (FuncInfo->isChainFunction() && !MFI.hasTailCall())
1588 return;
1589
1590 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1591 const SIInstrInfo *TII = ST.getInstrInfo();
1592 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1593 MachineRegisterInfo &MRI = MF.getRegInfo();
1594 LiveRegUnits LiveUnits;
1595 // Get the insert location for the epilogue. If there were no terminators in
1596 // the block, get the last instruction.
1597 MachineBasicBlock::iterator MBBI = MBB.end();
1598 DebugLoc DL;
1599 if (!MBB.empty()) {
1600 MBBI = MBB.getLastNonDebugInstr();
1601 if (MBBI != MBB.end())
1602 DL = MBBI->getDebugLoc();
1603
1604 MBBI = MBB.getFirstTerminator();
1605 }
1606
1607 uint32_t NumBytes = MFI.getStackSize();
1608 uint32_t RoundedSize = FuncInfo->isStackRealigned()
1609 ? NumBytes + MFI.getMaxAlign().value()
1610 : NumBytes;
1611 const Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
1612 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1613 bool FPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(Reg: FramePtrReg);
1614
1615 if (RoundedSize != 0) {
1616 if (TRI.hasBasePointer(MF)) {
1617 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: StackPtrReg)
1618 .addReg(RegNo: TRI.getBaseRegister())
1619 .setMIFlag(MachineInstr::FrameDestroy);
1620 } else if (hasFP(MF)) {
1621 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: StackPtrReg)
1622 .addReg(RegNo: FramePtrReg)
1623 .setMIFlag(MachineInstr::FrameDestroy);
1624 }
1625 }
1626
1627 Register FramePtrRegScratchCopy;
1628 Register SGPRForFPSaveRestoreCopy =
1629 FuncInfo->getScratchSGPRCopyDstReg(Reg: FramePtrReg);
1630 if (FPSaved) {
1631 // CSR spill restores should use FP as base register. If
1632 // SGPRForFPSaveRestoreCopy is not true, restore the previous value of FP
1633 // into a new scratch register and copy to FP later when other registers are
1634 // restored from the current stack frame.
1635 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ false);
1636 if (SGPRForFPSaveRestoreCopy) {
1637 LiveUnits.addReg(Reg: SGPRForFPSaveRestoreCopy);
1638 } else {
1639 FramePtrRegScratchCopy = findScratchNonCalleeSaveRegister(
1640 MRI, LiveUnits, RC: AMDGPU::SReg_32_XM0_XEXECRegClass);
1641 if (!FramePtrRegScratchCopy)
1642 report_fatal_error(reason: "failed to find free scratch register");
1643
1644 LiveUnits.addReg(Reg: FramePtrRegScratchCopy);
1645 }
1646
1647 emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveUnits, FrameReg: FramePtrReg,
1648 FramePtrRegScratchCopy);
1649 }
1650
1651 if (hasFP(MF) && MF.needsFrameMoves()) {
1652 emitDefCFA(MBB, MBBI, DL, StackPtrReg, /*AspaceAlreadyDefined=*/false,
1653 Flags: MachineInstr::FrameDestroy);
1654 }
1655
1656 if (FPSaved) {
1657 // Insert the copy to restore FP.
1658 Register SrcReg = SGPRForFPSaveRestoreCopy ? SGPRForFPSaveRestoreCopy
1659 : FramePtrRegScratchCopy;
1660 MachineInstrBuilder MIB =
1661 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: FramePtrReg)
1662 .addReg(RegNo: SrcReg);
1663 if (SGPRForFPSaveRestoreCopy)
1664 MIB.setMIFlag(MachineInstr::FrameDestroy);
1665 } else {
1666 // Insert the CSR spill restores with SP as the base register.
1667 emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveUnits, FrameReg: StackPtrReg,
1668 FramePtrRegScratchCopy);
1669 }
1670}
1671
1672#ifndef NDEBUG
1673static bool allSGPRSpillsAreDead(const MachineFunction &MF) {
1674 const MachineFrameInfo &MFI = MF.getFrameInfo();
1675 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1676 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
1677 I != E; ++I) {
1678 if (!MFI.isDeadObjectIndex(I) &&
1679 MFI.getStackID(I) == TargetStackID::SGPRSpill &&
1680 !FuncInfo->checkIndexInPrologEpilogSGPRSpills(I)) {
1681 return false;
1682 }
1683 }
1684
1685 return true;
1686}
1687#endif
1688
1689StackOffset SIFrameLowering::getFrameIndexReference(const MachineFunction &MF,
1690 int FI,
1691 Register &FrameReg) const {
1692 const SIRegisterInfo *RI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
1693
1694 FrameReg = RI->getFrameRegister(MF);
1695 return StackOffset::getFixed(Fixed: MF.getFrameInfo().getObjectOffset(ObjectIdx: FI));
1696}
1697
1698void SIFrameLowering::processFunctionBeforeFrameFinalized(
1699 MachineFunction &MF,
1700 RegScavenger *RS) const {
1701 MachineFrameInfo &MFI = MF.getFrameInfo();
1702
1703 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1704 const SIInstrInfo *TII = ST.getInstrInfo();
1705 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1706 MachineRegisterInfo &MRI = MF.getRegInfo();
1707 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1708
1709 const bool SpillVGPRToAGPR = ST.hasMAIInsts() && FuncInfo->hasSpilledVGPRs()
1710 && EnableSpillVGPRToAGPR;
1711
1712 if (SpillVGPRToAGPR) {
1713 // To track the spill frame indices handled in this pass.
1714 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
1715 BitVector NonVGPRSpillFIs(MFI.getObjectIndexEnd(), false);
1716
1717 bool SeenDbgInstr = false;
1718
1719 for (MachineBasicBlock &MBB : MF) {
1720 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
1721 int FrameIndex;
1722 if (MI.isDebugInstr())
1723 SeenDbgInstr = true;
1724
1725 if (TII->isVGPRSpill(MI)) {
1726 // Try to eliminate stack used by VGPR spills before frame
1727 // finalization.
1728 unsigned FIOp = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
1729 Name: AMDGPU::OpName::vaddr);
1730 int FI = MI.getOperand(i: FIOp).getIndex();
1731 Register VReg =
1732 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata)->getReg();
1733 if (FuncInfo->allocateVGPRSpillToAGPR(MF, FI,
1734 isAGPRtoVGPR: TRI->isAGPR(MRI, Reg: VReg))) {
1735 assert(RS != nullptr);
1736 RS->enterBasicBlockEnd(MBB);
1737 RS->backward(I: std::next(x: MI.getIterator()));
1738 TRI->eliminateFrameIndex(MI, SPAdj: 0, FIOperandNum: FIOp, RS);
1739 SpillFIs.set(FI);
1740 continue;
1741 }
1742 } else if (TII->isStoreToStackSlot(MI, FrameIndex) ||
1743 TII->isLoadFromStackSlot(MI, FrameIndex))
1744 if (!MFI.isFixedObjectIndex(ObjectIdx: FrameIndex))
1745 NonVGPRSpillFIs.set(FrameIndex);
1746 }
1747 }
1748
1749 // Stack slot coloring may assign different objects to the same stack slot.
1750 // If not, then the VGPR to AGPR spill slot is dead.
1751 for (unsigned FI : SpillFIs.set_bits())
1752 if (!NonVGPRSpillFIs.test(Idx: FI))
1753 FuncInfo->setVGPRToAGPRSpillDead(FI);
1754
1755 for (MachineBasicBlock &MBB : MF) {
1756 for (MCPhysReg Reg : FuncInfo->getVGPRSpillAGPRs())
1757 MBB.addLiveIn(PhysReg: Reg);
1758
1759 for (MCPhysReg Reg : FuncInfo->getAGPRSpillVGPRs())
1760 MBB.addLiveIn(PhysReg: Reg);
1761
1762 MBB.sortUniqueLiveIns();
1763
1764 if (!SpillFIs.empty() && SeenDbgInstr)
1765 clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
1766 }
1767 }
1768
1769 // At this point we've already allocated all spilled SGPRs to VGPRs if we
1770 // can. Any remaining SGPR spills will go to memory, so move them back to the
1771 // default stack.
1772 bool HaveSGPRToVMemSpill =
1773 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ true);
1774 assert(allSGPRSpillsAreDead(MF) &&
1775 "SGPR spill should have been removed in SILowerSGPRSpills");
1776
1777 // FIXME: The other checks should be redundant with allStackObjectsAreDead,
1778 // but currently hasNonSpillStackObjects is set only from source
1779 // allocas. Stack temps produced from legalization are not counted currently.
1780 if (!allStackObjectsAreDead(MFI)) {
1781 assert(RS && "RegScavenger required if spilling");
1782
1783 // Add an emergency spill slot
1784 RS->addScavengingFrameIndex(FI: FuncInfo->getScavengeFI(MFI, TRI: *TRI));
1785
1786 if (HaveSGPRToVMemSpill && FuncInfo->hasNoWWMPoolSGPRSpillFallback()) {
1787 // The no-WWM-pool fallback can reach SGPR-to-memory lowering while an
1788 // ordinary frame-index scavenge is live. It may then need one slot for
1789 // its temporary VGPR and another for recursive address materialization.
1790 RS->addScavengingFrameIndex(FI: MFI.CreateSpillStackObject(Size: 4, Alignment: Align(4)));
1791 RS->addScavengingFrameIndex(FI: MFI.CreateSpillStackObject(Size: 4, Alignment: Align(4)));
1792 } else if (HaveSGPRToVMemSpill &&
1793 allocateScavengingFrameIndexesNearIncomingSP(MF)) {
1794 // Existing large-frame SGPR-to-memory spills need one additional VGPR
1795 // emergency frame index.
1796 RS->addScavengingFrameIndex(FI: MFI.CreateSpillStackObject(Size: 4, Alignment: Align(4)));
1797 }
1798 }
1799}
1800
1801void SIFrameLowering::processFunctionBeforeFrameIndicesReplaced(
1802 MachineFunction &MF, RegScavenger *RS) const {
1803 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1804 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1805 MachineRegisterInfo &MRI = MF.getRegInfo();
1806 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1807
1808 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
1809 // On gfx908, we had initially reserved highest available VGPR for AGPR
1810 // copy. Now since we are done with RA, check if there exist an unused VGPR
1811 // which is lower than the eariler reserved VGPR before RA. If one exist,
1812 // use it for AGPR copy instead of one reserved before RA.
1813 Register VGPRForAGPRCopy = FuncInfo->getVGPRForAGPRCopy();
1814 Register UnusedLowVGPR =
1815 TRI->findUnusedRegister(MRI, RC: &AMDGPU::VGPR_32RegClass, MF);
1816 if (UnusedLowVGPR && (TRI->getHWRegIndex(Reg: UnusedLowVGPR) <
1817 TRI->getHWRegIndex(Reg: VGPRForAGPRCopy))) {
1818 // Reserve this newly identified VGPR (for AGPR copy)
1819 // reserved registers should already be frozen at this point
1820 // so we can avoid calling MRI.freezeReservedRegs and just use
1821 // MRI.reserveReg
1822 FuncInfo->setVGPRForAGPRCopy(UnusedLowVGPR);
1823 MRI.reserveReg(PhysReg: UnusedLowVGPR, TRI);
1824 }
1825 }
1826 // We initally reserved the highest available SGPR pair for long branches
1827 // now, after RA, we shift down to a lower unused one if one exists
1828 Register LongBranchReservedReg = FuncInfo->getLongBranchReservedReg();
1829 Register UnusedLowSGPR =
1830 TRI->findUnusedRegister(MRI, RC: &AMDGPU::SGPR_64RegClass, MF);
1831 // If LongBranchReservedReg is null then we didn't find a long branch
1832 // and never reserved a register to begin with so there is nothing to
1833 // shift down. Then if UnusedLowSGPR is null, there isn't available lower
1834 // register to use so just keep the original one we set.
1835 if (LongBranchReservedReg && UnusedLowSGPR) {
1836 FuncInfo->setLongBranchReservedReg(UnusedLowSGPR);
1837 MRI.reserveReg(PhysReg: UnusedLowSGPR, TRI);
1838 }
1839}
1840
1841// The special SGPR spills like the one needed for FP, BP or any reserved
1842// registers delayed until frame lowering.
1843void SIFrameLowering::determinePrologEpilogSGPRSaves(
1844 MachineFunction &MF, BitVector &SavedVGPRs,
1845 bool NeedExecCopyReservedReg) const {
1846 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
1847 MachineRegisterInfo &MRI = MF.getRegInfo();
1848 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1849 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1850 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1851 LiveRegUnits LiveUnits;
1852 LiveUnits.init(TRI: *TRI);
1853 // Initially mark callee saved registers as used so we will not choose them
1854 // while looking for scratch SGPRs.
1855 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
1856 for (unsigned I = 0; CSRegs[I]; ++I)
1857 LiveUnits.addReg(Reg: CSRegs[I]);
1858
1859 const TargetRegisterClass &RC = *TRI->getWaveMaskRegClass();
1860
1861 Register ReservedRegForExecCopy = MFI->getSGPRForEXECCopy();
1862 if (NeedExecCopyReservedReg ||
1863 (ReservedRegForExecCopy &&
1864 MRI.isPhysRegUsed(PhysReg: ReservedRegForExecCopy, /*SkipRegMaskTest=*/true))) {
1865 MRI.reserveReg(PhysReg: ReservedRegForExecCopy, TRI);
1866 Register UnusedScratchReg = findUnusedRegister(MRI, LiveUnits, RC);
1867 if (UnusedScratchReg) {
1868 // If found any unused scratch SGPR, reserve the register itself for Exec
1869 // copy and there is no need for any spills in that case.
1870 MFI->setSGPRForEXECCopy(UnusedScratchReg);
1871 MRI.replaceRegWith(FromReg: ReservedRegForExecCopy, ToReg: UnusedScratchReg);
1872 LiveUnits.addReg(Reg: UnusedScratchReg);
1873 } else {
1874 // Needs spill.
1875 assert(!MFI->hasPrologEpilogSGPRSpillEntry(ReservedRegForExecCopy) &&
1876 "Re-reserving spill slot for EXEC copy register");
1877 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, SGPR: ReservedRegForExecCopy, RC,
1878 /*IncludeScratchCopy=*/false);
1879 }
1880 } else if (ReservedRegForExecCopy) {
1881 // Reset it at this point. There are no whole-wave copies and spills
1882 // encountered.
1883 MFI->setSGPRForEXECCopy(AMDGPU::NoRegister);
1884 }
1885
1886 if (TRI->isCFISavedRegsSpillEnabled()) {
1887 Register Exec = TRI->getExec();
1888 assert(!MFI->hasPrologEpilogSGPRSpillEntry(Exec) &&
1889 "Re-reserving spill slot for EXEC");
1890 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, SGPR: Exec, RC);
1891 }
1892
1893 // Functions that don't return to the caller don't need to preserve
1894 // the FP and BP.
1895 const Function &F = MF.getFunction();
1896 if (F.hasFnAttribute(Kind: Attribute::NoReturn) ||
1897 AMDGPU::isChainCC(CC: F.getCallingConv()))
1898 return;
1899
1900 // hasFP only knows about stack objects that already exist. We're now
1901 // determining the stack slots that will be created, so we have to predict
1902 // them. Stack objects force FP usage with calls.
1903 //
1904 // Note a new VGPR CSR may be introduced if one is used for the spill, but we
1905 // don't want to report it here.
1906 //
1907 // FIXME: Is this really hasReservedCallFrame?
1908 const bool WillHaveFP =
1909 FrameInfo.hasCalls() &&
1910 (SavedVGPRs.any() || !allStackObjectsAreDead(MFI: FrameInfo));
1911
1912 if (WillHaveFP || hasFP(MF)) {
1913 Register FramePtrReg = MFI->getFrameOffsetReg();
1914 assert(!MFI->hasPrologEpilogSGPRSpillEntry(FramePtrReg) &&
1915 "Re-reserving spill slot for FP");
1916 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, SGPR: FramePtrReg);
1917 }
1918
1919 if (TRI->hasBasePointer(MF)) {
1920 Register BasePtrReg = TRI->getBaseRegister();
1921 assert(!MFI->hasPrologEpilogSGPRSpillEntry(BasePtrReg) &&
1922 "Re-reserving spill slot for BP");
1923 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, SGPR: BasePtrReg);
1924 }
1925}
1926
1927// Only report VGPRs to generic code.
1928void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
1929 BitVector &SavedVGPRs,
1930 RegScavenger *RS) const {
1931 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1932
1933 // If this is a function with the amdgpu_cs_chain[_preserve] calling
1934 // convention and it doesn't contain any calls to llvm.amdgcn.cs.chain, then
1935 // we don't need to save and restore anything.
1936 if (MFI->isChainFunction() && !MF.getFrameInfo().hasTailCall())
1937 return;
1938
1939 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs&: SavedVGPRs, RS);
1940
1941 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1942 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1943 const SIInstrInfo *TII = ST.getInstrInfo();
1944 bool NeedExecCopyReservedReg = false;
1945
1946 MachineInstr *ReturnMI = nullptr;
1947 for (MachineBasicBlock &MBB : MF) {
1948 for (MachineInstr &MI : MBB) {
1949 // TODO: Walking through all MBBs here would be a bad heuristic. Better
1950 // handle them elsewhere.
1951 if (TII->isWWMRegSpillOpcode(Opcode: MI.getOpcode()))
1952 NeedExecCopyReservedReg = true;
1953 else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
1954 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
1955 MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
1956 (MFI->isChainFunction() &&
1957 TII->isChainCallOpcode(Opcode: MI.getOpcode()))) {
1958 // We expect all return to be the same size.
1959 assert(!ReturnMI ||
1960 (count_if(MI.operands(), [](auto Op) { return Op.isReg(); }) ==
1961 count_if(ReturnMI->operands(), [](auto Op) { return Op.isReg(); })));
1962 ReturnMI = &MI;
1963 }
1964 }
1965 }
1966
1967 SmallVector<Register> SortedWWMVGPRs;
1968 for (Register Reg : MFI->getWWMReservedRegs()) {
1969 // The shift-back is needed only for the VGPRs used for SGPR spills and they
1970 // are of 32-bit size. SIPreAllocateWWMRegs pass can add tuples into WWM
1971 // reserved registers.
1972 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
1973 if (TRI->getRegSizeInBits(RC: *RC) != 32)
1974 continue;
1975 SortedWWMVGPRs.push_back(Elt: Reg);
1976 }
1977
1978 sort(C&: SortedWWMVGPRs, Comp: std::greater<Register>());
1979 MFI->shiftWwmVGPRsToLowestRange(MF, WWMVGPRs&: SortedWWMVGPRs, SavedVGPRs);
1980
1981 if (MFI->isEntryFunction())
1982 return;
1983
1984 if (MFI->isWholeWaveFunction()) {
1985 // In practice, all the VGPRs are WWM registers, and we will need to save at
1986 // least their inactive lanes. Add them to WWMReservedRegs.
1987 assert(!NeedExecCopyReservedReg &&
1988 "Whole wave functions can use the reg mapped for their i1 argument");
1989
1990 unsigned NumArchVGPRs = ST.getAddressableNumArchVGPRs();
1991 for (MCRegister Reg :
1992 AMDGPU::VGPR_32RegClass.getRegisters().take_front(N: NumArchVGPRs))
1993 if (MF.getRegInfo().isPhysRegModified(PhysReg: Reg)) {
1994 MFI->reserveWWMRegister(Reg);
1995 MF.begin()->addLiveIn(PhysReg: Reg);
1996 }
1997 MF.begin()->sortUniqueLiveIns();
1998 }
1999
2000 // Remove any VGPRs used in the return value because these do not need to be saved.
2001 // This prevents CSR restore from clobbering return VGPRs.
2002 if (ReturnMI) {
2003 for (auto &Op : ReturnMI->operands()) {
2004 if (Op.isReg())
2005 SavedVGPRs.reset(Idx: Op.getReg());
2006 }
2007 }
2008
2009 // Create the stack objects for WWM registers now.
2010 for (Register Reg : MFI->getWWMReservedRegs()) {
2011 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
2012 MFI->allocateWWMSpill(MF, VGPR: Reg, Size: TRI->getSpillSize(RC: *RC),
2013 Alignment: TRI->getSpillAlign(RC: *RC));
2014 }
2015
2016 // Ignore the SGPRs the default implementation found.
2017 SavedVGPRs.clearBitsNotInMask(Mask: TRI->getAllVectorRegMask());
2018
2019 // Do not save AGPRs prior to GFX90A because there was no easy way to do so.
2020 // In gfx908 there was do AGPR loads and stores and thus spilling also
2021 // require a temporary VGPR.
2022 if (!ST.hasGFX90AInsts())
2023 SavedVGPRs.clearBitsInMask(Mask: TRI->getAllAGPRRegMask());
2024
2025 determinePrologEpilogSGPRSaves(MF, SavedVGPRs, NeedExecCopyReservedReg);
2026
2027 // The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't
2028 // allow the default insertion to handle them.
2029 for (auto &Reg : MFI->getWWMSpills())
2030 SavedVGPRs.reset(Idx: Reg.first);
2031}
2032
2033void SIFrameLowering::determineCalleeSavesSGPR(MachineFunction &MF,
2034 BitVector &SavedRegs,
2035 RegScavenger *RS) const {
2036 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2037 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2038 if (MFI->isEntryFunction())
2039 return;
2040
2041 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2042 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2043
2044 // The SP is specifically managed and we don't want extra spills of it.
2045 SavedRegs.reset(Idx: MFI->getStackPtrOffsetReg());
2046
2047 const BitVector AllSavedRegs = SavedRegs;
2048 SavedRegs.clearBitsInMask(Mask: TRI->getAllVectorRegMask());
2049
2050 // We have to anticipate introducing CSR VGPR spills or spill of caller
2051 // save VGPR reserved for SGPR spills as we now always create stack entry
2052 // for it, if we don't have any stack objects already, since we require a FP
2053 // if there is a call and stack. We will allocate a VGPR for SGPR spills if
2054 // there are any SGPR spills. Whether they are CSR spills or otherwise.
2055 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
2056 const bool WillHaveFP =
2057 FrameInfo.hasCalls() && (AllSavedRegs.any() || MFI->hasSpilledSGPRs());
2058
2059 // FP will be specially managed like SP.
2060 if (WillHaveFP || hasFP(MF))
2061 SavedRegs.reset(Idx: MFI->getFrameOffsetReg());
2062
2063 // Return address use with return instruction is hidden through the SI_RETURN
2064 // pseudo. Given that and since the IPRA computes actual register usage and
2065 // does not use CSR list, the clobbering of return address by function calls
2066 // (D117243) or otherwise (D120922) is ignored/not seen by the IPRA's register
2067 // usage collection. This will ensure save/restore of return address happens
2068 // in those scenarios.
2069 const MachineRegisterInfo &MRI = MF.getRegInfo();
2070 Register RetAddrReg = TRI->getReturnAddressReg(MF);
2071 if (!MFI->isEntryFunction() &&
2072 (FrameInfo.hasCalls() || MRI.isPhysRegModified(PhysReg: RetAddrReg))) {
2073 SavedRegs.set(TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub0));
2074 SavedRegs.set(TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub1));
2075 }
2076}
2077
2078static void assignSlotsUsingVGPRBlocks(MachineFunction &MF,
2079 const GCNSubtarget &ST,
2080 std::vector<CalleeSavedInfo> &CSI) {
2081 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
2082 MachineFrameInfo &MFI = MF.getFrameInfo();
2083 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2084
2085 assert(
2086 llvm::is_sorted(CSI,
2087 [](const CalleeSavedInfo &A, const CalleeSavedInfo &B) {
2088 return A.getReg() < B.getReg();
2089 }) &&
2090 "Callee saved registers not sorted");
2091
2092 auto CanUseBlockOps = [&](const CalleeSavedInfo &CSI) {
2093 return !CSI.isSpilledToReg() &&
2094 TRI->getPhysRegBaseClass(Reg: CSI.getReg()) == &AMDGPU::VGPR_32RegClass &&
2095 !FuncInfo->isWWMReservedRegister(Reg: CSI.getReg());
2096 };
2097
2098 auto CSEnd = CSI.end();
2099 for (auto CSIt = CSI.begin(); CSIt != CSEnd; ++CSIt) {
2100 Register Reg = CSIt->getReg();
2101 if (!CanUseBlockOps(*CSIt))
2102 continue;
2103
2104 // Find all the regs that will fit in a 32-bit mask starting at the current
2105 // reg and build said mask. It should have 1 for every register that's
2106 // included, with the current register as the least significant bit.
2107 uint32_t Mask = 1;
2108 CSEnd = std::remove_if(
2109 first: CSIt + 1, last: CSEnd, pred: [&](const CalleeSavedInfo &CSI) -> bool {
2110 if (CanUseBlockOps(CSI) && CSI.getReg() < Reg + 32) {
2111 Mask |= 1 << (CSI.getReg() - Reg);
2112 return true;
2113 } else {
2114 return false;
2115 }
2116 });
2117
2118 const TargetRegisterClass *BlockRegClass = TRI->getRegClassForBlockOp(MF);
2119 Register RegBlock =
2120 TRI->getMatchingSuperReg(Reg, SubIdx: AMDGPU::sub0, RC: BlockRegClass);
2121 if (!RegBlock) {
2122 // We couldn't find a super register for the block. This can happen if
2123 // the register we started with is too high (e.g. v232 if the maximum is
2124 // v255). We therefore try to get the last register block and figure out
2125 // the mask from there.
2126 Register LastBlockStart =
2127 AMDGPU::VGPR0 + alignDown(Value: Reg - AMDGPU::VGPR0, Align: 32);
2128 RegBlock =
2129 TRI->getMatchingSuperReg(Reg: LastBlockStart, SubIdx: AMDGPU::sub0, RC: BlockRegClass);
2130 assert(RegBlock && TRI->isSubRegister(RegBlock, Reg) &&
2131 "Couldn't find super register");
2132 int RegDelta = Reg - LastBlockStart;
2133 assert(RegDelta > 0 && llvm::countl_zero(Mask) >= RegDelta &&
2134 "Bad shift amount");
2135 Mask <<= RegDelta;
2136 }
2137
2138 FuncInfo->setMaskForVGPRBlockOps(RegisterBlock: RegBlock, Mask);
2139
2140 // The stack objects can be a bit smaller than the register block if we know
2141 // some of the high bits of Mask are 0. This may happen often with calling
2142 // conventions where the caller and callee-saved VGPRs are interleaved at
2143 // a small boundary (e.g. 8 or 16).
2144 int UnusedBits = llvm::countl_zero(Val: Mask);
2145 unsigned BlockSize = TRI->getSpillSize(RC: *BlockRegClass) - UnusedBits * 4;
2146 int FrameIdx =
2147 MFI.CreateStackObject(Size: BlockSize, Alignment: TRI->getSpillAlign(RC: *BlockRegClass),
2148 /*isSpillSlot=*/true);
2149 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2150
2151 CSIt->setFrameIdx(FrameIdx);
2152 CSIt->setReg(RegBlock);
2153 }
2154 CSI.erase(first: CSEnd, last: CSI.end());
2155}
2156
2157bool SIFrameLowering::assignCalleeSavedSpillSlots(
2158 MachineFunction &MF, const TargetRegisterInfo *TRI,
2159 std::vector<CalleeSavedInfo> &CSI) const {
2160 if (CSI.empty())
2161 return true; // Early exit if no callee saved registers are modified!
2162
2163 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2164 bool UseVGPRBlocks = ST.useVGPRBlockOpsForCSR();
2165
2166 if (UseVGPRBlocks)
2167 assignSlotsUsingVGPRBlocks(MF, ST, CSI);
2168
2169 return assignCalleeSavedSpillSlotsImpl(MF, TRI, CSI) || UseVGPRBlocks;
2170}
2171
2172bool SIFrameLowering::assignCalleeSavedSpillSlotsImpl(
2173 MachineFunction &MF, const TargetRegisterInfo *TRI,
2174 std::vector<CalleeSavedInfo> &CSI) const {
2175 if (CSI.empty())
2176 return true; // Early exit if no callee saved registers are modified!
2177
2178 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
2179 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2180 const SIRegisterInfo *RI = ST.getRegisterInfo();
2181 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
2182 Register BasePtrReg = RI->getBaseRegister();
2183 Register SGPRForFPSaveRestoreCopy =
2184 FuncInfo->getScratchSGPRCopyDstReg(Reg: FramePtrReg);
2185 Register SGPRForBPSaveRestoreCopy =
2186 FuncInfo->getScratchSGPRCopyDstReg(Reg: BasePtrReg);
2187 if (!SGPRForFPSaveRestoreCopy && !SGPRForBPSaveRestoreCopy)
2188 return false;
2189
2190 unsigned NumModifiedRegs = 0;
2191
2192 if (SGPRForFPSaveRestoreCopy)
2193 NumModifiedRegs++;
2194 if (SGPRForBPSaveRestoreCopy)
2195 NumModifiedRegs++;
2196
2197 for (auto &CS : CSI) {
2198 if (CS.getReg() == FramePtrReg.asMCReg() && SGPRForFPSaveRestoreCopy) {
2199 CS.setDstReg(SGPRForFPSaveRestoreCopy);
2200 if (--NumModifiedRegs)
2201 break;
2202 } else if (CS.getReg() == BasePtrReg.asMCReg() &&
2203 SGPRForBPSaveRestoreCopy) {
2204 CS.setDstReg(SGPRForBPSaveRestoreCopy);
2205 if (--NumModifiedRegs)
2206 break;
2207 }
2208 }
2209
2210 return false;
2211}
2212
2213bool SIFrameLowering::allocateScavengingFrameIndexesNearIncomingSP(
2214 const MachineFunction &MF) const {
2215
2216 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2217 const MachineFrameInfo &MFI = MF.getFrameInfo();
2218 const SIInstrInfo *TII = ST.getInstrInfo();
2219 uint64_t EstStackSize = MFI.estimateStackSize(MF);
2220 uint64_t MaxOffset = EstStackSize - 1;
2221
2222 // We need the emergency stack slots to be allocated in range of the
2223 // MUBUF/flat scratch immediate offset from the base register, so assign these
2224 // first at the incoming SP position.
2225 //
2226 // TODO: We could try sorting the objects to find a hole in the first bytes
2227 // rather than allocating as close to possible. This could save a lot of space
2228 // on frames with alignment requirements.
2229 if (ST.hasFlatScratchEnabled()) {
2230 if (TII->isLegalFLATOffset(Offset: MaxOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
2231 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch))
2232 return false;
2233 } else {
2234 if (TII->isLegalMUBUFImmOffset(Imm: MaxOffset))
2235 return false;
2236 }
2237
2238 return true;
2239}
2240
2241/// Return the set of all root registers of regunits live-in to @p MBB.
2242///
2243/// Intended to avoid using the expensive @c MCRegAliasIterator when deciding
2244/// if a register to be spilled is already live-in (see @c isAnyRootLiveIn).
2245static SparseBitVector<> buildLiveInRoots(const MachineBasicBlock &MBB,
2246 const SIRegisterInfo &TRI) {
2247 SparseBitVector<> LiveInRoots;
2248 for (const auto &LI : MBB.liveins()) {
2249 for (MCRegUnitMaskIterator MI(LI.PhysReg, &TRI); MI.isValid(); ++MI) {
2250 auto [Unit, UnitLaneMask] = *MI;
2251 if ((LI.LaneMask & UnitLaneMask).none())
2252 continue;
2253 for (MCRegUnitRootIterator RI(Unit, &TRI); RI.isValid(); ++RI)
2254 LiveInRoots.set(*RI);
2255 }
2256 }
2257 return LiveInRoots;
2258}
2259
2260/// Returns true iff any root of @p Reg is in @p LiveInRoots
2261/// (see @c buildLiveInRoots).
2262static bool isAnyRootLiveIn(const SparseBitVector<> &LiveInRoots,
2263 const SIRegisterInfo &TRI, MCRegister Reg) {
2264 for (MCRegUnitIterator UI(Reg, &TRI); UI.isValid(); ++UI) {
2265 for (MCRegUnitRootIterator RI(*UI, &TRI); RI.isValid(); ++RI) {
2266 if (LiveInRoots.test(Idx: *RI))
2267 return true;
2268 }
2269 }
2270 return false;
2271}
2272
2273void SIFrameLowering::spillCalleeSavedRegisterWithoutBlockOps(
2274 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2275 const CalleeSavedInfo &CS, const SIInstrInfo *TII,
2276 const SIRegisterInfo &TRI,
2277 const std::optional<SparseBitVector<>> &LiveInRoots) const {
2278 MCRegister Reg = CS.getReg();
2279
2280 // We assume a sortUniqueLiveIns later
2281 MBB.addLiveIn(PhysReg: Reg);
2282
2283 if (CS.isSpilledToReg()) {
2284 BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: CS.getDstReg())
2285 .addReg(RegNo: Reg, Flags: getKillRegState(B: true));
2286 } else {
2287 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
2288 bool IsKill = true;
2289 // If this value was already livein, we probably have a direct use of
2290 // the incoming register value, so don't kill at the spill point. This
2291 // happens since we pass some special inputs (workgroup IDs) in the
2292 // callee saved range.
2293 if (LiveInRoots)
2294 IsKill = !isAnyRootLiveIn(LiveInRoots: *LiveInRoots, TRI, Reg);
2295 TII->storeRegToStackSlotCFI(MBB, MI, SrcReg: Reg, isKill: IsKill, FrameIndex: CS.getFrameIdx(), RC);
2296 }
2297}
2298
2299bool SIFrameLowering::spillCalleeSavedRegisters(
2300 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2301 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *OrigTRI) const {
2302 auto &TRI = *static_cast<const SIRegisterInfo *>(OrigTRI);
2303 MachineFunction *MF = MBB.getParent();
2304 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2305 const SIInstrInfo *TII = ST.getInstrInfo();
2306
2307 std::optional<SparseBitVector<>> LiveInRoots;
2308 if (MBB.getParent()->getRegInfo().tracksLiveness())
2309 LiveInRoots = buildLiveInRoots(MBB, TRI);
2310
2311 if (!ST.useVGPRBlockOpsForCSR()) {
2312 for (const CalleeSavedInfo &CS : CSI)
2313 spillCalleeSavedRegisterWithoutBlockOps(MBB, MI, CS, TII, TRI,
2314 LiveInRoots);
2315 if (LiveInRoots)
2316 MBB.sortUniqueLiveIns();
2317 return true;
2318 }
2319
2320 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
2321 SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
2322
2323 const TargetRegisterClass *BlockRegClass = TRI.getRegClassForBlockOp(MF: *MF);
2324 for (const CalleeSavedInfo &CS : CSI) {
2325 Register Reg = CS.getReg();
2326 if (!BlockRegClass->contains(Reg) ||
2327 !FuncInfo->hasMaskForVGPRBlockOps(RegisterBlock: Reg)) {
2328 spillCalleeSavedRegisterWithoutBlockOps(MBB, MI, CS, TII, TRI,
2329 LiveInRoots);
2330 continue;
2331 }
2332
2333 // Build a scratch block store.
2334 uint32_t Mask = FuncInfo->getMaskForVGPRBlockOps(RegisterBlock: Reg);
2335 int FrameIndex = CS.getFrameIdx();
2336 MachinePointerInfo PtrInfo =
2337 MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
2338 MachineMemOperand *MMO =
2339 MF->getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
2340 Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
2341 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
2342
2343 BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(),
2344 MCID: TII->get(Opcode: AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE))
2345 .addReg(RegNo: Reg, Flags: getKillRegState(B: false))
2346 .addFrameIndex(Idx: FrameIndex)
2347 .addReg(RegNo: FuncInfo->getStackPtrOffsetReg())
2348 .addImm(Val: 0)
2349 .addImm(Val: Mask)
2350 .addMemOperand(MMO);
2351
2352 FuncInfo->setHasSpilledVGPRs();
2353
2354 // Add the register to the liveins. This is necessary because if any of the
2355 // VGPRs in the register block is reserved (e.g. if it's a WWM register),
2356 // then the whole block will be marked as reserved and `updateLiveness` will
2357 // skip it.
2358 if (LiveInRoots)
2359 MBB.addLiveIn(PhysReg: Reg);
2360 }
2361 if (LiveInRoots)
2362 MBB.sortUniqueLiveIns();
2363
2364 return true;
2365}
2366
2367bool SIFrameLowering::restoreCalleeSavedRegisters(
2368 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2369 MutableArrayRef<CalleeSavedInfo> CSI,
2370 const TargetRegisterInfo *OrigTRI) const {
2371 auto &TRI = *static_cast<const SIRegisterInfo *>(OrigTRI);
2372 MachineFunction *MF = MBB.getParent();
2373 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2374 if (!ST.useVGPRBlockOpsForCSR())
2375 return false;
2376
2377 SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
2378 MachineFrameInfo &MFI = MF->getFrameInfo();
2379 const SIInstrInfo *TII = ST.getInstrInfo();
2380 const TargetRegisterClass *BlockRegClass = TRI.getRegClassForBlockOp(MF: *MF);
2381 for (const CalleeSavedInfo &CS : reverse(C&: CSI)) {
2382 Register Reg = CS.getReg();
2383 if (!BlockRegClass->contains(Reg) ||
2384 !FuncInfo->hasMaskForVGPRBlockOps(RegisterBlock: Reg)) {
2385 restoreCalleeSavedRegister(MBB, MI, CS, TII, TRI: &TRI);
2386 continue;
2387 }
2388
2389 // Build a scratch block load.
2390 uint32_t Mask = FuncInfo->getMaskForVGPRBlockOps(RegisterBlock: Reg);
2391 int FrameIndex = CS.getFrameIdx();
2392 MachinePointerInfo PtrInfo =
2393 MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
2394 MachineMemOperand *MMO = MF->getMachineMemOperand(
2395 PtrInfo, F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIndex),
2396 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIndex));
2397
2398 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(),
2399 MCID: TII->get(Opcode: AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE), DestReg: Reg)
2400 .addFrameIndex(Idx: FrameIndex)
2401 .addReg(RegNo: FuncInfo->getStackPtrOffsetReg())
2402 .addImm(Val: 0)
2403 .addImm(Val: Mask)
2404 .addMemOperand(MMO);
2405 TRI.addImplicitUsesForBlockCSRLoad(MIB, BlockReg: Reg);
2406
2407 // Add the register to the liveins. This is necessary because if any of the
2408 // VGPRs in the register block is reserved (e.g. if it's a WWM register),
2409 // then the whole block will be marked as reserved and `updateLiveness` will
2410 // skip it.
2411 MBB.addLiveIn(PhysReg: Reg);
2412 }
2413
2414 MBB.sortUniqueLiveIns();
2415 return true;
2416}
2417
2418MachineBasicBlock::iterator SIFrameLowering::eliminateCallFramePseudoInstr(
2419 MachineFunction &MF,
2420 MachineBasicBlock &MBB,
2421 MachineBasicBlock::iterator I) const {
2422 int64_t Amount = I->getOperand(i: 0).getImm();
2423 if (Amount == 0)
2424 return MBB.erase(I);
2425
2426 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2427 const SIInstrInfo *TII = ST.getInstrInfo();
2428 const DebugLoc &DL = I->getDebugLoc();
2429 unsigned Opc = I->getOpcode();
2430 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
2431 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(i: 1).getImm() : 0;
2432
2433 if (!hasReservedCallFrame(MF)) {
2434 Amount = alignTo(Size: Amount, A: getStackAlign());
2435 assert(isUInt<32>(Amount) && "exceeded stack address space size");
2436 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2437 Register SPReg = MFI->getStackPtrOffsetReg();
2438
2439 Amount *= getScratchScaleFactor(ST);
2440 if (IsDestroy)
2441 Amount = -Amount;
2442 auto Add = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: SPReg)
2443 .addReg(RegNo: SPReg)
2444 .addImm(Val: Amount);
2445 Add->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
2446 } else if (CalleePopAmount != 0) {
2447 llvm_unreachable("is this used?");
2448 }
2449
2450 return MBB.erase(I);
2451}
2452
2453/// Returns true if the frame will require a reference to the stack pointer.
2454///
2455/// This is the set of conditions common to setting up the stack pointer in a
2456/// kernel, and for using a frame pointer in a callable function.
2457///
2458/// FIXME: Should also check hasOpaqueSPAdjustment and if any inline asm
2459/// references SP.
2460static bool frameTriviallyRequiresSP(const MachineFrameInfo &MFI) {
2461 return MFI.hasVarSizedObjects() || MFI.hasStackMap() || MFI.hasPatchPoint();
2462}
2463
2464// The FP for kernels is always known 0, so we never really need to setup an
2465// explicit register for it. However, DisableFramePointerElim will force us to
2466// use a register for it.
2467bool SIFrameLowering::hasFPImpl(const MachineFunction &MF) const {
2468 const MachineFrameInfo &MFI = MF.getFrameInfo();
2469
2470 // For entry functions we can use an immediate offset in most cases,
2471 // so the presence of calls doesn't imply we need a distinct frame pointer.
2472 if (MFI.hasCalls() &&
2473 !MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
2474 // All offsets are unsigned, so need to be addressed in the same direction
2475 // as stack growth.
2476
2477 // FIXME: This function is pretty broken, since it can be called before the
2478 // frame layout is determined or CSR spills are inserted.
2479 return MFI.getStackSize() != 0;
2480 }
2481
2482 return frameTriviallyRequiresSP(MFI) || MFI.isFrameAddressTaken() ||
2483 MF.getSubtarget<GCNSubtarget>().getRegisterInfo()->hasStackRealignment(
2484 MF) ||
2485 mayReserveScratchForCWSR(MF) ||
2486 MF.getTarget().Options.DisableFramePointerElim(MF);
2487}
2488
2489bool SIFrameLowering::mayReserveScratchForCWSR(
2490 const MachineFunction &MF) const {
2491 return MF.getInfo<SIMachineFunctionInfo>()->isDynamicVGPREnabled() &&
2492 AMDGPU::isEntryFunctionCC(CC: MF.getFunction().getCallingConv()) &&
2493 AMDGPU::isCompute(CC: MF.getFunction().getCallingConv());
2494}
2495
2496// This is essentially a reduced version of hasFP for entry functions. Since the
2497// stack pointer is known 0 on entry to kernels, we never really need an FP
2498// register. We may need to initialize the stack pointer depending on the frame
2499// properties, which logically overlaps many of the cases where an ordinary
2500// function would require an FP.
2501bool SIFrameLowering::requiresStackPointerReference(
2502 const MachineFunction &MF) const {
2503 // Callable functions always require a stack pointer reference.
2504 assert(MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction() &&
2505 "only expected to call this for entry points functions");
2506
2507 const MachineFrameInfo &MFI = MF.getFrameInfo();
2508
2509 // Entry points ordinarily don't need to initialize SP. We have to set it up
2510 // for callees if there are any. Also note tail calls are only possible via
2511 // the `llvm.amdgcn.cs.chain` intrinsic.
2512 if (MFI.hasCalls() || MFI.hasTailCall())
2513 return true;
2514
2515 // We still need to initialize the SP if we're doing anything weird that
2516 // references the SP, like variable sized stack objects.
2517 return frameTriviallyRequiresSP(MFI);
2518}
2519
2520MachineInstr *SIFrameLowering::buildCFI(MachineBasicBlock &MBB,
2521 MachineBasicBlock::iterator MBBI,
2522 const DebugLoc &DL,
2523 const MCCFIInstruction &CFIInst,
2524 MachineInstr::MIFlag Flag) const {
2525 MachineFunction &MF = *MBB.getParent();
2526 const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
2527 return BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
2528 .addCFIIndex(CFIIndex: MF.addFrameInst(Inst: CFIInst))
2529 .setMIFlag(Flag);
2530}
2531
2532MachineInstr *SIFrameLowering::buildCFIForVRegToVRegSpill(
2533 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2534 const DebugLoc &DL, const MCRegister Reg, const MCRegister RegCopy) const {
2535 MachineFunction &MF = *MBB.getParent();
2536 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2537 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2538
2539 unsigned MaskReg = MCRI.getDwarfRegNum(
2540 Reg: ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC, isEH: false);
2541 auto CFIInst = MCCFIInstruction::createLLVMVectorRegisterMask(
2542 L: nullptr, Register: MCRI.getDwarfRegNum(Reg, isEH: false),
2543 SpillRegister: MCRI.getDwarfRegNum(Reg: RegCopy, isEH: false), SpillRegisterLaneSizeInBits: VGPRLaneBitSize, MaskRegister: MaskReg,
2544 MaskRegisterSizeInBits: ST.getWavefrontSize());
2545 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2546}
2547
2548MachineInstr *SIFrameLowering::buildCFIForSGPRToVGPRSpill(
2549 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2550 const DebugLoc &DL, const MCRegister SGPR, const MCRegister VGPR,
2551 const int Lane) const {
2552 const MachineFunction &MF = *MBB.getParent();
2553 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2554
2555 int DwarfSGPR = MCRI.getDwarfRegNum(Reg: SGPR, isEH: false);
2556 int DwarfVGPR = MCRI.getDwarfRegNum(Reg: VGPR, isEH: false);
2557 assert(DwarfSGPR != -1 && DwarfVGPR != -1);
2558 assert(Lane != -1 && "Expected a lane to be present");
2559
2560 // Build a CFI instruction that represents a SGPR spilled to a single lane of
2561 // a VGPR.
2562 MCCFIInstruction::VectorRegisterWithLane VR{.Register: unsigned(DwarfVGPR),
2563 .Lane: unsigned(Lane), .SizeInBits: VGPRLaneBitSize};
2564 auto CFIInst =
2565 MCCFIInstruction::createLLVMVectorRegisters(L: nullptr, Register: DwarfSGPR, VectorRegisters: {VR});
2566 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2567}
2568
2569MachineInstr *SIFrameLowering::buildCFIForSGPRToVGPRSpill(
2570 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2571 const DebugLoc &DL, MCRegister SGPR,
2572 ArrayRef<SIRegisterInfo::SpilledReg> VGPRSpills) const {
2573 if (VGPRSpills.size() == 1u)
2574 return buildCFIForSGPRToVGPRSpill(MBB, MBBI, DL, SGPR, VGPR: VGPRSpills[0].VGPR,
2575 Lane: VGPRSpills[0].Lane);
2576 const MachineFunction &MF = *MBB.getParent();
2577 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2578
2579 int DwarfSGPR = MCRI.getDwarfRegNum(Reg: SGPR, isEH: false);
2580 assert(DwarfSGPR != -1);
2581
2582 // Build a CFI instruction that represents a SGPR spilled to multiple lanes of
2583 // multiple VGPRs.
2584
2585 SmallVector<MCCFIInstruction::VectorRegisterWithLane> VGPRs;
2586 for (SIRegisterInfo::SpilledReg Spill : VGPRSpills) {
2587 int DwarfVGPR = MCRI.getDwarfRegNum(Reg: Spill.VGPR, isEH: false);
2588 assert(DwarfVGPR != -1);
2589 assert(Spill.hasLane() && "Expected a lane to be present");
2590 VGPRs.push_back(
2591 Elt: {.Register: unsigned(DwarfVGPR), .Lane: unsigned(Spill.Lane), .SizeInBits: VGPRLaneBitSize});
2592 }
2593
2594 auto CFIInst = MCCFIInstruction::createLLVMVectorRegisters(L: nullptr, Register: DwarfSGPR,
2595 VectorRegisters: std::move(VGPRs));
2596 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2597}
2598
2599MachineInstr *SIFrameLowering::buildCFIForSGPRToVMEMSpill(
2600 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2601 const DebugLoc &DL, MCRegister SGPR, int64_t Offset) const {
2602 MachineFunction &MF = *MBB.getParent();
2603 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2604 return buildCFI(MBB, MBBI, DL,
2605 CFIInst: llvm::MCCFIInstruction::createOffset(
2606 L: nullptr, Register: MCRI.getDwarfRegNum(Reg: SGPR, isEH: false), Offset));
2607}
2608
2609MachineInstr *SIFrameLowering::buildCFIForVGPRToVMEMSpill(
2610 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2611 const DebugLoc &DL, MCRegister VGPR, int64_t Offset) const {
2612 const MachineFunction &MF = *MBB.getParent();
2613 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2614 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2615
2616 int DwarfVGPR = MCRI.getDwarfRegNum(Reg: VGPR, isEH: false);
2617 assert(DwarfVGPR != -1);
2618
2619 unsigned MaskReg = MCRI.getDwarfRegNum(
2620 Reg: ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC, isEH: false);
2621 auto CFIInst = MCCFIInstruction::createLLVMVectorOffset(
2622 L: nullptr, Register: DwarfVGPR, RegisterSizeInBits: VGPRLaneBitSize, MaskRegister: MaskReg, MaskRegisterSizeInBits: ST.getWavefrontSize(),
2623 Offset);
2624 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2625}
2626
2627MachineInstr *SIFrameLowering::buildCFIForRegToSGPRPairSpill(
2628 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2629 const DebugLoc &DL, const MCRegister Reg, const MCRegister SGPRPair) const {
2630 const MachineFunction &MF = *MBB.getParent();
2631 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2632 const SIRegisterInfo &TRI = *ST.getRegisterInfo();
2633
2634 MCRegister SGPR0 = TRI.getSubReg(Reg: SGPRPair, Idx: AMDGPU::sub0);
2635 MCRegister SGPR1 = TRI.getSubReg(Reg: SGPRPair, Idx: AMDGPU::sub1);
2636
2637 int DwarfReg = TRI.getDwarfRegNum(Reg, isEH: false);
2638 int DwarfSGPR0 = TRI.getDwarfRegNum(Reg: SGPR0, isEH: false);
2639 int DwarfSGPR1 = TRI.getDwarfRegNum(Reg: SGPR1, isEH: false);
2640 assert(DwarfReg != -1 && DwarfSGPR0 != -1 && DwarfSGPR1 != -1);
2641
2642 auto CFIInst = MCCFIInstruction::createLLVMRegisterPair(
2643 L: nullptr, Register: DwarfReg, R1: DwarfSGPR0, R1SizeInBits: SGPRBitSize, R2: DwarfSGPR1, R2SizeInBits: SGPRBitSize);
2644 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2645}
2646
2647MachineInstr *SIFrameLowering::buildCFIForSameValue(
2648 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2649 const DebugLoc &DL, MCRegister Reg) const {
2650 const MachineFunction &MF = *MBB.getParent();
2651 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2652 int DwarfReg = MCRI.getDwarfRegNum(Reg, /*isEH=*/false);
2653 auto CFIInst = MCCFIInstruction::createSameValue(L: nullptr, Register: DwarfReg);
2654 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2655}
2656