1//===----------------------- SIFrameLowering.cpp --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8
9#include "SIFrameLowering.h"
10#include "AMDGPULaneMaskUtils.h"
11#include "GCNSubtarget.h"
12#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13#include "SIMachineFunctionInfo.h"
14#include "SISpillUtils.h"
15#include "llvm/BinaryFormat/Dwarf.h"
16#include "llvm/CodeGen/LiveRegUnits.h"
17#include "llvm/CodeGen/MachineFrameInfo.h"
18#include "llvm/CodeGen/MachineModuleInfo.h"
19#include "llvm/CodeGen/RegisterScavenging.h"
20#include "llvm/Support/LEB128.h"
21#include "llvm/Target/TargetMachine.h"
22
23using namespace llvm;
24
25#define DEBUG_TYPE "frame-info"
26
27static constexpr unsigned SGPRBitSize = 32;
28static constexpr unsigned SGPRByteSize = SGPRBitSize / 8;
29static constexpr unsigned VGPRLaneBitSize = 32;
30
31// The DWARF register numbers of vcc_lo and vcc_hi (VCC_32_LO and VCC_32_HI) are
32// only defined in wave32. In wave64 both halves are described as part of
33// VCC_64.
34static bool isWave64VCCHalf(const GCNSubtarget &ST, MCRegister Reg) {
35 return ST.isWave64() && (Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI);
36}
37
38// Find a register matching \p RC from \p LiveUnits which is unused and
39// available throughout the function. On failure, returns AMDGPU::NoRegister.
40// TODO: Rewrite the loop here to iterate over MCRegUnits instead of
41// MCRegisters. This should reduce the number of iterations and avoid redundant
42// checking.
43static MCRegister findUnusedRegister(MachineRegisterInfo &MRI,
44 const LiveRegUnits &LiveUnits,
45 const TargetRegisterClass &RC) {
46 for (MCRegister Reg : RC) {
47 if (!MRI.isPhysRegUsed(PhysReg: Reg) && LiveUnits.available(Reg) &&
48 !MRI.isReserved(PhysReg: Reg))
49 return Reg;
50 }
51 return MCRegister();
52}
53
54static void encodeDwarfRegisterLocation(int DwarfReg, raw_ostream &OS) {
55 assert(DwarfReg >= 0);
56 if (DwarfReg < 32) {
57 OS << uint8_t(dwarf::DW_OP_reg0 + DwarfReg);
58 } else {
59 OS << uint8_t(dwarf::DW_OP_regx);
60 encodeULEB128(Value: DwarfReg, OS);
61 }
62}
63
64static MCCFIInstruction createScaledCFAInPrivateWave(const GCNSubtarget &ST,
65 int64_t DwarfStackPtrReg) {
66 assert(ST.hasFlatScratchEnabled());
67
68 // When flat scratch is enabled, the stack pointer is an address in the
69 // private_lane DWARF address space (i.e. swizzled), but in order to
70 // accurately and efficiently describe things like masked spills of vector
71 // registers we want to define the CFA to be an address in the private_wave
72 // DWARF address space (i.e. unswizzled). To achieve this we scale the stack
73 // pointer by the wavefront size, implemented as (SP << wave_size_log2).
74 const unsigned WavefrontSizeLog2 = ST.getWavefrontSizeLog2();
75 assert(WavefrontSizeLog2 < 32);
76
77 SmallString<20> Block;
78 raw_svector_ostream OSBlock(Block);
79 encodeDwarfRegisterLocation(DwarfReg: DwarfStackPtrReg, OS&: OSBlock);
80 OSBlock << uint8_t(dwarf::DW_OP_deref_size) << uint8_t(SGPRByteSize)
81 << uint8_t(dwarf::DW_OP_lit0 + WavefrontSizeLog2)
82 << uint8_t(dwarf::DW_OP_shl)
83 << uint8_t(dwarf::DW_OP_lit0 +
84 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave)
85 << uint8_t(dwarf::DW_OP_LLVM_user)
86 << uint8_t(dwarf::DW_OP_LLVM_form_aspace_address);
87
88 SmallString<20> CFIInst;
89 raw_svector_ostream OSCFIInst(CFIInst);
90 OSCFIInst << uint8_t(dwarf::DW_CFA_def_cfa_expression);
91 encodeULEB128(Value: Block.size(), OS&: OSCFIInst);
92 OSCFIInst << Block;
93
94 return MCCFIInstruction::createEscape(L: nullptr, Vals: OSCFIInst.str());
95}
96
97void SIFrameLowering::emitDefCFA(MachineBasicBlock &MBB,
98 MachineBasicBlock::iterator MBBI,
99 DebugLoc const &DL, MCRegister StackPtrReg,
100 bool AspaceAlreadyDefined,
101 MachineInstr::MIFlag Flags) const {
102 MachineFunction &MF = *MBB.getParent();
103 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
104 const SIRegisterInfo *TRI = ST.getRegisterInfo();
105
106 int64_t DwarfStackPtrReg = TRI->getDwarfRegNum(Reg: StackPtrReg, isEH: false);
107 MCCFIInstruction CFIInst =
108 ST.hasFlatScratchEnabled()
109 ? createScaledCFAInPrivateWave(ST, DwarfStackPtrReg)
110 : (AspaceAlreadyDefined
111 ? MCCFIInstruction::createLLVMDefAspaceCfa(
112 L: nullptr, Register: DwarfStackPtrReg, Offset: 0,
113 AddressSpace: dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave, Loc: SMLoc())
114 : MCCFIInstruction::createDefCfaRegister(L: nullptr,
115 Register: DwarfStackPtrReg));
116 buildCFI(MBB, MBBI, DL, CFIInst, flag: Flags);
117}
118
119// Find a scratch register that we can use in the prologue. We avoid using
120// callee-save registers since they may appear to be free when this is called
121// from canUseAsPrologue (during shrink wrapping), but then no longer be free
122// when this is called from emitPrologue.
123static MCRegister findScratchNonCalleeSaveRegister(
124 MachineRegisterInfo &MRI, LiveRegUnits &LiveUnits,
125 const TargetRegisterClass &RC, bool Unused = false) {
126 // Mark callee saved registers as used so we will not choose them.
127 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
128 for (unsigned i = 0; CSRegs[i]; ++i)
129 LiveUnits.addReg(Reg: CSRegs[i]);
130
131 // We are looking for a register that can be used throughout the entire
132 // function, so any use is unacceptable.
133 if (Unused)
134 return findUnusedRegister(MRI, LiveUnits, RC);
135
136 for (MCRegister Reg : RC) {
137 if (LiveUnits.available(Reg) && !MRI.isReserved(PhysReg: Reg))
138 return Reg;
139 }
140
141 return MCRegister();
142}
143
144/// Query target location for spilling SGPRs
145/// \p IncludeScratchCopy : Also look for free scratch SGPRs
146static void getVGPRSpillLaneOrTempRegister(
147 MachineFunction &MF, LiveRegUnits &LiveUnits, Register SGPR,
148 const TargetRegisterClass &RC = AMDGPU::SReg_32_XM0_XEXECRegClass,
149 bool IncludeScratchCopy = true) {
150 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
151 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
152
153 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
154 const SIRegisterInfo *TRI = ST.getRegisterInfo();
155 unsigned Size = TRI->getSpillSize(RC);
156 Align Alignment = TRI->getSpillAlign(RC);
157
158 // We need to save and restore the given SGPR.
159
160 Register ScratchSGPR;
161 // 1: Try to save the given register into an unused scratch SGPR. The
162 // LiveUnits should have all the callee saved registers marked as used. For
163 // certain cases we skip copy to scratch SGPR.
164 if (IncludeScratchCopy)
165 ScratchSGPR = findUnusedRegister(MRI&: MF.getRegInfo(), LiveUnits, RC);
166
167 if (!ScratchSGPR) {
168 int FI = FrameInfo.CreateStackObject(Size, Alignment, isSpillSlot: true, Alloca: nullptr,
169 ID: TargetStackID::SGPRSpill);
170
171 if (TRI->spillSGPRToVGPR() &&
172 MFI->allocateSGPRSpillToVGPRLane(MF, FI, /*SpillToPhysVGPRLane=*/true,
173 /*IsPrologEpilog=*/true)) {
174 // 2: There's no free lane to spill, and no free register to save the
175 // SGPR, so we're forced to take another VGPR to use for the spill.
176 MFI->addToPrologEpilogSGPRSpills(
177 Reg: SGPR, SI: PrologEpilogSGPRSaveRestoreInfo(
178 SGPRSaveKind::SPILL_TO_VGPR_LANE, FI));
179
180 LLVM_DEBUG(auto Spill = MFI->getSGPRSpillToPhysicalVGPRLanes(FI).front();
181 dbgs() << printReg(SGPR, TRI) << " requires fallback spill to "
182 << printReg(Spill.VGPR, TRI) << ':' << Spill.Lane
183 << '\n';);
184 } else {
185 // Remove dead <FI> index
186 MF.getFrameInfo().RemoveStackObject(ObjectIdx: FI);
187 // 3: If all else fails, spill the register to memory.
188 FI = FrameInfo.CreateSpillStackObject(Size, Alignment);
189 MFI->addToPrologEpilogSGPRSpills(
190 Reg: SGPR,
191 SI: PrologEpilogSGPRSaveRestoreInfo(SGPRSaveKind::SPILL_TO_MEM, FI));
192 LLVM_DEBUG(dbgs() << "Reserved FI " << FI << " for spilling "
193 << printReg(SGPR, TRI) << '\n');
194 }
195 } else {
196 MFI->addToPrologEpilogSGPRSpills(
197 Reg: SGPR, SI: PrologEpilogSGPRSaveRestoreInfo(
198 SGPRSaveKind::COPY_TO_SCRATCH_SGPR, ScratchSGPR));
199 LiveUnits.addReg(Reg: ScratchSGPR);
200 LLVM_DEBUG(dbgs() << "Saving " << printReg(SGPR, TRI) << " with copy to "
201 << printReg(ScratchSGPR, TRI) << '\n');
202 }
203}
204
205// We need to specially emit stack operations here because a different frame
206// register is used than in the rest of the function, as getFrameRegister would
207// use.
208static void buildPrologSpill(const GCNSubtarget &ST, const SIRegisterInfo &TRI,
209 const SIMachineFunctionInfo &FuncInfo,
210 LiveRegUnits &LiveUnits, MachineFunction &MF,
211 MachineBasicBlock &MBB,
212 MachineBasicBlock::iterator I, const DebugLoc &DL,
213 Register SpillReg, int FI, Register FrameReg,
214 int64_t DwordOff = 0) {
215 unsigned Opc = ST.hasFlatScratchEnabled() ? AMDGPU::SCRATCH_STORE_DWORD_SADDR
216 : AMDGPU::BUFFER_STORE_DWORD_OFFSET;
217
218 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
219 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
220 MachineMemOperand *MMO = MF.getMachineMemOperand(
221 PtrInfo, F: MachineMemOperand::MOStore, Size: FrameInfo.getObjectSize(ObjectIdx: FI),
222 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FI));
223 LiveUnits.addReg(Reg: SpillReg);
224 bool IsKill = !MBB.isLiveIn(Reg: SpillReg);
225 TRI.buildSpillLoadStore(MBB, MI: I, DL, LoadStoreOp: Opc, Index: FI, ValueReg: SpillReg, ValueIsKill: IsKill, ScratchOffsetReg: FrameReg,
226 InstrOffset: DwordOff, MMO, RS: nullptr, LiveUnits: &LiveUnits);
227 if (IsKill)
228 LiveUnits.removeReg(Reg: SpillReg);
229}
230
231static void buildEpilogRestore(const GCNSubtarget &ST,
232 const SIRegisterInfo &TRI,
233 const SIMachineFunctionInfo &FuncInfo,
234 LiveRegUnits &LiveUnits, MachineFunction &MF,
235 MachineBasicBlock &MBB,
236 MachineBasicBlock::iterator I,
237 const DebugLoc &DL, Register SpillReg, int FI,
238 Register FrameReg, int64_t DwordOff = 0) {
239 unsigned Opc = ST.hasFlatScratchEnabled() ? AMDGPU::SCRATCH_LOAD_DWORD_SADDR
240 : AMDGPU::BUFFER_LOAD_DWORD_OFFSET;
241
242 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
243 MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI);
244 MachineMemOperand *MMO = MF.getMachineMemOperand(
245 PtrInfo, F: MachineMemOperand::MOLoad, Size: FrameInfo.getObjectSize(ObjectIdx: FI),
246 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FI));
247 TRI.buildSpillLoadStore(MBB, MI: I, DL, LoadStoreOp: Opc, Index: FI, ValueReg: SpillReg, ValueIsKill: false, ScratchOffsetReg: FrameReg,
248 InstrOffset: DwordOff, MMO, RS: nullptr, LiveUnits: &LiveUnits);
249}
250
251static void buildGitPtr(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
252 const DebugLoc &DL, const SIInstrInfo *TII,
253 Register TargetReg) {
254 MachineFunction *MF = MBB.getParent();
255 const SIMachineFunctionInfo *MFI = MF->getInfo<SIMachineFunctionInfo>();
256 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
257 const MCInstrDesc &SMovB32 = TII->get(Opcode: AMDGPU::S_MOV_B32);
258 Register TargetLo = TRI->getSubReg(Reg: TargetReg, Idx: AMDGPU::sub0);
259 Register TargetHi = TRI->getSubReg(Reg: TargetReg, Idx: AMDGPU::sub1);
260
261 if (MFI->getGITPtrHigh() != 0xffffffff) {
262 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: TargetHi)
263 .addImm(Val: MFI->getGITPtrHigh())
264 .addReg(RegNo: TargetReg, Flags: RegState::ImplicitDefine);
265 } else {
266 const MCInstrDesc &GetPC64 = TII->get(Opcode: AMDGPU::S_GETPC_B64_pseudo);
267 BuildMI(BB&: MBB, I, MIMD: DL, MCID: GetPC64, DestReg: TargetReg);
268 }
269 Register GitPtrLo = MFI->getGITPtrLoReg(MF: *MF);
270 MF->getRegInfo().addLiveIn(Reg: GitPtrLo);
271 MBB.addLiveIn(PhysReg: GitPtrLo);
272 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: TargetLo)
273 .addReg(RegNo: GitPtrLo);
274}
275
276static void initLiveUnits(LiveRegUnits &LiveUnits, const SIRegisterInfo &TRI,
277 const SIMachineFunctionInfo *FuncInfo,
278 MachineFunction &MF, MachineBasicBlock &MBB,
279 MachineBasicBlock::iterator MBBI, bool IsProlog) {
280 if (LiveUnits.empty()) {
281 LiveUnits.init(TRI);
282 if (IsProlog) {
283 LiveUnits.addLiveIns(MBB);
284 } else {
285 // In epilog.
286 LiveUnits.addLiveOuts(MBB);
287 LiveUnits.stepBackward(MI: *MBBI);
288 }
289 }
290}
291
292namespace llvm {
293
294// SpillBuilder to save/restore special SGPR spills like the one needed for FP,
295// BP, etc. These spills are delayed until the current function's frame is
296// finalized. For a given register, the builder uses the
297// PrologEpilogSGPRSaveRestoreInfo to decide the spill method.
298class PrologEpilogSGPRSpillBuilder {
299 MachineBasicBlock::iterator MI;
300 MachineBasicBlock &MBB;
301 MachineFunction &MF;
302 const GCNSubtarget &ST;
303 MachineFrameInfo &MFI;
304 SIMachineFunctionInfo *FuncInfo;
305 const SIInstrInfo *TII;
306 const SIRegisterInfo &TRI;
307 const MCRegisterInfo *MCRI;
308 const SIFrameLowering *TFI;
309 Register SuperReg;
310 const PrologEpilogSGPRSaveRestoreInfo SI;
311 LiveRegUnits &LiveUnits;
312 const DebugLoc &DL;
313 Register FrameReg;
314 ArrayRef<int16_t> SplitParts;
315 unsigned NumSubRegs;
316 unsigned EltSize = 4;
317 bool IsFramePtrPrologSpill;
318 bool NeedsFrameMoves;
319
320 static bool isExec(Register Reg) {
321 return Reg == AMDGPU::EXEC_LO || Reg == AMDGPU::EXEC;
322 }
323
324 /// If this builder requires SuperReg-based CFI, which is emitted after all
325 /// SubRegs are actually spilled, return the Register which should be used
326 /// as input to getDwarfRegNum. Otherwise, CFI should be generated per-SubReg.
327 ///
328 /// Note: Most spills handled by this builder generate CFI after each
329 /// SubReg spill, as each SubReg maps directly to a CFI register via
330 /// getDwarfRegNum(SubReg, false). All other cases currently currently
331 /// correspond to the SuperReg directly.
332 MCRegister getCFISuperReg() const {
333 if (IsFramePtrPrologSpill)
334 return FuncInfo->getFrameOffsetReg();
335 // FIXME: CFI for EXEC needs a fix by accurately computing the spill
336 // offset for both the low and high components.
337 if (isExec(Reg: SuperReg))
338 return AMDGPU::EXEC;
339 return {};
340 }
341
342 void saveToMemory(const int FI) const {
343 MachineRegisterInfo &MRI = MF.getRegInfo();
344 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
345 assert(!MFI.isDeadObjectIndex(FI));
346
347 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI: MI, /*IsProlog*/ true);
348
349 MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
350 MRI, LiveUnits, RC: AMDGPU::VGPR_32RegClass);
351 if (!TmpVGPR)
352 report_fatal_error(reason: "failed to find free scratch register");
353
354 auto BuildCFI = [&](Register Reg) {
355 TFI->buildCFI(MBB, MBBI: MI, DL,
356 CFIInst: MCCFIInstruction::createOffset(
357 L: nullptr, Register: MCRI->getDwarfRegNum(Reg, isEH: false),
358 Offset: MFI.getObjectOffset(ObjectIdx: FI) * ST.getWavefrontSize()));
359 };
360 MCRegister CFISuperReg = getCFISuperReg();
361 for (unsigned I = 0, DwordOff = 0; I < NumSubRegs; ++I) {
362 Register SubReg = NumSubRegs == 1
363 ? SuperReg
364 : Register(TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]));
365 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_MOV_B32_e32), DestReg: TmpVGPR)
366 .addReg(RegNo: SubReg);
367
368 buildPrologSpill(ST, TRI, FuncInfo: *FuncInfo, LiveUnits, MF, MBB, I: MI, DL, SpillReg: TmpVGPR,
369 FI, FrameReg, DwordOff);
370 if (NeedsFrameMoves && !CFISuperReg)
371 BuildCFI(SubReg);
372 DwordOff += 4;
373 }
374 if (NeedsFrameMoves && CFISuperReg)
375 BuildCFI(CFISuperReg);
376 }
377
378 void saveToVGPRLane(const int FI) const {
379 assert(!MFI.isDeadObjectIndex(FI));
380
381 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
382 ArrayRef<SIRegisterInfo::SpilledReg> Spill =
383 FuncInfo->getSGPRSpillToPhysicalVGPRLanes(FrameIndex: FI);
384 assert(Spill.size() == NumSubRegs);
385
386 MCRegister CFISuperReg = getCFISuperReg();
387 for (unsigned I = 0; I < NumSubRegs; ++I) {
388 Register SubReg = NumSubRegs == 1
389 ? SuperReg
390 : Register(TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]));
391 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_SPILL_S32_TO_VGPR),
392 DestReg: Spill[I].VGPR)
393 .addReg(RegNo: SubReg)
394 .addImm(Val: Spill[I].Lane)
395 .addReg(RegNo: Spill[I].VGPR, Flags: RegState::Undef);
396 if (NeedsFrameMoves && !CFISuperReg)
397 TFI->buildCFIForSGPRToVGPRSpill(MBB, MBBI: MI, DL, SGPR: SubReg, VGPR: Spill[I].VGPR,
398 Lane: Spill[I].Lane);
399 }
400 if (NeedsFrameMoves && CFISuperReg)
401 TFI->buildCFIForSGPRToVGPRSpill(MBB, MBBI: MI, DL, SGPR: CFISuperReg, VGPRSpills: Spill);
402 }
403
404 void copyToScratchSGPR(Register DstReg) const {
405 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: DstReg)
406 .addReg(RegNo: SuperReg)
407 .setMIFlag(MachineInstr::FrameSetup);
408 if (NeedsFrameMoves) {
409 MCRegister CFISuperReg = getCFISuperReg();
410 if (!CFISuperReg)
411 CFISuperReg = SuperReg;
412 if (isWave64VCCHalf(ST, Reg: DstReg)) {
413 assert(NumSubRegs == 1 && "Expected a 32-bit SGPR");
414 TFI->buildCFIForSGPRToVCCHalfCopy(MBB, MBBI: MI, DL, SGPR: CFISuperReg, VCCHalf: DstReg);
415 } else {
416 TFI->buildCFIForSRegToSRegSpill(MBB, MBBI: MI, DL, Reg: CFISuperReg, CopyReg: DstReg);
417 }
418 }
419 }
420
421 void restoreFromMemory(const int FI) {
422 MachineRegisterInfo &MRI = MF.getRegInfo();
423 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
424
425 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI: MI, /*IsProlog*/ false);
426 MCPhysReg TmpVGPR = findScratchNonCalleeSaveRegister(
427 MRI, LiveUnits, RC: AMDGPU::VGPR_32RegClass);
428 if (!TmpVGPR)
429 report_fatal_error(reason: "failed to find free scratch register");
430
431 for (unsigned I = 0, DwordOff = 0; I < NumSubRegs; ++I) {
432 MCRegister SubReg = NumSubRegs == 1
433 ? SuperReg.asMCReg()
434 : TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]);
435
436 buildEpilogRestore(ST, TRI, FuncInfo: *FuncInfo, LiveUnits, MF, MBB, I: MI, DL,
437 SpillReg: TmpVGPR, FI, FrameReg, DwordOff);
438 assert(SubReg.isPhysical());
439
440 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::V_READFIRSTLANE_B32), DestReg: SubReg)
441 .addReg(RegNo: TmpVGPR, Flags: RegState::Kill);
442 DwordOff += 4;
443 }
444 }
445
446 void restoreFromVGPRLane(const int FI) {
447 assert(MFI.getStackID(FI) == TargetStackID::SGPRSpill);
448 ArrayRef<SIRegisterInfo::SpilledReg> Spill =
449 FuncInfo->getSGPRSpillToPhysicalVGPRLanes(FrameIndex: FI);
450 assert(Spill.size() == NumSubRegs);
451
452 for (unsigned I = 0; I < NumSubRegs; ++I) {
453 MCRegister SubReg = NumSubRegs == 1
454 ? SuperReg.asMCReg()
455 : TRI.getSubReg(Reg: SuperReg, Idx: SplitParts[I]);
456 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::SI_RESTORE_S32_FROM_VGPR), DestReg: SubReg)
457 .addReg(RegNo: Spill[I].VGPR)
458 .addImm(Val: Spill[I].Lane);
459 }
460 }
461
462 void copyFromScratchSGPR(Register SrcReg) const {
463 BuildMI(BB&: MBB, I: MI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SuperReg)
464 .addReg(RegNo: SrcReg)
465 .setMIFlag(MachineInstr::FrameDestroy);
466 }
467
468public:
469 PrologEpilogSGPRSpillBuilder(Register Reg,
470 const PrologEpilogSGPRSaveRestoreInfo SI,
471 MachineBasicBlock &MBB,
472 MachineBasicBlock::iterator MI,
473 const DebugLoc &DL, const SIInstrInfo *TII,
474 const SIRegisterInfo &TRI,
475 LiveRegUnits &LiveUnits, Register FrameReg,
476 bool IsFramePtrPrologSpill = false)
477 : MI(MI), MBB(MBB), MF(*MBB.getParent()),
478 ST(MF.getSubtarget<GCNSubtarget>()), MFI(MF.getFrameInfo()),
479 FuncInfo(MF.getInfo<SIMachineFunctionInfo>()), TII(TII), TRI(TRI),
480 MCRI(MF.getContext().getRegisterInfo()), TFI(ST.getFrameLowering()),
481 SuperReg(Reg), SI(SI), LiveUnits(LiveUnits), DL(DL), FrameReg(FrameReg),
482 IsFramePtrPrologSpill(IsFramePtrPrologSpill),
483 NeedsFrameMoves(MF.needsFrameMoves()) {
484 const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg: SuperReg);
485 SplitParts = TRI.getRegSplitParts(RC, EltSize);
486 NumSubRegs = SplitParts.empty() ? 1 : SplitParts.size();
487
488 assert(SuperReg != AMDGPU::M0 && "m0 should never spill");
489 }
490
491 void save() {
492 switch (SI.getKind()) {
493 case SGPRSaveKind::SPILL_TO_MEM:
494 return saveToMemory(FI: SI.getIndex());
495 case SGPRSaveKind::SPILL_TO_VGPR_LANE:
496 return saveToVGPRLane(FI: SI.getIndex());
497 case SGPRSaveKind::COPY_TO_SCRATCH_SGPR:
498 return copyToScratchSGPR(DstReg: SI.getReg());
499 }
500 }
501
502 void restore() {
503 switch (SI.getKind()) {
504 case SGPRSaveKind::SPILL_TO_MEM:
505 return restoreFromMemory(FI: SI.getIndex());
506 case SGPRSaveKind::SPILL_TO_VGPR_LANE:
507 return restoreFromVGPRLane(FI: SI.getIndex());
508 case SGPRSaveKind::COPY_TO_SCRATCH_SGPR:
509 return copyFromScratchSGPR(SrcReg: SI.getReg());
510 }
511 }
512};
513
514} // namespace llvm
515
516// Emit flat scratch setup code, assuming `MFI->hasFlatScratchInit()`
517void SIFrameLowering::emitEntryFunctionFlatScratchInit(
518 MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
519 const DebugLoc &DL, Register ScratchWaveOffsetReg) const {
520 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
521 const SIInstrInfo *TII = ST.getInstrInfo();
522 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
523 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
524
525 // We don't need this if we only have spills since there is no user facing
526 // scratch.
527
528 // TODO: If we know we don't have flat instructions earlier, we can omit
529 // this from the input registers.
530 //
531 // TODO: We only need to know if we access scratch space through a flat
532 // pointer. Because we only detect if flat instructions are used at all,
533 // this will be used more often than necessary on VI.
534
535 Register FlatScrInitLo;
536 Register FlatScrInitHi;
537
538 if (ST.isAmdPalOS()) {
539 // Extract the scratch offset from the descriptor in the GIT
540 LiveRegUnits LiveUnits;
541 LiveUnits.init(TRI: *TRI);
542 LiveUnits.addLiveIns(MBB);
543
544 // Find unused reg to load flat scratch init into
545 MachineRegisterInfo &MRI = MF.getRegInfo();
546 Register FlatScrInit = AMDGPU::NoRegister;
547 ArrayRef<MCPhysReg> AllSGPR64s = TRI->getAllSGPR64(MF);
548 unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 1) / 2;
549 AllSGPR64s = AllSGPR64s.slice(
550 N: std::min(a: static_cast<unsigned>(AllSGPR64s.size()), b: NumPreloaded));
551 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
552 for (MCPhysReg Reg : AllSGPR64s) {
553 if (LiveUnits.available(Reg) && !MRI.isReserved(PhysReg: Reg) &&
554 MRI.isAllocatable(PhysReg: Reg) && !TRI->isSubRegisterEq(RegA: Reg, RegB: GITPtrLoReg)) {
555 FlatScrInit = Reg;
556 break;
557 }
558 }
559 assert(FlatScrInit && "Failed to find free register for scratch init");
560
561 FlatScrInitLo = TRI->getSubReg(Reg: FlatScrInit, Idx: AMDGPU::sub0);
562 FlatScrInitHi = TRI->getSubReg(Reg: FlatScrInit, Idx: AMDGPU::sub1);
563
564 buildGitPtr(MBB, I, DL, TII, TargetReg: FlatScrInit);
565
566 // We now have the GIT ptr - now get the scratch descriptor from the entry
567 // at offset 0 (or offset 16 for a compute shader).
568 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
569 const MCInstrDesc &LoadDwordX2 = TII->get(Opcode: AMDGPU::S_LOAD_DWORDX2_IMM);
570 auto *MMO = MF.getMachineMemOperand(
571 PtrInfo,
572 F: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
573 MachineMemOperand::MODereferenceable,
574 Size: 8, BaseAlignment: Align(4));
575 unsigned Offset =
576 MF.getFunction().getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
577 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
578 unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(ST: Subtarget, ByteOffset: Offset);
579 BuildMI(BB&: MBB, I, MIMD: DL, MCID: LoadDwordX2, DestReg: FlatScrInit)
580 .addReg(RegNo: FlatScrInit)
581 .addImm(Val: EncodedOffset) // offset
582 .addImm(Val: 0) // cpol
583 .addMemOperand(MMO);
584
585 // Mask the offset in [47:0] of the descriptor
586 const MCInstrDesc &SAndB32 = TII->get(Opcode: AMDGPU::S_AND_B32);
587 auto And = BuildMI(BB&: MBB, I, MIMD: DL, MCID: SAndB32, DestReg: FlatScrInitHi)
588 .addReg(RegNo: FlatScrInitHi)
589 .addImm(Val: 0xffff);
590 And->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
591 } else {
592 Register FlatScratchInitReg =
593 MFI->getPreloadedReg(Value: AMDGPUFunctionArgInfo::FLAT_SCRATCH_INIT);
594 assert(FlatScratchInitReg);
595
596 MachineRegisterInfo &MRI = MF.getRegInfo();
597 MRI.addLiveIn(Reg: FlatScratchInitReg);
598 MBB.addLiveIn(PhysReg: FlatScratchInitReg);
599
600 FlatScrInitLo = TRI->getSubReg(Reg: FlatScratchInitReg, Idx: AMDGPU::sub0);
601 FlatScrInitHi = TRI->getSubReg(Reg: FlatScratchInitReg, Idx: AMDGPU::sub1);
602 }
603
604 // Do a 64-bit pointer add.
605 if (ST.flatScratchIsPointer()) {
606 if (ST.getGeneration() >= AMDGPUSubtarget::GFX10) {
607 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: FlatScrInitLo)
608 .addReg(RegNo: FlatScrInitLo)
609 .addReg(RegNo: ScratchWaveOffsetReg);
610 auto Addc = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADDC_U32),
611 DestReg: FlatScrInitHi)
612 .addReg(RegNo: FlatScrInitHi)
613 .addImm(Val: 0);
614 Addc->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
615
616 using namespace AMDGPU::Hwreg;
617 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_B32))
618 .addReg(RegNo: FlatScrInitLo)
619 .addImm(Val: int16_t(HwregEncoding::encode(Values: ID_FLAT_SCR_LO, Values: 0, Values: 32)));
620 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_SETREG_B32))
621 .addReg(RegNo: FlatScrInitHi)
622 .addImm(Val: int16_t(HwregEncoding::encode(Values: ID_FLAT_SCR_HI, Values: 0, Values: 32)));
623 return;
624 }
625
626 // For GFX9.
627 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: AMDGPU::FLAT_SCR_LO)
628 .addReg(RegNo: FlatScrInitLo)
629 .addReg(RegNo: ScratchWaveOffsetReg);
630 auto Addc = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADDC_U32),
631 DestReg: AMDGPU::FLAT_SCR_HI)
632 .addReg(RegNo: FlatScrInitHi)
633 .addImm(Val: 0);
634 Addc->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
635
636 return;
637 }
638
639 assert(ST.getGeneration() < AMDGPUSubtarget::GFX9);
640
641 // Copy the size in bytes.
642 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: AMDGPU::FLAT_SCR_LO)
643 .addReg(RegNo: FlatScrInitHi, Flags: RegState::Kill);
644
645 // Add wave offset in bytes to private base offset.
646 // See comment in AMDKernelCodeT.h for enable_sgpr_flat_scratch_init.
647 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: FlatScrInitLo)
648 .addReg(RegNo: FlatScrInitLo)
649 .addReg(RegNo: ScratchWaveOffsetReg);
650
651 // Convert offset to 256-byte units.
652 auto LShr = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_LSHR_B32),
653 DestReg: AMDGPU::FLAT_SCR_HI)
654 .addReg(RegNo: FlatScrInitLo, Flags: RegState::Kill)
655 .addImm(Val: 8);
656 LShr->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
657}
658
659// Note SGPRSpill stack IDs should only be used for SGPR spilling to VGPRs, not
660// memory. They should have been removed by now.
661static bool allStackObjectsAreDead(const MachineFrameInfo &MFI) {
662 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
663 I != E; ++I) {
664 if (!MFI.isDeadObjectIndex(ObjectIdx: I))
665 return false;
666 }
667
668 return true;
669}
670
671// Shift down registers reserved for the scratch RSRC.
672Register SIFrameLowering::getEntryFunctionReservedScratchRsrcReg(
673 MachineFunction &MF) const {
674
675 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
676 const SIInstrInfo *TII = ST.getInstrInfo();
677 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
678 MachineRegisterInfo &MRI = MF.getRegInfo();
679 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
680
681 assert(MFI->isEntryFunction());
682
683 Register ScratchRsrcReg = MFI->getScratchRSrcReg();
684
685 if (!ScratchRsrcReg || (!MRI.isPhysRegUsed(PhysReg: ScratchRsrcReg) &&
686 allStackObjectsAreDead(MFI: MF.getFrameInfo())))
687 return Register();
688
689 if (ST.hasSGPRInitBug() ||
690 ScratchRsrcReg != TRI->reservedPrivateSegmentBufferReg(MF))
691 return ScratchRsrcReg;
692
693 // We reserved the last registers for this. Shift it down to the end of those
694 // which were actually used.
695 //
696 // FIXME: It might be safer to use a pseudoregister before replacement.
697
698 // FIXME: We should be able to eliminate unused input registers. We only
699 // cannot do this for the resources required for scratch access. For now we
700 // skip over user SGPRs and may leave unused holes.
701
702 unsigned NumPreloaded = (MFI->getNumPreloadedSGPRs() + 3) / 4;
703 ArrayRef<MCPhysReg> AllSGPR128s = TRI->getAllSGPR128(MF);
704 AllSGPR128s = AllSGPR128s.slice(N: std::min(a: static_cast<unsigned>(AllSGPR128s.size()), b: NumPreloaded));
705
706 // Skip the last N reserved elements because they should have already been
707 // reserved for VCC etc.
708 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
709 for (MCPhysReg Reg : AllSGPR128s) {
710 // Pick the first unallocated one. Make sure we don't clobber the other
711 // reserved input we needed. Also for PAL, make sure we don't clobber
712 // the GIT pointer passed in SGPR0 or SGPR8.
713 if (!MRI.isPhysRegUsed(PhysReg: Reg) && MRI.isAllocatable(PhysReg: Reg) &&
714 (!GITPtrLoReg || !TRI->isSubRegisterEq(RegA: Reg, RegB: GITPtrLoReg))) {
715 MRI.replaceRegWith(FromReg: ScratchRsrcReg, ToReg: Reg);
716 MFI->setScratchRSrcReg(Reg);
717 MRI.reserveReg(PhysReg: Reg, TRI);
718 return Reg;
719 }
720 }
721
722 return ScratchRsrcReg;
723}
724
725static unsigned getScratchScaleFactor(const GCNSubtarget &ST) {
726 return ST.hasFlatScratchEnabled() ? 1 : ST.getWavefrontSize();
727}
728
729void SIFrameLowering::emitEntryFunctionPrologue(MachineFunction &MF,
730 MachineBasicBlock &MBB) const {
731 assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported");
732
733 // FIXME: If we only have SGPR spills, we won't actually be using scratch
734 // memory since these spill to VGPRs. We should be cleaning up these unused
735 // SGPR spill frame indices somewhere.
736
737 // FIXME: We still have implicit uses on SGPR spill instructions in case they
738 // need to spill to vector memory. It's likely that will not happen, but at
739 // this point it appears we need the setup. This part of the prolog should be
740 // emitted after frame indices are eliminated.
741
742 // FIXME: Remove all of the isPhysRegUsed checks
743
744 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
745 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
746 const SIInstrInfo *TII = ST.getInstrInfo();
747 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
748 MachineRegisterInfo &MRI = MF.getRegInfo();
749 const Function &F = MF.getFunction();
750 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
751
752 assert(MFI->isEntryFunction());
753
754 // Debug location must be unknown since the first debug location is used to
755 // determine the end of the prologue.
756 DebugLoc DL;
757 MachineBasicBlock::iterator I = MBB.begin();
758
759 if (MF.needsFrameMoves()) {
760 // On entry the SP/FP are not set up, so we need to define the CFA in terms
761 // of a literal location expression.
762 static const char CFAEncodedInstUserOpsArr[] = {
763 dwarf::DW_CFA_def_cfa_expression,
764 4, // length
765 static_cast<char>(dwarf::DW_OP_lit0),
766 static_cast<char>(dwarf::DW_OP_lit0 +
767 dwarf::DW_ASPACE_LLVM_AMDGPU_private_wave),
768 static_cast<char>(dwarf::DW_OP_LLVM_user),
769 static_cast<char>(dwarf::DW_OP_LLVM_form_aspace_address)};
770 static StringRef CFAEncodedInstUserOps =
771 StringRef(CFAEncodedInstUserOpsArr, sizeof(CFAEncodedInstUserOpsArr));
772 buildCFI(MBB, MBBI: I, DL,
773 CFIInst: MCCFIInstruction::createEscape(L: nullptr, Vals: CFAEncodedInstUserOps,
774 Loc: SMLoc(),
775 Comment: "CFA is 0 in private_wave aspace"));
776 // Unwinding halts when the return address (PC) is undefined.
777 buildCFI(MBB, MBBI: I, DL,
778 CFIInst: MCCFIInstruction::createUndefined(
779 L: nullptr, Register: TRI->getDwarfRegNum(Reg: AMDGPU::PC_REG, isEH: false)));
780 }
781
782 Register PreloadedScratchWaveOffsetReg = MFI->getPreloadedReg(
783 Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_WAVE_BYTE_OFFSET);
784
785 // We need to do the replacement of the private segment buffer register even
786 // if there are no stack objects. There could be stores to undef or a
787 // constant without an associated object.
788 //
789 // This will return `Register()` in cases where there are no actual
790 // uses of the SRSRC.
791 Register ScratchRsrcReg;
792 if (!ST.hasFlatScratchEnabled())
793 ScratchRsrcReg = getEntryFunctionReservedScratchRsrcReg(MF);
794
795 // Make the selected register live throughout the function.
796 if (ScratchRsrcReg) {
797 for (MachineBasicBlock &OtherBB : MF) {
798 if (&OtherBB != &MBB) {
799 OtherBB.addLiveIn(PhysReg: ScratchRsrcReg);
800 }
801 }
802 }
803
804 // Now that we have fixed the reserved SRSRC we need to locate the
805 // (potentially) preloaded SRSRC.
806 Register PreloadedScratchRsrcReg;
807 if (ST.isAmdHsaOrMesa(F)) {
808 PreloadedScratchRsrcReg =
809 MFI->getPreloadedReg(Value: AMDGPUFunctionArgInfo::PRIVATE_SEGMENT_BUFFER);
810 if (ScratchRsrcReg && PreloadedScratchRsrcReg) {
811 // We added live-ins during argument lowering, but since they were not
812 // used they were deleted. We're adding the uses now, so add them back.
813 MRI.addLiveIn(Reg: PreloadedScratchRsrcReg);
814 MBB.addLiveIn(PhysReg: PreloadedScratchRsrcReg);
815 }
816 }
817
818 // We found the SRSRC first because it needs four registers and has an
819 // alignment requirement. If the SRSRC that we found is clobbering with
820 // the scratch wave offset, which may be in a fixed SGPR or a free SGPR
821 // chosen by SITargetLowering::allocateSystemSGPRs, COPY the scratch
822 // wave offset to a free SGPR.
823 Register ScratchWaveOffsetReg;
824 if (PreloadedScratchWaveOffsetReg &&
825 TRI->isSubRegisterEq(RegA: ScratchRsrcReg, RegB: PreloadedScratchWaveOffsetReg)) {
826 ArrayRef<MCPhysReg> AllSGPRs = TRI->getAllSGPR32(MF);
827 unsigned NumPreloaded = MFI->getNumPreloadedSGPRs();
828 AllSGPRs = AllSGPRs.slice(
829 N: std::min(a: static_cast<unsigned>(AllSGPRs.size()), b: NumPreloaded));
830 Register GITPtrLoReg = MFI->getGITPtrLoReg(MF);
831 for (MCPhysReg Reg : AllSGPRs) {
832 if (!MRI.isPhysRegUsed(PhysReg: Reg) && MRI.isAllocatable(PhysReg: Reg) &&
833 !TRI->isSubRegisterEq(RegA: ScratchRsrcReg, RegB: Reg) && GITPtrLoReg != Reg) {
834 ScratchWaveOffsetReg = Reg;
835 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: ScratchWaveOffsetReg)
836 .addReg(RegNo: PreloadedScratchWaveOffsetReg, Flags: RegState::Kill);
837 break;
838 }
839 }
840
841 // FIXME: We can spill incoming arguments and restore at the end of the
842 // prolog.
843 if (!ScratchWaveOffsetReg)
844 report_fatal_error(
845 reason: "could not find temporary scratch offset register in prolog");
846 } else {
847 ScratchWaveOffsetReg = PreloadedScratchWaveOffsetReg;
848 }
849 assert(ScratchWaveOffsetReg || !PreloadedScratchWaveOffsetReg);
850
851 unsigned Offset = FrameInfo.getStackSize() * getScratchScaleFactor(ST);
852 if (!mayReserveScratchForCWSR(MF)) {
853 if (hasFP(MF)) {
854 Register FPReg = MFI->getFrameOffsetReg();
855 assert(FPReg != AMDGPU::FP_REG);
856 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: FPReg).addImm(Val: 0);
857 }
858
859 if (requiresStackPointerReference(MF)) {
860 Register SPReg = MFI->getStackPtrOffsetReg();
861 assert(SPReg != AMDGPU::SP_REG);
862 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: SPReg).addImm(Val: Offset);
863 }
864 } else {
865 // We need to check if we're on a compute queue - if we are, then the CWSR
866 // trap handler may need to store some VGPRs on the stack. The first VGPR
867 // block is saved separately, so we only need to allocate space for any
868 // additional VGPR blocks used. For now, we will make sure there's enough
869 // room for the theoretical maximum number of VGPRs that can be allocated.
870 // FIXME: Figure out if the shader uses fewer VGPRs in practice.
871 assert(hasFP(MF));
872 Register FPReg = MFI->getFrameOffsetReg();
873 assert(FPReg != AMDGPU::FP_REG);
874 unsigned VGPRSize = llvm::alignTo(
875 Size: (ST.getAddressableNumVGPRs(DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize()) -
876 AMDGPU::IsaInfo::getVGPRAllocGranule(STI: ST,
877 DynamicVGPRBlockSize: MFI->getDynamicVGPRBlockSize())) *
878 4,
879 A: FrameInfo.getMaxAlign());
880 MFI->setScratchReservedForDynamicVGPRs(VGPRSize);
881
882 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::GET_STACK_BASE), DestReg: FPReg);
883 if (requiresStackPointerReference(MF)) {
884 Register SPReg = MFI->getStackPtrOffsetReg();
885 assert(SPReg != AMDGPU::SP_REG);
886
887 // If at least one of the constants can be inlined, then we can use
888 // s_cselect. Otherwise, use a mov and cmovk.
889 if (AMDGPU::isInlinableLiteral32(Literal: Offset, HasInv2Pi: ST.hasInv2PiInlineImm()) ||
890 AMDGPU::isInlinableLiteral32(Literal: Offset + VGPRSize,
891 HasInv2Pi: ST.hasInv2PiInlineImm())) {
892 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CSELECT_B32), DestReg: SPReg)
893 .addImm(Val: Offset + VGPRSize)
894 .addImm(Val: Offset);
895 } else {
896 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: SPReg).addImm(Val: Offset);
897 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_CMOVK_I32), DestReg: SPReg)
898 .addImm(Val: Offset + VGPRSize);
899 }
900 }
901 }
902
903 bool NeedsFlatScratchInit =
904 MFI->getUserSGPRInfo().hasFlatScratchInit() &&
905 (MRI.isPhysRegUsed(PhysReg: AMDGPU::FLAT_SCR) || FrameInfo.hasCalls() ||
906 (!allStackObjectsAreDead(MFI: FrameInfo) && ST.hasFlatScratchEnabled()));
907
908 if ((NeedsFlatScratchInit || ScratchRsrcReg) &&
909 PreloadedScratchWaveOffsetReg && !ST.hasArchitectedFlatScratch()) {
910 MRI.addLiveIn(Reg: PreloadedScratchWaveOffsetReg);
911 MBB.addLiveIn(PhysReg: PreloadedScratchWaveOffsetReg);
912 }
913
914 if (NeedsFlatScratchInit) {
915 emitEntryFunctionFlatScratchInit(MF, MBB, I, DL, ScratchWaveOffsetReg);
916 }
917
918 if (ScratchRsrcReg) {
919 emitEntryFunctionScratchRsrcRegSetup(MF, MBB, I, DL,
920 PreloadedPrivateBufferReg: PreloadedScratchRsrcReg,
921 ScratchRsrcReg, ScratchWaveOffsetReg);
922 }
923}
924
925void SIFrameLowering::emitEntryFunctionEpilogue(MachineFunction &MF,
926 MachineBasicBlock &MBB) const {
927 // Entry functions don't usually need an epilogue. The only exception is a
928 // block ending in a chain call (which is the only possible tail call in an
929 // entry function), in which case the current frame is becoming dead and we
930 // need to reset SP to point to the base of the stack. Other exits of the
931 // function, e.g. blocks ending in S_ENDPGM, don't need any of this.
932 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
933 const SIInstrInfo *TII = ST.getInstrInfo();
934 MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
935 assert(MBBI != MBB.end() && "return block should end in a terminator");
936 if (!TII->isTailCall(Inst: *MBBI))
937 return;
938
939 // If the prologue has set SP and there aren't any local variables, then we
940 // don't need to restore anything.
941 const MachineFrameInfo &FrameInfo = MF.getFrameInfo();
942 if (requiresStackPointerReference(MF) && FrameInfo.getStackSize() == 0)
943 return;
944
945 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
946 const DebugLoc &DL = MBBI->getDebugLoc();
947
948 Register SPReg = MFI->getStackPtrOffsetReg();
949 assert(SPReg != AMDGPU::SP_REG);
950
951 // We need to either reset SP to point before the local variables, or to
952 // initialize it in case we skipped it in the prologue. In either case,
953 // the SP needs to be either 0 or point right after the space reserved for
954 // CWSR (which the prologue conveniently puts in FP).
955 if (mayReserveScratchForCWSR(MF)) {
956 assert(hasFP(MF));
957 Register FPReg = MFI->getFrameOffsetReg();
958 assert(FPReg != AMDGPU::FP_REG);
959 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: SPReg)
960 .addReg(RegNo: FPReg)
961 .setMIFlag(MachineInstr::FrameDestroy);
962 } else {
963 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_MOV_B32), DestReg: SPReg)
964 .addImm(Val: 0)
965 .setMIFlag(MachineInstr::FrameDestroy);
966 }
967}
968
969// Emit scratch RSRC setup code, assuming `ScratchRsrcReg != AMDGPU::NoReg`
970void SIFrameLowering::emitEntryFunctionScratchRsrcRegSetup(
971 MachineFunction &MF, MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
972 const DebugLoc &DL, Register PreloadedScratchRsrcReg,
973 Register ScratchRsrcReg, Register ScratchWaveOffsetReg) const {
974
975 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
976 const SIInstrInfo *TII = ST.getInstrInfo();
977 const SIRegisterInfo *TRI = &TII->getRegisterInfo();
978 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
979 const Function &Fn = MF.getFunction();
980
981 if (ST.isAmdPalOS()) {
982 // The pointer to the GIT is formed from the offset passed in and either
983 // the amdgpu-git-ptr-high function attribute or the top part of the PC
984 Register Rsrc01 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub0_sub1);
985 Register Rsrc03 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub3);
986
987 buildGitPtr(MBB, I, DL, TII, TargetReg: Rsrc01);
988
989 // We now have the GIT ptr - now get the scratch descriptor from the entry
990 // at offset 0 (or offset 16 for a compute shader).
991 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
992 const MCInstrDesc &LoadDwordX4 = TII->get(Opcode: AMDGPU::S_LOAD_DWORDX4_IMM);
993 auto *MMO = MF.getMachineMemOperand(
994 PtrInfo,
995 F: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
996 MachineMemOperand::MODereferenceable,
997 Size: 16, BaseAlignment: Align(4));
998 unsigned Offset = Fn.getCallingConv() == CallingConv::AMDGPU_CS ? 16 : 0;
999 const GCNSubtarget &Subtarget = MF.getSubtarget<GCNSubtarget>();
1000 unsigned EncodedOffset = AMDGPU::convertSMRDOffsetUnits(ST: Subtarget, ByteOffset: Offset);
1001 BuildMI(BB&: MBB, I, MIMD: DL, MCID: LoadDwordX4, DestReg: ScratchRsrcReg)
1002 .addReg(RegNo: Rsrc01)
1003 .addImm(Val: EncodedOffset) // offset
1004 .addImm(Val: 0) // cpol
1005 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine)
1006 .addMemOperand(MMO);
1007
1008 // The driver will always set the SRD for wave 64 (bits 118:117 of
1009 // descriptor / bits 22:21 of third sub-reg will be 0b11)
1010 // If the shader is actually wave32 we have to modify the const_index_stride
1011 // field of the descriptor 3rd sub-reg (bits 22:21) to 0b10 (stride=32). The
1012 // reason the driver does this is that there can be cases where it presents
1013 // 2 shaders with different wave size (e.g. VsFs).
1014 // TODO: convert to using SCRATCH instructions or multiple SRD buffers
1015 if (ST.isWave32()) {
1016 const MCInstrDesc &SBitsetB32 = TII->get(Opcode: AMDGPU::S_BITSET0_B32);
1017 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SBitsetB32, DestReg: Rsrc03)
1018 .addImm(Val: 21)
1019 .addReg(RegNo: Rsrc03);
1020 }
1021 } else if (ST.isMesaGfxShader(F: Fn) || !PreloadedScratchRsrcReg) {
1022 assert(!ST.isAmdHsaOrMesa(Fn));
1023 const MCInstrDesc &SMovB32 = TII->get(Opcode: AMDGPU::S_MOV_B32);
1024
1025 Register Rsrc2 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub2);
1026 Register Rsrc3 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub3);
1027
1028 // Use relocations to get the pointer, and setup the other bits manually.
1029 uint64_t Rsrc23 = TII->getScratchRsrcWords23();
1030
1031 if (MFI->getUserSGPRInfo().hasImplicitBufferPtr()) {
1032 Register Rsrc01 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub0_sub1);
1033
1034 if (AMDGPU::isCompute(CC: MF.getFunction().getCallingConv())) {
1035 const MCInstrDesc &Mov64 = TII->get(Opcode: AMDGPU::S_MOV_B64);
1036
1037 BuildMI(BB&: MBB, I, MIMD: DL, MCID: Mov64, DestReg: Rsrc01)
1038 .addReg(RegNo: MFI->getImplicitBufferPtrUserSGPR())
1039 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1040 } else {
1041 const MCInstrDesc &LoadDwordX2 = TII->get(Opcode: AMDGPU::S_LOAD_DWORDX2_IMM);
1042
1043 MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS);
1044 auto *MMO = MF.getMachineMemOperand(
1045 PtrInfo,
1046 F: MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
1047 MachineMemOperand::MODereferenceable,
1048 Size: 8, BaseAlignment: Align(4));
1049 BuildMI(BB&: MBB, I, MIMD: DL, MCID: LoadDwordX2, DestReg: Rsrc01)
1050 .addReg(RegNo: MFI->getImplicitBufferPtrUserSGPR())
1051 .addImm(Val: 0) // offset
1052 .addImm(Val: 0) // cpol
1053 .addMemOperand(MMO)
1054 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1055 }
1056
1057 MF.getRegInfo().addLiveIn(Reg: MFI->getImplicitBufferPtrUserSGPR());
1058 MBB.addLiveIn(PhysReg: MFI->getImplicitBufferPtrUserSGPR());
1059 } else {
1060 Register Rsrc0 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub0);
1061 Register Rsrc1 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub1);
1062
1063 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: Rsrc0)
1064 .addExternalSymbol(FnName: "SCRATCH_RSRC_DWORD0")
1065 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1066
1067 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: Rsrc1)
1068 .addExternalSymbol(FnName: "SCRATCH_RSRC_DWORD1")
1069 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1070 }
1071
1072 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: Rsrc2)
1073 .addImm(Val: Lo_32(Value: Rsrc23))
1074 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1075
1076 BuildMI(BB&: MBB, I, MIMD: DL, MCID: SMovB32, DestReg: Rsrc3)
1077 .addImm(Val: Hi_32(Value: Rsrc23))
1078 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1079 } else if (ST.isAmdHsaOrMesa(F: Fn)) {
1080 assert(PreloadedScratchRsrcReg);
1081
1082 if (ScratchRsrcReg != PreloadedScratchRsrcReg) {
1083 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: ScratchRsrcReg)
1084 .addReg(RegNo: PreloadedScratchRsrcReg, Flags: RegState::Kill);
1085 }
1086 }
1087
1088 // Add the scratch wave offset into the scratch RSRC.
1089 //
1090 // We only want to update the first 48 bits, which is the base address
1091 // pointer, without touching the adjacent 16 bits of flags. We know this add
1092 // cannot carry-out from bit 47, otherwise the scratch allocation would be
1093 // impossible to fit in the 48-bit global address space.
1094 //
1095 // TODO: Evaluate if it is better to just construct an SRD using the flat
1096 // scratch init and some constants rather than update the one we are passed.
1097 Register ScratchRsrcSub0 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub0);
1098 Register ScratchRsrcSub1 = TRI->getSubReg(Reg: ScratchRsrcReg, Idx: AMDGPU::sub1);
1099
1100 // We cannot Kill ScratchWaveOffsetReg here because we allow it to be used in
1101 // the kernel body via inreg arguments.
1102 BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_U32), DestReg: ScratchRsrcSub0)
1103 .addReg(RegNo: ScratchRsrcSub0)
1104 .addReg(RegNo: ScratchWaveOffsetReg)
1105 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1106 auto Addc = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADDC_U32), DestReg: ScratchRsrcSub1)
1107 .addReg(RegNo: ScratchRsrcSub1)
1108 .addImm(Val: 0)
1109 .addReg(RegNo: ScratchRsrcReg, Flags: RegState::ImplicitDefine);
1110 Addc->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
1111}
1112
1113bool SIFrameLowering::isSupportedStackID(TargetStackID::Value ID) const {
1114 switch (ID) {
1115 case TargetStackID::Default:
1116 case TargetStackID::NoAlloc:
1117 case TargetStackID::SGPRSpill:
1118 return true;
1119 case TargetStackID::ScalableVector:
1120 case TargetStackID::WasmLocal:
1121 case TargetStackID::ScalablePredicateVector:
1122 case TargetStackID::AvrAlign:
1123 return false;
1124 }
1125 llvm_unreachable("Invalid TargetStackID::Value");
1126}
1127
1128void SIFrameLowering::emitPrologueEntryCFI(MachineBasicBlock &MBB,
1129 MachineBasicBlock::iterator MBBI,
1130 const DebugLoc &DL) const {
1131 const MachineFunction &MF = *MBB.getParent();
1132 const MachineRegisterInfo &MRI = MF.getRegInfo();
1133 const MCRegisterInfo *MCRI = MF.getContext().getRegisterInfo();
1134 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1135 const SIRegisterInfo &TRI = ST.getInstrInfo()->getRegisterInfo();
1136 MCRegister StackPtrReg =
1137 MF.getInfo<SIMachineFunctionInfo>()->getStackPtrOffsetReg();
1138
1139 emitDefCFA(MBB, MBBI, DL, StackPtrReg, /*AspaceAlreadyDefined=*/true,
1140 Flags: MachineInstr::FrameSetup);
1141
1142 buildCFIForSRegToSRegSpill(MBB, MBBI, DL, Reg: AMDGPU::PC_REG,
1143 CopyReg: TRI.getReturnAddressReg(MF));
1144
1145 BitVector IsCalleeSaved(TRI.getNumRegs());
1146 const MCPhysReg *CSRegs = MRI.getCalleeSavedRegs();
1147 for (unsigned I = 0; CSRegs[I]; ++I) {
1148 IsCalleeSaved.set(CSRegs[I]);
1149 }
1150 auto ProcessReg = [&](MCPhysReg Reg) {
1151 // VCC is not preserved across calls.
1152 if (Reg == AMDGPU::VCC || Reg == AMDGPU::VCC_LO || Reg == AMDGPU::VCC_HI)
1153 return;
1154 if (IsCalleeSaved.test(Idx: Reg) || !MRI.isPhysRegModified(PhysReg: Reg))
1155 return;
1156 unsigned DwarfReg = MCRI->getDwarfRegNum(Reg, isEH: false);
1157 buildCFI(MBB, MBBI, DL,
1158 CFIInst: MCCFIInstruction::createUndefined(L: nullptr, Register: DwarfReg));
1159 };
1160
1161 // Emit CFI rules for caller saved Arch VGPRs which are clobbered
1162 unsigned NumArchVGPRs = ST.getAddressableNumArchVGPRs();
1163 for_each(Range: AMDGPU::VGPR_32RegClass.getRegisters().take_front(N: NumArchVGPRs),
1164 F: ProcessReg);
1165
1166 // Emit CFI rules for caller saved Accum VGPRs which are clobbered
1167 if (ST.hasMAIInsts()) {
1168 for_each(Range: AMDGPU::AGPR_32RegClass.getRegisters(), F: ProcessReg);
1169 }
1170
1171 // Emit CFI rules for caller saved SGPRs which are clobbered
1172 for_each(Range: AMDGPU::SGPR_32RegClass.getRegisters(), F: ProcessReg);
1173}
1174
1175// Activate only the inactive lanes when \p EnableInactiveLanes is true.
1176// Otherwise, activate all lanes. It returns the saved exec.
1177static Register buildScratchExecCopy(LiveRegUnits &LiveUnits,
1178 MachineFunction &MF,
1179 MachineBasicBlock &MBB,
1180 MachineBasicBlock::iterator MBBI,
1181 const DebugLoc &DL, bool IsProlog,
1182 bool EnableInactiveLanes) {
1183 Register ScratchExecCopy;
1184 MachineRegisterInfo &MRI = MF.getRegInfo();
1185 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1186 const SIInstrInfo *TII = ST.getInstrInfo();
1187 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1188 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1189
1190 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, IsProlog);
1191
1192 if (FuncInfo->isWholeWaveFunction()) {
1193 // Whole wave functions already have a copy of the original EXEC mask that
1194 // we can use.
1195 assert(IsProlog && "Epilog should look at return, not setup");
1196 ScratchExecCopy =
1197 TII->getWholeWaveFunctionSetup(MF)->getOperand(i: 0).getReg();
1198 assert(ScratchExecCopy && "Couldn't find copy of EXEC");
1199 } else {
1200 ScratchExecCopy = findScratchNonCalleeSaveRegister(
1201 MRI, LiveUnits, RC: *TRI.getWaveMaskRegClass());
1202 }
1203
1204 if (!ScratchExecCopy)
1205 report_fatal_error(reason: "failed to find free scratch register");
1206
1207 LiveUnits.addReg(Reg: ScratchExecCopy);
1208
1209 const unsigned SaveExecOpc =
1210 ST.isWave32() ? (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B32
1211 : AMDGPU::S_OR_SAVEEXEC_B32)
1212 : (EnableInactiveLanes ? AMDGPU::S_XOR_SAVEEXEC_B64
1213 : AMDGPU::S_OR_SAVEEXEC_B64);
1214 auto SaveExec =
1215 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: SaveExecOpc), DestReg: ScratchExecCopy).addImm(Val: -1);
1216 SaveExec->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
1217
1218 return ScratchExecCopy;
1219}
1220
1221void SIFrameLowering::emitCSRSpillStores(
1222 MachineFunction &MF, MachineBasicBlock &MBB,
1223 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1224 LiveRegUnits &LiveUnits, Register FrameReg, Register FramePtrRegScratchCopy,
1225 const bool NeedsFrameMoves) const {
1226 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1227 MachineFrameInfo &MFI = MF.getFrameInfo();
1228 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1229 const SIInstrInfo *TII = ST.getInstrInfo();
1230 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1231 const MCRegisterInfo *MCRI = MF.getContext().getRegisterInfo();
1232 MachineRegisterInfo &MRI = MF.getRegInfo();
1233 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
1234
1235 // Spill Whole-Wave Mode VGPRs. Save only the inactive lanes of the scratch
1236 // registers. However, save all lanes of callee-saved VGPRs. Due to this, we
1237 // might end up flipping the EXEC bits twice.
1238 Register ScratchExecCopy;
1239 SmallVector<std::pair<Register, int>, 2> WWMCalleeSavedRegs, WWMScratchRegs;
1240 FuncInfo->splitWWMSpillRegisters(MF, CalleeSavedRegs&: WWMCalleeSavedRegs, ScratchRegs&: WWMScratchRegs);
1241 if (!WWMScratchRegs.empty())
1242 ScratchExecCopy =
1243 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1244 /*IsProlog*/ true, /*EnableInactiveLanes*/ true);
1245
1246 auto StoreWWMRegisters =
1247 [&](SmallVectorImpl<std::pair<Register, int>> &WWMRegs) {
1248 for (const auto &Reg : WWMRegs) {
1249 Register VGPR = Reg.first;
1250 int FI = Reg.second;
1251 buildPrologSpill(ST, TRI, FuncInfo: *FuncInfo, LiveUnits, MF, MBB, I: MBBI, DL,
1252 SpillReg: VGPR, FI, FrameReg);
1253 if (NeedsFrameMoves) {
1254 // We spill the entire VGPR, so we can get away with just cfi_offset
1255 buildCFI(MBB, MBBI, DL,
1256 CFIInst: MCCFIInstruction::createOffset(
1257 L: nullptr, Register: MCRI->getDwarfRegNum(Reg: VGPR, isEH: false),
1258 Offset: MFI.getObjectOffset(ObjectIdx: FI) * ST.getWavefrontSize()));
1259 }
1260 }
1261 };
1262
1263 for (const Register Reg : make_first_range(c&: WWMScratchRegs)) {
1264 if (!MRI.isReserved(PhysReg: Reg)) {
1265 MRI.addLiveIn(Reg);
1266 MBB.addLiveIn(PhysReg: Reg);
1267 }
1268 }
1269 StoreWWMRegisters(WWMScratchRegs);
1270
1271 auto EnableAllLanes = [&]() {
1272 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
1273 };
1274
1275 if (!WWMCalleeSavedRegs.empty()) {
1276 if (ScratchExecCopy) {
1277 EnableAllLanes();
1278 } else {
1279 ScratchExecCopy = buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1280 /*IsProlog*/ true,
1281 /*EnableInactiveLanes*/ false);
1282 }
1283 }
1284
1285 StoreWWMRegisters(WWMCalleeSavedRegs);
1286 if (FuncInfo->isWholeWaveFunction()) {
1287 // If we have already saved some WWM CSR registers, then the EXEC is already
1288 // -1 and we don't need to do anything else. Otherwise, save the original
1289 // EXEC into the setup register and set EXEC to -1 here.
1290 if (!ScratchExecCopy)
1291 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL, /*IsProlog*/ true,
1292 /*EnableInactiveLanes*/ false);
1293 else if (WWMCalleeSavedRegs.empty())
1294 EnableAllLanes();
1295 } else if (ScratchExecCopy) {
1296 // FIXME: Split block and make terminator.
1297 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
1298 .addReg(RegNo: ScratchExecCopy, Flags: RegState::Kill);
1299 LiveUnits.addReg(Reg: ScratchExecCopy);
1300 }
1301
1302 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1303
1304 for (const auto &Spill : FuncInfo->getPrologEpilogSGPRSpills()) {
1305 // Special handle FP spill:
1306 // Skip if FP is saved to a scratch SGPR, the save has already been emitted.
1307 // Otherwise, FP has been moved to a temporary register and spill it
1308 // instead.
1309 bool IsFramePtrPrologSpill = Spill.first == FramePtrReg;
1310 Register Reg = IsFramePtrPrologSpill ? FramePtrRegScratchCopy : Spill.first;
1311 if (!Reg)
1312 continue;
1313
1314 PrologEpilogSGPRSpillBuilder SB(Reg, Spill.second, MBB, MBBI, DL, TII, TRI,
1315 LiveUnits, FrameReg, IsFramePtrPrologSpill);
1316 SB.save();
1317 }
1318
1319 // If a copy to scratch SGPR has been chosen for any of the SGPR spills, make
1320 // such scratch registers live throughout the function.
1321 SmallVector<Register, 1> ScratchSGPRs;
1322 FuncInfo->getAllScratchSGPRCopyDstRegs(Regs&: ScratchSGPRs);
1323 if (!ScratchSGPRs.empty()) {
1324 for (MachineBasicBlock &MBB : MF) {
1325 for (MCPhysReg Reg : ScratchSGPRs)
1326 MBB.addLiveIn(PhysReg: Reg);
1327
1328 MBB.sortUniqueLiveIns();
1329 }
1330 if (!LiveUnits.empty()) {
1331 for (MCPhysReg Reg : ScratchSGPRs)
1332 LiveUnits.addReg(Reg);
1333 }
1334 }
1335
1336 // Remove the spill entry created for EXEC. It is needed only for CFISaves in
1337 // the prologue.
1338 if (TRI.isCFISavedRegsSpillEnabled())
1339 FuncInfo->removePrologEpilogSGPRSpillEntry(Reg: TRI.getExec());
1340}
1341
1342void SIFrameLowering::emitCSRSpillRestores(
1343 MachineFunction &MF, MachineBasicBlock &MBB,
1344 MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
1345 LiveRegUnits &LiveUnits, Register FrameReg,
1346 Register FramePtrRegScratchCopy) const {
1347 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1348 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1349 const SIInstrInfo *TII = ST.getInstrInfo();
1350 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1351 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
1352 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1353
1354 for (const auto &Spill : FuncInfo->getPrologEpilogSGPRSpills()) {
1355 // Special handle FP restore:
1356 // Skip if FP needs to be restored from the scratch SGPR. Otherwise, restore
1357 // the FP value to a temporary register. The frame pointer should be
1358 // overwritten only at the end when all other spills are restored from
1359 // current frame.
1360 Register Reg =
1361 Spill.first == FramePtrReg ? FramePtrRegScratchCopy : Spill.first;
1362 if (!Reg)
1363 continue;
1364
1365 PrologEpilogSGPRSpillBuilder SB(Reg, Spill.second, MBB, MBBI, DL, TII, TRI,
1366 LiveUnits, FrameReg);
1367 SB.restore();
1368 }
1369
1370 // Restore Whole-Wave Mode VGPRs. Restore only the inactive lanes of the
1371 // scratch registers. However, restore all lanes of callee-saved VGPRs. Due to
1372 // this, we might end up flipping the EXEC bits twice.
1373 Register ScratchExecCopy;
1374 SmallVector<std::pair<Register, int>, 2> WWMCalleeSavedRegs, WWMScratchRegs;
1375 FuncInfo->splitWWMSpillRegisters(MF, CalleeSavedRegs&: WWMCalleeSavedRegs, ScratchRegs&: WWMScratchRegs);
1376 auto RestoreWWMRegisters =
1377 [&](SmallVectorImpl<std::pair<Register, int>> &WWMRegs) {
1378 for (const auto &Reg : WWMRegs) {
1379 Register VGPR = Reg.first;
1380 int FI = Reg.second;
1381 buildEpilogRestore(ST, TRI, FuncInfo: *FuncInfo, LiveUnits, MF, MBB, I: MBBI, DL,
1382 SpillReg: VGPR, FI, FrameReg);
1383 }
1384 };
1385
1386 if (FuncInfo->isWholeWaveFunction()) {
1387 // For whole wave functions, the EXEC is already -1 at this point.
1388 // Therefore, we can restore the CSR WWM registers right away.
1389 RestoreWWMRegisters(WWMCalleeSavedRegs);
1390
1391 // The original EXEC is the first operand of the return instruction.
1392 MachineInstr &Return = MBB.instr_back();
1393 unsigned Opcode = Return.getOpcode();
1394 switch (Opcode) {
1395 case AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN:
1396 Opcode = AMDGPU::SI_RETURN;
1397 break;
1398 case AMDGPU::SI_TCRETURN_GFX_WholeWave:
1399 Opcode = AMDGPU::SI_TCRETURN_GFX;
1400 break;
1401 default:
1402 llvm_unreachable("Unexpected return inst");
1403 }
1404 Register OrigExec = Return.getOperand(i: 0).getReg();
1405
1406 if (!WWMScratchRegs.empty()) {
1407 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.XorOpc), DestReg: LMC.ExecReg)
1408 .addReg(RegNo: OrigExec)
1409 .addImm(Val: -1);
1410 RestoreWWMRegisters(WWMScratchRegs);
1411 }
1412
1413 // Restore original EXEC.
1414 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addReg(RegNo: OrigExec);
1415
1416 // Drop the first operand and update the opcode.
1417 Return.removeOperand(OpNo: 0);
1418 Return.setDesc(TII->get(Opcode));
1419
1420 return;
1421 }
1422
1423 if (!WWMScratchRegs.empty()) {
1424 ScratchExecCopy =
1425 buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1426 /*IsProlog=*/false, /*EnableInactiveLanes=*/true);
1427 }
1428 RestoreWWMRegisters(WWMScratchRegs);
1429 if (!WWMCalleeSavedRegs.empty()) {
1430 if (ScratchExecCopy) {
1431 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg).addImm(Val: -1);
1432 } else {
1433 ScratchExecCopy = buildScratchExecCopy(LiveUnits, MF, MBB, MBBI, DL,
1434 /*IsProlog*/ false,
1435 /*EnableInactiveLanes*/ false);
1436 }
1437 }
1438
1439 RestoreWWMRegisters(WWMCalleeSavedRegs);
1440 if (ScratchExecCopy) {
1441 // FIXME: Split block and make terminator.
1442 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: LMC.MovOpc), DestReg: LMC.ExecReg)
1443 .addReg(RegNo: ScratchExecCopy, Flags: RegState::Kill);
1444 }
1445}
1446
1447void SIFrameLowering::emitPrologue(MachineFunction &MF,
1448 MachineBasicBlock &MBB) const {
1449 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1450 if (FuncInfo->isEntryFunction()) {
1451 emitEntryFunctionPrologue(MF, MBB);
1452 return;
1453 }
1454
1455 MachineFrameInfo &MFI = MF.getFrameInfo();
1456 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1457 const SIInstrInfo *TII = ST.getInstrInfo();
1458 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1459 MachineRegisterInfo &MRI = MF.getRegInfo();
1460
1461 Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
1462 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1463 Register BasePtrReg =
1464 TRI.hasBasePointer(MF) ? TRI.getBaseRegister() : Register();
1465 LiveRegUnits LiveUnits;
1466
1467 MachineBasicBlock::iterator MBBI = MBB.begin();
1468 // DebugLoc must be unknown since the first instruction with DebugLoc is used
1469 // to determine the end of the prologue.
1470 DebugLoc DL;
1471
1472 bool HasFP = false;
1473 bool HasBP = false;
1474 int64_t NumBytes = MFI.getStackSize();
1475 int64_t RoundedSize = NumBytes;
1476
1477 // Functions that never return don't need to save and restore the FP or BP.
1478 const Function &F = MF.getFunction();
1479 bool SavesStackRegs =
1480 !F.hasFnAttribute(Kind: Attribute::NoReturn) && !FuncInfo->isChainFunction();
1481
1482 const bool NeedsFrameMoves = MF.needsFrameMoves();
1483
1484 if (NeedsFrameMoves)
1485 emitPrologueEntryCFI(MBB, MBBI, DL);
1486
1487 if (TRI.hasStackRealignment(MF))
1488 HasFP = true;
1489
1490 Register FramePtrRegScratchCopy;
1491 if (!HasFP && !hasFP(MF)) {
1492 // Emit the CSR spill stores with SP base register.
1493 emitCSRSpillStores(MF, MBB, MBBI, DL, LiveUnits, FrameReg: StackPtrReg,
1494 FramePtrRegScratchCopy, NeedsFrameMoves);
1495 } else if (SavesStackRegs) {
1496 // CSR spill stores will use FP as base register.
1497 Register SGPRForFPSaveRestoreCopy =
1498 FuncInfo->getScratchSGPRCopyDstReg(Reg: FramePtrReg);
1499
1500 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ true);
1501 if (SGPRForFPSaveRestoreCopy) {
1502 // Copy FP to the scratch register now and emit the CFI entry. It avoids
1503 // the extra FP copy needed in the other two cases when FP is spilled to
1504 // memory or to a VGPR lane.
1505 PrologEpilogSGPRSpillBuilder SB(
1506 FramePtrReg,
1507 FuncInfo->getPrologEpilogSGPRSaveRestoreInfo(Reg: FramePtrReg), MBB, MBBI,
1508 DL, TII, TRI, LiveUnits, FramePtrReg,
1509 /*IsFramePtrPrologSpill*/ true);
1510 SB.save();
1511 LiveUnits.addReg(Reg: SGPRForFPSaveRestoreCopy);
1512 } else {
1513 // Copy FP into a new scratch register so that its previous value can be
1514 // spilled after setting up the new frame.
1515 FramePtrRegScratchCopy = findScratchNonCalleeSaveRegister(
1516 MRI, LiveUnits, RC: AMDGPU::SReg_32_XM0_XEXECRegClass);
1517 if (!FramePtrRegScratchCopy)
1518 report_fatal_error(reason: "failed to find free scratch register");
1519
1520 LiveUnits.addReg(Reg: FramePtrRegScratchCopy);
1521 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: FramePtrRegScratchCopy)
1522 .addReg(RegNo: FramePtrReg);
1523 }
1524 }
1525
1526 if (HasFP) {
1527 const unsigned Alignment = MFI.getMaxAlign().value();
1528
1529 RoundedSize += Alignment;
1530 if (LiveUnits.empty()) {
1531 LiveUnits.init(TRI);
1532 LiveUnits.addLiveIns(MBB);
1533 }
1534
1535 // s_add_i32 s33, s32, NumBytes
1536 // s_and_b32 s33, s33, 0b111...0000
1537 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: FramePtrReg)
1538 .addReg(RegNo: StackPtrReg)
1539 .addImm(Val: (Alignment - 1) * getScratchScaleFactor(ST))
1540 .setMIFlag(MachineInstr::FrameSetup);
1541 auto And = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_AND_B32), DestReg: FramePtrReg)
1542 .addReg(RegNo: FramePtrReg, Flags: RegState::Kill)
1543 .addImm(Val: -Alignment * getScratchScaleFactor(ST))
1544 .setMIFlag(MachineInstr::FrameSetup);
1545 And->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
1546 FuncInfo->setIsStackRealigned(true);
1547 } else if ((HasFP = hasFP(MF))) {
1548 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: FramePtrReg)
1549 .addReg(RegNo: StackPtrReg)
1550 .setMIFlag(MachineInstr::FrameSetup);
1551 }
1552
1553 // If FP is used, emit the CSR spills with FP base register.
1554 if (HasFP) {
1555 emitCSRSpillStores(MF, MBB, MBBI, DL, LiveUnits, FrameReg: FramePtrReg,
1556 FramePtrRegScratchCopy, NeedsFrameMoves);
1557 if (FramePtrRegScratchCopy)
1558 LiveUnits.removeReg(Reg: FramePtrRegScratchCopy);
1559 }
1560
1561 // If we need a base pointer, set it up here. It's whatever the value of
1562 // the stack pointer is at this point. Any variable size objects will be
1563 // allocated after this, so we can still use the base pointer to reference
1564 // the incoming arguments.
1565 if ((HasBP = TRI.hasBasePointer(MF))) {
1566 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: BasePtrReg)
1567 .addReg(RegNo: StackPtrReg)
1568 .setMIFlag(MachineInstr::FrameSetup);
1569 }
1570
1571 if (HasFP) {
1572 if (NeedsFrameMoves)
1573 emitDefCFA(MBB, MBBI, DL, StackPtrReg: FramePtrReg, /*AspaceAlreadyDefined=*/false,
1574 Flags: MachineInstr::FrameSetup);
1575 }
1576
1577 if (HasFP && RoundedSize != 0) {
1578 auto Add = BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: StackPtrReg)
1579 .addReg(RegNo: StackPtrReg)
1580 .addImm(Val: RoundedSize * getScratchScaleFactor(ST))
1581 .setMIFlag(MachineInstr::FrameSetup);
1582 Add->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
1583 }
1584
1585 bool FPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(Reg: FramePtrReg);
1586 (void)FPSaved;
1587 assert((!HasFP || FPSaved || !SavesStackRegs) &&
1588 "Needed to save FP but didn't save it anywhere");
1589
1590 // If we allow spilling to AGPRs we may have saved FP but then spill
1591 // everything into AGPRs instead of the stack.
1592 assert((HasFP || !FPSaved || !SavesStackRegs ||
1593 ST.getCLOpts().spill_vgpr_to_agpr) &&
1594 "Saved FP but didn't need it");
1595
1596 bool BPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(Reg: BasePtrReg);
1597 (void)BPSaved;
1598 assert((!HasBP || BPSaved || !SavesStackRegs) &&
1599 "Needed to save BP but didn't save it anywhere");
1600
1601 assert((HasBP || !BPSaved) && "Saved BP but didn't need it");
1602
1603 if (FuncInfo->isWholeWaveFunction()) {
1604 // SI_WHOLE_WAVE_FUNC_SETUP has outlived its purpose.
1605 TII->getWholeWaveFunctionSetup(MF)->eraseFromParent();
1606 }
1607}
1608
1609void SIFrameLowering::emitEpilogue(MachineFunction &MF,
1610 MachineBasicBlock &MBB) const {
1611 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1612 if (FuncInfo->isEntryFunction()) {
1613 emitEntryFunctionEpilogue(MF, MBB);
1614 return;
1615 }
1616
1617 const MachineFrameInfo &MFI = MF.getFrameInfo();
1618 if (FuncInfo->isChainFunction() && !MFI.hasTailCall())
1619 return;
1620
1621 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1622 const SIInstrInfo *TII = ST.getInstrInfo();
1623 const SIRegisterInfo &TRI = TII->getRegisterInfo();
1624 MachineRegisterInfo &MRI = MF.getRegInfo();
1625 LiveRegUnits LiveUnits;
1626 // Get the insert location for the epilogue. If there were no terminators in
1627 // the block, get the last instruction.
1628 MachineBasicBlock::iterator MBBI = MBB.end();
1629 DebugLoc DL;
1630 if (!MBB.empty()) {
1631 MBBI = MBB.getLastNonDebugInstr();
1632 if (MBBI != MBB.end())
1633 DL = MBBI->getDebugLoc();
1634
1635 MBBI = MBB.getFirstTerminator();
1636 }
1637
1638 uint32_t NumBytes = MFI.getStackSize();
1639 uint32_t RoundedSize = FuncInfo->isStackRealigned()
1640 ? NumBytes + MFI.getMaxAlign().value()
1641 : NumBytes;
1642 const Register StackPtrReg = FuncInfo->getStackPtrOffsetReg();
1643 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
1644 bool FPSaved = FuncInfo->hasPrologEpilogSGPRSpillEntry(Reg: FramePtrReg);
1645
1646 if (RoundedSize != 0) {
1647 if (TRI.hasBasePointer(MF)) {
1648 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: StackPtrReg)
1649 .addReg(RegNo: TRI.getBaseRegister())
1650 .setMIFlag(MachineInstr::FrameDestroy);
1651 } else if (hasFP(MF)) {
1652 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: StackPtrReg)
1653 .addReg(RegNo: FramePtrReg)
1654 .setMIFlag(MachineInstr::FrameDestroy);
1655 }
1656 }
1657
1658 Register FramePtrRegScratchCopy;
1659 Register SGPRForFPSaveRestoreCopy =
1660 FuncInfo->getScratchSGPRCopyDstReg(Reg: FramePtrReg);
1661 if (FPSaved) {
1662 // CSR spill restores should use FP as base register. If
1663 // SGPRForFPSaveRestoreCopy is not true, restore the previous value of FP
1664 // into a new scratch register and copy to FP later when other registers are
1665 // restored from the current stack frame.
1666 initLiveUnits(LiveUnits, TRI, FuncInfo, MF, MBB, MBBI, /*IsProlog*/ false);
1667 if (SGPRForFPSaveRestoreCopy) {
1668 LiveUnits.addReg(Reg: SGPRForFPSaveRestoreCopy);
1669 } else {
1670 FramePtrRegScratchCopy = findScratchNonCalleeSaveRegister(
1671 MRI, LiveUnits, RC: AMDGPU::SReg_32_XM0_XEXECRegClass);
1672 if (!FramePtrRegScratchCopy)
1673 report_fatal_error(reason: "failed to find free scratch register");
1674
1675 LiveUnits.addReg(Reg: FramePtrRegScratchCopy);
1676 }
1677
1678 emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveUnits, FrameReg: FramePtrReg,
1679 FramePtrRegScratchCopy);
1680 }
1681
1682 if (hasFP(MF) && MF.needsFrameMoves()) {
1683 emitDefCFA(MBB, MBBI, DL, StackPtrReg, /*AspaceAlreadyDefined=*/false,
1684 Flags: MachineInstr::FrameDestroy);
1685 }
1686
1687 if (FPSaved) {
1688 // Insert the copy to restore FP.
1689 Register SrcReg = SGPRForFPSaveRestoreCopy ? SGPRForFPSaveRestoreCopy
1690 : FramePtrRegScratchCopy;
1691 MachineInstrBuilder MIB =
1692 BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::COPY), DestReg: FramePtrReg)
1693 .addReg(RegNo: SrcReg);
1694 if (SGPRForFPSaveRestoreCopy)
1695 MIB.setMIFlag(MachineInstr::FrameDestroy);
1696 } else {
1697 // Insert the CSR spill restores with SP as the base register.
1698 emitCSRSpillRestores(MF, MBB, MBBI, DL, LiveUnits, FrameReg: StackPtrReg,
1699 FramePtrRegScratchCopy);
1700 }
1701}
1702
1703#ifndef NDEBUG
1704static bool allSGPRSpillsAreDead(const MachineFunction &MF) {
1705 const MachineFrameInfo &MFI = MF.getFrameInfo();
1706 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1707 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd();
1708 I != E; ++I) {
1709 if (!MFI.isDeadObjectIndex(I) &&
1710 MFI.getStackID(I) == TargetStackID::SGPRSpill &&
1711 !FuncInfo->checkIndexInPrologEpilogSGPRSpills(I)) {
1712 return false;
1713 }
1714 }
1715
1716 return true;
1717}
1718#endif
1719
1720StackOffset SIFrameLowering::getFrameIndexReference(const MachineFunction &MF,
1721 int FI,
1722 Register &FrameReg) const {
1723 const SIRegisterInfo *RI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
1724
1725 FrameReg = RI->getFrameRegister(MF);
1726 return StackOffset::getFixed(Fixed: MF.getFrameInfo().getObjectOffset(ObjectIdx: FI));
1727}
1728
1729void SIFrameLowering::processFunctionBeforeFrameFinalized(
1730 MachineFunction &MF,
1731 RegScavenger *RS) const {
1732 MachineFrameInfo &MFI = MF.getFrameInfo();
1733
1734 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1735 const SIInstrInfo *TII = ST.getInstrInfo();
1736 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1737 MachineRegisterInfo &MRI = MF.getRegInfo();
1738 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1739
1740 const bool SpillVGPRToAGPR = ST.hasMAIInsts() &&
1741 FuncInfo->hasSpilledVGPRs() &&
1742 ST.getCLOpts().spill_vgpr_to_agpr;
1743
1744 if (SpillVGPRToAGPR) {
1745 // To track the spill frame indices handled in this pass.
1746 BitVector SpillFIs(MFI.getObjectIndexEnd(), false);
1747 BitVector NonVGPRSpillFIs(MFI.getObjectIndexEnd(), false);
1748
1749 bool SeenDbgInstr = false;
1750
1751 for (MachineBasicBlock &MBB : MF) {
1752 for (MachineInstr &MI : llvm::make_early_inc_range(Range&: MBB)) {
1753 int FrameIndex;
1754 if (MI.isDebugInstr())
1755 SeenDbgInstr = true;
1756
1757 if (TII->isVGPRSpill(MI)) {
1758 // Try to eliminate stack used by VGPR spills before frame
1759 // finalization.
1760 unsigned FIOp = AMDGPU::getNamedOperandIdx(Opcode: MI.getOpcode(),
1761 Name: AMDGPU::OpName::vaddr);
1762 int FI = MI.getOperand(i: FIOp).getIndex();
1763 Register VReg =
1764 TII->getNamedOperand(MI, OperandName: AMDGPU::OpName::vdata)->getReg();
1765 if (FuncInfo->allocateVGPRSpillToAGPR(MF, FI,
1766 isAGPRtoVGPR: TRI->isAGPR(MRI, Reg: VReg))) {
1767 assert(RS != nullptr);
1768 RS->enterBasicBlockEnd(MBB);
1769 RS->backward(I: std::next(x: MI.getIterator()));
1770 TRI->eliminateFrameIndex(MI, SPAdj: 0, FIOperandNum: FIOp, RS);
1771 SpillFIs.set(FI);
1772 continue;
1773 }
1774 } else if (TII->isStoreToStackSlot(MI, FrameIndex) ||
1775 TII->isLoadFromStackSlot(MI, FrameIndex))
1776 if (!MFI.isFixedObjectIndex(ObjectIdx: FrameIndex))
1777 NonVGPRSpillFIs.set(FrameIndex);
1778 }
1779 }
1780
1781 // Stack slot coloring may assign different objects to the same stack slot.
1782 // If not, then the VGPR to AGPR spill slot is dead.
1783 for (unsigned FI : SpillFIs.set_bits())
1784 if (!NonVGPRSpillFIs.test(Idx: FI))
1785 FuncInfo->setVGPRToAGPRSpillDead(FI);
1786
1787 for (MachineBasicBlock &MBB : MF) {
1788 for (MCPhysReg Reg : FuncInfo->getVGPRSpillAGPRs())
1789 MBB.addLiveIn(PhysReg: Reg);
1790
1791 for (MCPhysReg Reg : FuncInfo->getAGPRSpillVGPRs())
1792 MBB.addLiveIn(PhysReg: Reg);
1793
1794 MBB.sortUniqueLiveIns();
1795
1796 if (!SpillFIs.empty() && SeenDbgInstr)
1797 clearDebugInfoForSpillFIs(MFI, MBB, SpillFIs);
1798 }
1799 }
1800
1801 // At this point we've already allocated all spilled SGPRs to VGPRs if we
1802 // can. Any remaining SGPR spills will go to memory, so move them back to the
1803 // default stack.
1804 bool HaveSGPRToVMemSpill =
1805 FuncInfo->removeDeadFrameIndices(MFI, /*ResetSGPRSpillStackIDs*/ true);
1806 assert(allSGPRSpillsAreDead(MF) &&
1807 "SGPR spill should have been removed in SILowerSGPRSpills");
1808
1809 // FIXME: The other checks should be redundant with allStackObjectsAreDead,
1810 // but currently hasNonSpillStackObjects is set only from source
1811 // allocas. Stack temps produced from legalization are not counted currently.
1812 if (!allStackObjectsAreDead(MFI)) {
1813 assert(RS && "RegScavenger required if spilling");
1814
1815 // Add an emergency spill slot
1816 RS->addScavengingFrameIndex(FI: FuncInfo->getScavengeFI(MFI, TRI: *TRI));
1817
1818 if (HaveSGPRToVMemSpill && FuncInfo->hasNoWWMPoolSGPRSpillFallback()) {
1819 // The no-WWM-pool fallback can reach SGPR-to-memory lowering while an
1820 // ordinary frame-index scavenge is live. It may then need one slot for
1821 // its temporary VGPR and another for recursive address materialization.
1822 RS->addScavengingFrameIndex(FI: MFI.CreateSpillStackObject(Size: 4, Alignment: Align(4)));
1823 RS->addScavengingFrameIndex(FI: MFI.CreateSpillStackObject(Size: 4, Alignment: Align(4)));
1824 } else if (HaveSGPRToVMemSpill &&
1825 allocateScavengingFrameIndexesNearIncomingSP(MF)) {
1826 // Existing large-frame SGPR-to-memory spills need one additional VGPR
1827 // emergency frame index.
1828 RS->addScavengingFrameIndex(FI: MFI.CreateSpillStackObject(Size: 4, Alignment: Align(4)));
1829 }
1830 }
1831}
1832
1833void SIFrameLowering::processFunctionBeforeFrameIndicesReplaced(
1834 MachineFunction &MF, RegScavenger *RS) const {
1835 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1836 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1837 MachineRegisterInfo &MRI = MF.getRegInfo();
1838 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
1839
1840 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
1841 // On gfx908, we had initially reserved highest available VGPR for AGPR
1842 // copy. Now since we are done with RA, check if there exist an unused VGPR
1843 // which is lower than the eariler reserved VGPR before RA. If one exist,
1844 // use it for AGPR copy instead of one reserved before RA.
1845 Register VGPRForAGPRCopy = FuncInfo->getVGPRForAGPRCopy();
1846 Register UnusedLowVGPR =
1847 TRI->findUnusedRegister(MRI, RC: &AMDGPU::VGPR_32RegClass, MF);
1848 if (UnusedLowVGPR && (TRI->getHWRegIndex(Reg: UnusedLowVGPR) <
1849 TRI->getHWRegIndex(Reg: VGPRForAGPRCopy))) {
1850 // Reserve this newly identified VGPR (for AGPR copy)
1851 // reserved registers should already be frozen at this point
1852 // so we can avoid calling MRI.freezeReservedRegs and just use
1853 // MRI.reserveReg
1854 FuncInfo->setVGPRForAGPRCopy(UnusedLowVGPR);
1855 MRI.reserveReg(PhysReg: UnusedLowVGPR, TRI);
1856 }
1857 }
1858 // We initally reserved the highest available SGPR pair for long branches
1859 // now, after RA, we shift down to a lower unused one if one exists
1860 Register LongBranchReservedReg = FuncInfo->getLongBranchReservedReg();
1861 Register UnusedLowSGPR =
1862 TRI->findUnusedRegister(MRI, RC: &AMDGPU::SGPR_64RegClass, MF);
1863 // If LongBranchReservedReg is null then we didn't find a long branch
1864 // and never reserved a register to begin with so there is nothing to
1865 // shift down. Then if UnusedLowSGPR is null, there isn't available lower
1866 // register to use so just keep the original one we set.
1867 if (LongBranchReservedReg && UnusedLowSGPR) {
1868 FuncInfo->setLongBranchReservedReg(UnusedLowSGPR);
1869 MRI.reserveReg(PhysReg: UnusedLowSGPR, TRI);
1870 }
1871}
1872
1873// The special SGPR spills like the one needed for FP, BP or any reserved
1874// registers delayed until frame lowering.
1875void SIFrameLowering::determinePrologEpilogSGPRSaves(
1876 MachineFunction &MF, BitVector &SavedVGPRs,
1877 bool NeedExecCopyReservedReg) const {
1878 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
1879 MachineRegisterInfo &MRI = MF.getRegInfo();
1880 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1881 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1882 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1883 LiveRegUnits LiveUnits;
1884 LiveUnits.init(TRI: *TRI);
1885 // Initially mark callee saved registers as used so we will not choose them
1886 // while looking for scratch SGPRs.
1887 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
1888 for (unsigned I = 0; CSRegs[I]; ++I)
1889 LiveUnits.addReg(Reg: CSRegs[I]);
1890
1891 const TargetRegisterClass &RC = *TRI->getWaveMaskRegClass();
1892
1893 Register ReservedRegForExecCopy = MFI->getSGPRForEXECCopy();
1894 if (NeedExecCopyReservedReg ||
1895 (ReservedRegForExecCopy &&
1896 MRI.isPhysRegUsed(PhysReg: ReservedRegForExecCopy, /*SkipRegMaskTest=*/true))) {
1897 MRI.reserveReg(PhysReg: ReservedRegForExecCopy, TRI);
1898 Register UnusedScratchReg = findUnusedRegister(MRI, LiveUnits, RC);
1899 if (UnusedScratchReg) {
1900 // If found any unused scratch SGPR, reserve the register itself for Exec
1901 // copy and there is no need for any spills in that case.
1902 MFI->setSGPRForEXECCopy(UnusedScratchReg);
1903 // Only rewrite EXEC saves and restores. Other uses, such as inline
1904 // assembly operands and their callee saves, name fixed registers.
1905 const AMDGPU::LaneMaskConstants &LMC = AMDGPU::LaneMaskConstants::get(ST);
1906 for (MachineOperand &Op :
1907 make_early_inc_range(Range: MRI.reg_operands(Reg: ReservedRegForExecCopy))) {
1908 MachineInstr &MI = *Op.getParent();
1909 if (MI.getOpcode() != LMC.OrSaveExecOpc && MI.getOpcode() != LMC.MovOpc)
1910 continue;
1911 if (MI.readsRegister(Reg: LMC.ExecReg, TRI) ||
1912 MI.modifiesRegister(Reg: LMC.ExecReg, TRI))
1913 Op.setReg(UnusedScratchReg);
1914 }
1915 LiveUnits.addReg(Reg: UnusedScratchReg);
1916 } else {
1917 // Needs spill.
1918 assert(!MFI->hasPrologEpilogSGPRSpillEntry(ReservedRegForExecCopy) &&
1919 "Re-reserving spill slot for EXEC copy register");
1920 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, SGPR: ReservedRegForExecCopy, RC,
1921 /*IncludeScratchCopy=*/false);
1922 }
1923 } else if (ReservedRegForExecCopy) {
1924 // Reset it at this point. There are no whole-wave copies and spills
1925 // encountered.
1926 MFI->setSGPRForEXECCopy(AMDGPU::NoRegister);
1927 }
1928
1929 if (TRI->isCFISavedRegsSpillEnabled()) {
1930 Register Exec = TRI->getExec();
1931 assert(!MFI->hasPrologEpilogSGPRSpillEntry(Exec) &&
1932 "Re-reserving spill slot for EXEC");
1933 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, SGPR: Exec, RC);
1934 }
1935
1936 // Functions that don't return to the caller don't need to preserve
1937 // the FP and BP.
1938 const Function &F = MF.getFunction();
1939 if (F.hasFnAttribute(Kind: Attribute::NoReturn) ||
1940 AMDGPU::isChainCC(CC: F.getCallingConv()))
1941 return;
1942
1943 // hasFP only knows about stack objects that already exist. We're now
1944 // determining the stack slots that will be created, so we have to predict
1945 // them. Stack objects force FP usage with calls.
1946 //
1947 // Note a new VGPR CSR may be introduced if one is used for the spill, but we
1948 // don't want to report it here.
1949 //
1950 // FIXME: Is this really hasReservedCallFrame?
1951 const bool WillHaveFP =
1952 FrameInfo.hasCalls() &&
1953 (SavedVGPRs.any() || !allStackObjectsAreDead(MFI: FrameInfo));
1954
1955 if (WillHaveFP || hasFP(MF)) {
1956 Register FramePtrReg = MFI->getFrameOffsetReg();
1957 assert(!MFI->hasPrologEpilogSGPRSpillEntry(FramePtrReg) &&
1958 "Re-reserving spill slot for FP");
1959 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, SGPR: FramePtrReg);
1960 }
1961
1962 if (TRI->hasBasePointer(MF)) {
1963 Register BasePtrReg = TRI->getBaseRegister();
1964 assert(!MFI->hasPrologEpilogSGPRSpillEntry(BasePtrReg) &&
1965 "Re-reserving spill slot for BP");
1966 getVGPRSpillLaneOrTempRegister(MF, LiveUnits, SGPR: BasePtrReg);
1967 }
1968}
1969
1970// Only report VGPRs to generic code.
1971void SIFrameLowering::determineCalleeSaves(MachineFunction &MF,
1972 BitVector &SavedVGPRs,
1973 RegScavenger *RS) const {
1974 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
1975
1976 // If this is a function with the amdgpu_cs_chain[_preserve] calling
1977 // convention and it doesn't contain any calls to llvm.amdgcn.cs.chain, then
1978 // we don't need to save and restore anything.
1979 if (MFI->isChainFunction() && !MF.getFrameInfo().hasTailCall())
1980 return;
1981
1982 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs&: SavedVGPRs, RS);
1983
1984 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
1985 const SIRegisterInfo *TRI = ST.getRegisterInfo();
1986 const SIInstrInfo *TII = ST.getInstrInfo();
1987 bool NeedExecCopyReservedReg = false;
1988 bool NeedExecCopyForAsyncBarrier = false;
1989
1990 MachineInstr *ReturnMI = nullptr;
1991 for (MachineBasicBlock &MBB : MF) {
1992 for (MachineInstr &MI : MBB) {
1993 // TODO: Walking through all MBBs here would be a bad heuristic. Better
1994 // handle them elsewhere.
1995 if (TII->isWWMRegSpillOpcode(Opcode: MI.getOpcode())) {
1996 NeedExecCopyReservedReg = true;
1997 } else if (MI.getOpcode() == AMDGPU::DS_ATOMIC_ASYNC_BARRIER_ARRIVE_B64 &&
1998 ST.needsAsyncBarrierArriveReplayWorkaround()) {
1999 NeedExecCopyForAsyncBarrier = true;
2000 } else if (MI.getOpcode() == AMDGPU::SI_RETURN ||
2001 MI.getOpcode() == AMDGPU::SI_RETURN_TO_EPILOG ||
2002 MI.getOpcode() == AMDGPU::SI_WHOLE_WAVE_FUNC_RETURN ||
2003 (MFI->isChainFunction() &&
2004 TII->isChainCallOpcode(Opcode: MI.getOpcode()))) {
2005 // We expect all return to be the same size.
2006 assert(!ReturnMI ||
2007 (count_if(MI.operands(), [](auto Op) { return Op.isReg(); }) ==
2008 count_if(ReturnMI->operands(), [](auto Op) { return Op.isReg(); })));
2009 ReturnMI = &MI;
2010 }
2011 }
2012 }
2013
2014 SmallVector<Register> SortedWWMVGPRs;
2015 for (Register Reg : MFI->getWWMReservedRegs()) {
2016 // The shift-back is needed only for the VGPRs used for SGPR spills and they
2017 // are of 32-bit size. SIPreAllocateWWMRegs pass can add tuples into WWM
2018 // reserved registers.
2019 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
2020 if (TRI->getRegSizeInBits(RC: *RC) != 32)
2021 continue;
2022 SortedWWMVGPRs.push_back(Elt: Reg);
2023 }
2024
2025 sort(C&: SortedWWMVGPRs, Comp: std::greater<Register>());
2026 MFI->shiftWwmVGPRsToLowestRange(MF, WWMVGPRs&: SortedWWMVGPRs, SavedVGPRs);
2027
2028 if (MFI->isEntryFunction())
2029 return;
2030
2031 if (MFI->isWholeWaveFunction()) {
2032 // In practice, all the VGPRs are WWM registers, and we will need to save at
2033 // least their inactive lanes. Add them to WWMReservedRegs.
2034 assert(!NeedExecCopyReservedReg &&
2035 "Whole wave functions can use the reg mapped for their i1 argument");
2036
2037 unsigned NumArchVGPRs = ST.getAddressableNumArchVGPRs();
2038 for (MCRegister Reg :
2039 AMDGPU::VGPR_32RegClass.getRegisters().take_front(N: NumArchVGPRs))
2040 if (MF.getRegInfo().isPhysRegModified(PhysReg: Reg)) {
2041 MFI->reserveWWMRegister(Reg);
2042 MF.begin()->addLiveIn(PhysReg: Reg);
2043 }
2044 MF.begin()->sortUniqueLiveIns();
2045 }
2046
2047 // Remove any VGPRs used in the return value because these do not need to be saved.
2048 // This prevents CSR restore from clobbering return VGPRs.
2049 if (ReturnMI) {
2050 for (auto &Op : ReturnMI->operands()) {
2051 if (Op.isReg())
2052 SavedVGPRs.reset(Idx: Op.getReg());
2053 }
2054 }
2055
2056 // Create the stack objects for WWM registers now.
2057 for (Register Reg : MFI->getWWMReservedRegs()) {
2058 const TargetRegisterClass *RC = TRI->getPhysRegBaseClass(Reg);
2059 MFI->allocateWWMSpill(MF, VGPR: Reg, Size: TRI->getSpillSize(RC: *RC),
2060 Alignment: TRI->getSpillAlign(RC: *RC));
2061 }
2062
2063 // Ignore the SGPRs the default implementation found.
2064 SavedVGPRs.clearBitsNotInMask(Mask: TRI->getAllVectorRegMask());
2065
2066 // Do not save AGPRs prior to GFX90A because there was no easy way to do so.
2067 // In gfx908 there was do AGPR loads and stores and thus spilling also
2068 // require a temporary VGPR.
2069 if (!ST.hasGFX90AInsts())
2070 SavedVGPRs.clearBitsInMask(Mask: TRI->getAllAGPRRegMask());
2071
2072 // Whole-wave spills use the original EXEC mask from the first argument, but
2073 // the barrier workaround needs separate storage for the current EXEC mask.
2074 determinePrologEpilogSGPRSaves(
2075 MF, SavedVGPRs, NeedExecCopyReservedReg: NeedExecCopyReservedReg || NeedExecCopyForAsyncBarrier);
2076
2077 // The Whole-Wave VGPRs need to be specially inserted in the prolog, so don't
2078 // allow the default insertion to handle them.
2079 for (auto &Reg : MFI->getWWMSpills())
2080 SavedVGPRs.reset(Idx: Reg.first);
2081}
2082
2083void SIFrameLowering::determineCalleeSavesSGPR(MachineFunction &MF,
2084 BitVector &SavedRegs,
2085 RegScavenger *RS) const {
2086 TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS);
2087 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2088 if (MFI->isEntryFunction())
2089 return;
2090
2091 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2092 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2093
2094 // The SP is specifically managed and we don't want extra spills of it.
2095 SavedRegs.reset(Idx: MFI->getStackPtrOffsetReg());
2096
2097 const BitVector AllSavedRegs = SavedRegs;
2098 SavedRegs.clearBitsInMask(Mask: TRI->getAllVectorRegMask());
2099
2100 // We have to anticipate introducing CSR VGPR spills or spill of caller
2101 // save VGPR reserved for SGPR spills as we now always create stack entry
2102 // for it, if we don't have any stack objects already, since we require a FP
2103 // if there is a call and stack. We will allocate a VGPR for SGPR spills if
2104 // there are any SGPR spills. Whether they are CSR spills or otherwise.
2105 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
2106 const bool WillHaveFP =
2107 FrameInfo.hasCalls() && (AllSavedRegs.any() || MFI->hasSpilledSGPRs());
2108
2109 // FP will be specially managed like SP.
2110 if (WillHaveFP || hasFP(MF))
2111 SavedRegs.reset(Idx: MFI->getFrameOffsetReg());
2112
2113 // Return address use with return instruction is hidden through the SI_RETURN
2114 // pseudo. Given that and since the IPRA computes actual register usage and
2115 // does not use CSR list, the clobbering of return address by function calls
2116 // (D117243) or otherwise (D120922) is ignored/not seen by the IPRA's register
2117 // usage collection. This will ensure save/restore of return address happens
2118 // in those scenarios.
2119 const MachineRegisterInfo &MRI = MF.getRegInfo();
2120 Register RetAddrReg = TRI->getReturnAddressReg(MF);
2121 if (!MFI->isEntryFunction() &&
2122 (FrameInfo.hasCalls() || MRI.isPhysRegModified(PhysReg: RetAddrReg))) {
2123 SavedRegs.set(TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub0));
2124 SavedRegs.set(TRI->getSubReg(Reg: RetAddrReg, Idx: AMDGPU::sub1));
2125 }
2126}
2127
2128static void assignSlotsUsingVGPRBlocks(MachineFunction &MF,
2129 const GCNSubtarget &ST,
2130 std::vector<CalleeSavedInfo> &CSI) {
2131 SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
2132 MachineFrameInfo &MFI = MF.getFrameInfo();
2133 const SIRegisterInfo *TRI = ST.getRegisterInfo();
2134
2135 assert(
2136 llvm::is_sorted(CSI,
2137 [](const CalleeSavedInfo &A, const CalleeSavedInfo &B) {
2138 return A.getReg() < B.getReg();
2139 }) &&
2140 "Callee saved registers not sorted");
2141
2142 auto CanUseBlockOps = [&](const CalleeSavedInfo &CSI) {
2143 return !CSI.isSpilledToReg() &&
2144 TRI->getPhysRegBaseClass(Reg: CSI.getReg()) == &AMDGPU::VGPR_32RegClass &&
2145 !FuncInfo->isWWMReservedRegister(Reg: CSI.getReg());
2146 };
2147
2148 auto CSEnd = CSI.end();
2149 for (auto CSIt = CSI.begin(); CSIt != CSEnd; ++CSIt) {
2150 Register Reg = CSIt->getReg();
2151 if (!CanUseBlockOps(*CSIt))
2152 continue;
2153
2154 // Find all the regs that will fit in a 32-bit mask starting at the current
2155 // reg and build said mask. It should have 1 for every register that's
2156 // included, with the current register as the least significant bit.
2157 uint32_t Mask = 1;
2158 CSEnd = std::remove_if(
2159 first: CSIt + 1, last: CSEnd, pred: [&](const CalleeSavedInfo &CSI) -> bool {
2160 if (CanUseBlockOps(CSI) && CSI.getReg() < Reg + 32) {
2161 Mask |= 1 << (CSI.getReg() - Reg);
2162 return true;
2163 } else {
2164 return false;
2165 }
2166 });
2167
2168 const TargetRegisterClass *BlockRegClass = TRI->getRegClassForBlockOp(MF);
2169 Register RegBlock =
2170 TRI->getMatchingSuperReg(Reg, SubIdx: AMDGPU::sub0, RC: BlockRegClass);
2171 if (!RegBlock) {
2172 // We couldn't find a super register for the block. This can happen if
2173 // the register we started with is too high (e.g. v232 if the maximum is
2174 // v255). We therefore try to get the last register block and figure out
2175 // the mask from there.
2176 Register LastBlockStart =
2177 AMDGPU::VGPR0 + alignDown(Value: Reg - AMDGPU::VGPR0, Align: 32);
2178 RegBlock =
2179 TRI->getMatchingSuperReg(Reg: LastBlockStart, SubIdx: AMDGPU::sub0, RC: BlockRegClass);
2180 assert(RegBlock && TRI->isSubRegister(RegBlock, Reg) &&
2181 "Couldn't find super register");
2182 int RegDelta = Reg - LastBlockStart;
2183 assert(RegDelta > 0 && llvm::countl_zero(Mask) >= RegDelta &&
2184 "Bad shift amount");
2185 Mask <<= RegDelta;
2186 }
2187
2188 FuncInfo->setMaskForVGPRBlockOps(RegisterBlock: RegBlock, Mask);
2189
2190 // The stack objects can be a bit smaller than the register block if we know
2191 // some of the high bits of Mask are 0. This may happen often with calling
2192 // conventions where the caller and callee-saved VGPRs are interleaved at
2193 // a small boundary (e.g. 8 or 16).
2194 int UnusedBits = llvm::countl_zero(Val: Mask);
2195 unsigned BlockSize = TRI->getSpillSize(RC: *BlockRegClass) - UnusedBits * 4;
2196 int FrameIdx =
2197 MFI.CreateStackObject(Size: BlockSize, Alignment: TRI->getSpillAlign(RC: *BlockRegClass),
2198 /*isSpillSlot=*/true);
2199 MFI.setIsCalleeSavedObjectIndex(ObjectIdx: FrameIdx, IsCalleeSaved: true);
2200
2201 CSIt->setFrameIdx(FrameIdx);
2202 CSIt->setReg(RegBlock);
2203 }
2204 CSI.erase(first: CSEnd, last: CSI.end());
2205}
2206
2207bool SIFrameLowering::assignCalleeSavedSpillSlots(
2208 MachineFunction &MF, const TargetRegisterInfo *TRI,
2209 std::vector<CalleeSavedInfo> &CSI) const {
2210 if (CSI.empty())
2211 return true; // Early exit if no callee saved registers are modified!
2212
2213 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2214 bool UseVGPRBlocks = ST.useVGPRBlockOpsForCSR();
2215
2216 if (UseVGPRBlocks)
2217 assignSlotsUsingVGPRBlocks(MF, ST, CSI);
2218
2219 return assignCalleeSavedSpillSlotsImpl(MF, TRI, CSI) || UseVGPRBlocks;
2220}
2221
2222bool SIFrameLowering::assignCalleeSavedSpillSlotsImpl(
2223 MachineFunction &MF, const TargetRegisterInfo *TRI,
2224 std::vector<CalleeSavedInfo> &CSI) const {
2225 if (CSI.empty())
2226 return true; // Early exit if no callee saved registers are modified!
2227
2228 const SIMachineFunctionInfo *FuncInfo = MF.getInfo<SIMachineFunctionInfo>();
2229 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2230 const SIRegisterInfo *RI = ST.getRegisterInfo();
2231 Register FramePtrReg = FuncInfo->getFrameOffsetReg();
2232 Register BasePtrReg = RI->getBaseRegister();
2233 Register SGPRForFPSaveRestoreCopy =
2234 FuncInfo->getScratchSGPRCopyDstReg(Reg: FramePtrReg);
2235 Register SGPRForBPSaveRestoreCopy =
2236 FuncInfo->getScratchSGPRCopyDstReg(Reg: BasePtrReg);
2237 if (!SGPRForFPSaveRestoreCopy && !SGPRForBPSaveRestoreCopy)
2238 return false;
2239
2240 unsigned NumModifiedRegs = 0;
2241
2242 if (SGPRForFPSaveRestoreCopy)
2243 NumModifiedRegs++;
2244 if (SGPRForBPSaveRestoreCopy)
2245 NumModifiedRegs++;
2246
2247 for (auto &CS : CSI) {
2248 if (CS.getReg() == FramePtrReg.asMCReg() && SGPRForFPSaveRestoreCopy) {
2249 CS.setDstReg(SGPRForFPSaveRestoreCopy);
2250 if (--NumModifiedRegs)
2251 break;
2252 } else if (CS.getReg() == BasePtrReg.asMCReg() &&
2253 SGPRForBPSaveRestoreCopy) {
2254 CS.setDstReg(SGPRForBPSaveRestoreCopy);
2255 if (--NumModifiedRegs)
2256 break;
2257 }
2258 }
2259
2260 return false;
2261}
2262
2263bool SIFrameLowering::allocateScavengingFrameIndexesNearIncomingSP(
2264 const MachineFunction &MF) const {
2265
2266 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2267 const MachineFrameInfo &MFI = MF.getFrameInfo();
2268 const SIInstrInfo *TII = ST.getInstrInfo();
2269 uint64_t EstStackSize = MFI.estimateStackSize(MF);
2270 uint64_t MaxOffset = EstStackSize - 1;
2271
2272 // We need the emergency stack slots to be allocated in range of the
2273 // MUBUF/flat scratch immediate offset from the base register, so assign these
2274 // first at the incoming SP position.
2275 //
2276 // TODO: We could try sorting the objects to find a hole in the first bytes
2277 // rather than allocating as close to possible. This could save a lot of space
2278 // on frames with alignment requirements.
2279 if (ST.hasFlatScratchEnabled()) {
2280 if (TII->isLegalFLATOffset(Offset: MaxOffset, AddrSpace: AMDGPUAS::PRIVATE_ADDRESS,
2281 FlatVariant: AMDGPU::FlatAddrSpace::FlatScratch))
2282 return false;
2283 } else {
2284 if (TII->isLegalMUBUFImmOffset(Imm: MaxOffset))
2285 return false;
2286 }
2287
2288 return true;
2289}
2290
2291/// Return the set of all root registers of regunits live-in to @p MBB.
2292///
2293/// Intended to avoid using the expensive @c MCRegAliasIterator when deciding
2294/// if a register to be spilled is already live-in (see @c isAnyRootLiveIn).
2295static SparseBitVector<> buildLiveInRoots(const MachineBasicBlock &MBB,
2296 const SIRegisterInfo &TRI) {
2297 SparseBitVector<> LiveInRoots;
2298 for (const auto &LI : MBB.liveins()) {
2299 for (MCRegUnitMaskIterator MI(LI.PhysReg, &TRI); MI.isValid(); ++MI) {
2300 auto [Unit, UnitLaneMask] = *MI;
2301 if ((LI.LaneMask & UnitLaneMask).none())
2302 continue;
2303 for (MCRegUnitRootIterator RI(Unit, &TRI); RI.isValid(); ++RI)
2304 LiveInRoots.set(*RI);
2305 }
2306 }
2307 return LiveInRoots;
2308}
2309
2310/// Returns true iff any root of @p Reg is in @p LiveInRoots
2311/// (see @c buildLiveInRoots).
2312static bool isAnyRootLiveIn(const SparseBitVector<> &LiveInRoots,
2313 const SIRegisterInfo &TRI, MCRegister Reg) {
2314 for (MCRegUnitIterator UI(Reg, &TRI); UI.isValid(); ++UI) {
2315 for (MCRegUnitRootIterator RI(*UI, &TRI); RI.isValid(); ++RI) {
2316 if (LiveInRoots.test(Idx: *RI))
2317 return true;
2318 }
2319 }
2320 return false;
2321}
2322
2323void SIFrameLowering::spillCalleeSavedRegisterWithoutBlockOps(
2324 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2325 const CalleeSavedInfo &CS, const SIInstrInfo *TII,
2326 const SIRegisterInfo &TRI,
2327 const std::optional<SparseBitVector<>> &LiveInRoots) const {
2328 MCRegister Reg = CS.getReg();
2329
2330 // We assume a sortUniqueLiveIns later
2331 MBB.addLiveIn(PhysReg: Reg);
2332
2333 if (CS.isSpilledToReg()) {
2334 BuildMI(BB&: MBB, I: MI, MIMD: DebugLoc(), MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg: CS.getDstReg())
2335 .addReg(RegNo: Reg, Flags: getKillRegState(B: true));
2336 } else {
2337 const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(Reg);
2338 bool IsKill = true;
2339 // If this value was already livein, we probably have a direct use of
2340 // the incoming register value, so don't kill at the spill point. This
2341 // happens since we pass some special inputs (workgroup IDs) in the
2342 // callee saved range.
2343 if (LiveInRoots)
2344 IsKill = !isAnyRootLiveIn(LiveInRoots: *LiveInRoots, TRI, Reg);
2345 TII->storeRegToStackSlotCFI(MBB, MI, SrcReg: Reg, isKill: IsKill, FrameIndex: CS.getFrameIdx(), RC);
2346 }
2347}
2348
2349bool SIFrameLowering::spillCalleeSavedRegisters(
2350 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2351 ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *OrigTRI) const {
2352 auto &TRI = *static_cast<const SIRegisterInfo *>(OrigTRI);
2353 MachineFunction *MF = MBB.getParent();
2354 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2355 const SIInstrInfo *TII = ST.getInstrInfo();
2356
2357 std::optional<SparseBitVector<>> LiveInRoots;
2358 if (MBB.getParent()->getRegInfo().tracksLiveness())
2359 LiveInRoots = buildLiveInRoots(MBB, TRI);
2360
2361 if (!ST.useVGPRBlockOpsForCSR()) {
2362 for (const CalleeSavedInfo &CS : CSI)
2363 spillCalleeSavedRegisterWithoutBlockOps(MBB, MI, CS, TII, TRI,
2364 LiveInRoots);
2365 if (LiveInRoots)
2366 MBB.sortUniqueLiveIns();
2367 return true;
2368 }
2369
2370 MachineFrameInfo &FrameInfo = MF->getFrameInfo();
2371 SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
2372
2373 const TargetRegisterClass *BlockRegClass = TRI.getRegClassForBlockOp(MF: *MF);
2374 for (const CalleeSavedInfo &CS : CSI) {
2375 Register Reg = CS.getReg();
2376 if (!BlockRegClass->contains(Reg) ||
2377 !FuncInfo->hasMaskForVGPRBlockOps(RegisterBlock: Reg)) {
2378 spillCalleeSavedRegisterWithoutBlockOps(MBB, MI, CS, TII, TRI,
2379 LiveInRoots);
2380 continue;
2381 }
2382
2383 // Build a scratch block store.
2384 uint32_t Mask = FuncInfo->getMaskForVGPRBlockOps(RegisterBlock: Reg);
2385 int FrameIndex = CS.getFrameIdx();
2386 MachinePointerInfo PtrInfo =
2387 MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
2388 MachineMemOperand *MMO =
2389 MF->getMachineMemOperand(PtrInfo, F: MachineMemOperand::MOStore,
2390 Size: FrameInfo.getObjectSize(ObjectIdx: FrameIndex),
2391 BaseAlignment: FrameInfo.getObjectAlign(ObjectIdx: FrameIndex));
2392
2393 BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(),
2394 MCID: TII->get(Opcode: AMDGPU::SI_BLOCK_SPILL_V1024_CFI_SAVE))
2395 .addReg(RegNo: Reg, Flags: getKillRegState(B: false))
2396 .addFrameIndex(Idx: FrameIndex)
2397 .addReg(RegNo: FuncInfo->getStackPtrOffsetReg())
2398 .addImm(Val: 0)
2399 .addImm(Val: Mask)
2400 .addMemOperand(MMO);
2401
2402 FuncInfo->setHasSpilledVGPRs();
2403
2404 // Add the register to the liveins. This is necessary because if any of the
2405 // VGPRs in the register block is reserved (e.g. if it's a WWM register),
2406 // then the whole block will be marked as reserved and `updateLiveness` will
2407 // skip it.
2408 if (LiveInRoots)
2409 MBB.addLiveIn(PhysReg: Reg);
2410 }
2411 if (LiveInRoots)
2412 MBB.sortUniqueLiveIns();
2413
2414 return true;
2415}
2416
2417bool SIFrameLowering::restoreCalleeSavedRegisters(
2418 MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
2419 MutableArrayRef<CalleeSavedInfo> CSI,
2420 const TargetRegisterInfo *OrigTRI) const {
2421 auto &TRI = *static_cast<const SIRegisterInfo *>(OrigTRI);
2422 MachineFunction *MF = MBB.getParent();
2423 const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
2424 if (!ST.useVGPRBlockOpsForCSR())
2425 return false;
2426
2427 SIMachineFunctionInfo *FuncInfo = MF->getInfo<SIMachineFunctionInfo>();
2428 MachineFrameInfo &MFI = MF->getFrameInfo();
2429 const SIInstrInfo *TII = ST.getInstrInfo();
2430 const TargetRegisterClass *BlockRegClass = TRI.getRegClassForBlockOp(MF: *MF);
2431 for (const CalleeSavedInfo &CS : reverse(C&: CSI)) {
2432 Register Reg = CS.getReg();
2433 if (!BlockRegClass->contains(Reg) ||
2434 !FuncInfo->hasMaskForVGPRBlockOps(RegisterBlock: Reg)) {
2435 restoreCalleeSavedRegister(MBB, MI, CS, TII, TRI: &TRI);
2436 continue;
2437 }
2438
2439 // Build a scratch block load.
2440 uint32_t Mask = FuncInfo->getMaskForVGPRBlockOps(RegisterBlock: Reg);
2441 int FrameIndex = CS.getFrameIdx();
2442 MachinePointerInfo PtrInfo =
2443 MachinePointerInfo::getFixedStack(MF&: *MF, FI: FrameIndex);
2444 MachineMemOperand *MMO = MF->getMachineMemOperand(
2445 PtrInfo, F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIndex),
2446 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIndex));
2447
2448 auto MIB = BuildMI(BB&: MBB, I: MI, MIMD: MI->getDebugLoc(),
2449 MCID: TII->get(Opcode: AMDGPU::SI_BLOCK_SPILL_V1024_RESTORE), DestReg: Reg)
2450 .addFrameIndex(Idx: FrameIndex)
2451 .addReg(RegNo: FuncInfo->getStackPtrOffsetReg())
2452 .addImm(Val: 0)
2453 .addImm(Val: Mask)
2454 .addMemOperand(MMO);
2455 TRI.addImplicitUsesForBlockCSRLoad(MIB, BlockReg: Reg);
2456
2457 // Add the register to the liveins. This is necessary because if any of the
2458 // VGPRs in the register block is reserved (e.g. if it's a WWM register),
2459 // then the whole block will be marked as reserved and `updateLiveness` will
2460 // skip it.
2461 MBB.addLiveIn(PhysReg: Reg);
2462 }
2463
2464 MBB.sortUniqueLiveIns();
2465 return true;
2466}
2467
2468MachineBasicBlock::iterator SIFrameLowering::eliminateCallFramePseudoInstr(
2469 MachineFunction &MF,
2470 MachineBasicBlock &MBB,
2471 MachineBasicBlock::iterator I) const {
2472 int64_t Amount = I->getOperand(i: 0).getImm();
2473 if (Amount == 0)
2474 return MBB.erase(I);
2475
2476 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2477 const SIInstrInfo *TII = ST.getInstrInfo();
2478 const DebugLoc &DL = I->getDebugLoc();
2479 unsigned Opc = I->getOpcode();
2480 bool IsDestroy = Opc == TII->getCallFrameDestroyOpcode();
2481 uint64_t CalleePopAmount = IsDestroy ? I->getOperand(i: 1).getImm() : 0;
2482
2483 if (!hasReservedCallFrame(MF)) {
2484 Amount = alignTo(Size: Amount, A: getStackAlign());
2485 assert(isUInt<32>(Amount) && "exceeded stack address space size");
2486 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
2487 Register SPReg = MFI->getStackPtrOffsetReg();
2488
2489 Amount *= getScratchScaleFactor(ST);
2490 if (IsDestroy)
2491 Amount = -Amount;
2492 auto Add = BuildMI(BB&: MBB, I, MIMD: DL, MCID: TII->get(Opcode: AMDGPU::S_ADD_I32), DestReg: SPReg)
2493 .addReg(RegNo: SPReg)
2494 .addImm(Val: Amount);
2495 Add->getOperand(i: 3).setIsDead(); // Mark SCC as dead.
2496 } else if (CalleePopAmount != 0) {
2497 llvm_unreachable("is this used?");
2498 }
2499
2500 return MBB.erase(I);
2501}
2502
2503/// Returns true if the frame will require a reference to the stack pointer.
2504///
2505/// This is the set of conditions common to setting up the stack pointer in a
2506/// kernel, and for using a frame pointer in a callable function.
2507///
2508/// FIXME: Should also check hasOpaqueSPAdjustment and if any inline asm
2509/// references SP.
2510static bool frameTriviallyRequiresSP(const MachineFrameInfo &MFI) {
2511 return MFI.hasVarSizedObjects() || MFI.hasStackMap() || MFI.hasPatchPoint();
2512}
2513
2514// The FP for kernels is always known 0, so we never really need to setup an
2515// explicit register for it. However, DisableFramePointerElim will force us to
2516// use a register for it.
2517bool SIFrameLowering::hasFPImpl(const MachineFunction &MF) const {
2518 const MachineFrameInfo &MFI = MF.getFrameInfo();
2519
2520 // For entry functions we can use an immediate offset in most cases,
2521 // so the presence of calls doesn't imply we need a distinct frame pointer.
2522 if (MFI.hasCalls() &&
2523 !MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction()) {
2524 // All offsets are unsigned, so need to be addressed in the same direction
2525 // as stack growth.
2526
2527 // FIXME: This function is pretty broken, since it can be called before the
2528 // frame layout is determined or CSR spills are inserted.
2529 return MFI.getStackSize() != 0;
2530 }
2531
2532 return frameTriviallyRequiresSP(MFI) || MFI.isFrameAddressTaken() ||
2533 MF.getSubtarget<GCNSubtarget>().getRegisterInfo()->hasStackRealignment(
2534 MF) ||
2535 mayReserveScratchForCWSR(MF) || MF.disableFramePointerElim();
2536}
2537
2538bool SIFrameLowering::mayReserveScratchForCWSR(
2539 const MachineFunction &MF) const {
2540 return MF.getInfo<SIMachineFunctionInfo>()->isDynamicVGPREnabled() &&
2541 AMDGPU::isEntryFunctionCC(CC: MF.getFunction().getCallingConv()) &&
2542 AMDGPU::isCompute(CC: MF.getFunction().getCallingConv());
2543}
2544
2545// This is essentially a reduced version of hasFP for entry functions. Since the
2546// stack pointer is known 0 on entry to kernels, we never really need an FP
2547// register. We may need to initialize the stack pointer depending on the frame
2548// properties, which logically overlaps many of the cases where an ordinary
2549// function would require an FP.
2550bool SIFrameLowering::requiresStackPointerReference(
2551 const MachineFunction &MF) const {
2552 // Callable functions always require a stack pointer reference.
2553 assert(MF.getInfo<SIMachineFunctionInfo>()->isEntryFunction() &&
2554 "only expected to call this for entry points functions");
2555
2556 const MachineFrameInfo &MFI = MF.getFrameInfo();
2557
2558 // Entry points ordinarily don't need to initialize SP. We have to set it up
2559 // for callees if there are any.
2560 if (MFI.hasCalls())
2561 return true;
2562
2563 // We still need to initialize the SP if we're doing anything weird that
2564 // references the SP, like variable sized stack objects.
2565 return frameTriviallyRequiresSP(MFI);
2566}
2567
2568MachineInstr *SIFrameLowering::buildCFI(MachineBasicBlock &MBB,
2569 MachineBasicBlock::iterator MBBI,
2570 const DebugLoc &DL,
2571 const MCCFIInstruction &CFIInst,
2572 MachineInstr::MIFlag Flag) const {
2573 MachineFunction &MF = *MBB.getParent();
2574 const SIInstrInfo *TII = MF.getSubtarget<GCNSubtarget>().getInstrInfo();
2575 return BuildMI(BB&: MBB, I: MBBI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::CFI_INSTRUCTION))
2576 .addCFIIndex(CFIIndex: MF.addFrameInst(Inst: CFIInst))
2577 .setMIFlag(Flag);
2578}
2579
2580MachineInstr *SIFrameLowering::buildCFIForVRegToVRegSpill(
2581 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2582 const DebugLoc &DL, const MCRegister Reg, const MCRegister RegCopy) const {
2583 MachineFunction &MF = *MBB.getParent();
2584 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2585 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2586
2587 unsigned MaskReg = MCRI.getDwarfRegNum(
2588 Reg: ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC, isEH: false);
2589 auto CFIInst = MCCFIInstruction::createLLVMVectorRegisterMask(
2590 L: nullptr, Register: MCRI.getDwarfRegNum(Reg, isEH: false),
2591 SpillRegister: MCRI.getDwarfRegNum(Reg: RegCopy, isEH: false), SpillRegisterLaneSizeInBits: VGPRLaneBitSize, MaskRegister: MaskReg,
2592 MaskRegisterSizeInBits: ST.getWavefrontSize());
2593 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2594}
2595
2596MachineInstr *SIFrameLowering::buildCFIForSGPRToVGPRSpill(
2597 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2598 const DebugLoc &DL, const MCRegister SGPR, const MCRegister VGPR,
2599 const int Lane) const {
2600 const MachineFunction &MF = *MBB.getParent();
2601 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2602
2603 int DwarfSGPR = MCRI.getDwarfRegNum(Reg: SGPR, isEH: false);
2604 int DwarfVGPR = MCRI.getDwarfRegNum(Reg: VGPR, isEH: false);
2605 assert(DwarfSGPR != -1 && DwarfVGPR != -1);
2606 assert(Lane != -1 && "Expected a lane to be present");
2607
2608 // Build a CFI instruction that represents a SGPR spilled to a single lane of
2609 // a VGPR.
2610 MCCFIInstruction::VectorRegisterWithLane VR{.Register: unsigned(DwarfVGPR),
2611 .Lane: unsigned(Lane), .SizeInBits: VGPRLaneBitSize};
2612 auto CFIInst =
2613 MCCFIInstruction::createLLVMVectorRegisters(L: nullptr, Register: DwarfSGPR, VectorRegisters: {VR});
2614 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2615}
2616
2617MachineInstr *SIFrameLowering::buildCFIForSGPRToVGPRSpill(
2618 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2619 const DebugLoc &DL, MCRegister SGPR,
2620 ArrayRef<SIRegisterInfo::SpilledReg> VGPRSpills) const {
2621 if (VGPRSpills.size() == 1u)
2622 return buildCFIForSGPRToVGPRSpill(MBB, MBBI, DL, SGPR, VGPR: VGPRSpills[0].VGPR,
2623 Lane: VGPRSpills[0].Lane);
2624 const MachineFunction &MF = *MBB.getParent();
2625 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2626
2627 int DwarfSGPR = MCRI.getDwarfRegNum(Reg: SGPR, isEH: false);
2628 assert(DwarfSGPR != -1);
2629
2630 // Build a CFI instruction that represents a SGPR spilled to multiple lanes of
2631 // multiple VGPRs.
2632
2633 SmallVector<MCCFIInstruction::VectorRegisterWithLane> VGPRs;
2634 for (SIRegisterInfo::SpilledReg Spill : VGPRSpills) {
2635 int DwarfVGPR = MCRI.getDwarfRegNum(Reg: Spill.VGPR, isEH: false);
2636 assert(DwarfVGPR != -1);
2637 assert(Spill.hasLane() && "Expected a lane to be present");
2638 VGPRs.push_back(
2639 Elt: {.Register: unsigned(DwarfVGPR), .Lane: unsigned(Spill.Lane), .SizeInBits: VGPRLaneBitSize});
2640 }
2641
2642 auto CFIInst = MCCFIInstruction::createLLVMVectorRegisters(L: nullptr, Register: DwarfSGPR,
2643 VectorRegisters: std::move(VGPRs));
2644 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2645}
2646
2647MachineInstr *SIFrameLowering::buildCFIForSGPRToVMEMSpill(
2648 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2649 const DebugLoc &DL, MCRegister SGPR, int64_t Offset) const {
2650 MachineFunction &MF = *MBB.getParent();
2651 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2652 return buildCFI(MBB, MBBI, DL,
2653 CFIInst: llvm::MCCFIInstruction::createOffset(
2654 L: nullptr, Register: MCRI.getDwarfRegNum(Reg: SGPR, isEH: false), Offset));
2655}
2656
2657MachineInstr *SIFrameLowering::buildCFIForVGPRToVMEMSpill(
2658 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2659 const DebugLoc &DL, MCRegister VGPR, int64_t Offset) const {
2660 const MachineFunction &MF = *MBB.getParent();
2661 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2662 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2663
2664 int DwarfVGPR = MCRI.getDwarfRegNum(Reg: VGPR, isEH: false);
2665 assert(DwarfVGPR != -1);
2666
2667 unsigned MaskReg = MCRI.getDwarfRegNum(
2668 Reg: ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC, isEH: false);
2669 auto CFIInst = MCCFIInstruction::createLLVMVectorOffset(
2670 L: nullptr, Register: DwarfVGPR, RegisterSizeInBits: VGPRLaneBitSize, MaskRegister: MaskReg, MaskRegisterSizeInBits: ST.getWavefrontSize(),
2671 Offset);
2672 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2673}
2674
2675static void decomposeDwarfRegs(const SIRegisterInfo &TRI, MCRegister Reg,
2676 SmallVectorImpl<unsigned> &DwarfRegs) {
2677 int64_t DwarfReg = TRI.getDwarfRegNum(Reg, /*IsEH=*/isEH: false);
2678 if (DwarfReg >= 0) {
2679 DwarfRegs.push_back(Elt: unsigned(DwarfReg));
2680 return;
2681 }
2682
2683 const TargetRegisterClass *RC = TRI.getPhysRegBaseClass(Reg);
2684 ArrayRef<int16_t> SplitParts = TRI.getRegSplitParts(RC, EltSize: SGPRByteSize);
2685 for (unsigned I = 0; I < SplitParts.size(); ++I) {
2686 MCRegister SubReg = TRI.getSubReg(Reg, Idx: SplitParts[I]);
2687 DwarfReg = TRI.getDwarfRegNum(Reg: SubReg, /*IsEH=*/isEH: false);
2688 if (DwarfReg < 0) {
2689 DwarfRegs.clear();
2690 return;
2691 }
2692 DwarfRegs.push_back(Elt: unsigned(DwarfReg));
2693 }
2694}
2695
2696void SIFrameLowering::buildCFIForSRegToSRegSpill(
2697 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2698 const DebugLoc &DL, MCRegister Reg, MCRegister CopyReg) const {
2699 const MachineFunction &MF = *MBB.getParent();
2700 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
2701 const SIRegisterInfo &TRI = *ST.getRegisterInfo();
2702
2703 SmallVector<unsigned> DwarfRegs, DwarfCopyRegs;
2704 decomposeDwarfRegs(TRI, Reg, DwarfRegs);
2705 decomposeDwarfRegs(TRI, Reg: CopyReg, DwarfRegs&: DwarfCopyRegs);
2706
2707 if (DwarfRegs.size() == DwarfCopyRegs.size()) {
2708 for (unsigned I = 0, E = DwarfRegs.size(); I < E; ++I) {
2709 auto CFI = MCCFIInstruction::createRegister(L: nullptr, Register1: DwarfRegs[I],
2710 Register2: DwarfCopyRegs[I]);
2711 buildCFI(MBB, MBBI, DL, CFIInst: CFI);
2712 }
2713 return;
2714 }
2715
2716 if (DwarfRegs.size() == 1 && DwarfCopyRegs.size() == 2) {
2717 auto CFI = MCCFIInstruction::createLLVMRegisterPair(
2718 L: nullptr, Register: DwarfRegs[0], R1: DwarfCopyRegs[0], R1SizeInBits: SGPRBitSize, R2: DwarfCopyRegs[1],
2719 R2SizeInBits: SGPRBitSize);
2720 buildCFI(MBB, MBBI, DL, CFIInst: CFI);
2721 return;
2722 }
2723
2724 for (unsigned I = 0, E = DwarfRegs.size(); I < E; ++I) {
2725 auto CFI = MCCFIInstruction::createUndefined(L: nullptr, Register: DwarfRegs[I]);
2726 buildCFI(MBB, MBBI, DL, CFIInst: CFI);
2727 }
2728}
2729
2730MachineInstr *SIFrameLowering::buildCFIForSGPRToVCCHalfCopy(
2731 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2732 const DebugLoc &DL, MCRegister SGPR, MCRegister VCCHalf) const {
2733 const MachineFunction &MF = *MBB.getParent();
2734 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2735 assert(isWave64VCCHalf(MF.getSubtarget<GCNSubtarget>(), VCCHalf));
2736
2737 int DwarfSGPR = MCRI.getDwarfRegNum(Reg: SGPR, isEH: false);
2738 int DwarfVCC = MCRI.getDwarfRegNum(Reg: AMDGPU::VCC, isEH: false);
2739 assert(DwarfSGPR != -1 && DwarfVCC != -1);
2740
2741 // Describe the copy as one 32-bit lane of VCC_64: lane 0 is vcc_lo and lane 1
2742 // is vcc_hi.
2743 MCCFIInstruction::VectorRegisterWithLane Half{
2744 .Register: unsigned(DwarfVCC), .Lane: VCCHalf == AMDGPU::VCC_HI ? 1u : 0u, .SizeInBits: SGPRBitSize};
2745 auto CFIInst =
2746 MCCFIInstruction::createLLVMVectorRegisters(L: nullptr, Register: DwarfSGPR, VectorRegisters: {Half});
2747 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2748}
2749
2750MachineInstr *SIFrameLowering::buildCFIForSameValue(
2751 MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
2752 const DebugLoc &DL, MCRegister Reg) const {
2753 const MachineFunction &MF = *MBB.getParent();
2754 const MCRegisterInfo &MCRI = *MF.getContext().getRegisterInfo();
2755 int DwarfReg = MCRI.getDwarfRegNum(Reg, /*isEH=*/false);
2756 auto CFIInst = MCCFIInstruction::createSameValue(L: nullptr, Register: DwarfReg);
2757 return buildCFI(MBB, MBBI, DL, CFIInst: std::move(CFIInst));
2758}
2759