1//===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SIMachineFunctionInfo.h"
10#include "AMDGPUSubtarget.h"
11#include "GCNSubtarget.h"
12#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
13#include "SIRegisterInfo.h"
14#include "Utils/AMDGPUBaseInfo.h"
15#include "llvm/CodeGen/LiveIntervals.h"
16#include "llvm/CodeGen/MIRParser/MIParser.h"
17#include "llvm/CodeGen/MachineBasicBlock.h"
18#include "llvm/CodeGen/MachineFrameInfo.h"
19#include "llvm/CodeGen/MachineFunction.h"
20#include "llvm/CodeGen/MachineRegisterInfo.h"
21#include "llvm/IR/CallingConv.h"
22#include "llvm/IR/DiagnosticInfo.h"
23#include "llvm/IR/Function.h"
24#include <cassert>
25#include <optional>
26#include <vector>
27
28enum { MAX_LANES = 64 };
29
30using namespace llvm;
31
32// TODO -- delete this flag once we have more robust mechanisms to allocate the
33// optimal RC for Opc and Dest of MFMA. In particular, there are high RP cases
34// where it is better to produce the VGPR form (e.g. if there are VGPR users
35// of the MFMA result).
36static cl::opt<bool, true> MFMAVGPRFormOpt(
37 "amdgpu-mfma-vgpr-form",
38 cl::desc("Whether to force use VGPR for Opc and Dest of MFMA. If "
39 "unspecified, default to compiler heuristics"),
40 cl::location(L&: SIMachineFunctionInfo::MFMAVGPRForm), cl::init(Val: true),
41 cl::Hidden);
42
43const GCNTargetMachine &getTM(const GCNSubtarget *STI) {
44 const SITargetLowering *TLI = STI->getTargetLowering();
45 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
46}
47
48bool SIMachineFunctionInfo::MFMAVGPRForm = false;
49
50SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,
51 const GCNSubtarget *STI)
52 : AMDGPUMachineFunctionInfo(F, *STI), Mode(F, *STI),
53 GWSResourcePSV(getTM(STI)), UserSGPRInfo(F, *STI), WorkGroupIDX(false),
54 WorkGroupIDY(false), WorkGroupIDZ(false), WorkGroupInfo(false),
55 LDSKernelId(false), PrivateSegmentWaveByteOffset(false),
56 WorkItemIDX(false), WorkItemIDY(false), WorkItemIDZ(false),
57 ImplicitArgPtr(false), GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0),
58 IsWholeWaveFunction(F.getCallingConv() ==
59 CallingConv::AMDGPU_Gfx_WholeWave) {
60 const GCNSubtarget &ST = *STI;
61 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
62 WavesPerEU = ST.getWavesPerEU(F);
63 MaxNumWorkGroups = AMDGPU::getMaxNumWorkGroups(F);
64 assert(MaxNumWorkGroups.size() == 3);
65
66 DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
67 Occupancy = ST.computeOccupancy(F, LDSSize: getLDSSize()).second;
68 CallingConv::ID CC = F.getCallingConv();
69
70 VRegFlags.reserve(S: 1024);
71
72 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
73 CC == CallingConv::SPIR_KERNEL;
74
75 if (IsKernel) {
76 WorkGroupIDX = true;
77 WorkItemIDX = true;
78 } else if (CC == CallingConv::AMDGPU_PS) {
79 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
80 }
81
82 if (ST.hasGFX90AInsts()) {
83 // FIXME: Extract logic out of getMaxNumVectorRegs; we need to apply the
84 // allocation granule and clamping.
85 auto [MinNumAGPRAttr, MaxNumAGPRAttr] =
86 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-agpr-alloc", Default: {~0u, ~0u},
87 /*OnlyFirstRequired=*/true);
88 MinNumAGPRs = MinNumAGPRAttr;
89 }
90
91 if (!isEntryFunction()) {
92 if (CC != CallingConv::AMDGPU_Gfx &&
93 CC != CallingConv::AMDGPU_Gfx_WholeWave)
94 ArgInfo = AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
95
96 FrameOffsetReg = AMDGPU::SGPR33;
97 StackPtrOffsetReg = AMDGPU::SGPR32;
98
99 if (!ST.hasFlatScratchEnabled()) {
100 // Non-entry functions have no special inputs for now, other registers
101 // required for scratch access.
102 ScratchRSrcReg = AMDGPU::isChainCC(CC)
103 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
104 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
105
106 ArgInfo.PrivateSegmentBuffer =
107 ArgDescriptor::createRegister(Reg: ScratchRSrcReg);
108 }
109
110 if (!F.hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr") &&
111 !AMDGPU::isChainCC(CC))
112 ImplicitArgPtr = true;
113 } else {
114 ImplicitArgPtr = false;
115 MaxKernArgAlign =
116 std::max(a: ST.getAlignmentForImplicitArgPtr(), b: MaxKernArgAlign);
117 }
118
119 if (!AMDGPU::isGraphics(CC) ||
120 ((CC == CallingConv::AMDGPU_CS || CC == CallingConv::AMDGPU_Gfx) &&
121 ST.hasArchitectedSGPRs())) {
122 if (IsKernel || !F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-x") ||
123 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-x"))
124 WorkGroupIDX = true;
125
126 if (!F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-y") ||
127 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-y"))
128 WorkGroupIDY = true;
129
130 if (!F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-z") ||
131 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-z"))
132 WorkGroupIDZ = true;
133 }
134
135 if (!AMDGPU::isGraphics(CC)) {
136 if (IsKernel || !F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-x"))
137 WorkItemIDX = true;
138
139 if (!F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-y") &&
140 ST.getMaxWorkitemID(Kernel: F, Dimension: 1) != 0)
141 WorkItemIDY = true;
142
143 if (!F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-z") &&
144 ST.getMaxWorkitemID(Kernel: F, Dimension: 2) != 0)
145 WorkItemIDZ = true;
146
147 if (!IsKernel && !F.hasFnAttribute(Kind: "amdgpu-no-lds-kernel-id"))
148 LDSKernelId = true;
149 }
150
151 if (isEntryFunction()) {
152 // X, XY, and XYZ are the only supported combinations, so make sure Y is
153 // enabled if Z is.
154 if (WorkItemIDZ)
155 WorkItemIDY = true;
156
157 if (!ST.hasArchitectedFlatScratch()) {
158 PrivateSegmentWaveByteOffset = true;
159
160 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
161 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
162 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
163 ArgInfo.PrivateSegmentWaveByteOffset =
164 ArgDescriptor::createRegister(Reg: AMDGPU::SGPR5);
165 }
166 }
167
168 Attribute A = F.getFnAttribute(Kind: "amdgpu-git-ptr-high");
169 StringRef S = A.getValueAsString();
170 if (!S.empty())
171 S.consumeInteger(Radix: 0, Result&: GITPtrHigh);
172
173 A = F.getFnAttribute(Kind: "amdgpu-32bit-address-high-bits");
174 S = A.getValueAsString();
175 if (!S.empty())
176 S.consumeInteger(Radix: 0, Result&: HighBitsOf32BitAddress);
177
178 MaxMemoryClusterDWords = F.getFnAttributeAsParsedInteger(
179 Kind: "amdgpu-max-memory-cluster-dwords", Default: DefaultMemoryClusterDWordsLimit);
180
181 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
182 // VGPR available at all times. For now, reserve highest available VGPR. After
183 // RA, shift it to the lowest available unused VGPR if the one exist.
184 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
185 VGPRForAGPRCopy =
186 AMDGPU::VGPR_32RegClass.getRegister(i: ST.getMaxNumVGPRs(F) - 1);
187 }
188
189 ClusterDims = AMDGPU::ClusterDimsAttr::get(F);
190}
191
192MachineFunctionInfo *SIMachineFunctionInfo::clone(
193 BumpPtrAllocator &Allocator, MachineFunction &DestMF,
194 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
195 const {
196 return DestMF.cloneInfo<SIMachineFunctionInfo>(Old: *this);
197}
198
199void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
200 limitOccupancy(Limit: getMaxWavesPerEU());
201 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
202 limitOccupancy(Limit: ST.getOccupancyWithWorkGroupSizes(MF).second);
203}
204
205Register SIMachineFunctionInfo::addPrivateSegmentBuffer(
206 const SIRegisterInfo &TRI) {
207 ArgInfo.PrivateSegmentBuffer =
208 ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
209 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SGPR_128RegClass));
210 NumUserSGPRs += 4;
211 return ArgInfo.PrivateSegmentBuffer.getRegister();
212}
213
214Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
215 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
216 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
217 NumUserSGPRs += 2;
218 return ArgInfo.DispatchPtr.getRegister();
219}
220
221Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
222 ArgInfo.QueuePtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
223 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
224 NumUserSGPRs += 2;
225 return ArgInfo.QueuePtr.getRegister();
226}
227
228Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
229 ArgInfo.KernargSegmentPtr
230 = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
231 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
232 NumUserSGPRs += 2;
233 return ArgInfo.KernargSegmentPtr.getRegister();
234}
235
236Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
237 ArgInfo.DispatchID = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
238 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
239 NumUserSGPRs += 2;
240 return ArgInfo.DispatchID.getRegister();
241}
242
243Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
244 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
245 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
246 NumUserSGPRs += 2;
247 return ArgInfo.FlatScratchInit.getRegister();
248}
249
250Register SIMachineFunctionInfo::addPrivateSegmentSize(const SIRegisterInfo &TRI) {
251 ArgInfo.PrivateSegmentSize = ArgDescriptor::createRegister(Reg: getNextUserSGPR());
252 NumUserSGPRs += 1;
253 return ArgInfo.PrivateSegmentSize.getRegister();
254}
255
256Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
257 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
258 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
259 NumUserSGPRs += 2;
260 return ArgInfo.ImplicitBufferPtr.getRegister();
261}
262
263Register SIMachineFunctionInfo::addLDSKernelId() {
264 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(Reg: getNextUserSGPR());
265 NumUserSGPRs += 1;
266 return ArgInfo.LDSKernelId.getRegister();
267}
268
269SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg(
270 const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
271 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) {
272 auto [It, Inserted] = ArgInfo.PreloadKernArgs.try_emplace(Key: KernArgIdx);
273 assert(Inserted && "Preload kernel argument allocated twice.");
274 NumUserSGPRs += PaddingSGPRs;
275 // If the available register tuples are aligned with the kernarg to be
276 // preloaded use that register, otherwise we need to use a set of SGPRs and
277 // merge them.
278 if (!ArgInfo.FirstKernArgPreloadReg)
279 ArgInfo.FirstKernArgPreloadReg = getNextUserSGPR();
280 Register PreloadReg =
281 TRI.getMatchingSuperReg(Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC);
282 auto &Regs = It->second.Regs;
283 if (PreloadReg &&
284 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
285 Regs.push_back(Elt: PreloadReg);
286 NumUserSGPRs += AllocSizeDWord;
287 } else {
288 Regs.reserve(N: AllocSizeDWord);
289 for (unsigned I = 0; I < AllocSizeDWord; ++I) {
290 Regs.push_back(Elt: getNextUserSGPR());
291 NumUserSGPRs++;
292 }
293 }
294
295 // Track the actual number of SGPRs that HW will preload to.
296 UserSGPRInfo.allocKernargPreloadSGPRs(NumSGPRs: AllocSizeDWord + PaddingSGPRs);
297 return &Regs;
298}
299
300void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
301 uint64_t Size, Align Alignment) {
302 // Skip if it is an entry function or the register is already added.
303 if (isEntryFunction() || WWMSpills.count(Key: VGPR))
304 return;
305
306 // Skip if this is a function with the amdgpu_cs_chain or
307 // amdgpu_cs_chain_preserve calling convention and this is a scratch register.
308 // We never need to allocate a spill for these because we don't even need to
309 // restore the inactive lanes for them (they're scratchier than the usual
310 // scratch registers). We only need to do this if we have calls to
311 // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since
312 // chain functions do not return) and the function did not contain a call to
313 // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes
314 // when entering the function).
315 if (isChainFunction() &&
316 (SIRegisterInfo::isChainScratchRegister(VGPR) ||
317 !MF.getFrameInfo().hasTailCall() || hasInitWholeWave()))
318 return;
319
320 WWMSpills.insert(KV: std::make_pair(
321 x&: VGPR, y: MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
322}
323
324// Separate out the callee-saved and scratch registers.
325void SIMachineFunctionInfo::splitWWMSpillRegisters(
326 MachineFunction &MF,
327 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
328 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
329 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
330 for (auto &Reg : WWMSpills) {
331 if (isCalleeSavedReg(CSRegs, Reg: Reg.first))
332 CalleeSavedRegs.push_back(Elt: Reg);
333 else
334 ScratchRegs.push_back(Elt: Reg);
335 }
336}
337
338bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,
339 MCPhysReg Reg) const {
340 for (unsigned I = 0; CSRegs[I]; ++I) {
341 if (CSRegs[I] == Reg)
342 return true;
343 }
344
345 return false;
346}
347
348void SIMachineFunctionInfo::shiftWwmVGPRsToLowestRange(
349 MachineFunction &MF, SmallVectorImpl<Register> &WWMVGPRs,
350 BitVector &SavedVGPRs) {
351 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
352 MachineRegisterInfo &MRI = MF.getRegInfo();
353 for (unsigned I = 0, E = WWMVGPRs.size(); I < E; ++I) {
354 Register Reg = WWMVGPRs[I];
355 Register NewReg =
356 TRI->findUnusedRegister(MRI, RC: &AMDGPU::VGPR_32RegClass, MF);
357 if (!NewReg || NewReg >= Reg)
358 break;
359
360 MRI.replaceRegWith(FromReg: Reg, ToReg: NewReg);
361
362 // Update various tables with the new VGPR.
363 WWMVGPRs[I] = NewReg;
364 WWMReservedRegs.remove(X: Reg);
365 WWMReservedRegs.insert(X: NewReg);
366 MRI.reserveReg(PhysReg: NewReg, TRI);
367
368 // Replace the register in SpillPhysVGPRs. This is needed to look for free
369 // lanes while spilling special SGPRs like FP, BP, etc. during PEI.
370 auto *RegItr = llvm::find(Range&: SpillPhysVGPRs, Val: Reg);
371 if (RegItr != SpillPhysVGPRs.end()) {
372 unsigned Idx = std::distance(first: SpillPhysVGPRs.begin(), last: RegItr);
373 SpillPhysVGPRs[Idx] = NewReg;
374
375 // For replacing registers used in the CFI instructions.
376 MF.replaceFrameInstRegister(From: Reg, To: NewReg);
377 }
378
379 // The generic `determineCalleeSaves` might have set the old register if it
380 // is in the CSR range.
381 SavedVGPRs.reset(Idx: Reg);
382
383 for (MachineBasicBlock &MBB : MF) {
384 MBB.removeLiveIn(Reg);
385 MBB.sortUniqueLiveIns();
386 }
387
388 Reg = NewReg;
389 }
390}
391
392bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
393 MachineFunction &MF, int FI, unsigned LaneIndex) {
394 MachineRegisterInfo &MRI = MF.getRegInfo();
395 Register LaneVGPR;
396 if (!LaneIndex) {
397 LaneVGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
398 SpillVGPRs.push_back(Elt: LaneVGPR);
399 } else {
400 LaneVGPR = SpillVGPRs.back();
401 }
402
403 SGPRSpillsToVirtualVGPRLanes[FI].emplace_back(args&: LaneVGPR, args&: LaneIndex);
404 return true;
405}
406
407bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
408 MachineFunction &MF, int FI, unsigned LaneIndex, bool IsPrologEpilog) {
409 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
410 const SIRegisterInfo *TRI = ST.getRegisterInfo();
411 MachineRegisterInfo &MRI = MF.getRegInfo();
412 Register LaneVGPR;
413 if (!LaneIndex) {
414 // Find the highest available register if called before RA to ensure the
415 // lowest registers are available for allocation. The LaneVGPR, in that
416 // case, will be shifted back to the lowest range after VGPR allocation.
417 LaneVGPR = TRI->findUnusedRegister(MRI, RC: &AMDGPU::VGPR_32RegClass, MF,
418 ReserveHighestVGPR: !IsPrologEpilog);
419 if (LaneVGPR == AMDGPU::NoRegister) {
420 // We have no VGPRs left for spilling SGPRs. Reset because we will not
421 // partially spill the SGPR to VGPRs.
422 SGPRSpillsToPhysicalVGPRLanes.erase(Val: FI);
423 return false;
424 }
425
426 if (IsPrologEpilog)
427 allocateWWMSpill(MF, VGPR: LaneVGPR);
428
429 reserveWWMRegister(Reg: LaneVGPR);
430 for (MachineBasicBlock &MBB : MF) {
431 MBB.addLiveIn(PhysReg: LaneVGPR);
432 MBB.sortUniqueLiveIns();
433 }
434 SpillPhysVGPRs.push_back(Elt: LaneVGPR);
435 } else {
436 LaneVGPR = SpillPhysVGPRs.back();
437 }
438
439 SGPRSpillsToPhysicalVGPRLanes[FI].emplace_back(args&: LaneVGPR, args&: LaneIndex);
440 return true;
441}
442
443bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(
444 MachineFunction &MF, int FI, bool SpillToPhysVGPRLane,
445 bool IsPrologEpilog) {
446 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
447 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]
448 : SGPRSpillsToVirtualVGPRLanes[FI];
449
450 // This has already been allocated.
451 if (!SpillLanes.empty())
452 return true;
453
454 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
455 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
456 unsigned WaveSize = ST.getWavefrontSize();
457
458 unsigned Size = FrameInfo.getObjectSize(ObjectIdx: FI);
459 unsigned NumLanes = Size / 4;
460
461 if (NumLanes > WaveSize)
462 return false;
463
464 assert(Size >= 4 && "invalid sgpr spill size");
465 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
466 "not spilling SGPRs to VGPRs");
467
468 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes
469 : NumVirtualVGPRSpillLanes;
470
471 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
472 unsigned LaneIndex = (NumSpillLanes % WaveSize);
473
474 bool Allocated = SpillToPhysVGPRLane
475 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,
476 IsPrologEpilog)
477 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
478 if (!Allocated) {
479 NumSpillLanes -= I;
480 return false;
481 }
482 }
483
484 return true;
485}
486
487/// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
488/// Either AGPR is spilled to VGPR to vice versa.
489/// Returns true if a \p FI can be eliminated completely.
490bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
491 int FI,
492 bool isAGPRtoVGPR) {
493 MachineRegisterInfo &MRI = MF.getRegInfo();
494 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
495 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
496
497 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
498
499 auto &Spill = VGPRToAGPRSpills[FI];
500
501 // This has already been allocated.
502 if (!Spill.Lanes.empty())
503 return Spill.FullyAllocated;
504
505 unsigned Size = FrameInfo.getObjectSize(ObjectIdx: FI);
506 unsigned NumLanes = Size / 4;
507 Spill.Lanes.resize(N: NumLanes, NV: AMDGPU::NoRegister);
508
509 const TargetRegisterClass &RC =
510 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
511 auto Regs = RC.getRegisters();
512
513 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
514 const SIRegisterInfo *TRI = ST.getRegisterInfo();
515 Spill.FullyAllocated = true;
516
517 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
518 // once.
519 BitVector OtherUsedRegs;
520 OtherUsedRegs.resize(N: TRI->getNumRegs());
521
522 const uint32_t *CSRMask =
523 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
524 if (CSRMask)
525 OtherUsedRegs.setBitsInMask(Mask: CSRMask);
526
527 // TODO: Should include register tuples, but doesn't matter with current
528 // usage.
529 for (MCPhysReg Reg : SpillAGPR)
530 OtherUsedRegs.set(Reg);
531 for (MCPhysReg Reg : SpillVGPR)
532 OtherUsedRegs.set(Reg);
533
534 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
535 for (int I = NumLanes - 1; I >= 0; --I) {
536 NextSpillReg = std::find_if(
537 first: NextSpillReg, last: Regs.end(), pred: [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
538 return MRI.isAllocatable(PhysReg: Reg) && !MRI.isPhysRegUsed(PhysReg: Reg) &&
539 !OtherUsedRegs[Reg];
540 });
541
542 if (NextSpillReg == Regs.end()) { // Registers exhausted
543 Spill.FullyAllocated = false;
544 break;
545 }
546
547 OtherUsedRegs.set(*NextSpillReg);
548 SpillRegs.push_back(Elt: *NextSpillReg);
549 MRI.reserveReg(PhysReg: *NextSpillReg, TRI);
550 Spill.Lanes[I] = *NextSpillReg++;
551 }
552
553 return Spill.FullyAllocated;
554}
555
556bool SIMachineFunctionInfo::removeDeadFrameIndices(
557 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
558 // Remove dead frame indices from function frame, however keep FP & BP since
559 // spills for them haven't been inserted yet. And also make sure to remove the
560 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure,
561 // otherwise, it could result in an unexpected side effect and bug, in case of
562 // any re-mapping of freed frame indices by later pass(es) like "stack slot
563 // coloring".
564 for (auto &R : SGPRSpillsToVirtualVGPRLanes)
565 MFI.RemoveStackObject(ObjectIdx: R.first);
566 SGPRSpillsToVirtualVGPRLanes.clear();
567
568 // Remove the dead frame indices of CSR SGPRs which are spilled to physical
569 // VGPR lanes during SILowerSGPRSpills pass.
570 if (!ResetSGPRSpillStackIDs) {
571 for (auto &R : SGPRSpillsToPhysicalVGPRLanes)
572 MFI.RemoveStackObject(ObjectIdx: R.first);
573 SGPRSpillsToPhysicalVGPRLanes.clear();
574 }
575 bool HaveSGPRToMemory = false;
576
577 if (ResetSGPRSpillStackIDs) {
578 // All other SGPRs must be allocated on the default stack, so reset the
579 // stack ID.
580 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
581 ++I) {
582 if (!checkIndexInPrologEpilogSGPRSpills(FI: I)) {
583 if (MFI.getStackID(ObjectIdx: I) == TargetStackID::SGPRSpill) {
584 MFI.setStackID(ObjectIdx: I, ID: TargetStackID::Default);
585 HaveSGPRToMemory = true;
586 }
587 }
588 }
589 }
590
591 for (auto &R : VGPRToAGPRSpills) {
592 if (R.second.IsDead)
593 MFI.RemoveStackObject(ObjectIdx: R.first);
594 }
595
596 return HaveSGPRToMemory;
597}
598
599int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI,
600 const SIRegisterInfo &TRI) {
601 if (ScavengeFI)
602 return *ScavengeFI;
603
604 ScavengeFI =
605 MFI.CreateStackObject(Size: TRI.getSpillSize(RC: AMDGPU::SGPR_32RegClass),
606 Alignment: TRI.getSpillAlign(RC: AMDGPU::SGPR_32RegClass), isSpillSlot: false);
607 return *ScavengeFI;
608}
609
610MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
611 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
612 return AMDGPU::SGPR0 + NumUserSGPRs;
613}
614
615MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
616 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
617}
618
619void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {
620 VRegFlags.grow(N: Reg);
621}
622
623void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,
624 Register SrcReg) {
625 VRegFlags.grow(N: NewReg);
626 VRegFlags[NewReg] = VRegFlags[SrcReg];
627}
628
629Register
630SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const {
631 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
632 if (!ST.isAmdPalOS())
633 return Register();
634 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
635 if (ST.hasMergedShaders()) {
636 switch (MF.getFunction().getCallingConv()) {
637 case CallingConv::AMDGPU_HS:
638 case CallingConv::AMDGPU_GS:
639 // Low GIT address is passed in s8 rather than s0 for an LS+HS or
640 // ES+GS merged shader on gfx9+.
641 GitPtrLo = AMDGPU::SGPR8;
642 return GitPtrLo;
643 default:
644 return GitPtrLo;
645 }
646 }
647 return GitPtrLo;
648}
649
650static yaml::StringValue regToString(Register Reg,
651 const TargetRegisterInfo &TRI) {
652 yaml::StringValue Dest;
653 {
654 raw_string_ostream OS(Dest.Value);
655 OS << printReg(Reg, TRI: &TRI);
656 }
657 return Dest;
658}
659
660static std::optional<yaml::SIArgumentInfo>
661convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
662 const TargetRegisterInfo &TRI) {
663 yaml::SIArgumentInfo AI;
664
665 auto convertArg = [&](std::optional<yaml::SIArgument> &A,
666 const ArgDescriptor &Arg) {
667 if (!Arg)
668 return false;
669
670 // Create a register or stack argument.
671 yaml::SIArgument SA = yaml::SIArgument::createArgument(IsReg: Arg.isRegister());
672 if (Arg.isRegister()) {
673 raw_string_ostream OS(SA.RegisterName.Value);
674 OS << printReg(Reg: Arg.getRegister(), TRI: &TRI);
675 } else
676 SA.StackOffset = Arg.getStackOffset();
677 // Check and update the optional mask.
678 if (Arg.isMasked())
679 SA.Mask = Arg.getMask();
680
681 A = std::move(SA);
682 return true;
683 };
684
685 bool Any = false;
686 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
687 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
688 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
689 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
690 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
691 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
692 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
693 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
694 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
695 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
696 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
697 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
698 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
699 ArgInfo.PrivateSegmentWaveByteOffset);
700 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
701 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
702 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
703 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
704 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
705
706 // Write FirstKernArgPreloadReg separately, since it's a Register,
707 // not ArgDescriptor.
708 if (ArgInfo.FirstKernArgPreloadReg) {
709 Register Reg = ArgInfo.FirstKernArgPreloadReg;
710 assert(Reg.isPhysical() &&
711 "FirstKernArgPreloadReg must be a physical register");
712
713 yaml::SIArgument SA = yaml::SIArgument::createArgument(IsReg: true);
714 raw_string_ostream OS(SA.RegisterName.Value);
715 OS << printReg(Reg, TRI: &TRI);
716
717 AI.FirstKernArgPreloadReg = SA;
718 Any = true;
719 }
720
721 if (Any)
722 return AI;
723
724 return std::nullopt;
725}
726
727yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
728 const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI,
729 const llvm::MachineFunction &MF)
730 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
731 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
732 GDSSize(MFI.getGDSSize()), DynLDSAlign(MFI.getDynLDSAlign()),
733 IsEntryFunction(MFI.isEntryFunction()), MemoryBound(MFI.isMemoryBound()),
734 WaveLimiter(MFI.needsWaveLimiter()),
735 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
736 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
737 HasNoWWMPoolSGPRSpillFallback(MFI.hasNoWWMPoolSGPRSpillFallback()),
738 NumWaveDispatchSGPRs(MFI.getNumWaveDispatchSGPRs()),
739 NumWaveDispatchVGPRs(MFI.getNumWaveDispatchVGPRs()),
740 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
741 Occupancy(MFI.getOccupancy()),
742 ScratchRSrcReg(regToString(Reg: MFI.getScratchRSrcReg(), TRI)),
743 FrameOffsetReg(regToString(Reg: MFI.getFrameOffsetReg(), TRI)),
744 StackPtrOffsetReg(regToString(Reg: MFI.getStackPtrOffsetReg(), TRI)),
745 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
746 ReturnsVoid(MFI.returnsVoid()),
747 ArgInfo(convertArgumentInfo(ArgInfo: MFI.getArgInfo(), TRI)),
748 PSInputAddr(MFI.getPSInputAddr()), PSInputEnable(MFI.getPSInputEnable()),
749 MaxMemoryClusterDWords(MFI.getMaxMemoryClusterDWords()),
750 Mode(MFI.getMode()), HasInitWholeWave(MFI.hasInitWholeWave()),
751 IsWholeWaveFunction(MFI.isWholeWaveFunction()),
752 DynamicVGPRBlockSize(MFI.getDynamicVGPRBlockSize()),
753 ScratchReservedForDynamicVGPRs(MFI.getScratchReservedForDynamicVGPRs()),
754 NumKernargPreloadSGPRs(MFI.getNumKernargPreloadedSGPRs()),
755 MinNumAGPRs(MFI.getMinNumAGPRs()) {
756 for (Register Reg : MFI.getSGPRSpillPhysVGPRs())
757 SpillPhysVGPRS.push_back(Elt: regToString(Reg, TRI));
758
759 for (Register Reg : MFI.getWWMReservedRegs())
760 WWMReservedRegs.push_back(Elt: regToString(Reg, TRI));
761
762 if (MFI.getLongBranchReservedReg())
763 LongBranchReservedReg = regToString(Reg: MFI.getLongBranchReservedReg(), TRI);
764 if (MFI.getVGPRForAGPRCopy())
765 VGPRForAGPRCopy = regToString(Reg: MFI.getVGPRForAGPRCopy(), TRI);
766
767 if (MFI.getSGPRForEXECCopy())
768 SGPRForEXECCopy = regToString(Reg: MFI.getSGPRForEXECCopy(), TRI);
769
770 auto SFI = MFI.getOptionalScavengeFI();
771 if (SFI)
772 ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo());
773}
774
775void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
776 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, MFI&: *this);
777}
778
779bool SIMachineFunctionInfo::initializeBaseYamlFields(
780 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
781 PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) {
782 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
783 MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
784 LDSSize = YamlMFI.LDSSize;
785 GDSSize = YamlMFI.GDSSize;
786 DynLDSAlign = YamlMFI.DynLDSAlign;
787 PSInputAddr = YamlMFI.PSInputAddr;
788 PSInputEnable = YamlMFI.PSInputEnable;
789 MaxMemoryClusterDWords = YamlMFI.MaxMemoryClusterDWords;
790 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
791 Occupancy = YamlMFI.Occupancy;
792 IsEntryFunction = YamlMFI.IsEntryFunction;
793 MemoryBound = YamlMFI.MemoryBound;
794 WaveLimiter = YamlMFI.WaveLimiter;
795 HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
796 HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
797 HasNoWWMPoolSGPRSpillFallback = YamlMFI.HasNoWWMPoolSGPRSpillFallback;
798 NumWaveDispatchSGPRs = YamlMFI.NumWaveDispatchSGPRs;
799 NumWaveDispatchVGPRs = YamlMFI.NumWaveDispatchVGPRs;
800 BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
801 ReturnsVoid = YamlMFI.ReturnsVoid;
802 IsWholeWaveFunction = YamlMFI.IsWholeWaveFunction;
803 MinNumAGPRs = YamlMFI.MinNumAGPRs;
804 // This can also be set by the function attribute, MFI has higher precedence
805 // though.
806 if (YamlMFI.DynamicVGPRBlockSize != std::nullopt)
807 DynamicVGPRBlockSize = *YamlMFI.DynamicVGPRBlockSize;
808
809 UserSGPRInfo.allocKernargPreloadSGPRs(NumSGPRs: YamlMFI.NumKernargPreloadSGPRs);
810
811 if (YamlMFI.ScavengeFI) {
812 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MFI: MF.getFrameInfo());
813 if (!FIOrErr) {
814 // Create a diagnostic for a the frame index.
815 const MemoryBuffer &Buffer =
816 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
817
818 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
819 SourceMgr::DK_Error, toString(E: FIOrErr.takeError()),
820 "", {}, {});
821 SourceRange = YamlMFI.ScavengeFI->SourceRange;
822 return true;
823 }
824 ScavengeFI = *FIOrErr;
825 } else {
826 ScavengeFI = std::nullopt;
827 }
828 return false;
829}
830
831bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const {
832 auto [MinNumAGPR, MaxNumAGPR] =
833 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-agpr-alloc", Default: {~0u, ~0u},
834 /*OnlyFirstRequired=*/true);
835 return MinNumAGPR != 0u;
836}
837