1//===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SIMachineFunctionInfo.h"
10#include "AMDGPUSubtarget.h"
11#include "GCNSubtarget.h"
12#include "SIRegisterInfo.h"
13#include "Utils/AMDGPUBaseInfo.h"
14#include "llvm/CodeGen/LiveIntervals.h"
15#include "llvm/CodeGen/MIRParser/MIParser.h"
16#include "llvm/CodeGen/MachineBasicBlock.h"
17#include "llvm/CodeGen/MachineFrameInfo.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/IR/CallingConv.h"
21#include "llvm/IR/DiagnosticInfo.h"
22#include "llvm/IR/Function.h"
23#include <cassert>
24#include <optional>
25
26enum { MAX_LANES = 64 };
27
28using namespace llvm;
29
30// TODO -- delete this flag once we have more robust mechanisms to allocate the
31// optimal RC for Opc and Dest of MFMA. In particular, there are high RP cases
32// where it is better to produce the VGPR form (e.g. if there are VGPR users
33// of the MFMA result).
34static cl::opt<bool, true> MFMAVGPRFormOpt(
35 "amdgpu-mfma-vgpr-form",
36 cl::desc("Whether to force use VGPR for Opc and Dest of MFMA. If "
37 "unspecified, default to compiler heuristics"),
38 cl::location(L&: SIMachineFunctionInfo::MFMAVGPRForm), cl::init(Val: true),
39 cl::Hidden);
40
41const GCNTargetMachine &getTM(const GCNSubtarget *STI) {
42 const SITargetLowering *TLI = STI->getTargetLowering();
43 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
44}
45
46bool SIMachineFunctionInfo::MFMAVGPRForm = false;
47
48SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,
49 const GCNSubtarget *STI)
50 : AMDGPUMachineFunctionInfo(F, *STI), Mode(F, *STI),
51 GWSResourcePSV(getTM(STI)), UserSGPRInfo(F, *STI), WorkGroupIDX(false),
52 WorkGroupIDY(false), WorkGroupIDZ(false), WorkGroupInfo(false),
53 LDSKernelId(false), PrivateSegmentWaveByteOffset(false),
54 WorkItemIDX(false), WorkItemIDY(false), WorkItemIDZ(false),
55 ImplicitArgPtr(false), GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0),
56 IsWholeWaveFunction(F.getCallingConv() ==
57 CallingConv::AMDGPU_Gfx_WholeWave) {
58 const GCNSubtarget &ST = *STI;
59 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
60 WavesPerEU = ST.getWavesPerEU(F);
61 MaxNumWorkGroups = AMDGPU::getMaxNumWorkGroups(F);
62 assert(MaxNumWorkGroups.size() == 3);
63
64 DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
65 Occupancy = ST.computeOccupancy(F, LDSSize: getLDSSize()).second;
66 CallingConv::ID CC = F.getCallingConv();
67
68 VRegFlags.reserve(S: 1024);
69
70 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
71 CC == CallingConv::SPIR_KERNEL;
72
73 if (IsKernel) {
74 WorkGroupIDX = true;
75 WorkItemIDX = true;
76 } else if (CC == CallingConv::AMDGPU_PS) {
77 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
78 }
79
80 if (ST.hasGFX90AInsts()) {
81 // FIXME: Extract logic out of getMaxNumVectorRegs; we need to apply the
82 // allocation granule and clamping.
83 auto [MinNumAGPRAttr, MaxNumAGPRAttr] =
84 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-agpr-alloc", Default: {~0u, ~0u},
85 /*OnlyFirstRequired=*/true);
86 MinNumAGPRs = MinNumAGPRAttr;
87 }
88
89 if (!isEntryFunction()) {
90 if (CC != CallingConv::AMDGPU_Gfx &&
91 CC != CallingConv::AMDGPU_Gfx_WholeWave)
92 ArgInfo = AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
93
94 FrameOffsetReg = AMDGPU::SGPR33;
95 StackPtrOffsetReg = AMDGPU::SGPR32;
96
97 if (!ST.hasFlatScratchEnabled()) {
98 // Non-entry functions have no special inputs for now, other registers
99 // required for scratch access.
100 ScratchRSrcReg = AMDGPU::isChainCC(CC)
101 ? AMDGPU::SGPR48_SGPR49_SGPR50_SGPR51
102 : AMDGPU::SGPR0_SGPR1_SGPR2_SGPR3;
103
104 ArgInfo.PrivateSegmentBuffer =
105 ArgDescriptor::createRegister(Reg: ScratchRSrcReg);
106 }
107
108 if (!F.hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr") &&
109 !AMDGPU::isChainCC(CC))
110 ImplicitArgPtr = true;
111 } else {
112 ImplicitArgPtr = false;
113 MaxKernArgAlign =
114 std::max(a: ST.getAlignmentForImplicitArgPtr(), b: MaxKernArgAlign);
115 }
116
117 if (!AMDGPU::isGraphics(CC) ||
118 ((CC == CallingConv::AMDGPU_CS || CC == CallingConv::AMDGPU_Gfx) &&
119 ST.hasArchitectedSGPRs())) {
120 if (IsKernel || !F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-x") ||
121 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-x"))
122 WorkGroupIDX = true;
123
124 if (!F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-y") ||
125 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-y"))
126 WorkGroupIDY = true;
127
128 if (!F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-z") ||
129 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-z"))
130 WorkGroupIDZ = true;
131 }
132
133 if (!AMDGPU::isGraphics(CC)) {
134 if (IsKernel || !F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-x"))
135 WorkItemIDX = true;
136
137 if (!F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-y") &&
138 ST.getMaxWorkitemID(Kernel: F, Dimension: 1) != 0)
139 WorkItemIDY = true;
140
141 if (!F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-z") &&
142 ST.getMaxWorkitemID(Kernel: F, Dimension: 2) != 0)
143 WorkItemIDZ = true;
144
145 if (!IsKernel && !F.hasFnAttribute(Kind: "amdgpu-no-lds-kernel-id"))
146 LDSKernelId = true;
147 }
148
149 if (isEntryFunction()) {
150 // X, XY, and XYZ are the only supported combinations, so make sure Y is
151 // enabled if Z is.
152 if (WorkItemIDZ)
153 WorkItemIDY = true;
154
155 if (!ST.hasArchitectedFlatScratch()) {
156 PrivateSegmentWaveByteOffset = true;
157
158 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
159 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
160 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
161 ArgInfo.PrivateSegmentWaveByteOffset =
162 ArgDescriptor::createRegister(Reg: AMDGPU::SGPR5);
163 }
164 }
165
166 Attribute A = F.getFnAttribute(Kind: "amdgpu-git-ptr-high");
167 StringRef S = A.getValueAsString();
168 if (!S.empty())
169 S.consumeInteger(Radix: 0, Result&: GITPtrHigh);
170
171 A = F.getFnAttribute(Kind: "amdgpu-32bit-address-high-bits");
172 S = A.getValueAsString();
173 if (!S.empty())
174 S.consumeInteger(Radix: 0, Result&: HighBitsOf32BitAddress);
175
176 MaxMemoryClusterDWords = F.getFnAttributeAsParsedInteger(
177 Kind: "amdgpu-max-memory-cluster-dwords", Default: DefaultMemoryClusterDWordsLimit);
178
179 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
180 // VGPR available at all times. For now, reserve highest available VGPR. After
181 // RA, shift it to the lowest available unused VGPR if the one exist.
182 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
183 VGPRForAGPRCopy =
184 AMDGPU::VGPR_32RegClass.getRegister(i: ST.getMaxNumVGPRs(F) - 1);
185 }
186
187 ClusterDims = AMDGPU::ClusterDimsAttr::get(F);
188}
189
190MachineFunctionInfo *SIMachineFunctionInfo::clone(
191 BumpPtrAllocator &Allocator, MachineFunction &DestMF,
192 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
193 const {
194 return DestMF.cloneInfo<SIMachineFunctionInfo>(Old: *this);
195}
196
197void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
198 limitOccupancy(Limit: getMaxWavesPerEU());
199 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
200 limitOccupancy(Limit: ST.getOccupancyWithWorkGroupSizes(MF).second);
201}
202
203Register SIMachineFunctionInfo::addPrivateSegmentBuffer(
204 const SIRegisterInfo &TRI) {
205 ArgInfo.PrivateSegmentBuffer =
206 ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
207 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SGPR_128RegClass));
208 NumUserSGPRs += 4;
209 return ArgInfo.PrivateSegmentBuffer.getRegister();
210}
211
212Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
213 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
214 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
215 NumUserSGPRs += 2;
216 return ArgInfo.DispatchPtr.getRegister();
217}
218
219Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
220 ArgInfo.QueuePtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
221 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
222 NumUserSGPRs += 2;
223 return ArgInfo.QueuePtr.getRegister();
224}
225
226Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
227 ArgInfo.KernargSegmentPtr
228 = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
229 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
230 NumUserSGPRs += 2;
231 return ArgInfo.KernargSegmentPtr.getRegister();
232}
233
234Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
235 ArgInfo.DispatchID = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
236 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
237 NumUserSGPRs += 2;
238 return ArgInfo.DispatchID.getRegister();
239}
240
241Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
242 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
243 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
244 NumUserSGPRs += 2;
245 return ArgInfo.FlatScratchInit.getRegister();
246}
247
248Register SIMachineFunctionInfo::addPrivateSegmentSize(const SIRegisterInfo &TRI) {
249 ArgInfo.PrivateSegmentSize = ArgDescriptor::createRegister(Reg: getNextUserSGPR());
250 NumUserSGPRs += 1;
251 return ArgInfo.PrivateSegmentSize.getRegister();
252}
253
254Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
255 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
256 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
257 NumUserSGPRs += 2;
258 return ArgInfo.ImplicitBufferPtr.getRegister();
259}
260
261Register SIMachineFunctionInfo::addLDSKernelId() {
262 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(Reg: getNextUserSGPR());
263 NumUserSGPRs += 1;
264 return ArgInfo.LDSKernelId.getRegister();
265}
266
267SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg(
268 const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
269 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) {
270 auto [It, Inserted] = ArgInfo.PreloadKernArgs.try_emplace(Key: KernArgIdx);
271 assert(Inserted && "Preload kernel argument allocated twice.");
272 NumUserSGPRs += PaddingSGPRs;
273 // If the available register tuples are aligned with the kernarg to be
274 // preloaded use that register, otherwise we need to use a set of SGPRs and
275 // merge them.
276 if (!ArgInfo.FirstKernArgPreloadReg)
277 ArgInfo.FirstKernArgPreloadReg = getNextUserSGPR();
278 Register PreloadReg =
279 TRI.getMatchingSuperReg(Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC);
280 auto &Regs = It->second.Regs;
281 if (PreloadReg &&
282 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
283 Regs.push_back(Elt: PreloadReg);
284 NumUserSGPRs += AllocSizeDWord;
285 } else {
286 Regs.reserve(N: AllocSizeDWord);
287 for (unsigned I = 0; I < AllocSizeDWord; ++I) {
288 Regs.push_back(Elt: getNextUserSGPR());
289 NumUserSGPRs++;
290 }
291 }
292
293 // Track the actual number of SGPRs that HW will preload to.
294 UserSGPRInfo.allocKernargPreloadSGPRs(NumSGPRs: AllocSizeDWord + PaddingSGPRs);
295 return &Regs;
296}
297
298void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
299 uint64_t Size, Align Alignment) {
300 // Skip if it is an entry function or the register is already added.
301 if (isEntryFunction() || WWMSpills.count(Key: VGPR))
302 return;
303
304 // Skip if this is a function with the amdgpu_cs_chain or
305 // amdgpu_cs_chain_preserve calling convention and this is a scratch register.
306 // We never need to allocate a spill for these because we don't even need to
307 // restore the inactive lanes for them (they're scratchier than the usual
308 // scratch registers). We only need to do this if we have calls to
309 // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since
310 // chain functions do not return) and the function did not contain a call to
311 // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes
312 // when entering the function).
313 if (isChainFunction() &&
314 (SIRegisterInfo::isChainScratchRegister(VGPR) ||
315 !MF.getFrameInfo().hasTailCall() || hasInitWholeWave()))
316 return;
317
318 WWMSpills.insert(KV: std::make_pair(
319 x&: VGPR, y: MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
320}
321
322// Separate out the callee-saved and scratch registers.
323void SIMachineFunctionInfo::splitWWMSpillRegisters(
324 MachineFunction &MF,
325 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
326 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
327 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
328 for (auto &Reg : WWMSpills) {
329 if (isCalleeSavedReg(CSRegs, Reg: Reg.first))
330 CalleeSavedRegs.push_back(Elt: Reg);
331 else
332 ScratchRegs.push_back(Elt: Reg);
333 }
334}
335
336bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,
337 MCPhysReg Reg) const {
338 for (unsigned I = 0; CSRegs[I]; ++I) {
339 if (CSRegs[I] == Reg)
340 return true;
341 }
342
343 return false;
344}
345
346void SIMachineFunctionInfo::shiftWwmVGPRsToLowestRange(
347 MachineFunction &MF, SmallVectorImpl<Register> &WWMVGPRs,
348 BitVector &SavedVGPRs) {
349 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
350 MachineRegisterInfo &MRI = MF.getRegInfo();
351 for (unsigned I = 0, E = WWMVGPRs.size(); I < E; ++I) {
352 Register Reg = WWMVGPRs[I];
353 Register NewReg =
354 TRI->findUnusedRegister(MRI, RC: &AMDGPU::VGPR_32RegClass, MF);
355 if (!NewReg || NewReg >= Reg)
356 break;
357
358 MRI.replaceRegWith(FromReg: Reg, ToReg: NewReg);
359
360 // Update various tables with the new VGPR.
361 WWMVGPRs[I] = NewReg;
362 WWMReservedRegs.remove(X: Reg);
363 WWMReservedRegs.insert(X: NewReg);
364 MRI.reserveReg(PhysReg: NewReg, TRI);
365
366 // Replace the register in SpillPhysVGPRs. This is needed to look for free
367 // lanes while spilling special SGPRs like FP, BP, etc. during PEI.
368 auto *RegItr = llvm::find(Range&: SpillPhysVGPRs, Val: Reg);
369 if (RegItr != SpillPhysVGPRs.end()) {
370 unsigned Idx = std::distance(first: SpillPhysVGPRs.begin(), last: RegItr);
371 SpillPhysVGPRs[Idx] = NewReg;
372
373 // For replacing registers used in the CFI instructions.
374 MF.replaceFrameInstRegister(From: Reg, To: NewReg);
375 }
376
377 // The generic `determineCalleeSaves` might have set the old register if it
378 // is in the CSR range.
379 SavedVGPRs.reset(Idx: Reg);
380
381 for (MachineBasicBlock &MBB : MF) {
382 MBB.removeLiveIn(Reg);
383 MBB.sortUniqueLiveIns();
384 }
385
386 Reg = NewReg;
387 }
388}
389
390bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
391 MachineFunction &MF, int FI, unsigned LaneIndex) {
392 MachineRegisterInfo &MRI = MF.getRegInfo();
393 Register LaneVGPR;
394 if (!LaneIndex) {
395 LaneVGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
396 SpillVGPRs.push_back(Elt: LaneVGPR);
397 } else {
398 LaneVGPR = SpillVGPRs.back();
399 }
400
401 SGPRSpillsToVirtualVGPRLanes[FI].emplace_back(args&: LaneVGPR, args&: LaneIndex);
402 return true;
403}
404
405bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
406 MachineFunction &MF, int FI, unsigned LaneIndex, bool IsPrologEpilog) {
407 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
408 const SIRegisterInfo *TRI = ST.getRegisterInfo();
409 MachineRegisterInfo &MRI = MF.getRegInfo();
410 Register LaneVGPR;
411 if (!LaneIndex) {
412 // Find the highest available register if called before RA to ensure the
413 // lowest registers are available for allocation. The LaneVGPR, in that
414 // case, will be shifted back to the lowest range after VGPR allocation.
415 LaneVGPR = TRI->findUnusedRegister(MRI, RC: &AMDGPU::VGPR_32RegClass, MF,
416 ReserveHighestVGPR: !IsPrologEpilog);
417 if (LaneVGPR == AMDGPU::NoRegister) {
418 // We have no VGPRs left for spilling SGPRs. Reset because we will not
419 // partially spill the SGPR to VGPRs.
420 SGPRSpillsToPhysicalVGPRLanes.erase(Val: FI);
421 return false;
422 }
423
424 if (IsPrologEpilog)
425 allocateWWMSpill(MF, VGPR: LaneVGPR);
426
427 reserveWWMRegister(Reg: LaneVGPR);
428 for (MachineBasicBlock &MBB : MF) {
429 MBB.addLiveIn(PhysReg: LaneVGPR);
430 MBB.sortUniqueLiveIns();
431 }
432 SpillPhysVGPRs.push_back(Elt: LaneVGPR);
433 } else {
434 LaneVGPR = SpillPhysVGPRs.back();
435 }
436
437 SGPRSpillsToPhysicalVGPRLanes[FI].emplace_back(args&: LaneVGPR, args&: LaneIndex);
438 return true;
439}
440
441bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(
442 MachineFunction &MF, int FI, bool SpillToPhysVGPRLane,
443 bool IsPrologEpilog) {
444 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
445 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]
446 : SGPRSpillsToVirtualVGPRLanes[FI];
447
448 // This has already been allocated.
449 if (!SpillLanes.empty())
450 return true;
451
452 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
453 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
454 unsigned WaveSize = ST.getWavefrontSize();
455
456 unsigned Size = FrameInfo.getObjectSize(ObjectIdx: FI);
457 unsigned NumLanes = Size / 4;
458
459 if (NumLanes > WaveSize)
460 return false;
461
462 assert(Size >= 4 && "invalid sgpr spill size");
463 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
464 "not spilling SGPRs to VGPRs");
465
466 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes
467 : NumVirtualVGPRSpillLanes;
468
469 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
470 unsigned LaneIndex = (NumSpillLanes % WaveSize);
471
472 bool Allocated = SpillToPhysVGPRLane
473 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,
474 IsPrologEpilog)
475 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
476 if (!Allocated) {
477 NumSpillLanes -= I;
478 return false;
479 }
480 }
481
482 return true;
483}
484
485/// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
486/// Either AGPR is spilled to VGPR to vice versa.
487/// Returns true if a \p FI can be eliminated completely.
488bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
489 int FI,
490 bool isAGPRtoVGPR) {
491 MachineRegisterInfo &MRI = MF.getRegInfo();
492 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
493 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
494
495 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
496
497 auto &Spill = VGPRToAGPRSpills[FI];
498
499 // This has already been allocated.
500 if (!Spill.Lanes.empty())
501 return Spill.FullyAllocated;
502
503 unsigned Size = FrameInfo.getObjectSize(ObjectIdx: FI);
504 unsigned NumLanes = Size / 4;
505 Spill.Lanes.resize(N: NumLanes, NV: AMDGPU::NoRegister);
506
507 const TargetRegisterClass &RC =
508 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
509 auto Regs = RC.getRegisters();
510
511 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
512 const SIRegisterInfo *TRI = ST.getRegisterInfo();
513 Spill.FullyAllocated = true;
514
515 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
516 // once.
517 BitVector OtherUsedRegs;
518 OtherUsedRegs.resize(N: TRI->getNumRegs());
519
520 const uint32_t *CSRMask =
521 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
522 if (CSRMask)
523 OtherUsedRegs.setBitsInMask(Mask: CSRMask);
524
525 // TODO: Should include register tuples, but doesn't matter with current
526 // usage.
527 for (MCPhysReg Reg : SpillAGPR)
528 OtherUsedRegs.set(Reg);
529 for (MCPhysReg Reg : SpillVGPR)
530 OtherUsedRegs.set(Reg);
531
532 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
533 for (int I = NumLanes - 1; I >= 0; --I) {
534 NextSpillReg = std::find_if(
535 first: NextSpillReg, last: Regs.end(), pred: [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
536 return MRI.isAllocatable(PhysReg: Reg) && !MRI.isPhysRegUsed(PhysReg: Reg) &&
537 !OtherUsedRegs[Reg];
538 });
539
540 if (NextSpillReg == Regs.end()) { // Registers exhausted
541 Spill.FullyAllocated = false;
542 break;
543 }
544
545 OtherUsedRegs.set(*NextSpillReg);
546 SpillRegs.push_back(Elt: *NextSpillReg);
547 MRI.reserveReg(PhysReg: *NextSpillReg, TRI);
548 Spill.Lanes[I] = *NextSpillReg++;
549 }
550
551 return Spill.FullyAllocated;
552}
553
554bool SIMachineFunctionInfo::removeDeadFrameIndices(
555 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
556 // Remove dead frame indices from function frame, however keep FP & BP since
557 // spills for them haven't been inserted yet. And also make sure to remove the
558 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure,
559 // otherwise, it could result in an unexpected side effect and bug, in case of
560 // any re-mapping of freed frame indices by later pass(es) like "stack slot
561 // coloring".
562 for (auto &R : SGPRSpillsToVirtualVGPRLanes)
563 MFI.RemoveStackObject(ObjectIdx: R.first);
564 SGPRSpillsToVirtualVGPRLanes.clear();
565
566 // Remove the dead frame indices of CSR SGPRs which are spilled to physical
567 // VGPR lanes during SILowerSGPRSpills pass.
568 if (!ResetSGPRSpillStackIDs) {
569 for (auto &R : SGPRSpillsToPhysicalVGPRLanes)
570 MFI.RemoveStackObject(ObjectIdx: R.first);
571 SGPRSpillsToPhysicalVGPRLanes.clear();
572 }
573 bool HaveSGPRToMemory = false;
574
575 if (ResetSGPRSpillStackIDs) {
576 // All other SGPRs must be allocated on the default stack, so reset the
577 // stack ID.
578 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
579 ++I) {
580 if (!checkIndexInPrologEpilogSGPRSpills(FI: I)) {
581 if (MFI.getStackID(ObjectIdx: I) == TargetStackID::SGPRSpill) {
582 MFI.setStackID(ObjectIdx: I, ID: TargetStackID::Default);
583 HaveSGPRToMemory = true;
584 }
585 }
586 }
587 }
588
589 for (auto &R : VGPRToAGPRSpills) {
590 if (R.second.IsDead)
591 MFI.RemoveStackObject(ObjectIdx: R.first);
592 }
593
594 return HaveSGPRToMemory;
595}
596
597int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI,
598 const SIRegisterInfo &TRI) {
599 if (ScavengeFI)
600 return *ScavengeFI;
601
602 ScavengeFI =
603 MFI.CreateStackObject(Size: TRI.getSpillSize(RC: AMDGPU::SGPR_32RegClass),
604 Alignment: TRI.getSpillAlign(RC: AMDGPU::SGPR_32RegClass), isSpillSlot: false);
605 return *ScavengeFI;
606}
607
608MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
609 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
610 return AMDGPU::SGPR0 + NumUserSGPRs;
611}
612
613MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
614 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
615}
616
617void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {
618 VRegFlags.grow(N: Reg);
619}
620
621void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,
622 Register SrcReg) {
623 VRegFlags.grow(N: NewReg);
624 VRegFlags[NewReg] = VRegFlags[SrcReg];
625}
626
627Register
628SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const {
629 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
630 if (!ST.isAmdPalOS())
631 return Register();
632 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
633 if (ST.hasMergedShaders()) {
634 switch (MF.getFunction().getCallingConv()) {
635 case CallingConv::AMDGPU_HS:
636 case CallingConv::AMDGPU_GS:
637 // Low GIT address is passed in s8 rather than s0 for an LS+HS or
638 // ES+GS merged shader on gfx9+.
639 GitPtrLo = AMDGPU::SGPR8;
640 return GitPtrLo;
641 default:
642 return GitPtrLo;
643 }
644 }
645 return GitPtrLo;
646}
647
648static yaml::StringValue regToString(Register Reg,
649 const TargetRegisterInfo &TRI) {
650 yaml::StringValue Dest;
651 {
652 raw_string_ostream OS(Dest.Value);
653 OS << printReg(Reg, TRI: &TRI);
654 }
655 return Dest;
656}
657
658static std::optional<yaml::SIArgumentInfo>
659convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
660 const TargetRegisterInfo &TRI) {
661 yaml::SIArgumentInfo AI;
662
663 auto convertArg = [&](std::optional<yaml::SIArgument> &A,
664 const ArgDescriptor &Arg) {
665 if (!Arg)
666 return false;
667
668 // Create a register or stack argument.
669 yaml::SIArgument SA = yaml::SIArgument::createArgument(IsReg: Arg.isRegister());
670 if (Arg.isRegister()) {
671 raw_string_ostream OS(SA.RegisterName.Value);
672 OS << printReg(Reg: Arg.getRegister(), TRI: &TRI);
673 } else
674 SA.StackOffset = Arg.getStackOffset();
675 // Check and update the optional mask.
676 if (Arg.isMasked())
677 SA.Mask = Arg.getMask();
678
679 A = std::move(SA);
680 return true;
681 };
682
683 bool Any = false;
684 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
685 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
686 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
687 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
688 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
689 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
690 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
691 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
692 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
693 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
694 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
695 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
696 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
697 ArgInfo.PrivateSegmentWaveByteOffset);
698 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
699 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
700 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
701 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
702 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
703
704 // Write FirstKernArgPreloadReg separately, since it's a Register,
705 // not ArgDescriptor.
706 if (ArgInfo.FirstKernArgPreloadReg) {
707 Register Reg = ArgInfo.FirstKernArgPreloadReg;
708 assert(Reg.isPhysical() &&
709 "FirstKernArgPreloadReg must be a physical register");
710
711 yaml::SIArgument SA = yaml::SIArgument::createArgument(IsReg: true);
712 raw_string_ostream OS(SA.RegisterName.Value);
713 OS << printReg(Reg, TRI: &TRI);
714
715 AI.FirstKernArgPreloadReg = SA;
716 Any = true;
717 }
718
719 if (Any)
720 return AI;
721
722 return std::nullopt;
723}
724
725yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
726 const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI,
727 const llvm::MachineFunction &MF)
728 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
729 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
730 GDSSize(MFI.getGDSSize()), DynLDSAlign(MFI.getDynLDSAlign()),
731 IsEntryFunction(MFI.isEntryFunction()), MemoryBound(MFI.isMemoryBound()),
732 WaveLimiter(MFI.needsWaveLimiter()),
733 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
734 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
735 HasNoWWMPoolSGPRSpillFallback(MFI.hasNoWWMPoolSGPRSpillFallback()),
736 NumWaveDispatchSGPRs(MFI.getNumWaveDispatchSGPRs()),
737 NumWaveDispatchVGPRs(MFI.getNumWaveDispatchVGPRs()),
738 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
739 Occupancy(MFI.getOccupancy()),
740 ScratchRSrcReg(regToString(Reg: MFI.getScratchRSrcReg(), TRI)),
741 FrameOffsetReg(regToString(Reg: MFI.getFrameOffsetReg(), TRI)),
742 StackPtrOffsetReg(regToString(Reg: MFI.getStackPtrOffsetReg(), TRI)),
743 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
744 ReturnsVoid(MFI.returnsVoid()),
745 ArgInfo(convertArgumentInfo(ArgInfo: MFI.getArgInfo(), TRI)),
746 PSInputAddr(MFI.getPSInputAddr()), PSInputEnable(MFI.getPSInputEnable()),
747 MaxMemoryClusterDWords(MFI.getMaxMemoryClusterDWords()),
748 Mode(MFI.getMode()), HasInitWholeWave(MFI.hasInitWholeWave()),
749 IsWholeWaveFunction(MFI.isWholeWaveFunction()),
750 DynamicVGPRBlockSize(MFI.getDynamicVGPRBlockSize()),
751 ScratchReservedForDynamicVGPRs(MFI.getScratchReservedForDynamicVGPRs()),
752 NumKernargPreloadSGPRs(MFI.getNumKernargPreloadedSGPRs()),
753 MinNumAGPRs(MFI.getMinNumAGPRs()) {
754 for (Register Reg : MFI.getSGPRSpillPhysVGPRs())
755 SpillPhysVGPRS.push_back(Elt: regToString(Reg, TRI));
756
757 for (Register Reg : MFI.getWWMReservedRegs())
758 WWMReservedRegs.push_back(Elt: regToString(Reg, TRI));
759
760 if (MFI.getLongBranchReservedReg())
761 LongBranchReservedReg = regToString(Reg: MFI.getLongBranchReservedReg(), TRI);
762 if (MFI.getVGPRForAGPRCopy())
763 VGPRForAGPRCopy = regToString(Reg: MFI.getVGPRForAGPRCopy(), TRI);
764
765 if (MFI.getSGPRForEXECCopy())
766 SGPRForEXECCopy = regToString(Reg: MFI.getSGPRForEXECCopy(), TRI);
767
768 auto SFI = MFI.getOptionalScavengeFI();
769 if (SFI)
770 ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo());
771}
772
773void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
774 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, MFI&: *this);
775}
776
777bool SIMachineFunctionInfo::initializeBaseYamlFields(
778 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
779 PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) {
780 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
781 MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
782 LDSSize = YamlMFI.LDSSize;
783 GDSSize = YamlMFI.GDSSize;
784 DynLDSAlign = YamlMFI.DynLDSAlign;
785 PSInputAddr = YamlMFI.PSInputAddr;
786 PSInputEnable = YamlMFI.PSInputEnable;
787 MaxMemoryClusterDWords = YamlMFI.MaxMemoryClusterDWords;
788 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
789 Occupancy = YamlMFI.Occupancy;
790 IsEntryFunction = YamlMFI.IsEntryFunction;
791 MemoryBound = YamlMFI.MemoryBound;
792 WaveLimiter = YamlMFI.WaveLimiter;
793 HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
794 HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
795 HasNoWWMPoolSGPRSpillFallback = YamlMFI.HasNoWWMPoolSGPRSpillFallback;
796 NumWaveDispatchSGPRs = YamlMFI.NumWaveDispatchSGPRs;
797 NumWaveDispatchVGPRs = YamlMFI.NumWaveDispatchVGPRs;
798 BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
799 ReturnsVoid = YamlMFI.ReturnsVoid;
800 IsWholeWaveFunction = YamlMFI.IsWholeWaveFunction;
801 MinNumAGPRs = YamlMFI.MinNumAGPRs;
802 // This can also be set by the function attribute, MFI has higher precedence
803 // though.
804 if (YamlMFI.DynamicVGPRBlockSize != std::nullopt)
805 DynamicVGPRBlockSize = *YamlMFI.DynamicVGPRBlockSize;
806
807 UserSGPRInfo.allocKernargPreloadSGPRs(NumSGPRs: YamlMFI.NumKernargPreloadSGPRs);
808
809 if (YamlMFI.ScavengeFI) {
810 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MFI: MF.getFrameInfo());
811 if (!FIOrErr) {
812 // Create a diagnostic for a the frame index.
813 const MemoryBuffer &Buffer =
814 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
815
816 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
817 SourceMgr::DK_Error, toString(E: FIOrErr.takeError()),
818 "", {}, {});
819 SourceRange = YamlMFI.ScavengeFI->SourceRange;
820 return true;
821 }
822 ScavengeFI = *FIOrErr;
823 } else {
824 ScavengeFI = std::nullopt;
825 }
826 return false;
827}
828
829bool SIMachineFunctionInfo::mayUseAGPRs(const Function &F) const {
830 auto [MinNumAGPR, MaxNumAGPR] =
831 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-agpr-alloc", Default: {~0u, ~0u},
832 /*OnlyFirstRequired=*/true);
833 return MinNumAGPR != 0u;
834}
835