1//===- SIMachineFunctionInfo.cpp - SI Machine Function Info ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "SIMachineFunctionInfo.h"
10#include "AMDGPUSubtarget.h"
11#include "GCNSubtarget.h"
12#include "SIRegisterInfo.h"
13#include "Utils/AMDGPUBaseInfo.h"
14#include "llvm/CodeGen/LiveIntervals.h"
15#include "llvm/CodeGen/MIRParser/MIParser.h"
16#include "llvm/CodeGen/MachineBasicBlock.h"
17#include "llvm/CodeGen/MachineFrameInfo.h"
18#include "llvm/CodeGen/MachineFunction.h"
19#include "llvm/CodeGen/MachineRegisterInfo.h"
20#include "llvm/IR/CallingConv.h"
21#include "llvm/IR/DiagnosticInfo.h"
22#include "llvm/IR/Function.h"
23#include <cassert>
24#include <optional>
25
26enum { MAX_LANES = 64 };
27
28using namespace llvm;
29
30// TODO -- delete this flag once we have more robust mechanisms to allocate the
31// optimal RC for Opc and Dest of MFMA. In particular, there are high RP cases
32// where it is better to produce the VGPR form (e.g. if there are VGPR users
33// of the MFMA result).
34static cl::opt<bool, true> MFMAVGPRFormOpt(
35 "amdgpu-mfma-vgpr-form",
36 cl::desc("Whether to force use VGPR for Opc and Dest of MFMA. If "
37 "unspecified, default to compiler heuristics"),
38 cl::location(L&: SIMachineFunctionInfo::MFMAVGPRForm), cl::init(Val: true),
39 cl::Hidden);
40
41const GCNTargetMachine &getTM(const GCNSubtarget *STI) {
42 const SITargetLowering *TLI = STI->getTargetLowering();
43 return static_cast<const GCNTargetMachine &>(TLI->getTargetMachine());
44}
45
46bool SIMachineFunctionInfo::MFMAVGPRForm = false;
47
48SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,
49 const GCNSubtarget *STI)
50 : AMDGPUMachineFunctionInfo(F, *STI), Mode(F, *STI),
51 GWSResourcePSV(getTM(STI)), UserSGPRInfo(F, *STI), WorkGroupIDX(false),
52 WorkGroupIDY(false), WorkGroupIDZ(false), WorkGroupInfo(false),
53 LDSKernelId(false), PrivateSegmentWaveByteOffset(false),
54 WorkItemIDX(false), WorkItemIDY(false), WorkItemIDZ(false),
55 ImplicitArgPtr(false), GITPtrHigh(0xffffffff), HighBitsOf32BitAddress(0),
56 IsWholeWaveFunction(F.getCallingConv() ==
57 CallingConv::AMDGPU_Gfx_WholeWave) {
58 const GCNSubtarget &ST = *STI;
59 FlatWorkGroupSizes = ST.getFlatWorkGroupSizes(F);
60 WavesPerEU = ST.getWavesPerEU(F);
61 MaxNumWorkGroups = AMDGPU::getMaxNumWorkGroups(F);
62 assert(MaxNumWorkGroups.size() == 3);
63
64 DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
65 Occupancy = ST.computeOccupancy(F, LDSSize: getLDSSize()).second;
66 CallingConv::ID CC = F.getCallingConv();
67
68 VRegFlags.reserve(S: 1024);
69
70 const bool IsKernel = CC == CallingConv::AMDGPU_KERNEL ||
71 CC == CallingConv::SPIR_KERNEL;
72
73 if (IsKernel) {
74 WorkGroupIDX = true;
75 WorkItemIDX = true;
76 } else if (CC == CallingConv::AMDGPU_PS) {
77 PSInputAddr = AMDGPU::getInitialPSInputAddr(F);
78 }
79
80 if (ST.hasGFX90AInsts()) {
81 // FIXME: Extract logic out of getMaxNumVectorRegs; we need to apply the
82 // allocation granule and clamping.
83 auto [MinNumAGPRAttr, MaxNumAGPRAttr] =
84 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-agpr-alloc", Default: {~0u, ~0u},
85 /*OnlyFirstRequired=*/true);
86 MinNumAGPRs = MinNumAGPRAttr;
87 }
88
89 if (!isEntryFunction()) {
90 if (CC != CallingConv::AMDGPU_Gfx &&
91 CC != CallingConv::AMDGPU_Gfx_WholeWave)
92 ArgInfo = AMDGPUFunctionArgInfo::FixedABIFunctionInfo;
93
94 FrameOffsetReg = AMDGPU::SGPR33;
95 StackPtrOffsetReg = AMDGPU::SGPR32;
96
97 if (!ST.hasFlatScratchEnabled()) {
98 // Non-entry functions have no special inputs for now, other registers
99 // required for scratch access.
100 ScratchRSrcReg = AMDGPUFunctionArgInfo::getPrivateSegmentBufferReg(CC);
101
102 ArgInfo.PrivateSegmentBuffer =
103 ArgDescriptor::createRegister(Reg: ScratchRSrcReg);
104 }
105
106 if (!F.hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr") &&
107 !AMDGPU::isChainCC(CC))
108 ImplicitArgPtr = true;
109 } else {
110 ImplicitArgPtr = false;
111 MaxKernArgAlign =
112 std::max(a: ST.getAlignmentForImplicitArgPtr(), b: MaxKernArgAlign);
113 }
114
115 if (!AMDGPU::isGraphics(CC) ||
116 ((CC == CallingConv::AMDGPU_CS || CC == CallingConv::AMDGPU_Gfx) &&
117 ST.hasArchitectedSGPRs())) {
118 if (IsKernel || !F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-x") ||
119 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-x"))
120 WorkGroupIDX = true;
121
122 if (!F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-y") ||
123 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-y"))
124 WorkGroupIDY = true;
125
126 if (!F.hasFnAttribute(Kind: "amdgpu-no-workgroup-id-z") ||
127 !F.hasFnAttribute(Kind: "amdgpu-no-cluster-id-z"))
128 WorkGroupIDZ = true;
129 }
130
131 if (!AMDGPU::isGraphics(CC)) {
132 if (IsKernel || !F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-x"))
133 WorkItemIDX = true;
134
135 if (!F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-y") &&
136 ST.getMaxWorkitemID(Kernel: F, Dimension: 1) != 0)
137 WorkItemIDY = true;
138
139 if (!F.hasFnAttribute(Kind: "amdgpu-no-workitem-id-z") &&
140 ST.getMaxWorkitemID(Kernel: F, Dimension: 2) != 0)
141 WorkItemIDZ = true;
142
143 if (!IsKernel && !F.hasFnAttribute(Kind: "amdgpu-no-lds-kernel-id"))
144 LDSKernelId = true;
145 }
146
147 if (isEntryFunction()) {
148 // X, XY, and XYZ are the only supported combinations, so make sure Y is
149 // enabled if Z is.
150 if (WorkItemIDZ)
151 WorkItemIDY = true;
152
153 if (!ST.hasArchitectedFlatScratch()) {
154 PrivateSegmentWaveByteOffset = true;
155
156 // HS and GS always have the scratch wave offset in SGPR5 on GFX9.
157 if (ST.getGeneration() >= AMDGPUSubtarget::GFX9 &&
158 (CC == CallingConv::AMDGPU_HS || CC == CallingConv::AMDGPU_GS))
159 ArgInfo.PrivateSegmentWaveByteOffset =
160 ArgDescriptor::createRegister(Reg: AMDGPU::SGPR5);
161 }
162 }
163
164 Attribute A = F.getFnAttribute(Kind: "amdgpu-git-ptr-high");
165 StringRef S = A.getValueAsString();
166 if (!S.empty())
167 S.consumeInteger(Radix: 0, Result&: GITPtrHigh);
168
169 A = F.getFnAttribute(Kind: "amdgpu-32bit-address-high-bits");
170 S = A.getValueAsString();
171 if (!S.empty())
172 S.consumeInteger(Radix: 0, Result&: HighBitsOf32BitAddress);
173
174 MaxMemoryClusterDWords = F.getFnAttributeAsParsedInteger(
175 Kind: "amdgpu-max-memory-cluster-dwords", Default: DefaultMemoryClusterDWordsLimit);
176
177 // On GFX908, in order to guarantee copying between AGPRs, we need a scratch
178 // VGPR available at all times. For now, reserve highest available VGPR. After
179 // RA, shift it to the lowest available unused VGPR if the one exist.
180 if (ST.hasMAIInsts() && !ST.hasGFX90AInsts()) {
181 VGPRForAGPRCopy =
182 AMDGPU::VGPR_32RegClass.getRegister(i: ST.getMaxNumVGPRs(F) - 1);
183 }
184
185 ClusterDims = AMDGPU::ClusterDimsAttr::get(F);
186}
187
188MachineFunctionInfo *SIMachineFunctionInfo::clone(
189 BumpPtrAllocator &Allocator, MachineFunction &DestMF,
190 const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB)
191 const {
192 return DestMF.cloneInfo<SIMachineFunctionInfo>(Old: *this);
193}
194
195void SIMachineFunctionInfo::limitOccupancy(const MachineFunction &MF) {
196 limitOccupancy(Limit: getMaxWavesPerEU());
197 const GCNSubtarget& ST = MF.getSubtarget<GCNSubtarget>();
198 limitOccupancy(Limit: ST.getOccupancyWithWorkGroupSizes(MF).second);
199}
200
201Register SIMachineFunctionInfo::addPrivateSegmentBuffer(
202 const SIRegisterInfo &TRI) {
203 ArgInfo.PrivateSegmentBuffer =
204 ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
205 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SGPR_128RegClass));
206 NumUserSGPRs += 4;
207 return ArgInfo.PrivateSegmentBuffer.getRegister();
208}
209
210Register SIMachineFunctionInfo::addDispatchPtr(const SIRegisterInfo &TRI) {
211 ArgInfo.DispatchPtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
212 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
213 NumUserSGPRs += 2;
214 return ArgInfo.DispatchPtr.getRegister();
215}
216
217Register SIMachineFunctionInfo::addQueuePtr(const SIRegisterInfo &TRI) {
218 ArgInfo.QueuePtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
219 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
220 NumUserSGPRs += 2;
221 return ArgInfo.QueuePtr.getRegister();
222}
223
224Register SIMachineFunctionInfo::addKernargSegmentPtr(const SIRegisterInfo &TRI) {
225 ArgInfo.KernargSegmentPtr
226 = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
227 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
228 NumUserSGPRs += 2;
229 return ArgInfo.KernargSegmentPtr.getRegister();
230}
231
232Register SIMachineFunctionInfo::addDispatchID(const SIRegisterInfo &TRI) {
233 ArgInfo.DispatchID = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
234 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
235 NumUserSGPRs += 2;
236 return ArgInfo.DispatchID.getRegister();
237}
238
239Register SIMachineFunctionInfo::addFlatScratchInit(const SIRegisterInfo &TRI) {
240 ArgInfo.FlatScratchInit = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
241 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
242 NumUserSGPRs += 2;
243 return ArgInfo.FlatScratchInit.getRegister();
244}
245
246Register SIMachineFunctionInfo::addPrivateSegmentSize(const SIRegisterInfo &TRI) {
247 ArgInfo.PrivateSegmentSize = ArgDescriptor::createRegister(Reg: getNextUserSGPR());
248 NumUserSGPRs += 1;
249 return ArgInfo.PrivateSegmentSize.getRegister();
250}
251
252Register SIMachineFunctionInfo::addImplicitBufferPtr(const SIRegisterInfo &TRI) {
253 ArgInfo.ImplicitBufferPtr = ArgDescriptor::createRegister(Reg: TRI.getMatchingSuperReg(
254 Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC: &AMDGPU::SReg_64RegClass));
255 NumUserSGPRs += 2;
256 return ArgInfo.ImplicitBufferPtr.getRegister();
257}
258
259Register SIMachineFunctionInfo::addLDSKernelId() {
260 ArgInfo.LDSKernelId = ArgDescriptor::createRegister(Reg: getNextUserSGPR());
261 NumUserSGPRs += 1;
262 return ArgInfo.LDSKernelId.getRegister();
263}
264
265SmallVectorImpl<MCRegister> *SIMachineFunctionInfo::addPreloadedKernArg(
266 const SIRegisterInfo &TRI, const TargetRegisterClass *RC,
267 unsigned AllocSizeDWord, int KernArgIdx, int PaddingSGPRs) {
268 auto [It, Inserted] = ArgInfo.PreloadKernArgs.try_emplace(Key: KernArgIdx);
269 assert(Inserted && "Preload kernel argument allocated twice.");
270 NumUserSGPRs += PaddingSGPRs;
271 // If the available register tuples are aligned with the kernarg to be
272 // preloaded use that register, otherwise we need to use a set of SGPRs and
273 // merge them.
274 if (!ArgInfo.FirstKernArgPreloadReg)
275 ArgInfo.FirstKernArgPreloadReg = getNextUserSGPR();
276 Register PreloadReg =
277 TRI.getMatchingSuperReg(Reg: getNextUserSGPR(), SubIdx: AMDGPU::sub0, RC);
278 auto &Regs = It->second.Regs;
279 if (PreloadReg &&
280 (RC == &AMDGPU::SReg_32RegClass || RC == &AMDGPU::SReg_64RegClass)) {
281 Regs.push_back(Elt: PreloadReg);
282 NumUserSGPRs += AllocSizeDWord;
283 } else {
284 Regs.reserve(N: AllocSizeDWord);
285 for (unsigned I = 0; I < AllocSizeDWord; ++I) {
286 Regs.push_back(Elt: getNextUserSGPR());
287 NumUserSGPRs++;
288 }
289 }
290
291 // Track the actual number of SGPRs that HW will preload to.
292 UserSGPRInfo.allocKernargPreloadSGPRs(NumSGPRs: AllocSizeDWord + PaddingSGPRs);
293 return &Regs;
294}
295
296void SIMachineFunctionInfo::allocateWWMSpill(MachineFunction &MF, Register VGPR,
297 uint64_t Size, Align Alignment) {
298 // Skip if it is an entry function or the register is already added.
299 if (isEntryFunction() || WWMSpills.count(Key: VGPR))
300 return;
301
302 // Skip if this is a function with the amdgpu_cs_chain or
303 // amdgpu_cs_chain_preserve calling convention and this is a scratch register.
304 // We never need to allocate a spill for these because we don't even need to
305 // restore the inactive lanes for them (they're scratchier than the usual
306 // scratch registers). We only need to do this if we have calls to
307 // llvm.amdgcn.cs.chain (otherwise there's no one to save them for, since
308 // chain functions do not return) and the function did not contain a call to
309 // llvm.amdgcn.init.whole.wave (since in that case there are no inactive lanes
310 // when entering the function).
311 if (isChainFunction() &&
312 (SIRegisterInfo::isChainScratchRegister(VGPR) ||
313 !MF.getFrameInfo().hasTailCall() || hasInitWholeWave()))
314 return;
315
316 WWMSpills.insert(KV: std::make_pair(
317 x&: VGPR, y: MF.getFrameInfo().CreateSpillStackObject(Size, Alignment)));
318}
319
320// Separate out the callee-saved and scratch registers.
321void SIMachineFunctionInfo::splitWWMSpillRegisters(
322 MachineFunction &MF,
323 SmallVectorImpl<std::pair<Register, int>> &CalleeSavedRegs,
324 SmallVectorImpl<std::pair<Register, int>> &ScratchRegs) const {
325 const MCPhysReg *CSRegs = MF.getRegInfo().getCalleeSavedRegs();
326 for (auto &Reg : WWMSpills) {
327 if (isCalleeSavedReg(CSRegs, Reg: Reg.first))
328 CalleeSavedRegs.push_back(Elt: Reg);
329 else
330 ScratchRegs.push_back(Elt: Reg);
331 }
332}
333
334bool SIMachineFunctionInfo::isCalleeSavedReg(const MCPhysReg *CSRegs,
335 MCPhysReg Reg) const {
336 for (unsigned I = 0; CSRegs[I]; ++I) {
337 if (CSRegs[I] == Reg)
338 return true;
339 }
340
341 return false;
342}
343
344void SIMachineFunctionInfo::shiftWwmVGPRsToLowestRange(
345 MachineFunction &MF, SmallVectorImpl<Register> &WWMVGPRs,
346 BitVector &SavedVGPRs) {
347 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
348 MachineRegisterInfo &MRI = MF.getRegInfo();
349 for (unsigned I = 0, E = WWMVGPRs.size(); I < E; ++I) {
350 Register Reg = WWMVGPRs[I];
351 Register NewReg =
352 TRI->findUnusedRegister(MRI, RC: &AMDGPU::VGPR_32RegClass, MF);
353 if (!NewReg || NewReg >= Reg)
354 break;
355
356 MRI.replaceRegWith(FromReg: Reg, ToReg: NewReg);
357
358 // Update various tables with the new VGPR.
359 WWMVGPRs[I] = NewReg;
360 WWMReservedRegs.remove(X: Reg);
361 WWMReservedRegs.insert(X: NewReg);
362 MRI.reserveReg(PhysReg: NewReg, TRI);
363
364 // Replace the register in SpillPhysVGPRs. This is needed to look for free
365 // lanes while spilling special SGPRs like FP, BP, etc. during PEI.
366 auto *RegItr = llvm::find(Range&: SpillPhysVGPRs, Val: Reg);
367 if (RegItr != SpillPhysVGPRs.end()) {
368 unsigned Idx = std::distance(first: SpillPhysVGPRs.begin(), last: RegItr);
369 SpillPhysVGPRs[Idx] = NewReg;
370
371 // For replacing registers used in the CFI instructions.
372 MF.replaceFrameInstRegister(From: Reg, To: NewReg);
373 }
374
375 // The generic `determineCalleeSaves` might have set the old register if it
376 // is in the CSR range.
377 SavedVGPRs.reset(Idx: Reg);
378
379 for (MachineBasicBlock &MBB : MF) {
380 MBB.removeLiveIn(Reg);
381 MBB.sortUniqueLiveIns();
382 }
383
384 Reg = NewReg;
385 }
386}
387
388bool SIMachineFunctionInfo::allocateVirtualVGPRForSGPRSpills(
389 MachineFunction &MF, int FI, unsigned LaneIndex) {
390 MachineRegisterInfo &MRI = MF.getRegInfo();
391 Register LaneVGPR;
392 if (!LaneIndex) {
393 LaneVGPR = MRI.createVirtualRegister(RegClass: &AMDGPU::VGPR_32RegClass);
394 SpillVGPRs.push_back(Elt: LaneVGPR);
395 } else {
396 LaneVGPR = SpillVGPRs.back();
397 }
398
399 SGPRSpillsToVirtualVGPRLanes[FI].emplace_back(args&: LaneVGPR, args&: LaneIndex);
400 return true;
401}
402
403bool SIMachineFunctionInfo::allocatePhysicalVGPRForSGPRSpills(
404 MachineFunction &MF, int FI, unsigned LaneIndex, bool IsPrologEpilog) {
405 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
406 const SIRegisterInfo *TRI = ST.getRegisterInfo();
407 MachineRegisterInfo &MRI = MF.getRegInfo();
408 Register LaneVGPR;
409 if (!LaneIndex) {
410 // Find the highest available register if called before RA to ensure the
411 // lowest registers are available for allocation. The LaneVGPR, in that
412 // case, will be shifted back to the lowest range after VGPR allocation.
413 LaneVGPR = TRI->findUnusedRegister(MRI, RC: &AMDGPU::VGPR_32RegClass, MF,
414 ReserveHighestVGPR: !IsPrologEpilog);
415 if (LaneVGPR == AMDGPU::NoRegister) {
416 // We have no VGPRs left for spilling SGPRs. Reset because we will not
417 // partially spill the SGPR to VGPRs.
418 SGPRSpillsToPhysicalVGPRLanes.erase(Val: FI);
419 return false;
420 }
421
422 if (IsPrologEpilog)
423 allocateWWMSpill(MF, VGPR: LaneVGPR);
424
425 reserveWWMRegister(Reg: LaneVGPR);
426 for (MachineBasicBlock &MBB : MF) {
427 MBB.addLiveIn(PhysReg: LaneVGPR);
428 MBB.sortUniqueLiveIns();
429 }
430 SpillPhysVGPRs.push_back(Elt: LaneVGPR);
431 } else {
432 LaneVGPR = SpillPhysVGPRs.back();
433 }
434
435 SGPRSpillsToPhysicalVGPRLanes[FI].emplace_back(args&: LaneVGPR, args&: LaneIndex);
436 return true;
437}
438
439bool SIMachineFunctionInfo::allocateSGPRSpillToVGPRLane(
440 MachineFunction &MF, int FI, bool SpillToPhysVGPRLane,
441 bool IsPrologEpilog) {
442 std::vector<SIRegisterInfo::SpilledReg> &SpillLanes =
443 SpillToPhysVGPRLane ? SGPRSpillsToPhysicalVGPRLanes[FI]
444 : SGPRSpillsToVirtualVGPRLanes[FI];
445
446 // This has already been allocated.
447 if (!SpillLanes.empty())
448 return true;
449
450 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
451 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
452 unsigned WaveSize = ST.getWavefrontSize();
453
454 unsigned Size = FrameInfo.getObjectSize(ObjectIdx: FI);
455 unsigned NumLanes = Size / 4;
456
457 if (NumLanes > WaveSize)
458 return false;
459
460 assert(Size >= 4 && "invalid sgpr spill size");
461 assert(ST.getRegisterInfo()->spillSGPRToVGPR() &&
462 "not spilling SGPRs to VGPRs");
463
464 unsigned &NumSpillLanes = SpillToPhysVGPRLane ? NumPhysicalVGPRSpillLanes
465 : NumVirtualVGPRSpillLanes;
466
467 for (unsigned I = 0; I < NumLanes; ++I, ++NumSpillLanes) {
468 unsigned LaneIndex = (NumSpillLanes % WaveSize);
469
470 bool Allocated = SpillToPhysVGPRLane
471 ? allocatePhysicalVGPRForSGPRSpills(MF, FI, LaneIndex,
472 IsPrologEpilog)
473 : allocateVirtualVGPRForSGPRSpills(MF, FI, LaneIndex);
474 if (!Allocated) {
475 NumSpillLanes -= I;
476 return false;
477 }
478 }
479
480 return true;
481}
482
483/// Reserve AGPRs or VGPRs to support spilling for FrameIndex \p FI.
484/// Either AGPR is spilled to VGPR to vice versa.
485/// Returns true if a \p FI can be eliminated completely.
486bool SIMachineFunctionInfo::allocateVGPRSpillToAGPR(MachineFunction &MF,
487 int FI,
488 bool isAGPRtoVGPR) {
489 MachineRegisterInfo &MRI = MF.getRegInfo();
490 MachineFrameInfo &FrameInfo = MF.getFrameInfo();
491 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
492
493 assert(ST.hasMAIInsts() && FrameInfo.isSpillSlotObjectIndex(FI));
494
495 auto &Spill = VGPRToAGPRSpills[FI];
496
497 // This has already been allocated.
498 if (!Spill.Lanes.empty())
499 return Spill.FullyAllocated;
500
501 unsigned Size = FrameInfo.getObjectSize(ObjectIdx: FI);
502 unsigned NumLanes = Size / 4;
503 Spill.Lanes.resize(N: NumLanes, NV: AMDGPU::NoRegister);
504
505 const TargetRegisterClass &RC =
506 isAGPRtoVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::AGPR_32RegClass;
507 auto Regs = RC.getRegisters();
508
509 auto &SpillRegs = isAGPRtoVGPR ? SpillAGPR : SpillVGPR;
510 const SIRegisterInfo *TRI = ST.getRegisterInfo();
511 Spill.FullyAllocated = true;
512
513 // FIXME: Move allocation logic out of MachineFunctionInfo and initialize
514 // once.
515 BitVector OtherUsedRegs;
516 OtherUsedRegs.resize(N: TRI->getNumRegs());
517
518 const uint32_t *CSRMask =
519 TRI->getCallPreservedMask(MF, MF.getFunction().getCallingConv());
520 if (CSRMask)
521 OtherUsedRegs.setBitsInMask(Mask: CSRMask);
522
523 // TODO: Should include register tuples, but doesn't matter with current
524 // usage.
525 for (MCPhysReg Reg : SpillAGPR)
526 OtherUsedRegs.set(Reg);
527 for (MCPhysReg Reg : SpillVGPR)
528 OtherUsedRegs.set(Reg);
529
530 SmallVectorImpl<MCPhysReg>::const_iterator NextSpillReg = Regs.begin();
531 for (int I = NumLanes - 1; I >= 0; --I) {
532 NextSpillReg = std::find_if(
533 first: NextSpillReg, last: Regs.end(), pred: [&MRI, &OtherUsedRegs](MCPhysReg Reg) {
534 return MRI.isAllocatable(PhysReg: Reg) && !MRI.isPhysRegUsed(PhysReg: Reg) &&
535 !OtherUsedRegs[Reg];
536 });
537
538 if (NextSpillReg == Regs.end()) { // Registers exhausted
539 Spill.FullyAllocated = false;
540 break;
541 }
542
543 OtherUsedRegs.set(*NextSpillReg);
544 SpillRegs.push_back(Elt: *NextSpillReg);
545 MRI.reserveReg(PhysReg: *NextSpillReg, TRI);
546 Spill.Lanes[I] = *NextSpillReg++;
547 }
548
549 return Spill.FullyAllocated;
550}
551
552bool SIMachineFunctionInfo::removeDeadFrameIndices(
553 MachineFrameInfo &MFI, bool ResetSGPRSpillStackIDs) {
554 // Remove dead frame indices from function frame, however keep FP & BP since
555 // spills for them haven't been inserted yet. And also make sure to remove the
556 // frame indices from `SGPRSpillsToVirtualVGPRLanes` data structure,
557 // otherwise, it could result in an unexpected side effect and bug, in case of
558 // any re-mapping of freed frame indices by later pass(es) like "stack slot
559 // coloring".
560 for (auto &R : SGPRSpillsToVirtualVGPRLanes)
561 MFI.RemoveStackObject(ObjectIdx: R.first);
562 SGPRSpillsToVirtualVGPRLanes.clear();
563
564 // Remove the dead frame indices of CSR SGPRs which are spilled to physical
565 // VGPR lanes during SILowerSGPRSpills pass.
566 if (!ResetSGPRSpillStackIDs) {
567 for (auto &R : SGPRSpillsToPhysicalVGPRLanes)
568 MFI.RemoveStackObject(ObjectIdx: R.first);
569 SGPRSpillsToPhysicalVGPRLanes.clear();
570 }
571 bool HaveSGPRToMemory = false;
572
573 if (ResetSGPRSpillStackIDs) {
574 // All other SGPRs must be allocated on the default stack, so reset the
575 // stack ID.
576 for (int I = MFI.getObjectIndexBegin(), E = MFI.getObjectIndexEnd(); I != E;
577 ++I) {
578 if (!checkIndexInPrologEpilogSGPRSpills(FI: I)) {
579 if (MFI.getStackID(ObjectIdx: I) == TargetStackID::SGPRSpill) {
580 MFI.setStackID(ObjectIdx: I, ID: TargetStackID::Default);
581 HaveSGPRToMemory = true;
582 }
583 }
584 }
585 }
586
587 for (auto &R : VGPRToAGPRSpills) {
588 if (R.second.IsDead)
589 MFI.RemoveStackObject(ObjectIdx: R.first);
590 }
591
592 return HaveSGPRToMemory;
593}
594
595int SIMachineFunctionInfo::getScavengeFI(MachineFrameInfo &MFI,
596 const SIRegisterInfo &TRI) {
597 if (ScavengeFI)
598 return *ScavengeFI;
599
600 ScavengeFI =
601 MFI.CreateStackObject(Size: TRI.getSpillSize(RC: AMDGPU::SGPR_32RegClass),
602 Alignment: TRI.getSpillAlign(RC: AMDGPU::SGPR_32RegClass), isSpillSlot: false);
603 return *ScavengeFI;
604}
605
606MCPhysReg SIMachineFunctionInfo::getNextUserSGPR() const {
607 assert(NumSystemSGPRs == 0 && "System SGPRs must be added after user SGPRs");
608 return AMDGPU::SGPR0 + NumUserSGPRs;
609}
610
611MCPhysReg SIMachineFunctionInfo::getNextSystemSGPR() const {
612 return AMDGPU::SGPR0 + NumUserSGPRs + NumSystemSGPRs;
613}
614
615void SIMachineFunctionInfo::MRI_NoteNewVirtualRegister(Register Reg) {
616 VRegFlags.grow(N: Reg);
617}
618
619void SIMachineFunctionInfo::MRI_NoteCloneVirtualRegister(Register NewReg,
620 Register SrcReg) {
621 VRegFlags.grow(N: NewReg);
622 VRegFlags[NewReg] = VRegFlags[SrcReg];
623}
624
625Register
626SIMachineFunctionInfo::getGITPtrLoReg(const MachineFunction &MF) const {
627 const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
628 if (!ST.isAmdPalOS())
629 return Register();
630 Register GitPtrLo = AMDGPU::SGPR0; // Low GIT address passed in
631 if (ST.hasMergedShaders()) {
632 switch (MF.getFunction().getCallingConv()) {
633 case CallingConv::AMDGPU_HS:
634 case CallingConv::AMDGPU_GS:
635 // Low GIT address is passed in s8 rather than s0 for an LS+HS or
636 // ES+GS merged shader on gfx9+.
637 GitPtrLo = AMDGPU::SGPR8;
638 return GitPtrLo;
639 default:
640 return GitPtrLo;
641 }
642 }
643 return GitPtrLo;
644}
645
646static yaml::StringValue regToString(Register Reg,
647 const TargetRegisterInfo &TRI) {
648 yaml::StringValue Dest;
649 {
650 raw_string_ostream OS(Dest.Value);
651 OS << printReg(Reg, TRI: &TRI);
652 }
653 return Dest;
654}
655
656static std::optional<yaml::SIArgumentInfo>
657convertArgumentInfo(const AMDGPUFunctionArgInfo &ArgInfo,
658 const TargetRegisterInfo &TRI) {
659 yaml::SIArgumentInfo AI;
660
661 auto convertArg = [&](std::optional<yaml::SIArgument> &A,
662 const ArgDescriptor &Arg) {
663 if (!Arg)
664 return false;
665
666 // Create a register or stack argument.
667 yaml::SIArgument SA = yaml::SIArgument::createArgument(IsReg: Arg.isRegister());
668 if (Arg.isRegister()) {
669 raw_string_ostream OS(SA.RegisterName.Value);
670 OS << printReg(Reg: Arg.getRegister(), TRI: &TRI);
671 } else
672 SA.StackOffset = Arg.getStackOffset();
673 // Check and update the optional mask.
674 if (Arg.isMasked())
675 SA.Mask = Arg.getMask();
676
677 A = std::move(SA);
678 return true;
679 };
680
681 bool Any = false;
682 Any |= convertArg(AI.PrivateSegmentBuffer, ArgInfo.PrivateSegmentBuffer);
683 Any |= convertArg(AI.DispatchPtr, ArgInfo.DispatchPtr);
684 Any |= convertArg(AI.QueuePtr, ArgInfo.QueuePtr);
685 Any |= convertArg(AI.KernargSegmentPtr, ArgInfo.KernargSegmentPtr);
686 Any |= convertArg(AI.DispatchID, ArgInfo.DispatchID);
687 Any |= convertArg(AI.FlatScratchInit, ArgInfo.FlatScratchInit);
688 Any |= convertArg(AI.LDSKernelId, ArgInfo.LDSKernelId);
689 Any |= convertArg(AI.PrivateSegmentSize, ArgInfo.PrivateSegmentSize);
690 Any |= convertArg(AI.WorkGroupIDX, ArgInfo.WorkGroupIDX);
691 Any |= convertArg(AI.WorkGroupIDY, ArgInfo.WorkGroupIDY);
692 Any |= convertArg(AI.WorkGroupIDZ, ArgInfo.WorkGroupIDZ);
693 Any |= convertArg(AI.WorkGroupInfo, ArgInfo.WorkGroupInfo);
694 Any |= convertArg(AI.PrivateSegmentWaveByteOffset,
695 ArgInfo.PrivateSegmentWaveByteOffset);
696 Any |= convertArg(AI.ImplicitArgPtr, ArgInfo.ImplicitArgPtr);
697 Any |= convertArg(AI.ImplicitBufferPtr, ArgInfo.ImplicitBufferPtr);
698 Any |= convertArg(AI.WorkItemIDX, ArgInfo.WorkItemIDX);
699 Any |= convertArg(AI.WorkItemIDY, ArgInfo.WorkItemIDY);
700 Any |= convertArg(AI.WorkItemIDZ, ArgInfo.WorkItemIDZ);
701
702 // Write FirstKernArgPreloadReg separately, since it's a Register,
703 // not ArgDescriptor.
704 if (ArgInfo.FirstKernArgPreloadReg) {
705 Register Reg = ArgInfo.FirstKernArgPreloadReg;
706 assert(Reg.isPhysical() &&
707 "FirstKernArgPreloadReg must be a physical register");
708
709 yaml::SIArgument SA = yaml::SIArgument::createArgument(IsReg: true);
710 raw_string_ostream OS(SA.RegisterName.Value);
711 OS << printReg(Reg, TRI: &TRI);
712
713 AI.FirstKernArgPreloadReg = SA;
714 Any = true;
715 }
716
717 if (Any)
718 return AI;
719
720 return std::nullopt;
721}
722
723yaml::SIMachineFunctionInfo::SIMachineFunctionInfo(
724 const llvm::SIMachineFunctionInfo &MFI, const TargetRegisterInfo &TRI,
725 const llvm::MachineFunction &MF)
726 : ExplicitKernArgSize(MFI.getExplicitKernArgSize()),
727 MaxKernArgAlign(MFI.getMaxKernArgAlign()), LDSSize(MFI.getLDSSize()),
728 GDSSize(MFI.getGDSSize()), DynLDSAlign(MFI.getDynLDSAlign()),
729 IsEntryFunction(MFI.isEntryFunction()), MemoryBound(MFI.isMemoryBound()),
730 WaveLimiter(MFI.needsWaveLimiter()),
731 HasSpilledSGPRs(MFI.hasSpilledSGPRs()),
732 HasSpilledVGPRs(MFI.hasSpilledVGPRs()),
733 HasNoWWMPoolSGPRSpillFallback(MFI.hasNoWWMPoolSGPRSpillFallback()),
734 NumWaveDispatchSGPRs(MFI.getNumWaveDispatchSGPRs()),
735 NumWaveDispatchVGPRs(MFI.getNumWaveDispatchVGPRs()),
736 HighBitsOf32BitAddress(MFI.get32BitAddressHighBits()),
737 Occupancy(MFI.getOccupancy()),
738 ScratchRSrcReg(regToString(Reg: MFI.getScratchRSrcReg(), TRI)),
739 FrameOffsetReg(regToString(Reg: MFI.getFrameOffsetReg(), TRI)),
740 StackPtrOffsetReg(regToString(Reg: MFI.getStackPtrOffsetReg(), TRI)),
741 BytesInStackArgArea(MFI.getBytesInStackArgArea()),
742 ReturnsVoid(MFI.returnsVoid()),
743 ArgInfo(convertArgumentInfo(ArgInfo: MFI.getArgInfo(), TRI)),
744 PSInputAddr(MFI.getPSInputAddr()), PSInputEnable(MFI.getPSInputEnable()),
745 MaxMemoryClusterDWords(MFI.getMaxMemoryClusterDWords()),
746 Mode(MFI.getMode()), HasInitWholeWave(MFI.hasInitWholeWave()),
747 IsWholeWaveFunction(MFI.isWholeWaveFunction()),
748 DynamicVGPRBlockSize(MFI.getDynamicVGPRBlockSize()),
749 ScratchReservedForDynamicVGPRs(MFI.getScratchReservedForDynamicVGPRs()),
750 NumKernargPreloadSGPRs(MFI.getNumKernargPreloadedSGPRs()),
751 MinNumAGPRs(MFI.getMinNumAGPRs()) {
752 for (Register Reg : MFI.getSGPRSpillPhysVGPRs())
753 SpillPhysVGPRS.push_back(Elt: regToString(Reg, TRI));
754
755 for (Register Reg : MFI.getWWMReservedRegs())
756 WWMReservedRegs.push_back(Elt: regToString(Reg, TRI));
757
758 if (MFI.getLongBranchReservedReg())
759 LongBranchReservedReg = regToString(Reg: MFI.getLongBranchReservedReg(), TRI);
760 if (MFI.getVGPRForAGPRCopy())
761 VGPRForAGPRCopy = regToString(Reg: MFI.getVGPRForAGPRCopy(), TRI);
762
763 if (MFI.getSGPRForEXECCopy())
764 SGPRForEXECCopy = regToString(Reg: MFI.getSGPRForEXECCopy(), TRI);
765
766 auto SFI = MFI.getOptionalScavengeFI();
767 if (SFI)
768 ScavengeFI = yaml::FrameIndex(*SFI, MF.getFrameInfo());
769}
770
771void yaml::SIMachineFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
772 MappingTraits<SIMachineFunctionInfo>::mapping(YamlIO, MFI&: *this);
773}
774
775bool SIMachineFunctionInfo::initializeBaseYamlFields(
776 const yaml::SIMachineFunctionInfo &YamlMFI, const MachineFunction &MF,
777 PerFunctionMIParsingState &PFS, SMDiagnostic &Error, SMRange &SourceRange) {
778 ExplicitKernArgSize = YamlMFI.ExplicitKernArgSize;
779 MaxKernArgAlign = YamlMFI.MaxKernArgAlign;
780 LDSSize = YamlMFI.LDSSize;
781 GDSSize = YamlMFI.GDSSize;
782 DynLDSAlign = YamlMFI.DynLDSAlign;
783 PSInputAddr = YamlMFI.PSInputAddr;
784 PSInputEnable = YamlMFI.PSInputEnable;
785 MaxMemoryClusterDWords = YamlMFI.MaxMemoryClusterDWords;
786 HighBitsOf32BitAddress = YamlMFI.HighBitsOf32BitAddress;
787 Occupancy = YamlMFI.Occupancy;
788 IsEntryFunction = YamlMFI.IsEntryFunction;
789 MemoryBound = YamlMFI.MemoryBound;
790 WaveLimiter = YamlMFI.WaveLimiter;
791 HasSpilledSGPRs = YamlMFI.HasSpilledSGPRs;
792 HasSpilledVGPRs = YamlMFI.HasSpilledVGPRs;
793 HasNoWWMPoolSGPRSpillFallback = YamlMFI.HasNoWWMPoolSGPRSpillFallback;
794 NumWaveDispatchSGPRs = YamlMFI.NumWaveDispatchSGPRs;
795 NumWaveDispatchVGPRs = YamlMFI.NumWaveDispatchVGPRs;
796 BytesInStackArgArea = YamlMFI.BytesInStackArgArea;
797 ReturnsVoid = YamlMFI.ReturnsVoid;
798 IsWholeWaveFunction = YamlMFI.IsWholeWaveFunction;
799 MinNumAGPRs = YamlMFI.MinNumAGPRs;
800 // This can also be set by the function attribute, MFI has higher precedence
801 // though.
802 if (YamlMFI.DynamicVGPRBlockSize != std::nullopt)
803 DynamicVGPRBlockSize = *YamlMFI.DynamicVGPRBlockSize;
804
805 UserSGPRInfo.allocKernargPreloadSGPRs(NumSGPRs: YamlMFI.NumKernargPreloadSGPRs);
806
807 if (YamlMFI.ScavengeFI) {
808 auto FIOrErr = YamlMFI.ScavengeFI->getFI(MFI: MF.getFrameInfo());
809 if (!FIOrErr) {
810 // Create a diagnostic for a the frame index.
811 const MemoryBuffer &Buffer =
812 *PFS.SM->getMemoryBuffer(i: PFS.SM->getMainFileID());
813
814 Error = SMDiagnostic(*PFS.SM, SMLoc(), Buffer.getBufferIdentifier(), 1, 1,
815 SourceMgr::DK_Error, toString(E: FIOrErr.takeError()),
816 "", {}, {});
817 SourceRange = YamlMFI.ScavengeFI->SourceRange;
818 return true;
819 }
820 ScavengeFI = *FIOrErr;
821 } else {
822 ScavengeFI = std::nullopt;
823 }
824 return false;
825}
826