1//===-- GCNSubtarget.cpp - GCN Subtarget Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Implements the GCN specific subclass of TargetSubtarget.
11//
12//===----------------------------------------------------------------------===//
13
14#include "GCNSubtarget.h"
15#include "AMDGPUCallLowering.h"
16#include "AMDGPUInstructionSelector.h"
17#include "AMDGPULegalizerInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "AMDGPUSelectionDAGInfo.h"
20#include "AMDGPUTargetMachine.h"
21#include "SIMachineFunctionInfo.h"
22#include "Utils/AMDGPUBaseInfo.h"
23#include "llvm/ADT/SmallString.h"
24#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
25#include "llvm/CodeGen/MachineScheduler.h"
26#include "llvm/CodeGen/TargetFrameLowering.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/MDBuilder.h"
29#include "llvm/TargetParser/AMDGPUTargetParser.h"
30#include <algorithm>
31
32using namespace llvm;
33
34#define DEBUG_TYPE "gcn-subtarget"
35
36#define GET_SUBTARGETINFO_TARGET_DESC
37#define GET_SUBTARGETINFO_CTOR
38#define AMDGPUSubtarget GCNSubtarget
39#include "AMDGPUGenSubtargetInfo.inc"
40#undef AMDGPUSubtarget
41
42static cl::opt<bool> EnableVGPRIndexMode(
43 "amdgpu-vgpr-index-mode",
44 cl::desc("Use GPR indexing mode instead of movrel for vector indexing"),
45 cl::init(Val: false));
46
47static cl::opt<bool> UseAA("amdgpu-use-aa-in-codegen",
48 cl::desc("Enable the use of AA during codegen."),
49 cl::init(Val: true));
50
51static cl::opt<unsigned>
52 NSAThreshold("amdgpu-nsa-threshold",
53 cl::desc("Number of addresses from which to enable MIMG NSA."),
54 cl::init(Val: 2), cl::Hidden);
55
56GCNSubtarget::~GCNSubtarget() = default;
57
58static AMDGPUSubtarget::Generation computeDefaultGeneration(const Triple &TT) {
59 // Legacy triples without a subarch default to the first target that supports
60 // flat addressing for HSA, otherwise the first amdgcn target.
61 if (TT.getSubArch() == Triple::NoSubArch)
62 return TT.getOS() == Triple::AMDHSA ? AMDGPUSubtarget::SEA_ISLANDS
63 : AMDGPUSubtarget::SOUTHERN_ISLANDS;
64
65 switch (AMDGPU::getMajorSubArch(SubArch: TT.getSubArch())) {
66 case Triple::AMDGPUSubArch6:
67 return AMDGPUSubtarget::SOUTHERN_ISLANDS;
68 case Triple::AMDGPUSubArch7:
69 return AMDGPUSubtarget::SEA_ISLANDS;
70 case Triple::AMDGPUSubArch8:
71 case Triple::AMDGPUSubArch810:
72 return AMDGPUSubtarget::VOLCANIC_ISLANDS;
73 case Triple::AMDGPUSubArch9:
74 case Triple::AMDGPUSubArch908:
75 case Triple::AMDGPUSubArch90A:
76 case Triple::AMDGPUSubArch9_4:
77 return AMDGPUSubtarget::GFX9;
78 case Triple::AMDGPUSubArch10_1:
79 case Triple::AMDGPUSubArch10_3:
80 return AMDGPUSubtarget::GFX10;
81 case Triple::AMDGPUSubArch11:
82 case Triple::AMDGPUSubArch11_7:
83 return AMDGPUSubtarget::GFX11;
84 case Triple::AMDGPUSubArch12:
85 case Triple::AMDGPUSubArch12_5:
86 return AMDGPUSubtarget::GFX12;
87 case Triple::AMDGPUSubArch13:
88 return AMDGPUSubtarget::GFX13;
89 default:
90 reportFatalUsageError(reason: "invalid subarch for amdgpu");
91 }
92}
93
94GCNSubtarget &GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
95 StringRef GPU,
96 StringRef FS) {
97 // Determine default and user-specified characteristics
98 //
99 // We want to be able to turn these off, but making this a subtarget feature
100 // for SI has the unhelpful behavior that it unsets everything else if you
101 // disable it.
102 //
103 // Similarly we want enable-prt-strict-null to be on by default and not to
104 // unset everything else if it is disabled
105
106 SmallString<256> FullFS("+load-store-opt,+enable-ds128,");
107
108 // Turn on features that HSA ABI requires. Also turn on FlatForGlobal by
109 // default
110 if (isAmdHsaOS())
111 FullFS += "+flat-for-global,+unaligned-access-mode,+trap-handler,";
112
113 FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
114
115 // Disable mutually exclusive bits.
116 if (FS.contains_insensitive(Other: "+wavefrontsize")) {
117 if (!FS.contains_insensitive(Other: "wavefrontsize16"))
118 FullFS += "-wavefrontsize16,";
119 if (!FS.contains_insensitive(Other: "wavefrontsize32"))
120 FullFS += "-wavefrontsize32,";
121 if (!FS.contains_insensitive(Other: "wavefrontsize64"))
122 FullFS += "-wavefrontsize64,";
123 }
124
125 FullFS += FS;
126
127 ParseSubtargetFeatures(CPU: GPU, /*TuneCPU*/ GPU, FS: FullFS);
128
129 // Implement the "generic" processors, which acts as the default when no
130 // generation features are enabled (e.g for -mcpu=''). HSA OS defaults to
131 // the first amdgcn target that supports flat addressing. Other OSes defaults
132 // to the first amdgcn target.
133 if (Gen == AMDGPUSubtarget::INVALID) {
134 Gen = computeDefaultGeneration(TT);
135 // Assume wave64 for the unknown target, if not explicitly set.
136 if (getWavefrontSizeLog2() == 0)
137 WavefrontSizeLog2 = 6;
138 } else if (!hasFeature(Feature: AMDGPU::FeatureWavefrontSize32) &&
139 !hasFeature(Feature: AMDGPU::FeatureWavefrontSize64)) {
140 // If there is no default wave size it must be a generation before gfx10,
141 // these have FeatureWavefrontSize64 in their definition already. For gfx10+
142 // set wave32 as a default.
143 ToggleFeature(FB: AMDGPU::FeatureWavefrontSize32);
144 WavefrontSizeLog2 = getGeneration() >= AMDGPUSubtarget::GFX10 ? 5 : 6;
145 }
146
147 // We don't support FP64 for EG/NI atm.
148 assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS));
149
150 // Targets must either support 64-bit offsets for MUBUF instructions, and/or
151 // support flat operations, otherwise they cannot access a 64-bit global
152 // address space
153 assert(hasAddr64() || hasFlat());
154 // Unless +-flat-for-global is specified, turn on FlatForGlobal for targets
155 // that do not support ADDR64 variants of MUBUF instructions. Such targets
156 // cannot use a 64 bit offset with a MUBUF instruction to access the global
157 // address space
158 if (!hasAddr64() && !FS.contains(Other: "flat-for-global") && !UseFlatForGlobal) {
159 ToggleFeature(FB: AMDGPU::FeatureUseFlatForGlobal);
160 UseFlatForGlobal = true;
161 }
162 // Unless +-flat-for-global is specified, use MUBUF instructions for global
163 // address space access if flat operations are not available.
164 if (!hasFlat() && !FS.contains(Other: "flat-for-global") && UseFlatForGlobal) {
165 ToggleFeature(FB: AMDGPU::FeatureUseFlatForGlobal);
166 UseFlatForGlobal = false;
167 }
168
169 // Set defaults if needed.
170 if (MaxPrivateElementSize == 0)
171 MaxPrivateElementSize = 4;
172
173 if (LDSBankCount == 0)
174 LDSBankCount = 32;
175
176 if (AddressableLocalMemorySize == 0)
177 AddressableLocalMemorySize = 32768;
178
179 if (FlatOffsetBitWidth == 0)
180 FlatOffsetBitWidth = 13;
181
182 LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(STI: *this);
183 // LDS Allocation Granularity calculated in bytes from dwords
184 LDSAllocationGranularity =
185 AMDGPU::getLdsDwGranularity(ST: *this) * sizeof(uint32_t);
186
187 HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
188 HasSMulHi = getGeneration() >= AMDGPUSubtarget::GFX9;
189
190 // InstCacheLineSize is set from TableGen subtarget features
191 // (FeatureInstCacheLineSize64 / FeatureInstCacheLineSize128).
192 // Fall back to 64 if no feature was specified (e.g. generic targets).
193 if (InstCacheLineSize == 0)
194 InstCacheLineSize = 64;
195
196 assert(llvm::isPowerOf2_32(InstCacheLineSize) &&
197 "InstCacheLineSize must be a power of 2");
198
199 LLVM_DEBUG(dbgs() << "xnack setting for subtarget: "
200 << TargetID.getXnackSetting() << '\n');
201 LLVM_DEBUG(dbgs() << "sramecc setting for subtarget: "
202 << TargetID.getSramEccSetting() << '\n');
203
204 return *this;
205}
206
207void GCNSubtarget::checkSubtargetFeatures(const Function &F) const {
208 LLVMContext &Ctx = F.getContext();
209 if (hasFeature(Feature: AMDGPU::FeatureWavefrontSize32) &&
210 hasFeature(Feature: AMDGPU::FeatureWavefrontSize64)) {
211 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
212 F, "must specify exactly one of wavefrontsize32 and wavefrontsize64"));
213 }
214}
215
216// TODO: Validate subarch for subtarget
217
218GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
219 const GCNTargetMachine &TM, bool BufferOOBRelaxed,
220 bool TBufferOOBRelaxed)
221 : // clang-format off
222 AMDGPUGenSubtargetInfo(TT, GPU, /*TuneCPU*/ GPU, FS),
223 AMDGPUSubtarget(TT),
224 TargetID(AMDGPU::createAMDGPUTargetID(STI: *this, FeatureString: FS)),
225 InstrItins(getInstrItineraryForCPU(CPU: GPU)),
226 BufferOOBRelaxed(BufferOOBRelaxed),
227 TBufferOOBRelaxed(TBufferOOBRelaxed),
228 InstrInfo(initializeSubtargetDependencies(TT, GPU, FS)),
229 TLInfo(TM, *this),
230 // Frame index expansion sometimes assumes the low bit of SP is 0
231 FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0,
232 /*TransAl=*/Align(4)) {
233
234 // clang-format on
235 MaxWavesPerEU = AMDGPU::IsaInfo::getMaxWavesPerEU(STI: *this);
236 EUsPerCU = AMDGPU::IsaInfo::getEUsPerCU(STI: *this);
237
238 TSInfo = std::make_unique<AMDGPUSelectionDAGInfo>();
239
240 CallLoweringInfo = std::make_unique<AMDGPUCallLowering>(args: *getTargetLowering());
241 InlineAsmLoweringInfo =
242 std::make_unique<InlineAsmLowering>(args: getTargetLowering());
243 Legalizer = std::make_unique<AMDGPULegalizerInfo>(args&: *this, args: TM);
244 RegBankInfo = std::make_unique<AMDGPURegisterBankInfo>(args&: *this);
245 InstSelector =
246 std::make_unique<AMDGPUInstructionSelector>(args&: *this, args&: *RegBankInfo);
247}
248
249const SelectionDAGTargetInfo *GCNSubtarget::getSelectionDAGInfo() const {
250 return TSInfo.get();
251}
252
253unsigned GCNSubtarget::getConstantBusLimit(unsigned Opcode) const {
254 if (getGeneration() < GFX10)
255 return 1;
256
257 switch (Opcode) {
258 case AMDGPU::V_LSHLREV_B64_e64:
259 case AMDGPU::V_LSHLREV_B64_gfx10:
260 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
261 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
262 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
263 case AMDGPU::V_LSHL_B64_e64:
264 case AMDGPU::V_LSHRREV_B64_e64:
265 case AMDGPU::V_LSHRREV_B64_gfx10:
266 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
267 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
268 case AMDGPU::V_LSHR_B64_e64:
269 case AMDGPU::V_ASHRREV_I64_e64:
270 case AMDGPU::V_ASHRREV_I64_gfx10:
271 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
272 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
273 case AMDGPU::V_ASHR_I64_e64:
274 return 1;
275 }
276
277 return 2;
278}
279
280/// This list was mostly derived from experimentation.
281bool GCNSubtarget::zeroesHigh16BitsOfDest(unsigned Opcode) const {
282 switch (Opcode) {
283 case AMDGPU::V_CVT_F16_F32_e32:
284 case AMDGPU::V_CVT_F16_F32_e64:
285 case AMDGPU::V_CVT_F16_U16_e32:
286 case AMDGPU::V_CVT_F16_U16_e64:
287 case AMDGPU::V_CVT_F16_I16_e32:
288 case AMDGPU::V_CVT_F16_I16_e64:
289 case AMDGPU::V_RCP_F16_e64:
290 case AMDGPU::V_RCP_F16_e32:
291 case AMDGPU::V_RSQ_F16_e64:
292 case AMDGPU::V_RSQ_F16_e32:
293 case AMDGPU::V_SQRT_F16_e64:
294 case AMDGPU::V_SQRT_F16_e32:
295 case AMDGPU::V_LOG_F16_e64:
296 case AMDGPU::V_LOG_F16_e32:
297 case AMDGPU::V_EXP_F16_e64:
298 case AMDGPU::V_EXP_F16_e32:
299 case AMDGPU::V_SIN_F16_e64:
300 case AMDGPU::V_SIN_F16_e32:
301 case AMDGPU::V_COS_F16_e64:
302 case AMDGPU::V_COS_F16_e32:
303 case AMDGPU::V_FLOOR_F16_e64:
304 case AMDGPU::V_FLOOR_F16_e32:
305 case AMDGPU::V_CEIL_F16_e64:
306 case AMDGPU::V_CEIL_F16_e32:
307 case AMDGPU::V_TRUNC_F16_e64:
308 case AMDGPU::V_TRUNC_F16_e32:
309 case AMDGPU::V_RNDNE_F16_e64:
310 case AMDGPU::V_RNDNE_F16_e32:
311 case AMDGPU::V_FRACT_F16_e64:
312 case AMDGPU::V_FRACT_F16_e32:
313 case AMDGPU::V_FREXP_MANT_F16_e64:
314 case AMDGPU::V_FREXP_MANT_F16_e32:
315 case AMDGPU::V_FREXP_EXP_I16_F16_e64:
316 case AMDGPU::V_FREXP_EXP_I16_F16_e32:
317 case AMDGPU::V_LDEXP_F16_e64:
318 case AMDGPU::V_LDEXP_F16_e32:
319 case AMDGPU::V_LSHLREV_B16_e64:
320 case AMDGPU::V_LSHLREV_B16_e32:
321 case AMDGPU::V_LSHRREV_B16_e64:
322 case AMDGPU::V_LSHRREV_B16_e32:
323 case AMDGPU::V_ASHRREV_I16_e64:
324 case AMDGPU::V_ASHRREV_I16_e32:
325 case AMDGPU::V_ADD_U16_e64:
326 case AMDGPU::V_ADD_U16_e32:
327 case AMDGPU::V_SUB_U16_e64:
328 case AMDGPU::V_SUB_U16_e32:
329 case AMDGPU::V_SUBREV_U16_e64:
330 case AMDGPU::V_SUBREV_U16_e32:
331 case AMDGPU::V_MUL_LO_U16_e64:
332 case AMDGPU::V_MUL_LO_U16_e32:
333 case AMDGPU::V_ADD_F16_e64:
334 case AMDGPU::V_ADD_F16_e32:
335 case AMDGPU::V_SUB_F16_e64:
336 case AMDGPU::V_SUB_F16_e32:
337 case AMDGPU::V_SUBREV_F16_e64:
338 case AMDGPU::V_SUBREV_F16_e32:
339 case AMDGPU::V_MUL_F16_e64:
340 case AMDGPU::V_MUL_F16_e32:
341 case AMDGPU::V_MAX_F16_e64:
342 case AMDGPU::V_MAX_F16_e32:
343 case AMDGPU::V_MIN_F16_e64:
344 case AMDGPU::V_MIN_F16_e32:
345 case AMDGPU::V_MAX_U16_e64:
346 case AMDGPU::V_MAX_U16_e32:
347 case AMDGPU::V_MIN_U16_e64:
348 case AMDGPU::V_MIN_U16_e32:
349 case AMDGPU::V_MAX_I16_e64:
350 case AMDGPU::V_MAX_I16_e32:
351 case AMDGPU::V_MIN_I16_e64:
352 case AMDGPU::V_MIN_I16_e32:
353 case AMDGPU::V_MAD_F16_e64:
354 case AMDGPU::V_MAD_U16_e64:
355 case AMDGPU::V_MAD_I16_e64:
356 case AMDGPU::V_FMA_F16_e64:
357 case AMDGPU::V_DIV_FIXUP_F16_e64:
358 // On gfx10, all 16-bit instructions preserve the high bits.
359 return getGeneration() <= AMDGPUSubtarget::GFX9;
360 case AMDGPU::V_MADAK_F16:
361 case AMDGPU::V_MADMK_F16:
362 case AMDGPU::V_MAC_F16_e64:
363 case AMDGPU::V_MAC_F16_e32:
364 case AMDGPU::V_FMAMK_F16:
365 case AMDGPU::V_FMAAK_F16:
366 case AMDGPU::V_FMAC_F16_e64:
367 case AMDGPU::V_FMAC_F16_e32:
368 // In gfx9, the preferred handling of the unused high 16-bits changed. Most
369 // instructions maintain the legacy behavior of 0ing. Some instructions
370 // changed to preserving the high bits.
371 return getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
372 case AMDGPU::V_MAD_MIXLO_F16:
373 case AMDGPU::V_MAD_MIXHI_F16:
374 default:
375 return false;
376 }
377}
378
379void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
380 const SchedRegion &Region) const {
381 // Track register pressure so the scheduler can try to decrease
382 // pressure once register usage is above the threshold defined by
383 // SIRegisterInfo::getRegPressureSetLimit()
384 Policy.ShouldTrackPressure = true;
385
386 const Function &F = Region.RegionBegin->getMF()->getFunction();
387 if (AMDGPU::getSchedStrategy(F) == "coexec") {
388 Policy.OnlyTopDown = true;
389 Policy.OnlyBottomUp = false;
390 return;
391 }
392
393 // Enabling both top down and bottom up scheduling seems to give us less
394 // register spills than just using one of these approaches on its own.
395 Policy.OnlyTopDown = false;
396 Policy.OnlyBottomUp = false;
397
398 // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
399 if (!enableSIScheduler())
400 Policy.ShouldTrackLaneMasks = true;
401}
402
403void GCNSubtarget::overridePostRASchedPolicy(MachineSchedPolicy &Policy,
404 const SchedRegion &Region) const {
405 const Function &F = Region.RegionBegin->getMF()->getFunction();
406 Attribute PostRADirectionAttr = F.getFnAttribute(Kind: "amdgpu-post-ra-direction");
407 if (!PostRADirectionAttr.isValid())
408 return;
409
410 StringRef PostRADirectionStr = PostRADirectionAttr.getValueAsString();
411 if (PostRADirectionStr == "topdown") {
412 Policy.OnlyTopDown = true;
413 Policy.OnlyBottomUp = false;
414 } else if (PostRADirectionStr == "bottomup") {
415 Policy.OnlyTopDown = false;
416 Policy.OnlyBottomUp = true;
417 } else if (PostRADirectionStr == "bidirectional") {
418 Policy.OnlyTopDown = false;
419 Policy.OnlyBottomUp = false;
420 } else {
421 DiagnosticInfoOptimizationFailure Diag(
422 F, F.getSubprogram(), "invalid value for postRA direction attribute");
423 F.getContext().diagnose(DI: Diag);
424 }
425
426 LLVM_DEBUG({
427 const char *DirStr = "default";
428 if (Policy.OnlyTopDown && !Policy.OnlyBottomUp)
429 DirStr = "topdown";
430 else if (!Policy.OnlyTopDown && Policy.OnlyBottomUp)
431 DirStr = "bottomup";
432 else if (!Policy.OnlyTopDown && !Policy.OnlyBottomUp)
433 DirStr = "bidirectional";
434
435 dbgs() << "Post-MI-sched direction (" << F.getName() << "): " << DirStr
436 << '\n';
437 });
438}
439
440void GCNSubtarget::mirFileLoaded(MachineFunction &MF) const {
441 if (isWave32()) {
442 // Fix implicit $vcc operands after MIParser has verified that they match
443 // the instruction definitions.
444 for (auto &MBB : MF) {
445 for (auto &MI : MBB)
446 InstrInfo.fixImplicitOperands(MI);
447 }
448 }
449}
450
451bool GCNSubtarget::hasMadF16() const {
452 return InstrInfo.pseudoToMCOpcode(Opcode: AMDGPU::V_MAD_F16_e64) != -1;
453}
454
455bool GCNSubtarget::useVGPRIndexMode() const {
456 return hasVGPRIndexMode() && (!hasMovrel() || EnableVGPRIndexMode);
457}
458
459bool GCNSubtarget::useAA() const { return UseAA; }
460
461unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
462 return AMDGPU::IsaInfo::getOccupancyWithNumSGPRs(STI: *this, SGPRs);
463}
464
465unsigned
466GCNSubtarget::getOccupancyWithNumVGPRs(unsigned NumVGPRs,
467 unsigned DynamicVGPRBlockSize) const {
468 return AMDGPU::IsaInfo::getNumWavesPerEUWithNumVGPRs(STI: *this, NumVGPRs,
469 DynamicVGPRBlockSize);
470}
471
472unsigned
473GCNSubtarget::getBaseReservedNumSGPRs(const bool HasFlatScratch) const {
474 if (getGeneration() >= AMDGPUSubtarget::GFX10)
475 return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
476
477 if (HasFlatScratch || HasArchitectedFlatScratch) {
478 if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
479 return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
480 if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
481 return 4; // FLAT_SCRATCH, VCC (in that order).
482 }
483
484 if (isXNACKEnabled())
485 return 4; // XNACK, VCC (in that order).
486 return 2; // VCC.
487}
488
489unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
490 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
491 return getBaseReservedNumSGPRs(HasFlatScratch: MFI.getUserSGPRInfo().hasFlatScratchInit());
492}
493
494unsigned GCNSubtarget::getReservedNumSGPRs(const Function &F) const {
495 // In principle we do not need to reserve SGPR pair used for flat_scratch if
496 // we know flat instructions do not access the stack anywhere in the
497 // program. For now assume it's needed if we have flat instructions.
498 const bool KernelUsesFlatScratch = hasFlatAddressSpace();
499 return getBaseReservedNumSGPRs(HasFlatScratch: KernelUsesFlatScratch);
500}
501
502std::pair<unsigned, unsigned>
503GCNSubtarget::computeOccupancy(const Function &F, unsigned LDSSize,
504 unsigned NumSGPRs, unsigned NumVGPRs) const {
505 unsigned DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
506 // Temporarily check both the attribute and the subtarget feature until the
507 // latter is removed.
508 if (DynamicVGPRBlockSize == 0 && isDynamicVGPREnabled())
509 DynamicVGPRBlockSize = getDynamicVGPRBlockSize();
510
511 auto [MinOcc, MaxOcc] = getOccupancyWithWorkGroupSizes(LDSBytes: LDSSize, F);
512 unsigned SGPROcc = getOccupancyWithNumSGPRs(SGPRs: NumSGPRs);
513 unsigned VGPROcc = getOccupancyWithNumVGPRs(NumVGPRs, DynamicVGPRBlockSize);
514
515 // Maximum occupancy may be further limited by high SGPR/VGPR usage.
516 MaxOcc = std::min(l: {MaxOcc, SGPROcc, VGPROcc});
517 return {std::min(a: MinOcc, b: MaxOcc), MaxOcc};
518}
519
520unsigned GCNSubtarget::getBaseMaxNumSGPRs(
521 const Function &F, std::pair<unsigned, unsigned> WavesPerEU,
522 unsigned PreloadedSGPRs, unsigned ReservedNumSGPRs) const {
523 // Compute maximum number of SGPRs function can use using default/requested
524 // minimum number of waves per execution unit.
525 unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: false);
526 unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: true);
527
528 // Check if maximum number of SGPRs was explicitly requested using
529 // "amdgpu-num-sgpr" attribute.
530 unsigned Requested =
531 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-num-sgpr", Default: MaxNumSGPRs);
532
533 if (Requested != MaxNumSGPRs) {
534 // Make sure requested value does not violate subtarget's specifications.
535 if (Requested && (Requested <= ReservedNumSGPRs))
536 Requested = 0;
537
538 // If more SGPRs are required to support the input user/system SGPRs,
539 // increase to accommodate them.
540 //
541 // FIXME: This really ends up using the requested number of SGPRs + number
542 // of reserved special registers in total. Theoretically you could re-use
543 // the last input registers for these special registers, but this would
544 // require a lot of complexity to deal with the weird aliasing.
545 unsigned InputNumSGPRs = PreloadedSGPRs;
546 if (Requested && Requested < InputNumSGPRs)
547 Requested = InputNumSGPRs;
548
549 // Make sure requested value is compatible with values implied by
550 // default/requested minimum/maximum number of waves per execution unit.
551 if (Requested && Requested > getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: false))
552 Requested = 0;
553 if (WavesPerEU.second && Requested &&
554 Requested < getMinNumSGPRs(WavesPerEU: WavesPerEU.second))
555 Requested = 0;
556
557 if (Requested)
558 MaxNumSGPRs = Requested;
559 }
560
561 if (hasSGPRInitBug())
562 MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
563
564 return std::min(a: MaxNumSGPRs - ReservedNumSGPRs, b: MaxAddressableNumSGPRs);
565}
566
567unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
568 const Function &F = MF.getFunction();
569 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
570 return getBaseMaxNumSGPRs(F, WavesPerEU: MFI.getWavesPerEU(), PreloadedSGPRs: MFI.getNumPreloadedSGPRs(),
571 ReservedNumSGPRs: getReservedNumSGPRs(MF));
572}
573
574unsigned GCNSubtarget::getMaxNumPreloadedSGPRs() const {
575 using USI = GCNUserSGPRUsageInfo;
576 // Max number of user SGPRs
577 const unsigned MaxUserSGPRs =
578 USI::getNumUserSGPRForField(ID: USI::PrivateSegmentBufferID) +
579 USI::getNumUserSGPRForField(ID: USI::DispatchPtrID) +
580 USI::getNumUserSGPRForField(ID: USI::QueuePtrID) +
581 USI::getNumUserSGPRForField(ID: USI::KernargSegmentPtrID) +
582 USI::getNumUserSGPRForField(ID: USI::DispatchIdID) +
583 USI::getNumUserSGPRForField(ID: USI::FlatScratchInitID) +
584 USI::getNumUserSGPRForField(ID: USI::ImplicitBufferPtrID);
585
586 // Max number of system SGPRs
587 const unsigned MaxSystemSGPRs = 1 + // WorkGroupIDX
588 1 + // WorkGroupIDY
589 1 + // WorkGroupIDZ
590 1 + // WorkGroupInfo
591 1; // private segment wave byte offset
592
593 // Max number of synthetic SGPRs
594 const unsigned SyntheticSGPRs = 1; // LDSKernelId
595
596 return MaxUserSGPRs + MaxSystemSGPRs + SyntheticSGPRs;
597}
598
599unsigned GCNSubtarget::getMaxNumSGPRs(const Function &F) const {
600 return getBaseMaxNumSGPRs(F, WavesPerEU: getWavesPerEU(F), PreloadedSGPRs: getMaxNumPreloadedSGPRs(),
601 ReservedNumSGPRs: getReservedNumSGPRs(F));
602}
603
604unsigned GCNSubtarget::getBaseMaxNumVGPRs(
605 const Function &F, std::pair<unsigned, unsigned> NumVGPRBounds) const {
606 const auto [Min, Max] = NumVGPRBounds;
607
608 // Check if maximum number of VGPRs was explicitly requested using
609 // "amdgpu-num-vgpr" attribute.
610
611 unsigned Requested = F.getFnAttributeAsParsedInteger(Kind: "amdgpu-num-vgpr", Default: Max);
612 if (Requested != Max && hasGFX90AInsts())
613 Requested *= 2;
614
615 // Make sure requested value is inside the range of possible VGPR usage.
616 return std::clamp(val: Requested, lo: Min, hi: Max);
617}
618
619unsigned GCNSubtarget::getMaxNumVGPRs(const Function &F) const {
620 // Temporarily check both the attribute and the subtarget feature, until the
621 // latter is removed.
622 unsigned DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
623 if (DynamicVGPRBlockSize == 0 && isDynamicVGPREnabled())
624 DynamicVGPRBlockSize = getDynamicVGPRBlockSize();
625
626 std::pair<unsigned, unsigned> Waves = getWavesPerEU(F);
627 return getBaseMaxNumVGPRs(
628 F, NumVGPRBounds: {getMinNumVGPRs(WavesPerEU: Waves.second, DynamicVGPRBlockSize),
629 getMaxNumVGPRs(WavesPerEU: Waves.first, DynamicVGPRBlockSize)});
630}
631
632unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
633 return getMaxNumVGPRs(F: MF.getFunction());
634}
635
636std::pair<unsigned, unsigned>
637GCNSubtarget::getMaxNumVectorRegs(const Function &F) const {
638 const unsigned MaxVectorRegs = getMaxNumVGPRs(F);
639
640 unsigned MaxNumVGPRs = MaxVectorRegs;
641 unsigned MaxNumAGPRs = 0;
642 unsigned NumArchVGPRs = getAddressableNumArchVGPRs();
643
644 // On GFX90A, the number of VGPRs and AGPRs need not be equal. Theoretically,
645 // a wave may have up to 512 total vector registers combining together both
646 // VGPRs and AGPRs. Hence, in an entry function without calls and without
647 // AGPRs used within it, it is possible to use the whole vector register
648 // budget for VGPRs.
649 //
650 // TODO: it shall be possible to estimate maximum AGPR/VGPR pressure and split
651 // register file accordingly.
652 if (hasGFX90AInsts()) {
653 unsigned MinNumAGPRs = 0;
654 const unsigned TotalNumAGPRs = AMDGPU::AGPR_32RegClass.getNumRegs();
655
656 const std::pair<unsigned, unsigned> DefaultNumAGPR = {~0u, ~0u};
657
658 // TODO: The lower bound should probably force the number of required
659 // registers up, overriding amdgpu-waves-per-eu.
660 std::tie(args&: MinNumAGPRs, args&: MaxNumAGPRs) =
661 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-agpr-alloc", Default: DefaultNumAGPR,
662 /*OnlyFirstRequired=*/true);
663
664 if (MinNumAGPRs == DefaultNumAGPR.first) {
665 // Default to splitting half the registers if AGPRs are required.
666 MinNumAGPRs = MaxNumAGPRs = MaxVectorRegs / 2;
667 } else {
668 // Align to accum_offset's allocation granularity.
669 MinNumAGPRs = alignTo(Value: MinNumAGPRs, Align: 4);
670
671 MinNumAGPRs = std::min(a: MinNumAGPRs, b: TotalNumAGPRs);
672 }
673
674 // Clamp values to be inbounds of our limits, and ensure min <= max.
675
676 MaxNumAGPRs = std::min(a: std::max(a: MinNumAGPRs, b: MaxNumAGPRs), b: MaxVectorRegs);
677 MinNumAGPRs = std::min(l: {MinNumAGPRs, TotalNumAGPRs, MaxNumAGPRs});
678
679 MaxNumVGPRs = std::min(a: MaxVectorRegs - MinNumAGPRs, b: NumArchVGPRs);
680 MaxNumAGPRs = std::min(a: MaxVectorRegs - MaxNumVGPRs, b: MaxNumAGPRs);
681
682 assert(MaxNumVGPRs + MaxNumAGPRs <= MaxVectorRegs &&
683 MaxNumAGPRs <= TotalNumAGPRs && MaxNumVGPRs <= NumArchVGPRs &&
684 "invalid register counts");
685 } else if (hasMAIInsts()) {
686 // On gfx908 the number of AGPRs always equals the number of VGPRs.
687 MaxNumAGPRs = MaxNumVGPRs = MaxVectorRegs;
688 }
689
690 return std::pair(MaxNumVGPRs, MaxNumAGPRs);
691}
692
693// Check to which source operand UseOpIdx points to and return a pointer to the
694// operand of the corresponding source modifier.
695// Return nullptr if UseOpIdx either doesn't point to src0/1/2 or if there is no
696// operand for the corresponding source modifier.
697static const MachineOperand *
698getVOP3PSourceModifierFromOpIdx(const MachineInstr &UseI, int UseOpIdx,
699 const SIInstrInfo &InstrInfo) {
700 AMDGPU::OpName UseName =
701 AMDGPU::getOperandIdxName(Opcode: UseI.getOpcode(), Idx: UseOpIdx);
702 switch (UseName) {
703 case AMDGPU::OpName::src0:
704 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src0_modifiers);
705 case AMDGPU::OpName::src1:
706 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src1_modifiers);
707 case AMDGPU::OpName::src2:
708 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src2_modifiers);
709 default:
710 return nullptr;
711 }
712}
713
714// Get the subreg idx of the subreg that is used by the given instruction
715// operand, considering the given op_sel modifier.
716// Return 0 if the whole register is used or as a conservative fallback.
717static unsigned getEffectiveSubRegIdx(const SIRegisterInfo &TRI,
718 const SIInstrInfo &InstrInfo,
719 const MachineInstr &I,
720 const MachineOperand &Op) {
721 if (!InstrInfo.isVOP3P(MI: I) || InstrInfo.isWMMA(MI: I) || InstrInfo.isSWMMAC(MI: I))
722 return AMDGPU::NoSubRegister;
723
724 const MachineOperand *OpMod =
725 getVOP3PSourceModifierFromOpIdx(UseI: I, UseOpIdx: Op.getOperandNo(), InstrInfo);
726 if (!OpMod)
727 return AMDGPU::NoSubRegister;
728
729 // Note: the FMA_MIX* and MAD_MIX* instructions have different semantics for
730 // the op_sel and op_sel_hi source modifiers:
731 // - op_sel: selects low/high operand bits as input to the operation;
732 // has only meaning for 16-bit source operands
733 // - op_sel_hi: specifies the size of the source operands (16 or 32 bits);
734 // a value of 0 indicates 32 bit, 1 indicates 16 bit
735 // For the other VOP3P instructions, the semantics are:
736 // - op_sel: selects low/high operand bits as input to the operation which
737 // results in the lower-half of the destination
738 // - op_sel_hi: selects the low/high operand bits as input to the operation
739 // which results in the higher-half of the destination
740 int64_t OpSel = OpMod->getImm() & SISrcMods::OP_SEL_0;
741 int64_t OpSelHi = OpMod->getImm() & SISrcMods::OP_SEL_1;
742
743 // Check if all parts of the register are being used (= op_sel and op_sel_hi
744 // differ for VOP3P or op_sel_hi=0 for VOP3PMix). In that case we can return
745 // early.
746 if ((!InstrInfo.isVOP3PMix(MI: I) && (!OpSel || !OpSelHi) &&
747 (OpSel || OpSelHi)) ||
748 (InstrInfo.isVOP3PMix(MI: I) && !OpSelHi))
749 return AMDGPU::NoSubRegister;
750
751 const MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
752 const TargetRegisterClass *RC = TRI.getRegClassForOperandReg(MRI, MO: Op);
753
754 if (unsigned SubRegIdx = OpSel ? AMDGPU::sub1 : AMDGPU::sub0;
755 TRI.getSubClassWithSubReg(RC, SubRegIdx) == RC)
756 return SubRegIdx;
757 if (unsigned SubRegIdx = OpSel ? AMDGPU::hi16 : AMDGPU::lo16;
758 TRI.getSubClassWithSubReg(RC, SubRegIdx) == RC)
759 return SubRegIdx;
760
761 return AMDGPU::NoSubRegister;
762}
763
764Register GCNSubtarget::getRealSchedDependency(const MachineInstr &DefI,
765 int DefOpIdx,
766 const MachineInstr &UseI,
767 int UseOpIdx) const {
768 const SIRegisterInfo *TRI = getRegisterInfo();
769 const MachineOperand &DefOp = DefI.getOperand(i: DefOpIdx);
770 const MachineOperand &UseOp = UseI.getOperand(i: UseOpIdx);
771 Register DefReg = DefOp.getReg();
772 Register UseReg = UseOp.getReg();
773
774 // If the registers aren't restricted to a sub-register, there is no point in
775 // further analysis. This check makes only sense for virtual registers because
776 // physical registers may form a tuple and thus be part of a superregister
777 // although they are not a subregister themselves (vgpr0 is a "subreg" of
778 // vgpr0_vgpr1 without being a subreg in itself).
779 unsigned DefSubRegIdx = DefOp.getSubReg();
780 if (DefReg.isVirtual() && DefSubRegIdx == AMDGPU::NoSubRegister)
781 return DefReg;
782 unsigned UseSubRegIdx = getEffectiveSubRegIdx(TRI: *TRI, InstrInfo, I: UseI, Op: UseOp);
783 if (UseReg.isVirtual() && UseSubRegIdx == AMDGPU::NoSubRegister)
784 return DefReg;
785
786 if (!TRI->checkSubRegInterference(RegA: DefReg, SubA: DefSubRegIdx, RegB: UseReg, SubB: UseSubRegIdx))
787 return Register(); // No real dependency
788
789 // UseReg might be smaller or larger than DefReg, depending on the subreg and
790 // on whether DefReg is a subreg, too. -> Find the smaller one. This does not
791 // apply to virtual registers because we cannot construct a subreg for them.
792 if (DefReg.isVirtual())
793 return DefReg;
794 MCRegister DefMCReg =
795 DefSubRegIdx ? TRI->getSubReg(Reg: DefReg, Idx: DefSubRegIdx) : DefReg.asMCReg();
796 MCRegister UseMCReg =
797 UseSubRegIdx ? TRI->getSubReg(Reg: UseReg, Idx: UseSubRegIdx) : UseReg.asMCReg();
798 return TRI->isSubRegisterEq(RegA: DefMCReg, RegB: UseMCReg) ? UseMCReg : DefMCReg;
799}
800
801void GCNSubtarget::adjustSchedDependency(
802 SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx, SDep &Dep,
803 const TargetSchedModel *SchedModel) const {
804 if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() || !Def->isInstr() ||
805 !Use->isInstr())
806 return;
807
808 MachineInstr *DefI = Def->getInstr();
809 MachineInstr *UseI = Use->getInstr();
810
811 // Check for false latency on $tensorcnt / $asynccnt dependencies
812 if (Dep.getReg() == AMDGPU::TENSORcnt || Dep.getReg() == AMDGPU::ASYNCcnt) {
813 unsigned UseOp = UseI->getOpcode();
814 // Do not adjust latency for load->s_wait
815 bool IsBarrierCase =
816 InstrInfo.isLDSDMA(MI: *DefI) &&
817 (UseOp == AMDGPU::S_WAIT_TENSORCNT || UseOp == AMDGPU::S_WAIT_ASYNCCNT);
818 if (!IsBarrierCase) {
819 Dep.setLatency(1);
820 return;
821 }
822 }
823
824 if (Register Reg = getRealSchedDependency(DefI: *DefI, DefOpIdx, UseI: *UseI, UseOpIdx)) {
825 Dep.setReg(Reg);
826 } else {
827 Dep = SDep(Def, SDep::Artificial);
828 return; // This is not a data dependency anymore.
829 }
830
831 if (DefI->isBundle()) {
832 const SIRegisterInfo *TRI = getRegisterInfo();
833 auto Reg = Dep.getReg();
834 MachineBasicBlock::const_instr_iterator I(DefI->getIterator());
835 MachineBasicBlock::const_instr_iterator E(DefI->getParent()->instr_end());
836 unsigned Lat = 0;
837 for (++I; I != E && I->isBundledWithPred(); ++I) {
838 if (I->isMetaInstruction())
839 continue;
840 if (I->modifiesRegister(Reg, TRI))
841 Lat = InstrInfo.getInstrLatency(ItinData: getInstrItineraryData(), MI: *I);
842 else if (Lat)
843 --Lat;
844 }
845 Dep.setLatency(Lat);
846 } else if (UseI->isBundle()) {
847 const SIRegisterInfo *TRI = getRegisterInfo();
848 auto Reg = Dep.getReg();
849 MachineBasicBlock::const_instr_iterator I(UseI->getIterator());
850 MachineBasicBlock::const_instr_iterator E(UseI->getParent()->instr_end());
851 unsigned Lat = InstrInfo.getInstrLatency(ItinData: getInstrItineraryData(), MI: *DefI);
852 for (++I; I != E && I->isBundledWithPred() && Lat; ++I) {
853 if (I->isMetaInstruction())
854 continue;
855 if (I->readsRegister(Reg, TRI))
856 break;
857 --Lat;
858 }
859 Dep.setLatency(Lat);
860 } else if (Dep.getLatency() == 0 && Dep.getReg() == AMDGPU::VCC_LO) {
861 // Work around the fact that SIInstrInfo::fixImplicitOperands modifies
862 // implicit operands which come from the MCInstrDesc, which can fool
863 // ScheduleDAGInstrs::addPhysRegDataDeps into treating them as implicit
864 // pseudo operands.
865 Dep.setLatency(InstrInfo.getSchedModel().computeOperandLatency(
866 DefMI: DefI, DefOperIdx: DefOpIdx, UseMI: UseI, UseOperIdx: UseOpIdx));
867 }
868}
869
870unsigned GCNSubtarget::getNSAThreshold(const MachineFunction &MF) const {
871 if (getGeneration() >= AMDGPUSubtarget::GFX12)
872 return 0; // Not MIMG encoding.
873
874 if (NSAThreshold.getNumOccurrences() > 0)
875 return std::max(a: NSAThreshold.getValue(), b: 2u);
876
877 int Value = MF.getFunction().getFnAttributeAsParsedInteger(
878 Kind: "amdgpu-nsa-threshold", Default: -1);
879 if (Value > 0)
880 return std::max(a: Value, b: 2);
881
882 return NSAThreshold;
883}
884
885GCNUserSGPRUsageInfo::GCNUserSGPRUsageInfo(const Function &F,
886 const GCNSubtarget &ST)
887 : ST(ST) {
888 const CallingConv::ID CC = F.getCallingConv();
889 const bool IsKernel =
890 CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL;
891
892 if (IsKernel && (!F.arg_empty() || ST.getImplicitArgNumBytes(F) != 0))
893 KernargSegmentPtr = true;
894
895 bool IsAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
896 if (IsAmdHsaOrMesa && !ST.hasFlatScratchEnabled())
897 PrivateSegmentBuffer = true;
898 else if (ST.isMesaGfxShader(F))
899 ImplicitBufferPtr = true;
900
901 if (!AMDGPU::isGraphics(CC)) {
902 if (!F.hasFnAttribute(Kind: "amdgpu-no-dispatch-ptr"))
903 DispatchPtr = true;
904
905 // FIXME: Can this always be disabled with < COv5?
906 if (!F.hasFnAttribute(Kind: "amdgpu-no-queue-ptr"))
907 QueuePtr = true;
908
909 if (!F.hasFnAttribute(Kind: "amdgpu-no-dispatch-id"))
910 DispatchID = true;
911 }
912
913 if (ST.hasFlatAddressSpace() && AMDGPU::isEntryFunctionCC(CC) &&
914 (IsAmdHsaOrMesa || ST.hasFlatScratchEnabled()) &&
915 // FlatScratchInit cannot be true for graphics CC if
916 // hasFlatScratchEnabled() is false.
917 (ST.hasFlatScratchEnabled() ||
918 (!AMDGPU::isGraphics(CC) &&
919 !F.hasFnAttribute(Kind: "amdgpu-no-flat-scratch-init"))) &&
920 !ST.hasArchitectedFlatScratch()) {
921 FlatScratchInit = true;
922 }
923
924 if (hasImplicitBufferPtr())
925 NumUsedUserSGPRs += getNumUserSGPRForField(ID: ImplicitBufferPtrID);
926
927 if (hasPrivateSegmentBuffer())
928 NumUsedUserSGPRs += getNumUserSGPRForField(ID: PrivateSegmentBufferID);
929
930 if (hasDispatchPtr())
931 NumUsedUserSGPRs += getNumUserSGPRForField(ID: DispatchPtrID);
932
933 if (hasQueuePtr())
934 NumUsedUserSGPRs += getNumUserSGPRForField(ID: QueuePtrID);
935
936 if (hasKernargSegmentPtr())
937 NumUsedUserSGPRs += getNumUserSGPRForField(ID: KernargSegmentPtrID);
938
939 if (hasDispatchID())
940 NumUsedUserSGPRs += getNumUserSGPRForField(ID: DispatchIdID);
941
942 if (hasFlatScratchInit())
943 NumUsedUserSGPRs += getNumUserSGPRForField(ID: FlatScratchInitID);
944
945 if (hasPrivateSegmentSize())
946 NumUsedUserSGPRs += getNumUserSGPRForField(ID: PrivateSegmentSizeID);
947}
948
949void GCNUserSGPRUsageInfo::allocKernargPreloadSGPRs(unsigned NumSGPRs) {
950 assert(NumKernargPreloadSGPRs + NumSGPRs <= AMDGPU::getMaxNumUserSGPRs(ST));
951 NumKernargPreloadSGPRs += NumSGPRs;
952 NumUsedUserSGPRs += NumSGPRs;
953}
954
955unsigned GCNUserSGPRUsageInfo::getNumFreeUserSGPRs() {
956 return AMDGPU::getMaxNumUserSGPRs(STI: ST) - NumUsedUserSGPRs;
957}
958