1//===-- GCNSubtarget.cpp - GCN Subtarget Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Implements the GCN specific subclass of TargetSubtarget.
11//
12//===----------------------------------------------------------------------===//
13
14#include "GCNSubtarget.h"
15#include "AMDGPUCallLowering.h"
16#include "AMDGPUInstructionSelector.h"
17#include "AMDGPULegalizerInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "AMDGPUSelectionDAGInfo.h"
20#include "AMDGPUTargetMachine.h"
21#include "SIMachineFunctionInfo.h"
22#include "Utils/AMDGPUBaseInfo.h"
23#include "llvm/ADT/SmallString.h"
24#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
25#include "llvm/CodeGen/MachinePipeliner.h"
26#include "llvm/CodeGen/MachineScheduler.h"
27#include "llvm/CodeGen/TargetFrameLowering.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/TargetParser/AMDGPUTargetParser.h"
31#include <algorithm>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "gcn-subtarget"
36
37#define GET_SUBTARGETINFO_TARGET_DESC
38#define GET_SUBTARGETINFO_CTOR
39#define AMDGPUSubtarget GCNSubtarget
40#include "AMDGPUGenSubtargetInfo.inc"
41#undef AMDGPUSubtarget
42
43static cl::opt<bool> EnableVGPRIndexMode(
44 "amdgpu-vgpr-index-mode",
45 cl::desc("Use GPR indexing mode instead of movrel for vector indexing"),
46 cl::init(Val: false));
47
48static cl::opt<bool> UseAA("amdgpu-use-aa-in-codegen",
49 cl::desc("Enable the use of AA during codegen."),
50 cl::init(Val: true));
51
52static cl::opt<unsigned>
53 NSAThreshold("amdgpu-nsa-threshold",
54 cl::desc("Number of addresses from which to enable MIMG NSA."),
55 cl::init(Val: 2), cl::Hidden);
56
57GCNSubtarget::~GCNSubtarget() = default;
58
59static AMDGPUSubtarget::Generation computeDefaultGeneration(const Triple &TT) {
60 // Legacy triples without a subarch default to the first target that supports
61 // flat addressing for HSA, otherwise the first amdgcn target.
62 if (TT.getSubArch() == Triple::NoSubArch)
63 return TT.getOS() == Triple::AMDHSA ? AMDGPUSubtarget::SEA_ISLANDS
64 : AMDGPUSubtarget::SOUTHERN_ISLANDS;
65
66 switch (AMDGPU::getMajorSubArch(SubArch: TT.getSubArch())) {
67 case Triple::AMDGPUSubArch6:
68 return AMDGPUSubtarget::SOUTHERN_ISLANDS;
69 case Triple::AMDGPUSubArch7:
70 return AMDGPUSubtarget::SEA_ISLANDS;
71 case Triple::AMDGPUSubArch8:
72 case Triple::AMDGPUSubArch810:
73 return AMDGPUSubtarget::VOLCANIC_ISLANDS;
74 case Triple::AMDGPUSubArch9:
75 case Triple::AMDGPUSubArch908:
76 case Triple::AMDGPUSubArch90A:
77 case Triple::AMDGPUSubArch9_4:
78 return AMDGPUSubtarget::GFX9;
79 case Triple::AMDGPUSubArch10_1:
80 case Triple::AMDGPUSubArch10_3:
81 return AMDGPUSubtarget::GFX10;
82 case Triple::AMDGPUSubArch11:
83 case Triple::AMDGPUSubArch11_7:
84 return AMDGPUSubtarget::GFX11;
85 case Triple::AMDGPUSubArch12:
86 case Triple::AMDGPUSubArch12_5:
87 case Triple::AMDGPUSubArch1250S:
88 return AMDGPUSubtarget::GFX12;
89 case Triple::AMDGPUSubArch13:
90 return AMDGPUSubtarget::GFX13;
91 default:
92 reportFatalUsageError(reason: "invalid subarch for amdgpu");
93 }
94}
95
96GCNSubtarget &GCNSubtarget::initializeSubtargetDependencies(const Triple &TT,
97 StringRef GPU,
98 StringRef FS) {
99 // Determine default and user-specified characteristics
100 //
101 // We want to be able to turn these off, but making this a subtarget feature
102 // for SI has the unhelpful behavior that it unsets everything else if you
103 // disable it.
104 //
105 // Similarly we want enable-prt-strict-null to be on by default and not to
106 // unset everything else if it is disabled
107
108 SmallString<256> FullFS("+load-store-opt,+enable-ds128,");
109
110 // Turn on features that HSA ABI requires. Also turn on FlatForGlobal by
111 // default
112 if (isAmdHsaOS())
113 FullFS += "+flat-for-global,+unaligned-access-mode,+trap-handler,";
114
115 FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
116
117 // Disable mutually exclusive bits.
118 if (FS.contains_insensitive(Other: "+wavefrontsize")) {
119 if (!FS.contains_insensitive(Other: "wavefrontsize16"))
120 FullFS += "-wavefrontsize16,";
121 if (!FS.contains_insensitive(Other: "wavefrontsize32"))
122 FullFS += "-wavefrontsize32,";
123 if (!FS.contains_insensitive(Other: "wavefrontsize64"))
124 FullFS += "-wavefrontsize64,";
125 }
126
127 FullFS += FS;
128
129 ParseSubtargetFeatures(CPU: GPU, /*TuneCPU*/ GPU, FS: FullFS);
130
131 // Implement the "generic" processors, which acts as the default when no
132 // generation features are enabled (e.g for -mcpu=''). HSA OS defaults to
133 // the first amdgcn target that supports flat addressing. Other OSes defaults
134 // to the first amdgcn target.
135 if (Gen == AMDGPUSubtarget::INVALID) {
136 Gen = computeDefaultGeneration(TT);
137 // Assume wave64 for the unknown target, if not explicitly set.
138 if (getWavefrontSizeLog2() == 0)
139 WavefrontSizeLog2 = 6;
140 } else if (!hasFeature(Feature: AMDGPU::FeatureWavefrontSize32) &&
141 !hasFeature(Feature: AMDGPU::FeatureWavefrontSize64)) {
142 // If there is no default wave size it must be a generation before gfx10,
143 // these have FeatureWavefrontSize64 in their definition already. For gfx10+
144 // set wave32 as a default.
145 ToggleFeature(FB: AMDGPU::FeatureWavefrontSize32);
146 WavefrontSizeLog2 = getGeneration() >= AMDGPUSubtarget::GFX10 ? 5 : 6;
147 }
148
149 // We don't support FP64 for EG/NI atm.
150 assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS));
151
152 // Targets must either support 64-bit offsets for MUBUF instructions, and/or
153 // support flat operations, otherwise they cannot access a 64-bit global
154 // address space
155 assert(hasAddr64() || hasFlat());
156 // Unless +-flat-for-global is specified, turn on FlatForGlobal for targets
157 // that do not support ADDR64 variants of MUBUF instructions. Such targets
158 // cannot use a 64 bit offset with a MUBUF instruction to access the global
159 // address space
160 if (!hasAddr64() && !FS.contains(Other: "flat-for-global") && !UseFlatForGlobal) {
161 ToggleFeature(FB: AMDGPU::FeatureUseFlatForGlobal);
162 UseFlatForGlobal = true;
163 }
164 // Unless +-flat-for-global is specified, use MUBUF instructions for global
165 // address space access if flat operations are not available.
166 if (!hasFlat() && !FS.contains(Other: "flat-for-global") && UseFlatForGlobal) {
167 ToggleFeature(FB: AMDGPU::FeatureUseFlatForGlobal);
168 UseFlatForGlobal = false;
169 }
170
171 // Set defaults if needed.
172 if (MaxPrivateElementSize == 0)
173 MaxPrivateElementSize = 4;
174
175 if (LDSBankCount == 0)
176 LDSBankCount = 32;
177
178 if (MaxWavesPerEU == 0)
179 MaxWavesPerEU = 10;
180
181 if (FlatOffsetBitWidth == 0)
182 FlatOffsetBitWidth = 13;
183
184 LocalMemorySize = AMDGPU::IsaInfo::getLocalMemorySize(STI: *this);
185 AddressableLocalMemorySize =
186 AMDGPU::IsaInfo::getAddressableLocalMemorySize(STI: *this);
187 // LDS Allocation Granularity calculated in bytes from dwords
188 LDSAllocationGranularity =
189 AMDGPU::getLdsDwGranularity(ST: *this) * sizeof(uint32_t);
190
191 HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
192 HasSMulHi = getGeneration() >= AMDGPUSubtarget::GFX9;
193
194 // InstCacheLineSize is set from TableGen subtarget features
195 // (FeatureInstCacheLineSize64 / FeatureInstCacheLineSize128).
196 // Fall back to 64 if no feature was specified (e.g. generic targets).
197 if (InstCacheLineSize == 0)
198 InstCacheLineSize = 64;
199
200 assert(llvm::isPowerOf2_32(InstCacheLineSize) &&
201 "InstCacheLineSize must be a power of 2");
202
203 return *this;
204}
205
206void GCNSubtarget::checkSubtargetFeatures(const Function &F) const {
207 LLVMContext &Ctx = F.getContext();
208 if (hasFeature(Feature: AMDGPU::FeatureWavefrontSize32) &&
209 hasFeature(Feature: AMDGPU::FeatureWavefrontSize64)) {
210 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
211 F, "must specify exactly one of wavefrontsize32 and wavefrontsize64"));
212 }
213}
214
215// TODO: Validate subarch for subtarget
216
217GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
218 const GCNTargetMachine &TM, bool BufferOOBRelaxed,
219 bool TBufferOOBRelaxed,
220 AMDGPU::TargetIDSetting XnackSetting,
221 AMDGPU::TargetIDSetting SramEccSetting)
222 : // clang-format off
223 AMDGPUGenSubtargetInfo(TT, GPU, /*TuneCPU*/ GPU, FS),
224 AMDGPUSubtarget(TT),
225 TargetID(AMDGPU::createAMDGPUTargetID(STI: *this, FeatureString: "")),
226 InstrItins(getInstrItineraryForCPU(CPU: GPU)),
227 BufferOOBRelaxed(BufferOOBRelaxed),
228 TBufferOOBRelaxed(TBufferOOBRelaxed),
229 InstrInfo(initializeSubtargetDependencies(TT, GPU, FS)),
230 TLInfo(TM, *this),
231 // Frame index expansion sometimes assumes the low bit of SP is 0
232 FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0,
233 /*TransAl=*/Align(4)) {
234
235 // clang-format on
236
237 // Apply the module flag's xnack setting if the target supports on/off modes.
238 // Targets without on/off mode support have xnack always on and ignore module
239 // flags.
240 if (hasXNACKOnOffModes())
241 TargetID.setXnackSetting(XnackSetting);
242
243 // Apply the module flag's sramecc setting if the target supports it.
244 if (supportsSRAMECC())
245 TargetID.setSramEccSetting(SramEccSetting);
246
247 LLVM_DEBUG(dbgs() << "xnack setting for subtarget: "
248 << TargetID.getXnackSetting() << '\n');
249 LLVM_DEBUG(dbgs() << "sramecc setting for subtarget: "
250 << TargetID.getSramEccSetting() << '\n');
251
252 NumWorkGroupSIMDs =
253 AMDGPU::getNumWorkGroupSIMDs(FullSIMDMode: AMDGPU::isFullSIMDMode(STI: *this));
254
255 TSInfo = std::make_unique<AMDGPUSelectionDAGInfo>();
256
257 CallLoweringInfo = std::make_unique<AMDGPUCallLowering>(args: *getTargetLowering());
258 InlineAsmLoweringInfo =
259 std::make_unique<InlineAsmLowering>(args: getTargetLowering());
260 Legalizer = std::make_unique<AMDGPULegalizerInfo>(args&: *this, args: TM);
261 RegBankInfo = std::make_unique<AMDGPURegisterBankInfo>(args&: *this);
262 InstSelector =
263 std::make_unique<AMDGPUInstructionSelector>(args&: *this, args&: *RegBankInfo);
264}
265
266const SelectionDAGTargetInfo *GCNSubtarget::getSelectionDAGInfo() const {
267 return TSInfo.get();
268}
269
270unsigned GCNSubtarget::getConstantBusLimit(unsigned Opcode) const {
271 if (getGeneration() < GFX10)
272 return 1;
273
274 switch (Opcode) {
275 case AMDGPU::V_LSHLREV_B64_e64:
276 case AMDGPU::V_LSHLREV_B64_gfx10:
277 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
278 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
279 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
280 case AMDGPU::V_LSHL_B64_e64:
281 case AMDGPU::V_LSHRREV_B64_e64:
282 case AMDGPU::V_LSHRREV_B64_gfx10:
283 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
284 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
285 case AMDGPU::V_LSHR_B64_e64:
286 case AMDGPU::V_ASHRREV_I64_e64:
287 case AMDGPU::V_ASHRREV_I64_gfx10:
288 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
289 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
290 case AMDGPU::V_ASHR_I64_e64:
291 return 1;
292 }
293
294 return 2;
295}
296
297/// This list was mostly derived from experimentation.
298bool GCNSubtarget::zeroesHigh16BitsOfDest(unsigned Opcode) const {
299 switch (Opcode) {
300 case AMDGPU::V_CVT_F16_F32_e32:
301 case AMDGPU::V_CVT_F16_F32_e64:
302 case AMDGPU::V_CVT_F16_U16_e32:
303 case AMDGPU::V_CVT_F16_U16_e64:
304 case AMDGPU::V_CVT_F16_I16_e32:
305 case AMDGPU::V_CVT_F16_I16_e64:
306 case AMDGPU::V_RCP_F16_e64:
307 case AMDGPU::V_RCP_F16_e32:
308 case AMDGPU::V_RSQ_F16_e64:
309 case AMDGPU::V_RSQ_F16_e32:
310 case AMDGPU::V_SQRT_F16_e64:
311 case AMDGPU::V_SQRT_F16_e32:
312 case AMDGPU::V_LOG_F16_e64:
313 case AMDGPU::V_LOG_F16_e32:
314 case AMDGPU::V_EXP_F16_e64:
315 case AMDGPU::V_EXP_F16_e32:
316 case AMDGPU::V_SIN_F16_e64:
317 case AMDGPU::V_SIN_F16_e32:
318 case AMDGPU::V_COS_F16_e64:
319 case AMDGPU::V_COS_F16_e32:
320 case AMDGPU::V_FLOOR_F16_e64:
321 case AMDGPU::V_FLOOR_F16_e32:
322 case AMDGPU::V_CEIL_F16_e64:
323 case AMDGPU::V_CEIL_F16_e32:
324 case AMDGPU::V_TRUNC_F16_e64:
325 case AMDGPU::V_TRUNC_F16_e32:
326 case AMDGPU::V_RNDNE_F16_e64:
327 case AMDGPU::V_RNDNE_F16_e32:
328 case AMDGPU::V_FRACT_F16_e64:
329 case AMDGPU::V_FRACT_F16_e32:
330 case AMDGPU::V_FREXP_MANT_F16_e64:
331 case AMDGPU::V_FREXP_MANT_F16_e32:
332 case AMDGPU::V_FREXP_EXP_I16_F16_e64:
333 case AMDGPU::V_FREXP_EXP_I16_F16_e32:
334 case AMDGPU::V_LDEXP_F16_e64:
335 case AMDGPU::V_LDEXP_F16_e32:
336 case AMDGPU::V_LSHLREV_B16_e64:
337 case AMDGPU::V_LSHLREV_B16_e32:
338 case AMDGPU::V_LSHRREV_B16_e64:
339 case AMDGPU::V_LSHRREV_B16_e32:
340 case AMDGPU::V_ASHRREV_I16_e64:
341 case AMDGPU::V_ASHRREV_I16_e32:
342 case AMDGPU::V_ADD_U16_e64:
343 case AMDGPU::V_ADD_U16_e32:
344 case AMDGPU::V_SUB_U16_e64:
345 case AMDGPU::V_SUB_U16_e32:
346 case AMDGPU::V_SUBREV_U16_e64:
347 case AMDGPU::V_SUBREV_U16_e32:
348 case AMDGPU::V_MUL_LO_U16_e64:
349 case AMDGPU::V_MUL_LO_U16_e32:
350 case AMDGPU::V_ADD_F16_e64:
351 case AMDGPU::V_ADD_F16_e32:
352 case AMDGPU::V_SUB_F16_e64:
353 case AMDGPU::V_SUB_F16_e32:
354 case AMDGPU::V_SUBREV_F16_e64:
355 case AMDGPU::V_SUBREV_F16_e32:
356 case AMDGPU::V_MUL_F16_e64:
357 case AMDGPU::V_MUL_F16_e32:
358 case AMDGPU::V_MAX_F16_e64:
359 case AMDGPU::V_MAX_F16_e32:
360 case AMDGPU::V_MIN_F16_e64:
361 case AMDGPU::V_MIN_F16_e32:
362 case AMDGPU::V_MAX_U16_e64:
363 case AMDGPU::V_MAX_U16_e32:
364 case AMDGPU::V_MIN_U16_e64:
365 case AMDGPU::V_MIN_U16_e32:
366 case AMDGPU::V_MAX_I16_e64:
367 case AMDGPU::V_MAX_I16_e32:
368 case AMDGPU::V_MIN_I16_e64:
369 case AMDGPU::V_MIN_I16_e32:
370 case AMDGPU::V_MAD_F16_e64:
371 case AMDGPU::V_MAD_U16_e64:
372 case AMDGPU::V_MAD_I16_e64:
373 case AMDGPU::V_FMA_F16_e64:
374 case AMDGPU::V_DIV_FIXUP_F16_e64:
375 // On gfx10, all 16-bit instructions preserve the high bits.
376 return getGeneration() <= AMDGPUSubtarget::GFX9;
377 case AMDGPU::V_MADAK_F16:
378 case AMDGPU::V_MADMK_F16:
379 case AMDGPU::V_MAC_F16_e64:
380 case AMDGPU::V_MAC_F16_e32:
381 case AMDGPU::V_FMAMK_F16:
382 case AMDGPU::V_FMAAK_F16:
383 case AMDGPU::V_FMAC_F16_e64:
384 case AMDGPU::V_FMAC_F16_e32:
385 // In gfx9, the preferred handling of the unused high 16-bits changed. Most
386 // instructions maintain the legacy behavior of 0ing. Some instructions
387 // changed to preserving the high bits.
388 return getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
389 case AMDGPU::V_MAD_MIXLO_F16:
390 case AMDGPU::V_MAD_MIXHI_F16:
391 default:
392 return false;
393 }
394}
395
396void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
397 const SchedRegion &Region) const {
398 // Track register pressure so the scheduler can try to decrease
399 // pressure once register usage is above the threshold defined by
400 // SIRegisterInfo::getRegPressureSetLimit()
401 Policy.ShouldTrackPressure = true;
402
403 const Function &F = Region.RegionBegin->getMF()->getFunction();
404 if (AMDGPU::getSchedStrategy(F) == "coexec") {
405 Policy.OnlyTopDown = true;
406 Policy.OnlyBottomUp = false;
407 return;
408 }
409
410 // Enabling both top down and bottom up scheduling seems to give us less
411 // register spills than just using one of these approaches on its own.
412 Policy.OnlyTopDown = false;
413 Policy.OnlyBottomUp = false;
414
415 // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
416 if (!enableSIScheduler())
417 Policy.ShouldTrackLaneMasks = true;
418}
419
420void GCNSubtarget::overridePostRASchedPolicy(MachineSchedPolicy &Policy,
421 const SchedRegion &Region) const {
422 const Function &F = Region.RegionBegin->getMF()->getFunction();
423 Attribute PostRADirectionAttr = F.getFnAttribute(Kind: "amdgpu-post-ra-direction");
424 if (!PostRADirectionAttr.isValid())
425 return;
426
427 StringRef PostRADirectionStr = PostRADirectionAttr.getValueAsString();
428 if (PostRADirectionStr == "topdown") {
429 Policy.OnlyTopDown = true;
430 Policy.OnlyBottomUp = false;
431 } else if (PostRADirectionStr == "bottomup") {
432 Policy.OnlyTopDown = false;
433 Policy.OnlyBottomUp = true;
434 } else if (PostRADirectionStr == "bidirectional") {
435 Policy.OnlyTopDown = false;
436 Policy.OnlyBottomUp = false;
437 } else {
438 DiagnosticInfoOptimizationFailure Diag(
439 F, F.getSubprogram(), "invalid value for postRA direction attribute");
440 F.getContext().diagnose(DI: Diag);
441 }
442
443 LLVM_DEBUG({
444 const char *DirStr = "default";
445 if (Policy.OnlyTopDown && !Policy.OnlyBottomUp)
446 DirStr = "topdown";
447 else if (!Policy.OnlyTopDown && Policy.OnlyBottomUp)
448 DirStr = "bottomup";
449 else if (!Policy.OnlyTopDown && !Policy.OnlyBottomUp)
450 DirStr = "bidirectional";
451
452 dbgs() << "Post-MI-sched direction (" << F.getName() << "): " << DirStr
453 << '\n';
454 });
455}
456
457void GCNSubtarget::overridePipelinerPolicy(
458 MachinePipelinerPolicy &Policy) const {
459 Policy.ShouldLimitRegPressure = true;
460}
461
462void GCNSubtarget::mirFileLoaded(MachineFunction &MF) const {
463 if (isWave32()) {
464 // Fix implicit $vcc operands after MIParser has verified that they match
465 // the instruction definitions.
466 for (auto &MBB : MF) {
467 for (auto &MI : MBB)
468 InstrInfo.fixImplicitOperands(MI);
469 }
470 }
471}
472
473bool GCNSubtarget::hasMadF16() const {
474 return InstrInfo.pseudoToMCOpcode(Opcode: AMDGPU::V_MAD_F16_e64) != -1;
475}
476
477bool GCNSubtarget::useVGPRIndexMode() const {
478 return hasVGPRIndexMode() && (!hasMovrel() || EnableVGPRIndexMode);
479}
480
481bool GCNSubtarget::useAA() const { return UseAA; }
482
483unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
484 return AMDGPU::IsaInfo::getOccupancyWithNumSGPRs(STI: *this, SGPRs);
485}
486
487unsigned
488GCNSubtarget::getOccupancyWithNumVGPRs(unsigned NumVGPRs,
489 unsigned DynamicVGPRBlockSize) const {
490 return AMDGPU::IsaInfo::getNumWavesPerEUWithNumVGPRs(STI: *this, NumVGPRs,
491 DynamicVGPRBlockSize);
492}
493
494unsigned
495GCNSubtarget::getBaseReservedNumSGPRs(const bool HasFlatScratch) const {
496 if (getGeneration() >= AMDGPUSubtarget::GFX10)
497 return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
498
499 if (HasFlatScratch || HasArchitectedFlatScratch) {
500 if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
501 return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
502 if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
503 return 4; // FLAT_SCRATCH, VCC (in that order).
504 }
505
506 if (isXNACKEnabled())
507 return 4; // XNACK, VCC (in that order).
508 return 2; // VCC.
509}
510
511unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
512 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
513 return getBaseReservedNumSGPRs(HasFlatScratch: MFI.getUserSGPRInfo().hasFlatScratchInit());
514}
515
516unsigned GCNSubtarget::getReservedNumSGPRs(const Function &F) const {
517 // In principle we do not need to reserve SGPR pair used for flat_scratch if
518 // we know flat instructions do not access the stack anywhere in the
519 // program. For now assume it's needed if we have flat instructions.
520 const bool KernelUsesFlatScratch = hasFlatAddressSpace();
521 return getBaseReservedNumSGPRs(HasFlatScratch: KernelUsesFlatScratch);
522}
523
524std::pair<unsigned, unsigned>
525GCNSubtarget::computeOccupancy(const Function &F, unsigned LDSSize,
526 unsigned NumSGPRs, unsigned NumVGPRs) const {
527 unsigned DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
528 auto [MinOcc, MaxOcc] = getOccupancyWithWorkGroupSizes(LDSBytes: LDSSize, F);
529 unsigned SGPROcc = getOccupancyWithNumSGPRs(SGPRs: NumSGPRs);
530 unsigned VGPROcc = getOccupancyWithNumVGPRs(NumVGPRs, DynamicVGPRBlockSize);
531
532 // Maximum occupancy may be further limited by high SGPR/VGPR usage.
533 MaxOcc = std::min(l: {MaxOcc, SGPROcc, VGPROcc});
534 return {std::min(a: MinOcc, b: MaxOcc), MaxOcc};
535}
536
537unsigned GCNSubtarget::getBaseMaxNumSGPRs(
538 const Function &F, std::pair<unsigned, unsigned> WavesPerEU,
539 unsigned PreloadedSGPRs, unsigned ReservedNumSGPRs) const {
540 // Compute maximum number of SGPRs function can use using default/requested
541 // minimum number of waves per execution unit.
542 unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: false);
543 unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: true);
544
545 // Check if maximum number of SGPRs was explicitly requested using
546 // "amdgpu-num-sgpr" attribute.
547 unsigned Requested =
548 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-num-sgpr", Default: MaxNumSGPRs);
549
550 if (Requested != MaxNumSGPRs) {
551 // Make sure requested value does not violate subtarget's specifications.
552 if (Requested && (Requested <= ReservedNumSGPRs))
553 Requested = 0;
554
555 // If more SGPRs are required to support the input user/system SGPRs,
556 // increase to accommodate them.
557 //
558 // FIXME: This really ends up using the requested number of SGPRs + number
559 // of reserved special registers in total. Theoretically you could re-use
560 // the last input registers for these special registers, but this would
561 // require a lot of complexity to deal with the weird aliasing.
562 unsigned InputNumSGPRs = PreloadedSGPRs;
563 if (Requested && Requested < InputNumSGPRs)
564 Requested = InputNumSGPRs;
565
566 // Make sure requested value is compatible with values implied by
567 // default/requested minimum/maximum number of waves per execution unit.
568 if (Requested && Requested > getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: false))
569 Requested = 0;
570 if (WavesPerEU.second && Requested &&
571 Requested < getMinNumSGPRs(WavesPerEU: WavesPerEU.second))
572 Requested = 0;
573
574 if (Requested)
575 MaxNumSGPRs = Requested;
576 }
577
578 if (hasSGPRInitBug())
579 MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
580
581 return std::min(a: MaxNumSGPRs - ReservedNumSGPRs, b: MaxAddressableNumSGPRs);
582}
583
584unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
585 const Function &F = MF.getFunction();
586 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
587 return getBaseMaxNumSGPRs(F, WavesPerEU: MFI.getWavesPerEU(), PreloadedSGPRs: MFI.getNumPreloadedSGPRs(),
588 ReservedNumSGPRs: getReservedNumSGPRs(MF));
589}
590
591unsigned GCNSubtarget::getMaxNumPreloadedSGPRs() const {
592 using USI = GCNUserSGPRUsageInfo;
593 // Max number of user SGPRs
594 const unsigned MaxUserSGPRs =
595 USI::getNumUserSGPRForField(ID: USI::PrivateSegmentBufferID) +
596 USI::getNumUserSGPRForField(ID: USI::DispatchPtrID) +
597 USI::getNumUserSGPRForField(ID: USI::QueuePtrID) +
598 USI::getNumUserSGPRForField(ID: USI::KernargSegmentPtrID) +
599 USI::getNumUserSGPRForField(ID: USI::DispatchIdID) +
600 USI::getNumUserSGPRForField(ID: USI::FlatScratchInitID) +
601 USI::getNumUserSGPRForField(ID: USI::ImplicitBufferPtrID);
602
603 // Max number of system SGPRs
604 const unsigned MaxSystemSGPRs = 1 + // WorkGroupIDX
605 1 + // WorkGroupIDY
606 1 + // WorkGroupIDZ
607 1 + // WorkGroupInfo
608 1; // private segment wave byte offset
609
610 // Max number of synthetic SGPRs
611 const unsigned SyntheticSGPRs = 1; // LDSKernelId
612
613 return MaxUserSGPRs + MaxSystemSGPRs + SyntheticSGPRs;
614}
615
616unsigned GCNSubtarget::getMaxNumSGPRs(const Function &F) const {
617 return getBaseMaxNumSGPRs(F, WavesPerEU: getWavesPerEU(F), PreloadedSGPRs: getMaxNumPreloadedSGPRs(),
618 ReservedNumSGPRs: getReservedNumSGPRs(F));
619}
620
621unsigned GCNSubtarget::getBaseMaxNumVGPRs(
622 const Function &F, std::pair<unsigned, unsigned> NumVGPRBounds) const {
623 const auto [Min, Max] = NumVGPRBounds;
624
625 // Check if maximum number of VGPRs was explicitly requested using
626 // "amdgpu-num-vgpr" attribute.
627
628 unsigned Requested = F.getFnAttributeAsParsedInteger(Kind: "amdgpu-num-vgpr", Default: Max);
629 if (Requested != Max && hasGFX90AInsts())
630 Requested *= 2;
631
632 // Make sure requested value is inside the range of possible VGPR usage.
633 return std::clamp(val: Requested, lo: Min, hi: Max);
634}
635
636unsigned GCNSubtarget::getMaxNumVGPRs(const Function &F) const {
637 unsigned DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
638 std::pair<unsigned, unsigned> Waves = getWavesPerEU(F);
639 return getBaseMaxNumVGPRs(
640 F, NumVGPRBounds: {getMinNumVGPRs(WavesPerEU: Waves.second, DynamicVGPRBlockSize),
641 getMaxNumVGPRs(WavesPerEU: Waves.first, DynamicVGPRBlockSize)});
642}
643
644unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
645 return getMaxNumVGPRs(F: MF.getFunction());
646}
647
648std::pair<unsigned, unsigned>
649GCNSubtarget::getMaxNumVectorRegs(const Function &F) const {
650 const unsigned MaxVectorRegs = getMaxNumVGPRs(F);
651
652 unsigned MaxNumVGPRs = MaxVectorRegs;
653 unsigned MaxNumAGPRs = 0;
654 unsigned NumArchVGPRs = getAddressableNumArchVGPRs();
655
656 // On GFX90A, the number of VGPRs and AGPRs need not be equal. Theoretically,
657 // a wave may have up to 512 total vector registers combining together both
658 // VGPRs and AGPRs. Hence, in an entry function without calls and without
659 // AGPRs used within it, it is possible to use the whole vector register
660 // budget for VGPRs.
661 //
662 // TODO: it shall be possible to estimate maximum AGPR/VGPR pressure and split
663 // register file accordingly.
664 if (hasGFX90AInsts()) {
665 unsigned MinNumAGPRs = 0;
666 const unsigned TotalNumAGPRs = AMDGPU::AGPR_32RegClass.getNumRegs();
667
668 const std::pair<unsigned, unsigned> DefaultNumAGPR = {~0u, ~0u};
669
670 // TODO: The lower bound should probably force the number of required
671 // registers up, overriding amdgpu-waves-per-eu.
672 std::tie(args&: MinNumAGPRs, args&: MaxNumAGPRs) =
673 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-agpr-alloc", Default: DefaultNumAGPR,
674 /*OnlyFirstRequired=*/true);
675
676 if (MinNumAGPRs == DefaultNumAGPR.first) {
677 // Default to splitting half the registers if AGPRs are required.
678 MinNumAGPRs = MaxNumAGPRs = MaxVectorRegs / 2;
679 } else {
680 // Align to accum_offset's allocation granularity.
681 MinNumAGPRs = alignTo(Value: MinNumAGPRs, Align: 4);
682
683 MinNumAGPRs = std::min(a: MinNumAGPRs, b: TotalNumAGPRs);
684 }
685
686 // Clamp values to be inbounds of our limits, and ensure min <= max.
687
688 MaxNumAGPRs = std::min(a: std::max(a: MinNumAGPRs, b: MaxNumAGPRs), b: MaxVectorRegs);
689 MinNumAGPRs = std::min(l: {MinNumAGPRs, TotalNumAGPRs, MaxNumAGPRs});
690
691 MaxNumVGPRs = std::min(a: MaxVectorRegs - MinNumAGPRs, b: NumArchVGPRs);
692 MaxNumAGPRs = std::min(a: MaxVectorRegs - MaxNumVGPRs, b: MaxNumAGPRs);
693
694 assert(MaxNumVGPRs + MaxNumAGPRs <= MaxVectorRegs &&
695 MaxNumAGPRs <= TotalNumAGPRs && MaxNumVGPRs <= NumArchVGPRs &&
696 "invalid register counts");
697 } else if (hasMAIInsts()) {
698 // On gfx908 the number of AGPRs always equals the number of VGPRs.
699 MaxNumAGPRs = MaxNumVGPRs = MaxVectorRegs;
700 }
701
702 return std::pair(MaxNumVGPRs, MaxNumAGPRs);
703}
704
705// Check to which source operand UseOpIdx points to and return a pointer to the
706// operand of the corresponding source modifier.
707// Return nullptr if UseOpIdx either doesn't point to src0/1/2 or if there is no
708// operand for the corresponding source modifier.
709static const MachineOperand *
710getVOP3PSourceModifierFromOpIdx(const MachineInstr &UseI, int UseOpIdx,
711 const SIInstrInfo &InstrInfo) {
712 AMDGPU::OpName UseName =
713 AMDGPU::getOperandIdxName(Opcode: UseI.getOpcode(), Idx: UseOpIdx);
714 switch (UseName) {
715 case AMDGPU::OpName::src0:
716 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src0_modifiers);
717 case AMDGPU::OpName::src1:
718 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src1_modifiers);
719 case AMDGPU::OpName::src2:
720 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src2_modifiers);
721 default:
722 return nullptr;
723 }
724}
725
726// Get the subreg idx of the subreg that is used by the given instruction
727// operand, considering the given op_sel modifier.
728// Return 0 if the whole register is used or as a conservative fallback.
729static unsigned getEffectiveSubRegIdx(const SIRegisterInfo &TRI,
730 const SIInstrInfo &InstrInfo,
731 const MachineInstr &I,
732 const MachineOperand &Op) {
733 if (!InstrInfo.isVOP3P(MI: I) || InstrInfo.isWMMA(MI: I) || InstrInfo.isSWMMAC(MI: I))
734 return AMDGPU::NoSubRegister;
735
736 const MachineOperand *OpMod =
737 getVOP3PSourceModifierFromOpIdx(UseI: I, UseOpIdx: Op.getOperandNo(), InstrInfo);
738 if (!OpMod)
739 return AMDGPU::NoSubRegister;
740
741 // Note: the FMA_MIX* and MAD_MIX* instructions have different semantics for
742 // the op_sel and op_sel_hi source modifiers:
743 // - op_sel: selects low/high operand bits as input to the operation;
744 // has only meaning for 16-bit source operands
745 // - op_sel_hi: specifies the size of the source operands (16 or 32 bits);
746 // a value of 0 indicates 32 bit, 1 indicates 16 bit
747 // For the other VOP3P instructions, the semantics are:
748 // - op_sel: selects low/high operand bits as input to the operation which
749 // results in the lower-half of the destination
750 // - op_sel_hi: selects the low/high operand bits as input to the operation
751 // which results in the higher-half of the destination
752 int64_t OpSel = OpMod->getImm() & SISrcMods::OP_SEL_0;
753 int64_t OpSelHi = OpMod->getImm() & SISrcMods::OP_SEL_1;
754
755 // Check if all parts of the register are being used (= op_sel and op_sel_hi
756 // differ for VOP3P or op_sel_hi=0 for VOP3PMix). In that case we can return
757 // early.
758 if ((!InstrInfo.isVOP3PMix(MI: I) && (!OpSel || !OpSelHi) &&
759 (OpSel || OpSelHi)) ||
760 (InstrInfo.isVOP3PMix(MI: I) && !OpSelHi))
761 return AMDGPU::NoSubRegister;
762
763 const MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
764 const TargetRegisterClass *RC = TRI.getRegClassForOperandReg(MRI, MO: Op);
765
766 if (unsigned SubRegIdx = OpSel ? AMDGPU::sub1 : AMDGPU::sub0;
767 TRI.getSubClassWithSubReg(RC, SubRegIdx) == RC)
768 return SubRegIdx;
769 if (unsigned SubRegIdx = OpSel ? AMDGPU::hi16 : AMDGPU::lo16;
770 TRI.getSubClassWithSubReg(RC, SubRegIdx) == RC)
771 return SubRegIdx;
772
773 return AMDGPU::NoSubRegister;
774}
775
776Register GCNSubtarget::getRealSchedDependency(const MachineInstr &DefI,
777 int DefOpIdx,
778 const MachineInstr &UseI,
779 int UseOpIdx) const {
780 const SIRegisterInfo *TRI = getRegisterInfo();
781 const MachineOperand &DefOp = DefI.getOperand(i: DefOpIdx);
782 const MachineOperand &UseOp = UseI.getOperand(i: UseOpIdx);
783 Register DefReg = DefOp.getReg();
784 Register UseReg = UseOp.getReg();
785
786 // If the registers aren't restricted to a sub-register, there is no point in
787 // further analysis. This check makes only sense for virtual registers because
788 // physical registers may form a tuple and thus be part of a superregister
789 // although they are not a subregister themselves (vgpr0 is a "subreg" of
790 // vgpr0_vgpr1 without being a subreg in itself).
791 unsigned DefSubRegIdx = DefOp.getSubReg();
792 if (DefReg.isVirtual() && DefSubRegIdx == AMDGPU::NoSubRegister)
793 return DefReg;
794 unsigned UseSubRegIdx = getEffectiveSubRegIdx(TRI: *TRI, InstrInfo, I: UseI, Op: UseOp);
795 if (UseReg.isVirtual() && UseSubRegIdx == AMDGPU::NoSubRegister)
796 return DefReg;
797
798 if (!TRI->checkSubRegInterference(RegA: DefReg, SubA: DefSubRegIdx, RegB: UseReg, SubB: UseSubRegIdx))
799 return Register(); // No real dependency
800
801 // UseReg might be smaller or larger than DefReg, depending on the subreg and
802 // on whether DefReg is a subreg, too. -> Find the smaller one. This does not
803 // apply to virtual registers because we cannot construct a subreg for them.
804 if (DefReg.isVirtual())
805 return DefReg;
806 MCRegister DefMCReg =
807 DefSubRegIdx ? TRI->getSubReg(Reg: DefReg, Idx: DefSubRegIdx) : DefReg.asMCReg();
808 MCRegister UseMCReg =
809 UseSubRegIdx ? TRI->getSubReg(Reg: UseReg, Idx: UseSubRegIdx) : UseReg.asMCReg();
810 return TRI->isSubRegisterEq(RegA: DefMCReg, RegB: UseMCReg) ? UseMCReg : DefMCReg;
811}
812
813void GCNSubtarget::adjustSchedDependency(
814 SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx, SDep &Dep,
815 const TargetSchedModel *SchedModel) const {
816 if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() || !Def->isInstr() ||
817 !Use->isInstr())
818 return;
819
820 MachineInstr *DefI = Def->getInstr();
821 MachineInstr *UseI = Use->getInstr();
822
823 // Check for false latency on $tensorcnt / $asynccnt dependencies
824 if (Dep.getReg() == AMDGPU::TENSORcnt || Dep.getReg() == AMDGPU::ASYNCcnt) {
825 unsigned UseOp = UseI->getOpcode();
826 // Do not adjust latency for load->s_wait
827 bool IsBarrierCase =
828 InstrInfo.isLDSDMA(MI: *DefI) &&
829 (UseOp == AMDGPU::S_WAIT_TENSORCNT || UseOp == AMDGPU::S_WAIT_ASYNCCNT);
830 if (!IsBarrierCase) {
831 Dep.setLatency(1);
832 return;
833 }
834 }
835
836 if (Register Reg = getRealSchedDependency(DefI: *DefI, DefOpIdx, UseI: *UseI, UseOpIdx)) {
837 Dep.setReg(Reg);
838 } else {
839 Dep = SDep(Def, SDep::Artificial);
840 return; // This is not a data dependency anymore.
841 }
842
843 if (DefI->isBundle()) {
844 const SIRegisterInfo *TRI = getRegisterInfo();
845 auto Reg = Dep.getReg();
846 MachineBasicBlock::const_instr_iterator I(DefI->getIterator());
847 MachineBasicBlock::const_instr_iterator E(DefI->getParent()->instr_end());
848 unsigned Lat = 0;
849 for (++I; I != E && I->isBundledWithPred(); ++I) {
850 if (I->isMetaInstruction())
851 continue;
852 if (I->modifiesRegister(Reg, TRI))
853 Lat = InstrInfo.getInstrLatency(ItinData: getInstrItineraryData(), MI: *I);
854 else if (Lat)
855 --Lat;
856 }
857 Dep.setLatency(Lat);
858 } else if (UseI->isBundle()) {
859 const SIRegisterInfo *TRI = getRegisterInfo();
860 auto Reg = Dep.getReg();
861 MachineBasicBlock::const_instr_iterator I(UseI->getIterator());
862 MachineBasicBlock::const_instr_iterator E(UseI->getParent()->instr_end());
863 unsigned Lat = InstrInfo.getInstrLatency(ItinData: getInstrItineraryData(), MI: *DefI);
864 for (++I; I != E && I->isBundledWithPred() && Lat; ++I) {
865 if (I->isMetaInstruction())
866 continue;
867 if (I->readsRegister(Reg, TRI))
868 break;
869 --Lat;
870 }
871 Dep.setLatency(Lat);
872 } else if (Dep.getLatency() == 0 && Dep.getReg() == AMDGPU::VCC_LO) {
873 // Work around the fact that SIInstrInfo::fixImplicitOperands modifies
874 // implicit operands which come from the MCInstrDesc, which can fool
875 // ScheduleDAGInstrs::addPhysRegDataDeps into treating them as implicit
876 // pseudo operands.
877 Dep.setLatency(InstrInfo.getSchedModel().computeOperandLatency(
878 DefMI: DefI, DefOperIdx: DefOpIdx, UseMI: UseI, UseOperIdx: UseOpIdx));
879 }
880}
881
882unsigned GCNSubtarget::getNSAThreshold(const MachineFunction &MF) const {
883 if (getGeneration() >= AMDGPUSubtarget::GFX12)
884 return 0; // Not MIMG encoding.
885
886 if (NSAThreshold.getNumOccurrences() > 0)
887 return std::max(a: NSAThreshold.getValue(), b: 2u);
888
889 int Value = MF.getFunction().getFnAttributeAsParsedInteger(
890 Kind: "amdgpu-nsa-threshold", Default: -1);
891 if (Value > 0)
892 return std::max(a: Value, b: 2);
893
894 return NSAThreshold;
895}
896
897GCNUserSGPRUsageInfo::GCNUserSGPRUsageInfo(const Function &F,
898 const GCNSubtarget &ST)
899 : ST(ST) {
900 const CallingConv::ID CC = F.getCallingConv();
901 const bool IsKernel =
902 CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL;
903
904 if (IsKernel && (!F.arg_empty() || ST.getImplicitArgNumBytes(F) != 0))
905 KernargSegmentPtr = true;
906
907 bool IsAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
908 if (IsAmdHsaOrMesa && !ST.hasFlatScratchEnabled())
909 PrivateSegmentBuffer = true;
910 else if (ST.isMesaGfxShader(F))
911 ImplicitBufferPtr = true;
912
913 if (!AMDGPU::isGraphics(CC)) {
914 if (!F.hasFnAttribute(Kind: "amdgpu-no-dispatch-ptr"))
915 DispatchPtr = true;
916
917 // FIXME: Can this always be disabled with < COv5?
918 if (!F.hasFnAttribute(Kind: "amdgpu-no-queue-ptr"))
919 QueuePtr = true;
920
921 if (!F.hasFnAttribute(Kind: "amdgpu-no-dispatch-id"))
922 DispatchID = true;
923 }
924
925 if (ST.hasFlatAddressSpace() && AMDGPU::isEntryFunctionCC(CC) &&
926 (IsAmdHsaOrMesa || ST.hasFlatScratchEnabled()) &&
927 // FlatScratchInit cannot be true for graphics CC if
928 // hasFlatScratchEnabled() is false.
929 (ST.hasFlatScratchEnabled() ||
930 (!AMDGPU::isGraphics(CC) &&
931 !F.hasFnAttribute(Kind: "amdgpu-no-flat-scratch-init"))) &&
932 !ST.hasArchitectedFlatScratch()) {
933 FlatScratchInit = true;
934 }
935
936 if (hasImplicitBufferPtr())
937 NumUsedUserSGPRs += getNumUserSGPRForField(ID: ImplicitBufferPtrID);
938
939 if (hasPrivateSegmentBuffer())
940 NumUsedUserSGPRs += getNumUserSGPRForField(ID: PrivateSegmentBufferID);
941
942 if (hasDispatchPtr())
943 NumUsedUserSGPRs += getNumUserSGPRForField(ID: DispatchPtrID);
944
945 if (hasQueuePtr())
946 NumUsedUserSGPRs += getNumUserSGPRForField(ID: QueuePtrID);
947
948 if (hasKernargSegmentPtr())
949 NumUsedUserSGPRs += getNumUserSGPRForField(ID: KernargSegmentPtrID);
950
951 if (hasDispatchID())
952 NumUsedUserSGPRs += getNumUserSGPRForField(ID: DispatchIdID);
953
954 if (hasFlatScratchInit())
955 NumUsedUserSGPRs += getNumUserSGPRForField(ID: FlatScratchInitID);
956
957 if (hasPrivateSegmentSize())
958 NumUsedUserSGPRs += getNumUserSGPRForField(ID: PrivateSegmentSizeID);
959}
960
961void GCNUserSGPRUsageInfo::allocKernargPreloadSGPRs(unsigned NumSGPRs) {
962 assert(NumKernargPreloadSGPRs + NumSGPRs <= AMDGPU::getMaxNumUserSGPRs(ST));
963 NumKernargPreloadSGPRs += NumSGPRs;
964 NumUsedUserSGPRs += NumSGPRs;
965}
966
967unsigned GCNUserSGPRUsageInfo::getNumFreeUserSGPRs() {
968 return AMDGPU::getMaxNumUserSGPRs(STI: ST) - NumUsedUserSGPRs;
969}
970