1//===-- GCNSubtarget.cpp - GCN Subtarget Information ----------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Implements the GCN specific subclass of TargetSubtarget.
11//
12//===----------------------------------------------------------------------===//
13
14#include "GCNSubtarget.h"
15#include "AMDGPUCallLowering.h"
16#include "AMDGPUInstructionSelector.h"
17#include "AMDGPULegalizerInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "AMDGPUSelectionDAGInfo.h"
20#include "AMDGPUTargetMachine.h"
21#include "SIMachineFunctionInfo.h"
22#include "Utils/AMDGPUBaseInfo.h"
23#include "llvm/ADT/SmallString.h"
24#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
25#include "llvm/CodeGen/MachinePipeliner.h"
26#include "llvm/CodeGen/MachineScheduler.h"
27#include "llvm/CodeGen/TargetFrameLowering.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/MDBuilder.h"
30#include "llvm/TargetParser/AMDGPUTargetParser.h"
31#include <algorithm>
32
33using namespace llvm;
34
35#define DEBUG_TYPE "gcn-subtarget"
36
37#define GET_SUBTARGETINFO_TARGET_DESC
38#define GET_SUBTARGETINFO_CTOR
39#define AMDGPUSubtarget GCNSubtarget
40#include "AMDGPUGenSubtargetInfo.inc"
41#undef AMDGPUSubtarget
42
43GCNSubtarget::~GCNSubtarget() = default;
44
45static AMDGPUSubtarget::Generation computeDefaultGeneration(const Triple &TT) {
46 // Legacy triples without a subarch default to the first target that supports
47 // flat addressing for HSA, otherwise the first amdgcn target.
48 if (TT.getSubArch() == Triple::NoSubArch)
49 return TT.getOS() == Triple::AMDHSA ? AMDGPUSubtarget::SEA_ISLANDS
50 : AMDGPUSubtarget::SOUTHERN_ISLANDS;
51
52 switch (AMDGPU::getMajorSubArch(SubArch: TT.getSubArch())) {
53 case Triple::AMDGPUSubArch6:
54 return AMDGPUSubtarget::SOUTHERN_ISLANDS;
55 case Triple::AMDGPUSubArch7:
56 return AMDGPUSubtarget::SEA_ISLANDS;
57 case Triple::AMDGPUSubArch8:
58 case Triple::AMDGPUSubArch810:
59 return AMDGPUSubtarget::VOLCANIC_ISLANDS;
60 case Triple::AMDGPUSubArch9:
61 case Triple::AMDGPUSubArch908:
62 case Triple::AMDGPUSubArch90A:
63 case Triple::AMDGPUSubArch9_4:
64 return AMDGPUSubtarget::GFX9;
65 case Triple::AMDGPUSubArch10_1:
66 case Triple::AMDGPUSubArch10_3:
67 return AMDGPUSubtarget::GFX10;
68 case Triple::AMDGPUSubArch11:
69 case Triple::AMDGPUSubArch11_7:
70 return AMDGPUSubtarget::GFX11;
71 case Triple::AMDGPUSubArch12:
72 case Triple::AMDGPUSubArch12_5:
73 case Triple::AMDGPUSubArch1250S:
74 return AMDGPUSubtarget::GFX12;
75 case Triple::AMDGPUSubArch13:
76 return AMDGPUSubtarget::GFX13;
77 default:
78 reportFatalUsageError(reason: "invalid subarch for amdgpu");
79 }
80}
81
82GCNSubtarget &GCNSubtarget::initializeSubtargetDependencies(
83 const Triple &TT, StringRef GPU, StringRef FS,
84 AMDGPU::TargetIDSetting XnackSetting,
85 AMDGPU::TargetIDSetting SramEccSetting) {
86 // Determine default and user-specified characteristics
87 //
88 // We want to be able to turn these off, but making this a subtarget feature
89 // for SI has the unhelpful behavior that it unsets everything else if you
90 // disable it.
91 //
92 // Similarly we want enable-prt-strict-null to be on by default and not to
93 // unset everything else if it is disabled
94
95 SmallString<256> FullFS("+load-store-opt,+enable-ds128,");
96
97 // Turn on features that HSA ABI requires. Also turn on FlatForGlobal by
98 // default
99 if (isAmdHsaOS())
100 FullFS += "+flat-for-global,+unaligned-access-mode,+trap-handler,";
101
102 FullFS += "+enable-prt-strict-null,"; // This is overridden by a disable in FS
103
104 // Disable mutually exclusive bits.
105 if (FS.contains_insensitive(Other: "+wavefrontsize")) {
106 if (!FS.contains_insensitive(Other: "wavefrontsize16"))
107 FullFS += "-wavefrontsize16,";
108 if (!FS.contains_insensitive(Other: "wavefrontsize32"))
109 FullFS += "-wavefrontsize32,";
110 if (!FS.contains_insensitive(Other: "wavefrontsize64"))
111 FullFS += "-wavefrontsize64,";
112 }
113
114 FullFS += FS;
115
116 ParseSubtargetFeatures(CPU: GPU, /*TuneCPU*/ GPU, FS: FullFS);
117
118 // Implement the "generic" processors, which acts as the default when no
119 // generation features are enabled (e.g for -mcpu=''). HSA OS defaults to
120 // the first amdgcn target that supports flat addressing. Other OSes defaults
121 // to the first amdgcn target.
122 if (Gen == AMDGPUSubtarget::INVALID) {
123 Gen = computeDefaultGeneration(TT);
124 // Assume wave64 for the unknown target, if not explicitly set.
125 if (getWavefrontSizeLog2() == 0)
126 WavefrontSizeLog2 = 6;
127 } else if (!hasFeature(Feature: AMDGPU::FeatureWavefrontSize32) &&
128 !hasFeature(Feature: AMDGPU::FeatureWavefrontSize64)) {
129 // If there is no default wave size it must be a generation before gfx10,
130 // these have FeatureWavefrontSize64 in their definition already. For gfx10+
131 // set wave32 as a default.
132 ToggleFeature(FB: AMDGPU::FeatureWavefrontSize32);
133 WavefrontSizeLog2 = getGeneration() >= AMDGPUSubtarget::GFX10 ? 5 : 6;
134 }
135
136 // We don't support FP64 for EG/NI atm.
137 assert(!hasFP64() || (getGeneration() >= AMDGPUSubtarget::SOUTHERN_ISLANDS));
138
139 // Targets must either support 64-bit offsets for MUBUF instructions, and/or
140 // support flat operations, otherwise they cannot access a 64-bit global
141 // address space
142 assert(hasAddr64() || hasFlat());
143 // Unless +-flat-for-global is specified, turn on FlatForGlobal for targets
144 // that do not support ADDR64 variants of MUBUF instructions. Such targets
145 // cannot use a 64 bit offset with a MUBUF instruction to access the global
146 // address space
147 if (!hasAddr64() && !FS.contains(Other: "flat-for-global") && !UseFlatForGlobal) {
148 ToggleFeature(FB: AMDGPU::FeatureUseFlatForGlobal);
149 UseFlatForGlobal = true;
150 }
151 // Unless +-flat-for-global is specified, use MUBUF instructions for global
152 // address space access if flat operations are not available.
153 if (!hasFlat() && !FS.contains(Other: "flat-for-global") && UseFlatForGlobal) {
154 ToggleFeature(FB: AMDGPU::FeatureUseFlatForGlobal);
155 UseFlatForGlobal = false;
156 }
157
158 // Set defaults if needed.
159 if (MaxPrivateElementSize == 0)
160 MaxPrivateElementSize = 4;
161
162 if (LDSBankCount == 0)
163 LDSBankCount = 32;
164
165 if (MaxWavesPerEU == 0)
166 MaxWavesPerEU = 10;
167
168 if (FlatOffsetBitWidth == 0)
169 FlatOffsetBitWidth = 13;
170
171 LocalMemorySize =
172 AMDGPU::getLocalMemorySize(AK: getTargetID().getGPUKind(), FullSIMDMode: isFullSIMDMode());
173 AddressableLocalMemorySize = AMDGPU::getAddressableLocalMemorySize(
174 AK: getTargetID().getGPUKind(), FullSIMDMode: isFullSIMDMode());
175 // LDS allocation granularity is in bytes.
176 LDSAllocationGranularity =
177 AMDGPU::getLDSAllocGranule(AK: getTargetID().getGPUKind());
178
179 HasFminFmaxLegacy = getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS;
180 HasSMulHi = getGeneration() >= AMDGPUSubtarget::GFX9;
181
182 // InstCacheLineSize is set from TableGen subtarget features
183 // (FeatureInstCacheLineSize64 / FeatureInstCacheLineSize128).
184 // Fall back to 64 if no feature was specified (e.g. generic targets).
185 if (InstCacheLineSize == 0)
186 InstCacheLineSize = 64;
187
188 assert(llvm::isPowerOf2_32(InstCacheLineSize) &&
189 "InstCacheLineSize must be a power of 2");
190
191 // Apply the module flag's xnack setting if the target supports on/off modes.
192 // Targets without on/off mode support have xnack always on and ignore module
193 // flags.
194 if (hasXNACKOnOffModes())
195 TargetID.setXnackSetting(XnackSetting);
196
197 // Apply the module flag's sramecc setting if the target supports on/off
198 // modes. Targets with sramecc hardwired on ignore module flags.
199 if (hasSRAMECCOnOffModes())
200 TargetID.setSramEccSetting(SramEccSetting);
201
202 return *this;
203}
204
205void GCNSubtarget::checkSubtargetFeatures(const Function &F) const {
206 LLVMContext &Ctx = F.getContext();
207 if (hasFeature(Feature: AMDGPU::FeatureWavefrontSize32) &&
208 hasFeature(Feature: AMDGPU::FeatureWavefrontSize64)) {
209 Ctx.diagnose(DI: DiagnosticInfoUnsupported(
210 F, "must specify exactly one of wavefrontsize32 and wavefrontsize64"));
211 }
212}
213
214// TODO: Validate subarch for subtarget
215
216GCNSubtarget::GCNSubtarget(const Triple &TT, StringRef GPU, StringRef FS,
217 const GCNTargetMachine &TM, bool BufferOOBRelaxed,
218 bool TBufferOOBRelaxed,
219 AMDGPU::TargetIDSetting XnackSetting,
220 AMDGPU::TargetIDSetting SramEccSetting)
221 : // clang-format off
222 AMDGPUGenSubtargetInfo(TT, GPU, /*TuneCPU*/ GPU, FS),
223 AMDGPUSubtarget(TT, TM.getCLOpts()),
224 TargetID(AMDGPU::createAMDGPUTargetID(STI: *this, FeatureString: "")),
225 InstrItins(getInstrItineraryForCPU(CPU: GPU)),
226 BufferOOBRelaxed(BufferOOBRelaxed),
227 TBufferOOBRelaxed(TBufferOOBRelaxed),
228 InstrInfo(initializeSubtargetDependencies(TT, GPU, FS, XnackSetting,
229 SramEccSetting)),
230 TLInfo(TM, *this),
231 // Frame index expansion sometimes assumes the low bit of SP is 0
232 FrameLowering(TargetFrameLowering::StackGrowsUp, getStackAlignment(), 0,
233 /*TransAl=*/Align(4)) {
234
235 // clang-format on
236
237 LLVM_DEBUG(dbgs() << "xnack setting for subtarget: "
238 << TargetID.getXnackSetting() << '\n');
239 LLVM_DEBUG(dbgs() << "sramecc setting for subtarget: "
240 << TargetID.getSramEccSetting() << '\n');
241
242 NumWorkGroupSIMDs = AMDGPU::getNumWorkGroupSIMDs(FullSIMDMode: isFullSIMDMode());
243
244 TSInfo = std::make_unique<AMDGPUSelectionDAGInfo>();
245
246 CallLoweringInfo = std::make_unique<AMDGPUCallLowering>(args: *getTargetLowering());
247 InlineAsmLoweringInfo =
248 std::make_unique<InlineAsmLowering>(args: getTargetLowering());
249 Legalizer = std::make_unique<AMDGPULegalizerInfo>(args&: *this, args: TM);
250 RegBankInfo = std::make_unique<AMDGPURegisterBankInfo>(args&: *this);
251 InstSelector =
252 std::make_unique<AMDGPUInstructionSelector>(args&: *this, args&: *RegBankInfo);
253}
254
255const SelectionDAGTargetInfo *GCNSubtarget::getSelectionDAGInfo() const {
256 return TSInfo.get();
257}
258
259unsigned GCNSubtarget::getConstantBusLimit(unsigned Opcode) const {
260 if (getGeneration() < GFX10)
261 return 1;
262
263 switch (Opcode) {
264 case AMDGPU::V_LSHLREV_B64_e64:
265 case AMDGPU::V_LSHLREV_B64_gfx10:
266 case AMDGPU::V_LSHLREV_B64_e64_gfx11:
267 case AMDGPU::V_LSHLREV_B64_e32_gfx12:
268 case AMDGPU::V_LSHLREV_B64_e64_gfx12:
269 case AMDGPU::V_LSHL_B64_e64:
270 case AMDGPU::V_LSHRREV_B64_e64:
271 case AMDGPU::V_LSHRREV_B64_gfx10:
272 case AMDGPU::V_LSHRREV_B64_e64_gfx11:
273 case AMDGPU::V_LSHRREV_B64_e64_gfx12:
274 case AMDGPU::V_LSHR_B64_e64:
275 case AMDGPU::V_ASHRREV_I64_e64:
276 case AMDGPU::V_ASHRREV_I64_gfx10:
277 case AMDGPU::V_ASHRREV_I64_e64_gfx11:
278 case AMDGPU::V_ASHRREV_I64_e64_gfx12:
279 case AMDGPU::V_ASHR_I64_e64:
280 return 1;
281 }
282
283 return 2;
284}
285
286/// This list was mostly derived from experimentation.
287bool GCNSubtarget::zeroesHigh16BitsOfDest(unsigned Opcode) const {
288 switch (Opcode) {
289 case AMDGPU::V_CVT_F16_F32_e32:
290 case AMDGPU::V_CVT_F16_F32_e64:
291 case AMDGPU::V_CVT_F16_U16_e32:
292 case AMDGPU::V_CVT_F16_U16_e64:
293 case AMDGPU::V_CVT_F16_I16_e32:
294 case AMDGPU::V_CVT_F16_I16_e64:
295 case AMDGPU::V_RCP_F16_e64:
296 case AMDGPU::V_RCP_F16_e32:
297 case AMDGPU::V_RSQ_F16_e64:
298 case AMDGPU::V_RSQ_F16_e32:
299 case AMDGPU::V_SQRT_F16_e64:
300 case AMDGPU::V_SQRT_F16_e32:
301 case AMDGPU::V_LOG_F16_e64:
302 case AMDGPU::V_LOG_F16_e32:
303 case AMDGPU::V_EXP_F16_e64:
304 case AMDGPU::V_EXP_F16_e32:
305 case AMDGPU::V_SIN_F16_e64:
306 case AMDGPU::V_SIN_F16_e32:
307 case AMDGPU::V_COS_F16_e64:
308 case AMDGPU::V_COS_F16_e32:
309 case AMDGPU::V_FLOOR_F16_e64:
310 case AMDGPU::V_FLOOR_F16_e32:
311 case AMDGPU::V_CEIL_F16_e64:
312 case AMDGPU::V_CEIL_F16_e32:
313 case AMDGPU::V_TRUNC_F16_e64:
314 case AMDGPU::V_TRUNC_F16_e32:
315 case AMDGPU::V_RNDNE_F16_e64:
316 case AMDGPU::V_RNDNE_F16_e32:
317 case AMDGPU::V_FRACT_F16_e64:
318 case AMDGPU::V_FRACT_F16_e32:
319 case AMDGPU::V_FREXP_MANT_F16_e64:
320 case AMDGPU::V_FREXP_MANT_F16_e32:
321 case AMDGPU::V_FREXP_EXP_I16_F16_e64:
322 case AMDGPU::V_FREXP_EXP_I16_F16_e32:
323 case AMDGPU::V_LDEXP_F16_e64:
324 case AMDGPU::V_LDEXP_F16_e32:
325 case AMDGPU::V_LSHLREV_B16_e64:
326 case AMDGPU::V_LSHLREV_B16_e32:
327 case AMDGPU::V_LSHRREV_B16_e64:
328 case AMDGPU::V_LSHRREV_B16_e32:
329 case AMDGPU::V_ASHRREV_I16_e64:
330 case AMDGPU::V_ASHRREV_I16_e32:
331 case AMDGPU::V_ADD_U16_e64:
332 case AMDGPU::V_ADD_U16_e32:
333 case AMDGPU::V_SUB_U16_e64:
334 case AMDGPU::V_SUB_U16_e32:
335 case AMDGPU::V_SUBREV_U16_e64:
336 case AMDGPU::V_SUBREV_U16_e32:
337 case AMDGPU::V_MUL_LO_U16_e64:
338 case AMDGPU::V_MUL_LO_U16_e32:
339 case AMDGPU::V_ADD_F16_e64:
340 case AMDGPU::V_ADD_F16_e32:
341 case AMDGPU::V_SUB_F16_e64:
342 case AMDGPU::V_SUB_F16_e32:
343 case AMDGPU::V_SUBREV_F16_e64:
344 case AMDGPU::V_SUBREV_F16_e32:
345 case AMDGPU::V_MUL_F16_e64:
346 case AMDGPU::V_MUL_F16_e32:
347 case AMDGPU::V_MAX_F16_e64:
348 case AMDGPU::V_MAX_F16_e32:
349 case AMDGPU::V_MIN_F16_e64:
350 case AMDGPU::V_MIN_F16_e32:
351 case AMDGPU::V_MAX_U16_e64:
352 case AMDGPU::V_MAX_U16_e32:
353 case AMDGPU::V_MIN_U16_e64:
354 case AMDGPU::V_MIN_U16_e32:
355 case AMDGPU::V_MAX_I16_e64:
356 case AMDGPU::V_MAX_I16_e32:
357 case AMDGPU::V_MIN_I16_e64:
358 case AMDGPU::V_MIN_I16_e32:
359 case AMDGPU::V_MAD_F16_e64:
360 case AMDGPU::V_MAD_U16_e64:
361 case AMDGPU::V_MAD_I16_e64:
362 case AMDGPU::V_FMA_F16_e64:
363 case AMDGPU::V_DIV_FIXUP_F16_e64:
364 // On gfx10, all 16-bit instructions preserve the high bits.
365 return getGeneration() <= AMDGPUSubtarget::GFX9;
366 case AMDGPU::V_MADAK_F16:
367 case AMDGPU::V_MADMK_F16:
368 case AMDGPU::V_MAC_F16_e64:
369 case AMDGPU::V_MAC_F16_e32:
370 case AMDGPU::V_FMAMK_F16:
371 case AMDGPU::V_FMAAK_F16:
372 case AMDGPU::V_FMAC_F16_e64:
373 case AMDGPU::V_FMAC_F16_e32:
374 // In gfx9, the preferred handling of the unused high 16-bits changed. Most
375 // instructions maintain the legacy behavior of 0ing. Some instructions
376 // changed to preserving the high bits.
377 return getGeneration() == AMDGPUSubtarget::VOLCANIC_ISLANDS;
378 case AMDGPU::V_MAD_MIXLO_F16:
379 case AMDGPU::V_MAD_MIXHI_F16:
380 default:
381 return false;
382 }
383}
384
385void GCNSubtarget::overrideSchedPolicy(MachineSchedPolicy &Policy,
386 const SchedRegion &Region) const {
387 // Track register pressure so the scheduler can try to decrease
388 // pressure once register usage is above the threshold defined by
389 // SIRegisterInfo::getRegPressureSetLimit()
390 Policy.ShouldTrackPressure = true;
391
392 const Function &F = Region.RegionBegin->getMF()->getFunction();
393 if (AMDGPU::getSchedStrategy(CLOpts: getCLOpts(), F) == "coexec") {
394 Policy.OnlyTopDown = true;
395 Policy.OnlyBottomUp = false;
396 return;
397 }
398
399 // Enabling both top down and bottom up scheduling seems to give us less
400 // register spills than just using one of these approaches on its own.
401 Policy.OnlyTopDown = false;
402 Policy.OnlyBottomUp = false;
403
404 // Enabling ShouldTrackLaneMasks crashes the SI Machine Scheduler.
405 if (!enableSIScheduler())
406 Policy.ShouldTrackLaneMasks = true;
407}
408
409void GCNSubtarget::overridePostRASchedPolicy(MachineSchedPolicy &Policy,
410 const SchedRegion &Region) const {
411 const Function &F = Region.RegionBegin->getMF()->getFunction();
412 Attribute PostRADirectionAttr = F.getFnAttribute(Kind: "amdgpu-post-ra-direction");
413 if (!PostRADirectionAttr.isValid())
414 return;
415
416 StringRef PostRADirectionStr = PostRADirectionAttr.getValueAsString();
417 if (PostRADirectionStr == "topdown") {
418 Policy.OnlyTopDown = true;
419 Policy.OnlyBottomUp = false;
420 } else if (PostRADirectionStr == "bottomup") {
421 Policy.OnlyTopDown = false;
422 Policy.OnlyBottomUp = true;
423 } else if (PostRADirectionStr == "bidirectional") {
424 Policy.OnlyTopDown = false;
425 Policy.OnlyBottomUp = false;
426 } else {
427 DiagnosticInfoOptimizationFailure Diag(
428 F, F.getSubprogram(), "invalid value for postRA direction attribute");
429 F.getContext().diagnose(DI: Diag);
430 }
431
432 LLVM_DEBUG({
433 const char *DirStr = "default";
434 if (Policy.OnlyTopDown && !Policy.OnlyBottomUp)
435 DirStr = "topdown";
436 else if (!Policy.OnlyTopDown && Policy.OnlyBottomUp)
437 DirStr = "bottomup";
438 else if (!Policy.OnlyTopDown && !Policy.OnlyBottomUp)
439 DirStr = "bidirectional";
440
441 dbgs() << "Post-MI-sched direction (" << F.getName() << "): " << DirStr
442 << '\n';
443 });
444}
445
446void GCNSubtarget::overridePipelinerPolicy(
447 MachinePipelinerPolicy &Policy) const {
448 Policy.ShouldLimitRegPressure = true;
449}
450
451void GCNSubtarget::mirFileLoaded(MachineFunction &MF) const {
452 if (isWave32()) {
453 // Fix implicit $vcc operands after MIParser has verified that they match
454 // the instruction definitions.
455 for (auto &MBB : MF) {
456 for (auto &MI : MBB)
457 InstrInfo.fixImplicitOperands(MI);
458 }
459 }
460}
461
462bool GCNSubtarget::hasMadF16() const {
463 return InstrInfo.pseudoToMCOpcode(Opcode: AMDGPU::V_MAD_F16_e64) != -1;
464}
465
466bool GCNSubtarget::useVGPRIndexMode() const {
467 return hasVGPRIndexMode() && (!hasMovrel() || getCLOpts().vgpr_index_mode);
468}
469
470bool GCNSubtarget::useAA() const { return getCLOpts().use_aa_in_codegen; }
471
472uint64_t GCNSubtarget::getMaxBufferIntrinsicWidth(const DataLayout &DL, Type *T,
473 Align A, bool NoWrapFromMax,
474 bool NoPartialOOB) const {
475 Align Result(16);
476 if (!hasUnalignedBufferAccessEnabled() && A < Align(4))
477 Result = A;
478 auto *VT = dyn_cast<VectorType>(Val: T);
479 if (!hasRelaxedBufferOOBMode() && VT) {
480 TypeSize ElemBits = DL.getTypeSizeInBits(Ty: VT->getElementType());
481 if (ElemBits.isKnownMultipleOf(RHS: 32)) {
482 // Word-sized operations are bounds-checked per word. So, the only case we
483 // have to worry about is stores that start out of bounds and then go in,
484 // and those can only become in-bounds on a multiple of their alignment.
485 // Therefore, we can use the declared alignment of the operation as the
486 // maximum width, rounding up to 4.
487 if (!NoWrapFromMax)
488 Result = std::min(a: Result, b: std::max(a: A, b: Align(4)));
489 } else if ((ElemBits.isKnownMultipleOf(RHS: 8) ||
490 isPowerOf2_64(Value: ElemBits.getKnownMinValue()))) {
491 // To ensure correct behavior for sub-word types, we must always scalarize
492 // unaligned loads of sub-word types. For example, if you load
493 // a <4 x i8> from offset 7 in an 8-byte buffer, expecting the vector
494 // to be padded out with 0s after that last byte, you'll get all 0s
495 // instead. To prevent this behavior when not requested, de-vectorize such
496 // loads.
497 //
498 // If we knew that the value that triggers bounds checks was a multiple of
499 // 4 along with the access being word-aligned, we could avoid the
500 // scalarization here, as the bitcast wouldn't change any check behavior,
501 // but we don't currently try to analyze this.
502 //
503 // Strict OOB checking isn't supported if the size of each element is a
504 // non-power-of-2 value less than 8, since there's no feasible way to
505 // apply such a strict bounds check.
506 if (!NoPartialOOB)
507 Result =
508 commonAlignment(A: Result, Offset: divideCeil(Numerator: ElemBits.getKnownMinValue(), Denominator: 8));
509 }
510 }
511 return Result.value() * 8;
512}
513
514unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const {
515 return AMDGPU::IsaInfo::getOccupancyWithNumSGPRs(STI: *this, SGPRs);
516}
517
518unsigned
519GCNSubtarget::getOccupancyWithNumVGPRs(unsigned NumVGPRs,
520 unsigned DynamicVGPRBlockSize) const {
521 return AMDGPU::IsaInfo::getNumWavesPerEUWithNumVGPRs(STI: *this, NumVGPRs,
522 DynamicVGPRBlockSize);
523}
524
525unsigned
526GCNSubtarget::getBaseReservedNumSGPRs(const bool HasFlatScratch) const {
527 if (getGeneration() >= AMDGPUSubtarget::GFX10)
528 return 2; // VCC. FLAT_SCRATCH and XNACK are no longer in SGPRs.
529
530 if (HasFlatScratch || HasArchitectedFlatScratch) {
531 if (getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS)
532 return 6; // FLAT_SCRATCH, XNACK, VCC (in that order).
533 if (getGeneration() == AMDGPUSubtarget::SEA_ISLANDS)
534 return 4; // FLAT_SCRATCH, VCC (in that order).
535 }
536
537 if (isXNACKEnabled())
538 return 4; // XNACK, VCC (in that order).
539 return 2; // VCC.
540}
541
542unsigned GCNSubtarget::getReservedNumSGPRs(const MachineFunction &MF) const {
543 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
544 return getBaseReservedNumSGPRs(HasFlatScratch: MFI.getUserSGPRInfo().hasFlatScratchInit());
545}
546
547unsigned GCNSubtarget::getReservedNumSGPRs(const Function &F) const {
548 // In principle we do not need to reserve SGPR pair used for flat_scratch if
549 // we know flat instructions do not access the stack anywhere in the
550 // program. For now assume it's needed if we have flat instructions.
551 const bool KernelUsesFlatScratch = hasFlatAddressSpace();
552 return getBaseReservedNumSGPRs(HasFlatScratch: KernelUsesFlatScratch);
553}
554
555std::pair<unsigned, unsigned>
556GCNSubtarget::computeOccupancy(const Function &F, unsigned LDSSize,
557 unsigned NumSGPRs, unsigned NumVGPRs) const {
558 unsigned DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
559 auto [MinOcc, MaxOcc] = getOccupancyWithWorkGroupSizes(LDSBytes: LDSSize, F);
560 unsigned SGPROcc = getOccupancyWithNumSGPRs(SGPRs: NumSGPRs);
561 unsigned VGPROcc = getOccupancyWithNumVGPRs(NumVGPRs, DynamicVGPRBlockSize);
562
563 // Maximum occupancy may be further limited by high SGPR/VGPR usage.
564 MaxOcc = std::min(l: {MaxOcc, SGPROcc, VGPROcc});
565 return {std::min(a: MinOcc, b: MaxOcc), MaxOcc};
566}
567
568unsigned GCNSubtarget::getBaseMaxNumSGPRs(
569 const Function &F, std::pair<unsigned, unsigned> WavesPerEU,
570 unsigned PreloadedSGPRs, unsigned ReservedNumSGPRs) const {
571 // Compute maximum number of SGPRs function can use using default/requested
572 // minimum number of waves per execution unit.
573 unsigned MaxNumSGPRs = getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: false);
574 unsigned MaxAddressableNumSGPRs = getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: true);
575
576 // Check if maximum number of SGPRs was explicitly requested using
577 // "amdgpu-num-sgpr" attribute.
578 unsigned Requested = static_cast<unsigned>(
579 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-num-sgpr", Default: MaxNumSGPRs));
580
581 if (Requested != MaxNumSGPRs) {
582 // Make sure requested value does not violate subtarget's specifications.
583 if (Requested && (Requested <= ReservedNumSGPRs))
584 Requested = 0;
585
586 // If more SGPRs are required to support the input user/system SGPRs,
587 // increase to accommodate them.
588 //
589 // FIXME: This really ends up using the requested number of SGPRs + number
590 // of reserved special registers in total. Theoretically you could re-use
591 // the last input registers for these special registers, but this would
592 // require a lot of complexity to deal with the weird aliasing.
593 unsigned InputNumSGPRs = PreloadedSGPRs;
594 if (Requested && Requested < InputNumSGPRs)
595 Requested = InputNumSGPRs;
596
597 // Make sure requested value is compatible with values implied by
598 // default/requested minimum/maximum number of waves per execution unit.
599 if (Requested && Requested > getMaxNumSGPRs(WavesPerEU: WavesPerEU.first, Addressable: false))
600 Requested = 0;
601 if (WavesPerEU.second && Requested &&
602 Requested < getMinNumSGPRs(WavesPerEU: WavesPerEU.second))
603 Requested = 0;
604
605 if (Requested)
606 MaxNumSGPRs = Requested;
607 }
608
609 if (hasSGPRInitBug())
610 MaxNumSGPRs = AMDGPU::IsaInfo::FIXED_NUM_SGPRS_FOR_INIT_BUG;
611
612 return std::min(a: MaxNumSGPRs - ReservedNumSGPRs, b: MaxAddressableNumSGPRs);
613}
614
615unsigned GCNSubtarget::getMaxNumSGPRs(const MachineFunction &MF) const {
616 const Function &F = MF.getFunction();
617 const SIMachineFunctionInfo &MFI = *MF.getInfo<SIMachineFunctionInfo>();
618 return getBaseMaxNumSGPRs(F, WavesPerEU: MFI.getWavesPerEU(), PreloadedSGPRs: MFI.getNumPreloadedSGPRs(),
619 ReservedNumSGPRs: getReservedNumSGPRs(MF));
620}
621
622unsigned GCNSubtarget::getMaxNumPreloadedSGPRs() const {
623 using USI = GCNUserSGPRUsageInfo;
624 // Max number of user SGPRs
625 const unsigned MaxUserSGPRs =
626 USI::getNumUserSGPRForField(ID: USI::PrivateSegmentBufferID) +
627 USI::getNumUserSGPRForField(ID: USI::DispatchPtrID) +
628 USI::getNumUserSGPRForField(ID: USI::QueuePtrID) +
629 USI::getNumUserSGPRForField(ID: USI::KernargSegmentPtrID) +
630 USI::getNumUserSGPRForField(ID: USI::DispatchIdID) +
631 USI::getNumUserSGPRForField(ID: USI::FlatScratchInitID) +
632 USI::getNumUserSGPRForField(ID: USI::ImplicitBufferPtrID);
633
634 // Max number of system SGPRs
635 const unsigned MaxSystemSGPRs = 1 + // WorkGroupIDX
636 1 + // WorkGroupIDY
637 1 + // WorkGroupIDZ
638 1 + // WorkGroupInfo
639 1; // private segment wave byte offset
640
641 // Max number of synthetic SGPRs
642 const unsigned SyntheticSGPRs = 1; // LDSKernelId
643
644 return MaxUserSGPRs + MaxSystemSGPRs + SyntheticSGPRs;
645}
646
647unsigned GCNSubtarget::getMaxNumSGPRs(const Function &F) const {
648 return getBaseMaxNumSGPRs(F, WavesPerEU: getWavesPerEU(F), PreloadedSGPRs: getMaxNumPreloadedSGPRs(),
649 ReservedNumSGPRs: getReservedNumSGPRs(F));
650}
651
652unsigned GCNSubtarget::getBaseMaxNumVGPRs(
653 const Function &F, std::pair<unsigned, unsigned> NumVGPRBounds) const {
654 const auto [Min, Max] = NumVGPRBounds;
655
656 // Check if maximum number of VGPRs was explicitly requested using
657 // "amdgpu-num-vgpr" attribute.
658
659 unsigned Requested = static_cast<unsigned>(
660 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-num-vgpr", Default: Max));
661 if (Requested != Max && hasGFX90AInsts())
662 Requested *= 2;
663
664 // Make sure requested value is inside the range of possible VGPR usage.
665 return std::clamp(val: Requested, lo: Min, hi: Max);
666}
667
668unsigned GCNSubtarget::getMaxNumVGPRs(const Function &F) const {
669 unsigned DynamicVGPRBlockSize = AMDGPU::getDynamicVGPRBlockSize(F);
670 std::pair<unsigned, unsigned> Waves = getWavesPerEU(F);
671
672 unsigned MaxNumVGPRs = getBaseMaxNumVGPRs(
673 F, NumVGPRBounds: {getMinNumVGPRs(WavesPerEU: Waves.second, DynamicVGPRBlockSize),
674 getMaxNumVGPRs(WavesPerEU: Waves.first, DynamicVGPRBlockSize)});
675
676 // In DVGPR mode, a wave launches with a single VGPR block allocated. Applied
677 // after getBaseMaxNumVGPRs so "amdgpu-num-vgpr" cannot raise it back up.
678 if (DynamicVGPRBlockSize != 0 &&
679 AMDGPU::isEntryFunctionCC(CC: F.getCallingConv()))
680 MaxNumVGPRs = std::min(a: MaxNumVGPRs, b: DynamicVGPRBlockSize);
681
682 return MaxNumVGPRs;
683}
684
685unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const {
686 return getMaxNumVGPRs(F: MF.getFunction());
687}
688
689std::pair<unsigned, unsigned>
690GCNSubtarget::getMaxNumVectorRegs(const Function &F) const {
691 const unsigned MaxVectorRegs = getMaxNumVGPRs(F);
692
693 unsigned MaxNumVGPRs = MaxVectorRegs;
694 unsigned MaxNumAGPRs = 0;
695 unsigned NumArchVGPRs = getAddressableNumArchVGPRs();
696
697 // On GFX90A, the number of VGPRs and AGPRs need not be equal. Theoretically,
698 // a wave may have up to 512 total vector registers combining together both
699 // VGPRs and AGPRs. Hence, in an entry function without calls and without
700 // AGPRs used within it, it is possible to use the whole vector register
701 // budget for VGPRs.
702 //
703 // TODO: it shall be possible to estimate maximum AGPR/VGPR pressure and split
704 // register file accordingly.
705 if (hasGFX90AInsts()) {
706 unsigned MinNumAGPRs = 0;
707 const unsigned TotalNumAGPRs = AMDGPU::AGPR_32RegClass.getNumRegs();
708
709 const std::pair<unsigned, unsigned> DefaultNumAGPR = {~0u, ~0u};
710
711 // TODO: The lower bound should probably force the number of required
712 // registers up, overriding amdgpu-waves-per-eu.
713 std::tie(args&: MinNumAGPRs, args&: MaxNumAGPRs) =
714 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-agpr-alloc", Default: DefaultNumAGPR,
715 /*OnlyFirstRequired=*/true);
716
717 if (MinNumAGPRs == DefaultNumAGPR.first) {
718 // Default to splitting half the registers if AGPRs are required.
719 MinNumAGPRs = MaxNumAGPRs = MaxVectorRegs / 2;
720 } else {
721 // Align to accum_offset's allocation granularity.
722 MinNumAGPRs = alignTo(Value: MinNumAGPRs, Align: 4);
723
724 MinNumAGPRs = std::min(a: MinNumAGPRs, b: TotalNumAGPRs);
725 }
726
727 // Clamp values to be inbounds of our limits, and ensure min <= max.
728
729 MaxNumAGPRs = std::min(a: std::max(a: MinNumAGPRs, b: MaxNumAGPRs), b: MaxVectorRegs);
730 MinNumAGPRs = std::min(l: {MinNumAGPRs, TotalNumAGPRs, MaxNumAGPRs});
731
732 MaxNumVGPRs = std::min(a: MaxVectorRegs - MinNumAGPRs, b: NumArchVGPRs);
733 MaxNumAGPRs = std::min(a: MaxVectorRegs - MaxNumVGPRs, b: MaxNumAGPRs);
734
735 assert(MaxNumVGPRs + MaxNumAGPRs <= MaxVectorRegs &&
736 MaxNumAGPRs <= TotalNumAGPRs && MaxNumVGPRs <= NumArchVGPRs &&
737 "invalid register counts");
738 } else if (hasMAIInsts()) {
739 // On gfx908 the number of AGPRs always equals the number of VGPRs.
740 MaxNumAGPRs = MaxNumVGPRs = MaxVectorRegs;
741 }
742
743 return std::pair(MaxNumVGPRs, MaxNumAGPRs);
744}
745
746// Check to which source operand UseOpIdx points to and return a pointer to the
747// operand of the corresponding source modifier.
748// Return nullptr if UseOpIdx either doesn't point to src0/1/2 or if there is no
749// operand for the corresponding source modifier.
750static const MachineOperand *
751getVOP3PSourceModifierFromOpIdx(const MachineInstr &UseI, int UseOpIdx,
752 const SIInstrInfo &InstrInfo) {
753 AMDGPU::OpName UseName = AMDGPU::getOperandIdxName(
754 Opcode: UseI.getOpcode(), Idx: static_cast<int16_t>(UseOpIdx));
755 switch (UseName) {
756 case AMDGPU::OpName::src0:
757 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src0_modifiers);
758 case AMDGPU::OpName::src1:
759 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src1_modifiers);
760 case AMDGPU::OpName::src2:
761 return InstrInfo.getNamedOperand(MI: UseI, OperandName: AMDGPU::OpName::src2_modifiers);
762 default:
763 return nullptr;
764 }
765}
766
767// Get the subreg idx of the subreg that is used by the given instruction
768// operand, considering the given op_sel modifier.
769// Return 0 if the whole register is used or as a conservative fallback.
770static unsigned getEffectiveSubRegIdx(const SIRegisterInfo &TRI,
771 const SIInstrInfo &InstrInfo,
772 const MachineInstr &I,
773 const MachineOperand &Op) {
774 if (!InstrInfo.isVOP3P(MI: I) || InstrInfo.isWMMA(MI: I) || InstrInfo.isSWMMAC(MI: I))
775 return AMDGPU::NoSubRegister;
776
777 const MachineOperand *OpMod =
778 getVOP3PSourceModifierFromOpIdx(UseI: I, UseOpIdx: Op.getOperandNo(), InstrInfo);
779 if (!OpMod)
780 return AMDGPU::NoSubRegister;
781
782 // Note: the FMA_MIX* and MAD_MIX* instructions have different semantics for
783 // the op_sel and op_sel_hi source modifiers:
784 // - op_sel: selects low/high operand bits as input to the operation;
785 // has only meaning for 16-bit source operands
786 // - op_sel_hi: specifies the size of the source operands (16 or 32 bits);
787 // a value of 0 indicates 32 bit, 1 indicates 16 bit
788 // For the other VOP3P instructions, the semantics are:
789 // - op_sel: selects low/high operand bits as input to the operation which
790 // results in the lower-half of the destination
791 // - op_sel_hi: selects the low/high operand bits as input to the operation
792 // which results in the higher-half of the destination
793 int64_t OpSel = OpMod->getImm() & SISrcMods::OP_SEL_0;
794 int64_t OpSelHi = OpMod->getImm() & SISrcMods::OP_SEL_1;
795
796 // Check if all parts of the register are being used (= op_sel and op_sel_hi
797 // differ for VOP3P or op_sel_hi=0 for VOP3PMix). In that case we can return
798 // early.
799 if ((!InstrInfo.isVOP3PMix(MI: I) && (!OpSel || !OpSelHi) &&
800 (OpSel || OpSelHi)) ||
801 (InstrInfo.isVOP3PMix(MI: I) && !OpSelHi))
802 return AMDGPU::NoSubRegister;
803
804 const MachineRegisterInfo &MRI = I.getParent()->getParent()->getRegInfo();
805 const TargetRegisterClass *RC = TRI.getRegClassForOperandReg(MRI, MO: Op);
806
807 if (unsigned SubRegIdx = OpSel ? AMDGPU::sub1 : AMDGPU::sub0;
808 TRI.getSubClassWithSubReg(RC, SubRegIdx) == RC)
809 return SubRegIdx;
810 if (unsigned SubRegIdx = OpSel ? AMDGPU::hi16 : AMDGPU::lo16;
811 TRI.getSubClassWithSubReg(RC, SubRegIdx) == RC)
812 return SubRegIdx;
813
814 return AMDGPU::NoSubRegister;
815}
816
817Register GCNSubtarget::getRealSchedDependency(const MachineInstr &DefI,
818 int DefOpIdx,
819 const MachineInstr &UseI,
820 int UseOpIdx) const {
821 const SIRegisterInfo *TRI = getRegisterInfo();
822 const MachineOperand &DefOp = DefI.getOperand(i: DefOpIdx);
823 const MachineOperand &UseOp = UseI.getOperand(i: UseOpIdx);
824 Register DefReg = DefOp.getReg();
825 Register UseReg = UseOp.getReg();
826
827 // If the registers aren't restricted to a sub-register, there is no point in
828 // further analysis. This check makes only sense for virtual registers because
829 // physical registers may form a tuple and thus be part of a superregister
830 // although they are not a subregister themselves (vgpr0 is a "subreg" of
831 // vgpr0_vgpr1 without being a subreg in itself).
832 unsigned DefSubRegIdx = DefOp.getSubReg();
833 if (DefReg.isVirtual() && DefSubRegIdx == AMDGPU::NoSubRegister)
834 return DefReg;
835 unsigned UseSubRegIdx = getEffectiveSubRegIdx(TRI: *TRI, InstrInfo, I: UseI, Op: UseOp);
836 if (UseReg.isVirtual() && UseSubRegIdx == AMDGPU::NoSubRegister)
837 return DefReg;
838
839 if (!TRI->checkSubRegInterference(RegA: DefReg, SubA: DefSubRegIdx, RegB: UseReg, SubB: UseSubRegIdx))
840 return Register(); // No real dependency
841
842 // UseReg might be smaller or larger than DefReg, depending on the subreg and
843 // on whether DefReg is a subreg, too. -> Find the smaller one. This does not
844 // apply to virtual registers because we cannot construct a subreg for them.
845 if (DefReg.isVirtual())
846 return DefReg;
847 MCRegister DefMCReg =
848 DefSubRegIdx ? TRI->getSubReg(Reg: DefReg, Idx: DefSubRegIdx) : DefReg.asMCReg();
849 MCRegister UseMCReg =
850 UseSubRegIdx ? TRI->getSubReg(Reg: UseReg, Idx: UseSubRegIdx) : UseReg.asMCReg();
851 return TRI->isSubRegisterEq(RegA: DefMCReg, RegB: UseMCReg) ? UseMCReg : DefMCReg;
852}
853
854void GCNSubtarget::adjustSchedDependency(
855 SUnit *Def, int DefOpIdx, SUnit *Use, int UseOpIdx, SDep &Dep,
856 const TargetSchedModel *SchedModel) const {
857 if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() || !Def->isInstr() ||
858 !Use->isInstr())
859 return;
860
861 MachineInstr *DefI = Def->getInstr();
862 MachineInstr *UseI = Use->getInstr();
863
864 // Check for false latency on $tensorcnt / $asynccnt dependencies
865 if (Dep.getReg() == AMDGPU::TENSORcnt || Dep.getReg() == AMDGPU::ASYNCcnt) {
866 unsigned UseOp = UseI->getOpcode();
867 // Do not adjust latency for load->s_wait
868 bool IsBarrierCase =
869 InstrInfo.isLDSDMA(MI: *DefI) &&
870 (UseOp == AMDGPU::S_WAIT_TENSORCNT || UseOp == AMDGPU::S_WAIT_ASYNCCNT);
871 if (!IsBarrierCase) {
872 Dep.setLatency(1);
873 return;
874 }
875 }
876
877 if (Register Reg = getRealSchedDependency(DefI: *DefI, DefOpIdx, UseI: *UseI, UseOpIdx)) {
878 Dep.setReg(Reg);
879 } else {
880 Dep = SDep(Def, SDep::Artificial);
881 return; // This is not a data dependency anymore.
882 }
883
884 if (DefI->isBundle()) {
885 const SIRegisterInfo *TRI = getRegisterInfo();
886 auto Reg = Dep.getReg();
887 MachineBasicBlock::const_instr_iterator I(DefI->getIterator());
888 MachineBasicBlock::const_instr_iterator E(DefI->getParent()->instr_end());
889 unsigned Lat = 0;
890 for (++I; I != E && I->isBundledWithPred(); ++I) {
891 if (I->isMetaInstruction())
892 continue;
893 if (I->modifiesRegister(Reg, TRI))
894 Lat = InstrInfo.getInstrLatency(ItinData: getInstrItineraryData(), MI: *I);
895 else if (Lat)
896 --Lat;
897 }
898 Dep.setLatency(Lat);
899 } else if (UseI->isBundle()) {
900 const SIRegisterInfo *TRI = getRegisterInfo();
901 auto Reg = Dep.getReg();
902 MachineBasicBlock::const_instr_iterator I(UseI->getIterator());
903 MachineBasicBlock::const_instr_iterator E(UseI->getParent()->instr_end());
904 unsigned Lat = InstrInfo.getInstrLatency(ItinData: getInstrItineraryData(), MI: *DefI);
905 for (++I; I != E && I->isBundledWithPred() && Lat; ++I) {
906 if (I->isMetaInstruction())
907 continue;
908 if (I->readsRegister(Reg, TRI))
909 break;
910 --Lat;
911 }
912 Dep.setLatency(Lat);
913 } else if (Dep.getLatency() == 0 && Dep.getReg() == AMDGPU::VCC_LO) {
914 // Work around the fact that SIInstrInfo::fixImplicitOperands modifies
915 // implicit operands which come from the MCInstrDesc, which can fool
916 // ScheduleDAGInstrs::addPhysRegDataDeps into treating them as implicit
917 // pseudo operands.
918 Dep.setLatency(InstrInfo.getSchedModel().computeOperandLatency(
919 DefMI: DefI, DefOperIdx: DefOpIdx, UseMI: UseI, UseOperIdx: UseOpIdx));
920 }
921}
922
923unsigned GCNSubtarget::getNSAThreshold(const MachineFunction &MF) const {
924 if (getGeneration() >= AMDGPUSubtarget::GFX12)
925 return 0; // Not MIMG encoding.
926
927 if (getCLOpts().nsa_threshold)
928 return std::max(a: *getCLOpts().nsa_threshold, b: 2u);
929
930 int Value = static_cast<int>(MF.getFunction().getFnAttributeAsParsedInteger(
931 Kind: "amdgpu-nsa-threshold", Default: -1));
932 if (Value > 0)
933 return std::max(a: Value, b: 2);
934
935 return 2;
936}
937
938GCNUserSGPRUsageInfo::GCNUserSGPRUsageInfo(const Function &F,
939 const GCNSubtarget &ST)
940 : ST(ST) {
941 const CallingConv::ID CC = F.getCallingConv();
942 const bool IsKernel =
943 CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL;
944
945 if (IsKernel && (!F.arg_empty() || ST.getImplicitArgNumBytes(F) != 0))
946 KernargSegmentPtr = true;
947
948 bool IsAmdHsaOrMesa = ST.isAmdHsaOrMesa(F);
949 if (IsAmdHsaOrMesa && !ST.hasFlatScratchEnabled())
950 PrivateSegmentBuffer = true;
951 else if (!ST.hasArchitectedFlatScratch() && ST.isMesaGfxShader(F))
952 ImplicitBufferPtr = true; // Only needed to set up scratch.
953
954 if (!AMDGPU::isGraphics(CC)) {
955 if (!F.hasFnAttribute(Kind: "amdgpu-no-dispatch-ptr"))
956 DispatchPtr = true;
957
958 // FIXME: Can this always be disabled with < COv5?
959 if (!F.hasFnAttribute(Kind: "amdgpu-no-queue-ptr"))
960 QueuePtr = true;
961
962 if (!F.hasFnAttribute(Kind: "amdgpu-no-dispatch-id"))
963 DispatchID = true;
964 }
965
966 if (ST.hasFlatAddressSpace() && AMDGPU::isEntryFunctionCC(CC) &&
967 (IsAmdHsaOrMesa || ST.hasFlatScratchEnabled()) &&
968 // FlatScratchInit cannot be true for graphics CC if
969 // hasFlatScratchEnabled() is false.
970 (ST.hasFlatScratchEnabled() ||
971 (!AMDGPU::isGraphics(CC) &&
972 !F.hasFnAttribute(Kind: "amdgpu-no-flat-scratch-init"))) &&
973 !ST.hasArchitectedFlatScratch()) {
974 FlatScratchInit = true;
975 }
976
977 if (hasImplicitBufferPtr())
978 NumUsedUserSGPRs += getNumUserSGPRForField(ID: ImplicitBufferPtrID);
979
980 if (hasPrivateSegmentBuffer())
981 NumUsedUserSGPRs += getNumUserSGPRForField(ID: PrivateSegmentBufferID);
982
983 if (hasDispatchPtr())
984 NumUsedUserSGPRs += getNumUserSGPRForField(ID: DispatchPtrID);
985
986 if (hasQueuePtr())
987 NumUsedUserSGPRs += getNumUserSGPRForField(ID: QueuePtrID);
988
989 if (hasKernargSegmentPtr())
990 NumUsedUserSGPRs += getNumUserSGPRForField(ID: KernargSegmentPtrID);
991
992 if (hasDispatchID())
993 NumUsedUserSGPRs += getNumUserSGPRForField(ID: DispatchIdID);
994
995 if (hasFlatScratchInit())
996 NumUsedUserSGPRs += getNumUserSGPRForField(ID: FlatScratchInitID);
997
998 if (hasPrivateSegmentSize())
999 NumUsedUserSGPRs += getNumUserSGPRForField(ID: PrivateSegmentSizeID);
1000}
1001
1002void GCNUserSGPRUsageInfo::allocKernargPreloadSGPRs(unsigned NumSGPRs) {
1003 assert(NumKernargPreloadSGPRs + NumSGPRs <= AMDGPU::getMaxNumUserSGPRs(ST));
1004 NumKernargPreloadSGPRs += NumSGPRs;
1005 NumUsedUserSGPRs += NumSGPRs;
1006}
1007
1008unsigned GCNUserSGPRUsageInfo::getNumFreeUserSGPRs() {
1009 return AMDGPU::getMaxNumUserSGPRs(STI: ST) - NumUsedUserSGPRs;
1010}
1011