1//===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Implements the AMDGPU specific subclass of TargetSubtarget.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUSubtarget.h"
15#include "AMDGPUCallLowering.h"
16#include "AMDGPUInstructionSelector.h"
17#include "AMDGPULegalizerInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "R600Subtarget.h"
20#include "SIMachineFunctionInfo.h"
21#include "Utils/AMDGPUBaseInfo.h"
22#include "llvm/CodeGen/MachineScheduler.h"
23#include "llvm/CodeGen/TargetFrameLowering.h"
24#include "llvm/IR/DiagnosticInfo.h"
25#include "llvm/IR/IntrinsicsAMDGPU.h"
26#include "llvm/IR/IntrinsicsR600.h"
27#include "llvm/IR/MDBuilder.h"
28#include "llvm/Option/LibraryOptions.h"
29#include <algorithm>
30
31#define OPTIONS_STRUCT_DEFS
32#include "AMDGPUOptions.inc"
33
34using namespace llvm;
35
36#define DEBUG_TYPE "amdgpu-subtarget"
37
38// Returns the maximum per-workgroup LDS allocation size (in bytes) that still
39// allows the given function to achieve an occupancy of NWaves waves per
40// SIMD / EU, taking into account only the function's *maximum* workgroup size.
41unsigned
42AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
43 const Function &F) const {
44 const unsigned WaveSize = getWavefrontSize();
45 const unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
46 const unsigned WavesPerWorkgroup =
47 std::max(a: 1u, b: (WorkGroupSize + WaveSize - 1) / WaveSize);
48
49 const unsigned WorkGroupsPerCU =
50 std::max(a: 1u, b: (NWaves * getNumWorkGroupSIMDs()) / WavesPerWorkgroup);
51
52 const unsigned Granularity = std::max(a: LDSAllocationGranularity, b: 1u);
53 return alignDown(Value: std::min(a: getLocalMemorySize() / WorkGroupsPerCU,
54 b: getAddressableLocalMemorySize()),
55 Align: Granularity);
56}
57
58std::pair<unsigned, unsigned> AMDGPUSubtarget::getOccupancyWithWorkGroupSizes(
59 uint32_t LDSBytes, std::pair<unsigned, unsigned> FlatWorkGroupSizes) const {
60
61 // LDS granularity accounted for by aligning the queried LDS size to the
62 // allocation block size.
63 const unsigned Granularity = std::max(a: LDSAllocationGranularity, b: 1u);
64 LDSBytes = alignTo(Value: LDSBytes, Align: Granularity);
65 const unsigned MaxWGsLDS = getLocalMemorySize() / std::max(a: LDSBytes, b: 1u);
66
67 // Queried LDS size may be larger than available on a CU, in which case we
68 // consider the only achievable occupancy to be 1, in line with what we
69 // consider the occupancy to be when the number of requested registers in a
70 // particular bank is higher than the number of available ones in that bank.
71 if (!MaxWGsLDS)
72 return {1, 1};
73
74 const unsigned WaveSize = getWavefrontSize(), WavesPerEU = getMaxWavesPerEU();
75
76 auto PropsFromWGSize = [=](unsigned WGSize)
77 -> std::tuple<const unsigned, const unsigned, unsigned> {
78 unsigned WavesPerWG = divideCeil(Numerator: WGSize, Denominator: WaveSize);
79 unsigned WGsPerCU = std::min(a: getMaxWorkGroupsPerCU(FlatWorkGroupSize: WGSize), b: MaxWGsLDS);
80 return {WavesPerWG, WGsPerCU, WavesPerWG * WGsPerCU};
81 };
82
83 // The maximum group size will generally yield the minimum number of
84 // workgroups, maximum number of waves, and minimum occupancy. The opposite is
85 // generally true for the minimum group size. LDS or barrier ressource
86 // limitations can flip those minimums/maximums.
87 const auto [MinWGSize, MaxWGSize] = FlatWorkGroupSizes;
88 auto [MinWavesPerWG, MaxWGsPerCU, MaxWavesPerCU] = PropsFromWGSize(MinWGSize);
89 auto [MaxWavesPerWG, MinWGsPerCU, MinWavesPerCU] = PropsFromWGSize(MaxWGSize);
90
91 // It is possible that we end up with flipped minimum and maximum number of
92 // waves per CU when the number of minimum/maximum concurrent groups on the CU
93 // is limited by LDS usage or barrier resources.
94 if (MinWavesPerCU >= MaxWavesPerCU) {
95 std::swap(a&: MinWavesPerCU, b&: MaxWavesPerCU);
96 } else {
97 const unsigned WaveSlotsPerCU = WavesPerEU * getNumWorkGroupSIMDs();
98
99 // Look for a potential smaller group size than the maximum which decreases
100 // the concurrent number of waves on the CU for the same number of
101 // concurrent workgroups on the CU.
102 unsigned MinWavesPerCUForWGSize =
103 divideCeil(Numerator: WaveSlotsPerCU, Denominator: MinWGsPerCU + 1) * MinWGsPerCU;
104 if (MinWavesPerCU > MinWavesPerCUForWGSize) {
105 unsigned ExcessSlots = MinWavesPerCU - MinWavesPerCUForWGSize;
106 if (unsigned ExcessSlotsPerWG = ExcessSlots / MinWGsPerCU) {
107 // There may exist a smaller group size than the maximum that achieves
108 // the minimum number of waves per CU. This group size is the largest
109 // possible size that requires MaxWavesPerWG - E waves where E is
110 // maximized under the following constraints.
111 // 1. 0 <= E <= ExcessSlotsPerWG
112 // 2. (MaxWavesPerWG - E) * WaveSize >= MinWGSize
113 MinWavesPerCU -= MinWGsPerCU * std::min(a: ExcessSlotsPerWG,
114 b: MaxWavesPerWG - MinWavesPerWG);
115 }
116 }
117
118 // Look for a potential larger group size than the minimum which increases
119 // the concurrent number of waves on the CU for the same number of
120 // concurrent workgroups on the CU.
121 unsigned LeftoverSlots = WaveSlotsPerCU - MaxWGsPerCU * MinWavesPerWG;
122 if (unsigned LeftoverSlotsPerWG = LeftoverSlots / MaxWGsPerCU) {
123 // There may exist a larger group size than the minimum that achieves the
124 // maximum number of waves per CU. This group size is the smallest
125 // possible size that requires MinWavesPerWG + L waves where L is
126 // maximized under the following constraints.
127 // 1. 0 <= L <= LeftoverSlotsPerWG
128 // 2. (MinWavesPerWG + L - 1) * WaveSize <= MaxWGSize
129 MaxWavesPerCU += MaxWGsPerCU * std::min(a: LeftoverSlotsPerWG,
130 b: ((MaxWGSize - 1) / WaveSize) + 1 -
131 MinWavesPerWG);
132 }
133 }
134
135 // Return the minimum/maximum number of waves on any EU, assuming that all
136 // wavefronts are spread across all EUs as evenly as possible.
137 return {std::clamp(val: MinWavesPerCU / getNumWorkGroupSIMDs(), lo: 1U, hi: WavesPerEU),
138 std::clamp(val: divideCeil(Numerator: MaxWavesPerCU, Denominator: getNumWorkGroupSIMDs()), lo: 1U,
139 hi: WavesPerEU)};
140}
141
142std::pair<unsigned, unsigned> AMDGPUSubtarget::getOccupancyWithWorkGroupSizes(
143 const MachineFunction &MF) const {
144 const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
145 return getOccupancyWithWorkGroupSizes(LDSBytes: MFI->getLDSSize(), F: MF.getFunction());
146}
147
148std::pair<unsigned, unsigned>
149AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
150 switch (CC) {
151 case CallingConv::AMDGPU_VS:
152 case CallingConv::AMDGPU_LS:
153 case CallingConv::AMDGPU_HS:
154 case CallingConv::AMDGPU_ES:
155 case CallingConv::AMDGPU_GS:
156 case CallingConv::AMDGPU_PS:
157 return std::pair(1, getWavefrontSize());
158 default:
159 return std::pair(1u, getMaxFlatWorkGroupSize());
160 }
161}
162
163std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
164 const Function &F) const {
165 // Default minimum/maximum flat work group sizes.
166 std::pair<unsigned, unsigned> Default =
167 getDefaultFlatWorkGroupSize(CC: F.getCallingConv());
168
169 // Requested minimum/maximum flat work group sizes.
170 std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
171 F, Name: "amdgpu-flat-work-group-size", Default);
172
173 // Make sure requested minimum is less than requested maximum.
174 if (Requested.first > Requested.second)
175 return Default;
176
177 // Make sure requested values do not violate subtarget's specifications.
178 if (Requested.first < getMinFlatWorkGroupSize())
179 return Default;
180 if (Requested.second > getMaxFlatWorkGroupSize())
181 return Default;
182
183 return Requested;
184}
185
186bool AMDGPUSubtarget::isSingleWavefrontWorkgroup(const Function &F) const {
187 return getFlatWorkGroupSizes(F).second <= getWavefrontSize();
188}
189
190std::pair<unsigned, unsigned> AMDGPUSubtarget::getEffectiveWavesPerEU(
191 std::pair<unsigned, unsigned> RequestedWavesPerEU,
192 std::pair<unsigned, unsigned> FlatWorkGroupSizes, unsigned LDSBytes) const {
193 // Default minimum/maximum number of waves per EU. The range of flat workgroup
194 // sizes limits the achievable maximum, and we aim to support enough waves per
195 // EU so that we can concurrently execute all waves of a single workgroup of
196 // maximum size on a CU.
197 std::pair<unsigned, unsigned> Default = {
198 getWavesPerEUForWorkGroup(FlatWorkGroupSize: FlatWorkGroupSizes.second),
199 getOccupancyWithWorkGroupSizes(LDSBytes, FlatWorkGroupSizes).second};
200 Default.first = std::min(a: Default.first, b: Default.second);
201
202 // Make sure requested minimum is within the default range and lower than the
203 // requested maximum. The latter must not violate target specification.
204 if (RequestedWavesPerEU.first < Default.first ||
205 RequestedWavesPerEU.first > Default.second ||
206 RequestedWavesPerEU.first > RequestedWavesPerEU.second ||
207 RequestedWavesPerEU.second > getMaxWavesPerEU())
208 return Default;
209
210 // We cannot exceed maximum occupancy implied by flat workgroup size and LDS.
211 RequestedWavesPerEU.second =
212 std::min(a: RequestedWavesPerEU.second, b: Default.second);
213 return RequestedWavesPerEU;
214}
215
216std::pair<unsigned, unsigned>
217AMDGPUSubtarget::getWavesPerEU(const Function &F) const {
218 // Default/requested minimum/maximum flat work group sizes.
219 std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
220 // Minimum number of bytes allocated in the LDS.
221 unsigned LDSBytes =
222 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-lds-size", Default: {0, UINT32_MAX},
223 /*OnlyFirstRequired=*/true)
224 .first;
225 return getWavesPerEU(FlatWorkGroupSizes, LDSBytes, F);
226}
227
228std::pair<unsigned, unsigned>
229AMDGPUSubtarget::getWavesPerEU(std::pair<unsigned, unsigned> FlatWorkGroupSizes,
230 unsigned LDSBytes, const Function &F) const {
231 // Default minimum/maximum number of waves per execution unit.
232 std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
233
234 // Requested minimum/maximum number of waves per execution unit.
235 std::pair<unsigned, unsigned> Requested =
236 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu", Default, OnlyFirstRequired: true);
237 return getEffectiveWavesPerEU(RequestedWavesPerEU: Requested, FlatWorkGroupSizes, LDSBytes);
238}
239
240std::optional<unsigned>
241AMDGPUSubtarget::getReqdWorkGroupSize(const Function &Kernel,
242 unsigned Dim) const {
243 auto *Node = Kernel.getMetadata(Kind: "reqd_work_group_size");
244 if (Node && Node->getNumOperands() == 3)
245 return static_cast<unsigned>(
246 mdconst::extract<ConstantInt>(MD: Node->getOperand(I: Dim))->getZExtValue());
247 return std::nullopt;
248}
249
250bool AMDGPUSubtarget::hasWavefrontsEvenlySplittingXDim(
251 const Function &F, bool RequiresUniformYZ) const {
252 auto *Node = F.getMetadata(Kind: "reqd_work_group_size");
253 if (!Node || Node->getNumOperands() != 3)
254 return false;
255 uint64_t XLen =
256 mdconst::extract<ConstantInt>(MD: Node->getOperand(I: 0))->getZExtValue();
257 uint64_t YLen =
258 mdconst::extract<ConstantInt>(MD: Node->getOperand(I: 1))->getZExtValue();
259 uint64_t ZLen =
260 mdconst::extract<ConstantInt>(MD: Node->getOperand(I: 2))->getZExtValue();
261
262 bool Is1D = YLen <= 1 && ZLen <= 1;
263 bool IsXLargeEnough =
264 isPowerOf2_64(Value: XLen) && (!RequiresUniformYZ || XLen >= getWavefrontSize());
265 return Is1D || IsXLargeEnough;
266}
267
268bool AMDGPUSubtarget::isMesaKernel(const Function &F) const {
269 return isMesa3DOS() && !AMDGPU::isShader(CC: F.getCallingConv());
270}
271
272unsigned AMDGPUSubtarget::getMaxWorkitemID(const Function &Kernel,
273 unsigned Dimension) const {
274 std::optional<unsigned> ReqdSize = getReqdWorkGroupSize(Kernel, Dim: Dimension);
275 if (ReqdSize)
276 return *ReqdSize - 1;
277 return getFlatWorkGroupSizes(F: Kernel).second - 1;
278}
279
280bool AMDGPUSubtarget::isSingleLaneExecution(const Function &Func) const {
281 for (int I = 0; I < 3; ++I) {
282 if (getMaxWorkitemID(Kernel: Func, Dimension: I) > 0)
283 return false;
284 }
285
286 // If the function may call the WWM intrinsic, just return false as
287 // all threads will be active at some point
288 if (!Func.hasFnAttribute(Kind: "amdgpu-no-wwm"))
289 return false;
290
291 return true;
292}
293
294bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
295 Function *Kernel = I->getFunction();
296 unsigned MinSize = 0;
297 unsigned MaxSize = getFlatWorkGroupSizes(F: *Kernel).second;
298 bool IdQuery = false;
299
300 // If reqd_work_group_size is present it narrows value down.
301 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
302 const Function *F = CI->getCalledFunction();
303 if (F) {
304 unsigned Dim = UINT_MAX;
305 switch (F->getIntrinsicID()) {
306 case Intrinsic::amdgcn_workitem_id_x:
307 case Intrinsic::r600_read_tidig_x:
308 IdQuery = true;
309 [[fallthrough]];
310 case Intrinsic::r600_read_local_size_x:
311 Dim = 0;
312 break;
313 case Intrinsic::amdgcn_workitem_id_y:
314 case Intrinsic::r600_read_tidig_y:
315 IdQuery = true;
316 [[fallthrough]];
317 case Intrinsic::r600_read_local_size_y:
318 Dim = 1;
319 break;
320 case Intrinsic::amdgcn_workitem_id_z:
321 case Intrinsic::r600_read_tidig_z:
322 IdQuery = true;
323 [[fallthrough]];
324 case Intrinsic::r600_read_local_size_z:
325 Dim = 2;
326 break;
327 default:
328 break;
329 }
330
331 if (Dim <= 3) {
332 std::optional<unsigned> ReqdSize = getReqdWorkGroupSize(Kernel: *Kernel, Dim);
333 if (ReqdSize)
334 MinSize = MaxSize = *ReqdSize;
335 }
336 }
337 }
338
339 if (!MaxSize)
340 return false;
341
342 // Range metadata is [Lo, Hi). For ID query we need to pass max size
343 // as Hi. For size query we need to pass Hi + 1.
344 if (IdQuery)
345 MinSize = 0;
346 else
347 ++MaxSize;
348
349 APInt Lower{32, MinSize};
350 APInt Upper{32, MaxSize};
351 if (auto *CI = dyn_cast<CallBase>(Val: I)) {
352 ConstantRange Range(Lower, Upper);
353 CI->addRangeRetAttr(CR: Range);
354 } else {
355 MDBuilder MDB(I->getContext());
356 MDNode *MaxWorkGroupSizeRange = MDB.createRange(Lo: Lower, Hi: Upper);
357 I->setMetadata(KindID: LLVMContext::MD_range, Node: MaxWorkGroupSizeRange);
358 }
359 return true;
360}
361
362unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const {
363
364 // We don't allocate the segment if we know the implicit arguments weren't
365 // used, even if the ABI implies we need them.
366 if (F.hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr"))
367 return 0;
368
369 if (isMesaKernel(F))
370 return 16;
371
372 // Assume all implicit inputs are used by default
373 const Module *M = F.getParent();
374 unsigned NBytes =
375 AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5 ? 256 : 56;
376 return static_cast<unsigned>(
377 F.getFnAttributeAsParsedInteger(Kind: "amdgpu-implicitarg-num-bytes", Default: NBytes));
378}
379
380uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
381 Align &MaxAlign) const {
382 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
383 F.getCallingConv() == CallingConv::SPIR_KERNEL);
384
385 const DataLayout &DL = F.getDataLayout();
386 uint64_t ExplicitArgBytes = 0;
387 MaxAlign = Align(1);
388
389 for (const Argument &Arg : F.args()) {
390 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument"))
391 continue;
392
393 const bool IsByRef = Arg.hasByRefAttr();
394 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
395 Align Alignment = DL.getValueOrABITypeAlignment(
396 Alignment: IsByRef ? Arg.getParamAlign() : std::nullopt, Ty: ArgTy);
397 uint64_t AllocSize = DL.getTypeAllocSize(Ty: ArgTy);
398 ExplicitArgBytes = alignTo(Size: ExplicitArgBytes, A: Alignment) + AllocSize;
399 MaxAlign = std::max(a: MaxAlign, b: Alignment);
400 }
401
402 return ExplicitArgBytes;
403}
404
405unsigned AMDGPUSubtarget::getKernArgSegmentSize(const Function &F,
406 Align &MaxAlign) const {
407 if (F.getCallingConv() != CallingConv::AMDGPU_KERNEL &&
408 F.getCallingConv() != CallingConv::SPIR_KERNEL)
409 return 0;
410
411 uint64_t ExplicitArgBytes = getExplicitKernArgSize(F, MaxAlign);
412
413 unsigned ExplicitOffset = getExplicitKernelArgOffset();
414
415 uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
416 unsigned ImplicitBytes = getImplicitArgNumBytes(F);
417 if (ImplicitBytes != 0) {
418 const Align Alignment = getAlignmentForImplicitArgPtr();
419 TotalSize = alignTo(Size: ExplicitArgBytes, A: Alignment) + ImplicitBytes;
420 MaxAlign = std::max(a: MaxAlign, b: Alignment);
421 }
422
423 // Being able to dereference past the end is useful for emitting scalar loads.
424 return static_cast<unsigned>(alignTo(Value: TotalSize, Align: 4));
425}
426
427AMDGPUDwarfFlavour AMDGPUSubtarget::getAMDGPUDwarfFlavour() const {
428 return getWavefrontSize() == 32 ? AMDGPUDwarfFlavour::Wave32
429 : AMDGPUDwarfFlavour::Wave64;
430}
431
432const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
433 if (MF.getTarget().getTargetTriple().isAMDGCN())
434 return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
435 return static_cast<const AMDGPUSubtarget &>(MF.getSubtarget<R600Subtarget>());
436}
437
438const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {
439 if (TM.getTargetTriple().isAMDGCN())
440 return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));
441 return static_cast<const AMDGPUSubtarget &>(
442 TM.getSubtarget<R600Subtarget>(F));
443}
444