1//===-- AMDGPUSubtarget.cpp - AMDGPU Subtarget Information ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file
10/// Implements the AMDGPU specific subclass of TargetSubtarget.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPUSubtarget.h"
15#include "AMDGPUCallLowering.h"
16#include "AMDGPUInstructionSelector.h"
17#include "AMDGPULegalizerInfo.h"
18#include "AMDGPURegisterBankInfo.h"
19#include "R600Subtarget.h"
20#include "SIMachineFunctionInfo.h"
21#include "Utils/AMDGPUBaseInfo.h"
22#include "llvm/CodeGen/GlobalISel/InlineAsmLowering.h"
23#include "llvm/CodeGen/MachineScheduler.h"
24#include "llvm/CodeGen/TargetFrameLowering.h"
25#include "llvm/IR/DiagnosticInfo.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/IR/IntrinsicsR600.h"
28#include "llvm/IR/MDBuilder.h"
29#include <algorithm>
30
31using namespace llvm;
32
33#define DEBUG_TYPE "amdgpu-subtarget"
34
35// Returns the maximum per-workgroup LDS allocation size (in bytes) that still
36// allows the given function to achieve an occupancy of NWaves waves per
37// SIMD / EU, taking into account only the function's *maximum* workgroup size.
38unsigned
39AMDGPUSubtarget::getMaxLocalMemSizeWithWaveCount(unsigned NWaves,
40 const Function &F) const {
41 const unsigned WaveSize = getWavefrontSize();
42 const unsigned WorkGroupSize = getFlatWorkGroupSizes(F).second;
43 const unsigned WavesPerWorkgroup =
44 std::max(a: 1u, b: (WorkGroupSize + WaveSize - 1) / WaveSize);
45
46 const unsigned WorkGroupsPerCU =
47 std::max(a: 1u, b: (NWaves * getNumWorkGroupSIMDs()) / WavesPerWorkgroup);
48
49 const unsigned Granularity = std::max(a: LDSAllocationGranularity, b: 1u);
50 return alignDown(Value: getLocalMemorySize() / WorkGroupsPerCU, Align: Granularity);
51}
52
53std::pair<unsigned, unsigned> AMDGPUSubtarget::getOccupancyWithWorkGroupSizes(
54 uint32_t LDSBytes, std::pair<unsigned, unsigned> FlatWorkGroupSizes) const {
55
56 // LDS granularity accounted for by aligning the queried LDS size to the
57 // allocation block size.
58 const unsigned Granularity = std::max(a: LDSAllocationGranularity, b: 1u);
59 LDSBytes = alignTo(Value: LDSBytes, Align: Granularity);
60 const unsigned MaxWGsLDS = getLocalMemorySize() / std::max(a: LDSBytes, b: 1u);
61
62 // Queried LDS size may be larger than available on a CU, in which case we
63 // consider the only achievable occupancy to be 1, in line with what we
64 // consider the occupancy to be when the number of requested registers in a
65 // particular bank is higher than the number of available ones in that bank.
66 if (!MaxWGsLDS)
67 return {1, 1};
68
69 const unsigned WaveSize = getWavefrontSize(), WavesPerEU = getMaxWavesPerEU();
70
71 auto PropsFromWGSize = [=](unsigned WGSize)
72 -> std::tuple<const unsigned, const unsigned, unsigned> {
73 unsigned WavesPerWG = divideCeil(Numerator: WGSize, Denominator: WaveSize);
74 unsigned WGsPerCU = std::min(a: getMaxWorkGroupsPerCU(FlatWorkGroupSize: WGSize), b: MaxWGsLDS);
75 return {WavesPerWG, WGsPerCU, WavesPerWG * WGsPerCU};
76 };
77
78 // The maximum group size will generally yield the minimum number of
79 // workgroups, maximum number of waves, and minimum occupancy. The opposite is
80 // generally true for the minimum group size. LDS or barrier ressource
81 // limitations can flip those minimums/maximums.
82 const auto [MinWGSize, MaxWGSize] = FlatWorkGroupSizes;
83 auto [MinWavesPerWG, MaxWGsPerCU, MaxWavesPerCU] = PropsFromWGSize(MinWGSize);
84 auto [MaxWavesPerWG, MinWGsPerCU, MinWavesPerCU] = PropsFromWGSize(MaxWGSize);
85
86 // It is possible that we end up with flipped minimum and maximum number of
87 // waves per CU when the number of minimum/maximum concurrent groups on the CU
88 // is limited by LDS usage or barrier resources.
89 if (MinWavesPerCU >= MaxWavesPerCU) {
90 std::swap(a&: MinWavesPerCU, b&: MaxWavesPerCU);
91 } else {
92 const unsigned WaveSlotsPerCU = WavesPerEU * getNumWorkGroupSIMDs();
93
94 // Look for a potential smaller group size than the maximum which decreases
95 // the concurrent number of waves on the CU for the same number of
96 // concurrent workgroups on the CU.
97 unsigned MinWavesPerCUForWGSize =
98 divideCeil(Numerator: WaveSlotsPerCU, Denominator: MinWGsPerCU + 1) * MinWGsPerCU;
99 if (MinWavesPerCU > MinWavesPerCUForWGSize) {
100 unsigned ExcessSlots = MinWavesPerCU - MinWavesPerCUForWGSize;
101 if (unsigned ExcessSlotsPerWG = ExcessSlots / MinWGsPerCU) {
102 // There may exist a smaller group size than the maximum that achieves
103 // the minimum number of waves per CU. This group size is the largest
104 // possible size that requires MaxWavesPerWG - E waves where E is
105 // maximized under the following constraints.
106 // 1. 0 <= E <= ExcessSlotsPerWG
107 // 2. (MaxWavesPerWG - E) * WaveSize >= MinWGSize
108 MinWavesPerCU -= MinWGsPerCU * std::min(a: ExcessSlotsPerWG,
109 b: MaxWavesPerWG - MinWavesPerWG);
110 }
111 }
112
113 // Look for a potential larger group size than the minimum which increases
114 // the concurrent number of waves on the CU for the same number of
115 // concurrent workgroups on the CU.
116 unsigned LeftoverSlots = WaveSlotsPerCU - MaxWGsPerCU * MinWavesPerWG;
117 if (unsigned LeftoverSlotsPerWG = LeftoverSlots / MaxWGsPerCU) {
118 // There may exist a larger group size than the minimum that achieves the
119 // maximum number of waves per CU. This group size is the smallest
120 // possible size that requires MinWavesPerWG + L waves where L is
121 // maximized under the following constraints.
122 // 1. 0 <= L <= LeftoverSlotsPerWG
123 // 2. (MinWavesPerWG + L - 1) * WaveSize <= MaxWGSize
124 MaxWavesPerCU += MaxWGsPerCU * std::min(a: LeftoverSlotsPerWG,
125 b: ((MaxWGSize - 1) / WaveSize) + 1 -
126 MinWavesPerWG);
127 }
128 }
129
130 // Return the minimum/maximum number of waves on any EU, assuming that all
131 // wavefronts are spread across all EUs as evenly as possible.
132 return {std::clamp(val: MinWavesPerCU / getNumWorkGroupSIMDs(), lo: 1U, hi: WavesPerEU),
133 std::clamp(val: divideCeil(Numerator: MaxWavesPerCU, Denominator: getNumWorkGroupSIMDs()), lo: 1U,
134 hi: WavesPerEU)};
135}
136
137std::pair<unsigned, unsigned> AMDGPUSubtarget::getOccupancyWithWorkGroupSizes(
138 const MachineFunction &MF) const {
139 const auto *MFI = MF.getInfo<SIMachineFunctionInfo>();
140 return getOccupancyWithWorkGroupSizes(LDSBytes: MFI->getLDSSize(), F: MF.getFunction());
141}
142
143std::pair<unsigned, unsigned>
144AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const {
145 switch (CC) {
146 case CallingConv::AMDGPU_VS:
147 case CallingConv::AMDGPU_LS:
148 case CallingConv::AMDGPU_HS:
149 case CallingConv::AMDGPU_ES:
150 case CallingConv::AMDGPU_GS:
151 case CallingConv::AMDGPU_PS:
152 return std::pair(1, getWavefrontSize());
153 default:
154 return std::pair(1u, getMaxFlatWorkGroupSize());
155 }
156}
157
158std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes(
159 const Function &F) const {
160 // Default minimum/maximum flat work group sizes.
161 std::pair<unsigned, unsigned> Default =
162 getDefaultFlatWorkGroupSize(CC: F.getCallingConv());
163
164 // Requested minimum/maximum flat work group sizes.
165 std::pair<unsigned, unsigned> Requested = AMDGPU::getIntegerPairAttribute(
166 F, Name: "amdgpu-flat-work-group-size", Default);
167
168 // Make sure requested minimum is less than requested maximum.
169 if (Requested.first > Requested.second)
170 return Default;
171
172 // Make sure requested values do not violate subtarget's specifications.
173 if (Requested.first < getMinFlatWorkGroupSize())
174 return Default;
175 if (Requested.second > getMaxFlatWorkGroupSize())
176 return Default;
177
178 return Requested;
179}
180
181std::pair<unsigned, unsigned> AMDGPUSubtarget::getEffectiveWavesPerEU(
182 std::pair<unsigned, unsigned> RequestedWavesPerEU,
183 std::pair<unsigned, unsigned> FlatWorkGroupSizes, unsigned LDSBytes) const {
184 // Default minimum/maximum number of waves per EU. The range of flat workgroup
185 // sizes limits the achievable maximum, and we aim to support enough waves per
186 // EU so that we can concurrently execute all waves of a single workgroup of
187 // maximum size on a CU.
188 std::pair<unsigned, unsigned> Default = {
189 getWavesPerEUForWorkGroup(FlatWorkGroupSize: FlatWorkGroupSizes.second),
190 getOccupancyWithWorkGroupSizes(LDSBytes, FlatWorkGroupSizes).second};
191 Default.first = std::min(a: Default.first, b: Default.second);
192
193 // Make sure requested minimum is within the default range and lower than the
194 // requested maximum. The latter must not violate target specification.
195 if (RequestedWavesPerEU.first < Default.first ||
196 RequestedWavesPerEU.first > Default.second ||
197 RequestedWavesPerEU.first > RequestedWavesPerEU.second ||
198 RequestedWavesPerEU.second > getMaxWavesPerEU())
199 return Default;
200
201 // We cannot exceed maximum occupancy implied by flat workgroup size and LDS.
202 RequestedWavesPerEU.second =
203 std::min(a: RequestedWavesPerEU.second, b: Default.second);
204 return RequestedWavesPerEU;
205}
206
207std::pair<unsigned, unsigned>
208AMDGPUSubtarget::getWavesPerEU(const Function &F) const {
209 // Default/requested minimum/maximum flat work group sizes.
210 std::pair<unsigned, unsigned> FlatWorkGroupSizes = getFlatWorkGroupSizes(F);
211 // Minimum number of bytes allocated in the LDS.
212 unsigned LDSBytes =
213 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-lds-size", Default: {0, UINT32_MAX},
214 /*OnlyFirstRequired=*/true)
215 .first;
216 return getWavesPerEU(FlatWorkGroupSizes, LDSBytes, F);
217}
218
219std::pair<unsigned, unsigned>
220AMDGPUSubtarget::getWavesPerEU(std::pair<unsigned, unsigned> FlatWorkGroupSizes,
221 unsigned LDSBytes, const Function &F) const {
222 // Default minimum/maximum number of waves per execution unit.
223 std::pair<unsigned, unsigned> Default(1, getMaxWavesPerEU());
224
225 // Requested minimum/maximum number of waves per execution unit.
226 std::pair<unsigned, unsigned> Requested =
227 AMDGPU::getIntegerPairAttribute(F, Name: "amdgpu-waves-per-eu", Default, OnlyFirstRequired: true);
228 return getEffectiveWavesPerEU(RequestedWavesPerEU: Requested, FlatWorkGroupSizes, LDSBytes);
229}
230
231std::optional<unsigned>
232AMDGPUSubtarget::getReqdWorkGroupSize(const Function &Kernel,
233 unsigned Dim) const {
234 auto *Node = Kernel.getMetadata(Kind: "reqd_work_group_size");
235 if (Node && Node->getNumOperands() == 3)
236 return mdconst::extract<ConstantInt>(MD: Node->getOperand(I: Dim))->getZExtValue();
237 return std::nullopt;
238}
239
240bool AMDGPUSubtarget::hasWavefrontsEvenlySplittingXDim(
241 const Function &F, bool RequiresUniformYZ) const {
242 auto *Node = F.getMetadata(Kind: "reqd_work_group_size");
243 if (!Node || Node->getNumOperands() != 3)
244 return false;
245 unsigned XLen =
246 mdconst::extract<ConstantInt>(MD: Node->getOperand(I: 0))->getZExtValue();
247 unsigned YLen =
248 mdconst::extract<ConstantInt>(MD: Node->getOperand(I: 1))->getZExtValue();
249 unsigned ZLen =
250 mdconst::extract<ConstantInt>(MD: Node->getOperand(I: 2))->getZExtValue();
251
252 bool Is1D = YLen <= 1 && ZLen <= 1;
253 bool IsXLargeEnough =
254 isPowerOf2_32(Value: XLen) && (!RequiresUniformYZ || XLen >= getWavefrontSize());
255 return Is1D || IsXLargeEnough;
256}
257
258bool AMDGPUSubtarget::isMesaKernel(const Function &F) const {
259 return isMesa3DOS() && !AMDGPU::isShader(CC: F.getCallingConv());
260}
261
262unsigned AMDGPUSubtarget::getMaxWorkitemID(const Function &Kernel,
263 unsigned Dimension) const {
264 std::optional<unsigned> ReqdSize = getReqdWorkGroupSize(Kernel, Dim: Dimension);
265 if (ReqdSize)
266 return *ReqdSize - 1;
267 return getFlatWorkGroupSizes(F: Kernel).second - 1;
268}
269
270bool AMDGPUSubtarget::isSingleLaneExecution(const Function &Func) const {
271 for (int I = 0; I < 3; ++I) {
272 if (getMaxWorkitemID(Kernel: Func, Dimension: I) > 0)
273 return false;
274 }
275
276 // If the function may call the WWM intrinsic, just return false as
277 // all threads will be active at some point
278 if (!Func.hasFnAttribute(Kind: "amdgpu-no-wwm"))
279 return false;
280
281 return true;
282}
283
284bool AMDGPUSubtarget::makeLIDRangeMetadata(Instruction *I) const {
285 Function *Kernel = I->getFunction();
286 unsigned MinSize = 0;
287 unsigned MaxSize = getFlatWorkGroupSizes(F: *Kernel).second;
288 bool IdQuery = false;
289
290 // If reqd_work_group_size is present it narrows value down.
291 if (auto *CI = dyn_cast<CallInst>(Val: I)) {
292 const Function *F = CI->getCalledFunction();
293 if (F) {
294 unsigned Dim = UINT_MAX;
295 switch (F->getIntrinsicID()) {
296 case Intrinsic::amdgcn_workitem_id_x:
297 case Intrinsic::r600_read_tidig_x:
298 IdQuery = true;
299 [[fallthrough]];
300 case Intrinsic::r600_read_local_size_x:
301 Dim = 0;
302 break;
303 case Intrinsic::amdgcn_workitem_id_y:
304 case Intrinsic::r600_read_tidig_y:
305 IdQuery = true;
306 [[fallthrough]];
307 case Intrinsic::r600_read_local_size_y:
308 Dim = 1;
309 break;
310 case Intrinsic::amdgcn_workitem_id_z:
311 case Intrinsic::r600_read_tidig_z:
312 IdQuery = true;
313 [[fallthrough]];
314 case Intrinsic::r600_read_local_size_z:
315 Dim = 2;
316 break;
317 default:
318 break;
319 }
320
321 if (Dim <= 3) {
322 std::optional<unsigned> ReqdSize = getReqdWorkGroupSize(Kernel: *Kernel, Dim);
323 if (ReqdSize)
324 MinSize = MaxSize = *ReqdSize;
325 }
326 }
327 }
328
329 if (!MaxSize)
330 return false;
331
332 // Range metadata is [Lo, Hi). For ID query we need to pass max size
333 // as Hi. For size query we need to pass Hi + 1.
334 if (IdQuery)
335 MinSize = 0;
336 else
337 ++MaxSize;
338
339 APInt Lower{32, MinSize};
340 APInt Upper{32, MaxSize};
341 if (auto *CI = dyn_cast<CallBase>(Val: I)) {
342 ConstantRange Range(Lower, Upper);
343 CI->addRangeRetAttr(CR: Range);
344 } else {
345 MDBuilder MDB(I->getContext());
346 MDNode *MaxWorkGroupSizeRange = MDB.createRange(Lo: Lower, Hi: Upper);
347 I->setMetadata(KindID: LLVMContext::MD_range, Node: MaxWorkGroupSizeRange);
348 }
349 return true;
350}
351
352unsigned AMDGPUSubtarget::getImplicitArgNumBytes(const Function &F) const {
353
354 // We don't allocate the segment if we know the implicit arguments weren't
355 // used, even if the ABI implies we need them.
356 if (F.hasFnAttribute(Kind: "amdgpu-no-implicitarg-ptr"))
357 return 0;
358
359 if (isMesaKernel(F))
360 return 16;
361
362 // Assume all implicit inputs are used by default
363 const Module *M = F.getParent();
364 unsigned NBytes =
365 AMDGPU::getAMDHSACodeObjectVersion(M: *M) >= AMDGPU::AMDHSA_COV5 ? 256 : 56;
366 return F.getFnAttributeAsParsedInteger(Kind: "amdgpu-implicitarg-num-bytes",
367 Default: NBytes);
368}
369
370uint64_t AMDGPUSubtarget::getExplicitKernArgSize(const Function &F,
371 Align &MaxAlign) const {
372 assert(F.getCallingConv() == CallingConv::AMDGPU_KERNEL ||
373 F.getCallingConv() == CallingConv::SPIR_KERNEL);
374
375 const DataLayout &DL = F.getDataLayout();
376 uint64_t ExplicitArgBytes = 0;
377 MaxAlign = Align(1);
378
379 for (const Argument &Arg : F.args()) {
380 if (Arg.hasAttribute(Kind: "amdgpu-hidden-argument"))
381 continue;
382
383 const bool IsByRef = Arg.hasByRefAttr();
384 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
385 Align Alignment = DL.getValueOrABITypeAlignment(
386 Alignment: IsByRef ? Arg.getParamAlign() : std::nullopt, Ty: ArgTy);
387 uint64_t AllocSize = DL.getTypeAllocSize(Ty: ArgTy);
388 ExplicitArgBytes = alignTo(Size: ExplicitArgBytes, A: Alignment) + AllocSize;
389 MaxAlign = std::max(a: MaxAlign, b: Alignment);
390 }
391
392 return ExplicitArgBytes;
393}
394
395unsigned AMDGPUSubtarget::getKernArgSegmentSize(const Function &F,
396 Align &MaxAlign) const {
397 if (F.getCallingConv() != CallingConv::AMDGPU_KERNEL &&
398 F.getCallingConv() != CallingConv::SPIR_KERNEL)
399 return 0;
400
401 uint64_t ExplicitArgBytes = getExplicitKernArgSize(F, MaxAlign);
402
403 unsigned ExplicitOffset = getExplicitKernelArgOffset();
404
405 uint64_t TotalSize = ExplicitOffset + ExplicitArgBytes;
406 unsigned ImplicitBytes = getImplicitArgNumBytes(F);
407 if (ImplicitBytes != 0) {
408 const Align Alignment = getAlignmentForImplicitArgPtr();
409 TotalSize = alignTo(Size: ExplicitArgBytes, A: Alignment) + ImplicitBytes;
410 MaxAlign = std::max(a: MaxAlign, b: Alignment);
411 }
412
413 // Being able to dereference past the end is useful for emitting scalar loads.
414 return alignTo(Value: TotalSize, Align: 4);
415}
416
417AMDGPUDwarfFlavour AMDGPUSubtarget::getAMDGPUDwarfFlavour() const {
418 return getWavefrontSize() == 32 ? AMDGPUDwarfFlavour::Wave32
419 : AMDGPUDwarfFlavour::Wave64;
420}
421
422const AMDGPUSubtarget &AMDGPUSubtarget::get(const MachineFunction &MF) {
423 if (MF.getTarget().getTargetTriple().isAMDGCN())
424 return static_cast<const AMDGPUSubtarget&>(MF.getSubtarget<GCNSubtarget>());
425 return static_cast<const AMDGPUSubtarget &>(MF.getSubtarget<R600Subtarget>());
426}
427
428const AMDGPUSubtarget &AMDGPUSubtarget::get(const TargetMachine &TM, const Function &F) {
429 if (TM.getTargetTriple().isAMDGCN())
430 return static_cast<const AMDGPUSubtarget&>(TM.getSubtarget<GCNSubtarget>(F));
431 return static_cast<const AMDGPUSubtarget &>(
432 TM.getSubtarget<R600Subtarget>(F));
433}
434