1//===-- AMDGPUMachineFunctionInfo.cpp ---------------------------------------=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUMachineFunctionInfo.h"
10#include "AMDGPUMemoryUtils.h"
11#include "AMDGPUSubtarget.h"
12#include "Utils/AMDGPUBaseInfo.h"
13#include "llvm/CodeGen/MachineModuleInfo.h"
14#include "llvm/IR/ConstantRange.h"
15#include "llvm/IR/Constants.h"
16#include "llvm/IR/Metadata.h"
17#include "llvm/Support/AMDGPUAddrSpace.h"
18#include "llvm/Target/TargetMachine.h"
19
20using namespace llvm;
21
22static const GlobalVariable *
23getKernelDynLDSGlobalFromFunction(const Function &F) {
24 const Module *M = F.getParent();
25 SmallString<64> KernelDynLDSName("llvm.amdgcn.");
26 KernelDynLDSName += F.getName();
27 KernelDynLDSName += ".dynlds";
28 return M->getNamedGlobal(Name: KernelDynLDSName);
29}
30
31static bool hasLDSKernelArgument(const Function &F) {
32 for (const Argument &Arg : F.args()) {
33 Type *ArgTy = Arg.getType();
34 if (auto *PtrTy = dyn_cast<PointerType>(Val: ArgTy)) {
35 if (PtrTy->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
36 return true;
37 }
38 }
39 return false;
40}
41
42AMDGPUMachineFunctionInfo::AMDGPUMachineFunctionInfo(const Function &F,
43 const AMDGPUSubtarget &ST)
44 : IsEntryFunction(AMDGPU::isEntryFunctionCC(CC: F.getCallingConv())),
45 IsModuleEntryFunction(
46 AMDGPU::isModuleEntryFunctionCC(CC: F.getCallingConv())),
47 IsChainFunction(AMDGPU::isChainCC(CC: F.getCallingConv())) {
48
49 // FIXME: Should initialize KernArgSize based on ExplicitKernelArgOffset,
50 // except reserved size is not correctly aligned.
51
52 Attribute MemBoundAttr = F.getFnAttribute(Kind: "amdgpu-memory-bound");
53 MemoryBound = MemBoundAttr.getValueAsBool();
54
55 Attribute WaveLimitAttr = F.getFnAttribute(Kind: "amdgpu-wave-limiter");
56 WaveLimiter = WaveLimitAttr.getValueAsBool();
57
58 // FIXME: How is this attribute supposed to interact with statically known
59 // global sizes?
60 StringRef S = F.getFnAttribute(Kind: "amdgpu-gds-size").getValueAsString();
61 if (!S.empty())
62 S.consumeInteger(Radix: 0, Result&: GDSSize);
63
64 // Assume the attribute allocates before any known GDS globals.
65 StaticGDSSize = GDSSize;
66
67 // Second value, if present, is the maximum value that can be assigned.
68 // Useful in PromoteAlloca or for LDS spills. Could be used for diagnostics
69 // during codegen.
70 std::pair<unsigned, unsigned> LDSSizeRange = AMDGPU::getIntegerPairAttribute(
71 F, Name: "amdgpu-lds-size", Default: {0, UINT32_MAX}, OnlyFirstRequired: true);
72
73 // The two separate variables are only profitable when the LDS module lowering
74 // pass is disabled. If graphics does not use dynamic LDS, this is never
75 // profitable. Leaving cleanup for a later change.
76 LDSSize = LDSSizeRange.first;
77 StaticLDSSize = LDSSize;
78
79 CallingConv::ID CC = F.getCallingConv();
80 if (CC == CallingConv::AMDGPU_KERNEL || CC == CallingConv::SPIR_KERNEL)
81 ExplicitKernArgSize = ST.getExplicitKernArgSize(F, MaxAlign&: MaxKernArgAlign);
82
83 const GlobalVariable *DynLdsGlobal = getKernelDynLDSGlobalFromFunction(F);
84 if (DynLdsGlobal || hasLDSKernelArgument(F))
85 UsesDynamicLDS = true;
86}
87
88unsigned AMDGPUMachineFunctionInfo::allocateLDSGlobal(const DataLayout &DL,
89 const GlobalVariable &GV,
90 Align Trailing) {
91 auto Entry = LocalMemoryObjects.insert(KV: std::pair(&GV, 0));
92 if (!Entry.second)
93 return Entry.first->second;
94
95 Align Alignment =
96 DL.getValueOrABITypeAlignment(Alignment: GV.getAlign(), Ty: GV.getValueType());
97
98 unsigned Offset;
99 if (GV.getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
100 std::optional<uint32_t> MaybeAbs =
101 get32BitAbsoluteAddress(GV, AS: AMDGPUAS::LOCAL_ADDRESS);
102 if (MaybeAbs) {
103 // Absolute address LDS variables that exist prior to the LDS lowering
104 // pass raise a fatal error in that pass. These failure modes are only
105 // reachable if that lowering pass is disabled or broken. If/when adding
106 // support for absolute addresses on user specified variables, the
107 // alignment check moves to the lowering pass and the frame calculation
108 // needs to take the user variables into consideration.
109
110 uint32_t ObjectStart = *MaybeAbs;
111
112 if (ObjectStart != alignTo(Size: ObjectStart, A: Alignment)) {
113 report_fatal_error(reason: "Absolute address LDS variable inconsistent with "
114 "variable alignment");
115 }
116
117 if (isModuleEntryFunction()) {
118 // If this is a module entry function, we can also sanity check against
119 // the static frame. Strictly it would be better to check against the
120 // attribute, i.e. that the variable is within the always-allocated
121 // section, and not within some other non-absolute-address object
122 // allocated here, but the extra error detection is minimal and we would
123 // have to pass the Function around or cache the attribute value.
124 uint32_t ObjectEnd = ObjectStart + GV.getGlobalSize(DL);
125 if (ObjectEnd > StaticLDSSize) {
126 report_fatal_error(
127 reason: "Absolute address LDS variable outside of static frame");
128 }
129 }
130
131 Entry.first->second = ObjectStart;
132 return ObjectStart;
133 }
134
135 /// TODO: We should sort these to minimize wasted space due to alignment
136 /// padding. Currently the padding is decided by the first encountered use
137 /// during lowering.
138 Offset = StaticLDSSize = alignTo(Size: StaticLDSSize, A: Alignment);
139
140 StaticLDSSize += GV.getGlobalSize(DL);
141
142 // Align LDS size to trailing, e.g. for aligning dynamic shared memory
143 LDSSize = alignTo(Size: StaticLDSSize, A: Trailing);
144 } else {
145 assert(GV.getAddressSpace() == AMDGPUAS::REGION_ADDRESS &&
146 "expected region address space");
147
148 Offset = StaticGDSSize = alignTo(Size: StaticGDSSize, A: Alignment);
149 StaticGDSSize += GV.getGlobalSize(DL);
150
151 // FIXME: Apply alignment of dynamic GDS
152 GDSSize = StaticGDSSize;
153 }
154
155 Entry.first->second = Offset;
156 return Offset;
157}
158
159unsigned
160AMDGPUMachineFunctionInfo::allocateBarrierGlobal(const DataLayout &DL,
161 const GlobalVariable &GV) {
162 assert(AMDGPU::isNamedBarrier(GV));
163 std::optional<unsigned> BarAddr =
164 get32BitAbsoluteAddress(GV, AS: AMDGPUAS::BARRIER);
165 assert(BarAddr && "Expected named barrier global to have an address!");
166
167 if (*BarAddr == 0) {
168 // We cannot allow this because some places in CodeGen (rightfully) assume a
169 // GV address is never null. For example, there are no null checks on
170 // addrspacecast if the pointer is a GV pointer.
171 reportFatalInternalError(reason: "named barrier global variable '" + GV.getName() +
172 "' has a NULL address, which is not supported");
173 }
174
175 unsigned BarCnt = AMDGPU::getNumNamedBarriersDeclared(DL, GV);
176 recordNumNamedBarriers(GVAddr: BarAddr.value(), BarCnt);
177 return BarAddr.value();
178}
179
180std::optional<uint32_t>
181AMDGPUMachineFunctionInfo::getLDSKernelIdMetadata(const Function &F) {
182 // TODO: Would be more consistent with the abs symbols to use a range
183 MDNode *MD = F.getMetadata(Kind: "llvm.amdgcn.lds.kernel.id");
184 if (MD && MD->getNumOperands() == 1) {
185 if (ConstantInt *KnownSize =
186 mdconst::extract<ConstantInt>(MD: MD->getOperand(I: 0))) {
187 uint64_t ZExt = KnownSize->getZExtValue();
188 if (ZExt <= UINT32_MAX) {
189 return ZExt;
190 }
191 }
192 }
193 return {};
194}
195
196std::optional<uint32_t>
197AMDGPUMachineFunctionInfo::get32BitAbsoluteAddress(const GlobalValue &GV,
198 unsigned AS) {
199 if (GV.getAddressSpace() != AS)
200 return {};
201
202 std::optional<ConstantRange> AbsSymRange = GV.getAbsoluteSymbolRange();
203 if (!AbsSymRange)
204 return {};
205
206 if (const APInt *V = AbsSymRange->getSingleElement()) {
207 std::optional<uint64_t> ZExt = V->tryZExtValue();
208 if (ZExt && (*ZExt <= UINT32_MAX)) {
209 return *ZExt;
210 }
211 }
212
213 return {};
214}
215
216void AMDGPUMachineFunctionInfo::setDynLDSAlign(const Function &F,
217 const GlobalVariable &GV) {
218 const Module *M = F.getParent();
219 const DataLayout &DL = M->getDataLayout();
220 assert(GV.getGlobalSize(DL) == 0);
221
222 Align Alignment =
223 DL.getValueOrABITypeAlignment(Alignment: GV.getAlign(), Ty: GV.getValueType());
224 if (Alignment <= DynLDSAlign)
225 return;
226
227 LDSSize = alignTo(Size: StaticLDSSize, A: Alignment);
228 DynLDSAlign = Alignment;
229
230 // If there is a dynamic LDS variable associated with this function F, every
231 // further dynamic LDS instance (allocated by calling setDynLDSAlign) must
232 // map to the same address. This holds because no LDS is allocated after the
233 // lowering pass if there are dynamic LDS variables present.
234 const GlobalVariable *Dyn = getKernelDynLDSGlobalFromFunction(F);
235 if (Dyn) {
236 unsigned Offset = LDSSize; // return this?
237 std::optional<uint32_t> Expect =
238 get32BitAbsoluteAddress(GV, AS: AMDGPUAS::LOCAL_ADDRESS);
239 if (!Expect || (Offset != *Expect)) {
240 report_fatal_error(reason: "Inconsistent metadata on dynamic LDS variable");
241 }
242 }
243}
244
245void AMDGPUMachineFunctionInfo::setUsesDynamicLDS(bool DynLDS) {
246 UsesDynamicLDS = DynLDS;
247}
248
249bool AMDGPUMachineFunctionInfo::isDynamicLDSUsed() const {
250 return UsesDynamicLDS;
251}
252