1//===- AMDGPUPreloadKernelArguments.cpp - Preload Kernel Arguments --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass preloads kernel arguments into user_data SGPRs before kernel
10/// execution begins. The number of registers available for preloading depends
11/// on the number of free user SGPRs, up to the hardware's maximum limit.
12/// Implicit arguments enabled in the kernel descriptor are allocated first,
13/// followed by SGPRs used for preloaded kernel arguments. (Reference:
14/// https://llvm.org/docs/AMDGPUUsage.html#initial-kernel-execution-state)
15/// Additionally, hidden kernel arguments may be preloaded, in which case they
16/// are appended to the kernel signature after explicit arguments. Preloaded
17/// arguments will be marked with `inreg`.
18//
19//===----------------------------------------------------------------------===//
20
21#include "AMDGPU.h"
22#include "AMDGPUTargetMachine.h"
23#include "llvm/Analysis/ValueTracking.h"
24#include "llvm/IR/Function.h"
25#include "llvm/IR/Instructions.h"
26#include "llvm/IR/IntrinsicsAMDGPU.h"
27#include "llvm/IR/Module.h"
28#include "llvm/IR/PassManager.h"
29#include "llvm/IR/Verifier.h"
30#include "llvm/Pass.h"
31
32#define DEBUG_TYPE "amdgpu-preload-kernel-arguments"
33
34using namespace llvm;
35
36namespace {
37
38class AMDGPUPreloadKernelArgumentsLegacy : public ModulePass {
39 const GCNTargetMachine *TM;
40
41public:
42 static char ID;
43 explicit AMDGPUPreloadKernelArgumentsLegacy(
44 const GCNTargetMachine *TM = nullptr);
45
46 StringRef getPassName() const override {
47 return "AMDGPU Preload Kernel Arguments";
48 }
49
50 bool runOnModule(Module &M) override;
51};
52
53class PreloadKernelArgInfo {
54private:
55 Function &F;
56 const GCNSubtarget &ST;
57 unsigned NumFreeUserSGPRs;
58
59 enum HiddenArg : unsigned {
60 HIDDEN_BLOCK_COUNT_X,
61 HIDDEN_BLOCK_COUNT_Y,
62 HIDDEN_BLOCK_COUNT_Z,
63 HIDDEN_GROUP_SIZE_X,
64 HIDDEN_GROUP_SIZE_Y,
65 HIDDEN_GROUP_SIZE_Z,
66 HIDDEN_REMAINDER_X,
67 HIDDEN_REMAINDER_Y,
68 HIDDEN_REMAINDER_Z,
69 END_HIDDEN_ARGS
70 };
71
72 // Stores information about a specific hidden argument.
73 struct HiddenArgInfo {
74 // Offset in bytes from the location in the kernearg segment pointed to by
75 // the implicitarg pointer.
76 uint8_t Offset;
77 // The size of the hidden argument in bytes.
78 uint8_t Size;
79 // The name of the hidden argument in the kernel signature.
80 const char *Name;
81 };
82
83 static constexpr HiddenArgInfo HiddenArgs[END_HIDDEN_ARGS] = {
84 {.Offset: 0, .Size: 4, .Name: "_hidden_block_count_x"}, {.Offset: 4, .Size: 4, .Name: "_hidden_block_count_y"},
85 {.Offset: 8, .Size: 4, .Name: "_hidden_block_count_z"}, {.Offset: 12, .Size: 2, .Name: "_hidden_group_size_x"},
86 {.Offset: 14, .Size: 2, .Name: "_hidden_group_size_y"}, {.Offset: 16, .Size: 2, .Name: "_hidden_group_size_z"},
87 {.Offset: 18, .Size: 2, .Name: "_hidden_remainder_x"}, {.Offset: 20, .Size: 2, .Name: "_hidden_remainder_y"},
88 {.Offset: 22, .Size: 2, .Name: "_hidden_remainder_z"}};
89
90 static HiddenArg getHiddenArgFromOffset(unsigned Offset) {
91 for (unsigned I = 0; I < END_HIDDEN_ARGS; ++I)
92 if (HiddenArgs[I].Offset == Offset)
93 return static_cast<HiddenArg>(I);
94
95 return END_HIDDEN_ARGS;
96 }
97
98 static Type *getHiddenArgType(LLVMContext &Ctx, HiddenArg HA) {
99 if (HA < END_HIDDEN_ARGS)
100 return Type::getIntNTy(C&: Ctx, N: HiddenArgs[HA].Size * 8);
101
102 llvm_unreachable("Unexpected hidden argument.");
103 }
104
105 static const char *getHiddenArgName(HiddenArg HA) {
106 if (HA < END_HIDDEN_ARGS)
107 return HiddenArgs[HA].Name;
108
109 llvm_unreachable("Unexpected hidden argument.");
110 }
111
112 // Clones the function after adding implicit arguments to the argument list
113 // and returns the new updated function. Preloaded implicit arguments are
114 // added up to and including the last one that will be preloaded, indicated by
115 // LastPreloadIndex. Currently preloading is only performed on the totality of
116 // sequential data from the kernarg segment including implicit (hidden)
117 // arguments. This means that all arguments up to the last preloaded argument
118 // will also be preloaded even if that data is unused.
119 Function *cloneFunctionWithPreloadImplicitArgs(unsigned LastPreloadIndex) {
120 FunctionType *FT = F.getFunctionType();
121 LLVMContext &Ctx = F.getContext();
122 SmallVector<Type *, 16> FTypes(FT->param_begin(), FT->param_end());
123 for (unsigned I = 0; I <= LastPreloadIndex; ++I)
124 FTypes.push_back(Elt: getHiddenArgType(Ctx, HA: HiddenArg(I)));
125
126 FunctionType *NFT =
127 FunctionType::get(Result: FT->getReturnType(), Params: FTypes, isVarArg: FT->isVarArg());
128 Function *NF =
129 Function::Create(Ty: NFT, Linkage: F.getLinkage(), AddrSpace: F.getAddressSpace(), N: F.getName());
130
131 NF->copyAttributesFrom(Src: &F);
132 NF->copyMetadata(Src: &F, Offset: 0);
133
134 F.getParent()->getFunctionList().insert(where: F.getIterator(), New: NF);
135 NF->takeName(V: &F);
136 NF->splice(ToIt: NF->begin(), FromF: &F);
137
138 Function::arg_iterator NFArg = NF->arg_begin();
139 for (Argument &Arg : F.args()) {
140 Arg.replaceAllUsesWith(V: &*NFArg);
141 NFArg->takeName(V: &Arg);
142 ++NFArg;
143 }
144
145 AttrBuilder AB(Ctx);
146 AB.addAttribute(Val: Attribute::InReg);
147 AB.addAttribute(A: "amdgpu-hidden-argument");
148 AttributeList AL = NF->getAttributes();
149 for (unsigned I = 0; I <= LastPreloadIndex; ++I) {
150 AL = AL.addParamAttributes(C&: Ctx, ArgNo: NFArg->getArgNo(), B: AB);
151 NFArg++->setName(getHiddenArgName(HA: HiddenArg(I)));
152 }
153
154 NF->setAttributes(AL);
155 F.replaceAllUsesWith(V: NF);
156
157 return NF;
158 }
159
160public:
161 PreloadKernelArgInfo(Function &F, const GCNSubtarget &ST) : F(F), ST(ST) {
162 setInitialFreeUserSGPRsCount();
163 }
164
165 // Returns the maximum number of user SGPRs that we have available to preload
166 // arguments.
167 void setInitialFreeUserSGPRsCount() {
168 GCNUserSGPRUsageInfo UserSGPRInfo(F, ST);
169 NumFreeUserSGPRs = UserSGPRInfo.getNumFreeUserSGPRs();
170 }
171
172 bool canPreloadKernArgAtOffset(uint64_t ExplicitArgOffset) {
173 return ExplicitArgOffset <= NumFreeUserSGPRs * 4;
174 }
175
176 // Try to allocate SGPRs to preload hidden kernel arguments.
177 void
178 tryAllocHiddenArgPreloadSGPRs(uint64_t ImplicitArgsBaseOffset,
179 SmallVectorImpl<Function *> &FunctionsToErase) {
180 Function *ImplicitArgPtr = Intrinsic::getDeclarationIfExists(
181 M: F.getParent(), id: Intrinsic::amdgcn_implicitarg_ptr);
182 if (!ImplicitArgPtr)
183 return;
184
185 const DataLayout &DL = F.getDataLayout();
186 // Pair is the load and the load offset.
187 SmallVector<std::pair<LoadInst *, unsigned>, 4> ImplicitArgLoads;
188 for (auto *U : ImplicitArgPtr->users()) {
189 Instruction *CI = dyn_cast<Instruction>(Val: U);
190 if (!CI || CI->getFunction() != &F)
191 continue;
192
193 for (auto *U : CI->users()) {
194 int64_t Offset = 0;
195 auto *Load = dyn_cast<LoadInst>(Val: U); // Load from ImplicitArgPtr?
196 if (!Load) {
197 if (U->user_empty() ||
198 GetPointerBaseWithConstantOffset(Ptr: U, Offset, DL) != CI)
199 continue;
200
201 Load = dyn_cast<LoadInst>(Val: *U->user_begin()); // Load from GEP?
202 }
203
204 if (!Load || !Load->isSimple())
205 continue;
206
207 // FIXME: Expand handle merged loads.
208 LLVMContext &Ctx = F.getContext();
209 Type *LoadTy = Load->getType();
210 HiddenArg HA = getHiddenArgFromOffset(Offset);
211 if (HA == END_HIDDEN_ARGS || LoadTy != getHiddenArgType(Ctx, HA))
212 continue;
213
214 ImplicitArgLoads.push_back(Elt: std::make_pair(x&: Load, y&: Offset));
215 }
216 }
217
218 if (ImplicitArgLoads.empty())
219 return;
220
221 // Allocate loads in order of offset. We need to be sure that the implicit
222 // argument can actually be preloaded.
223 std::sort(first: ImplicitArgLoads.begin(), last: ImplicitArgLoads.end(), comp: less_second());
224
225 // If we fail to preload any implicit argument we know we don't have SGPRs
226 // to preload any subsequent ones with larger offsets. Find the first
227 // argument that we cannot preload.
228 auto *PreloadEnd = llvm::find_if(
229 Range&: ImplicitArgLoads, P: [&](const std::pair<LoadInst *, unsigned> &Load) {
230 unsigned LoadSize = DL.getTypeStoreSize(Ty: Load.first->getType());
231 unsigned LoadOffset = Load.second;
232 if (!canPreloadKernArgAtOffset(ExplicitArgOffset: LoadOffset + LoadSize +
233 ImplicitArgsBaseOffset))
234 return true;
235
236 return false;
237 });
238
239 if (PreloadEnd == ImplicitArgLoads.begin())
240 return;
241
242 unsigned LastHiddenArgIndex = getHiddenArgFromOffset(Offset: PreloadEnd[-1].second);
243 Function *NF = cloneFunctionWithPreloadImplicitArgs(LastPreloadIndex: LastHiddenArgIndex);
244 assert(NF);
245 FunctionsToErase.push_back(Elt: &F);
246 for (const auto *I = ImplicitArgLoads.begin(); I != PreloadEnd; ++I) {
247 LoadInst *LoadInst = I->first;
248 unsigned LoadOffset = I->second;
249 unsigned HiddenArgIndex = getHiddenArgFromOffset(Offset: LoadOffset);
250 unsigned Index = NF->arg_size() - LastHiddenArgIndex + HiddenArgIndex - 1;
251 Argument *Arg = NF->getArg(i: Index);
252 LoadInst->replaceAllUsesWith(V: Arg);
253 }
254 }
255};
256
257} // end anonymous namespace
258
259char AMDGPUPreloadKernelArgumentsLegacy::ID = 0;
260
261INITIALIZE_PASS(AMDGPUPreloadKernelArgumentsLegacy, DEBUG_TYPE,
262 "AMDGPU Preload Kernel Arguments", false, false)
263
264ModulePass *
265llvm::createAMDGPUPreloadKernelArgumentsLegacyPass(const TargetMachine *TM) {
266 return new AMDGPUPreloadKernelArgumentsLegacy(
267 static_cast<const GCNTargetMachine *>(TM));
268}
269
270AMDGPUPreloadKernelArgumentsLegacy::AMDGPUPreloadKernelArgumentsLegacy(
271 const GCNTargetMachine *TM)
272 : ModulePass(ID), TM(TM) {}
273
274static bool markKernelArgsAsInreg(Module &M, const TargetMachine &TM) {
275 const AMDGPUOptions &CLOpts =
276 static_cast<const AMDGPUTargetMachine &>(TM).getCLOpts();
277 if (!CLOpts.kernarg_preload)
278 return false;
279
280 SmallVector<Function *, 4> FunctionsToErase;
281 bool Changed = false;
282 for (auto &F : M) {
283 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
284 if (!ST.hasKernargPreload() ||
285 F.getCallingConv() != CallingConv::AMDGPU_KERNEL)
286 continue;
287
288 PreloadKernelArgInfo PreloadInfo(F, ST);
289 uint64_t ExplicitArgOffset = 0;
290 const DataLayout &DL = F.getDataLayout();
291 const uint64_t BaseOffset = ST.getExplicitKernelArgOffset();
292 unsigned NumPreloadsRequested = CLOpts.kernarg_preload_count;
293 unsigned NumPreloadedExplicitArgs = 0;
294 for (Argument &Arg : F.args()) {
295 // Avoid incompatible attributes and guard against running this pass
296 // twice.
297 //
298 // TODO: Preload byref kernel arguments
299 if (Arg.hasByRefAttr() || Arg.hasNestAttr() ||
300 Arg.hasAttribute(Kind: "amdgpu-hidden-argument"))
301 break;
302
303 // Inreg may be pre-existing on some arguments, try to preload these.
304 if (NumPreloadsRequested == 0 && !Arg.hasInRegAttr())
305 break;
306
307 // FIXME: Preload aggregates.
308 if (Arg.getType()->isAggregateType())
309 break;
310
311 Type *ArgTy = Arg.getType();
312 Align ABITypeAlign = DL.getABITypeAlign(Ty: ArgTy);
313 uint64_t AllocSize = DL.getTypeAllocSize(Ty: ArgTy);
314 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: ABITypeAlign) + AllocSize;
315
316 if (!PreloadInfo.canPreloadKernArgAtOffset(ExplicitArgOffset))
317 break;
318
319 Arg.addAttr(Kind: Attribute::InReg);
320 NumPreloadedExplicitArgs++;
321 if (NumPreloadsRequested > 0)
322 NumPreloadsRequested--;
323 }
324
325 // Only try preloading hidden arguments if we can successfully preload the
326 // last explicit argument.
327 if (NumPreloadedExplicitArgs == F.arg_size()) {
328 uint64_t ImplicitArgsBaseOffset =
329 alignTo(Size: ExplicitArgOffset, A: ST.getAlignmentForImplicitArgPtr()) +
330 BaseOffset;
331 PreloadInfo.tryAllocHiddenArgPreloadSGPRs(ImplicitArgsBaseOffset,
332 FunctionsToErase);
333 }
334
335 Changed |= NumPreloadedExplicitArgs > 0;
336 }
337
338 Changed |= !FunctionsToErase.empty();
339 // Erase cloned functions if we needed to update the kernel signature to
340 // support preloading hidden kernel arguments.
341 for (auto *F : FunctionsToErase)
342 F->eraseFromParent();
343
344 return Changed;
345}
346
347bool AMDGPUPreloadKernelArgumentsLegacy::runOnModule(Module &M) {
348 if (skipModule(M) || !TM)
349 return false;
350
351 return markKernelArgsAsInreg(M, TM: *TM);
352}
353
354PreservedAnalyses
355AMDGPUPreloadKernelArgumentsPass::run(Module &M, ModuleAnalysisManager &AM) {
356 bool Changed = markKernelArgsAsInreg(M, TM);
357 return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
358}
359