1//===-- AMDGPULowerKernelArguments.cpp ------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9/// \file This pass replaces accesses to kernel arguments with loads from
10/// offsets from the kernarg base pointer.
11//
12//===----------------------------------------------------------------------===//
13
14#include "AMDGPU.h"
15#include "GCNSubtarget.h"
16#include "llvm/Analysis/AliasAnalysis.h"
17#include "llvm/Analysis/CaptureTracking.h"
18#include "llvm/Analysis/ScopedNoAliasAA.h"
19#include "llvm/Analysis/ValueTracking.h"
20#include "llvm/CodeGen/TargetPassConfig.h"
21#include "llvm/IR/Argument.h"
22#include "llvm/IR/Attributes.h"
23#include "llvm/IR/Dominators.h"
24#include "llvm/IR/IRBuilder.h"
25#include "llvm/IR/InstIterator.h"
26#include "llvm/IR/Instruction.h"
27#include "llvm/IR/Instructions.h"
28#include "llvm/IR/IntrinsicsAMDGPU.h"
29#include "llvm/IR/LLVMContext.h"
30#include "llvm/IR/MDBuilder.h"
31#include "llvm/Target/TargetMachine.h"
32#include <optional>
33
34#define DEBUG_TYPE "amdgpu-lower-kernel-arguments"
35
36using namespace llvm;
37
38namespace {
39
40class AMDGPULowerKernelArguments : public FunctionPass {
41public:
42 static char ID;
43
44 AMDGPULowerKernelArguments() : FunctionPass(ID) {}
45
46 bool runOnFunction(Function &F) override;
47
48 void getAnalysisUsage(AnalysisUsage &AU) const override {
49 AU.addRequired<TargetPassConfig>();
50 AU.addRequired<DominatorTreeWrapperPass>();
51 AU.setPreservesAll();
52 }
53};
54
55} // end anonymous namespace
56
57// skip allocas
58static BasicBlock::iterator getInsertPt(BasicBlock &BB) {
59 BasicBlock::iterator InsPt = BB.getFirstInsertionPt();
60 for (BasicBlock::iterator E = BB.end(); InsPt != E; ++InsPt) {
61 AllocaInst *AI = dyn_cast<AllocaInst>(Val: &*InsPt);
62
63 // If this is a dynamic alloca, the value may depend on the loaded kernargs,
64 // so loads will need to be inserted before it.
65 if (!AI || !AI->isStaticAlloca())
66 break;
67 }
68
69 return InsPt;
70}
71
72static void addAliasScopeMetadata(Function &F, const DataLayout &DL,
73 DominatorTree &DT) {
74 // Collect noalias arguments.
75 SmallVector<const Argument *, 4u> NoAliasArgs;
76
77 for (Argument &Arg : F.args())
78 if (Arg.hasNoAliasAttr() && !Arg.use_empty())
79 NoAliasArgs.push_back(Elt: &Arg);
80
81 if (NoAliasArgs.empty())
82 return;
83
84 // Add alias scopes for each noalias argument. The scopes are disjoint, since
85 // an access based on one noalias argument can't modify memory an access based
86 // on another one reads.
87 MDBuilder MDB(F.getContext());
88 DenseMap<const Argument *, MDNode *> NewScopes;
89 MDNode *NewDomain =
90 MDB.createAnonymousAliasScopeDomain(Description: F.getName(), /*DisjointScopes=*/true);
91
92 for (unsigned I = 0u; I < NoAliasArgs.size(); ++I) {
93 const Argument *Arg = NoAliasArgs[I];
94 MDNode *NewScope = MDB.createAnonymousAliasScope(Domain: NewDomain, Name: Arg->getName());
95 NewScopes.insert(KV: {Arg, NewScope});
96 }
97
98 // Iterate over all instructions.
99 for (inst_iterator Inst = inst_begin(F), InstEnd = inst_end(F);
100 Inst != InstEnd; ++Inst) {
101 // If instruction accesses memory, collect its pointer arguments.
102 Instruction *I = &(*Inst);
103 SmallVector<const Value *, 2u> PtrArgs;
104 // May reach a noalias argument via a copy captured before the call.
105 bool IsUnrestrictedCall = false;
106
107 if (std::optional<MemoryLocation> MO = MemoryLocation::getOrNone(Inst: I))
108 PtrArgs.push_back(Elt: MO->Ptr);
109 else if (const CallBase *Call = dyn_cast<CallBase>(Val: I)) {
110 MemoryEffects ME = Call->getMemoryEffects();
111 if (ME.doesNotAccessMemory())
112 continue;
113
114 // Inaccessible memory cannot alias any IR-visible pointer.
115 if (ME.onlyAccessesInaccessibleMem())
116 continue;
117 IsUnrestrictedCall = !ME.onlyAccessesArgPointees();
118
119 for (Value *Arg : Call->args()) {
120 if (!Arg->getType()->isPointerTy())
121 continue;
122
123 PtrArgs.push_back(Elt: Arg);
124 }
125 } else {
126 // Not a memory access and not a call — nothing to annotate.
127 continue;
128 }
129
130 // Collect underlying objects of pointer arguments.
131 SmallVector<Metadata *, 4u> Scopes;
132 SmallPtrSet<const Value *, 4u> ObjSet;
133 SmallVector<Metadata *, 4u> NoAliases;
134
135 for (const Value *Val : PtrArgs) {
136 SmallVector<const Value *, 4u> Objects;
137 getUnderlyingObjects(V: Val, Objects);
138 ObjSet.insert_range(R&: Objects);
139 }
140
141 bool RequiresNoCaptureBefore = false;
142 bool UsesUnknownObject = false;
143 bool UsesAliasingPtr = false;
144
145 for (const Value *Val : ObjSet) {
146 if (isa<ConstantData>(Val))
147 continue;
148
149 if (const Argument *Arg = dyn_cast<Argument>(Val)) {
150 if (!Arg->hasAttribute(Kind: Attribute::NoAlias))
151 UsesAliasingPtr = true;
152 } else
153 UsesAliasingPtr = true;
154
155 if (isEscapeSource(V: Val)) {
156 // Can only alias a noalias argument if captured beforehand.
157 RequiresNoCaptureBefore = true;
158 } else if (!isa<Argument>(Val) && !isIdentifiedObject(V: Val)) {
159 // Unknown provenance: assume nothing.
160 UsesUnknownObject = true;
161 }
162 }
163
164 if (UsesUnknownObject)
165 continue;
166
167 if (IsUnrestrictedCall)
168 RequiresNoCaptureBefore = true;
169
170 // Collect scopes for alias.scope metadata. Skip unrestricted calls: they
171 // may touch memory beyond their pointer arguments' pointees.
172 if (!UsesAliasingPtr && !IsUnrestrictedCall)
173 for (const Argument *Arg : NoAliasArgs) {
174 if (ObjSet.count(Ptr: Arg))
175 Scopes.push_back(Elt: NewScopes[Arg]);
176 }
177
178 // Collect noalias scopes for the instruction not implied by its scopes.
179 // Scopes is only non-empty when no underlying object is an escape source
180 // and the instruction isn't an unrestricted call.
181 if (Scopes.empty())
182 for (const Argument *Arg : NoAliasArgs) {
183 if (ObjSet.contains(Ptr: Arg))
184 continue;
185
186 if (!RequiresNoCaptureBefore ||
187 !capturesAnything(CC: PointerMayBeCapturedBefore(
188 V: Arg, ReturnCaptures: false, I, DT: &DT, IncludeI: false, Mask: CaptureComponents::Provenance)))
189 NoAliases.push_back(Elt: NewScopes[Arg]);
190 }
191
192 // Add noalias metadata to instruction.
193 if (!NoAliases.empty()) {
194 MDNode *NewMD =
195 MDNode::concatenate(A: Inst->getMetadata(KindID: LLVMContext::MD_noalias),
196 B: MDNode::get(Context&: F.getContext(), MDs: NoAliases));
197 Inst->setMetadata(KindID: LLVMContext::MD_noalias, Node: NewMD);
198 }
199
200 // Add alias.scope metadata to instruction.
201 if (!Scopes.empty()) {
202 MDNode *NewMD =
203 MDNode::concatenate(A: Inst->getMetadata(KindID: LLVMContext::MD_alias_scope),
204 B: MDNode::get(Context&: F.getContext(), MDs: Scopes));
205 Inst->setMetadata(KindID: LLVMContext::MD_alias_scope, Node: NewMD);
206 }
207 }
208}
209
210static bool lowerKernelArguments(Function &F, const TargetMachine &TM,
211 DominatorTree &DT) {
212 CallingConv::ID CC = F.getCallingConv();
213 if (CC != CallingConv::AMDGPU_KERNEL || F.arg_empty())
214 return false;
215
216 const GCNSubtarget &ST = TM.getSubtarget<GCNSubtarget>(F);
217 LLVMContext &Ctx = F.getContext();
218 const DataLayout &DL = F.getDataLayout();
219 BasicBlock &EntryBlock = *F.begin();
220 IRBuilder<> Builder(getInsertPt(BB&: EntryBlock));
221
222 const Align KernArgBaseAlign(16); // FIXME: Increase if necessary
223 const uint64_t BaseOffset = ST.getExplicitKernelArgOffset();
224
225 Align MaxAlign;
226 // FIXME: Alignment is broken with explicit arg offset.;
227 const uint64_t TotalKernArgSize = ST.getKernArgSegmentSize(F, MaxAlign);
228 if (TotalKernArgSize == 0)
229 return false;
230
231 CallInst *KernArgSegment = Builder.CreateIntrinsicWithoutFolding(
232 ID: Intrinsic::amdgcn_kernarg_segment_ptr, Args: {}, FMFSource: nullptr,
233 Name: F.getName() + ".kernarg.segment");
234 KernArgSegment->addRetAttr(Kind: Attribute::NonNull);
235 KernArgSegment->addRetAttr(
236 Attr: Attribute::getWithDereferenceableBytes(Context&: Ctx, Bytes: TotalKernArgSize));
237
238 uint64_t ExplicitArgOffset = 0;
239
240 addAliasScopeMetadata(F, DL: F.getDataLayout(), DT);
241
242 for (Argument &Arg : F.args()) {
243 const bool IsByRef = Arg.hasByRefAttr();
244 Type *ArgTy = IsByRef ? Arg.getParamByRefType() : Arg.getType();
245 MaybeAlign ParamAlign = IsByRef ? Arg.getParamAlign() : std::nullopt;
246 Align ABITypeAlign = DL.getValueOrABITypeAlignment(Alignment: ParamAlign, Ty: ArgTy);
247
248 uint64_t Size = DL.getTypeSizeInBits(Ty: ArgTy);
249 uint64_t AllocSize = DL.getTypeAllocSize(Ty: ArgTy);
250
251 uint64_t EltOffset = alignTo(Size: ExplicitArgOffset, A: ABITypeAlign) + BaseOffset;
252 ExplicitArgOffset = alignTo(Size: ExplicitArgOffset, A: ABITypeAlign) + AllocSize;
253
254 // Skip inreg arguments which should be preloaded.
255 if (Arg.use_empty() || Arg.hasInRegAttr())
256 continue;
257
258 // If this is byval, the loads are already explicit in the function. We just
259 // need to rewrite the pointer values.
260 if (IsByRef) {
261 Value *ArgOffsetPtr = Builder.CreateConstInBoundsGEP1_64(
262 Ty: Builder.getInt8Ty(), Ptr: KernArgSegment, Idx0: EltOffset,
263 Name: Arg.getName() + ".byval.kernarg.offset");
264
265 Value *CastOffsetPtr =
266 Builder.CreateAddrSpaceCast(V: ArgOffsetPtr, DestTy: Arg.getType());
267 Arg.replaceAllUsesWith(V: CastOffsetPtr);
268 continue;
269 }
270
271 if (PointerType *PT = dyn_cast<PointerType>(Val: ArgTy)) {
272 // FIXME: Hack. We rely on AssertZext to be able to fold DS addressing
273 // modes on SI to know the high bits are 0 so pointer adds don't wrap. We
274 // can't represent this with range metadata because it's only allowed for
275 // integer types.
276 if ((PT->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS ||
277 PT->getAddressSpace() == AMDGPUAS::REGION_ADDRESS) &&
278 !ST.hasUsableDSOffset())
279 continue;
280 }
281
282 auto *VT = dyn_cast<FixedVectorType>(Val: ArgTy);
283 bool IsV3 = VT && VT->getNumElements() == 3;
284 bool DoShiftOpt = Size < 32 && !ArgTy->isAggregateType();
285
286 VectorType *V4Ty = nullptr;
287
288 int64_t AlignDownOffset = alignDown(Value: EltOffset, Align: 4);
289 int64_t OffsetDiff = EltOffset - AlignDownOffset;
290 Align AdjustedAlign = commonAlignment(
291 A: KernArgBaseAlign, Offset: DoShiftOpt ? AlignDownOffset : EltOffset);
292
293 Value *ArgPtr;
294 Type *AdjustedArgTy;
295 if (DoShiftOpt) { // FIXME: Handle aggregate types
296 // Since we don't have sub-dword scalar loads, avoid doing an extload by
297 // loading earlier than the argument address, and extracting the relevant
298 // bits.
299 // TODO: Update this for GFX12 which does have scalar sub-dword loads.
300 //
301 // Additionally widen any sub-dword load to i32 even if suitably aligned,
302 // so that CSE between different argument loads works easily.
303 ArgPtr = Builder.CreateConstInBoundsGEP1_64(
304 Ty: Builder.getInt8Ty(), Ptr: KernArgSegment, Idx0: AlignDownOffset,
305 Name: Arg.getName() + ".kernarg.offset.align.down");
306 AdjustedArgTy = Builder.getInt32Ty();
307 } else {
308 ArgPtr = Builder.CreateConstInBoundsGEP1_64(
309 Ty: Builder.getInt8Ty(), Ptr: KernArgSegment, Idx0: EltOffset,
310 Name: Arg.getName() + ".kernarg.offset");
311 AdjustedArgTy = ArgTy;
312 }
313
314 if (IsV3 && Size >= 32) {
315 V4Ty = FixedVectorType::get(ElementType: VT->getElementType(), NumElts: 4);
316 // Use the hack that clang uses to avoid SelectionDAG ruining v3 loads
317 AdjustedArgTy = V4Ty;
318 }
319
320 LoadInst *Load =
321 Builder.CreateAlignedLoad(Ty: AdjustedArgTy, Ptr: ArgPtr, Align: AdjustedAlign);
322 Load->setMetadata(KindID: LLVMContext::MD_invariant_load, Node: MDNode::get(Context&: Ctx, MDs: {}));
323
324 MDBuilder MDB(Ctx);
325
326 if (Arg.hasAttribute(Kind: Attribute::NoUndef) && AdjustedArgTy == ArgTy)
327 Load->setMetadata(KindID: LLVMContext::MD_noundef, Node: MDNode::get(Context&: Ctx, MDs: {}));
328
329 if (Arg.hasAttribute(Kind: Attribute::Range) && AdjustedArgTy == ArgTy) {
330 const ConstantRange &Range =
331 Arg.getAttribute(Kind: Attribute::Range).getValueAsConstantRange();
332 Load->setMetadata(KindID: LLVMContext::MD_range,
333 Node: MDB.createRange(Lo: Range.getLower(), Hi: Range.getUpper()));
334 }
335
336 if (Arg.hasAttribute(Kind: Attribute::NoFPClass) && AdjustedArgTy == ArgTy) {
337 FPClassTest Mask = Arg.getNoFPClass();
338 Load->setMetadata(
339 KindID: LLVMContext::MD_nofpclass,
340 Node: MDNode::get(Context&: Ctx, MDs: ConstantAsMetadata::get(
341 C: ConstantInt::get(Ty: Type::getInt32Ty(C&: Ctx), V: Mask))));
342 }
343
344 if (isa<PointerType>(Val: ArgTy)) {
345 if (Arg.hasNonNullAttr())
346 Load->setMetadata(KindID: LLVMContext::MD_nonnull, Node: MDNode::get(Context&: Ctx, MDs: {}));
347
348 uint64_t DerefBytes = Arg.getDereferenceableBytes();
349 if (DerefBytes != 0) {
350 Load->setMetadata(
351 KindID: LLVMContext::MD_dereferenceable,
352 Node: MDNode::get(Context&: Ctx,
353 MDs: MDB.createConstant(
354 C: ConstantInt::get(Ty: Builder.getInt64Ty(), V: DerefBytes))));
355 }
356
357 uint64_t DerefOrNullBytes = Arg.getDereferenceableOrNullBytes();
358 if (DerefOrNullBytes != 0) {
359 Load->setMetadata(
360 KindID: LLVMContext::MD_dereferenceable_or_null,
361 Node: MDNode::get(Context&: Ctx,
362 MDs: MDB.createConstant(C: ConstantInt::get(Ty: Builder.getInt64Ty(),
363 V: DerefOrNullBytes))));
364 }
365
366 if (MaybeAlign ParamAlign = Arg.getParamAlign()) {
367 Load->setMetadata(
368 KindID: LLVMContext::MD_align,
369 Node: MDNode::get(Context&: Ctx, MDs: MDB.createConstant(C: ConstantInt::get(
370 Ty: Builder.getInt64Ty(), V: ParamAlign->value()))));
371 }
372 }
373
374 if (DoShiftOpt) {
375 Value *ExtractBits = OffsetDiff == 0 ?
376 Load : Builder.CreateLShr(LHS: Load, RHS: OffsetDiff * 8);
377
378 IntegerType *ArgIntTy = Builder.getIntNTy(N: Size);
379 Value *Trunc = Builder.CreateTrunc(V: ExtractBits, DestTy: ArgIntTy);
380 Value *NewVal = Builder.CreateBitCast(V: Trunc, DestTy: ArgTy,
381 Name: Arg.getName() + ".load");
382 Arg.replaceAllUsesWith(V: NewVal);
383 } else if (IsV3) {
384 Value *Shuf = Builder.CreateShuffleVector(V: Load, Mask: ArrayRef<int>{0, 1, 2},
385 Name: Arg.getName() + ".load");
386 Arg.replaceAllUsesWith(V: Shuf);
387 } else {
388 Load->setName(Arg.getName() + ".load");
389 Arg.replaceAllUsesWith(V: Load);
390 }
391 }
392
393 KernArgSegment->addRetAttr(
394 Attr: Attribute::getWithAlignment(Context&: Ctx, Alignment: std::max(a: KernArgBaseAlign, b: MaxAlign)));
395
396 return true;
397}
398
399bool AMDGPULowerKernelArguments::runOnFunction(Function &F) {
400 auto &TPC = getAnalysis<TargetPassConfig>();
401 const TargetMachine &TM = TPC.getTM<TargetMachine>();
402 DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
403 return lowerKernelArguments(F, TM, DT);
404}
405
406INITIALIZE_PASS_BEGIN(AMDGPULowerKernelArguments, DEBUG_TYPE,
407 "AMDGPU Lower Kernel Arguments", false, false)
408INITIALIZE_PASS_END(AMDGPULowerKernelArguments, DEBUG_TYPE, "AMDGPU Lower Kernel Arguments",
409 false, false)
410
411char AMDGPULowerKernelArguments::ID = 0;
412
413FunctionPass *llvm::createAMDGPULowerKernelArgumentsPass() {
414 return new AMDGPULowerKernelArguments();
415}
416
417PreservedAnalyses
418AMDGPULowerKernelArgumentsPass::run(Function &F, FunctionAnalysisManager &AM) {
419 DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(IR&: F);
420 bool Changed = lowerKernelArguments(F, TM, DT);
421 if (Changed) {
422 // TODO: Preserves a lot more.
423 PreservedAnalyses PA;
424 PA.preserveSet<CFGAnalyses>();
425 return PA;
426 }
427
428 return PreservedAnalyses::all();
429}
430