1//===-- AMDGPULowerModuleLDSPass.cpp ------------------------------*- C++ -*-=//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass eliminates local data store, LDS, uses from non-kernel functions.
10// LDS is contiguous memory allocated per kernel execution.
11//
12// Background.
13//
14// The programming model is global variables, or equivalently function local
15// static variables, accessible from kernels or other functions. For uses from
16// kernels this is straightforward - assign an integer to the kernel for the
17// memory required by all the variables combined, allocate them within that.
18// For uses from functions there are performance tradeoffs to choose between.
19//
20// This model means the GPU runtime can specify the amount of memory allocated.
21// If this is more than the kernel assumed, the excess can be made available
22// using a language specific feature, which IR represents as a variable with
23// no initializer. This feature is referred to here as "Dynamic LDS" and is
24// lowered slightly differently to the normal case.
25//
26// Consequences of this GPU feature:
27// - memory is limited and exceeding it halts compilation
28// - a global accessed by one kernel exists independent of other kernels
29// - a global exists independent of simultaneous execution of the same kernel
30// - the address of the global may be different from different kernels as they
31// do not alias, which permits only allocating variables they use
32// - if the address is allowed to differ, functions need help to find it
33//
34// Uses from kernels are implemented here by grouping them in a per-kernel
35// struct instance. This duplicates the variables, accurately modelling their
36// aliasing properties relative to a single global representation. It also
37// permits control over alignment via padding.
38//
39// Uses from functions are more complicated and the primary purpose of this
40// IR pass. Several different lowering are chosen between to meet requirements
41// to avoid allocating any LDS where it is not necessary, as that impacts
42// occupancy and may fail the compilation, while not imposing overhead on a
43// feature whose primary advantage over global memory is performance. The basic
44// design goal is to avoid one kernel imposing overhead on another.
45//
46// Implementation.
47//
48// LDS variables with constant annotation or non-undef initializer are passed
49// through unchanged for simplification or error diagnostics in later passes.
50// Non-undef initializers are not yet implemented for LDS.
51//
52// LDS variables that are always allocated at the same address can be found
53// by lookup at that address. Otherwise runtime information/cost is required.
54//
55// The simplest strategy possible is to group all LDS variables in a single
56// struct and allocate that struct in every kernel such that the original
57// variables are always at the same address. LDS is however a limited resource
58// so this strategy is unusable in practice. It is not implemented here.
59//
60// Strategy | Precise allocation | Zero runtime cost | General purpose |
61// --------+--------------------+-------------------+-----------------+
62// Module | No | Yes | Yes |
63// Table | Yes | No | Yes |
64// Kernel | Yes | Yes | No |
65// Hybrid | Yes | Partial | Yes |
66//
67// "Module" spends LDS memory to save cycles. "Table" spends cycles and global
68// memory to save LDS. "Kernel" is as fast as kernel allocation but only works
69// for variables that are known reachable from a single kernel. "Hybrid" picks
70// between all three. When forced to choose between LDS and cycles we minimise
71// LDS use.
72
73// The "module" lowering implemented here finds LDS variables which are used by
74// non-kernel functions and creates a new struct with a field for each of those
75// LDS variables. Variables that are only used from kernels are excluded.
76//
77// The "table" lowering implemented here has three components.
78// First kernels are assigned a unique integer identifier which is available in
79// functions it calls through the intrinsic amdgcn_lds_kernel_id. The integer
80// is passed through a specific SGPR, thus works with indirect calls.
81// Second, each kernel allocates LDS variables independent of other kernels and
82// writes the addresses it chose for each variable into an array in consistent
83// order. If the kernel does not allocate a given variable, it writes undef to
84// the corresponding array location. These arrays are written to a constant
85// table in the order matching the kernel unique integer identifier.
86// Third, uses from non-kernel functions are replaced with a table lookup using
87// the intrinsic function to find the address of the variable.
88//
89// "Kernel" lowering is only applicable for variables that are unambiguously
90// reachable from exactly one kernel. For those cases, accesses to the variable
91// can be lowered to ConstantExpr address of a struct instance specific to that
92// one kernel. This is zero cost in space and in compute. It will raise a fatal
93// error on any variable that might be reachable from multiple kernels and is
94// thus most easily used as part of the hybrid lowering strategy.
95//
96// Hybrid lowering is a mixture of the above. It uses the zero cost kernel
97// lowering where it can. It lowers the variable accessed by the greatest
98// number of kernels using the module strategy as that is free for the first
99// variable. Any futher variables that can be lowered with the module strategy
100// without incurring LDS memory overhead are. The remaining ones are lowered
101// via table.
102//
103// Consequences
104// - No heuristics or user controlled magic numbers, hybrid is the right choice
105// - Kernels that don't use functions (or have had them all inlined) are not
106// affected by any lowering for kernels that do.
107// - Kernels that don't make indirect function calls are not affected by those
108// that do.
109// - Variables which are used by lots of kernels, e.g. those injected by a
110// language runtime in most kernels, are expected to have no overhead
111// - Implementations that instantiate templates per-kernel where those templates
112// use LDS are expected to hit the "Kernel" lowering strategy
113// - The runtime properties impose a cost in compiler implementation complexity
114//
115// Dynamic LDS implementation
116// Dynamic LDS is lowered similarly to the "table" strategy above and uses the
117// same intrinsic to identify which kernel is at the root of the dynamic call
118// graph. This relies on the specified behaviour that all dynamic LDS variables
119// alias one another, i.e. are at the same address, with respect to a given
120// kernel. Therefore this pass creates new dynamic LDS variables for each kernel
121// that allocates any dynamic LDS and builds a table of addresses out of those.
122// The AMDGPUPromoteAlloca pass skips kernels that use dynamic LDS.
123// The corresponding optimisation for "kernel" lowering where the table lookup
124// is elided is not implemented.
125//
126//
127// Implementation notes / limitations
128// A single LDS global variable represents an instance per kernel that can reach
129// said variables. This pass essentially specialises said variables per kernel.
130// Handling ConstantExpr during the pass complicated this significantly so now
131// all ConstantExpr uses of LDS variables are expanded to instructions. This
132// may need amending when implementing non-undef initialisers.
133//
134// Lowering is split between this IR pass and the back end. This pass chooses
135// where given variables should be allocated and marks them with metadata,
136// MD_absolute_symbol. The backend places the variables in coincidentally the
137// same location and raises a fatal error if something has gone awry. This works
138// in practice because the only pass between this one and the backend that
139// changes LDS is PromoteAlloca and the changes it makes do not conflict.
140//
141// Addresses are written to constant global arrays based on the same metadata.
142//
143// The backend lowers LDS variables in the order of traversal of the function.
144// This is at odds with the deterministic layout required. The workaround is to
145// allocate the fixed-address variables immediately upon starting the function
146// where they can be placed as intended. This requires a means of mapping from
147// the function to the variables that it allocates. For the module scope lds,
148// this is via metadata indicating whether the variable is not required. If a
149// pass deletes that metadata, a fatal error on disagreement with the absolute
150// symbol metadata will occur. For kernel scope and dynamic, this is by _name_
151// correspondence between the function and the variable. It requires the
152// kernel to have a name (which is only a limitation for tests in practice) and
153// for nothing to rename the corresponding symbols. This is a hazard if the pass
154// is run multiple times during debugging. Alternative schemes considered all
155// involve bespoke metadata.
156//
157// If the name correspondence can be replaced, multiple distinct kernels that
158// have the same memory layout can map to the same kernel id (as the address
159// itself is handled by the absolute symbol metadata) and that will allow more
160// uses of the "kernel" style faster lowering and reduce the size of the lookup
161// tables.
162//
163// There is a test that checks this does not fire for a graphics shader. This
164// lowering is expected to work for graphics if the isKernel test is changed.
165//
166// The current markUsedByKernel is sufficient for PromoteAlloca but is elided
167// before codegen. Replacing this with an equivalent intrinsic which lasts until
168// shortly after the machine function lowering of LDS would help break the name
169// mapping. The other part needed is probably to amend PromoteAlloca to embed
170// the LDS variables it creates in the same struct created here. That avoids the
171// current hazard where a PromoteAlloca LDS variable might be allocated before
172// the kernel scope (and thus error on the address check). Given a new invariant
173// that no LDS variables exist outside of the structs managed here, and an
174// intrinsic that lasts until after the LDS frame lowering, it should be
175// possible to drop the name mapping and fold equivalent memory layouts.
176//
177//===----------------------------------------------------------------------===//
178
179#include "AMDGPU.h"
180#include "AMDGPUMemoryUtils.h"
181#include "AMDGPUTargetMachine.h"
182#include "Utils/AMDGPUBaseInfo.h"
183#include "llvm/ADT/BitVector.h"
184#include "llvm/ADT/DenseMap.h"
185#include "llvm/ADT/DenseSet.h"
186#include "llvm/ADT/STLExtras.h"
187#include "llvm/ADT/SetOperations.h"
188#include "llvm/Analysis/CallGraph.h"
189#include "llvm/Analysis/ScopedNoAliasAA.h"
190#include "llvm/CodeGen/TargetPassConfig.h"
191#include "llvm/IR/Constants.h"
192#include "llvm/IR/DerivedTypes.h"
193#include "llvm/IR/Dominators.h"
194#include "llvm/IR/IRBuilder.h"
195#include "llvm/IR/InlineAsm.h"
196#include "llvm/IR/Instructions.h"
197#include "llvm/IR/IntrinsicsAMDGPU.h"
198#include "llvm/IR/MDBuilder.h"
199#include "llvm/IR/ReplaceConstant.h"
200#include "llvm/InitializePasses.h"
201#include "llvm/Pass.h"
202#include "llvm/Support/CommandLine.h"
203#include "llvm/Support/Debug.h"
204#include "llvm/Support/Format.h"
205#include "llvm/Support/OptimizedStructLayout.h"
206#include "llvm/Support/raw_ostream.h"
207#include "llvm/Transforms/Utils/BasicBlockUtils.h"
208#include "llvm/Transforms/Utils/ModuleUtils.h"
209
210#include <vector>
211
212#include <cstdio>
213
214#define DEBUG_TYPE "amdgpu-lower-module-lds"
215
216using namespace llvm;
217using namespace AMDGPU;
218
219namespace {
220
221cl::opt<bool> SuperAlignLDSGlobals(
222 "amdgpu-super-align-lds-globals",
223 cl::desc("Increase alignment of LDS if it is not on align boundary"),
224 cl::init(Val: true), cl::Hidden);
225
226enum class LoweringKind { module, table, kernel, hybrid };
227cl::opt<LoweringKind> LoweringKindLoc(
228 "amdgpu-lower-module-lds-strategy",
229 cl::desc("Specify lowering strategy for function LDS access:"), cl::Hidden,
230 cl::init(Val: LoweringKind::hybrid),
231 cl::values(
232 clEnumValN(LoweringKind::table, "table", "Lower via table lookup"),
233 clEnumValN(LoweringKind::module, "module", "Lower via module struct"),
234 clEnumValN(
235 LoweringKind::kernel, "kernel",
236 "Lower variables reachable from one kernel, otherwise abort"),
237 clEnumValN(LoweringKind::hybrid, "hybrid",
238 "Lower via mixture of above strategies")));
239
240template <typename T> std::vector<T> sortByName(std::vector<T> &&V) {
241 llvm::sort(V, [](const auto *L, const auto *R) {
242 return L->getName() < R->getName();
243 });
244 return {std::move(V)};
245}
246
247class AMDGPULowerModuleLDS {
248 const AMDGPUTargetMachine &TM;
249
250 static void
251 removeLocalVarsFromUsedLists(Module &M,
252 const DenseSet<GlobalVariable *> &LocalVars) {
253 // The verifier rejects used lists containing an inttoptr of a constant
254 // so remove the variables from these lists before replaceAllUsesWith
255 SmallPtrSet<Constant *, 8> LocalVarsSet;
256 for (GlobalVariable *LocalVar : LocalVars)
257 LocalVarsSet.insert(Ptr: cast<Constant>(Val: LocalVar->stripPointerCasts()));
258
259 removeFromUsedLists(
260 M, ShouldRemove: [&LocalVarsSet](Constant *C) { return LocalVarsSet.count(Ptr: C); });
261
262 for (GlobalVariable *LocalVar : LocalVars)
263 LocalVar->removeDeadConstantUsers();
264 }
265
266 static void markUsedByKernel(Function *Func, GlobalVariable *SGV) {
267 // The llvm.amdgcn.module.lds instance is implicitly used by all kernels
268 // that might call a function which accesses a field within it. This is
269 // presently approximated to 'all kernels' if there are any such functions
270 // in the module. This implicit use is redefined as an explicit use here so
271 // that later passes, specifically PromoteAlloca, account for the required
272 // memory without any knowledge of this transform.
273
274 // An operand bundle on llvm.donothing works because the call instruction
275 // survives until after the last pass that needs to account for LDS. It is
276 // better than inline asm as the latter survives until the end of codegen. A
277 // totally robust solution would be a function with the same semantics as
278 // llvm.donothing that takes a pointer to the instance and is lowered to a
279 // no-op after LDS is allocated, but that is not presently necessary.
280
281 // This intrinsic is eliminated shortly before instruction selection. It
282 // does not suffice to indicate to ISel that a given global which is not
283 // immediately used by the kernel must still be allocated by it. An
284 // equivalent target specific intrinsic which lasts until immediately after
285 // codegen would suffice for that, but one would still need to ensure that
286 // the variables are allocated in the anticipated order.
287 BasicBlock *Entry = &Func->getEntryBlock();
288 IRBuilder<> Builder(Entry, Entry->getFirstNonPHIIt());
289
290 Function *Decl = Intrinsic::getOrInsertDeclaration(
291 M: Func->getParent(), id: Intrinsic::donothing, Tys: {});
292
293 Value *UseInstance[1] = {
294 Builder.CreateConstInBoundsGEP1_32(Ty: SGV->getValueType(), Ptr: SGV, Idx0: 0)};
295
296 Builder.CreateCall(
297 Callee: Decl, Args: {}, OpBundles: {OperandBundleDefT<Value *>("ExplicitUse", UseInstance)});
298 }
299
300public:
301 AMDGPULowerModuleLDS(const AMDGPUTargetMachine &TM_) : TM(TM_) {}
302
303 struct LDSVariableReplacement {
304 GlobalVariable *SGV = nullptr;
305 DenseMap<GlobalVariable *, Constant *> LDSVarsToConstantGEP;
306 };
307
308 // remap from lds global to a constantexpr gep to where it has been moved to
309 // for each kernel
310 // an array with an element for each kernel containing where the corresponding
311 // variable was remapped to
312
313 static Constant *getAddressesOfVariablesInKernel(
314 LLVMContext &Ctx, ArrayRef<GlobalVariable *> Variables,
315 const DenseMap<GlobalVariable *, Constant *> &LDSVarsToConstantGEP) {
316 // Create a ConstantArray containing the address of each Variable within the
317 // kernel corresponding to LDSVarsToConstantGEP, or poison if that kernel
318 // does not allocate it
319 // TODO: Drop the ptrtoint conversion
320
321 Type *I32 = Type::getInt32Ty(C&: Ctx);
322
323 ArrayType *KernelOffsetsType = ArrayType::get(ElementType: I32, NumElements: Variables.size());
324
325 SmallVector<Constant *> Elements;
326 for (GlobalVariable *GV : Variables) {
327 auto ConstantGepIt = LDSVarsToConstantGEP.find(Val: GV);
328 if (ConstantGepIt != LDSVarsToConstantGEP.end()) {
329 auto *elt = ConstantExpr::getPtrToInt(C: ConstantGepIt->second, Ty: I32);
330 Elements.push_back(Elt: elt);
331 } else {
332 Elements.push_back(Elt: PoisonValue::get(T: I32));
333 }
334 }
335 return ConstantArray::get(T: KernelOffsetsType, V: Elements);
336 }
337
338 static GlobalVariable *buildLookupTable(
339 Module &M, ArrayRef<GlobalVariable *> Variables,
340 ArrayRef<Function *> kernels,
341 DenseMap<Function *, LDSVariableReplacement> &KernelToReplacement) {
342 if (Variables.empty()) {
343 return nullptr;
344 }
345 LLVMContext &Ctx = M.getContext();
346
347 const size_t NumberVariables = Variables.size();
348 const size_t NumberKernels = kernels.size();
349
350 ArrayType *KernelOffsetsType =
351 ArrayType::get(ElementType: Type::getInt32Ty(C&: Ctx), NumElements: NumberVariables);
352
353 ArrayType *AllKernelsOffsetsType =
354 ArrayType::get(ElementType: KernelOffsetsType, NumElements: NumberKernels);
355
356 Constant *Missing = PoisonValue::get(T: KernelOffsetsType);
357 std::vector<Constant *> overallConstantExprElts(NumberKernels);
358 for (size_t i = 0; i < NumberKernels; i++) {
359 auto Replacement = KernelToReplacement.find(Val: kernels[i]);
360 overallConstantExprElts[i] =
361 (Replacement == KernelToReplacement.end())
362 ? Missing
363 : getAddressesOfVariablesInKernel(
364 Ctx, Variables, LDSVarsToConstantGEP: Replacement->second.LDSVarsToConstantGEP);
365 }
366
367 Constant *init =
368 ConstantArray::get(T: AllKernelsOffsetsType, V: overallConstantExprElts);
369
370 return new GlobalVariable(
371 M, AllKernelsOffsetsType, true, GlobalValue::InternalLinkage, init,
372 "llvm.amdgcn.lds.offset.table", nullptr, GlobalValue::NotThreadLocal,
373 AMDGPUAS::CONSTANT_ADDRESS);
374 }
375
376 void replaceUseWithTableLookup(Module &M, IRBuilder<> &Builder,
377 GlobalVariable *LookupTable,
378 GlobalVariable *GV, Use &U,
379 Value *OptionalIndex) {
380 // Table is a constant array of the same length as OrderedKernels
381 LLVMContext &Ctx = M.getContext();
382 Type *I32 = Type::getInt32Ty(C&: Ctx);
383 auto *I = cast<Instruction>(Val: U.getUser());
384
385 Value *tableKernelIndex = getTableLookupKernelIndex(M, F: I->getFunction());
386
387 if (auto *Phi = dyn_cast<PHINode>(Val: I)) {
388 BasicBlock *BB = Phi->getIncomingBlock(U);
389 Builder.SetInsertPoint(&(*(BB->getFirstInsertionPt())));
390 } else {
391 Builder.SetInsertPoint(I);
392 }
393
394 SmallVector<Value *, 3> GEPIdx = {
395 ConstantInt::get(Ty: I32, V: 0),
396 tableKernelIndex,
397 };
398 if (OptionalIndex)
399 GEPIdx.push_back(Elt: OptionalIndex);
400
401 Value *Address = Builder.CreateInBoundsGEP(
402 Ty: LookupTable->getValueType(), Ptr: LookupTable, IdxList: GEPIdx, Name: GV->getName());
403
404 Value *loaded = Builder.CreateLoad(Ty: I32, Ptr: Address);
405
406 Value *replacement =
407 Builder.CreateIntToPtr(V: loaded, DestTy: GV->getType(), Name: GV->getName());
408
409 U.set(replacement);
410 }
411
412 void replaceUsesInInstructionsWithTableLookup(
413 Module &M, ArrayRef<GlobalVariable *> ModuleScopeVariables,
414 GlobalVariable *LookupTable) {
415
416 LLVMContext &Ctx = M.getContext();
417 IRBuilder<> Builder(Ctx);
418 Type *I32 = Type::getInt32Ty(C&: Ctx);
419
420 for (size_t Index = 0; Index < ModuleScopeVariables.size(); Index++) {
421 auto *GV = ModuleScopeVariables[Index];
422
423 for (Use &U : make_early_inc_range(Range: GV->uses())) {
424 auto *I = dyn_cast<Instruction>(Val: U.getUser());
425 if (!I)
426 continue;
427
428 replaceUseWithTableLookup(M, Builder, LookupTable, GV, U,
429 OptionalIndex: ConstantInt::get(Ty: I32, V: Index));
430 }
431 }
432 }
433
434 static DenseSet<Function *> kernelsThatIndirectlyAccessAnyOfPassedVariables(
435 Module &M, LDSUsesInfoTy &LDSUsesInfo,
436 DenseSet<GlobalVariable *> const &VariableSet) {
437
438 DenseSet<Function *> KernelSet;
439
440 if (VariableSet.empty())
441 return KernelSet;
442
443 for (Function &Func : M.functions()) {
444 if (Func.isDeclaration() || !isKernel(F: Func))
445 continue;
446 for (GlobalVariable *GV : LDSUsesInfo.indirect_access[&Func]) {
447 if (VariableSet.contains(V: GV)) {
448 KernelSet.insert(V: &Func);
449 break;
450 }
451 }
452 }
453
454 return KernelSet;
455 }
456
457 static GlobalVariable *
458 chooseBestVariableForModuleStrategy(const DataLayout &DL,
459 VariableFunctionMap &LDSVars) {
460 // Find the global variable with the most indirect uses from kernels
461
462 struct CandidateTy {
463 GlobalVariable *GV = nullptr;
464 size_t UserCount = 0;
465 size_t Size = 0;
466
467 CandidateTy() = default;
468
469 CandidateTy(GlobalVariable *GV, uint64_t UserCount, uint64_t AllocSize)
470 : GV(GV), UserCount(UserCount), Size(AllocSize) {}
471
472 bool operator<(const CandidateTy &Other) const {
473 // Fewer users makes module scope variable less attractive
474 if (UserCount < Other.UserCount) {
475 return true;
476 }
477 if (UserCount > Other.UserCount) {
478 return false;
479 }
480
481 // Bigger makes module scope variable less attractive
482 if (Size < Other.Size) {
483 return false;
484 }
485
486 if (Size > Other.Size) {
487 return true;
488 }
489
490 // Arbitrary but consistent
491 return GV->getName() < Other.GV->getName();
492 }
493 };
494
495 CandidateTy MostUsed;
496
497 for (auto &K : LDSVars) {
498 GlobalVariable *GV = K.first;
499 if (K.second.size() <= 1) {
500 // A variable reachable by only one kernel is best lowered with kernel
501 // strategy
502 continue;
503 }
504 CandidateTy Candidate(GV, K.second.size(), GV->getGlobalSize(DL));
505 if (MostUsed < Candidate)
506 MostUsed = Candidate;
507 }
508
509 return MostUsed.GV;
510 }
511
512 static void recordLDSAbsoluteAddress(Module *M, GlobalVariable *GV,
513 uint32_t Address) {
514 // Write the specified address into metadata where it can be retrieved by
515 // the assembler. Format is a half open range, [Address Address+1)
516 LLVMContext &Ctx = M->getContext();
517 auto *IntTy =
518 M->getDataLayout().getIntPtrType(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS);
519 auto *MinC = ConstantAsMetadata::get(C: ConstantInt::get(Ty: IntTy, V: Address));
520 auto *MaxC = ConstantAsMetadata::get(C: ConstantInt::get(Ty: IntTy, V: Address + 1));
521 GV->setMetadata(KindID: LLVMContext::MD_absolute_symbol,
522 Node: MDNode::get(Context&: Ctx, MDs: {MinC, MaxC}));
523 }
524
525 DenseMap<Function *, Value *> tableKernelIndexCache;
526 Value *getTableLookupKernelIndex(Module &M, Function *F) {
527 // Accesses from a function use the amdgcn_lds_kernel_id intrinsic which
528 // lowers to a read from a live in register. Emit it once in the entry
529 // block to spare deduplicating it later.
530 auto [It, Inserted] = tableKernelIndexCache.try_emplace(Key: F);
531 if (Inserted) {
532 auto InsertAt = F->getEntryBlock().getFirstNonPHIOrDbgOrAlloca();
533 IRBuilder<> Builder(&*InsertAt);
534
535 It->second = Builder.CreateIntrinsic(ID: Intrinsic::amdgcn_lds_kernel_id, Args: {});
536 }
537
538 return It->second;
539 }
540
541 static std::vector<Function *> assignLDSKernelIDToEachKernel(
542 Module *M, DenseSet<Function *> const &KernelsThatAllocateTableLDS,
543 DenseSet<Function *> const &KernelsThatIndirectlyAllocateDynamicLDS) {
544 // Associate kernels in the set with an arbitrary but reproducible order and
545 // annotate them with that order in metadata. This metadata is recognised by
546 // the backend and lowered to a SGPR which can be read from using
547 // amdgcn_lds_kernel_id.
548
549 std::vector<Function *> OrderedKernels;
550 if (!KernelsThatAllocateTableLDS.empty() ||
551 !KernelsThatIndirectlyAllocateDynamicLDS.empty()) {
552
553 for (Function &Func : M->functions()) {
554 if (Func.isDeclaration())
555 continue;
556 if (!isKernel(F: Func))
557 continue;
558
559 if (KernelsThatAllocateTableLDS.contains(V: &Func) ||
560 KernelsThatIndirectlyAllocateDynamicLDS.contains(V: &Func)) {
561 assert(Func.hasName()); // else fatal error earlier
562 OrderedKernels.push_back(x: &Func);
563 }
564 }
565
566 // Put them in an arbitrary but reproducible order
567 OrderedKernels = sortByName(V: std::move(OrderedKernels));
568
569 // Annotate the kernels with their order in this vector
570 LLVMContext &Ctx = M->getContext();
571 IRBuilder<> Builder(Ctx);
572
573 if (OrderedKernels.size() > UINT32_MAX) {
574 // 32 bit keeps it in one SGPR. > 2**32 kernels won't fit on the GPU
575 reportFatalUsageError(reason: "unimplemented LDS lowering for > 2**32 kernels");
576 }
577
578 for (size_t i = 0; i < OrderedKernels.size(); i++) {
579 Metadata *AttrMDArgs[1] = {
580 ConstantAsMetadata::get(C: Builder.getInt32(C: i)),
581 };
582 OrderedKernels[i]->setMetadata(Kind: "llvm.amdgcn.lds.kernel.id",
583 Node: MDNode::get(Context&: Ctx, MDs: AttrMDArgs));
584 }
585 }
586 return OrderedKernels;
587 }
588
589 static void partitionVariablesIntoIndirectStrategies(
590 Module &M, LDSUsesInfoTy const &LDSUsesInfo,
591 VariableFunctionMap &LDSToKernelsThatNeedToAccessItIndirectly,
592 DenseSet<GlobalVariable *> &ModuleScopeVariables,
593 DenseSet<GlobalVariable *> &TableLookupVariables,
594 DenseSet<GlobalVariable *> &KernelAccessVariables,
595 DenseSet<GlobalVariable *> &DynamicVariables) {
596
597 GlobalVariable *HybridModuleRoot =
598 LoweringKindLoc != LoweringKind::hybrid
599 ? nullptr
600 : chooseBestVariableForModuleStrategy(
601 DL: M.getDataLayout(), LDSVars&: LDSToKernelsThatNeedToAccessItIndirectly);
602
603 DenseSet<Function *> const EmptySet;
604 DenseSet<Function *> const &HybridModuleRootKernels =
605 HybridModuleRoot
606 ? LDSToKernelsThatNeedToAccessItIndirectly[HybridModuleRoot]
607 : EmptySet;
608
609 for (auto &K : LDSToKernelsThatNeedToAccessItIndirectly) {
610 // Each iteration of this loop assigns exactly one global variable to
611 // exactly one of the implementation strategies.
612
613 GlobalVariable *GV = K.first;
614 assert(AMDGPU::isLDSVariableToLower(*GV));
615 assert(K.second.size() != 0);
616
617 if (AMDGPU::isDynamicLDS(GV: *GV)) {
618 DynamicVariables.insert(V: GV);
619 continue;
620 }
621
622 switch (LoweringKindLoc) {
623 case LoweringKind::module:
624 ModuleScopeVariables.insert(V: GV);
625 break;
626
627 case LoweringKind::table:
628 TableLookupVariables.insert(V: GV);
629 break;
630
631 case LoweringKind::kernel:
632 if (K.second.size() == 1) {
633 KernelAccessVariables.insert(V: GV);
634 } else {
635 // FIXME: This should use DiagnosticInfo
636 reportFatalUsageError(
637 reason: "cannot lower LDS '" + GV->getName() +
638 "' to kernel access as it is reachable from multiple kernels");
639 }
640 break;
641
642 case LoweringKind::hybrid: {
643 if (GV == HybridModuleRoot) {
644 assert(K.second.size() != 1);
645 ModuleScopeVariables.insert(V: GV);
646 } else if (K.second.size() == 1) {
647 KernelAccessVariables.insert(V: GV);
648 } else if (K.second == HybridModuleRootKernels) {
649 ModuleScopeVariables.insert(V: GV);
650 } else {
651 TableLookupVariables.insert(V: GV);
652 }
653 break;
654 }
655 }
656 }
657
658 // All LDS variables accessed indirectly have now been partitioned into
659 // the distinct lowering strategies.
660 assert(ModuleScopeVariables.size() + TableLookupVariables.size() +
661 KernelAccessVariables.size() + DynamicVariables.size() ==
662 LDSToKernelsThatNeedToAccessItIndirectly.size());
663 }
664
665 static GlobalVariable *lowerModuleScopeStructVariables(
666 Module &M, DenseSet<GlobalVariable *> const &ModuleScopeVariables,
667 DenseSet<Function *> const &KernelsThatAllocateModuleLDS) {
668 // Create a struct to hold the ModuleScopeVariables
669 // Replace all uses of those variables from non-kernel functions with the
670 // new struct instance Replace only the uses from kernel functions that will
671 // allocate this instance. That is a space optimisation - kernels that use a
672 // subset of the module scope struct and do not need to allocate it for
673 // indirect calls will only allocate the subset they use (they do so as part
674 // of the per-kernel lowering).
675 if (ModuleScopeVariables.empty()) {
676 return nullptr;
677 }
678
679 LLVMContext &Ctx = M.getContext();
680
681 LDSVariableReplacement ModuleScopeReplacement =
682 createLDSVariableReplacement(M, VarName: "llvm.amdgcn.module.lds",
683 LDSVarsToTransform: ModuleScopeVariables);
684
685 appendToCompilerUsed(M, Values: {static_cast<GlobalValue *>(
686 ConstantExpr::getPointerBitCastOrAddrSpaceCast(
687 C: cast<Constant>(Val: ModuleScopeReplacement.SGV),
688 Ty: PointerType::getUnqual(C&: Ctx)))});
689
690 // module.lds will be allocated at zero in any kernel that allocates it
691 recordLDSAbsoluteAddress(M: &M, GV: ModuleScopeReplacement.SGV, Address: 0);
692
693 // historic
694 removeLocalVarsFromUsedLists(M, LocalVars: ModuleScopeVariables);
695
696 // Replace all uses of module scope variable from non-kernel functions
697 replaceLDSVariablesWithStruct(
698 M, LDSVarsToTransformArg: ModuleScopeVariables, Replacement: ModuleScopeReplacement, Predicate: [&](Use &U) {
699 Instruction *I = dyn_cast<Instruction>(Val: U.getUser());
700 if (!I) {
701 return false;
702 }
703 Function *F = I->getFunction();
704 return !isKernel(F: *F);
705 });
706
707 // Replace uses of module scope variable from kernel functions that
708 // allocate the module scope variable, otherwise leave them unchanged
709 // Record on each kernel whether the module scope global is used by it
710
711 for (Function &Func : M.functions()) {
712 if (Func.isDeclaration() || !isKernel(F: Func))
713 continue;
714
715 if (KernelsThatAllocateModuleLDS.contains(V: &Func)) {
716 replaceLDSVariablesWithStruct(
717 M, LDSVarsToTransformArg: ModuleScopeVariables, Replacement: ModuleScopeReplacement, Predicate: [&](Use &U) {
718 Instruction *I = dyn_cast<Instruction>(Val: U.getUser());
719 if (!I) {
720 return false;
721 }
722 Function *F = I->getFunction();
723 return F == &Func;
724 });
725
726 markUsedByKernel(Func: &Func, SGV: ModuleScopeReplacement.SGV);
727 }
728 }
729
730 return ModuleScopeReplacement.SGV;
731 }
732
733 static DenseMap<Function *, LDSVariableReplacement>
734 lowerKernelScopeStructVariables(
735 Module &M, LDSUsesInfoTy &LDSUsesInfo,
736 DenseSet<GlobalVariable *> const &ModuleScopeVariables,
737 DenseSet<Function *> const &KernelsThatAllocateModuleLDS,
738 GlobalVariable *MaybeModuleScopeStruct) {
739
740 // Create a struct for each kernel for the non-module-scope variables.
741
742 DenseMap<Function *, LDSVariableReplacement> KernelToReplacement;
743 for (Function &Func : M.functions()) {
744 if (Func.isDeclaration() || !isKernel(F: Func))
745 continue;
746
747 DenseSet<GlobalVariable *> KernelUsedVariables;
748 // Allocating variables that are used directly in this struct to get
749 // alignment aware allocation and predictable frame size.
750 for (auto &v : LDSUsesInfo.direct_access[&Func]) {
751 if (!AMDGPU::isDynamicLDS(GV: *v)) {
752 KernelUsedVariables.insert(V: v);
753 }
754 }
755
756 // Allocating variables that are accessed indirectly so that a lookup of
757 // this struct instance can find them from nested functions.
758 for (auto &v : LDSUsesInfo.indirect_access[&Func]) {
759 if (!AMDGPU::isDynamicLDS(GV: *v)) {
760 KernelUsedVariables.insert(V: v);
761 }
762 }
763
764 // Variables allocated in module lds must all resolve to that struct,
765 // not to the per-kernel instance.
766 if (KernelsThatAllocateModuleLDS.contains(V: &Func)) {
767 for (GlobalVariable *v : ModuleScopeVariables) {
768 KernelUsedVariables.erase(V: v);
769 }
770 }
771
772 if (KernelUsedVariables.empty()) {
773 // Either used no LDS, or the LDS it used was all in the module struct
774 // or dynamically sized
775 continue;
776 }
777
778 // The association between kernel function and LDS struct is done by
779 // symbol name, which only works if the function in question has a
780 // name This is not expected to be a problem in practice as kernels
781 // are called by name making anonymous ones (which are named by the
782 // backend) difficult to use. This does mean that llvm test cases need
783 // to name the kernels.
784 if (!Func.hasName()) {
785 reportFatalUsageError(reason: "anonymous kernels cannot use LDS variables");
786 }
787
788 std::string VarName =
789 (Twine("llvm.amdgcn.kernel.") + Func.getName() + ".lds").str();
790
791 auto Replacement =
792 createLDSVariableReplacement(M, VarName, LDSVarsToTransform: KernelUsedVariables);
793
794 // If any indirect uses, create a direct use to ensure allocation
795 // TODO: Simpler to unconditionally mark used but that regresses
796 // codegen in test/CodeGen/AMDGPU/noclobber-barrier.ll
797 auto Accesses = LDSUsesInfo.indirect_access.find(Val: &Func);
798 if ((Accesses != LDSUsesInfo.indirect_access.end()) &&
799 !Accesses->second.empty())
800 markUsedByKernel(Func: &Func, SGV: Replacement.SGV);
801
802 // remove preserves existing codegen
803 removeLocalVarsFromUsedLists(M, LocalVars: KernelUsedVariables);
804 KernelToReplacement[&Func] = Replacement;
805
806 // Rewrite uses within kernel to the new struct
807 replaceLDSVariablesWithStruct(
808 M, LDSVarsToTransformArg: KernelUsedVariables, Replacement, Predicate: [&Func](Use &U) {
809 Instruction *I = dyn_cast<Instruction>(Val: U.getUser());
810 return I && I->getFunction() == &Func;
811 });
812 }
813 return KernelToReplacement;
814 }
815
816 static GlobalVariable *
817 buildRepresentativeDynamicLDSInstance(Module &M, LDSUsesInfoTy &LDSUsesInfo,
818 Function *func) {
819 // Create a dynamic lds variable with a name associated with the passed
820 // function that has the maximum alignment of any dynamic lds variable
821 // reachable from this kernel. Dynamic LDS is allocated after the static LDS
822 // allocation, possibly after alignment padding. The representative variable
823 // created here has the maximum alignment of any other dynamic variable
824 // reachable by that kernel. All dynamic LDS variables are allocated at the
825 // same address in each kernel in order to provide the documented aliasing
826 // semantics. Setting the alignment here allows this IR pass to accurately
827 // predict the exact constant at which it will be allocated.
828
829 assert(isKernel(*func));
830
831 LLVMContext &Ctx = M.getContext();
832 const DataLayout &DL = M.getDataLayout();
833 Align MaxDynamicAlignment(1);
834
835 auto UpdateMaxAlignment = [&MaxDynamicAlignment, &DL](GlobalVariable *GV) {
836 if (AMDGPU::isDynamicLDS(GV: *GV)) {
837 MaxDynamicAlignment =
838 std::max(a: MaxDynamicAlignment, b: AMDGPU::getAlign(DL, GV));
839 }
840 };
841
842 for (GlobalVariable *GV : LDSUsesInfo.indirect_access[func]) {
843 UpdateMaxAlignment(GV);
844 }
845
846 for (GlobalVariable *GV : LDSUsesInfo.direct_access[func]) {
847 UpdateMaxAlignment(GV);
848 }
849
850 assert(func->hasName()); // Checked by caller
851 auto *emptyCharArray = ArrayType::get(ElementType: Type::getInt8Ty(C&: Ctx), NumElements: 0);
852 GlobalVariable *N = new GlobalVariable(
853 M, emptyCharArray, false, GlobalValue::ExternalLinkage, nullptr,
854 Twine("llvm.amdgcn." + func->getName() + ".dynlds"), nullptr, GlobalValue::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS,
855 false);
856 N->setAlignment(MaxDynamicAlignment);
857
858 assert(AMDGPU::isDynamicLDS(*N));
859 return N;
860 }
861
862 DenseMap<Function *, GlobalVariable *> lowerDynamicLDSVariables(
863 Module &M, LDSUsesInfoTy &LDSUsesInfo,
864 DenseSet<Function *> const &KernelsThatIndirectlyAllocateDynamicLDS,
865 DenseSet<GlobalVariable *> const &DynamicVariables,
866 std::vector<Function *> const &OrderedKernels) {
867 DenseMap<Function *, GlobalVariable *> KernelToCreatedDynamicLDS;
868 if (!KernelsThatIndirectlyAllocateDynamicLDS.empty()) {
869 LLVMContext &Ctx = M.getContext();
870 IRBuilder<> Builder(Ctx);
871 Type *I32 = Type::getInt32Ty(C&: Ctx);
872
873 std::vector<Constant *> newDynamicLDS;
874
875 // Table is built in the same order as OrderedKernels
876 for (auto &func : OrderedKernels) {
877
878 if (KernelsThatIndirectlyAllocateDynamicLDS.contains(V: func)) {
879 assert(isKernel(*func));
880 if (!func->hasName()) {
881 reportFatalUsageError(reason: "anonymous kernels cannot use LDS variables");
882 }
883
884 GlobalVariable *N =
885 buildRepresentativeDynamicLDSInstance(M, LDSUsesInfo, func);
886
887 KernelToCreatedDynamicLDS[func] = N;
888
889 markUsedByKernel(Func: func, SGV: N);
890
891 auto *emptyCharArray = ArrayType::get(ElementType: Type::getInt8Ty(C&: Ctx), NumElements: 0);
892 auto *GEP = ConstantExpr::getGetElementPtr(
893 Ty: emptyCharArray, C: N, Idx: ConstantInt::get(Ty: I32, V: 0), NW: true);
894 newDynamicLDS.push_back(x: ConstantExpr::getPtrToInt(C: GEP, Ty: I32));
895 } else {
896 newDynamicLDS.push_back(x: PoisonValue::get(T: I32));
897 }
898 }
899 assert(OrderedKernels.size() == newDynamicLDS.size());
900
901 ArrayType *t = ArrayType::get(ElementType: I32, NumElements: newDynamicLDS.size());
902 Constant *init = ConstantArray::get(T: t, V: newDynamicLDS);
903 GlobalVariable *table = new GlobalVariable(
904 M, t, true, GlobalValue::InternalLinkage, init,
905 "llvm.amdgcn.dynlds.offset.table", nullptr,
906 GlobalValue::NotThreadLocal, AMDGPUAS::CONSTANT_ADDRESS);
907
908 for (GlobalVariable *GV : DynamicVariables) {
909 for (Use &U : make_early_inc_range(Range: GV->uses())) {
910 auto *I = dyn_cast<Instruction>(Val: U.getUser());
911 if (!I)
912 continue;
913 if (isKernel(F: *I->getFunction()))
914 continue;
915
916 replaceUseWithTableLookup(M, Builder, LookupTable: table, GV, U, OptionalIndex: nullptr);
917 }
918 }
919 }
920 return KernelToCreatedDynamicLDS;
921 }
922
923 bool runOnModule(Module &M) {
924 CallGraph CG = CallGraph(M);
925 bool Changed = superAlignLDSGlobals(M);
926
927 Changed |= eliminateConstantExprUsesOfLDSFromAllInstructions(M);
928
929 Changed = true; // todo: narrow this down
930
931 // For each kernel, what variables does it access directly or through
932 // callees
933 LDSUsesInfoTy LDSUsesInfo = getTransitiveUsesOfLDS(CG, M);
934
935 // For each variable accessed through callees, which kernels access it
936 VariableFunctionMap LDSToKernelsThatNeedToAccessItIndirectly;
937 for (auto &K : LDSUsesInfo.indirect_access) {
938 Function *F = K.first;
939 assert(isKernel(*F));
940 for (GlobalVariable *GV : K.second) {
941 LDSToKernelsThatNeedToAccessItIndirectly[GV].insert(V: F);
942 }
943 }
944
945 // Partition variables accessed indirectly into the different strategies
946 DenseSet<GlobalVariable *> ModuleScopeVariables;
947 DenseSet<GlobalVariable *> TableLookupVariables;
948 DenseSet<GlobalVariable *> KernelAccessVariables;
949 DenseSet<GlobalVariable *> DynamicVariables;
950 partitionVariablesIntoIndirectStrategies(
951 M, LDSUsesInfo, LDSToKernelsThatNeedToAccessItIndirectly,
952 ModuleScopeVariables, TableLookupVariables, KernelAccessVariables,
953 DynamicVariables);
954
955 // If the kernel accesses a variable that is going to be stored in the
956 // module instance through a call then that kernel needs to allocate the
957 // module instance
958 const DenseSet<Function *> KernelsThatAllocateModuleLDS =
959 kernelsThatIndirectlyAccessAnyOfPassedVariables(M, LDSUsesInfo,
960 VariableSet: ModuleScopeVariables);
961 const DenseSet<Function *> KernelsThatAllocateTableLDS =
962 kernelsThatIndirectlyAccessAnyOfPassedVariables(M, LDSUsesInfo,
963 VariableSet: TableLookupVariables);
964
965 const DenseSet<Function *> KernelsThatIndirectlyAllocateDynamicLDS =
966 kernelsThatIndirectlyAccessAnyOfPassedVariables(M, LDSUsesInfo,
967 VariableSet: DynamicVariables);
968
969 GlobalVariable *MaybeModuleScopeStruct = lowerModuleScopeStructVariables(
970 M, ModuleScopeVariables, KernelsThatAllocateModuleLDS);
971
972 DenseMap<Function *, LDSVariableReplacement> KernelToReplacement =
973 lowerKernelScopeStructVariables(M, LDSUsesInfo, ModuleScopeVariables,
974 KernelsThatAllocateModuleLDS,
975 MaybeModuleScopeStruct);
976
977 // Lower zero cost accesses to the kernel instances just created
978 for (auto &GV : KernelAccessVariables) {
979 auto &funcs = LDSToKernelsThatNeedToAccessItIndirectly[GV];
980 assert(funcs.size() == 1); // Only one kernel can access it
981 LDSVariableReplacement Replacement =
982 KernelToReplacement[*(funcs.begin())];
983
984 DenseSet<GlobalVariable *> Vec;
985 Vec.insert(V: GV);
986
987 replaceLDSVariablesWithStruct(M, LDSVarsToTransformArg: Vec, Replacement, Predicate: [](Use &U) {
988 return isa<Instruction>(Val: U.getUser());
989 });
990 }
991
992 // The ith element of this vector is kernel id i
993 std::vector<Function *> OrderedKernels =
994 assignLDSKernelIDToEachKernel(M: &M, KernelsThatAllocateTableLDS,
995 KernelsThatIndirectlyAllocateDynamicLDS);
996
997 if (!KernelsThatAllocateTableLDS.empty()) {
998 LLVMContext &Ctx = M.getContext();
999 IRBuilder<> Builder(Ctx);
1000
1001 // The order must be consistent between lookup table and accesses to
1002 // lookup table
1003 auto TableLookupVariablesOrdered =
1004 sortByName(V: std::vector<GlobalVariable *>(TableLookupVariables.begin(),
1005 TableLookupVariables.end()));
1006
1007 GlobalVariable *LookupTable = buildLookupTable(
1008 M, Variables: TableLookupVariablesOrdered, kernels: OrderedKernels, KernelToReplacement);
1009 replaceUsesInInstructionsWithTableLookup(M, ModuleScopeVariables: TableLookupVariablesOrdered,
1010 LookupTable);
1011 }
1012
1013 DenseMap<Function *, GlobalVariable *> KernelToCreatedDynamicLDS =
1014 lowerDynamicLDSVariables(M, LDSUsesInfo,
1015 KernelsThatIndirectlyAllocateDynamicLDS,
1016 DynamicVariables, OrderedKernels);
1017
1018 // Strip amdgpu-no-lds-kernel-id from all functions reachable from the
1019 // kernel. We may have inferred this wasn't used prior to the pass.
1020 // TODO: We could filter out subgraphs that do not access LDS globals.
1021 for (auto *KernelSet : {&KernelsThatIndirectlyAllocateDynamicLDS,
1022 &KernelsThatAllocateTableLDS})
1023 for (Function *F : *KernelSet)
1024 removeFnAttrFromReachable(CG, KernelRoot: F, FnAttrs: {"amdgpu-no-lds-kernel-id"});
1025
1026 // All kernel frames have been allocated. Calculate and record the
1027 // addresses.
1028 {
1029 const DataLayout &DL = M.getDataLayout();
1030
1031 for (Function &Func : M.functions()) {
1032 if (Func.isDeclaration() || !isKernel(F: Func))
1033 continue;
1034
1035 // All three of these are optional. The first variable is allocated at
1036 // zero. They are allocated by AMDGPUMachineFunction as one block.
1037 // Layout:
1038 //{
1039 // module.lds
1040 // alignment padding
1041 // kernel instance
1042 // alignment padding
1043 // dynamic lds variables
1044 //}
1045
1046 const bool AllocateModuleScopeStruct =
1047 MaybeModuleScopeStruct &&
1048 KernelsThatAllocateModuleLDS.contains(V: &Func);
1049
1050 auto Replacement = KernelToReplacement.find(Val: &Func);
1051 const bool AllocateKernelScopeStruct =
1052 Replacement != KernelToReplacement.end();
1053
1054 const bool AllocateDynamicVariable =
1055 KernelToCreatedDynamicLDS.contains(Val: &Func);
1056
1057 uint32_t Offset = 0;
1058
1059 if (AllocateModuleScopeStruct) {
1060 // Allocated at zero, recorded once on construction, not once per
1061 // kernel
1062 Offset += MaybeModuleScopeStruct->getGlobalSize(DL);
1063 }
1064
1065 if (AllocateKernelScopeStruct) {
1066 GlobalVariable *KernelStruct = Replacement->second.SGV;
1067 Offset = alignTo(Size: Offset, A: AMDGPU::getAlign(DL, GV: KernelStruct));
1068 recordLDSAbsoluteAddress(M: &M, GV: KernelStruct, Address: Offset);
1069 Offset += KernelStruct->getGlobalSize(DL);
1070 }
1071
1072 // If there is dynamic allocation, the alignment needed is included in
1073 // the static frame size. There may be no reference to the dynamic
1074 // variable in the kernel itself, so without including it here, that
1075 // alignment padding could be missed.
1076 if (AllocateDynamicVariable) {
1077 GlobalVariable *DynamicVariable = KernelToCreatedDynamicLDS[&Func];
1078 Offset = alignTo(Size: Offset, A: AMDGPU::getAlign(DL, GV: DynamicVariable));
1079 recordLDSAbsoluteAddress(M: &M, GV: DynamicVariable, Address: Offset);
1080 }
1081
1082 if (Offset != 0) {
1083 (void)TM; // TODO: Account for target maximum LDS
1084 std::string Buffer;
1085 raw_string_ostream SS{Buffer};
1086 SS << format(Fmt: "%u", Vals: Offset);
1087
1088 // Instead of explicitly marking kernels that access dynamic variables
1089 // using special case metadata, annotate with min-lds == max-lds, i.e.
1090 // that there is no more space available for allocating more static
1091 // LDS variables. That is the right condition to prevent allocating
1092 // more variables which would collide with the addresses assigned to
1093 // dynamic variables.
1094 if (AllocateDynamicVariable)
1095 SS << format(Fmt: ",%u", Vals: Offset);
1096
1097 Func.addFnAttr(Kind: "amdgpu-lds-size", Val: Buffer);
1098 }
1099 }
1100 }
1101
1102 for (auto &GV : make_early_inc_range(Range: M.globals()))
1103 if (AMDGPU::isLDSVariableToLower(GV)) {
1104 // probably want to remove from used lists
1105 GV.removeDeadConstantUsers();
1106 if (GV.use_empty())
1107 GV.eraseFromParent();
1108 }
1109
1110 return Changed;
1111 }
1112
1113private:
1114 // Increase the alignment of LDS globals if necessary to maximise the chance
1115 // that we can use aligned LDS instructions to access them.
1116 static bool superAlignLDSGlobals(Module &M) {
1117 const DataLayout &DL = M.getDataLayout();
1118 bool Changed = false;
1119 if (!SuperAlignLDSGlobals) {
1120 return Changed;
1121 }
1122
1123 for (auto &GV : M.globals()) {
1124 if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) {
1125 // Only changing alignment of LDS variables
1126 continue;
1127 }
1128 if (!GV.hasInitializer()) {
1129 // cuda/hip extern __shared__ variable, leave alignment alone
1130 continue;
1131 }
1132
1133 if (GV.isAbsoluteSymbolRef()) {
1134 // If the variable is already allocated, don't change the alignment
1135 continue;
1136 }
1137
1138 Align Alignment = AMDGPU::getAlign(DL, GV: &GV);
1139 uint64_t GVSize = GV.getGlobalSize(DL);
1140
1141 if (GVSize > 8) {
1142 // We might want to use a b96 or b128 load/store
1143 Alignment = std::max(a: Alignment, b: Align(16));
1144 } else if (GVSize > 4) {
1145 // We might want to use a b64 load/store
1146 Alignment = std::max(a: Alignment, b: Align(8));
1147 } else if (GVSize > 2) {
1148 // We might want to use a b32 load/store
1149 Alignment = std::max(a: Alignment, b: Align(4));
1150 } else if (GVSize > 1) {
1151 // We might want to use a b16 load/store
1152 Alignment = std::max(a: Alignment, b: Align(2));
1153 }
1154
1155 if (Alignment != AMDGPU::getAlign(DL, GV: &GV)) {
1156 Changed = true;
1157 GV.setAlignment(Alignment);
1158 }
1159 }
1160 return Changed;
1161 }
1162
1163 static LDSVariableReplacement createLDSVariableReplacement(
1164 Module &M, std::string VarName,
1165 DenseSet<GlobalVariable *> const &LDSVarsToTransform) {
1166 // Create a struct instance containing LDSVarsToTransform and map from those
1167 // variables to ConstantExprGEP
1168 // Variables may be introduced to meet alignment requirements. No aliasing
1169 // metadata is useful for these as they have no uses. Erased before return.
1170
1171 LLVMContext &Ctx = M.getContext();
1172 const DataLayout &DL = M.getDataLayout();
1173 assert(!LDSVarsToTransform.empty());
1174
1175 SmallVector<OptimizedStructLayoutField, 8> LayoutFields;
1176 LayoutFields.reserve(N: LDSVarsToTransform.size());
1177 {
1178 // The order of fields in this struct depends on the order of
1179 // variables in the argument which varies when changing how they
1180 // are identified, leading to spurious test breakage.
1181 auto Sorted = sortByName(V: std::vector<GlobalVariable *>(
1182 LDSVarsToTransform.begin(), LDSVarsToTransform.end()));
1183
1184 for (GlobalVariable *GV : Sorted) {
1185 OptimizedStructLayoutField F(GV, GV->getGlobalSize(DL),
1186 AMDGPU::getAlign(DL, GV));
1187 LayoutFields.emplace_back(Args&: F);
1188 }
1189 }
1190
1191 performOptimizedStructLayout(Fields: LayoutFields);
1192
1193 std::vector<GlobalVariable *> LocalVars;
1194 BitVector IsPaddingField;
1195 LocalVars.reserve(n: LDSVarsToTransform.size()); // will be at least this large
1196 IsPaddingField.reserve(N: LDSVarsToTransform.size());
1197 {
1198 uint64_t CurrentOffset = 0;
1199 for (auto &F : LayoutFields) {
1200 GlobalVariable *FGV =
1201 static_cast<GlobalVariable *>(const_cast<void *>(F.Id));
1202 Align DataAlign = F.Alignment;
1203
1204 uint64_t DataAlignV = DataAlign.value();
1205 if (uint64_t Rem = CurrentOffset % DataAlignV) {
1206 uint64_t Padding = DataAlignV - Rem;
1207
1208 // Append an array of padding bytes to meet alignment requested
1209 // Note (o + (a - (o % a)) ) % a == 0
1210 // (offset + Padding ) % align == 0
1211
1212 Type *ATy = ArrayType::get(ElementType: Type::getInt8Ty(C&: Ctx), NumElements: Padding);
1213 LocalVars.push_back(x: new GlobalVariable(
1214 M, ATy, false, GlobalValue::InternalLinkage,
1215 PoisonValue::get(T: ATy), "", nullptr, GlobalValue::NotThreadLocal,
1216 AMDGPUAS::LOCAL_ADDRESS, false));
1217 IsPaddingField.push_back(Val: true);
1218 CurrentOffset += Padding;
1219 }
1220
1221 LocalVars.push_back(x: FGV);
1222 IsPaddingField.push_back(Val: false);
1223 CurrentOffset += F.Size;
1224 }
1225 }
1226
1227 std::vector<Type *> LocalVarTypes;
1228 LocalVarTypes.reserve(n: LocalVars.size());
1229 std::transform(
1230 first: LocalVars.cbegin(), last: LocalVars.cend(), result: std::back_inserter(x&: LocalVarTypes),
1231 unary_op: [](const GlobalVariable *V) -> Type * { return V->getValueType(); });
1232
1233 StructType *LDSTy = StructType::create(Context&: Ctx, Elements: LocalVarTypes, Name: VarName + ".t");
1234
1235 Align StructAlign = AMDGPU::getAlign(DL, GV: LocalVars[0]);
1236
1237 GlobalVariable *SGV = new GlobalVariable(
1238 M, LDSTy, false, GlobalValue::InternalLinkage, PoisonValue::get(T: LDSTy),
1239 VarName, nullptr, GlobalValue::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS,
1240 false);
1241 SGV->setAlignment(StructAlign);
1242
1243 DenseMap<GlobalVariable *, Constant *> Map;
1244 Type *I32 = Type::getInt32Ty(C&: Ctx);
1245 for (size_t I = 0; I < LocalVars.size(); I++) {
1246 GlobalVariable *GV = LocalVars[I];
1247 Constant *GEPIdx[] = {ConstantInt::get(Ty: I32, V: 0), ConstantInt::get(Ty: I32, V: I)};
1248 Constant *GEP = ConstantExpr::getGetElementPtr(Ty: LDSTy, C: SGV, IdxList: GEPIdx, NW: true);
1249 if (IsPaddingField[I]) {
1250 assert(GV->use_empty());
1251 GV->eraseFromParent();
1252 } else {
1253 Map[GV] = GEP;
1254 }
1255 }
1256 assert(Map.size() == LDSVarsToTransform.size());
1257 return {.SGV: SGV, .LDSVarsToConstantGEP: std::move(Map)};
1258 }
1259
1260 template <typename PredicateTy>
1261 static void replaceLDSVariablesWithStruct(
1262 Module &M, DenseSet<GlobalVariable *> const &LDSVarsToTransformArg,
1263 const LDSVariableReplacement &Replacement, PredicateTy Predicate) {
1264 LLVMContext &Ctx = M.getContext();
1265 const DataLayout &DL = M.getDataLayout();
1266
1267 // A hack... we need to insert the aliasing info in a predictable order for
1268 // lit tests. Would like to have them in a stable order already, ideally the
1269 // same order they get allocated, which might mean an ordered set container
1270 auto LDSVarsToTransform = sortByName(V: std::vector<GlobalVariable *>(
1271 LDSVarsToTransformArg.begin(), LDSVarsToTransformArg.end()));
1272
1273 // Create alias.scope and their lists. Each field in the new structure
1274 // does not alias with all other fields.
1275 SmallVector<MDNode *> AliasScopes;
1276 SmallVector<Metadata *> NoAliasList;
1277 const size_t NumberVars = LDSVarsToTransform.size();
1278 if (NumberVars > 1) {
1279 MDBuilder MDB(Ctx);
1280 AliasScopes.reserve(N: NumberVars);
1281 MDNode *Domain = MDB.createAnonymousAliasScopeDomain();
1282 for (size_t I = 0; I < NumberVars; I++) {
1283 MDNode *Scope = MDB.createAnonymousAliasScope(Domain);
1284 AliasScopes.push_back(Elt: Scope);
1285 }
1286 NoAliasList.append(in_start: &AliasScopes[1], in_end: AliasScopes.end());
1287 }
1288
1289 // Replace uses of ith variable with a constantexpr to the corresponding
1290 // field of the instance that will be allocated by AMDGPUMachineFunction
1291 for (size_t I = 0; I < NumberVars; I++) {
1292 GlobalVariable *GV = LDSVarsToTransform[I];
1293 Constant *GEP = Replacement.LDSVarsToConstantGEP.at(Val: GV);
1294
1295 GV->replaceUsesWithIf(New: GEP, ShouldReplace: Predicate);
1296
1297 APInt APOff(DL.getIndexTypeSizeInBits(Ty: GEP->getType()), 0);
1298 GEP->stripAndAccumulateInBoundsConstantOffsets(DL, Offset&: APOff);
1299 uint64_t Offset = APOff.getZExtValue();
1300
1301 Align A =
1302 commonAlignment(A: Replacement.SGV->getAlign().valueOrOne(), Offset);
1303
1304 if (I)
1305 NoAliasList[I - 1] = AliasScopes[I - 1];
1306 MDNode *NoAlias =
1307 NoAliasList.empty() ? nullptr : MDNode::get(Context&: Ctx, MDs: NoAliasList);
1308 MDNode *AliasScope =
1309 AliasScopes.empty() ? nullptr : MDNode::get(Context&: Ctx, MDs: {AliasScopes[I]});
1310
1311 refineUsesAlignmentAndAA(Ptr: GEP, A, DL, AliasScope, NoAlias);
1312 }
1313 }
1314
1315 static void refineUsesAlignmentAndAA(Value *Ptr, Align A,
1316 const DataLayout &DL, MDNode *AliasScope,
1317 MDNode *NoAlias, unsigned MaxDepth = 5) {
1318 if (!MaxDepth || (A == 1 && !AliasScope))
1319 return;
1320
1321 ScopedNoAliasAAResult ScopedNoAlias;
1322
1323 for (User *U : Ptr->users()) {
1324 if (auto *I = dyn_cast<Instruction>(Val: U)) {
1325 if (AliasScope && I->mayReadOrWriteMemory()) {
1326 MDNode *AS = I->getMetadata(KindID: LLVMContext::MD_alias_scope);
1327 AS = (AS ? MDNode::getMostGenericAliasScope(A: AS, B: AliasScope)
1328 : AliasScope);
1329 I->setMetadata(KindID: LLVMContext::MD_alias_scope, Node: AS);
1330
1331 MDNode *NA = I->getMetadata(KindID: LLVMContext::MD_noalias);
1332
1333 // Scoped aliases can originate from two different domains.
1334 // First domain would be from LDS domain (created by this pass).
1335 // All entries (LDS vars) into LDS struct will have same domain.
1336
1337 // Second domain could be existing scoped aliases that are the
1338 // results of noalias params and subsequent optimizations that
1339 // may alter thesse sets.
1340
1341 // We need to be careful how we create new alias sets, and
1342 // have right scopes and domains for loads/stores of these new
1343 // LDS variables. We intersect NoAlias set if alias sets belong
1344 // to the same domain. This is the case if we have memcpy using
1345 // LDS variables. Both src and dst of memcpy would belong to
1346 // LDS struct, they donot alias.
1347 // On the other hand, if one of the domains is LDS and other is
1348 // existing domain prior to LDS, we need to have a union of all
1349 // these aliases set to preserve existing aliasing information.
1350
1351 SmallPtrSet<const MDNode *, 16> ExistingDomains, LDSDomains;
1352 ScopedNoAlias.collectScopedDomains(NoAlias: NA, Domains&: ExistingDomains);
1353 ScopedNoAlias.collectScopedDomains(NoAlias, Domains&: LDSDomains);
1354 auto Intersection = set_intersection(S1: ExistingDomains, S2: LDSDomains);
1355 if (Intersection.empty()) {
1356 NA = NA ? MDNode::concatenate(A: NA, B: NoAlias) : NoAlias;
1357 } else {
1358 NA = NA ? MDNode::intersect(A: NA, B: NoAlias) : NoAlias;
1359 }
1360 I->setMetadata(KindID: LLVMContext::MD_noalias, Node: NA);
1361 }
1362 }
1363
1364 if (auto *LI = dyn_cast<LoadInst>(Val: U)) {
1365 LI->setAlignment(std::max(a: A, b: LI->getAlign()));
1366 continue;
1367 }
1368 if (auto *SI = dyn_cast<StoreInst>(Val: U)) {
1369 if (SI->getPointerOperand() == Ptr)
1370 SI->setAlignment(std::max(a: A, b: SI->getAlign()));
1371 continue;
1372 }
1373 if (auto *AI = dyn_cast<AtomicRMWInst>(Val: U)) {
1374 // None of atomicrmw operations can work on pointers, but let's
1375 // check it anyway in case it will or we will process ConstantExpr.
1376 if (AI->getPointerOperand() == Ptr)
1377 AI->setAlignment(std::max(a: A, b: AI->getAlign()));
1378 continue;
1379 }
1380 if (auto *AI = dyn_cast<AtomicCmpXchgInst>(Val: U)) {
1381 if (AI->getPointerOperand() == Ptr)
1382 AI->setAlignment(std::max(a: A, b: AI->getAlign()));
1383 continue;
1384 }
1385 if (auto *GEP = dyn_cast<GetElementPtrInst>(Val: U)) {
1386 unsigned BitWidth = DL.getIndexTypeSizeInBits(Ty: GEP->getType());
1387 APInt Off(BitWidth, 0);
1388 if (GEP->getPointerOperand() == Ptr) {
1389 Align GA;
1390 if (GEP->accumulateConstantOffset(DL, Offset&: Off))
1391 GA = commonAlignment(A, Offset: Off.getLimitedValue());
1392 refineUsesAlignmentAndAA(Ptr: GEP, A: GA, DL, AliasScope, NoAlias,
1393 MaxDepth: MaxDepth - 1);
1394 }
1395 continue;
1396 }
1397 if (auto *I = dyn_cast<Instruction>(Val: U)) {
1398 if (I->getOpcode() == Instruction::BitCast ||
1399 I->getOpcode() == Instruction::AddrSpaceCast)
1400 refineUsesAlignmentAndAA(Ptr: I, A, DL, AliasScope, NoAlias, MaxDepth: MaxDepth - 1);
1401 }
1402 }
1403 }
1404};
1405
1406class AMDGPULowerModuleLDSLegacy : public ModulePass {
1407public:
1408 const AMDGPUTargetMachine *TM;
1409 static char ID;
1410
1411 AMDGPULowerModuleLDSLegacy(const AMDGPUTargetMachine *TM = nullptr)
1412 : ModulePass(ID), TM(TM) {}
1413
1414 void getAnalysisUsage(AnalysisUsage &AU) const override {
1415 if (!TM)
1416 AU.addRequired<TargetPassConfig>();
1417 }
1418
1419 bool runOnModule(Module &M) override {
1420 if (!TM) {
1421 auto &TPC = getAnalysis<TargetPassConfig>();
1422 TM = &TPC.getTM<AMDGPUTargetMachine>();
1423 }
1424
1425 return AMDGPULowerModuleLDS(*TM).runOnModule(M);
1426 }
1427};
1428
1429} // namespace
1430char AMDGPULowerModuleLDSLegacy::ID = 0;
1431
1432char &llvm::AMDGPULowerModuleLDSLegacyPassID = AMDGPULowerModuleLDSLegacy::ID;
1433
1434INITIALIZE_PASS_BEGIN(AMDGPULowerModuleLDSLegacy, DEBUG_TYPE,
1435 "Lower uses of LDS variables from non-kernel functions",
1436 false, false)
1437INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
1438INITIALIZE_PASS_END(AMDGPULowerModuleLDSLegacy, DEBUG_TYPE,
1439 "Lower uses of LDS variables from non-kernel functions",
1440 false, false)
1441
1442ModulePass *
1443llvm::createAMDGPULowerModuleLDSLegacyPass(const AMDGPUTargetMachine *TM) {
1444 return new AMDGPULowerModuleLDSLegacy(TM);
1445}
1446
1447PreservedAnalyses AMDGPULowerModuleLDSPass::run(Module &M,
1448 ModuleAnalysisManager &) {
1449 return AMDGPULowerModuleLDS(TM).runOnModule(M) ? PreservedAnalyses::none()
1450 : PreservedAnalyses::all();
1451}
1452