1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Lower global variables with target extension type "amdgpu.named.barrier"
10// that require specialized address assignment. It assigns a unique
11// barrier identifier to each named-barrier variable and encodes
12// this identifier within the !absolute_symbol metadata of that global.
13//
14//===----------------------------------------------------------------------===//
15
16#include "AMDGPU.h"
17#include "AMDGPUMemoryUtils.h"
18#include "AMDGPUTargetMachine.h"
19#include "llvm/Analysis/CallGraph.h"
20#include "llvm/IR/Constants.h"
21#include "llvm/IR/Instructions.h"
22#include "llvm/IR/ReplaceConstant.h"
23#include "llvm/InitializePasses.h"
24#include "llvm/Pass.h"
25#include "llvm/Transforms/Utils/ModuleUtils.h"
26
27#define DEBUG_TYPE "amdgpu-lower-exec-sync"
28
29using namespace llvm;
30using namespace AMDGPU;
31
32namespace {
33
34static bool isNamedBarrierToLower(const GlobalVariable &GV) {
35 return isNamedBarrier(GV) && !GV.isAbsoluteSymbolRef();
36}
37
38// Write the specified address into metadata where it can be retrieved by
39// the assembler. Format is a half open range, [Address Address+1)
40static void recordAbsoluteAddress(Module *M, GlobalVariable *GV,
41 uint32_t Address) {
42 LLVMContext &Ctx = M->getContext();
43 auto *IntTy = M->getDataLayout().getIntPtrType(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS);
44 auto *MinC = ConstantAsMetadata::get(C: ConstantInt::get(Ty: IntTy, V: Address));
45 auto *MaxC = ConstantAsMetadata::get(C: ConstantInt::get(Ty: IntTy, V: Address + 1));
46 GV->setMetadata(KindID: LLVMContext::MD_absolute_symbol,
47 Node: MDNode::get(Context&: Ctx, MDs: {MinC, MaxC}));
48}
49
50/// Get next available ID for sync object. The ID allocation is tracked in \p
51/// MaxNumGroup groups by \p NextAvailableIDTracker. Each call of the function
52/// will ask for \p IDCnt against all the \p Kernels, it will return the
53/// maximum of the available ones and update the ID tracker.
54template <typename T>
55unsigned allocateExecSyncID(T &NextAvailableIDTracker,
56 ArrayRef<Function *> Kernels, unsigned GroupID,
57 unsigned MaxNumGroup, unsigned IDCnt) {
58 constexpr unsigned InitialVal = 1;
59 unsigned NextID = InitialVal;
60 for (Function *F : Kernels) {
61 const SmallVectorImpl<unsigned> &NextAvailableID =
62 NextAvailableIDTracker.lookup(F);
63 unsigned ID = InitialVal;
64 if (!NextAvailableID.empty())
65 ID = NextAvailableID[GroupID];
66
67 if (ID > NextID)
68 NextID = ID;
69 }
70
71 // Bump the next available id for the kernels.
72 for (Function *F : Kernels) {
73 auto Inserted = NextAvailableIDTracker.try_emplace(F);
74 // Initialize on first insertion.
75 if (Inserted.second)
76 Inserted.first->second.assign(MaxNumGroup, InitialVal);
77 // Update the available ID.
78 Inserted.first->second[GroupID] = NextID + IDCnt;
79 }
80 return NextID;
81}
82
83// Main utility function for special LDS variables lowering.
84static bool lowerExecSyncGlobalVariables(Module &M, GVUsesInfoTy &GVUsesInfo) {
85 bool Changed = false;
86 const DataLayout &DL = M.getDataLayout();
87
88 constexpr unsigned NumBarScopes = 1;
89 MapVector<GlobalVariable *, SmallVector<Function *>> AllocationQ;
90 DenseMap<Function *, SmallVector<unsigned, NumBarScopes>> KernelBarrierIDs;
91
92 for (auto &[F, GVs] : GVUsesInfo.IndirectAccess) {
93 for (auto *GV : GVs) {
94 if (!isNamedBarrier(GV: *GV) || GV->isAbsoluteSymbolRef())
95 continue;
96 auto Iter = AllocationQ.find(Key: GV);
97 if (Iter == AllocationQ.end())
98 AllocationQ.insert(KV: {GV, {F}});
99 else
100 Iter->second.push_back(Elt: F);
101 }
102 }
103
104 for (auto &[F, GVs] : GVUsesInfo.DirectAccess) {
105 for (auto *GV : GVs) {
106 if (!isNamedBarrier(GV: *GV) || GV->isAbsoluteSymbolRef())
107 continue;
108 auto Iter = AllocationQ.find(Key: GV);
109 if (Iter == AllocationQ.end())
110 AllocationQ.insert(KV: {GV, {F}});
111 else
112 Iter->second.push_back(Elt: F);
113 }
114 }
115
116 sort(C&: AllocationQ, Comp: [](std::pair<GlobalVariable *, SmallVector<Function *>> A,
117 std::pair<GlobalVariable *, SmallVector<Function *>> B) {
118 // First order by number of kernels that access the GlobalVariable.
119 if (A.second.size() != B.second.size())
120 return A.second.size() > B.second.size();
121
122 // Then order by their names so we always get a deterministic order.
123 return A.first->getName() < B.first->getName();
124 });
125
126 for (auto &[GV, Kernels] : AllocationQ) {
127 unsigned Offset;
128 if (TargetExtType *ExtTy = isNamedBarrier(GV: *GV)) {
129 unsigned BarrierScope = ExtTy->getIntParameter(i: 0);
130 unsigned BarCnt = GV->getGlobalSize(DL) / 16;
131
132 unsigned BarID = allocateExecSyncID(NextAvailableIDTracker&: KernelBarrierIDs, Kernels,
133 GroupID: BarrierScope, MaxNumGroup: NumBarScopes, IDCnt: BarCnt);
134
135 LLVM_DEBUG(GV->printAsOperand(dbgs(), false);
136 dbgs() << " was assigned barrier id: " << BarID
137 << " id-count: " << BarCnt << "\n");
138 Offset = BarID;
139 } else {
140 llvm_unreachable("Unhandled special variable type.");
141 }
142
143 recordAbsoluteAddress(M: &M, GV, Address: Offset);
144 }
145
146 // Also erase those special LDS variables from indirect_access.
147 for (auto &K : GVUsesInfo.IndirectAccess) {
148 assert(isKernel(*K.first));
149 K.second.remove_if(Pred: [](GlobalVariable *GV) { return isNamedBarrier(GV: *GV); });
150 }
151 return Changed;
152}
153
154static bool hasBarrierToLower(const GVUsesInfoTy &GVUsesInfo) {
155 for (auto &Map : {GVUsesInfo.DirectAccess, GVUsesInfo.IndirectAccess}) {
156 for (auto &[Fn, GVs] : Map) {
157 for (auto &GV : GVs) {
158 if (AMDGPU::isNamedBarrier(GV: *GV))
159 return true;
160 }
161 }
162 }
163 return false;
164}
165
166// With object linking, barrier ID assignment is deferred to the linker.
167// Externalize named barrier globals and emit self-contained metadata so the
168// AsmPrinter can generate the callgraph entries the linker needs.
169static bool handleNamedBarriersForObjectLinking(Module &M) {
170 DenseMap<GlobalVariable *, DenseSet<Function *>> BarrierToFuncs;
171 for (GlobalVariable &GV : M.globals()) {
172 if (!isNamedBarrier(GV) || GV.use_empty())
173 continue;
174 for (User *U : GV.users()) {
175 if (auto *I = dyn_cast<Instruction>(Val: U))
176 BarrierToFuncs[&GV].insert(V: I->getFunction());
177 }
178 }
179 if (BarrierToFuncs.empty())
180 return false;
181
182 LLVMContext &Ctx = M.getContext();
183 NamedMDNode *BarMD = M.getOrInsertNamedMetadata(Name: "amdgpu.named_barrier.uses");
184
185 std::string ModuleId;
186 ModuleId = getUniqueModuleId(M: &M);
187 assert(!ModuleId.empty() &&
188 "modules with named barriers should have a unique ID");
189 for (auto &[V, Funcs] : BarrierToFuncs) {
190 if (V->hasLocalLinkage())
191 V->setName("__amdgpu_named_barrier." + V->getName() + ModuleId);
192 else if (!V->getName().starts_with(Prefix: "__amdgpu_named_barrier"))
193 V->setName("__amdgpu_named_barrier." + V->getName());
194 V->setInitializer(nullptr);
195 V->setLinkage(GlobalValue::ExternalLinkage);
196
197 SmallVector<Metadata *, 4> Ops;
198 Ops.push_back(Elt: ValueAsMetadata::get(V));
199 for (Function *F : Funcs)
200 Ops.push_back(Elt: ValueAsMetadata::get(V: F));
201 BarMD->addOperand(M: MDNode::get(Context&: Ctx, MDs: Ops));
202 }
203 return true;
204}
205
206static bool runLowerExecSyncGlobals(Module &M) {
207 if (AMDGPUTargetMachine::EnableObjectLinking)
208 return handleNamedBarriersForObjectLinking(M);
209
210 CallGraph CG = CallGraph(M);
211 bool Changed = false;
212 Changed |=
213 eliminateGVConstantExprUsesFromAllInstructions(M, Filter: isNamedBarrierToLower);
214
215 // For each kernel, what variables does it access directly or through
216 // callees
217 GVUsesInfoTy BarrierUsesInfo =
218 getTransitiveUsesOfGV(CG, M, Filter: isNamedBarrierToLower);
219
220 if (hasBarrierToLower(GVUsesInfo: BarrierUsesInfo)) {
221 // Special LDS variables need special address assignment
222 Changed |= lowerExecSyncGlobalVariables(M, GVUsesInfo&: BarrierUsesInfo);
223 }
224
225 return Changed;
226}
227
228class AMDGPULowerExecSyncLegacy : public ModulePass {
229public:
230 static char ID;
231 AMDGPULowerExecSyncLegacy() : ModulePass(ID) {}
232 bool runOnModule(Module &M) override;
233};
234
235} // namespace
236
237char AMDGPULowerExecSyncLegacy::ID = 0;
238char &llvm::AMDGPULowerExecSyncLegacyPassID = AMDGPULowerExecSyncLegacy::ID;
239
240INITIALIZE_PASS_BEGIN(AMDGPULowerExecSyncLegacy, DEBUG_TYPE,
241 "AMDGPU lowering of execution synchronization", false,
242 false)
243INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
244INITIALIZE_PASS_END(AMDGPULowerExecSyncLegacy, DEBUG_TYPE,
245 "AMDGPU lowering of execution synchronization", false,
246 false)
247
248bool AMDGPULowerExecSyncLegacy::runOnModule(Module &M) {
249 return runLowerExecSyncGlobals(M);
250}
251
252ModulePass *llvm::createAMDGPULowerExecSyncLegacyPass() {
253 return new AMDGPULowerExecSyncLegacy();
254}
255
256PreservedAnalyses AMDGPULowerExecSyncPass::run(Module &M,
257 ModuleAnalysisManager &AM) {
258 return runLowerExecSyncGlobals(M) ? PreservedAnalyses::none()
259 : PreservedAnalyses::all();
260}
261