1//===-- AMDGPUSwLowerLDS.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers the local data store, LDS, uses in kernel and non-kernel
10// functions in module to use dynamically allocated global memory.
11// Packed LDS Layout is emulated in the global memory.
12// The lowered memory instructions from LDS to global memory are then
13// instrumented for address sanitizer, to catch addressing errors.
14// This pass only work when address sanitizer has been enabled and has
15// instrumented the IR. It identifies that IR has been instrumented using
16// "nosanitize_address" module flag.
17//
18// Replacement of Kernel LDS accesses:
19// For a kernel, LDS access can be static or dynamic which are direct
20// (accessed within kernel) and indirect (accessed through non-kernels).
21// All these LDS accesses corresponding to kernel will be packed together,
22// where all static LDS accesses will be allocated first and then dynamic
23// LDS follows. The total size with alignment is calculated. A new LDS global
24// will be created for the kernel called "SW LDS" and it will have the
25// attribute "amdgpu-lds-size" attached with value of the size calculated.
26// All the LDS accesses in the module will be replaced by GEP with offset
27// into the "Sw LDS".
28// A new "llvm.amdgcn.<kernel>.dynlds" is created per kernel accessing
29// the dynamic LDS. This will be marked used by kernel and will have
30// MD_absolue_symbol metadata set to total static LDS size, Since dynamic
31// LDS allocation starts after all static LDS allocation.
32//
33// A device global memory equal to the total LDS size will be allocated.
34// At the prologue of the kernel, a single work-item from the
35// work-group, does a "malloc" and stores the pointer of the
36// allocation in "SW LDS".
37//
38// To store the offsets corresponding to all LDS accesses, another global
39// variable is created which will be called "SW LDS metadata" in this pass.
40// - SW LDS Global:
41// It is LDS global of ptr type with name
42// "llvm.amdgcn.sw.lds.<kernel-name>".
43// - Metadata Global:
44// It is of struct type, with n members. n equals the number of LDS
45// globals accessed by the kernel(direct and indirect). Each member of
46// struct is another struct of type {i32, i32, i32}. First member
47// corresponds to offset, second member corresponds to size of LDS global
48// being replaced and third represents the total aligned size. It will
49// have name "llvm.amdgcn.sw.lds.<kernel-name>.md". This global will have
50// an initializer with static LDS related offsets and sizes initialized.
51// But for dynamic LDS related entries, offsets will be initialized to
52// previous static LDS allocation end offset. Sizes for them will be zero
53// initially. These dynamic LDS offset and size values will be updated
54// within the kernel, since kernel can read the dynamic LDS size
55// allocation done at runtime with query to "hidden_dynamic_lds_size"
56// hidden kernel argument.
57//
58// At the epilogue of kernel, allocated memory would be made free by the same
59// single work-item.
60//
61// Replacement of non-kernel LDS accesses:
62// Multiple kernels can access the same non-kernel function.
63// All the kernels accessing LDS through non-kernels, or reaching
64// non-kernels with LDS instructions, are sorted and assigned a kernel-id.
65// All the LDS globals accessed by non-kernels are sorted. This
66// information is used to build two tables:
67// - Base table:
68// Base table will have single row, with elements of the row
69// placed as per kernel ID. Each element in the row corresponds
70// to ptr of "SW LDS" variable created for that kernel.
71// - Offset table:
72// Offset table will have multiple rows and columns.
73// Rows are assumed to be from 0 to (n-1). n is total number
74// of kernels accessing the LDS through non-kernels.
75// Each row will have m elements. m is the total number of
76// unique LDS globals accessed by all non-kernels.
77// Each element in the row correspond to the ptr of
78// the replacement of LDS global done by that particular kernel.
79// A LDS variable in non-kernel will be replaced based on the information
80// from base and offset tables. Based on kernel-id query, ptr of "SW
81// LDS" for that corresponding kernel is obtained from base table.
82// The Offset into the base "SW LDS" is obtained from
83// corresponding element in offset table. With this information, replacement
84// value is obtained.
85//===----------------------------------------------------------------------===//
86
87#include "AMDGPU.h"
88#include "AMDGPUAsanInstrumentation.h"
89#include "AMDGPUMemoryUtils.h"
90#include "llvm/ADT/StringExtras.h"
91#include "llvm/ADT/StringRef.h"
92#include "llvm/Analysis/CallGraph.h"
93#include "llvm/Analysis/DomTreeUpdater.h"
94#include "llvm/IR/Constants.h"
95#include "llvm/IR/DIBuilder.h"
96#include "llvm/IR/DebugInfo.h"
97#include "llvm/IR/DebugInfoMetadata.h"
98#include "llvm/IR/IRBuilder.h"
99#include "llvm/IR/InstIterator.h"
100#include "llvm/IR/Instructions.h"
101#include "llvm/IR/MDBuilder.h"
102#include "llvm/IR/ReplaceConstant.h"
103#include "llvm/Pass.h"
104#include "llvm/Support/raw_ostream.h"
105#include "llvm/Transforms/Instrumentation/AddressSanitizerCommon.h"
106#include "llvm/Transforms/Utils/ModuleUtils.h"
107
108#include <algorithm>
109
110#define DEBUG_TYPE "amdgpu-sw-lower-lds"
111#define COV5_HIDDEN_DYN_LDS_SIZE_ARG 15
112
113using namespace llvm;
114using namespace AMDGPU;
115
116namespace {
117
118using DomTreeCallback = function_ref<DominatorTree *(Function &F)>;
119
120struct LDSAccessTypeInfo {
121 SetVector<GlobalVariable *> StaticLDSGlobals;
122 SetVector<GlobalVariable *> DynamicLDSGlobals;
123};
124
125// Struct to hold all the Metadata required for a kernel
126// to replace a LDS global uses with corresponding offset
127// in to device global memory.
128struct KernelLDSParameters {
129 GlobalVariable *SwLDS = nullptr;
130 GlobalVariable *SwDynLDS = nullptr;
131 GlobalVariable *SwLDSMetadata = nullptr;
132 LDSAccessTypeInfo DirectAccess;
133 LDSAccessTypeInfo IndirectAccess;
134 DenseMap<GlobalVariable *, SmallVector<uint32_t, 3>>
135 LDSToReplacementIndicesMap;
136 uint32_t MallocSize = 0;
137 uint32_t LDSSize = 0;
138 SmallVector<std::pair<uint32_t, uint32_t>, 64> RedzoneOffsetAndSizeVector;
139};
140
141// Struct to store information for creation of offset table
142// for all the non-kernel LDS accesses.
143struct NonKernelLDSParameters {
144 GlobalVariable *LDSBaseTable = nullptr;
145 GlobalVariable *LDSOffsetTable = nullptr;
146 SetVector<Function *> OrderedKernels;
147 SetVector<GlobalVariable *> OrdereLDSGlobals;
148};
149
150struct AsanInstrumentInfo {
151 int Scale = 0;
152 uint32_t Offset = 0;
153 SetVector<Instruction *> Instructions;
154};
155
156struct FunctionsAndLDSAccess {
157 MapVector<Function *, KernelLDSParameters> KernelToLDSParametersMap;
158 SetVector<Function *> KernelsWithIndirectLDSAccess;
159 SetVector<Function *> NonKernelsWithLDSInstructions;
160 SetVector<GlobalVariable *> AllNonKernelLDSAccess;
161 FunctionVariableMap NonKernelToLDSAccessMap;
162};
163
164class AMDGPUSwLowerLDS {
165public:
166 AMDGPUSwLowerLDS(Module &Mod, DomTreeCallback Callback)
167 : M(Mod), IRB(M), DTCallback(Callback) {}
168 bool run();
169 void getUsesOfLDSByNonKernels();
170 void getNonKernelsWithLDSInstructions(const CallGraph &CG);
171 SetVector<Function *>
172 getOrderedIndirectLDSAccessingKernels(SetVector<Function *> &Kernels);
173 SetVector<GlobalVariable *>
174 getOrderedNonKernelAllLDSGlobals(SetVector<GlobalVariable *> &Variables);
175 void buildSwLDSGlobal(Function *Func);
176 void buildSwDynLDSGlobal(Function *Func);
177 void populateSwMetadataGlobal(Function *Func);
178 void populateSwLDSAttributeAndMetadata(Function *Func);
179 void populateLDSToReplacementIndicesMap(Function *Func);
180 void getLDSMemoryInstructions(Function *Func,
181 SetVector<Instruction *> &LDSInstructions);
182 void replaceKernelLDSAccesses(Function *Func);
183 Value *getTranslatedGlobalMemoryPtrOfLDS(Value *LoadMallocPtr, Value *LDSPtr);
184 Value *getTranslatedLDSPtrOfFlat(Value *LoadMallocPtr, Value *FlatPtr,
185 Type *LDSPtrTy);
186 void translateLDSMemoryOperationsToGlobalMemory(
187 Function *Func, Value *LoadMallocPtr,
188 SetVector<Instruction *> &LDSInstructions);
189 void poisonRedzones(Function *Func, Value *MallocPtr);
190 void lowerKernelLDSAccesses(Function *Func, DomTreeUpdater &DTU);
191 void buildNonKernelLDSOffsetTable(NonKernelLDSParameters &NKLDSParams);
192 void buildNonKernelLDSBaseTable(NonKernelLDSParameters &NKLDSParams);
193 Constant *
194 getAddressesOfVariablesInKernel(Function *Func,
195 SetVector<GlobalVariable *> &Variables);
196 void lowerNonKernelLDSAccesses(Function *Func,
197 SetVector<GlobalVariable *> &LDSGlobals,
198 NonKernelLDSParameters &NKLDSParams);
199 void
200 updateMallocSizeForDynamicLDS(Function *Func, Value **CurrMallocSize,
201 Value *HiddenDynLDSSize,
202 SetVector<GlobalVariable *> &DynamicLDSGlobals);
203 void initAsanInfo();
204
205private:
206 Module &M;
207 IRBuilder<> IRB;
208 DomTreeCallback DTCallback;
209 FunctionsAndLDSAccess FuncLDSAccessInfo;
210 AsanInstrumentInfo AsanInfo;
211};
212
213template <typename T> SetVector<T> sortByName(std::vector<T> &&V) {
214 // Sort the vector of globals or Functions based on their name.
215 // Returns a SetVector of globals/Functions.
216 sort(V, [](const auto *L, const auto *R) {
217 return L->getName() < R->getName();
218 });
219 return {SetVector<T>(llvm::from_range, V)};
220}
221
222SetVector<GlobalVariable *> AMDGPUSwLowerLDS::getOrderedNonKernelAllLDSGlobals(
223 SetVector<GlobalVariable *> &Variables) {
224 // Sort all the non-kernel LDS accesses based on their name.
225 return sortByName(
226 V: std::vector<GlobalVariable *>(Variables.begin(), Variables.end()));
227}
228
229SetVector<Function *> AMDGPUSwLowerLDS::getOrderedIndirectLDSAccessingKernels(
230 SetVector<Function *> &Kernels) {
231 // Sort the non-kernels accessing LDS based on their name.
232 // Also assign a kernel ID metadata based on the sorted order.
233 LLVMContext &Ctx = M.getContext();
234 if (Kernels.size() > UINT32_MAX) {
235 report_fatal_error(reason: "Unimplemented SW LDS lowering for > 2**32 kernels");
236 }
237 SetVector<Function *> OrderedKernels =
238 sortByName(V: std::vector<Function *>(Kernels.begin(), Kernels.end()));
239 for (size_t i = 0; i < Kernels.size(); i++) {
240 Metadata *AttrMDArgs[1] = {
241 ConstantAsMetadata::get(C: IRB.getInt32(C: i)),
242 };
243 Function *Func = OrderedKernels[i];
244 Func->setMetadata(Kind: "llvm.amdgcn.lds.kernel.id",
245 Node: MDNode::get(Context&: Ctx, MDs: AttrMDArgs));
246 }
247 return OrderedKernels;
248}
249
250// LDS memory operations and casts between flat and LDS.
251static bool isLDSMemoryInstruction(const Instruction &Inst) {
252 if (auto *LI = dyn_cast<LoadInst>(Val: &Inst))
253 return LI->getPointerAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
254 if (auto *SI = dyn_cast<StoreInst>(Val: &Inst))
255 return SI->getPointerAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
256 if (auto *RMW = dyn_cast<AtomicRMWInst>(Val: &Inst))
257 return RMW->getPointerAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
258 if (auto *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: &Inst))
259 return XCHG->getPointerAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
260 if (auto *ASC = dyn_cast<AddrSpaceCastInst>(Val: &Inst)) {
261 unsigned SrcAS = ASC->getSrcAddressSpace();
262 unsigned DstAS = ASC->getDestAddressSpace();
263 return (SrcAS == AMDGPUAS::LOCAL_ADDRESS &&
264 DstAS == AMDGPUAS::FLAT_ADDRESS) ||
265 (SrcAS == AMDGPUAS::FLAT_ADDRESS &&
266 DstAS == AMDGPUAS::LOCAL_ADDRESS);
267 }
268 if (auto *MI = dyn_cast<AnyMemIntrinsic>(Val: &Inst)) {
269 if (MI->getDestAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
270 return true;
271 auto *MTI = dyn_cast<AnyMemTransferInst>(Val: MI);
272 return MTI && MTI->getSourceAddressSpace() == AMDGPUAS::LOCAL_ADDRESS;
273 }
274 return false;
275}
276
277void AMDGPUSwLowerLDS::getNonKernelsWithLDSInstructions(const CallGraph &CG) {
278 // Non-kernels with LDS instructions find the malloc buffer through the base
279 // table, so every kernel reaching them needs a kernel ID, even one without
280 // LDS.
281 DenseMap<Function *, bool> HasLDSInstructionsCache;
282 auto HasLDSInstructions = [&](Function *F) {
283 auto [It, Inserted] = HasLDSInstructionsCache.try_emplace(Key: F, Args: false);
284 if (Inserted) {
285 It->second = any_of(Range: instructions(F&: *F), P: isLDSMemoryInstruction);
286 if (It->second)
287 FuncLDSAccessInfo.NonKernelsWithLDSInstructions.insert(X: F);
288 }
289 return It->second;
290 };
291
292 // Indirect calls may reach any address taken non-kernel.
293 SmallVector<Function *> AddressTakenFuncs;
294 for (Function &F : M)
295 if (!F.isDeclaration() && !isKernel(F) &&
296 F.hasAddressTaken(nullptr, /*IgnoreCallbackUses=*/false,
297 /*IgnoreAssumeLikeCalls=*/false,
298 /*IgnoreLLVMUsed=*/IngoreLLVMUsed: true))
299 AddressTakenFuncs.push_back(Elt: &F);
300
301 for (Function &Kernel : M) {
302 if (Kernel.isDeclaration() || !isKernel(F: Kernel))
303 continue;
304
305 bool ReachesLDSInstructions = false;
306 bool SeenUnknownCall = false;
307 SmallVector<Function *> WorkList = {&Kernel};
308 SmallPtrSet<Function *, 8> Visited;
309 auto Visit = [&](Function *Callee) {
310 if (Callee->isDeclaration() || isKernel(F: *Callee) ||
311 !Visited.insert(Ptr: Callee).second)
312 return;
313 ReachesLDSInstructions |= HasLDSInstructions(Callee);
314 WorkList.push_back(Elt: Callee);
315 };
316
317 while (!WorkList.empty()) {
318 Function *F = WorkList.pop_back_val();
319 for (const CallGraphNode::CallRecord &CallRecord : *CG[F]) {
320 if (!CallRecord.second)
321 continue;
322 if (Function *Callee = CallRecord.second->getFunction()) {
323 Visit(Callee);
324 continue;
325 }
326 if (SeenUnknownCall)
327 continue;
328 SeenUnknownCall = true;
329 for (Function *PotentialCallee : AddressTakenFuncs)
330 Visit(PotentialCallee);
331 }
332 }
333
334 if (ReachesLDSInstructions)
335 FuncLDSAccessInfo.KernelsWithIndirectLDSAccess.insert(X: &Kernel);
336 }
337}
338
339void AMDGPUSwLowerLDS::getUsesOfLDSByNonKernels() {
340 for (GlobalVariable *GV : FuncLDSAccessInfo.AllNonKernelLDSAccess) {
341 if (!AMDGPU::isLDSVariableToLower(GV: *GV))
342 continue;
343
344 for (User *V : GV->users()) {
345 if (auto *I = dyn_cast<Instruction>(Val: V)) {
346 Function *F = I->getFunction();
347 if (!isKernel(F: *F) && !F->isDeclaration())
348 FuncLDSAccessInfo.NonKernelToLDSAccessMap[F].insert(V: GV);
349 }
350 }
351 }
352}
353
354static void recordLDSAbsoluteAddress(Module &M, GlobalVariable *GV,
355 uint32_t Address) {
356 // Write the specified address into metadata where it can be retrieved by
357 // the assembler. Format is a half open range, [Address Address+1)
358 LLVMContext &Ctx = M.getContext();
359 auto *IntTy = M.getDataLayout().getIntPtrType(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS);
360 MDBuilder MDB(Ctx);
361 MDNode *MetadataNode = MDB.createRange(Lo: ConstantInt::get(Ty: IntTy, V: Address),
362 Hi: ConstantInt::get(Ty: IntTy, V: Address + 1));
363 GV->setMetadata(KindID: LLVMContext::MD_absolute_symbol, Node: MetadataNode);
364}
365
366static void addLDSSizeAttribute(Function *Func, uint32_t Offset,
367 bool IsDynLDS) {
368 if (Offset != 0) {
369 std::string Buffer;
370 raw_string_ostream SS{Buffer};
371 SS << Offset;
372 if (IsDynLDS)
373 SS << "," << Offset;
374 Func->addFnAttr(Kind: "amdgpu-lds-size", Val: Buffer);
375 }
376}
377
378static void markUsedByKernel(Function *Func, GlobalVariable *SGV) {
379 BasicBlock *Entry = &Func->getEntryBlock();
380 IRBuilder<> Builder(Entry->getFirstNonPHIIt());
381
382 Function *Decl = Intrinsic::getOrInsertDeclaration(M: Func->getParent(),
383 id: Intrinsic::donothing, OverloadTys: {});
384
385 Value *UseInstance[] = {SGV};
386
387 Builder.CreateCall(Callee: Decl, Args: {},
388 OpBundles: {OperandBundleDefT<Value *>("ExplicitUse", UseInstance)});
389}
390
391void AMDGPUSwLowerLDS::buildSwLDSGlobal(Function *Func) {
392 // Create new LDS global required for each kernel to store
393 // device global memory pointer.
394 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
395 // Create new global pointer variable
396 LDSParams.SwLDS = new GlobalVariable(
397 M, IRB.getPtrTy(), false, GlobalValue::InternalLinkage,
398 PoisonValue::get(T: IRB.getPtrTy()), "llvm.amdgcn.sw.lds." + Func->getName(),
399 nullptr, GlobalValue::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS, false);
400 GlobalValue::SanitizerMetadata MD;
401 MD.NoAddress = true;
402 LDSParams.SwLDS->setSanitizerMetadata(MD);
403}
404
405void AMDGPUSwLowerLDS::buildSwDynLDSGlobal(Function *Func) {
406 // Create new Dyn LDS global if kernel accesses dyn LDS.
407 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
408 if (LDSParams.DirectAccess.DynamicLDSGlobals.empty() &&
409 LDSParams.IndirectAccess.DynamicLDSGlobals.empty())
410 return;
411 // Create new global pointer variable
412 auto *emptyCharArray = ArrayType::get(ElementType: IRB.getInt8Ty(), NumElements: 0);
413 LDSParams.SwDynLDS = new GlobalVariable(
414 M, emptyCharArray, false, GlobalValue::ExternalLinkage, nullptr,
415 "llvm.amdgcn." + Func->getName() + ".dynlds", nullptr,
416 GlobalValue::NotThreadLocal, AMDGPUAS::LOCAL_ADDRESS, false);
417 markUsedByKernel(Func, SGV: LDSParams.SwDynLDS);
418 GlobalValue::SanitizerMetadata MD;
419 MD.NoAddress = true;
420 LDSParams.SwDynLDS->setSanitizerMetadata(MD);
421}
422
423void AMDGPUSwLowerLDS::populateSwLDSAttributeAndMetadata(Function *Func) {
424 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
425 bool IsDynLDSUsed = LDSParams.SwDynLDS;
426 uint32_t Offset = LDSParams.LDSSize;
427 recordLDSAbsoluteAddress(M, GV: LDSParams.SwLDS, Address: 0);
428 addLDSSizeAttribute(Func, Offset, IsDynLDS: IsDynLDSUsed);
429 if (LDSParams.SwDynLDS)
430 recordLDSAbsoluteAddress(M, GV: LDSParams.SwDynLDS, Address: Offset);
431}
432
433void AMDGPUSwLowerLDS::populateSwMetadataGlobal(Function *Func) {
434 // Create new metadata global for every kernel and initialize the
435 // start offsets and sizes corresponding to each LDS accesses.
436 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
437 auto &Ctx = M.getContext();
438 auto &DL = M.getDataLayout();
439 std::vector<Type *> Items;
440 Type *Int32Ty = IRB.getInt32Ty();
441 std::vector<Constant *> Initializers;
442 Align MaxAlignment(1);
443 auto UpdateMaxAlignment = [&MaxAlignment, &DL](GlobalVariable *GV) {
444 Align GVAlign = AMDGPU::getAlign(DL, GV);
445 MaxAlignment = std::max(a: MaxAlignment, b: GVAlign);
446 };
447
448 for (GlobalVariable *GV : LDSParams.DirectAccess.StaticLDSGlobals)
449 UpdateMaxAlignment(GV);
450
451 for (GlobalVariable *GV : LDSParams.DirectAccess.DynamicLDSGlobals)
452 UpdateMaxAlignment(GV);
453
454 for (GlobalVariable *GV : LDSParams.IndirectAccess.StaticLDSGlobals)
455 UpdateMaxAlignment(GV);
456
457 for (GlobalVariable *GV : LDSParams.IndirectAccess.DynamicLDSGlobals)
458 UpdateMaxAlignment(GV);
459
460 //{StartOffset, AlignedSizeInBytes}
461 SmallString<128> MDItemStr;
462 raw_svector_ostream MDItemOS(MDItemStr);
463 MDItemOS << "llvm.amdgcn.sw.lds." << Func->getName() << ".md.item";
464
465 StructType *LDSItemTy =
466 StructType::create(Context&: Ctx, Elements: {Int32Ty, Int32Ty, Int32Ty}, Name: MDItemOS.str());
467 uint32_t &MallocSize = LDSParams.MallocSize;
468 SetVector<GlobalVariable *> UniqueLDSGlobals;
469 int AsanScale = AsanInfo.Scale;
470 auto buildInitializerForSwLDSMD =
471 [&](SetVector<GlobalVariable *> &LDSGlobals) {
472 for (auto &GV : LDSGlobals) {
473 if (is_contained(Range&: UniqueLDSGlobals, Element: GV))
474 continue;
475 UniqueLDSGlobals.insert(X: GV);
476
477 Type *Ty = GV->getValueType();
478 const uint64_t SizeInBytes = DL.getTypeAllocSize(Ty);
479 Items.push_back(x: LDSItemTy);
480 Constant *ItemStartOffset = ConstantInt::get(Ty: Int32Ty, V: MallocSize);
481 Constant *SizeInBytesConst = ConstantInt::get(Ty: Int32Ty, V: SizeInBytes);
482 // Get redzone size corresponding a size.
483 const uint64_t RightRedzoneSize =
484 AMDGPU::getRedzoneSizeForGlobal(Scale: AsanScale, SizeInBytes);
485 // Update MallocSize with current size and redzone size.
486 MallocSize += SizeInBytes;
487 if (!AMDGPU::isDynamicLDS(GV: *GV))
488 LDSParams.RedzoneOffsetAndSizeVector.emplace_back(Args&: MallocSize,
489 Args: RightRedzoneSize);
490 MallocSize += RightRedzoneSize;
491 // Align current size plus redzone.
492 uint64_t AlignedSize =
493 alignTo(Size: SizeInBytes + RightRedzoneSize, A: MaxAlignment);
494 Constant *AlignedSizeInBytesConst =
495 ConstantInt::get(Ty: Int32Ty, V: AlignedSize);
496 // Align MallocSize
497 MallocSize = alignTo(Size: MallocSize, A: MaxAlignment);
498 Constant *InitItem =
499 ConstantStruct::get(T: LDSItemTy, V: {ItemStartOffset, SizeInBytesConst,
500 AlignedSizeInBytesConst});
501 Initializers.push_back(x: InitItem);
502 }
503 };
504 SetVector<GlobalVariable *> SwLDSVector;
505 SwLDSVector.insert(X: LDSParams.SwLDS);
506 buildInitializerForSwLDSMD(SwLDSVector);
507 buildInitializerForSwLDSMD(LDSParams.DirectAccess.StaticLDSGlobals);
508 buildInitializerForSwLDSMD(LDSParams.IndirectAccess.StaticLDSGlobals);
509 buildInitializerForSwLDSMD(LDSParams.DirectAccess.DynamicLDSGlobals);
510 buildInitializerForSwLDSMD(LDSParams.IndirectAccess.DynamicLDSGlobals);
511
512 // Update the LDS size used by the kernel.
513 Type *Ty = LDSParams.SwLDS->getValueType();
514 const uint64_t SizeInBytes = DL.getTypeAllocSize(Ty);
515 uint64_t AlignedSize = alignTo(Size: SizeInBytes, A: MaxAlignment);
516 LDSParams.LDSSize = AlignedSize;
517 SmallString<128> MDTypeStr;
518 raw_svector_ostream MDTypeOS(MDTypeStr);
519 MDTypeOS << "llvm.amdgcn.sw.lds." << Func->getName() << ".md.type";
520 StructType *MetadataStructType =
521 StructType::create(Context&: Ctx, Elements: Items, Name: MDTypeOS.str());
522 SmallString<128> MDStr;
523 raw_svector_ostream MDOS(MDStr);
524 MDOS << "llvm.amdgcn.sw.lds." << Func->getName() << ".md";
525 LDSParams.SwLDSMetadata = new GlobalVariable(
526 M, MetadataStructType, false, GlobalValue::InternalLinkage,
527 PoisonValue::get(T: MetadataStructType), MDOS.str(), nullptr,
528 GlobalValue::NotThreadLocal, AMDGPUAS::GLOBAL_ADDRESS, false);
529 Constant *data = ConstantStruct::get(T: MetadataStructType, V: Initializers);
530 LDSParams.SwLDSMetadata->setInitializer(data);
531 assert(LDSParams.SwLDS);
532 // Set the alignment to MaxAlignment for SwLDS.
533 LDSParams.SwLDS->setAlignment(MaxAlignment);
534 if (LDSParams.SwDynLDS)
535 LDSParams.SwDynLDS->setAlignment(MaxAlignment);
536 GlobalValue::SanitizerMetadata MD;
537 MD.NoAddress = true;
538 LDSParams.SwLDSMetadata->setSanitizerMetadata(MD);
539}
540
541void AMDGPUSwLowerLDS::populateLDSToReplacementIndicesMap(Function *Func) {
542 // Fill the corresponding LDS replacement indices for each LDS access
543 // related to this kernel.
544 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
545 SetVector<GlobalVariable *> UniqueLDSGlobals;
546 auto PopulateIndices = [&](SetVector<GlobalVariable *> &LDSGlobals,
547 uint32_t &Idx) {
548 for (auto &GV : LDSGlobals) {
549 if (is_contained(Range&: UniqueLDSGlobals, Element: GV))
550 continue;
551 UniqueLDSGlobals.insert(X: GV);
552 LDSParams.LDSToReplacementIndicesMap[GV] = {0, Idx, 0};
553 ++Idx;
554 }
555 };
556 uint32_t Idx = 0;
557 SetVector<GlobalVariable *> SwLDSVector;
558 SwLDSVector.insert(X: LDSParams.SwLDS);
559 PopulateIndices(SwLDSVector, Idx);
560 PopulateIndices(LDSParams.DirectAccess.StaticLDSGlobals, Idx);
561 PopulateIndices(LDSParams.IndirectAccess.StaticLDSGlobals, Idx);
562 PopulateIndices(LDSParams.DirectAccess.DynamicLDSGlobals, Idx);
563 PopulateIndices(LDSParams.IndirectAccess.DynamicLDSGlobals, Idx);
564}
565
566static void replacesUsesOfGlobalInFunction(Function *Func, GlobalVariable *GV,
567 Value *Replacement) {
568 // Replace all uses of LDS global in this Function with a Replacement.
569 auto ReplaceUsesLambda = [Func](const Use &U) -> bool {
570 auto *V = U.getUser();
571 if (auto *Inst = dyn_cast<Instruction>(Val: V)) {
572 auto *Func1 = Inst->getFunction();
573 if (Func == Func1)
574 return true;
575 }
576 return false;
577 };
578 GV->replaceUsesWithIf(New: Replacement, ShouldReplace: ReplaceUsesLambda);
579}
580
581void AMDGPUSwLowerLDS::replaceKernelLDSAccesses(Function *Func) {
582 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
583 GlobalVariable *SwLDS = LDSParams.SwLDS;
584 assert(SwLDS);
585 GlobalVariable *SwLDSMetadata = LDSParams.SwLDSMetadata;
586 assert(SwLDSMetadata);
587 StructType *SwLDSMetadataStructType =
588 cast<StructType>(Val: SwLDSMetadata->getValueType());
589 Type *Int32Ty = IRB.getInt32Ty();
590 auto &IndirectAccess = LDSParams.IndirectAccess;
591 auto &DirectAccess = LDSParams.DirectAccess;
592 // Replace all uses of LDS global in this Function with a Replacement.
593 SetVector<GlobalVariable *> UniqueLDSGlobals;
594 auto ReplaceLDSGlobalUses = [&](SetVector<GlobalVariable *> &LDSGlobals) {
595 for (auto &GV : LDSGlobals) {
596 // Do not generate instructions if LDS access is in non-kernel
597 // i.e indirect-access.
598 if ((IndirectAccess.StaticLDSGlobals.contains(key: GV) ||
599 IndirectAccess.DynamicLDSGlobals.contains(key: GV)) &&
600 (!DirectAccess.StaticLDSGlobals.contains(key: GV) &&
601 !DirectAccess.DynamicLDSGlobals.contains(key: GV)))
602 continue;
603 if (is_contained(Range&: UniqueLDSGlobals, Element: GV))
604 continue;
605 UniqueLDSGlobals.insert(X: GV);
606 auto &Indices = LDSParams.LDSToReplacementIndicesMap[GV];
607 assert(Indices.size() == 3);
608 Constant *GEPIdx[] = {ConstantInt::get(Ty: Int32Ty, V: Indices[0]),
609 ConstantInt::get(Ty: Int32Ty, V: Indices[1]),
610 ConstantInt::get(Ty: Int32Ty, V: Indices[2])};
611 Constant *GEP = ConstantExpr::getGetElementPtr(
612 DL: Func->getDataLayout(), Ty: SwLDSMetadataStructType, C: SwLDSMetadata, IdxList: GEPIdx,
613 NW: GEPNoWrapFlags::inBounds());
614 Value *Offset = IRB.CreateLoad(Ty: Int32Ty, Ptr: GEP);
615 Value *BasePlusOffset =
616 IRB.CreateInBoundsGEP(Ty: IRB.getInt8Ty(), Ptr: SwLDS, IdxList: {Offset});
617 LLVM_DEBUG(GV->printAsOperand(dbgs() << "Sw LDS Lowering, Replacing LDS ",
618 false));
619 replacesUsesOfGlobalInFunction(Func, GV, Replacement: BasePlusOffset);
620 }
621 };
622 ReplaceLDSGlobalUses(DirectAccess.StaticLDSGlobals);
623 ReplaceLDSGlobalUses(IndirectAccess.StaticLDSGlobals);
624 ReplaceLDSGlobalUses(DirectAccess.DynamicLDSGlobals);
625 ReplaceLDSGlobalUses(IndirectAccess.DynamicLDSGlobals);
626}
627
628void AMDGPUSwLowerLDS::updateMallocSizeForDynamicLDS(
629 Function *Func, Value **CurrMallocSize, Value *HiddenDynLDSSize,
630 SetVector<GlobalVariable *> &DynamicLDSGlobals) {
631 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
632 Type *Int32Ty = IRB.getInt32Ty();
633
634 GlobalVariable *SwLDS = LDSParams.SwLDS;
635 GlobalVariable *SwLDSMetadata = LDSParams.SwLDSMetadata;
636 assert(SwLDS && SwLDSMetadata);
637 StructType *MetadataStructType =
638 cast<StructType>(Val: SwLDSMetadata->getValueType());
639 unsigned MaxAlignment = SwLDS->getAlign().valueOrOne().value();
640 Value *MaxAlignValue = IRB.getInt32(C: MaxAlignment);
641 Value *MaxAlignValueMinusOne = IRB.getInt32(C: MaxAlignment - 1);
642
643 for (GlobalVariable *DynGV : DynamicLDSGlobals) {
644 auto &Indices = LDSParams.LDSToReplacementIndicesMap[DynGV];
645 // Update the Offset metadata.
646 Constant *Index0 = ConstantInt::get(Ty: Int32Ty, V: 0);
647 Constant *Index1 = ConstantInt::get(Ty: Int32Ty, V: Indices[1]);
648
649 Constant *Index2Offset = ConstantInt::get(Ty: Int32Ty, V: 0);
650 auto *GEPForOffset = IRB.CreateInBoundsGEP(
651 Ty: MetadataStructType, Ptr: SwLDSMetadata, IdxList: {Index0, Index1, Index2Offset});
652
653 IRB.CreateStore(Val: *CurrMallocSize, Ptr: GEPForOffset);
654 // Update the size and Aligned Size metadata.
655 Constant *Index2Size = ConstantInt::get(Ty: Int32Ty, V: 1);
656 auto *GEPForSize = IRB.CreateInBoundsGEP(Ty: MetadataStructType, Ptr: SwLDSMetadata,
657 IdxList: {Index0, Index1, Index2Size});
658
659 Value *CurrDynLDSSize = IRB.CreateLoad(Ty: Int32Ty, Ptr: HiddenDynLDSSize);
660 IRB.CreateStore(Val: CurrDynLDSSize, Ptr: GEPForSize);
661 Constant *Index2AlignedSize = ConstantInt::get(Ty: Int32Ty, V: 2);
662 auto *GEPForAlignedSize = IRB.CreateInBoundsGEP(
663 Ty: MetadataStructType, Ptr: SwLDSMetadata, IdxList: {Index0, Index1, Index2AlignedSize});
664
665 Value *AlignedDynLDSSize =
666 IRB.CreateAdd(LHS: CurrDynLDSSize, RHS: MaxAlignValueMinusOne);
667 AlignedDynLDSSize = IRB.CreateUDiv(LHS: AlignedDynLDSSize, RHS: MaxAlignValue);
668 AlignedDynLDSSize = IRB.CreateMul(LHS: AlignedDynLDSSize, RHS: MaxAlignValue);
669 IRB.CreateStore(Val: AlignedDynLDSSize, Ptr: GEPForAlignedSize);
670
671 // Update the Current Malloc Size
672 *CurrMallocSize = IRB.CreateAdd(LHS: *CurrMallocSize, RHS: AlignedDynLDSSize);
673 }
674}
675
676static DebugLoc getOrCreateDebugLoc(const Instruction *InsertBefore,
677 DISubprogram *SP) {
678 assert(InsertBefore);
679 if (InsertBefore->getDebugLoc())
680 return InsertBefore->getDebugLoc();
681 if (SP)
682 return DILocation::get(Context&: SP->getContext(), Line: SP->getLine(), Column: 1, Scope: SP);
683 return DebugLoc();
684}
685
686void AMDGPUSwLowerLDS::getLDSMemoryInstructions(
687 Function *Func, SetVector<Instruction *> &LDSInstructions) {
688 for (Instruction &Inst : instructions(F: Func))
689 if (isLDSMemoryInstruction(Inst))
690 LDSInstructions.insert(X: &Inst);
691}
692
693Value *AMDGPUSwLowerLDS::getTranslatedGlobalMemoryPtrOfLDS(Value *LoadMallocPtr,
694 Value *LDSPtr) {
695 assert(LDSPtr && "Invalid LDS pointer operand");
696 Type *LDSPtrType = LDSPtr->getType();
697 LLVMContext &Ctx = M.getContext();
698 const DataLayout &DL = M.getDataLayout();
699 Type *IntTy = DL.getIntPtrType(C&: Ctx, AddressSpace: AMDGPUAS::LOCAL_ADDRESS);
700 if (auto *VecPtrTy = dyn_cast<VectorType>(Val: LDSPtrType)) {
701 // Handle vector of pointers
702 ElementCount NumElements = VecPtrTy->getElementCount();
703 IntTy = VectorType::get(ElementType: IntTy, EC: NumElements);
704 }
705 Value *GepIndex = IRB.CreatePtrToInt(V: LDSPtr, DestTy: IntTy);
706 return IRB.CreateInBoundsGEP(Ty: IRB.getInt8Ty(), Ptr: LoadMallocPtr, IdxList: {GepIndex});
707}
708
709// A flat pointer to lowered LDS holds a global address inside the malloc
710// buffer, but LDS values in this pass are offsets into that buffer.
711// Rebase it as inttoptr(trunc(ptrtoaddr(flat) - ptrtoaddr(buf))).
712Value *AMDGPUSwLowerLDS::getTranslatedLDSPtrOfFlat(Value *LoadMallocPtr,
713 Value *FlatPtr,
714 Type *LDSPtrTy) {
715 const DataLayout &DL = M.getDataLayout();
716 Value *FlatAddr = IRB.CreatePtrToAddr(V: FlatPtr);
717 Value *Base = IRB.CreatePtrToAddr(V: LoadMallocPtr);
718 if (auto *VecTy = dyn_cast<VectorType>(Val: FlatAddr->getType()))
719 Base = IRB.CreateVectorSplat(EC: VecTy->getElementCount(), V: Base);
720 Value *Diff = IRB.CreateSub(LHS: FlatAddr, RHS: Base);
721 Value *Offset = IRB.CreateTrunc(V: Diff, DestTy: DL.getAddressType(PtrTy: LDSPtrTy));
722 Value *LDSPtr = IRB.CreateIntToPtr(V: Offset, DestTy: LDSPtrTy);
723 // Flat null is 0 but LDS null is -1, which the rebased offset cannot keep.
724 Constant *LDSNull = ConstantExpr::getAddrSpaceCast(
725 C: Constant::getNullValue(Ty: FlatPtr->getType()), Ty: LDSPtrTy);
726 return IRB.CreateSelect(C: IRB.CreateIsNull(Arg: FlatPtr), True: LDSNull, False: LDSPtr);
727}
728
729void AMDGPUSwLowerLDS::translateLDSMemoryOperationsToGlobalMemory(
730 Function *Func, Value *LoadMallocPtr,
731 SetVector<Instruction *> &LDSInstructions) {
732 LLVM_DEBUG(dbgs() << "Translating LDS memory operations to global memory : "
733 << Func->getName());
734 for (Instruction *Inst : LDSInstructions) {
735 IRB.SetInsertPoint(Inst);
736 if (LoadInst *LI = dyn_cast<LoadInst>(Val: Inst)) {
737 Value *LIOperand = LI->getPointerOperand();
738 Value *Replacement =
739 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, LDSPtr: LIOperand);
740 LoadInst *NewLI =
741 IRB.CreateLoad(Ty: LI->getType(), Ptr: Replacement, Props: LI->getProperties());
742 AsanInfo.Instructions.insert(X: NewLI);
743 LI->replaceAllUsesWith(V: NewLI);
744 LI->eraseFromParent();
745 } else if (StoreInst *SI = dyn_cast<StoreInst>(Val: Inst)) {
746 Value *SIOperand = SI->getPointerOperand();
747 Value *Replacement =
748 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, LDSPtr: SIOperand);
749 StoreInst *NewSI = IRB.CreateStore(Val: SI->getValueOperand(), Ptr: Replacement,
750 Props: SI->getProperties());
751 AsanInfo.Instructions.insert(X: NewSI);
752 SI->replaceAllUsesWith(V: NewSI);
753 SI->eraseFromParent();
754 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(Val: Inst)) {
755 Value *RMWPtrOperand = RMW->getPointerOperand();
756 Value *RMWValOperand = RMW->getValOperand();
757 Value *Replacement =
758 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, LDSPtr: RMWPtrOperand);
759 AtomicRMWInst *NewRMW = IRB.CreateAtomicRMW(
760 Op: RMW->getOperation(), Ptr: Replacement, Val: RMWValOperand, Align: RMW->getAlign(),
761 Ordering: RMW->getOrdering(), SSID: RMW->getSyncScopeID());
762 NewRMW->setVolatile(RMW->isVolatile());
763 AsanInfo.Instructions.insert(X: NewRMW);
764 RMW->replaceAllUsesWith(V: NewRMW);
765 RMW->eraseFromParent();
766 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(Val: Inst)) {
767 Value *XCHGPtrOperand = XCHG->getPointerOperand();
768 Value *Replacement =
769 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, LDSPtr: XCHGPtrOperand);
770 AtomicCmpXchgInst *NewXCHG = IRB.CreateAtomicCmpXchg(
771 Ptr: Replacement, Cmp: XCHG->getCompareOperand(), New: XCHG->getNewValOperand(),
772 Align: XCHG->getAlign(), SuccessOrdering: XCHG->getSuccessOrdering(),
773 FailureOrdering: XCHG->getFailureOrdering(), SSID: XCHG->getSyncScopeID());
774 NewXCHG->setVolatile(XCHG->isVolatile());
775 AsanInfo.Instructions.insert(X: NewXCHG);
776 XCHG->replaceAllUsesWith(V: NewXCHG);
777 XCHG->eraseFromParent();
778 } else if (AnyMemIntrinsic *MI = dyn_cast<AnyMemIntrinsic>(Val: Inst)) {
779 Value *NewDest = MI->getRawDest();
780 if (MI->getDestAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
781 NewDest = getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, LDSPtr: NewDest);
782 CallInst *NewMI = nullptr;
783 if (AnyMemSetInst *MSI = dyn_cast<AnyMemSetInst>(Val: MI)) {
784 if (MI->isAtomic()) {
785 NewMI = IRB.CreateElementUnorderedAtomicMemSet(
786 Ptr: NewDest, Val: MSI->getValue(), Size: MSI->getLength(),
787 Alignment: MSI->getDestAlign().valueOrOne(), ElementSize: MSI->getElementSizeInBytes());
788 } else {
789 NewMI = IRB.CreateMemSet(Ptr: NewDest, Val: MSI->getValue(), Size: MSI->getLength(),
790 Align: MSI->getDestAlign(),
791 isVolatile: cast<MemSetInst>(Val: MI)->isVolatile());
792 }
793 } else if (AnyMemTransferInst *MTI = dyn_cast<AnyMemTransferInst>(Val: MI)) {
794 Value *NewSrc = MTI->getRawSource();
795 if (MTI->getSourceAddressSpace() == AMDGPUAS::LOCAL_ADDRESS)
796 NewSrc = getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, LDSPtr: NewSrc);
797 if (MI->isAtomic()) {
798 if (MI->getIntrinsicID() ==
799 Intrinsic::memmove_element_unordered_atomic) {
800 NewMI = IRB.CreateElementUnorderedAtomicMemMove(
801 Dst: NewDest, DstAlign: MTI->getDestAlign().valueOrOne(), Src: NewSrc,
802 SrcAlign: MTI->getSourceAlign().valueOrOne(), Size: MTI->getLength(),
803 ElementSize: MTI->getElementSizeInBytes());
804 } else {
805 NewMI = IRB.CreateElementUnorderedAtomicMemCpy(
806 Dst: NewDest, DstAlign: MTI->getDestAlign().valueOrOne(), Src: NewSrc,
807 SrcAlign: MTI->getSourceAlign().valueOrOne(), Size: MTI->getLength(),
808 ElementSize: MTI->getElementSizeInBytes());
809 }
810 } else {
811 NewMI = IRB.CreateMemTransferInst(
812 IntrID: MI->getIntrinsicID(), Dst: NewDest, DstAlign: MTI->getDestAlign(), Src: NewSrc,
813 SrcAlign: MTI->getSourceAlign(), Size: MTI->getLength(),
814 isVolatile: cast<MemTransferInst>(Val: MI)->isVolatile());
815 }
816 } else
817 reportFatalUsageError(reason: "Unimplemented LDS lowering memory intrinsic");
818 AsanInfo.Instructions.insert(X: NewMI);
819 MI->replaceAllUsesWith(V: NewMI);
820 MI->eraseFromParent();
821 } else if (AddrSpaceCastInst *ASC = dyn_cast<AddrSpaceCastInst>(Val: Inst)) {
822 Value *AIOperand = ASC->getPointerOperand();
823 Value *NewAI = nullptr;
824 if (ASC->getDestAddressSpace() == AMDGPUAS::LOCAL_ADDRESS) {
825 NewAI =
826 getTranslatedLDSPtrOfFlat(LoadMallocPtr, FlatPtr: AIOperand, LDSPtrTy: ASC->getType());
827 } else {
828 Value *Replacement =
829 getTranslatedGlobalMemoryPtrOfLDS(LoadMallocPtr, LDSPtr: AIOperand);
830 // Note: No need to add the instruction to AsanInfo instructions to be
831 // instrumented list. FLAT_ADDRESS ptr would have been already
832 // instrumented by asan pass prior to this pass.
833 NewAI = IRB.CreateAddrSpaceCast(V: Replacement, DestTy: ASC->getType());
834 }
835 ASC->replaceAllUsesWith(V: NewAI);
836 ASC->eraseFromParent();
837 } else
838 report_fatal_error(reason: "Unimplemented LDS lowering instruction");
839 }
840}
841
842void AMDGPUSwLowerLDS::poisonRedzones(Function *Func, Value *MallocPtr) {
843 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
844 Type *Int64Ty = IRB.getInt64Ty();
845 Type *VoidTy = IRB.getVoidTy();
846 FunctionCallee AsanPoisonRegion = M.getOrInsertFunction(
847 Name: "__asan_poison_region",
848 T: FunctionType::get(Result: VoidTy, Params: {Int64Ty, Int64Ty}, isVarArg: false));
849
850 auto RedzonesVec = LDSParams.RedzoneOffsetAndSizeVector;
851 size_t VecSize = RedzonesVec.size();
852 for (unsigned i = 0; i < VecSize; i++) {
853 auto &RedzonePair = RedzonesVec[i];
854 uint64_t RedzoneOffset = RedzonePair.first;
855 uint64_t RedzoneSize = RedzonePair.second;
856 Value *RedzoneAddrOffset = IRB.CreateInBoundsGEP(
857 Ty: IRB.getInt8Ty(), Ptr: MallocPtr, IdxList: {IRB.getInt64(C: RedzoneOffset)});
858 Value *RedzoneAddress = IRB.CreatePtrToInt(V: RedzoneAddrOffset, DestTy: Int64Ty);
859 IRB.CreateCall(Callee: AsanPoisonRegion,
860 Args: {RedzoneAddress, IRB.getInt64(C: RedzoneSize)});
861 }
862}
863
864void AMDGPUSwLowerLDS::lowerKernelLDSAccesses(Function *Func,
865 DomTreeUpdater &DTU) {
866 LLVM_DEBUG(dbgs() << "Sw Lowering Kernel LDS for : " << Func->getName());
867 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
868 auto &Ctx = M.getContext();
869 auto *PrevEntryBlock = &Func->getEntryBlock();
870 SetVector<Instruction *> LDSInstructions;
871 getLDSMemoryInstructions(Func, LDSInstructions);
872 const DataLayout &DL = M.getDataLayout();
873
874 // Create malloc block.
875 auto *MallocBlock = BasicBlock::Create(Context&: Ctx, Name: "Malloc", Parent: Func, InsertBefore: PrevEntryBlock);
876
877 // Create WIdBlock block which has instructions related to selection of
878 // {0,0,0} indiex work item in the work group.
879 auto *WIdBlock = BasicBlock::Create(Context&: Ctx, Name: "WId", Parent: Func, InsertBefore: MallocBlock);
880
881 // Move constant-size allocas from the original entry block to the new entry
882 // block (WIdBlock) so they remain static allocas. Splice the leading cluster
883 // in bulk, then move any stragglers that are interleaved with other
884 // instructions.
885 auto SplitIt = PrevEntryBlock->getFirstNonPHIOrAlloca();
886 WIdBlock->splice(ToIt: WIdBlock->end(), FromBB: PrevEntryBlock, FromBeginIt: PrevEntryBlock->begin(),
887 FromEndIt: SplitIt);
888 for (Instruction &I : make_early_inc_range(Range&: *PrevEntryBlock))
889 if (auto *AI = dyn_cast<AllocaInst>(Val: &I))
890 if (isa<ConstantInt>(Val: AI->getArraySize()))
891 AI->moveBefore(BB&: *WIdBlock, I: WIdBlock->end());
892
893 IRB.SetInsertPoint(WIdBlock->end());
894 DebugLoc FirstDL =
895 getOrCreateDebugLoc(InsertBefore: &*PrevEntryBlock->begin(), SP: Func->getSubprogram());
896 IRB.SetCurrentDebugLocation(FirstDL);
897 Value *WIdx = IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_workitem_id_x, Args: {});
898 Value *WIdy = IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_workitem_id_y, Args: {});
899 Value *WIdz = IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_workitem_id_z, Args: {});
900 Value *XYOr = IRB.CreateOr(LHS: WIdx, RHS: WIdy);
901 Value *XYZOr = IRB.CreateOr(LHS: XYOr, RHS: WIdz);
902 Value *WIdzCond = IRB.CreateICmpEQ(LHS: XYZOr, RHS: IRB.getInt32(C: 0));
903
904 // All work items will branch to PrevEntryBlock except {0,0,0} index
905 // work item which will branch to malloc block.
906 IRB.CreateCondBr(Cond: WIdzCond, True: MallocBlock, False: PrevEntryBlock);
907
908 // Malloc block
909 IRB.SetInsertPoint(MallocBlock->begin());
910
911 // If Dynamic LDS globals are accessed by the kernel,
912 // Get the size of dyn lds from hidden dyn_lds_size kernel arg.
913 // Update the corresponding metadata global entries for this dyn lds global.
914 GlobalVariable *SwLDS = LDSParams.SwLDS;
915 GlobalVariable *SwLDSMetadata = LDSParams.SwLDSMetadata;
916 assert(SwLDS && SwLDSMetadata);
917 StructType *MetadataStructType =
918 cast<StructType>(Val: SwLDSMetadata->getValueType());
919 Type *Int32Ty = IRB.getInt32Ty();
920 Type *Int64Ty = IRB.getInt64Ty();
921
922 SetVector<GlobalVariable *> UniqueLDSGlobals;
923 auto GetUniqueLDSGlobals = [&](SetVector<GlobalVariable *> &LDSGlobals) {
924 for (auto &GV : LDSGlobals) {
925 if (is_contained(Range&: UniqueLDSGlobals, Element: GV))
926 continue;
927 UniqueLDSGlobals.insert(X: GV);
928 }
929 };
930
931 GetUniqueLDSGlobals(LDSParams.DirectAccess.StaticLDSGlobals);
932 GetUniqueLDSGlobals(LDSParams.IndirectAccess.StaticLDSGlobals);
933 // The metadata global always has an item for the SwLDS pointer itself, so
934 // there is at least one static item and the last one ends the static region.
935 unsigned LastStaticLDSIdx = UniqueLDSGlobals.size();
936 UniqueLDSGlobals.clear();
937
938 auto *GEPForEndStaticLDSOffset =
939 IRB.CreateInBoundsGEP(Ty: MetadataStructType, Ptr: SwLDSMetadata,
940 IdxList: {ConstantInt::get(Ty: Int32Ty, V: 0),
941 ConstantInt::get(Ty: Int32Ty, V: LastStaticLDSIdx),
942 ConstantInt::get(Ty: Int32Ty, V: 0)});
943
944 auto *GEPForEndStaticLDSSize =
945 IRB.CreateInBoundsGEP(Ty: MetadataStructType, Ptr: SwLDSMetadata,
946 IdxList: {ConstantInt::get(Ty: Int32Ty, V: 0),
947 ConstantInt::get(Ty: Int32Ty, V: LastStaticLDSIdx),
948 ConstantInt::get(Ty: Int32Ty, V: 2)});
949
950 Value *EndStaticLDSOffset = IRB.CreateLoad(Ty: Int32Ty, Ptr: GEPForEndStaticLDSOffset);
951 Value *EndStaticLDSSize = IRB.CreateLoad(Ty: Int32Ty, Ptr: GEPForEndStaticLDSSize);
952 Value *CurrMallocSize = IRB.CreateAdd(LHS: EndStaticLDSOffset, RHS: EndStaticLDSSize);
953
954 if (LDSParams.SwDynLDS) {
955 if (!(AMDGPU::getAMDHSACodeObjectVersion(M) >= AMDGPU::AMDHSA_COV5))
956 report_fatal_error(
957 reason: "Dynamic LDS size query is only supported for CO V5 and later.");
958 // Get size from hidden dyn_lds_size argument of kernel
959 Value *ImplicitArg =
960 IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_implicitarg_ptr, Args: {});
961 Value *HiddenDynLDSSize = IRB.CreateInBoundsGEP(
962 Ty: ImplicitArg->getType(), Ptr: ImplicitArg,
963 IdxList: {ConstantInt::get(Ty: Int64Ty, COV5_HIDDEN_DYN_LDS_SIZE_ARG)});
964 UniqueLDSGlobals.clear();
965 GetUniqueLDSGlobals(LDSParams.DirectAccess.DynamicLDSGlobals);
966 GetUniqueLDSGlobals(LDSParams.IndirectAccess.DynamicLDSGlobals);
967 updateMallocSizeForDynamicLDS(Func, CurrMallocSize: &CurrMallocSize, HiddenDynLDSSize,
968 DynamicLDSGlobals&: UniqueLDSGlobals);
969 }
970
971 CurrMallocSize = IRB.CreateZExt(V: CurrMallocSize, DestTy: Int64Ty);
972
973 // Create a call to malloc function which does device global memory allocation
974 // with size equals to all LDS global accesses size in this kernel.
975 Value *ReturnAddress = IRB.CreateIntrinsic(
976 ID: Intrinsic::returnaddress, OverloadTypes: IRB.getPtrTy(AddrSpace: DL.getProgramAddressSpace()),
977 Args: {IRB.getInt32(C: 0)});
978 FunctionCallee MallocFunc = M.getOrInsertFunction(
979 Name: StringRef("__asan_malloc_impl"),
980 T: FunctionType::get(Result: Int64Ty, Params: {Int64Ty, Int64Ty}, isVarArg: false));
981 Value *RAPtrToInt = IRB.CreatePtrToInt(V: ReturnAddress, DestTy: Int64Ty);
982 Value *MallocCall = IRB.CreateCall(Callee: MallocFunc, Args: {CurrMallocSize, RAPtrToInt});
983
984 Value *MallocPtr =
985 IRB.CreateIntToPtr(V: MallocCall, DestTy: IRB.getPtrTy(AddrSpace: AMDGPUAS::GLOBAL_ADDRESS));
986
987 // Create store of malloc to new global
988 IRB.CreateStore(Val: MallocPtr, Ptr: SwLDS);
989
990 // Create calls to __asan_poison_region to poison redzones.
991 poisonRedzones(Func, MallocPtr);
992
993 // Create branch to PrevEntryBlock
994 IRB.CreateBr(Dest: PrevEntryBlock);
995
996 // Create wave-group barrier at the starting of Previous entry block
997 Type *Int1Ty = IRB.getInt1Ty();
998 IRB.SetInsertPoint(PrevEntryBlock->begin());
999 auto *XYZCondPhi = IRB.CreatePHI(Ty: Int1Ty, NumReservedValues: 2, Name: "xyzCond");
1000 XYZCondPhi->addIncoming(V: IRB.getInt1(V: 0), BB: WIdBlock);
1001 XYZCondPhi->addIncoming(V: IRB.getInt1(V: 1), BB: MallocBlock);
1002
1003 IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_s_barrier, Args: {});
1004
1005 // Load malloc pointer from Sw LDS.
1006 Value *LoadMallocPtr =
1007 IRB.CreateLoad(Ty: IRB.getPtrTy(AddrSpace: AMDGPUAS::GLOBAL_ADDRESS), Ptr: SwLDS);
1008
1009 // Replace All uses of LDS globals with new LDS pointers.
1010 replaceKernelLDSAccesses(Func);
1011
1012 // Replace Memory Operations on LDS with corresponding
1013 // global memory pointers.
1014 translateLDSMemoryOperationsToGlobalMemory(Func, LoadMallocPtr,
1015 LDSInstructions);
1016
1017 auto *CondFreeBlock = BasicBlock::Create(Context&: Ctx, Name: "CondFree", Parent: Func);
1018 auto *FreeBlock = BasicBlock::Create(Context&: Ctx, Name: "Free", Parent: Func);
1019 auto *EndBlock = BasicBlock::Create(Context&: Ctx, Name: "End", Parent: Func);
1020 for (BasicBlock &BB : *Func) {
1021 if (!BB.empty()) {
1022 if (ReturnInst *RI = dyn_cast<ReturnInst>(Val: &BB.back())) {
1023 RI->eraseFromParent();
1024 IRB.SetInsertPoint(BB.end());
1025 IRB.CreateBr(Dest: CondFreeBlock);
1026 }
1027 }
1028 }
1029
1030 // Cond Free Block
1031 IRB.SetInsertPoint(CondFreeBlock->begin());
1032 IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_s_barrier, Args: {});
1033 IRB.CreateCondBr(Cond: XYZCondPhi, True: FreeBlock, False: EndBlock);
1034
1035 // Free Block
1036 IRB.SetInsertPoint(FreeBlock->begin());
1037
1038 // Free the previously allocate device global memory.
1039 FunctionCallee AsanFreeFunc = M.getOrInsertFunction(
1040 Name: StringRef("__asan_free_impl"),
1041 T: FunctionType::get(Result: IRB.getVoidTy(), Params: {Int64Ty, Int64Ty}, isVarArg: false));
1042 Value *ReturnAddr = IRB.CreateIntrinsic(
1043 ID: Intrinsic::returnaddress, OverloadTypes: IRB.getPtrTy(AddrSpace: DL.getProgramAddressSpace()),
1044 Args: IRB.getInt32(C: 0));
1045 Value *RAPToInt = IRB.CreatePtrToInt(V: ReturnAddr, DestTy: Int64Ty);
1046 Value *MallocPtrToInt = IRB.CreatePtrToInt(V: LoadMallocPtr, DestTy: Int64Ty);
1047 IRB.CreateCall(Callee: AsanFreeFunc, Args: {MallocPtrToInt, RAPToInt});
1048
1049 IRB.CreateBr(Dest: EndBlock);
1050
1051 // End Block
1052 IRB.SetInsertPoint(EndBlock->begin());
1053 IRB.CreateRetVoid();
1054 // Update the DomTree with corresponding links to basic blocks.
1055 DTU.applyUpdates(Updates: {{DominatorTree::Insert, WIdBlock, MallocBlock},
1056 {DominatorTree::Insert, MallocBlock, PrevEntryBlock},
1057 {DominatorTree::Insert, CondFreeBlock, FreeBlock},
1058 {DominatorTree::Insert, FreeBlock, EndBlock}});
1059}
1060
1061Constant *AMDGPUSwLowerLDS::getAddressesOfVariablesInKernel(
1062 Function *Func, SetVector<GlobalVariable *> &Variables) {
1063 Type *Int32Ty = IRB.getInt32Ty();
1064 ArrayType *KernelOffsetsType =
1065 ArrayType::get(ElementType: IRB.getPtrTy(AddrSpace: AMDGPUAS::GLOBAL_ADDRESS), NumElements: Variables.size());
1066
1067 // Kernels with no direct or indirect LDS uses never read their offset row.
1068 auto KernelIt = FuncLDSAccessInfo.KernelToLDSParametersMap.find(Key: Func);
1069 if (KernelIt == FuncLDSAccessInfo.KernelToLDSParametersMap.end() ||
1070 !KernelIt->second.SwLDSMetadata)
1071 return PoisonValue::get(T: KernelOffsetsType);
1072
1073 auto &LDSParams = KernelIt->second;
1074 GlobalVariable *SwLDSMetadata = LDSParams.SwLDSMetadata;
1075 auto *SwLDSMetadataStructType =
1076 cast<StructType>(Val: SwLDSMetadata->getValueType());
1077
1078 SmallVector<Constant *> Elements;
1079 for (auto *GV : Variables) {
1080 auto It = LDSParams.LDSToReplacementIndicesMap.find(Val: GV);
1081 if (It == LDSParams.LDSToReplacementIndicesMap.end()) {
1082 Elements.push_back(
1083 Elt: PoisonValue::get(T: IRB.getPtrTy(AddrSpace: AMDGPUAS::GLOBAL_ADDRESS)));
1084 continue;
1085 }
1086 auto &Indices = It->second;
1087 Constant *GEPIdx[] = {ConstantInt::get(Ty: Int32Ty, V: Indices[0]),
1088 ConstantInt::get(Ty: Int32Ty, V: Indices[1]),
1089 ConstantInt::get(Ty: Int32Ty, V: Indices[2])};
1090 Constant *GEP = ConstantExpr::getGetElementPtr(
1091 DL: Func->getDataLayout(), Ty: SwLDSMetadataStructType, C: SwLDSMetadata, IdxList: GEPIdx,
1092 NW: GEPNoWrapFlags::inBounds());
1093 Elements.push_back(Elt: GEP);
1094 }
1095 return ConstantArray::get(T: KernelOffsetsType, V: Elements);
1096}
1097
1098void AMDGPUSwLowerLDS::buildNonKernelLDSBaseTable(
1099 NonKernelLDSParameters &NKLDSParams) {
1100 // Base table will have single row, with elements of the row
1101 // placed as per kernel ID. Each element in the row corresponds
1102 // to addresss of "SW LDS" global of the kernel.
1103 auto &Kernels = NKLDSParams.OrderedKernels;
1104 if (Kernels.empty())
1105 return;
1106 const size_t NumberKernels = Kernels.size();
1107 PointerType *LDSPtrTy = IRB.getPtrTy(AddrSpace: AMDGPUAS::LOCAL_ADDRESS);
1108 ArrayType *AllKernelsOffsetsType = ArrayType::get(ElementType: LDSPtrTy, NumElements: NumberKernels);
1109 std::vector<Constant *> OverallConstantExprElts(NumberKernels);
1110 for (size_t i = 0; i < NumberKernels; i++) {
1111 // Null for kernels without LDS. Lowered functions still load this entry
1112 // on entry, but only use it on LDS paths these kernels never take.
1113 Constant *SwLDS = ConstantPointerNull::get(T: LDSPtrTy);
1114 auto It = FuncLDSAccessInfo.KernelToLDSParametersMap.find(Key: Kernels[i]);
1115 if (It != FuncLDSAccessInfo.KernelToLDSParametersMap.end() &&
1116 It->second.SwLDS)
1117 SwLDS = It->second.SwLDS;
1118 OverallConstantExprElts[i] = SwLDS;
1119 }
1120 Constant *init =
1121 ConstantArray::get(T: AllKernelsOffsetsType, V: OverallConstantExprElts);
1122 NKLDSParams.LDSBaseTable = new GlobalVariable(
1123 M, AllKernelsOffsetsType, true, GlobalValue::InternalLinkage, init,
1124 "llvm.amdgcn.sw.lds.base.table", nullptr, GlobalValue::NotThreadLocal,
1125 AMDGPUAS::GLOBAL_ADDRESS);
1126 GlobalValue::SanitizerMetadata MD;
1127 MD.NoAddress = true;
1128 NKLDSParams.LDSBaseTable->setSanitizerMetadata(MD);
1129}
1130
1131void AMDGPUSwLowerLDS::buildNonKernelLDSOffsetTable(
1132 NonKernelLDSParameters &NKLDSParams) {
1133 // Offset table will have multiple rows and columns.
1134 // Rows are assumed to be from 0 to (n-1). n is total number
1135 // of kernels accessing the LDS through non-kernels.
1136 // Each row will have m elements. m is the total number of
1137 // unique LDS globals accessed by non-kernels.
1138 // Each element in the row correspond to the address of
1139 // the replacement of LDS global done by that particular kernel.
1140 auto &Variables = NKLDSParams.OrdereLDSGlobals;
1141 auto &Kernels = NKLDSParams.OrderedKernels;
1142 if (Variables.empty() || Kernels.empty())
1143 return;
1144 const size_t NumberVariables = Variables.size();
1145 const size_t NumberKernels = Kernels.size();
1146
1147 ArrayType *KernelOffsetsType =
1148 ArrayType::get(ElementType: IRB.getPtrTy(AddrSpace: AMDGPUAS::GLOBAL_ADDRESS), NumElements: NumberVariables);
1149
1150 ArrayType *AllKernelsOffsetsType =
1151 ArrayType::get(ElementType: KernelOffsetsType, NumElements: NumberKernels);
1152 std::vector<Constant *> overallConstantExprElts(NumberKernels);
1153 for (size_t i = 0; i < NumberKernels; i++) {
1154 Function *Func = Kernels[i];
1155 overallConstantExprElts[i] =
1156 getAddressesOfVariablesInKernel(Func, Variables);
1157 }
1158 Constant *Init =
1159 ConstantArray::get(T: AllKernelsOffsetsType, V: overallConstantExprElts);
1160 NKLDSParams.LDSOffsetTable = new GlobalVariable(
1161 M, AllKernelsOffsetsType, true, GlobalValue::InternalLinkage, Init,
1162 "llvm.amdgcn.sw.lds.offset.table", nullptr, GlobalValue::NotThreadLocal,
1163 AMDGPUAS::GLOBAL_ADDRESS);
1164 GlobalValue::SanitizerMetadata MD;
1165 MD.NoAddress = true;
1166 NKLDSParams.LDSOffsetTable->setSanitizerMetadata(MD);
1167}
1168
1169void AMDGPUSwLowerLDS::lowerNonKernelLDSAccesses(
1170 Function *Func, SetVector<GlobalVariable *> &LDSGlobals,
1171 NonKernelLDSParameters &NKLDSParams) {
1172 // Replace LDS access in non-kernel with replacement queried from
1173 // Base table and offset from offset table.
1174 LLVM_DEBUG(dbgs() << "Sw LDS lowering, lower non-kernel access for : "
1175 << Func->getName());
1176 auto InsertAt = Func->getEntryBlock().getFirstNonPHIOrAlloca();
1177 IRB.SetInsertPoint(InsertAt);
1178
1179 // Get LDS memory instructions.
1180 SetVector<Instruction *> LDSInstructions;
1181 getLDSMemoryInstructions(Func, LDSInstructions);
1182
1183 auto *KernelId = IRB.CreateIntrinsic(ID: Intrinsic::amdgcn_lds_kernel_id, Args: {});
1184 GlobalVariable *LDSBaseTable = NKLDSParams.LDSBaseTable;
1185 GlobalVariable *LDSOffsetTable = NKLDSParams.LDSOffsetTable;
1186 auto &OrdereLDSGlobals = NKLDSParams.OrdereLDSGlobals;
1187 Value *BaseGEP = IRB.CreateInBoundsGEP(
1188 Ty: LDSBaseTable->getValueType(), Ptr: LDSBaseTable, IdxList: {IRB.getInt32(C: 0), KernelId});
1189 Value *BaseLoad =
1190 IRB.CreateLoad(Ty: IRB.getPtrTy(AddrSpace: AMDGPUAS::LOCAL_ADDRESS), Ptr: BaseGEP);
1191 Value *LoadMallocPtr =
1192 IRB.CreateLoad(Ty: IRB.getPtrTy(AddrSpace: AMDGPUAS::GLOBAL_ADDRESS), Ptr: BaseLoad);
1193
1194 for (GlobalVariable *GV : LDSGlobals) {
1195 const auto *GVIt = llvm::find(Range&: OrdereLDSGlobals, Val: GV);
1196 assert(GVIt != OrdereLDSGlobals.end());
1197 uint32_t GVOffset = std::distance(first: OrdereLDSGlobals.begin(), last: GVIt);
1198
1199 Value *OffsetGEP = IRB.CreateInBoundsGEP(
1200 Ty: LDSOffsetTable->getValueType(), Ptr: LDSOffsetTable,
1201 IdxList: {IRB.getInt32(C: 0), KernelId, IRB.getInt32(C: GVOffset)});
1202 Value *OffsetLoad =
1203 IRB.CreateLoad(Ty: IRB.getPtrTy(AddrSpace: AMDGPUAS::GLOBAL_ADDRESS), Ptr: OffsetGEP);
1204 Value *Offset = IRB.CreateLoad(Ty: IRB.getInt32Ty(), Ptr: OffsetLoad);
1205 Value *BasePlusOffset =
1206 IRB.CreateInBoundsGEP(Ty: IRB.getInt8Ty(), Ptr: BaseLoad, IdxList: {Offset});
1207 LLVM_DEBUG(dbgs() << "Sw LDS Lowering, Replace non-kernel LDS for "
1208 << GV->getName());
1209 replacesUsesOfGlobalInFunction(Func, GV, Replacement: BasePlusOffset);
1210 }
1211 translateLDSMemoryOperationsToGlobalMemory(Func, LoadMallocPtr,
1212 LDSInstructions);
1213}
1214
1215static void reorderStaticDynamicIndirectLDSSet(KernelLDSParameters &LDSParams) {
1216 // Sort Static, dynamic LDS globals which are either
1217 // direct or indirect access on basis of name.
1218 auto &DirectAccess = LDSParams.DirectAccess;
1219 auto &IndirectAccess = LDSParams.IndirectAccess;
1220 LDSParams.DirectAccess.StaticLDSGlobals = sortByName(
1221 V: std::vector<GlobalVariable *>(DirectAccess.StaticLDSGlobals.begin(),
1222 DirectAccess.StaticLDSGlobals.end()));
1223 LDSParams.DirectAccess.DynamicLDSGlobals = sortByName(
1224 V: std::vector<GlobalVariable *>(DirectAccess.DynamicLDSGlobals.begin(),
1225 DirectAccess.DynamicLDSGlobals.end()));
1226 LDSParams.IndirectAccess.StaticLDSGlobals = sortByName(
1227 V: std::vector<GlobalVariable *>(IndirectAccess.StaticLDSGlobals.begin(),
1228 IndirectAccess.StaticLDSGlobals.end()));
1229 LDSParams.IndirectAccess.DynamicLDSGlobals = sortByName(
1230 V: std::vector<GlobalVariable *>(IndirectAccess.DynamicLDSGlobals.begin(),
1231 IndirectAccess.DynamicLDSGlobals.end()));
1232}
1233
1234void AMDGPUSwLowerLDS::initAsanInfo() {
1235 // Get Shadow mapping scale and offset.
1236 unsigned LongSize =
1237 M.getDataLayout().getPointerSizeInBits(AS: AMDGPUAS::GLOBAL_ADDRESS);
1238 uint64_t Offset;
1239 int Scale;
1240 bool OrShadowOffset;
1241 llvm::getAddressSanitizerParams(Ctx: M.getContext(), TargetTriple: M.getTargetTriple(), LongSize,
1242 IsKasan: false, ShadowBase: &Offset, MappingScale: &Scale, OrShadowOffset: &OrShadowOffset);
1243 AsanInfo.Scale = Scale;
1244 AsanInfo.Offset = Offset;
1245}
1246
1247static bool hasFnWithSanitizeAddressAttr(FunctionVariableMap &LDSAccesses) {
1248 for (auto &K : LDSAccesses) {
1249 Function *F = K.first;
1250 if (!F)
1251 continue;
1252 if (F->hasFnAttribute(Kind: Attribute::SanitizeAddress))
1253 return true;
1254 }
1255 return false;
1256}
1257
1258bool AMDGPUSwLowerLDS::run() {
1259 bool Changed = false;
1260
1261 CallGraph CG = CallGraph(M);
1262
1263 Changed |=
1264 eliminateGVConstantExprUsesFromAllInstructions(M, Filter: isLDSVariableToLower);
1265
1266 // Get all the direct and indirect access of LDS for all the kernels.
1267 GVUsesInfoTy LDSUsesInfo = getTransitiveUsesOfLDSForLowering(CG, M);
1268
1269 // Flag to decide whether to lower all the LDS accesses
1270 // based on sanitize_address attribute.
1271 bool LowerAllLDS = hasFnWithSanitizeAddressAttr(LDSAccesses&: LDSUsesInfo.DirectAccess) ||
1272 hasFnWithSanitizeAddressAttr(LDSAccesses&: LDSUsesInfo.IndirectAccess);
1273
1274 if (!LowerAllLDS)
1275 return Changed;
1276
1277 // Utility to group LDS access into direct, indirect, static and dynamic.
1278 auto PopulateKernelStaticDynamicLDS = [&](FunctionVariableMap &LDSAccesses,
1279 bool DirectAccess) {
1280 for (auto &K : LDSAccesses) {
1281 Function *F = K.first;
1282 if (!F || K.second.empty())
1283 continue;
1284
1285 assert(isKernel(*F));
1286
1287 // Only inserts if key isn't already in the map.
1288 FuncLDSAccessInfo.KernelToLDSParametersMap.insert(
1289 KV: {F, KernelLDSParameters()});
1290
1291 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[F];
1292 if (!DirectAccess)
1293 FuncLDSAccessInfo.KernelsWithIndirectLDSAccess.insert(X: F);
1294 for (GlobalVariable *GV : K.second) {
1295 if (!DirectAccess) {
1296 if (AMDGPU::isDynamicLDS(GV: *GV))
1297 LDSParams.IndirectAccess.DynamicLDSGlobals.insert(X: GV);
1298 else
1299 LDSParams.IndirectAccess.StaticLDSGlobals.insert(X: GV);
1300 FuncLDSAccessInfo.AllNonKernelLDSAccess.insert(X: GV);
1301 } else {
1302 if (AMDGPU::isDynamicLDS(GV: *GV))
1303 LDSParams.DirectAccess.DynamicLDSGlobals.insert(X: GV);
1304 else
1305 LDSParams.DirectAccess.StaticLDSGlobals.insert(X: GV);
1306 }
1307 }
1308 }
1309 };
1310
1311 PopulateKernelStaticDynamicLDS(LDSUsesInfo.DirectAccess, true);
1312 PopulateKernelStaticDynamicLDS(LDSUsesInfo.IndirectAccess, false);
1313
1314 // Get address sanitizer scale.
1315 initAsanInfo();
1316
1317 for (auto &K : FuncLDSAccessInfo.KernelToLDSParametersMap) {
1318 Function *Func = K.first;
1319 auto &LDSParams = FuncLDSAccessInfo.KernelToLDSParametersMap[Func];
1320 if (LDSParams.DirectAccess.StaticLDSGlobals.empty() &&
1321 LDSParams.DirectAccess.DynamicLDSGlobals.empty() &&
1322 LDSParams.IndirectAccess.StaticLDSGlobals.empty() &&
1323 LDSParams.IndirectAccess.DynamicLDSGlobals.empty())
1324 continue;
1325
1326 removeFnAttrFromReachable(
1327 CG, KernelRoot: Func,
1328 FnAttrs: {"amdgpu-no-workitem-id-x", "amdgpu-no-workitem-id-y",
1329 "amdgpu-no-workitem-id-z", "amdgpu-no-heap-ptr"});
1330 reorderStaticDynamicIndirectLDSSet(LDSParams);
1331 buildSwLDSGlobal(Func);
1332 buildSwDynLDSGlobal(Func);
1333 populateSwMetadataGlobal(Func);
1334 populateSwLDSAttributeAndMetadata(Func);
1335 populateLDSToReplacementIndicesMap(Func);
1336 DomTreeUpdater DTU(DTCallback(*Func), DomTreeUpdater::UpdateStrategy::Lazy);
1337 lowerKernelLDSAccesses(Func, DTU);
1338 Changed = true;
1339 }
1340
1341 // Get the Uses of LDS from non-kernels.
1342 getUsesOfLDSByNonKernels();
1343
1344 // Get non-kernels with LDS instructions and the kernels reaching them.
1345 getNonKernelsWithLDSInstructions(CG);
1346
1347 // TODO: removeFnAttrFromReachable does not visit callees of indirect call
1348 // targets.
1349 for (Function *Func : FuncLDSAccessInfo.KernelsWithIndirectLDSAccess)
1350 removeFnAttrFromReachable(CG, KernelRoot: Func, FnAttrs: {"amdgpu-no-lds-kernel-id"});
1351
1352 // Lower LDS accesses in non-kernels.
1353 if (!FuncLDSAccessInfo.NonKernelToLDSAccessMap.empty() ||
1354 !FuncLDSAccessInfo.NonKernelsWithLDSInstructions.empty()) {
1355 NonKernelLDSParameters NKLDSParams;
1356 NKLDSParams.OrderedKernels = getOrderedIndirectLDSAccessingKernels(
1357 Kernels&: FuncLDSAccessInfo.KernelsWithIndirectLDSAccess);
1358 NKLDSParams.OrdereLDSGlobals = getOrderedNonKernelAllLDSGlobals(
1359 Variables&: FuncLDSAccessInfo.AllNonKernelLDSAccess);
1360 buildNonKernelLDSBaseTable(NKLDSParams);
1361 buildNonKernelLDSOffsetTable(NKLDSParams);
1362 for (auto &K : FuncLDSAccessInfo.NonKernelToLDSAccessMap) {
1363 Function *Func = K.first;
1364 DenseSet<GlobalVariable *> &LDSGlobals = K.second;
1365 SetVector<GlobalVariable *> OrderedLDSGlobals = sortByName(
1366 V: std::vector<GlobalVariable *>(LDSGlobals.begin(), LDSGlobals.end()));
1367 lowerNonKernelLDSAccesses(Func, LDSGlobals&: OrderedLDSGlobals, NKLDSParams);
1368 }
1369 for (Function *Func : FuncLDSAccessInfo.NonKernelsWithLDSInstructions) {
1370 if (FuncLDSAccessInfo.NonKernelToLDSAccessMap.contains(Key: Func))
1371 continue;
1372 SetVector<GlobalVariable *> NoLDSGlobals;
1373 lowerNonKernelLDSAccesses(Func, LDSGlobals&: NoLDSGlobals, NKLDSParams);
1374 }
1375 Changed = true;
1376 }
1377
1378 if (!Changed)
1379 return Changed;
1380
1381 for (auto &GV : make_early_inc_range(Range: M.globals())) {
1382 if (AMDGPU::isLDSVariableToLower(GV)) {
1383 // probably want to remove from used lists
1384 GV.removeDeadConstantUsers();
1385 if (GV.use_empty())
1386 GV.eraseFromParent();
1387 }
1388 }
1389
1390 if (M.getContext()
1391 .getLibraryOptions()
1392 .get<AMDGPUOptions>()
1393 .asan_instrument_lds) {
1394 SmallVector<InterestingMemoryOperand, 16> OperandsToInstrument;
1395 for (Instruction *Inst : AsanInfo.Instructions) {
1396 SmallVector<InterestingMemoryOperand, 1> InterestingOperands;
1397 getInterestingMemoryOperands(M, I: Inst, Interesting&: InterestingOperands);
1398 llvm::append_range(C&: OperandsToInstrument, R&: InterestingOperands);
1399 }
1400 for (auto &Operand : OperandsToInstrument) {
1401 Value *Addr = Operand.getPtr();
1402 instrumentAddress(M, IRB, OrigIns: Operand.getInsn(), InsertBefore: Operand.getInsn(), Addr,
1403 Alignment: Operand.Alignment.valueOrOne(), TypeStoreSize: Operand.TypeStoreSize,
1404 IsWrite: Operand.IsWrite, SizeArgument: nullptr, UseCalls: false, Recover: false, Scale: AsanInfo.Scale,
1405 Offset: AsanInfo.Offset);
1406 Changed = true;
1407 }
1408 }
1409
1410 return Changed;
1411}
1412
1413class AMDGPUSwLowerLDSLegacy : public ModulePass {
1414public:
1415 static char ID;
1416 AMDGPUSwLowerLDSLegacy() : ModulePass(ID) {}
1417 bool runOnModule(Module &M) override;
1418 void getAnalysisUsage(AnalysisUsage &AU) const override {
1419 AU.addPreserved<DominatorTreeWrapperPass>();
1420 }
1421};
1422} // namespace
1423
1424char AMDGPUSwLowerLDSLegacy::ID = 0;
1425char &llvm::AMDGPUSwLowerLDSLegacyPassID = AMDGPUSwLowerLDSLegacy::ID;
1426
1427INITIALIZE_PASS_BEGIN(AMDGPUSwLowerLDSLegacy, "amdgpu-sw-lower-lds",
1428 "AMDGPU Software lowering of LDS", false, false)
1429INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
1430INITIALIZE_PASS_END(AMDGPUSwLowerLDSLegacy, "amdgpu-sw-lower-lds",
1431 "AMDGPU Software lowering of LDS", false, false)
1432
1433bool AMDGPUSwLowerLDSLegacy::runOnModule(Module &M) {
1434 // AddressSanitizer pass adds "nosanitize_address" module flag if it has
1435 // instrumented the IR. Return early if the flag is not present.
1436 if (!M.getModuleFlag(Key: "nosanitize_address"))
1437 return false;
1438 DominatorTreeWrapperPass *const DTW =
1439 getAnalysisIfAvailable<DominatorTreeWrapperPass>();
1440 auto DTCallback = [&DTW](Function &F) -> DominatorTree * {
1441 return DTW ? &DTW->getDomTree() : nullptr;
1442 };
1443
1444 AMDGPUSwLowerLDS SwLowerLDSImpl(M, DTCallback);
1445 bool IsChanged = SwLowerLDSImpl.run();
1446 return IsChanged;
1447}
1448
1449ModulePass *llvm::createAMDGPUSwLowerLDSLegacyPass() {
1450 return new AMDGPUSwLowerLDSLegacy();
1451}
1452
1453PreservedAnalyses AMDGPUSwLowerLDSPass::run(Module &M,
1454 ModuleAnalysisManager &AM) {
1455 // AddressSanitizer pass adds "nosanitize_address" module flag if it has
1456 // instrumented the IR. Return early if the flag is not present.
1457 if (!M.getModuleFlag(Key: "nosanitize_address"))
1458 return PreservedAnalyses::all();
1459 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(IR&: M).getManager();
1460 auto DTCallback = [&FAM](Function &F) -> DominatorTree * {
1461 return &FAM.getResult<DominatorTreeAnalysis>(IR&: F);
1462 };
1463 AMDGPUSwLowerLDS SwLowerLDSImpl(M, DTCallback);
1464 bool IsChanged = SwLowerLDSImpl.run();
1465 if (!IsChanged)
1466 return PreservedAnalyses::all();
1467
1468 PreservedAnalyses PA;
1469 PA.preserve<DominatorTreeAnalysis>();
1470 return PA;
1471}
1472