1//===-- AMDGPUMemoryUtils.cpp - -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "AMDGPUMemoryUtils.h"
10#include "Utils/AMDGPUBaseInfo.h"
11#include "llvm/ADT/SetOperations.h"
12#include "llvm/ADT/StringRef.h"
13#include "llvm/Analysis/AliasAnalysis.h"
14#include "llvm/Analysis/CallGraph.h"
15#include "llvm/Analysis/MemorySSA.h"
16#include "llvm/IR/DataLayout.h"
17#include "llvm/IR/DiagnosticInfo.h"
18#include "llvm/IR/Instructions.h"
19#include "llvm/IR/IntrinsicInst.h"
20#include "llvm/IR/IntrinsicsAMDGPU.h"
21#include "llvm/IR/LLVMContext.h"
22#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
23#include "llvm/IR/ReplaceConstant.h"
24#include "llvm/Support/AMDGPUAddrSpace.h"
25#include "llvm/Support/AtomicOrdering.h"
26
27#define DEBUG_TYPE "amdgpu-memory-utils"
28
29using namespace llvm;
30
31namespace llvm::AMDGPU {
32
33Align getAlign(const DataLayout &DL, const GlobalVariable *GV) {
34 return DL.getValueOrABITypeAlignment(Alignment: GV->getPointerAlignment(DL),
35 Ty: GV->getValueType());
36}
37
38unsigned getSyntheticApertureNumber(unsigned AS) {
39 switch (AS) {
40 case AMDGPUAS::BARRIER:
41 return SyntheticAperture::BARRIER;
42 case AMDGPUAS::VGPR:
43 return SyntheticAperture::VGPR;
44 default:
45 return SyntheticAperture::None;
46 }
47}
48
49void copyMetadataForWidenedLoad(LoadInst &Dest, const LoadInst &Source) {
50 SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
51 Source.getAllMetadata(MDs&: MD);
52 for (const auto &[ID, N] : MD) {
53 switch (ID) {
54 case LLVMContext::MD_dbg:
55 case LLVMContext::MD_invariant_load:
56 case LLVMContext::MD_nontemporal:
57 Dest.setMetadata(KindID: ID, Node: N);
58 break;
59 default:
60 break;
61 }
62 }
63}
64
65// Returns the target extension type of a global variable,
66// which can only be a TargetExtType, an array or single-element struct of it,
67// or their nesting combination.
68// TODO: allow struct of multiple TargetExtType elements of the same type.
69// TODO: Disallow other uses of target("amdgcn.named.barrier") including:
70// - Structs containing barriers in different scope/rank
71// - Structs containing a mixture of barriers and other data.
72// - Globals in other address spaces.
73// - Allocas.
74static TargetExtType *getTargetExtType(const GlobalVariable &GV) {
75 Type *Ty = GV.getValueType();
76 while (true) {
77 if (auto *TTy = dyn_cast<TargetExtType>(Val: Ty))
78 return TTy;
79 if (auto *STy = dyn_cast<StructType>(Val: Ty)) {
80 if (STy->getNumElements() != 1)
81 return nullptr;
82 Ty = STy->getElementType(N: 0);
83 continue;
84 }
85 if (auto *ATy = dyn_cast<ArrayType>(Val: Ty)) {
86 Ty = ATy->getElementType();
87 continue;
88 }
89 return nullptr;
90 }
91}
92
93TargetExtType *isNamedBarrier(const GlobalVariable &GV) {
94 if (GV.getAddressSpace() != AMDGPUAS::BARRIER)
95 return nullptr;
96 if (TargetExtType *Ty = getTargetExtType(GV))
97 return Ty->getName() == "amdgcn.named.barrier" ? Ty : nullptr;
98 return nullptr;
99}
100
101unsigned getNumNamedBarriersDeclared(const DataLayout &DL,
102 const GlobalVariable &GV) {
103 assert(isNamedBarrier(GV));
104 unsigned GVSize = GV.getGlobalSize(DL);
105 assert(GVSize && (GVSize % NamedBarrierTypeSizeInBytes == 0));
106 return GVSize / NamedBarrierTypeSizeInBytes;
107}
108
109bool isDynamicLDS(const GlobalVariable &GV) {
110 // external zero size addrspace(3) without initializer is dynlds.
111 const Module *M = GV.getParent();
112 const DataLayout &DL = M->getDataLayout();
113 if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS)
114 return false;
115 return GV.getGlobalSize(DL) == 0;
116}
117
118bool isLDSVariableToLower(const GlobalVariable &GV) {
119 if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) {
120 return false;
121 }
122 if (isDynamicLDS(GV)) {
123 return true;
124 }
125 if (GV.isConstant()) {
126 // A constant undef variable can't be written to, and any load is
127 // undef, so it should be eliminated by the optimizer. It could be
128 // dropped by the back end if not. This pass skips over it.
129 return false;
130 }
131 if (GV.hasInitializer() && !isa<UndefValue>(Val: GV.getInitializer())) {
132 // Initializers are unimplemented for LDS address space.
133 // Leave such variables in place for consistent error reporting.
134 return false;
135 }
136 return true;
137}
138
139bool eliminateGVConstantExprUsesFromAllInstructions(
140 Module &M, function_ref<bool(const GlobalVariable &)> Filter) {
141 SmallVector<Constant *> Worklist;
142 for (auto &GV : M.globals())
143 if (Filter(GV))
144 Worklist.push_back(Elt: &GV);
145 return convertUsersOfConstantsToInstructions(Consts: Worklist);
146}
147
148void getUsesOfGVByFunction(const CallGraph &CG, Module &M,
149 function_ref<bool(const GlobalVariable &)> Filter,
150 FunctionVariableMap &Kernels,
151 FunctionVariableMap &Functions) {
152 // Get uses from the current function, excluding uses by called Functions
153 // Two output variables to avoid walking the globals list twice
154 for (auto &GV : M.globals()) {
155 if (!Filter(GV))
156 continue;
157 for (User *V : GV.users()) {
158 if (auto *I = dyn_cast<Instruction>(Val: V)) {
159 Function *F = I->getFunction();
160 if (isKernel(F: *F))
161 Kernels[F].insert(V: &GV);
162 else
163 Functions[F].insert(V: &GV);
164 }
165 }
166 }
167}
168
169GVUsesInfoTy
170getTransitiveUsesOfGV(const CallGraph &CG, Module &M,
171 function_ref<bool(const GlobalVariable &)> Filter) {
172
173 FunctionVariableMap DirectMapKernel;
174 FunctionVariableMap DirectMapFunction;
175 getUsesOfGVByFunction(CG, M, Filter, Kernels&: DirectMapKernel, Functions&: DirectMapFunction);
176
177 // Collect functions whose address has escaped
178 DenseSet<Function *> AddressTakenFuncs;
179 for (Function &F : M.functions()) {
180 if (!isKernel(F))
181 if (F.hasAddressTaken(nullptr,
182 /* IgnoreCallbackUses */ false,
183 /* IgnoreAssumeLikeCalls */ false,
184 /* IgnoreLLVMUsed */ IngoreLLVMUsed: true,
185 /* IgnoreArcAttachedCall */ IgnoreARCAttachedCall: false)) {
186 AddressTakenFuncs.insert(V: &F);
187 }
188 }
189
190 // Collect variables that are used by functions whose address has escaped
191 DenseSet<GlobalVariable *> VariablesReachableThroughFunctionPointer;
192 for (Function *F : AddressTakenFuncs) {
193 set_union(S1&: VariablesReachableThroughFunctionPointer, S2: DirectMapFunction[F]);
194 }
195
196 auto FunctionMakesUnknownCall = [&](const Function *F) -> bool {
197 assert(!F->isDeclaration());
198 for (const CallGraphNode::CallRecord &R : *CG[F]) {
199 if (!R.second->getFunction())
200 return true;
201 }
202 return false;
203 };
204
205 // Work out which variables are reachable through function calls
206 FunctionVariableMap TransitiveMapFunction = DirectMapFunction;
207
208 // If the function makes any unknown call, assume the worst case that it can
209 // access all variables accessed by functions whose address escaped
210 for (Function &F : M.functions()) {
211 if (!F.isDeclaration() && FunctionMakesUnknownCall(&F)) {
212 if (!isKernel(F)) {
213 set_union(S1&: TransitiveMapFunction[&F],
214 S2: VariablesReachableThroughFunctionPointer);
215 }
216 }
217 }
218
219 // Direct implementation of collecting all variables reachable from each
220 // function
221 for (Function &Func : M.functions()) {
222 if (Func.isDeclaration() || isKernel(F: Func))
223 continue;
224
225 DenseSet<Function *> seen; // catches cycles
226 SmallVector<Function *, 4> wip = {&Func};
227
228 while (!wip.empty()) {
229 Function *F = wip.pop_back_val();
230
231 // Can accelerate this by referring to transitive map for functions that
232 // have already been computed, with more care than this
233 set_union(S1&: TransitiveMapFunction[&Func], S2: DirectMapFunction[F]);
234
235 for (const CallGraphNode::CallRecord &R : *CG[F]) {
236 Function *Ith = R.second->getFunction();
237 if (Ith) {
238 if (!seen.contains(V: Ith)) {
239 seen.insert(V: Ith);
240 wip.push_back(Elt: Ith);
241 }
242 }
243 }
244 }
245 }
246
247 // Collect variables that are transitively used by functions whose address has
248 // escaped
249 for (Function *F : AddressTakenFuncs) {
250 set_union(S1&: VariablesReachableThroughFunctionPointer,
251 S2: TransitiveMapFunction[F]);
252 }
253
254 // DirectMapKernel lists which variables are used by the kernel
255 // find the variables which are used through a function call
256 FunctionVariableMap IndirectMapKernel;
257
258 for (Function &Func : M.functions()) {
259 if (Func.isDeclaration() || !isKernel(F: Func))
260 continue;
261
262 for (const CallGraphNode::CallRecord &R : *CG[&Func]) {
263 Function *Ith = R.second->getFunction();
264 if (Ith) {
265 set_union(S1&: IndirectMapKernel[&Func], S2: TransitiveMapFunction[Ith]);
266 }
267 }
268
269 // Check if the kernel encounters unknows calls, wheher directly or
270 // indirectly.
271 bool SeesUnknownCalls = [&]() {
272 SmallVector<Function *> WorkList = {CG[&Func]->getFunction()};
273 SmallPtrSet<Function *, 8> Visited;
274
275 while (!WorkList.empty()) {
276 Function *F = WorkList.pop_back_val();
277
278 for (const CallGraphNode::CallRecord &CallRecord : *CG[F]) {
279 if (!CallRecord.second)
280 continue;
281
282 Function *Callee = CallRecord.second->getFunction();
283 if (!Callee)
284 return true;
285
286 if (Visited.insert(Ptr: Callee).second)
287 WorkList.push_back(Elt: Callee);
288 }
289 }
290 return false;
291 }();
292
293 if (SeesUnknownCalls) {
294 set_union(S1&: IndirectMapKernel[&Func],
295 S2: VariablesReachableThroughFunctionPointer);
296 }
297 }
298
299 return {.DirectAccess: std::move(DirectMapKernel), .IndirectAccess: std::move(IndirectMapKernel)};
300}
301
302GVUsesInfoTy getTransitiveUsesOfLDSForLowering(const CallGraph &CG, Module &M) {
303 GVUsesInfoTy UsesInfo = getTransitiveUsesOfGV(CG, M, Filter: isLDSVariableToLower);
304 // Verify that we fall into one of 2 cases:
305 // - All variables are either absolute
306 // or direct mapped dynamic LDS that is not lowered.
307 // - No variables are absolute.
308 // Named-barriers which are absolute symbols are removed
309 // from the maps.
310 std::optional<bool> HasAbsoluteGVs;
311 for (auto &Map : {UsesInfo.DirectAccess, UsesInfo.IndirectAccess}) {
312 for (auto &[Fn, GVs] : Map) {
313 for (auto *GV : GVs) {
314 bool IsAbsolute = GV->isAbsoluteSymbolRef();
315 bool IsDirectMapDynLDSGV =
316 AMDGPU::isDynamicLDS(GV: *GV) && UsesInfo.DirectAccess.contains(Key: Fn);
317 if (IsDirectMapDynLDSGV)
318 continue;
319
320 if (HasAbsoluteGVs.has_value()) {
321 if (*HasAbsoluteGVs != IsAbsolute) {
322 reportFatalUsageError(
323 reason: "module cannot mix absolute and non-absolute LDS GVs");
324 }
325 } else
326 HasAbsoluteGVs = IsAbsolute;
327 }
328 }
329 }
330
331 // If we only had absolute GVs, we have nothing to do, return an empty
332 // result.
333 if (HasAbsoluteGVs && *HasAbsoluteGVs)
334 return GVUsesInfoTy();
335
336 return UsesInfo;
337}
338
339void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot,
340 ArrayRef<StringRef> FnAttrs) {
341 for (StringRef Attr : FnAttrs)
342 KernelRoot->removeFnAttr(Kind: Attr);
343
344 SmallVector<Function *> WorkList = {CG[KernelRoot]->getFunction()};
345 SmallPtrSet<Function *, 8> Visited;
346 bool SeenUnknownCall = false;
347
348 while (!WorkList.empty()) {
349 Function *F = WorkList.pop_back_val();
350
351 for (auto &CallRecord : *CG[F]) {
352 if (!CallRecord.second)
353 continue;
354
355 Function *Callee = CallRecord.second->getFunction();
356 if (!Callee) {
357 if (!SeenUnknownCall) {
358 SeenUnknownCall = true;
359
360 // If we see any indirect calls, assume nothing about potential
361 // targets.
362 // TODO: This could be refined to possible LDS global users.
363 for (auto &ExternalCallRecord : *CG.getExternalCallingNode()) {
364 Function *PotentialCallee =
365 ExternalCallRecord.second->getFunction();
366 assert(PotentialCallee);
367 if (!isKernel(F: *PotentialCallee)) {
368 for (StringRef Attr : FnAttrs)
369 PotentialCallee->removeFnAttr(Kind: Attr);
370 }
371 }
372 }
373 } else {
374 for (StringRef Attr : FnAttrs)
375 Callee->removeFnAttr(Kind: Attr);
376 if (Visited.insert(Ptr: Callee).second)
377 WorkList.push_back(Elt: Callee);
378 }
379 }
380 }
381}
382
383bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) {
384 Instruction *DefInst = Def->getMemoryInst();
385
386 if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: DefInst)) {
387 switch (II->getIntrinsicID()) {
388 case Intrinsic::amdgcn_s_barrier:
389 case Intrinsic::amdgcn_s_cluster_barrier:
390 case Intrinsic::amdgcn_s_barrier_signal:
391 case Intrinsic::amdgcn_s_barrier_signal_var:
392 case Intrinsic::amdgcn_s_barrier_signal_isfirst:
393 case Intrinsic::amdgcn_s_barrier_init:
394 case Intrinsic::amdgcn_s_barrier_join:
395 case Intrinsic::amdgcn_s_barrier_wait:
396 case Intrinsic::amdgcn_s_barrier_leave:
397 case Intrinsic::amdgcn_s_get_barrier_state:
398 case Intrinsic::amdgcn_s_wakeup_barrier:
399 case Intrinsic::amdgcn_wave_barrier:
400 case Intrinsic::amdgcn_sched_barrier:
401 case Intrinsic::amdgcn_sched_group_barrier:
402 case Intrinsic::amdgcn_iglp_opt:
403 return false;
404 default:
405 break;
406 }
407 }
408
409 // Ignore non-acquire fences/atomics not aliasing with the original load; MSSA
410 // counts any atomic or fence as a universal MemoryDef.
411 // Acquire (or stronger) fences/atomics act as clobbers because they can bring
412 // in effects from other threads.
413 const auto MayAlias = [AA, Ptr](auto I) -> bool {
414 return !AA->isNoAlias(I->getPointerOperand(), Ptr);
415 };
416
417 if (const auto *F = dyn_cast<FenceInst>(Val: DefInst)) {
418 if (!isAcquireOrStronger(AO: F->getOrdering()))
419 return false;
420 SIAtomicAddrSpace AccessedAS =
421 toSIAtomicAddrSpace(AS: Ptr->getType()->getPointerAddressSpace());
422 return (AccessedAS & getSynchronizeAddrSpace(I: *F)) !=
423 SIAtomicAddrSpace::NONE;
424 }
425
426 if (const auto *I = dyn_cast<AtomicRMWInst>(Val: DefInst))
427 return isAcquireOrStronger(AO: I->getOrdering()) || MayAlias(I);
428
429 if (const auto *I = dyn_cast<AtomicCmpXchgInst>(Val: DefInst))
430 return isAcquireOrStronger(AO: I->getMergedOrdering()) || MayAlias(I);
431
432 // TODO This is pessimistic for one-as load-acquires from a different AS.
433 if (const auto *I = dyn_cast<LoadInst>(Val: DefInst))
434 return isAcquireOrStronger(AO: I->getOrdering());
435
436 // (Atomic) stores that don't alias do not clobber.
437 if (const auto *I = dyn_cast<StoreInst>(Val: DefInst))
438 return MayAlias(I);
439
440 return true;
441}
442
443bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA,
444 AAResults *AA) {
445 MemorySSAWalker *Walker = MSSA->getWalker();
446 MemoryLocation Loc(MemoryLocation::get(LI: Load));
447 MemoryUseOrDef *Use = MSSA->getMemoryAccess(I: Load);
448 SmallVector<MemoryAccess *> WorkList{
449 Walker->getClobberingMemoryAccess(MA: Use->getDefiningAccess(), Loc)};
450 SmallPtrSet<MemoryAccess *, 8> Visited;
451
452 LLVM_DEBUG(dbgs() << "Checking clobbering of: " << *Load << '\n');
453
454 // Start with a nearest dominating clobbering access, it will be either
455 // live on entry (nothing to do, load is not clobbered), MemoryDef, or
456 // MemoryPhi if several MemoryDefs can define this memory state. In that
457 // case add all Defs to WorkList and continue going up and checking all
458 // the definitions of this memory location until the root. When all the
459 // defs are exhausted and came to the entry state we have no clobber.
460 // Along the scan ignore barriers which are considered clobbers by the
461 // MemorySSA, but not really writing anything into the memory.
462 while (!WorkList.empty()) {
463 MemoryAccess *MA = WorkList.pop_back_val();
464 if (!Visited.insert(Ptr: MA).second)
465 continue;
466
467 if (MSSA->isLiveOnEntryDef(MA))
468 continue;
469
470 if (MemoryDef *Def = dyn_cast<MemoryDef>(Val: MA)) {
471 LLVM_DEBUG(dbgs() << " Def: " << *Def->getMemoryInst() << '\n');
472
473 if (isReallyAClobber(Ptr: Load->getPointerOperand(), Def, AA)) {
474 LLVM_DEBUG(dbgs() << " -> load is clobbered\n");
475 return true;
476 }
477
478 WorkList.push_back(
479 Elt: Walker->getClobberingMemoryAccess(MA: Def->getDefiningAccess(), Loc));
480 continue;
481 }
482
483 const MemoryPhi *Phi = cast<MemoryPhi>(Val: MA);
484 for (const auto &Use : Phi->incoming_values())
485 WorkList.push_back(
486 Elt: Walker->getClobberingMemoryAccess(MA: cast<MemoryAccess>(Val: &Use), Loc));
487 }
488
489 LLVM_DEBUG(dbgs() << " -> no clobber\n");
490 return false;
491}
492
493static constexpr SIAtomicAddrSpace LLVMToSIAtomicAddrSpaceMapping[] = {
494 SIAtomicAddrSpace::FLAT, //< AMDGPUAS::FLAT_ADDRESS
495 SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::GLOBAL_ADDRESS
496 SIAtomicAddrSpace::GDS, //< AMDGPUAS::REGION_ADDRESS
497 SIAtomicAddrSpace::LDS, //< AMDGPUAS::LOCAL_ADDRESS
498 SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::CONSTANT_ADDRESS
499 SIAtomicAddrSpace::SCRATCH, //< AMDGPUAS::PRIVATE_ADDRESS
500 SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::CONSTANT_ADDRESS_32BIT
501 SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_FAT_POINTER
502 SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_RESOURCE
503 SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_STRIDED_POINTER
504};
505
506SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned LLVMAddrSpace) {
507 constexpr unsigned SizeOfLLVMToSIAtomicAddrSpaceMapping =
508 sizeof(LLVMToSIAtomicAddrSpaceMapping) /
509 sizeof(LLVMToSIAtomicAddrSpaceMapping[0]);
510 if (LLVMAddrSpace > AMDGPUAS::MAX_AMDGPU_ADDRESS)
511 return SIAtomicAddrSpace::GLOBAL;
512 if (LLVMAddrSpace < SizeOfLLVMToSIAtomicAddrSpaceMapping)
513 return LLVMToSIAtomicAddrSpaceMapping[LLVMAddrSpace];
514 return SIAtomicAddrSpace::OTHER;
515}
516
517static const StringMap<SIAtomicAddrSpace> ASNames = {{
518 {"global", SIAtomicAddrSpace::GLOBAL},
519 {"local", SIAtomicAddrSpace::LDS},
520}};
521
522SIAtomicAddrSpace getSynchronizeAddrSpace(const MMRAMetadata &MMRA,
523 const Function *DiagFn,
524 const DebugLoc &DL) {
525 static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as";
526
527 if (!MMRA)
528 return SIAtomicAddrSpace::ATOMIC;
529
530 SIAtomicAddrSpace Result{};
531 for (const auto &[Prefix, Suffix] : MMRA) {
532 if (Prefix != FenceASPrefix)
533 continue;
534
535 if (auto It = ASNames.find(Key: Suffix); It != ASNames.end()) {
536 Result |= It->second;
537 } else if (DiagFn) {
538 SmallString<128> Str;
539 raw_svector_ostream OS(Str);
540 OS << "unknown address space '" << Suffix << "'; expected one of ";
541 ListSeparator LS;
542 for (const auto &[Name, Val] : ASNames)
543 OS << LS << '\'' << Name << '\'';
544 DiagFn->getContext().diagnose(
545 DI: DiagnosticInfoUnsupported(*DiagFn, Str, DL, DS_Warning));
546 }
547 }
548
549 if (Result == SIAtomicAddrSpace::NONE)
550 return SIAtomicAddrSpace::ATOMIC;
551
552 return Result;
553}
554
555SIAtomicAddrSpace getSynchronizeAddrSpace(const Instruction &I) {
556 return getSynchronizeAddrSpace(MMRA: MMRAMetadata(I));
557}
558
559} // end namespace llvm::AMDGPU
560