| 1 | //===-- AMDGPUMemoryUtils.cpp - -------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "AMDGPUMemoryUtils.h" |
| 10 | #include "Utils/AMDGPUBaseInfo.h" |
| 11 | #include "llvm/ADT/SetOperations.h" |
| 12 | #include "llvm/ADT/StringRef.h" |
| 13 | #include "llvm/Analysis/AliasAnalysis.h" |
| 14 | #include "llvm/Analysis/CallGraph.h" |
| 15 | #include "llvm/Analysis/MemorySSA.h" |
| 16 | #include "llvm/IR/DataLayout.h" |
| 17 | #include "llvm/IR/DiagnosticInfo.h" |
| 18 | #include "llvm/IR/Instructions.h" |
| 19 | #include "llvm/IR/IntrinsicInst.h" |
| 20 | #include "llvm/IR/IntrinsicsAMDGPU.h" |
| 21 | #include "llvm/IR/LLVMContext.h" |
| 22 | #include "llvm/IR/MemoryModelRelaxationAnnotations.h" |
| 23 | #include "llvm/IR/ReplaceConstant.h" |
| 24 | #include "llvm/Support/AMDGPUAddrSpace.h" |
| 25 | #include "llvm/Support/AtomicOrdering.h" |
| 26 | |
| 27 | #define DEBUG_TYPE "amdgpu-memory-utils" |
| 28 | |
| 29 | using namespace llvm; |
| 30 | |
| 31 | namespace llvm::AMDGPU { |
| 32 | |
| 33 | Align getAlign(const DataLayout &DL, const GlobalVariable *GV) { |
| 34 | return DL.getValueOrABITypeAlignment(Alignment: GV->getPointerAlignment(DL), |
| 35 | Ty: GV->getValueType()); |
| 36 | } |
| 37 | |
| 38 | unsigned getSyntheticApertureNumber(unsigned AS) { |
| 39 | switch (AS) { |
| 40 | case AMDGPUAS::BARRIER: |
| 41 | return SyntheticAperture::BARRIER; |
| 42 | case AMDGPUAS::VGPR: |
| 43 | return SyntheticAperture::VGPR; |
| 44 | default: |
| 45 | return SyntheticAperture::None; |
| 46 | } |
| 47 | } |
| 48 | |
| 49 | void copyMetadataForWidenedLoad(LoadInst &Dest, const LoadInst &Source) { |
| 50 | SmallVector<std::pair<unsigned, MDNode *>, 8> MD; |
| 51 | Source.getAllMetadata(MDs&: MD); |
| 52 | for (const auto &[ID, N] : MD) { |
| 53 | switch (ID) { |
| 54 | case LLVMContext::MD_dbg: |
| 55 | case LLVMContext::MD_invariant_load: |
| 56 | case LLVMContext::MD_nontemporal: |
| 57 | Dest.setMetadata(KindID: ID, Node: N); |
| 58 | break; |
| 59 | default: |
| 60 | break; |
| 61 | } |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | // Returns the target extension type of a global variable, |
| 66 | // which can only be a TargetExtType, an array or single-element struct of it, |
| 67 | // or their nesting combination. |
| 68 | // TODO: allow struct of multiple TargetExtType elements of the same type. |
| 69 | // TODO: Disallow other uses of target("amdgcn.named.barrier") including: |
| 70 | // - Structs containing barriers in different scope/rank |
| 71 | // - Structs containing a mixture of barriers and other data. |
| 72 | // - Globals in other address spaces. |
| 73 | // - Allocas. |
| 74 | static TargetExtType *getTargetExtType(const GlobalVariable &GV) { |
| 75 | Type *Ty = GV.getValueType(); |
| 76 | while (true) { |
| 77 | if (auto *TTy = dyn_cast<TargetExtType>(Val: Ty)) |
| 78 | return TTy; |
| 79 | if (auto *STy = dyn_cast<StructType>(Val: Ty)) { |
| 80 | if (STy->getNumElements() != 1) |
| 81 | return nullptr; |
| 82 | Ty = STy->getElementType(N: 0); |
| 83 | continue; |
| 84 | } |
| 85 | if (auto *ATy = dyn_cast<ArrayType>(Val: Ty)) { |
| 86 | Ty = ATy->getElementType(); |
| 87 | continue; |
| 88 | } |
| 89 | return nullptr; |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | TargetExtType *isNamedBarrier(const GlobalVariable &GV) { |
| 94 | if (GV.getAddressSpace() != AMDGPUAS::BARRIER) |
| 95 | return nullptr; |
| 96 | if (TargetExtType *Ty = getTargetExtType(GV)) |
| 97 | return Ty->getName() == "amdgcn.named.barrier" ? Ty : nullptr; |
| 98 | return nullptr; |
| 99 | } |
| 100 | |
| 101 | unsigned getNumNamedBarriersDeclared(const DataLayout &DL, |
| 102 | const GlobalVariable &GV) { |
| 103 | assert(isNamedBarrier(GV)); |
| 104 | unsigned GVSize = GV.getGlobalSize(DL); |
| 105 | assert(GVSize && (GVSize % NamedBarrierTypeSizeInBytes == 0)); |
| 106 | return GVSize / NamedBarrierTypeSizeInBytes; |
| 107 | } |
| 108 | |
| 109 | bool isDynamicLDS(const GlobalVariable &GV) { |
| 110 | // external zero size addrspace(3) without initializer is dynlds. |
| 111 | const Module *M = GV.getParent(); |
| 112 | const DataLayout &DL = M->getDataLayout(); |
| 113 | if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) |
| 114 | return false; |
| 115 | return GV.getGlobalSize(DL) == 0; |
| 116 | } |
| 117 | |
| 118 | bool isLDSVariableToLower(const GlobalVariable &GV) { |
| 119 | if (GV.getType()->getPointerAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) { |
| 120 | return false; |
| 121 | } |
| 122 | if (isDynamicLDS(GV)) { |
| 123 | return true; |
| 124 | } |
| 125 | if (GV.isConstant()) { |
| 126 | // A constant undef variable can't be written to, and any load is |
| 127 | // undef, so it should be eliminated by the optimizer. It could be |
| 128 | // dropped by the back end if not. This pass skips over it. |
| 129 | return false; |
| 130 | } |
| 131 | if (GV.hasInitializer() && !isa<UndefValue>(Val: GV.getInitializer())) { |
| 132 | // Initializers are unimplemented for LDS address space. |
| 133 | // Leave such variables in place for consistent error reporting. |
| 134 | return false; |
| 135 | } |
| 136 | return true; |
| 137 | } |
| 138 | |
| 139 | bool eliminateGVConstantExprUsesFromAllInstructions( |
| 140 | Module &M, function_ref<bool(const GlobalVariable &)> Filter) { |
| 141 | SmallVector<Constant *> Worklist; |
| 142 | for (auto &GV : M.globals()) |
| 143 | if (Filter(GV)) |
| 144 | Worklist.push_back(Elt: &GV); |
| 145 | return convertUsersOfConstantsToInstructions(Consts: Worklist); |
| 146 | } |
| 147 | |
| 148 | void getUsesOfGVByFunction(const CallGraph &CG, Module &M, |
| 149 | function_ref<bool(const GlobalVariable &)> Filter, |
| 150 | FunctionVariableMap &Kernels, |
| 151 | FunctionVariableMap &Functions) { |
| 152 | // Get uses from the current function, excluding uses by called Functions |
| 153 | // Two output variables to avoid walking the globals list twice |
| 154 | for (auto &GV : M.globals()) { |
| 155 | if (!Filter(GV)) |
| 156 | continue; |
| 157 | for (User *V : GV.users()) { |
| 158 | if (auto *I = dyn_cast<Instruction>(Val: V)) { |
| 159 | Function *F = I->getFunction(); |
| 160 | if (isKernel(F: *F)) |
| 161 | Kernels[F].insert(V: &GV); |
| 162 | else |
| 163 | Functions[F].insert(V: &GV); |
| 164 | } |
| 165 | } |
| 166 | } |
| 167 | } |
| 168 | |
| 169 | GVUsesInfoTy |
| 170 | getTransitiveUsesOfGV(const CallGraph &CG, Module &M, |
| 171 | function_ref<bool(const GlobalVariable &)> Filter) { |
| 172 | |
| 173 | FunctionVariableMap DirectMapKernel; |
| 174 | FunctionVariableMap DirectMapFunction; |
| 175 | getUsesOfGVByFunction(CG, M, Filter, Kernels&: DirectMapKernel, Functions&: DirectMapFunction); |
| 176 | |
| 177 | // Collect functions whose address has escaped |
| 178 | DenseSet<Function *> AddressTakenFuncs; |
| 179 | for (Function &F : M.functions()) { |
| 180 | if (!isKernel(F)) |
| 181 | if (F.hasAddressTaken(nullptr, |
| 182 | /* IgnoreCallbackUses */ false, |
| 183 | /* IgnoreAssumeLikeCalls */ false, |
| 184 | /* IgnoreLLVMUsed */ IngoreLLVMUsed: true, |
| 185 | /* IgnoreArcAttachedCall */ IgnoreARCAttachedCall: false)) { |
| 186 | AddressTakenFuncs.insert(V: &F); |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | // Collect variables that are used by functions whose address has escaped |
| 191 | DenseSet<GlobalVariable *> VariablesReachableThroughFunctionPointer; |
| 192 | for (Function *F : AddressTakenFuncs) { |
| 193 | set_union(S1&: VariablesReachableThroughFunctionPointer, S2: DirectMapFunction[F]); |
| 194 | } |
| 195 | |
| 196 | auto FunctionMakesUnknownCall = [&](const Function *F) -> bool { |
| 197 | assert(!F->isDeclaration()); |
| 198 | for (const CallGraphNode::CallRecord &R : *CG[F]) { |
| 199 | if (!R.second->getFunction()) |
| 200 | return true; |
| 201 | } |
| 202 | return false; |
| 203 | }; |
| 204 | |
| 205 | // Work out which variables are reachable through function calls |
| 206 | FunctionVariableMap TransitiveMapFunction = DirectMapFunction; |
| 207 | |
| 208 | // If the function makes any unknown call, assume the worst case that it can |
| 209 | // access all variables accessed by functions whose address escaped |
| 210 | for (Function &F : M.functions()) { |
| 211 | if (!F.isDeclaration() && FunctionMakesUnknownCall(&F)) { |
| 212 | if (!isKernel(F)) { |
| 213 | set_union(S1&: TransitiveMapFunction[&F], |
| 214 | S2: VariablesReachableThroughFunctionPointer); |
| 215 | } |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | // Direct implementation of collecting all variables reachable from each |
| 220 | // function |
| 221 | for (Function &Func : M.functions()) { |
| 222 | if (Func.isDeclaration() || isKernel(F: Func)) |
| 223 | continue; |
| 224 | |
| 225 | DenseSet<Function *> seen; // catches cycles |
| 226 | SmallVector<Function *, 4> wip = {&Func}; |
| 227 | |
| 228 | while (!wip.empty()) { |
| 229 | Function *F = wip.pop_back_val(); |
| 230 | |
| 231 | // Can accelerate this by referring to transitive map for functions that |
| 232 | // have already been computed, with more care than this |
| 233 | set_union(S1&: TransitiveMapFunction[&Func], S2: DirectMapFunction[F]); |
| 234 | |
| 235 | for (const CallGraphNode::CallRecord &R : *CG[F]) { |
| 236 | Function *Ith = R.second->getFunction(); |
| 237 | if (Ith) { |
| 238 | if (!seen.contains(V: Ith)) { |
| 239 | seen.insert(V: Ith); |
| 240 | wip.push_back(Elt: Ith); |
| 241 | } |
| 242 | } |
| 243 | } |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | // Collect variables that are transitively used by functions whose address has |
| 248 | // escaped |
| 249 | for (Function *F : AddressTakenFuncs) { |
| 250 | set_union(S1&: VariablesReachableThroughFunctionPointer, |
| 251 | S2: TransitiveMapFunction[F]); |
| 252 | } |
| 253 | |
| 254 | // DirectMapKernel lists which variables are used by the kernel |
| 255 | // find the variables which are used through a function call |
| 256 | FunctionVariableMap IndirectMapKernel; |
| 257 | |
| 258 | for (Function &Func : M.functions()) { |
| 259 | if (Func.isDeclaration() || !isKernel(F: Func)) |
| 260 | continue; |
| 261 | |
| 262 | for (const CallGraphNode::CallRecord &R : *CG[&Func]) { |
| 263 | Function *Ith = R.second->getFunction(); |
| 264 | if (Ith) { |
| 265 | set_union(S1&: IndirectMapKernel[&Func], S2: TransitiveMapFunction[Ith]); |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | // Check if the kernel encounters unknows calls, wheher directly or |
| 270 | // indirectly. |
| 271 | bool SeesUnknownCalls = [&]() { |
| 272 | SmallVector<Function *> WorkList = {CG[&Func]->getFunction()}; |
| 273 | SmallPtrSet<Function *, 8> Visited; |
| 274 | |
| 275 | while (!WorkList.empty()) { |
| 276 | Function *F = WorkList.pop_back_val(); |
| 277 | |
| 278 | for (const CallGraphNode::CallRecord &CallRecord : *CG[F]) { |
| 279 | if (!CallRecord.second) |
| 280 | continue; |
| 281 | |
| 282 | Function *Callee = CallRecord.second->getFunction(); |
| 283 | if (!Callee) |
| 284 | return true; |
| 285 | |
| 286 | if (Visited.insert(Ptr: Callee).second) |
| 287 | WorkList.push_back(Elt: Callee); |
| 288 | } |
| 289 | } |
| 290 | return false; |
| 291 | }(); |
| 292 | |
| 293 | if (SeesUnknownCalls) { |
| 294 | set_union(S1&: IndirectMapKernel[&Func], |
| 295 | S2: VariablesReachableThroughFunctionPointer); |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | return {.DirectAccess: std::move(DirectMapKernel), .IndirectAccess: std::move(IndirectMapKernel)}; |
| 300 | } |
| 301 | |
| 302 | GVUsesInfoTy getTransitiveUsesOfLDSForLowering(const CallGraph &CG, Module &M) { |
| 303 | GVUsesInfoTy UsesInfo = getTransitiveUsesOfGV(CG, M, Filter: isLDSVariableToLower); |
| 304 | // Verify that we fall into one of 2 cases: |
| 305 | // - All variables are either absolute |
| 306 | // or direct mapped dynamic LDS that is not lowered. |
| 307 | // - No variables are absolute. |
| 308 | // Named-barriers which are absolute symbols are removed |
| 309 | // from the maps. |
| 310 | std::optional<bool> HasAbsoluteGVs; |
| 311 | for (auto &Map : {UsesInfo.DirectAccess, UsesInfo.IndirectAccess}) { |
| 312 | for (auto &[Fn, GVs] : Map) { |
| 313 | for (auto *GV : GVs) { |
| 314 | bool IsAbsolute = GV->isAbsoluteSymbolRef(); |
| 315 | bool IsDirectMapDynLDSGV = |
| 316 | AMDGPU::isDynamicLDS(GV: *GV) && UsesInfo.DirectAccess.contains(Key: Fn); |
| 317 | if (IsDirectMapDynLDSGV) |
| 318 | continue; |
| 319 | |
| 320 | if (HasAbsoluteGVs.has_value()) { |
| 321 | if (*HasAbsoluteGVs != IsAbsolute) { |
| 322 | reportFatalUsageError( |
| 323 | reason: "module cannot mix absolute and non-absolute LDS GVs" ); |
| 324 | } |
| 325 | } else |
| 326 | HasAbsoluteGVs = IsAbsolute; |
| 327 | } |
| 328 | } |
| 329 | } |
| 330 | |
| 331 | // If we only had absolute GVs, we have nothing to do, return an empty |
| 332 | // result. |
| 333 | if (HasAbsoluteGVs && *HasAbsoluteGVs) |
| 334 | return GVUsesInfoTy(); |
| 335 | |
| 336 | return UsesInfo; |
| 337 | } |
| 338 | |
| 339 | void removeFnAttrFromReachable(CallGraph &CG, Function *KernelRoot, |
| 340 | ArrayRef<StringRef> FnAttrs) { |
| 341 | for (StringRef Attr : FnAttrs) |
| 342 | KernelRoot->removeFnAttr(Kind: Attr); |
| 343 | |
| 344 | SmallVector<Function *> WorkList = {CG[KernelRoot]->getFunction()}; |
| 345 | SmallPtrSet<Function *, 8> Visited; |
| 346 | bool SeenUnknownCall = false; |
| 347 | |
| 348 | while (!WorkList.empty()) { |
| 349 | Function *F = WorkList.pop_back_val(); |
| 350 | |
| 351 | for (auto &CallRecord : *CG[F]) { |
| 352 | if (!CallRecord.second) |
| 353 | continue; |
| 354 | |
| 355 | Function *Callee = CallRecord.second->getFunction(); |
| 356 | if (!Callee) { |
| 357 | if (!SeenUnknownCall) { |
| 358 | SeenUnknownCall = true; |
| 359 | |
| 360 | // If we see any indirect calls, assume nothing about potential |
| 361 | // targets. |
| 362 | // TODO: This could be refined to possible LDS global users. |
| 363 | for (auto &ExternalCallRecord : *CG.getExternalCallingNode()) { |
| 364 | Function *PotentialCallee = |
| 365 | ExternalCallRecord.second->getFunction(); |
| 366 | assert(PotentialCallee); |
| 367 | if (!isKernel(F: *PotentialCallee)) { |
| 368 | for (StringRef Attr : FnAttrs) |
| 369 | PotentialCallee->removeFnAttr(Kind: Attr); |
| 370 | } |
| 371 | } |
| 372 | } |
| 373 | } else { |
| 374 | for (StringRef Attr : FnAttrs) |
| 375 | Callee->removeFnAttr(Kind: Attr); |
| 376 | if (Visited.insert(Ptr: Callee).second) |
| 377 | WorkList.push_back(Elt: Callee); |
| 378 | } |
| 379 | } |
| 380 | } |
| 381 | } |
| 382 | |
| 383 | bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) { |
| 384 | Instruction *DefInst = Def->getMemoryInst(); |
| 385 | |
| 386 | if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(Val: DefInst)) { |
| 387 | switch (II->getIntrinsicID()) { |
| 388 | case Intrinsic::amdgcn_s_barrier: |
| 389 | case Intrinsic::amdgcn_s_cluster_barrier: |
| 390 | case Intrinsic::amdgcn_s_barrier_signal: |
| 391 | case Intrinsic::amdgcn_s_barrier_signal_var: |
| 392 | case Intrinsic::amdgcn_s_barrier_signal_isfirst: |
| 393 | case Intrinsic::amdgcn_s_barrier_init: |
| 394 | case Intrinsic::amdgcn_s_barrier_join: |
| 395 | case Intrinsic::amdgcn_s_barrier_wait: |
| 396 | case Intrinsic::amdgcn_s_barrier_leave: |
| 397 | case Intrinsic::amdgcn_s_get_barrier_state: |
| 398 | case Intrinsic::amdgcn_s_wakeup_barrier: |
| 399 | case Intrinsic::amdgcn_wave_barrier: |
| 400 | case Intrinsic::amdgcn_sched_barrier: |
| 401 | case Intrinsic::amdgcn_sched_group_barrier: |
| 402 | case Intrinsic::amdgcn_iglp_opt: |
| 403 | return false; |
| 404 | default: |
| 405 | break; |
| 406 | } |
| 407 | } |
| 408 | |
| 409 | // Ignore non-acquire fences/atomics not aliasing with the original load; MSSA |
| 410 | // counts any atomic or fence as a universal MemoryDef. |
| 411 | // Acquire (or stronger) fences/atomics act as clobbers because they can bring |
| 412 | // in effects from other threads. |
| 413 | const auto MayAlias = [AA, Ptr](auto I) -> bool { |
| 414 | return !AA->isNoAlias(I->getPointerOperand(), Ptr); |
| 415 | }; |
| 416 | |
| 417 | if (const auto *F = dyn_cast<FenceInst>(Val: DefInst)) { |
| 418 | if (!isAcquireOrStronger(AO: F->getOrdering())) |
| 419 | return false; |
| 420 | SIAtomicAddrSpace AccessedAS = |
| 421 | toSIAtomicAddrSpace(AS: Ptr->getType()->getPointerAddressSpace()); |
| 422 | return (AccessedAS & getSynchronizeAddrSpace(I: *F)) != |
| 423 | SIAtomicAddrSpace::NONE; |
| 424 | } |
| 425 | |
| 426 | if (const auto *I = dyn_cast<AtomicRMWInst>(Val: DefInst)) |
| 427 | return isAcquireOrStronger(AO: I->getOrdering()) || MayAlias(I); |
| 428 | |
| 429 | if (const auto *I = dyn_cast<AtomicCmpXchgInst>(Val: DefInst)) |
| 430 | return isAcquireOrStronger(AO: I->getMergedOrdering()) || MayAlias(I); |
| 431 | |
| 432 | // TODO This is pessimistic for one-as load-acquires from a different AS. |
| 433 | if (const auto *I = dyn_cast<LoadInst>(Val: DefInst)) |
| 434 | return isAcquireOrStronger(AO: I->getOrdering()); |
| 435 | |
| 436 | // (Atomic) stores that don't alias do not clobber. |
| 437 | if (const auto *I = dyn_cast<StoreInst>(Val: DefInst)) |
| 438 | return MayAlias(I); |
| 439 | |
| 440 | return true; |
| 441 | } |
| 442 | |
| 443 | bool isClobberedInFunction(const LoadInst *Load, MemorySSA *MSSA, |
| 444 | AAResults *AA) { |
| 445 | MemorySSAWalker *Walker = MSSA->getWalker(); |
| 446 | MemoryLocation Loc(MemoryLocation::get(LI: Load)); |
| 447 | MemoryUseOrDef *Use = MSSA->getMemoryAccess(I: Load); |
| 448 | SmallVector<MemoryAccess *> WorkList{ |
| 449 | Walker->getClobberingMemoryAccess(MA: Use->getDefiningAccess(), Loc)}; |
| 450 | SmallPtrSet<MemoryAccess *, 8> Visited; |
| 451 | |
| 452 | LLVM_DEBUG(dbgs() << "Checking clobbering of: " << *Load << '\n'); |
| 453 | |
| 454 | // Start with a nearest dominating clobbering access, it will be either |
| 455 | // live on entry (nothing to do, load is not clobbered), MemoryDef, or |
| 456 | // MemoryPhi if several MemoryDefs can define this memory state. In that |
| 457 | // case add all Defs to WorkList and continue going up and checking all |
| 458 | // the definitions of this memory location until the root. When all the |
| 459 | // defs are exhausted and came to the entry state we have no clobber. |
| 460 | // Along the scan ignore barriers which are considered clobbers by the |
| 461 | // MemorySSA, but not really writing anything into the memory. |
| 462 | while (!WorkList.empty()) { |
| 463 | MemoryAccess *MA = WorkList.pop_back_val(); |
| 464 | if (!Visited.insert(Ptr: MA).second) |
| 465 | continue; |
| 466 | |
| 467 | if (MSSA->isLiveOnEntryDef(MA)) |
| 468 | continue; |
| 469 | |
| 470 | if (MemoryDef *Def = dyn_cast<MemoryDef>(Val: MA)) { |
| 471 | LLVM_DEBUG(dbgs() << " Def: " << *Def->getMemoryInst() << '\n'); |
| 472 | |
| 473 | if (isReallyAClobber(Ptr: Load->getPointerOperand(), Def, AA)) { |
| 474 | LLVM_DEBUG(dbgs() << " -> load is clobbered\n" ); |
| 475 | return true; |
| 476 | } |
| 477 | |
| 478 | WorkList.push_back( |
| 479 | Elt: Walker->getClobberingMemoryAccess(MA: Def->getDefiningAccess(), Loc)); |
| 480 | continue; |
| 481 | } |
| 482 | |
| 483 | const MemoryPhi *Phi = cast<MemoryPhi>(Val: MA); |
| 484 | for (const auto &Use : Phi->incoming_values()) |
| 485 | WorkList.push_back( |
| 486 | Elt: Walker->getClobberingMemoryAccess(MA: cast<MemoryAccess>(Val: &Use), Loc)); |
| 487 | } |
| 488 | |
| 489 | LLVM_DEBUG(dbgs() << " -> no clobber\n" ); |
| 490 | return false; |
| 491 | } |
| 492 | |
| 493 | static constexpr SIAtomicAddrSpace LLVMToSIAtomicAddrSpaceMapping[] = { |
| 494 | SIAtomicAddrSpace::FLAT, //< AMDGPUAS::FLAT_ADDRESS |
| 495 | SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::GLOBAL_ADDRESS |
| 496 | SIAtomicAddrSpace::GDS, //< AMDGPUAS::REGION_ADDRESS |
| 497 | SIAtomicAddrSpace::LDS, //< AMDGPUAS::LOCAL_ADDRESS |
| 498 | SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::CONSTANT_ADDRESS |
| 499 | SIAtomicAddrSpace::SCRATCH, //< AMDGPUAS::PRIVATE_ADDRESS |
| 500 | SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::CONSTANT_ADDRESS_32BIT |
| 501 | SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_FAT_POINTER |
| 502 | SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_RESOURCE |
| 503 | SIAtomicAddrSpace::GLOBAL, //< AMDGPUAS::BUFFER_STRIDED_POINTER |
| 504 | }; |
| 505 | |
| 506 | SIAtomicAddrSpace toSIAtomicAddrSpace(unsigned LLVMAddrSpace) { |
| 507 | constexpr unsigned SizeOfLLVMToSIAtomicAddrSpaceMapping = |
| 508 | sizeof(LLVMToSIAtomicAddrSpaceMapping) / |
| 509 | sizeof(LLVMToSIAtomicAddrSpaceMapping[0]); |
| 510 | if (LLVMAddrSpace > AMDGPUAS::MAX_AMDGPU_ADDRESS) |
| 511 | return SIAtomicAddrSpace::GLOBAL; |
| 512 | if (LLVMAddrSpace < SizeOfLLVMToSIAtomicAddrSpaceMapping) |
| 513 | return LLVMToSIAtomicAddrSpaceMapping[LLVMAddrSpace]; |
| 514 | return SIAtomicAddrSpace::OTHER; |
| 515 | } |
| 516 | |
| 517 | static const StringMap<SIAtomicAddrSpace> ASNames = {{ |
| 518 | {"global" , SIAtomicAddrSpace::GLOBAL}, |
| 519 | {"local" , SIAtomicAddrSpace::LDS}, |
| 520 | }}; |
| 521 | |
| 522 | SIAtomicAddrSpace getSynchronizeAddrSpace(const MMRAMetadata &MMRA, |
| 523 | const Function *DiagFn, |
| 524 | const DebugLoc &DL) { |
| 525 | static constexpr StringLiteral FenceASPrefix = "amdgpu-synchronize-as" ; |
| 526 | |
| 527 | if (!MMRA) |
| 528 | return SIAtomicAddrSpace::ATOMIC; |
| 529 | |
| 530 | SIAtomicAddrSpace Result{}; |
| 531 | for (const auto &[Prefix, Suffix] : MMRA) { |
| 532 | if (Prefix != FenceASPrefix) |
| 533 | continue; |
| 534 | |
| 535 | if (auto It = ASNames.find(Key: Suffix); It != ASNames.end()) { |
| 536 | Result |= It->second; |
| 537 | } else if (DiagFn) { |
| 538 | SmallString<128> Str; |
| 539 | raw_svector_ostream OS(Str); |
| 540 | OS << "unknown address space '" << Suffix << "'; expected one of " ; |
| 541 | ListSeparator LS; |
| 542 | for (const auto &[Name, Val] : ASNames) |
| 543 | OS << LS << '\'' << Name << '\''; |
| 544 | DiagFn->getContext().diagnose( |
| 545 | DI: DiagnosticInfoUnsupported(*DiagFn, Str, DL, DS_Warning)); |
| 546 | } |
| 547 | } |
| 548 | |
| 549 | if (Result == SIAtomicAddrSpace::NONE) |
| 550 | return SIAtomicAddrSpace::ATOMIC; |
| 551 | |
| 552 | return Result; |
| 553 | } |
| 554 | |
| 555 | SIAtomicAddrSpace getSynchronizeAddrSpace(const Instruction &I) { |
| 556 | return getSynchronizeAddrSpace(MMRA: MMRAMetadata(I)); |
| 557 | } |
| 558 | |
| 559 | } // end namespace llvm::AMDGPU |
| 560 | |