| 1 | //===-- NVPTXCanonicalizeDynamicShared.cpp - Merge dynamic shared decls ---===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Every unsized `.extern .shared` array names the start of the same dynamic |
| 10 | // shared memory allocation, so `extern __shared__ int a[]` and |
| 11 | // `extern __shared__ float b[]` are the same address. LLVM sees two distinct |
| 12 | // global variables, which alias analysis treats as disjoint objects, and can |
| 13 | // then reorder or forward a store through @a past a load through @b. |
| 14 | // |
| 15 | // This pass gives all such declarations one underlying object. It redirects |
| 16 | // their uses to a single declaration that carries the strongest alignment and |
| 17 | // the debug info of each source variable, and erases the others unless a used |
| 18 | // list names them. |
| 19 | // |
| 20 | //===----------------------------------------------------------------------===// |
| 21 | |
| 22 | #include "NVPTX.h" |
| 23 | #include "llvm/ADT/SmallPtrSet.h" |
| 24 | #include "llvm/IR/Constants.h" |
| 25 | #include "llvm/IR/DebugInfoMetadata.h" |
| 26 | #include "llvm/IR/GlobalVariable.h" |
| 27 | #include "llvm/IR/Module.h" |
| 28 | #include "llvm/Pass.h" |
| 29 | |
| 30 | using namespace llvm; |
| 31 | |
| 32 | static bool isDynamicSharedDecl(const GlobalVariable &GV) { |
| 33 | const auto *AT = dyn_cast<ArrayType>(Val: GV.getValueType()); |
| 34 | return GV.isDeclaration() && GV.hasExternalLinkage() && AT && |
| 35 | AT->getNumElements() == 0 && |
| 36 | GV.getAddressSpace() == NVPTX::AddressSpace::Shared; |
| 37 | } |
| 38 | |
| 39 | static void restoreUsedList(GlobalVariable *List, |
| 40 | ArrayRef<GlobalValue *> Values) { |
| 41 | if (!List) |
| 42 | return; |
| 43 | auto *AT = cast<ArrayType>(Val: List->getValueType()); |
| 44 | SmallVector<Constant *, 8> Init; |
| 45 | for (GlobalValue *V : Values) |
| 46 | Init.push_back(Elt: ConstantExpr::getPointerBitCastOrAddrSpaceCast( |
| 47 | C: V, Ty: AT->getElementType())); |
| 48 | List->setInitializer(ConstantArray::get(T: AT, V: Init)); |
| 49 | } |
| 50 | |
| 51 | static bool canonicalizeDynamicShared(Module &M) { |
| 52 | SmallVector<GlobalVariable *, 8> DynSharedDecls; |
| 53 | for (GlobalVariable &GV : M.globals()) |
| 54 | if (isDynamicSharedDecl(GV)) |
| 55 | DynSharedDecls.push_back(Elt: &GV); |
| 56 | if (DynSharedDecls.size() < 2) |
| 57 | return false; |
| 58 | |
| 59 | // Redirecting uses also rewrites the used lists. Rebuild them at the end so |
| 60 | // that they keep naming the original symbols. |
| 61 | SmallVector<GlobalValue *, 8> Used, CompilerUsed; |
| 62 | GlobalVariable *UsedList = collectUsedGlobalVariables(M, Vec&: Used, CompilerUsed: false); |
| 63 | GlobalVariable *CompilerUsedList = |
| 64 | collectUsedGlobalVariables(M, Vec&: CompilerUsed, CompilerUsed: true); |
| 65 | SmallPtrSet<GlobalValue *, 8> InUsedList(llvm::from_range, Used); |
| 66 | InUsedList.insert_range(R&: CompilerUsed); |
| 67 | |
| 68 | const DataLayout &DL = M.getDataLayout(); |
| 69 | GlobalVariable *Canonical = nullptr; |
| 70 | Align MaxAlign; |
| 71 | bool Changed = false; |
| 72 | for (GlobalVariable *GV : DynSharedDecls) { |
| 73 | GV->removeDeadConstantUsers(); |
| 74 | if (GV->use_empty()) |
| 75 | continue; |
| 76 | MaxAlign = std::max(a: MaxAlign, b: DL.getValueOrABITypeAlignment( |
| 77 | Alignment: GV->getAlign(), Ty: GV->getValueType())); |
| 78 | if (!Canonical) { |
| 79 | Canonical = GV; |
| 80 | continue; |
| 81 | } |
| 82 | |
| 83 | // A write through any name may change what the others read. |
| 84 | if (!GV->isConstant()) |
| 85 | Canonical->setConstant(false); |
| 86 | |
| 87 | // Each source variable keeps its name and type at the common address. |
| 88 | SmallVector<DIGlobalVariableExpression *, 4> GVEs; |
| 89 | GV->getDebugInfo(GVs&: GVEs); |
| 90 | for (DIGlobalVariableExpression *GVE : GVEs) |
| 91 | Canonical->addDebugInfo(GV: GVE); |
| 92 | GV->eraseMetadata(KindID: LLVMContext::MD_dbg); |
| 93 | |
| 94 | GV->replaceAllUsesWith(V: Canonical); |
| 95 | if (!InUsedList.contains(Ptr: GV)) |
| 96 | GV->eraseFromParent(); |
| 97 | Changed = true; |
| 98 | } |
| 99 | if (Changed) |
| 100 | Canonical->setAlignment(MaxAlign); |
| 101 | |
| 102 | restoreUsedList(List: UsedList, Values: Used); |
| 103 | restoreUsedList(List: CompilerUsedList, Values: CompilerUsed); |
| 104 | return Changed; |
| 105 | } |
| 106 | |
| 107 | namespace { |
| 108 | class NVPTXCanonicalizeDynamicSharedLegacyPass : public ModulePass { |
| 109 | public: |
| 110 | static char ID; |
| 111 | NVPTXCanonicalizeDynamicSharedLegacyPass() : ModulePass(ID) {} |
| 112 | |
| 113 | bool runOnModule(Module &M) override { return canonicalizeDynamicShared(M); } |
| 114 | }; |
| 115 | } // namespace |
| 116 | |
| 117 | char NVPTXCanonicalizeDynamicSharedLegacyPass::ID = 0; |
| 118 | |
| 119 | INITIALIZE_PASS(NVPTXCanonicalizeDynamicSharedLegacyPass, |
| 120 | "nvptx-canonicalize-dynamic-shared" , |
| 121 | "Merge NVPTX dynamic shared memory declarations" , false, false) |
| 122 | |
| 123 | ModulePass *llvm::createNVPTXCanonicalizeDynamicSharedLegacyPass() { |
| 124 | return new NVPTXCanonicalizeDynamicSharedLegacyPass(); |
| 125 | } |
| 126 | |
| 127 | PreservedAnalyses |
| 128 | NVPTXCanonicalizeDynamicSharedPass::run(Module &M, ModuleAnalysisManager &) { |
| 129 | if (!canonicalizeDynamicShared(M)) |
| 130 | return PreservedAnalyses::all(); |
| 131 | return PreservedAnalyses::none().preserveSet<CFGAnalyses>(); |
| 132 | } |
| 133 | |