1//===- ACCDeclareGPUModuleInsertion.cpp -----------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass copies globals marked with the `acc.declare` attribute into the
10// GPU module so that device code (e.g. acc routine, compute regions) can
11// reference them.
12//
13// Overview:
14// ---------
15// Globals that have the `acc.declare` attribute (from the OpenACC declare
16// directive or from the `ACCImplicitDeclare` pass) must be present in the
17// GPU module for device code to use them. This pass inserts copies of those
18// globals into the GPU module, creating the module if it does not yet exist.
19// The host copy of each global remains in the parent module.
20//
21// Example:
22// --------
23//
24// Before:
25// module {
26// memref.global @arr : memref<7xf32> = dense<0.0>
27// {acc.declare = #acc.declare<dataClause = acc_create>}
28// }
29//
30// After:
31// module attributes {gpu.container_module} {
32// memref.global @arr : memref<7xf32> = dense<0.0>
33// {acc.declare = #acc.declare<dataClause = acc_create>}
34// gpu.module @acc_gpu_module {
35// memref.global @arr : memref<7xf32> = dense<0.0>
36// {acc.declare = #acc.declare<dataClause = acc_create>}
37// }
38// }
39//
40// Requirements:
41// -------------
42// The pass uses the `acc::OpenACCSupport` for:
43// - getOrCreateGPUModule: to obtain or create the GPU module.
44// - emitNYI: to report failure when GPU module creation is not supported.
45// If no custom implementation is registered, the default implementation is
46// used (see OpenACCSupport).
47//
48//===----------------------------------------------------------------------===//
49
50#include "mlir/Dialect/GPU/IR/GPUDialect.h"
51#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
52#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
53#include "mlir/Dialect/OpenACC/OpenACC.h"
54#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
55#include "mlir/IR/BuiltinOps.h"
56#include "mlir/IR/Operation.h"
57#include "mlir/IR/OperationSupport.h"
58#include "mlir/IR/SymbolTable.h"
59
60namespace mlir {
61namespace acc {
62#define GEN_PASS_DEF_ACCDECLAREGPUMODULEINSERTION
63#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
64} // namespace acc
65} // namespace mlir
66
67#define DEBUG_TYPE "acc-declare-gpu-module-insertion"
68
69using namespace mlir;
70
71namespace {
72static bool hasAccDeclareGlobals(ModuleOp mod) {
73 for (Operation &op : mod.getBody()->getOperations())
74 if (op.getDiscardableAttr(name: acc::getDeclareAttrName()))
75 return true;
76 return false;
77}
78
79static void makeDeviceGlobalDeclaration(Operation &globalOp) {
80 globalOp.setInherentAttr(name: StringAttr::get(context: globalOp.getContext(), bytes: "initVal"),
81 value: {});
82 globalOp.setInherentAttr(name: StringAttr::get(context: globalOp.getContext(), bytes: "linkage"),
83 value: {});
84 for (Region &region : globalOp.getRegions()) {
85 region.dropAllReferences();
86 region.getBlocks().clear();
87 }
88}
89
90class ACCDeclareGPUModuleInsertion
91 : public acc::impl::ACCDeclareGPUModuleInsertionBase<
92 ACCDeclareGPUModuleInsertion> {
93public:
94 using acc::impl::ACCDeclareGPUModuleInsertionBase<
95 ACCDeclareGPUModuleInsertion>::ACCDeclareGPUModuleInsertionBase;
96
97 void getDependentDialects(DialectRegistry &registry) const override {
98 acc::impl::ACCDeclareGPUModuleInsertionBase<
99 ACCDeclareGPUModuleInsertion>::getDependentDialects(registry);
100 acc::registerAllExtensions(registry);
101 }
102
103 LogicalResult copyGlobalsToGPUModule(gpu::GPUModuleOp gpuMod, ModuleOp mod,
104 acc::OpenACCSupport &accSupport) const {
105 SymbolTable gpuSymTable(gpuMod);
106
107 for (Operation &globalOp : mod.getBody()->getOperations()) {
108 if (!globalOp.getDiscardableAttr(name: acc::getDeclareAttrName()))
109 continue;
110
111 auto symOp = dyn_cast<SymbolOpInterface>(Val: &globalOp);
112 if (!symOp)
113 continue;
114
115 StringAttr name = symOp.getNameAttr();
116 Operation *deviceGlobal = globalOp.clone();
117 auto declareAttr = globalOp.getDiscardableAttrOfType<acc::DeclareAttr>(
118 name: acc::getDeclareAttrName());
119 auto globalVar = dyn_cast<acc::GlobalVariableOpInterface>(Val: &globalOp);
120 bool makeUnifiedDeclaration =
121 cudaUnified &&
122 declareAttr.getDataClause().getValue() !=
123 acc::DataClause::acc_declare_device_resident &&
124 (!globalVar || !globalVar.isConstant() ||
125 globalVar.isCompilerGenerated());
126 if (makeUnifiedDeclaration)
127 makeDeviceGlobalDeclaration(globalOp&: *deviceGlobal);
128
129 if (Operation *existing = gpuSymTable.lookup(name: name.getValue())) {
130 // Reuse when structurally equivalent ignoring locations and discardable
131 // attrs such as `acc.declare` attributes. Only a different op type or a
132 // true definition mismatch is a conflict.
133 auto isEquivalent = [](Operation *lhs, Operation *rhs) {
134 return lhs->getName() == rhs->getName() &&
135 OperationEquivalence::isEquivalentTo(
136 lhs, rhs, checkEquivalent: OperationEquivalence::ignoreValueEquivalence,
137 /*markEquivalent=*/nullptr,
138 flags: OperationEquivalence::IgnoreLocations |
139 OperationEquivalence::IgnoreDiscardableAttrs);
140 };
141 if (!isEquivalent(existing, deviceGlobal)) {
142 // Earlier GPU lowering can create a global in the GPU module before
143 // this pass. In unified memory, convert an equivalent pre-existing
144 // global to the declaration form expected for an OpenACC global.
145 if (makeUnifiedDeclaration) {
146 Operation *normalizedExisting = existing->clone();
147 makeDeviceGlobalDeclaration(globalOp&: *normalizedExisting);
148 bool canReuse = isEquivalent(normalizedExisting, deviceGlobal);
149 normalizedExisting->destroy();
150 if (canReuse)
151 makeDeviceGlobalDeclaration(globalOp&: *existing);
152 }
153 }
154 if (!isEquivalent(existing, deviceGlobal)) {
155 deviceGlobal->destroy();
156 accSupport.emitNYI(loc: globalOp.getLoc(),
157 message: llvm::Twine("duplicate global symbol '") +
158 name.getValue() + "' in gpu module");
159 return failure();
160 }
161 // Propagate acc.declare onto the GPU copy if it was cloned before the
162 // host global was marked.
163 if (!existing->getDiscardableAttr(name: acc::getDeclareAttrName()))
164 if (Attribute declareAttr =
165 globalOp.getDiscardableAttr(name: acc::getDeclareAttrName()))
166 existing->setDiscardableAttr(name: acc::getDeclareAttrName(),
167 value: declareAttr);
168 deviceGlobal->destroy();
169 continue;
170 }
171
172 gpuSymTable.insert(symbol: deviceGlobal);
173 }
174 return success();
175 }
176
177 void runOnOperation() override {
178 ModuleOp mod = getOperation();
179
180 // Check for any candidates first - do this to avoid creating the GPU module
181 // if there are no candidates.
182 if (!hasAccDeclareGlobals(mod))
183 return;
184
185 acc::OpenACCSupport &accSupport = getAnalysis<acc::OpenACCSupport>();
186 std::optional<gpu::GPUModuleOp> gpuMod =
187 accSupport.getOrCreateGPUModule(mod);
188 if (!gpuMod) {
189 accSupport.emitNYI(loc: mod.getLoc(), message: "Failed to create GPU module");
190 return;
191 }
192
193 if (failed(Result: copyGlobalsToGPUModule(gpuMod: *gpuMod, mod, accSupport)))
194 return;
195 }
196};
197} // namespace
198