1//===- ACCImplicitDeclare.cpp ---------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass applies implicit `acc declare` actions to global variables
10// referenced in OpenACC compute regions and routine functions.
11//
12// Overview:
13// ---------
14// Global references in an acc regions (for globals not marked with `acc
15// declare` by the user) can be handled in one of two ways:
16// - Mapped through data clauses
17// - Implicitly marked as `acc declare` (this pass)
18//
19// Thus, the OpenACC specification focuses solely on implicit data mapping rules
20// whose implementation is captured in `ACCImplicitData` pass.
21//
22// However, it is both advantageous and required for certain cases to
23// use implicit `acc declare` instead:
24// - Any functions that are implicitly marked as `acc routine` through
25// `ACCImplicitRoutine` may reference globals. Since data mapping
26// is only possible for compute regions, such globals can only be
27// made available on device through `acc declare`.
28// - Compiler can generate and use globals for cases needed in IR
29// representation such as type descriptors or various names needed for
30// runtime calls and error reporting - such cases often are introduced
31// after a frontend semantic checking is done since it is related to
32// implementation detail. Thus, such compiler generated globals would
33// not have been visible for a user to mark with `acc declare`.
34// - Constant globals such as filename strings or data initialization values
35// are values that do not get mutated but are still needed for appropriate
36// runtime execution. If a kernel is launched 1000 times, it is not a
37// good idea to map such a global 1000 times. Therefore, such globals
38// benefit from being marked with `acc declare`.
39//
40// This pass automatically
41// marks global variables with the `acc.declare` attribute when they are
42// referenced in OpenACC compute constructs or routine functions and meet
43// the criteria noted above, ensuring
44// they are properly handled for device execution.
45//
46// The pass performs two main optimizations:
47//
48// 1. Hoisting: For non-constant globals and constant globals without a local
49// initializer referenced in compute regions, the pass hoists the address-of
50// operation out of the region when possible, allowing them to be implicitly
51// mapped through normal data clause mechanisms rather than requiring
52// declare marking.
53//
54// 2. Declaration: For globals that must be available on the device (constants,
55// globals in routines, globals in recipe operations), the pass adds the
56// `acc.declare` attribute with the copyin data clause.
57//
58// Requirements:
59// -------------
60// To use this pass in a pipeline, the following requirements must be met:
61//
62// 1. Operation Interface Implementation: Operations that compute addresses
63// of global variables must implement the `acc::AddressOfGlobalOpInterface`
64// and those that represent globals must implement the
65// `acc::GlobalOpInterface`. Additionally, any operations that indirectly
66// access globals must implement the `acc::IndirectGlobalAccessOpInterface`.
67//
68// 2. Analysis Registration (Optional): If custom behavior is needed for
69// determining if a symbol use is valid within GPU regions, the dialect
70// should pre-register the `acc::OpenACCSupport` analysis.
71//
72// Examples:
73// ---------
74//
75// Example 1: Non-constant global in compute region (hoisted)
76//
77// Before:
78// memref.global @g_scalar : memref<f32> = dense<0.0>
79// func.func @test() {
80// acc.serial {
81// %addr = memref.get_global @g_scalar : memref<f32>
82// %val = memref.load %addr[] : memref<f32>
83// acc.yield
84// }
85// }
86//
87// After:
88// memref.global @g_scalar : memref<f32> = dense<0.0>
89// func.func @test() {
90// %addr = memref.get_global @g_scalar : memref<f32>
91// acc.serial {
92// %val = memref.load %addr[] : memref<f32>
93// acc.yield
94// }
95// }
96//
97// Example 2: Constant global in compute region (declared)
98//
99// Before:
100// memref.global constant @g_const : memref<f32> = dense<1.0>
101// func.func @test() {
102// acc.serial {
103// %addr = memref.get_global @g_const : memref<f32>
104// %val = memref.load %addr[] : memref<f32>
105// acc.yield
106// }
107// }
108//
109// After:
110// memref.global constant @g_const : memref<f32> = dense<1.0>
111// {acc.declare = #acc.declare<dataClause = acc_copyin>}
112// func.func @test() {
113// acc.serial {
114// %addr = memref.get_global @g_const : memref<f32>
115// %val = memref.load %addr[] : memref<f32>
116// acc.yield
117// }
118// }
119//
120// Example 3: Global in acc routine (declared)
121//
122// Before:
123// memref.global @g_data : memref<f32> = dense<0.0>
124// acc.routine @routine_0 func(@device_func)
125// func.func @device_func() attributes {acc.routine_info = ...} {
126// %addr = memref.get_global @g_data : memref<f32>
127// %val = memref.load %addr[] : memref<f32>
128// }
129//
130// After:
131// memref.global @g_data : memref<f32> = dense<0.0>
132// {acc.declare = #acc.declare<dataClause = acc_copyin>}
133// acc.routine @routine_0 func(@device_func)
134// func.func @device_func() attributes {acc.routine_info = ...} {
135// %addr = memref.get_global @g_data : memref<f32>
136// %val = memref.load %addr[] : memref<f32>
137// }
138//
139// Example 4: Global in private recipe (declared if recipe is used)
140//
141// Before:
142// memref.global @g_init : memref<f32> = dense<0.0>
143// acc.private.recipe @priv_recipe : memref<f32> init {
144// ^bb0(%arg0: memref<f32>):
145// %alloc = memref.alloc() : memref<f32>
146// %global = memref.get_global @g_init : memref<f32>
147// %val = memref.load %global[] : memref<f32>
148// memref.store %val, %alloc[] : memref<f32>
149// acc.yield %alloc : memref<f32>
150// } destroy { ... }
151// func.func @test() {
152// %var = memref.alloc() : memref<f32>
153// %priv = acc.private varPtr(%var : memref<f32>)
154// recipe(@priv_recipe) -> memref<f32>
155// acc.parallel private(%priv : memref<f32>) { ... }
156// }
157//
158// After:
159// memref.global @g_init : memref<f32> = dense<0.0>
160// {acc.declare = #acc.declare<dataClause = acc_copyin>}
161// acc.private.recipe @priv_recipe : memref<f32> init {
162// ^bb0(%arg0: memref<f32>):
163// %alloc = memref.alloc() : memref<f32>
164// %global = memref.get_global @g_init : memref<f32>
165// %val = memref.load %global[] : memref<f32>
166// memref.store %val, %alloc[] : memref<f32>
167// acc.yield %alloc : memref<f32>
168// } destroy { ... }
169// func.func @test() {
170// %var = memref.alloc() : memref<f32>
171// %priv = acc.private varPtr(%var : memref<f32>)
172// recipe(@priv_recipe) -> memref<f32>
173// acc.parallel private(%priv : memref<f32>) { ... }
174// }
175//
176//===----------------------------------------------------------------------===//
177
178#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
179#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
180
181#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
182#include "mlir/Dialect/OpenACC/OpenACC.h"
183#include "mlir/IR/Builders.h"
184#include "mlir/IR/BuiltinAttributes.h"
185#include "mlir/IR/BuiltinOps.h"
186#include "mlir/IR/Operation.h"
187#include "mlir/IR/Value.h"
188#include "mlir/Interfaces/FunctionInterfaces.h"
189#include "llvm/ADT/DenseSet.h"
190#include "llvm/ADT/SmallVector.h"
191#include "llvm/ADT/TypeSwitch.h"
192
193namespace mlir {
194namespace acc {
195#define GEN_PASS_DEF_ACCIMPLICITDECLARE
196#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
197} // namespace acc
198} // namespace mlir
199
200#define DEBUG_TYPE "acc-implicit-declare"
201
202using namespace mlir;
203
204namespace {
205using GlobalOpSetT = llvm::SmallSetVector<Operation *, 16>;
206
207/// Checks whether a use of the requested `globalOp` should be considered
208/// for hoisting out of acc region due to avoid `acc declare`ing something
209/// that instead should be implicitly mapped.
210static bool isGlobalUseCandidateForHoisting(Operation *globalOp,
211 Operation *user,
212 SymbolRefAttr symbol,
213 acc::OpenACCSupport &accSupport) {
214 // This symbol is valid in GPU region. This means semantics
215 // would change if moved to host - therefore it is not a candidate.
216 if (accSupport.isValidSymbolUse(user, symbol))
217 return false;
218
219 bool isInitializedConstant = false;
220 bool isFunction = false;
221
222 if (auto globalVarOp = dyn_cast<acc::GlobalVariableOpInterface>(Val: globalOp))
223 isInitializedConstant =
224 globalVarOp.isConstant() && globalVarOp.hasInitializer();
225
226 if (isa<FunctionOpInterface>(Val: globalOp))
227 isFunction = true;
228
229 // Initialized constants should be kept in device code so their definitions
230 // can be duplicated in the device image. An initializer-less constant is an
231 // external declaration, so keeping its address-of in device code would
232 // create a device symbol that a separately compiled defining translation
233 // unit may not provide. Hoist it so normal implicit mapping passes its host
234 // definition to the compute region instead.
235 //
236 // Function references should be kept in device code to ensure their device
237 // addresses are computed. Everything else should be hoisted since we already
238 // proved it is not a valid symbol in the GPU region.
239 return !isInitializedConstant && !isFunction;
240}
241
242/// Checks whether it is valid to use acc.declare marking on the global.
243bool isValidForAccDeclare(Operation *globalOp) {
244 // For functions - we use acc.routine marking instead.
245 return !isa<FunctionOpInterface>(Val: globalOp);
246}
247
248/// Collect remaining symbol uses with a single walk. Asking whether each
249/// recipe is referenced via getSymbolUses walks the whole module again, and
250/// recipes are generated per type, so there can be many of them. A recipe
251/// referring only to its own symbol is not a "relevant" use (it does not
252/// mean the recipe is attached to a compute construct).
253static std::optional<llvm::DenseSet<StringAttr>>
254collectUsedSymbolsExcludingRecipeSelfUses(ModuleOp mod) {
255 // The module region, not the module op, is the symbol table scope: asking
256 // for the uses on the op itself would not walk into the body.
257 std::optional<SymbolTable::UseRange> uses =
258 SymbolTable::getSymbolUses(from: &mod.getBodyRegion());
259 if (!uses)
260 return std::nullopt;
261
262 llvm::DenseSet<StringAttr> usedSymbols;
263 auto isRecipeSelfUse = [](Operation *user, StringAttr name) {
264 if (auto recipe = dyn_cast<acc::PrivateRecipeOp>(Val: user))
265 return recipe.getNameAttr() == name;
266 if (auto recipe = dyn_cast<acc::FirstprivateRecipeOp>(Val: user))
267 return recipe.getNameAttr() == name;
268 if (auto recipe = dyn_cast<acc::ReductionRecipeOp>(Val: user))
269 return recipe.getNameAttr() == name;
270 return false;
271 };
272 for (const SymbolTable::SymbolUse &use : *uses) {
273 StringAttr name = use.getSymbolRef().getLeafReference();
274 if (!isRecipeSelfUse(use.getUser(), name))
275 usedSymbols.insert(V: name);
276 }
277 return usedSymbols;
278}
279
280/// Checks whether a recipe operation has meaningful use of its symbol that
281/// justifies processing its regions for global references. Returns false if:
282/// 1. The recipe has no symbol uses at all, or
283/// 2. The only symbol use is the recipe's own symbol definition
284template <typename RecipeOpT>
285static bool hasRelevantRecipeUse(
286 RecipeOpT &recipeOp, ModuleOp &mod,
287 const std::optional<llvm::DenseSet<StringAttr>> &usedSymbols) {
288 auto recipeName = recipeOp.getNameAttr();
289 if (usedSymbols)
290 return usedSymbols->contains(V: recipeName);
291
292 std::optional<SymbolTable::UseRange> symbolUses = recipeOp.getSymbolUses(mod);
293
294 // No recipe symbol uses.
295 if (!symbolUses.has_value() || symbolUses->empty())
296 return false;
297
298 // If more than one use, assume it's used.
299 auto begin = symbolUses->begin();
300 auto end = symbolUses->end();
301 if (begin != end && std::next(x: begin) != end)
302 return true;
303
304 // If single use, check if the use is the recipe itself.
305 const SymbolTable::SymbolUse &use = *symbolUses->begin();
306 return use.getUser() != recipeOp.getOperation();
307}
308
309// Hoists addr_of operations for globals that cannot be defined in this
310// translation unit out of OpenACC regions. This way they are implicitly mapped
311// instead of being considered for implicit declare.
312template <typename AccConstructT>
313static void hoistNonConstantDirectUses(AccConstructT accOp,
314 acc::OpenACCSupport &accSupport) {
315 accOp.walk([&](acc::AddressOfGlobalOpInterface addrOfOp) {
316 SymbolRefAttr symRef = addrOfOp.getSymbol();
317 if (symRef) {
318 Operation *globalOp =
319 SymbolTable::lookupNearestSymbolFrom(from: addrOfOp, symbol: symRef);
320 if (isGlobalUseCandidateForHoisting(globalOp, user: addrOfOp, symbol: symRef,
321 accSupport)) {
322 auto computeRegionParent =
323 addrOfOp->getParentOfType<acc::ComputeRegionOp>();
324 addrOfOp->moveBefore(accOp);
325 if (computeRegionParent)
326 for (Value v : addrOfOp->getResults())
327 computeRegionParent.wireHoistedValueThroughIns(value: v);
328 LLVM_DEBUG(
329 llvm::dbgs() << "Hoisted:\n\t" << addrOfOp << "\n\tfrom:\n\t";
330 accOp->print(llvm::dbgs(),
331 OpPrintingFlags{}.skipRegions().enableDebugInfo());
332 llvm::dbgs() << "\n");
333 }
334 }
335 });
336}
337
338// Collects the globals referenced in a device region
339static void collectGlobalsFromDeviceRegion(Region &region,
340 GlobalOpSetT &globals,
341 acc::OpenACCSupport &accSupport,
342 SymbolTable &symTab) {
343 region.walk(callback: [&](Operation *op) {
344 // 1) Only consider relevant operations which use symbols
345 auto addrOfOp = dyn_cast<acc::AddressOfGlobalOpInterface>(Val: op);
346 if (addrOfOp) {
347 SymbolRefAttr symRef = addrOfOp.getSymbol();
348 // 2) Found an operation which uses the symbol. Next determine if it
349 // is a candidate for `acc declare`. Some of the criteria considered
350 // is whether this symbol is not already a device one (either because
351 // acc declare is already used or this is a CUF global).
352 Operation *globalOp = nullptr;
353 bool isCandidate = !accSupport.isValidSymbolUse(user: op, symbol: symRef, definingOpPtr: &globalOp);
354 // 3) Add the candidate to the set of globals to be `acc declare`d.
355 if (isCandidate && globalOp && isValidForAccDeclare(globalOp))
356 globals.insert(X: globalOp);
357 } else if (auto indirectAccessOp =
358 dyn_cast<acc::IndirectGlobalAccessOpInterface>(Val: op)) {
359 // Process operations that indirectly access globals
360 llvm::SmallVector<SymbolRefAttr> symbols;
361 indirectAccessOp.getReferencedSymbols(symbols, symbolTable: &symTab);
362 for (SymbolRefAttr symRef : symbols)
363 if (Operation *globalOp = symTab.lookup(name: symRef.getLeafReference()))
364 if (isValidForAccDeclare(globalOp))
365 globals.insert(X: globalOp);
366 }
367 });
368}
369
370// Adds the declare attribute to the operation `op`.
371static void addDeclareAttr(MLIRContext *context, Operation *op,
372 acc::DataClause clause) {
373 op->setDiscardableAttr(
374 name: acc::getDeclareAttrName(),
375 value: acc::DeclareAttr::get(context,
376 dataClause: acc::DataClauseAttr::get(context, value: clause)));
377}
378
379// This pass applies implicit declare actions for globals referenced in
380// OpenACC compute and routine regions.
381class ACCImplicitDeclare
382 : public acc::impl::ACCImplicitDeclareBase<ACCImplicitDeclare> {
383public:
384 using ACCImplicitDeclareBase<ACCImplicitDeclare>::ACCImplicitDeclareBase;
385
386 void getDependentDialects(DialectRegistry &registry) const override {
387 ACCImplicitDeclareBase<ACCImplicitDeclare>::getDependentDialects(registry);
388 acc::registerAllExtensions(registry);
389 }
390
391 void runOnOperation() override {
392 ModuleOp mod = getOperation();
393 MLIRContext *context = &getContext();
394 acc::OpenACCSupport &accSupport = getAnalysis<acc::OpenACCSupport>();
395
396 // 1) Start off by hoisting any AddressOf operations out of acc region
397 // for any cases we do not want to `acc declare`. This is because we can
398 // rely on implicit data mapping in majority of cases without uselessly
399 // polluting the device globals.
400 mod.walk(callback: [&](Operation *op) {
401 TypeSwitch<Operation *, void>(op)
402 .Case<ACC_COMPUTE_CONSTRUCT_OPS, acc::ComputeRegionOp>(
403 caseFn: [&](auto accOp) {
404 hoistNonConstantDirectUses(accOp, accSupport);
405 });
406 });
407
408 // 2) Collect global symbols which need to be `acc declare`d. Do it for
409 // compute regions, acc routine, and existing globals with the declare
410 // attribute.
411 SymbolTable symTab(mod);
412 GlobalOpSetT globalsToAccDeclare;
413 std::optional<llvm::DenseSet<StringAttr>> usedSymbols =
414 collectUsedSymbolsExcludingRecipeSelfUses(mod);
415 mod.walk(callback: [&](Operation *op) {
416 TypeSwitch<Operation *, void>(op)
417 .Case<ACC_COMPUTE_CONSTRUCT_OPS, acc::ComputeRegionOp>(
418 caseFn: [&](auto accOp) {
419 collectGlobalsFromDeviceRegion(
420 accOp.getRegion(), globalsToAccDeclare, accSupport, symTab);
421 })
422 .Case(caseFn: [&](FunctionOpInterface func) {
423 if ((acc::isAccRoutine(op: func) ||
424 acc::isSpecializedAccRoutine(op: func)) &&
425 !func.isExternal())
426 collectGlobalsFromDeviceRegion(region&: func.getFunctionBody(),
427 globals&: globalsToAccDeclare, accSupport,
428 symTab);
429 })
430 .Case(caseFn: [&](acc::GlobalVariableOpInterface globalVarOp) {
431 if (globalVarOp->getDiscardableAttr(name: acc::getDeclareAttrName()))
432 if (Region *initRegion = globalVarOp.getInitRegion())
433 collectGlobalsFromDeviceRegion(region&: *initRegion, globals&: globalsToAccDeclare,
434 accSupport, symTab);
435 })
436 .Case(caseFn: [&](acc::PrivateRecipeOp privateRecipe) {
437 if (hasRelevantRecipeUse(recipeOp&: privateRecipe, mod, usedSymbols)) {
438 collectGlobalsFromDeviceRegion(region&: privateRecipe.getInitRegion(),
439 globals&: globalsToAccDeclare, accSupport,
440 symTab);
441 collectGlobalsFromDeviceRegion(region&: privateRecipe.getDestroyRegion(),
442 globals&: globalsToAccDeclare, accSupport,
443 symTab);
444 }
445 })
446 .Case(caseFn: [&](acc::FirstprivateRecipeOp firstprivateRecipe) {
447 if (hasRelevantRecipeUse(recipeOp&: firstprivateRecipe, mod, usedSymbols)) {
448 collectGlobalsFromDeviceRegion(region&: firstprivateRecipe.getInitRegion(),
449 globals&: globalsToAccDeclare, accSupport,
450 symTab);
451 collectGlobalsFromDeviceRegion(
452 region&: firstprivateRecipe.getDestroyRegion(), globals&: globalsToAccDeclare,
453 accSupport, symTab);
454 collectGlobalsFromDeviceRegion(region&: firstprivateRecipe.getCopyRegion(),
455 globals&: globalsToAccDeclare, accSupport,
456 symTab);
457 }
458 })
459 .Case(caseFn: [&](acc::ReductionRecipeOp reductionRecipe) {
460 if (hasRelevantRecipeUse(recipeOp&: reductionRecipe, mod, usedSymbols)) {
461 collectGlobalsFromDeviceRegion(region&: reductionRecipe.getInitRegion(),
462 globals&: globalsToAccDeclare, accSupport,
463 symTab);
464 collectGlobalsFromDeviceRegion(
465 region&: reductionRecipe.getCombinerRegion(), globals&: globalsToAccDeclare,
466 accSupport, symTab);
467 }
468 });
469 });
470
471 // 3) Finally, generate the appropriate declare actions needed to ensure
472 // this is considered for device global.
473 for (Operation *globalOp : globalsToAccDeclare) {
474 LLVM_DEBUG(
475 llvm::dbgs() << "Global is being `acc declare copyin`d: ";
476 globalOp->print(llvm::dbgs(),
477 OpPrintingFlags{}.skipRegions().enableDebugInfo());
478 llvm::dbgs() << "\n");
479
480 // Mark it as declare copyin.
481 addDeclareAttr(context, op: globalOp, clause: acc::DataClause::acc_copyin);
482
483 // TODO: May need to create the global constructor which does the mapping
484 // action. It is not yet clear if this is needed yet (since the globals
485 // might just end up in the GPU image without requiring mapping via
486 // runtime).
487 }
488 }
489};
490} // namespace
491