1//===- ACCRoutineToGPUFunc.cpp - Move ACC routines to GPU module ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// The OpenACC `routine` directive defines functions that may be invoked from
10// device code. Those functions need to be available in the device compilation
11// unit. This pass moves materialized acc routines into the GPU module as
12// gpu.func operations so they can be compiled for the device.
13//
14// Overview:
15// ---------
16// For each acc.routine that is not bound by name, the corresponding
17// specialized function (created by ACCRoutineLowering) or the original
18// host function (in case of seq) is cloned into theGPU module as a gpu.func.
19// Callees referenced from those routines are processed: device-valid callees
20// (runtime, intrinsics, other acc routines) are added to the GPU module as
21// declarations or full clones as needed. Bind-name routines are not moved;
22// their acc.routine ops are erased. After cloning, the host copies of
23// specialized device functions and nohost routines are removed.
24//
25// Approach:
26// ----------------
27// 1. Collect materialized routines (acc.routine without bind(name)); record
28// bind-name routines for erasure. Emit remarks for materialized routines.
29//
30// 2. Process calls: walk each materialized function; for each call, if the
31// callee is already in the GPU module or is an acc routine (or specialized
32// acc routine), skip; otherwise require OpenACCSupport::isValidSymbolUse.
33// Valid callees are added to the clone set (as declaration or full clone).
34//
35// 3. Clone into GPU module: each function in the clone set is turned into a
36// gpu.func (body cloned or declaration only). acc.specialized_routine is
37// preserved and symbol uses are updated so the routine name is unchanged.
38//
39// 4. Cleanup: erase from the host module the specialized device function
40// bodies and any nohost routine (host copy removed after move to device).
41//
42// Example:
43// --------
44// Before (after ACCRoutineLowering):
45// acc.routine @r_seq func(@foo) seq
46// func.func @foo() attributes {acc.specialized_routine = ...} { ... }
47//
48// After:
49// acc.routine @r_seq func(@foo) seq
50// gpu.module @acc_gpu_module {
51// gpu.func @foo() attributes {acc.specialized_routine = ...} { ... }
52// }
53// (host @foo erased)
54//
55// Requirements:
56// -------------
57// - Must run after `ACCRoutineLowering` pass which ensures variants for all
58// levels of parallelism are created.
59// - Uses OpenACCSupport: getOrCreateGPUModule, isValidSymbolUse, emitRemark,
60// emitNYI. If no custom implementation is registered, the default is used.
61//
62//===----------------------------------------------------------------------===//
63
64#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
65#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
66
67#include "mlir/Dialect/Func/IR/FuncOps.h"
68#include "mlir/Dialect/GPU/IR/GPUDialect.h"
69#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
70#include "mlir/Dialect/OpenACC/OpenACC.h"
71#include "mlir/IR/AttrTypeSubElements.h"
72#include "mlir/IR/BuiltinAttributes.h"
73#include "mlir/IR/IRMapping.h"
74#include "mlir/IR/SymbolTable.h"
75#include "mlir/Interfaces/CallInterfaces.h"
76#include "llvm/ADT/DenseMap.h"
77#include "llvm/ADT/SetVector.h"
78#include <string>
79
80namespace mlir {
81namespace acc {
82#define GEN_PASS_DEF_ACCROUTINETOGPUFUNC
83#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
84} // namespace acc
85} // namespace mlir
86
87#define DEBUG_TYPE "acc-routine-to-gpu-func"
88
89using namespace mlir;
90using namespace mlir::acc;
91
92namespace {
93/// Create a gpu.func from a func.func by cloning the body.
94static gpu::GPUFuncOp createGPUFuncFromFunc(OpBuilder &builder,
95 func::FuncOp sourceFunc) {
96 Location loc = sourceFunc.getLoc();
97 StringRef name = sourceFunc.getName();
98 FunctionType type = sourceFunc.getFunctionType();
99 // Do not copy any attributes from the source; specialized_routine is set
100 // later when applicable.
101 gpu::GPUFuncOp gpuFunc =
102 gpu::GPUFuncOp::create(builder, location: loc, name, type,
103 /*workgroupAttributions=*/TypeRange(),
104 /*privateAttributions=*/TypeRange(), /*attrs=*/{});
105
106 Region &sourceBody = sourceFunc.getBody();
107 Region &deviceBody = gpuFunc.getBody();
108 Block &deviceEntryBlock = deviceBody.front();
109
110 // Map source block arguments to the GPU func's entry block arguments (which
111 // GPUFuncOp::create already created).
112 IRMapping mapping;
113 Block &sourceEntryBlock = sourceBody.front();
114 for (auto [srcArg, destArg] : llvm::zip(t: sourceEntryBlock.getArguments(),
115 u: deviceEntryBlock.getArguments()))
116 mapping.map(from: srcArg, to: destArg);
117
118 sourceBody.cloneInto(dest: &deviceBody, mapper&: mapping);
119
120 // Replace func.return with gpu.return in the cloned blocks.
121 gpuFunc.walk(callback: [](func::ReturnOp op) {
122 OpBuilder replacer(op);
123 gpu::ReturnOp gpuReturn = gpu::ReturnOp::create(builder&: replacer, location: op.getLoc());
124 gpuReturn->setOperands(op.getOperands());
125 op.erase();
126 });
127
128 // Splice the cloned entry block's operations into the GPU func's entry block
129 // (cloneInto created a separate block for the cloned content), then remove
130 // the now-empty cloned block.
131 Block *clonedSourceEntry = mapping.lookup(from: &sourceEntryBlock);
132 deviceEntryBlock.getOperations().splice(
133 where: deviceEntryBlock.getOperations().end(),
134 L2&: clonedSourceEntry->getOperations());
135 clonedSourceEntry->erase();
136
137 return gpuFunc;
138}
139
140using CloneCandidate = std::pair<func::FuncOp, RoutineOp>;
141
142/// Collect materialized and bind routines; fill candidate func names and
143/// materialized routine set. Emit remarks for materialized routines.
144static void collectRoutineCandidates(
145 ModuleOp mod, SymbolTable &symTab, acc::DeviceType deviceType,
146 OpenACCSupport &accSupport,
147 llvm::SmallSetVector<llvm::StringRef, 4> &funcsToCloneCandidates,
148 llvm::SmallSetVector<RoutineOp, 4> &materializedAccRoutines,
149 llvm::SmallSetVector<RoutineOp, 4> &bindAccRoutines) {
150 auto isParallelRoutine = [deviceType](RoutineOp routineOp) {
151 return routineOp.hasGang(deviceType) || routineOp.hasGang() ||
152 routineOp.hasWorker(deviceType) || routineOp.hasWorker() ||
153 routineOp.hasVector(deviceType) || routineOp.hasVector() ||
154 routineOp.getGangDimValue(deviceType) || routineOp.getGangDimValue();
155 };
156
157 mod.walk(callback: [&](RoutineOp op) {
158 if (op.getEffectiveBindName(targetDeviceType: deviceType)) {
159 bindAccRoutines.insert(X: op);
160 return;
161 }
162 func::FuncOp callee =
163 symTab.lookup<func::FuncOp>(name: op.getFuncName().getLeafReference());
164 accSupport.emitRemark(
165 op: callee ? callee.getOperation() : op.getOperation(),
166 messageFn: [&op, &isParallelRoutine]() {
167 std::string msg = "Generating";
168 if (op.getImplicitAttr())
169 msg += " implicit";
170 msg += " acc routine";
171 if (!isParallelRoutine(op))
172 msg += " seq";
173 return msg;
174 },
175 DEBUG_TYPE);
176 funcsToCloneCandidates.insert(X: op.getFuncName().getLeafReference());
177 materializedAccRoutines.insert(X: op);
178 });
179}
180
181/// Process calls in ACC routines: add valid callees to funcsToClone (for
182/// declaration or clone). Returns failure() if any call is unsupported.
183static LogicalResult processCallsInRoutines(
184 SymbolTable &symTab, SymbolTable &gpuSymTab, OpenACCSupport &accSupport,
185 const llvm::SmallSetVector<llvm::StringRef, 4> &funcsToCloneCandidates,
186 const llvm::SmallSetVector<RoutineOp, 4> &materializedAccRoutines,
187 llvm::SmallSetVector<CloneCandidate, 4> &funcsToClone) {
188 LogicalResult callCheckResult = success();
189 auto processCalls = [&](CallOpInterface callOp) {
190 if (!callOp.getCallableForCallee())
191 return;
192 auto calleeSymbolRef =
193 dyn_cast<SymbolRefAttr>(Val: callOp.getCallableForCallee());
194 if (!calleeSymbolRef)
195 return;
196
197 auto callee =
198 symTab.lookup<func::FuncOp>(name: calleeSymbolRef.getLeafReference());
199 if (!callee)
200 return;
201
202 if (gpuSymTab.lookup(name: callee.getName()))
203 return;
204 if (isAccRoutine(op: callee) || isSpecializedAccRoutine(op: callee))
205 return;
206
207 if (!accSupport.isValidSymbolUse(user: callOp.getOperation(), symbol: calleeSymbolRef)) {
208 accSupport.emitNYI(loc: callOp->getLoc(), message: "Unsupported call in acc routine");
209 callCheckResult = failure();
210 return;
211 }
212 funcsToClone.insert(X: {callee, RoutineOp{}});
213 };
214
215 for (auto [funcName, accRoutine] :
216 llvm::zip(t: funcsToCloneCandidates, u: materializedAccRoutines)) {
217 func::FuncOp func = symTab.lookup<func::FuncOp>(name: funcName);
218 if (!func)
219 continue;
220 if (!gpuSymTab.lookup(name: funcName))
221 funcsToClone.insert(X: {func, accRoutine});
222 func.walk(callback: [&](CallOpInterface callOp) { processCalls(callOp); });
223 if (failed(Result: callCheckResult))
224 return failure();
225 }
226 return success();
227}
228
229/// Rewrite symbol uses of uniqued specialized names back to the original
230/// routine names. One walk of the module body instead of
231/// replaceAllSymbolUses per routine (which walks all of the IR each time).
232static void
233rewriteSpecializedSymbolUses(ModuleOp mod,
234 const DenseMap<StringAttr, StringAttr> &renames) {
235 if (renames.empty())
236 return;
237
238 AttrTypeReplacer replacer;
239 replacer.addReplacement(
240 callback: [&](SymbolRefAttr attr) -> std::pair<Attribute, WalkResult> {
241 auto it = renames.find(Val: attr.getRootReference());
242 if (it == renames.end())
243 return {attr, WalkResult::skip()};
244 if (isa<FlatSymbolRefAttr>(Val: attr))
245 return {FlatSymbolRefAttr::get(value: it->second), WalkResult::skip()};
246 return {SymbolRefAttr::get(rootReference: it->second, nestedReferences: attr.getNestedReferences()),
247 WalkResult::skip()};
248 });
249
250 // Match SymbolTable::replaceAllSymbolUses: walk the module body and do
251 // not enter nested symbol tables (e.g. gpu.module).
252 for (Region &region : mod->getRegions()) {
253 region.walk(callback: [&](Operation *op) {
254 if (op->hasTrait<OpTrait::SymbolTable>())
255 return WalkResult::skip();
256 replacer.replaceElementsIn(op);
257 return WalkResult::advance();
258 });
259 }
260}
261
262/// Clone each function in funcsToClone into the GPU module (declaration or
263/// full body). Fix up symbol names and specialized_routine attr for ACC
264/// routines.
265static LogicalResult cloneFuncsToGPUModule(
266 ModuleOp mod, SymbolTable &gpuSymTab,
267 const llvm::SmallSetVector<CloneCandidate, 4> &funcsToClone) {
268 OpBuilder builder(mod.getContext());
269
270 // ACCRoutineLowering inserts the device copy next to the host function, so
271 // the symbol table uniqued its name while acc.specialized_routine still
272 // holds the original one. Collect all of those renames and apply them in a
273 // single walk below.
274 DenseMap<StringAttr, StringAttr> specializedRenames;
275 for (CloneCandidate candidate : funcsToClone) {
276 func::FuncOp srcFunc = candidate.first;
277 if (srcFunc.isDeclaration())
278 continue;
279 if (auto specAttr =
280 srcFunc->getDiscardableAttrOfType<SpecializedRoutineAttr>(
281 name: getSpecializedRoutineAttrName())) {
282 StringAttr destName = specAttr.getFuncName();
283 if (srcFunc.getNameAttr() != destName)
284 specializedRenames.try_emplace(Key: srcFunc.getNameAttr(), Args&: destName);
285 }
286 }
287 rewriteSpecializedSymbolUses(mod, renames: specializedRenames);
288
289 for (CloneCandidate candidate : funcsToClone) {
290 func::FuncOp srcFunc = candidate.first;
291
292 if (srcFunc.isDeclaration()) {
293 Operation *cloned = srcFunc->clone();
294 gpuSymTab.insert(symbol: cloned);
295 continue;
296 }
297
298 gpu::GPUFuncOp deviceFuncOp = createGPUFuncFromFunc(builder, sourceFunc: srcFunc);
299
300 if (auto specAttr =
301 srcFunc->getDiscardableAttrOfType<SpecializedRoutineAttr>(
302 name: getSpecializedRoutineAttrName())) {
303 deviceFuncOp.setName(specAttr.getFuncName());
304 deviceFuncOp->setDiscardableAttr(name: getSpecializedRoutineAttrName(),
305 value: specAttr);
306 }
307
308 gpuSymTab.insert(symbol: deviceFuncOp);
309 }
310 return success();
311}
312
313/// Remove specialized device copies and nohost routines from the host module.
314static void
315cleanupHostModule(const llvm::SmallSetVector<CloneCandidate, 4> &funcsToClone) {
316 for (CloneCandidate candidate : funcsToClone) {
317 func::FuncOp funcCandidate = candidate.first;
318 RoutineOp routineCandidate = candidate.second;
319 if ((routineCandidate && routineCandidate.getNohost()) ||
320 acc::isSpecializedAccRoutine(op: funcCandidate))
321 funcCandidate.erase();
322 }
323}
324
325class ACCRoutineToGPUFunc
326 : public acc::impl::ACCRoutineToGPUFuncBase<ACCRoutineToGPUFunc> {
327public:
328 using acc::impl::ACCRoutineToGPUFuncBase<
329 ACCRoutineToGPUFunc>::ACCRoutineToGPUFuncBase;
330
331 void getDependentDialects(DialectRegistry &registry) const override {
332 acc::impl::ACCRoutineToGPUFuncBase<
333 ACCRoutineToGPUFunc>::getDependentDialects(registry);
334 acc::registerAllExtensions(registry);
335 }
336
337 void runOnOperation() override {
338 ModuleOp mod = getOperation();
339 if (mod.getOps<RoutineOp>().empty()) {
340 LLVM_DEBUG(llvm::dbgs()
341 << "Skipping ACCRoutineToGPUFunc - no acc.routine ops\n");
342 return;
343 }
344
345 OpenACCSupport &accSupport = getAnalysis<OpenACCSupport>();
346 std::optional<gpu::GPUModuleOp> gpuModOpt =
347 accSupport.getOrCreateGPUModule(mod);
348 if (!gpuModOpt) {
349 accSupport.emitNYI(loc: mod.getLoc(), message: "Failed to create GPU module");
350 return signalPassFailure();
351 }
352 gpu::GPUModuleOp gpuMod = *gpuModOpt;
353
354 SymbolTable symTab(mod);
355 SymbolTable gpuSymTab(gpuMod);
356
357 llvm::SmallSetVector<llvm::StringRef, 4> funcsToCloneCandidates;
358 llvm::SmallSetVector<RoutineOp, 4> materializedAccRoutines;
359 llvm::SmallSetVector<RoutineOp, 4> bindAccRoutines;
360
361 collectRoutineCandidates(mod, symTab, deviceType: this->deviceType, accSupport,
362 funcsToCloneCandidates, materializedAccRoutines,
363 bindAccRoutines);
364
365 llvm::SmallSetVector<CloneCandidate, 4> funcsToClone;
366 if (failed(Result: processCallsInRoutines(symTab, gpuSymTab, accSupport,
367 funcsToCloneCandidates,
368 materializedAccRoutines, funcsToClone)))
369 return signalPassFailure();
370
371 if (failed(Result: cloneFuncsToGPUModule(mod, gpuSymTab, funcsToClone)))
372 return signalPassFailure();
373
374 cleanupHostModule(funcsToClone);
375 }
376};
377} // namespace
378