1//===- ACCRoutineLowering.cpp - Wrap ACC routines in compute_region -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass handles `acc routine` directive by creating specialized
10// functions with appropriate parallelism information that can be used for
11// eventual creation of device function.
12//
13// Overview:
14// ---------
15// For each acc.routine that is not bound by name, the pass creates a new
16// function (the "device" copy) whose body is a single acc.compute_region
17// containing a clone of the original (host) function body. Parallelism is
18// expressed by one acc.par_width derived from the routine's clauses (seq,
19// vector, worker, gang). The device copy created is simply a staging
20// place for eventual move to device module level function.
21//
22// Transformations:
23// ----------------
24// 1. Device function: Same signature as the host; attributes copied except
25// acc.routine_info. The acc.specialized_routine attribute is set with the
26// routine symbol, par level, and original function name.
27//
28// 2. Body: One acc.par_width, one acc.compute_region that clones the host
29// body. Multi-block host bodies are wrapped in scf.execute_region inside
30// the compute_region.
31//
32// 3. Finalization: acc.routine's func_name is updated to the device function.
33//
34//===----------------------------------------------------------------------===//
35
36#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
37#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
38
39#include "mlir/Dialect/Func/IR/FuncOps.h"
40#include "mlir/Dialect/OpenACC/OpenACC.h"
41#include "mlir/Dialect/OpenACC/OpenACCParMapping.h"
42#include "mlir/Dialect/OpenACC/OpenACCUtilsCG.h"
43#include "mlir/IR/BuiltinAttributes.h"
44#include "mlir/IR/IRMapping.h"
45#include "mlir/IR/PatternMatch.h"
46#include "mlir/IR/SymbolTable.h"
47#include "mlir/IR/Value.h"
48
49namespace mlir {
50namespace acc {
51#define GEN_PASS_DEF_ACCROUTINELOWERING
52#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
53} // namespace acc
54} // namespace mlir
55
56#define DEBUG_TYPE "acc-routine-lowering"
57
58using namespace mlir;
59using namespace mlir::acc;
60
61namespace {
62/// Compute the ParLevel from an acc.routine op for specialization.
63static ParLevel computeParLevel(RoutineOp routineOp, DeviceType deviceType) {
64 auto gangDim = routineOp.getGangDimValue(deviceType);
65 if (!gangDim)
66 gangDim = routineOp.getGangDimValue();
67 if (gangDim) {
68 switch (*gangDim) {
69 case 1:
70 return ParLevel::gang_dim1;
71 case 2:
72 return ParLevel::gang_dim2;
73 case 3:
74 return ParLevel::gang_dim3;
75 default:
76 break;
77 }
78 }
79 if (routineOp.hasGang(deviceType) || routineOp.hasGang())
80 return ParLevel::gang_dim1;
81 if (routineOp.hasWorker(deviceType) || routineOp.hasWorker())
82 return ParLevel::worker;
83 if (routineOp.hasVector(deviceType) || routineOp.hasVector())
84 return ParLevel::vector;
85 return ParLevel::seq;
86}
87
88/// Collect return operands from the function (first block with func.return).
89static void getReturnValues(func::FuncOp func, SmallVectorImpl<Value> &result) {
90 result.clear();
91 for (Block &block : func.getBody().getBlocks()) {
92 if (auto returnOp = dyn_cast<func::ReturnOp>(Val: block.getTerminator())) {
93 result.assign(in_start: returnOp.operand_begin(), in_end: returnOp.operand_end());
94 break;
95 }
96 }
97}
98
99/// Create the device function with the same signature as the host, set
100/// specialized_routine, and add a single block with the same block arguments.
101static func::FuncOp createFunctionForDeviceStaging(func::FuncOp hostFunc,
102 RoutineOp routineOp,
103 ParLevel parLevel,
104 MLIRContext *ctx,
105 IRRewriter &rewriter) {
106 Location loc = hostFunc.getLoc();
107 FunctionType funcType = hostFunc.getFunctionType();
108 func::FuncOp deviceFunc =
109 func::FuncOp::create(builder&: rewriter, location: loc, name: hostFunc.getName(), type: funcType);
110 deviceFunc->copyProperties(rhs: hostFunc->getPropertiesStorage());
111 deviceFunc->setDiscardableAttrs(hostFunc->getDiscardableAttrDictionary());
112 deviceFunc->removeDiscardableAttr(name: getRoutineInfoAttrName());
113 deviceFunc->setDiscardableAttr(
114 name: getSpecializedRoutineAttrName(),
115 value: SpecializedRoutineAttr::get(
116 context: ctx, routine: SymbolRefAttr::get(ctx, value: routineOp.getSymName()),
117 level: ParLevelAttr::get(context: ctx, value: parLevel),
118 funcName: StringAttr::get(context: ctx, bytes: hostFunc.getName())));
119
120 Block *sourceBlock = &hostFunc.getBody().front();
121 Block *newBlock = rewriter.createBlock(parent: &deviceFunc.getRegion());
122 for (BlockArgument arg : sourceBlock->getArguments())
123 newBlock->addArgument(type: arg.getType(), loc: hostFunc.getLoc());
124
125 return deviceFunc;
126}
127
128/// Fill the device function body: one acc.par_width, one acc.compute_region
129/// (cloning the host body with inputArgsToMap), then func.return.
130static LogicalResult
131buildRoutineBody(func::FuncOp deviceFunc, func::FuncOp hostFunc,
132 ArrayRef<Value> funcReturnVals, ParLevel parLevel,
133 DefaultACCToGPUMappingPolicy &policy, IRRewriter &rewriter) {
134 Block *newBlock = &deviceFunc.getBody().front();
135 Block *sourceBlock = &hostFunc.getBody().front();
136 Location loc = hostFunc.getLoc();
137 MLIRContext *ctx = rewriter.getContext();
138
139 rewriter.setInsertionPointToStart(newBlock);
140 GPUParallelDimAttr parDim = policy.map(ctx, level: parLevel);
141 Value parWidthVal = ParWidthOp::create(builder&: rewriter, location: loc, launchArg: Value(), par_dim: parDim);
142 SmallVector<Value, 4> inputArgs(newBlock->getArguments().begin(),
143 newBlock->getArguments().end());
144
145 // Normally the region passed to buildComputeRegion is something in the
146 // current function. Here we pass the body of the original (host) function as
147 // an optimization to avoid cloning twice (once for a staged device copy and
148 // again when creating the compute region). Since we clone only once, we must
149 // also provide the original function's arguments so the mapping is correct
150 // when cloning the body.
151 ValueRange sourceArgsToMap = sourceBlock->getArguments();
152
153 IRMapping mapping;
154 rewriter.setInsertionPointAfter(parWidthVal.getDefiningOp());
155 ComputeRegionOp computeRegion = buildComputeRegion(
156 loc, launchArgs: {parWidthVal}, inputArgs, origin: RoutineOp::getOperationName(),
157 regionToClone&: hostFunc.getBody(), rewriter, mapping,
158 /*output=*/funcReturnVals, /*kernelFuncName=*/{},
159 /*kernelModuleName=*/{}, /*stream=*/{}, inputArgsToMap: sourceArgsToMap);
160 if (!computeRegion)
161 return failure();
162
163 rewriter.setInsertionPointAfter(computeRegion);
164 if (funcReturnVals.empty())
165 func::ReturnOp::create(builder&: rewriter, location: loc);
166 else
167 func::ReturnOp::create(builder&: rewriter, location: loc, operands: computeRegion.getResults());
168
169 return success();
170}
171
172/// Update acc.routine refs
173static void finalizeRoutines(
174 SmallVectorImpl<std::pair<func::FuncOp, RoutineOp>> &accRoutineInfo,
175 MLIRContext *ctx) {
176 for (auto &[deviceFunc, routineOp] : accRoutineInfo) {
177 routineOp.setFuncNameAttr(SymbolRefAttr::get(ctx, value: deviceFunc.getName()));
178 routineOp->moveBefore(existingOp: deviceFunc);
179 }
180}
181
182class ACCRoutineLowering
183 : public acc::impl::ACCRoutineLoweringBase<ACCRoutineLowering> {
184public:
185 using ACCRoutineLoweringBase::ACCRoutineLoweringBase;
186
187 void getDependentDialects(DialectRegistry &registry) const override {
188 ACCRoutineLoweringBase::getDependentDialects(registry);
189 acc::registerAllExtensions(registry);
190 }
191
192 void runOnOperation() override {
193 ModuleOp mod = getOperation();
194 if (mod.getOps<RoutineOp>().empty()) {
195 LLVM_DEBUG(llvm::dbgs()
196 << "Skipping ACCRoutineLowering - no acc.routine ops\n");
197 return;
198 }
199
200 SymbolTable symTab(mod);
201 MLIRContext *ctx = mod.getContext();
202 IRRewriter rewriter(ctx);
203 DefaultACCToGPUMappingPolicy policy;
204
205 // Pair: device function, routine operation
206 SmallVector<std::pair<func::FuncOp, RoutineOp>, 4> accRoutineInfo;
207
208 for (RoutineOp routineOp : mod.getOps<RoutineOp>()) {
209 if (routineOp.getEffectiveBindName(targetDeviceType: deviceType))
210 continue;
211
212 func::FuncOp hostFunc = symTab.lookup<func::FuncOp>(
213 name: routineOp.getFuncName().getLeafReference());
214 if (!hostFunc) {
215 routineOp.emitError(message: "acc routine function not found in symbol table");
216 return signalPassFailure();
217 }
218 if (hostFunc.isExternal())
219 continue;
220
221 SmallVector<Value, 4> funcReturnVals;
222 getReturnValues(func: hostFunc, result&: funcReturnVals);
223
224 OpBuilder::InsertionGuard guard(rewriter);
225 ParLevel parLevel = computeParLevel(routineOp, deviceType);
226 func::FuncOp deviceFunc = createFunctionForDeviceStaging(
227 hostFunc, routineOp, parLevel, ctx, rewriter);
228 if (failed(Result: buildRoutineBody(deviceFunc, hostFunc, funcReturnVals,
229 parLevel, policy, rewriter)))
230 return signalPassFailure();
231
232 accRoutineInfo.push_back(Elt: {deviceFunc, routineOp});
233 symTab.insert(symbol: deviceFunc);
234 }
235
236 finalizeRoutines(accRoutineInfo, ctx);
237 }
238};
239} // namespace
240