1//===- ACCSpecializeForDevice.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass strips OpenACC constructs that are invalid or unnecessary inside
10// device code (specialized acc routines or compute construct regions).
11//
12// Overview:
13// ---------
14// In a specialized acc routine or compute construct, many OpenACC operations
15// do not make sense because they are host-side constructs. This pass removes
16// or transforms these operations appropriately:
17//
18// - Data operations that manage device memory from host perspective
19// - Compute constructs that launch kernels (we're already on device)
20// - Runtime operations like init/shutdown/set/wait
21//
22// Transformations:
23// ----------------
24// The pass applies the following transformations:
25//
26// 1. Data Entry Ops (replaced with var operand):
27// acc.attach, acc.copyin, acc.create, acc.declare_device_resident,
28// acc.declare_link, acc.deviceptr, acc.get_deviceptr, acc.nocreate,
29// acc.present, acc.update_device, acc.use_device
30//
31// 2. Data Exit Ops (erased):
32// acc.copyout, acc.delete, acc.detach, acc.update_host
33//
34// 3. Structured Data/Compute Constructs (region inlined):
35// acc.data, acc.host_data, acc.kernel_environment, acc.parallel,
36// acc.serial, acc.kernels
37//
38// 4. Unstructured Data Ops (erased):
39// acc.enter_data, acc.exit_data, acc.update, acc.declare_enter,
40// acc.declare_exit
41//
42// 5. Runtime Ops (erased):
43// acc.init, acc.shutdown, acc.set, acc.wait
44//
45// 6. acc.on_device (folded):
46// acc.on_device with constant device type is folded to a boolean constant.
47// With a device type only known at runtime it is replaced by comparisons
48// against the device types being specialized for.
49//
50// Scope of Application:
51// ---------------------
52// - For functions with `acc.specialized_routine` attribute: patterns are
53// applied to the entire function body.
54// - For non-specialized functions: patterns are applied only to ACC
55// operations INSIDE compute constructs (parallel, serial, kernels),
56// not to the compute constructs themselves or their data operands.
57//
58// Note: acc.cache, acc.private, acc.reduction, acc.firstprivate are NOT
59// transformed by this pass as they are valid in device code.
60//
61//===----------------------------------------------------------------------===//
62
63#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
64#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
65
66#include "mlir/Dialect/Arith/IR/Arith.h"
67#include "mlir/Dialect/Func/IR/FuncOps.h"
68#include "mlir/Dialect/OpenACC/OpenACC.h"
69#include "mlir/Dialect/OpenACC/Transforms/ACCSpecializePatterns.h"
70#include "mlir/Dialect/SCF/IR/SCF.h"
71#include "mlir/IR/MLIRContext.h"
72#include "mlir/IR/PatternMatch.h"
73#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
74
75namespace mlir {
76namespace acc {
77#define GEN_PASS_DEF_ACCSPECIALIZEFORDEVICE
78#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
79} // namespace acc
80} // namespace mlir
81
82using namespace mlir;
83using namespace mlir::acc;
84
85namespace {
86/// Fold acc.on_device in device code. A constant device type folds to a
87/// boolean constant, any other device type to a comparison against the device
88/// types being specialized for.
89class FoldAccOnDeviceOpConversion : public OpRewritePattern<acc::OnDeviceOp> {
90 using OpRewritePattern<acc::OnDeviceOp>::OpRewritePattern;
91
92 const TypesForDevice &theDeviceTypes;
93
94public:
95 FoldAccOnDeviceOpConversion(MLIRContext *context,
96 const TypesForDevice &theDeviceTypes)
97 : OpRewritePattern<acc::OnDeviceOp>(context),
98 theDeviceTypes(theDeviceTypes) {}
99
100 LogicalResult matchAndRewrite(acc::OnDeviceOp op,
101 PatternRewriter &rewriter) const override {
102 Value deviceType = op.getDeviceType();
103 APInt constVal;
104 if (matchPattern(value: deviceType, pattern: m_ConstantInt(bind_value: &constVal))) {
105 bool result = theDeviceTypes.contains(key: constVal.getSExtValue());
106 rewriter.replaceOpWithNewOp<arith::ConstantOp>(
107 op, args: rewriter.getI1Type(), args: rewriter.getBoolAttr(value: result));
108 return success();
109 }
110
111 // The device type is only known at runtime, inline the definition of
112 // `on_device`.
113 Location loc = op.getLoc();
114 Value result;
115 for (int64_t type : theDeviceTypes) {
116 Value typeValue = arith::ConstantOp::create(
117 builder&: rewriter, location: loc, result: deviceType.getType(),
118 value: rewriter.getIntegerAttr(type: deviceType.getType(), value: type));
119 Value isType = arith::CmpIOp::create(
120 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: deviceType, rhs: typeValue);
121 if (result)
122 result = arith::OrIOp::create(builder&: rewriter, location: loc, lhs: result, rhs: isType);
123 else
124 result = isType;
125 }
126 assert(result && "At least one device type expected.");
127 rewriter.replaceOp(op, newValues: result);
128 return success();
129 }
130};
131
132class ACCSpecializeForDevice
133 : public acc::impl::ACCSpecializeForDeviceBase<ACCSpecializeForDevice> {
134public:
135 using ACCSpecializeForDeviceBase<
136 ACCSpecializeForDevice>::ACCSpecializeForDeviceBase;
137
138 void getDependentDialects(DialectRegistry &registry) const override {
139 ACCSpecializeForDeviceBase<ACCSpecializeForDevice>::getDependentDialects(
140 registry);
141 acc::registerAllExtensions(registry);
142 }
143
144 void runOnOperation() override {
145 func::FuncOp func = getOperation();
146
147 TypesForDevice types{theDeviceTypes.begin(), theDeviceTypes.end()};
148
149 RewritePatternSet patterns(&getContext());
150 acc::populateACCSpecializeForDevicePatterns(patterns, theDeviceTypes: types);
151 GreedyRewriteConfig config;
152 config.setUseTopDownTraversal(true);
153
154 if (acc::isSpecializedAccRoutine(op: func)) {
155 // For specialized acc routines, apply patterns to the entire function
156 (void)applyPatternsGreedily(op: func, patterns: std::move(patterns), config);
157 } else {
158 // For non-specialized functions, apply patterns only to ACC operations
159 // inside compute constructs (not to the compute constructs themselves).
160 // Use ExistingOps strictness so the greedy driver does not expand the
161 // worklist to parent ops, which would accidentally unwrap the compute
162 // construct (e.g. after inlining acc routines with their own data
163 // regions).
164 config.setStrictness(GreedyRewriteStrictness::ExistingOps);
165 SmallVector<Operation *> opsToTransform;
166 func.walk(callback: [&](Operation *op) {
167 if (isa<ACC_COMPUTE_CONSTRUCT_OPS>(Val: op)) {
168 // Walk inside the compute construct and collect ACC ops
169 op->walk(callback: [&](Operation *innerOp) {
170 // Skip the compute construct itself
171 if (innerOp == op)
172 return;
173 if (isa<acc::OpenACCDialect>(Val: innerOp->getDialect()))
174 opsToTransform.push_back(Elt: innerOp);
175 });
176 }
177 });
178 if (!opsToTransform.empty())
179 (void)applyOpPatternsGreedily(ops: opsToTransform, patterns: std::move(patterns),
180 config);
181 }
182 }
183};
184} // namespace
185
186//===----------------------------------------------------------------------===//
187// Pattern population functions
188//===----------------------------------------------------------------------===//
189
190void mlir::acc::populateACCSpecializeForDevicePatterns(
191 RewritePatternSet &patterns, const TypesForDevice &theDeviceTypes) {
192 MLIRContext *context = patterns.getContext();
193
194 // Declare patterns - erase declare_enter and its associated declare_exit
195 patterns.insert<ACCDeclareEnterOpConversion>(arg&: context);
196
197 // Data entry ops - replaced with their var operand
198 // Note: acc.cache, acc.private, acc.reduction, acc.firstprivate are NOT
199 // included here - they are valid in device code
200 patterns.insert<ACCOpReplaceWithVarConversion<acc::AttachOp>,
201 ACCOpReplaceWithVarConversion<acc::CopyinOp>,
202 ACCOpReplaceWithVarConversion<acc::CreateOp>,
203 ACCOpReplaceWithVarConversion<acc::DeclareDeviceResidentOp>,
204 ACCOpReplaceWithVarConversion<acc::DeclareLinkOp>,
205 ACCOpReplaceWithVarConversion<acc::DevicePtrOp>,
206 ACCOpReplaceWithVarConversion<acc::GetDevicePtrOp>,
207 ACCOpReplaceWithVarConversion<acc::NoCreateOp>,
208 ACCOpReplaceWithVarConversion<acc::PresentOp>,
209 ACCOpReplaceWithVarConversion<acc::UpdateDeviceOp>,
210 ACCOpReplaceWithVarConversion<acc::UseDeviceOp>>(arg&: context);
211
212 // Data exit ops - simply erased (no results)
213 patterns.insert<ACCOpEraseConversion<acc::CopyoutOp>,
214 ACCOpEraseConversion<acc::DeleteOp>,
215 ACCOpEraseConversion<acc::DetachOp>,
216 ACCOpEraseConversion<acc::UpdateHostOp>>(arg&: context);
217
218 // Structured data constructs - unwrap their regions
219 patterns.insert<ACCRegionUnwrapConversion<acc::DataOp>,
220 ACCRegionUnwrapConversion<acc::HostDataOp>,
221 ACCRegionUnwrapConversion<acc::KernelEnvironmentOp>>(arg&: context);
222
223 // Compute constructs - unwrap their regions
224 patterns.insert<ACCRegionUnwrapConversion<acc::ParallelOp>,
225 ACCRegionUnwrapConversion<acc::SerialOp>,
226 ACCRegionUnwrapConversion<acc::KernelsOp>>(arg&: context);
227
228 // Unstructured data operations - erase them
229 patterns.insert<ACCOpEraseConversion<acc::EnterDataOp>,
230 ACCOpEraseConversion<acc::ExitDataOp>,
231 ACCOpEraseConversion<acc::UpdateOp>>(arg&: context);
232
233 // Runtime operations - erase them
234 patterns.insert<
235 ACCOpEraseConversion<acc::InitOp>, ACCOpEraseConversion<acc::ShutdownOp>,
236 ACCOpEraseConversion<acc::SetOp>, ACCOpEraseConversion<acc::WaitOp>>(
237 arg&: context);
238
239 // Fold acc.on_device calls so dead-code elimination can remove host-only
240 // code paths in device code.
241 patterns.insert<FoldAccOnDeviceOpConversion>(arg&: context, args: theDeviceTypes);
242}
243