1//===- ACCSpecializeForHost.cpp -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass converts OpenACC operations to host-compatible representations,
10// enabling execution on the host rather than on accelerator devices.
11//
12// Overview:
13// ---------
14// The pass operates in two modes depending on the `enableHostFallback` option:
15//
16// 1. Default Mode (Orphan Operations Only):
17// Only converts "orphan" ACC operations that are not inside or attached to
18// compute regions. This is used for host routines (acc routine marked for
19// host) where structured/unstructured data constructs, compute constructs,
20// and their associated data operations should be preserved.
21//
22// 2. Host Fallback Mode (enableHostFallback=true):
23// Converts ALL ACC operations within the region to host equivalents. This
24// is used when the `if` clause evaluates to false at runtime and the
25// entire ACC region needs to fall back to host execution.
26//
27// Transformations (Orphan Mode):
28// ------------------------------
29// The following orphan operations are converted:
30//
31// 1. Atomic Ops (converted to load/store):
32// acc.atomic.update -> load + compute + store
33// acc.atomic.read -> load + store (copy)
34// acc.atomic.write -> store
35// acc.atomic.capture -> inline region contents
36//
37// 2. Loop Ops (converted to SCF):
38// acc.loop (structured) -> scf.for
39// acc.loop (unstructured) -> scf.execute_region
40//
41// 3. Orphan Data Entry Ops (replaced with var operand):
42// acc.cache, acc.private, acc.firstprivate, acc.reduction
43// (only if NOT connected to compute constructs or loop)
44//
45// Transformations (Host Fallback Mode):
46// -------------------------------------
47// In addition to orphan transformations, ALL of the following are converted:
48//
49// 1. Data Entry Ops (replaced with var operand):
50// acc.copyin, acc.create, acc.attach, acc.present, acc.deviceptr,
51// acc.get_deviceptr, acc.nocreate, acc.declare_device_resident,
52// acc.declare_link, acc.use_device, acc.update_device
53//
54// 2. Data Exit Ops (erased):
55// acc.copyout, acc.delete, acc.detach, acc.update_host
56//
57// 3. Structured Data/Compute Constructs (region inlined):
58// acc.data, acc.host_data, acc.kernel_environment, acc.declare,
59// acc.parallel, acc.serial, acc.kernels
60//
61// 4. Unstructured Data Ops (erased):
62// acc.enter_data, acc.exit_data, acc.update
63//
64// 5. Declare Ops (erased):
65// acc.declare_enter, acc.declare_exit
66//
67// 6. Runtime Ops (erased):
68// acc.init, acc.shutdown, acc.set, acc.wait, acc.terminator
69//
70// Requirements:
71// -------------
72// For atomic operation conversion, variables must implement the
73// `acc::PointerLikeType` interface to enable generating load/store operations.
74//
75// The pass uses `OpenACCSupport::emitNYI()` to report unsupported cases.
76//
77//===----------------------------------------------------------------------===//
78
79#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
80#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
81
82#include "mlir/Dialect/Func/IR/FuncOps.h"
83#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
84#include "mlir/Dialect/OpenACC/OpenACC.h"
85#include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h"
86#include "mlir/Dialect/OpenACC/Transforms/ACCSpecializePatterns.h"
87#include "mlir/IR/BuiltinAttributes.h"
88#include "mlir/IR/BuiltinOps.h"
89#include "mlir/IR/BuiltinTypes.h"
90#include "mlir/IR/Operation.h"
91#include "mlir/IR/PatternMatch.h"
92#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
93
94namespace mlir {
95namespace acc {
96#define GEN_PASS_DEF_ACCSPECIALIZEFORHOST
97#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
98} // namespace acc
99} // namespace mlir
100
101#define DEBUG_TYPE "acc-specialize-for-host"
102
103using namespace mlir;
104using namespace mlir::acc;
105
106/// Check if an operation is inside an ACC compute construct.
107static bool isInsideACCComputeConstruct(Operation *op) {
108 while ((op = op->getParentOp()))
109 if (isa<ACC_COMPUTE_CONSTRUCT_OPS>(Val: op))
110 return true;
111 return false;
112}
113
114/// Return true if an enclosing compute construct or capture must be removed
115/// before converting an atomic operation.
116static bool isAtomicConversionDeferred(Operation *op) {
117 return isInsideACCComputeConstruct(op) ||
118 op->getParentOfType<acc::AtomicCaptureOp>();
119}
120
121namespace {
122// Lower orphan acc.atomic.update by: load from addr, clone region expr with
123// the loaded value, then store the computed result back to addr.
124// Only matches outside compute regions and atomic captures.
125class ACCOrphanAtomicUpdateOpConversion
126 : public OpRewritePattern<acc::AtomicUpdateOp> {
127public:
128 ACCOrphanAtomicUpdateOpConversion(MLIRContext *ctx, OpenACCSupport &support)
129 : OpRewritePattern<acc::AtomicUpdateOp>(ctx), accSupport(support) {}
130
131 LogicalResult matchAndRewrite(acc::AtomicUpdateOp atomicUpdateOp,
132 PatternRewriter &rewriter) const override {
133 if (isAtomicConversionDeferred(op: atomicUpdateOp))
134 return failure();
135
136 Value x = atomicUpdateOp.getX();
137 Type type = x.getType();
138 auto ptrLikeType = dyn_cast<acc::PointerLikeType>(Val&: type);
139 if (ptrLikeType) {
140 auto xTyped = cast<TypedValue<acc::PointerLikeType>>(Val&: x);
141 rewriter.setInsertionPointAfter(atomicUpdateOp);
142 Value loadOp =
143 ptrLikeType.genLoad(builder&: rewriter, loc: atomicUpdateOp.getLoc(), srcPtr: xTyped, valueType: {});
144 if (!loadOp) {
145 accSupport.emitNYI(loc: atomicUpdateOp.getLoc(),
146 message: "failed to generate load for atomic update");
147 return failure();
148 }
149 IRMapping mapping;
150 mapping.map(from: atomicUpdateOp.getRegion().front().getArgument(i: 0), to: loadOp);
151 Block &block = atomicUpdateOp.getRegion().front();
152 for (Operation &op : block.without_terminator())
153 rewriter.clone(op, mapper&: mapping);
154 auto yieldOp = cast<acc::YieldOp>(Val: block.getTerminator());
155 Value result = mapping.lookup(from: yieldOp.getOperand(i: 0));
156 if (!ptrLikeType.genStore(builder&: rewriter, loc: atomicUpdateOp.getLoc(), valueToStore: result,
157 destPtr: xTyped)) {
158 accSupport.emitNYI(loc: atomicUpdateOp.getLoc(),
159 message: "failed to generate store for atomic update");
160 return failure();
161 }
162 rewriter.eraseOp(op: atomicUpdateOp);
163 } else {
164 accSupport.emitNYI(loc: atomicUpdateOp.getLoc(),
165 message: "unsupported type for atomic update");
166 return failure();
167 }
168 return success();
169 }
170
171private:
172 OpenACCSupport &accSupport;
173};
174
175// Lower orphan acc.atomic.read by: load from src, then store into dst.
176// Only matches outside compute regions and atomic captures.
177class ACCOrphanAtomicReadOpConversion
178 : public OpRewritePattern<acc::AtomicReadOp> {
179public:
180 ACCOrphanAtomicReadOpConversion(MLIRContext *ctx, OpenACCSupport &support)
181 : OpRewritePattern<acc::AtomicReadOp>(ctx), accSupport(support) {}
182
183 LogicalResult matchAndRewrite(acc::AtomicReadOp readOp,
184 PatternRewriter &rewriter) const override {
185 if (isAtomicConversionDeferred(op: readOp))
186 return failure();
187
188 Value x = readOp.getX();
189 Value v = readOp.getV();
190 auto xPtrType = dyn_cast<acc::PointerLikeType>(Val: x.getType());
191 auto vPtrType = dyn_cast<acc::PointerLikeType>(Val: v.getType());
192 if (xPtrType && vPtrType) {
193 auto xTyped = cast<TypedValue<acc::PointerLikeType>>(Val&: x);
194 auto vTyped = cast<TypedValue<acc::PointerLikeType>>(Val&: v);
195 rewriter.setInsertionPointAfter(readOp);
196
197 // Use genCopy which does load + store
198 if (!xPtrType.genCopy(builder&: rewriter, loc: readOp.getLoc(), destination: vTyped, source: xTyped, varType: {})) {
199 accSupport.emitNYI(loc: readOp.getLoc(),
200 message: "failed to generate copy for atomic read");
201 return failure();
202 }
203 rewriter.eraseOp(op: readOp);
204 } else {
205 accSupport.emitNYI(loc: readOp.getLoc(), message: "unsupported type for atomic read");
206 return failure();
207 }
208 return success();
209 }
210
211private:
212 OpenACCSupport &accSupport;
213};
214
215// Lower orphan acc.atomic.write by: store value into addr.
216// Only matches outside compute regions and atomic captures.
217class ACCOrphanAtomicWriteOpConversion
218 : public OpRewritePattern<acc::AtomicWriteOp> {
219public:
220 ACCOrphanAtomicWriteOpConversion(MLIRContext *ctx, OpenACCSupport &support)
221 : OpRewritePattern<acc::AtomicWriteOp>(ctx), accSupport(support) {}
222
223 LogicalResult matchAndRewrite(acc::AtomicWriteOp writeOp,
224 PatternRewriter &rewriter) const override {
225 if (isAtomicConversionDeferred(op: writeOp))
226 return failure();
227
228 Value x = writeOp.getX();
229 Value expr = writeOp.getExpr();
230 auto ptrLikeType = dyn_cast<acc::PointerLikeType>(Val: x.getType());
231 if (ptrLikeType) {
232 auto xTyped = cast<TypedValue<acc::PointerLikeType>>(Val&: x);
233 rewriter.setInsertionPointAfter(writeOp);
234 if (!ptrLikeType.genStore(builder&: rewriter, loc: writeOp.getLoc(), valueToStore: expr, destPtr: xTyped)) {
235 accSupport.emitNYI(loc: writeOp.getLoc(),
236 message: "failed to generate store for atomic write");
237 return failure();
238 }
239 rewriter.eraseOp(op: writeOp);
240 } else {
241 accSupport.emitNYI(loc: writeOp.getLoc(), message: "unsupported type for atomic write");
242 return failure();
243 }
244 return success();
245 }
246
247private:
248 OpenACCSupport &accSupport;
249};
250
251// Lower orphan acc.atomic.capture by: unwrap the capture region and erase the
252// wrapper; inner ops are lowered in-order (e.g., read+update becomes load/store
253// to dst then load/compute/store to addr).
254// Only matches if NOT inside an ACC compute construct.
255class ACCOrphanAtomicCaptureOpConversion
256 : public OpRewritePattern<acc::AtomicCaptureOp> {
257 using OpRewritePattern<acc::AtomicCaptureOp>::OpRewritePattern;
258
259 LogicalResult matchAndRewrite(acc::AtomicCaptureOp captureOp,
260 PatternRewriter &rewriter) const override {
261 // Only convert if this op is not inside an ACC compute construct
262 if (isInsideACCComputeConstruct(op: captureOp))
263 return failure();
264
265 assert(captureOp.getRegion().hasOneBlock() && "expected one block");
266 Block *block = &captureOp.getRegion().front();
267 // Remove the terminator before inlining
268 rewriter.eraseOp(op: block->getTerminator());
269 rewriter.inlineBlockBefore(source: block, op: captureOp);
270 rewriter.eraseOp(op: captureOp);
271 return success();
272 }
273};
274
275// Convert orphan acc.loop to scf.for or scf.execute_region.
276// Only matches if NOT inside an ACC compute construct.
277class ACCOrphanLoopOpConversion : public OpRewritePattern<acc::LoopOp> {
278 using OpRewritePattern<acc::LoopOp>::OpRewritePattern;
279
280 LogicalResult matchAndRewrite(acc::LoopOp loopOp,
281 PatternRewriter &rewriter) const override {
282 // Only convert if this op is not inside an ACC compute construct
283 if (isInsideACCComputeConstruct(op: loopOp))
284 return failure();
285
286 if (loopOp.getUnstructured()) {
287 auto executeRegion =
288 acc::convertUnstructuredACCLoopToSCFExecuteRegion(loopOp, rewriter);
289 if (!executeRegion)
290 return failure();
291 rewriter.replaceOp(op: loopOp, newOp: executeRegion);
292 } else {
293 auto forOp = acc::convertACCLoopToSCFFor(loopOp, rewriter,
294 /*enableCollapse=*/false);
295 if (!forOp)
296 return failure();
297 rewriter.replaceOp(op: loopOp, newOp: forOp);
298 }
299 return success();
300 }
301};
302
303/// Check if an operation is used by a compute construct or loop op
304static bool isUsedByComputeOrLoop(Operation *op) {
305 for (auto *user : op->getUsers())
306 if (isa<acc::ParallelOp, acc::SerialOp, acc::KernelsOp, acc::LoopOp>(Val: user))
307 return true;
308 return false;
309}
310
311/// Orphan data entry ops - only match if NOT connected to compute/loop and
312/// NOT inside a compute region. Used for acc.cache, acc.private,
313/// acc.firstprivate, acc.reduction.
314template <typename OpTy>
315class ACCOrphanDataEntryConversion : public OpRewritePattern<OpTy> {
316 using OpRewritePattern<OpTy>::OpRewritePattern;
317
318 LogicalResult matchAndRewrite(OpTy op,
319 PatternRewriter &rewriter) const override {
320 // Only convert if this op is not used by a compute construct or loop,
321 // and not inside an ACC compute construct.
322 if (isUsedByComputeOrLoop(op) || isInsideACCComputeConstruct(op))
323 return failure();
324
325 if (op->use_empty())
326 rewriter.eraseOp(op);
327 else
328 rewriter.replaceOp(op, op.getVar());
329 return success();
330 }
331};
332
333class ACCSpecializeForHost
334 : public acc::impl::ACCSpecializeForHostBase<ACCSpecializeForHost> {
335public:
336 using ACCSpecializeForHostBase<
337 ACCSpecializeForHost>::ACCSpecializeForHostBase;
338
339 void getDependentDialects(DialectRegistry &registry) const override {
340 ACCSpecializeForHostBase<ACCSpecializeForHost>::getDependentDialects(
341 registry);
342 acc::registerAllExtensions(registry);
343 }
344
345 void runOnOperation() override {
346 LLVM_DEBUG(llvm::dbgs() << "Enter ACCSpecializeForHost()\n");
347
348 func::FuncOp funcOp = getOperation();
349 if (!acc::isSpecializedAccRoutine(op: funcOp)) {
350 // Convert orphan operations to host, or all ACC operations if
351 // host fallback patterns are enabled.
352 auto *context = &getContext();
353 RewritePatternSet patterns(context);
354 OpenACCSupport &accSupport = getAnalysis<OpenACCSupport>();
355 if (enableHostFallback)
356 populateACCHostFallbackPatterns(patterns, accSupport);
357 else
358 populateACCOrphanToHostPatterns(patterns, accSupport);
359 GreedyRewriteConfig config;
360 config.setUseTopDownTraversal(true);
361 // Deeply nested orphan acc.loops can need more than the default
362 // iteration cap to converge; lift it to avoid spurious pass failure.
363 config.setMaxIterations(GreedyRewriteConfig::kNoLimit);
364 if (failed(Result: applyPatternsGreedily(op: funcOp, patterns: std::move(patterns), config)))
365 signalPassFailure();
366 }
367
368 LLVM_DEBUG(llvm::dbgs() << "Exit ACCSpecializeForHost()\n");
369 }
370};
371} // namespace
372
373//===----------------------------------------------------------------------===//
374// Pattern population functions
375//===----------------------------------------------------------------------===//
376
377void mlir::acc::populateACCOrphanToHostPatterns(RewritePatternSet &patterns,
378 OpenACCSupport &accSupport,
379 bool enableLoopConversion) {
380 MLIRContext *context = patterns.getContext();
381
382 // For host routines (acc routine marked for host), we only convert orphan
383 // operations that are not allowed outside compute regions. All patterns
384 // here check that the operation is NOT inside a compute region before
385 // converting:
386 // - acc.atomic.* -> load/store operations
387 // - acc.loop -> scf.for or scf.execute_region
388 // - acc.cache -> replaced with var
389 // - acc.private, acc.reduction, acc.firstprivate -> replaced with var
390 // (only if NOT connected to compute constructs or loop)
391 //
392 // We do NOT remove structured/unstructured data constructs, compute
393 // constructs, or their associated data operations - those are valid
394 // in host routines and will be processed by other passes.
395
396 // Loop conversion (orphan only)
397 if (enableLoopConversion)
398 patterns.insert<ACCOrphanLoopOpConversion>(arg&: context);
399
400 // Atomic operations - convert to non-atomic load/store (orphan only)
401 patterns.insert<ACCOrphanAtomicUpdateOpConversion>(arg&: context, args&: accSupport);
402 patterns.insert<ACCOrphanAtomicReadOpConversion>(arg&: context, args&: accSupport);
403 patterns.insert<ACCOrphanAtomicWriteOpConversion>(arg&: context, args&: accSupport);
404 patterns.insert<ACCOrphanAtomicCaptureOpConversion>(arg&: context);
405
406 // Orphan data entry ops - only convert if NOT connected to compute/loop
407 // and NOT inside a compute region
408 patterns.insert<ACCOrphanDataEntryConversion<acc::CacheOp>,
409 ACCOrphanDataEntryConversion<acc::PrivateOp>,
410 ACCOrphanDataEntryConversion<acc::FirstprivateOp>,
411 ACCOrphanDataEntryConversion<acc::ReductionOp>>(arg&: context);
412}
413
414void mlir::acc::populateACCHostFallbackPatterns(RewritePatternSet &patterns,
415 OpenACCSupport &accSupport,
416 bool enableLoopConversion) {
417 MLIRContext *context = patterns.getContext();
418
419 // For host fallback path (when `if` clause evaluates to false), ALL ACC
420 // operations within the region should be converted to host equivalents.
421 // This includes structured/unstructured data, compute constructs, and
422 // their associated data operations.
423
424 // Loop conversion - OK to use the orphan loop conversion pattern here
425 // because the parent compute constructs will also be converted.
426 if (enableLoopConversion)
427 patterns.insert<ACCOrphanLoopOpConversion>(arg&: context);
428
429 // Atomic operations - convert to non-atomic load/store. OK to use the orphan
430 // atomic conversion patterns here because the parent compute constructs will
431 // also be converted.
432 patterns.insert<ACCOrphanAtomicUpdateOpConversion>(arg&: context, args&: accSupport);
433 patterns.insert<ACCOrphanAtomicReadOpConversion>(arg&: context, args&: accSupport);
434 patterns.insert<ACCOrphanAtomicWriteOpConversion>(arg&: context, args&: accSupport);
435 patterns.insert<ACCOrphanAtomicCaptureOpConversion>(arg&: context);
436
437 // acc.cache - convert ALL cache ops (including those inside compute regions)
438 patterns.insert<ACCOpReplaceWithVarConversion<acc::CacheOp>>(arg&: context);
439
440 // Privatization ops - convert ALL (including those attached to compute/loop)
441 patterns.insert<ACCOpReplaceWithVarConversion<acc::PrivateOp>,
442 ACCOpReplaceWithVarConversion<acc::FirstprivateOp>,
443 ACCOpReplaceWithVarConversion<acc::ReductionOp>>(arg&: context);
444
445 // Data entry ops - replaced with their var operand
446 patterns.insert<ACCOpReplaceWithVarConversion<acc::CopyinOp>,
447 ACCOpReplaceWithVarConversion<acc::CreateOp>,
448 ACCOpReplaceWithVarConversion<acc::AttachOp>,
449 ACCOpReplaceWithVarConversion<acc::PresentOp>,
450 ACCOpReplaceWithVarConversion<acc::DevicePtrOp>,
451 ACCOpReplaceWithVarConversion<acc::GetDevicePtrOp>,
452 ACCOpReplaceWithVarConversion<acc::NoCreateOp>,
453 ACCOpReplaceWithVarConversion<acc::DeclareDeviceResidentOp>,
454 ACCOpReplaceWithVarConversion<acc::DeclareLinkOp>,
455 ACCOpReplaceWithVarConversion<acc::UseDeviceOp>,
456 ACCOpReplaceWithVarConversion<acc::UpdateDeviceOp>>(arg&: context);
457
458 // Data exit ops - simply erased (no results)
459 patterns.insert<ACCOpEraseConversion<acc::CopyoutOp>,
460 ACCOpEraseConversion<acc::DeleteOp>,
461 ACCOpEraseConversion<acc::DetachOp>,
462 ACCOpEraseConversion<acc::UpdateHostOp>>(arg&: context);
463
464 // Structured data constructs - unwrap their regions
465 patterns.insert<ACCRegionUnwrapConversion<acc::DataOp>,
466 ACCRegionUnwrapConversion<acc::HostDataOp>,
467 ACCRegionUnwrapConversion<acc::KernelEnvironmentOp>>(arg&: context);
468
469 // Declare ops
470 patterns.insert<ACCDeclareEnterOpConversion,
471 ACCRegionUnwrapConversion<acc::DeclareOp>>(arg&: context);
472
473 // Unstructured data operations - erase them
474 patterns.insert<ACCOpEraseConversion<acc::EnterDataOp>,
475 ACCOpEraseConversion<acc::ExitDataOp>,
476 ACCOpEraseConversion<acc::UpdateOp>>(arg&: context);
477
478 // Runtime operations - erase them
479 patterns.insert<
480 ACCOpEraseConversion<acc::InitOp>, ACCOpEraseConversion<acc::ShutdownOp>,
481 ACCOpEraseConversion<acc::SetOp>, ACCOpEraseConversion<acc::WaitOp>>(
482 arg&: context);
483
484 // Compute constructs - unwrap their regions
485 patterns.insert<ACCRegionUnwrapConversion<acc::ParallelOp>,
486 ACCRegionUnwrapConversion<acc::SerialOp>,
487 ACCRegionUnwrapConversion<acc::KernelsOp>>(arg&: context);
488}
489