| 1 | //===- ACCSpecializeForHost.cpp -------------------------------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass converts OpenACC operations to host-compatible representations, |
| 10 | // enabling execution on the host rather than on accelerator devices. |
| 11 | // |
| 12 | // Overview: |
| 13 | // --------- |
| 14 | // The pass operates in two modes depending on the `enableHostFallback` option: |
| 15 | // |
| 16 | // 1. Default Mode (Orphan Operations Only): |
| 17 | // Only converts "orphan" ACC operations that are not inside or attached to |
| 18 | // compute regions. This is used for host routines (acc routine marked for |
| 19 | // host) where structured/unstructured data constructs, compute constructs, |
| 20 | // and their associated data operations should be preserved. |
| 21 | // |
| 22 | // 2. Host Fallback Mode (enableHostFallback=true): |
| 23 | // Converts ALL ACC operations within the region to host equivalents. This |
| 24 | // is used when the `if` clause evaluates to false at runtime and the |
| 25 | // entire ACC region needs to fall back to host execution. |
| 26 | // |
| 27 | // Transformations (Orphan Mode): |
| 28 | // ------------------------------ |
| 29 | // The following orphan operations are converted: |
| 30 | // |
| 31 | // 1. Atomic Ops (converted to load/store): |
| 32 | // acc.atomic.update -> load + compute + store |
| 33 | // acc.atomic.read -> load + store (copy) |
| 34 | // acc.atomic.write -> store |
| 35 | // acc.atomic.capture -> inline region contents |
| 36 | // |
| 37 | // 2. Loop Ops (converted to SCF): |
| 38 | // acc.loop (structured) -> scf.for |
| 39 | // acc.loop (unstructured) -> scf.execute_region |
| 40 | // |
| 41 | // 3. Orphan Data Entry Ops (replaced with var operand): |
| 42 | // acc.cache, acc.private, acc.firstprivate, acc.reduction |
| 43 | // (only if NOT connected to compute constructs or loop) |
| 44 | // |
| 45 | // Transformations (Host Fallback Mode): |
| 46 | // ------------------------------------- |
| 47 | // In addition to orphan transformations, ALL of the following are converted: |
| 48 | // |
| 49 | // 1. Data Entry Ops (replaced with var operand): |
| 50 | // acc.copyin, acc.create, acc.attach, acc.present, acc.deviceptr, |
| 51 | // acc.get_deviceptr, acc.nocreate, acc.declare_device_resident, |
| 52 | // acc.declare_link, acc.use_device, acc.update_device |
| 53 | // |
| 54 | // 2. Data Exit Ops (erased): |
| 55 | // acc.copyout, acc.delete, acc.detach, acc.update_host |
| 56 | // |
| 57 | // 3. Structured Data/Compute Constructs (region inlined): |
| 58 | // acc.data, acc.host_data, acc.kernel_environment, acc.declare, |
| 59 | // acc.parallel, acc.serial, acc.kernels |
| 60 | // |
| 61 | // 4. Unstructured Data Ops (erased): |
| 62 | // acc.enter_data, acc.exit_data, acc.update |
| 63 | // |
| 64 | // 5. Declare Ops (erased): |
| 65 | // acc.declare_enter, acc.declare_exit |
| 66 | // |
| 67 | // 6. Runtime Ops (erased): |
| 68 | // acc.init, acc.shutdown, acc.set, acc.wait, acc.terminator |
| 69 | // |
| 70 | // Requirements: |
| 71 | // ------------- |
| 72 | // For atomic operation conversion, variables must implement the |
| 73 | // `acc::PointerLikeType` interface to enable generating load/store operations. |
| 74 | // |
| 75 | // The pass uses `OpenACCSupport::emitNYI()` to report unsupported cases. |
| 76 | // |
| 77 | //===----------------------------------------------------------------------===// |
| 78 | |
| 79 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 80 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 81 | |
| 82 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 83 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 84 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 85 | #include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h" |
| 86 | #include "mlir/Dialect/OpenACC/Transforms/ACCSpecializePatterns.h" |
| 87 | #include "mlir/IR/BuiltinAttributes.h" |
| 88 | #include "mlir/IR/BuiltinOps.h" |
| 89 | #include "mlir/IR/BuiltinTypes.h" |
| 90 | #include "mlir/IR/Operation.h" |
| 91 | #include "mlir/IR/PatternMatch.h" |
| 92 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 93 | |
| 94 | namespace mlir { |
| 95 | namespace acc { |
| 96 | #define GEN_PASS_DEF_ACCSPECIALIZEFORHOST |
| 97 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 98 | } // namespace acc |
| 99 | } // namespace mlir |
| 100 | |
| 101 | #define DEBUG_TYPE "acc-specialize-for-host" |
| 102 | |
| 103 | using namespace mlir; |
| 104 | using namespace mlir::acc; |
| 105 | |
| 106 | /// Check if an operation is inside an ACC compute construct. |
| 107 | static bool isInsideACCComputeConstruct(Operation *op) { |
| 108 | while ((op = op->getParentOp())) |
| 109 | if (isa<ACC_COMPUTE_CONSTRUCT_OPS>(Val: op)) |
| 110 | return true; |
| 111 | return false; |
| 112 | } |
| 113 | |
| 114 | /// Return true if an enclosing compute construct or capture must be removed |
| 115 | /// before converting an atomic operation. |
| 116 | static bool isAtomicConversionDeferred(Operation *op) { |
| 117 | return isInsideACCComputeConstruct(op) || |
| 118 | op->getParentOfType<acc::AtomicCaptureOp>(); |
| 119 | } |
| 120 | |
| 121 | namespace { |
| 122 | // Lower orphan acc.atomic.update by: load from addr, clone region expr with |
| 123 | // the loaded value, then store the computed result back to addr. |
| 124 | // Only matches outside compute regions and atomic captures. |
| 125 | class ACCOrphanAtomicUpdateOpConversion |
| 126 | : public OpRewritePattern<acc::AtomicUpdateOp> { |
| 127 | public: |
| 128 | ACCOrphanAtomicUpdateOpConversion(MLIRContext *ctx, OpenACCSupport &support) |
| 129 | : OpRewritePattern<acc::AtomicUpdateOp>(ctx), accSupport(support) {} |
| 130 | |
| 131 | LogicalResult matchAndRewrite(acc::AtomicUpdateOp atomicUpdateOp, |
| 132 | PatternRewriter &rewriter) const override { |
| 133 | if (isAtomicConversionDeferred(op: atomicUpdateOp)) |
| 134 | return failure(); |
| 135 | |
| 136 | Value x = atomicUpdateOp.getX(); |
| 137 | Type type = x.getType(); |
| 138 | auto ptrLikeType = dyn_cast<acc::PointerLikeType>(Val&: type); |
| 139 | if (ptrLikeType) { |
| 140 | auto xTyped = cast<TypedValue<acc::PointerLikeType>>(Val&: x); |
| 141 | rewriter.setInsertionPointAfter(atomicUpdateOp); |
| 142 | Value loadOp = |
| 143 | ptrLikeType.genLoad(builder&: rewriter, loc: atomicUpdateOp.getLoc(), srcPtr: xTyped, valueType: {}); |
| 144 | if (!loadOp) { |
| 145 | accSupport.emitNYI(loc: atomicUpdateOp.getLoc(), |
| 146 | message: "failed to generate load for atomic update" ); |
| 147 | return failure(); |
| 148 | } |
| 149 | IRMapping mapping; |
| 150 | mapping.map(from: atomicUpdateOp.getRegion().front().getArgument(i: 0), to: loadOp); |
| 151 | Block &block = atomicUpdateOp.getRegion().front(); |
| 152 | for (Operation &op : block.without_terminator()) |
| 153 | rewriter.clone(op, mapper&: mapping); |
| 154 | auto yieldOp = cast<acc::YieldOp>(Val: block.getTerminator()); |
| 155 | Value result = mapping.lookup(from: yieldOp.getOperand(i: 0)); |
| 156 | if (!ptrLikeType.genStore(builder&: rewriter, loc: atomicUpdateOp.getLoc(), valueToStore: result, |
| 157 | destPtr: xTyped)) { |
| 158 | accSupport.emitNYI(loc: atomicUpdateOp.getLoc(), |
| 159 | message: "failed to generate store for atomic update" ); |
| 160 | return failure(); |
| 161 | } |
| 162 | rewriter.eraseOp(op: atomicUpdateOp); |
| 163 | } else { |
| 164 | accSupport.emitNYI(loc: atomicUpdateOp.getLoc(), |
| 165 | message: "unsupported type for atomic update" ); |
| 166 | return failure(); |
| 167 | } |
| 168 | return success(); |
| 169 | } |
| 170 | |
| 171 | private: |
| 172 | OpenACCSupport &accSupport; |
| 173 | }; |
| 174 | |
| 175 | // Lower orphan acc.atomic.read by: load from src, then store into dst. |
| 176 | // Only matches outside compute regions and atomic captures. |
| 177 | class ACCOrphanAtomicReadOpConversion |
| 178 | : public OpRewritePattern<acc::AtomicReadOp> { |
| 179 | public: |
| 180 | ACCOrphanAtomicReadOpConversion(MLIRContext *ctx, OpenACCSupport &support) |
| 181 | : OpRewritePattern<acc::AtomicReadOp>(ctx), accSupport(support) {} |
| 182 | |
| 183 | LogicalResult matchAndRewrite(acc::AtomicReadOp readOp, |
| 184 | PatternRewriter &rewriter) const override { |
| 185 | if (isAtomicConversionDeferred(op: readOp)) |
| 186 | return failure(); |
| 187 | |
| 188 | Value x = readOp.getX(); |
| 189 | Value v = readOp.getV(); |
| 190 | auto xPtrType = dyn_cast<acc::PointerLikeType>(Val: x.getType()); |
| 191 | auto vPtrType = dyn_cast<acc::PointerLikeType>(Val: v.getType()); |
| 192 | if (xPtrType && vPtrType) { |
| 193 | auto xTyped = cast<TypedValue<acc::PointerLikeType>>(Val&: x); |
| 194 | auto vTyped = cast<TypedValue<acc::PointerLikeType>>(Val&: v); |
| 195 | rewriter.setInsertionPointAfter(readOp); |
| 196 | |
| 197 | // Use genCopy which does load + store |
| 198 | if (!xPtrType.genCopy(builder&: rewriter, loc: readOp.getLoc(), destination: vTyped, source: xTyped, varType: {})) { |
| 199 | accSupport.emitNYI(loc: readOp.getLoc(), |
| 200 | message: "failed to generate copy for atomic read" ); |
| 201 | return failure(); |
| 202 | } |
| 203 | rewriter.eraseOp(op: readOp); |
| 204 | } else { |
| 205 | accSupport.emitNYI(loc: readOp.getLoc(), message: "unsupported type for atomic read" ); |
| 206 | return failure(); |
| 207 | } |
| 208 | return success(); |
| 209 | } |
| 210 | |
| 211 | private: |
| 212 | OpenACCSupport &accSupport; |
| 213 | }; |
| 214 | |
| 215 | // Lower orphan acc.atomic.write by: store value into addr. |
| 216 | // Only matches outside compute regions and atomic captures. |
| 217 | class ACCOrphanAtomicWriteOpConversion |
| 218 | : public OpRewritePattern<acc::AtomicWriteOp> { |
| 219 | public: |
| 220 | ACCOrphanAtomicWriteOpConversion(MLIRContext *ctx, OpenACCSupport &support) |
| 221 | : OpRewritePattern<acc::AtomicWriteOp>(ctx), accSupport(support) {} |
| 222 | |
| 223 | LogicalResult matchAndRewrite(acc::AtomicWriteOp writeOp, |
| 224 | PatternRewriter &rewriter) const override { |
| 225 | if (isAtomicConversionDeferred(op: writeOp)) |
| 226 | return failure(); |
| 227 | |
| 228 | Value x = writeOp.getX(); |
| 229 | Value expr = writeOp.getExpr(); |
| 230 | auto ptrLikeType = dyn_cast<acc::PointerLikeType>(Val: x.getType()); |
| 231 | if (ptrLikeType) { |
| 232 | auto xTyped = cast<TypedValue<acc::PointerLikeType>>(Val&: x); |
| 233 | rewriter.setInsertionPointAfter(writeOp); |
| 234 | if (!ptrLikeType.genStore(builder&: rewriter, loc: writeOp.getLoc(), valueToStore: expr, destPtr: xTyped)) { |
| 235 | accSupport.emitNYI(loc: writeOp.getLoc(), |
| 236 | message: "failed to generate store for atomic write" ); |
| 237 | return failure(); |
| 238 | } |
| 239 | rewriter.eraseOp(op: writeOp); |
| 240 | } else { |
| 241 | accSupport.emitNYI(loc: writeOp.getLoc(), message: "unsupported type for atomic write" ); |
| 242 | return failure(); |
| 243 | } |
| 244 | return success(); |
| 245 | } |
| 246 | |
| 247 | private: |
| 248 | OpenACCSupport &accSupport; |
| 249 | }; |
| 250 | |
| 251 | // Lower orphan acc.atomic.capture by: unwrap the capture region and erase the |
| 252 | // wrapper; inner ops are lowered in-order (e.g., read+update becomes load/store |
| 253 | // to dst then load/compute/store to addr). |
| 254 | // Only matches if NOT inside an ACC compute construct. |
| 255 | class ACCOrphanAtomicCaptureOpConversion |
| 256 | : public OpRewritePattern<acc::AtomicCaptureOp> { |
| 257 | using OpRewritePattern<acc::AtomicCaptureOp>::OpRewritePattern; |
| 258 | |
| 259 | LogicalResult matchAndRewrite(acc::AtomicCaptureOp captureOp, |
| 260 | PatternRewriter &rewriter) const override { |
| 261 | // Only convert if this op is not inside an ACC compute construct |
| 262 | if (isInsideACCComputeConstruct(op: captureOp)) |
| 263 | return failure(); |
| 264 | |
| 265 | assert(captureOp.getRegion().hasOneBlock() && "expected one block" ); |
| 266 | Block *block = &captureOp.getRegion().front(); |
| 267 | // Remove the terminator before inlining |
| 268 | rewriter.eraseOp(op: block->getTerminator()); |
| 269 | rewriter.inlineBlockBefore(source: block, op: captureOp); |
| 270 | rewriter.eraseOp(op: captureOp); |
| 271 | return success(); |
| 272 | } |
| 273 | }; |
| 274 | |
| 275 | // Convert orphan acc.loop to scf.for or scf.execute_region. |
| 276 | // Only matches if NOT inside an ACC compute construct. |
| 277 | class ACCOrphanLoopOpConversion : public OpRewritePattern<acc::LoopOp> { |
| 278 | using OpRewritePattern<acc::LoopOp>::OpRewritePattern; |
| 279 | |
| 280 | LogicalResult matchAndRewrite(acc::LoopOp loopOp, |
| 281 | PatternRewriter &rewriter) const override { |
| 282 | // Only convert if this op is not inside an ACC compute construct |
| 283 | if (isInsideACCComputeConstruct(op: loopOp)) |
| 284 | return failure(); |
| 285 | |
| 286 | if (loopOp.getUnstructured()) { |
| 287 | auto executeRegion = |
| 288 | acc::convertUnstructuredACCLoopToSCFExecuteRegion(loopOp, rewriter); |
| 289 | if (!executeRegion) |
| 290 | return failure(); |
| 291 | rewriter.replaceOp(op: loopOp, newOp: executeRegion); |
| 292 | } else { |
| 293 | auto forOp = acc::convertACCLoopToSCFFor(loopOp, rewriter, |
| 294 | /*enableCollapse=*/false); |
| 295 | if (!forOp) |
| 296 | return failure(); |
| 297 | rewriter.replaceOp(op: loopOp, newOp: forOp); |
| 298 | } |
| 299 | return success(); |
| 300 | } |
| 301 | }; |
| 302 | |
| 303 | /// Check if an operation is used by a compute construct or loop op |
| 304 | static bool isUsedByComputeOrLoop(Operation *op) { |
| 305 | for (auto *user : op->getUsers()) |
| 306 | if (isa<acc::ParallelOp, acc::SerialOp, acc::KernelsOp, acc::LoopOp>(Val: user)) |
| 307 | return true; |
| 308 | return false; |
| 309 | } |
| 310 | |
| 311 | /// Orphan data entry ops - only match if NOT connected to compute/loop and |
| 312 | /// NOT inside a compute region. Used for acc.cache, acc.private, |
| 313 | /// acc.firstprivate, acc.reduction. |
| 314 | template <typename OpTy> |
| 315 | class ACCOrphanDataEntryConversion : public OpRewritePattern<OpTy> { |
| 316 | using OpRewritePattern<OpTy>::OpRewritePattern; |
| 317 | |
| 318 | LogicalResult matchAndRewrite(OpTy op, |
| 319 | PatternRewriter &rewriter) const override { |
| 320 | // Only convert if this op is not used by a compute construct or loop, |
| 321 | // and not inside an ACC compute construct. |
| 322 | if (isUsedByComputeOrLoop(op) || isInsideACCComputeConstruct(op)) |
| 323 | return failure(); |
| 324 | |
| 325 | if (op->use_empty()) |
| 326 | rewriter.eraseOp(op); |
| 327 | else |
| 328 | rewriter.replaceOp(op, op.getVar()); |
| 329 | return success(); |
| 330 | } |
| 331 | }; |
| 332 | |
| 333 | class ACCSpecializeForHost |
| 334 | : public acc::impl::ACCSpecializeForHostBase<ACCSpecializeForHost> { |
| 335 | public: |
| 336 | using ACCSpecializeForHostBase< |
| 337 | ACCSpecializeForHost>::ACCSpecializeForHostBase; |
| 338 | |
| 339 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 340 | ACCSpecializeForHostBase<ACCSpecializeForHost>::getDependentDialects( |
| 341 | registry); |
| 342 | acc::registerAllExtensions(registry); |
| 343 | } |
| 344 | |
| 345 | void runOnOperation() override { |
| 346 | LLVM_DEBUG(llvm::dbgs() << "Enter ACCSpecializeForHost()\n" ); |
| 347 | |
| 348 | func::FuncOp funcOp = getOperation(); |
| 349 | if (!acc::isSpecializedAccRoutine(op: funcOp)) { |
| 350 | // Convert orphan operations to host, or all ACC operations if |
| 351 | // host fallback patterns are enabled. |
| 352 | auto *context = &getContext(); |
| 353 | RewritePatternSet patterns(context); |
| 354 | OpenACCSupport &accSupport = getAnalysis<OpenACCSupport>(); |
| 355 | if (enableHostFallback) |
| 356 | populateACCHostFallbackPatterns(patterns, accSupport); |
| 357 | else |
| 358 | populateACCOrphanToHostPatterns(patterns, accSupport); |
| 359 | GreedyRewriteConfig config; |
| 360 | config.setUseTopDownTraversal(true); |
| 361 | // Deeply nested orphan acc.loops can need more than the default |
| 362 | // iteration cap to converge; lift it to avoid spurious pass failure. |
| 363 | config.setMaxIterations(GreedyRewriteConfig::kNoLimit); |
| 364 | if (failed(Result: applyPatternsGreedily(op: funcOp, patterns: std::move(patterns), config))) |
| 365 | signalPassFailure(); |
| 366 | } |
| 367 | |
| 368 | LLVM_DEBUG(llvm::dbgs() << "Exit ACCSpecializeForHost()\n" ); |
| 369 | } |
| 370 | }; |
| 371 | } // namespace |
| 372 | |
| 373 | //===----------------------------------------------------------------------===// |
| 374 | // Pattern population functions |
| 375 | //===----------------------------------------------------------------------===// |
| 376 | |
| 377 | void mlir::acc::populateACCOrphanToHostPatterns(RewritePatternSet &patterns, |
| 378 | OpenACCSupport &accSupport, |
| 379 | bool enableLoopConversion) { |
| 380 | MLIRContext *context = patterns.getContext(); |
| 381 | |
| 382 | // For host routines (acc routine marked for host), we only convert orphan |
| 383 | // operations that are not allowed outside compute regions. All patterns |
| 384 | // here check that the operation is NOT inside a compute region before |
| 385 | // converting: |
| 386 | // - acc.atomic.* -> load/store operations |
| 387 | // - acc.loop -> scf.for or scf.execute_region |
| 388 | // - acc.cache -> replaced with var |
| 389 | // - acc.private, acc.reduction, acc.firstprivate -> replaced with var |
| 390 | // (only if NOT connected to compute constructs or loop) |
| 391 | // |
| 392 | // We do NOT remove structured/unstructured data constructs, compute |
| 393 | // constructs, or their associated data operations - those are valid |
| 394 | // in host routines and will be processed by other passes. |
| 395 | |
| 396 | // Loop conversion (orphan only) |
| 397 | if (enableLoopConversion) |
| 398 | patterns.insert<ACCOrphanLoopOpConversion>(arg&: context); |
| 399 | |
| 400 | // Atomic operations - convert to non-atomic load/store (orphan only) |
| 401 | patterns.insert<ACCOrphanAtomicUpdateOpConversion>(arg&: context, args&: accSupport); |
| 402 | patterns.insert<ACCOrphanAtomicReadOpConversion>(arg&: context, args&: accSupport); |
| 403 | patterns.insert<ACCOrphanAtomicWriteOpConversion>(arg&: context, args&: accSupport); |
| 404 | patterns.insert<ACCOrphanAtomicCaptureOpConversion>(arg&: context); |
| 405 | |
| 406 | // Orphan data entry ops - only convert if NOT connected to compute/loop |
| 407 | // and NOT inside a compute region |
| 408 | patterns.insert<ACCOrphanDataEntryConversion<acc::CacheOp>, |
| 409 | ACCOrphanDataEntryConversion<acc::PrivateOp>, |
| 410 | ACCOrphanDataEntryConversion<acc::FirstprivateOp>, |
| 411 | ACCOrphanDataEntryConversion<acc::ReductionOp>>(arg&: context); |
| 412 | } |
| 413 | |
| 414 | void mlir::acc::populateACCHostFallbackPatterns(RewritePatternSet &patterns, |
| 415 | OpenACCSupport &accSupport, |
| 416 | bool enableLoopConversion) { |
| 417 | MLIRContext *context = patterns.getContext(); |
| 418 | |
| 419 | // For host fallback path (when `if` clause evaluates to false), ALL ACC |
| 420 | // operations within the region should be converted to host equivalents. |
| 421 | // This includes structured/unstructured data, compute constructs, and |
| 422 | // their associated data operations. |
| 423 | |
| 424 | // Loop conversion - OK to use the orphan loop conversion pattern here |
| 425 | // because the parent compute constructs will also be converted. |
| 426 | if (enableLoopConversion) |
| 427 | patterns.insert<ACCOrphanLoopOpConversion>(arg&: context); |
| 428 | |
| 429 | // Atomic operations - convert to non-atomic load/store. OK to use the orphan |
| 430 | // atomic conversion patterns here because the parent compute constructs will |
| 431 | // also be converted. |
| 432 | patterns.insert<ACCOrphanAtomicUpdateOpConversion>(arg&: context, args&: accSupport); |
| 433 | patterns.insert<ACCOrphanAtomicReadOpConversion>(arg&: context, args&: accSupport); |
| 434 | patterns.insert<ACCOrphanAtomicWriteOpConversion>(arg&: context, args&: accSupport); |
| 435 | patterns.insert<ACCOrphanAtomicCaptureOpConversion>(arg&: context); |
| 436 | |
| 437 | // acc.cache - convert ALL cache ops (including those inside compute regions) |
| 438 | patterns.insert<ACCOpReplaceWithVarConversion<acc::CacheOp>>(arg&: context); |
| 439 | |
| 440 | // Privatization ops - convert ALL (including those attached to compute/loop) |
| 441 | patterns.insert<ACCOpReplaceWithVarConversion<acc::PrivateOp>, |
| 442 | ACCOpReplaceWithVarConversion<acc::FirstprivateOp>, |
| 443 | ACCOpReplaceWithVarConversion<acc::ReductionOp>>(arg&: context); |
| 444 | |
| 445 | // Data entry ops - replaced with their var operand |
| 446 | patterns.insert<ACCOpReplaceWithVarConversion<acc::CopyinOp>, |
| 447 | ACCOpReplaceWithVarConversion<acc::CreateOp>, |
| 448 | ACCOpReplaceWithVarConversion<acc::AttachOp>, |
| 449 | ACCOpReplaceWithVarConversion<acc::PresentOp>, |
| 450 | ACCOpReplaceWithVarConversion<acc::DevicePtrOp>, |
| 451 | ACCOpReplaceWithVarConversion<acc::GetDevicePtrOp>, |
| 452 | ACCOpReplaceWithVarConversion<acc::NoCreateOp>, |
| 453 | ACCOpReplaceWithVarConversion<acc::DeclareDeviceResidentOp>, |
| 454 | ACCOpReplaceWithVarConversion<acc::DeclareLinkOp>, |
| 455 | ACCOpReplaceWithVarConversion<acc::UseDeviceOp>, |
| 456 | ACCOpReplaceWithVarConversion<acc::UpdateDeviceOp>>(arg&: context); |
| 457 | |
| 458 | // Data exit ops - simply erased (no results) |
| 459 | patterns.insert<ACCOpEraseConversion<acc::CopyoutOp>, |
| 460 | ACCOpEraseConversion<acc::DeleteOp>, |
| 461 | ACCOpEraseConversion<acc::DetachOp>, |
| 462 | ACCOpEraseConversion<acc::UpdateHostOp>>(arg&: context); |
| 463 | |
| 464 | // Structured data constructs - unwrap their regions |
| 465 | patterns.insert<ACCRegionUnwrapConversion<acc::DataOp>, |
| 466 | ACCRegionUnwrapConversion<acc::HostDataOp>, |
| 467 | ACCRegionUnwrapConversion<acc::KernelEnvironmentOp>>(arg&: context); |
| 468 | |
| 469 | // Declare ops |
| 470 | patterns.insert<ACCDeclareEnterOpConversion, |
| 471 | ACCRegionUnwrapConversion<acc::DeclareOp>>(arg&: context); |
| 472 | |
| 473 | // Unstructured data operations - erase them |
| 474 | patterns.insert<ACCOpEraseConversion<acc::EnterDataOp>, |
| 475 | ACCOpEraseConversion<acc::ExitDataOp>, |
| 476 | ACCOpEraseConversion<acc::UpdateOp>>(arg&: context); |
| 477 | |
| 478 | // Runtime operations - erase them |
| 479 | patterns.insert< |
| 480 | ACCOpEraseConversion<acc::InitOp>, ACCOpEraseConversion<acc::ShutdownOp>, |
| 481 | ACCOpEraseConversion<acc::SetOp>, ACCOpEraseConversion<acc::WaitOp>>( |
| 482 | arg&: context); |
| 483 | |
| 484 | // Compute constructs - unwrap their regions |
| 485 | patterns.insert<ACCRegionUnwrapConversion<acc::ParallelOp>, |
| 486 | ACCRegionUnwrapConversion<acc::SerialOp>, |
| 487 | ACCRegionUnwrapConversion<acc::KernelsOp>>(arg&: context); |
| 488 | } |
| 489 | |