| 1 | //===- ACCIfClauseLowering.cpp - Lower ACC compute construct if clauses --===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass lowers OpenACC compute constructs (parallel, kernels, serial) with |
| 10 | // `if` clauses using region specialization. It creates two execution paths: |
| 11 | // device execution when the condition is true, host execution when false. |
| 12 | // |
| 13 | // Overview: |
| 14 | // --------- |
| 15 | // When an ACC compute construct has an `if` clause, the construct should only |
| 16 | // execute on the device when the condition is true. If the condition is false, |
| 17 | // the code should execute on the host instead. This pass transforms: |
| 18 | // |
| 19 | // acc.parallel if(%cond) { ... } |
| 20 | // |
| 21 | // Into: |
| 22 | // |
| 23 | // scf.if %cond { |
| 24 | // // Device path: clone data ops, compute construct without if, exit ops |
| 25 | // acc.parallel { ... } |
| 26 | // } else { |
| 27 | // // Host path: original region body with ACC ops converted to host |
| 28 | // } |
| 29 | // |
| 30 | // Transformations: |
| 31 | // ---------------- |
| 32 | // For each compute construct with an `if` clause: |
| 33 | // |
| 34 | // 1. Device Path (true branch): |
| 35 | // - Clone data entry operations (acc.copyin, acc.create, etc.) |
| 36 | // - Clone the compute construct without the `if` clause |
| 37 | // - Clone data exit operations (acc.copyout, acc.delete, etc.) |
| 38 | // |
| 39 | // 2. Host Path (false branch): |
| 40 | // - Move the original region body to the else branch |
| 41 | // - Apply host fallback patterns to convert ACC ops to host equivalents |
| 42 | // |
| 43 | // 3. Cleanup: |
| 44 | // - Erase the original compute construct and data operations |
| 45 | // - Replace uses of ACC variables with host variables in the else branch |
| 46 | // |
| 47 | // Requirements: |
| 48 | // ------------- |
| 49 | // To use this pass in a pipeline, the following requirements exist: |
| 50 | // |
| 51 | // 1. Analysis Registration (Optional): If custom behavior is needed for |
| 52 | // emitting not-yet-implemented messages for unsupported cases, the pipeline |
| 53 | // should pre-register the `acc::OpenACCSupport` analysis. |
| 54 | // |
| 55 | //===----------------------------------------------------------------------===// |
| 56 | |
| 57 | #include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h" |
| 58 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h" |
| 59 | |
| 60 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 61 | #include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h" |
| 62 | #include "mlir/Dialect/OpenACC/OpenACC.h" |
| 63 | #include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h" |
| 64 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 65 | #include "mlir/IR/Builders.h" |
| 66 | #include "mlir/IR/IRMapping.h" |
| 67 | #include "mlir/IR/PatternMatch.h" |
| 68 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 69 | #include "llvm/ADT/STLExtras.h" |
| 70 | #include "llvm/ADT/SetVector.h" |
| 71 | #include "llvm/Support/Debug.h" |
| 72 | |
| 73 | namespace mlir { |
| 74 | namespace acc { |
| 75 | #define GEN_PASS_DEF_ACCIFCLAUSELOWERING |
| 76 | #include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc" |
| 77 | } // namespace acc |
| 78 | } // namespace mlir |
| 79 | |
| 80 | #define DEBUG_TYPE "acc-if-clause-lowering" |
| 81 | |
| 82 | using namespace mlir; |
| 83 | using namespace mlir::acc; |
| 84 | |
| 85 | namespace { |
| 86 | class ACCIfClauseLowering |
| 87 | : public acc::impl::ACCIfClauseLoweringBase<ACCIfClauseLowering> { |
| 88 | using ACCIfClauseLoweringBase<ACCIfClauseLowering>::ACCIfClauseLoweringBase; |
| 89 | |
| 90 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 91 | ACCIfClauseLoweringBase<ACCIfClauseLowering>::getDependentDialects( |
| 92 | registry); |
| 93 | acc::registerAllExtensions(registry); |
| 94 | } |
| 95 | |
| 96 | private: |
| 97 | OpenACCSupport *accSupport = nullptr; |
| 98 | |
| 99 | void convertHostRegion(Operation *computeOp, Region ®ion); |
| 100 | |
| 101 | template <typename OpTy> |
| 102 | void |
| 103 | lowerIfClauseForComputeConstruct(OpTy computeConstructOp, |
| 104 | llvm::SetVector<Operation *> &eraseOps, |
| 105 | llvm::SetVector<Operation *> &condEraseOps); |
| 106 | |
| 107 | public: |
| 108 | void runOnOperation() override; |
| 109 | }; |
| 110 | |
| 111 | void ACCIfClauseLowering::convertHostRegion(Operation *computeOp, |
| 112 | Region ®ion) { |
| 113 | // Only collect ACC dialect operations - other ops don't need conversion |
| 114 | SmallVector<Operation *> hostOps; |
| 115 | region.walk<WalkOrder::PreOrder>(callback: [&](Operation *op) { |
| 116 | if (isa<acc::OpenACCDialect>(Val: op->getDialect())) |
| 117 | hostOps.push_back(Elt: op); |
| 118 | }); |
| 119 | |
| 120 | RewritePatternSet patterns(computeOp->getContext()); |
| 121 | populateACCHostFallbackPatterns(patterns, accSupport&: *accSupport); |
| 122 | |
| 123 | GreedyRewriteConfig config; |
| 124 | config.setUseTopDownTraversal(true); |
| 125 | config.setStrictness(GreedyRewriteStrictness::ExistingOps); |
| 126 | if (failed(Result: applyOpPatternsGreedily(ops: hostOps, patterns: std::move(patterns), config))) |
| 127 | accSupport->emitNYI(loc: computeOp->getLoc(), message: "failed to convert host region" ); |
| 128 | } |
| 129 | |
| 130 | // Template function to handle if condition conversion for ACC compute |
| 131 | // constructs |
| 132 | template <typename OpTy> |
| 133 | void ACCIfClauseLowering::lowerIfClauseForComputeConstruct( |
| 134 | OpTy computeConstructOp, llvm::SetVector<Operation *> &eraseOps, |
| 135 | llvm::SetVector<Operation *> &condEraseOps) { |
| 136 | Value ifCond = computeConstructOp.getIfCond(); |
| 137 | if (!ifCond) |
| 138 | return; |
| 139 | |
| 140 | IRRewriter rewriter(computeConstructOp); |
| 141 | |
| 142 | LLVM_DEBUG(llvm::dbgs() << "Converting " << computeConstructOp->getName() |
| 143 | << " with if condition: " << computeConstructOp |
| 144 | << "\n" ); |
| 145 | |
| 146 | // Collect data clause operations that need to be recreated in the if |
| 147 | // condition |
| 148 | llvm::SetVector<Operation *> dataEntryOps; |
| 149 | llvm::SetVector<Operation *> dataExitOps; |
| 150 | SmallVector<Operation *> firstprivateOps; |
| 151 | SmallVector<Operation *> privateOps; |
| 152 | SmallVector<Operation *> reductionOps; |
| 153 | |
| 154 | // Entries used outside this construct belong to the surrounding scope. |
| 155 | // Their entry and exit operations must remain outside the conditional. |
| 156 | auto isExternallyOwned = [&](Operation *dataOp) { |
| 157 | for (Operation *user : dataOp->getUsers()) |
| 158 | if (!isa<ACC_DATA_EXIT_OPS>(Val: user) && user != computeConstructOp && |
| 159 | !computeConstructOp->isAncestor(user)) |
| 160 | return true; |
| 161 | return false; |
| 162 | }; |
| 163 | |
| 164 | // Collect data entry operations |
| 165 | for (Value operand : computeConstructOp.getDataClauseOperands()) |
| 166 | if (Operation *defOp = operand.getDefiningOp()) |
| 167 | if (isa<ACC_DATA_ENTRY_OPS>(Val: defOp)) |
| 168 | dataEntryOps.insert(X: defOp); |
| 169 | |
| 170 | // Find corresponding exit operations for each local entry operation. Exit ops |
| 171 | // of externally owned entry ops belong to the enclosing construct. |
| 172 | // Iterate backwards through entry ops since exit ops appear in reverse order. |
| 173 | for (Operation *dataEntryOp : llvm::reverse(C&: dataEntryOps)) |
| 174 | if (!isExternallyOwned(dataEntryOp)) |
| 175 | for (Operation *user : dataEntryOp->getUsers()) |
| 176 | if (isa<ACC_DATA_EXIT_OPS>(Val: user)) |
| 177 | dataExitOps.insert(X: user); |
| 178 | |
| 179 | // Collect firstprivate, private, and reduction operations |
| 180 | auto collectOps = [&](SmallVector<Operation *> &ops, OperandRange operands) { |
| 181 | for (Value operand : operands) |
| 182 | if (Operation *defOp = operand.getDefiningOp()) |
| 183 | ops.push_back(Elt: defOp); |
| 184 | }; |
| 185 | collectOps(firstprivateOps, computeConstructOp.getFirstprivateOperands()); |
| 186 | collectOps(privateOps, computeConstructOp.getPrivateOperands()); |
| 187 | collectOps(reductionOps, computeConstructOp.getReductionOperands()); |
| 188 | |
| 189 | // Create scf.if with device and host execution paths |
| 190 | auto ifOp = scf::IfOp::create(rewriter, computeConstructOp.getLoc(), |
| 191 | TypeRange{}, ifCond, /*withElseRegion=*/true); |
| 192 | |
| 193 | LLVM_DEBUG(llvm::dbgs() << "Cloning " << dataEntryOps.size() |
| 194 | << " data entry operations for device path\n" ); |
| 195 | |
| 196 | // Device execution path (true branch) |
| 197 | Block &thenBlock = ifOp.getThenRegion().front(); |
| 198 | rewriter.setInsertionPointToStart(&thenBlock); |
| 199 | |
| 200 | // Clone data entry operations |
| 201 | SmallVector<Value> deviceDataOperands; |
| 202 | SmallVector<Value> firstprivateOperands; |
| 203 | SmallVector<Value> privateOperands; |
| 204 | SmallVector<Value> reductionOperands; |
| 205 | |
| 206 | // Map the data entry and firstprivate ops for the cloned region |
| 207 | IRMapping deviceMapping; |
| 208 | auto cloneAndMapOps = [&](SmallVector<Operation *> &ops, |
| 209 | SmallVector<Value> &operands) { |
| 210 | for (Operation *op : ops) { |
| 211 | Operation *clonedOp = rewriter.clone(op&: *op, mapper&: deviceMapping); |
| 212 | operands.push_back(Elt: clonedOp->getResult(idx: 0)); |
| 213 | deviceMapping.map(from: op->getResult(idx: 0), to: clonedOp->getResult(idx: 0)); |
| 214 | } |
| 215 | }; |
| 216 | // Clone each local data entry op once. Externally owned ops (mapped by a |
| 217 | // surrounding scope) are referenced directly. |
| 218 | for (Operation *op : dataEntryOps) { |
| 219 | if (isExternallyOwned(op)) |
| 220 | continue; |
| 221 | Operation *clonedOp = rewriter.clone(op&: *op, mapper&: deviceMapping); |
| 222 | deviceMapping.map(from: op->getResult(idx: 0), to: clonedOp->getResult(idx: 0)); |
| 223 | } |
| 224 | // Preserve the original operand order and multiplicity while using the |
| 225 | // cloned value for local entries and the original value for external ones. |
| 226 | for (Value operand : computeConstructOp.getDataClauseOperands()) |
| 227 | deviceDataOperands.push_back(Elt: deviceMapping.lookupOrDefault(from: operand)); |
| 228 | cloneAndMapOps(firstprivateOps, firstprivateOperands); |
| 229 | cloneAndMapOps(privateOps, privateOperands); |
| 230 | cloneAndMapOps(reductionOps, reductionOperands); |
| 231 | |
| 232 | // Create new compute op without if condition for device execution by |
| 233 | // cloning |
| 234 | OpTy newComputeOp = cast<OpTy>( |
| 235 | rewriter.clone(*computeConstructOp.getOperation(), deviceMapping)); |
| 236 | newComputeOp.getIfCondMutable().clear(); |
| 237 | newComputeOp.getDataClauseOperandsMutable().assign(deviceDataOperands); |
| 238 | newComputeOp.getFirstprivateOperandsMutable().assign(firstprivateOperands); |
| 239 | newComputeOp.getPrivateOperandsMutable().assign(privateOperands); |
| 240 | newComputeOp.getReductionOperandsMutable().assign(reductionOperands); |
| 241 | |
| 242 | // Clone data exit operations |
| 243 | rewriter.setInsertionPointAfter(newComputeOp); |
| 244 | for (Operation *dataOp : dataExitOps) |
| 245 | rewriter.clone(op&: *dataOp, mapper&: deviceMapping); |
| 246 | |
| 247 | rewriter.setInsertionPointToEnd(&thenBlock); |
| 248 | if (!thenBlock.getTerminator()) |
| 249 | scf::YieldOp::create(rewriter, computeConstructOp.getLoc()); |
| 250 | |
| 251 | // Host execution path (false branch) |
| 252 | Region &hostRegion = computeConstructOp.getRegion(); |
| 253 | if (hostRegion.hasOneBlock()) { |
| 254 | // Don't need to clone original ops, just take them and legalize for host. |
| 255 | ifOp.getElseRegion().takeBody(hostRegion); |
| 256 | |
| 257 | // Swap acc yield for scf yield. |
| 258 | Block &elseBlock = ifOp.getElseRegion().front(); |
| 259 | elseBlock.getTerminator()->erase(); |
| 260 | rewriter.setInsertionPointToEnd(&elseBlock); |
| 261 | scf::YieldOp::create(rewriter, computeConstructOp.getLoc()); |
| 262 | |
| 263 | convertHostRegion(computeOp: computeConstructOp, region&: ifOp.getElseRegion()); |
| 264 | } else { |
| 265 | // scf.if regions must stay single-block. Wrap the original multi-block ACC |
| 266 | // body in scf.execute_region so it can be hosted in the else branch. |
| 267 | Block &elseBlock = ifOp.getElseRegion().front(); |
| 268 | rewriter.setInsertionPoint(elseBlock.getTerminator()); |
| 269 | IRMapping hostMapping; |
| 270 | auto hostExecuteRegion = wrapMultiBlockRegionWithSCFExecuteRegion( |
| 271 | hostRegion, hostMapping, computeConstructOp.getLoc(), rewriter); |
| 272 | convertHostRegion(computeOp: computeConstructOp, region&: hostExecuteRegion.getRegion()); |
| 273 | } |
| 274 | |
| 275 | // The original op is now empty and can be erased |
| 276 | eraseOps.insert(computeConstructOp); |
| 277 | |
| 278 | // TODO: Can probably 'move' the data ops instead of cloning them |
| 279 | // which would eliminate need to explicitly erase |
| 280 | for (Operation *dataOp : dataExitOps) |
| 281 | eraseOps.insert(X: dataOp); |
| 282 | |
| 283 | // Redirect host-side uses while preserving uses outside this construct. |
| 284 | // Shared operations are erased only after their users are gone. |
| 285 | Region &elseRegion = ifOp.getElseRegion(); |
| 286 | auto replaceHostUsesAndScheduleErase = [&](auto &ops) { |
| 287 | for (Operation *op : ops) { |
| 288 | getAccVar(accDataClauseOp: op).replaceUsesWithIf(newValue: getVar(accDataClauseOp: op), shouldReplace: [&](OpOperand &use) { |
| 289 | return elseRegion.isAncestor(other: use.getOwner()->getParentRegion()); |
| 290 | }); |
| 291 | condEraseOps.insert(X: op); |
| 292 | } |
| 293 | }; |
| 294 | replaceHostUsesAndScheduleErase(dataEntryOps); |
| 295 | replaceHostUsesAndScheduleErase(firstprivateOps); |
| 296 | replaceHostUsesAndScheduleErase(privateOps); |
| 297 | replaceHostUsesAndScheduleErase(reductionOps); |
| 298 | } |
| 299 | |
| 300 | void ACCIfClauseLowering::runOnOperation() { |
| 301 | func::FuncOp funcOp = getOperation(); |
| 302 | accSupport = &getAnalysis<OpenACCSupport>(); |
| 303 | |
| 304 | llvm::SetVector<Operation *> eraseOps; |
| 305 | llvm::SetVector<Operation *> condEraseOps; |
| 306 | funcOp.walk(callback: [&](Operation *op) { |
| 307 | if (auto parallelOp = dyn_cast<acc::ParallelOp>(Val: op)) |
| 308 | lowerIfClauseForComputeConstruct(computeConstructOp: parallelOp, eraseOps, condEraseOps); |
| 309 | else if (auto kernelsOp = dyn_cast<acc::KernelsOp>(Val: op)) |
| 310 | lowerIfClauseForComputeConstruct(computeConstructOp: kernelsOp, eraseOps, condEraseOps); |
| 311 | else if (auto serialOp = dyn_cast<acc::SerialOp>(Val: op)) |
| 312 | lowerIfClauseForComputeConstruct(computeConstructOp: serialOp, eraseOps, condEraseOps); |
| 313 | }); |
| 314 | |
| 315 | for (Operation *op : llvm::reverse(C&: eraseOps)) |
| 316 | op->erase(); |
| 317 | // Shared entry/private/reduction ops can become dead in stages. |
| 318 | // Revisit deferred producers after their consumers are erased. |
| 319 | SmallVector<Operation *> pendingEraseOps(condEraseOps.begin(), |
| 320 | condEraseOps.end()); |
| 321 | bool erased; |
| 322 | do { |
| 323 | erased = false; |
| 324 | for (size_t i = pendingEraseOps.size(); i > 0; --i) { |
| 325 | Operation *op = pendingEraseOps[i - 1]; |
| 326 | if (!op->use_empty()) |
| 327 | continue; |
| 328 | pendingEraseOps.erase(CI: pendingEraseOps.begin() + i - 1); |
| 329 | op->erase(); |
| 330 | erased = true; |
| 331 | } |
| 332 | } while (erased); |
| 333 | } |
| 334 | } // namespace |
| 335 | |