1//===- ACCIfClauseLowering.cpp - Lower ACC compute construct if clauses --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers OpenACC compute constructs (parallel, kernels, serial) with
10// `if` clauses using region specialization. It creates two execution paths:
11// device execution when the condition is true, host execution when false.
12//
13// Overview:
14// ---------
15// When an ACC compute construct has an `if` clause, the construct should only
16// execute on the device when the condition is true. If the condition is false,
17// the code should execute on the host instead. This pass transforms:
18//
19// acc.parallel if(%cond) { ... }
20//
21// Into:
22//
23// scf.if %cond {
24// // Device path: clone data ops, compute construct without if, exit ops
25// acc.parallel { ... }
26// } else {
27// // Host path: original region body with ACC ops converted to host
28// }
29//
30// Transformations:
31// ----------------
32// For each compute construct with an `if` clause:
33//
34// 1. Device Path (true branch):
35// - Clone data entry operations (acc.copyin, acc.create, etc.)
36// - Clone the compute construct without the `if` clause
37// - Clone data exit operations (acc.copyout, acc.delete, etc.)
38//
39// 2. Host Path (false branch):
40// - Move the original region body to the else branch
41// - Apply host fallback patterns to convert ACC ops to host equivalents
42//
43// 3. Cleanup:
44// - Erase the original compute construct and data operations
45// - Replace uses of ACC variables with host variables in the else branch
46//
47// Requirements:
48// -------------
49// To use this pass in a pipeline, the following requirements exist:
50//
51// 1. Analysis Registration (Optional): If custom behavior is needed for
52// emitting not-yet-implemented messages for unsupported cases, the pipeline
53// should pre-register the `acc::OpenACCSupport` analysis.
54//
55//===----------------------------------------------------------------------===//
56
57#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
58#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
59
60#include "mlir/Dialect/Func/IR/FuncOps.h"
61#include "mlir/Dialect/OpenACC/Analysis/OpenACCSupport.h"
62#include "mlir/Dialect/OpenACC/OpenACC.h"
63#include "mlir/Dialect/OpenACC/OpenACCUtilsLoop.h"
64#include "mlir/Dialect/SCF/IR/SCF.h"
65#include "mlir/IR/Builders.h"
66#include "mlir/IR/IRMapping.h"
67#include "mlir/IR/PatternMatch.h"
68#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
69#include "llvm/ADT/STLExtras.h"
70#include "llvm/ADT/SetVector.h"
71#include "llvm/Support/Debug.h"
72
73namespace mlir {
74namespace acc {
75#define GEN_PASS_DEF_ACCIFCLAUSELOWERING
76#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
77} // namespace acc
78} // namespace mlir
79
80#define DEBUG_TYPE "acc-if-clause-lowering"
81
82using namespace mlir;
83using namespace mlir::acc;
84
85namespace {
86class ACCIfClauseLowering
87 : public acc::impl::ACCIfClauseLoweringBase<ACCIfClauseLowering> {
88 using ACCIfClauseLoweringBase<ACCIfClauseLowering>::ACCIfClauseLoweringBase;
89
90 void getDependentDialects(DialectRegistry &registry) const override {
91 ACCIfClauseLoweringBase<ACCIfClauseLowering>::getDependentDialects(
92 registry);
93 acc::registerAllExtensions(registry);
94 }
95
96private:
97 OpenACCSupport *accSupport = nullptr;
98
99 void convertHostRegion(Operation *computeOp, Region &region);
100
101 template <typename OpTy>
102 void
103 lowerIfClauseForComputeConstruct(OpTy computeConstructOp,
104 llvm::SetVector<Operation *> &eraseOps,
105 llvm::SetVector<Operation *> &condEraseOps);
106
107public:
108 void runOnOperation() override;
109};
110
111void ACCIfClauseLowering::convertHostRegion(Operation *computeOp,
112 Region &region) {
113 // Only collect ACC dialect operations - other ops don't need conversion
114 SmallVector<Operation *> hostOps;
115 region.walk<WalkOrder::PreOrder>(callback: [&](Operation *op) {
116 if (isa<acc::OpenACCDialect>(Val: op->getDialect()))
117 hostOps.push_back(Elt: op);
118 });
119
120 RewritePatternSet patterns(computeOp->getContext());
121 populateACCHostFallbackPatterns(patterns, accSupport&: *accSupport);
122
123 GreedyRewriteConfig config;
124 config.setUseTopDownTraversal(true);
125 config.setStrictness(GreedyRewriteStrictness::ExistingOps);
126 if (failed(Result: applyOpPatternsGreedily(ops: hostOps, patterns: std::move(patterns), config)))
127 accSupport->emitNYI(loc: computeOp->getLoc(), message: "failed to convert host region");
128}
129
130// Template function to handle if condition conversion for ACC compute
131// constructs
132template <typename OpTy>
133void ACCIfClauseLowering::lowerIfClauseForComputeConstruct(
134 OpTy computeConstructOp, llvm::SetVector<Operation *> &eraseOps,
135 llvm::SetVector<Operation *> &condEraseOps) {
136 Value ifCond = computeConstructOp.getIfCond();
137 if (!ifCond)
138 return;
139
140 IRRewriter rewriter(computeConstructOp);
141
142 LLVM_DEBUG(llvm::dbgs() << "Converting " << computeConstructOp->getName()
143 << " with if condition: " << computeConstructOp
144 << "\n");
145
146 // Collect data clause operations that need to be recreated in the if
147 // condition
148 llvm::SetVector<Operation *> dataEntryOps;
149 llvm::SetVector<Operation *> dataExitOps;
150 SmallVector<Operation *> firstprivateOps;
151 SmallVector<Operation *> privateOps;
152 SmallVector<Operation *> reductionOps;
153
154 // Entries used outside this construct belong to the surrounding scope.
155 // Their entry and exit operations must remain outside the conditional.
156 auto isExternallyOwned = [&](Operation *dataOp) {
157 for (Operation *user : dataOp->getUsers())
158 if (!isa<ACC_DATA_EXIT_OPS>(Val: user) && user != computeConstructOp &&
159 !computeConstructOp->isAncestor(user))
160 return true;
161 return false;
162 };
163
164 // Collect data entry operations
165 for (Value operand : computeConstructOp.getDataClauseOperands())
166 if (Operation *defOp = operand.getDefiningOp())
167 if (isa<ACC_DATA_ENTRY_OPS>(Val: defOp))
168 dataEntryOps.insert(X: defOp);
169
170 // Find corresponding exit operations for each local entry operation. Exit ops
171 // of externally owned entry ops belong to the enclosing construct.
172 // Iterate backwards through entry ops since exit ops appear in reverse order.
173 for (Operation *dataEntryOp : llvm::reverse(C&: dataEntryOps))
174 if (!isExternallyOwned(dataEntryOp))
175 for (Operation *user : dataEntryOp->getUsers())
176 if (isa<ACC_DATA_EXIT_OPS>(Val: user))
177 dataExitOps.insert(X: user);
178
179 // Collect firstprivate, private, and reduction operations
180 auto collectOps = [&](SmallVector<Operation *> &ops, OperandRange operands) {
181 for (Value operand : operands)
182 if (Operation *defOp = operand.getDefiningOp())
183 ops.push_back(Elt: defOp);
184 };
185 collectOps(firstprivateOps, computeConstructOp.getFirstprivateOperands());
186 collectOps(privateOps, computeConstructOp.getPrivateOperands());
187 collectOps(reductionOps, computeConstructOp.getReductionOperands());
188
189 // Create scf.if with device and host execution paths
190 auto ifOp = scf::IfOp::create(rewriter, computeConstructOp.getLoc(),
191 TypeRange{}, ifCond, /*withElseRegion=*/true);
192
193 LLVM_DEBUG(llvm::dbgs() << "Cloning " << dataEntryOps.size()
194 << " data entry operations for device path\n");
195
196 // Device execution path (true branch)
197 Block &thenBlock = ifOp.getThenRegion().front();
198 rewriter.setInsertionPointToStart(&thenBlock);
199
200 // Clone data entry operations
201 SmallVector<Value> deviceDataOperands;
202 SmallVector<Value> firstprivateOperands;
203 SmallVector<Value> privateOperands;
204 SmallVector<Value> reductionOperands;
205
206 // Map the data entry and firstprivate ops for the cloned region
207 IRMapping deviceMapping;
208 auto cloneAndMapOps = [&](SmallVector<Operation *> &ops,
209 SmallVector<Value> &operands) {
210 for (Operation *op : ops) {
211 Operation *clonedOp = rewriter.clone(op&: *op, mapper&: deviceMapping);
212 operands.push_back(Elt: clonedOp->getResult(idx: 0));
213 deviceMapping.map(from: op->getResult(idx: 0), to: clonedOp->getResult(idx: 0));
214 }
215 };
216 // Clone each local data entry op once. Externally owned ops (mapped by a
217 // surrounding scope) are referenced directly.
218 for (Operation *op : dataEntryOps) {
219 if (isExternallyOwned(op))
220 continue;
221 Operation *clonedOp = rewriter.clone(op&: *op, mapper&: deviceMapping);
222 deviceMapping.map(from: op->getResult(idx: 0), to: clonedOp->getResult(idx: 0));
223 }
224 // Preserve the original operand order and multiplicity while using the
225 // cloned value for local entries and the original value for external ones.
226 for (Value operand : computeConstructOp.getDataClauseOperands())
227 deviceDataOperands.push_back(Elt: deviceMapping.lookupOrDefault(from: operand));
228 cloneAndMapOps(firstprivateOps, firstprivateOperands);
229 cloneAndMapOps(privateOps, privateOperands);
230 cloneAndMapOps(reductionOps, reductionOperands);
231
232 // Create new compute op without if condition for device execution by
233 // cloning
234 OpTy newComputeOp = cast<OpTy>(
235 rewriter.clone(*computeConstructOp.getOperation(), deviceMapping));
236 newComputeOp.getIfCondMutable().clear();
237 newComputeOp.getDataClauseOperandsMutable().assign(deviceDataOperands);
238 newComputeOp.getFirstprivateOperandsMutable().assign(firstprivateOperands);
239 newComputeOp.getPrivateOperandsMutable().assign(privateOperands);
240 newComputeOp.getReductionOperandsMutable().assign(reductionOperands);
241
242 // Clone data exit operations
243 rewriter.setInsertionPointAfter(newComputeOp);
244 for (Operation *dataOp : dataExitOps)
245 rewriter.clone(op&: *dataOp, mapper&: deviceMapping);
246
247 rewriter.setInsertionPointToEnd(&thenBlock);
248 if (!thenBlock.getTerminator())
249 scf::YieldOp::create(rewriter, computeConstructOp.getLoc());
250
251 // Host execution path (false branch)
252 Region &hostRegion = computeConstructOp.getRegion();
253 if (hostRegion.hasOneBlock()) {
254 // Don't need to clone original ops, just take them and legalize for host.
255 ifOp.getElseRegion().takeBody(hostRegion);
256
257 // Swap acc yield for scf yield.
258 Block &elseBlock = ifOp.getElseRegion().front();
259 elseBlock.getTerminator()->erase();
260 rewriter.setInsertionPointToEnd(&elseBlock);
261 scf::YieldOp::create(rewriter, computeConstructOp.getLoc());
262
263 convertHostRegion(computeOp: computeConstructOp, region&: ifOp.getElseRegion());
264 } else {
265 // scf.if regions must stay single-block. Wrap the original multi-block ACC
266 // body in scf.execute_region so it can be hosted in the else branch.
267 Block &elseBlock = ifOp.getElseRegion().front();
268 rewriter.setInsertionPoint(elseBlock.getTerminator());
269 IRMapping hostMapping;
270 auto hostExecuteRegion = wrapMultiBlockRegionWithSCFExecuteRegion(
271 hostRegion, hostMapping, computeConstructOp.getLoc(), rewriter);
272 convertHostRegion(computeOp: computeConstructOp, region&: hostExecuteRegion.getRegion());
273 }
274
275 // The original op is now empty and can be erased
276 eraseOps.insert(computeConstructOp);
277
278 // TODO: Can probably 'move' the data ops instead of cloning them
279 // which would eliminate need to explicitly erase
280 for (Operation *dataOp : dataExitOps)
281 eraseOps.insert(X: dataOp);
282
283 // Redirect host-side uses while preserving uses outside this construct.
284 // Shared operations are erased only after their users are gone.
285 Region &elseRegion = ifOp.getElseRegion();
286 auto replaceHostUsesAndScheduleErase = [&](auto &ops) {
287 for (Operation *op : ops) {
288 getAccVar(accDataClauseOp: op).replaceUsesWithIf(newValue: getVar(accDataClauseOp: op), shouldReplace: [&](OpOperand &use) {
289 return elseRegion.isAncestor(other: use.getOwner()->getParentRegion());
290 });
291 condEraseOps.insert(X: op);
292 }
293 };
294 replaceHostUsesAndScheduleErase(dataEntryOps);
295 replaceHostUsesAndScheduleErase(firstprivateOps);
296 replaceHostUsesAndScheduleErase(privateOps);
297 replaceHostUsesAndScheduleErase(reductionOps);
298}
299
300void ACCIfClauseLowering::runOnOperation() {
301 func::FuncOp funcOp = getOperation();
302 accSupport = &getAnalysis<OpenACCSupport>();
303
304 llvm::SetVector<Operation *> eraseOps;
305 llvm::SetVector<Operation *> condEraseOps;
306 funcOp.walk(callback: [&](Operation *op) {
307 if (auto parallelOp = dyn_cast<acc::ParallelOp>(Val: op))
308 lowerIfClauseForComputeConstruct(computeConstructOp: parallelOp, eraseOps, condEraseOps);
309 else if (auto kernelsOp = dyn_cast<acc::KernelsOp>(Val: op))
310 lowerIfClauseForComputeConstruct(computeConstructOp: kernelsOp, eraseOps, condEraseOps);
311 else if (auto serialOp = dyn_cast<acc::SerialOp>(Val: op))
312 lowerIfClauseForComputeConstruct(computeConstructOp: serialOp, eraseOps, condEraseOps);
313 });
314
315 for (Operation *op : llvm::reverse(C&: eraseOps))
316 op->erase();
317 // Shared entry/private/reduction ops can become dead in stages.
318 // Revisit deferred producers after their consumers are erased.
319 SmallVector<Operation *> pendingEraseOps(condEraseOps.begin(),
320 condEraseOps.end());
321 bool erased;
322 do {
323 erased = false;
324 for (size_t i = pendingEraseOps.size(); i > 0; --i) {
325 Operation *op = pendingEraseOps[i - 1];
326 if (!op->use_empty())
327 continue;
328 pendingEraseOps.erase(CI: pendingEraseOps.begin() + i - 1);
329 op->erase();
330 erased = true;
331 }
332 } while (erased);
333}
334} // namespace
335