1//===- LegalizeDataValues.cpp - -------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/OpenACC/Extensions/AllExtensions.h"
10#include "mlir/Dialect/OpenACC/Transforms/Passes.h"
11
12#include "mlir/Dialect/Func/IR/FuncOps.h"
13#include "mlir/Dialect/OpenACC/OpenACC.h"
14#include "mlir/IR/Dominance.h"
15#include "mlir/Pass/Pass.h"
16#include "mlir/Transforms/RegionUtils.h"
17#include "llvm/Support/ErrorHandling.h"
18
19namespace mlir {
20namespace acc {
21#define GEN_PASS_DEF_LEGALIZEDATAVALUESINREGION
22#include "mlir/Dialect/OpenACC/Transforms/Passes.h.inc"
23} // namespace acc
24} // namespace mlir
25
26using namespace mlir;
27
28namespace {
29static bool insideAccComputeRegion(mlir::Operation *op) {
30 return op->getParentOfType<ACC_COMPUTE_CONSTRUCT_OPS>();
31}
32
33static void collectVars(mlir::ValueRange operands,
34 llvm::SmallVector<std::pair<Value, Value>> &values,
35 bool hostToDevice) {
36 for (auto operand : operands) {
37 Value var = acc::getVar(accDataClauseOp: operand.getDefiningOp());
38 Value accVar = acc::getAccVar(accDataClauseOp: operand.getDefiningOp());
39 if (var && accVar) {
40 if (hostToDevice)
41 values.push_back(Elt: {var, accVar});
42 else
43 values.push_back(Elt: {accVar, var});
44 }
45 }
46}
47
48template <typename Op>
49static void replaceAllUsesInAccComputeRegionsWith(Value orig, Value replacement,
50 Region &outerRegion) {
51 for (auto &use : llvm::make_early_inc_range(Range: orig.getUses())) {
52 if (outerRegion.isAncestor(other: use.getOwner()->getParentRegion())) {
53 if constexpr (std::is_same_v<Op, acc::DataOp> ||
54 std::is_same_v<Op, acc::DeclareOp>) {
55 // For data construct regions, only replace uses in contained compute
56 // regions.
57 if (insideAccComputeRegion(op: use.getOwner())) {
58 use.set(replacement);
59 }
60 } else {
61 use.set(replacement);
62 }
63 }
64 }
65}
66
67template <typename Op>
68static void replaceAllUsesInUnstructuredComputeRegionWith(
69 Op &op, llvm::SmallVector<std::pair<Value, Value>> &values,
70 DominanceInfo &domInfo, PostDominanceInfo &postDomInfo) {
71
72 SmallVector<Operation *> exitOps;
73 if constexpr (std::is_same_v<Op, acc::DeclareEnterOp>) {
74 // For declare enter/exit pairs, collect all exit ops
75 for (auto *user : op.getToken().getUsers()) {
76 if (auto declareExit = dyn_cast<acc::DeclareExitOp>(user))
77 exitOps.push_back(Elt: declareExit);
78 }
79 if (exitOps.empty())
80 return;
81 }
82
83 for (auto p : values) {
84 Value hostVal = std::get<0>(in&: p);
85 Value deviceVal = std::get<1>(in&: p);
86 for (auto &use : llvm::make_early_inc_range(Range: hostVal.getUses())) {
87 Operation *owner = use.getOwner();
88
89 // Check It's the case that the acc entry operation dominates the use.
90 if (!domInfo.dominates(op.getOperation(), owner))
91 continue;
92
93 // Check It's the case that at least one of the acc exit operations
94 // post-dominates the use
95 bool hasPostDominatingExit = false;
96 for (auto *exit : exitOps) {
97 if (postDomInfo.postDominates(a: exit, b: owner)) {
98 hasPostDominatingExit = true;
99 break;
100 }
101 }
102
103 if (!hasPostDominatingExit)
104 continue;
105
106 if (insideAccComputeRegion(op: owner))
107 use.set(deviceVal);
108 }
109 }
110}
111
112template <typename Op>
113static void
114collectAndReplaceInRegion(Op &op, bool hostToDevice,
115 DominanceInfo *domInfo = nullptr,
116 PostDominanceInfo *postDomInfo = nullptr) {
117 llvm::SmallVector<std::pair<Value, Value>> values;
118
119 if constexpr (std::is_same_v<Op, acc::LoopOp>) {
120 collectVars(op.getReductionOperands(), values, hostToDevice);
121 collectVars(op.getPrivateOperands(), values, hostToDevice);
122 } else {
123 collectVars(op.getDataClauseOperands(), values, hostToDevice);
124 if constexpr (!std::is_same_v<Op, acc::KernelsOp> &&
125 !std::is_same_v<Op, acc::DataOp> &&
126 !std::is_same_v<Op, acc::DeclareOp> &&
127 !std::is_same_v<Op, acc::HostDataOp> &&
128 !std::is_same_v<Op, acc::DeclareEnterOp>) {
129 collectVars(op.getReductionOperands(), values, hostToDevice);
130 collectVars(op.getPrivateOperands(), values, hostToDevice);
131 collectVars(op.getFirstprivateOperands(), values, hostToDevice);
132 }
133 }
134
135 if constexpr (std::is_same_v<Op, acc::DeclareEnterOp>) {
136 assert(domInfo && postDomInfo &&
137 "Dominance info required for DeclareEnterOp");
138 replaceAllUsesInUnstructuredComputeRegionWith<Op>(op, values, *domInfo,
139 *postDomInfo);
140 } else {
141 for (auto p : values) {
142 replaceAllUsesInAccComputeRegionsWith<Op>(std::get<0>(in&: p), std::get<1>(in&: p),
143 op.getRegion());
144 }
145 }
146}
147
148class LegalizeDataValuesInRegion
149 : public acc::impl::LegalizeDataValuesInRegionBase<
150 LegalizeDataValuesInRegion> {
151public:
152 using LegalizeDataValuesInRegionBase<
153 LegalizeDataValuesInRegion>::LegalizeDataValuesInRegionBase;
154
155 void getDependentDialects(DialectRegistry &registry) const override {
156 LegalizeDataValuesInRegionBase<
157 LegalizeDataValuesInRegion>::getDependentDialects(registry);
158 acc::registerAllExtensions(registry);
159 }
160
161 void runOnOperation() override {
162 func::FuncOp funcOp = getOperation();
163 bool replaceHostVsDevice = this->hostToDevice.getValue();
164
165 // Initialize dominance info
166 DominanceInfo domInfo;
167 PostDominanceInfo postDomInfo;
168 bool computedDomInfo = false;
169
170 funcOp.walk(callback: [&](Operation *op) {
171 if (!isa<ACC_COMPUTE_CONSTRUCT_AND_LOOP_OPS>(Val: *op) &&
172 !(isa<ACC_DATA_CONSTRUCT_STRUCTURED_OPS>(Val: *op) &&
173 applyToAccDataConstruct) &&
174 !isa<acc::DeclareEnterOp>(Val: *op))
175 return;
176
177 if (auto parallelOp = dyn_cast<acc::ParallelOp>(Val&: *op)) {
178 collectAndReplaceInRegion(op&: parallelOp, hostToDevice: replaceHostVsDevice);
179 } else if (auto serialOp = dyn_cast<acc::SerialOp>(Val&: *op)) {
180 collectAndReplaceInRegion(op&: serialOp, hostToDevice: replaceHostVsDevice);
181 } else if (auto kernelsOp = dyn_cast<acc::KernelsOp>(Val&: *op)) {
182 collectAndReplaceInRegion(op&: kernelsOp, hostToDevice: replaceHostVsDevice);
183 } else if (auto loopOp = dyn_cast<acc::LoopOp>(Val&: *op)) {
184 collectAndReplaceInRegion(op&: loopOp, hostToDevice: replaceHostVsDevice);
185 } else if (auto dataOp = dyn_cast<acc::DataOp>(Val&: *op)) {
186 collectAndReplaceInRegion(op&: dataOp, hostToDevice: replaceHostVsDevice);
187 } else if (auto declareOp = dyn_cast<acc::DeclareOp>(Val&: *op)) {
188 collectAndReplaceInRegion(op&: declareOp, hostToDevice: replaceHostVsDevice);
189 } else if (auto hostDataOp = dyn_cast<acc::HostDataOp>(Val&: *op)) {
190 collectAndReplaceInRegion(op&: hostDataOp, hostToDevice: replaceHostVsDevice);
191 } else if (auto declareEnterOp = dyn_cast<acc::DeclareEnterOp>(Val&: *op)) {
192 if (!computedDomInfo) {
193 domInfo = DominanceInfo(funcOp);
194 postDomInfo = PostDominanceInfo(funcOp);
195 computedDomInfo = true;
196 }
197 collectAndReplaceInRegion(op&: declareEnterOp, hostToDevice: replaceHostVsDevice, domInfo: &domInfo,
198 postDomInfo: &postDomInfo);
199 } else {
200 llvm_unreachable("unsupported acc region op");
201 }
202 });
203 }
204};
205} // end anonymous namespace
206