1//===- TestSCFUtils.cpp --- Pass to test independent SCF dialect utils ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass to test SCF dialect utils.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Func/IR/FuncOps.h"
15#include "mlir/Dialect/MemRef/IR/MemRef.h"
16#include "mlir/Dialect/SCF/IR/SCF.h"
17#include "mlir/Dialect/SCF/Transforms/Patterns.h"
18#include "mlir/Dialect/SCF/Utils/Utils.h"
19#include "mlir/IR/Builders.h"
20#include "mlir/IR/PatternMatch.h"
21#include "mlir/Pass/Pass.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24#include <limits>
25
26using namespace mlir;
27
28namespace {
29struct TestSCFForUtilsPass
30 : public PassWrapper<TestSCFForUtilsPass, OperationPass<func::FuncOp>> {
31 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSCFForUtilsPass)
32
33 StringRef getArgument() const final { return "test-scf-for-utils"; }
34 StringRef getDescription() const final { return "test scf.for utils"; }
35 explicit TestSCFForUtilsPass() = default;
36 TestSCFForUtilsPass(const TestSCFForUtilsPass &pass) : PassWrapper(pass) {}
37
38 Option<bool> testReplaceWithNewYields{
39 *this, "test-replace-with-new-yields",
40 llvm::cl::desc("Test replacing a loop with a new loop that returns new "
41 "additional yield values"),
42 llvm::cl::init(Val: false)};
43
44 void runOnOperation() override {
45 func::FuncOp func = getOperation();
46
47 // Annotate every loop-like operation with the static trip count.
48 func.walk(callback: [&](LoopLikeOpInterface loopOp) {
49 std::optional<APInt> tripCount = loopOp.getStaticTripCount();
50 if (tripCount.has_value())
51 loopOp->setDiscardableAttr(
52 name: "test.trip-count",
53 value: IntegerAttr::get(type: IntegerType::get(context: &getContext(),
54 width: tripCount.value().getBitWidth()),
55 value: tripCount.value().getZExtValue()));
56 else
57 loopOp->setDiscardableAttr(name: "test.trip-count",
58 value: StringAttr::get(context: &getContext(), bytes: "none"));
59 });
60
61 if (testReplaceWithNewYields) {
62 func.walk(callback: [&](scf::ForOp forOp) {
63 if (forOp.getNumResults() == 0)
64 return;
65 auto newInitValues = forOp.getInitArgs();
66 if (newInitValues.empty())
67 return;
68 SmallVector<Value> oldYieldValues =
69 llvm::to_vector(Range: forOp.getYieldedValues());
70 NewYieldValuesFn fn = [&](OpBuilder &b, Location loc,
71 ArrayRef<BlockArgument> newBBArgs) {
72 SmallVector<Value> newYieldValues;
73 for (auto yieldVal : oldYieldValues) {
74 newYieldValues.push_back(
75 Elt: arith::AddFOp::create(builder&: b, location: loc, lhs: yieldVal, rhs: yieldVal));
76 }
77 return newYieldValues;
78 };
79 IRRewriter rewriter(forOp.getContext());
80 if (failed(Result: forOp.replaceWithAdditionalYields(
81 rewriter, newInitOperands: newInitValues, /*replaceInitOperandUsesInLoop=*/true,
82 newYieldValuesFn: fn)))
83 signalPassFailure();
84 });
85 }
86 }
87};
88
89struct TestSCFIfUtilsPass
90 : public PassWrapper<TestSCFIfUtilsPass, OperationPass<ModuleOp>> {
91 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSCFIfUtilsPass)
92
93 StringRef getArgument() const final { return "test-scf-if-utils"; }
94 StringRef getDescription() const final { return "test scf.if utils"; }
95 explicit TestSCFIfUtilsPass() = default;
96
97 void getDependentDialects(DialectRegistry &registry) const override {
98 registry.insert<func::FuncDialect>();
99 }
100
101 void runOnOperation() override {
102 int count = 0;
103 getOperation().walk(callback: [&](scf::IfOp ifOp) {
104 auto strCount = std::to_string(val: count++);
105 func::FuncOp thenFn, elseFn;
106 OpBuilder b(ifOp);
107 IRRewriter rewriter(b);
108 if (failed(Result: outlineIfOp(b&: rewriter, ifOp, thenFn: &thenFn,
109 thenFnName: std::string("outlined_then") + strCount, elseFn: &elseFn,
110 elseFnName: std::string("outlined_else") + strCount))) {
111 this->signalPassFailure();
112 return WalkResult::interrupt();
113 }
114 return WalkResult::advance();
115 });
116 }
117};
118
119static const StringLiteral kTestPipeliningLoopMarker =
120 "__test_pipelining_loop__";
121static const StringLiteral kTestPipeliningStageMarker =
122 "__test_pipelining_stage__";
123/// Marker to express the order in which operations should be after
124/// pipelining.
125static const StringLiteral kTestPipeliningOpOrderMarker =
126 "__test_pipelining_op_order__";
127
128static const StringLiteral kTestPipeliningAnnotationPart =
129 "__test_pipelining_part";
130static const StringLiteral kTestPipeliningAnnotationIteration =
131 "__test_pipelining_iteration";
132
133struct TestSCFPipeliningPass
134 : public PassWrapper<TestSCFPipeliningPass, OperationPass<func::FuncOp>> {
135 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSCFPipeliningPass)
136
137 TestSCFPipeliningPass() = default;
138 TestSCFPipeliningPass(const TestSCFPipeliningPass &) {}
139 StringRef getArgument() const final { return "test-scf-pipelining"; }
140 StringRef getDescription() const final { return "test scf.forOp pipelining"; }
141
142 Option<bool> annotatePipeline{
143 *this, "annotate",
144 llvm::cl::desc("Annote operations during loop pipelining transformation"),
145 llvm::cl::init(Val: false)};
146
147 Option<bool> noEpiloguePeeling{
148 *this, "no-epilogue-peeling",
149 llvm::cl::desc("Use predicates instead of peeling the epilogue."),
150 llvm::cl::init(Val: false)};
151
152 static void
153 getSchedule(scf::ForOp forOp,
154 std::vector<std::pair<Operation *, unsigned>> &schedule) {
155 if (!forOp->hasDiscardableAttr(name: kTestPipeliningLoopMarker))
156 return;
157
158 schedule.resize(new_size: forOp.getBody()->getOperations().size() - 1);
159 WalkResult result = forOp.walk(callback: [&schedule](Operation *op) {
160 auto attrStage =
161 op->getDiscardableAttrOfType<IntegerAttr>(name: kTestPipeliningStageMarker);
162 auto attrCycle = op->getDiscardableAttrOfType<IntegerAttr>(
163 name: kTestPipeliningOpOrderMarker);
164 if (attrCycle && attrStage) {
165 const APInt &stage = attrStage.getValue();
166 if (stage.isNegative() ||
167 stage.getActiveBits() > std::numeric_limits<unsigned>::digits) {
168 op->emitOpError(message: "invalid pipeline stage");
169 return WalkResult::interrupt();
170 }
171 const APInt &order = attrCycle.getValue();
172 if (order.isNegative() || order.getActiveBits() > 64 ||
173 order.getZExtValue() >= schedule.size()) {
174 op->emitOpError(message: "invalid pipeline op order");
175 return WalkResult::interrupt();
176 }
177 size_t orderIndex = static_cast<size_t>(order.getZExtValue());
178 if (schedule[orderIndex].first) {
179 op->emitOpError(message: "duplicate pipeline op order");
180 return WalkResult::interrupt();
181 }
182 schedule[orderIndex] =
183 std::make_pair(x&: op, y: static_cast<unsigned>(stage.getZExtValue()));
184 }
185 return WalkResult::advance();
186 });
187 if (result.wasInterrupted())
188 schedule.clear();
189 }
190
191 /// Helper to generate "predicated" version of `op`. For simplicity we just
192 /// wrap the operation in a scf.ifOp operation.
193 static Operation *predicateOp(RewriterBase &rewriter, Operation *op,
194 Value pred) {
195 Location loc = op->getLoc();
196 auto ifOp =
197 scf::IfOp::create(builder&: rewriter, location: loc, resultTypes: op->getResultTypes(), cond: pred, withElseRegion: true);
198 // True branch.
199 rewriter.moveOpBefore(op, block: &ifOp.getThenRegion().front(),
200 iterator: ifOp.getThenRegion().front().begin());
201 rewriter.setInsertionPointAfter(op);
202 if (op->getNumResults() > 0)
203 scf::YieldOp::create(builder&: rewriter, location: loc, results: op->getResults());
204 // False branch.
205 rewriter.setInsertionPointToStart(&ifOp.getElseRegion().front());
206 SmallVector<Value> elseYieldOperands;
207 elseYieldOperands.reserve(N: ifOp.getNumResults());
208 if (auto viewOp = dyn_cast<memref::SubViewOp>(Val: op)) {
209 // For sub-views, just clone the op.
210 // NOTE: This is okay in the test because we use dynamic memref sizes, so
211 // the verifier will not complain. Otherwise, we may create a logically
212 // out-of-bounds view and a different technique should be used.
213 Operation *opClone = rewriter.clone(op&: *op);
214 elseYieldOperands.append(in_start: opClone->result_begin(), in_end: opClone->result_end());
215 } else {
216 // Default to assuming constant numeric values.
217 for (Type type : op->getResultTypes()) {
218 elseYieldOperands.push_back(Elt: arith::ConstantOp::create(
219 builder&: rewriter, location: loc, value: rewriter.getZeroAttr(type)));
220 }
221 }
222 if (op->getNumResults() > 0)
223 scf::YieldOp::create(builder&: rewriter, location: loc, results: elseYieldOperands);
224 return ifOp.getOperation();
225 }
226
227 static void annotate(Operation *op,
228 mlir::scf::PipeliningOption::PipelinerPart part,
229 unsigned iteration) {
230 OpBuilder b(op);
231 switch (part) {
232 case mlir::scf::PipeliningOption::PipelinerPart::Prologue:
233 op->setDiscardableAttr(name: kTestPipeliningAnnotationPart,
234 value: b.getStringAttr(bytes: "prologue"));
235 break;
236 case mlir::scf::PipeliningOption::PipelinerPart::Kernel:
237 op->setDiscardableAttr(name: kTestPipeliningAnnotationPart,
238 value: b.getStringAttr(bytes: "kernel"));
239 break;
240 case mlir::scf::PipeliningOption::PipelinerPart::Epilogue:
241 op->setDiscardableAttr(name: kTestPipeliningAnnotationPart,
242 value: b.getStringAttr(bytes: "epilogue"));
243 break;
244 }
245 op->setDiscardableAttr(name: kTestPipeliningAnnotationIteration,
246 value: b.getI32IntegerAttr(value: iteration));
247 }
248
249 void getDependentDialects(DialectRegistry &registry) const override {
250 registry.insert<arith::ArithDialect, memref::MemRefDialect>();
251 }
252
253 void runOnOperation() override {
254 RewritePatternSet patterns(&getContext());
255 mlir::scf::PipeliningOption options;
256 options.getScheduleFn = getSchedule;
257 options.supportDynamicLoops = true;
258 options.predicateFn = predicateOp;
259 if (annotatePipeline)
260 options.annotateFn = annotate;
261 if (noEpiloguePeeling) {
262 options.peelEpilogue = false;
263 }
264 scf::populateSCFLoopPipeliningPatterns(patterns, options);
265 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
266 getOperation().walk(callback: [](Operation *op) {
267 // Clean up the markers.
268 op->removeDiscardableAttr(name: kTestPipeliningStageMarker);
269 op->removeDiscardableAttr(name: kTestPipeliningOpOrderMarker);
270 });
271 }
272};
273
274static constexpr StringLiteral kSplitAtAttr = "test.split_at";
275static constexpr StringLiteral kSplitArgAttr = "test.split_arg";
276
277struct TestSplitForOpAtPointPass
278 : public PassWrapper<TestSplitForOpAtPointPass,
279 OperationPass<func::FuncOp>> {
280 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestSplitForOpAtPointPass)
281
282 StringRef getArgument() const final { return "test-split-for-op-at-point"; }
283
284 StringRef getDescription() const final { return "test splitForOpAtPoint"; }
285
286 TestSplitForOpAtPointPass() = default;
287 TestSplitForOpAtPointPass(const TestSplitForOpAtPointPass &) {}
288
289 void getDependentDialects(DialectRegistry &registry) const override {
290 registry.insert<arith::ArithDialect>();
291 }
292
293 void runOnOperation() override {
294 func::FuncOp func = getOperation();
295 SmallVector<scf::ForOp> loopsToSplit;
296 func.walk(callback: [&](scf::ForOp forOp) {
297 if (forOp->hasDiscardableAttr(name: kSplitAtAttr) ||
298 forOp->hasDiscardableAttr(name: kSplitArgAttr))
299 loopsToSplit.push_back(Elt: forOp);
300 });
301
302 IRRewriter rewriter(func.getContext());
303 for (scf::ForOp forOp : loopsToSplit) {
304 Value splitPoint;
305 if (auto splitAttr =
306 forOp->getDiscardableAttrOfType<IntegerAttr>(name: kSplitAtAttr)) {
307 rewriter.setInsertionPoint(forOp);
308 splitPoint =
309 arith::ConstantOp::create(builder&: rewriter, location: forOp.getLoc(), value: splitAttr);
310 rewriter.modifyOpInPlace(
311 root: forOp, callable: [&] { forOp->removeDiscardableAttr(name: kSplitAtAttr); });
312 } else if (auto argAttr = forOp->getDiscardableAttrOfType<IntegerAttr>(
313 name: kSplitArgAttr)) {
314 int64_t argNo = argAttr.getInt();
315 if (argNo < 0 ||
316 static_cast<unsigned>(argNo) >= func.getNumArguments()) {
317 emitError(loc: forOp.getLoc(), message: "test.split_arg is out of range");
318 return signalPassFailure();
319 }
320 splitPoint = func.getArgument(idx: argNo);
321 rewriter.modifyOpInPlace(
322 root: forOp, callable: [&] { forOp->removeDiscardableAttr(name: kSplitArgAttr); });
323 } else {
324 continue;
325 }
326 if (failed(Result: splitForOpAtPoint(rewriter, forOp, splitPoint))) {
327 emitError(loc: forOp.getLoc(), message: "failed to split scf.for");
328 return signalPassFailure();
329 }
330 }
331 }
332};
333} // namespace
334
335namespace mlir {
336namespace test {
337void registerTestSCFUtilsPass() {
338 PassRegistration<TestSCFForUtilsPass>();
339 PassRegistration<TestSCFIfUtilsPass>();
340 PassRegistration<TestSCFPipeliningPass>();
341 PassRegistration<TestSplitForOpAtPointPass>();
342}
343} // namespace test
344} // namespace mlir
345