1//===- TestLinalgElementwiseFusion.cpp - Test Linalg elementwise fusion ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass for testing fusion of elementwise operations in
10// Linalg, mainly linalg options.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Affine/IR/AffineDialect.h"
15#include "mlir/Dialect/Func/IR/FuncOps.h"
16#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
17#include "mlir/Pass/Pass.h"
18#include "mlir/Pass/PassManager.h"
19#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
20#include "llvm/ADT/TypeSwitch.h"
21
22using namespace mlir;
23
24static void addOperands(Operation *op, SetVector<Value> &operandSet) {
25 if (!op)
26 return;
27 TypeSwitch<Operation *, void>(op)
28 .Case(caseFn: [&](linalg::LinalgOp linalgOp) {
29 SmallVector<Value> inputOperands = linalgOp.getDpsInputs();
30 operandSet.insert_range(R&: inputOperands);
31 })
32 .Default(defaultFn: [&](Operation *operation) {
33 operandSet.insert(Start: operation->operand_begin(), End: operation->operand_end());
34 });
35}
36
37template <int limit = 3>
38static bool setFusedOpOperandLimit(OpOperand *fusedOperand) {
39 Operation *producer = fusedOperand->get().getDefiningOp();
40 if (!producer)
41 return false;
42
43 Operation *consumer = fusedOperand->getOwner();
44 SetVector<Value> fusedOpOperands;
45 if (producer->getNumResults() != 1)
46 return false;
47 addOperands(op: consumer, operandSet&: fusedOpOperands);
48 fusedOpOperands.remove(X: producer->getResult(idx: 0));
49 addOperands(op: producer, operandSet&: fusedOpOperands);
50 return fusedOpOperands.size() <= limit;
51}
52
53namespace {
54
55/// Pattern to test fusion of producer with consumer, even if producer has
56/// multiple uses.
57struct TestMultiUseProducerFusion : public OpRewritePattern<linalg::GenericOp> {
58 using OpRewritePattern<linalg::GenericOp>::OpRewritePattern;
59
60 LogicalResult matchAndRewrite(linalg::GenericOp genericOp,
61 PatternRewriter &rewriter) const override {
62 OpOperand *fusableOperand = nullptr;
63 for (OpOperand &operand : genericOp->getOpOperands()) {
64 if (linalg::areElementwiseOpsFusable(fusedOperand: &operand)) {
65 fusableOperand = &operand;
66 break;
67 }
68 }
69 if (!fusableOperand) {
70 return rewriter.notifyMatchFailure(arg&: genericOp, msg: "no fusable operand found");
71 }
72 std::optional<linalg::ElementwiseOpFusionResult> fusionResult =
73 linalg::fuseElementwiseOps(rewriter, fusedOperand: fusableOperand);
74 if (!fusionResult)
75 return rewriter.notifyMatchFailure(arg&: genericOp, msg: "fusion failed");
76 for (auto [origValue, replacement] : fusionResult->replacements) {
77 rewriter.replaceUsesWithIf(from: origValue, to: replacement, functor: [&](OpOperand &use) {
78 return use.getOwner() != genericOp.getOperation();
79 });
80 }
81 rewriter.eraseOp(op: genericOp);
82 return success();
83 }
84};
85
86struct TestLinalgElementwiseFusion
87 : public PassWrapper<TestLinalgElementwiseFusion,
88 OperationPass<func::FuncOp>> {
89 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestLinalgElementwiseFusion)
90
91 TestLinalgElementwiseFusion() = default;
92 TestLinalgElementwiseFusion(const TestLinalgElementwiseFusion &pass)
93 : PassWrapper(pass) {}
94 void getDependentDialects(DialectRegistry &registry) const override {
95 registry.insert<affine::AffineDialect, linalg::LinalgDialect,
96 memref::MemRefDialect, tensor::TensorDialect>();
97 }
98 StringRef getArgument() const final {
99 return "test-linalg-elementwise-fusion-patterns";
100 }
101 StringRef getDescription() const final {
102 return "Test Linalg element wise operation fusion patterns";
103 }
104
105 Option<bool> fuseGenericOps{
106 *this, "fuse-generic-ops",
107 llvm::cl::desc("Test fusion of generic operations."),
108 llvm::cl::init(Val: false)};
109
110 Option<bool> fuseGenericOpsControl{
111 *this, "fuse-generic-ops-control",
112 llvm::cl::desc(
113 "Test fusion of generic operations with a control function."),
114 llvm::cl::init(Val: false)};
115
116 Option<bool> fuseWithReshapeByExpansion{
117 *this, "fuse-with-reshape-by-expansion",
118 llvm::cl::desc(
119 "Test fusion of generic operations with reshape by expansion"),
120 llvm::cl::init(Val: false)};
121
122 Option<bool> controlFuseByExpansion{
123 *this, "control-fusion-by-expansion",
124 llvm::cl::desc(
125 "Test controlling fusion of reshape with generic op by expansion"),
126 llvm::cl::init(Val: false)};
127
128 Option<bool> fuseWithReshapeByCollapsing{
129 *this, "fuse-with-reshape-by-collapsing",
130 llvm::cl::desc("Test linalg expand_shape -> generic fusion patterns that "
131 "collapse the iteration space of the consumer"),
132 llvm::cl::init(Val: false)};
133
134 Option<bool> fuseWithReshapeByCollapsingWithControlFn{
135 *this, "fuse-with-reshape-by-collapsing-control",
136 llvm::cl::desc("Test controlling the linalg expand_shape -> generic "
137 "fusion patterns that "
138 "collapse the iteration space of the consumer"),
139 llvm::cl::init(Val: false)};
140
141 Option<bool> fuseMultiUseProducer{
142 *this, "fuse-multiuse-producer",
143 llvm::cl::desc("Test fusion of producer ops with multiple uses"),
144 llvm::cl::init(Val: false)};
145
146 Option<bool> testInvalidElementwiseKindBuilder{
147 *this, "test-invalid-elementwise-kind-builder",
148 llvm::cl::desc("Test building an elementwise op with an invalid kind"),
149 llvm::cl::init(Val: false)};
150
151 ListOption<int64_t> collapseDimensions{
152 *this, "collapse-dimensions-control",
153 llvm::cl::desc("Test controlling dimension collapse pattern")};
154
155 void runOnOperation() override {
156 MLIRContext *context = &this->getContext();
157 func::FuncOp funcOp = this->getOperation();
158
159 if (testInvalidElementwiseKindBuilder) {
160 OpBuilder builder = OpBuilder::atBlockBegin(block: &funcOp.front());
161 linalg::ElementwiseOp missingKindOp = linalg::ElementwiseOp::create(
162 builder, location: funcOp.getLoc(), inputs: ValueRange{}, outputs: ValueRange{});
163 missingKindOp.erase();
164
165 NamedAttribute kind = builder.getNamedAttr(name: "kind", val: builder.getUnitAttr());
166 linalg::ElementwiseOp wrongKindOp = linalg::ElementwiseOp::create(
167 builder, location: funcOp.getLoc(), inputs: ValueRange{}, outputs: ValueRange{}, attributes: kind);
168 wrongKindOp.erase();
169 return;
170 }
171
172 if (fuseGenericOps) {
173 RewritePatternSet fusionPatterns(context);
174 auto controlFn = [](OpOperand *operand) { return true; };
175 linalg::populateElementwiseOpsFusionPatterns(patterns&: fusionPatterns, controlElementwiseOpFusion: controlFn);
176 if (failed(Result: applyPatternsGreedily(region&: funcOp.getBody(),
177 patterns: std::move(fusionPatterns))))
178 return signalPassFailure();
179 return;
180 }
181
182 if (fuseGenericOpsControl) {
183 RewritePatternSet fusionPatterns(context);
184 linalg::populateElementwiseOpsFusionPatterns(patterns&: fusionPatterns,
185 controlElementwiseOpFusion: setFusedOpOperandLimit<4>);
186
187 if (failed(Result: applyPatternsGreedily(region&: funcOp.getBody(),
188 patterns: std::move(fusionPatterns))))
189 return signalPassFailure();
190 return;
191 }
192
193 if (fuseWithReshapeByExpansion) {
194 RewritePatternSet fusionPatterns(context);
195 linalg::populateFoldReshapeOpsByExpansionPatterns(
196 patterns&: fusionPatterns, controlFoldingReshapes: [](OpOperand * /*fusedOperand*/) { return true; });
197 if (failed(Result: applyPatternsGreedily(region&: funcOp.getBody(),
198 patterns: std::move(fusionPatterns))))
199 return signalPassFailure();
200 return;
201 }
202
203 if (controlFuseByExpansion) {
204 RewritePatternSet fusionPatterns(context);
205
206 linalg::ControlFusionFn controlReshapeFusionFn =
207 [](OpOperand *fusedOperand) {
208 Operation *producer = fusedOperand->get().getDefiningOp();
209 if (!producer)
210 return false;
211
212 if (auto collapseOp = dyn_cast<tensor::CollapseShapeOp>(Val: producer)) {
213 if (!collapseOp.getSrc().getDefiningOp<linalg::LinalgOp>()) {
214 return false;
215 }
216 }
217
218 Operation *consumer = fusedOperand->getOwner();
219 if (auto expandOp = dyn_cast<tensor::ExpandShapeOp>(Val: consumer)) {
220 if (expandOp->hasOneUse()) {
221 OpOperand &use = *expandOp->getUses().begin();
222 auto linalgOp = dyn_cast<linalg::LinalgOp>(Val: use.getOwner());
223 if (linalgOp && linalgOp.isDpsInit(opOperand: &use))
224 return true;
225 }
226 return false;
227 }
228 return true;
229 };
230
231 linalg::populateFoldReshapeOpsByExpansionPatterns(patterns&: fusionPatterns,
232 controlFoldingReshapes: controlReshapeFusionFn);
233 if (failed(Result: applyPatternsGreedily(region&: funcOp.getBody(),
234 patterns: std::move(fusionPatterns))))
235 return signalPassFailure();
236 return;
237 }
238
239 if (fuseWithReshapeByCollapsing) {
240 RewritePatternSet patterns(context);
241 linalg::populateFoldReshapeOpsByCollapsingPatterns(
242 patterns, controlFoldingReshapes: [](OpOperand * /*fusedOperand */) { return true; });
243 if (failed(Result: applyPatternsGreedily(region&: funcOp.getBody(), patterns: std::move(patterns))))
244 return signalPassFailure();
245 return;
246 }
247
248 if (fuseWithReshapeByCollapsingWithControlFn) {
249 RewritePatternSet patterns(context);
250 linalg::ControlFusionFn controlFn = [](OpOperand *fusedOperand) -> bool {
251 Operation *producer = fusedOperand->get().getDefiningOp();
252 if (isa<tensor::ExpandShapeOp>(Val: producer)) {
253 // Skip fusing the first operand.
254 return fusedOperand->getOperandNumber();
255 }
256 Operation *consumer = fusedOperand->getOwner();
257 if (auto collapseOp = dyn_cast<tensor::CollapseShapeOp>(Val: consumer)) {
258 auto producerResult = dyn_cast<OpResult>(Val: collapseOp.getSrc());
259 // skip fusing first result.
260 return producerResult.getResultNumber();
261 }
262 return true;
263 };
264 linalg::populateFoldReshapeOpsByCollapsingPatterns(patterns, controlFoldingReshapes: controlFn);
265 if (failed(Result: applyPatternsGreedily(region&: funcOp.getBody(), patterns: std::move(patterns))))
266 return signalPassFailure();
267 return;
268 }
269
270 if (fuseMultiUseProducer) {
271 RewritePatternSet patterns(context);
272 patterns.insert<TestMultiUseProducerFusion>(arg&: context);
273 if (failed(Result: applyPatternsGreedily(
274 region&: funcOp.getBody(), patterns: std::move(patterns),
275 config: GreedyRewriteConfig().setUseTopDownTraversal(true))))
276 return signalPassFailure();
277 return;
278 }
279
280 if (!collapseDimensions.empty()) {
281 SmallVector<int64_t, 2> dims(collapseDimensions.begin(),
282 collapseDimensions.end());
283 linalg::GetCollapsableDimensionsFn collapseFn =
284 [&dims](linalg::LinalgOp op) {
285 SmallVector<ReassociationIndices> reassociations;
286 reassociations.emplace_back(Args&: dims);
287 return reassociations;
288 };
289 RewritePatternSet patterns(context);
290 linalg::populateCollapseDimensions(patterns, controlCollapseDimensions: collapseFn);
291 if (failed(Result: applyPatternsGreedily(region&: funcOp.getBody(), patterns: std::move(patterns))))
292 return signalPassFailure();
293 return;
294 }
295 }
296};
297
298} // namespace
299
300namespace mlir {
301namespace test {
302void registerTestLinalgElementwiseFusion() {
303 PassRegistration<TestLinalgElementwiseFusion>();
304}
305} // namespace test
306} // namespace mlir
307