1//===- FoldIntoElementwise.cpp - Fold Ops into elementwise if possible ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements folding ops such as transpose and broadcast into the
10// affine maps of elementwise consumers.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Linalg/IR/Linalg.h"
15#include "mlir/Dialect/Linalg/Passes.h"
16#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
17#include "mlir/Dialect/Linalg/Utils/Utils.h"
18#include "mlir/IR/PatternMatch.h"
19#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/SmallVector.h"
22
23namespace mlir {
24#define GEN_PASS_DEF_LINALGFOLDINTOELEMENTWISEPASS
25#include "mlir/Dialect/Linalg/Passes.h.inc"
26} // namespace mlir
27
28using namespace mlir;
29using namespace mlir::linalg;
30
31#define DEBUG_TYPE "linalg-fold-into-elementwise"
32
33namespace {
34template <typename ProducerOpTy>
35struct ElementwiseOpFolder {
36 // Helper function to fold broadcast etc. into a consumer operand.
37 static bool fold(OpOperand *elwiseOperand, AffineMap elwiseMap,
38 SmallVector<Value> &newIns,
39 SmallVector<AffineMap> &newMaps) {
40 auto producerOp = elwiseOperand->get().getDefiningOp<ProducerOpTy>();
41 if (!producerOp || !elwiseMap.isProjectedPermutation())
42 return false;
43 newIns.push_back(Elt: producerOp.getInput());
44 // push in the new composed affine map
45 newMaps.push_back(
46 Elt: producerOp.getMatchingIndexingMap(producerOp.getDpsInputOperand(0))
47 .compose(elwiseMap));
48 return true;
49 }
50};
51
52template <typename... ProducerOps>
53struct FoldIntoElementwisePattern : public OpInterfaceRewritePattern<LinalgOp> {
54 using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern;
55
56 LogicalResult matchAndRewrite(LinalgOp op,
57 PatternRewriter &rewriter) const override {
58 if (!isa<GenericOp, ElementwiseOp>(Val: op.getOperation()) || !isElementwise(op))
59 return failure();
60
61 bool changed = false;
62 SmallVector<Value> newIns;
63 SmallVector<AffineMap> newMaps;
64 for (OpOperand *operand : op.getDpsInputOperands()) {
65 AffineMap consumerMap = op.getMatchingIndexingMap(opOperand: operand);
66 const bool folded = (ElementwiseOpFolder<ProducerOps>::fold(
67 operand, consumerMap, newIns, newMaps) ||
68 ...);
69 if (folded) {
70 changed = true;
71 } else {
72 // push in original operand and its map.
73 newIns.push_back(Elt: operand->get());
74 newMaps.push_back(Elt: consumerMap);
75 }
76 }
77 if (!changed)
78 return failure();
79
80 // Keep all output operands and their maps unchanged.
81 SmallVector<AffineMap> originalMaps = op.getIndexingMapsArray();
82 newMaps.append(in_start: originalMaps.begin() + op.getNumDpsInputs(),
83 in_end: originalMaps.end());
84
85 // The maps of the rewritten op must still determine bounds for every loop
86 // dimension. Folding a broadcast can otherwise drop the only map result
87 // that covers a dimension.
88 // See `generic_broadcast_not_folded_non_invertible` in
89 // mlir/test/Dialect/Linalg/elementwise/fold.mlir for an example.
90 if (!inversePermutation(map: concatAffineMaps(maps: newMaps, context: op.getContext())))
91 return failure();
92
93 rewriter.modifyOpInPlace(op, [&] {
94 for (auto [index, operand] : llvm::enumerate(First: op.getDpsInputOperands()))
95 op->setOperand(idx: operand->getOperandNumber(), value: newIns[index]);
96 op->setInherentAttr(name: rewriter.getStringAttr(bytes: "indexing_maps"),
97 value: rewriter.getAffineMapArrayAttr(values: newMaps));
98 });
99 return success();
100 }
101};
102
103struct LinalgFoldIntoElementwisePass
104 : public impl::LinalgFoldIntoElementwisePassBase<
105 LinalgFoldIntoElementwisePass> {
106 using impl::LinalgFoldIntoElementwisePassBase<
107 LinalgFoldIntoElementwisePass>::LinalgFoldIntoElementwisePassBase;
108
109 void runOnOperation() override {
110 Operation *op = getOperation();
111 RewritePatternSet patterns(op->getContext());
112 populateLinalgFoldIntoElementwisePatterns(patterns);
113
114 if (failed(Result: applyPatternsGreedily(op, patterns: std::move(patterns))))
115 return signalPassFailure();
116 }
117};
118} // namespace
119
120void mlir::linalg::populateLinalgFoldIntoElementwisePatterns(
121 RewritePatternSet &patterns) {
122 patterns.add<FoldIntoElementwisePattern<TransposeOp, BroadcastOp>>(
123 arg: patterns.getContext());
124}
125