| 1 | //===- FoldIntoElementwise.cpp - Fold Ops into elementwise if possible ---===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements folding ops such as transpose and broadcast into the |
| 10 | // affine maps of elementwise consumers. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/Linalg/IR/Linalg.h" |
| 15 | #include "mlir/Dialect/Linalg/Passes.h" |
| 16 | #include "mlir/Dialect/Linalg/Transforms/Transforms.h" |
| 17 | #include "mlir/Dialect/Linalg/Utils/Utils.h" |
| 18 | #include "mlir/IR/PatternMatch.h" |
| 19 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 20 | #include "llvm/ADT/STLExtras.h" |
| 21 | #include "llvm/ADT/SmallVector.h" |
| 22 | |
| 23 | namespace mlir { |
| 24 | #define GEN_PASS_DEF_LINALGFOLDINTOELEMENTWISEPASS |
| 25 | #include "mlir/Dialect/Linalg/Passes.h.inc" |
| 26 | } // namespace mlir |
| 27 | |
| 28 | using namespace mlir; |
| 29 | using namespace mlir::linalg; |
| 30 | |
| 31 | #define DEBUG_TYPE "linalg-fold-into-elementwise" |
| 32 | |
| 33 | namespace { |
| 34 | template <typename ProducerOpTy> |
| 35 | struct ElementwiseOpFolder { |
| 36 | // Helper function to fold broadcast etc. into a consumer operand. |
| 37 | static bool fold(OpOperand *elwiseOperand, AffineMap elwiseMap, |
| 38 | SmallVector<Value> &newIns, |
| 39 | SmallVector<AffineMap> &newMaps) { |
| 40 | auto producerOp = elwiseOperand->get().getDefiningOp<ProducerOpTy>(); |
| 41 | if (!producerOp || !elwiseMap.isProjectedPermutation()) |
| 42 | return false; |
| 43 | newIns.push_back(Elt: producerOp.getInput()); |
| 44 | // push in the new composed affine map |
| 45 | newMaps.push_back( |
| 46 | Elt: producerOp.getMatchingIndexingMap(producerOp.getDpsInputOperand(0)) |
| 47 | .compose(elwiseMap)); |
| 48 | return true; |
| 49 | } |
| 50 | }; |
| 51 | |
| 52 | template <typename... ProducerOps> |
| 53 | struct FoldIntoElementwisePattern : public OpInterfaceRewritePattern<LinalgOp> { |
| 54 | using OpInterfaceRewritePattern<LinalgOp>::OpInterfaceRewritePattern; |
| 55 | |
| 56 | LogicalResult matchAndRewrite(LinalgOp op, |
| 57 | PatternRewriter &rewriter) const override { |
| 58 | if (!isa<GenericOp, ElementwiseOp>(Val: op.getOperation()) || !isElementwise(op)) |
| 59 | return failure(); |
| 60 | |
| 61 | bool changed = false; |
| 62 | SmallVector<Value> newIns; |
| 63 | SmallVector<AffineMap> newMaps; |
| 64 | for (OpOperand *operand : op.getDpsInputOperands()) { |
| 65 | AffineMap consumerMap = op.getMatchingIndexingMap(opOperand: operand); |
| 66 | const bool folded = (ElementwiseOpFolder<ProducerOps>::fold( |
| 67 | operand, consumerMap, newIns, newMaps) || |
| 68 | ...); |
| 69 | if (folded) { |
| 70 | changed = true; |
| 71 | } else { |
| 72 | // push in original operand and its map. |
| 73 | newIns.push_back(Elt: operand->get()); |
| 74 | newMaps.push_back(Elt: consumerMap); |
| 75 | } |
| 76 | } |
| 77 | if (!changed) |
| 78 | return failure(); |
| 79 | |
| 80 | // Keep all output operands and their maps unchanged. |
| 81 | SmallVector<AffineMap> originalMaps = op.getIndexingMapsArray(); |
| 82 | newMaps.append(in_start: originalMaps.begin() + op.getNumDpsInputs(), |
| 83 | in_end: originalMaps.end()); |
| 84 | |
| 85 | // The maps of the rewritten op must still determine bounds for every loop |
| 86 | // dimension. Folding a broadcast can otherwise drop the only map result |
| 87 | // that covers a dimension. |
| 88 | // See `generic_broadcast_not_folded_non_invertible` in |
| 89 | // mlir/test/Dialect/Linalg/elementwise/fold.mlir for an example. |
| 90 | if (!inversePermutation(map: concatAffineMaps(maps: newMaps, context: op.getContext()))) |
| 91 | return failure(); |
| 92 | |
| 93 | rewriter.modifyOpInPlace(op, [&] { |
| 94 | for (auto [index, operand] : llvm::enumerate(First: op.getDpsInputOperands())) |
| 95 | op->setOperand(idx: operand->getOperandNumber(), value: newIns[index]); |
| 96 | op->setInherentAttr(name: rewriter.getStringAttr(bytes: "indexing_maps" ), |
| 97 | value: rewriter.getAffineMapArrayAttr(values: newMaps)); |
| 98 | }); |
| 99 | return success(); |
| 100 | } |
| 101 | }; |
| 102 | |
| 103 | struct LinalgFoldIntoElementwisePass |
| 104 | : public impl::LinalgFoldIntoElementwisePassBase< |
| 105 | LinalgFoldIntoElementwisePass> { |
| 106 | using impl::LinalgFoldIntoElementwisePassBase< |
| 107 | LinalgFoldIntoElementwisePass>::LinalgFoldIntoElementwisePassBase; |
| 108 | |
| 109 | void runOnOperation() override { |
| 110 | Operation *op = getOperation(); |
| 111 | RewritePatternSet patterns(op->getContext()); |
| 112 | populateLinalgFoldIntoElementwisePatterns(patterns); |
| 113 | |
| 114 | if (failed(Result: applyPatternsGreedily(op, patterns: std::move(patterns)))) |
| 115 | return signalPassFailure(); |
| 116 | } |
| 117 | }; |
| 118 | } // namespace |
| 119 | |
| 120 | void mlir::linalg::populateLinalgFoldIntoElementwisePatterns( |
| 121 | RewritePatternSet &patterns) { |
| 122 | patterns.add<FoldIntoElementwisePattern<TransposeOp, BroadcastOp>>( |
| 123 | arg: patterns.getContext()); |
| 124 | } |
| 125 | |