| 1 | //===- EmptyOpPatterns.cpp - Patterns related to tensor.empty folding ----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 10 | #include "mlir/Dialect/Tensor/Transforms/Transforms.h" |
| 11 | #include "mlir/IR/PatternMatch.h" |
| 12 | |
| 13 | using namespace mlir; |
| 14 | using namespace mlir::tensor; |
| 15 | |
| 16 | namespace { |
| 17 | |
| 18 | template <typename ReshapeOp> |
| 19 | struct FoldEmptyTensorWithReshapeOp : public OpRewritePattern<ReshapeOp> { |
| 20 | FoldEmptyTensorWithReshapeOp(MLIRContext *ctx, PatternBenefit benefit = 1, |
| 21 | bool foldSingleUseOnly = false) |
| 22 | : OpRewritePattern<ReshapeOp>(ctx, benefit), |
| 23 | foldSingleUseOnly(foldSingleUseOnly) {} |
| 24 | |
| 25 | LogicalResult matchAndRewrite(ReshapeOp reshapeOp, |
| 26 | PatternRewriter &rewriter) const override { |
| 27 | // Check for tensor.empty source. |
| 28 | auto emptyOp = reshapeOp.getSrc().template getDefiningOp<EmptyOp>(); |
| 29 | if (!emptyOp) |
| 30 | return failure(); |
| 31 | |
| 32 | // Check for single use. |
| 33 | if (foldSingleUseOnly && !llvm::hasSingleElement(emptyOp->getUses())) |
| 34 | return failure(); |
| 35 | |
| 36 | // Reify result shape. |
| 37 | Location loc = reshapeOp.getLoc(); |
| 38 | ReifiedRankedShapedTypeDims resultShapes; |
| 39 | if (failed(reifyResultShapes(rewriter, reshapeOp, resultShapes)) || |
| 40 | !llvm::hasSingleElement(C&: resultShapes)) |
| 41 | return failure(); |
| 42 | |
| 43 | Attribute encoding; |
| 44 | if (auto tensorTy = dyn_cast<RankedTensorType>(reshapeOp.getResultType())) |
| 45 | encoding = tensorTy.getEncoding(); |
| 46 | |
| 47 | // Create new tensor.empty op. |
| 48 | Value emptyTensor = |
| 49 | EmptyOp::create(rewriter, loc, resultShapes[0], |
| 50 | reshapeOp.getResultType().getElementType(), encoding); |
| 51 | if (emptyTensor.getType() != reshapeOp.getResultType()) { |
| 52 | rewriter.replaceOpWithNewOp<tensor::CastOp>( |
| 53 | reshapeOp, reshapeOp.getResultType(), emptyTensor); |
| 54 | } else { |
| 55 | rewriter.replaceOp(reshapeOp, emptyTensor); |
| 56 | } |
| 57 | return success(); |
| 58 | } |
| 59 | |
| 60 | private: |
| 61 | bool foldSingleUseOnly = false; |
| 62 | }; |
| 63 | |
| 64 | /// tensor.empty does not define any tensor contents, so a slice of a |
| 65 | /// tensor.empty can be folded to a smaller tensor.empty. |
| 66 | struct |
| 67 | : public OpRewritePattern<ExtractSliceOp> { |
| 68 | (MLIRContext *ctx, |
| 69 | PatternBenefit benefit = 1, |
| 70 | bool foldSingleUseOnly = false) |
| 71 | : OpRewritePattern<ExtractSliceOp>(ctx, benefit), |
| 72 | foldSingleUseOnly(foldSingleUseOnly) {} |
| 73 | |
| 74 | LogicalResult matchAndRewrite(ExtractSliceOp sliceOp, |
| 75 | PatternRewriter &rewriter) const override { |
| 76 | // Check for tensor.empty source. |
| 77 | auto emptyOp = sliceOp.getSource().template getDefiningOp<EmptyOp>(); |
| 78 | if (!emptyOp) |
| 79 | return failure(); |
| 80 | |
| 81 | // Check for single use. |
| 82 | if (foldSingleUseOnly && !llvm::hasSingleElement(C: emptyOp->getUses())) |
| 83 | return failure(); |
| 84 | |
| 85 | // Create new tensor.empty op. tensor.extract_slice may be rank-reducing; |
| 86 | // its dynamic sizes must be preserved as well as its result type. |
| 87 | auto tensorType = RankedTensorType::get(shape: sliceOp.getType().getShape(), |
| 88 | elementType: sliceOp.getType().getElementType(), |
| 89 | encoding: sliceOp.getType().getEncoding()); |
| 90 | rewriter.replaceOpWithNewOp<EmptyOp>(op: sliceOp, args&: tensorType, |
| 91 | args: sliceOp.getSizes()); |
| 92 | return success(); |
| 93 | } |
| 94 | |
| 95 | private: |
| 96 | bool = false; |
| 97 | }; |
| 98 | |
| 99 | // Fold concat operation where all the operands are empty. |
| 100 | struct FoldConcatsOfEmpty : public OpRewritePattern<ConcatOp> { |
| 101 | using OpRewritePattern<ConcatOp>::OpRewritePattern; |
| 102 | |
| 103 | LogicalResult matchAndRewrite(tensor::ConcatOp concatOp, |
| 104 | PatternRewriter &rewriter) const override { |
| 105 | auto concatOperands = concatOp.getInputs(); |
| 106 | if (concatOperands.empty()) { |
| 107 | return failure(); |
| 108 | } |
| 109 | auto firstEmptyOp = concatOperands.front().getDefiningOp<tensor::EmptyOp>(); |
| 110 | if (!firstEmptyOp) { |
| 111 | return failure(); |
| 112 | } |
| 113 | auto isDefinedByEmptyOp = [](Value v) -> bool { |
| 114 | return v.getDefiningOp<tensor::EmptyOp>(); |
| 115 | }; |
| 116 | if (!llvm::all_of(Range: concatOperands.drop_front(), P: isDefinedByEmptyOp)) { |
| 117 | return rewriter.notifyMatchFailure( |
| 118 | arg&: concatOp, msg: "not all operands are defined by an empty op" ); |
| 119 | } |
| 120 | SmallVector<SmallVector<OpFoldResult>> resultShape; |
| 121 | if (failed(Result: concatOp.reifyResultShapes(builder&: rewriter, reifiedReturnShapes&: resultShape))) { |
| 122 | return rewriter.notifyMatchFailure(arg&: concatOp, |
| 123 | msg: "failed to get result shape" ); |
| 124 | } |
| 125 | auto resultType = concatOp.getResultType(); |
| 126 | rewriter.replaceOpWithNewOp<tensor::EmptyOp>(op: concatOp, args&: resultShape[0], |
| 127 | args: resultType.getElementType(), |
| 128 | args: resultType.getEncoding()); |
| 129 | return success(); |
| 130 | } |
| 131 | }; |
| 132 | |
| 133 | } // namespace |
| 134 | |
| 135 | void mlir::tensor::populateFoldTensorEmptyPatterns(RewritePatternSet &patterns, |
| 136 | bool foldSingleUseOnly) { |
| 137 | patterns.add<FoldEmptyTensorWithExtractSliceOp, |
| 138 | FoldEmptyTensorWithReshapeOp<tensor::ExpandShapeOp>, |
| 139 | FoldEmptyTensorWithReshapeOp<tensor::CollapseShapeOp>>( |
| 140 | arg: patterns.getContext(), /*benefit=*/args: 1, args&: foldSingleUseOnly); |
| 141 | patterns.add<FoldConcatsOfEmpty>(arg: patterns.getContext(), |
| 142 | /*benefit=*/args: 1); |
| 143 | } |
| 144 | |