1//===- FusePadOpWithLinalgProducer.cpp ---- Fuse pad with linalg producer -===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements patterns that fuses a linalg.generic -> tensor.pad op
10// chain into a tensor.extract_slice -> linalg.generic -> tensor.insert_slice
11// op chain.
12//
13//===----------------------------------------------------------------------===//
14
15#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
16
17#include "mlir/Dialect/Linalg/IR/Linalg.h"
18
19using namespace mlir;
20
21namespace {
22/// A sequence of operations
23///
24/// ```mlir
25/// %0 = linalg. ...
26/// %1 = tensor.pad %0 ...
27/// ```
28///
29/// can be replaced with
30///
31/// ```mlir
32/// %0 = linalg.fill
33/// %1 = tensor.extract_slice %0 ...
34/// %2 = linalg. .... outs(..., %1, ....) ....
35/// %3 = tensor.insert_slice %2 into %1 ...
36/// ```
37///
38/// if the `linalg.generic` has all parallel iterator types.
39struct FusePadOp : OpRewritePattern<tensor::PadOp> {
40 using OpRewritePattern<tensor::PadOp>::OpRewritePattern;
41
42 LogicalResult matchAndRewrite(tensor::PadOp padOp,
43 PatternRewriter &rewriter) const override {
44 // Only works on padding op that sets the padded value to a constant.
45 Value padValue = padOp.getConstantPaddingValue();
46 if (!padValue)
47 return rewriter.notifyMatchFailure(arg&: padOp, msg: "non constant padding");
48
49 // This pattern could work for any Linalg op. For now restrict it to generic
50 // ops.
51 Value source = padOp.getSource();
52 auto linalgOp = source.getDefiningOp<linalg::GenericOp>();
53 if (!linalgOp) {
54 return rewriter.notifyMatchFailure(
55 arg&: padOp, msg: "expected source to be linalg.generic op");
56 }
57 // All iterator types need to be parallel.
58 if (linalgOp.getNumLoops() != linalgOp.getNumParallelLoops()) {
59 return rewriter.notifyMatchFailure(
60 arg&: padOp, msg: "only supported for ops with all parallel iterator types");
61 }
62 ReifiedRankedShapedTypeDims resultShape;
63 if (failed(Result: reifyResultShapes(b&: rewriter, op: padOp, reifiedReturnShapes&: resultShape)) ||
64 resultShape.size() != 1) {
65 return rewriter.notifyMatchFailure(
66 arg&: padOp, msg: "failed to get shape of pad op result");
67 }
68
69 Location loc = padOp.getLoc();
70
71 // Create the tensor of same size as output of the pad op.
72 RankedTensorType padResultType = padOp.getResultType();
73 auto resultSizes = resultShape[0];
74 auto emptyTensor = tensor::EmptyOp::create(builder&: rewriter, location: loc, sizes: resultSizes,
75 elementType: padResultType.getElementType());
76
77 // Fill the tensor with the pad value.
78 // TODO: There is an option to fill only the boundaries. For now just
79 // filling the whole tensor.
80 // Move the padding value defined inside the PadOp block to outside.
81 if (padValue.getParentBlock() == &padOp.getRegion().front())
82 rewriter.moveOpBefore(op: padValue.getDefiningOp(), existingOp: padOp);
83 auto fillTensor = linalg::FillOp::create(builder&: rewriter, location: loc, inputs: padValue,
84 outputs: emptyTensor.getResult());
85
86 // Construct a slice of the fill result that is to be replaced with the
87 // result of the generic op. The low pad values are the offsets, the size of
88 // the source is the size of the slice.
89 // TODO: This insert/extract could be potentially made a utility method.
90 unsigned resultNumber = cast<OpResult>(Val&: source).getResultNumber();
91 SmallVector<OpFoldResult> offsets = padOp.getMixedLowPad();
92 SmallVector<OpFoldResult> sizes;
93 sizes.reserve(N: offsets.size());
94 for (const auto &shape :
95 llvm::enumerate(First: cast<RankedTensorType>(Val: source.getType()).getShape())) {
96 if (ShapedType::isDynamic(dValue: shape.value())) {
97 sizes.push_back(
98 Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source, index: shape.index())
99 .getResult());
100 } else {
101 sizes.push_back(Elt: rewriter.getIndexAttr(value: shape.value()));
102 }
103 }
104 SmallVector<OpFoldResult> strides(offsets.size(), rewriter.getIndexAttr(value: 1));
105 auto slice = tensor::ExtractSliceOp::create(
106 builder&: rewriter, location: loc, source: fillTensor.getResult(i: 0), offsets, sizes, strides);
107
108 // Clone the generic op.
109 auto clonedOp =
110 cast<linalg::GenericOp>(Val: rewriter.clone(op&: *linalgOp.getOperation()));
111 clonedOp.setDpsInitOperand(i: resultNumber, value: slice.getResult());
112
113 // Insert it back into the result of the fill.
114 rewriter.replaceOpWithNewOp<tensor::InsertSliceOp>(
115 op: padOp, args: clonedOp.getResult(i: resultNumber), args: fillTensor.getResult(i: 0),
116 args&: offsets, args&: sizes, args&: strides);
117 return success();
118 }
119};
120} // namespace
121
122void mlir::linalg::populateFuseTensorPadWithProducerLinalgOpPatterns(
123 RewritePatternSet &patterns) {
124 patterns.add<FusePadOp>(arg: patterns.getContext());
125}
126