1//===- FoldTensorSubsetOps.cpp - Fold tensor subset ops -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Fold tensor subset ops with producer / consumers.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Affine/IR/AffineOps.h"
14#include "mlir/Dialect/Affine/ViewLikeInterfaceUtils.h"
15#include "mlir/Dialect/SCF/IR/SCF.h"
16#include "mlir/Dialect/Tensor/IR/Tensor.h"
17#include "mlir/Dialect/Tensor/Transforms/Passes.h"
18#include "mlir/Dialect/Tensor/Transforms/Transforms.h"
19#include "mlir/Dialect/Vector/IR/VectorOps.h"
20#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
21#include "mlir/IR/AffineMap.h"
22#include "mlir/IR/BuiltinAttributes.h"
23#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
24#include <type_traits>
25
26namespace mlir {
27namespace tensor {
28#define GEN_PASS_DEF_FOLDTENSORSUBSETOPSPASS
29#include "mlir/Dialect/Tensor/Transforms/Passes.h.inc"
30} // namespace tensor
31} // namespace mlir
32
33using namespace mlir;
34
35static Value getTensorOperand(vector::TransferReadOp op) {
36 return op.getBase();
37}
38
39static Value getTensorOperand(tensor::InsertSliceOp op) {
40 return op.getSource();
41}
42
43//===----------------------------------------------------------------------===//
44// Patterns
45//===----------------------------------------------------------------------===//
46
47namespace {
48/// Merge extract_slice operation with load/transferRead operation.
49class TransferReadOfExtractSliceOpFolder final
50 : public vector::MaskableOpRewritePattern<vector::TransferReadOp> {
51public:
52 using MaskableOpRewritePattern::MaskableOpRewritePattern;
53
54 FailureOr<mlir::Value>
55 matchAndRewriteMaskableOp(vector::TransferReadOp readOp,
56 vector::MaskingOpInterface maskOp,
57 PatternRewriter &rewriter) const override;
58};
59
60/// Merge insert_slice operation with store/transferWriteOp operation.
61class InsertSliceOfTransferWriteOpFolder final
62 : public OpRewritePattern<tensor::InsertSliceOp> {
63public:
64 using OpRewritePattern<tensor::InsertSliceOp>::OpRewritePattern;
65
66 LogicalResult matchAndRewrite(tensor::InsertSliceOp insertSliceOp,
67 PatternRewriter &rewriter) const override;
68
69private:
70 static bool
71 doesTransferWriteCoverInsertSlice(vector::TransferWriteOp writeOp);
72};
73} // namespace
74
75template <typename XferOp, typename ExtractOrInsertOp>
76static LogicalResult preconditionsFoldExtractOrInsertWithTransferOp(
77 RewriterBase &rewriter, XferOp xferOp,
78 ExtractOrInsertOp extractOrInsertSliceOp) {
79 if (xferOp.hasOutOfBoundsDim())
80 return rewriter.notifyMatchFailure(xferOp, "out of bounds transfer dim");
81 if (xferOp.getMask())
82 return rewriter.notifyMatchFailure(xferOp, "masked transfer");
83 if (!extractOrInsertSliceOp.hasUnitStride()) {
84 return rewriter.notifyMatchFailure(
85 xferOp, "non-1 stride insert/extract, requires keeping track of "
86 "strides, this may result in needing to insert "
87 "vector.insert_strided_slice/extract_strided_slice ops");
88 }
89 return success();
90}
91
92FailureOr<mlir::Value>
93TransferReadOfExtractSliceOpFolder::matchAndRewriteMaskableOp(
94 vector::TransferReadOp readOp, vector::MaskingOpInterface maskOp,
95 PatternRewriter &rewriter) const {
96 auto extractSliceOp =
97 getTensorOperand(op: readOp).getDefiningOp<tensor::ExtractSliceOp>();
98 if (!extractSliceOp)
99 return rewriter.notifyMatchFailure(arg&: readOp, msg: "not an extract_slice");
100
101 LogicalResult preconditionResult =
102 preconditionsFoldExtractOrInsertWithTransferOp(rewriter, xferOp: readOp,
103 extractOrInsertSliceOp: extractSliceOp);
104 if (failed(Result: preconditionResult))
105 return rewriter.notifyMatchFailure(arg&: readOp, msg: "Failed preconditions");
106
107 SmallVector<Value> indices(readOp.getIndices().begin(),
108 readOp.getIndices().end());
109 SmallVector<Value> sourceIndices;
110 affine::resolveIndicesIntoOpWithOffsetsAndStrides(
111 rewriter, loc: readOp.getLoc(), mixedSourceOffsets: extractSliceOp.getMixedOffsets(),
112 mixedSourceStrides: extractSliceOp.getMixedStrides(), rankReducedDims: extractSliceOp.getDroppedDims(),
113 consumerIndices: indices, resolvedIndices&: sourceIndices);
114
115 Operation *newOp = vector::TransferReadOp::create(
116 builder&: rewriter, location: readOp.getLoc(), result: readOp.getVectorType(),
117 base: extractSliceOp.getSource(), indices: sourceIndices,
118 permutation_map: AffineMapAttr::get(value: expandDimsToRank(
119 map: readOp.getPermutationMap(), rank: extractSliceOp.getSourceType().getRank(),
120 projectedDimensions: extractSliceOp.getDroppedDims())),
121 padding: readOp.getPadding(),
122 /*mask=*/Value(), in_bounds: readOp.getInBoundsAttr());
123 if (maskOp)
124 newOp = mlir::vector::maskOperation(builder&: rewriter, maskableOp: newOp, mask: maskOp.getMask());
125 return newOp->getResults()[0];
126}
127
128LogicalResult InsertSliceOfTransferWriteOpFolder::matchAndRewrite(
129 tensor::InsertSliceOp insertSliceOp, PatternRewriter &rewriter) const {
130 auto writeOp = getTensorOperand(op: insertSliceOp)
131 .template getDefiningOp<vector::TransferWriteOp>();
132 if (!writeOp)
133 return rewriter.notifyMatchFailure(arg&: insertSliceOp, msg: "not a transfer_write");
134
135 LogicalResult preconditionResult =
136 preconditionsFoldExtractOrInsertWithTransferOp(rewriter, xferOp: writeOp,
137 extractOrInsertSliceOp: insertSliceOp);
138 if (failed(Result: preconditionResult))
139 return preconditionResult;
140
141 if (!doesTransferWriteCoverInsertSlice(writeOp))
142 return rewriter.notifyMatchFailure(
143 arg&: insertSliceOp, msg: "transfer_write does not cover insert_slice");
144
145 SmallVector<Value> indices(writeOp.getIndices().begin(),
146 writeOp.getIndices().end());
147 SmallVector<Value> sourceIndices;
148 affine::resolveIndicesIntoOpWithOffsetsAndStrides(
149 rewriter, loc: writeOp.getLoc(), mixedSourceOffsets: insertSliceOp.getMixedOffsets(),
150 mixedSourceStrides: insertSliceOp.getMixedStrides(), rankReducedDims: insertSliceOp.getDroppedDims(), consumerIndices: indices,
151 resolvedIndices&: sourceIndices);
152
153 rewriter.replaceOpWithNewOp<vector::TransferWriteOp>(
154 op: insertSliceOp, args: writeOp.getValue(), args: insertSliceOp.getDest(), args&: sourceIndices,
155 args: AffineMapAttr::get(value: expandDimsToRank(map: writeOp.getPermutationMap(),
156 rank: insertSliceOp.getDestType().getRank(),
157 projectedDimensions: insertSliceOp.getDroppedDims())),
158 args: writeOp.getInBoundsAttr());
159
160 return success();
161}
162
163bool InsertSliceOfTransferWriteOpFolder::doesTransferWriteCoverInsertSlice(
164 vector::TransferWriteOp writeOp) {
165 if (writeOp.getShapedType().hasStaticShape())
166 return llvm::equal(LRange: writeOp.getVectorType().getShape(),
167 RRange: writeOp.getShapedType().getShape());
168
169 // TODO: Use ValueBoundsConstraintSet for dynamic shapes.
170
171 return false;
172}
173
174template <typename OpTy>
175struct InsertSliceOfInsertSliceFolder : public OpRewritePattern<OpTy> {
176 using OpRewritePattern<OpTy>::OpRewritePattern;
177
178 LogicalResult matchAndRewrite(OpTy insertSliceOp,
179 PatternRewriter &rewriter) const override {
180 auto sourceInsertSliceOp =
181 insertSliceOp.getSource()
182 .template getDefiningOp<tensor::InsertSliceOp>();
183 if (!sourceInsertSliceOp)
184 return failure();
185
186 int64_t srcDim = 0;
187 llvm::SmallBitVector droppedDims = insertSliceOp.getDroppedDims();
188 for (int64_t d = 0, e = insertSliceOp.getDestType().getRank(); d < e; ++d) {
189 if (droppedDims[d])
190 continue;
191 if (insertSliceOp.getMixedSizes()[d] !=
192 sourceInsertSliceOp.getMixedSizes()[srcDim++]) {
193 return rewriter.notifyMatchFailure(
194 sourceInsertSliceOp,
195 "requires matching sizes to fold, otherwise a copy is needed");
196 }
197 }
198
199 // If we are inside a ParallelCombining region, temporarily set the
200 // insertion point outside: only ops of ParallelCombiningOpInterface are
201 // allowed in there.
202 if (isa<mlir::ParallelCombiningOpInterface>(insertSliceOp.getOperation())) {
203 rewriter.setInsertionPoint(insertSliceOp->getParentOp());
204 }
205
206 SmallVector<OpFoldResult> newOffsets, newSizes, newStrides;
207 if (failed(affine::mergeOffsetsSizesAndStrides(
208 rewriter, insertSliceOp.getLoc(), insertSliceOp,
209 sourceInsertSliceOp, droppedDims, newOffsets, newSizes,
210 newStrides)))
211 return failure();
212
213 // Reset the insertion point.
214 rewriter.setInsertionPoint(insertSliceOp);
215 // Replace original op.
216 rewriter.replaceOpWithNewOp<OpTy>(
217 insertSliceOp, sourceInsertSliceOp.getSource(), insertSliceOp.getDest(),
218 newOffsets, newSizes, newStrides);
219 return success();
220 }
221};
222
223struct MergeConsecutiveExtractSlice
224 : public OpRewritePattern<tensor::ExtractSliceOp> {
225 using Base::Base;
226
227 LogicalResult matchAndRewrite(tensor::ExtractSliceOp nextOp,
228 PatternRewriter &rewriter) const override {
229 auto prevOp = nextOp.getSource().getDefiningOp<tensor::ExtractSliceOp>();
230 if (!prevOp)
231 return failure();
232
233 SmallVector<OpFoldResult> newOffsets, newSizes, newStrides;
234 if (failed(Result: affine::mergeOffsetsSizesAndStrides(
235 builder&: rewriter, loc: nextOp.getLoc(), producer: prevOp, consumer: nextOp, droppedProducerDims: prevOp.getDroppedDims(),
236 combinedOffsets&: newOffsets, combinedSizes&: newSizes, combinedStrides&: newStrides)))
237 return failure();
238
239 rewriter.replaceOpWithNewOp<tensor::ExtractSliceOp>(
240 op: nextOp, args: nextOp.getType(), args: prevOp.getSource(), args&: newOffsets, args&: newSizes,
241 args&: newStrides);
242 return success();
243 }
244};
245
246void tensor::populateFoldTensorSubsetIntoVectorTransferPatterns(
247 RewritePatternSet &patterns) {
248 patterns.add<TransferReadOfExtractSliceOpFolder,
249 InsertSliceOfTransferWriteOpFolder>(arg: patterns.getContext());
250}
251
252void tensor::populateMergeConsecutiveInsertExtractSlicePatterns(
253 RewritePatternSet &patterns) {
254 patterns.add<MergeConsecutiveExtractSlice,
255 InsertSliceOfInsertSliceFolder<tensor::InsertSliceOp>,
256 InsertSliceOfInsertSliceFolder<tensor::ParallelInsertSliceOp>>(
257 arg: patterns.getContext());
258}
259
260void tensor::populateFoldTensorSubsetOpPatterns(RewritePatternSet &patterns) {
261 populateFoldTensorSubsetIntoVectorTransferPatterns(patterns);
262 populateMergeConsecutiveInsertExtractSlicePatterns(patterns);
263}
264
265//===----------------------------------------------------------------------===//
266// Pass registration
267//===----------------------------------------------------------------------===//
268
269namespace {
270
271struct FoldTensorSubsetOpsPass final
272 : public tensor::impl::FoldTensorSubsetOpsPassBase<
273 FoldTensorSubsetOpsPass> {
274 void runOnOperation() override;
275};
276
277} // namespace
278
279void FoldTensorSubsetOpsPass::runOnOperation() {
280 RewritePatternSet patterns(&getContext());
281 tensor::populateFoldTensorSubsetOpPatterns(patterns);
282 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
283}
284