1//===- FoldMemRefAliasOps.cpp - Fold memref alias ops for affine ops ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass contains affine-specif versions of the folding patterns for
10// memref.expand_shape, memref.collapse_shape, and memref.subview, since
11// those all need affine-specific handling that won't fit a general interface.
12//
13//===----------------------------------------------------------------------===//
14
15#include "mlir/Dialect/Affine/IR/AffineOps.h"
16#include "mlir/Dialect/Affine/Transforms/Passes.h"
17#include "mlir/Dialect/Affine/ViewLikeInterfaceUtils.h"
18#include "mlir/Dialect/Arith/Utils/Utils.h"
19#include "mlir/Dialect/MemRef/IR/MemRef.h"
20#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
21#include "mlir/IR/AffineMap.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24namespace mlir {
25namespace affine {
26#define GEN_PASS_DEF_AFFINEFOLDMEMREFALIASOPS
27#include "mlir/Dialect/Affine/Transforms/Passes.h.inc"
28} // namespace affine
29} // namespace mlir
30
31using namespace mlir;
32using namespace mlir::affine;
33
34//===----------------------------------------------------------------------===//
35// Utility functions
36//===----------------------------------------------------------------------===//
37
38/// Given an AffineMap and a list of indices, apply the map to get the
39/// underlying indices (expanding the affine map).
40static void expandToUnderlyingIndices(AffineMap affineMap, ValueRange indices,
41 Location loc, PatternRewriter &rewriter,
42 SmallVectorImpl<Value> &result) {
43 SmallVector<OpFoldResult> indicesOfr(
44 llvm::map_to_vector(C&: indices, F: [](Value v) -> OpFoldResult { return v; }));
45 for (unsigned i : llvm::seq(Begin: 0u, End: affineMap.getNumResults())) {
46 OpFoldResult ofr = affine::makeComposedFoldedAffineApply(
47 b&: rewriter, loc, map: affineMap.getSubMap(resultPos: {i}), operands: indicesOfr);
48 result.push_back(Elt: getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr));
49 }
50}
51
52//===----------------------------------------------------------------------===//
53// Patterns
54//===----------------------------------------------------------------------===//
55
56namespace {
57
58struct AffineLoadOpOfSubViewOpFolder final : OpRewritePattern<AffineLoadOp> {
59 using Base::Base;
60
61 LogicalResult matchAndRewrite(AffineLoadOp loadOp,
62 PatternRewriter &rewriter) const override {
63 auto subViewOp = loadOp.getMemref().getDefiningOp<memref::SubViewOp>();
64
65 if (!subViewOp)
66 return rewriter.notifyMatchFailure(arg&: loadOp, msg: "not a subview producer");
67
68 SmallVector<Value> indices;
69 expandToUnderlyingIndices(affineMap: loadOp.getAffineMap(), indices: loadOp.getIndices(),
70 loc: loadOp.getLoc(), rewriter, result&: indices);
71
72 SmallVector<Value> sourceIndices;
73 affine::resolveIndicesIntoOpWithOffsetsAndStrides(
74 rewriter, loc: loadOp.getLoc(), mixedSourceOffsets: subViewOp.getMixedOffsets(),
75 mixedSourceStrides: subViewOp.getMixedStrides(), rankReducedDims: subViewOp.getDroppedDims(), consumerIndices: indices,
76 resolvedIndices&: sourceIndices);
77
78 rewriter.replaceOpWithNewOp<AffineLoadOp>(op: loadOp, args: subViewOp.getSource(),
79 args&: sourceIndices);
80 return success();
81 }
82};
83
84struct AffineLoadOpOfExpandShapeOpFolder final
85 : OpRewritePattern<AffineLoadOp> {
86 using Base::Base;
87
88 LogicalResult matchAndRewrite(AffineLoadOp loadOp,
89 PatternRewriter &rewriter) const override {
90 auto expandShapeOp =
91 loadOp.getMemref().getDefiningOp<memref::ExpandShapeOp>();
92
93 if (!expandShapeOp)
94 return failure();
95
96 SmallVector<Value> indices;
97 expandToUnderlyingIndices(affineMap: loadOp.getAffineMap(), indices: loadOp.getIndices(),
98 loc: loadOp.getLoc(), rewriter, result&: indices);
99
100 SmallVector<Value> sourceIndices;
101 // affine.load guarantees that indexes start inbounds, which impacts if our
102 // linearization is `disjoint`.
103 memref::resolveSourceIndicesExpandShape(
104 loc: loadOp.getLoc(), rewriter, expandShapeOp, indices, sourceIndices,
105 /*startsInbounds=*/true);
106
107 rewriter.replaceOpWithNewOp<AffineLoadOp>(
108 op: loadOp, args: expandShapeOp.getViewSource(), args&: sourceIndices);
109 return success();
110 }
111};
112
113struct AffineLoadOpOfCollapseShapeOpFolder final
114 : OpRewritePattern<AffineLoadOp> {
115 using Base::Base;
116
117 LogicalResult matchAndRewrite(AffineLoadOp loadOp,
118 PatternRewriter &rewriter) const override {
119 auto collapseShapeOp =
120 loadOp.getMemref().getDefiningOp<memref::CollapseShapeOp>();
121
122 if (!collapseShapeOp)
123 return failure();
124
125 SmallVector<Value> indices;
126 expandToUnderlyingIndices(affineMap: loadOp.getAffineMap(), indices: loadOp.getIndices(),
127 loc: loadOp.getLoc(), rewriter, result&: indices);
128
129 SmallVector<Value> sourceIndices;
130 memref::resolveSourceIndicesCollapseShape(
131 loc: loadOp.getLoc(), rewriter, collapseShapeOp, indices, sourceIndices,
132 /*startsInbounds=*/true);
133
134 rewriter.replaceOpWithNewOp<AffineLoadOp>(
135 op: loadOp, args: collapseShapeOp.getViewSource(), args&: sourceIndices);
136 return success();
137 }
138};
139
140struct AffineStoreOpOfSubViewOpFolder final : OpRewritePattern<AffineStoreOp> {
141 using Base::Base;
142
143 LogicalResult matchAndRewrite(AffineStoreOp storeOp,
144 PatternRewriter &rewriter) const override {
145 auto subViewOp = storeOp.getMemref().getDefiningOp<memref::SubViewOp>();
146
147 if (!subViewOp)
148 return rewriter.notifyMatchFailure(arg&: storeOp, msg: "not a subview producer");
149
150 // For affine ops, we need to apply the map to get the "actual" indices.
151 SmallVector<Value> indices;
152 expandToUnderlyingIndices(affineMap: storeOp.getAffineMap(), indices: storeOp.getIndices(),
153 loc: storeOp.getLoc(), rewriter, result&: indices);
154
155 SmallVector<Value> sourceIndices;
156 affine::resolveIndicesIntoOpWithOffsetsAndStrides(
157 rewriter, loc: storeOp.getLoc(), mixedSourceOffsets: subViewOp.getMixedOffsets(),
158 mixedSourceStrides: subViewOp.getMixedStrides(), rankReducedDims: subViewOp.getDroppedDims(), consumerIndices: indices,
159 resolvedIndices&: sourceIndices);
160
161 rewriter.replaceOpWithNewOp<AffineStoreOp>(
162 op: storeOp, args: storeOp.getValue(), args: subViewOp.getSource(), args&: sourceIndices);
163 return success();
164 }
165};
166
167struct AffineStoreOpOfExpandShapeOpFolder final
168 : OpRewritePattern<AffineStoreOp> {
169 using Base::Base;
170
171 LogicalResult matchAndRewrite(AffineStoreOp storeOp,
172 PatternRewriter &rewriter) const override {
173 auto expandShapeOp =
174 storeOp.getMemref().getDefiningOp<memref::ExpandShapeOp>();
175
176 if (!expandShapeOp)
177 return failure();
178
179 SmallVector<Value> indices;
180 expandToUnderlyingIndices(affineMap: storeOp.getAffineMap(), indices: storeOp.getIndices(),
181 loc: storeOp.getLoc(), rewriter, result&: indices);
182
183 SmallVector<Value> sourceIndices;
184 // affine.store guarantees that indexes start inbounds, which impacts if our
185 // linearization is `disjoint`.
186 memref::resolveSourceIndicesExpandShape(
187 loc: storeOp.getLoc(), rewriter, expandShapeOp, indices, sourceIndices,
188 /*startsInbounds=*/true);
189
190 rewriter.replaceOpWithNewOp<AffineStoreOp>(
191 op: storeOp, args: storeOp.getValueToStore(), args: expandShapeOp.getViewSource(),
192 args&: sourceIndices);
193 return success();
194 }
195};
196
197struct AffineStoreOpOfCollapseShapeOpFolder final
198 : OpRewritePattern<AffineStoreOp> {
199 using Base::Base;
200
201 LogicalResult matchAndRewrite(AffineStoreOp storeOp,
202 PatternRewriter &rewriter) const override {
203 auto collapseShapeOp =
204 storeOp.getMemref().getDefiningOp<memref::CollapseShapeOp>();
205
206 if (!collapseShapeOp)
207 return failure();
208
209 // For affine ops, we need to apply the map to get the "actual" indices.
210 SmallVector<Value> indices;
211 expandToUnderlyingIndices(affineMap: storeOp.getAffineMap(), indices: storeOp.getIndices(),
212 loc: storeOp.getLoc(), rewriter, result&: indices);
213
214 SmallVector<Value> sourceIndices;
215 memref::resolveSourceIndicesCollapseShape(
216 loc: storeOp.getLoc(), rewriter, collapseShapeOp, indices, sourceIndices,
217 /*startsInbounds=*/true);
218
219 rewriter.replaceOpWithNewOp<AffineStoreOp>(
220 op: storeOp, args: storeOp.getValueToStore(), args: collapseShapeOp.getViewSource(),
221 args&: sourceIndices);
222 return success();
223 }
224};
225
226} // namespace
227
228void affine::populateAffineFoldMemRefAliasOpPatterns(
229 RewritePatternSet &patterns) {
230 patterns
231 .add<AffineLoadOpOfSubViewOpFolder, AffineLoadOpOfExpandShapeOpFolder,
232 AffineLoadOpOfCollapseShapeOpFolder, AffineStoreOpOfSubViewOpFolder,
233 AffineStoreOpOfExpandShapeOpFolder,
234 AffineStoreOpOfCollapseShapeOpFolder>(arg: patterns.getContext());
235}
236
237//===----------------------------------------------------------------------===//
238// Pass registration
239//===----------------------------------------------------------------------===//
240
241namespace {
242
243struct AffineFoldMemRefAliasOpsPass final
244 : public affine::impl::AffineFoldMemRefAliasOpsBase<
245 AffineFoldMemRefAliasOpsPass> {
246 void runOnOperation() override;
247};
248
249} // namespace
250
251void AffineFoldMemRefAliasOpsPass::runOnOperation() {
252 RewritePatternSet patterns(&getContext());
253 affine::populateAffineFoldMemRefAliasOpPatterns(patterns);
254 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
255}
256