| 1 | //===- FoldMemRefAliasOps.cpp - Fold memref alias ops for affine ops ------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This pass contains affine-specif versions of the folding patterns for |
| 10 | // memref.expand_shape, memref.collapse_shape, and memref.subview, since |
| 11 | // those all need affine-specific handling that won't fit a general interface. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #include "mlir/Dialect/Affine/IR/AffineOps.h" |
| 16 | #include "mlir/Dialect/Affine/Transforms/Passes.h" |
| 17 | #include "mlir/Dialect/Affine/ViewLikeInterfaceUtils.h" |
| 18 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 19 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 20 | #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h" |
| 21 | #include "mlir/IR/AffineMap.h" |
| 22 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 23 | |
| 24 | namespace mlir { |
| 25 | namespace affine { |
| 26 | #define GEN_PASS_DEF_AFFINEFOLDMEMREFALIASOPS |
| 27 | #include "mlir/Dialect/Affine/Transforms/Passes.h.inc" |
| 28 | } // namespace affine |
| 29 | } // namespace mlir |
| 30 | |
| 31 | using namespace mlir; |
| 32 | using namespace mlir::affine; |
| 33 | |
| 34 | //===----------------------------------------------------------------------===// |
| 35 | // Utility functions |
| 36 | //===----------------------------------------------------------------------===// |
| 37 | |
| 38 | /// Given an AffineMap and a list of indices, apply the map to get the |
| 39 | /// underlying indices (expanding the affine map). |
| 40 | static void expandToUnderlyingIndices(AffineMap affineMap, ValueRange indices, |
| 41 | Location loc, PatternRewriter &rewriter, |
| 42 | SmallVectorImpl<Value> &result) { |
| 43 | SmallVector<OpFoldResult> indicesOfr( |
| 44 | llvm::map_to_vector(C&: indices, F: [](Value v) -> OpFoldResult { return v; })); |
| 45 | for (unsigned i : llvm::seq(Begin: 0u, End: affineMap.getNumResults())) { |
| 46 | OpFoldResult ofr = affine::makeComposedFoldedAffineApply( |
| 47 | b&: rewriter, loc, map: affineMap.getSubMap(resultPos: {i}), operands: indicesOfr); |
| 48 | result.push_back(Elt: getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr)); |
| 49 | } |
| 50 | } |
| 51 | |
| 52 | //===----------------------------------------------------------------------===// |
| 53 | // Patterns |
| 54 | //===----------------------------------------------------------------------===// |
| 55 | |
| 56 | namespace { |
| 57 | |
| 58 | struct AffineLoadOpOfSubViewOpFolder final : OpRewritePattern<AffineLoadOp> { |
| 59 | using Base::Base; |
| 60 | |
| 61 | LogicalResult matchAndRewrite(AffineLoadOp loadOp, |
| 62 | PatternRewriter &rewriter) const override { |
| 63 | auto subViewOp = loadOp.getMemref().getDefiningOp<memref::SubViewOp>(); |
| 64 | |
| 65 | if (!subViewOp) |
| 66 | return rewriter.notifyMatchFailure(arg&: loadOp, msg: "not a subview producer" ); |
| 67 | |
| 68 | SmallVector<Value> indices; |
| 69 | expandToUnderlyingIndices(affineMap: loadOp.getAffineMap(), indices: loadOp.getIndices(), |
| 70 | loc: loadOp.getLoc(), rewriter, result&: indices); |
| 71 | |
| 72 | SmallVector<Value> sourceIndices; |
| 73 | affine::resolveIndicesIntoOpWithOffsetsAndStrides( |
| 74 | rewriter, loc: loadOp.getLoc(), mixedSourceOffsets: subViewOp.getMixedOffsets(), |
| 75 | mixedSourceStrides: subViewOp.getMixedStrides(), rankReducedDims: subViewOp.getDroppedDims(), consumerIndices: indices, |
| 76 | resolvedIndices&: sourceIndices); |
| 77 | |
| 78 | rewriter.replaceOpWithNewOp<AffineLoadOp>(op: loadOp, args: subViewOp.getSource(), |
| 79 | args&: sourceIndices); |
| 80 | return success(); |
| 81 | } |
| 82 | }; |
| 83 | |
| 84 | struct AffineLoadOpOfExpandShapeOpFolder final |
| 85 | : OpRewritePattern<AffineLoadOp> { |
| 86 | using Base::Base; |
| 87 | |
| 88 | LogicalResult matchAndRewrite(AffineLoadOp loadOp, |
| 89 | PatternRewriter &rewriter) const override { |
| 90 | auto expandShapeOp = |
| 91 | loadOp.getMemref().getDefiningOp<memref::ExpandShapeOp>(); |
| 92 | |
| 93 | if (!expandShapeOp) |
| 94 | return failure(); |
| 95 | |
| 96 | SmallVector<Value> indices; |
| 97 | expandToUnderlyingIndices(affineMap: loadOp.getAffineMap(), indices: loadOp.getIndices(), |
| 98 | loc: loadOp.getLoc(), rewriter, result&: indices); |
| 99 | |
| 100 | SmallVector<Value> sourceIndices; |
| 101 | // affine.load guarantees that indexes start inbounds, which impacts if our |
| 102 | // linearization is `disjoint`. |
| 103 | memref::resolveSourceIndicesExpandShape( |
| 104 | loc: loadOp.getLoc(), rewriter, expandShapeOp, indices, sourceIndices, |
| 105 | /*startsInbounds=*/true); |
| 106 | |
| 107 | rewriter.replaceOpWithNewOp<AffineLoadOp>( |
| 108 | op: loadOp, args: expandShapeOp.getViewSource(), args&: sourceIndices); |
| 109 | return success(); |
| 110 | } |
| 111 | }; |
| 112 | |
| 113 | struct AffineLoadOpOfCollapseShapeOpFolder final |
| 114 | : OpRewritePattern<AffineLoadOp> { |
| 115 | using Base::Base; |
| 116 | |
| 117 | LogicalResult matchAndRewrite(AffineLoadOp loadOp, |
| 118 | PatternRewriter &rewriter) const override { |
| 119 | auto collapseShapeOp = |
| 120 | loadOp.getMemref().getDefiningOp<memref::CollapseShapeOp>(); |
| 121 | |
| 122 | if (!collapseShapeOp) |
| 123 | return failure(); |
| 124 | |
| 125 | SmallVector<Value> indices; |
| 126 | expandToUnderlyingIndices(affineMap: loadOp.getAffineMap(), indices: loadOp.getIndices(), |
| 127 | loc: loadOp.getLoc(), rewriter, result&: indices); |
| 128 | |
| 129 | SmallVector<Value> sourceIndices; |
| 130 | memref::resolveSourceIndicesCollapseShape( |
| 131 | loc: loadOp.getLoc(), rewriter, collapseShapeOp, indices, sourceIndices, |
| 132 | /*startsInbounds=*/true); |
| 133 | |
| 134 | rewriter.replaceOpWithNewOp<AffineLoadOp>( |
| 135 | op: loadOp, args: collapseShapeOp.getViewSource(), args&: sourceIndices); |
| 136 | return success(); |
| 137 | } |
| 138 | }; |
| 139 | |
| 140 | struct AffineStoreOpOfSubViewOpFolder final : OpRewritePattern<AffineStoreOp> { |
| 141 | using Base::Base; |
| 142 | |
| 143 | LogicalResult matchAndRewrite(AffineStoreOp storeOp, |
| 144 | PatternRewriter &rewriter) const override { |
| 145 | auto subViewOp = storeOp.getMemref().getDefiningOp<memref::SubViewOp>(); |
| 146 | |
| 147 | if (!subViewOp) |
| 148 | return rewriter.notifyMatchFailure(arg&: storeOp, msg: "not a subview producer" ); |
| 149 | |
| 150 | // For affine ops, we need to apply the map to get the "actual" indices. |
| 151 | SmallVector<Value> indices; |
| 152 | expandToUnderlyingIndices(affineMap: storeOp.getAffineMap(), indices: storeOp.getIndices(), |
| 153 | loc: storeOp.getLoc(), rewriter, result&: indices); |
| 154 | |
| 155 | SmallVector<Value> sourceIndices; |
| 156 | affine::resolveIndicesIntoOpWithOffsetsAndStrides( |
| 157 | rewriter, loc: storeOp.getLoc(), mixedSourceOffsets: subViewOp.getMixedOffsets(), |
| 158 | mixedSourceStrides: subViewOp.getMixedStrides(), rankReducedDims: subViewOp.getDroppedDims(), consumerIndices: indices, |
| 159 | resolvedIndices&: sourceIndices); |
| 160 | |
| 161 | rewriter.replaceOpWithNewOp<AffineStoreOp>( |
| 162 | op: storeOp, args: storeOp.getValue(), args: subViewOp.getSource(), args&: sourceIndices); |
| 163 | return success(); |
| 164 | } |
| 165 | }; |
| 166 | |
| 167 | struct AffineStoreOpOfExpandShapeOpFolder final |
| 168 | : OpRewritePattern<AffineStoreOp> { |
| 169 | using Base::Base; |
| 170 | |
| 171 | LogicalResult matchAndRewrite(AffineStoreOp storeOp, |
| 172 | PatternRewriter &rewriter) const override { |
| 173 | auto expandShapeOp = |
| 174 | storeOp.getMemref().getDefiningOp<memref::ExpandShapeOp>(); |
| 175 | |
| 176 | if (!expandShapeOp) |
| 177 | return failure(); |
| 178 | |
| 179 | SmallVector<Value> indices; |
| 180 | expandToUnderlyingIndices(affineMap: storeOp.getAffineMap(), indices: storeOp.getIndices(), |
| 181 | loc: storeOp.getLoc(), rewriter, result&: indices); |
| 182 | |
| 183 | SmallVector<Value> sourceIndices; |
| 184 | // affine.store guarantees that indexes start inbounds, which impacts if our |
| 185 | // linearization is `disjoint`. |
| 186 | memref::resolveSourceIndicesExpandShape( |
| 187 | loc: storeOp.getLoc(), rewriter, expandShapeOp, indices, sourceIndices, |
| 188 | /*startsInbounds=*/true); |
| 189 | |
| 190 | rewriter.replaceOpWithNewOp<AffineStoreOp>( |
| 191 | op: storeOp, args: storeOp.getValueToStore(), args: expandShapeOp.getViewSource(), |
| 192 | args&: sourceIndices); |
| 193 | return success(); |
| 194 | } |
| 195 | }; |
| 196 | |
| 197 | struct AffineStoreOpOfCollapseShapeOpFolder final |
| 198 | : OpRewritePattern<AffineStoreOp> { |
| 199 | using Base::Base; |
| 200 | |
| 201 | LogicalResult matchAndRewrite(AffineStoreOp storeOp, |
| 202 | PatternRewriter &rewriter) const override { |
| 203 | auto collapseShapeOp = |
| 204 | storeOp.getMemref().getDefiningOp<memref::CollapseShapeOp>(); |
| 205 | |
| 206 | if (!collapseShapeOp) |
| 207 | return failure(); |
| 208 | |
| 209 | // For affine ops, we need to apply the map to get the "actual" indices. |
| 210 | SmallVector<Value> indices; |
| 211 | expandToUnderlyingIndices(affineMap: storeOp.getAffineMap(), indices: storeOp.getIndices(), |
| 212 | loc: storeOp.getLoc(), rewriter, result&: indices); |
| 213 | |
| 214 | SmallVector<Value> sourceIndices; |
| 215 | memref::resolveSourceIndicesCollapseShape( |
| 216 | loc: storeOp.getLoc(), rewriter, collapseShapeOp, indices, sourceIndices, |
| 217 | /*startsInbounds=*/true); |
| 218 | |
| 219 | rewriter.replaceOpWithNewOp<AffineStoreOp>( |
| 220 | op: storeOp, args: storeOp.getValueToStore(), args: collapseShapeOp.getViewSource(), |
| 221 | args&: sourceIndices); |
| 222 | return success(); |
| 223 | } |
| 224 | }; |
| 225 | |
| 226 | } // namespace |
| 227 | |
| 228 | void affine::populateAffineFoldMemRefAliasOpPatterns( |
| 229 | RewritePatternSet &patterns) { |
| 230 | patterns |
| 231 | .add<AffineLoadOpOfSubViewOpFolder, AffineLoadOpOfExpandShapeOpFolder, |
| 232 | AffineLoadOpOfCollapseShapeOpFolder, AffineStoreOpOfSubViewOpFolder, |
| 233 | AffineStoreOpOfExpandShapeOpFolder, |
| 234 | AffineStoreOpOfCollapseShapeOpFolder>(arg: patterns.getContext()); |
| 235 | } |
| 236 | |
| 237 | //===----------------------------------------------------------------------===// |
| 238 | // Pass registration |
| 239 | //===----------------------------------------------------------------------===// |
| 240 | |
| 241 | namespace { |
| 242 | |
| 243 | struct AffineFoldMemRefAliasOpsPass final |
| 244 | : public affine::impl::AffineFoldMemRefAliasOpsBase< |
| 245 | AffineFoldMemRefAliasOpsPass> { |
| 246 | void runOnOperation() override; |
| 247 | }; |
| 248 | |
| 249 | } // namespace |
| 250 | |
| 251 | void AffineFoldMemRefAliasOpsPass::runOnOperation() { |
| 252 | RewritePatternSet patterns(&getContext()); |
| 253 | affine::populateAffineFoldMemRefAliasOpPatterns(patterns); |
| 254 | (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)); |
| 255 | } |
| 256 | |