1//===- DecomposeMemRefs.cpp - Decompose memrefs pass implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements decompose memrefs pass.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Affine/IR/AffineOps.h"
14#include "mlir/Dialect/GPU/IR/GPUDialect.h"
15#include "mlir/Dialect/GPU/Transforms/Passes.h"
16#include "mlir/Dialect/MemRef/IR/MemRef.h"
17#include "mlir/Dialect/Utils/IndexingUtils.h"
18#include "mlir/Dialect/Utils/StaticValueUtils.h"
19#include "mlir/IR/AffineExpr.h"
20#include "mlir/IR/Builders.h"
21#include "mlir/IR/PatternMatch.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23
24namespace mlir {
25#define GEN_PASS_DEF_GPUDECOMPOSEMEMREFSPASS
26#include "mlir/Dialect/GPU/Transforms/Passes.h.inc"
27} // namespace mlir
28
29using namespace mlir;
30
31static MemRefType inferCastResultType(Value source, OpFoldResult offset) {
32 auto sourceType = cast<BaseMemRefType>(Val: source.getType());
33 SmallVector<int64_t> staticOffsets;
34 SmallVector<Value> dynamicOffsets;
35 dispatchIndexOpFoldResults(ofrs: offset, dynamicVec&: dynamicOffsets, staticVec&: staticOffsets);
36 auto stridedLayout =
37 StridedLayoutAttr::get(context: source.getContext(), offset: staticOffsets.front(), strides: {});
38 return MemRefType::get(shape: {}, elementType: sourceType.getElementType(), layout: stridedLayout,
39 memorySpace: sourceType.getMemorySpace());
40}
41
42static void setInsertionPointToStart(OpBuilder &builder, Value val) {
43 if (auto *parentOp = val.getDefiningOp()) {
44 builder.setInsertionPointAfter(parentOp);
45 } else {
46 builder.setInsertionPointToStart(val.getParentBlock());
47 }
48}
49
50static bool isInsideLaunch(Operation *op) {
51 return op->getParentOfType<gpu::LaunchOp>();
52}
53
54static std::tuple<Value, OpFoldResult, SmallVector<OpFoldResult>>
55getFlatOffsetAndStrides(OpBuilder &rewriter, Location loc, Value source,
56 ArrayRef<OpFoldResult> subOffsets,
57 ArrayRef<OpFoldResult> subStrides = {}) {
58 auto sourceType = cast<MemRefType>(Val: source.getType());
59 auto sourceRank = static_cast<unsigned>(sourceType.getRank());
60
61 memref::ExtractStridedMetadataOp newExtractStridedMetadata;
62 {
63 OpBuilder::InsertionGuard g(rewriter);
64 setInsertionPointToStart(builder&: rewriter, val: source);
65 newExtractStridedMetadata =
66 memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source);
67 }
68
69 auto &&[sourceStrides, sourceOffset] = sourceType.getStridesAndOffset();
70
71 auto getDim = [&](int64_t dim, Value dimVal) -> OpFoldResult {
72 return ShapedType::isDynamic(dValue: dim) ? getAsOpFoldResult(val: dimVal)
73 : rewriter.getIndexAttr(value: dim);
74 };
75
76 OpFoldResult origOffset =
77 getDim(sourceOffset, newExtractStridedMetadata.getOffset());
78 ValueRange sourceStridesVals = newExtractStridedMetadata.getStrides();
79
80 SmallVector<OpFoldResult> origStrides;
81 origStrides.reserve(N: sourceRank);
82
83 SmallVector<OpFoldResult> strides;
84 strides.reserve(N: sourceRank);
85
86 AffineExpr s0 = rewriter.getAffineSymbolExpr(position: 0);
87 AffineExpr s1 = rewriter.getAffineSymbolExpr(position: 1);
88 for (auto i : llvm::seq(Begin: 0u, End: sourceRank)) {
89 OpFoldResult origStride = getDim(sourceStrides[i], sourceStridesVals[i]);
90
91 if (!subStrides.empty()) {
92 strides.push_back(Elt: affine::makeComposedFoldedAffineApply(
93 b&: rewriter, loc, expr: s0 * s1, operands: {subStrides[i], origStride}));
94 }
95
96 origStrides.emplace_back(Args&: origStride);
97 }
98
99 auto &&[expr, values] =
100 computeLinearIndex(sourceOffset: origOffset, strides: origStrides, indices: subOffsets);
101 OpFoldResult finalOffset =
102 affine::makeComposedFoldedAffineApply(b&: rewriter, loc, expr, operands: values);
103 return {newExtractStridedMetadata.getBaseBuffer(), finalOffset, strides};
104}
105
106static Value getFlatMemref(OpBuilder &rewriter, Location loc, Value source,
107 ValueRange offsets) {
108 SmallVector<OpFoldResult> offsetsTemp = getAsOpFoldResult(values: offsets);
109 auto &&[base, offset, ignore] =
110 getFlatOffsetAndStrides(rewriter, loc, source, subOffsets: offsetsTemp);
111 MemRefType retType = inferCastResultType(source: base, offset);
112 return memref::ReinterpretCastOp::create(builder&: rewriter, location: loc, resultType: retType, source: base, offset,
113 sizes: ArrayRef<OpFoldResult>(),
114 strides: ArrayRef<OpFoldResult>());
115}
116
117static bool needFlatten(Value val) {
118 auto type = cast<MemRefType>(Val: val.getType());
119 return type.getRank() != 0;
120}
121
122static bool checkLayout(Value val) {
123 auto type = cast<MemRefType>(Val: val.getType());
124 return type.getLayout().isIdentity() ||
125 isa<StridedLayoutAttr>(Val: type.getLayout());
126}
127
128namespace {
129struct FlattenLoad : public OpRewritePattern<memref::LoadOp> {
130 using Base::Base;
131
132 LogicalResult matchAndRewrite(memref::LoadOp op,
133 PatternRewriter &rewriter) const override {
134 if (!isInsideLaunch(op))
135 return rewriter.notifyMatchFailure(arg&: op, msg: "not inside gpu.launch");
136
137 Value memref = op.getMemref();
138 if (!needFlatten(val: memref))
139 return rewriter.notifyMatchFailure(arg&: op, msg: "nothing to do");
140
141 if (!checkLayout(val: memref))
142 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported layout");
143
144 Location loc = op.getLoc();
145 Value flatMemref = getFlatMemref(rewriter, loc, source: memref, offsets: op.getIndices());
146 rewriter.replaceOpWithNewOp<memref::LoadOp>(
147 op, args&: flatMemref, args: ValueRange{}, args: op.getNontemporalAttr(),
148 args: op.getAlignmentAttr(), args: op.getInvariantAttr());
149 return success();
150 }
151};
152
153struct FlattenStore : public OpRewritePattern<memref::StoreOp> {
154 using Base::Base;
155
156 LogicalResult matchAndRewrite(memref::StoreOp op,
157 PatternRewriter &rewriter) const override {
158 if (!isInsideLaunch(op))
159 return rewriter.notifyMatchFailure(arg&: op, msg: "not inside gpu.launch");
160
161 Value memref = op.getMemref();
162 if (!needFlatten(val: memref))
163 return rewriter.notifyMatchFailure(arg&: op, msg: "nothing to do");
164
165 if (!checkLayout(val: memref))
166 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported layout");
167
168 Location loc = op.getLoc();
169 Value flatMemref = getFlatMemref(rewriter, loc, source: memref, offsets: op.getIndices());
170 Value value = op.getValue();
171 rewriter.replaceOpWithNewOp<memref::StoreOp>(
172 op, args&: value, args&: flatMemref, args: ValueRange{}, args: op.getNontemporalAttr(),
173 args: op.getAlignmentAttr());
174 return success();
175 }
176};
177
178struct FlattenSubview : public OpRewritePattern<memref::SubViewOp> {
179 using Base::Base;
180
181 LogicalResult matchAndRewrite(memref::SubViewOp op,
182 PatternRewriter &rewriter) const override {
183 if (!isInsideLaunch(op))
184 return rewriter.notifyMatchFailure(arg&: op, msg: "not inside gpu.launch");
185
186 Value memref = op.getSource();
187 if (!needFlatten(val: memref))
188 return rewriter.notifyMatchFailure(arg&: op, msg: "nothing to do");
189
190 if (!checkLayout(val: memref))
191 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported layout");
192
193 Location loc = op.getLoc();
194 SmallVector<OpFoldResult> subOffsets = op.getMixedOffsets();
195 SmallVector<OpFoldResult> subSizes = op.getMixedSizes();
196 SmallVector<OpFoldResult> subStrides = op.getMixedStrides();
197 auto &&[base, finalOffset, strides] =
198 getFlatOffsetAndStrides(rewriter, loc, source: memref, subOffsets, subStrides);
199
200 auto srcType = cast<MemRefType>(Val: memref.getType());
201 auto resultType = cast<MemRefType>(Val: op.getType());
202 unsigned subRank = static_cast<unsigned>(resultType.getRank());
203
204 llvm::SmallBitVector droppedDims = op.getDroppedDims();
205
206 SmallVector<OpFoldResult> finalSizes;
207 finalSizes.reserve(N: subRank);
208
209 SmallVector<OpFoldResult> finalStrides;
210 finalStrides.reserve(N: subRank);
211
212 for (auto i : llvm::seq(Begin: 0u, End: static_cast<unsigned>(srcType.getRank()))) {
213 if (droppedDims.test(Idx: i))
214 continue;
215
216 finalSizes.push_back(Elt: subSizes[i]);
217 finalStrides.push_back(Elt: strides[i]);
218 }
219
220 resultType = updateTypeFromMetadata(type: resultType, offset: finalOffset, sizes: finalSizes,
221 strides: finalStrides);
222 auto flattenedSubview = memref::ReinterpretCastOp::create(
223 builder&: rewriter, location: op.getLoc(), resultType, source: base, offset: finalOffset, sizes: finalSizes,
224 strides: finalStrides);
225 if (resultType == op.getType()) {
226 rewriter.replaceOp(op, newOp: flattenedSubview);
227 return success();
228 }
229 // Preserve the original result type expected by existing users.
230 rewriter.replaceOpWithNewOp<memref::CastOp>(op, args: op.getType(),
231 args&: flattenedSubview);
232 return success();
233 }
234};
235
236struct GpuDecomposeMemrefsPass
237 : public impl::GpuDecomposeMemrefsPassBase<GpuDecomposeMemrefsPass> {
238
239 void runOnOperation() override {
240 RewritePatternSet patterns(&getContext());
241
242 populateGpuDecomposeMemrefsPatterns(patterns);
243
244 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
245 return signalPassFailure();
246 }
247};
248
249} // namespace
250
251void mlir::populateGpuDecomposeMemrefsPatterns(RewritePatternSet &patterns) {
252 patterns.insert<FlattenLoad, FlattenStore, FlattenSubview>(
253 arg: patterns.getContext());
254}
255