1//===- MaskedloadToLoad.cpp - Lowers maskedload to load -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/AMDGPU/Transforms/Passes.h"
10
11#include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h"
12#include "mlir/Dialect/Arith/IR/Arith.h"
13#include "mlir/Dialect/MemRef/IR/MemRef.h"
14#include "mlir/Dialect/MemRef/Utils/MemRefUtils.h"
15#include "mlir/Dialect/SCF/IR/SCF.h"
16#include "mlir/Dialect/Vector/IR/VectorOps.h"
17#include "mlir/IR/BuiltinTypes.h"
18#include "mlir/IR/OpDefinition.h"
19#include "mlir/IR/PatternMatch.h"
20#include "mlir/IR/TypeUtilities.h"
21#include "mlir/Support/LogicalResult.h"
22#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
23#include "llvm/Support/MathExtras.h"
24
25namespace mlir::amdgpu {
26#define GEN_PASS_DEF_AMDGPUMASKEDLOADTOLOADPASS
27#include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc"
28} // namespace mlir::amdgpu
29
30using namespace mlir;
31using namespace mlir::amdgpu;
32
33/// This pattern supports lowering of: `vector.maskedload` to `vector.load`
34/// and `arith.select` if the memref is in buffer address space.
35static LogicalResult hasBufferAddressSpace(Type type) {
36 auto memRefType = dyn_cast<MemRefType>(Val&: type);
37 if (!memRefType)
38 return failure();
39
40 Attribute addrSpace = memRefType.getMemorySpace();
41 if (!isa_and_nonnull<amdgpu::AddressSpaceAttr>(Val: addrSpace))
42 return failure();
43
44 if (dyn_cast<amdgpu::AddressSpaceAttr>(Val&: addrSpace).getValue() !=
45 amdgpu::AddressSpace::FatRawBuffer)
46 return failure();
47
48 return success();
49}
50
51static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc,
52 vector::MaskedLoadOp maskedOp,
53 bool passthru) {
54 VectorType vectorType = maskedOp.getVectorType();
55 Value load = vector::LoadOp::create(
56 builder, location: loc, resultType: vectorType, base: maskedOp.getBase(), indices: maskedOp.getIndices());
57 if (passthru)
58 load = arith::SelectOp::create(builder, location: loc, result: vectorType, condition: maskedOp.getMask(),
59 true_value: load, false_value: maskedOp.getPassThru());
60 return load;
61}
62
63/// Check if the given value comes from a broadcasted i1 condition.
64static FailureOr<Value> matchFullMask(OpBuilder &b, Value val) {
65 auto broadcastOp = val.getDefiningOp<vector::BroadcastOp>();
66 if (!broadcastOp)
67 return failure();
68 if (isa<VectorType>(Val: broadcastOp.getSourceType()))
69 return failure();
70 return broadcastOp.getSource();
71}
72
73static constexpr char kMaskedloadNeedsMask[] =
74 "amdgpu.buffer_maskedload_needs_mask";
75
76namespace {
77
78struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> {
79 using Base::Base;
80
81 LogicalResult matchAndRewrite(vector::MaskedLoadOp maskedOp,
82 PatternRewriter &rewriter) const override {
83 if (maskedOp->hasDiscardableAttr(name: kMaskedloadNeedsMask))
84 return rewriter.notifyMatchFailure(arg&: maskedOp, msg: "already rewritten");
85
86 if (failed(Result: hasBufferAddressSpace(type: maskedOp.getBase().getType()))) {
87 return rewriter.notifyMatchFailure(
88 arg&: maskedOp, msg: "isn't a load from a fat buffer resource");
89 }
90
91 // Check if this is either a full inbounds load or an empty, oob load. If
92 // so, take the fast path and don't generate an if condition, because we
93 // know doing the oob load is always safe.
94 if (succeeded(Result: matchFullMask(b&: rewriter, val: maskedOp.getMask()))) {
95 Value load = createVectorLoadForMaskedLoad(builder&: rewriter, loc: maskedOp.getLoc(),
96 maskedOp, /*passthru=*/true);
97 rewriter.replaceOp(op: maskedOp, newValues: load);
98 return success();
99 }
100
101 Location loc = maskedOp.getLoc();
102 Value src = maskedOp.getBase();
103
104 VectorType vectorType = maskedOp.getVectorType();
105 int64_t vectorSize = vectorType.getNumElements();
106 int64_t elementBitWidth = vectorType.getElementTypeBitWidth();
107 SmallVector<OpFoldResult> indices = maskedOp.getIndices();
108
109 auto stridedMetadata =
110 memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source: src);
111 SmallVector<OpFoldResult> strides =
112 stridedMetadata.getConstifiedMixedStrides();
113 SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes();
114 OpFoldResult offset = stridedMetadata.getConstifiedMixedOffset();
115 memref::LinearizedMemRefInfo linearizedInfo;
116 OpFoldResult linearizedIndices;
117 std::tie(args&: linearizedInfo, args&: linearizedIndices) =
118 memref::getLinearizedMemRefOffsetAndSize(builder&: rewriter, loc, srcBits: elementBitWidth,
119 dstBits: elementBitWidth, offset, sizes,
120 strides, indices);
121
122 // delta = bufferSize - linearizedOffset
123 Value vectorSizeOffset =
124 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: vectorSize);
125 Value linearIndex =
126 getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: linearizedIndices);
127 Value totalSize = getValueOrCreateConstantIndexOp(
128 b&: rewriter, loc, ofr: linearizedInfo.linearizedSize);
129 Value delta = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: totalSize, rhs: linearIndex);
130
131 // 1) check if delta < vectorSize
132 Value isOutofBounds = arith::CmpIOp::create(
133 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ult, lhs: delta, rhs: vectorSizeOffset);
134
135 // 2) check if (detla % elements_per_word != 0)
136 Value elementsPerWord = arith::ConstantIndexOp::create(
137 builder&: rewriter, location: loc, value: llvm::divideCeil(Numerator: 32, Denominator: elementBitWidth));
138 Value isNotWordAligned = arith::CmpIOp::create(
139 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ne,
140 lhs: arith::RemUIOp::create(builder&: rewriter, location: loc, lhs: delta, rhs: elementsPerWord),
141 rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0));
142
143 // We take the fallback of maskedload default lowering only it is both
144 // out-of-bounds and not word aligned. The fallback ensures correct results
145 // when loading at the boundary of the buffer since buffer load returns
146 // inconsistent zeros for the whole word when boundary is crossed.
147 Value ifCondition =
148 arith::AndIOp::create(builder&: rewriter, location: loc, lhs: isOutofBounds, rhs: isNotWordAligned);
149
150 auto thenBuilder = [&](OpBuilder &builder, Location loc) {
151 Operation *read = builder.clone(op&: *maskedOp.getOperation());
152 read->setDiscardableAttr(name: kMaskedloadNeedsMask, value: builder.getUnitAttr());
153 Value readResult = read->getResult(idx: 0);
154 scf::YieldOp::create(builder, location: loc, results: readResult);
155 };
156
157 auto elseBuilder = [&](OpBuilder &builder, Location loc) {
158 Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp,
159 /*passthru=*/true);
160 scf::YieldOp::create(builder&: rewriter, location: loc, results: res);
161 };
162
163 auto ifOp =
164 scf::IfOp::create(builder&: rewriter, location: loc, cond: ifCondition, thenBuilder, elseBuilder);
165
166 rewriter.replaceOp(op: maskedOp, newOp: ifOp);
167
168 return success();
169 }
170};
171
172struct FullMaskedLoadToConditionalLoad
173 : OpRewritePattern<vector::MaskedLoadOp> {
174 using Base::Base;
175
176 LogicalResult matchAndRewrite(vector::MaskedLoadOp loadOp,
177 PatternRewriter &rewriter) const override {
178 if (succeeded(Result: hasBufferAddressSpace(type: loadOp.getBase().getType())))
179 return rewriter.notifyMatchFailure(
180 arg&: loadOp, msg: "buffer loads are handled by a more specialized pattern");
181
182 FailureOr<Value> maybeCond = matchFullMask(b&: rewriter, val: loadOp.getMask());
183 if (failed(Result: maybeCond)) {
184 return rewriter.notifyMatchFailure(arg&: loadOp,
185 msg: "isn't loading a broadcasted scalar");
186 }
187
188 Value cond = maybeCond.value();
189 auto trueBuilder = [&](OpBuilder &builder, Location loc) {
190 Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp: loadOp,
191 /*passthru=*/false);
192 scf::YieldOp::create(builder&: rewriter, location: loc, results: res);
193 };
194 auto falseBuilder = [&](OpBuilder &builder, Location loc) {
195 scf::YieldOp::create(builder&: rewriter, location: loc, results: loadOp.getPassThru());
196 };
197 auto ifOp = scf::IfOp::create(builder&: rewriter, location: loadOp.getLoc(), cond, thenBuilder: trueBuilder,
198 elseBuilder: falseBuilder);
199 rewriter.replaceOp(op: loadOp, newOp: ifOp);
200 return success();
201 }
202};
203
204struct FullMaskedStoreToConditionalStore
205 : OpRewritePattern<vector::MaskedStoreOp> {
206 using Base::Base;
207
208 LogicalResult matchAndRewrite(vector::MaskedStoreOp storeOp,
209 PatternRewriter &rewriter) const override {
210 // A condition-free implementation of fully masked stores requires
211 // 1) an accessor for the num_records field on buffer resources/fat pointers
212 // 2) knowledge that said field will always be set accurately - that is,
213 // that writes to x < num_records of offset wouldn't trap, which is
214 // something a pattern user would need to assert or we'd need to prove.
215 //
216 // Therefore, conditional stores to buffers still go down this path at
217 // present.
218
219 FailureOr<Value> maybeCond = matchFullMask(b&: rewriter, val: storeOp.getMask());
220 if (failed(Result: maybeCond)) {
221 return failure();
222 }
223 Value cond = maybeCond.value();
224
225 auto trueBuilder = [&](OpBuilder &builder, Location loc) {
226 vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: storeOp.getValueToStore(),
227 base: storeOp.getBase(), indices: storeOp.getIndices());
228 scf::YieldOp::create(builder&: rewriter, location: loc);
229 };
230 auto ifOp =
231 scf::IfOp::create(builder&: rewriter, location: storeOp.getLoc(), cond, thenBuilder: trueBuilder);
232 rewriter.replaceOp(op: storeOp, newOp: ifOp);
233 return success();
234 }
235};
236
237} // namespace
238
239void mlir::amdgpu::populateAmdgpuMaskedloadToLoadPatterns(
240 RewritePatternSet &patterns, PatternBenefit benefit) {
241 patterns.add<MaskedLoadLowering, FullMaskedLoadToConditionalLoad,
242 FullMaskedStoreToConditionalStore>(arg: patterns.getContext(),
243 args&: benefit);
244}
245
246struct AmdgpuMaskedloadToLoadPass final
247 : amdgpu::impl::AmdgpuMaskedloadToLoadPassBase<AmdgpuMaskedloadToLoadPass> {
248 void runOnOperation() override {
249 RewritePatternSet patterns(&getContext());
250 populateAmdgpuMaskedloadToLoadPatterns(patterns);
251 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)))) {
252 return signalPassFailure();
253 }
254 }
255};
256