| 1 | //===- MaskedloadToLoad.cpp - Lowers maskedload to load -------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/AMDGPU/Transforms/Passes.h" |
| 10 | |
| 11 | #include "mlir/Dialect/AMDGPU/IR/AMDGPUDialect.h" |
| 12 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 13 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 14 | #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h" |
| 15 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 16 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 17 | #include "mlir/IR/BuiltinTypes.h" |
| 18 | #include "mlir/IR/OpDefinition.h" |
| 19 | #include "mlir/IR/PatternMatch.h" |
| 20 | #include "mlir/IR/TypeUtilities.h" |
| 21 | #include "mlir/Support/LogicalResult.h" |
| 22 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 23 | #include "llvm/Support/MathExtras.h" |
| 24 | |
| 25 | namespace mlir::amdgpu { |
| 26 | #define GEN_PASS_DEF_AMDGPUMASKEDLOADTOLOADPASS |
| 27 | #include "mlir/Dialect/AMDGPU/Transforms/Passes.h.inc" |
| 28 | } // namespace mlir::amdgpu |
| 29 | |
| 30 | using namespace mlir; |
| 31 | using namespace mlir::amdgpu; |
| 32 | |
| 33 | /// This pattern supports lowering of: `vector.maskedload` to `vector.load` |
| 34 | /// and `arith.select` if the memref is in buffer address space. |
| 35 | static LogicalResult hasBufferAddressSpace(Type type) { |
| 36 | auto memRefType = dyn_cast<MemRefType>(Val&: type); |
| 37 | if (!memRefType) |
| 38 | return failure(); |
| 39 | |
| 40 | Attribute addrSpace = memRefType.getMemorySpace(); |
| 41 | if (!isa_and_nonnull<amdgpu::AddressSpaceAttr>(Val: addrSpace)) |
| 42 | return failure(); |
| 43 | |
| 44 | if (dyn_cast<amdgpu::AddressSpaceAttr>(Val&: addrSpace).getValue() != |
| 45 | amdgpu::AddressSpace::FatRawBuffer) |
| 46 | return failure(); |
| 47 | |
| 48 | return success(); |
| 49 | } |
| 50 | |
| 51 | static Value createVectorLoadForMaskedLoad(OpBuilder &builder, Location loc, |
| 52 | vector::MaskedLoadOp maskedOp, |
| 53 | bool passthru) { |
| 54 | VectorType vectorType = maskedOp.getVectorType(); |
| 55 | Value load = vector::LoadOp::create( |
| 56 | builder, location: loc, resultType: vectorType, base: maskedOp.getBase(), indices: maskedOp.getIndices()); |
| 57 | if (passthru) |
| 58 | load = arith::SelectOp::create(builder, location: loc, result: vectorType, condition: maskedOp.getMask(), |
| 59 | true_value: load, false_value: maskedOp.getPassThru()); |
| 60 | return load; |
| 61 | } |
| 62 | |
| 63 | /// Check if the given value comes from a broadcasted i1 condition. |
| 64 | static FailureOr<Value> matchFullMask(OpBuilder &b, Value val) { |
| 65 | auto broadcastOp = val.getDefiningOp<vector::BroadcastOp>(); |
| 66 | if (!broadcastOp) |
| 67 | return failure(); |
| 68 | if (isa<VectorType>(Val: broadcastOp.getSourceType())) |
| 69 | return failure(); |
| 70 | return broadcastOp.getSource(); |
| 71 | } |
| 72 | |
| 73 | static constexpr char kMaskedloadNeedsMask[] = |
| 74 | "amdgpu.buffer_maskedload_needs_mask" ; |
| 75 | |
| 76 | namespace { |
| 77 | |
| 78 | struct MaskedLoadLowering final : OpRewritePattern<vector::MaskedLoadOp> { |
| 79 | using Base::Base; |
| 80 | |
| 81 | LogicalResult matchAndRewrite(vector::MaskedLoadOp maskedOp, |
| 82 | PatternRewriter &rewriter) const override { |
| 83 | if (maskedOp->hasDiscardableAttr(name: kMaskedloadNeedsMask)) |
| 84 | return rewriter.notifyMatchFailure(arg&: maskedOp, msg: "already rewritten" ); |
| 85 | |
| 86 | if (failed(Result: hasBufferAddressSpace(type: maskedOp.getBase().getType()))) { |
| 87 | return rewriter.notifyMatchFailure( |
| 88 | arg&: maskedOp, msg: "isn't a load from a fat buffer resource" ); |
| 89 | } |
| 90 | |
| 91 | // Check if this is either a full inbounds load or an empty, oob load. If |
| 92 | // so, take the fast path and don't generate an if condition, because we |
| 93 | // know doing the oob load is always safe. |
| 94 | if (succeeded(Result: matchFullMask(b&: rewriter, val: maskedOp.getMask()))) { |
| 95 | Value load = createVectorLoadForMaskedLoad(builder&: rewriter, loc: maskedOp.getLoc(), |
| 96 | maskedOp, /*passthru=*/true); |
| 97 | rewriter.replaceOp(op: maskedOp, newValues: load); |
| 98 | return success(); |
| 99 | } |
| 100 | |
| 101 | Location loc = maskedOp.getLoc(); |
| 102 | Value src = maskedOp.getBase(); |
| 103 | |
| 104 | VectorType vectorType = maskedOp.getVectorType(); |
| 105 | int64_t vectorSize = vectorType.getNumElements(); |
| 106 | int64_t elementBitWidth = vectorType.getElementTypeBitWidth(); |
| 107 | SmallVector<OpFoldResult> indices = maskedOp.getIndices(); |
| 108 | |
| 109 | auto stridedMetadata = |
| 110 | memref::ExtractStridedMetadataOp::create(builder&: rewriter, location: loc, source: src); |
| 111 | SmallVector<OpFoldResult> strides = |
| 112 | stridedMetadata.getConstifiedMixedStrides(); |
| 113 | SmallVector<OpFoldResult> sizes = stridedMetadata.getConstifiedMixedSizes(); |
| 114 | OpFoldResult offset = stridedMetadata.getConstifiedMixedOffset(); |
| 115 | memref::LinearizedMemRefInfo linearizedInfo; |
| 116 | OpFoldResult linearizedIndices; |
| 117 | std::tie(args&: linearizedInfo, args&: linearizedIndices) = |
| 118 | memref::getLinearizedMemRefOffsetAndSize(builder&: rewriter, loc, srcBits: elementBitWidth, |
| 119 | dstBits: elementBitWidth, offset, sizes, |
| 120 | strides, indices); |
| 121 | |
| 122 | // delta = bufferSize - linearizedOffset |
| 123 | Value vectorSizeOffset = |
| 124 | arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: vectorSize); |
| 125 | Value linearIndex = |
| 126 | getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: linearizedIndices); |
| 127 | Value totalSize = getValueOrCreateConstantIndexOp( |
| 128 | b&: rewriter, loc, ofr: linearizedInfo.linearizedSize); |
| 129 | Value delta = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: totalSize, rhs: linearIndex); |
| 130 | |
| 131 | // 1) check if delta < vectorSize |
| 132 | Value isOutofBounds = arith::CmpIOp::create( |
| 133 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ult, lhs: delta, rhs: vectorSizeOffset); |
| 134 | |
| 135 | // 2) check if (detla % elements_per_word != 0) |
| 136 | Value elementsPerWord = arith::ConstantIndexOp::create( |
| 137 | builder&: rewriter, location: loc, value: llvm::divideCeil(Numerator: 32, Denominator: elementBitWidth)); |
| 138 | Value isNotWordAligned = arith::CmpIOp::create( |
| 139 | builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ne, |
| 140 | lhs: arith::RemUIOp::create(builder&: rewriter, location: loc, lhs: delta, rhs: elementsPerWord), |
| 141 | rhs: arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0)); |
| 142 | |
| 143 | // We take the fallback of maskedload default lowering only it is both |
| 144 | // out-of-bounds and not word aligned. The fallback ensures correct results |
| 145 | // when loading at the boundary of the buffer since buffer load returns |
| 146 | // inconsistent zeros for the whole word when boundary is crossed. |
| 147 | Value ifCondition = |
| 148 | arith::AndIOp::create(builder&: rewriter, location: loc, lhs: isOutofBounds, rhs: isNotWordAligned); |
| 149 | |
| 150 | auto thenBuilder = [&](OpBuilder &builder, Location loc) { |
| 151 | Operation *read = builder.clone(op&: *maskedOp.getOperation()); |
| 152 | read->setDiscardableAttr(name: kMaskedloadNeedsMask, value: builder.getUnitAttr()); |
| 153 | Value readResult = read->getResult(idx: 0); |
| 154 | scf::YieldOp::create(builder, location: loc, results: readResult); |
| 155 | }; |
| 156 | |
| 157 | auto elseBuilder = [&](OpBuilder &builder, Location loc) { |
| 158 | Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp, |
| 159 | /*passthru=*/true); |
| 160 | scf::YieldOp::create(builder&: rewriter, location: loc, results: res); |
| 161 | }; |
| 162 | |
| 163 | auto ifOp = |
| 164 | scf::IfOp::create(builder&: rewriter, location: loc, cond: ifCondition, thenBuilder, elseBuilder); |
| 165 | |
| 166 | rewriter.replaceOp(op: maskedOp, newOp: ifOp); |
| 167 | |
| 168 | return success(); |
| 169 | } |
| 170 | }; |
| 171 | |
| 172 | struct FullMaskedLoadToConditionalLoad |
| 173 | : OpRewritePattern<vector::MaskedLoadOp> { |
| 174 | using Base::Base; |
| 175 | |
| 176 | LogicalResult matchAndRewrite(vector::MaskedLoadOp loadOp, |
| 177 | PatternRewriter &rewriter) const override { |
| 178 | if (succeeded(Result: hasBufferAddressSpace(type: loadOp.getBase().getType()))) |
| 179 | return rewriter.notifyMatchFailure( |
| 180 | arg&: loadOp, msg: "buffer loads are handled by a more specialized pattern" ); |
| 181 | |
| 182 | FailureOr<Value> maybeCond = matchFullMask(b&: rewriter, val: loadOp.getMask()); |
| 183 | if (failed(Result: maybeCond)) { |
| 184 | return rewriter.notifyMatchFailure(arg&: loadOp, |
| 185 | msg: "isn't loading a broadcasted scalar" ); |
| 186 | } |
| 187 | |
| 188 | Value cond = maybeCond.value(); |
| 189 | auto trueBuilder = [&](OpBuilder &builder, Location loc) { |
| 190 | Value res = createVectorLoadForMaskedLoad(builder, loc, maskedOp: loadOp, |
| 191 | /*passthru=*/false); |
| 192 | scf::YieldOp::create(builder&: rewriter, location: loc, results: res); |
| 193 | }; |
| 194 | auto falseBuilder = [&](OpBuilder &builder, Location loc) { |
| 195 | scf::YieldOp::create(builder&: rewriter, location: loc, results: loadOp.getPassThru()); |
| 196 | }; |
| 197 | auto ifOp = scf::IfOp::create(builder&: rewriter, location: loadOp.getLoc(), cond, thenBuilder: trueBuilder, |
| 198 | elseBuilder: falseBuilder); |
| 199 | rewriter.replaceOp(op: loadOp, newOp: ifOp); |
| 200 | return success(); |
| 201 | } |
| 202 | }; |
| 203 | |
| 204 | struct FullMaskedStoreToConditionalStore |
| 205 | : OpRewritePattern<vector::MaskedStoreOp> { |
| 206 | using Base::Base; |
| 207 | |
| 208 | LogicalResult matchAndRewrite(vector::MaskedStoreOp storeOp, |
| 209 | PatternRewriter &rewriter) const override { |
| 210 | // A condition-free implementation of fully masked stores requires |
| 211 | // 1) an accessor for the num_records field on buffer resources/fat pointers |
| 212 | // 2) knowledge that said field will always be set accurately - that is, |
| 213 | // that writes to x < num_records of offset wouldn't trap, which is |
| 214 | // something a pattern user would need to assert or we'd need to prove. |
| 215 | // |
| 216 | // Therefore, conditional stores to buffers still go down this path at |
| 217 | // present. |
| 218 | |
| 219 | FailureOr<Value> maybeCond = matchFullMask(b&: rewriter, val: storeOp.getMask()); |
| 220 | if (failed(Result: maybeCond)) { |
| 221 | return failure(); |
| 222 | } |
| 223 | Value cond = maybeCond.value(); |
| 224 | |
| 225 | auto trueBuilder = [&](OpBuilder &builder, Location loc) { |
| 226 | vector::StoreOp::create(builder&: rewriter, location: loc, valueToStore: storeOp.getValueToStore(), |
| 227 | base: storeOp.getBase(), indices: storeOp.getIndices()); |
| 228 | scf::YieldOp::create(builder&: rewriter, location: loc); |
| 229 | }; |
| 230 | auto ifOp = |
| 231 | scf::IfOp::create(builder&: rewriter, location: storeOp.getLoc(), cond, thenBuilder: trueBuilder); |
| 232 | rewriter.replaceOp(op: storeOp, newOp: ifOp); |
| 233 | return success(); |
| 234 | } |
| 235 | }; |
| 236 | |
| 237 | } // namespace |
| 238 | |
| 239 | void mlir::amdgpu::populateAmdgpuMaskedloadToLoadPatterns( |
| 240 | RewritePatternSet &patterns, PatternBenefit benefit) { |
| 241 | patterns.add<MaskedLoadLowering, FullMaskedLoadToConditionalLoad, |
| 242 | FullMaskedStoreToConditionalStore>(arg: patterns.getContext(), |
| 243 | args&: benefit); |
| 244 | } |
| 245 | |
| 246 | struct AmdgpuMaskedloadToLoadPass final |
| 247 | : amdgpu::impl::AmdgpuMaskedloadToLoadPassBase<AmdgpuMaskedloadToLoadPass> { |
| 248 | void runOnOperation() override { |
| 249 | RewritePatternSet patterns(&getContext()); |
| 250 | populateAmdgpuMaskedloadToLoadPatterns(patterns); |
| 251 | if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)))) { |
| 252 | return signalPassFailure(); |
| 253 | } |
| 254 | } |
| 255 | }; |
| 256 | |