| 1 | //=- VectorEmulateMaskedLoadStore.cpp - Emulate 'vector.maskedload/store' op =// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements target-independent rewrites and utilities to emulate the |
| 10 | // 'vector.maskedload' and 'vector.maskedstore' operation. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 15 | #include "mlir/Dialect/SCF/IR/SCF.h" |
| 16 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
| 17 | |
| 18 | using namespace mlir; |
| 19 | |
| 20 | namespace { |
| 21 | |
| 22 | /// Convert vector.maskedload |
| 23 | /// |
| 24 | /// Before: |
| 25 | /// |
| 26 | /// vector.maskedload %base[%idx_0, %idx_1], %mask, %pass_thru |
| 27 | /// |
| 28 | /// After: |
| 29 | /// |
| 30 | /// %ivalue = %pass_thru |
| 31 | /// %m = vector.extract %mask[0] |
| 32 | /// %result0 = scf.if %m { |
| 33 | /// %v = memref.load %base[%idx_0, %idx_1] |
| 34 | /// %combined = vector.insert %v, %ivalue[0] |
| 35 | /// scf.yield %combined |
| 36 | /// } else { |
| 37 | /// scf.yield %ivalue |
| 38 | /// } |
| 39 | /// %m = vector.extract %mask[1] |
| 40 | /// %result1 = scf.if %m { |
| 41 | /// %v = memref.load %base[%idx_0, %idx_1 + 1] |
| 42 | /// %combined = vector.insert %v, %result0[1] |
| 43 | /// scf.yield %combined |
| 44 | /// } else { |
| 45 | /// scf.yield %result0 |
| 46 | /// } |
| 47 | /// ... |
| 48 | /// |
| 49 | struct VectorMaskedLoadOpConverter final |
| 50 | : OpRewritePattern<vector::MaskedLoadOp> { |
| 51 | using Base::Base; |
| 52 | |
| 53 | LogicalResult matchAndRewrite(vector::MaskedLoadOp maskedLoadOp, |
| 54 | PatternRewriter &rewriter) const override { |
| 55 | VectorType maskVType = maskedLoadOp.getMaskVectorType(); |
| 56 | if (maskVType.getShape().size() != 1) |
| 57 | return rewriter.notifyMatchFailure( |
| 58 | arg&: maskedLoadOp, msg: "expected vector.maskedload with 1-D mask" ); |
| 59 | if (maskVType.isScalable()) |
| 60 | return rewriter.notifyMatchFailure(arg&: maskedLoadOp, |
| 61 | msg: "cannot scalarize a scalable mask" ); |
| 62 | |
| 63 | Location loc = maskedLoadOp.getLoc(); |
| 64 | int64_t maskLength = maskVType.getShape()[0]; |
| 65 | |
| 66 | Type indexType = rewriter.getIndexType(); |
| 67 | Value mask = maskedLoadOp.getMask(); |
| 68 | Value base = maskedLoadOp.getBase(); |
| 69 | Value iValue = maskedLoadOp.getPassThru(); |
| 70 | auto indices = llvm::to_vector_of<Value>(Range: maskedLoadOp.getIndices()); |
| 71 | Value one = arith::ConstantOp::create(builder&: rewriter, location: loc, result: indexType, |
| 72 | value: IntegerAttr::get(type: indexType, value: 1)); |
| 73 | for (int64_t i = 0; i < maskLength; ++i) { |
| 74 | auto maskBit = vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask, position: i); |
| 75 | |
| 76 | auto ifOp = scf::IfOp::create( |
| 77 | builder&: rewriter, location: loc, cond: maskBit, |
| 78 | thenBuilder: [&](OpBuilder &builder, Location loc) { |
| 79 | auto loadedValue = memref::LoadOp::create( |
| 80 | builder, location: loc, memref: base, indices, /*nontemporal=*/false, |
| 81 | alignment: llvm::MaybeAlign(maskedLoadOp.getAlignment().value_or(u: 0))); |
| 82 | auto combinedValue = |
| 83 | vector::InsertOp::create(builder, location: loc, valueToStore: loadedValue, dest: iValue, position: i); |
| 84 | scf::YieldOp::create(builder, location: loc, results: combinedValue.getResult()); |
| 85 | }, |
| 86 | elseBuilder: [&](OpBuilder &builder, Location loc) { |
| 87 | scf::YieldOp::create(builder, location: loc, results: iValue); |
| 88 | }); |
| 89 | iValue = ifOp.getResult(i: 0); |
| 90 | if (!indices.empty()) |
| 91 | indices.back() = |
| 92 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: indices.back(), rhs: one); |
| 93 | } |
| 94 | |
| 95 | rewriter.replaceOp(op: maskedLoadOp, newValues: iValue); |
| 96 | |
| 97 | return success(); |
| 98 | } |
| 99 | }; |
| 100 | |
| 101 | /// Convert vector.maskedstore |
| 102 | /// |
| 103 | /// Before: |
| 104 | /// |
| 105 | /// vector.maskedstore %base[%idx_0, %idx_1], %mask, %value |
| 106 | /// |
| 107 | /// After: |
| 108 | /// |
| 109 | /// %m = vector.extract %mask[0] |
| 110 | /// scf.if %m { |
| 111 | /// %extracted = vector.extract %value[0] |
| 112 | /// memref.store %extracted, %base[%idx_0, %idx_1] |
| 113 | /// } |
| 114 | /// %m = vector.extract %mask[1] |
| 115 | /// scf.if %m { |
| 116 | /// %extracted = vector.extract %value[1] |
| 117 | /// memref.store %extracted, %base[%idx_0, %idx_1 + 1] |
| 118 | /// } |
| 119 | /// ... |
| 120 | /// |
| 121 | struct VectorMaskedStoreOpConverter final |
| 122 | : OpRewritePattern<vector::MaskedStoreOp> { |
| 123 | using Base::Base; |
| 124 | |
| 125 | LogicalResult matchAndRewrite(vector::MaskedStoreOp maskedStoreOp, |
| 126 | PatternRewriter &rewriter) const override { |
| 127 | VectorType maskVType = maskedStoreOp.getMaskVectorType(); |
| 128 | if (maskVType.getShape().size() != 1) |
| 129 | return rewriter.notifyMatchFailure( |
| 130 | arg&: maskedStoreOp, msg: "expected vector.maskedstore with 1-D mask" ); |
| 131 | if (maskVType.isScalable()) |
| 132 | return rewriter.notifyMatchFailure(arg&: maskedStoreOp, |
| 133 | msg: "cannot scalarize a scalable mask" ); |
| 134 | |
| 135 | Location loc = maskedStoreOp.getLoc(); |
| 136 | int64_t maskLength = maskVType.getShape()[0]; |
| 137 | |
| 138 | Type indexType = rewriter.getIndexType(); |
| 139 | Value mask = maskedStoreOp.getMask(); |
| 140 | Value base = maskedStoreOp.getBase(); |
| 141 | Value value = maskedStoreOp.getValueToStore(); |
| 142 | bool nontemporal = false; |
| 143 | auto indices = llvm::to_vector_of<Value>(Range: maskedStoreOp.getIndices()); |
| 144 | Value one = arith::ConstantOp::create(builder&: rewriter, location: loc, result: indexType, |
| 145 | value: IntegerAttr::get(type: indexType, value: 1)); |
| 146 | for (int64_t i = 0; i < maskLength; ++i) { |
| 147 | auto maskBit = vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask, position: i); |
| 148 | |
| 149 | auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: maskBit, /*else=*/withElseRegion: false); |
| 150 | rewriter.setInsertionPointToStart(&ifOp.getThenRegion().front()); |
| 151 | auto = vector::ExtractOp::create(builder&: rewriter, location: loc, source: value, position: i); |
| 152 | memref::StoreOp::create( |
| 153 | builder&: rewriter, location: loc, valueToStore: extractedValue, memref: base, indices, nontemporal, |
| 154 | alignment: llvm::MaybeAlign(maskedStoreOp.getAlignment().value_or(u: 0))); |
| 155 | |
| 156 | rewriter.setInsertionPointAfter(ifOp); |
| 157 | if (!indices.empty()) |
| 158 | indices.back() = |
| 159 | arith::AddIOp::create(builder&: rewriter, location: loc, lhs: indices.back(), rhs: one); |
| 160 | } |
| 161 | |
| 162 | rewriter.eraseOp(op: maskedStoreOp); |
| 163 | |
| 164 | return success(); |
| 165 | } |
| 166 | }; |
| 167 | |
| 168 | } // namespace |
| 169 | |
| 170 | void mlir::vector::populateVectorMaskedLoadStoreEmulationPatterns( |
| 171 | RewritePatternSet &patterns, PatternBenefit benefit) { |
| 172 | patterns.add<VectorMaskedLoadOpConverter, VectorMaskedStoreOpConverter>( |
| 173 | arg: patterns.getContext(), args&: benefit); |
| 174 | } |
| 175 | |