| 1 | //===- MemorySlotInterfacesImpl.cpp - Mem2Reg for Vector ops --------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements memory slot interface models for |
| 10 | // `vector.transfer_read`, `vector.transfer_write`, and `vector.mask`. They |
| 11 | // implement `PromotableMemOpInterface` to replace vector transfers with uses |
| 12 | // and definitions of a vector SSA value. The models for `memref.copy` and |
| 13 | // `memref.subview` live in the MemRef dialect. |
| 14 | // |
| 15 | // A read uses the slot's current value; a write defines its next value. |
| 16 | // Static subviews use a vector matching the subview's shape: reads extract that |
| 17 | // sub-vector from the parent, and writes insert it back into the parent. |
| 18 | // Supported transfers through dynamic subviews use a vector type matching the |
| 19 | // whole static parent, and the subview's slot holds that whole vector. Writes |
| 20 | // preserve the previous value of lanes outside the runtime extent. Transfers |
| 21 | // must be in-bounds, so lanes past the extent are expected to be masked off. |
| 22 | // `arith.select` implements masking by a transfer's mask operand. |
| 23 | // |
| 24 | // `vector.mask` owns its wrapped transfer's blocking use and is promoted as a |
| 25 | // whole. Its region cannot accept extra operations, so `arith.select` is built |
| 26 | // outside it. |
| 27 | // |
| 28 | //===----------------------------------------------------------------------===// |
| 29 | |
| 30 | #include "mlir/Dialect/Vector/Transforms/MemorySlotInterfacesImpl.h" |
| 31 | |
| 32 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 33 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 34 | #include "mlir/Dialect/Utils/StaticValueUtils.h" |
| 35 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 36 | #include "mlir/IR/BuiltinTypes.h" |
| 37 | #include "mlir/Interfaces/MemorySlotInterfaces.h" |
| 38 | |
| 39 | using namespace mlir; |
| 40 | using namespace mlir::vector; |
| 41 | |
| 42 | //===----------------------------------------------------------------------===// |
| 43 | // Utilities |
| 44 | //===----------------------------------------------------------------------===// |
| 45 | |
| 46 | /// Returns whether `xferOp` can be promoted to use or define `slot`'s vector |
| 47 | /// value. The transfer's memref base must be the slot pointer, its vector type |
| 48 | /// must equal `slot.valueType`, its indices must be zero, and its permutation |
| 49 | /// map must be the identity, and it must be in-bounds in every dimension. |
| 50 | /// |
| 51 | /// More general transfers may be supportable, but their value depends on |
| 52 | /// motivating use cases, implementation complexity, and generated code quality. |
| 53 | static bool |
| 54 | isPromotableTransfer(VectorTransferOpInterface xferOp, const MemorySlot &slot, |
| 55 | const SmallPtrSetImpl<OpOperand *> &blockingUses) { |
| 56 | // The transfer's memref base must be the slot pointer. |
| 57 | if (blockingUses.size() != 1) |
| 58 | return false; |
| 59 | Value blockingUse = (*blockingUses.begin())->get(); |
| 60 | if (blockingUse != slot.ptr || xferOp.getBase() != slot.ptr) |
| 61 | return false; |
| 62 | |
| 63 | // Reject the tensor form (already implied, since slot pointers are memrefs). |
| 64 | if (!isa<MemRefType>(Val: xferOp.getBase().getType())) |
| 65 | return false; |
| 66 | |
| 67 | // Exact type match pins rank/extents/element type/scalable dims. |
| 68 | if (xferOp.getVectorType() != slot.valueType) |
| 69 | return false; |
| 70 | |
| 71 | // Access must start at the buffer origin in every dimension. |
| 72 | for (Value index : xferOp.getIndices()) { |
| 73 | std::optional<int64_t> constIndex = getConstantIntValue(ofr: index); |
| 74 | if (!constIndex || *constIndex != 0) |
| 75 | return false; |
| 76 | } |
| 77 | |
| 78 | // Identity map: no broadcast or transpose. |
| 79 | if (!xferOp.getPermutationMap().isIdentity()) |
| 80 | return false; |
| 81 | |
| 82 | // Out-of-bounds transfers (`in_bounds = [false]`) are not supported. |
| 83 | // Out-of-bounds lanes, e.g. past a dynamic subview's extent, must be masked |
| 84 | // off instead. |
| 85 | if (xferOp.hasOutOfBoundsDim()) |
| 86 | return false; |
| 87 | |
| 88 | return true; |
| 89 | } |
| 90 | |
| 91 | //===----------------------------------------------------------------------===// |
| 92 | // PromotableMemOpInterface models |
| 93 | //===----------------------------------------------------------------------===// |
| 94 | |
| 95 | namespace { |
| 96 | /// `PromotableMemOpInterface` model for `vector.transfer_read`. |
| 97 | struct TransferReadOpMemOpModel |
| 98 | : public PromotableMemOpInterface::ExternalModel<TransferReadOpMemOpModel, |
| 99 | vector::TransferReadOp> { |
| 100 | bool loadsFrom(Operation *op, const MemorySlot &slot) const { |
| 101 | return cast<vector::TransferReadOp>(Val: op).getBase() == slot.ptr; |
| 102 | } |
| 103 | |
| 104 | bool storesTo(Operation *op, const MemorySlot &slot) const { return false; } |
| 105 | |
| 106 | Value getStored(Operation *op, const MemorySlot &slot, OpBuilder &builder, |
| 107 | Value reachingDef, const DataLayout &dataLayout) const { |
| 108 | llvm_unreachable("getStored should not be called on TransferReadOp" ); |
| 109 | } |
| 110 | |
| 111 | bool canUsesBeRemoved(Operation *op, const MemorySlot &slot, |
| 112 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 113 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 114 | const DataLayout &dataLayout) const { |
| 115 | return isPromotableTransfer(xferOp: cast<VectorTransferOpInterface>(Val: op), slot, |
| 116 | blockingUses); |
| 117 | } |
| 118 | |
| 119 | // Replace the read with the reaching definition, selecting padding for lanes |
| 120 | // disabled by the transfer's mask. |
| 121 | DeletionKind |
| 122 | removeBlockingUses(Operation *op, const MemorySlot &slot, |
| 123 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 124 | OpBuilder &builder, Value reachingDefinition, |
| 125 | const DataLayout &dataLayout) const { |
| 126 | auto readOp = cast<vector::TransferReadOp>(Val: op); |
| 127 | Location loc = op->getLoc(); |
| 128 | Value result = reachingDefinition; |
| 129 | if (Value mask = readOp.getMask()) { |
| 130 | Value padSplat = vector::BroadcastOp::create( |
| 131 | builder, location: loc, result: readOp.getVectorType(), source: readOp.getPadding()); |
| 132 | result = arith::SelectOp::create(builder, location: loc, condition: mask, true_value: reachingDefinition, |
| 133 | false_value: padSplat); |
| 134 | } |
| 135 | readOp.getResult().replaceAllUsesWith(newValue: result); |
| 136 | return DeletionKind::Delete; |
| 137 | } |
| 138 | }; |
| 139 | |
| 140 | /// `PromotableMemOpInterface` model for `vector.transfer_write`. |
| 141 | struct TransferWriteOpMemOpModel |
| 142 | : public PromotableMemOpInterface::ExternalModel<TransferWriteOpMemOpModel, |
| 143 | vector::TransferWriteOp> { |
| 144 | bool loadsFrom(Operation *op, const MemorySlot &slot) const { return false; } |
| 145 | |
| 146 | bool storesTo(Operation *op, const MemorySlot &slot) const { |
| 147 | return cast<vector::TransferWriteOp>(Val: op).getBase() == slot.ptr; |
| 148 | } |
| 149 | |
| 150 | // The stored value is the transfer's vector, with inactive lanes preserved |
| 151 | // from the reaching definition when the transfer has a mask. |
| 152 | Value getStored(Operation *op, const MemorySlot &slot, OpBuilder &builder, |
| 153 | Value reachingDef, const DataLayout &dataLayout) const { |
| 154 | auto writeOp = cast<vector::TransferWriteOp>(Val: op); |
| 155 | Value stored = writeOp.getValueToStore(); |
| 156 | // PromotableAliaserInterface::projectAliasValueToSlotValue separately |
| 157 | // preserves parent lanes outside a dynamic subview's extent. |
| 158 | if (Value mask = writeOp.getMask()) |
| 159 | stored = arith::SelectOp::create(builder, location: op->getLoc(), condition: mask, true_value: stored, |
| 160 | false_value: reachingDef); |
| 161 | return stored; |
| 162 | } |
| 163 | |
| 164 | bool canUsesBeRemoved(Operation *op, const MemorySlot &slot, |
| 165 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 166 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 167 | const DataLayout &dataLayout) const { |
| 168 | return isPromotableTransfer(xferOp: cast<VectorTransferOpInterface>(Val: op), slot, |
| 169 | blockingUses); |
| 170 | } |
| 171 | |
| 172 | // `getStored` already provided the value for later uses, so the write can |
| 173 | // simply be erased. |
| 174 | DeletionKind |
| 175 | removeBlockingUses(Operation *op, const MemorySlot &slot, |
| 176 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 177 | OpBuilder &builder, Value reachingDefinition, |
| 178 | const DataLayout &dataLayout) const { |
| 179 | return DeletionKind::Delete; |
| 180 | } |
| 181 | }; |
| 182 | |
| 183 | /// Returns the operation wrapped by the `vector.mask` `op`. |
| 184 | PromotableMemOpInterface getWrappedMemOp(Operation *op) { |
| 185 | Operation *maskableOp = cast<vector::MaskOp>(Val: op).getMaskableOp(); |
| 186 | if (!maskableOp) |
| 187 | return {}; |
| 188 | if (!isa<vector::TransferReadOp, vector::TransferWriteOp>(Val: maskableOp)) |
| 189 | return {}; |
| 190 | return cast<PromotableMemOpInterface>(Val: maskableOp); |
| 191 | } |
| 192 | |
| 193 | /// `PromotableMemOpInterface` model for `vector.mask` wrapping a promotable |
| 194 | /// memory operation. `vector.mask` owns the wrapped operation's slot use, |
| 195 | /// delegates promotion to it, and applies its mask to the result. |
| 196 | struct MaskOpMemOpModel |
| 197 | : public PromotableMemOpInterface::ExternalModel<MaskOpMemOpModel, |
| 198 | vector::MaskOp> { |
| 199 | bool ownsWrappedBlockingUses(Operation *op) const { |
| 200 | return static_cast<bool>(getWrappedMemOp(op)); |
| 201 | } |
| 202 | |
| 203 | bool loadsFrom(Operation *op, const MemorySlot &slot) const { |
| 204 | PromotableMemOpInterface memOp = getWrappedMemOp(op); |
| 205 | return memOp && memOp.loadsFrom(slot); |
| 206 | } |
| 207 | |
| 208 | bool storesTo(Operation *op, const MemorySlot &slot) const { |
| 209 | PromotableMemOpInterface memOp = getWrappedMemOp(op); |
| 210 | return memOp && memOp.storesTo(slot); |
| 211 | } |
| 212 | |
| 213 | Value getStored(Operation *op, const MemorySlot &slot, OpBuilder &builder, |
| 214 | Value reachingDef, const DataLayout &dataLayout) const { |
| 215 | Value stored = |
| 216 | getWrappedMemOp(op).getStored(slot, builder, reachingDef, dataLayout); |
| 217 | return arith::SelectOp::create(builder, location: op->getLoc(), |
| 218 | condition: cast<vector::MaskOp>(Val: op).getMask(), true_value: stored, |
| 219 | false_value: reachingDef); |
| 220 | } |
| 221 | |
| 222 | bool canUsesBeRemoved(Operation *op, const MemorySlot &slot, |
| 223 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 224 | SmallVectorImpl<OpOperand *> &newBlockingUses, |
| 225 | const DataLayout &dataLayout) const { |
| 226 | PromotableMemOpInterface memOp = getWrappedMemOp(op); |
| 227 | // Transfers with their own mask operand are unsupported. |
| 228 | if (!memOp || cast<VectorTransferOpInterface>(Val&: *memOp).getMask()) |
| 229 | return false; |
| 230 | return memOp.canUsesBeRemoved(slot, blockingUses, newBlockingUses, |
| 231 | datalayout: dataLayout); |
| 232 | } |
| 233 | |
| 234 | // Let the wrapped memory op promote itself, then apply the wrapper's mask to |
| 235 | // the value it now yields. |
| 236 | DeletionKind |
| 237 | removeBlockingUses(Operation *op, const MemorySlot &slot, |
| 238 | const SmallPtrSetImpl<OpOperand *> &blockingUses, |
| 239 | OpBuilder &builder, Value reachingDefinition, |
| 240 | const DataLayout &dataLayout) const { |
| 241 | auto maskOp = cast<vector::MaskOp>(Val: op); |
| 242 | PromotableMemOpInterface memOp = getWrappedMemOp(op); |
| 243 | memOp.removeBlockingUses(slot, blockingUses, builder, reachingDefinition, |
| 244 | dataLayout); |
| 245 | auto readOp = dyn_cast<vector::TransferReadOp>(Val&: *memOp); |
| 246 | if (!readOp) |
| 247 | return DeletionKind::Delete; |
| 248 | |
| 249 | Location loc = op->getLoc(); |
| 250 | Value enabled = maskOp.getMaskBlock()->getTerminator()->getOperand(idx: 0); |
| 251 | Value disabled = |
| 252 | maskOp.hasPassthru() |
| 253 | ? maskOp.getPassthru() |
| 254 | : vector::BroadcastOp::create(builder, location: loc, result: readOp.getVectorType(), |
| 255 | source: readOp.getPadding()); |
| 256 | Value result = arith::SelectOp::create(builder, location: loc, condition: maskOp.getMask(), |
| 257 | true_value: enabled, false_value: disabled); |
| 258 | maskOp->getResult(idx: 0).replaceAllUsesWith(newValue: result); |
| 259 | return DeletionKind::Delete; |
| 260 | } |
| 261 | }; |
| 262 | } // namespace |
| 263 | |
| 264 | //===----------------------------------------------------------------------===// |
| 265 | // Register external models |
| 266 | //===----------------------------------------------------------------------===// |
| 267 | |
| 268 | void mlir::vector::registerMemorySlotInterfacesExternalModels( |
| 269 | DialectRegistry ®istry) { |
| 270 | registry.addExtension(extensionFn: +[](MLIRContext *ctx, vector::VectorDialect *dialect) { |
| 271 | TransferReadOp::attachInterface<TransferReadOpMemOpModel>(context&: *ctx); |
| 272 | TransferWriteOp::attachInterface<TransferWriteOpMemOpModel>(context&: *ctx); |
| 273 | MaskOp::attachInterface<MaskOpMemOpModel>(context&: *ctx); |
| 274 | }); |
| 275 | } |
| 276 | |