1//===- MemorySlotInterfacesImpl.cpp - Mem2Reg for Vector ops --------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements memory slot interface models for
10// `vector.transfer_read`, `vector.transfer_write`, and `vector.mask`. They
11// implement `PromotableMemOpInterface` to replace vector transfers with uses
12// and definitions of a vector SSA value. The models for `memref.copy` and
13// `memref.subview` live in the MemRef dialect.
14//
15// A read uses the slot's current value; a write defines its next value.
16// Static subviews use a vector matching the subview's shape: reads extract that
17// sub-vector from the parent, and writes insert it back into the parent.
18// Supported transfers through dynamic subviews use a vector type matching the
19// whole static parent, and the subview's slot holds that whole vector. Writes
20// preserve the previous value of lanes outside the runtime extent. Transfers
21// must be in-bounds, so lanes past the extent are expected to be masked off.
22// `arith.select` implements masking by a transfer's mask operand.
23//
24// `vector.mask` owns its wrapped transfer's blocking use and is promoted as a
25// whole. Its region cannot accept extra operations, so `arith.select` is built
26// outside it.
27//
28//===----------------------------------------------------------------------===//
29
30#include "mlir/Dialect/Vector/Transforms/MemorySlotInterfacesImpl.h"
31
32#include "mlir/Dialect/Arith/IR/Arith.h"
33#include "mlir/Dialect/Arith/Utils/Utils.h"
34#include "mlir/Dialect/Utils/StaticValueUtils.h"
35#include "mlir/Dialect/Vector/IR/VectorOps.h"
36#include "mlir/IR/BuiltinTypes.h"
37#include "mlir/Interfaces/MemorySlotInterfaces.h"
38
39using namespace mlir;
40using namespace mlir::vector;
41
42//===----------------------------------------------------------------------===//
43// Utilities
44//===----------------------------------------------------------------------===//
45
46/// Returns whether `xferOp` can be promoted to use or define `slot`'s vector
47/// value. The transfer's memref base must be the slot pointer, its vector type
48/// must equal `slot.valueType`, its indices must be zero, and its permutation
49/// map must be the identity, and it must be in-bounds in every dimension.
50///
51/// More general transfers may be supportable, but their value depends on
52/// motivating use cases, implementation complexity, and generated code quality.
53static bool
54isPromotableTransfer(VectorTransferOpInterface xferOp, const MemorySlot &slot,
55 const SmallPtrSetImpl<OpOperand *> &blockingUses) {
56 // The transfer's memref base must be the slot pointer.
57 if (blockingUses.size() != 1)
58 return false;
59 Value blockingUse = (*blockingUses.begin())->get();
60 if (blockingUse != slot.ptr || xferOp.getBase() != slot.ptr)
61 return false;
62
63 // Reject the tensor form (already implied, since slot pointers are memrefs).
64 if (!isa<MemRefType>(Val: xferOp.getBase().getType()))
65 return false;
66
67 // Exact type match pins rank/extents/element type/scalable dims.
68 if (xferOp.getVectorType() != slot.valueType)
69 return false;
70
71 // Access must start at the buffer origin in every dimension.
72 for (Value index : xferOp.getIndices()) {
73 std::optional<int64_t> constIndex = getConstantIntValue(ofr: index);
74 if (!constIndex || *constIndex != 0)
75 return false;
76 }
77
78 // Identity map: no broadcast or transpose.
79 if (!xferOp.getPermutationMap().isIdentity())
80 return false;
81
82 // Out-of-bounds transfers (`in_bounds = [false]`) are not supported.
83 // Out-of-bounds lanes, e.g. past a dynamic subview's extent, must be masked
84 // off instead.
85 if (xferOp.hasOutOfBoundsDim())
86 return false;
87
88 return true;
89}
90
91//===----------------------------------------------------------------------===//
92// PromotableMemOpInterface models
93//===----------------------------------------------------------------------===//
94
95namespace {
96/// `PromotableMemOpInterface` model for `vector.transfer_read`.
97struct TransferReadOpMemOpModel
98 : public PromotableMemOpInterface::ExternalModel<TransferReadOpMemOpModel,
99 vector::TransferReadOp> {
100 bool loadsFrom(Operation *op, const MemorySlot &slot) const {
101 return cast<vector::TransferReadOp>(Val: op).getBase() == slot.ptr;
102 }
103
104 bool storesTo(Operation *op, const MemorySlot &slot) const { return false; }
105
106 Value getStored(Operation *op, const MemorySlot &slot, OpBuilder &builder,
107 Value reachingDef, const DataLayout &dataLayout) const {
108 llvm_unreachable("getStored should not be called on TransferReadOp");
109 }
110
111 bool canUsesBeRemoved(Operation *op, const MemorySlot &slot,
112 const SmallPtrSetImpl<OpOperand *> &blockingUses,
113 SmallVectorImpl<OpOperand *> &newBlockingUses,
114 const DataLayout &dataLayout) const {
115 return isPromotableTransfer(xferOp: cast<VectorTransferOpInterface>(Val: op), slot,
116 blockingUses);
117 }
118
119 // Replace the read with the reaching definition, selecting padding for lanes
120 // disabled by the transfer's mask.
121 DeletionKind
122 removeBlockingUses(Operation *op, const MemorySlot &slot,
123 const SmallPtrSetImpl<OpOperand *> &blockingUses,
124 OpBuilder &builder, Value reachingDefinition,
125 const DataLayout &dataLayout) const {
126 auto readOp = cast<vector::TransferReadOp>(Val: op);
127 Location loc = op->getLoc();
128 Value result = reachingDefinition;
129 if (Value mask = readOp.getMask()) {
130 Value padSplat = vector::BroadcastOp::create(
131 builder, location: loc, result: readOp.getVectorType(), source: readOp.getPadding());
132 result = arith::SelectOp::create(builder, location: loc, condition: mask, true_value: reachingDefinition,
133 false_value: padSplat);
134 }
135 readOp.getResult().replaceAllUsesWith(newValue: result);
136 return DeletionKind::Delete;
137 }
138};
139
140/// `PromotableMemOpInterface` model for `vector.transfer_write`.
141struct TransferWriteOpMemOpModel
142 : public PromotableMemOpInterface::ExternalModel<TransferWriteOpMemOpModel,
143 vector::TransferWriteOp> {
144 bool loadsFrom(Operation *op, const MemorySlot &slot) const { return false; }
145
146 bool storesTo(Operation *op, const MemorySlot &slot) const {
147 return cast<vector::TransferWriteOp>(Val: op).getBase() == slot.ptr;
148 }
149
150 // The stored value is the transfer's vector, with inactive lanes preserved
151 // from the reaching definition when the transfer has a mask.
152 Value getStored(Operation *op, const MemorySlot &slot, OpBuilder &builder,
153 Value reachingDef, const DataLayout &dataLayout) const {
154 auto writeOp = cast<vector::TransferWriteOp>(Val: op);
155 Value stored = writeOp.getValueToStore();
156 // PromotableAliaserInterface::projectAliasValueToSlotValue separately
157 // preserves parent lanes outside a dynamic subview's extent.
158 if (Value mask = writeOp.getMask())
159 stored = arith::SelectOp::create(builder, location: op->getLoc(), condition: mask, true_value: stored,
160 false_value: reachingDef);
161 return stored;
162 }
163
164 bool canUsesBeRemoved(Operation *op, const MemorySlot &slot,
165 const SmallPtrSetImpl<OpOperand *> &blockingUses,
166 SmallVectorImpl<OpOperand *> &newBlockingUses,
167 const DataLayout &dataLayout) const {
168 return isPromotableTransfer(xferOp: cast<VectorTransferOpInterface>(Val: op), slot,
169 blockingUses);
170 }
171
172 // `getStored` already provided the value for later uses, so the write can
173 // simply be erased.
174 DeletionKind
175 removeBlockingUses(Operation *op, const MemorySlot &slot,
176 const SmallPtrSetImpl<OpOperand *> &blockingUses,
177 OpBuilder &builder, Value reachingDefinition,
178 const DataLayout &dataLayout) const {
179 return DeletionKind::Delete;
180 }
181};
182
183/// Returns the operation wrapped by the `vector.mask` `op`.
184PromotableMemOpInterface getWrappedMemOp(Operation *op) {
185 Operation *maskableOp = cast<vector::MaskOp>(Val: op).getMaskableOp();
186 if (!maskableOp)
187 return {};
188 if (!isa<vector::TransferReadOp, vector::TransferWriteOp>(Val: maskableOp))
189 return {};
190 return cast<PromotableMemOpInterface>(Val: maskableOp);
191}
192
193/// `PromotableMemOpInterface` model for `vector.mask` wrapping a promotable
194/// memory operation. `vector.mask` owns the wrapped operation's slot use,
195/// delegates promotion to it, and applies its mask to the result.
196struct MaskOpMemOpModel
197 : public PromotableMemOpInterface::ExternalModel<MaskOpMemOpModel,
198 vector::MaskOp> {
199 bool ownsWrappedBlockingUses(Operation *op) const {
200 return static_cast<bool>(getWrappedMemOp(op));
201 }
202
203 bool loadsFrom(Operation *op, const MemorySlot &slot) const {
204 PromotableMemOpInterface memOp = getWrappedMemOp(op);
205 return memOp && memOp.loadsFrom(slot);
206 }
207
208 bool storesTo(Operation *op, const MemorySlot &slot) const {
209 PromotableMemOpInterface memOp = getWrappedMemOp(op);
210 return memOp && memOp.storesTo(slot);
211 }
212
213 Value getStored(Operation *op, const MemorySlot &slot, OpBuilder &builder,
214 Value reachingDef, const DataLayout &dataLayout) const {
215 Value stored =
216 getWrappedMemOp(op).getStored(slot, builder, reachingDef, dataLayout);
217 return arith::SelectOp::create(builder, location: op->getLoc(),
218 condition: cast<vector::MaskOp>(Val: op).getMask(), true_value: stored,
219 false_value: reachingDef);
220 }
221
222 bool canUsesBeRemoved(Operation *op, const MemorySlot &slot,
223 const SmallPtrSetImpl<OpOperand *> &blockingUses,
224 SmallVectorImpl<OpOperand *> &newBlockingUses,
225 const DataLayout &dataLayout) const {
226 PromotableMemOpInterface memOp = getWrappedMemOp(op);
227 // Transfers with their own mask operand are unsupported.
228 if (!memOp || cast<VectorTransferOpInterface>(Val&: *memOp).getMask())
229 return false;
230 return memOp.canUsesBeRemoved(slot, blockingUses, newBlockingUses,
231 datalayout: dataLayout);
232 }
233
234 // Let the wrapped memory op promote itself, then apply the wrapper's mask to
235 // the value it now yields.
236 DeletionKind
237 removeBlockingUses(Operation *op, const MemorySlot &slot,
238 const SmallPtrSetImpl<OpOperand *> &blockingUses,
239 OpBuilder &builder, Value reachingDefinition,
240 const DataLayout &dataLayout) const {
241 auto maskOp = cast<vector::MaskOp>(Val: op);
242 PromotableMemOpInterface memOp = getWrappedMemOp(op);
243 memOp.removeBlockingUses(slot, blockingUses, builder, reachingDefinition,
244 dataLayout);
245 auto readOp = dyn_cast<vector::TransferReadOp>(Val&: *memOp);
246 if (!readOp)
247 return DeletionKind::Delete;
248
249 Location loc = op->getLoc();
250 Value enabled = maskOp.getMaskBlock()->getTerminator()->getOperand(idx: 0);
251 Value disabled =
252 maskOp.hasPassthru()
253 ? maskOp.getPassthru()
254 : vector::BroadcastOp::create(builder, location: loc, result: readOp.getVectorType(),
255 source: readOp.getPadding());
256 Value result = arith::SelectOp::create(builder, location: loc, condition: maskOp.getMask(),
257 true_value: enabled, false_value: disabled);
258 maskOp->getResult(idx: 0).replaceAllUsesWith(newValue: result);
259 return DeletionKind::Delete;
260 }
261};
262} // namespace
263
264//===----------------------------------------------------------------------===//
265// Register external models
266//===----------------------------------------------------------------------===//
267
268void mlir::vector::registerMemorySlotInterfacesExternalModels(
269 DialectRegistry &registry) {
270 registry.addExtension(extensionFn: +[](MLIRContext *ctx, vector::VectorDialect *dialect) {
271 TransferReadOp::attachInterface<TransferReadOpMemOpModel>(context&: *ctx);
272 TransferWriteOp::attachInterface<TransferWriteOpMemOpModel>(context&: *ctx);
273 MaskOp::attachInterface<MaskOpMemOpModel>(context&: *ctx);
274 });
275}
276