1//===- VectorTransferSplitRewritePatterns.cpp - Transfer Split Rewrites ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements target-independent patterns to rewrite a vector.transfer
10// op into a fully in-bounds part and a partial part.
11//
12//===----------------------------------------------------------------------===//
13
14#include <optional>
15
16#include "mlir/Dialect/Affine/IR/AffineOps.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/MemRef/IR/MemRef.h"
19#include "mlir/Dialect/SCF/IR/SCF.h"
20#include "mlir/Dialect/Utils/StructuredOpsUtils.h"
21#include "llvm/ADT/SmallVectorExtras.h"
22
23#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
24#include "mlir/IR/PatternMatch.h"
25#include "mlir/Interfaces/VectorInterfaces.h"
26
27#include "llvm/ADT/STLExtras.h"
28#include "llvm/ADT/SmallVectorExtras.h"
29
30#define DEBUG_TYPE "vector-transfer-split"
31
32using namespace mlir;
33using namespace mlir::vector;
34
35/// Build the condition to ensure that a particular VectorTransferOpInterface
36/// is in-bounds.
37static Value createInBoundsCond(RewriterBase &b,
38 VectorTransferOpInterface xferOp) {
39 assert(xferOp.getPermutationMap().isMinorIdentity() &&
40 "Expected minor identity map");
41 Value inBoundsCond;
42 xferOp.zipResultAndIndexing(fun: [&](int64_t resultIdx, int64_t indicesIdx) {
43 // Zip over the resulting vector shape and memref indices.
44 // If the dimension is known to be in-bounds, it does not participate in
45 // the construction of `inBoundsCond`.
46 if (xferOp.isDimInBounds(dim: resultIdx))
47 return;
48 // Fold or create the check that `index + vector_size` <= `memref_size`.
49 Location loc = xferOp.getLoc();
50 int64_t vectorSize = xferOp.getVectorType().getDimSize(idx: resultIdx);
51 OpFoldResult sum = affine::makeComposedFoldedAffineApply(
52 b, loc, expr: b.getAffineDimExpr(position: 0) + b.getAffineConstantExpr(constant: vectorSize),
53 operands: {xferOp.getIndices()[indicesIdx]});
54 OpFoldResult dimSz =
55 memref::getMixedSize(builder&: b, loc, value: xferOp.getBase(), dim: indicesIdx);
56 auto maybeCstSum = getConstantIntValue(ofr: sum);
57 auto maybeCstDimSz = getConstantIntValue(ofr: dimSz);
58 if (maybeCstSum && maybeCstDimSz && *maybeCstSum <= *maybeCstDimSz)
59 return;
60 Value cond =
61 arith::CmpIOp::create(builder&: b, location: loc, predicate: arith::CmpIPredicate::sle,
62 lhs: getValueOrCreateConstantIndexOp(b, loc, ofr: sum),
63 rhs: getValueOrCreateConstantIndexOp(b, loc, ofr: dimSz));
64 // Conjunction over all dims for which we are in-bounds.
65 if (inBoundsCond)
66 inBoundsCond = arith::AndIOp::create(builder&: b, location: loc, lhs: inBoundsCond, rhs: cond);
67 else
68 inBoundsCond = cond;
69 });
70 return inBoundsCond;
71}
72
73/// Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds
74/// masking) fast path and a slow path.
75/// If `ifOp` is not null and the result is `success, the `ifOp` points to the
76/// newly created conditional upon function return.
77/// To accommodate for the fact that the original vector.transfer indexing may
78/// be arbitrary and the slow path indexes @[0...0] in the temporary buffer, the
79/// scf.if op returns a view and values of type index.
80/// At this time, only vector.transfer_read case is implemented.
81///
82/// Example (a 2-D vector.transfer_read):
83/// ```
84/// %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
85/// ```
86/// is transformed into:
87/// ```
88/// %1:3 = scf.if (%inBounds) {
89/// // fast path, direct cast
90/// memref.cast %A: memref<A...> to compatibleMemRefType
91/// scf.yield %view : compatibleMemRefType, index, index
92/// } else {
93/// // slow path, not in-bounds vector.transfer or memref.copy.
94/// memref.cast %alloc: memref<B...> to compatibleMemRefType
95/// scf.yield %4 : compatibleMemRefType, index, index
96// }
97/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]}
98/// ```
99/// where `alloc` is a top of the function alloca'ed buffer of one vector.
100///
101/// Preconditions:
102/// 1. `xferOp.getPermutationMap()` must be a minor identity map
103/// 2. the rank of the `xferOp.memref()` and the rank of the
104/// `xferOp.getVector()` must be equal. This will be relaxed in the future
105/// but requires rank-reducing subviews.
106static LogicalResult
107splitFullAndPartialTransferPrecondition(VectorTransferOpInterface xferOp) {
108 // TODO: support 0-d corner case.
109 if (xferOp.getTransferRank() == 0)
110 return failure();
111
112 // TODO: expand support to these 2 cases.
113 if (!xferOp.getPermutationMap().isMinorIdentity())
114 return failure();
115 // Must have some out-of-bounds dimension to be a candidate for splitting.
116 if (!xferOp.hasOutOfBoundsDim())
117 return failure();
118 // Don't split transfer operations directly under IfOp, this avoids applying
119 // the pattern recursively.
120 // TODO: improve the filtering condition to make it more applicable.
121 if (isa<scf::IfOp>(Val: xferOp->getParentOp()))
122 return failure();
123 return success();
124}
125
126/// Given two MemRefTypes `aT` and `bT`, return a MemRefType to which both can
127/// be cast. If the MemRefTypes don't have the same rank or are not strided,
128/// return null; otherwise:
129/// 1. if `aT` and `bT` are cast-compatible, return `aT`.
130/// 2. else return a new MemRefType obtained by iterating over the shape and
131/// strides and:
132/// a. keeping the ones that are static and equal across `aT` and `bT`.
133/// b. using a dynamic shape and/or stride for the dimensions that don't
134/// agree.
135static MemRefType getCastCompatibleMemRefType(MemRefType aT, MemRefType bT) {
136 if (memref::CastOp::areCastCompatible(inputs: aT, outputs: bT))
137 return aT;
138 if (aT.getRank() != bT.getRank())
139 return MemRefType();
140 int64_t aOffset, bOffset;
141 SmallVector<int64_t, 4> aStrides, bStrides;
142 if (failed(Result: aT.getStridesAndOffset(strides&: aStrides, offset&: aOffset)) ||
143 failed(Result: bT.getStridesAndOffset(strides&: bStrides, offset&: bOffset)) ||
144 aStrides.size() != bStrides.size())
145 return MemRefType();
146
147 ArrayRef<int64_t> aShape = aT.getShape(), bShape = bT.getShape();
148 int64_t resOffset;
149 SmallVector<int64_t, 4> resShape(aT.getRank(), 0),
150 resStrides(bT.getRank(), 0);
151 for (int64_t idx = 0, e = aT.getRank(); idx < e; ++idx) {
152 resShape[idx] =
153 (aShape[idx] == bShape[idx]) ? aShape[idx] : ShapedType::kDynamic;
154 resStrides[idx] =
155 (aStrides[idx] == bStrides[idx]) ? aStrides[idx] : ShapedType::kDynamic;
156 }
157 resOffset = (aOffset == bOffset) ? aOffset : ShapedType::kDynamic;
158 return MemRefType::get(
159 shape: resShape, elementType: aT.getElementType(),
160 layout: StridedLayoutAttr::get(context: aT.getContext(), offset: resOffset, strides: resStrides));
161}
162
163/// Casts the given memref to a compatible memref type. If the source memref has
164/// a different address space than the target type, a `memref.memory_space_cast`
165/// is first inserted, followed by a `memref.cast`.
166static Value castToCompatibleMemRefType(OpBuilder &b, Value memref,
167 MemRefType compatibleMemRefType) {
168 MemRefType sourceType = cast<MemRefType>(Val: memref.getType());
169 Value res = memref;
170 if (sourceType.getMemorySpace() != compatibleMemRefType.getMemorySpace()) {
171 sourceType = MemRefType::get(
172 shape: sourceType.getShape(), elementType: sourceType.getElementType(),
173 layout: sourceType.getLayout(), memorySpace: compatibleMemRefType.getMemorySpace());
174 res =
175 memref::MemorySpaceCastOp::create(builder&: b, location: memref.getLoc(), dest: sourceType, source: res);
176 }
177 if (sourceType == compatibleMemRefType)
178 return res;
179 return memref::CastOp::create(builder&: b, location: memref.getLoc(), dest: compatibleMemRefType, source: res);
180}
181
182/// Operates under a scoped context to build the intersection between the
183/// view `xferOp.getbase()` @ `xferOp.getIndices()` and the view `alloc`.
184// TODO: view intersection/union/differences should be a proper std op.
185static std::pair<Value, Value>
186createSubViewIntersection(RewriterBase &b, VectorTransferOpInterface xferOp,
187 Value alloc) {
188 Location loc = xferOp.getLoc();
189 int64_t memrefRank = xferOp.getShapedType().getRank();
190 // TODO: relax this precondition, will require rank-reducing subviews.
191 assert(memrefRank == cast<MemRefType>(alloc.getType()).getRank() &&
192 "Expected memref rank to match the alloc rank");
193 ValueRange leadingIndices =
194 xferOp.getIndices().take_front(n: xferOp.getLeadingShapedRank());
195 SmallVector<OpFoldResult, 4> sizes;
196 sizes.append(in_start: leadingIndices.begin(), in_end: leadingIndices.end());
197 auto isaWrite = isa<vector::TransferWriteOp>(Val: xferOp);
198 xferOp.zipResultAndIndexing(fun: [&](int64_t resultIdx, int64_t indicesIdx) {
199 using MapList = ArrayRef<ArrayRef<AffineExpr>>;
200 Value dimMemRef =
201 memref::DimOp::create(builder&: b, location: xferOp.getLoc(), source: xferOp.getBase(), index: indicesIdx);
202 Value dimAlloc = memref::DimOp::create(builder&: b, location: loc, source: alloc, index: resultIdx);
203 Value index = xferOp.getIndices()[indicesIdx];
204 AffineExpr i, j, k;
205 bindDims(ctx: xferOp.getContext(), exprs&: i, exprs&: j, exprs&: k);
206 SmallVector<AffineMap, 4> maps =
207 AffineMap::inferFromExprList(exprsList: MapList{{i - j, k}}, context: b.getContext());
208 // affine_min(%dimMemRef - %index, %dimAlloc)
209 Value affineMin =
210 affine::AffineMinOp::create(builder&: b, location: loc, resultType0: index.getType(), map: maps[0],
211 operands: ValueRange{dimMemRef, index, dimAlloc});
212 sizes.push_back(Elt: affineMin);
213 });
214
215 SmallVector<OpFoldResult> srcIndices = llvm::map_to_vector<4>(
216 C: xferOp.getIndices(), F: [](Value idx) -> OpFoldResult { return idx; });
217 SmallVector<OpFoldResult> destIndices(memrefRank, b.getIndexAttr(value: 0));
218 SmallVector<OpFoldResult> strides(memrefRank, b.getIndexAttr(value: 1));
219 auto copySrc = memref::SubViewOp::create(
220 builder&: b, location: loc, source: isaWrite ? alloc : xferOp.getBase(), offsets: srcIndices, sizes, strides);
221 auto copyDest = memref::SubViewOp::create(
222 builder&: b, location: loc, source: isaWrite ? xferOp.getBase() : alloc, offsets: destIndices, sizes, strides);
223 return std::make_pair(x&: copySrc, y&: copyDest);
224}
225
226/// Given an `xferOp` for which:
227/// 1. `inBoundsCond` and a `compatibleMemRefType` have been computed.
228/// 2. a memref of single vector `alloc` has been allocated.
229/// Produce IR resembling:
230/// ```
231/// %1:3 = scf.if (%inBounds) {
232/// (memref.memory_space_cast %A: memref<A..., addr_space> to memref<A...>)
233/// %view = memref.cast %A: memref<A...> to compatibleMemRefType
234/// scf.yield %view, ... : compatibleMemRefType, index, index
235/// } else {
236/// %2 = vector.broadcast %pad : elt to vector<...>
237/// %3 = vector.type_cast %alloc : memref<B...> to memref<vector<...>>
238/// memref.store %2, %3[] : memref<vector<...>>
239/// %4 = subview %view [...][...][...]
240/// %5 = subview %alloc [0, 0] [...] [...]
241/// memref.copy %4, %5
242/// %6 = memref.cast %alloc: memref<B...> to compatibleMemRefType
243/// scf.yield %6, ... : compatibleMemRefType, index, index
244/// }
245/// ```
246/// Return the produced scf::IfOp.
247static scf::IfOp
248createFullPartialMemRefCopy(RewriterBase &b, vector::TransferReadOp xferOp,
249 TypeRange returnTypes, Value inBoundsCond,
250 MemRefType compatibleMemRefType, Value alloc) {
251 Location loc = xferOp.getLoc();
252 Value zero = arith::ConstantIndexOp::create(builder&: b, location: loc, value: 0);
253 Value memref = xferOp.getBase();
254 return scf::IfOp::create(
255 builder&: b, location: loc, cond: inBoundsCond,
256 thenBuilder: [&](OpBuilder &b, Location loc) {
257 Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
258 scf::ValueVector viewAndIndices{res};
259 llvm::append_range(C&: viewAndIndices, R: xferOp.getIndices());
260 scf::YieldOp::create(builder&: b, location: loc, results: viewAndIndices);
261 },
262 elseBuilder: [&](OpBuilder &b, Location loc) {
263 // Fill the whole buffer with the padding value. The buffer holds
264 // exactly one vector, so a single vector store covers it.
265 Value padding = vector::BroadcastOp::create(
266 builder&: b, location: loc, result: xferOp.getVectorType(), source: xferOp.getPadding());
267 memref::StoreOp::create(
268 builder&: b, location: loc, valueToStore: padding,
269 memref: vector::TypeCastOp::create(
270 builder&: b, location: loc, result: MemRefType::get(shape: {}, elementType: padding.getType()), memref: alloc));
271 // Take partial subview of memref which guarantees no dimension
272 // overflows.
273 IRRewriter rewriter(b);
274 std::pair<Value, Value> copyArgs = createSubViewIntersection(
275 b&: rewriter, xferOp: cast<VectorTransferOpInterface>(Val: xferOp.getOperation()),
276 alloc);
277 memref::CopyOp::create(builder&: b, location: loc, source: copyArgs.first, target: copyArgs.second);
278 Value casted =
279 castToCompatibleMemRefType(b, memref: alloc, compatibleMemRefType);
280 scf::ValueVector viewAndIndices{casted};
281 viewAndIndices.insert(I: viewAndIndices.end(), NumToInsert: xferOp.getTransferRank(),
282 Elt: zero);
283 scf::YieldOp::create(builder&: b, location: loc, results: viewAndIndices);
284 });
285}
286
287/// Given an `xferOp` for which:
288/// 1. `inBoundsCond` and a `compatibleMemRefType` have been computed.
289/// 2. a memref of single vector `alloc` has been allocated.
290/// Produce IR resembling:
291/// ```
292/// %1:3 = scf.if (%inBounds) {
293/// (memref.memory_space_cast %A: memref<A..., addr_space> to memref<A...>)
294/// memref.cast %A: memref<A...> to compatibleMemRefType
295/// scf.yield %view, ... : compatibleMemRefType, index, index
296/// } else {
297/// %2 = vector.transfer_read %view[...], %pad : memref<A...>, vector<...>
298/// %3 = vector.type_cast %extra_alloc :
299/// memref<...> to memref<vector<...>>
300/// store %2, %3[] : memref<vector<...>>
301/// %4 = memref.cast %alloc: memref<B...> to compatibleMemRefType
302/// scf.yield %4, ... : compatibleMemRefType, index, index
303/// }
304/// ```
305/// Return the produced scf::IfOp.
306static scf::IfOp createFullPartialVectorTransferRead(
307 RewriterBase &b, vector::TransferReadOp xferOp, TypeRange returnTypes,
308 Value inBoundsCond, MemRefType compatibleMemRefType, Value alloc) {
309 Location loc = xferOp.getLoc();
310 scf::IfOp fullPartialIfOp;
311 Value zero = arith::ConstantIndexOp::create(builder&: b, location: loc, value: 0);
312 Value memref = xferOp.getBase();
313 return scf::IfOp::create(
314 builder&: b, location: loc, cond: inBoundsCond,
315 thenBuilder: [&](OpBuilder &b, Location loc) {
316 Value res = castToCompatibleMemRefType(b, memref, compatibleMemRefType);
317 scf::ValueVector viewAndIndices{res};
318 llvm::append_range(C&: viewAndIndices, R: xferOp.getIndices());
319 scf::YieldOp::create(builder&: b, location: loc, results: viewAndIndices);
320 },
321 elseBuilder: [&](OpBuilder &b, Location loc) {
322 Operation *newXfer = b.clone(op&: *xferOp.getOperation());
323 Value vector = cast<VectorTransferOpInterface>(Val: newXfer).getVector();
324 memref::StoreOp::create(
325 builder&: b, location: loc, valueToStore: vector,
326 memref: vector::TypeCastOp::create(
327 builder&: b, location: loc, result: MemRefType::get(shape: {}, elementType: vector.getType()), memref: alloc));
328
329 Value casted =
330 castToCompatibleMemRefType(b, memref: alloc, compatibleMemRefType);
331 scf::ValueVector viewAndIndices{casted};
332 viewAndIndices.insert(I: viewAndIndices.end(), NumToInsert: xferOp.getTransferRank(),
333 Elt: zero);
334 scf::YieldOp::create(builder&: b, location: loc, results: viewAndIndices);
335 });
336}
337
338/// Given an `xferOp` for which:
339/// 1. `inBoundsCond` and a `compatibleMemRefType` have been computed.
340/// 2. a memref of single vector `alloc` has been allocated.
341/// Produce IR resembling:
342/// ```
343/// %1:3 = scf.if (%inBounds) {
344/// memref.cast %A: memref<A...> to compatibleMemRefType
345/// scf.yield %view, ... : compatibleMemRefType, index, index
346/// } else {
347/// %3 = vector.type_cast %extra_alloc :
348/// memref<...> to memref<vector<...>>
349/// %4 = memref.cast %alloc: memref<B...> to compatibleMemRefType
350/// scf.yield %4, ... : compatibleMemRefType, index, index
351/// }
352/// ```
353static ValueRange
354getLocationToWriteFullVec(RewriterBase &b, vector::TransferWriteOp xferOp,
355 TypeRange returnTypes, Value inBoundsCond,
356 MemRefType compatibleMemRefType, Value alloc) {
357 Location loc = xferOp.getLoc();
358 Value zero = arith::ConstantIndexOp::create(builder&: b, location: loc, value: 0);
359 Value memref = xferOp.getBase();
360 return scf::IfOp::create(
361 builder&: b, location: loc, cond: inBoundsCond,
362 thenBuilder: [&](OpBuilder &b, Location loc) {
363 Value res =
364 castToCompatibleMemRefType(b, memref, compatibleMemRefType);
365 scf::ValueVector viewAndIndices{res};
366 llvm::append_range(C&: viewAndIndices, R: xferOp.getIndices());
367 scf::YieldOp::create(builder&: b, location: loc, results: viewAndIndices);
368 },
369 elseBuilder: [&](OpBuilder &b, Location loc) {
370 Value casted =
371 castToCompatibleMemRefType(b, memref: alloc, compatibleMemRefType);
372 scf::ValueVector viewAndIndices{casted};
373 viewAndIndices.insert(I: viewAndIndices.end(),
374 NumToInsert: xferOp.getTransferRank(), Elt: zero);
375 scf::YieldOp::create(builder&: b, location: loc, results: viewAndIndices);
376 })
377 ->getResults();
378}
379
380/// Given an `xferOp` for which:
381/// 1. `inBoundsCond` has been computed.
382/// 2. a memref of single vector `alloc` has been allocated.
383/// 3. it originally wrote to %view
384/// Produce IR resembling:
385/// ```
386/// %notInBounds = arith.xori %inBounds, %true
387/// scf.if (%notInBounds) {
388/// %3 = subview %alloc [...][...][...]
389/// %4 = subview %view [0, 0][...][...]
390/// memref.copy %3, %4
391/// }
392/// ```
393static void createFullPartialMemRefCopy(RewriterBase &b,
394 vector::TransferWriteOp xferOp,
395 Value inBoundsCond, Value alloc) {
396 Location loc = xferOp.getLoc();
397 auto notInBounds = arith::XOrIOp::create(
398 builder&: b, location: loc, lhs: inBoundsCond, rhs: arith::ConstantIntOp::create(builder&: b, location: loc, value: true, width: 1));
399 scf::IfOp::create(builder&: b, location: loc, cond: notInBounds, thenBuilder: [&](OpBuilder &b, Location loc) {
400 IRRewriter rewriter(b);
401 std::pair<Value, Value> copyArgs = createSubViewIntersection(
402 b&: rewriter, xferOp: cast<VectorTransferOpInterface>(Val: xferOp.getOperation()),
403 alloc);
404 memref::CopyOp::create(builder&: b, location: loc, source: copyArgs.first, target: copyArgs.second);
405 scf::YieldOp::create(builder&: b, location: loc, results: ValueRange{});
406 });
407}
408
409/// Given an `xferOp` for which:
410/// 1. `inBoundsCond` has been computed.
411/// 2. a memref of single vector `alloc` has been allocated.
412/// 3. it originally wrote to %view
413/// Produce IR resembling:
414/// ```
415/// %notInBounds = arith.xori %inBounds, %true
416/// scf.if (%notInBounds) {
417/// %2 = load %alloc : memref<vector<...>>
418/// vector.transfer_write %2, %view[...] : memref<A...>, vector<...>
419/// }
420/// ```
421static void createFullPartialVectorTransferWrite(RewriterBase &b,
422 vector::TransferWriteOp xferOp,
423 Value inBoundsCond,
424 Value alloc) {
425 Location loc = xferOp.getLoc();
426 auto notInBounds = arith::XOrIOp::create(
427 builder&: b, location: loc, lhs: inBoundsCond, rhs: arith::ConstantIntOp::create(builder&: b, location: loc, value: true, width: 1));
428 scf::IfOp::create(builder&: b, location: loc, cond: notInBounds, thenBuilder: [&](OpBuilder &b, Location loc) {
429 IRMapping mapping;
430 Value load = memref::LoadOp::create(
431 builder&: b, location: loc,
432 memref: vector::TypeCastOp::create(
433 builder&: b, location: loc, result: MemRefType::get(shape: {}, elementType: xferOp.getVector().getType()), memref: alloc),
434 indices: ValueRange());
435 mapping.map(from: xferOp.getVector(), to: load);
436 b.clone(op&: *xferOp.getOperation(), mapper&: mapping);
437 scf::YieldOp::create(builder&: b, location: loc, results: ValueRange{});
438 });
439}
440
441// TODO: Parallelism and threadlocal considerations with a ParallelScope trait.
442static Operation *getAutomaticAllocationScope(Operation *op) {
443 // Find the closest surrounding allocation scope that is not a known looping
444 // construct (putting alloca's in loops doesn't always lower to deallocation
445 // until the end of the loop).
446 Operation *scope = nullptr;
447 for (Operation *parent = op->getParentOp(); parent != nullptr;
448 parent = parent->getParentOp()) {
449 if (parent->hasTrait<OpTrait::AutomaticAllocationScope>())
450 scope = parent;
451 if (!isa<scf::ForOp, affine::AffineForOp>(Val: parent))
452 break;
453 }
454 assert(scope && "Expected op to be inside automatic allocation scope");
455 return scope;
456}
457
458/// Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds
459/// masking) fastpath and a slowpath.
460///
461/// For vector.transfer_read:
462/// If `ifOp` is not null and the result is `success, the `ifOp` points to the
463/// newly created conditional upon function return.
464/// To accomodate for the fact that the original vector.transfer indexing may be
465/// arbitrary and the slow path indexes @[0...0] in the temporary buffer, the
466/// scf.if op returns a view and values of type index.
467///
468/// Example (a 2-D vector.transfer_read):
469/// ```
470/// %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...>
471/// ```
472/// is transformed into:
473/// ```
474/// %1:3 = scf.if (%inBounds) {
475/// // fastpath, direct cast
476/// memref.cast %A: memref<A...> to compatibleMemRefType
477/// scf.yield %view : compatibleMemRefType, index, index
478/// } else {
479/// // slowpath, not in-bounds vector.transfer or memref.copy.
480/// memref.cast %alloc: memref<B...> to compatibleMemRefType
481/// scf.yield %4 : compatibleMemRefType, index, index
482// }
483/// %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]}
484/// ```
485/// where `alloc` is a top of the function alloca'ed buffer of one vector.
486///
487/// For vector.transfer_write:
488/// There are 2 conditional blocks. First a block to decide which memref and
489/// indices to use for an unmasked, inbounds write. Then a conditional block to
490/// further copy a partial buffer into the final result in the slow path case.
491///
492/// Example (a 2-D vector.transfer_write):
493/// ```
494/// vector.transfer_write %arg, %0[...], %pad : memref<A...>, vector<...>
495/// ```
496/// is transformed into:
497/// ```
498/// %1:3 = scf.if (%inBounds) {
499/// memref.cast %A: memref<A...> to compatibleMemRefType
500/// scf.yield %view : compatibleMemRefType, index, index
501/// } else {
502/// memref.cast %alloc: memref<B...> to compatibleMemRefType
503/// scf.yield %4 : compatibleMemRefType, index, index
504/// }
505/// %0 = vector.transfer_write %arg, %1#0[%1#1, %1#2] {in_bounds = [true ...
506/// true]}
507/// scf.if (%notInBounds) {
508/// // slowpath: not in-bounds vector.transfer or memref.copy.
509/// }
510/// ```
511/// where `alloc` is a top of the function alloca'ed buffer of one vector.
512///
513/// Preconditions:
514/// 1. `xferOp.getPermutationMap()` must be a minor identity map
515/// 2. the rank of the `xferOp.getBase()` and the rank of the
516/// `xferOp.getVector()` must be equal. This will be relaxed in the future
517/// but requires rank-reducing subviews.
518LogicalResult mlir::vector::splitFullAndPartialTransfer(
519 RewriterBase &b, VectorTransferOpInterface xferOp,
520 VectorTransformsOptions options, scf::IfOp *ifOp) {
521 if (options.vectorTransferSplit == VectorTransferSplit::None)
522 return failure();
523
524 SmallVector<bool, 4> bools(xferOp.getTransferRank(), true);
525 auto inBoundsAttr = b.getBoolArrayAttr(values: bools);
526 if (options.vectorTransferSplit == VectorTransferSplit::ForceInBounds) {
527 b.modifyOpInPlace(root: xferOp, callable: [&]() {
528 xferOp->setInherentAttr(name: b.getStringAttr(bytes: xferOp.getInBoundsAttrName()),
529 value: inBoundsAttr);
530 });
531 return success();
532 }
533
534 // Assert preconditions. Additionally, keep the variables in an inner scope to
535 // ensure they aren't used in the wrong scopes further down.
536 {
537 assert(succeeded(splitFullAndPartialTransferPrecondition(xferOp)) &&
538 "Expected splitFullAndPartialTransferPrecondition to hold");
539
540 auto xferReadOp = dyn_cast<vector::TransferReadOp>(Val: xferOp.getOperation());
541 auto xferWriteOp = dyn_cast<vector::TransferWriteOp>(Val: xferOp.getOperation());
542
543 if (!(xferReadOp || xferWriteOp))
544 return failure();
545 if (xferWriteOp && xferWriteOp.getMask())
546 return failure();
547 if (xferReadOp && xferReadOp.getMask())
548 return failure();
549 }
550
551 RewriterBase::InsertionGuard guard(b);
552 b.setInsertionPoint(xferOp);
553 Value inBoundsCond = createInBoundsCond(
554 b, xferOp: cast<VectorTransferOpInterface>(Val: xferOp.getOperation()));
555 if (!inBoundsCond)
556 return failure();
557
558 // Top of the function `alloc` for transient storage.
559 Value alloc;
560 {
561 RewriterBase::InsertionGuard guard(b);
562 Operation *scope = getAutomaticAllocationScope(op: xferOp);
563 assert(scope->getNumRegions() == 1 &&
564 "AutomaticAllocationScope with >1 regions");
565 b.setInsertionPointToStart(&scope->getRegion(index: 0).front());
566 auto shape = xferOp.getVectorType().getShape();
567 Type elementType = xferOp.getVectorType().getElementType();
568 alloc = memref::AllocaOp::create(builder&: b, location: scope->getLoc(),
569 memrefType: MemRefType::get(shape, elementType),
570 dynamicSizes: ValueRange{}, alignment: b.getI64IntegerAttr(value: 32));
571 }
572
573 MemRefType compatibleMemRefType =
574 getCastCompatibleMemRefType(aT: cast<MemRefType>(Val: xferOp.getShapedType()),
575 bT: cast<MemRefType>(Val: alloc.getType()));
576 if (!compatibleMemRefType)
577 return failure();
578
579 SmallVector<Type, 4> returnTypes(1 + xferOp.getTransferRank(),
580 b.getIndexType());
581 returnTypes[0] = compatibleMemRefType;
582
583 if (auto xferReadOp =
584 dyn_cast<vector::TransferReadOp>(Val: xferOp.getOperation())) {
585 // Read case: full fill + partial copy -> in-bounds vector.xfer_read.
586 scf::IfOp fullPartialIfOp =
587 options.vectorTransferSplit == VectorTransferSplit::VectorTransfer
588 ? createFullPartialVectorTransferRead(b, xferOp: xferReadOp, returnTypes,
589 inBoundsCond,
590 compatibleMemRefType, alloc)
591 : createFullPartialMemRefCopy(b, xferOp: xferReadOp, returnTypes,
592 inBoundsCond, compatibleMemRefType,
593 alloc);
594 if (ifOp)
595 *ifOp = fullPartialIfOp;
596
597 // Set existing read op to in-bounds, it always reads from a full buffer.
598 for (unsigned i = 0, e = returnTypes.size(); i != e; ++i)
599 xferReadOp.setOperand(i, value: fullPartialIfOp.getResult(i));
600
601 b.modifyOpInPlace(root: xferOp, callable: [&]() {
602 xferOp->setInherentAttr(name: b.getStringAttr(bytes: xferOp.getInBoundsAttrName()),
603 value: inBoundsAttr);
604 });
605
606 return success();
607 }
608
609 auto xferWriteOp = cast<vector::TransferWriteOp>(Val: xferOp.getOperation());
610
611 // Decide which location to write the entire vector to.
612 auto memrefAndIndices = getLocationToWriteFullVec(
613 b, xferOp: xferWriteOp, returnTypes, inBoundsCond, compatibleMemRefType, alloc);
614
615 // Do an in bounds write to either the output or the extra allocated buffer.
616 // The operation is cloned to prevent deleting information needed for the
617 // later IR creation.
618 IRMapping mapping;
619 mapping.map(from: xferWriteOp.getBase(), to: memrefAndIndices.front());
620 mapping.map(from: xferWriteOp.getIndices(), to: memrefAndIndices.drop_front());
621 auto *clone = b.clone(op&: *xferWriteOp, mapper&: mapping);
622 clone->setInherentAttr(name: xferWriteOp.getInBoundsAttrName(), value: inBoundsAttr);
623
624 // Create a potential copy from the allocated buffer to the final output in
625 // the slow path case.
626 if (options.vectorTransferSplit == VectorTransferSplit::VectorTransfer)
627 createFullPartialVectorTransferWrite(b, xferOp: xferWriteOp, inBoundsCond, alloc);
628 else
629 createFullPartialMemRefCopy(b, xferOp: xferWriteOp, inBoundsCond, alloc);
630
631 b.eraseOp(op: xferOp);
632
633 return success();
634}
635
636namespace {
637/// Apply `splitFullAndPartialTransfer` selectively via a pattern. This pattern
638/// may take an extra filter to perform selection at a finer granularity.
639struct VectorTransferFullPartialRewriter : public RewritePattern {
640 using FilterConstraintType =
641 std::function<LogicalResult(VectorTransferOpInterface op)>;
642
643 explicit VectorTransferFullPartialRewriter(
644 MLIRContext *context,
645 VectorTransformsOptions options = VectorTransformsOptions(),
646 FilterConstraintType filter =
647 [](VectorTransferOpInterface op) { return success(); },
648 PatternBenefit benefit = 1)
649 : RewritePattern(MatchAnyOpTypeTag(), benefit, context), options(options),
650 filter(std::move(filter)) {}
651
652 /// Performs the rewrite.
653 LogicalResult matchAndRewrite(Operation *op,
654 PatternRewriter &rewriter) const override;
655
656private:
657 VectorTransformsOptions options;
658 FilterConstraintType filter;
659};
660
661} // namespace
662
663LogicalResult VectorTransferFullPartialRewriter::matchAndRewrite(
664 Operation *op, PatternRewriter &rewriter) const {
665 auto xferOp = dyn_cast<VectorTransferOpInterface>(Val: op);
666 if (!xferOp || failed(Result: splitFullAndPartialTransferPrecondition(xferOp)) ||
667 failed(Result: filter(xferOp)))
668 return failure();
669 return splitFullAndPartialTransfer(b&: rewriter, xferOp, options);
670}
671
672void mlir::vector::populateVectorTransferFullPartialPatterns(
673 RewritePatternSet &patterns, const VectorTransformsOptions &options) {
674 patterns.add<VectorTransferFullPartialRewriter>(arg: patterns.getContext(),
675 args: options);
676}
677