1//===- VectorUtils.cpp - MLIR Utilities for VectorOps ------------------===//
2//
3// Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements utility methods for working with the Vector dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
14
15#include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h"
16#include "mlir/Dialect/Affine/IR/AffineOps.h"
17#include "mlir/Dialect/Arith/IR/Arith.h"
18#include "mlir/Dialect/Arith/Utils/Utils.h"
19#include "mlir/Dialect/Func/IR/FuncOps.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/Dialect/Tensor/IR/Tensor.h"
22#include "mlir/Dialect/Utils/IndexingUtils.h"
23#include "mlir/Dialect/Vector/IR/VectorOps.h"
24#include "mlir/IR/Builders.h"
25#include "mlir/IR/IntegerSet.h"
26#include "mlir/IR/Matchers.h"
27#include "mlir/IR/Operation.h"
28#include "mlir/IR/TypeUtilities.h"
29#include "mlir/Support/LLVM.h"
30
31#include "llvm/ADT/DenseSet.h"
32#include "llvm/Support/DebugLog.h"
33#include "llvm/Support/InterleavedRange.h"
34
35#define DEBUG_TYPE "vector-utils"
36
37using namespace mlir;
38
39/// Helper function that creates a memref::DimOp or tensor::DimOp depending on
40/// the type of `source`.
41Value mlir::vector::createOrFoldDimOp(OpBuilder &b, Location loc, Value source,
42 int64_t dim) {
43 if (isa<UnrankedMemRefType, MemRefType>(Val: source.getType()))
44 return b.createOrFold<memref::DimOp>(location: loc, args&: source, args&: dim);
45 if (isa<UnrankedTensorType, RankedTensorType>(Val: source.getType()))
46 return b.createOrFold<tensor::DimOp>(location: loc, args&: source, args&: dim);
47 llvm_unreachable("Expected MemRefType or TensorType");
48}
49
50/// Given the n-D transpose pattern 'transp', return true if 'dim0' and 'dim1'
51/// should be transposed with each other within the context of their 2D
52/// transposition slice.
53///
54/// Example 1: dim0 = 0, dim1 = 2, transp = [2, 1, 0]
55/// Return true: dim0 and dim1 are transposed within the context of their 2D
56/// transposition slice ([1, 0]).
57///
58/// Example 2: dim0 = 0, dim1 = 1, transp = [2, 1, 0]
59/// Return true: dim0 and dim1 are transposed within the context of their 2D
60/// transposition slice ([1, 0]). Paradoxically, note how dim1 (1) is *not*
61/// transposed within the full context of the transposition.
62///
63/// Example 3: dim0 = 0, dim1 = 1, transp = [2, 0, 1]
64/// Return false: dim0 and dim1 are *not* transposed within the context of
65/// their 2D transposition slice ([0, 1]). Paradoxically, note how dim0 (0)
66/// and dim1 (1) are transposed within the full context of the of the
67/// transposition.
68static bool areDimsTransposedIn2DSlice(int64_t dim0, int64_t dim1,
69 ArrayRef<int64_t> transp) {
70 // Perform a linear scan along the dimensions of the transposed pattern. If
71 // dim0 is found first, dim0 and dim1 are not transposed within the context of
72 // their 2D slice. Otherwise, 'dim1' is found first and they are transposed.
73 for (int64_t permDim : transp) {
74 if (permDim == dim0)
75 return false;
76 if (permDim == dim1)
77 return true;
78 }
79
80 llvm_unreachable("Ill-formed transpose pattern");
81}
82
83FailureOr<std::pair<int, int>>
84mlir::vector::isTranspose2DSlice(vector::TransposeOp op) {
85 VectorType srcType = op.getSourceVectorType();
86 SmallVector<int64_t> srcGtOneDims;
87 for (auto [index, size] : llvm::enumerate(First: srcType.getShape()))
88 if (size > 1)
89 srcGtOneDims.push_back(Elt: index);
90
91 if (srcGtOneDims.size() != 2)
92 return failure();
93
94 // Check whether the two source vector dimensions that are greater than one
95 // must be transposed with each other so that we can apply one of the 2-D
96 // transpose patterns. Otherwise, these patterns are not applicable.
97 if (!areDimsTransposedIn2DSlice(dim0: srcGtOneDims[0], dim1: srcGtOneDims[1],
98 transp: op.getPermutation()))
99 return failure();
100
101 return std::pair<int, int>(srcGtOneDims[0], srcGtOneDims[1]);
102}
103
104/// Constructs a permutation map from memref indices to vector dimension.
105///
106/// The implementation uses the knowledge of the mapping of enclosing loop to
107/// vector dimension. `enclosingLoopToVectorDim` carries this information as a
108/// map with:
109/// - keys representing "vectorized enclosing loops";
110/// - values representing the corresponding vector dimension.
111/// The algorithm traverses "vectorized enclosing loops" and extracts the
112/// at-most-one MemRef index that is invariant along said loop. This index is
113/// guaranteed to be at most one by construction: otherwise the MemRef is not
114/// vectorizable.
115/// If this invariant index is found, it is added to the permutation_map at the
116/// proper vector dimension.
117/// If no index is found to be invariant, 0 is added to the permutation_map and
118/// corresponds to a vector broadcast along that dimension.
119///
120/// Returns an empty AffineMap if `enclosingLoopToVectorDim` is empty,
121/// signalling that no permutation map can be constructed given
122/// `enclosingLoopToVectorDim`.
123///
124/// Examples can be found in the documentation of `makePermutationMap`, in the
125/// header file.
126static AffineMap makePermutationMap(
127 ArrayRef<Value> indices,
128 const DenseMap<Operation *, unsigned> &enclosingLoopToVectorDim) {
129 if (enclosingLoopToVectorDim.empty())
130 return AffineMap();
131 MLIRContext *context =
132 enclosingLoopToVectorDim.begin()->getFirst()->getContext();
133 SmallVector<AffineExpr> perm(enclosingLoopToVectorDim.size(),
134 getAffineConstantExpr(constant: 0, context));
135
136 for (auto kvp : enclosingLoopToVectorDim) {
137 assert(kvp.second < perm.size());
138 auto invariants = affine::getInvariantAccesses(
139 iv: cast<affine::AffineForOp>(Val: kvp.first).getInductionVar(), indices);
140 unsigned numIndices = indices.size();
141 unsigned countInvariantIndices = 0;
142 for (unsigned dim = 0; dim < numIndices; ++dim) {
143 if (!invariants.count(V: indices[dim])) {
144 assert(perm[kvp.second] == getAffineConstantExpr(0, context) &&
145 "permutationMap already has an entry along dim");
146 perm[kvp.second] = getAffineDimExpr(position: dim, context);
147 } else {
148 ++countInvariantIndices;
149 }
150 }
151 assert((countInvariantIndices == numIndices ||
152 countInvariantIndices == numIndices - 1) &&
153 "Vectorization prerequisite violated: at most 1 index may be "
154 "invariant wrt a vectorized loop");
155 (void)countInvariantIndices;
156 }
157 return AffineMap::get(dimCount: indices.size(), symbolCount: 0, results: perm, context);
158}
159
160/// Implementation detail that walks up the parents and records the ones with
161/// the specified type.
162/// TODO: could also be implemented as a collect parents followed by a
163/// filter and made available outside this file.
164template <typename T>
165static SetVector<Operation *> getParentsOfType(Block *block) {
166 SetVector<Operation *> res;
167 auto *current = block->getParentOp();
168 while (current) {
169 if ([[maybe_unused]] auto typedParent = dyn_cast<T>(current)) {
170 assert(res.count(current) == 0 && "Already inserted");
171 res.insert(X: current);
172 }
173 current = current->getParentOp();
174 }
175 return res;
176}
177
178/// Returns the enclosing AffineForOp, from closest to farthest.
179static SetVector<Operation *> getEnclosingforOps(Block *block) {
180 return getParentsOfType<affine::AffineForOp>(block);
181}
182
183AffineMap mlir::makePermutationMap(
184 Block *insertPoint, ArrayRef<Value> indices,
185 const DenseMap<Operation *, unsigned> &loopToVectorDim) {
186 DenseMap<Operation *, unsigned> enclosingLoopToVectorDim;
187 auto enclosingLoops = getEnclosingforOps(block: insertPoint);
188 for (auto *forInst : enclosingLoops) {
189 auto it = loopToVectorDim.find(Val: forInst);
190 if (it != loopToVectorDim.end()) {
191 enclosingLoopToVectorDim.insert(KV: *it);
192 }
193 }
194 return ::makePermutationMap(indices, enclosingLoopToVectorDim);
195}
196
197AffineMap mlir::makePermutationMap(
198 Operation *op, ArrayRef<Value> indices,
199 const DenseMap<Operation *, unsigned> &loopToVectorDim) {
200 return makePermutationMap(insertPoint: op->getBlock(), indices, loopToVectorDim);
201}
202
203bool matcher::operatesOnSuperVectorsOf(Operation &op,
204 VectorType subVectorType) {
205 // First, extract the vector type and distinguish between:
206 // a. ops that *must* lower a super-vector (i.e. vector.transfer_read,
207 // vector.transfer_write); and
208 // b. ops that *may* lower a super-vector (all other ops).
209 // The ops that *may* lower a super-vector only do so if the super-vector to
210 // sub-vector ratio exists. The ops that *must* lower a super-vector are
211 // explicitly checked for this property.
212 /// TODO: there should be a single function for all ops to do this so we
213 /// do not have to special case. Maybe a trait, or just a method, unclear atm.
214 VectorType superVectorType;
215 if (auto transfer = dyn_cast<VectorTransferOpInterface>(Val&: op)) {
216 superVectorType = transfer.getVectorType();
217 } else if (op.getNumResults() == 0) {
218 if (!isa<func::ReturnOp>(Val: op)) {
219 op.emitError(message: "NYI: assuming only return operations can have 0 "
220 " results at this point");
221 }
222 return false;
223 } else if (op.getNumResults() == 1) {
224 if (auto v = dyn_cast<VectorType>(Val: op.getResult(idx: 0).getType())) {
225 superVectorType = v;
226 } else {
227 // Not a vector type.
228 return false;
229 }
230 } else {
231 // Not a vector.transfer and has more than 1 result, fail hard for now to
232 // wake us up when something changes.
233 op.emitError(message: "NYI: operation has more than 1 result");
234 return false;
235 }
236
237 // Get the ratio. If the shapes are incompatible (e.g., different ranks or
238 // non-integer divisibility), the operation does not operate on a super-vector
239 // of the given sub-vector type.
240 auto ratio =
241 computeShapeRatio(shape: superVectorType.getShape(), subShape: subVectorType.getShape());
242 return ratio.has_value();
243}
244
245bool vector::isContiguousSlice(MemRefType memrefType, VectorType vectorType) {
246 if (vectorType.isScalable())
247 return false;
248
249 // Ignore a leading sequence of adjacent unit dimensions in the vector.
250 ArrayRef<int64_t> vectorShape =
251 vectorType.getShape().drop_while(Pred: [](auto v) { return v == 1; });
252 auto vecRank = vectorShape.size();
253
254 // A single element is always contiguous.
255 if (vecRank == 0)
256 return true;
257
258 if (!memrefType.areTrailingDimsContiguous(n: vecRank))
259 return false;
260
261 // Extract the trailing dims of the input memref
262 auto memrefShape = memrefType.getShape().take_back(N: vecRank);
263
264 // Compare the dims of `vectorType` against `memrefType`.
265 // All of the dimensions, except the first must match.
266 return llvm::equal(LRange: vectorShape.drop_front(), RRange: memrefShape.drop_front());
267}
268
269std::optional<StaticTileOffsetRange>
270vector::createUnrollIterator(VectorType vType, int64_t targetRank) {
271 if (vType.getRank() <= targetRank)
272 return {};
273 // Attempt to unroll until targetRank or the first scalable dimension (which
274 // cannot be unrolled).
275 auto shapeToUnroll = vType.getShape().drop_back(N: targetRank);
276 auto inputScalableVecDimsToUnroll =
277 vType.getScalableDims().drop_back(N: targetRank);
278 const auto *it = llvm::find(Range&: inputScalableVecDimsToUnroll, Val: true);
279 auto firstScalableDim = it - inputScalableVecDimsToUnroll.begin();
280 if (firstScalableDim == 0)
281 return {};
282 // All scalable dimensions should be removed now.
283 inputScalableVecDimsToUnroll =
284 inputScalableVecDimsToUnroll.slice(N: 0, M: firstScalableDim);
285 assert(!llvm::is_contained(inputScalableVecDimsToUnroll, true) &&
286 "unexpected leading scalable dimension");
287 // Create an unroll iterator for leading dimensions.
288 shapeToUnroll = shapeToUnroll.slice(N: 0, M: firstScalableDim);
289 return StaticTileOffsetRange(shapeToUnroll, /*unrollStep=*/1);
290}
291
292SmallVector<OpFoldResult> vector::getMixedSizesXfer(bool hasTensorSemantics,
293 Operation *xfer,
294 RewriterBase &rewriter) {
295 auto loc = xfer->getLoc();
296
297 Value base =
298 TypeSwitch<Operation *, Value>(xfer)
299 .Case(caseFn: [&](vector::TransferReadOp readOp) { return readOp.getBase(); })
300 .Case(caseFn: [&](vector::TransferWriteOp writeOp) {
301 return writeOp.getOperand(i: 1);
302 });
303
304 SmallVector<OpFoldResult> mixedSourceDims =
305 hasTensorSemantics ? tensor::getMixedSizes(builder&: rewriter, loc, value: base)
306 : memref::getMixedSizes(builder&: rewriter, loc, value: base);
307 return mixedSourceDims;
308}
309
310bool vector::isLinearizableVector(VectorType type) {
311 return (type.getRank() > 1) && (type.getNumScalableDims() <= 1);
312}
313
314/// Determines whether a mask for xfer_read/write is trivially "all true"
315///
316/// Given all the inputs required to generate a mask (mask sizes and shapes),
317/// and an xfer_read/write operation (indices and the source/destination tensor
318/// shape), determines whether the corresponding mask would be trivially
319/// foldable (i.e., trivially "all true").
320///
321/// Use this method to avoid generating spurious masks and relying on
322/// vectorization post-processing to remove them.
323///
324/// Pre-conditions for a mask to be trivially foldable:
325/// * All involved shapes (mask + destination tensor) are static.
326/// * All indices are constant.
327/// * All mask sizes are constant (including `arith.constant`).
328///
329/// If the pre-conditions are met, the method checks for each destination
330/// dimension `d`:
331/// (1) maskShape[d] <= destDimSize[rankDiff + d]
332/// (2) index[d] + maskSize[d] <= destDimSize[rankDiff + d]
333/// (3) maskShape[d] <= maskSize[d]
334///
335/// rankDiff = rank(dest) - rank(mask).
336///
337/// This method takes a conservative view: it may return false even if the mask
338/// is technically foldable.
339///
340/// EXAMPLE 1 (trivially foldable, all shapes match, mask sizes match the shape
341/// of the dest tensor):
342/// %c0 = arith.constant 0 : index
343/// %mask = vector.create_mask 5, 1
344/// vector.mask %mask {
345/// vector.transfer_write %vecToStore_1, %dest{[%c0, %c0]
346/// {in_bounds = [true, true]}
347/// : vector<5x1xi32>, tensor<5x1xi32>
348/// }
349///
350/// EXAMPLE 2 (not trivially foldable - vector shape exceeds the tensor shape,
351/// mask is required to avoid out-of-bounds write):
352/// %c0 = arith.constant 0 : index
353/// %mask = vector.create_mask 5, 1
354/// vector.mask %mask {
355/// vector.transfer_write %vecToStore_2, %dest[%c0, %c0]
356/// {in_bounds = [true, true]}
357/// : vector<8x1xi32>, tensor<5x1xi32>
358/// }
359static bool isMaskTriviallyFoldable(SmallVector<OpFoldResult> &maskSizes,
360 SmallVector<Value> &indices,
361 ArrayRef<int64_t> baseShape,
362 ArrayRef<int64_t> maskShape) {
363 // Masking is unavoidable in the case of dynamic tensors.
364 if (ShapedType::isDynamicShape(dSizes: baseShape))
365 return false;
366
367 // Collect all constant mask sizes.
368 SmallVector<int64_t, 4> cstMaskSizes;
369 for (auto [i, dimSize] : llvm::enumerate(First&: maskSizes)) {
370 if (auto intSize = getConstantIntValue(ofr: dimSize)) {
371 cstMaskSizes.push_back(Elt: *intSize);
372 }
373 }
374
375 // If any of the mask sizes is non-constant, bail out.
376 if (cstMaskSizes.size() != maskShape.size())
377 return false;
378
379 // Collect all constant indices.
380 SmallVector<int64_t, 4> cstIndices;
381 for (auto [i, idx] : llvm::enumerate(First&: indices)) {
382 APSInt intVal;
383 if (matchPattern(value: idx, pattern: m_ConstantInt(bind_value: &intVal))) {
384 cstIndices.push_back(Elt: intVal.getSExtValue());
385 }
386 }
387
388 // If any of the indices is non-constant, bail out.
389 if (cstIndices.size() != baseShape.size())
390 return false;
391
392 // Go over all destination dims and check (1), (2) and (3). Take into account
393 // that:
394 // * The number of mask sizes will match the rank of the vector to
395 // load/store. This could be lower than the rank of the destination tensor.
396 // * Mask sizes could be larger than the corresponding mask shape (hence
397 // `clamp`).
398 // TODO: The 2nd item should be rejected by the verifier.
399 int64_t rankDiff = baseShape.size() - cstMaskSizes.size();
400 for (auto [i, idx] : llvm::enumerate(First&: cstMaskSizes)) {
401 if (/*(1)*/ maskShape[i] > baseShape[rankDiff + i] ||
402 /*(2)*/ baseShape[rankDiff + i] <
403 (std::clamp(val: cstMaskSizes[i], lo: int64_t(0), hi: maskShape[i]) +
404 cstIndices[i]) ||
405 /*(3)*/ maskShape[i] > cstMaskSizes[i])
406 return false;
407 }
408
409 return true;
410}
411
412Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
413 Value source,
414 ArrayRef<int64_t> inputVectorSizes,
415 std::optional<Value> padValue,
416 bool useInBoundsInsteadOfMasking,
417 ArrayRef<bool> inputScalableVecDims) {
418 VectorType vecToReadTy = VectorType::get(
419 shape: inputVectorSizes, elementType: cast<ShapedType>(Val: source.getType()).getElementType(),
420 scalableDims: inputScalableVecDims);
421
422 return createReadOrMaskedRead(builder, loc, source, vecToReadTy, padValue,
423 useInBoundsInsteadOfMasking);
424}
425
426/// Compute the in_bounds attribute for a transfer op given its permutation map
427/// and the source being accessed. Dimension i is in-bounds when the map result
428/// is an AffineDimExpr pointing to a static source dimension divisible by the
429/// vector size, or an AffineConstantExpr (broadcast).
430static SmallVector<bool> computeInBoundsFromPermutationMap(
431 AffineMap permutationMap, VectorType vectorType, ShapedType sourceType) {
432 SmallVector<bool> inBounds(vectorType.getRank(), false);
433 for (unsigned i = 0; i < (unsigned)vectorType.getRank(); ++i) {
434 AffineExpr expr = permutationMap.getResult(idx: i);
435 if (auto dimExpr = dyn_cast<AffineDimExpr>(Val&: expr)) {
436 unsigned memDim = dimExpr.getPosition();
437 if (!sourceType.isDynamicDim(idx: memDim) &&
438 sourceType.getDimSize(idx: memDim) % vectorType.getDimSize(idx: i) == 0)
439 inBounds[i] = true;
440 } else if (isa<AffineConstantExpr>(Val: expr)) {
441 inBounds[i] = true;
442 }
443 }
444 return inBounds;
445}
446
447Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc,
448 Value source,
449 const VectorType &vecToReadTy,
450 std::optional<Value> padValue,
451 bool useInBoundsInsteadOfMasking,
452 ArrayRef<Value> customIndices,
453 AffineMap permutationMap) {
454 assert(!llvm::is_contained(vecToReadTy.getScalableDims(),
455 ShapedType::kDynamic) &&
456 "invalid input vector sizes");
457 auto sourceShapedType = cast<ShapedType>(Val: source.getType());
458 auto sourceShape = sourceShapedType.getShape();
459
460 int64_t vecToReadRank = vecToReadTy.getRank();
461 auto vecToReadShape = vecToReadTy.getShape();
462
463 // The permutation map maps the source's index space to the vector's, so its
464 // dims must match the source rank and its results the vector rank. Without a
465 // map, a minor identity is implied, requiring the two ranks to match.
466 assert(sourceShape.size() == (permutationMap
467 ? permutationMap.getNumDims()
468 : static_cast<size_t>(vecToReadRank)) &&
469 "expected source rank to match permutation map dims or vector rank.");
470 assert((!permutationMap || permutationMap.getNumResults() ==
471 static_cast<size_t>(vecToReadRank)) &&
472 "expected permutation map results to match vector rank.");
473 assert((!padValue.has_value() ||
474 padValue.value().getType() == sourceShapedType.getElementType()) &&
475 "expected same pad element type to match source element type");
476
477 SmallVector<bool> inBoundsVal(vecToReadRank, true);
478
479 if (useInBoundsInsteadOfMasking) {
480 if (permutationMap) {
481 // Update the inBounds attribute.
482 // FIXME: This computation is too weak - it ignores the read indices.
483 inBoundsVal = computeInBoundsFromPermutationMap(
484 permutationMap, vectorType: vecToReadTy, sourceType: cast<ShapedType>(Val: source.getType()));
485 } else {
486 // Update the inBounds attribute.
487 // FIXME: This computation is too weak - it ignores the read indices.
488 for (unsigned i = 0; i < vecToReadRank; i++)
489 inBoundsVal[i] = (sourceShape[i] == vecToReadShape[i]) &&
490 ShapedType::isStatic(dValue: sourceShape[i]);
491 }
492 }
493 // The transfer op expects one index per source dimension.
494 assert(
495 (customIndices.empty() || customIndices.size() == sourceShape.size()) &&
496 "expected as many custom indices as source dims.");
497 SmallVector<Value> indices;
498 customIndices.empty()
499 ? indices.assign(NumElts: sourceShape.size(),
500 Elt: arith::ConstantIndexOp::create(builder, location: loc, value: 0))
501 : indices.assign(in_start: customIndices.begin(), in_end: customIndices.end());
502
503 // A null permutation map means the builder defaults to a minor identity map.
504 auto transferReadOp =
505 vector::TransferReadOp::create(builder, location: loc, /*vectorType=*/vecToReadTy,
506 /*source=*/source,
507 /*indices=*/indices,
508 /*padding=*/padValue,
509 /*permutationMap=*/permutationMap,
510 /*inBounds=*/inBoundsVal);
511
512 if (useInBoundsInsteadOfMasking)
513 return transferReadOp;
514
515 SmallVector<OpFoldResult> mixedSourceDims =
516 isa<MemRefType>(Val: source.getType())
517 ? memref::getMixedSizes(builder, loc, value: source)
518 : tensor::getMixedSizes(builder, loc, value: source);
519
520 if (isMaskTriviallyFoldable(maskSizes&: mixedSourceDims, indices, baseShape: sourceShape,
521 maskShape: vecToReadShape))
522 return transferReadOp;
523
524 auto maskType = vecToReadTy.cloneWith(/*shape=*/{}, elementType: builder.getI1Type());
525 Value mask =
526 vector::CreateMaskOp::create(builder, location: loc, type: maskType, mixedOperands: mixedSourceDims);
527 return mlir::vector::maskOperation(builder, maskableOp: transferReadOp, mask)
528 ->getResult(idx: 0);
529}
530
531Operation *vector::createWriteOrMaskedWrite(OpBuilder &builder, Location loc,
532 Value vecToStore, Value dest,
533 SmallVector<Value> writeIndices,
534 bool useInBoundsInsteadOfMasking,
535 AffineMap permutationMap) {
536
537 ShapedType destType = cast<ShapedType>(Val: dest.getType());
538 int64_t destRank = destType.getRank();
539 auto destShape = destType.getShape();
540
541 VectorType vecToStoreType = cast<VectorType>(Val: vecToStore.getType());
542 int64_t vecToStoreRank = vecToStoreType.getRank();
543 auto vecToStoreShape = vecToStoreType.getShape();
544
545 // Compute the in_bounds attribute
546 SmallVector<bool> inBoundsVal(vecToStoreRank, true);
547 if (useInBoundsInsteadOfMasking) {
548 if (permutationMap) {
549 // Update the inBounds attribute.
550 // FIXME: This computation is too weak - it ignores the write indices.
551 inBoundsVal = computeInBoundsFromPermutationMap(
552 permutationMap, vectorType: vecToStoreType, sourceType: cast<ShapedType>(Val: dest.getType()));
553 } else {
554 // Update the inBounds attribute.
555 // FIXME: This computation is too weak - it ignores the write indices.
556 for (unsigned i = 0; i < vecToStoreRank; i++)
557 inBoundsVal[i] =
558 (destShape[destRank - vecToStoreRank + i] >= vecToStoreShape[i]) &&
559 ShapedType::isStatic(dValue: destShape[destRank - vecToStoreRank + i]);
560 }
561 }
562
563 // If missing, initialize the write indices to 0.
564 bool useDefaultWriteIdxs = writeIndices.empty();
565 assert((useDefaultWriteIdxs ||
566 writeIndices.size() == static_cast<size_t>(destRank)) &&
567 "Invalid number of write indices!");
568 if (useDefaultWriteIdxs) {
569 auto zero = arith::ConstantIndexOp::create(builder, location: loc, value: 0);
570 writeIndices.assign(NumElts: destRank, Elt: zero);
571 }
572
573 // Generate the xfer_write Op. A null permutation map means the builder
574 // defaults to a minor identity map.
575 Operation *write =
576 vector::TransferWriteOp::create(builder, location: loc,
577 /*vector=*/vecToStore,
578 /*dest=*/dest,
579 /*indices=*/writeIndices,
580 /*permutationMap=*/permutationMap,
581 /*inBounds=*/inBoundsVal);
582
583 // If masking is disabled, exit.
584 if (useInBoundsInsteadOfMasking)
585 return write;
586
587 // The dest dims written to are those the permutation map selects. A null map
588 // means a minor identity: the trailing dims.
589 AffineMap map = permutationMap
590 ? permutationMap
591 : AffineMap::getMinorIdentityMap(dims: destRank, results: vecToStoreRank,
592 context: builder.getContext());
593
594 // A write that starts at 0 and fills the dims it writes to needs no mask.
595 bool startsAtZero = llvm::all_of(Range: map.getResults(), P: [&](AffineExpr expr) {
596 return isZeroInteger(v: writeIndices[cast<AffineDimExpr>(Val&: expr).getPosition()]);
597 });
598 if (startsAtZero &&
599 llvm::equal(LRange&: vecToStoreShape, RRange: applyPermutationMap(map, source: destShape)))
600 return write;
601
602 // Compute the mask and mask the write Op.
603 VectorType writeMaskType = inferTransferOpMaskType(vecType: vecToStoreType, permMap: map);
604
605 SmallVector<OpFoldResult> destSizes =
606 isa<MemRefType>(Val: dest.getType())
607 ? memref::getMixedSizes(builder, loc, value: dest)
608 : tensor::getMixedSizes(builder, loc, value: dest);
609
610 // The map has a dim per dest dim but a result per vector dim, and the vector
611 // can have fewer dims than the dest (e.g. rank-reducing `insert_slice`). The
612 // mask has a dim for each dest dim that the map selects, in increasing order.
613 // For each, collect the mask size (`dim - index`), write index and dest size.
614 SmallVector<int64_t> maskedDestDims;
615 for (AffineExpr expr : map.getResults())
616 maskedDestDims.push_back(Elt: cast<AffineDimExpr>(Val&: expr).getPosition());
617 // A transposing map selects them out of order.
618 llvm::sort(C&: maskedDestDims);
619 SmallVector<OpFoldResult> maskSizes;
620 SmallVector<Value> maskedDestIndices;
621 SmallVector<int64_t> maskedDestShape;
622 for (int64_t dim : maskedDestDims) {
623 Value size = getValueOrCreateConstantIndexOp(b&: builder, loc, ofr: destSizes[dim]);
624 maskSizes.push_back(
625 Elt: builder.createOrFold<arith::SubIOp>(location: loc, args&: size, args&: writeIndices[dim]));
626 maskedDestIndices.push_back(Elt: writeIndices[dim]);
627 maskedDestShape.push_back(Elt: destShape[dim]);
628 }
629
630 if (isMaskTriviallyFoldable(maskSizes, indices&: maskedDestIndices, baseShape: maskedDestShape,
631 maskShape: writeMaskType.getShape()))
632 return write;
633
634 Value maskForWrite =
635 builder.createOrFold<vector::CreateMaskOp>(location: loc, args&: writeMaskType, args&: maskSizes);
636 return mlir::vector::maskOperation(builder, maskableOp: write, mask: maskForWrite);
637}
638
639LogicalResult
640vector::isValidMaskedInputVector(ArrayRef<int64_t> shape,
641 ArrayRef<int64_t> inputVectorSizes) {
642 LDBG() << "Iteration space static sizes:" << llvm::interleaved(R: shape);
643
644 if (inputVectorSizes.size() != shape.size()) {
645 LDBG() << "Input vector sizes don't match the number of loops";
646 return failure();
647 }
648 if (ShapedType::isDynamicShape(dSizes: inputVectorSizes)) {
649 LDBG() << "Input vector sizes can't have dynamic dimensions";
650 return failure();
651 }
652 if (!llvm::all_of(Range: llvm::zip(t&: shape, u&: inputVectorSizes),
653 P: [](std::tuple<int64_t, int64_t> sizePair) {
654 int64_t staticSize = std::get<0>(t&: sizePair);
655 int64_t inputSize = std::get<1>(t&: sizePair);
656 return ShapedType::isDynamic(dValue: staticSize) ||
657 staticSize <= inputSize;
658 })) {
659 LDBG() << "Input vector sizes must be greater than or equal to iteration "
660 "space static sizes";
661 return failure();
662 }
663 return success();
664}
665
666/// Takes a 2+ dimensional vector as an input
667/// returns n vector values produced by n vector.extract operations.
668/// I.e. calling unrollVectorValue([[%v]], rewriter) such that
669///
670/// %v : vector<nxaxb...>
671///
672/// will produce the following IR changes
673///
674/// %v0 = vector.extract %v[0] : vector<axbx...> from vector<nxaxb...>
675/// %v1 = vector.extract %v[1] : vector<axbx...> from vector<nxaxb...>
676/// ...
677/// %vnminusone = vector.extract %v[n-1] : vector<axbx...> from ...
678///
679/// and returns SmallVector<Value> r = {[[%v0]], [[%v1]], ..., [[%vnminusone]]}
680FailureOr<SmallVector<Value>>
681vector::unrollVectorValue(TypedValue<VectorType> vector,
682 RewriterBase &rewriter) {
683 SmallVector<Value> subvectors;
684 VectorType ty = cast<VectorType>(Val: vector.getType());
685 Location loc = vector.getLoc();
686 if (ty.getRank() < 2)
687 return rewriter.notifyMatchFailure(arg&: loc, msg: "already 1-D");
688
689 // Unrolling doesn't take vscale into account. Pattern is disabled for
690 // vectors with leading scalable dim(s).
691 if (ty.getScalableDims().front())
692 return rewriter.notifyMatchFailure(arg&: loc, msg: "cannot unroll scalable dim");
693
694 for (int64_t i = 0, e = ty.getShape().front(); i < e; ++i) {
695 subvectors.push_back(Elt: vector::ExtractOp::create(builder&: rewriter, location: loc, source: vector, position: i));
696 }
697
698 return subvectors;
699}
700
701LogicalResult vector::unrollVectorOp(Operation *op, PatternRewriter &rewriter,
702 vector::UnrollVectorOpFn unrollFn) {
703 assert(op->getNumResults() == 1 && "expected single result");
704 assert(isa<VectorType>(op->getResult(0).getType()) && "expected vector type");
705 VectorType resultTy = cast<VectorType>(Val: op->getResult(idx: 0).getType());
706 if (resultTy.getRank() < 2)
707 return rewriter.notifyMatchFailure(arg&: op, msg: "already 1-D");
708
709 // Unrolling doesn't take vscale into account. Pattern is disabled for
710 // vectors with leading scalable dim(s).
711 if (resultTy.getScalableDims().front())
712 return rewriter.notifyMatchFailure(arg&: op, msg: "cannot unroll scalable dim");
713
714 Location loc = op->getLoc();
715 Value result = ub::PoisonOp::create(builder&: rewriter, location: loc, result: resultTy);
716 VectorType subTy = VectorType::Builder(resultTy).dropDim(pos: 0);
717
718 for (int64_t i = 0, e = resultTy.getShape().front(); i < e; ++i) {
719 Value subVector = unrollFn(rewriter, loc, subTy, i);
720 result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: subVector, dest: result, position: i);
721 }
722
723 rewriter.replaceOp(op, newValues: result);
724 return success();
725}
726