| 1 | //===- VectorUtils.cpp - MLIR Utilities for VectorOps ------------------===// |
| 2 | // |
| 3 | // Part of the MLIR Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements utility methods for working with the Vector dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/Vector/Utils/VectorUtils.h" |
| 14 | |
| 15 | #include "mlir/Dialect/Affine/Analysis/LoopAnalysis.h" |
| 16 | #include "mlir/Dialect/Affine/IR/AffineOps.h" |
| 17 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 18 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 19 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 20 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 21 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 22 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 23 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 24 | #include "mlir/IR/Builders.h" |
| 25 | #include "mlir/IR/IntegerSet.h" |
| 26 | #include "mlir/IR/Matchers.h" |
| 27 | #include "mlir/IR/Operation.h" |
| 28 | #include "mlir/IR/TypeUtilities.h" |
| 29 | #include "mlir/Support/LLVM.h" |
| 30 | |
| 31 | #include "llvm/ADT/DenseSet.h" |
| 32 | #include "llvm/Support/DebugLog.h" |
| 33 | #include "llvm/Support/InterleavedRange.h" |
| 34 | |
| 35 | #define DEBUG_TYPE "vector-utils" |
| 36 | |
| 37 | using namespace mlir; |
| 38 | |
| 39 | /// Helper function that creates a memref::DimOp or tensor::DimOp depending on |
| 40 | /// the type of `source`. |
| 41 | Value mlir::vector::createOrFoldDimOp(OpBuilder &b, Location loc, Value source, |
| 42 | int64_t dim) { |
| 43 | if (isa<UnrankedMemRefType, MemRefType>(Val: source.getType())) |
| 44 | return b.createOrFold<memref::DimOp>(location: loc, args&: source, args&: dim); |
| 45 | if (isa<UnrankedTensorType, RankedTensorType>(Val: source.getType())) |
| 46 | return b.createOrFold<tensor::DimOp>(location: loc, args&: source, args&: dim); |
| 47 | llvm_unreachable("Expected MemRefType or TensorType" ); |
| 48 | } |
| 49 | |
| 50 | /// Given the n-D transpose pattern 'transp', return true if 'dim0' and 'dim1' |
| 51 | /// should be transposed with each other within the context of their 2D |
| 52 | /// transposition slice. |
| 53 | /// |
| 54 | /// Example 1: dim0 = 0, dim1 = 2, transp = [2, 1, 0] |
| 55 | /// Return true: dim0 and dim1 are transposed within the context of their 2D |
| 56 | /// transposition slice ([1, 0]). |
| 57 | /// |
| 58 | /// Example 2: dim0 = 0, dim1 = 1, transp = [2, 1, 0] |
| 59 | /// Return true: dim0 and dim1 are transposed within the context of their 2D |
| 60 | /// transposition slice ([1, 0]). Paradoxically, note how dim1 (1) is *not* |
| 61 | /// transposed within the full context of the transposition. |
| 62 | /// |
| 63 | /// Example 3: dim0 = 0, dim1 = 1, transp = [2, 0, 1] |
| 64 | /// Return false: dim0 and dim1 are *not* transposed within the context of |
| 65 | /// their 2D transposition slice ([0, 1]). Paradoxically, note how dim0 (0) |
| 66 | /// and dim1 (1) are transposed within the full context of the of the |
| 67 | /// transposition. |
| 68 | static bool areDimsTransposedIn2DSlice(int64_t dim0, int64_t dim1, |
| 69 | ArrayRef<int64_t> transp) { |
| 70 | // Perform a linear scan along the dimensions of the transposed pattern. If |
| 71 | // dim0 is found first, dim0 and dim1 are not transposed within the context of |
| 72 | // their 2D slice. Otherwise, 'dim1' is found first and they are transposed. |
| 73 | for (int64_t permDim : transp) { |
| 74 | if (permDim == dim0) |
| 75 | return false; |
| 76 | if (permDim == dim1) |
| 77 | return true; |
| 78 | } |
| 79 | |
| 80 | llvm_unreachable("Ill-formed transpose pattern" ); |
| 81 | } |
| 82 | |
| 83 | FailureOr<std::pair<int, int>> |
| 84 | mlir::vector::isTranspose2DSlice(vector::TransposeOp op) { |
| 85 | VectorType srcType = op.getSourceVectorType(); |
| 86 | SmallVector<int64_t> srcGtOneDims; |
| 87 | for (auto [index, size] : llvm::enumerate(First: srcType.getShape())) |
| 88 | if (size > 1) |
| 89 | srcGtOneDims.push_back(Elt: index); |
| 90 | |
| 91 | if (srcGtOneDims.size() != 2) |
| 92 | return failure(); |
| 93 | |
| 94 | // Check whether the two source vector dimensions that are greater than one |
| 95 | // must be transposed with each other so that we can apply one of the 2-D |
| 96 | // transpose patterns. Otherwise, these patterns are not applicable. |
| 97 | if (!areDimsTransposedIn2DSlice(dim0: srcGtOneDims[0], dim1: srcGtOneDims[1], |
| 98 | transp: op.getPermutation())) |
| 99 | return failure(); |
| 100 | |
| 101 | return std::pair<int, int>(srcGtOneDims[0], srcGtOneDims[1]); |
| 102 | } |
| 103 | |
| 104 | /// Constructs a permutation map from memref indices to vector dimension. |
| 105 | /// |
| 106 | /// The implementation uses the knowledge of the mapping of enclosing loop to |
| 107 | /// vector dimension. `enclosingLoopToVectorDim` carries this information as a |
| 108 | /// map with: |
| 109 | /// - keys representing "vectorized enclosing loops"; |
| 110 | /// - values representing the corresponding vector dimension. |
| 111 | /// The algorithm traverses "vectorized enclosing loops" and extracts the |
| 112 | /// at-most-one MemRef index that is invariant along said loop. This index is |
| 113 | /// guaranteed to be at most one by construction: otherwise the MemRef is not |
| 114 | /// vectorizable. |
| 115 | /// If this invariant index is found, it is added to the permutation_map at the |
| 116 | /// proper vector dimension. |
| 117 | /// If no index is found to be invariant, 0 is added to the permutation_map and |
| 118 | /// corresponds to a vector broadcast along that dimension. |
| 119 | /// |
| 120 | /// Returns an empty AffineMap if `enclosingLoopToVectorDim` is empty, |
| 121 | /// signalling that no permutation map can be constructed given |
| 122 | /// `enclosingLoopToVectorDim`. |
| 123 | /// |
| 124 | /// Examples can be found in the documentation of `makePermutationMap`, in the |
| 125 | /// header file. |
| 126 | static AffineMap makePermutationMap( |
| 127 | ArrayRef<Value> indices, |
| 128 | const DenseMap<Operation *, unsigned> &enclosingLoopToVectorDim) { |
| 129 | if (enclosingLoopToVectorDim.empty()) |
| 130 | return AffineMap(); |
| 131 | MLIRContext *context = |
| 132 | enclosingLoopToVectorDim.begin()->getFirst()->getContext(); |
| 133 | SmallVector<AffineExpr> perm(enclosingLoopToVectorDim.size(), |
| 134 | getAffineConstantExpr(constant: 0, context)); |
| 135 | |
| 136 | for (auto kvp : enclosingLoopToVectorDim) { |
| 137 | assert(kvp.second < perm.size()); |
| 138 | auto invariants = affine::getInvariantAccesses( |
| 139 | iv: cast<affine::AffineForOp>(Val: kvp.first).getInductionVar(), indices); |
| 140 | unsigned numIndices = indices.size(); |
| 141 | unsigned countInvariantIndices = 0; |
| 142 | for (unsigned dim = 0; dim < numIndices; ++dim) { |
| 143 | if (!invariants.count(V: indices[dim])) { |
| 144 | assert(perm[kvp.second] == getAffineConstantExpr(0, context) && |
| 145 | "permutationMap already has an entry along dim" ); |
| 146 | perm[kvp.second] = getAffineDimExpr(position: dim, context); |
| 147 | } else { |
| 148 | ++countInvariantIndices; |
| 149 | } |
| 150 | } |
| 151 | assert((countInvariantIndices == numIndices || |
| 152 | countInvariantIndices == numIndices - 1) && |
| 153 | "Vectorization prerequisite violated: at most 1 index may be " |
| 154 | "invariant wrt a vectorized loop" ); |
| 155 | (void)countInvariantIndices; |
| 156 | } |
| 157 | return AffineMap::get(dimCount: indices.size(), symbolCount: 0, results: perm, context); |
| 158 | } |
| 159 | |
| 160 | /// Implementation detail that walks up the parents and records the ones with |
| 161 | /// the specified type. |
| 162 | /// TODO: could also be implemented as a collect parents followed by a |
| 163 | /// filter and made available outside this file. |
| 164 | template <typename T> |
| 165 | static SetVector<Operation *> getParentsOfType(Block *block) { |
| 166 | SetVector<Operation *> res; |
| 167 | auto *current = block->getParentOp(); |
| 168 | while (current) { |
| 169 | if ([[maybe_unused]] auto typedParent = dyn_cast<T>(current)) { |
| 170 | assert(res.count(current) == 0 && "Already inserted" ); |
| 171 | res.insert(X: current); |
| 172 | } |
| 173 | current = current->getParentOp(); |
| 174 | } |
| 175 | return res; |
| 176 | } |
| 177 | |
| 178 | /// Returns the enclosing AffineForOp, from closest to farthest. |
| 179 | static SetVector<Operation *> getEnclosingforOps(Block *block) { |
| 180 | return getParentsOfType<affine::AffineForOp>(block); |
| 181 | } |
| 182 | |
| 183 | AffineMap mlir::makePermutationMap( |
| 184 | Block *insertPoint, ArrayRef<Value> indices, |
| 185 | const DenseMap<Operation *, unsigned> &loopToVectorDim) { |
| 186 | DenseMap<Operation *, unsigned> enclosingLoopToVectorDim; |
| 187 | auto enclosingLoops = getEnclosingforOps(block: insertPoint); |
| 188 | for (auto *forInst : enclosingLoops) { |
| 189 | auto it = loopToVectorDim.find(Val: forInst); |
| 190 | if (it != loopToVectorDim.end()) { |
| 191 | enclosingLoopToVectorDim.insert(KV: *it); |
| 192 | } |
| 193 | } |
| 194 | return ::makePermutationMap(indices, enclosingLoopToVectorDim); |
| 195 | } |
| 196 | |
| 197 | AffineMap mlir::makePermutationMap( |
| 198 | Operation *op, ArrayRef<Value> indices, |
| 199 | const DenseMap<Operation *, unsigned> &loopToVectorDim) { |
| 200 | return makePermutationMap(insertPoint: op->getBlock(), indices, loopToVectorDim); |
| 201 | } |
| 202 | |
| 203 | bool matcher::operatesOnSuperVectorsOf(Operation &op, |
| 204 | VectorType subVectorType) { |
| 205 | // First, extract the vector type and distinguish between: |
| 206 | // a. ops that *must* lower a super-vector (i.e. vector.transfer_read, |
| 207 | // vector.transfer_write); and |
| 208 | // b. ops that *may* lower a super-vector (all other ops). |
| 209 | // The ops that *may* lower a super-vector only do so if the super-vector to |
| 210 | // sub-vector ratio exists. The ops that *must* lower a super-vector are |
| 211 | // explicitly checked for this property. |
| 212 | /// TODO: there should be a single function for all ops to do this so we |
| 213 | /// do not have to special case. Maybe a trait, or just a method, unclear atm. |
| 214 | VectorType superVectorType; |
| 215 | if (auto transfer = dyn_cast<VectorTransferOpInterface>(Val&: op)) { |
| 216 | superVectorType = transfer.getVectorType(); |
| 217 | } else if (op.getNumResults() == 0) { |
| 218 | if (!isa<func::ReturnOp>(Val: op)) { |
| 219 | op.emitError(message: "NYI: assuming only return operations can have 0 " |
| 220 | " results at this point" ); |
| 221 | } |
| 222 | return false; |
| 223 | } else if (op.getNumResults() == 1) { |
| 224 | if (auto v = dyn_cast<VectorType>(Val: op.getResult(idx: 0).getType())) { |
| 225 | superVectorType = v; |
| 226 | } else { |
| 227 | // Not a vector type. |
| 228 | return false; |
| 229 | } |
| 230 | } else { |
| 231 | // Not a vector.transfer and has more than 1 result, fail hard for now to |
| 232 | // wake us up when something changes. |
| 233 | op.emitError(message: "NYI: operation has more than 1 result" ); |
| 234 | return false; |
| 235 | } |
| 236 | |
| 237 | // Get the ratio. If the shapes are incompatible (e.g., different ranks or |
| 238 | // non-integer divisibility), the operation does not operate on a super-vector |
| 239 | // of the given sub-vector type. |
| 240 | auto ratio = |
| 241 | computeShapeRatio(shape: superVectorType.getShape(), subShape: subVectorType.getShape()); |
| 242 | return ratio.has_value(); |
| 243 | } |
| 244 | |
| 245 | bool vector::isContiguousSlice(MemRefType memrefType, VectorType vectorType) { |
| 246 | if (vectorType.isScalable()) |
| 247 | return false; |
| 248 | |
| 249 | // Ignore a leading sequence of adjacent unit dimensions in the vector. |
| 250 | ArrayRef<int64_t> vectorShape = |
| 251 | vectorType.getShape().drop_while(Pred: [](auto v) { return v == 1; }); |
| 252 | auto vecRank = vectorShape.size(); |
| 253 | |
| 254 | // A single element is always contiguous. |
| 255 | if (vecRank == 0) |
| 256 | return true; |
| 257 | |
| 258 | if (!memrefType.areTrailingDimsContiguous(n: vecRank)) |
| 259 | return false; |
| 260 | |
| 261 | // Extract the trailing dims of the input memref |
| 262 | auto memrefShape = memrefType.getShape().take_back(N: vecRank); |
| 263 | |
| 264 | // Compare the dims of `vectorType` against `memrefType`. |
| 265 | // All of the dimensions, except the first must match. |
| 266 | return llvm::equal(LRange: vectorShape.drop_front(), RRange: memrefShape.drop_front()); |
| 267 | } |
| 268 | |
| 269 | std::optional<StaticTileOffsetRange> |
| 270 | vector::createUnrollIterator(VectorType vType, int64_t targetRank) { |
| 271 | if (vType.getRank() <= targetRank) |
| 272 | return {}; |
| 273 | // Attempt to unroll until targetRank or the first scalable dimension (which |
| 274 | // cannot be unrolled). |
| 275 | auto shapeToUnroll = vType.getShape().drop_back(N: targetRank); |
| 276 | auto inputScalableVecDimsToUnroll = |
| 277 | vType.getScalableDims().drop_back(N: targetRank); |
| 278 | const auto *it = llvm::find(Range&: inputScalableVecDimsToUnroll, Val: true); |
| 279 | auto firstScalableDim = it - inputScalableVecDimsToUnroll.begin(); |
| 280 | if (firstScalableDim == 0) |
| 281 | return {}; |
| 282 | // All scalable dimensions should be removed now. |
| 283 | inputScalableVecDimsToUnroll = |
| 284 | inputScalableVecDimsToUnroll.slice(N: 0, M: firstScalableDim); |
| 285 | assert(!llvm::is_contained(inputScalableVecDimsToUnroll, true) && |
| 286 | "unexpected leading scalable dimension" ); |
| 287 | // Create an unroll iterator for leading dimensions. |
| 288 | shapeToUnroll = shapeToUnroll.slice(N: 0, M: firstScalableDim); |
| 289 | return StaticTileOffsetRange(shapeToUnroll, /*unrollStep=*/1); |
| 290 | } |
| 291 | |
| 292 | SmallVector<OpFoldResult> vector::getMixedSizesXfer(bool hasTensorSemantics, |
| 293 | Operation *xfer, |
| 294 | RewriterBase &rewriter) { |
| 295 | auto loc = xfer->getLoc(); |
| 296 | |
| 297 | Value base = |
| 298 | TypeSwitch<Operation *, Value>(xfer) |
| 299 | .Case(caseFn: [&](vector::TransferReadOp readOp) { return readOp.getBase(); }) |
| 300 | .Case(caseFn: [&](vector::TransferWriteOp writeOp) { |
| 301 | return writeOp.getOperand(i: 1); |
| 302 | }); |
| 303 | |
| 304 | SmallVector<OpFoldResult> mixedSourceDims = |
| 305 | hasTensorSemantics ? tensor::getMixedSizes(builder&: rewriter, loc, value: base) |
| 306 | : memref::getMixedSizes(builder&: rewriter, loc, value: base); |
| 307 | return mixedSourceDims; |
| 308 | } |
| 309 | |
| 310 | bool vector::isLinearizableVector(VectorType type) { |
| 311 | return (type.getRank() > 1) && (type.getNumScalableDims() <= 1); |
| 312 | } |
| 313 | |
| 314 | /// Determines whether a mask for xfer_read/write is trivially "all true" |
| 315 | /// |
| 316 | /// Given all the inputs required to generate a mask (mask sizes and shapes), |
| 317 | /// and an xfer_read/write operation (indices and the source/destination tensor |
| 318 | /// shape), determines whether the corresponding mask would be trivially |
| 319 | /// foldable (i.e., trivially "all true"). |
| 320 | /// |
| 321 | /// Use this method to avoid generating spurious masks and relying on |
| 322 | /// vectorization post-processing to remove them. |
| 323 | /// |
| 324 | /// Pre-conditions for a mask to be trivially foldable: |
| 325 | /// * All involved shapes (mask + destination tensor) are static. |
| 326 | /// * All indices are constant. |
| 327 | /// * All mask sizes are constant (including `arith.constant`). |
| 328 | /// |
| 329 | /// If the pre-conditions are met, the method checks for each destination |
| 330 | /// dimension `d`: |
| 331 | /// (1) maskShape[d] <= destDimSize[rankDiff + d] |
| 332 | /// (2) index[d] + maskSize[d] <= destDimSize[rankDiff + d] |
| 333 | /// (3) maskShape[d] <= maskSize[d] |
| 334 | /// |
| 335 | /// rankDiff = rank(dest) - rank(mask). |
| 336 | /// |
| 337 | /// This method takes a conservative view: it may return false even if the mask |
| 338 | /// is technically foldable. |
| 339 | /// |
| 340 | /// EXAMPLE 1 (trivially foldable, all shapes match, mask sizes match the shape |
| 341 | /// of the dest tensor): |
| 342 | /// %c0 = arith.constant 0 : index |
| 343 | /// %mask = vector.create_mask 5, 1 |
| 344 | /// vector.mask %mask { |
| 345 | /// vector.transfer_write %vecToStore_1, %dest{[%c0, %c0] |
| 346 | /// {in_bounds = [true, true]} |
| 347 | /// : vector<5x1xi32>, tensor<5x1xi32> |
| 348 | /// } |
| 349 | /// |
| 350 | /// EXAMPLE 2 (not trivially foldable - vector shape exceeds the tensor shape, |
| 351 | /// mask is required to avoid out-of-bounds write): |
| 352 | /// %c0 = arith.constant 0 : index |
| 353 | /// %mask = vector.create_mask 5, 1 |
| 354 | /// vector.mask %mask { |
| 355 | /// vector.transfer_write %vecToStore_2, %dest[%c0, %c0] |
| 356 | /// {in_bounds = [true, true]} |
| 357 | /// : vector<8x1xi32>, tensor<5x1xi32> |
| 358 | /// } |
| 359 | static bool isMaskTriviallyFoldable(SmallVector<OpFoldResult> &maskSizes, |
| 360 | SmallVector<Value> &indices, |
| 361 | ArrayRef<int64_t> baseShape, |
| 362 | ArrayRef<int64_t> maskShape) { |
| 363 | // Masking is unavoidable in the case of dynamic tensors. |
| 364 | if (ShapedType::isDynamicShape(dSizes: baseShape)) |
| 365 | return false; |
| 366 | |
| 367 | // Collect all constant mask sizes. |
| 368 | SmallVector<int64_t, 4> cstMaskSizes; |
| 369 | for (auto [i, dimSize] : llvm::enumerate(First&: maskSizes)) { |
| 370 | if (auto intSize = getConstantIntValue(ofr: dimSize)) { |
| 371 | cstMaskSizes.push_back(Elt: *intSize); |
| 372 | } |
| 373 | } |
| 374 | |
| 375 | // If any of the mask sizes is non-constant, bail out. |
| 376 | if (cstMaskSizes.size() != maskShape.size()) |
| 377 | return false; |
| 378 | |
| 379 | // Collect all constant indices. |
| 380 | SmallVector<int64_t, 4> cstIndices; |
| 381 | for (auto [i, idx] : llvm::enumerate(First&: indices)) { |
| 382 | APSInt intVal; |
| 383 | if (matchPattern(value: idx, pattern: m_ConstantInt(bind_value: &intVal))) { |
| 384 | cstIndices.push_back(Elt: intVal.getSExtValue()); |
| 385 | } |
| 386 | } |
| 387 | |
| 388 | // If any of the indices is non-constant, bail out. |
| 389 | if (cstIndices.size() != baseShape.size()) |
| 390 | return false; |
| 391 | |
| 392 | // Go over all destination dims and check (1), (2) and (3). Take into account |
| 393 | // that: |
| 394 | // * The number of mask sizes will match the rank of the vector to |
| 395 | // load/store. This could be lower than the rank of the destination tensor. |
| 396 | // * Mask sizes could be larger than the corresponding mask shape (hence |
| 397 | // `clamp`). |
| 398 | // TODO: The 2nd item should be rejected by the verifier. |
| 399 | int64_t rankDiff = baseShape.size() - cstMaskSizes.size(); |
| 400 | for (auto [i, idx] : llvm::enumerate(First&: cstMaskSizes)) { |
| 401 | if (/*(1)*/ maskShape[i] > baseShape[rankDiff + i] || |
| 402 | /*(2)*/ baseShape[rankDiff + i] < |
| 403 | (std::clamp(val: cstMaskSizes[i], lo: int64_t(0), hi: maskShape[i]) + |
| 404 | cstIndices[i]) || |
| 405 | /*(3)*/ maskShape[i] > cstMaskSizes[i]) |
| 406 | return false; |
| 407 | } |
| 408 | |
| 409 | return true; |
| 410 | } |
| 411 | |
| 412 | Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc, |
| 413 | Value source, |
| 414 | ArrayRef<int64_t> inputVectorSizes, |
| 415 | std::optional<Value> padValue, |
| 416 | bool useInBoundsInsteadOfMasking, |
| 417 | ArrayRef<bool> inputScalableVecDims) { |
| 418 | VectorType vecToReadTy = VectorType::get( |
| 419 | shape: inputVectorSizes, elementType: cast<ShapedType>(Val: source.getType()).getElementType(), |
| 420 | scalableDims: inputScalableVecDims); |
| 421 | |
| 422 | return createReadOrMaskedRead(builder, loc, source, vecToReadTy, padValue, |
| 423 | useInBoundsInsteadOfMasking); |
| 424 | } |
| 425 | |
| 426 | /// Compute the in_bounds attribute for a transfer op given its permutation map |
| 427 | /// and the source being accessed. Dimension i is in-bounds when the map result |
| 428 | /// is an AffineDimExpr pointing to a static source dimension divisible by the |
| 429 | /// vector size, or an AffineConstantExpr (broadcast). |
| 430 | static SmallVector<bool> computeInBoundsFromPermutationMap( |
| 431 | AffineMap permutationMap, VectorType vectorType, ShapedType sourceType) { |
| 432 | SmallVector<bool> inBounds(vectorType.getRank(), false); |
| 433 | for (unsigned i = 0; i < (unsigned)vectorType.getRank(); ++i) { |
| 434 | AffineExpr expr = permutationMap.getResult(idx: i); |
| 435 | if (auto dimExpr = dyn_cast<AffineDimExpr>(Val&: expr)) { |
| 436 | unsigned memDim = dimExpr.getPosition(); |
| 437 | if (!sourceType.isDynamicDim(idx: memDim) && |
| 438 | sourceType.getDimSize(idx: memDim) % vectorType.getDimSize(idx: i) == 0) |
| 439 | inBounds[i] = true; |
| 440 | } else if (isa<AffineConstantExpr>(Val: expr)) { |
| 441 | inBounds[i] = true; |
| 442 | } |
| 443 | } |
| 444 | return inBounds; |
| 445 | } |
| 446 | |
| 447 | Value vector::createReadOrMaskedRead(OpBuilder &builder, Location loc, |
| 448 | Value source, |
| 449 | const VectorType &vecToReadTy, |
| 450 | std::optional<Value> padValue, |
| 451 | bool useInBoundsInsteadOfMasking, |
| 452 | ArrayRef<Value> customIndices, |
| 453 | AffineMap permutationMap) { |
| 454 | assert(!llvm::is_contained(vecToReadTy.getScalableDims(), |
| 455 | ShapedType::kDynamic) && |
| 456 | "invalid input vector sizes" ); |
| 457 | auto sourceShapedType = cast<ShapedType>(Val: source.getType()); |
| 458 | auto sourceShape = sourceShapedType.getShape(); |
| 459 | |
| 460 | int64_t vecToReadRank = vecToReadTy.getRank(); |
| 461 | auto vecToReadShape = vecToReadTy.getShape(); |
| 462 | |
| 463 | // The permutation map maps the source's index space to the vector's, so its |
| 464 | // dims must match the source rank and its results the vector rank. Without a |
| 465 | // map, a minor identity is implied, requiring the two ranks to match. |
| 466 | assert(sourceShape.size() == (permutationMap |
| 467 | ? permutationMap.getNumDims() |
| 468 | : static_cast<size_t>(vecToReadRank)) && |
| 469 | "expected source rank to match permutation map dims or vector rank." ); |
| 470 | assert((!permutationMap || permutationMap.getNumResults() == |
| 471 | static_cast<size_t>(vecToReadRank)) && |
| 472 | "expected permutation map results to match vector rank." ); |
| 473 | assert((!padValue.has_value() || |
| 474 | padValue.value().getType() == sourceShapedType.getElementType()) && |
| 475 | "expected same pad element type to match source element type" ); |
| 476 | |
| 477 | SmallVector<bool> inBoundsVal(vecToReadRank, true); |
| 478 | |
| 479 | if (useInBoundsInsteadOfMasking) { |
| 480 | if (permutationMap) { |
| 481 | // Update the inBounds attribute. |
| 482 | // FIXME: This computation is too weak - it ignores the read indices. |
| 483 | inBoundsVal = computeInBoundsFromPermutationMap( |
| 484 | permutationMap, vectorType: vecToReadTy, sourceType: cast<ShapedType>(Val: source.getType())); |
| 485 | } else { |
| 486 | // Update the inBounds attribute. |
| 487 | // FIXME: This computation is too weak - it ignores the read indices. |
| 488 | for (unsigned i = 0; i < vecToReadRank; i++) |
| 489 | inBoundsVal[i] = (sourceShape[i] == vecToReadShape[i]) && |
| 490 | ShapedType::isStatic(dValue: sourceShape[i]); |
| 491 | } |
| 492 | } |
| 493 | // The transfer op expects one index per source dimension. |
| 494 | assert( |
| 495 | (customIndices.empty() || customIndices.size() == sourceShape.size()) && |
| 496 | "expected as many custom indices as source dims." ); |
| 497 | SmallVector<Value> indices; |
| 498 | customIndices.empty() |
| 499 | ? indices.assign(NumElts: sourceShape.size(), |
| 500 | Elt: arith::ConstantIndexOp::create(builder, location: loc, value: 0)) |
| 501 | : indices.assign(in_start: customIndices.begin(), in_end: customIndices.end()); |
| 502 | |
| 503 | // A null permutation map means the builder defaults to a minor identity map. |
| 504 | auto transferReadOp = |
| 505 | vector::TransferReadOp::create(builder, location: loc, /*vectorType=*/vecToReadTy, |
| 506 | /*source=*/source, |
| 507 | /*indices=*/indices, |
| 508 | /*padding=*/padValue, |
| 509 | /*permutationMap=*/permutationMap, |
| 510 | /*inBounds=*/inBoundsVal); |
| 511 | |
| 512 | if (useInBoundsInsteadOfMasking) |
| 513 | return transferReadOp; |
| 514 | |
| 515 | SmallVector<OpFoldResult> mixedSourceDims = |
| 516 | isa<MemRefType>(Val: source.getType()) |
| 517 | ? memref::getMixedSizes(builder, loc, value: source) |
| 518 | : tensor::getMixedSizes(builder, loc, value: source); |
| 519 | |
| 520 | if (isMaskTriviallyFoldable(maskSizes&: mixedSourceDims, indices, baseShape: sourceShape, |
| 521 | maskShape: vecToReadShape)) |
| 522 | return transferReadOp; |
| 523 | |
| 524 | auto maskType = vecToReadTy.cloneWith(/*shape=*/{}, elementType: builder.getI1Type()); |
| 525 | Value mask = |
| 526 | vector::CreateMaskOp::create(builder, location: loc, type: maskType, mixedOperands: mixedSourceDims); |
| 527 | return mlir::vector::maskOperation(builder, maskableOp: transferReadOp, mask) |
| 528 | ->getResult(idx: 0); |
| 529 | } |
| 530 | |
| 531 | Operation *vector::createWriteOrMaskedWrite(OpBuilder &builder, Location loc, |
| 532 | Value vecToStore, Value dest, |
| 533 | SmallVector<Value> writeIndices, |
| 534 | bool useInBoundsInsteadOfMasking, |
| 535 | AffineMap permutationMap) { |
| 536 | |
| 537 | ShapedType destType = cast<ShapedType>(Val: dest.getType()); |
| 538 | int64_t destRank = destType.getRank(); |
| 539 | auto destShape = destType.getShape(); |
| 540 | |
| 541 | VectorType vecToStoreType = cast<VectorType>(Val: vecToStore.getType()); |
| 542 | int64_t vecToStoreRank = vecToStoreType.getRank(); |
| 543 | auto vecToStoreShape = vecToStoreType.getShape(); |
| 544 | |
| 545 | // Compute the in_bounds attribute |
| 546 | SmallVector<bool> inBoundsVal(vecToStoreRank, true); |
| 547 | if (useInBoundsInsteadOfMasking) { |
| 548 | if (permutationMap) { |
| 549 | // Update the inBounds attribute. |
| 550 | // FIXME: This computation is too weak - it ignores the write indices. |
| 551 | inBoundsVal = computeInBoundsFromPermutationMap( |
| 552 | permutationMap, vectorType: vecToStoreType, sourceType: cast<ShapedType>(Val: dest.getType())); |
| 553 | } else { |
| 554 | // Update the inBounds attribute. |
| 555 | // FIXME: This computation is too weak - it ignores the write indices. |
| 556 | for (unsigned i = 0; i < vecToStoreRank; i++) |
| 557 | inBoundsVal[i] = |
| 558 | (destShape[destRank - vecToStoreRank + i] >= vecToStoreShape[i]) && |
| 559 | ShapedType::isStatic(dValue: destShape[destRank - vecToStoreRank + i]); |
| 560 | } |
| 561 | } |
| 562 | |
| 563 | // If missing, initialize the write indices to 0. |
| 564 | bool useDefaultWriteIdxs = writeIndices.empty(); |
| 565 | assert((useDefaultWriteIdxs || |
| 566 | writeIndices.size() == static_cast<size_t>(destRank)) && |
| 567 | "Invalid number of write indices!" ); |
| 568 | if (useDefaultWriteIdxs) { |
| 569 | auto zero = arith::ConstantIndexOp::create(builder, location: loc, value: 0); |
| 570 | writeIndices.assign(NumElts: destRank, Elt: zero); |
| 571 | } |
| 572 | |
| 573 | // Generate the xfer_write Op. A null permutation map means the builder |
| 574 | // defaults to a minor identity map. |
| 575 | Operation *write = |
| 576 | vector::TransferWriteOp::create(builder, location: loc, |
| 577 | /*vector=*/vecToStore, |
| 578 | /*dest=*/dest, |
| 579 | /*indices=*/writeIndices, |
| 580 | /*permutationMap=*/permutationMap, |
| 581 | /*inBounds=*/inBoundsVal); |
| 582 | |
| 583 | // If masking is disabled, exit. |
| 584 | if (useInBoundsInsteadOfMasking) |
| 585 | return write; |
| 586 | |
| 587 | // The dest dims written to are those the permutation map selects. A null map |
| 588 | // means a minor identity: the trailing dims. |
| 589 | AffineMap map = permutationMap |
| 590 | ? permutationMap |
| 591 | : AffineMap::getMinorIdentityMap(dims: destRank, results: vecToStoreRank, |
| 592 | context: builder.getContext()); |
| 593 | |
| 594 | // A write that starts at 0 and fills the dims it writes to needs no mask. |
| 595 | bool startsAtZero = llvm::all_of(Range: map.getResults(), P: [&](AffineExpr expr) { |
| 596 | return isZeroInteger(v: writeIndices[cast<AffineDimExpr>(Val&: expr).getPosition()]); |
| 597 | }); |
| 598 | if (startsAtZero && |
| 599 | llvm::equal(LRange&: vecToStoreShape, RRange: applyPermutationMap(map, source: destShape))) |
| 600 | return write; |
| 601 | |
| 602 | // Compute the mask and mask the write Op. |
| 603 | VectorType writeMaskType = inferTransferOpMaskType(vecType: vecToStoreType, permMap: map); |
| 604 | |
| 605 | SmallVector<OpFoldResult> destSizes = |
| 606 | isa<MemRefType>(Val: dest.getType()) |
| 607 | ? memref::getMixedSizes(builder, loc, value: dest) |
| 608 | : tensor::getMixedSizes(builder, loc, value: dest); |
| 609 | |
| 610 | // The map has a dim per dest dim but a result per vector dim, and the vector |
| 611 | // can have fewer dims than the dest (e.g. rank-reducing `insert_slice`). The |
| 612 | // mask has a dim for each dest dim that the map selects, in increasing order. |
| 613 | // For each, collect the mask size (`dim - index`), write index and dest size. |
| 614 | SmallVector<int64_t> maskedDestDims; |
| 615 | for (AffineExpr expr : map.getResults()) |
| 616 | maskedDestDims.push_back(Elt: cast<AffineDimExpr>(Val&: expr).getPosition()); |
| 617 | // A transposing map selects them out of order. |
| 618 | llvm::sort(C&: maskedDestDims); |
| 619 | SmallVector<OpFoldResult> maskSizes; |
| 620 | SmallVector<Value> maskedDestIndices; |
| 621 | SmallVector<int64_t> maskedDestShape; |
| 622 | for (int64_t dim : maskedDestDims) { |
| 623 | Value size = getValueOrCreateConstantIndexOp(b&: builder, loc, ofr: destSizes[dim]); |
| 624 | maskSizes.push_back( |
| 625 | Elt: builder.createOrFold<arith::SubIOp>(location: loc, args&: size, args&: writeIndices[dim])); |
| 626 | maskedDestIndices.push_back(Elt: writeIndices[dim]); |
| 627 | maskedDestShape.push_back(Elt: destShape[dim]); |
| 628 | } |
| 629 | |
| 630 | if (isMaskTriviallyFoldable(maskSizes, indices&: maskedDestIndices, baseShape: maskedDestShape, |
| 631 | maskShape: writeMaskType.getShape())) |
| 632 | return write; |
| 633 | |
| 634 | Value maskForWrite = |
| 635 | builder.createOrFold<vector::CreateMaskOp>(location: loc, args&: writeMaskType, args&: maskSizes); |
| 636 | return mlir::vector::maskOperation(builder, maskableOp: write, mask: maskForWrite); |
| 637 | } |
| 638 | |
| 639 | LogicalResult |
| 640 | vector::isValidMaskedInputVector(ArrayRef<int64_t> shape, |
| 641 | ArrayRef<int64_t> inputVectorSizes) { |
| 642 | LDBG() << "Iteration space static sizes:" << llvm::interleaved(R: shape); |
| 643 | |
| 644 | if (inputVectorSizes.size() != shape.size()) { |
| 645 | LDBG() << "Input vector sizes don't match the number of loops" ; |
| 646 | return failure(); |
| 647 | } |
| 648 | if (ShapedType::isDynamicShape(dSizes: inputVectorSizes)) { |
| 649 | LDBG() << "Input vector sizes can't have dynamic dimensions" ; |
| 650 | return failure(); |
| 651 | } |
| 652 | if (!llvm::all_of(Range: llvm::zip(t&: shape, u&: inputVectorSizes), |
| 653 | P: [](std::tuple<int64_t, int64_t> sizePair) { |
| 654 | int64_t staticSize = std::get<0>(t&: sizePair); |
| 655 | int64_t inputSize = std::get<1>(t&: sizePair); |
| 656 | return ShapedType::isDynamic(dValue: staticSize) || |
| 657 | staticSize <= inputSize; |
| 658 | })) { |
| 659 | LDBG() << "Input vector sizes must be greater than or equal to iteration " |
| 660 | "space static sizes" ; |
| 661 | return failure(); |
| 662 | } |
| 663 | return success(); |
| 664 | } |
| 665 | |
| 666 | /// Takes a 2+ dimensional vector as an input |
| 667 | /// returns n vector values produced by n vector.extract operations. |
| 668 | /// I.e. calling unrollVectorValue([[%v]], rewriter) such that |
| 669 | /// |
| 670 | /// %v : vector<nxaxb...> |
| 671 | /// |
| 672 | /// will produce the following IR changes |
| 673 | /// |
| 674 | /// %v0 = vector.extract %v[0] : vector<axbx...> from vector<nxaxb...> |
| 675 | /// %v1 = vector.extract %v[1] : vector<axbx...> from vector<nxaxb...> |
| 676 | /// ... |
| 677 | /// %vnminusone = vector.extract %v[n-1] : vector<axbx...> from ... |
| 678 | /// |
| 679 | /// and returns SmallVector<Value> r = {[[%v0]], [[%v1]], ..., [[%vnminusone]]} |
| 680 | FailureOr<SmallVector<Value>> |
| 681 | vector::unrollVectorValue(TypedValue<VectorType> vector, |
| 682 | RewriterBase &rewriter) { |
| 683 | SmallVector<Value> subvectors; |
| 684 | VectorType ty = cast<VectorType>(Val: vector.getType()); |
| 685 | Location loc = vector.getLoc(); |
| 686 | if (ty.getRank() < 2) |
| 687 | return rewriter.notifyMatchFailure(arg&: loc, msg: "already 1-D" ); |
| 688 | |
| 689 | // Unrolling doesn't take vscale into account. Pattern is disabled for |
| 690 | // vectors with leading scalable dim(s). |
| 691 | if (ty.getScalableDims().front()) |
| 692 | return rewriter.notifyMatchFailure(arg&: loc, msg: "cannot unroll scalable dim" ); |
| 693 | |
| 694 | for (int64_t i = 0, e = ty.getShape().front(); i < e; ++i) { |
| 695 | subvectors.push_back(Elt: vector::ExtractOp::create(builder&: rewriter, location: loc, source: vector, position: i)); |
| 696 | } |
| 697 | |
| 698 | return subvectors; |
| 699 | } |
| 700 | |
| 701 | LogicalResult vector::unrollVectorOp(Operation *op, PatternRewriter &rewriter, |
| 702 | vector::UnrollVectorOpFn unrollFn) { |
| 703 | assert(op->getNumResults() == 1 && "expected single result" ); |
| 704 | assert(isa<VectorType>(op->getResult(0).getType()) && "expected vector type" ); |
| 705 | VectorType resultTy = cast<VectorType>(Val: op->getResult(idx: 0).getType()); |
| 706 | if (resultTy.getRank() < 2) |
| 707 | return rewriter.notifyMatchFailure(arg&: op, msg: "already 1-D" ); |
| 708 | |
| 709 | // Unrolling doesn't take vscale into account. Pattern is disabled for |
| 710 | // vectors with leading scalable dim(s). |
| 711 | if (resultTy.getScalableDims().front()) |
| 712 | return rewriter.notifyMatchFailure(arg&: op, msg: "cannot unroll scalable dim" ); |
| 713 | |
| 714 | Location loc = op->getLoc(); |
| 715 | Value result = ub::PoisonOp::create(builder&: rewriter, location: loc, result: resultTy); |
| 716 | VectorType subTy = VectorType::Builder(resultTy).dropDim(pos: 0); |
| 717 | |
| 718 | for (int64_t i = 0, e = resultTy.getShape().front(); i < e; ++i) { |
| 719 | Value subVector = unrollFn(rewriter, loc, subTy, i); |
| 720 | result = vector::InsertOp::create(builder&: rewriter, location: loc, valueToStore: subVector, dest: result, position: i); |
| 721 | } |
| 722 | |
| 723 | rewriter.replaceOp(op, newValues: result); |
| 724 | return success(); |
| 725 | } |
| 726 | |