| 1 | //===- VectorOps.cpp - MLIR Vector Dialect Operations ---------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements convenience types for working with super-vectorization |
| 10 | // operations, in particular super-vector loads and stores. |
| 11 | // |
| 12 | //===----------------------------------------------------------------------===// |
| 13 | |
| 14 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 15 | |
| 16 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h" |
| 17 | #include "mlir/Dialect/Affine/IR/ValueBoundsOpInterfaceImpl.h" |
| 18 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 19 | #include "mlir/Dialect/Arith/Utils/Utils.h" |
| 20 | #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h" |
| 21 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 22 | #include "mlir/Dialect/MemRef/IR/MemoryAccessOpInterfaces.h" |
| 23 | #include "mlir/Dialect/MemRef/Utils/MemRefUtils.h" |
| 24 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 25 | #include "mlir/Dialect/UB/IR/UBMatchers.h" |
| 26 | #include "mlir/Dialect/Utils/IndexingUtils.h" |
| 27 | #include "mlir/Dialect/Utils/StructuredOpsUtils.h" |
| 28 | #include "mlir/Dialect/Utils/VerificationUtils.h" |
| 29 | #include "mlir/IR/AffineExpr.h" |
| 30 | #include "mlir/IR/AffineMap.h" |
| 31 | #include "mlir/IR/Builders.h" |
| 32 | #include "mlir/IR/BuiltinAttributes.h" |
| 33 | #include "mlir/IR/BuiltinTypes.h" |
| 34 | #include "mlir/IR/IRMapping.h" |
| 35 | #include "mlir/IR/OpImplementation.h" |
| 36 | #include "mlir/IR/PatternMatch.h" |
| 37 | #include "mlir/IR/TypeUtilities.h" |
| 38 | #include "mlir/IR/ValueRange.h" |
| 39 | #include "mlir/Interfaces/SubsetOpInterface.h" |
| 40 | #include "mlir/Interfaces/ValueBoundsOpInterface.h" |
| 41 | #include "mlir/Interfaces/VectorInterfaces.h" |
| 42 | #include "mlir/Support/LLVM.h" |
| 43 | #include "mlir/Transforms/InliningUtils.h" |
| 44 | #include "llvm/ADT/ArrayRef.h" |
| 45 | #include "llvm/ADT/Repeated.h" |
| 46 | #include "llvm/ADT/STLExtras.h" |
| 47 | #include "llvm/ADT/SmallVector.h" |
| 48 | #include "llvm/ADT/SmallVectorExtras.h" |
| 49 | #include "llvm/ADT/StringSet.h" |
| 50 | #include "llvm/Support/Casting.h" |
| 51 | |
| 52 | #include <cassert> |
| 53 | #include <cstdint> |
| 54 | #include <numeric> |
| 55 | |
| 56 | #include "mlir/Dialect/Vector/IR/VectorDialect.cpp.inc" |
| 57 | // Pull in all enum type and utility function definitions. |
| 58 | #include "mlir/Dialect/Vector/IR/VectorEnums.cpp.inc" |
| 59 | |
| 60 | using namespace mlir; |
| 61 | using namespace mlir::vector; |
| 62 | |
| 63 | /// Helper enum to classify mask value. |
| 64 | enum class MaskFormat { |
| 65 | AllTrue = 0, |
| 66 | AllFalse = 1, |
| 67 | Unknown = 2, |
| 68 | }; |
| 69 | |
| 70 | // Default target shape for unrolling CreateMaskOp and ConstantMaskOp |
| 71 | static std::optional<SmallVector<int64_t>> |
| 72 | getDefaultTargetMaskShapeForUnroll() { |
| 73 | return SmallVector<int64_t>{8, 8}; |
| 74 | } |
| 75 | |
| 76 | // Default target shape for unrolling InterleaveOp and DeinterleaveOp |
| 77 | static std::optional<SmallVector<int64_t>> |
| 78 | getDefaultTargetInterleaveShapeForUnroll() { |
| 79 | return SmallVector<int64_t>{2, 4}; |
| 80 | } |
| 81 | |
| 82 | /// Helper method to classify a mask value. Currently, the method |
| 83 | /// looks "under the hood" of a constant value with dense attributes |
| 84 | /// and a constant mask operation (since the client may be called at |
| 85 | /// various stages during progressive lowering). |
| 86 | static MaskFormat getMaskFormat(Value mask) { |
| 87 | if (auto c = mask.getDefiningOp<arith::ConstantOp>()) { |
| 88 | // Inspect constant dense values. We count up for bits that |
| 89 | // are set, count down for bits that are cleared, and bail |
| 90 | // when a mix is detected. |
| 91 | if (auto denseElts = llvm::dyn_cast<DenseIntElementsAttr>(Val: c.getValue())) { |
| 92 | int64_t val = 0; |
| 93 | for (bool b : denseElts.getValues<bool>()) |
| 94 | if (b && val >= 0) |
| 95 | val++; |
| 96 | else if (!b && val <= 0) |
| 97 | val--; |
| 98 | else |
| 99 | return MaskFormat::Unknown; |
| 100 | if (val > 0) |
| 101 | return MaskFormat::AllTrue; |
| 102 | if (val < 0) |
| 103 | return MaskFormat::AllFalse; |
| 104 | } |
| 105 | } else if (auto m = mask.getDefiningOp<ConstantMaskOp>()) { |
| 106 | // Inspect constant mask index. If the index exceeds the |
| 107 | // dimension size, all bits are set. If the index is zero |
| 108 | // or less, no bits are set. |
| 109 | ArrayRef<int64_t> masks = m.getMaskDimSizes(); |
| 110 | auto shape = m.getType().getShape(); |
| 111 | bool allTrue = true; |
| 112 | bool allFalse = true; |
| 113 | for (auto [maskIdx, dimSize] : llvm::zip_equal(t&: masks, u&: shape)) { |
| 114 | if (maskIdx < dimSize) |
| 115 | allTrue = false; |
| 116 | if (maskIdx > 0) |
| 117 | allFalse = false; |
| 118 | } |
| 119 | if (allTrue) |
| 120 | return MaskFormat::AllTrue; |
| 121 | if (allFalse) |
| 122 | return MaskFormat::AllFalse; |
| 123 | } else if (auto m = mask.getDefiningOp<CreateMaskOp>()) { |
| 124 | // Finds all-false create_masks. An all-true create_mask requires all |
| 125 | // dims to be constants, so that'll be folded to a constant_mask, then |
| 126 | // detected in the constant_mask case. |
| 127 | auto maskOperands = m.getOperands(); |
| 128 | for (Value operand : maskOperands) { |
| 129 | if (auto constantOp = operand.getDefiningOp<arith::ConstantOp>()) { |
| 130 | int64_t dimSize = |
| 131 | llvm::cast<IntegerAttr>(Val: constantOp.getValue()).getInt(); |
| 132 | if (dimSize <= 0) |
| 133 | return MaskFormat::AllFalse; |
| 134 | } |
| 135 | } |
| 136 | return MaskFormat::Unknown; |
| 137 | } |
| 138 | return MaskFormat::Unknown; |
| 139 | } |
| 140 | |
| 141 | /// Default callback to build a region with a 'vector.yield' terminator with no |
| 142 | /// arguments. |
| 143 | void mlir::vector::buildTerminatedBody(OpBuilder &builder, Location loc) { |
| 144 | vector::YieldOp::create(builder, location: loc); |
| 145 | } |
| 146 | |
| 147 | // Helper for verifying combining kinds in contractions and reductions. |
| 148 | static bool isSupportedCombiningKind(CombiningKind combiningKind, |
| 149 | Type elementType) { |
| 150 | switch (combiningKind) { |
| 151 | case CombiningKind::ADD: |
| 152 | case CombiningKind::MUL: |
| 153 | return elementType.isIntOrIndexOrFloat(); |
| 154 | case CombiningKind::MINUI: |
| 155 | case CombiningKind::MINSI: |
| 156 | case CombiningKind::MAXUI: |
| 157 | case CombiningKind::MAXSI: |
| 158 | case CombiningKind::AND: |
| 159 | case CombiningKind::OR: |
| 160 | case CombiningKind::XOR: |
| 161 | return elementType.isIntOrIndex(); |
| 162 | case CombiningKind::MINNUMF: |
| 163 | case CombiningKind::MAXNUMF: |
| 164 | case CombiningKind::MINIMUMF: |
| 165 | case CombiningKind::MAXIMUMF: |
| 166 | case CombiningKind::MINIMUMNUMF: |
| 167 | case CombiningKind::MAXIMUMNUMF: |
| 168 | return llvm::isa<FloatType>(Val: elementType); |
| 169 | } |
| 170 | return false; |
| 171 | } |
| 172 | |
| 173 | /// Returns the effective rank of the vector to read/write for Xfer Ops |
| 174 | /// |
| 175 | /// When the element type of the shaped type is _a scalar_, this will simply |
| 176 | /// return the rank of the vector ( the result for xfer_read or the value to |
| 177 | /// store for xfer_write). |
| 178 | /// |
| 179 | /// When the element type of the base shaped type is _a vector_, returns the |
| 180 | /// difference between the original vector type and the element type of the |
| 181 | /// shaped type. |
| 182 | /// |
| 183 | /// EXAMPLE 1 (element type is _a scalar_): |
| 184 | /// - shapedType = tensor<10x20xf32>, vectorType = vector<2x4xf32> |
| 185 | /// - shapedType.getElementType() = f32 (rank 0) |
| 186 | /// - vectorType.getRank() = 2 |
| 187 | /// - Result = 2 - 0 = 2 |
| 188 | /// |
| 189 | /// EXAMPLE 2 (element type is _a vector_): |
| 190 | /// - shapedType = tensor<10xvector<20xf32>>, vectorType = vector<20xf32> |
| 191 | /// - shapedType.getElementType() = vector<20xf32> (rank 1) |
| 192 | /// - vectorType.getRank() = 1 |
| 193 | /// - Result = 1 - 1 = 0 |
| 194 | /// |
| 195 | /// This is used to determine the number of minor dimensions for identity maps |
| 196 | /// in vector transfer Ops. |
| 197 | static unsigned getEffectiveVectorRankForXferOp(ShapedType shapedType, |
| 198 | VectorType vectorType) { |
| 199 | unsigned elementVectorRank = 0; |
| 200 | VectorType elementVectorType = |
| 201 | llvm::dyn_cast<VectorType>(Val: shapedType.getElementType()); |
| 202 | if (elementVectorType) |
| 203 | elementVectorRank += elementVectorType.getRank(); |
| 204 | return vectorType.getRank() - elementVectorRank; |
| 205 | } |
| 206 | |
| 207 | AffineMap mlir::vector::getTransferMinorIdentityMap(ShapedType shapedType, |
| 208 | VectorType vectorType) { |
| 209 | // 0-d transfers are to/from tensor<t>/memref<t> and vector<1xt>. |
| 210 | // TODO: replace once we have 0-d vectors. |
| 211 | if (shapedType.getRank() == 0 && |
| 212 | vectorType.getShape() == ArrayRef<int64_t>{1}) |
| 213 | return AffineMap::get( |
| 214 | /*numDims=*/dimCount: 0, /*numSymbols=*/symbolCount: 0, |
| 215 | result: getAffineConstantExpr(constant: 0, context: shapedType.getContext())); |
| 216 | return AffineMap::getMinorIdentityMap( |
| 217 | dims: shapedType.getRank(), |
| 218 | results: getEffectiveVectorRankForXferOp(shapedType, vectorType), |
| 219 | context: shapedType.getContext()); |
| 220 | } |
| 221 | |
| 222 | /// Check if `write` is of a constant splat and the masked `read` is padded with |
| 223 | /// the same splat value -- meaning it could be the same value as the initial |
| 224 | /// constant splat. |
| 225 | static bool isSplatWriteConsistentWithMaskedRead(vector::TransferWriteOp write, |
| 226 | vector::TransferReadOp read) { |
| 227 | auto readMask = read.getMask(); |
| 228 | auto writeMask = write.getMask(); |
| 229 | // Check if the masks are consistent. The splat value could be the same if the |
| 230 | // read is masked (and padded with the splat value), and the write is unmasked |
| 231 | // or has the same mask. Note this does not allow the case where the write is |
| 232 | // masked and the read is unmasked, as then the read could be of more elements |
| 233 | // than the write (which may not be the same value). |
| 234 | bool couldBeSameSplat = readMask && (!writeMask || writeMask == readMask); |
| 235 | if (!couldBeSameSplat) |
| 236 | return false; |
| 237 | // Check for constant splat (as the source of the write). |
| 238 | DenseElementsAttr splatAttr; |
| 239 | if (!matchPattern(value: write.getVector(), |
| 240 | pattern: m_Constant<DenseElementsAttr>(bind_value: &splatAttr)) || |
| 241 | !splatAttr.isSplat()) { |
| 242 | return false; |
| 243 | } |
| 244 | // The padding of the read and the constant splat value must be the same. |
| 245 | Attribute padAttr; |
| 246 | if (!matchPattern(value: read.getPadding(), pattern: m_Constant(bind_value: &padAttr))) |
| 247 | return false; |
| 248 | return padAttr == splatAttr.getSplatValue<Attribute>(); |
| 249 | } |
| 250 | |
| 251 | bool mlir::vector::checkSameValueRAW(vector::TransferWriteOp defWrite, |
| 252 | vector::TransferReadOp read) { |
| 253 | // An enclosing vector.mask may leave some lanes unwritten or padded, so the |
| 254 | // read may not see the written vector as a whole. This conservatively |
| 255 | // includes all-true masks. MaskOp::fold removes them during canonicalization, |
| 256 | // so this only matters when a caller such as transferOpflowOpt runs first. |
| 257 | // TODO: Account for compatible region masks instead of conservatively |
| 258 | // rejecting all region-masked operations. |
| 259 | if (defWrite.isMasked() || read.isMasked()) |
| 260 | return false; |
| 261 | return !defWrite.hasOutOfBoundsDim() && |
| 262 | defWrite.getIndices() == read.getIndices() && |
| 263 | defWrite.getVectorType() == read.getVectorType() && |
| 264 | defWrite.getPermutationMap() == read.getPermutationMap() && |
| 265 | ((!defWrite.getMask() && !read.getMask()) || |
| 266 | isSplatWriteConsistentWithMaskedRead(write: defWrite, read)); |
| 267 | } |
| 268 | |
| 269 | bool mlir::vector::checkSameValueWAW(vector::TransferWriteOp write, |
| 270 | vector::TransferWriteOp priorWrite) { |
| 271 | // A write under an enclosing vector.mask may not overwrite all lanes of the |
| 272 | // prior write. This conservatively includes all-true masks, which |
| 273 | // MaskOp::fold removes during canonicalization. Only the later write needs |
| 274 | // checking: a region-masked prior write is still dead if the later write |
| 275 | // fully overwrites it. |
| 276 | // TODO: Handle a region-masked later write when it is known to fully |
| 277 | // overwrite the prior write. |
| 278 | if (write.isMasked()) |
| 279 | return false; |
| 280 | return priorWrite.getIndices() == write.getIndices() && |
| 281 | priorWrite.getMask() == write.getMask() && |
| 282 | priorWrite.getVectorType() == write.getVectorType() && |
| 283 | priorWrite.getPermutationMap() == write.getPermutationMap(); |
| 284 | } |
| 285 | |
| 286 | bool mlir::vector::isDisjointTransferIndices( |
| 287 | VectorTransferOpInterface transferA, VectorTransferOpInterface transferB, |
| 288 | bool testDynamicValueUsingBounds) { |
| 289 | // For simplicity only look at transfer of same type. |
| 290 | if (transferA.getVectorType() != transferB.getVectorType()) |
| 291 | return false; |
| 292 | unsigned rankOffset = transferA.getLeadingShapedRank(); |
| 293 | for (unsigned i = 0, e = transferA.getIndices().size(); i < e; i++) { |
| 294 | Value indexA = transferA.getIndices()[i]; |
| 295 | Value indexB = transferB.getIndices()[i]; |
| 296 | std::optional<int64_t> cstIndexA = getConstantIntValue(ofr: indexA); |
| 297 | std::optional<int64_t> cstIndexB = getConstantIntValue(ofr: indexB); |
| 298 | |
| 299 | if (i < rankOffset) { |
| 300 | // For leading dimensions, if we can prove that index are different we |
| 301 | // know we are accessing disjoint slices. |
| 302 | if (cstIndexA.has_value() && cstIndexB.has_value()) { |
| 303 | if (*cstIndexA != *cstIndexB) |
| 304 | return true; |
| 305 | continue; |
| 306 | } |
| 307 | if (testDynamicValueUsingBounds) { |
| 308 | // First try to see if we can fully compose and simplify the affine |
| 309 | // expression as a fast track. |
| 310 | FailureOr<uint64_t> delta = |
| 311 | affine::fullyComposeAndComputeConstantDelta(value1: indexA, value2: indexB); |
| 312 | if (succeeded(Result: delta) && *delta != 0) |
| 313 | return true; |
| 314 | |
| 315 | FailureOr<bool> testEqual = |
| 316 | ValueBoundsConstraintSet::areEqual(var1: indexA, var2: indexB); |
| 317 | if (succeeded(Result: testEqual) && !testEqual.value()) |
| 318 | return true; |
| 319 | } |
| 320 | } else { |
| 321 | // For this dimension, we slice a part of the memref we need to make sure |
| 322 | // the intervals accessed don't overlap. |
| 323 | int64_t vectorDim = transferA.getVectorType().getDimSize(idx: i - rankOffset); |
| 324 | if (cstIndexA.has_value() && cstIndexB.has_value()) { |
| 325 | int64_t distance = std::abs(i: *cstIndexA - *cstIndexB); |
| 326 | if (distance >= vectorDim) |
| 327 | return true; |
| 328 | continue; |
| 329 | } |
| 330 | if (testDynamicValueUsingBounds) { |
| 331 | // First try to see if we can fully compose and simplify the affine |
| 332 | // expression as a fast track. |
| 333 | FailureOr<int64_t> delta = |
| 334 | affine::fullyComposeAndComputeConstantDelta(value1: indexA, value2: indexB); |
| 335 | if (succeeded(Result: delta) && std::abs(i: *delta) >= vectorDim) |
| 336 | return true; |
| 337 | |
| 338 | FailureOr<int64_t> computeDelta = |
| 339 | ValueBoundsConstraintSet::computeConstantDelta(value1: indexA, value2: indexB); |
| 340 | if (succeeded(Result: computeDelta)) { |
| 341 | if (std::abs(i: computeDelta.value()) >= vectorDim) |
| 342 | return true; |
| 343 | } |
| 344 | } |
| 345 | } |
| 346 | } |
| 347 | return false; |
| 348 | } |
| 349 | |
| 350 | bool mlir::vector::isDisjointTransferSet(VectorTransferOpInterface transferA, |
| 351 | VectorTransferOpInterface transferB, |
| 352 | bool testDynamicValueUsingBounds) { |
| 353 | if (transferA.getBase() != transferB.getBase()) |
| 354 | return false; |
| 355 | return isDisjointTransferIndices(transferA, transferB, |
| 356 | testDynamicValueUsingBounds); |
| 357 | } |
| 358 | |
| 359 | // Helper to iterate over n-D vector slice elements. Calculate the next |
| 360 | // `position` in the n-D vector of size `shape`, applying an offset `offsets`. |
| 361 | // Modifies the `position` in place. Returns a failure when `position` becomes |
| 362 | // the end position. |
| 363 | static LogicalResult incSlicePosition(MutableArrayRef<int64_t> position, |
| 364 | ArrayRef<int64_t> shape, |
| 365 | ArrayRef<int64_t> offsets) { |
| 366 | for (auto [posInDim, dimSize, offsetInDim] : |
| 367 | llvm::reverse(C: llvm::zip_equal(t&: position, u&: shape, args&: offsets))) { |
| 368 | ++posInDim; |
| 369 | if (posInDim < dimSize + offsetInDim) |
| 370 | return success(); |
| 371 | |
| 372 | // Carry the overflow to the next loop iteration. |
| 373 | posInDim = offsetInDim; |
| 374 | } |
| 375 | |
| 376 | return failure(); |
| 377 | } |
| 378 | |
| 379 | /// Returns the integer numbers in `values`. `values` are expected to be |
| 380 | /// constant operations. |
| 381 | SmallVector<int64_t> vector::getAsIntegers(ArrayRef<Value> values) { |
| 382 | SmallVector<int64_t> ints; |
| 383 | llvm::transform(Range&: values, d_first: std::back_inserter(x&: ints), F: [](Value value) { |
| 384 | auto constOp = value.getDefiningOp<arith::ConstantIndexOp>(); |
| 385 | assert(constOp && "Unexpected non-constant index" ); |
| 386 | return constOp.value(); |
| 387 | }); |
| 388 | return ints; |
| 389 | } |
| 390 | |
| 391 | /// Returns the integer numbers in `foldResults`. `foldResults` are expected to |
| 392 | /// be constant operations. |
| 393 | SmallVector<int64_t> vector::getAsIntegers(ArrayRef<OpFoldResult> foldResults) { |
| 394 | SmallVector<int64_t> ints; |
| 395 | llvm::transform( |
| 396 | Range&: foldResults, d_first: std::back_inserter(x&: ints), F: [](OpFoldResult foldResult) { |
| 397 | assert(isa<Attribute>(foldResult) && "Unexpected non-constant index" ); |
| 398 | return cast<IntegerAttr>(Val: cast<Attribute>(Val&: foldResult)).getInt(); |
| 399 | }); |
| 400 | return ints; |
| 401 | } |
| 402 | |
| 403 | /// Convert `foldResults` into Values. Integer attributes are converted to |
| 404 | /// constant op. |
| 405 | SmallVector<Value> vector::getAsValues(OpBuilder &builder, Location loc, |
| 406 | ArrayRef<OpFoldResult> foldResults) { |
| 407 | SmallVector<Value> values; |
| 408 | llvm::transform(Range&: foldResults, d_first: std::back_inserter(x&: values), |
| 409 | F: [&](OpFoldResult foldResult) { |
| 410 | if (auto attr = dyn_cast<Attribute>(Val&: foldResult)) |
| 411 | return arith::ConstantIndexOp::create( |
| 412 | builder, location: loc, value: cast<IntegerAttr>(Val&: attr).getInt()) |
| 413 | .getResult(); |
| 414 | |
| 415 | return cast<Value>(Val&: foldResult); |
| 416 | }); |
| 417 | return values; |
| 418 | } |
| 419 | |
| 420 | std::optional<int64_t> vector::getConstantVscaleMultiplier(Value value) { |
| 421 | if (value.getDefiningOp<vector::VectorScaleOp>()) |
| 422 | return 1; |
| 423 | auto mul = value.getDefiningOp<arith::MulIOp>(); |
| 424 | if (!mul) |
| 425 | return {}; |
| 426 | auto lhs = mul.getLhs(); |
| 427 | auto rhs = mul.getRhs(); |
| 428 | if (lhs.getDefiningOp<vector::VectorScaleOp>()) |
| 429 | return getConstantIntValue(ofr: rhs); |
| 430 | if (rhs.getDefiningOp<vector::VectorScaleOp>()) |
| 431 | return getConstantIntValue(ofr: lhs); |
| 432 | return {}; |
| 433 | } |
| 434 | |
| 435 | /// Converts numeric attributes to the expected type. Supports |
| 436 | /// integer-to-integer and float-to-integer conversions. Returns the original |
| 437 | /// attribute if no conversion is needed or supported. |
| 438 | static Attribute convertNumericAttr(Attribute attr, Type expectedType) { |
| 439 | // Integer-to-integer conversion |
| 440 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) { |
| 441 | if (auto intType = dyn_cast<IntegerType>(Val&: expectedType)) { |
| 442 | if (intAttr.getType() != expectedType) |
| 443 | return IntegerAttr::get(type: expectedType, value: intAttr.getInt()); |
| 444 | } |
| 445 | return attr; |
| 446 | } |
| 447 | |
| 448 | // Float-to-integer bitcast (preserves bit representation) |
| 449 | if (auto floatAttr = dyn_cast<FloatAttr>(Val&: attr)) { |
| 450 | auto intType = dyn_cast<IntegerType>(Val&: expectedType); |
| 451 | if (!intType) |
| 452 | return attr; |
| 453 | |
| 454 | APFloat floatVal = floatAttr.getValue(); |
| 455 | APInt intVal = floatVal.bitcastToAPInt(); |
| 456 | return IntegerAttr::get(type: expectedType, value: intVal); |
| 457 | } |
| 458 | |
| 459 | return attr; |
| 460 | } |
| 461 | |
| 462 | /// Return whether `srcType` can be broadcast to `dstVectorType` under the |
| 463 | /// semantics of the `vector.broadcast` op. |
| 464 | BroadcastableToResult mlir::vector::isBroadcastableTo( |
| 465 | Type srcType, VectorType dstVectorType, |
| 466 | std::pair<VectorDim, VectorDim> *mismatchingDims) { |
| 467 | // Broadcast scalar to vector of the same element type. |
| 468 | if (isa<VectorElementTypeInterface>(Val: srcType) && dstVectorType && |
| 469 | srcType == getElementTypeOrSelf(type: dstVectorType)) |
| 470 | return BroadcastableToResult::Success; |
| 471 | // From now on, only vectors broadcast. |
| 472 | VectorType srcVectorType = llvm::dyn_cast<VectorType>(Val&: srcType); |
| 473 | if (!srcVectorType) |
| 474 | return BroadcastableToResult::SourceTypeNotAVector; |
| 475 | |
| 476 | int64_t srcRank = srcVectorType.getRank(); |
| 477 | int64_t dstRank = dstVectorType.getRank(); |
| 478 | if (srcRank > dstRank) |
| 479 | return BroadcastableToResult::SourceRankHigher; |
| 480 | // Source has an exact match or singleton value for all trailing dimensions |
| 481 | // (all leading dimensions are simply duplicated). |
| 482 | int64_t lead = dstRank - srcRank; |
| 483 | for (int64_t dimIdx = 0; dimIdx < srcRank; ++dimIdx) { |
| 484 | // Have mismatching dims (in the sense of vector.broadcast semantics) been |
| 485 | // encountered? |
| 486 | bool foundMismatchingDims = false; |
| 487 | |
| 488 | // Check fixed-width dims. |
| 489 | int64_t srcDim = srcVectorType.getDimSize(idx: dimIdx); |
| 490 | int64_t dstDim = dstVectorType.getDimSize(idx: lead + dimIdx); |
| 491 | if (srcDim != 1 && srcDim != dstDim) |
| 492 | foundMismatchingDims = true; |
| 493 | |
| 494 | // Check scalable flags. |
| 495 | bool srcDimScalableFlag = srcVectorType.getScalableDims()[dimIdx]; |
| 496 | bool dstDimScalableFlag = dstVectorType.getScalableDims()[lead + dimIdx]; |
| 497 | if ((srcDim == 1 && srcDimScalableFlag && dstDim != 1) || |
| 498 | // 1 -> [N] is fine, everything else should be rejected when mixing |
| 499 | // fixed-width and scalable dims |
| 500 | (srcDimScalableFlag != dstDimScalableFlag && |
| 501 | (srcDim != 1 || srcDimScalableFlag))) |
| 502 | foundMismatchingDims = true; |
| 503 | |
| 504 | if (foundMismatchingDims) { |
| 505 | if (mismatchingDims != nullptr) { |
| 506 | mismatchingDims->first.dim = srcDim; |
| 507 | mismatchingDims->first.isScalable = srcDimScalableFlag; |
| 508 | |
| 509 | mismatchingDims->second.dim = dstDim; |
| 510 | mismatchingDims->second.isScalable = dstDimScalableFlag; |
| 511 | } |
| 512 | return BroadcastableToResult::DimensionMismatch; |
| 513 | } |
| 514 | } |
| 515 | |
| 516 | return BroadcastableToResult::Success; |
| 517 | } |
| 518 | |
| 519 | //===----------------------------------------------------------------------===// |
| 520 | // CombiningKindAttr |
| 521 | //===----------------------------------------------------------------------===// |
| 522 | |
| 523 | namespace mlir { |
| 524 | namespace vector { |
| 525 | namespace detail { |
| 526 | struct BitmaskEnumStorage : public AttributeStorage { |
| 527 | using KeyTy = uint64_t; |
| 528 | |
| 529 | BitmaskEnumStorage(KeyTy val) : value(val) {} |
| 530 | |
| 531 | bool operator==(const KeyTy &key) const { return value == key; } |
| 532 | |
| 533 | static BitmaskEnumStorage *construct(AttributeStorageAllocator &allocator, |
| 534 | const KeyTy &key) { |
| 535 | return new (allocator.allocate<BitmaskEnumStorage>()) |
| 536 | BitmaskEnumStorage(key); |
| 537 | } |
| 538 | |
| 539 | KeyTy value = 0; |
| 540 | }; |
| 541 | } // namespace detail |
| 542 | } // namespace vector |
| 543 | } // namespace mlir |
| 544 | |
| 545 | //===----------------------------------------------------------------------===// |
| 546 | // VectorDialect |
| 547 | //===----------------------------------------------------------------------===// |
| 548 | |
| 549 | namespace { |
| 550 | /// This class defines the interface for handling inlining with vector dialect |
| 551 | /// operations. |
| 552 | struct VectorInlinerInterface : public DialectInlinerInterface { |
| 553 | using DialectInlinerInterface::DialectInlinerInterface; |
| 554 | |
| 555 | /// All vector dialect ops can be inlined. |
| 556 | bool isLegalToInline(Operation *, Region *, bool, IRMapping &) const final { |
| 557 | return true; |
| 558 | } |
| 559 | }; |
| 560 | } // namespace |
| 561 | |
| 562 | void VectorDialect::initialize() { |
| 563 | addAttributes< |
| 564 | #define GET_ATTRDEF_LIST |
| 565 | #include "mlir/Dialect/Vector/IR/VectorAttributes.cpp.inc" |
| 566 | >(); |
| 567 | |
| 568 | addOperations< |
| 569 | #define GET_OP_LIST |
| 570 | #include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc" |
| 571 | >(); |
| 572 | |
| 573 | addInterfaces<VectorInlinerInterface>(); |
| 574 | |
| 575 | declarePromisedInterfaces<memref::IndexedAccessOpInterface, LoadOp, StoreOp, |
| 576 | MaskedLoadOp, MaskedStoreOp, ExpandLoadOp, |
| 577 | CompressStoreOp>(); |
| 578 | declarePromisedInterfaces<bufferization::BufferizableOpInterface, |
| 579 | TransferReadOp, TransferWriteOp, GatherOp, MaskOp, |
| 580 | YieldOp>(); |
| 581 | declarePromisedInterfaces<SubsetOpInterface, TransferReadOp, |
| 582 | TransferWriteOp>(); |
| 583 | declarePromisedInterface<SubsetExtractionOpInterface, TransferReadOp>(); |
| 584 | declarePromisedInterface<SubsetInsertionOpInterface, TransferWriteOp>(); |
| 585 | declarePromisedInterface<ConvertToLLVMPatternInterface, VectorDialect>(); |
| 586 | } |
| 587 | |
| 588 | /// Materialize a single constant operation from a given attribute value with |
| 589 | /// the desired resultant type. |
| 590 | Operation *VectorDialect::materializeConstant(OpBuilder &builder, |
| 591 | Attribute value, Type type, |
| 592 | Location loc) { |
| 593 | if (matchPattern(attr: value, pattern: ub::m_Poison())) |
| 594 | return value.getDialect().materializeConstant(builder, value, type, loc); |
| 595 | |
| 596 | return arith::ConstantOp::materialize(builder, value, type, loc); |
| 597 | } |
| 598 | |
| 599 | IntegerType vector::getVectorSubscriptType(Builder &builder) { |
| 600 | return builder.getIntegerType(width: 64); |
| 601 | } |
| 602 | |
| 603 | ArrayAttr vector::getVectorSubscriptAttr(Builder &builder, |
| 604 | ArrayRef<int64_t> values) { |
| 605 | return builder.getI64ArrayAttr(values); |
| 606 | } |
| 607 | |
| 608 | //===----------------------------------------------------------------------===// |
| 609 | // MultiDimReductionOp |
| 610 | //===----------------------------------------------------------------------===// |
| 611 | |
| 612 | void vector::MultiDimReductionOp::build(OpBuilder &builder, |
| 613 | OperationState &result, Value source, |
| 614 | Value acc, ArrayRef<bool> reductionMask, |
| 615 | CombiningKind kind, |
| 616 | arith::FastMathFlags fastMathFlags) { |
| 617 | SmallVector<int64_t> reductionDims; |
| 618 | for (const auto &en : llvm::enumerate(First&: reductionMask)) |
| 619 | if (en.value()) |
| 620 | reductionDims.push_back(Elt: en.index()); |
| 621 | build(odsBuilder&: builder, odsState&: result, kind, source, acc, reduction_dims: reductionDims, fastmath: fastMathFlags); |
| 622 | } |
| 623 | |
| 624 | OpFoldResult MultiDimReductionOp::fold(FoldAdaptor adaptor) { |
| 625 | // No reduction dims: this is a noop regardless of rank. |
| 626 | if (getReductionDims().empty()) |
| 627 | return getSource(); |
| 628 | return {}; |
| 629 | } |
| 630 | |
| 631 | LogicalResult MultiDimReductionOp::verify() { |
| 632 | // Verify the reduction dimensions. |
| 633 | int64_t sourceRank = getSourceVectorType().getRank(); |
| 634 | SmallVector<bool> isReduced(sourceRank, false); |
| 635 | for (int64_t dim : getReductionDims()) { |
| 636 | if (dim < 0 || dim >= sourceRank) |
| 637 | return emitOpError(message: "reduction dimension out of range: " ) << dim; |
| 638 | if (isReduced[dim]) |
| 639 | return emitOpError(message: "duplicate reduction dimension: " ) << dim; |
| 640 | isReduced[dim] = true; |
| 641 | } |
| 642 | |
| 643 | SmallVector<int64_t> targetShape; |
| 644 | SmallVector<bool> scalableDims; |
| 645 | Type inferredReturnType; |
| 646 | auto sourceScalableDims = getSourceVectorType().getScalableDims(); |
| 647 | for (auto [dimIdx, dimSize] : |
| 648 | llvm::enumerate(First: getSourceVectorType().getShape())) |
| 649 | if (!isReduced[dimIdx]) { |
| 650 | targetShape.push_back(Elt: dimSize); |
| 651 | scalableDims.push_back(Elt: sourceScalableDims[dimIdx]); |
| 652 | } |
| 653 | // TODO: update to also allow 0-d vectors when available. |
| 654 | if (targetShape.empty()) |
| 655 | inferredReturnType = getSourceVectorType().getElementType(); |
| 656 | else |
| 657 | inferredReturnType = VectorType::get( |
| 658 | shape: targetShape, elementType: getSourceVectorType().getElementType(), scalableDims); |
| 659 | if (getType() != inferredReturnType) |
| 660 | return emitOpError() << "destination type " << getType() |
| 661 | << " is incompatible with source type " |
| 662 | << getSourceVectorType(); |
| 663 | |
| 664 | return success(); |
| 665 | } |
| 666 | |
| 667 | /// Returns the mask type expected by this operation. |
| 668 | Type MultiDimReductionOp::getExpectedMaskType() { |
| 669 | auto vecType = getSourceVectorType(); |
| 670 | return VectorType::get(shape: vecType.getShape(), |
| 671 | elementType: IntegerType::get(context: vecType.getContext(), /*width=*/1), |
| 672 | scalableDims: vecType.getScalableDims()); |
| 673 | } |
| 674 | |
| 675 | //----------------------------------------------------------------------------- |
| 676 | // VectorUnrollOpInterface methods |
| 677 | //----------------------------------------------------------------------------- |
| 678 | std::optional<SmallVector<int64_t, 4>> |
| 679 | MultiDimReductionOp::getIterationShapeToUnroll() { |
| 680 | return llvm::to_vector<4>(Range: getSourceVectorType().getShape()); |
| 681 | } |
| 682 | |
| 683 | namespace { |
| 684 | // Only unit dimensions that are being reduced are folded. If the dimension is |
| 685 | // unit, but not reduced, it is not folded, thereby keeping the output type the |
| 686 | // same. If not all dimensions which are reduced are of unit dimension, this |
| 687 | // transformation does nothing. This is just a generalization of |
| 688 | // ElideSingleElementReduction for ReduceOp. |
| 689 | struct ElideUnitDimsInMultiDimReduction |
| 690 | : public OpRewritePattern<MultiDimReductionOp> { |
| 691 | using Base::Base; |
| 692 | |
| 693 | LogicalResult matchAndRewrite(MultiDimReductionOp reductionOp, |
| 694 | PatternRewriter &rewriter) const override { |
| 695 | ArrayRef<int64_t> shape = reductionOp.getSourceVectorType().getShape(); |
| 696 | for (const auto &dim : enumerate(First&: shape)) { |
| 697 | if (reductionOp.isReducedDim(d: dim.index()) && dim.value() != 1) |
| 698 | return failure(); |
| 699 | } |
| 700 | |
| 701 | // Vector mask setup. |
| 702 | OpBuilder::InsertionGuard guard(rewriter); |
| 703 | Operation *rootOp; |
| 704 | Value mask; |
| 705 | if (reductionOp.isMasked()) { |
| 706 | rewriter.setInsertionPoint(reductionOp.getMaskingOp()); |
| 707 | rootOp = reductionOp.getMaskingOp(); |
| 708 | mask = reductionOp.getMaskingOp().getMask(); |
| 709 | } else { |
| 710 | rootOp = reductionOp; |
| 711 | } |
| 712 | |
| 713 | Location loc = reductionOp.getLoc(); |
| 714 | Value acc = reductionOp.getAcc(); |
| 715 | Value cast; |
| 716 | if (auto dstVecType = dyn_cast<VectorType>(Val: reductionOp.getDestType())) { |
| 717 | if (mask) { |
| 718 | VectorType newMaskType = |
| 719 | VectorType::get(shape: dstVecType.getShape(), elementType: rewriter.getI1Type(), |
| 720 | scalableDims: dstVecType.getScalableDims()); |
| 721 | mask = vector::ShapeCastOp::create(builder&: rewriter, location: loc, result: newMaskType, source: mask); |
| 722 | } |
| 723 | cast = vector::ShapeCastOp::create( |
| 724 | builder&: rewriter, location: loc, result: reductionOp.getDestType(), source: reductionOp.getSource()); |
| 725 | } else { |
| 726 | // This means we are reducing all the dimensions, and all reduction |
| 727 | // dimensions are of size 1. So a simple extraction would do. |
| 728 | if (mask) |
| 729 | mask = vector::ExtractOp::create(builder&: rewriter, location: loc, source: mask); |
| 730 | cast = vector::ExtractOp::create(builder&: rewriter, location: loc, source: reductionOp.getSource()); |
| 731 | } |
| 732 | |
| 733 | Value result = |
| 734 | vector::makeArithReduction(b&: rewriter, loc, kind: reductionOp.getKind(), v1: cast, |
| 735 | acc, fastmath: reductionOp.getFastmathAttr(), mask); |
| 736 | rewriter.replaceOp(op: rootOp, newValues: result); |
| 737 | return success(); |
| 738 | } |
| 739 | }; |
| 740 | } // namespace |
| 741 | |
| 742 | void MultiDimReductionOp::getCanonicalizationPatterns( |
| 743 | RewritePatternSet &results, MLIRContext *context) { |
| 744 | results.add<ElideUnitDimsInMultiDimReduction>(arg&: context); |
| 745 | } |
| 746 | |
| 747 | //===----------------------------------------------------------------------===// |
| 748 | // ReductionOp |
| 749 | //===----------------------------------------------------------------------===// |
| 750 | |
| 751 | void vector::ReductionOp::build(OpBuilder &builder, OperationState &result, |
| 752 | CombiningKind kind, Value vector, |
| 753 | arith::FastMathFlags fastMathFlags) { |
| 754 | build(odsBuilder&: builder, odsState&: result, kind, vector, /*acc=*/Value(), fastMathFlags); |
| 755 | } |
| 756 | |
| 757 | void vector::ReductionOp::build(OpBuilder &builder, OperationState &result, |
| 758 | CombiningKind kind, Value vector, Value acc, |
| 759 | arith::FastMathFlags fastMathFlags) { |
| 760 | build(odsBuilder&: builder, odsState&: result, |
| 761 | result: llvm::cast<VectorType>(Val: vector.getType()).getElementType(), kind, vector, |
| 762 | acc, fastmath: fastMathFlags); |
| 763 | } |
| 764 | |
| 765 | LogicalResult ReductionOp::verify() { |
| 766 | // Verify for 0-D and 1-D vector. |
| 767 | int64_t rank = getSourceVectorType().getRank(); |
| 768 | if (rank > 1) |
| 769 | return emitOpError(message: "unsupported reduction rank: " ) << rank; |
| 770 | |
| 771 | // Verify supported reduction kind. |
| 772 | Type eltType = getResult().getType(); |
| 773 | if (!isSupportedCombiningKind(combiningKind: getKind(), elementType: eltType)) |
| 774 | return emitOpError(message: "unsupported reduction type '" ) |
| 775 | << eltType << "' for kind '" << stringifyCombiningKind(val: getKind()) |
| 776 | << "'" ; |
| 777 | |
| 778 | return success(); |
| 779 | } |
| 780 | |
| 781 | // MaskableOpInterface methods. |
| 782 | |
| 783 | /// Returns the mask type expected by this operation. |
| 784 | Type ReductionOp::getExpectedMaskType() { |
| 785 | auto vecType = getSourceVectorType(); |
| 786 | return VectorType::get(shape: vecType.getShape(), |
| 787 | elementType: IntegerType::get(context: vecType.getContext(), /*width=*/1), |
| 788 | scalableDims: vecType.getScalableDims()); |
| 789 | } |
| 790 | |
| 791 | Value mlir::vector::getVectorReductionOp(arith::AtomicRMWKind op, |
| 792 | OpBuilder &builder, Location loc, |
| 793 | Value vector) { |
| 794 | switch (op) { |
| 795 | case arith::AtomicRMWKind::addf: |
| 796 | case arith::AtomicRMWKind::addi: |
| 797 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 798 | kind: CombiningKind::ADD, vector); |
| 799 | case arith::AtomicRMWKind::mulf: |
| 800 | case arith::AtomicRMWKind::muli: |
| 801 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 802 | kind: CombiningKind::MUL, vector); |
| 803 | case arith::AtomicRMWKind::minimumf: |
| 804 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 805 | kind: CombiningKind::MINIMUMF, vector); |
| 806 | case arith::AtomicRMWKind::mins: |
| 807 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 808 | kind: CombiningKind::MINSI, vector); |
| 809 | case arith::AtomicRMWKind::minu: |
| 810 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 811 | kind: CombiningKind::MINUI, vector); |
| 812 | case arith::AtomicRMWKind::maximumf: |
| 813 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 814 | kind: CombiningKind::MAXIMUMF, vector); |
| 815 | case arith::AtomicRMWKind::maxs: |
| 816 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 817 | kind: CombiningKind::MAXSI, vector); |
| 818 | case arith::AtomicRMWKind::maxu: |
| 819 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 820 | kind: CombiningKind::MAXUI, vector); |
| 821 | case arith::AtomicRMWKind::andi: |
| 822 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 823 | kind: CombiningKind::AND, vector); |
| 824 | case arith::AtomicRMWKind::ori: |
| 825 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 826 | kind: CombiningKind::OR, vector); |
| 827 | case arith::AtomicRMWKind::minnumf: |
| 828 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 829 | kind: CombiningKind::MINNUMF, vector); |
| 830 | case arith::AtomicRMWKind::maxnumf: |
| 831 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 832 | kind: CombiningKind::MAXNUMF, vector); |
| 833 | case arith::AtomicRMWKind::xori: |
| 834 | return vector::ReductionOp::create(builder, location: vector.getLoc(), |
| 835 | kind: CombiningKind::XOR, vector); |
| 836 | default: |
| 837 | (void)emitOptionalError(loc, args: "Reduction operation type not supported" ); |
| 838 | break; |
| 839 | } |
| 840 | return nullptr; |
| 841 | } |
| 842 | |
| 843 | //----------------------------------------------------------------------------- |
| 844 | // VectorUnrollOpInterface methods |
| 845 | //----------------------------------------------------------------------------- |
| 846 | std::optional<SmallVector<int64_t, 4>> |
| 847 | ReductionOp::getIterationShapeToUnroll() { |
| 848 | return llvm::to_vector<4>(Range: getSourceVectorType().getShape()); |
| 849 | } |
| 850 | std::optional<SmallVector<int64_t>> ReductionOp::getDefaultUnrollTargetShape() { |
| 851 | return SmallVector<int64_t>{2}; |
| 852 | } |
| 853 | |
| 854 | namespace { |
| 855 | struct ElideSingleElementReduction : public OpRewritePattern<ReductionOp> { |
| 856 | using Base::Base; |
| 857 | |
| 858 | LogicalResult matchAndRewrite(ReductionOp reductionOp, |
| 859 | PatternRewriter &rewriter) const override { |
| 860 | // Vector mask setup. |
| 861 | OpBuilder::InsertionGuard guard(rewriter); |
| 862 | auto maskableOp = |
| 863 | cast<vector::MaskableOpInterface>(Val: reductionOp.getOperation()); |
| 864 | Operation *rootOp; |
| 865 | Value mask; |
| 866 | if (maskableOp.isMasked()) { |
| 867 | rewriter.setInsertionPoint(maskableOp.getMaskingOp()); |
| 868 | rootOp = maskableOp.getMaskingOp(); |
| 869 | mask = maskableOp.getMaskingOp().getMask(); |
| 870 | } else { |
| 871 | rootOp = reductionOp; |
| 872 | } |
| 873 | |
| 874 | auto vectorType = reductionOp.getSourceVectorType(); |
| 875 | if (vectorType.getRank() != 0 && vectorType.getDimSize(idx: 0) != 1) |
| 876 | return failure(); |
| 877 | |
| 878 | Location loc = reductionOp.getLoc(); |
| 879 | if (mask) |
| 880 | mask = ExtractOp::create(builder&: rewriter, location: loc, source: mask); |
| 881 | Value result = ExtractOp::create(builder&: rewriter, location: loc, source: reductionOp.getVector()); |
| 882 | |
| 883 | if (Value acc = reductionOp.getAcc()) |
| 884 | result = vector::makeArithReduction(b&: rewriter, loc, kind: reductionOp.getKind(), |
| 885 | v1: result, acc, |
| 886 | fastmath: reductionOp.getFastmathAttr(), mask); |
| 887 | |
| 888 | rewriter.replaceOp(op: rootOp, newValues: result); |
| 889 | return success(); |
| 890 | } |
| 891 | }; |
| 892 | } // namespace |
| 893 | |
| 894 | void ReductionOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 895 | MLIRContext *context) { |
| 896 | results.add<ElideSingleElementReduction>(arg&: context); |
| 897 | } |
| 898 | |
| 899 | //===----------------------------------------------------------------------===// |
| 900 | // ContractionOp |
| 901 | //===----------------------------------------------------------------------===// |
| 902 | |
| 903 | void vector::ContractionOp::build(OpBuilder &builder, OperationState &result, |
| 904 | Value lhs, Value rhs, Value acc, |
| 905 | ArrayRef<ArrayRef<AffineExpr>> indexingExprs, |
| 906 | ArrayRef<IteratorType> iteratorTypes) { |
| 907 | result.addOperands(newOperands: {lhs, rhs, acc}); |
| 908 | result.addTypes(newTypes: acc.getType()); |
| 909 | result.addAttribute( |
| 910 | name: getIndexingMapsAttrName(name: result.name), |
| 911 | attr: builder.getAffineMapArrayAttr( |
| 912 | values: AffineMap::inferFromExprList(exprsList: indexingExprs, context: builder.getContext()))); |
| 913 | result.addAttribute( |
| 914 | name: getIteratorTypesAttrName(name: result.name), |
| 915 | attr: builder.getArrayAttr(value: llvm::map_to_vector( |
| 916 | C&: iteratorTypes, F: [&](IteratorType t) -> mlir::Attribute { |
| 917 | return IteratorTypeAttr::get(context: builder.getContext(), value: t); |
| 918 | }))); |
| 919 | } |
| 920 | |
| 921 | void vector::ContractionOp::build(OpBuilder &builder, OperationState &result, |
| 922 | Value lhs, Value rhs, Value acc, |
| 923 | ArrayAttr indexingMaps, |
| 924 | ArrayAttr iteratorTypes) { |
| 925 | build(odsBuilder&: builder, odsState&: result, lhs, rhs, acc, indexingMaps, iteratorTypes, |
| 926 | kind: ContractionOp::getDefaultKind()); |
| 927 | } |
| 928 | |
| 929 | void vector::ContractionOp::build(OpBuilder &builder, OperationState &result, |
| 930 | Value lhs, Value rhs, Value acc, |
| 931 | ArrayAttr indexingMaps, |
| 932 | ArrayAttr iteratorTypes, CombiningKind kind, |
| 933 | arith::FastMathFlags fastMathFlags) { |
| 934 | result.addOperands(newOperands: {lhs, rhs, acc}); |
| 935 | result.addTypes(newTypes: acc.getType()); |
| 936 | result.addAttribute(name: getIndexingMapsAttrName(name: result.name), attr: indexingMaps); |
| 937 | result.addAttribute(name: getIteratorTypesAttrName(name: result.name), attr: iteratorTypes); |
| 938 | result.addAttribute(name: getKindAttrName(name: result.name), |
| 939 | attr: CombiningKindAttr::get(context: builder.getContext(), value: kind)); |
| 940 | if (fastMathFlags != arith::FastMathFlags::none) |
| 941 | result.addAttribute( |
| 942 | name: getFastmathAttrName(name: result.name), |
| 943 | attr: arith::FastMathFlagsAttr::get(context: builder.getContext(), value: fastMathFlags)); |
| 944 | } |
| 945 | |
| 946 | ParseResult ContractionOp::parse(OpAsmParser &parser, OperationState &result) { |
| 947 | OpAsmParser::UnresolvedOperand lhsInfo; |
| 948 | OpAsmParser::UnresolvedOperand rhsInfo; |
| 949 | OpAsmParser::UnresolvedOperand accInfo; |
| 950 | SmallVector<OpAsmParser::UnresolvedOperand, 2> masksInfo; |
| 951 | SmallVector<Type, 2> types; |
| 952 | Type resultType; |
| 953 | auto loc = parser.getCurrentLocation(); |
| 954 | DictionaryAttr dictAttr; |
| 955 | // TODO: Unify linalg op attribute parsing. |
| 956 | if (parser.parseAttribute(result&: dictAttr) || parser.parseOperand(result&: lhsInfo) || |
| 957 | parser.parseComma() || parser.parseOperand(result&: rhsInfo) || |
| 958 | parser.parseComma() || parser.parseOperand(result&: accInfo) || |
| 959 | parser.parseTrailingOperandList(result&: masksInfo) || |
| 960 | parser.parseOptionalAttrDict(result&: result.attributes) || |
| 961 | parser.parseColonTypeList(result&: types) || |
| 962 | parser.parseKeywordType(keyword: "into" , result&: resultType) || |
| 963 | parser.resolveOperand(operand: lhsInfo, type: types[0], result&: result.operands) || |
| 964 | parser.resolveOperand(operand: rhsInfo, type: types[1], result&: result.operands) || |
| 965 | parser.resolveOperand(operand: accInfo, type: resultType, result&: result.operands) || |
| 966 | parser.addTypeToList(type: resultType, result&: result.types)) |
| 967 | return failure(); |
| 968 | result.attributes.append(inStart: dictAttr.getValue().begin(), |
| 969 | inEnd: dictAttr.getValue().end()); |
| 970 | |
| 971 | // Convert array of string into an array of IteratyType enums. This is needed, |
| 972 | // because tests still use the old format when 'iterator_types' attribute is |
| 973 | // represented as an array of strings. |
| 974 | // TODO: Remove this conversion once tests are fixed. |
| 975 | auto iteratorTypes = dyn_cast_or_null<ArrayAttr>( |
| 976 | Val: result.attributes.get(name: getIteratorTypesAttrName(name: result.name))); |
| 977 | if (!iteratorTypes) { |
| 978 | return parser.emitError(loc) |
| 979 | << "expected " << getIteratorTypesAttrName(name: result.name) |
| 980 | << " array attribute" ; |
| 981 | } |
| 982 | |
| 983 | SmallVector<Attribute> iteratorTypeAttrs; |
| 984 | |
| 985 | for (StringRef s : iteratorTypes.getAsValueRange<StringAttr>()) { |
| 986 | auto maybeIteratorType = symbolizeIteratorType(str: s); |
| 987 | if (!maybeIteratorType.has_value()) |
| 988 | return parser.emitError(loc) << "unexpected iterator_type (" << s << ")" ; |
| 989 | |
| 990 | iteratorTypeAttrs.push_back( |
| 991 | Elt: IteratorTypeAttr::get(context: parser.getContext(), value: maybeIteratorType.value())); |
| 992 | } |
| 993 | result.attributes.set(name: getIteratorTypesAttrName(name: result.name), |
| 994 | value: parser.getBuilder().getArrayAttr(value: iteratorTypeAttrs)); |
| 995 | |
| 996 | if (!result.attributes.get(name: getKindAttrName(name: result.name))) { |
| 997 | result.addAttribute( |
| 998 | name: getKindAttrName(name: result.name), |
| 999 | attr: CombiningKindAttr::get(context: result.getContext(), |
| 1000 | value: ContractionOp::getDefaultKind())); |
| 1001 | } |
| 1002 | if (masksInfo.empty()) |
| 1003 | return success(); |
| 1004 | if (masksInfo.size() != 2) |
| 1005 | return parser.emitError(loc: parser.getNameLoc(), |
| 1006 | message: "expected zero or exactly 2 vector mask operands" ); |
| 1007 | auto lhsType = llvm::cast<VectorType>(Val&: types[0]); |
| 1008 | auto rhsType = llvm::cast<VectorType>(Val&: types[1]); |
| 1009 | auto maskElementType = parser.getBuilder().getI1Type(); |
| 1010 | std::array<VectorType, 2> maskTypes = { |
| 1011 | VectorType::Builder(lhsType).setElementType(maskElementType), |
| 1012 | VectorType::Builder(rhsType).setElementType(maskElementType)}; |
| 1013 | if (parser.resolveOperands(operands&: masksInfo, types&: maskTypes, loc, result&: result.operands)) |
| 1014 | return failure(); |
| 1015 | return success(); |
| 1016 | } |
| 1017 | |
| 1018 | void ContractionOp::print(OpAsmPrinter &p) { |
| 1019 | // TODO: Unify printing code with linalg ops. |
| 1020 | auto attrNames = getTraitAttrNames(); |
| 1021 | llvm::StringSet<> ; |
| 1022 | traitAttrsSet.insert_range(R&: attrNames); |
| 1023 | NamedAttrList allAttrs(getOperation()->getRawDictionaryAttrs()); |
| 1024 | getOperation()->getName().walkInherentAttrs( |
| 1025 | op: getOperation(), |
| 1026 | visitor: [&](StringRef name, Attribute &attr) { allAttrs.append(name, attr); }); |
| 1027 | SmallVector<NamedAttribute, 8> attrs; |
| 1028 | for (auto attr : allAttrs) { |
| 1029 | if (attr.getName() == getIteratorTypesAttrName()) { |
| 1030 | auto iteratorTypes = |
| 1031 | llvm::cast<ArrayAttr>(Val: attr.getValue()) |
| 1032 | .getAsValueRange<IteratorTypeAttr, IteratorType>(); |
| 1033 | // Convert IteratorType enums into the string representation. This is |
| 1034 | // needed, because tests still use the old format when 'iterator_types' |
| 1035 | // attribute is represented as an array of strings. |
| 1036 | // TODO: Remove this conversion once tests are fixed. |
| 1037 | SmallVector<Attribute> iteratorTypeNames = |
| 1038 | llvm::map_to_vector(C&: iteratorTypes, F: [&](IteratorType t) -> Attribute { |
| 1039 | return StringAttr::get(context: getContext(), bytes: stringifyIteratorType(val: t)); |
| 1040 | }); |
| 1041 | |
| 1042 | attrs.emplace_back(Args: getIteratorTypesAttrName(), |
| 1043 | Args: ArrayAttr::get(context: getContext(), value: iteratorTypeNames)); |
| 1044 | } else if (traitAttrsSet.count(Key: attr.getName().strref()) > 0) { |
| 1045 | // Omit fastmath when it equals the default (none) to keep output clean. |
| 1046 | if (attr.getName() == getFastmathAttrName() && |
| 1047 | llvm::cast<arith::FastMathFlagsAttr>(Val: attr.getValue()).getValue() == |
| 1048 | arith::FastMathFlags::none) |
| 1049 | continue; |
| 1050 | attrs.push_back(Elt: attr); |
| 1051 | } |
| 1052 | } |
| 1053 | |
| 1054 | auto dictAttr = DictionaryAttr::get(context: getContext(), value: attrs); |
| 1055 | p << " " << dictAttr << " " << getLhs() << ", " ; |
| 1056 | p << getRhs() << ", " << getAcc(); |
| 1057 | |
| 1058 | p.printOptionalAttrDict(attrs: allAttrs.getAttrs(), elidedAttrs: attrNames); |
| 1059 | p << " : " << getLhs().getType() << ", " << getRhs().getType() << " into " |
| 1060 | << getResultType(); |
| 1061 | } |
| 1062 | |
| 1063 | static bool verifyDimMap(VectorType lhsType, VectorType rhsType, |
| 1064 | const std::vector<std::pair<int64_t, int64_t>> &map) { |
| 1065 | for (auto &dimPair : map) { |
| 1066 | if (dimPair.first < 0 || dimPair.first >= lhsType.getRank() || |
| 1067 | dimPair.second < 0 || dimPair.second >= rhsType.getRank() || |
| 1068 | lhsType.getDimSize(idx: dimPair.first) != rhsType.getDimSize(idx: dimPair.second)) |
| 1069 | return false; |
| 1070 | } |
| 1071 | return true; |
| 1072 | } |
| 1073 | |
| 1074 | static LogicalResult verifyOutputShape( |
| 1075 | ContractionOp op, VectorType lhsType, VectorType rhsType, Type accType, |
| 1076 | Type resType, |
| 1077 | const std::vector<std::pair<int64_t, int64_t>> &contractingDimMap, |
| 1078 | const std::vector<std::pair<int64_t, int64_t>> &batchDimMap) { |
| 1079 | DenseSet<int64_t> lhsContractingDimSet; |
| 1080 | DenseSet<int64_t> rhsContractingDimSet; |
| 1081 | for (auto &dimPair : contractingDimMap) { |
| 1082 | lhsContractingDimSet.insert(V: dimPair.first); |
| 1083 | rhsContractingDimSet.insert(V: dimPair.second); |
| 1084 | } |
| 1085 | DenseSet<int64_t> rhsBatchDimSet(llvm::from_range, |
| 1086 | llvm::make_second_range(c: batchDimMap)); |
| 1087 | |
| 1088 | // Add free and batch dimensions from 'lhsType' to 'expectedResultDims'. |
| 1089 | SmallVector<int64_t, 4> expectedResultDims; |
| 1090 | for (int64_t i = 0, e = lhsType.getRank(); i < e; ++i) { |
| 1091 | if (lhsContractingDimSet.count(V: i) > 0) |
| 1092 | continue; |
| 1093 | expectedResultDims.push_back(Elt: lhsType.getDimSize(idx: i)); |
| 1094 | } |
| 1095 | |
| 1096 | // Add free dimensions from 'rhsType' to 'expectedResultDims'. |
| 1097 | for (int64_t i = 0, e = rhsType.getRank(); i < e; ++i) { |
| 1098 | if (rhsContractingDimSet.count(V: i) > 0 || rhsBatchDimSet.count(V: i) > 0) |
| 1099 | continue; |
| 1100 | expectedResultDims.push_back(Elt: rhsType.getDimSize(idx: i)); |
| 1101 | } |
| 1102 | |
| 1103 | // The accumulator and the result must agree, regardless of the rank. |
| 1104 | if (resType != accType) |
| 1105 | return op.emitOpError(message: "expected same accumulator and result type" ); |
| 1106 | |
| 1107 | // Verify 'expectedResultDims'. |
| 1108 | if (expectedResultDims.empty()) { |
| 1109 | // No batch or free dimension implies a scalar or 0-D vector result. |
| 1110 | if (auto resVectorType = llvm::dyn_cast<VectorType>(Val&: resType)) |
| 1111 | if (resVectorType.getRank() != 0) |
| 1112 | return op.emitOpError(message: "expected scalar or 0-D vector result when " |
| 1113 | "there are no batch or free dimensions" ); |
| 1114 | } else { |
| 1115 | // At least one batch or free dimension implies a vector result. |
| 1116 | auto resVectorType = llvm::dyn_cast<VectorType>(Val&: resType); |
| 1117 | auto accVectorType = llvm::dyn_cast<VectorType>(Val&: accType); |
| 1118 | if (!resVectorType || !accVectorType) |
| 1119 | return op.emitOpError(message: "invalid accumulator/result vector shape" ); |
| 1120 | |
| 1121 | // Infer expected result vector type. Lhs + rhs map and lhs + rhs vector |
| 1122 | // types fully define the result vector type. This assumes the affine maps |
| 1123 | // are well-formed, which must have been verified already. |
| 1124 | MLIRContext *ctx = op.getContext(); |
| 1125 | AffineMap lhsMap = op.getIndexingMapsArray()[0]; |
| 1126 | AffineMap rhsMap = op.getIndexingMapsArray()[1]; |
| 1127 | if (getUnusedDimsBitVector(maps: {lhsMap, rhsMap}).any()) |
| 1128 | return op.emitOpError( |
| 1129 | message: "expected all dimensions to be either a LHS or a RHS dimension" ); |
| 1130 | SmallVector<AffineExpr, 4> extents(lhsMap.getNumInputs()); |
| 1131 | for (auto pair : |
| 1132 | {std::make_pair(x&: lhsType, y&: lhsMap), std::make_pair(x&: rhsType, y&: rhsMap)}) { |
| 1133 | VectorType v = pair.first; |
| 1134 | auto map = pair.second; |
| 1135 | for (unsigned idx = 0, e = v.getRank(); idx < e; ++idx) { |
| 1136 | unsigned pos = map.getDimPosition(idx); |
| 1137 | if (!extents[pos]) |
| 1138 | extents[pos] = getAffineConstantExpr(constant: v.getShape()[idx], context: ctx); |
| 1139 | } |
| 1140 | } |
| 1141 | if (!llvm::all_of(Range&: extents, P: [](AffineExpr e) { return e; })) |
| 1142 | return op.emitOpError(message: "expected all dimensions to get an extent as " |
| 1143 | "either a LHS or a RHS dimension" ); |
| 1144 | |
| 1145 | AffineMap resMap = op.getIndexingMapsArray()[2]; |
| 1146 | auto extentsMap = AffineMap::get(/*dimCount=*/extents.size(), |
| 1147 | /*symbolCount=*/0, results: extents, context: ctx); |
| 1148 | // Compose the resMap with the extentsMap, which is a constant map. |
| 1149 | AffineMap expectedMap = simplifyAffineMap(map: resMap.compose(map: extentsMap)); |
| 1150 | assert(llvm::all_of(expectedMap.getResults(), |
| 1151 | llvm::IsaPred<AffineConstantExpr>) && |
| 1152 | "expected constant extent along all dimensions." ); |
| 1153 | // Extract the expected shape and build the type. |
| 1154 | auto expectedShape = |
| 1155 | llvm::map_to_vector<4>(C: expectedMap.getResults(), F: [](AffineExpr e) { |
| 1156 | return cast<AffineConstantExpr>(Val&: e).getValue(); |
| 1157 | }); |
| 1158 | auto expected = |
| 1159 | VectorType::get(shape: expectedShape, elementType: resVectorType.getElementType(), |
| 1160 | scalableDims: resVectorType.getScalableDims()); |
| 1161 | if (resVectorType != expected || accVectorType != expected) |
| 1162 | return op.emitOpError( |
| 1163 | message: "invalid accumulator/result vector shape, expected: " ) |
| 1164 | << expected; |
| 1165 | } |
| 1166 | return success(); |
| 1167 | } |
| 1168 | |
| 1169 | // Find a one-sided reduction dim, i.e. a reduction dim that is present in only |
| 1170 | // one of the inputs. |
| 1171 | static std::optional<int64_t> |
| 1172 | getOneSidedReductionDim(vector::ContractionOp *contract) { |
| 1173 | auto maps = contract->getIndexingMapsArray(); |
| 1174 | AffineMap lhsMap = maps[0]; |
| 1175 | AffineMap rhsMap = maps[1]; |
| 1176 | |
| 1177 | for (auto [dim, attr] : llvm::enumerate(First: contract->getIteratorTypes())) { |
| 1178 | auto iteratorType = cast<vector::IteratorTypeAttr>(Val: attr).getValue(); |
| 1179 | if (iteratorType == vector::IteratorType::reduction && |
| 1180 | (!lhsMap.isFunctionOfDim(position: dim) || !rhsMap.isFunctionOfDim(position: dim))) |
| 1181 | return dim; |
| 1182 | } |
| 1183 | |
| 1184 | return std::nullopt; |
| 1185 | } |
| 1186 | |
| 1187 | LogicalResult ContractionOp::verify() { |
| 1188 | VectorType lhsType = getLhsType(); |
| 1189 | VectorType rhsType = getRhsType(); |
| 1190 | Type accType = getAccType(); |
| 1191 | Type resType = getResultType(); |
| 1192 | |
| 1193 | if (llvm::isa<IntegerType>(Val: lhsType.getElementType())) { |
| 1194 | if (!lhsType.getElementType().isSignlessInteger()) |
| 1195 | return emitOpError(message: "only supports signless integer types" ); |
| 1196 | } |
| 1197 | |
| 1198 | // Verify that an indexing map was specified for each vector operand. |
| 1199 | if (getIndexingMapsArray().size() != 3) |
| 1200 | return emitOpError(message: "expected an indexing map for each vector operand" ); |
| 1201 | |
| 1202 | // Verify that each index map has 'numIterators' inputs, no symbols, and |
| 1203 | // that the number of map outputs equals the rank of its associated |
| 1204 | // vector operand. |
| 1205 | unsigned numIterators = getIteratorTypes().getValue().size(); |
| 1206 | for (const auto &it : llvm::enumerate(First: getIndexingMapsArray())) { |
| 1207 | auto index = it.index(); |
| 1208 | auto map = it.value(); |
| 1209 | if (map.getNumSymbols() != 0) |
| 1210 | return emitOpError(message: "expected indexing map " ) |
| 1211 | << index << " to have no symbols" ; |
| 1212 | auto vectorType = llvm::dyn_cast<VectorType>(Val: getOperand(i: index).getType()); |
| 1213 | unsigned rank = vectorType ? vectorType.getShape().size() : 0; |
| 1214 | // Verify that the map has the right number of inputs, outputs, and indices. |
| 1215 | // This also correctly accounts for (..) -> () for rank-0 results. |
| 1216 | if (map.getNumDims() != numIterators) |
| 1217 | return emitOpError(message: "expected indexing map " ) |
| 1218 | << index << " to have " << numIterators << " number of inputs" ; |
| 1219 | if (map.getNumResults() != rank) |
| 1220 | return emitOpError(message: "expected indexing map " ) |
| 1221 | << index << " to have " << rank << " number of outputs" ; |
| 1222 | if (!map.isProjectedPermutation()) |
| 1223 | return emitOpError(message: "expected indexing map " ) |
| 1224 | << index << " to be a projected permutation of its inputs" ; |
| 1225 | } |
| 1226 | |
| 1227 | // TODO: For better code re-use, unify this with linalg.contract. |
| 1228 | auto dimIdx = getOneSidedReductionDim(contract: this); |
| 1229 | if (dimIdx.has_value()) |
| 1230 | return emitOpError(message: "reduction dim at index " ) |
| 1231 | << dimIdx.value() << " must appear in both LHS and RHS" ; |
| 1232 | |
| 1233 | auto contractingDimMap = getContractingDimMap(); |
| 1234 | auto batchDimMap = getBatchDimMap(); |
| 1235 | |
| 1236 | // Verify at least one contracting dimension pair was specified. |
| 1237 | if (contractingDimMap.empty()) |
| 1238 | return emitOpError(message: "expected at least one contracting dimension pair" ); |
| 1239 | |
| 1240 | // Verify contracting dimension map was properly constructed. |
| 1241 | if (!verifyDimMap(lhsType, rhsType, map: contractingDimMap)) |
| 1242 | return emitOpError(message: "invalid contracting dimension map" ); |
| 1243 | |
| 1244 | // Verify batch dimension map was properly constructed. |
| 1245 | if (!verifyDimMap(lhsType, rhsType, map: batchDimMap)) |
| 1246 | return emitOpError(message: "invalid batch dimension map" ); |
| 1247 | |
| 1248 | // Verify 'accType' and 'resType' shape. |
| 1249 | if (failed(Result: verifyOutputShape(op: *this, lhsType, rhsType, accType, resType, |
| 1250 | contractingDimMap, batchDimMap))) |
| 1251 | return failure(); |
| 1252 | |
| 1253 | if (!getKindAttr()) { |
| 1254 | return emitOpError(message: "expected 'kind' attribute of type CombiningKind (e.g. " |
| 1255 | "'vector.kind<add>')" ); |
| 1256 | } |
| 1257 | |
| 1258 | // Verify supported combining kind. |
| 1259 | auto vectorType = llvm::dyn_cast<VectorType>(Val&: resType); |
| 1260 | auto elementType = vectorType ? vectorType.getElementType() : resType; |
| 1261 | if (!isSupportedCombiningKind(combiningKind: getKind(), elementType)) |
| 1262 | return emitOpError(message: "unsupported contraction type" ); |
| 1263 | |
| 1264 | // Delayed calling of IndexingMapOpInterface::verifyImpl. |
| 1265 | return cast<IndexingMapOpInterface>(Val: this->getOperation()).verifyImpl(); |
| 1266 | } |
| 1267 | |
| 1268 | // MaskableOpInterface methods. |
| 1269 | |
| 1270 | /// Returns the mask type expected by this operation. Mostly used for |
| 1271 | /// verification purposes. It requires the operation to be vectorized." |
| 1272 | Type ContractionOp::getExpectedMaskType() { |
| 1273 | auto indexingMaps = this->getIndexingMapsArray(); |
| 1274 | AffineMap lhsIdxMap = indexingMaps[0]; |
| 1275 | AffineMap rhsIdxMap = indexingMaps[1]; |
| 1276 | VectorType lhsType = this->getLhsType(); |
| 1277 | VectorType rhsType = this->getRhsType(); |
| 1278 | |
| 1279 | unsigned numVecDims = lhsIdxMap.getNumDims(); |
| 1280 | SmallVector<int64_t> maskShape(numVecDims, ShapedType::kDynamic); |
| 1281 | SmallVector<bool> maskShapeScalableDims(numVecDims, false); |
| 1282 | |
| 1283 | // Using the information in the indexing maps, extract the size of each |
| 1284 | // dimension in the vector.contract operation from the two input operands. |
| 1285 | for (auto [dimIdx, dimSize] : llvm::enumerate(First: lhsType.getShape())) { |
| 1286 | maskShape[lhsIdxMap.getDimPosition(idx: dimIdx)] = dimSize; |
| 1287 | maskShapeScalableDims[lhsIdxMap.getDimPosition(idx: dimIdx)] = |
| 1288 | lhsType.getScalableDims()[dimIdx]; |
| 1289 | } |
| 1290 | for (auto [dimIdx, dimSize] : llvm::enumerate(First: rhsType.getShape())) { |
| 1291 | maskShape[rhsIdxMap.getDimPosition(idx: dimIdx)] = dimSize; |
| 1292 | maskShapeScalableDims[rhsIdxMap.getDimPosition(idx: dimIdx)] = |
| 1293 | rhsType.getScalableDims()[dimIdx]; |
| 1294 | } |
| 1295 | |
| 1296 | assert(ShapedType::isStaticShape(maskShape) && |
| 1297 | "Mask shape couldn't be computed" ); |
| 1298 | |
| 1299 | return VectorType::get(shape: maskShape, |
| 1300 | elementType: IntegerType::get(context: lhsType.getContext(), /*width=*/1), |
| 1301 | scalableDims: maskShapeScalableDims); |
| 1302 | } |
| 1303 | |
| 1304 | SmallVector<StringRef> ContractionOp::getTraitAttrNames() { |
| 1305 | return SmallVector<StringRef>{getIndexingMapsAttrName(), |
| 1306 | getIteratorTypesAttrName(), getKindAttrName(), |
| 1307 | getFastmathAttrName()}; |
| 1308 | } |
| 1309 | |
| 1310 | static int64_t getResultIndex(AffineMap map, AffineExpr targetExpr) { |
| 1311 | for (int64_t i = 0, e = map.getNumResults(); i < e; ++i) |
| 1312 | if (targetExpr == map.getResult(idx: i)) |
| 1313 | return i; |
| 1314 | return -1; |
| 1315 | } |
| 1316 | |
| 1317 | // For every contraction `dim` for which: |
| 1318 | // * the corresponding iteratory type matches `targetIteratorType`, |
| 1319 | // returns a pair: <lhsIdx, rhsIdx>, where: |
| 1320 | // * {lhs|rhs}Idx is the index of `dim` in the (result of) indexing map for |
| 1321 | // {lhs|rhs}. |
| 1322 | // If `oneSidedDim` is true, only dims indexed on exactly one of the `contract` |
| 1323 | // operands are returned. |
| 1324 | static std::vector<std::pair<int64_t, int64_t>> |
| 1325 | getDimPairForIterTy(ContractionOp *contract, IteratorType targetIteratorType, |
| 1326 | bool oneSidedDim = false) { |
| 1327 | SmallVector<AffineMap, 4> indexingMaps(contract->getIndexingMapsArray()); |
| 1328 | ArrayAttr iteratorTypes = contract->getIteratorTypes(); |
| 1329 | std::vector<std::pair<int64_t, int64_t>> dimMap; |
| 1330 | for (const auto &it : llvm::enumerate(First&: iteratorTypes)) { |
| 1331 | auto iteratorType = llvm::cast<IteratorTypeAttr>(Val: it.value()).getValue(); |
| 1332 | if (iteratorType != targetIteratorType) |
| 1333 | continue; |
| 1334 | // Search lhs/rhs map results for 'targetExpr'. |
| 1335 | auto targetExpr = getAffineDimExpr(position: it.index(), context: contract->getContext()); |
| 1336 | int64_t lhsDim = getResultIndex(map: indexingMaps[0], targetExpr); |
| 1337 | int64_t rhsDim = getResultIndex(map: indexingMaps[1], targetExpr); |
| 1338 | bool isOneSidedDim = (lhsDim >= 0) != (rhsDim >= 0); |
| 1339 | bool isTwoSidedDim = lhsDim >= 0 && rhsDim >= 0; |
| 1340 | if (oneSidedDim ? isOneSidedDim : isTwoSidedDim) |
| 1341 | dimMap.emplace_back(args&: lhsDim, args&: rhsDim); |
| 1342 | } |
| 1343 | return dimMap; |
| 1344 | } |
| 1345 | |
| 1346 | std::vector<std::pair<int64_t, int64_t>> ContractionOp::getContractingDimMap() { |
| 1347 | return getDimPairForIterTy(contract: this, targetIteratorType: IteratorType::reduction); |
| 1348 | } |
| 1349 | |
| 1350 | std::vector<std::pair<int64_t, int64_t>> ContractionOp::getBatchDimMap() { |
| 1351 | return getDimPairForIterTy(contract: this, targetIteratorType: IteratorType::parallel); |
| 1352 | } |
| 1353 | |
| 1354 | std::vector<std::pair<int64_t, int64_t>> ContractionOp::getFreeDimMap() { |
| 1355 | return getDimPairForIterTy(contract: this, targetIteratorType: IteratorType::parallel, |
| 1356 | /*oneSidedDim=*/true); |
| 1357 | } |
| 1358 | |
| 1359 | void ContractionOp::getIterationBounds( |
| 1360 | SmallVectorImpl<int64_t> &iterationBounds) { |
| 1361 | auto lhsShape = getLhsType().getShape(); |
| 1362 | auto resVectorType = llvm::dyn_cast<VectorType>(Val: getResultType()); |
| 1363 | SmallVector<AffineMap, 4> indexingMaps(getIndexingMapsArray()); |
| 1364 | for (const auto &it : llvm::enumerate(First: getIteratorTypes())) { |
| 1365 | // Search lhs/rhs map results for 'targetExpr'. |
| 1366 | auto targetExpr = getAffineDimExpr(position: it.index(), context: getContext()); |
| 1367 | auto iteratorType = llvm::cast<IteratorTypeAttr>(Val: it.value()).getValue(); |
| 1368 | if (iteratorType == IteratorType::reduction) { |
| 1369 | // Get reduction dim size from lhs shape (same size in rhsShape). |
| 1370 | int64_t lhsDimIndex = getResultIndex(map: indexingMaps[0], targetExpr); |
| 1371 | assert(lhsDimIndex >= 0); |
| 1372 | iterationBounds.push_back(Elt: lhsShape[lhsDimIndex]); |
| 1373 | continue; |
| 1374 | } |
| 1375 | // Get parallel dimension size from result shape. |
| 1376 | int64_t resDimIndex = getResultIndex(map: indexingMaps[2], targetExpr); |
| 1377 | assert(resDimIndex >= 0); |
| 1378 | assert(resVectorType != nullptr); |
| 1379 | iterationBounds.push_back(Elt: resVectorType.getShape()[resDimIndex]); |
| 1380 | } |
| 1381 | } |
| 1382 | |
| 1383 | void ContractionOp::getIterationIndexMap( |
| 1384 | std::vector<DenseMap<int64_t, int64_t>> &iterationIndexMap) { |
| 1385 | unsigned numMaps = getIndexingMapsArray().size(); |
| 1386 | iterationIndexMap.resize(new_size: numMaps); |
| 1387 | for (const auto &it : llvm::enumerate(First: getIndexingMapsArray())) { |
| 1388 | auto index = it.index(); |
| 1389 | auto map = it.value(); |
| 1390 | for (unsigned i = 0, e = map.getNumResults(); i < e; ++i) { |
| 1391 | auto dim = cast<AffineDimExpr>(Val: map.getResult(idx: i)); |
| 1392 | iterationIndexMap[index][dim.getPosition()] = i; |
| 1393 | } |
| 1394 | } |
| 1395 | } |
| 1396 | |
| 1397 | //----------------------------------------------------------------------------- |
| 1398 | // VectorUnrollOpInterface methods |
| 1399 | //----------------------------------------------------------------------------- |
| 1400 | std::optional<SmallVector<int64_t, 4>> |
| 1401 | ContractionOp::getIterationShapeToUnroll() { |
| 1402 | SmallVector<int64_t, 4> shape; |
| 1403 | getIterationBounds(iterationBounds&: shape); |
| 1404 | return shape; |
| 1405 | } |
| 1406 | |
| 1407 | std::optional<SmallVector<int64_t>> |
| 1408 | ContractionOp::getDefaultUnrollTargetShape() { |
| 1409 | SmallVector<int64_t> nativeShape(this->getIteratorTypes().size(), 4); |
| 1410 | Type lhsType = this->getLhsType().getElementType(); |
| 1411 | nativeShape[nativeShape.size() - 1] = lhsType.isF16() ? 4 : 2; |
| 1412 | return nativeShape; |
| 1413 | } |
| 1414 | |
| 1415 | /// Return a fused vector::ContractionOp which represents a patterns such as: |
| 1416 | /// |
| 1417 | /// ```mlir |
| 1418 | /// %c0 = vector.constant 0: ... |
| 1419 | /// %c = vector.contract %a, %b, %c0: ... |
| 1420 | /// %e = add %c, %d: ... |
| 1421 | /// ``` |
| 1422 | /// |
| 1423 | /// by: |
| 1424 | /// |
| 1425 | /// ```mlir |
| 1426 | /// %e = vector.contract %a, %b, %d: ... |
| 1427 | /// ``` |
| 1428 | /// |
| 1429 | /// Return null if the canonicalization does not apply. |
| 1430 | // TODO: This should be a folding of Add into Contract in core but while they |
| 1431 | // live in different dialects, it is not possible without unnatural |
| 1432 | // dependencies. |
| 1433 | template <typename AddOpType> |
| 1434 | struct CanonicalizeContractAdd : public OpRewritePattern<AddOpType> { |
| 1435 | using OpRewritePattern<AddOpType>::OpRewritePattern; |
| 1436 | |
| 1437 | LogicalResult matchAndRewrite(AddOpType addOp, |
| 1438 | PatternRewriter &rewriter) const override { |
| 1439 | auto canonicalize = [&](Value maybeContraction, |
| 1440 | Value otherOperand) -> vector::ContractionOp { |
| 1441 | vector::ContractionOp contractionOp = |
| 1442 | dyn_cast_or_null<vector::ContractionOp>( |
| 1443 | Val: maybeContraction.getDefiningOp()); |
| 1444 | if (!contractionOp) |
| 1445 | return vector::ContractionOp(); |
| 1446 | if (auto maybeZero = dyn_cast_or_null<arith::ConstantOp>( |
| 1447 | Val: contractionOp.getAcc().getDefiningOp())) { |
| 1448 | if (maybeZero.getValue() == |
| 1449 | rewriter.getZeroAttr(type: contractionOp.getAcc().getType())) { |
| 1450 | IRMapping bvm; |
| 1451 | bvm.map(from: contractionOp.getAcc(), to: otherOperand); |
| 1452 | auto newContraction = |
| 1453 | cast<vector::ContractionOp>(Val: rewriter.clone(op&: *contractionOp, mapper&: bvm)); |
| 1454 | rewriter.replaceOp(addOp, newContraction.getResult()); |
| 1455 | return newContraction; |
| 1456 | } |
| 1457 | } |
| 1458 | return vector::ContractionOp(); |
| 1459 | }; |
| 1460 | |
| 1461 | Value a = addOp->getOperand(0), b = addOp->getOperand(1); |
| 1462 | vector::ContractionOp contract = canonicalize(a, b); |
| 1463 | contract = contract ? contract : canonicalize(b, a); |
| 1464 | return contract ? success() : failure(); |
| 1465 | } |
| 1466 | }; |
| 1467 | |
| 1468 | void ContractionOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 1469 | MLIRContext *context) { |
| 1470 | results.add<CanonicalizeContractAdd<arith::AddIOp>, |
| 1471 | CanonicalizeContractAdd<arith::AddFOp>>(arg&: context); |
| 1472 | } |
| 1473 | |
| 1474 | // Returns `true` if `index` is either within [0, maxIndex) or equal to |
| 1475 | // `poisonValue`. |
| 1476 | static bool isValidPositiveIndexOrPoison(int64_t index, int64_t poisonValue, |
| 1477 | int64_t maxIndex) { |
| 1478 | return index == poisonValue || (index >= 0 && index < maxIndex); |
| 1479 | } |
| 1480 | |
| 1481 | //===----------------------------------------------------------------------===// |
| 1482 | // ExtractOp |
| 1483 | //===----------------------------------------------------------------------===// |
| 1484 | |
| 1485 | void ExtractOp::(ArrayRef<ConstantIntRanges> argRanges, |
| 1486 | SetIntRangeFn setResultRanges) { |
| 1487 | setResultRanges(getResult(), argRanges.front()); |
| 1488 | } |
| 1489 | |
| 1490 | void vector::ExtractOp::(OpBuilder &builder, OperationState &result, |
| 1491 | Value source) { |
| 1492 | auto vectorTy = cast<VectorType>(Val: source.getType()); |
| 1493 | build(odsBuilder&: builder, odsState&: result, source, position: SmallVector<int64_t>(vectorTy.getRank(), 0)); |
| 1494 | } |
| 1495 | |
| 1496 | void vector::ExtractOp::(OpBuilder &builder, OperationState &result, |
| 1497 | Value source, int64_t position) { |
| 1498 | build(odsBuilder&: builder, odsState&: result, source, position: ArrayRef<int64_t>{position}); |
| 1499 | } |
| 1500 | |
| 1501 | void vector::ExtractOp::(OpBuilder &builder, OperationState &result, |
| 1502 | Value source, OpFoldResult position) { |
| 1503 | build(odsBuilder&: builder, odsState&: result, source, position: ArrayRef<OpFoldResult>{position}); |
| 1504 | } |
| 1505 | |
| 1506 | void vector::ExtractOp::(OpBuilder &builder, OperationState &result, |
| 1507 | Value source, ArrayRef<int64_t> position) { |
| 1508 | build(odsBuilder&: builder, odsState&: result, source, /*dynamic_position=*/ArrayRef<Value>(), |
| 1509 | static_position: builder.getDenseI64ArrayAttr(values: position)); |
| 1510 | } |
| 1511 | |
| 1512 | void vector::ExtractOp::(OpBuilder &builder, OperationState &result, |
| 1513 | Value source, ArrayRef<OpFoldResult> position) { |
| 1514 | SmallVector<int64_t> staticPos; |
| 1515 | SmallVector<Value> dynamicPos; |
| 1516 | dispatchIndexOpFoldResults(ofrs: position, dynamicVec&: dynamicPos, staticVec&: staticPos); |
| 1517 | build(odsBuilder&: builder, odsState&: result, source, dynamic_position: dynamicPos, |
| 1518 | static_position: builder.getDenseI64ArrayAttr(values: staticPos)); |
| 1519 | } |
| 1520 | |
| 1521 | LogicalResult |
| 1522 | ExtractOp::(MLIRContext *, std::optional<Location>, |
| 1523 | ExtractOp::Adaptor adaptor, |
| 1524 | SmallVectorImpl<Type> &inferredReturnTypes) { |
| 1525 | auto vectorType = llvm::cast<VectorType>(Val: adaptor.getSource().getType()); |
| 1526 | if (static_cast<int64_t>(adaptor.getStaticPosition().size()) == |
| 1527 | vectorType.getRank()) { |
| 1528 | inferredReturnTypes.push_back(Elt: vectorType.getElementType()); |
| 1529 | } else { |
| 1530 | auto n = std::min<size_t>(a: adaptor.getStaticPosition().size(), |
| 1531 | b: vectorType.getRank()); |
| 1532 | inferredReturnTypes.push_back(Elt: VectorType::get( |
| 1533 | shape: vectorType.getShape().drop_front(N: n), elementType: vectorType.getElementType(), |
| 1534 | scalableDims: vectorType.getScalableDims().drop_front(N: n))); |
| 1535 | } |
| 1536 | return success(); |
| 1537 | } |
| 1538 | |
| 1539 | LogicalResult vector::ExtractOp::() { |
| 1540 | if (auto resTy = dyn_cast<VectorType>(Val: getResult().getType())) |
| 1541 | if (resTy.getRank() == 0) |
| 1542 | return emitError( |
| 1543 | message: "expected a scalar instead of a 0-d vector as the result type" ); |
| 1544 | |
| 1545 | // Note: This check must come before getMixedPosition() to prevent a crash. |
| 1546 | auto dynamicMarkersCount = |
| 1547 | llvm::count_if(Range: getStaticPosition(), P: ShapedType::isDynamic); |
| 1548 | if (static_cast<size_t>(dynamicMarkersCount) != getDynamicPosition().size()) |
| 1549 | return emitOpError( |
| 1550 | message: "mismatch between dynamic and static positions (kDynamic marker but no " |
| 1551 | "corresponding dynamic position) -- this can only happen due to an " |
| 1552 | "incorrect fold/rewrite" ); |
| 1553 | auto position = getMixedPosition(); |
| 1554 | if (position.size() > static_cast<unsigned>(getSourceVectorType().getRank())) |
| 1555 | return emitOpError( |
| 1556 | message: "expected position attribute of rank no greater than vector rank" ); |
| 1557 | for (auto [idx, pos] : llvm::enumerate(First&: position)) { |
| 1558 | if (auto attr = dyn_cast<Attribute>(Val&: pos)) { |
| 1559 | int64_t constIdx = cast<IntegerAttr>(Val&: attr).getInt(); |
| 1560 | if (!isValidPositiveIndexOrPoison( |
| 1561 | index: constIdx, poisonValue: kPoisonIndex, maxIndex: getSourceVectorType().getDimSize(idx))) { |
| 1562 | return emitOpError(message: "expected position attribute #" ) |
| 1563 | << (idx + 1) |
| 1564 | << " to be a non-negative integer smaller than the " |
| 1565 | "corresponding vector dimension or poison (-1)" ; |
| 1566 | } |
| 1567 | } |
| 1568 | } |
| 1569 | return success(); |
| 1570 | } |
| 1571 | |
| 1572 | template <typename IntType> |
| 1573 | static SmallVector<IntType> (ArrayAttr arrayAttr) { |
| 1574 | return llvm::map_to_vector<4>( |
| 1575 | arrayAttr.getAsRange<IntegerAttr>(), |
| 1576 | [](IntegerAttr attr) { return static_cast<IntType>(attr.getInt()); }); |
| 1577 | } |
| 1578 | |
| 1579 | /// Fold the result of chains of ExtractOp in place by simply concatenating the |
| 1580 | /// positions. Both static and dynamic positions are supported. |
| 1581 | /// |
| 1582 | /// BEFORE: |
| 1583 | /// ```mlir |
| 1584 | /// %0 = vector.extract %v[1, 2] : vector<3x4x5xf32> |
| 1585 | /// %1 = vector.extract %0[%idx, 3] : vector<5xf32> |
| 1586 | /// ``` |
| 1587 | /// |
| 1588 | /// AFTER: |
| 1589 | /// ```mlir |
| 1590 | /// %1 = vector.extract %v[1, 2, %idx, 3] : vector<3x4x5xf32> |
| 1591 | /// ``` |
| 1592 | static LogicalResult (ExtractOp ) { |
| 1593 | if (!extractOp.getSource().getDefiningOp<ExtractOp>()) |
| 1594 | return failure(); |
| 1595 | |
| 1596 | SmallVector<OpFoldResult> globalPosition = extractOp.getMixedPosition(); |
| 1597 | ExtractOp currentOp = extractOp; |
| 1598 | while (ExtractOp nextOp = currentOp.getSource().getDefiningOp<ExtractOp>()) { |
| 1599 | currentOp = nextOp; |
| 1600 | SmallVector<OpFoldResult> pos = currentOp.getMixedPosition(); |
| 1601 | globalPosition.insert(I: globalPosition.begin(), From: pos.begin(), To: pos.end()); |
| 1602 | } |
| 1603 | auto [staticPos, dynPos] = decomposeMixedValues(mixedValues: globalPosition); |
| 1604 | Value source = currentOp.getSource(); |
| 1605 | extractOp->setOperands( |
| 1606 | llvm::to_vector(Range: llvm::concat<Value>(Ranges: ValueRange(source), Ranges&: dynPos))); |
| 1607 | extractOp.setStaticPosition(staticPos); |
| 1608 | return success(); |
| 1609 | } |
| 1610 | |
| 1611 | namespace { |
| 1612 | /// Fold an ExtractOp that is fed by a chain of InsertOps and TransposeOps. |
| 1613 | /// Walk back a chain of InsertOp/TransposeOp until we hit a match. |
| 1614 | /// Compose TransposeOp permutations as we walk back. |
| 1615 | /// This helper class keeps an updated extraction position `extractPosition` |
| 1616 | /// with extra trailing sentinels. |
| 1617 | /// The sentinels encode the internal transposition status of the result vector. |
| 1618 | /// As we iterate, extractPosition is permuted and updated. |
| 1619 | class { |
| 1620 | public: |
| 1621 | ExtractFromInsertTransposeChainState(ExtractOp e); |
| 1622 | |
| 1623 | /// Iterate over producing insert and transpose ops until we find a fold. |
| 1624 | Value fold(); |
| 1625 | |
| 1626 | private: |
| 1627 | /// Return true if the vector at position `a` is contained within the vector |
| 1628 | /// at position `b`. Under insert/extract semantics, this is the same as `a` |
| 1629 | /// is a prefix of `b`. |
| 1630 | template <typename ContainerA, typename ContainerB> |
| 1631 | bool (const ContainerA &a, const ContainerB &b) { |
| 1632 | return a.size() <= b.size() && |
| 1633 | std::equal(a.begin(), a.begin() + a.size(), b.begin()); |
| 1634 | } |
| 1635 | |
| 1636 | /// Return true if the vector at position `a` intersects the vector at |
| 1637 | /// position `b`. Under insert/extract semantics, this is the same as equality |
| 1638 | /// of all entries of `a` that are >=0 with the corresponding entries of b. |
| 1639 | /// Comparison is on the common prefix (i.e. zip). |
| 1640 | template <typename ContainerA, typename ContainerB> |
| 1641 | bool (const ContainerA &a, const ContainerB &b) { |
| 1642 | for (auto [elemA, elemB] : llvm::zip(a, b)) { |
| 1643 | if (elemA < 0 || elemB < 0) |
| 1644 | continue; |
| 1645 | if (elemA != elemB) |
| 1646 | return false; |
| 1647 | } |
| 1648 | return true; |
| 1649 | } |
| 1650 | |
| 1651 | /// Folding is only possible in the absence of an internal permutation in the |
| 1652 | /// result vector. |
| 1653 | bool () { |
| 1654 | return (sentinels == ArrayRef(extractPosition).drop_front(N: extractedRank)); |
| 1655 | } |
| 1656 | |
| 1657 | // Helper to get the next defining op of interest. |
| 1658 | void (Value v) { |
| 1659 | nextInsertOp = v.getDefiningOp<vector::InsertOp>(); |
| 1660 | nextTransposeOp = v.getDefiningOp<vector::TransposeOp>(); |
| 1661 | }; |
| 1662 | |
| 1663 | // Case 1. If we hit a transpose, just compose the map and iterate. |
| 1664 | // Invariant: insert + transpose do not change rank, we can always compose. |
| 1665 | LogicalResult handleTransposeOp(); |
| 1666 | |
| 1667 | // Case 2: the insert position matches extractPosition exactly, early return. |
| 1668 | LogicalResult handleInsertOpWithMatchingPos(Value &res); |
| 1669 | |
| 1670 | /// Case 3: if the insert position is a prefix of extractPosition, extract a |
| 1671 | /// portion of the source of the insert. |
| 1672 | /// Example: |
| 1673 | /// ``` |
| 1674 | /// %ins = vector.insert %source, %vest[1]: vector<3x4> into vector<2x3x4x5> |
| 1675 | /// // extractPosition == [1, 2, 3] |
| 1676 | /// %ext = vector.extract %ins[1, 0]: vector<5> from vector<3x4x5> |
| 1677 | /// // can fold to vector.extract %source[0, 3] |
| 1678 | /// %ext = vector.extract %source[3]: vector<6> from vector<5x6> |
| 1679 | /// ``` |
| 1680 | /// To traverse through %source, we need to set the leading dims to 0 and |
| 1681 | /// drop the extra leading dims. |
| 1682 | /// This method updates the internal state. |
| 1683 | LogicalResult handleInsertOpWithPrefixPos(Value &res); |
| 1684 | |
| 1685 | /// Try to fold in place to extract(source, extractPosition) and return the |
| 1686 | /// folded result. Return null if folding is not possible (e.g. due to an |
| 1687 | /// internal transposition in the result). |
| 1688 | Value tryToFoldExtractOpInPlace(Value source); |
| 1689 | |
| 1690 | ExtractOp ; |
| 1691 | int64_t ; |
| 1692 | int64_t ; |
| 1693 | |
| 1694 | InsertOp ; |
| 1695 | TransposeOp ; |
| 1696 | |
| 1697 | /// Sentinel values that encode the internal permutation status of the result. |
| 1698 | /// They are set to (-1, ... , -k) at the beginning and appended to |
| 1699 | /// `extractPosition`. |
| 1700 | /// In the end, the tail of `extractPosition` must be exactly `sentinels` to |
| 1701 | /// ensure that there is no internal transposition. |
| 1702 | /// Internal transposition cannot be accounted for with a folding pattern. |
| 1703 | // TODO: We could relax the internal transposition with an extra transposition |
| 1704 | // operation in a future canonicalizer. |
| 1705 | SmallVector<int64_t> ; |
| 1706 | SmallVector<int64_t> ; |
| 1707 | }; |
| 1708 | } // namespace |
| 1709 | |
| 1710 | ExtractFromInsertTransposeChainState::( |
| 1711 | ExtractOp e) |
| 1712 | : extractOp(e), vectorRank(extractOp.getSourceVectorType().getRank()), |
| 1713 | extractedRank(extractOp.getNumIndices()) { |
| 1714 | assert(vectorRank >= extractedRank && "Extracted position overflow" ); |
| 1715 | sentinels.reserve(N: vectorRank - extractedRank); |
| 1716 | for (int64_t i = 0, e = vectorRank - extractedRank; i < e; ++i) |
| 1717 | sentinels.push_back(Elt: -(i + 1)); |
| 1718 | extractPosition.assign(in_start: extractOp.getStaticPosition().begin(), |
| 1719 | in_end: extractOp.getStaticPosition().end()); |
| 1720 | llvm::append_range(C&: extractPosition, R&: sentinels); |
| 1721 | } |
| 1722 | |
| 1723 | // Case 1. If we hit a transpose, just compose the map and iterate. |
| 1724 | // Invariant: insert + transpose do not change rank, we can always compose. |
| 1725 | LogicalResult ExtractFromInsertTransposeChainState::handleTransposeOp() { |
| 1726 | // TODO: Canonicalization for dynamic position not implemented yet. |
| 1727 | if (extractOp.hasDynamicPosition()) |
| 1728 | return failure(); |
| 1729 | |
| 1730 | if (!nextTransposeOp) |
| 1731 | return failure(); |
| 1732 | |
| 1733 | if (nextTransposeOp.getResultVectorType().getRank() == 0) |
| 1734 | return success(); |
| 1735 | |
| 1736 | AffineMap m = inversePermutation(map: AffineMap::getPermutationMap( |
| 1737 | permutation: nextTransposeOp.getPermutation(), context: extractOp.getContext())); |
| 1738 | extractPosition = applyPermutationMap(map: m, source: ArrayRef(extractPosition)); |
| 1739 | return success(); |
| 1740 | } |
| 1741 | |
| 1742 | // Case 2: the insert position matches extractPosition exactly, early return. |
| 1743 | LogicalResult |
| 1744 | ExtractFromInsertTransposeChainState::handleInsertOpWithMatchingPos( |
| 1745 | Value &res) { |
| 1746 | // TODO: Canonicalization for dynamic position not implemented yet. |
| 1747 | if (extractOp.hasDynamicPosition() || nextInsertOp.hasDynamicPosition()) |
| 1748 | return failure(); |
| 1749 | |
| 1750 | ArrayRef<int64_t> insertedPos = nextInsertOp.getStaticPosition(); |
| 1751 | if (insertedPos != llvm::ArrayRef(extractPosition).take_front(N: extractedRank)) |
| 1752 | return failure(); |
| 1753 | // Case 2.a. early-exit fold. |
| 1754 | res = nextInsertOp.getValueToStore(); |
| 1755 | // Case 2.b. if internal transposition is present, canFold will be false. |
| 1756 | return success(IsSuccess: canFold()); |
| 1757 | } |
| 1758 | |
| 1759 | /// Case 3: if inserted position is a prefix of extractPosition, |
| 1760 | /// extract a portion of the source of the insertion. |
| 1761 | /// This method updates the internal state. |
| 1762 | LogicalResult |
| 1763 | ExtractFromInsertTransposeChainState::handleInsertOpWithPrefixPos(Value &res) { |
| 1764 | // TODO: Canonicalization for dynamic position not implemented yet. |
| 1765 | if (extractOp.hasDynamicPosition() || nextInsertOp.hasDynamicPosition()) |
| 1766 | return failure(); |
| 1767 | |
| 1768 | ArrayRef<int64_t> insertedPos = nextInsertOp.getStaticPosition(); |
| 1769 | if (!isContainedWithin(a: insertedPos, b: extractPosition)) |
| 1770 | return failure(); |
| 1771 | // Set leading dims to zero. |
| 1772 | std::fill_n(first: extractPosition.begin(), n: insertedPos.size(), value: 0); |
| 1773 | // Drop extra leading dims. |
| 1774 | extractPosition.erase(CS: extractPosition.begin(), |
| 1775 | CE: extractPosition.begin() + insertedPos.size()); |
| 1776 | extractedRank = extractPosition.size() - sentinels.size(); |
| 1777 | // Case 3.a. early-exit fold (break and delegate to post-while path). |
| 1778 | res = nextInsertOp.getValueToStore(); |
| 1779 | // Case 3.b. if internal transposition is present, canFold will be false. |
| 1780 | return success(); |
| 1781 | } |
| 1782 | |
| 1783 | /// Try to fold in place to extract(source, extractPosition) and return the |
| 1784 | /// folded result. Return null if folding is not possible (e.g. due to an |
| 1785 | /// internal transposition in the result). |
| 1786 | Value ExtractFromInsertTransposeChainState::( |
| 1787 | Value source) { |
| 1788 | // TODO: Canonicalization for dynamic position not implemented yet. |
| 1789 | if (extractOp.hasDynamicPosition()) |
| 1790 | return Value(); |
| 1791 | |
| 1792 | // If we can't fold (either internal transposition, or nothing to fold), bail. |
| 1793 | bool nothingToFold = (source == extractOp.getSource()); |
| 1794 | if (nothingToFold || !canFold()) |
| 1795 | return Value(); |
| 1796 | |
| 1797 | // Otherwise, fold by updating the op inplace and return its result. |
| 1798 | OpBuilder b(extractOp.getContext()); |
| 1799 | extractOp.setStaticPosition( |
| 1800 | ArrayRef(extractPosition).take_front(N: extractedRank)); |
| 1801 | extractOp.getSourceMutable().assign(value: source); |
| 1802 | return extractOp.getResult(); |
| 1803 | } |
| 1804 | |
| 1805 | /// Iterate over producing insert and transpose ops until we find a fold. |
| 1806 | Value ExtractFromInsertTransposeChainState::() { |
| 1807 | // TODO: Canonicalization for dynamic position not implemented yet. |
| 1808 | if (extractOp.hasDynamicPosition()) |
| 1809 | return Value(); |
| 1810 | |
| 1811 | Value = extractOp.getSource(); |
| 1812 | updateStateForNextIteration(v: valueToExtractFrom); |
| 1813 | while (nextInsertOp || nextTransposeOp) { |
| 1814 | // Case 1. If we hit a transpose, just compose the map and iterate. |
| 1815 | // Invariant: insert + transpose do not change rank, we can always compose. |
| 1816 | if (succeeded(Result: handleTransposeOp())) { |
| 1817 | valueToExtractFrom = nextTransposeOp.getVector(); |
| 1818 | updateStateForNextIteration(v: valueToExtractFrom); |
| 1819 | continue; |
| 1820 | } |
| 1821 | |
| 1822 | Value result; |
| 1823 | // Case 2: the position match exactly. |
| 1824 | if (succeeded(Result: handleInsertOpWithMatchingPos(res&: result))) |
| 1825 | return result; |
| 1826 | |
| 1827 | // Case 3: if the inserted position is a prefix of extractPosition, we can |
| 1828 | // just extract a portion of the source of the insert. |
| 1829 | if (succeeded(Result: handleInsertOpWithPrefixPos(res&: result))) |
| 1830 | return tryToFoldExtractOpInPlace(source: result); |
| 1831 | |
| 1832 | // Case 4: extractPositionRef intersects insertedPosRef on non-sentinel |
| 1833 | // values. This is a more difficult case and we bail. |
| 1834 | ArrayRef<int64_t> insertedPos = nextInsertOp.getStaticPosition(); |
| 1835 | if (isContainedWithin(a: extractPosition, b: insertedPos) || |
| 1836 | intersectsWhereNonNegative(a: extractPosition, b: insertedPos)) |
| 1837 | return Value(); |
| 1838 | |
| 1839 | // Case 5: No intersection, we forward the extract to insertOp.dest(). |
| 1840 | valueToExtractFrom = nextInsertOp.getDest(); |
| 1841 | updateStateForNextIteration(v: valueToExtractFrom); |
| 1842 | } |
| 1843 | // If after all this we can fold, go for it. |
| 1844 | return tryToFoldExtractOpInPlace(source: valueToExtractFrom); |
| 1845 | } |
| 1846 | |
| 1847 | /// Returns true if the operation has a 0-D vector type operand or result. |
| 1848 | static bool hasZeroDimVectors(Operation *op) { |
| 1849 | auto hasZeroDimVectorType = [](Type type) -> bool { |
| 1850 | auto vecType = dyn_cast<VectorType>(Val&: type); |
| 1851 | return vecType && vecType.getRank() == 0; |
| 1852 | }; |
| 1853 | |
| 1854 | return llvm::any_of(Range: op->getOperandTypes(), P: hasZeroDimVectorType) || |
| 1855 | llvm::any_of(Range: op->getResultTypes(), P: hasZeroDimVectorType); |
| 1856 | } |
| 1857 | |
| 1858 | /// All BroadcastOps, as well as ShapeCastOps that only prepend 1s, are |
| 1859 | /// considered to be 'broadcastlike'. |
| 1860 | static bool isBroadcastLike(Operation *op) { |
| 1861 | if (isa<BroadcastOp>(Val: op)) |
| 1862 | return true; |
| 1863 | |
| 1864 | auto shapeCast = dyn_cast<ShapeCastOp>(Val: op); |
| 1865 | if (!shapeCast) |
| 1866 | return false; |
| 1867 | |
| 1868 | // Check that shape_cast **only** prepends 1s, like (2,3) -> (1,1,2,3). |
| 1869 | // Checking that the destination shape has a prefix of 1s is not sufficient, |
| 1870 | // for example (2,3) -> (1,3,2) is not broadcastlike. A sufficient condition |
| 1871 | // is that the source shape is a suffix of the destination shape. |
| 1872 | VectorType srcType = shapeCast.getSourceVectorType(); |
| 1873 | ArrayRef<int64_t> srcShape = srcType.getShape(); |
| 1874 | uint64_t srcRank = srcType.getRank(); |
| 1875 | ArrayRef<int64_t> dstShape = shapeCast.getType().getShape(); |
| 1876 | return dstShape.size() >= srcRank && dstShape.take_back(N: srcRank) == srcShape; |
| 1877 | } |
| 1878 | |
| 1879 | /// Fold extract(broadcast(X)) to either extract(X) or just X. |
| 1880 | /// |
| 1881 | /// Example: |
| 1882 | /// |
| 1883 | /// broadcast extract [1][2] |
| 1884 | /// (3, 4) --------> (2, 3, 4) ----------------> (4) |
| 1885 | /// |
| 1886 | /// becomes |
| 1887 | /// extract [1] |
| 1888 | /// (3,4) -------------------------------------> (4) |
| 1889 | /// |
| 1890 | /// |
| 1891 | /// The variable names used in this implementation correspond to the above |
| 1892 | /// shapes as, |
| 1893 | /// |
| 1894 | /// - (3, 4) is `input` shape. |
| 1895 | /// - (2, 3, 4) is `broadcast` shape. |
| 1896 | /// - (4) is `extract` shape. |
| 1897 | /// |
| 1898 | /// This folding is possible when the suffix of `input` shape is the same as |
| 1899 | /// `extract` shape. |
| 1900 | static Value (ExtractOp ) { |
| 1901 | |
| 1902 | Operation *defOp = extractOp.getSource().getDefiningOp(); |
| 1903 | if (!defOp || !isBroadcastLike(op: defOp)) |
| 1904 | return Value(); |
| 1905 | |
| 1906 | Value input = defOp->getOperand(idx: 0); |
| 1907 | |
| 1908 | // Replace extract(broadcast(X)) with X |
| 1909 | if (extractOp.getType() == input.getType()) |
| 1910 | return input; |
| 1911 | |
| 1912 | // Get required types and ranks in the chain |
| 1913 | // input -> broadcast -> extract |
| 1914 | // (scalars are treated as rank-0). |
| 1915 | auto inputType = llvm::dyn_cast<VectorType>(Val: input.getType()); |
| 1916 | auto = llvm::dyn_cast<VectorType>(Val: extractOp.getType()); |
| 1917 | unsigned inputRank = inputType ? inputType.getRank() : 0; |
| 1918 | unsigned broadcastRank = extractOp.getSourceVectorType().getRank(); |
| 1919 | unsigned = extractType ? extractType.getRank() : 0; |
| 1920 | |
| 1921 | // Cannot do without the broadcast if overall the rank increases. |
| 1922 | if (extractRank > inputRank) |
| 1923 | return Value(); |
| 1924 | |
| 1925 | // The above condition guarantees that input is a vector. |
| 1926 | assert(inputType && "input must be a vector type because of previous checks" ); |
| 1927 | ArrayRef<int64_t> inputShape = inputType.getShape(); |
| 1928 | |
| 1929 | // In the case where there is a broadcast dimension in the suffix, it is not |
| 1930 | // possible to replace extract(broadcast(X)) with extract(X). Example: |
| 1931 | // |
| 1932 | // broadcast extract |
| 1933 | // (1) --------> (3,4) ------> (4) |
| 1934 | if (extractType && |
| 1935 | extractType.getShape() != inputShape.take_back(N: extractRank)) |
| 1936 | return Value(); |
| 1937 | |
| 1938 | // Replace extract(broadcast(X)) with extract(X). |
| 1939 | // First, determine the new extraction position. |
| 1940 | unsigned deltaOverall = inputRank - extractRank; |
| 1941 | unsigned deltaBroadcast = broadcastRank - inputRank; |
| 1942 | SmallVector<OpFoldResult> oldPositions = extractOp.getMixedPosition(); |
| 1943 | SmallVector<OpFoldResult> newPositions(deltaOverall); |
| 1944 | IntegerAttr zero = OpBuilder(extractOp.getContext()).getIndexAttr(value: 0); |
| 1945 | for (auto [i, size] : llvm::enumerate(First: inputShape.take_front(N: deltaOverall))) { |
| 1946 | newPositions[i] = size == 1 ? zero : oldPositions[i + deltaBroadcast]; |
| 1947 | } |
| 1948 | auto [staticPos, dynPos] = decomposeMixedValues(mixedValues: newPositions); |
| 1949 | extractOp->setOperands( |
| 1950 | llvm::to_vector(Range: llvm::concat<Value>(Ranges: ValueRange(input), Ranges&: dynPos))); |
| 1951 | extractOp.setStaticPosition(staticPos); |
| 1952 | return extractOp.getResult(); |
| 1953 | } |
| 1954 | |
| 1955 | /// Fold extractOp coming from ShuffleOp. |
| 1956 | /// |
| 1957 | /// Example: |
| 1958 | /// |
| 1959 | /// %shuffle = vector.shuffle %a, %b [0, 8, 7, 15] |
| 1960 | /// : vector<8xf32>, vector<8xf32> |
| 1961 | /// %extract = vector.extract %shuffle[3] : f32 from vector<4xf32> |
| 1962 | /// -> |
| 1963 | /// %extract = vector.extract %b[7] : f32 from vector<8xf32> |
| 1964 | /// |
| 1965 | static Value (ExtractOp ) { |
| 1966 | // Dynamic positions are not folded as the resulting code would be more |
| 1967 | // complex than the input code. |
| 1968 | if (extractOp.hasDynamicPosition()) |
| 1969 | return Value(); |
| 1970 | |
| 1971 | auto shuffleOp = extractOp.getSource().getDefiningOp<ShuffleOp>(); |
| 1972 | if (!shuffleOp) |
| 1973 | return Value(); |
| 1974 | |
| 1975 | // TODO: 0-D or multi-dimensional vectors not supported yet. |
| 1976 | if (shuffleOp.getResultVectorType().getRank() != 1) |
| 1977 | return Value(); |
| 1978 | |
| 1979 | int64_t inputVecSize = shuffleOp.getV1().getType().getShape()[0]; |
| 1980 | auto shuffleMask = shuffleOp.getMask(); |
| 1981 | int64_t = extractOp.getStaticPosition()[0]; |
| 1982 | int64_t shuffleIdx = shuffleMask[extractIdx]; |
| 1983 | |
| 1984 | // Find the shuffled vector to extract from based on the shuffle index. |
| 1985 | if (shuffleIdx < inputVecSize) { |
| 1986 | extractOp.setOperand(i: 0, value: shuffleOp.getV1()); |
| 1987 | extractOp.setStaticPosition({shuffleIdx}); |
| 1988 | } else { |
| 1989 | extractOp.setOperand(i: 0, value: shuffleOp.getV2()); |
| 1990 | extractOp.setStaticPosition({shuffleIdx - inputVecSize}); |
| 1991 | } |
| 1992 | |
| 1993 | return extractOp.getResult(); |
| 1994 | } |
| 1995 | |
| 1996 | // Fold extractOp with source coming from ShapeCast op. |
| 1997 | static Value (ExtractOp ) { |
| 1998 | // TODO: Canonicalization for dynamic position not implemented yet. |
| 1999 | if (extractOp.hasDynamicPosition()) |
| 2000 | return Value(); |
| 2001 | |
| 2002 | auto shapeCastOp = extractOp.getSource().getDefiningOp<vector::ShapeCastOp>(); |
| 2003 | if (!shapeCastOp) |
| 2004 | return Value(); |
| 2005 | |
| 2006 | // Get the nth dimension size starting from lowest dimension. |
| 2007 | auto getDimReverse = [](VectorType type, int64_t n) { |
| 2008 | return type.getShape().take_back(N: n + 1).front(); |
| 2009 | }; |
| 2010 | int64_t destinationRank = |
| 2011 | llvm::isa<VectorType>(Val: extractOp.getType()) |
| 2012 | ? llvm::cast<VectorType>(Val: extractOp.getType()).getRank() |
| 2013 | : 0; |
| 2014 | if (destinationRank > shapeCastOp.getSourceVectorType().getRank()) |
| 2015 | return Value(); |
| 2016 | if (destinationRank > 0) { |
| 2017 | auto destinationType = |
| 2018 | llvm::cast<VectorType>(Val: extractOp.getResult().getType()); |
| 2019 | for (int64_t i = 0; i < destinationRank; i++) { |
| 2020 | // The lowest dimension of the destination must match the lowest |
| 2021 | // dimension of the shapecast op source. |
| 2022 | // TODO: This case could be support in a canonicalization pattern. |
| 2023 | if (getDimReverse(shapeCastOp.getSourceVectorType(), i) != |
| 2024 | getDimReverse(destinationType, i)) |
| 2025 | return Value(); |
| 2026 | } |
| 2027 | } |
| 2028 | // Extract the strides associated with the extract op vector source. Then use |
| 2029 | // this to calculate a linearized position for the extract. |
| 2030 | SmallVector<int64_t> (extractOp.getStaticPosition()); |
| 2031 | std::reverse(first: extractedPos.begin(), last: extractedPos.end()); |
| 2032 | SmallVector<int64_t, 4> strides; |
| 2033 | int64_t stride = 1; |
| 2034 | for (int64_t i = 0, e = extractedPos.size(); i < e; i++) { |
| 2035 | strides.push_back(Elt: stride); |
| 2036 | stride *= |
| 2037 | getDimReverse(extractOp.getSourceVectorType(), i + destinationRank); |
| 2038 | } |
| 2039 | |
| 2040 | int64_t position = linearize(offsets: extractedPos, basis: strides); |
| 2041 | // Then extract the strides associated to the shapeCast op vector source and |
| 2042 | // delinearize the position using those strides. |
| 2043 | SmallVector<int64_t, 4> newStrides; |
| 2044 | int64_t numDimension = |
| 2045 | shapeCastOp.getSourceVectorType().getRank() - destinationRank; |
| 2046 | stride = 1; |
| 2047 | for (int64_t i = 0; i < numDimension; i++) { |
| 2048 | newStrides.push_back(Elt: stride); |
| 2049 | stride *= |
| 2050 | getDimReverse(shapeCastOp.getSourceVectorType(), i + destinationRank); |
| 2051 | } |
| 2052 | std::reverse(first: newStrides.begin(), last: newStrides.end()); |
| 2053 | SmallVector<int64_t, 4> newPosition = delinearize(linearIndex: position, strides: newStrides); |
| 2054 | // OpBuilder is only used as a helper to build an I64ArrayAttr. |
| 2055 | OpBuilder b(extractOp.getContext()); |
| 2056 | extractOp.setStaticPosition(newPosition); |
| 2057 | extractOp.setOperand(i: 0, value: shapeCastOp.getSource()); |
| 2058 | return extractOp.getResult(); |
| 2059 | } |
| 2060 | |
| 2061 | /// Fold an ExtractOp from ExtractStridedSliceOp. |
| 2062 | static Value (ExtractOp ) { |
| 2063 | // TODO: Canonicalization for dynamic position not implemented yet. |
| 2064 | if (extractOp.hasDynamicPosition()) |
| 2065 | return Value(); |
| 2066 | |
| 2067 | auto = |
| 2068 | extractOp.getSource().getDefiningOp<vector::ExtractStridedSliceOp>(); |
| 2069 | if (!extractStridedSliceOp) |
| 2070 | return Value(); |
| 2071 | |
| 2072 | // 0-D vectors not supported. |
| 2073 | assert(!hasZeroDimVectors(extractOp) && "0-D vectors not supported" ); |
| 2074 | if (hasZeroDimVectors(op: extractStridedSliceOp)) |
| 2075 | return Value(); |
| 2076 | |
| 2077 | // Return if 'extractStridedSliceOp' has non-unit strides. |
| 2078 | if (extractStridedSliceOp.hasNonUnitStrides()) |
| 2079 | return Value(); |
| 2080 | |
| 2081 | // Trim offsets for dimensions fully extracted. |
| 2082 | auto sliceOffsets = |
| 2083 | extractVector<int64_t>(arrayAttr: extractStridedSliceOp.getOffsets()); |
| 2084 | while (!sliceOffsets.empty()) { |
| 2085 | size_t lastOffset = sliceOffsets.size() - 1; |
| 2086 | if (sliceOffsets.back() != 0 || |
| 2087 | extractStridedSliceOp.getType().getDimSize(idx: lastOffset) != |
| 2088 | extractStridedSliceOp.getSourceVectorType().getDimSize(idx: lastOffset)) |
| 2089 | break; |
| 2090 | sliceOffsets.pop_back(); |
| 2091 | } |
| 2092 | unsigned destinationRank = 0; |
| 2093 | if (auto vecType = llvm::dyn_cast<VectorType>(Val: extractOp.getType())) |
| 2094 | destinationRank = vecType.getRank(); |
| 2095 | // The dimensions of the result need to be untouched by the |
| 2096 | // extractStridedSlice op. |
| 2097 | if (destinationRank > extractStridedSliceOp.getSourceVectorType().getRank() - |
| 2098 | sliceOffsets.size()) |
| 2099 | return Value(); |
| 2100 | |
| 2101 | SmallVector<int64_t> (extractOp.getStaticPosition()); |
| 2102 | assert(extractedPos.size() >= sliceOffsets.size()); |
| 2103 | for (size_t i = 0, e = sliceOffsets.size(); i < e; i++) |
| 2104 | extractedPos[i] = extractedPos[i] + sliceOffsets[i]; |
| 2105 | extractOp.getSourceMutable().assign(value: extractStridedSliceOp.getSource()); |
| 2106 | |
| 2107 | // OpBuilder is only used as a helper to build an I64ArrayAttr. |
| 2108 | OpBuilder b(extractOp.getContext()); |
| 2109 | extractOp.setStaticPosition(extractedPos); |
| 2110 | return extractOp.getResult(); |
| 2111 | } |
| 2112 | |
| 2113 | /// Fold extract_op fed from a chain of insertStridedSlice ops. |
| 2114 | static Value (ExtractOp ) { |
| 2115 | // TODO: Canonicalization for dynamic position not implemented yet. |
| 2116 | if (extractOp.hasDynamicPosition()) |
| 2117 | return Value(); |
| 2118 | |
| 2119 | int64_t destinationRank = |
| 2120 | llvm::isa<VectorType>(Val: extractOp.getType()) |
| 2121 | ? llvm::cast<VectorType>(Val: extractOp.getType()).getRank() |
| 2122 | : 0; |
| 2123 | auto insertOp = extractOp.getSource().getDefiningOp<InsertStridedSliceOp>(); |
| 2124 | if (!insertOp) |
| 2125 | return Value(); |
| 2126 | |
| 2127 | // 0-D vectors not supported. |
| 2128 | assert(!hasZeroDimVectors(extractOp) && "0-D vectors not supported" ); |
| 2129 | if (hasZeroDimVectors(op: insertOp)) |
| 2130 | return Value(); |
| 2131 | |
| 2132 | while (insertOp) { |
| 2133 | int64_t insertRankDiff = insertOp.getDestVectorType().getRank() - |
| 2134 | insertOp.getSourceVectorType().getRank(); |
| 2135 | if (destinationRank > insertOp.getSourceVectorType().getRank()) |
| 2136 | return Value(); |
| 2137 | auto insertOffsets = extractVector<int64_t>(arrayAttr: insertOp.getOffsets()); |
| 2138 | ArrayRef<int64_t> = extractOp.getStaticPosition(); |
| 2139 | |
| 2140 | if (llvm::any_of(Range: insertOp.getStrides(), P: [](Attribute attr) { |
| 2141 | return llvm::cast<IntegerAttr>(Val&: attr).getInt() != 1; |
| 2142 | })) |
| 2143 | return Value(); |
| 2144 | bool disjoint = false; |
| 2145 | SmallVector<int64_t, 4> offsetDiffs; |
| 2146 | for (unsigned dim = 0, e = extractOffsets.size(); dim < e; ++dim) { |
| 2147 | int64_t start = insertOffsets[dim]; |
| 2148 | int64_t size = |
| 2149 | (dim < insertRankDiff) |
| 2150 | ? 1 |
| 2151 | : insertOp.getSourceVectorType().getDimSize(idx: dim - insertRankDiff); |
| 2152 | int64_t end = start + size; |
| 2153 | int64_t offset = extractOffsets[dim]; |
| 2154 | // Check if the start of the extract offset is in the interval inserted. |
| 2155 | if (start <= offset && offset < end) { |
| 2156 | if (dim >= insertRankDiff) |
| 2157 | offsetDiffs.push_back(Elt: offset - start); |
| 2158 | continue; |
| 2159 | } |
| 2160 | disjoint = true; |
| 2161 | break; |
| 2162 | } |
| 2163 | // The extract element chunk overlap with the vector inserted. |
| 2164 | if (!disjoint) { |
| 2165 | // If any of the inner dimensions are only partially inserted we have a |
| 2166 | // partial overlap. |
| 2167 | int64_t srcRankDiff = |
| 2168 | insertOp.getSourceVectorType().getRank() - destinationRank; |
| 2169 | for (int64_t i = 0; i < destinationRank; i++) { |
| 2170 | if (insertOp.getSourceVectorType().getDimSize(idx: i + srcRankDiff) != |
| 2171 | insertOp.getDestVectorType().getDimSize(idx: i + srcRankDiff + |
| 2172 | insertRankDiff)) |
| 2173 | return Value(); |
| 2174 | } |
| 2175 | extractOp.getSourceMutable().assign(value: insertOp.getValueToStore()); |
| 2176 | // OpBuilder is only used as a helper to build an I64ArrayAttr. |
| 2177 | OpBuilder b(extractOp.getContext()); |
| 2178 | extractOp.setStaticPosition(offsetDiffs); |
| 2179 | return extractOp.getResult(); |
| 2180 | } |
| 2181 | // If the chunk extracted is disjoint from the chunk inserted, keep |
| 2182 | // looking in the insert chain. |
| 2183 | insertOp = insertOp.getDest().getDefiningOp<InsertStridedSliceOp>(); |
| 2184 | } |
| 2185 | return Value(); |
| 2186 | } |
| 2187 | |
| 2188 | /// Try to fold the extraction of a scalar from a vector defined by |
| 2189 | /// vector.from_elements. E.g.: |
| 2190 | /// |
| 2191 | /// %0 = vector.from_elements %a, %b : vector<2xf32> |
| 2192 | /// %1 = vector.extract %0[0] : f32 from vector<2xf32> |
| 2193 | /// ==> fold to %a |
| 2194 | static Value (ExtractOp ) { |
| 2195 | // Dynamic extractions cannot be folded. |
| 2196 | if (extractOp.hasDynamicPosition()) |
| 2197 | return {}; |
| 2198 | |
| 2199 | // Look for extract(from_elements). |
| 2200 | auto fromElementsOp = extractOp.getSource().getDefiningOp<FromElementsOp>(); |
| 2201 | if (!fromElementsOp) |
| 2202 | return {}; |
| 2203 | |
| 2204 | // Scalable vectors are not supported. |
| 2205 | auto vecType = llvm::cast<VectorType>(Val: fromElementsOp.getType()); |
| 2206 | if (vecType.isScalable()) |
| 2207 | return {}; |
| 2208 | |
| 2209 | // Only extractions of scalars are supported. |
| 2210 | int64_t rank = vecType.getRank(); |
| 2211 | ArrayRef<int64_t> indices = extractOp.getStaticPosition(); |
| 2212 | if (extractOp.getType() != vecType.getElementType()) |
| 2213 | return {}; |
| 2214 | assert(static_cast<int64_t>(indices.size()) == rank && |
| 2215 | "unexpected number of indices" ); |
| 2216 | |
| 2217 | // Compute flattened/linearized index and fold to operand. |
| 2218 | int flatIndex = 0; |
| 2219 | int stride = 1; |
| 2220 | for (int i = rank - 1; i >= 0; --i) { |
| 2221 | flatIndex += indices[i] * stride; |
| 2222 | stride *= vecType.getDimSize(idx: i); |
| 2223 | } |
| 2224 | return fromElementsOp.getElements()[flatIndex]; |
| 2225 | } |
| 2226 | |
| 2227 | /// If the dynamic indices of `extractOp` or `insertOp` are in fact constants, |
| 2228 | /// then fold it. |
| 2229 | template <typename OpType, typename AdaptorType> |
| 2230 | static Value (OpType op, AdaptorType adaptor, |
| 2231 | SmallVectorImpl<Value> &operands) { |
| 2232 | std::vector<int64_t> staticPosition = op.getStaticPosition().vec(); |
| 2233 | OperandRange dynamicPosition = op.getDynamicPosition(); |
| 2234 | ArrayRef<Attribute> dynamicPositionAttr = adaptor.getDynamicPosition(); |
| 2235 | ArrayRef<int64_t> vectorShape; |
| 2236 | if constexpr (std::is_same_v<OpType, ExtractOp>) |
| 2237 | vectorShape = op.getSourceVectorType().getShape(); |
| 2238 | else |
| 2239 | vectorShape = op.getDestVectorType().getShape(); |
| 2240 | |
| 2241 | // If the dynamic operands is empty, it is returned directly. |
| 2242 | if (!dynamicPosition.size()) |
| 2243 | return {}; |
| 2244 | |
| 2245 | // `index` is used to iterate over the `dynamicPosition`. |
| 2246 | unsigned index = 0; |
| 2247 | |
| 2248 | // `opChange` is a flag. If it is true, it means to update `op` in place. |
| 2249 | bool opChange = false; |
| 2250 | for (unsigned i = 0, e = staticPosition.size(); i < e; ++i) { |
| 2251 | if (ShapedType::isStatic(dValue: staticPosition[i])) |
| 2252 | continue; |
| 2253 | Attribute positionAttr = dynamicPositionAttr[index]; |
| 2254 | Value position = dynamicPosition[index++]; |
| 2255 | if (auto attr = mlir::dyn_cast_if_present<IntegerAttr>(Val&: positionAttr)) { |
| 2256 | int64_t value = attr.getInt(); |
| 2257 | // Do not fold if the value is out of bounds (-1 signifies a poison |
| 2258 | // value rather than OOB index). |
| 2259 | if (value >= -1 && value < vectorShape[i]) { |
| 2260 | staticPosition[i] = attr.getInt(); |
| 2261 | opChange = true; |
| 2262 | continue; |
| 2263 | } |
| 2264 | } |
| 2265 | operands.push_back(Elt: position); |
| 2266 | } |
| 2267 | |
| 2268 | if (opChange) { |
| 2269 | op.setStaticPosition(staticPosition); |
| 2270 | op.getOperation()->setOperands(operands); |
| 2271 | // Return the original result to indicate an in-place folding happened. |
| 2272 | return op.getResult(); |
| 2273 | } |
| 2274 | return {}; |
| 2275 | } |
| 2276 | |
| 2277 | /// Fold an insert or extract operation into an poison value when a poison index |
| 2278 | /// is found at any dimension of the static position. |
| 2279 | static Attribute (MLIRContext *context, |
| 2280 | ArrayRef<int64_t> staticPos, |
| 2281 | int64_t poisonVal) { |
| 2282 | if (!is_contained(Range&: staticPos, Element: poisonVal)) |
| 2283 | return {}; |
| 2284 | |
| 2285 | return ub::PoisonAttr::get(ctx: context); |
| 2286 | } |
| 2287 | |
| 2288 | /// Fold a vector extract from is a poison source. |
| 2289 | static Attribute (Attribute srcAttr) { |
| 2290 | if (matchPattern(attr: srcAttr, pattern: ub::m_Poison())) |
| 2291 | return srcAttr; |
| 2292 | |
| 2293 | return {}; |
| 2294 | } |
| 2295 | |
| 2296 | /// Fold a vector extract extracting from a DenseElementsAttr. |
| 2297 | static Attribute (ExtractOp , |
| 2298 | Attribute srcAttr) { |
| 2299 | auto denseAttr = dyn_cast_if_present<DenseElementsAttr>(Val&: srcAttr); |
| 2300 | if (!denseAttr) { |
| 2301 | return {}; |
| 2302 | } |
| 2303 | |
| 2304 | if (denseAttr.isSplat()) { |
| 2305 | Attribute newAttr = denseAttr.getSplatValue<Attribute>(); |
| 2306 | if (auto vecDstType = dyn_cast<VectorType>(Val: extractOp.getType())) |
| 2307 | newAttr = DenseElementsAttr::get(type: vecDstType, values: newAttr); |
| 2308 | return newAttr; |
| 2309 | } |
| 2310 | |
| 2311 | auto vecTy = cast<VectorType>(Val: extractOp.getSourceVectorType()); |
| 2312 | if (vecTy.isScalable()) |
| 2313 | return {}; |
| 2314 | |
| 2315 | if (extractOp.hasDynamicPosition()) { |
| 2316 | return {}; |
| 2317 | } |
| 2318 | |
| 2319 | // Materializing subsets of a large constant array can generally lead to |
| 2320 | // explosion in IR size because of different combination of subsets that |
| 2321 | // can exist. However, vector.extract is a restricted form of subset |
| 2322 | // extract where you can only extract non-overlapping (or the same) subset for |
| 2323 | // a given rank of the subset. Because of this property, the IR size can only |
| 2324 | // increase at most by `rank * size(array)` from a single constant array being |
| 2325 | // extracted by multiple extracts. |
| 2326 | |
| 2327 | // Calculate the linearized position of the continuous chunk of elements to |
| 2328 | // extract. |
| 2329 | SmallVector<int64_t> completePositions(vecTy.getRank(), 0); |
| 2330 | copy(Range: extractOp.getStaticPosition(), Out: completePositions.begin()); |
| 2331 | int64_t startPos = |
| 2332 | linearize(offsets: completePositions, basis: computeStrides(sizes: vecTy.getShape())); |
| 2333 | auto denseValuesBegin = denseAttr.value_begin<TypedAttr>() + startPos; |
| 2334 | |
| 2335 | TypedAttr newAttr; |
| 2336 | if (auto resVecTy = dyn_cast<VectorType>(Val: extractOp.getType())) { |
| 2337 | SmallVector<Attribute> elementValues( |
| 2338 | denseValuesBegin, denseValuesBegin + resVecTy.getNumElements()); |
| 2339 | newAttr = DenseElementsAttr::get(type: resVecTy, values: elementValues); |
| 2340 | } else { |
| 2341 | newAttr = *denseValuesBegin; |
| 2342 | } |
| 2343 | |
| 2344 | return newAttr; |
| 2345 | } |
| 2346 | |
| 2347 | OpFoldResult ExtractOp::(FoldAdaptor adaptor) { |
| 2348 | // Fold "vector.extract %v[] : vector<2x2xf32> from vector<2x2xf32>" to %v. |
| 2349 | // Note: Do not fold "vector.extract %v[] : f32 from vector<f32>" (type |
| 2350 | // mismatch). |
| 2351 | if (getNumIndices() == 0 && getSource().getType() == getResult().getType()) |
| 2352 | return getSource(); |
| 2353 | if (auto res = foldPoisonSrcExtractOp(srcAttr: adaptor.getSource())) |
| 2354 | return res; |
| 2355 | // Fold `arith.constant` indices into the `vector.extract` operation. |
| 2356 | // Do not stop here as this fold may enable subsequent folds that require |
| 2357 | // constant indices. |
| 2358 | SmallVector<Value> operands = {getSource()}; |
| 2359 | auto inplaceFolded = extractInsertFoldConstantOp(op: *this, adaptor, operands); |
| 2360 | |
| 2361 | if (auto res = foldPoisonIndexInsertExtractOp( |
| 2362 | context: getContext(), staticPos: adaptor.getStaticPosition(), poisonVal: kPoisonIndex)) |
| 2363 | return res; |
| 2364 | if (auto res = foldDenseElementsAttrSrcExtractOp(extractOp: *this, srcAttr: adaptor.getSource())) |
| 2365 | return res; |
| 2366 | if (succeeded(Result: foldExtractOpFromExtractChain(extractOp: *this))) |
| 2367 | return getResult(); |
| 2368 | if (auto res = ExtractFromInsertTransposeChainState(*this).fold()) |
| 2369 | return res; |
| 2370 | if (auto res = foldExtractFromBroadcast(extractOp: *this)) |
| 2371 | return res; |
| 2372 | if (auto res = foldExtractFromShuffle(extractOp: *this)) |
| 2373 | return res; |
| 2374 | if (auto res = foldExtractFromShapeCast(extractOp: *this)) |
| 2375 | return res; |
| 2376 | if (auto val = foldExtractFromExtractStrided(extractOp: *this)) |
| 2377 | return val; |
| 2378 | if (auto val = foldExtractStridedOpFromInsertChain(extractOp: *this)) |
| 2379 | return val; |
| 2380 | if (auto val = foldScalarExtractFromFromElements(extractOp: *this)) |
| 2381 | return val; |
| 2382 | |
| 2383 | return inplaceFolded; |
| 2384 | } |
| 2385 | |
| 2386 | namespace { |
| 2387 | |
| 2388 | // Pattern to rewrite a ExtractOp(Broadcast) -> Broadcast. |
| 2389 | class final : public OpRewritePattern<ExtractOp> { |
| 2390 | public: |
| 2391 | using Base::Base; |
| 2392 | |
| 2393 | LogicalResult matchAndRewrite(ExtractOp , |
| 2394 | PatternRewriter &rewriter) const override { |
| 2395 | |
| 2396 | Operation *defOp = extractOp.getSource().getDefiningOp(); |
| 2397 | VectorType outType = dyn_cast<VectorType>(Val: extractOp.getType()); |
| 2398 | if (!defOp || !isBroadcastLike(op: defOp) || !outType) |
| 2399 | return failure(); |
| 2400 | |
| 2401 | Value source = defOp->getOperand(idx: 0); |
| 2402 | if (isBroadcastableTo(srcType: source.getType(), dstVectorType: outType) != |
| 2403 | BroadcastableToResult::Success) |
| 2404 | return failure(); |
| 2405 | |
| 2406 | rewriter.replaceOpWithNewOp<BroadcastOp>(op: extractOp, args&: outType, args&: source); |
| 2407 | return success(); |
| 2408 | } |
| 2409 | }; |
| 2410 | |
| 2411 | // Pattern to rewrite a ExtractOp(CreateMask) -> CreateMask. |
| 2412 | class final : public OpRewritePattern<ExtractOp> { |
| 2413 | public: |
| 2414 | using Base::Base; |
| 2415 | |
| 2416 | LogicalResult matchAndRewrite(ExtractOp , |
| 2417 | PatternRewriter &rewriter) const override { |
| 2418 | auto createMaskOp = |
| 2419 | extractOp.getSource().getDefiningOp<vector::CreateMaskOp>(); |
| 2420 | if (!createMaskOp) |
| 2421 | return failure(); |
| 2422 | |
| 2423 | VectorType = |
| 2424 | llvm::dyn_cast<VectorType>(Val: extractOp.getResult().getType()); |
| 2425 | |
| 2426 | if (!extractedMaskType) |
| 2427 | return failure(); |
| 2428 | |
| 2429 | auto maskOperands = createMaskOp.getOperands(); |
| 2430 | ArrayRef<int64_t> = extractOp.getStaticPosition(); |
| 2431 | VectorType maskType = createMaskOp.getVectorType(); |
| 2432 | |
| 2433 | bool containsUnknownDims = false; |
| 2434 | bool allFalse = getMaskFormat(mask: createMaskOp) == MaskFormat::AllFalse; |
| 2435 | |
| 2436 | for (size_t dimIdx = 0; !allFalse && dimIdx < extractOpPos.size(); |
| 2437 | dimIdx++) { |
| 2438 | int64_t pos = extractOpPos[dimIdx]; |
| 2439 | Value operand = maskOperands[dimIdx]; |
| 2440 | auto constantOp = operand.getDefiningOp<arith::ConstantOp>(); |
| 2441 | if (!constantOp) { |
| 2442 | // Bounds of this dim unknown. |
| 2443 | containsUnknownDims = true; |
| 2444 | continue; |
| 2445 | } |
| 2446 | |
| 2447 | int64_t createMaskBound = |
| 2448 | llvm::cast<IntegerAttr>(Val: constantOp.getValue()).getInt(); |
| 2449 | |
| 2450 | if (pos != ShapedType::kDynamic) { |
| 2451 | // If any position is outside the range from the `create_mask`, then the |
| 2452 | // extracted mask will be all-false. |
| 2453 | allFalse |= pos >= createMaskBound; |
| 2454 | } else if (createMaskBound < maskType.getDimSize(idx: dimIdx)) { |
| 2455 | // This dim is not all-true and since this is a dynamic index we don't |
| 2456 | // know if the extraction is within the true or false region. |
| 2457 | // Note: Zero dims have already handled via getMaskFormat(). |
| 2458 | containsUnknownDims = true; |
| 2459 | } |
| 2460 | } |
| 2461 | |
| 2462 | if (allFalse) { |
| 2463 | rewriter.replaceOpWithNewOp<arith::ConstantOp>( |
| 2464 | op: extractOp, args: DenseElementsAttr::get(type: extractedMaskType, value: false)); |
| 2465 | } else if (!containsUnknownDims) { |
| 2466 | rewriter.replaceOpWithNewOp<vector::CreateMaskOp>( |
| 2467 | op: extractOp, args&: extractedMaskType, |
| 2468 | args: maskOperands.drop_front(n: extractOpPos.size())); |
| 2469 | } else { |
| 2470 | return failure(); |
| 2471 | } |
| 2472 | return success(); |
| 2473 | } |
| 2474 | }; |
| 2475 | |
| 2476 | // Pattern to rewrite a ExtractOp(ConstantMask) -> ConstantMask. |
| 2477 | class final : public OpRewritePattern<ExtractOp> { |
| 2478 | public: |
| 2479 | using Base::Base; |
| 2480 | |
| 2481 | LogicalResult matchAndRewrite(ExtractOp , |
| 2482 | PatternRewriter &rewriter) const override { |
| 2483 | auto constantMaskOp = |
| 2484 | extractOp.getSource().getDefiningOp<vector::ConstantMaskOp>(); |
| 2485 | if (!constantMaskOp) |
| 2486 | return failure(); |
| 2487 | |
| 2488 | Type resultType = extractOp.getResult().getType(); |
| 2489 | auto = dyn_cast<VectorType>(Val&: resultType); |
| 2490 | |
| 2491 | ArrayRef<int64_t> = extractOp.getStaticPosition(); |
| 2492 | ArrayRef<int64_t> maskDimSizes = constantMaskOp.getMaskDimSizes(); |
| 2493 | |
| 2494 | VectorType maskType = constantMaskOp.getVectorType(); |
| 2495 | |
| 2496 | // Check if any extracted position is outside the mask bounds. |
| 2497 | for (size_t dimIdx = 0; dimIdx < extractOpPos.size(); dimIdx++) { |
| 2498 | int64_t pos = extractOpPos[dimIdx]; |
| 2499 | if (pos == ShapedType::kDynamic) { |
| 2500 | // If the dim is all-true, a dynamic index is fine — any position |
| 2501 | // is within the masked region. |
| 2502 | if (maskDimSizes[dimIdx] == maskType.getDimSize(idx: dimIdx)) |
| 2503 | continue; |
| 2504 | // Otherwise we don't know if the position is inside or outside of |
| 2505 | // the masked area, so bail out. |
| 2506 | return failure(); |
| 2507 | } |
| 2508 | |
| 2509 | // If the position is statically outside of the masked area, the result |
| 2510 | // will be all-false. |
| 2511 | if (pos >= maskDimSizes[dimIdx]) { |
| 2512 | if (extractedMaskType) { |
| 2513 | rewriter.replaceOpWithNewOp<arith::ConstantOp>( |
| 2514 | op: extractOp, args: DenseElementsAttr::get(type: extractedMaskType, value: false)); |
| 2515 | } else { |
| 2516 | rewriter.replaceOpWithNewOp<arith::ConstantOp>( |
| 2517 | op: extractOp, args: rewriter.getIntegerAttr(type: resultType, value: false)); |
| 2518 | } |
| 2519 | return success(); |
| 2520 | } |
| 2521 | } |
| 2522 | |
| 2523 | // All positions are within the mask bounds. |
| 2524 | if (extractedMaskType) { |
| 2525 | // Vector result: the result is a constant_mask with the remaining |
| 2526 | // dimensions. |
| 2527 | rewriter.replaceOpWithNewOp<vector::ConstantMaskOp>( |
| 2528 | op: extractOp, args&: extractedMaskType, |
| 2529 | args: maskDimSizes.drop_front(N: extractOpPos.size())); |
| 2530 | } else { |
| 2531 | // Scalar result: all positions are within the masked region, so the |
| 2532 | // result is true. |
| 2533 | rewriter.replaceOpWithNewOp<arith::ConstantOp>( |
| 2534 | op: extractOp, args: rewriter.getIntegerAttr(type: resultType, value: true)); |
| 2535 | } |
| 2536 | return success(); |
| 2537 | } |
| 2538 | }; |
| 2539 | |
| 2540 | // Folds extract(shape_cast(..)) into shape_cast when the total element count |
| 2541 | // does not change. |
| 2542 | LogicalResult (ExtractOp , |
| 2543 | PatternRewriter &rewriter) { |
| 2544 | auto castOp = extractOp.getSource().getDefiningOp<ShapeCastOp>(); |
| 2545 | if (!castOp) |
| 2546 | return failure(); |
| 2547 | |
| 2548 | VectorType sourceType = castOp.getSourceVectorType(); |
| 2549 | auto targetType = dyn_cast<VectorType>(Val: extractOp.getResult().getType()); |
| 2550 | if (!targetType) |
| 2551 | return failure(); |
| 2552 | |
| 2553 | if (sourceType.getNumElements() != targetType.getNumElements()) |
| 2554 | return failure(); |
| 2555 | |
| 2556 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: extractOp, args&: targetType, |
| 2557 | args: castOp.getSource()); |
| 2558 | return success(); |
| 2559 | } |
| 2560 | |
| 2561 | /// Try to canonicalize the extraction of a subvector from a vector defined by |
| 2562 | /// vector.from_elements. E.g.: |
| 2563 | /// |
| 2564 | /// %0 = vector.from_elements %a, %b, %a, %a : vector<2x2xf32> |
| 2565 | /// %1 = vector.extract %0[0] : vector<2xf32> from vector<2x2xf32> |
| 2566 | /// ==> canonicalize to vector.from_elements %a, %b : vector<2xf32> |
| 2567 | LogicalResult (ExtractOp , |
| 2568 | PatternRewriter &rewriter) { |
| 2569 | // Dynamic positions are not supported. |
| 2570 | if (extractOp.hasDynamicPosition()) |
| 2571 | return failure(); |
| 2572 | |
| 2573 | // Scalar extracts are handled by the folder. |
| 2574 | auto resultType = dyn_cast<VectorType>(Val: extractOp.getType()); |
| 2575 | if (!resultType) |
| 2576 | return failure(); |
| 2577 | |
| 2578 | // Look for extracts from a from_elements op. |
| 2579 | auto fromElementsOp = extractOp.getSource().getDefiningOp<FromElementsOp>(); |
| 2580 | if (!fromElementsOp) |
| 2581 | return failure(); |
| 2582 | VectorType inputType = fromElementsOp.getType(); |
| 2583 | |
| 2584 | // Scalable vectors are not supported. |
| 2585 | if (resultType.isScalable() || inputType.isScalable()) |
| 2586 | return failure(); |
| 2587 | |
| 2588 | // Compute the position of first extracted element and flatten/linearize the |
| 2589 | // position. |
| 2590 | SmallVector<int64_t> firstElementPos = |
| 2591 | llvm::to_vector(Range: extractOp.getStaticPosition()); |
| 2592 | firstElementPos.append(/*NumInputs=*/resultType.getRank(), /*Elt=*/0); |
| 2593 | int flatIndex = 0; |
| 2594 | int stride = 1; |
| 2595 | for (int64_t i = inputType.getRank() - 1; i >= 0; --i) { |
| 2596 | flatIndex += firstElementPos[i] * stride; |
| 2597 | stride *= inputType.getDimSize(idx: i); |
| 2598 | } |
| 2599 | |
| 2600 | // Replace the op with a smaller from_elements op. |
| 2601 | rewriter.replaceOpWithNewOp<FromElementsOp>( |
| 2602 | op: extractOp, args&: resultType, |
| 2603 | args: fromElementsOp.getElements().slice(n: flatIndex, |
| 2604 | m: resultType.getNumElements())); |
| 2605 | return success(); |
| 2606 | } |
| 2607 | |
| 2608 | /// Replace `vector.extract` with `vector.shape_cast`. |
| 2609 | /// |
| 2610 | /// BEFORE: |
| 2611 | /// %0 = vector.extract %arg0[0] : vector<4xf32> from vector<1x4xf32> |
| 2612 | /// AFTER: |
| 2613 | /// %0 = vector.shape_cast %arg0 : vector<1x4xf32> to vector<4xf32> |
| 2614 | /// |
| 2615 | /// The canonical form of vector operations that reshape vectors is shape_cast. |
| 2616 | struct final : OpRewritePattern<vector::ExtractOp> { |
| 2617 | using Base::Base; |
| 2618 | LogicalResult matchAndRewrite(vector::ExtractOp , |
| 2619 | PatternRewriter &rewriter) const override { |
| 2620 | VectorType sourceType = extractOp.getSourceVectorType(); |
| 2621 | VectorType outType = dyn_cast<VectorType>(Val: extractOp.getType()); |
| 2622 | if (!outType) |
| 2623 | return failure(); |
| 2624 | |
| 2625 | if (sourceType.getNumElements() != outType.getNumElements()) |
| 2626 | return rewriter.notifyMatchFailure( |
| 2627 | arg&: extractOp, msg: "extract to vector with fewer elements" ); |
| 2628 | |
| 2629 | // Negative values in `position` means that the extacted value is poison. |
| 2630 | // There is a vector.extract folder for this. |
| 2631 | if (llvm::any_of(Range: extractOp.getMixedPosition(), |
| 2632 | P: [](OpFoldResult v) { return !isConstantIntValue(ofr: v, value: 0); })) |
| 2633 | return rewriter.notifyMatchFailure(arg&: extractOp, |
| 2634 | msg: "leaving for extract poison folder" ); |
| 2635 | |
| 2636 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: extractOp, args&: outType, |
| 2637 | args: extractOp.getSource()); |
| 2638 | |
| 2639 | return success(); |
| 2640 | } |
| 2641 | }; |
| 2642 | |
| 2643 | /// Folds vector.extract from vector.insert when the extract position is a |
| 2644 | /// prefix of the insert position and the remaining (un-indexed) dimensions |
| 2645 | /// of the extracted sub-vector are all size 1. In that case the extracted |
| 2646 | /// value is fully determined by the inserted value. |
| 2647 | /// |
| 2648 | /// Examples: |
| 2649 | /// %ins = vector.insert %s, %v [3, 0] : f32 into vector<16x1xf32> |
| 2650 | /// %ext = vector.extract %ins [3] : vector<1xf32> from vector<16x1xf32> |
| 2651 | /// folds to: |
| 2652 | /// %ext = vector.broadcast %s : f32 to vector<1xf32> |
| 2653 | /// |
| 2654 | /// %ins = vector.insert %s, %v [0, 0] : vector<1xf32> into vector<16x1x1xf32> |
| 2655 | // %ext = vector.extract %ins [0] : vector<1x1xf32> from vector<16x1x1xf32> |
| 2656 | /// folds to: |
| 2657 | /// %ext = vector.shape_cast %arg0 : vector<1xf32> to vector<1x1xf32> |
| 2658 | struct final |
| 2659 | : OpRewritePattern<vector::ExtractOp> { |
| 2660 | using Base::Base; |
| 2661 | |
| 2662 | LogicalResult matchAndRewrite(vector::ExtractOp , |
| 2663 | PatternRewriter &rewriter) const override { |
| 2664 | if (extractOp.hasDynamicPosition()) |
| 2665 | return failure(); |
| 2666 | |
| 2667 | auto insertOp = extractOp.getSource().getDefiningOp<vector::InsertOp>(); |
| 2668 | if (!insertOp || insertOp.hasDynamicPosition()) |
| 2669 | return failure(); |
| 2670 | |
| 2671 | ArrayRef<int64_t> = extractOp.getStaticPosition(); |
| 2672 | ArrayRef<int64_t> insertPos = insertOp.getStaticPosition(); |
| 2673 | |
| 2674 | // The extract position must be a strict prefix of the insert position. |
| 2675 | if (extractPos.size() >= insertPos.size() || |
| 2676 | extractPos != insertPos.take_front(N: extractPos.size())) |
| 2677 | return failure(); |
| 2678 | |
| 2679 | // The remaining dimensions (those not indexed by the extract) must all |
| 2680 | // be size 1 in the source vector type. This guarantees that the inserted |
| 2681 | // value fully determines the extracted sub-vector. |
| 2682 | auto srcVecType = extractOp.getSourceVectorType(); |
| 2683 | for (int64_t i = extractPos.size(), e = srcVecType.getRank(); i < e; ++i) |
| 2684 | if (srcVecType.getDimSize(idx: i) != 1) |
| 2685 | return failure(); |
| 2686 | |
| 2687 | Value inserted = insertOp.getValueToStore(); |
| 2688 | Type = extractOp.getResult().getType(); |
| 2689 | if (isa<VectorType>(Val: inserted.getType())) { |
| 2690 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: extractOp, args&: extractedType, |
| 2691 | args&: inserted); |
| 2692 | } else { |
| 2693 | // The inserted value fully determines the extracted sub-vector; broadcast |
| 2694 | // it to the extracted type. |
| 2695 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>( |
| 2696 | op: extractOp, args: extractOp.getResult().getType(), |
| 2697 | args: insertOp.getValueToStore()); |
| 2698 | } |
| 2699 | return success(); |
| 2700 | } |
| 2701 | }; |
| 2702 | |
| 2703 | } // namespace |
| 2704 | |
| 2705 | void ExtractOp::(RewritePatternSet &results, |
| 2706 | MLIRContext *context) { |
| 2707 | results.add<ExtractOpFromBroadcast, ExtractOpFromCreateMask, |
| 2708 | ExtractOpFromConstantMask, ExtractToShapeCast, |
| 2709 | FoldExtractFromInsertUnitDim>(arg&: context); |
| 2710 | results.add(implFn: foldExtractFromShapeCastToShapeCast); |
| 2711 | results.add(implFn: foldExtractFromFromElements); |
| 2712 | } |
| 2713 | |
| 2714 | static void populateFromInt64AttrArray(ArrayAttr arrayAttr, |
| 2715 | SmallVectorImpl<int64_t> &results) { |
| 2716 | for (auto attr : arrayAttr) |
| 2717 | results.push_back(Elt: llvm::cast<IntegerAttr>(Val&: attr).getInt()); |
| 2718 | } |
| 2719 | |
| 2720 | //===----------------------------------------------------------------------===// |
| 2721 | // FmaOp |
| 2722 | //===----------------------------------------------------------------------===// |
| 2723 | |
| 2724 | //----------------------------------------------------------------------------- |
| 2725 | // VectorUnrollOpInterface methods |
| 2726 | //----------------------------------------------------------------------------- |
| 2727 | std::optional<SmallVector<int64_t, 4>> FMAOp::getIterationShapeToUnroll() { |
| 2728 | return llvm::to_vector<4>(Range: getVectorType().getShape()); |
| 2729 | } |
| 2730 | |
| 2731 | //===----------------------------------------------------------------------===// |
| 2732 | // ToElementsOp |
| 2733 | //===----------------------------------------------------------------------===// |
| 2734 | |
| 2735 | /// Returns true if all the `operands` are defined by `defOp`. |
| 2736 | /// Otherwise, returns false. |
| 2737 | static bool haveSameDefiningOp(OperandRange operands, Operation *defOp) { |
| 2738 | if (operands.empty()) |
| 2739 | return false; |
| 2740 | |
| 2741 | return llvm::all_of(Range&: operands, P: [&](Value operand) { |
| 2742 | Operation *currentDef = operand.getDefiningOp(); |
| 2743 | return currentDef == defOp; |
| 2744 | }); |
| 2745 | } |
| 2746 | |
| 2747 | /// Folds vector.to_elements(vector.from_elements(%e0, %e1, ...)) into |
| 2748 | /// (%e0, %e1, ...). For example: |
| 2749 | /// |
| 2750 | /// %0 = vector.from_elements %a, %b, %c : vector<3xf32> |
| 2751 | /// %1:3 = vector.to_elements %0 : vector<3xf32> |
| 2752 | /// user_op %1#0, %1#1, %1#2 |
| 2753 | /// |
| 2754 | /// becomes: |
| 2755 | /// |
| 2756 | /// user_op %a, %b, %c |
| 2757 | /// |
| 2758 | static LogicalResult |
| 2759 | foldToElementsFromElements(ToElementsOp toElementsOp, |
| 2760 | SmallVectorImpl<OpFoldResult> &results) { |
| 2761 | auto fromElementsOp = |
| 2762 | toElementsOp.getSource().getDefiningOp<FromElementsOp>(); |
| 2763 | if (!fromElementsOp) |
| 2764 | return failure(); |
| 2765 | |
| 2766 | llvm::append_range(C&: results, R: fromElementsOp.getElements()); |
| 2767 | return success(); |
| 2768 | } |
| 2769 | |
| 2770 | /// Folds vector.to_elements(vector.broadcast(%x)) for the scalar case only. |
| 2771 | /// |
| 2772 | /// Example: |
| 2773 | /// %b = vector.broadcast %x : i32 to vector<3xf32> |
| 2774 | /// %e:3 = vector.to_elements %b : vector<3xf32> |
| 2775 | /// user_op %e#0, %e#1, %e#2 |
| 2776 | /// becomes: |
| 2777 | /// user_op %x, %x, %x |
| 2778 | /// |
| 2779 | /// The vector source case is handled by a canonicalization pattern. |
| 2780 | static LogicalResult |
| 2781 | foldToElementsOfBroadcast(ToElementsOp toElementsOp, |
| 2782 | SmallVectorImpl<OpFoldResult> &results) { |
| 2783 | auto bcastOp = toElementsOp.getSource().getDefiningOp<BroadcastOp>(); |
| 2784 | if (!bcastOp) |
| 2785 | return failure(); |
| 2786 | // Vectors are handled in the ToElementsOfBroadcast RewritePattern. |
| 2787 | if (isa<VectorType>(Val: bcastOp.getSource().getType())) |
| 2788 | return failure(); |
| 2789 | |
| 2790 | auto resultVecType = cast<VectorType>(Val: toElementsOp.getSource().getType()); |
| 2791 | |
| 2792 | Value scalar = bcastOp.getSource(); |
| 2793 | results.assign(NumElts: resultVecType.getNumElements(), Elt: scalar); |
| 2794 | return success(); |
| 2795 | } |
| 2796 | |
| 2797 | LogicalResult ToElementsOp::fold(FoldAdaptor adaptor, |
| 2798 | SmallVectorImpl<OpFoldResult> &results) { |
| 2799 | if (succeeded(Result: foldToElementsFromElements(toElementsOp: *this, results))) |
| 2800 | return success(); |
| 2801 | |
| 2802 | // Y = ToElements(ShapeCast(X)) -> Y = ToElements(X) |
| 2803 | if (auto shapeCast = getSource().getDefiningOp<ShapeCastOp>()) { |
| 2804 | setOperand(shapeCast.getSource()); |
| 2805 | return success(); |
| 2806 | } |
| 2807 | |
| 2808 | return foldToElementsOfBroadcast(toElementsOp: *this, results); |
| 2809 | } |
| 2810 | |
| 2811 | LogicalResult |
| 2812 | ToElementsOp::inferReturnTypes(MLIRContext *ctx, std::optional<Location> loc, |
| 2813 | ToElementsOp::Adaptor adaptor, |
| 2814 | SmallVectorImpl<Type> &inferredReturnTypes) { |
| 2815 | auto vecType = cast<VectorType>(Val: adaptor.getSource().getType()); |
| 2816 | Type elType = vecType.getElementType(); |
| 2817 | inferredReturnTypes.append(NumInputs: vecType.getNumElements(), Elt: elType); |
| 2818 | return success(); |
| 2819 | } |
| 2820 | |
| 2821 | /// Canonicalize `vector.to_elements(vector.broadcast(%v))` where `%v` is a |
| 2822 | /// vector. |
| 2823 | /// - Build `vector.to_elements %v` and remap each destination element to the |
| 2824 | /// corresponding source element using broadcast rules (match or 1 → |
| 2825 | /// replicate). |
| 2826 | /// |
| 2827 | /// Example: |
| 2828 | /// %v = vector.broadcast %src : vector<2xf32> to vector<3x2xf32> |
| 2829 | /// %e:6 = vector.to_elements %v : vector<3x2xf32> |
| 2830 | /// becomes: |
| 2831 | /// %src_elems:2 = vector.to_elements %src : vector<2xf32> |
| 2832 | /// // uses: %src_elems#0, %src_elems#1, %src_elems#0, |
| 2833 | /// // %src_elems#1, %src_elems#0, %src_elems#1 |
| 2834 | struct ToElementsOfBroadcast final : OpRewritePattern<ToElementsOp> { |
| 2835 | using Base::Base; |
| 2836 | |
| 2837 | LogicalResult matchAndRewrite(ToElementsOp toElementsOp, |
| 2838 | PatternRewriter &rewriter) const override { |
| 2839 | auto bcastOp = toElementsOp.getSource().getDefiningOp<BroadcastOp>(); |
| 2840 | if (!bcastOp) |
| 2841 | return failure(); |
| 2842 | |
| 2843 | // Only handle broadcasts from a vector source here. |
| 2844 | auto srcType = dyn_cast<VectorType>(Val: bcastOp.getSource().getType()); |
| 2845 | if (!srcType) |
| 2846 | return failure(); |
| 2847 | |
| 2848 | auto dstType = cast<VectorType>(Val: toElementsOp.getSource().getType()); |
| 2849 | |
| 2850 | ArrayRef<int64_t> dstShape = dstType.getShape(); |
| 2851 | ArrayRef<int64_t> srcShape = srcType.getShape(); |
| 2852 | |
| 2853 | int64_t dstRank = dstShape.size(); |
| 2854 | int64_t srcRank = srcShape.size(); |
| 2855 | |
| 2856 | // Create elements for the broadcast source vector. |
| 2857 | auto srcElems = vector::ToElementsOp::create( |
| 2858 | builder&: rewriter, location: toElementsOp.getLoc(), source: bcastOp.getSource()); |
| 2859 | |
| 2860 | int64_t dstCount = llvm::product_of(Range&: dstShape); |
| 2861 | |
| 2862 | SmallVector<Value> replacements; |
| 2863 | replacements.reserve(N: dstCount); |
| 2864 | |
| 2865 | // For each element of the destination, determine which element of the |
| 2866 | // source should be used. We walk all destination positions using a single |
| 2867 | // counter, decode it into per-dimension indices, then build the matching |
| 2868 | // source position: use the same index where sizes match, and use 0 where |
| 2869 | // the source size is 1 (replication). This mapping is needed so we can |
| 2870 | // replace each result of to_elements with the corresponding element from |
| 2871 | // the broadcast source. |
| 2872 | // Inner-dimension stretch example: |
| 2873 | // %v = vector.broadcast %src : vector<2x1x2xf32> to vector<2x3x2xf32> |
| 2874 | // %e:12 = vector.to_elements %v : vector<2x3x2xf32> |
| 2875 | // becomes: |
| 2876 | // %src_elems:4 = vector.to_elements %src : vector<2x1x2xf32> |
| 2877 | // // uses: %src_elems#0, %src_elems#1, %src_elems#0, |
| 2878 | // // %src_elems#1, %src_elems#0, %src_elems#1, |
| 2879 | // // %src_elems#2, %src_elems#3, %src_elems#2, |
| 2880 | // // %src_elems#3, %src_elems#2, %src_elems#3 |
| 2881 | |
| 2882 | // Row-major strides for the destination shape. |
| 2883 | SmallVector<int64_t> dstStrides = computeStrides(sizes: dstShape); |
| 2884 | // Row-major strides for the source shape. |
| 2885 | SmallVector<int64_t> srcStrides = computeStrides(sizes: srcShape); |
| 2886 | SmallVector<int64_t> dstIdx(dstRank); |
| 2887 | SmallVector<int64_t> srcIdx(srcRank); |
| 2888 | for (int64_t lin = 0; lin < dstCount; ++lin) { |
| 2889 | // Convert linear destination index to per-dimension indices. |
| 2890 | dstIdx = delinearize(linearIndex: lin, strides: dstStrides); |
| 2891 | for (int64_t k = 0; k < srcRank; ++k) |
| 2892 | srcIdx[k] = (srcShape[k] == 1) ? 0 : dstIdx[dstRank - srcRank + k]; |
| 2893 | // Convert per-dimension source indices back to a linear index. |
| 2894 | int64_t srcLin = linearize(offsets: srcIdx, basis: srcStrides); |
| 2895 | replacements.push_back(Elt: srcElems.getResult(i: srcLin)); |
| 2896 | } |
| 2897 | |
| 2898 | rewriter.replaceOp(op: toElementsOp, newValues: replacements); |
| 2899 | return success(); |
| 2900 | } |
| 2901 | }; |
| 2902 | |
| 2903 | void ToElementsOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 2904 | MLIRContext *context) { |
| 2905 | results.add<ToElementsOfBroadcast>(arg&: context); |
| 2906 | } |
| 2907 | |
| 2908 | //===----------------------------------------------------------------------===// |
| 2909 | // FromElementsOp |
| 2910 | //===----------------------------------------------------------------------===// |
| 2911 | |
| 2912 | /// Folds vector.from_elements(vector.to_elements(%vector)) into %vector. |
| 2913 | /// |
| 2914 | /// Case #1: Input and output vectors are the same. |
| 2915 | /// |
| 2916 | /// %0:3 = vector.to_elements %a : vector<3xf32> |
| 2917 | /// %1 = vector.from_elements %0#0, %0#1, %0#2 : vector<3xf32> |
| 2918 | /// user_op %1 |
| 2919 | /// |
| 2920 | /// becomes: |
| 2921 | /// |
| 2922 | /// user_op %a |
| 2923 | /// |
| 2924 | static OpFoldResult foldFromElementsToElements(FromElementsOp fromElementsOp) { |
| 2925 | OperandRange fromElemsOperands = fromElementsOp.getElements(); |
| 2926 | if (fromElemsOperands.empty()) |
| 2927 | return {}; |
| 2928 | |
| 2929 | auto toElementsOp = fromElemsOperands[0].getDefiningOp<ToElementsOp>(); |
| 2930 | if (!toElementsOp) |
| 2931 | return {}; |
| 2932 | |
| 2933 | if (!haveSameDefiningOp(operands: fromElemsOperands, defOp: toElementsOp)) |
| 2934 | return {}; |
| 2935 | |
| 2936 | // Case #1: Input and output vectors are the same. Forward the input vector. |
| 2937 | Value toElementsInput = toElementsOp.getSource(); |
| 2938 | if (fromElementsOp.getType() == toElementsInput.getType() && |
| 2939 | llvm::equal(LRange&: fromElemsOperands, RRange: toElementsOp.getResults())) { |
| 2940 | return toElementsInput; |
| 2941 | } |
| 2942 | |
| 2943 | // TODO: Support cases with different input and output shapes and different |
| 2944 | // number of elements. |
| 2945 | |
| 2946 | return {}; |
| 2947 | } |
| 2948 | |
| 2949 | /// Fold vector.from_elements to a constant when all operands are constants. |
| 2950 | /// Example: |
| 2951 | /// %c1 = arith.constant 1 : i32 |
| 2952 | /// %c2 = arith.constant 2 : i32 |
| 2953 | /// %v = vector.from_elements %c1, %c2 : vector<2xi32> |
| 2954 | /// => |
| 2955 | /// %v = arith.constant dense<[1, 2]> : vector<2xi32> |
| 2956 | /// |
| 2957 | static OpFoldResult foldFromElementsToConstant(FromElementsOp fromElementsOp, |
| 2958 | ArrayRef<Attribute> elements) { |
| 2959 | // Check for null or poison attributes before any processing. |
| 2960 | if (llvm::any_of(Range&: elements, P: [](Attribute attr) { |
| 2961 | return !attr || matchPattern(attr, pattern: ub::m_Poison()); |
| 2962 | })) |
| 2963 | return {}; |
| 2964 | |
| 2965 | // DenseElementsAttr only supports int/index/float/complex types. |
| 2966 | auto resVecType = fromElementsOp.getResult().getType(); |
| 2967 | auto resEltType = resVecType.getElementType(); |
| 2968 | if (!resEltType.isIntOrIndexOrFloat() && !isa<ComplexType>(Val: resEltType)) |
| 2969 | return {}; |
| 2970 | |
| 2971 | // Constant attributes might have a different type than the return type. |
| 2972 | // Convert them before creating the dense elements attribute. |
| 2973 | auto convertedElements = llvm::map_to_vector(C&: elements, F: [&](Attribute attr) { |
| 2974 | return convertNumericAttr(attr, expectedType: resEltType); |
| 2975 | }); |
| 2976 | |
| 2977 | return DenseElementsAttr::get(type: resVecType, values: convertedElements); |
| 2978 | } |
| 2979 | |
| 2980 | OpFoldResult FromElementsOp::fold(FoldAdaptor adaptor) { |
| 2981 | if (auto res = foldFromElementsToElements(fromElementsOp: *this)) |
| 2982 | return res; |
| 2983 | if (auto res = foldFromElementsToConstant(fromElementsOp: *this, elements: adaptor.getElements())) |
| 2984 | return res; |
| 2985 | |
| 2986 | return {}; |
| 2987 | } |
| 2988 | |
| 2989 | /// Rewrite vector.from_elements as vector.broadcast if the elements are the |
| 2990 | /// same. Example: |
| 2991 | /// %0 = vector.from_elements %a, %a, %a : vector<3xf32> |
| 2992 | /// => |
| 2993 | /// %0 = vector.broadcast %a : f32 to vector<3xf32> |
| 2994 | static LogicalResult |
| 2995 | rewriteFromElementsAsBroadcast(FromElementsOp fromElementsOp, |
| 2996 | PatternRewriter &rewriter) { |
| 2997 | if (!llvm::all_equal(Range: fromElementsOp.getElements())) |
| 2998 | return failure(); |
| 2999 | rewriter.replaceOpWithNewOp<BroadcastOp>( |
| 3000 | op: fromElementsOp, args: fromElementsOp.getType(), |
| 3001 | args: fromElementsOp.getElements().front()); |
| 3002 | return success(); |
| 3003 | } |
| 3004 | |
| 3005 | /// Rewrite from_elements on multiple scalar extracts as a shape_cast |
| 3006 | /// on a single extract. Example: |
| 3007 | /// %0 = vector.extract %source[0, 0] : i8 from vector<2x2xi8> |
| 3008 | /// %1 = vector.extract %source[0, 1] : i8 from vector<2x2xi8> |
| 3009 | /// %2 = vector.from_elements %0, %1 : vector<2xi8> |
| 3010 | /// |
| 3011 | /// becomes |
| 3012 | /// %1 = vector.extract %source[0] : vector<1x2xi8> from vector<2x2xi8> |
| 3013 | /// %2 = vector.shape_cast %1 : vector<1x2xi8> to vector<2xi8> |
| 3014 | /// |
| 3015 | /// The requirements for this to be valid are |
| 3016 | /// |
| 3017 | /// i) The elements are extracted from the same vector (%source). |
| 3018 | /// |
| 3019 | /// ii) The elements form a suffix of %source. Specifically, the number |
| 3020 | /// of elements is the same as the product of the last N dimension sizes |
| 3021 | /// of %source, for some N. |
| 3022 | /// |
| 3023 | /// iii) The elements are extracted contiguously in ascending order. |
| 3024 | |
| 3025 | class FromElementsToShapeCast : public OpRewritePattern<FromElementsOp> { |
| 3026 | |
| 3027 | using Base::Base; |
| 3028 | |
| 3029 | LogicalResult matchAndRewrite(FromElementsOp fromElements, |
| 3030 | PatternRewriter &rewriter) const override { |
| 3031 | |
| 3032 | // Handled by `rewriteFromElementsAsBroadcast`. |
| 3033 | if (fromElements.getType().getNumElements() == 1) |
| 3034 | return failure(); |
| 3035 | |
| 3036 | // The common source that all elements are extracted from, if one exists. |
| 3037 | TypedValue<VectorType> source; |
| 3038 | // The position of the combined extract operation, if one is created. |
| 3039 | ArrayRef<int64_t> combinedPosition; |
| 3040 | // The expected index of extraction of the current element in the loop, if |
| 3041 | // elements are extracted contiguously in ascending order. |
| 3042 | SmallVector<int64_t> expectedPosition; |
| 3043 | |
| 3044 | for (auto [insertIndex, element] : |
| 3045 | llvm::enumerate(First: fromElements.getElements())) { |
| 3046 | |
| 3047 | // Check that the element is from a vector.extract operation. |
| 3048 | auto = element.getDefiningOp<vector::ExtractOp>(); |
| 3049 | if (!extractOp) { |
| 3050 | return rewriter.notifyMatchFailure(arg&: fromElements, |
| 3051 | msg: "element not from vector.extract" ); |
| 3052 | } |
| 3053 | |
| 3054 | // Check condition (i) by checking that all elements have the same source |
| 3055 | // as the first element. |
| 3056 | if (insertIndex == 0) { |
| 3057 | source = extractOp.getSource(); |
| 3058 | } else if (extractOp.getSource() != source) { |
| 3059 | return rewriter.notifyMatchFailure(arg&: fromElements, |
| 3060 | msg: "element from different vector" ); |
| 3061 | } |
| 3062 | |
| 3063 | ArrayRef<int64_t> position = extractOp.getStaticPosition(); |
| 3064 | int64_t rank = position.size(); |
| 3065 | assert(rank == source.getType().getRank() && |
| 3066 | "scalar extract must have full rank position" ); |
| 3067 | |
| 3068 | // Check condition (ii) by checking that the position that the first |
| 3069 | // element is extracted from has sufficient trailing 0s. For example, in |
| 3070 | // |
| 3071 | // %elm0 = vector.extract %source[1, 0, 0] : i8 from vector<2x3x4xi8> |
| 3072 | // [...] |
| 3073 | // %elms = vector.from_elements %elm0, [...] : vector<12xi8> |
| 3074 | // |
| 3075 | // The 2 trailing 0s in the position of extraction of %elm0 cover 3*4 = 12 |
| 3076 | // elements, which is the number of elements of %n, so this is valid. |
| 3077 | if (insertIndex == 0) { |
| 3078 | const int64_t numElms = fromElements.getType().getNumElements(); |
| 3079 | int64_t numSuffixElms = 1; |
| 3080 | int64_t index = rank; |
| 3081 | while (index > 0 && position[index - 1] == 0 && |
| 3082 | numSuffixElms < numElms) { |
| 3083 | numSuffixElms *= source.getType().getDimSize(idx: index - 1); |
| 3084 | --index; |
| 3085 | } |
| 3086 | if (numSuffixElms != numElms) { |
| 3087 | return rewriter.notifyMatchFailure( |
| 3088 | arg&: fromElements, msg: "elements do not form a suffix of source" ); |
| 3089 | } |
| 3090 | expectedPosition = llvm::to_vector(Range&: position); |
| 3091 | combinedPosition = position.drop_back(N: rank - index); |
| 3092 | } |
| 3093 | |
| 3094 | // Check condition (iii). |
| 3095 | else if (expectedPosition != position) { |
| 3096 | return rewriter.notifyMatchFailure( |
| 3097 | arg&: fromElements, msg: "elements not in ascending order (static order)" ); |
| 3098 | } |
| 3099 | increment(indices: expectedPosition, shape: source.getType().getShape()); |
| 3100 | } |
| 3101 | |
| 3102 | auto = rewriter.createOrFold<vector::ExtractOp>( |
| 3103 | location: fromElements.getLoc(), args&: source, args&: combinedPosition); |
| 3104 | |
| 3105 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>( |
| 3106 | op: fromElements, args: fromElements.getType(), args&: extracted); |
| 3107 | |
| 3108 | return success(); |
| 3109 | } |
| 3110 | |
| 3111 | /// Increments n-D `indices` by 1 starting from the innermost dimension. |
| 3112 | static void increment(MutableArrayRef<int64_t> indices, |
| 3113 | ArrayRef<int64_t> shape) { |
| 3114 | for (int dim : llvm::reverse(C: llvm::seq<int>(Begin: 0, End: indices.size()))) { |
| 3115 | indices[dim] += 1; |
| 3116 | if (indices[dim] < shape[dim]) |
| 3117 | break; |
| 3118 | indices[dim] = 0; |
| 3119 | } |
| 3120 | } |
| 3121 | }; |
| 3122 | |
| 3123 | void FromElementsOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 3124 | MLIRContext *context) { |
| 3125 | results.add(implFn: rewriteFromElementsAsBroadcast); |
| 3126 | results.add<FromElementsToShapeCast>(arg&: context); |
| 3127 | } |
| 3128 | |
| 3129 | //===----------------------------------------------------------------------===// |
| 3130 | // BroadcastOp |
| 3131 | //===----------------------------------------------------------------------===// |
| 3132 | |
| 3133 | void BroadcastOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges, |
| 3134 | SetIntRangeFn setResultRanges) { |
| 3135 | setResultRanges(getResult(), argRanges.front()); |
| 3136 | } |
| 3137 | |
| 3138 | //----------------------------------------------------------------------------- |
| 3139 | // VectorUnrollOpInterface methods |
| 3140 | //----------------------------------------------------------------------------- |
| 3141 | std::optional<SmallVector<int64_t, 4>> |
| 3142 | BroadcastOp::getIterationShapeToUnroll() { |
| 3143 | return llvm::to_vector<4>(Range: getResultVectorType().getShape()); |
| 3144 | } |
| 3145 | |
| 3146 | /// Return the dimensions of the result vector that were formerly ones in the |
| 3147 | /// source tensor and thus correspond to "dim-1" broadcasting. |
| 3148 | static llvm::SetVector<int64_t> |
| 3149 | computeBroadcastedUnitDims(ArrayRef<int64_t> srcShape, |
| 3150 | ArrayRef<int64_t> dstShape) { |
| 3151 | int64_t rankDiff = dstShape.size() - srcShape.size(); |
| 3152 | int64_t dstDim = rankDiff; |
| 3153 | llvm::SetVector<int64_t> res; |
| 3154 | for (auto [s1, s2] : |
| 3155 | llvm::zip_equal(t&: srcShape, u: dstShape.drop_front(N: rankDiff))) { |
| 3156 | if (s1 != s2) { |
| 3157 | assert(s1 == 1 && "expected \"dim-1\" broadcasting" ); |
| 3158 | res.insert(X: dstDim); |
| 3159 | } |
| 3160 | ++dstDim; |
| 3161 | } |
| 3162 | return res; |
| 3163 | } |
| 3164 | |
| 3165 | llvm::SetVector<int64_t> BroadcastOp::computeBroadcastedUnitDims() { |
| 3166 | // Scalar broadcast is without any unit dim broadcast. |
| 3167 | auto srcVectorType = llvm::dyn_cast<VectorType>(Val: getSourceType()); |
| 3168 | if (!srcVectorType) |
| 3169 | return {}; |
| 3170 | return ::computeBroadcastedUnitDims(srcShape: srcVectorType.getShape(), |
| 3171 | dstShape: getResultVectorType().getShape()); |
| 3172 | } |
| 3173 | |
| 3174 | /// Broadcast `value` to a vector of `dstShape`, knowing that exactly the |
| 3175 | /// `broadcastedDims` dimensions in the dstShape are broadcasted. |
| 3176 | /// This requires (and asserts) that the broadcast is free of "dim-1" |
| 3177 | /// broadcasting. |
| 3178 | /// Since vector.broadcast only allows expanding leading dimensions, an extra |
| 3179 | /// vector.transpose may be inserted to make the broadcast possible. |
| 3180 | /// `value`, `dstShape` and `broadcastedDims` must be properly specified or |
| 3181 | /// the helper will assert. This means: |
| 3182 | /// 1. `dstShape` must not be empty. |
| 3183 | /// 2. `broadcastedDims` must be confined to [0 .. rank(value.getVectorType)] |
| 3184 | /// 2. `dstShape` trimmed of the dimensions specified in `broadcastedDims` |
| 3185 | // must match the `value` shape. |
| 3186 | Value BroadcastOp::createOrFoldBroadcastOp( |
| 3187 | OpBuilder &b, Value value, ArrayRef<int64_t> dstShape, |
| 3188 | const llvm::SetVector<int64_t> &broadcastedDims) { |
| 3189 | assert(!dstShape.empty() && "unexpected empty dst shape" ); |
| 3190 | |
| 3191 | // Well-formedness check. |
| 3192 | SmallVector<int64_t> checkShape; |
| 3193 | for (int i = 0, e = dstShape.size(); i < e; ++i) { |
| 3194 | if (broadcastedDims.contains(key: i)) |
| 3195 | continue; |
| 3196 | checkShape.push_back(Elt: dstShape[i]); |
| 3197 | } |
| 3198 | assert(broadcastedDims.size() == dstShape.size() - checkShape.size() && |
| 3199 | "ill-formed broadcastedDims contains values not confined to " |
| 3200 | "destVectorShape" ); |
| 3201 | |
| 3202 | Location loc = value.getLoc(); |
| 3203 | Type elementType = getElementTypeOrSelf(type: value.getType()); |
| 3204 | VectorType srcVectorType = llvm::dyn_cast<VectorType>(Val: value.getType()); |
| 3205 | VectorType dstVectorType = VectorType::get(shape: dstShape, elementType); |
| 3206 | |
| 3207 | // Step 2. If scalar -> dstShape broadcast, just do it. |
| 3208 | if (!srcVectorType) { |
| 3209 | assert(checkShape.empty() && |
| 3210 | "ill-formed createOrFoldBroadcastOp arguments" ); |
| 3211 | return b.createOrFold<vector::BroadcastOp>(location: loc, args&: dstVectorType, args&: value); |
| 3212 | } |
| 3213 | |
| 3214 | assert(srcVectorType.getShape().equals(checkShape) && |
| 3215 | "ill-formed createOrFoldBroadcastOp arguments" ); |
| 3216 | |
| 3217 | // Step 3. Since vector.broadcast only allows creating leading dims, |
| 3218 | // vector -> dstShape broadcast may require a transpose. |
| 3219 | // Traverse the dims in order and construct: |
| 3220 | // 1. The leading entries of the broadcastShape that is guaranteed to be |
| 3221 | // achievable by a simple broadcast. |
| 3222 | // 2. The induced permutation for the subsequent vector.transpose that will |
| 3223 | // bring us from `broadcastShape` back to he desired `dstShape`. |
| 3224 | // If the induced permutation is not the identity, create a vector.transpose. |
| 3225 | SmallVector<int64_t> broadcastShape, permutation(dstShape.size(), -1); |
| 3226 | broadcastShape.reserve(N: dstShape.size()); |
| 3227 | // Consider the example: |
| 3228 | // srcShape = 2x4 |
| 3229 | // dstShape = 1x2x3x4x5 |
| 3230 | // broadcastedDims = [0, 2, 4] |
| 3231 | // |
| 3232 | // We want to build: |
| 3233 | // broadcastShape = 1x3x5x2x4 |
| 3234 | // permutation = [0, 2, 4, 1, 3] |
| 3235 | // ---V--- -----V----- |
| 3236 | // leading broadcast part src shape part |
| 3237 | // |
| 3238 | // Note that the trailing dims of broadcastShape are exactly the srcShape |
| 3239 | // by construction. |
| 3240 | // nextSrcShapeDim is used to keep track of where in the permutation the |
| 3241 | // "src shape part" occurs. |
| 3242 | int64_t nextSrcShapeDim = broadcastedDims.size(); |
| 3243 | for (int64_t i = 0, e = dstShape.size(); i < e; ++i) { |
| 3244 | if (broadcastedDims.contains(key: i)) { |
| 3245 | // 3.a. For each dim in the dst shape, if it is a broadcasted dim, |
| 3246 | // bring it to the head of the broadcastShape. |
| 3247 | // It will need to be permuted back from `broadcastShape.size() - 1` into |
| 3248 | // position `i`. |
| 3249 | broadcastShape.push_back(Elt: dstShape[i]); |
| 3250 | permutation[i] = broadcastShape.size() - 1; |
| 3251 | } else { |
| 3252 | // 3.b. Otherwise, the dim is not broadcasted, it comes from the src |
| 3253 | // shape and needs to be permuted into position `i`. |
| 3254 | // Don't touch `broadcastShape` here, the whole srcShape will be |
| 3255 | // appended after. |
| 3256 | permutation[i] = nextSrcShapeDim++; |
| 3257 | } |
| 3258 | } |
| 3259 | // 3.c. Append the srcShape. |
| 3260 | llvm::append_range(C&: broadcastShape, R: srcVectorType.getShape()); |
| 3261 | |
| 3262 | // Ensure there are no "dim-1" broadcasts. |
| 3263 | assert(::computeBroadcastedUnitDims(srcVectorType.getShape(), broadcastShape) |
| 3264 | .empty() && |
| 3265 | "unexpected \"dim-1\" broadcast" ); |
| 3266 | |
| 3267 | VectorType broadcastType = VectorType::get(shape: broadcastShape, elementType); |
| 3268 | assert(vector::isBroadcastableTo(value.getType(), broadcastType) == |
| 3269 | vector::BroadcastableToResult::Success && |
| 3270 | "must be broadcastable" ); |
| 3271 | Value res = b.createOrFold<vector::BroadcastOp>(location: loc, args&: broadcastType, args&: value); |
| 3272 | // Step 4. If we find any dimension that indeed needs to be permuted, |
| 3273 | // immediately return a new vector.transpose. |
| 3274 | for (int64_t i = 0, e = permutation.size(); i < e; ++i) |
| 3275 | if (permutation[i] != i) |
| 3276 | return b.createOrFold<vector::TransposeOp>(location: loc, args&: res, args&: permutation); |
| 3277 | // Otherwise return res. |
| 3278 | return res; |
| 3279 | } |
| 3280 | |
| 3281 | LogicalResult BroadcastOp::verify() { |
| 3282 | std::pair<VectorDim, VectorDim> mismatchingDims; |
| 3283 | BroadcastableToResult res = isBroadcastableTo( |
| 3284 | srcType: getSourceType(), dstVectorType: getResultVectorType(), mismatchingDims: &mismatchingDims); |
| 3285 | if (res == BroadcastableToResult::Success) |
| 3286 | return success(); |
| 3287 | if (res == BroadcastableToResult::SourceRankHigher) |
| 3288 | return emitOpError(message: "source rank higher than destination rank" ); |
| 3289 | if (res == BroadcastableToResult::DimensionMismatch) { |
| 3290 | return emitOpError(message: "dimension mismatch (" ) |
| 3291 | << (mismatchingDims.first.isScalable ? "[" : "" ) |
| 3292 | << mismatchingDims.first.dim |
| 3293 | << (mismatchingDims.first.isScalable ? "]" : "" ) << " vs. " |
| 3294 | << (mismatchingDims.second.isScalable ? "[" : "" ) |
| 3295 | << mismatchingDims.second.dim |
| 3296 | << (mismatchingDims.second.isScalable ? "]" : "" ) << ")" ; |
| 3297 | } |
| 3298 | if (res == BroadcastableToResult::SourceTypeNotAVector) |
| 3299 | return emitOpError(message: "source type is not a vector" ); |
| 3300 | llvm_unreachable("unexpected vector.broadcast op error" ); |
| 3301 | } |
| 3302 | |
| 3303 | // Fold broadcast(shape_cast(x)) into broadcast(x) if x's type is compatible |
| 3304 | // with broadcast's result type and shape_cast only adds or removes ones in the |
| 3305 | // leading dimensions. |
| 3306 | static LogicalResult foldBroadcastOfShapeCast(BroadcastOp broadcastOp) { |
| 3307 | auto srcShapeCast = broadcastOp.getSource().getDefiningOp<ShapeCastOp>(); |
| 3308 | if (!srcShapeCast) |
| 3309 | return failure(); |
| 3310 | |
| 3311 | VectorType srcType = srcShapeCast.getSourceVectorType(); |
| 3312 | VectorType destType = broadcastOp.getResultVectorType(); |
| 3313 | // Check type compatibility. |
| 3314 | if (vector::isBroadcastableTo(srcType, dstVectorType: destType) != |
| 3315 | BroadcastableToResult::Success) |
| 3316 | return failure(); |
| 3317 | |
| 3318 | ArrayRef<int64_t> srcShape = srcType.getShape(); |
| 3319 | ArrayRef<int64_t> shapecastShape = |
| 3320 | srcShapeCast.getResultVectorType().getShape(); |
| 3321 | // Trailing dimensions should be the same if shape_cast only alters the |
| 3322 | // leading dimensions. |
| 3323 | unsigned numTrailingDims = std::min(a: srcShape.size(), b: shapecastShape.size()); |
| 3324 | if (!llvm::equal(LRange: srcShape.take_back(N: numTrailingDims), |
| 3325 | RRange: shapecastShape.take_back(N: numTrailingDims))) |
| 3326 | return failure(); |
| 3327 | |
| 3328 | assert(all_of(srcShape.drop_back(numTrailingDims), |
| 3329 | [](int64_t E) { return E == 1; }) && |
| 3330 | all_of(shapecastShape.drop_back(numTrailingDims), |
| 3331 | [](int64_t E) { return E == 1; }) && |
| 3332 | "ill-formed shape_cast" ); |
| 3333 | |
| 3334 | broadcastOp.getSourceMutable().assign(value: srcShapeCast.getSource()); |
| 3335 | return success(); |
| 3336 | } |
| 3337 | |
| 3338 | OpFoldResult BroadcastOp::fold(FoldAdaptor adaptor) { |
| 3339 | if (getSourceType() == getResultVectorType()) |
| 3340 | return getSource(); |
| 3341 | if (succeeded(Result: foldBroadcastOfShapeCast(broadcastOp: *this))) |
| 3342 | return getResult(); |
| 3343 | |
| 3344 | if (!adaptor.getSource()) |
| 3345 | return {}; |
| 3346 | auto vectorType = getResultVectorType(); |
| 3347 | if (auto attr = llvm::dyn_cast<IntegerAttr>(Val: adaptor.getSource())) { |
| 3348 | if (vectorType.getElementType() != attr.getType()) |
| 3349 | return {}; |
| 3350 | return DenseElementsAttr::get(type: vectorType, values: attr); |
| 3351 | } |
| 3352 | if (auto attr = llvm::dyn_cast<FloatAttr>(Val: adaptor.getSource())) { |
| 3353 | if (vectorType.getElementType() != attr.getType()) |
| 3354 | return {}; |
| 3355 | return DenseElementsAttr::get(type: vectorType, values: attr); |
| 3356 | } |
| 3357 | if (auto attr = llvm::dyn_cast<SplatElementsAttr>(Val: adaptor.getSource())) |
| 3358 | return DenseElementsAttr::get(type: vectorType, values: attr.getSplatValue<Attribute>()); |
| 3359 | if (matchPattern(attr: adaptor.getSource(), pattern: ub::m_Poison())) |
| 3360 | return ub::PoisonAttr::get(ctx: getContext()); |
| 3361 | return {}; |
| 3362 | } |
| 3363 | |
| 3364 | namespace { |
| 3365 | |
| 3366 | // Fold broadcast1(broadcast2(x)) into broadcast1(x). |
| 3367 | struct BroadcastFolder : public OpRewritePattern<BroadcastOp> { |
| 3368 | using Base::Base; |
| 3369 | |
| 3370 | LogicalResult matchAndRewrite(BroadcastOp broadcastOp, |
| 3371 | PatternRewriter &rewriter) const override { |
| 3372 | auto srcBroadcast = broadcastOp.getSource().getDefiningOp<BroadcastOp>(); |
| 3373 | if (!srcBroadcast) |
| 3374 | return failure(); |
| 3375 | rewriter.replaceOpWithNewOp<BroadcastOp>(op: broadcastOp, |
| 3376 | args: broadcastOp.getResultVectorType(), |
| 3377 | args: srcBroadcast.getSource()); |
| 3378 | return success(); |
| 3379 | } |
| 3380 | }; |
| 3381 | |
| 3382 | /// Replace `vector.broadcast` with `vector.shape_cast`. |
| 3383 | /// |
| 3384 | /// BEFORE: |
| 3385 | /// %0 = vector.broadcast %arg0 : vector<4xi8> to vector<1x1x4xi8> |
| 3386 | /// AFTER: |
| 3387 | /// %0 = vector.shape_cast %arg0 : vector<4xi8> to vector<1x1x4xi8> |
| 3388 | /// |
| 3389 | /// The canonical form of vector operations that reshape vectors is shape_cast. |
| 3390 | struct BroadcastToShapeCast final |
| 3391 | : public OpRewritePattern<vector::BroadcastOp> { |
| 3392 | using Base::Base; |
| 3393 | LogicalResult matchAndRewrite(vector::BroadcastOp broadcast, |
| 3394 | PatternRewriter &rewriter) const override { |
| 3395 | |
| 3396 | auto sourceType = dyn_cast<VectorType>(Val: broadcast.getSourceType()); |
| 3397 | if (!sourceType) { |
| 3398 | return rewriter.notifyMatchFailure( |
| 3399 | arg&: broadcast, msg: "source is a scalar, shape_cast doesn't support scalar" ); |
| 3400 | } |
| 3401 | |
| 3402 | VectorType outType = broadcast.getType(); |
| 3403 | if (sourceType.getNumElements() != outType.getNumElements()) { |
| 3404 | return rewriter.notifyMatchFailure( |
| 3405 | arg&: broadcast, msg: "broadcast to a greater number of elements" ); |
| 3406 | } |
| 3407 | |
| 3408 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: broadcast, args&: outType, |
| 3409 | args: broadcast.getSource()); |
| 3410 | return success(); |
| 3411 | } |
| 3412 | }; |
| 3413 | } // namespace |
| 3414 | |
| 3415 | void BroadcastOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 3416 | MLIRContext *context) { |
| 3417 | results.add<BroadcastFolder, BroadcastToShapeCast>(arg&: context); |
| 3418 | } |
| 3419 | |
| 3420 | //===----------------------------------------------------------------------===// |
| 3421 | // ShuffleOp |
| 3422 | //===----------------------------------------------------------------------===// |
| 3423 | |
| 3424 | LogicalResult ShuffleOp::verify() { |
| 3425 | VectorType resultType = getResultVectorType(); |
| 3426 | VectorType v1Type = getV1VectorType(); |
| 3427 | VectorType v2Type = getV2VectorType(); |
| 3428 | // Verify ranks. |
| 3429 | int64_t resRank = resultType.getRank(); |
| 3430 | int64_t v1Rank = v1Type.getRank(); |
| 3431 | int64_t v2Rank = v2Type.getRank(); |
| 3432 | bool wellFormed0DCase = v1Rank == 0 && v2Rank == 0 && resRank == 1; |
| 3433 | bool wellFormedNDCase = v1Rank == resRank && v2Rank == resRank; |
| 3434 | if (!wellFormed0DCase && !wellFormedNDCase) |
| 3435 | return emitOpError(message: "rank mismatch" ); |
| 3436 | |
| 3437 | // Verify all but leading dimension sizes. |
| 3438 | for (int64_t r = 1; r < v1Rank; ++r) { |
| 3439 | int64_t resDim = resultType.getDimSize(idx: r); |
| 3440 | int64_t v1Dim = v1Type.getDimSize(idx: r); |
| 3441 | int64_t v2Dim = v2Type.getDimSize(idx: r); |
| 3442 | if (resDim != v1Dim || v1Dim != v2Dim) |
| 3443 | return emitOpError(message: "dimension mismatch" ); |
| 3444 | } |
| 3445 | // Verify mask length. |
| 3446 | ArrayRef<int64_t> mask = getMask(); |
| 3447 | int64_t maskLength = mask.size(); |
| 3448 | if (maskLength <= 0) |
| 3449 | return emitOpError(message: "invalid mask length" ); |
| 3450 | if (maskLength != resultType.getDimSize(idx: 0)) |
| 3451 | return emitOpError(message: "mask length mismatch" ); |
| 3452 | // Verify all indices. |
| 3453 | int64_t indexSize = (v1Type.getRank() == 0 ? 1 : v1Type.getDimSize(idx: 0)) + |
| 3454 | (v2Type.getRank() == 0 ? 1 : v2Type.getDimSize(idx: 0)); |
| 3455 | for (auto [idx, maskPos] : llvm::enumerate(First&: mask)) { |
| 3456 | if (!isValidPositiveIndexOrPoison(index: maskPos, poisonValue: kPoisonIndex, maxIndex: indexSize)) |
| 3457 | return emitOpError(message: "mask index #" ) << (idx + 1) << " out of range" ; |
| 3458 | } |
| 3459 | return success(); |
| 3460 | } |
| 3461 | |
| 3462 | LogicalResult |
| 3463 | ShuffleOp::inferReturnTypes(MLIRContext *, std::optional<Location> loc, |
| 3464 | ShuffleOp::Adaptor adaptor, |
| 3465 | SmallVectorImpl<Type> &inferredReturnTypes) { |
| 3466 | auto v1Type = llvm::dyn_cast<VectorType>(Val: adaptor.getV1().getType()); |
| 3467 | if (!v1Type) { |
| 3468 | return emitOptionalError(loc, args: "expected vector type" ); |
| 3469 | } |
| 3470 | auto v1Rank = v1Type.getRank(); |
| 3471 | // Construct resulting type: leading dimension matches mask |
| 3472 | // length, all trailing dimensions match the operands. |
| 3473 | SmallVector<int64_t, 4> shape; |
| 3474 | shape.reserve(N: v1Rank); |
| 3475 | shape.push_back(Elt: std::max<size_t>(a: 1, b: adaptor.getMask().size())); |
| 3476 | // In the 0-D case there is no trailing shape to append. |
| 3477 | if (v1Rank > 0) |
| 3478 | llvm::append_range(C&: shape, R: v1Type.getShape().drop_front()); |
| 3479 | inferredReturnTypes.push_back( |
| 3480 | Elt: VectorType::get(shape, elementType: v1Type.getElementType())); |
| 3481 | return success(); |
| 3482 | } |
| 3483 | |
| 3484 | template <typename T> |
| 3485 | static bool isStepIndexArray(ArrayRef<T> idxArr, uint64_t begin, size_t width) { |
| 3486 | T expected = begin; |
| 3487 | return idxArr.size() == width && llvm::all_of(idxArr, [&expected](T value) { |
| 3488 | return value == expected++; |
| 3489 | }); |
| 3490 | } |
| 3491 | |
| 3492 | /// Fold shuffle V1, V2, [0, 1, 2, 3] : <4xi32>, <2xi32> -> V1. |
| 3493 | /// Fold shuffle V1, V2, [4, 5] : <4xi32>, <2xi32> -> V2. |
| 3494 | static OpFoldResult foldShuffleIdentityMask(ShuffleOp op) { |
| 3495 | auto v1Type = op.getV1VectorType(); |
| 3496 | auto v2Type = op.getV2VectorType(); |
| 3497 | auto mask = op.getMask(); |
| 3498 | if (isStepIndexArray(idxArr: mask, begin: 0, width: v1Type.getDimSize(idx: 0))) |
| 3499 | return op.getV1(); |
| 3500 | if (isStepIndexArray(idxArr: mask, begin: v1Type.getDimSize(idx: 0), width: v2Type.getDimSize(idx: 0))) |
| 3501 | return op.getV2(); |
| 3502 | return {}; |
| 3503 | } |
| 3504 | |
| 3505 | /// If a shuffle operand is poison, replace all mask indices that reference it |
| 3506 | /// with kPoisonIndex. This is an in-place fold. |
| 3507 | static OpFoldResult foldShufflePoisonOperandToMask(ShuffleOp op) { |
| 3508 | bool isV1Poison = matchPattern(value: op.getV1(), pattern: ub::m_Poison()); |
| 3509 | bool isV2Poison = matchPattern(value: op.getV2(), pattern: ub::m_Poison()); |
| 3510 | if (!isV1Poison && !isV2Poison) |
| 3511 | return {}; |
| 3512 | |
| 3513 | int64_t v1Size = op.getV1VectorType().getDimSize(idx: 0); |
| 3514 | bool changed = false; |
| 3515 | SmallVector<int64_t> newMask = llvm::to_vector(Range: op.getMask()); |
| 3516 | for (int64_t &idx : newMask) { |
| 3517 | if (idx == ShuffleOp::kPoisonIndex) |
| 3518 | continue; |
| 3519 | if ((isV1Poison && idx < v1Size) || (isV2Poison && idx >= v1Size)) { |
| 3520 | idx = ShuffleOp::kPoisonIndex; |
| 3521 | changed = true; |
| 3522 | } |
| 3523 | } |
| 3524 | |
| 3525 | if (!changed) |
| 3526 | return {}; |
| 3527 | |
| 3528 | op.setMask(newMask); |
| 3529 | return op.getResult(); |
| 3530 | } |
| 3531 | |
| 3532 | /// Fold shuffle poison, poison -> poison. |
| 3533 | static OpFoldResult foldShufflePoisonInputs(MLIRContext *context, |
| 3534 | Attribute v1Attr, |
| 3535 | Attribute v2Attr) { |
| 3536 | if (matchPattern(attr: v1Attr, pattern: ub::m_Poison()) && |
| 3537 | matchPattern(attr: v2Attr, pattern: ub::m_Poison())) |
| 3538 | return ub::PoisonAttr::get(ctx: context); |
| 3539 | return {}; |
| 3540 | } |
| 3541 | |
| 3542 | /// Fold a shuffle of constant 1-D inputs by evaluating the mask. |
| 3543 | static OpFoldResult foldShuffleConstantInputs(ShuffleOp op, Attribute v1Attr, |
| 3544 | Attribute v2Attr) { |
| 3545 | auto v1Type = op.getV1VectorType(); |
| 3546 | if (v1Type.getRank() != 1) |
| 3547 | return {}; |
| 3548 | |
| 3549 | bool isV1Poison = matchPattern(attr: v1Attr, pattern: ub::m_Poison()); |
| 3550 | bool isV2Poison = matchPattern(attr: v2Attr, pattern: ub::m_Poison()); |
| 3551 | |
| 3552 | // Poison input attributes need special handling as they are not |
| 3553 | // DenseElementsAttr. If an index is poison, we select the first element of |
| 3554 | // the first non-poison input. |
| 3555 | SmallVector<Attribute> v1Elements, v2Elements; |
| 3556 | Attribute poisonElement; |
| 3557 | if (!isV2Poison) { |
| 3558 | auto v2DenseAttr = dyn_cast<DenseElementsAttr>(Val&: v2Attr); |
| 3559 | if (!v2DenseAttr) |
| 3560 | return {}; |
| 3561 | v2Elements = to_vector(Range: v2DenseAttr.getValues<Attribute>()); |
| 3562 | poisonElement = v2Elements[0]; |
| 3563 | } |
| 3564 | if (!isV1Poison) { |
| 3565 | auto v1DenseAttr = dyn_cast<DenseElementsAttr>(Val&: v1Attr); |
| 3566 | if (!v1DenseAttr) |
| 3567 | return {}; |
| 3568 | v1Elements = to_vector(Range: v1DenseAttr.getValues<Attribute>()); |
| 3569 | poisonElement = v1Elements[0]; |
| 3570 | } |
| 3571 | |
| 3572 | ArrayRef<int64_t> mask = op.getMask(); |
| 3573 | SmallVector<Attribute> results; |
| 3574 | int64_t v1Size = v1Type.getDimSize(idx: 0); |
| 3575 | for (int64_t maskIdx : mask) { |
| 3576 | Attribute indexedElm; |
| 3577 | // TODO: Return a partial poison vector when supported by the UB dialect. |
| 3578 | if (maskIdx == ShuffleOp::kPoisonIndex) { |
| 3579 | indexedElm = poisonElement; |
| 3580 | } else { |
| 3581 | if (maskIdx < v1Size) |
| 3582 | indexedElm = isV1Poison ? poisonElement : v1Elements[maskIdx]; |
| 3583 | else |
| 3584 | indexedElm = isV2Poison ? poisonElement : v2Elements[maskIdx - v1Size]; |
| 3585 | } |
| 3586 | |
| 3587 | results.push_back(Elt: indexedElm); |
| 3588 | } |
| 3589 | |
| 3590 | return DenseElementsAttr::get(type: op.getResultVectorType(), values: results); |
| 3591 | } |
| 3592 | |
| 3593 | OpFoldResult vector::ShuffleOp::fold(FoldAdaptor adaptor) { |
| 3594 | auto v1Type = getV1VectorType(); |
| 3595 | |
| 3596 | assert(!v1Type.isScalable() && !getV2VectorType().isScalable() && |
| 3597 | "Vector shuffle does not support scalable vectors" ); |
| 3598 | |
| 3599 | // For consistency: 0-D shuffle return type is 1-D, this cannot be a folding |
| 3600 | // but must be a canonicalization into a vector.broadcast. |
| 3601 | if (v1Type.getRank() == 0) |
| 3602 | return {}; |
| 3603 | |
| 3604 | if (auto res = foldShuffleIdentityMask(op: *this)) |
| 3605 | return res; |
| 3606 | if (auto res = foldShufflePoisonOperandToMask(op: *this)) |
| 3607 | return res; |
| 3608 | |
| 3609 | Attribute v1Attr = adaptor.getV1(), v2Attr = adaptor.getV2(); |
| 3610 | if (!v1Attr || !v2Attr) |
| 3611 | return {}; |
| 3612 | |
| 3613 | if (auto res = foldShufflePoisonInputs(context: getContext(), v1Attr, v2Attr)) |
| 3614 | return res; |
| 3615 | if (auto res = foldShuffleConstantInputs(op: *this, v1Attr, v2Attr)) |
| 3616 | return res; |
| 3617 | |
| 3618 | return {}; |
| 3619 | } |
| 3620 | |
| 3621 | namespace { |
| 3622 | |
| 3623 | // Pattern to rewrite a 0-D shuffle with [0] or [1] mask returning a 1-D vector |
| 3624 | // to a broadcast. |
| 3625 | struct Canonicalize0DShuffleOp : public OpRewritePattern<ShuffleOp> { |
| 3626 | using Base::Base; |
| 3627 | |
| 3628 | LogicalResult matchAndRewrite(ShuffleOp shuffleOp, |
| 3629 | PatternRewriter &rewriter) const override { |
| 3630 | VectorType v1VectorType = shuffleOp.getV1VectorType(); |
| 3631 | ArrayRef<int64_t> mask = shuffleOp.getMask(); |
| 3632 | if (v1VectorType.getRank() > 0) |
| 3633 | return failure(); |
| 3634 | if (mask.size() != 1) |
| 3635 | return failure(); |
| 3636 | VectorType resType = VectorType::Builder(v1VectorType).setShape(newShape: {1}); |
| 3637 | if (mask[0] == 0) |
| 3638 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: shuffleOp, args&: resType, |
| 3639 | args: shuffleOp.getV1()); |
| 3640 | else |
| 3641 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: shuffleOp, args&: resType, |
| 3642 | args: shuffleOp.getV2()); |
| 3643 | return success(); |
| 3644 | } |
| 3645 | }; |
| 3646 | |
| 3647 | /// Consider the defining operation `defOp` of `value`. If `defOp` is a |
| 3648 | /// vector.broadcast with a scalar operand, return the scalar value that is |
| 3649 | /// splatted. Otherwise return null. |
| 3650 | /// |
| 3651 | /// Example: |
| 3652 | /// |
| 3653 | /// scalar_source --> vector.broadcast --> value - return scalar_source |
| 3654 | static Value getScalarSplatSource(Value value) { |
| 3655 | // Block argument: |
| 3656 | Operation *defOp = value.getDefiningOp(); |
| 3657 | if (!defOp) |
| 3658 | return {}; |
| 3659 | |
| 3660 | auto broadcast = dyn_cast<vector::BroadcastOp>(Val: defOp); |
| 3661 | |
| 3662 | // Not broadcast (and not splat): |
| 3663 | if (!broadcast) |
| 3664 | return {}; |
| 3665 | |
| 3666 | // Broadcast of a vector: |
| 3667 | if (isa<VectorType>(Val: broadcast.getSourceType())) |
| 3668 | return {}; |
| 3669 | |
| 3670 | // Broadcast of a scalar: |
| 3671 | return broadcast.getSource(); |
| 3672 | } |
| 3673 | |
| 3674 | /// Pattern to rewrite shuffle(splat-like(v), splat-like(v)) as broadcast(v). |
| 3675 | class ShuffleSplat final : public OpRewritePattern<ShuffleOp> { |
| 3676 | public: |
| 3677 | using Base::Base; |
| 3678 | |
| 3679 | LogicalResult matchAndRewrite(ShuffleOp op, |
| 3680 | PatternRewriter &rewriter) const override { |
| 3681 | Value splat = getScalarSplatSource(value: op.getV1()); |
| 3682 | if (!splat || getScalarSplatSource(value: op.getV2()) != splat) |
| 3683 | return failure(); |
| 3684 | |
| 3685 | rewriter.replaceOpWithNewOp<BroadcastOp>(op, args: op.getType(), args&: splat); |
| 3686 | return success(); |
| 3687 | } |
| 3688 | }; |
| 3689 | |
| 3690 | /// Pattern to rewrite a fixed-size interleave via vector.shuffle to |
| 3691 | /// vector.interleave. |
| 3692 | class ShuffleInterleave : public OpRewritePattern<ShuffleOp> { |
| 3693 | public: |
| 3694 | using Base::Base; |
| 3695 | |
| 3696 | LogicalResult matchAndRewrite(ShuffleOp op, |
| 3697 | PatternRewriter &rewriter) const override { |
| 3698 | VectorType resultType = op.getResultVectorType(); |
| 3699 | if (resultType.isScalable()) |
| 3700 | return rewriter.notifyMatchFailure( |
| 3701 | arg&: op, msg: "ShuffleOp can't represent a scalable interleave" ); |
| 3702 | |
| 3703 | if (resultType.getRank() != 1) |
| 3704 | return rewriter.notifyMatchFailure( |
| 3705 | arg&: op, msg: "ShuffleOp can't represent an n-D interleave" ); |
| 3706 | |
| 3707 | VectorType sourceType = op.getV1VectorType(); |
| 3708 | if (sourceType != op.getV2VectorType() || |
| 3709 | sourceType.getNumElements() * 2 != resultType.getNumElements()) { |
| 3710 | return rewriter.notifyMatchFailure( |
| 3711 | arg&: op, msg: "ShuffleOp types don't match an interleave" ); |
| 3712 | } |
| 3713 | |
| 3714 | ArrayRef<int64_t> shuffleMask = op.getMask(); |
| 3715 | int64_t resultVectorSize = resultType.getNumElements(); |
| 3716 | for (int i = 0, e = resultVectorSize / 2; i < e; ++i) { |
| 3717 | int64_t maskValueA = shuffleMask[i * 2]; |
| 3718 | int64_t maskValueB = shuffleMask[(i * 2) + 1]; |
| 3719 | if (maskValueA != i || maskValueB != (resultVectorSize / 2) + i) |
| 3720 | return rewriter.notifyMatchFailure(arg&: op, |
| 3721 | msg: "ShuffleOp mask not interleaving" ); |
| 3722 | } |
| 3723 | |
| 3724 | rewriter.replaceOpWithNewOp<InterleaveOp>(op, args: op.getV1(), args: op.getV2()); |
| 3725 | return success(); |
| 3726 | } |
| 3727 | }; |
| 3728 | |
| 3729 | /// Pattern to replace usused shuffle operands / results with poison. |
| 3730 | /// |
| 3731 | /// Example Input: |
| 3732 | /// %r = vector.shuffle %v1, %v2 [2, 3, 3, 3] : vector<2xi32>, vector<2xi32> |
| 3733 | /// |
| 3734 | /// Example Output: |
| 3735 | /// %0 = ub.poison : vector<2xi32> |
| 3736 | /// %r = vector.shuffle %0, %v2 [2, 3, 3, 3] : vector<2xi32>, vector<2xi32> |
| 3737 | class FoldUnusedShuffleOperand final : public OpRewritePattern<ShuffleOp> { |
| 3738 | public: |
| 3739 | using Base::Base; |
| 3740 | |
| 3741 | LogicalResult matchAndRewrite(ShuffleOp op, |
| 3742 | PatternRewriter &rewriter) const override { |
| 3743 | // Replace with poison if all mask elements are poison. |
| 3744 | if (llvm::all_of(Range: op.getMask(), P: [](int64_t mask) { |
| 3745 | return mask == ShuffleOp::kPoisonIndex; |
| 3746 | })) { |
| 3747 | rewriter.replaceOpWithNewOp<ub::PoisonOp>(op, args: op.getType()); |
| 3748 | return success(); |
| 3749 | } |
| 3750 | |
| 3751 | // Helper function to replace an operand with poison. |
| 3752 | auto replaceOperandWithPoison = [&](OpOperand &operand) { |
| 3753 | // Do not replace if the operand is already poison. |
| 3754 | if (!matchPattern(value: operand.get(), pattern: ub::m_Poison())) { |
| 3755 | Value poison = ub::PoisonOp::create(builder&: rewriter, location: op.getLoc(), |
| 3756 | result: operand.get().getType()); |
| 3757 | rewriter.modifyOpInPlace(root: op, callable: [&]() { operand.set(poison); }); |
| 3758 | return success(); |
| 3759 | } |
| 3760 | return failure(); |
| 3761 | }; |
| 3762 | |
| 3763 | // Replace V1 with poison if it is not used. |
| 3764 | int64_t leadingV1Size = op.getV1VectorType().getRank() > 0 |
| 3765 | ? op.getV1VectorType().getDimSize(idx: 0) |
| 3766 | : 1; |
| 3767 | bool isV1Used = llvm::any_of(Range: op.getMask(), P: [&](int64_t mask) { |
| 3768 | return mask != ShuffleOp::kPoisonIndex && mask < leadingV1Size; |
| 3769 | }); |
| 3770 | if (!isV1Used && succeeded(Result: replaceOperandWithPoison(op.getV1Mutable()))) |
| 3771 | return success(); |
| 3772 | |
| 3773 | // Replace V2 with poison if it is not used. |
| 3774 | bool isV2Used = llvm::any_of(Range: op.getMask(), P: [&](int64_t mask) { |
| 3775 | return mask != ShuffleOp::kPoisonIndex && mask >= leadingV1Size; |
| 3776 | }); |
| 3777 | if (!isV2Used && succeeded(Result: replaceOperandWithPoison(op.getV2Mutable()))) |
| 3778 | return success(); |
| 3779 | |
| 3780 | return failure(); |
| 3781 | } |
| 3782 | }; |
| 3783 | } // namespace |
| 3784 | |
| 3785 | void ShuffleOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 3786 | MLIRContext *context) { |
| 3787 | results.add<ShuffleSplat, ShuffleInterleave, Canonicalize0DShuffleOp, |
| 3788 | FoldUnusedShuffleOperand>(arg&: context); |
| 3789 | } |
| 3790 | |
| 3791 | //===----------------------------------------------------------------------===// |
| 3792 | // InsertOp |
| 3793 | //===----------------------------------------------------------------------===// |
| 3794 | |
| 3795 | void vector::InsertOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges, |
| 3796 | SetIntRangeFn setResultRanges) { |
| 3797 | setResultRanges(getResult(), argRanges[0].rangeUnion(other: argRanges[1])); |
| 3798 | } |
| 3799 | |
| 3800 | void vector::InsertOp::build(OpBuilder &builder, OperationState &result, |
| 3801 | Value source, Value dest) { |
| 3802 | auto vectorTy = cast<VectorType>(Val: dest.getType()); |
| 3803 | build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, |
| 3804 | position: SmallVector<int64_t>(vectorTy.getRank(), 0)); |
| 3805 | } |
| 3806 | |
| 3807 | void vector::InsertOp::build(OpBuilder &builder, OperationState &result, |
| 3808 | Value source, Value dest, int64_t position) { |
| 3809 | build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, position: ArrayRef<int64_t>{position}); |
| 3810 | } |
| 3811 | |
| 3812 | void vector::InsertOp::build(OpBuilder &builder, OperationState &result, |
| 3813 | Value source, Value dest, OpFoldResult position) { |
| 3814 | build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, position: ArrayRef<OpFoldResult>{position}); |
| 3815 | } |
| 3816 | |
| 3817 | void vector::InsertOp::build(OpBuilder &builder, OperationState &result, |
| 3818 | Value source, Value dest, |
| 3819 | ArrayRef<int64_t> position) { |
| 3820 | SmallVector<OpFoldResult> posVals; |
| 3821 | posVals.reserve(N: position.size()); |
| 3822 | llvm::transform(Range&: position, d_first: std::back_inserter(x&: posVals), |
| 3823 | F: [&](int64_t pos) { return builder.getI64IntegerAttr(value: pos); }); |
| 3824 | build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, position: posVals); |
| 3825 | } |
| 3826 | |
| 3827 | void vector::InsertOp::build(OpBuilder &builder, OperationState &result, |
| 3828 | Value source, Value dest, |
| 3829 | ArrayRef<OpFoldResult> position) { |
| 3830 | SmallVector<int64_t> staticPos; |
| 3831 | SmallVector<Value> dynamicPos; |
| 3832 | dispatchIndexOpFoldResults(ofrs: position, dynamicVec&: dynamicPos, staticVec&: staticPos); |
| 3833 | build(odsBuilder&: builder, odsState&: result, valueToStore: source, dest, dynamic_position: dynamicPos, |
| 3834 | static_position: builder.getDenseI64ArrayAttr(values: staticPos)); |
| 3835 | } |
| 3836 | |
| 3837 | LogicalResult InsertOp::verify() { |
| 3838 | if (auto srcTy = dyn_cast<VectorType>(Val: getValueToStoreType())) |
| 3839 | if (srcTy.getRank() == 0) |
| 3840 | return emitError( |
| 3841 | message: "expected a scalar instead of a 0-d vector as the source operand" ); |
| 3842 | |
| 3843 | SmallVector<OpFoldResult> position = getMixedPosition(); |
| 3844 | auto destVectorType = getDestVectorType(); |
| 3845 | if (position.size() > static_cast<unsigned>(destVectorType.getRank())) |
| 3846 | return emitOpError( |
| 3847 | message: "expected position attribute of rank no greater than dest vector rank" ); |
| 3848 | auto srcVectorType = llvm::dyn_cast<VectorType>(Val: getValueToStoreType()); |
| 3849 | if (srcVectorType && |
| 3850 | (static_cast<unsigned>(srcVectorType.getRank()) + position.size() != |
| 3851 | static_cast<unsigned>(destVectorType.getRank()))) |
| 3852 | return emitOpError(message: "expected position attribute rank + source rank to " |
| 3853 | "match dest vector rank" ); |
| 3854 | if (!srcVectorType && |
| 3855 | (position.size() != static_cast<unsigned>(destVectorType.getRank()))) |
| 3856 | return emitOpError( |
| 3857 | message: "expected position attribute rank to match the dest vector rank" ); |
| 3858 | for (auto [idx, pos] : llvm::enumerate(First&: position)) { |
| 3859 | if (auto attr = dyn_cast<Attribute>(Val&: pos)) { |
| 3860 | int64_t constIdx = cast<IntegerAttr>(Val&: attr).getInt(); |
| 3861 | if (!isValidPositiveIndexOrPoison(index: constIdx, poisonValue: kPoisonIndex, |
| 3862 | maxIndex: destVectorType.getDimSize(idx))) { |
| 3863 | return emitOpError(message: "expected position attribute #" ) |
| 3864 | << (idx + 1) |
| 3865 | << " to be a non-negative integer smaller than the " |
| 3866 | "corresponding " |
| 3867 | "dest vector dimension" ; |
| 3868 | } |
| 3869 | } |
| 3870 | } |
| 3871 | return success(); |
| 3872 | } |
| 3873 | |
| 3874 | // Calculate the linearized position of the continuous chunk of elements to |
| 3875 | // insert, based on the shape of the value to insert and the positions to insert |
| 3876 | // at. |
| 3877 | static int64_t calculateInsertPosition(VectorType destTy, |
| 3878 | ArrayRef<int64_t> positions) { |
| 3879 | llvm::SmallVector<int64_t> completePositions(destTy.getRank(), 0); |
| 3880 | assert(positions.size() <= completePositions.size() && |
| 3881 | "positions size must be less than or equal to destTy rank" ); |
| 3882 | copy(Range&: positions, Out: completePositions.begin()); |
| 3883 | return linearize(offsets: completePositions, basis: computeStrides(sizes: destTy.getShape())); |
| 3884 | } |
| 3885 | |
| 3886 | namespace { |
| 3887 | |
| 3888 | // If insertOp is only inserting unit dimensions it can be transformed to a |
| 3889 | // broadcast. |
| 3890 | class InsertToBroadcast final : public OpRewritePattern<InsertOp> { |
| 3891 | public: |
| 3892 | using Base::Base; |
| 3893 | |
| 3894 | LogicalResult matchAndRewrite(InsertOp insertOp, |
| 3895 | PatternRewriter &rewriter) const override { |
| 3896 | auto srcVecType = |
| 3897 | llvm::dyn_cast<VectorType>(Val: insertOp.getValueToStoreType()); |
| 3898 | if (!srcVecType || insertOp.getDestVectorType().getNumElements() != |
| 3899 | srcVecType.getNumElements()) |
| 3900 | return failure(); |
| 3901 | rewriter.replaceOpWithNewOp<BroadcastOp>( |
| 3902 | op: insertOp, args: insertOp.getDestVectorType(), args: insertOp.getValueToStore()); |
| 3903 | return success(); |
| 3904 | } |
| 3905 | }; |
| 3906 | |
| 3907 | /// Pattern to rewrite a insert(splat-like(v), splat-like(v)) as broadcast(v). |
| 3908 | class InsertSplatToSplat final : public OpRewritePattern<InsertOp> { |
| 3909 | public: |
| 3910 | using Base::Base; |
| 3911 | |
| 3912 | LogicalResult matchAndRewrite(InsertOp op, |
| 3913 | PatternRewriter &rewriter) const override { |
| 3914 | |
| 3915 | Value splat = getScalarSplatSource(value: op.getValueToStore()); |
| 3916 | if (!splat || getScalarSplatSource(value: op.getDest()) != splat) |
| 3917 | return failure(); |
| 3918 | |
| 3919 | rewriter.replaceOpWithNewOp<BroadcastOp>(op, args: op.getType(), args&: splat); |
| 3920 | return success(); |
| 3921 | } |
| 3922 | }; |
| 3923 | |
| 3924 | /// Pattern to optimize a chain of insertions. |
| 3925 | /// |
| 3926 | /// This pattern identifies chains of vector.insert operations that: |
| 3927 | /// 1. Only insert values at static positions. |
| 3928 | /// 2. Completely initialize all elements in the resulting vector. |
| 3929 | /// 3. All intermediate insert operations have only one use. |
| 3930 | /// |
| 3931 | /// When these conditions are met, the entire chain can be replaced with a |
| 3932 | /// single vector.from_elements operation. |
| 3933 | /// |
| 3934 | /// To keep this pattern simple, and avoid spending too much time on matching |
| 3935 | /// fragmented insert chains, this pattern only considers the last insert op in |
| 3936 | /// the chain. |
| 3937 | /// |
| 3938 | /// Example transformation: |
| 3939 | /// %poison = ub.poison : vector<2xi32> |
| 3940 | /// %0 = vector.insert %c1, %poison[0] : i32 into vector<2xi32> |
| 3941 | /// %1 = vector.insert %c2, %0[1] : i32 into vector<2xi32> |
| 3942 | /// -> |
| 3943 | /// %result = vector.from_elements %c1, %c2 : vector<2xi32> |
| 3944 | class InsertChainFullyInitialized final : public OpRewritePattern<InsertOp> { |
| 3945 | public: |
| 3946 | using Base::Base; |
| 3947 | LogicalResult matchAndRewrite(InsertOp op, |
| 3948 | PatternRewriter &rewriter) const override { |
| 3949 | |
| 3950 | VectorType destTy = op.getDestVectorType(); |
| 3951 | if (destTy.isScalable()) |
| 3952 | return failure(); |
| 3953 | // Ensure this is the trailing vector.insert op in a chain of inserts. |
| 3954 | for (Operation *user : op.getResult().getUsers()) |
| 3955 | if (auto insertOp = dyn_cast<InsertOp>(Val: user)) |
| 3956 | if (insertOp.getDest() == op.getResult()) |
| 3957 | return failure(); |
| 3958 | |
| 3959 | InsertOp currentOp = op; |
| 3960 | SmallVector<InsertOp> chainInsertOps; |
| 3961 | while (currentOp) { |
| 3962 | // Check cond 1: Dynamic position is not supported. |
| 3963 | if (currentOp.hasDynamicPosition()) |
| 3964 | return failure(); |
| 3965 | |
| 3966 | chainInsertOps.push_back(Elt: currentOp); |
| 3967 | currentOp = currentOp.getDest().getDefiningOp<InsertOp>(); |
| 3968 | // Check cond 3: Intermediate inserts have only one use to avoid an |
| 3969 | // explosion of vectors. |
| 3970 | if (currentOp && !currentOp->hasOneUse()) |
| 3971 | return failure(); |
| 3972 | } |
| 3973 | |
| 3974 | int64_t vectorSize = destTy.getNumElements(); |
| 3975 | int64_t initializedCount = 0; |
| 3976 | SmallVector<bool> initializedDestIdxs(vectorSize, false); |
| 3977 | SmallVector<int64_t> pendingInsertPos; |
| 3978 | SmallVector<int64_t> pendingInsertSize; |
| 3979 | SmallVector<Value> pendingInsertValues; |
| 3980 | |
| 3981 | for (auto insertOp : chainInsertOps) { |
| 3982 | // This pattern can do nothing with poison index. |
| 3983 | if (is_contained(Range: insertOp.getStaticPosition(), Element: InsertOp::kPoisonIndex)) |
| 3984 | return failure(); |
| 3985 | |
| 3986 | // Calculate the linearized position for inserting elements. |
| 3987 | int64_t insertBeginPosition = |
| 3988 | calculateInsertPosition(destTy, positions: insertOp.getStaticPosition()); |
| 3989 | |
| 3990 | // The valueToStore operand may be a vector or a scalar. Need to handle |
| 3991 | // both cases. |
| 3992 | int64_t insertSize = 1; |
| 3993 | if (auto srcVectorType = |
| 3994 | llvm::dyn_cast<VectorType>(Val: insertOp.getValueToStoreType())) |
| 3995 | insertSize = srcVectorType.getNumElements(); |
| 3996 | |
| 3997 | assert(insertBeginPosition + insertSize <= vectorSize && |
| 3998 | "insert would overflow the vector" ); |
| 3999 | |
| 4000 | for (auto index : llvm::seq<int64_t>(Begin: insertBeginPosition, |
| 4001 | End: insertBeginPosition + insertSize)) { |
| 4002 | if (initializedDestIdxs[index]) |
| 4003 | continue; |
| 4004 | initializedDestIdxs[index] = true; |
| 4005 | ++initializedCount; |
| 4006 | } |
| 4007 | |
| 4008 | // Defer the creation of ops before we can make sure the pattern can |
| 4009 | // succeed. |
| 4010 | pendingInsertPos.push_back(Elt: insertBeginPosition); |
| 4011 | pendingInsertSize.push_back(Elt: insertSize); |
| 4012 | pendingInsertValues.push_back(Elt: insertOp.getValueToStore()); |
| 4013 | |
| 4014 | if (initializedCount == vectorSize) |
| 4015 | break; |
| 4016 | } |
| 4017 | |
| 4018 | // Check cond 2: all positions must be initialized. |
| 4019 | if (initializedCount != vectorSize) |
| 4020 | return failure(); |
| 4021 | |
| 4022 | SmallVector<Value> elements(vectorSize); |
| 4023 | for (auto [insertBeginPosition, insertSize, valueToStore] : |
| 4024 | llvm::reverse(C: llvm::zip(t&: pendingInsertPos, u&: pendingInsertSize, |
| 4025 | args&: pendingInsertValues))) { |
| 4026 | auto srcVectorType = llvm::dyn_cast<VectorType>(Val: valueToStore.getType()); |
| 4027 | |
| 4028 | if (!srcVectorType) { |
| 4029 | elements[insertBeginPosition] = valueToStore; |
| 4030 | continue; |
| 4031 | } |
| 4032 | |
| 4033 | Repeated<Type> elementToInsertTypes(insertSize, |
| 4034 | srcVectorType.getElementType()); |
| 4035 | // Get all elements from the vector in row-major order. |
| 4036 | auto elementsToInsert = vector::ToElementsOp::create( |
| 4037 | builder&: rewriter, location: op.getLoc(), elements: elementToInsertTypes, source: valueToStore); |
| 4038 | for (int64_t linearIdx = 0; linearIdx < insertSize; linearIdx++) { |
| 4039 | elements[insertBeginPosition + linearIdx] = |
| 4040 | elementsToInsert.getResult(i: linearIdx); |
| 4041 | } |
| 4042 | } |
| 4043 | |
| 4044 | rewriter.replaceOpWithNewOp<vector::FromElementsOp>(op, args&: destTy, args&: elements); |
| 4045 | return success(); |
| 4046 | } |
| 4047 | }; |
| 4048 | |
| 4049 | } // namespace |
| 4050 | |
| 4051 | static Attribute |
| 4052 | foldDenseElementsAttrDestInsertOp(InsertOp insertOp, Attribute srcAttr, |
| 4053 | Attribute dstAttr, |
| 4054 | int64_t maxVectorSizeFoldThreshold) { |
| 4055 | if (insertOp.hasDynamicPosition()) |
| 4056 | return {}; |
| 4057 | |
| 4058 | auto denseDst = llvm::dyn_cast_if_present<DenseElementsAttr>(Val&: dstAttr); |
| 4059 | if (!denseDst) |
| 4060 | return {}; |
| 4061 | |
| 4062 | if (!srcAttr) { |
| 4063 | return {}; |
| 4064 | } |
| 4065 | |
| 4066 | VectorType destTy = insertOp.getDestVectorType(); |
| 4067 | if (destTy.isScalable()) |
| 4068 | return {}; |
| 4069 | |
| 4070 | // Make sure we do not create too many large constants. |
| 4071 | if (destTy.getNumElements() > maxVectorSizeFoldThreshold && |
| 4072 | !insertOp->hasOneUse()) |
| 4073 | return {}; |
| 4074 | |
| 4075 | // Bail out on poison indices (kPoisonIndex = -1) to avoid computing an |
| 4076 | // invalid (negative) linearized position which would cause UB below. |
| 4077 | if (is_contained(Range: insertOp.getStaticPosition(), Element: InsertOp::kPoisonIndex)) |
| 4078 | return {}; |
| 4079 | |
| 4080 | // Calculate the linearized position for inserting elements. |
| 4081 | int64_t insertBeginPosition = |
| 4082 | calculateInsertPosition(destTy, positions: insertOp.getStaticPosition()); |
| 4083 | SmallVector<Attribute> insertedValues; |
| 4084 | Type destEltType = destTy.getElementType(); |
| 4085 | |
| 4086 | /// Converts attribute to the expected type if there's |
| 4087 | /// a mismatch. |
| 4088 | if (auto denseSource = llvm::dyn_cast<DenseElementsAttr>(Val&: srcAttr)) { |
| 4089 | for (auto value : denseSource.getValues<Attribute>()) |
| 4090 | insertedValues.push_back(Elt: convertNumericAttr(attr: value, expectedType: destEltType)); |
| 4091 | } else { |
| 4092 | insertedValues.push_back(Elt: convertNumericAttr(attr: srcAttr, expectedType: destEltType)); |
| 4093 | } |
| 4094 | |
| 4095 | auto allValues = llvm::to_vector(Range: denseDst.getValues<Attribute>()); |
| 4096 | copy(Range&: insertedValues, Out: allValues.begin() + insertBeginPosition); |
| 4097 | auto newAttr = DenseElementsAttr::get(type: destTy, values: allValues); |
| 4098 | |
| 4099 | return newAttr; |
| 4100 | } |
| 4101 | |
| 4102 | /// Folder to replace the `dest` operand of the insert op with the root dest of |
| 4103 | /// the insert op use chain. |
| 4104 | static Value foldInsertUseChain(InsertOp insertOp) { |
| 4105 | auto destInsert = insertOp.getDest().getDefiningOp<InsertOp>(); |
| 4106 | if (!destInsert) |
| 4107 | return {}; |
| 4108 | |
| 4109 | if (insertOp.getMixedPosition() != destInsert.getMixedPosition()) |
| 4110 | return {}; |
| 4111 | |
| 4112 | insertOp.setOperand(i: 1, value: destInsert.getDest()); |
| 4113 | return insertOp.getResult(); |
| 4114 | } |
| 4115 | |
| 4116 | void InsertOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 4117 | MLIRContext *context) { |
| 4118 | results.add<InsertToBroadcast, BroadcastFolder, InsertSplatToSplat, |
| 4119 | InsertChainFullyInitialized>(arg&: context); |
| 4120 | } |
| 4121 | |
| 4122 | OpFoldResult InsertOp::fold(FoldAdaptor adaptor) { |
| 4123 | // Do not create constants with more than `vectorSizeFoldThreashold` elements, |
| 4124 | // unless the source vector constant has a single use. |
| 4125 | constexpr int64_t vectorSizeFoldThreshold = 256; |
| 4126 | // Fold "vector.insert %v, %dest [] : vector<2x2xf32> from vector<2x2xf32>" to |
| 4127 | // %v. Note: Do not fold "vector.insert %v, %dest [] : f32 into vector<f32>" |
| 4128 | // (type mismatch). |
| 4129 | if (getNumIndices() == 0 && getValueToStoreType() == getType()) |
| 4130 | return getValueToStore(); |
| 4131 | // Fold `arith.constant` indices into the `vector.insert` operation. |
| 4132 | // Do not stop here as this fold may enable subsequent folds that require |
| 4133 | // constant indices. |
| 4134 | SmallVector<Value> operands = {getValueToStore(), getDest()}; |
| 4135 | auto inplaceFolded = extractInsertFoldConstantOp(op: *this, adaptor, operands); |
| 4136 | |
| 4137 | if (auto res = foldInsertUseChain(insertOp: *this)) |
| 4138 | return res; |
| 4139 | if (auto res = foldPoisonIndexInsertExtractOp( |
| 4140 | context: getContext(), staticPos: adaptor.getStaticPosition(), poisonVal: kPoisonIndex)) |
| 4141 | return res; |
| 4142 | if (auto res = foldDenseElementsAttrDestInsertOp( |
| 4143 | insertOp: *this, srcAttr: adaptor.getValueToStore(), dstAttr: adaptor.getDest(), |
| 4144 | maxVectorSizeFoldThreshold: vectorSizeFoldThreshold)) { |
| 4145 | return res; |
| 4146 | } |
| 4147 | |
| 4148 | return inplaceFolded; |
| 4149 | } |
| 4150 | |
| 4151 | //===----------------------------------------------------------------------===// |
| 4152 | // InsertStridedSliceOp |
| 4153 | //===----------------------------------------------------------------------===// |
| 4154 | |
| 4155 | void InsertStridedSliceOp::build(OpBuilder &builder, OperationState &result, |
| 4156 | Value source, Value dest, |
| 4157 | ArrayRef<int64_t> offsets, |
| 4158 | ArrayRef<int64_t> strides) { |
| 4159 | result.addOperands(newOperands: {source, dest}); |
| 4160 | auto offsetsAttr = getVectorSubscriptAttr(builder, values: offsets); |
| 4161 | auto stridesAttr = getVectorSubscriptAttr(builder, values: strides); |
| 4162 | result.addTypes(newTypes: dest.getType()); |
| 4163 | result.addAttribute(name: InsertStridedSliceOp::getOffsetsAttrName(name: result.name), |
| 4164 | attr: offsetsAttr); |
| 4165 | result.addAttribute(name: InsertStridedSliceOp::getStridesAttrName(name: result.name), |
| 4166 | attr: stridesAttr); |
| 4167 | } |
| 4168 | |
| 4169 | // TODO: Should be moved to Tablegen ConfinedAttr attributes. |
| 4170 | template <typename OpType> |
| 4171 | static LogicalResult isIntegerArrayAttrSmallerThanShape(OpType op, |
| 4172 | ArrayAttr arrayAttr, |
| 4173 | ArrayRef<int64_t> shape, |
| 4174 | StringRef attrName) { |
| 4175 | if (arrayAttr.size() > shape.size()) |
| 4176 | return op.emitOpError("expected " ) |
| 4177 | << attrName << " attribute of rank no greater than vector rank" ; |
| 4178 | return success(); |
| 4179 | } |
| 4180 | |
| 4181 | // Returns true if all integers in `arrayAttr` are in the half-open [min, max} |
| 4182 | // interval. If `halfOpen` is true then the admissible interval is [min, max). |
| 4183 | // Otherwise, the admissible interval is [min, max]. |
| 4184 | template <typename OpType> |
| 4185 | static LogicalResult |
| 4186 | isIntegerArrayAttrConfinedToRange(OpType op, ArrayAttr arrayAttr, int64_t min, |
| 4187 | int64_t max, StringRef attrName, |
| 4188 | bool halfOpen = true) { |
| 4189 | for (auto attr : arrayAttr) { |
| 4190 | auto val = llvm::cast<IntegerAttr>(Val&: attr).getInt(); |
| 4191 | auto upper = max; |
| 4192 | if (!halfOpen) |
| 4193 | upper += 1; |
| 4194 | if (val < min || val >= upper) |
| 4195 | return op.emitOpError("expected " ) << attrName << " to be confined to [" |
| 4196 | << min << ", " << upper << ")" ; |
| 4197 | } |
| 4198 | return success(); |
| 4199 | } |
| 4200 | |
| 4201 | // Returns true if all integers in `arrayAttr` are in the half-open [min, max} |
| 4202 | // interval. If `halfOpen` is true then the admissible interval is [min, max). |
| 4203 | // Otherwise, the admissible interval is [min, max]. |
| 4204 | template <typename OpType> |
| 4205 | static LogicalResult |
| 4206 | isIntegerArrayAttrConfinedToShape(OpType op, ArrayAttr arrayAttr, |
| 4207 | ArrayRef<int64_t> shape, StringRef attrName, |
| 4208 | bool halfOpen = true, int64_t min = 0) { |
| 4209 | for (auto [index, attrDimPair] : |
| 4210 | llvm::enumerate(First: llvm::zip_first(t&: arrayAttr, u&: shape))) { |
| 4211 | int64_t val = llvm::cast<IntegerAttr>(Val: std::get<0>(t&: attrDimPair)).getInt(); |
| 4212 | int64_t max = std::get<1>(t&: attrDimPair); |
| 4213 | if (!halfOpen) |
| 4214 | max += 1; |
| 4215 | if (val < min || val >= max) |
| 4216 | return op.emitOpError("expected " ) |
| 4217 | << attrName << " dimension " << index << " to be confined to [" |
| 4218 | << min << ", " << max << ")" ; |
| 4219 | } |
| 4220 | return success(); |
| 4221 | } |
| 4222 | |
| 4223 | // Returns true if, for all indices i = 0..shape.size()-1, val is in the |
| 4224 | // [min, max} interval: |
| 4225 | // val = `arrayAttr1[i]` + `arrayAttr2[i]`, |
| 4226 | // If `halfOpen` is true then the admissible interval is [min, max). Otherwise, |
| 4227 | // the admissible interval is [min, max]. |
| 4228 | template <typename OpType> |
| 4229 | static LogicalResult isSumOfIntegerArrayAttrConfinedToShape( |
| 4230 | OpType op, ArrayAttr arrayAttr1, ArrayAttr arrayAttr2, |
| 4231 | ArrayRef<int64_t> shape, StringRef attrName1, StringRef attrName2, |
| 4232 | bool halfOpen = true, int64_t min = 1) { |
| 4233 | assert(arrayAttr1.size() <= shape.size()); |
| 4234 | assert(arrayAttr2.size() <= shape.size()); |
| 4235 | for (auto [index, it] : |
| 4236 | llvm::enumerate(First: llvm::zip(t&: arrayAttr1, u&: arrayAttr2, args&: shape))) { |
| 4237 | auto val1 = llvm::cast<IntegerAttr>(Val: std::get<0>(t&: it)).getInt(); |
| 4238 | auto val2 = llvm::cast<IntegerAttr>(Val: std::get<1>(t&: it)).getInt(); |
| 4239 | int64_t max = std::get<2>(t&: it); |
| 4240 | if (!halfOpen) |
| 4241 | max += 1; |
| 4242 | if (val1 + val2 < 0 || val1 + val2 >= max) |
| 4243 | return op.emitOpError("expected sum(" ) |
| 4244 | << attrName1 << ", " << attrName2 << ") dimension " << index |
| 4245 | << " to be confined to [" << min << ", " << max << ")" ; |
| 4246 | } |
| 4247 | return success(); |
| 4248 | } |
| 4249 | |
| 4250 | static ArrayAttr makeI64ArrayAttr(ArrayRef<int64_t> values, |
| 4251 | MLIRContext *context) { |
| 4252 | auto attrs = llvm::map_range(C&: values, F: [context](int64_t v) -> Attribute { |
| 4253 | return IntegerAttr::get(type: IntegerType::get(context, width: 64), value: APInt(64, v)); |
| 4254 | }); |
| 4255 | return ArrayAttr::get(context, value: llvm::to_vector<8>(Range&: attrs)); |
| 4256 | } |
| 4257 | |
| 4258 | LogicalResult InsertStridedSliceOp::verify() { |
| 4259 | auto sourceVectorType = getSourceVectorType(); |
| 4260 | auto destVectorType = getDestVectorType(); |
| 4261 | auto offsets = getOffsetsAttr(); |
| 4262 | auto strides = getStridesAttr(); |
| 4263 | if (offsets.size() != static_cast<unsigned>(destVectorType.getRank())) |
| 4264 | return emitOpError( |
| 4265 | message: "expected offsets of same size as destination vector rank" ); |
| 4266 | if (strides.size() != static_cast<unsigned>(sourceVectorType.getRank())) |
| 4267 | return emitOpError(message: "expected strides of same size as source vector rank" ); |
| 4268 | if (sourceVectorType.getRank() > destVectorType.getRank()) |
| 4269 | return emitOpError( |
| 4270 | message: "expected source rank to be no greater than destination rank" ); |
| 4271 | |
| 4272 | auto sourceShape = sourceVectorType.getShape(); |
| 4273 | auto destShape = destVectorType.getShape(); |
| 4274 | SmallVector<int64_t, 4> sourceShapeAsDestShape( |
| 4275 | destShape.size() - sourceShape.size(), 0); |
| 4276 | sourceShapeAsDestShape.append(in_start: sourceShape.begin(), in_end: sourceShape.end()); |
| 4277 | auto offName = InsertStridedSliceOp::getOffsetsAttrName(); |
| 4278 | auto stridesName = InsertStridedSliceOp::getStridesAttrName(); |
| 4279 | if (failed(Result: isIntegerArrayAttrConfinedToShape(op: *this, arrayAttr: offsets, shape: destShape, |
| 4280 | attrName: offName)) || |
| 4281 | failed(Result: isIntegerArrayAttrConfinedToRange(op: *this, arrayAttr: strides, /*min=*/1, |
| 4282 | /*max=*/1, attrName: stridesName, |
| 4283 | /*halfOpen=*/false)) || |
| 4284 | failed(Result: isSumOfIntegerArrayAttrConfinedToShape( |
| 4285 | op: *this, arrayAttr1: offsets, |
| 4286 | arrayAttr2: makeI64ArrayAttr(values: sourceShapeAsDestShape, context: getContext()), shape: destShape, |
| 4287 | attrName1: offName, attrName2: "source vector shape" , |
| 4288 | /*halfOpen=*/false, /*min=*/1))) |
| 4289 | return failure(); |
| 4290 | |
| 4291 | unsigned rankDiff = destShape.size() - sourceShape.size(); |
| 4292 | for (unsigned idx = 0; idx < sourceShape.size(); ++idx) { |
| 4293 | if (sourceVectorType.getScalableDims()[idx] != |
| 4294 | destVectorType.getScalableDims()[idx + rankDiff]) { |
| 4295 | return emitOpError(message: "mismatching scalable flags (at source vector idx=" ) |
| 4296 | << idx << ")" ; |
| 4297 | } |
| 4298 | if (sourceVectorType.getScalableDims()[idx]) { |
| 4299 | auto sourceSize = sourceShape[idx]; |
| 4300 | auto destSize = destShape[idx + rankDiff]; |
| 4301 | if (sourceSize != destSize) { |
| 4302 | return emitOpError(message: "expected size at idx=" ) |
| 4303 | << idx |
| 4304 | << (" to match the corresponding base size from the input " |
| 4305 | "vector (" ) |
| 4306 | << sourceSize << (" vs " ) << destSize << (")" ); |
| 4307 | } |
| 4308 | } |
| 4309 | } |
| 4310 | |
| 4311 | return success(); |
| 4312 | } |
| 4313 | |
| 4314 | namespace { |
| 4315 | /// Rewrite insert_strided_slice(splat-like(v), splat-like(v)) as v. |
| 4316 | class FoldInsertStridedSliceSplat final |
| 4317 | : public OpRewritePattern<InsertStridedSliceOp> { |
| 4318 | public: |
| 4319 | using Base::Base; |
| 4320 | |
| 4321 | LogicalResult matchAndRewrite(InsertStridedSliceOp insertStridedSliceOp, |
| 4322 | PatternRewriter &rewriter) const override { |
| 4323 | |
| 4324 | auto dst = insertStridedSliceOp.getDest(); |
| 4325 | auto splat = getScalarSplatSource(value: insertStridedSliceOp.getValueToStore()); |
| 4326 | if (!splat || getScalarSplatSource(value: dst) != splat) |
| 4327 | return failure(); |
| 4328 | |
| 4329 | rewriter.replaceOp(op: insertStridedSliceOp, newValues: dst); |
| 4330 | return success(); |
| 4331 | } |
| 4332 | }; |
| 4333 | |
| 4334 | /// Pattern to rewrite an InsertStridedSliceOp(ExtractStridedSliceOp(dst), dst) |
| 4335 | /// to dst. |
| 4336 | class final |
| 4337 | : public OpRewritePattern<InsertStridedSliceOp> { |
| 4338 | public: |
| 4339 | using Base::Base; |
| 4340 | |
| 4341 | LogicalResult matchAndRewrite(InsertStridedSliceOp insertStridedSliceOp, |
| 4342 | PatternRewriter &rewriter) const override { |
| 4343 | auto = |
| 4344 | insertStridedSliceOp.getValueToStore() |
| 4345 | .getDefiningOp<vector::ExtractStridedSliceOp>(); |
| 4346 | |
| 4347 | if (!extractStridedSliceOp) |
| 4348 | return failure(); |
| 4349 | |
| 4350 | if (extractStridedSliceOp.getOperand() != insertStridedSliceOp.getDest()) |
| 4351 | return failure(); |
| 4352 | |
| 4353 | // Check if have the same strides and offsets. |
| 4354 | if (extractStridedSliceOp.getStrides() != |
| 4355 | insertStridedSliceOp.getStrides() || |
| 4356 | extractStridedSliceOp.getOffsets() != insertStridedSliceOp.getOffsets()) |
| 4357 | return failure(); |
| 4358 | |
| 4359 | rewriter.replaceOp(op: insertStridedSliceOp, newValues: insertStridedSliceOp.getDest()); |
| 4360 | return success(); |
| 4361 | } |
| 4362 | }; |
| 4363 | |
| 4364 | // Pattern to rewrite an InsertStridedSliceOp(ConstantOp into ConstantOp) -> |
| 4365 | // ConstantOp. |
| 4366 | class InsertStridedSliceConstantFolder final |
| 4367 | : public OpRewritePattern<InsertStridedSliceOp> { |
| 4368 | public: |
| 4369 | using Base::Base; |
| 4370 | |
| 4371 | // Do not create constants with more than `vectorSizeFoldThreashold` elements, |
| 4372 | // unless the source vector constant has a single use. |
| 4373 | static constexpr int64_t vectorSizeFoldThreshold = 256; |
| 4374 | |
| 4375 | LogicalResult matchAndRewrite(InsertStridedSliceOp op, |
| 4376 | PatternRewriter &rewriter) const override { |
| 4377 | // Return if 'InsertOp' operand is not defined by a compatible vector |
| 4378 | // ConstantOp. |
| 4379 | TypedValue<VectorType> destVector = op.getDest(); |
| 4380 | Attribute vectorDestCst; |
| 4381 | if (!matchPattern(value: destVector, pattern: m_Constant(bind_value: &vectorDestCst))) |
| 4382 | return failure(); |
| 4383 | |
| 4384 | VectorType destTy = destVector.getType(); |
| 4385 | if (destTy.isScalable()) |
| 4386 | return failure(); |
| 4387 | |
| 4388 | // Make sure we do not create too many large constants. |
| 4389 | if (destTy.getNumElements() > vectorSizeFoldThreshold && |
| 4390 | !destVector.hasOneUse()) |
| 4391 | return failure(); |
| 4392 | |
| 4393 | TypedValue<VectorType> sourceValue = op.getValueToStore(); |
| 4394 | Attribute sourceCst; |
| 4395 | if (!matchPattern(value: sourceValue, pattern: m_Constant(bind_value: &sourceCst))) |
| 4396 | return failure(); |
| 4397 | |
| 4398 | // TODO: Support poison. |
| 4399 | if (matchPattern(attr: vectorDestCst, pattern: ub::m_Poison()) || |
| 4400 | matchPattern(attr: sourceCst, pattern: ub::m_Poison())) |
| 4401 | return failure(); |
| 4402 | |
| 4403 | // TODO: Handle non-unit strides when they become available. |
| 4404 | if (op.hasNonUnitStrides()) |
| 4405 | return failure(); |
| 4406 | |
| 4407 | VectorType sliceVecTy = sourceValue.getType(); |
| 4408 | ArrayRef<int64_t> sliceShape = sliceVecTy.getShape(); |
| 4409 | int64_t rankDifference = destTy.getRank() - sliceVecTy.getRank(); |
| 4410 | SmallVector<int64_t, 4> offsets = getI64SubArray(arrayAttr: op.getOffsets()); |
| 4411 | SmallVector<int64_t, 4> destStrides = computeStrides(sizes: destTy.getShape()); |
| 4412 | |
| 4413 | // Calcualte the destination element indices by enumerating all slice |
| 4414 | // positions within the destination and linearizing them. The enumeration |
| 4415 | // order is lexicographic which yields a sequence of monotonically |
| 4416 | // increasing linearized position indices. |
| 4417 | // Because the destination may have higher dimensionality then the slice, |
| 4418 | // we keep track of two overlapping sets of positions and offsets. |
| 4419 | auto denseDest = llvm::cast<DenseElementsAttr>(Val&: vectorDestCst); |
| 4420 | auto denseSlice = llvm::cast<DenseElementsAttr>(Val&: sourceCst); |
| 4421 | auto sliceValuesIt = denseSlice.value_begin<Attribute>(); |
| 4422 | auto newValues = llvm::to_vector(Range: denseDest.getValues<Attribute>()); |
| 4423 | SmallVector<int64_t> currDestPosition(offsets.begin(), offsets.end()); |
| 4424 | MutableArrayRef<int64_t> currSlicePosition( |
| 4425 | currDestPosition.begin() + rankDifference, currDestPosition.end()); |
| 4426 | ArrayRef<int64_t> sliceOffsets(offsets.begin() + rankDifference, |
| 4427 | offsets.end()); |
| 4428 | do { |
| 4429 | int64_t linearizedPosition = linearize(offsets: currDestPosition, basis: destStrides); |
| 4430 | assert(linearizedPosition < destTy.getNumElements() && "Invalid index" ); |
| 4431 | assert(sliceValuesIt != denseSlice.value_end<Attribute>() && |
| 4432 | "Invalid slice element" ); |
| 4433 | newValues[linearizedPosition] = *sliceValuesIt; |
| 4434 | ++sliceValuesIt; |
| 4435 | } while (succeeded( |
| 4436 | Result: incSlicePosition(position: currSlicePosition, shape: sliceShape, offsets: sliceOffsets))); |
| 4437 | |
| 4438 | auto newAttr = DenseElementsAttr::get(type: destTy, values: newValues); |
| 4439 | rewriter.replaceOpWithNewOp<arith::ConstantOp>(op, args&: newAttr); |
| 4440 | return success(); |
| 4441 | } |
| 4442 | }; |
| 4443 | |
| 4444 | } // namespace |
| 4445 | |
| 4446 | void vector::InsertStridedSliceOp::getCanonicalizationPatterns( |
| 4447 | RewritePatternSet &results, MLIRContext *context) { |
| 4448 | results.add<FoldInsertStridedSliceSplat, FoldInsertStridedSliceOfExtract, |
| 4449 | InsertStridedSliceConstantFolder>(arg&: context); |
| 4450 | } |
| 4451 | |
| 4452 | OpFoldResult InsertStridedSliceOp::fold(FoldAdaptor adaptor) { |
| 4453 | if (getSourceVectorType() == getDestVectorType()) |
| 4454 | return getValueToStore(); |
| 4455 | return {}; |
| 4456 | } |
| 4457 | |
| 4458 | //===----------------------------------------------------------------------===// |
| 4459 | // OuterProductOp |
| 4460 | //===----------------------------------------------------------------------===// |
| 4461 | |
| 4462 | /// Build an op without mask, use the type of `acc` as the return type. |
| 4463 | void OuterProductOp::build(OpBuilder &builder, OperationState &result, |
| 4464 | Value lhs, Value rhs, Value acc) { |
| 4465 | result.addOperands(newOperands: {lhs, rhs, acc}); |
| 4466 | result.addTypes(newTypes: acc.getType()); |
| 4467 | } |
| 4468 | |
| 4469 | void OuterProductOp::print(OpAsmPrinter &p) { |
| 4470 | p << " " << getLhs() << ", " << getRhs(); |
| 4471 | if (getAcc()) { |
| 4472 | p << ", " << getAcc(); |
| 4473 | SmallVector<NamedAttribute> attrs((*this)->getDiscardableAttrs()); |
| 4474 | attrs.emplace_back(Args: getKindAttrName(), Args: getKindAttr()); |
| 4475 | llvm::sort(C&: attrs); |
| 4476 | p.printOptionalAttrDict(attrs); |
| 4477 | } |
| 4478 | p << " : " << getLhs().getType() << ", " << getRhs().getType(); |
| 4479 | } |
| 4480 | |
| 4481 | ParseResult OuterProductOp::parse(OpAsmParser &parser, OperationState &result) { |
| 4482 | SmallVector<OpAsmParser::UnresolvedOperand, 3> operandsInfo; |
| 4483 | Type tLHS, tRHS; |
| 4484 | if (parser.parseOperandList(result&: operandsInfo) || |
| 4485 | parser.parseOptionalAttrDict(result&: result.attributes) || |
| 4486 | parser.parseColonType(result&: tLHS) || parser.parseComma() || |
| 4487 | parser.parseType(result&: tRHS)) |
| 4488 | return failure(); |
| 4489 | if (operandsInfo.size() < 2) |
| 4490 | return parser.emitError(loc: parser.getNameLoc(), |
| 4491 | message: "expected at least 2 operands" ); |
| 4492 | VectorType vLHS = llvm::dyn_cast<VectorType>(Val&: tLHS); |
| 4493 | VectorType vRHS = llvm::dyn_cast<VectorType>(Val&: tRHS); |
| 4494 | if (!vLHS) |
| 4495 | return parser.emitError(loc: parser.getNameLoc(), |
| 4496 | message: "expected vector type for operand #1" ); |
| 4497 | // The result type is built below from dimension 0 of the operands, which a |
| 4498 | // 0-d vector does not have. Only that case has to be caught here; a higher |
| 4499 | // rank still reaches the verifier, which rejects it with the same wording. |
| 4500 | if (vLHS.getRank() == 0) |
| 4501 | return parser.emitError(loc: parser.getNameLoc(), |
| 4502 | message: "expected 1-d vector for operand #1" ); |
| 4503 | if (vRHS && vRHS.getRank() == 0) |
| 4504 | return parser.emitError(loc: parser.getNameLoc(), |
| 4505 | message: "expected 1-d vector for operand #2" ); |
| 4506 | |
| 4507 | VectorType resType; |
| 4508 | if (vRHS) { |
| 4509 | SmallVector<bool> scalableDimsRes{vLHS.getScalableDims()[0], |
| 4510 | vRHS.getScalableDims()[0]}; |
| 4511 | resType = VectorType::get(shape: {vLHS.getDimSize(idx: 0), vRHS.getDimSize(idx: 0)}, |
| 4512 | elementType: vLHS.getElementType(), scalableDims: scalableDimsRes); |
| 4513 | } else { |
| 4514 | // Scalar RHS operand |
| 4515 | SmallVector<bool> scalableDimsRes{vLHS.getScalableDims()[0]}; |
| 4516 | resType = VectorType::get(shape: {vLHS.getDimSize(idx: 0)}, elementType: vLHS.getElementType(), |
| 4517 | scalableDims: scalableDimsRes); |
| 4518 | } |
| 4519 | |
| 4520 | if (!result.attributes.get(name: OuterProductOp::getKindAttrName(name: result.name))) { |
| 4521 | result.attributes.append( |
| 4522 | name: OuterProductOp::getKindAttrName(name: result.name), |
| 4523 | attr: CombiningKindAttr::get(context: result.getContext(), |
| 4524 | value: OuterProductOp::getDefaultKind())); |
| 4525 | } |
| 4526 | |
| 4527 | return failure( |
| 4528 | IsFailure: parser.resolveOperand(operand: operandsInfo[0], type: tLHS, result&: result.operands) || |
| 4529 | parser.resolveOperand(operand: operandsInfo[1], type: tRHS, result&: result.operands) || |
| 4530 | (operandsInfo.size() > 2 && |
| 4531 | parser.resolveOperand(operand: operandsInfo[2], type: resType, result&: result.operands)) || |
| 4532 | parser.addTypeToList(type: resType, result&: result.types)); |
| 4533 | } |
| 4534 | |
| 4535 | LogicalResult OuterProductOp::verify() { |
| 4536 | Type tRHS = getOperandTypeRHS(); |
| 4537 | VectorType vLHS = getOperandVectorTypeLHS(), |
| 4538 | vRHS = llvm::dyn_cast<VectorType>(Val&: tRHS), |
| 4539 | vACC = getOperandVectorTypeACC(), vRES = getResultVectorType(); |
| 4540 | |
| 4541 | if (vLHS.getRank() != 1) |
| 4542 | return emitOpError(message: "expected 1-d vector for operand #1" ); |
| 4543 | |
| 4544 | if (vRHS) { |
| 4545 | // Proper OUTER operation. |
| 4546 | if (vRHS.getRank() != 1) |
| 4547 | return emitOpError(message: "expected 1-d vector for operand #2" ); |
| 4548 | if (vRES.getRank() != 2) |
| 4549 | return emitOpError(message: "expected 2-d vector result" ); |
| 4550 | if (vLHS.getDimSize(idx: 0) != vRES.getDimSize(idx: 0)) |
| 4551 | return emitOpError(message: "expected #1 operand dim to match result dim #1" ); |
| 4552 | if (vRHS.getDimSize(idx: 0) != vRES.getDimSize(idx: 1)) |
| 4553 | return emitOpError(message: "expected #2 operand dim to match result dim #2" ); |
| 4554 | if (vLHS.isScalable() && !vRHS.isScalable()) { |
| 4555 | // This restriction reflects what's currently supported in terms of |
| 4556 | // scalable vectors. However, we could relax this if there's a use case. |
| 4557 | return emitOpError( |
| 4558 | message: "expected either both or only #2 operand dim to be scalable" ); |
| 4559 | } |
| 4560 | } else { |
| 4561 | // An AXPY operation. |
| 4562 | if (vRES.getRank() != 1) |
| 4563 | return emitOpError(message: "expected 1-d vector result" ); |
| 4564 | if (vLHS.getDimSize(idx: 0) != vRES.getDimSize(idx: 0)) |
| 4565 | return emitOpError(message: "expected #1 operand dim to match result dim #1" ); |
| 4566 | } |
| 4567 | |
| 4568 | if (vACC && vACC != vRES) |
| 4569 | return emitOpError(message: "expected operand #3 of same type as result type" ); |
| 4570 | |
| 4571 | if (!getKindAttr()) { |
| 4572 | return emitOpError(message: "expected 'kind' attribute of type CombiningKind (e.g. " |
| 4573 | "'vector.kind<add>')" ); |
| 4574 | } |
| 4575 | |
| 4576 | // Verify supported combining kind. |
| 4577 | if (!isSupportedCombiningKind(combiningKind: getKind(), elementType: vRES.getElementType())) |
| 4578 | return emitOpError(message: "unsupported outerproduct type" ); |
| 4579 | |
| 4580 | return success(); |
| 4581 | } |
| 4582 | |
| 4583 | // MaskableOpInterface methods. |
| 4584 | |
| 4585 | /// Returns the mask type expected by this operation. Mostly used for |
| 4586 | /// verification purposes. It requires the operation to be vectorized." |
| 4587 | Type OuterProductOp::getExpectedMaskType() { |
| 4588 | auto vecType = this->getResultVectorType(); |
| 4589 | return VectorType::get(shape: vecType.getShape(), |
| 4590 | elementType: IntegerType::get(context: vecType.getContext(), /*width=*/1), |
| 4591 | scalableDims: vecType.getScalableDims()); |
| 4592 | } |
| 4593 | |
| 4594 | //===----------------------------------------------------------------------===// |
| 4595 | // ExtractStridedSliceOp |
| 4596 | //===----------------------------------------------------------------------===// |
| 4597 | |
| 4598 | // Inference works as follows: |
| 4599 | // 1. Add 'sizes' from prefix of dims in 'offsets'. |
| 4600 | // 2. Add sizes from 'vectorType' for remaining dims. |
| 4601 | // Scalable flags are inherited from 'vectorType'. |
| 4602 | static Type inferStridedSliceOpResultType(VectorType vectorType, |
| 4603 | ArrayAttr offsets, ArrayAttr sizes, |
| 4604 | ArrayAttr strides) { |
| 4605 | assert(offsets.size() == sizes.size() && offsets.size() == strides.size()); |
| 4606 | SmallVector<int64_t, 4> shape; |
| 4607 | shape.reserve(N: vectorType.getRank()); |
| 4608 | unsigned idx = 0; |
| 4609 | for (unsigned e = offsets.size(); idx < e; ++idx) |
| 4610 | shape.push_back(Elt: llvm::cast<IntegerAttr>(Val: sizes[idx]).getInt()); |
| 4611 | for (unsigned e = vectorType.getShape().size(); idx < e; ++idx) |
| 4612 | shape.push_back(Elt: vectorType.getShape()[idx]); |
| 4613 | |
| 4614 | return VectorType::get(shape, elementType: vectorType.getElementType(), |
| 4615 | scalableDims: vectorType.getScalableDims()); |
| 4616 | } |
| 4617 | |
| 4618 | void ExtractStridedSliceOp::(OpBuilder &builder, OperationState &result, |
| 4619 | Value source, ArrayRef<int64_t> offsets, |
| 4620 | ArrayRef<int64_t> sizes, |
| 4621 | ArrayRef<int64_t> strides) { |
| 4622 | result.addOperands(newOperands: source); |
| 4623 | auto offsetsAttr = getVectorSubscriptAttr(builder, values: offsets); |
| 4624 | auto sizesAttr = getVectorSubscriptAttr(builder, values: sizes); |
| 4625 | auto stridesAttr = getVectorSubscriptAttr(builder, values: strides); |
| 4626 | result.addTypes( |
| 4627 | newTypes: inferStridedSliceOpResultType(vectorType: llvm::cast<VectorType>(Val: source.getType()), |
| 4628 | offsets: offsetsAttr, sizes: sizesAttr, strides: stridesAttr)); |
| 4629 | result.addAttribute(name: ExtractStridedSliceOp::getOffsetsAttrName(name: result.name), |
| 4630 | attr: offsetsAttr); |
| 4631 | result.addAttribute(name: ExtractStridedSliceOp::getSizesAttrName(name: result.name), |
| 4632 | attr: sizesAttr); |
| 4633 | result.addAttribute(name: ExtractStridedSliceOp::getStridesAttrName(name: result.name), |
| 4634 | attr: stridesAttr); |
| 4635 | } |
| 4636 | |
| 4637 | LogicalResult ExtractStridedSliceOp::() { |
| 4638 | auto type = getSourceVectorType(); |
| 4639 | auto offsets = getOffsetsAttr(); |
| 4640 | auto sizes = getSizesAttr(); |
| 4641 | auto strides = getStridesAttr(); |
| 4642 | if (offsets.size() != sizes.size() || offsets.size() != strides.size()) |
| 4643 | return emitOpError( |
| 4644 | message: "expected offsets, sizes and strides attributes of same size" ); |
| 4645 | |
| 4646 | auto shape = type.getShape(); |
| 4647 | auto offName = getOffsetsAttrName(); |
| 4648 | auto sizesName = getSizesAttrName(); |
| 4649 | auto stridesName = getStridesAttrName(); |
| 4650 | if (failed( |
| 4651 | Result: isIntegerArrayAttrSmallerThanShape(op: *this, arrayAttr: offsets, shape, attrName: offName)) || |
| 4652 | failed( |
| 4653 | Result: isIntegerArrayAttrSmallerThanShape(op: *this, arrayAttr: sizes, shape, attrName: sizesName)) || |
| 4654 | failed(Result: isIntegerArrayAttrSmallerThanShape(op: *this, arrayAttr: strides, shape, |
| 4655 | attrName: stridesName)) || |
| 4656 | failed( |
| 4657 | Result: isIntegerArrayAttrConfinedToShape(op: *this, arrayAttr: offsets, shape, attrName: offName)) || |
| 4658 | failed(Result: isIntegerArrayAttrConfinedToShape(op: *this, arrayAttr: sizes, shape, attrName: sizesName, |
| 4659 | /*halfOpen=*/false, |
| 4660 | /*min=*/1)) || |
| 4661 | failed(Result: isIntegerArrayAttrConfinedToRange(op: *this, arrayAttr: strides, /*min=*/1, |
| 4662 | /*max=*/1, attrName: stridesName, |
| 4663 | /*halfOpen=*/false)) || |
| 4664 | failed(Result: isSumOfIntegerArrayAttrConfinedToShape(op: *this, arrayAttr1: offsets, arrayAttr2: sizes, |
| 4665 | shape, attrName1: offName, attrName2: sizesName, |
| 4666 | /*halfOpen=*/false))) |
| 4667 | return failure(); |
| 4668 | |
| 4669 | auto resultType = inferStridedSliceOpResultType(vectorType: getSourceVectorType(), |
| 4670 | offsets, sizes, strides); |
| 4671 | if (getResult().getType() != resultType) |
| 4672 | return emitOpError(message: "expected result type to be " ) << resultType; |
| 4673 | |
| 4674 | for (unsigned idx = 0; idx < sizes.size(); ++idx) { |
| 4675 | if (type.getScalableDims()[idx]) { |
| 4676 | auto inputDim = type.getShape()[idx]; |
| 4677 | auto inputSize = llvm::cast<IntegerAttr>(Val: sizes[idx]).getInt(); |
| 4678 | if (inputDim != inputSize) |
| 4679 | return emitOpError(message: "expected size at idx=" ) |
| 4680 | << idx |
| 4681 | << (" to match the corresponding base size from the input " |
| 4682 | "vector (" ) |
| 4683 | << inputSize << (" vs " ) << inputDim << (")" ); |
| 4684 | } |
| 4685 | } |
| 4686 | |
| 4687 | return success(); |
| 4688 | } |
| 4689 | |
| 4690 | // When the source of ExtractStrided comes from a chain of InsertStrided ops try |
| 4691 | // to use the source of the InsertStrided ops if we can detect that the |
| 4692 | // extracted vector is a subset of one of the vector inserted. |
| 4693 | static LogicalResult |
| 4694 | (ExtractStridedSliceOp op) { |
| 4695 | // Helper to extract integer out of ArrayAttr. |
| 4696 | auto getElement = [](ArrayAttr array, int idx) { |
| 4697 | return llvm::cast<IntegerAttr>(Val: array[idx]).getInt(); |
| 4698 | }; |
| 4699 | ArrayAttr = op.getOffsets(); |
| 4700 | ArrayAttr = op.getStrides(); |
| 4701 | ArrayAttr = op.getSizes(); |
| 4702 | auto insertOp = op.getSource().getDefiningOp<InsertStridedSliceOp>(); |
| 4703 | while (insertOp) { |
| 4704 | if (op.getSourceVectorType().getRank() != |
| 4705 | insertOp.getSourceVectorType().getRank()) |
| 4706 | return failure(); |
| 4707 | ArrayAttr insertOffsets = insertOp.getOffsets(); |
| 4708 | ArrayAttr insertStrides = insertOp.getStrides(); |
| 4709 | // If the rank of extract is greater than the rank of insert, we are likely |
| 4710 | // extracting a partial chunk of the vector inserted. |
| 4711 | if (extractOffsets.size() > insertOffsets.size()) |
| 4712 | return failure(); |
| 4713 | bool patialoverlap = false; |
| 4714 | bool disjoint = false; |
| 4715 | SmallVector<int64_t, 4> offsetDiffs; |
| 4716 | for (unsigned dim = 0, e = extractOffsets.size(); dim < e; ++dim) { |
| 4717 | if (getElement(extractStrides, dim) != getElement(insertStrides, dim)) |
| 4718 | return failure(); |
| 4719 | int64_t start = getElement(insertOffsets, dim); |
| 4720 | int64_t end = start + insertOp.getSourceVectorType().getDimSize(idx: dim); |
| 4721 | int64_t offset = getElement(extractOffsets, dim); |
| 4722 | int64_t size = getElement(extractSizes, dim); |
| 4723 | // Check if the start of the extract offset is in the interval inserted. |
| 4724 | if (start <= offset && offset < end) { |
| 4725 | // If the extract interval overlaps but is not fully included we may |
| 4726 | // have a partial overlap that will prevent any folding. |
| 4727 | if (offset + size > end) |
| 4728 | patialoverlap = true; |
| 4729 | offsetDiffs.push_back(Elt: offset - start); |
| 4730 | continue; |
| 4731 | } |
| 4732 | disjoint = true; |
| 4733 | break; |
| 4734 | } |
| 4735 | // The extract element chunk is a subset of the insert element. |
| 4736 | if (!disjoint && !patialoverlap) { |
| 4737 | op.setOperand(insertOp.getValueToStore()); |
| 4738 | // OpBuilder is only used as a helper to build an I64ArrayAttr. |
| 4739 | OpBuilder b(op.getContext()); |
| 4740 | op.setOffsetsAttr(b.getI64ArrayAttr(values: offsetDiffs)); |
| 4741 | return success(); |
| 4742 | } |
| 4743 | // If the chunk extracted is disjoint from the chunk inserted, keep looking |
| 4744 | // in the insert chain. |
| 4745 | if (disjoint) |
| 4746 | insertOp = insertOp.getDest().getDefiningOp<InsertStridedSliceOp>(); |
| 4747 | else { |
| 4748 | // The extracted vector partially overlap the inserted vector, we cannot |
| 4749 | // fold. |
| 4750 | return failure(); |
| 4751 | } |
| 4752 | } |
| 4753 | return failure(); |
| 4754 | } |
| 4755 | |
| 4756 | // ExtractStridedSliceOp(non-splat ConstantOp) -> ConstantOp. |
| 4757 | static OpFoldResult |
| 4758 | (ExtractStridedSliceOp op, |
| 4759 | Attribute foldInput) { |
| 4760 | |
| 4761 | auto dense = llvm::dyn_cast_if_present<DenseElementsAttr>(Val&: foldInput); |
| 4762 | if (!dense) |
| 4763 | return {}; |
| 4764 | |
| 4765 | // TODO: Handle non-unit strides when they become available. |
| 4766 | if (op.hasNonUnitStrides()) |
| 4767 | return {}; |
| 4768 | |
| 4769 | VectorType sourceVecTy = op.getSourceVectorType(); |
| 4770 | ArrayRef<int64_t> sourceShape = sourceVecTy.getShape(); |
| 4771 | SmallVector<int64_t, 4> sourceStrides = computeStrides(sizes: sourceShape); |
| 4772 | |
| 4773 | VectorType sliceVecTy = op.getType(); |
| 4774 | ArrayRef<int64_t> sliceShape = sliceVecTy.getShape(); |
| 4775 | int64_t rank = sliceVecTy.getRank(); |
| 4776 | |
| 4777 | // Expand offsets and sizes to match the vector rank. |
| 4778 | SmallVector<int64_t, 4> offsets(rank, 0); |
| 4779 | copy(Range: getI64SubArray(arrayAttr: op.getOffsets()), Out: offsets.begin()); |
| 4780 | |
| 4781 | SmallVector<int64_t, 4> sizes(sourceShape); |
| 4782 | copy(Range: getI64SubArray(arrayAttr: op.getSizes()), Out: sizes.begin()); |
| 4783 | |
| 4784 | // Calculate the slice elements by enumerating all slice positions and |
| 4785 | // linearizing them. The enumeration order is lexicographic which yields a |
| 4786 | // sequence of monotonically increasing linearized position indices. |
| 4787 | const auto denseValuesBegin = dense.value_begin<Attribute>(); |
| 4788 | SmallVector<Attribute> sliceValues; |
| 4789 | sliceValues.reserve(N: sliceVecTy.getNumElements()); |
| 4790 | SmallVector<int64_t> currSlicePosition(offsets.begin(), offsets.end()); |
| 4791 | do { |
| 4792 | int64_t linearizedPosition = linearize(offsets: currSlicePosition, basis: sourceStrides); |
| 4793 | assert(linearizedPosition < sourceVecTy.getNumElements() && |
| 4794 | "Invalid index" ); |
| 4795 | sliceValues.push_back(Elt: *(denseValuesBegin + linearizedPosition)); |
| 4796 | } while (succeeded(Result: incSlicePosition(position: currSlicePosition, shape: sliceShape, offsets))); |
| 4797 | |
| 4798 | assert(static_cast<int64_t>(sliceValues.size()) == |
| 4799 | sliceVecTy.getNumElements() && |
| 4800 | "Invalid number of slice elements" ); |
| 4801 | return DenseElementsAttr::get(type: sliceVecTy, values: sliceValues); |
| 4802 | } |
| 4803 | |
| 4804 | OpFoldResult ExtractStridedSliceOp::(FoldAdaptor adaptor) { |
| 4805 | if (getSourceVectorType() == getResult().getType()) |
| 4806 | return getSource(); |
| 4807 | if (succeeded(Result: foldExtractStridedOpFromInsertChain(op: *this))) |
| 4808 | return getResult(); |
| 4809 | |
| 4810 | // ExtractStridedSliceOp(splat ConstantOp) -> ConstantOp. |
| 4811 | if (auto splat = |
| 4812 | llvm::dyn_cast_if_present<SplatElementsAttr>(Val: adaptor.getSource())) |
| 4813 | return DenseElementsAttr::get(type: getType(), values: splat.getSplatValue<Attribute>()); |
| 4814 | |
| 4815 | // ExtractStridedSliceOp(non-splat ConstantOp) -> ConstantOp. |
| 4816 | return foldExtractStridedSliceNonSplatConstant(op: *this, foldInput: adaptor.getSource()); |
| 4817 | } |
| 4818 | |
| 4819 | void ExtractStridedSliceOp::(SmallVectorImpl<int64_t> &results) { |
| 4820 | populateFromInt64AttrArray(arrayAttr: getOffsets(), results); |
| 4821 | } |
| 4822 | |
| 4823 | namespace { |
| 4824 | |
| 4825 | // Pattern to rewrite nested ExtractStridedSliceOp into a single one. |
| 4826 | // |
| 4827 | // Example: |
| 4828 | // |
| 4829 | // %0 = vector.extract_strided_slice %arg0 |
| 4830 | // {offsets = [1, 2], sizes = [3, 4], strides = [1, 1]} |
| 4831 | // : vector<4x8x16xf32> to vector<3x4x16xf32> |
| 4832 | // %1 = vector.extract_strided_slice %0 |
| 4833 | // {offsets = [0, 1], sizes = [2, 2], strides = [1, 1]} |
| 4834 | // : vector<3x4x16xf32> to vector<2x2x16xf32> |
| 4835 | // |
| 4836 | // to |
| 4837 | // |
| 4838 | // %1 = vector.extract_strided_slice %arg0 |
| 4839 | // {offsets = [1, 3], sizes = [2, 2], strides = [1, 1]} |
| 4840 | // : vector<4x8x16xf32> to vector<2x2x16xf32> |
| 4841 | class StridedSliceFolder final |
| 4842 | : public OpRewritePattern<ExtractStridedSliceOp> { |
| 4843 | public: |
| 4844 | using OpRewritePattern<ExtractStridedSliceOp>::OpRewritePattern; |
| 4845 | |
| 4846 | LogicalResult matchAndRewrite(ExtractStridedSliceOp secondOp, |
| 4847 | PatternRewriter &rewriter) const override { |
| 4848 | auto firstOp = secondOp.getSource().getDefiningOp<ExtractStridedSliceOp>(); |
| 4849 | if (!firstOp) |
| 4850 | return failure(); |
| 4851 | |
| 4852 | if (secondOp.hasNonUnitStrides() || firstOp.hasNonUnitStrides()) |
| 4853 | return failure(); |
| 4854 | |
| 4855 | SmallVector<int64_t> firstOffsets = getI64SubArray(arrayAttr: firstOp.getOffsets()); |
| 4856 | SmallVector<int64_t> firstSizes = getI64SubArray(arrayAttr: firstOp.getSizes()); |
| 4857 | SmallVector<int64_t> secondOffsets = getI64SubArray(arrayAttr: secondOp.getOffsets()); |
| 4858 | SmallVector<int64_t> secondSizes = getI64SubArray(arrayAttr: secondOp.getSizes()); |
| 4859 | |
| 4860 | unsigned newRank = std::max(a: firstOffsets.size(), b: secondOffsets.size()); |
| 4861 | SmallVector<int64_t> combinedOffsets(newRank, 0); |
| 4862 | SmallVector<int64_t> combinedSizes(newRank); |
| 4863 | ArrayRef<int64_t> firstSourceShape = |
| 4864 | firstOp.getSourceVectorType().getShape(); |
| 4865 | for (unsigned i = 0; i < newRank; ++i) { |
| 4866 | int64_t off1 = (i < firstOffsets.size()) ? firstOffsets[i] : 0; |
| 4867 | int64_t off2 = (i < secondOffsets.size()) ? secondOffsets[i] : 0; |
| 4868 | combinedOffsets[i] = off1 + off2; |
| 4869 | |
| 4870 | if (i < secondSizes.size()) { |
| 4871 | combinedSizes[i] = secondSizes[i]; |
| 4872 | } else if (i < firstSizes.size()) { |
| 4873 | combinedSizes[i] = firstSizes[i]; |
| 4874 | } else { |
| 4875 | combinedSizes[i] = firstSourceShape[i]; |
| 4876 | } |
| 4877 | } |
| 4878 | |
| 4879 | SmallVector<int64_t> combinedStrides(newRank, 1); |
| 4880 | rewriter.replaceOpWithNewOp<ExtractStridedSliceOp>( |
| 4881 | op: secondOp, args: firstOp.getSource(), args&: combinedOffsets, args&: combinedSizes, |
| 4882 | args&: combinedStrides); |
| 4883 | return success(); |
| 4884 | } |
| 4885 | }; |
| 4886 | |
| 4887 | // Pattern to rewrite an ExtractStridedSliceOp(CreateMaskOp) to |
| 4888 | // CreateMaskOp. |
| 4889 | // |
| 4890 | // Example: |
| 4891 | // |
| 4892 | // %mask = vector.create_mask %ub : vector<16xi1> |
| 4893 | // %slice = vector.extract_strided_slice [%offset] [8] [1] |
| 4894 | // |
| 4895 | // to |
| 4896 | // |
| 4897 | // %new_ub = arith.subi %ub, %offset |
| 4898 | // %mask = vector.create_mask %new_ub : vector<8xi1> |
| 4899 | class StridedSliceCreateMaskFolder final |
| 4900 | : public OpRewritePattern<ExtractStridedSliceOp> { |
| 4901 | using Base::Base; |
| 4902 | |
| 4903 | public: |
| 4904 | LogicalResult matchAndRewrite(ExtractStridedSliceOp , |
| 4905 | PatternRewriter &rewriter) const override { |
| 4906 | Location loc = extractStridedSliceOp.getLoc(); |
| 4907 | // Return if 'extractStridedSliceOp' operand is not defined by a |
| 4908 | // CreateMaskOp. |
| 4909 | auto createMaskOp = |
| 4910 | extractStridedSliceOp.getSource().getDefiningOp<CreateMaskOp>(); |
| 4911 | if (!createMaskOp) |
| 4912 | return failure(); |
| 4913 | // Return if 'extractStridedSliceOp' has non-unit strides. |
| 4914 | if (extractStridedSliceOp.hasNonUnitStrides()) |
| 4915 | return failure(); |
| 4916 | // Gather constant mask dimension sizes. |
| 4917 | SmallVector<Value> maskDimSizes(createMaskOp.getOperands()); |
| 4918 | // Gather strided slice offsets and sizes. |
| 4919 | SmallVector<int64_t> sliceOffsets; |
| 4920 | populateFromInt64AttrArray(arrayAttr: extractStridedSliceOp.getOffsets(), |
| 4921 | results&: sliceOffsets); |
| 4922 | SmallVector<int64_t> sliceSizes; |
| 4923 | populateFromInt64AttrArray(arrayAttr: extractStridedSliceOp.getSizes(), results&: sliceSizes); |
| 4924 | |
| 4925 | // Compute slice of vector mask region. |
| 4926 | SmallVector<Value> sliceMaskDimSizes; |
| 4927 | sliceMaskDimSizes.reserve(N: maskDimSizes.size()); |
| 4928 | // sliceOffsets.size() <= maskDimSizes.size(), so we use llvm::zip and |
| 4929 | // only iterate on the leading dim sizes. The tail accounts for the |
| 4930 | // remaining dim sizes. |
| 4931 | for (auto [maskDimSize, sliceOffset, sliceSize] : |
| 4932 | llvm::zip(t&: maskDimSizes, u&: sliceOffsets, args&: sliceSizes)) { |
| 4933 | // No need to clamp on min/max values, because create_mask has clamping |
| 4934 | // semantics, i.e. the sliceMaskDimSize is allowed to be negative or |
| 4935 | // greater than the vector dim size. |
| 4936 | IntegerAttr offsetAttr = |
| 4937 | rewriter.getIntegerAttr(type: maskDimSize.getType(), value: sliceOffset); |
| 4938 | Value offset = arith::ConstantOp::create(builder&: rewriter, location: loc, value: offsetAttr); |
| 4939 | Value sliceMaskDimSize = |
| 4940 | arith::SubIOp::create(builder&: rewriter, location: loc, lhs: maskDimSize, rhs: offset); |
| 4941 | sliceMaskDimSizes.push_back(Elt: sliceMaskDimSize); |
| 4942 | } |
| 4943 | // Add unchanged dimensions. |
| 4944 | llvm::append_range( |
| 4945 | C&: sliceMaskDimSizes, |
| 4946 | R: llvm::drop_begin(RangeOrContainer&: maskDimSizes, N: sliceMaskDimSizes.size())); |
| 4947 | // Replace 'extractStridedSliceOp' with CreateMaskOp with sliced mask |
| 4948 | // region. |
| 4949 | rewriter.replaceOpWithNewOp<CreateMaskOp>( |
| 4950 | op: extractStridedSliceOp, args: extractStridedSliceOp.getResult().getType(), |
| 4951 | args&: sliceMaskDimSizes); |
| 4952 | return success(); |
| 4953 | } |
| 4954 | }; |
| 4955 | |
| 4956 | // Pattern to rewrite an ExtractStridedSliceOp(ConstantMaskOp) to |
| 4957 | // ConstantMaskOp. |
| 4958 | class StridedSliceConstantMaskFolder final |
| 4959 | : public OpRewritePattern<ExtractStridedSliceOp> { |
| 4960 | public: |
| 4961 | using Base::Base; |
| 4962 | |
| 4963 | LogicalResult matchAndRewrite(ExtractStridedSliceOp , |
| 4964 | PatternRewriter &rewriter) const override { |
| 4965 | // Return if 'extractStridedSliceOp' operand is not defined by a |
| 4966 | // ConstantMaskOp. |
| 4967 | auto *defOp = extractStridedSliceOp.getSource().getDefiningOp(); |
| 4968 | auto constantMaskOp = dyn_cast_or_null<ConstantMaskOp>(Val: defOp); |
| 4969 | if (!constantMaskOp) |
| 4970 | return failure(); |
| 4971 | // Return if 'extractStridedSliceOp' has non-unit strides. |
| 4972 | if (extractStridedSliceOp.hasNonUnitStrides()) |
| 4973 | return failure(); |
| 4974 | // Gather constant mask dimension sizes. |
| 4975 | ArrayRef<int64_t> maskDimSizes = constantMaskOp.getMaskDimSizes(); |
| 4976 | // Gather strided slice offsets and sizes. |
| 4977 | SmallVector<int64_t> sliceOffsets; |
| 4978 | populateFromInt64AttrArray(arrayAttr: extractStridedSliceOp.getOffsets(), |
| 4979 | results&: sliceOffsets); |
| 4980 | SmallVector<int64_t> sliceSizes; |
| 4981 | populateFromInt64AttrArray(arrayAttr: extractStridedSliceOp.getSizes(), results&: sliceSizes); |
| 4982 | |
| 4983 | // Compute slice of vector mask region. |
| 4984 | SmallVector<int64_t> sliceMaskDimSizes; |
| 4985 | sliceMaskDimSizes.reserve(N: maskDimSizes.size()); |
| 4986 | for (auto [maskDimSize, sliceOffset, sliceSize] : |
| 4987 | llvm::zip(t&: maskDimSizes, u&: sliceOffsets, args&: sliceSizes)) { |
| 4988 | int64_t sliceMaskDimSize = std::max( |
| 4989 | a: static_cast<int64_t>(0), |
| 4990 | b: std::min(a: sliceOffset + sliceSize, b: maskDimSize) - sliceOffset); |
| 4991 | sliceMaskDimSizes.push_back(Elt: sliceMaskDimSize); |
| 4992 | } |
| 4993 | // Add unchanged dimensions. |
| 4994 | if (sliceMaskDimSizes.size() < maskDimSizes.size()) |
| 4995 | for (size_t i = sliceMaskDimSizes.size(); i < maskDimSizes.size(); ++i) |
| 4996 | sliceMaskDimSizes.push_back(Elt: maskDimSizes[i]); |
| 4997 | // If any of 'sliceMaskDimSizes' are zero, then set all to zero (masked |
| 4998 | // region is a conjunction of mask dim intervals). |
| 4999 | if (llvm::is_contained(Range&: sliceMaskDimSizes, Element: 0)) |
| 5000 | sliceMaskDimSizes.assign(NumElts: maskDimSizes.size(), Elt: 0); |
| 5001 | |
| 5002 | // Replace 'extractStridedSliceOp' with ConstantMaskOp with sliced mask |
| 5003 | // region. |
| 5004 | rewriter.replaceOpWithNewOp<ConstantMaskOp>( |
| 5005 | op: extractStridedSliceOp, args: extractStridedSliceOp.getResult().getType(), |
| 5006 | args&: sliceMaskDimSizes); |
| 5007 | return success(); |
| 5008 | } |
| 5009 | }; |
| 5010 | |
| 5011 | // Pattern to rewrite an ExtractStridedSliceOp(BroadcastOp) to |
| 5012 | // BroadcastOp(ExtractStrideSliceOp). |
| 5013 | class StridedSliceBroadcast final |
| 5014 | : public OpRewritePattern<ExtractStridedSliceOp> { |
| 5015 | public: |
| 5016 | using Base::Base; |
| 5017 | |
| 5018 | LogicalResult matchAndRewrite(ExtractStridedSliceOp op, |
| 5019 | PatternRewriter &rewriter) const override { |
| 5020 | auto broadcast = op.getSource().getDefiningOp<BroadcastOp>(); |
| 5021 | if (!broadcast) |
| 5022 | return failure(); |
| 5023 | auto srcVecType = |
| 5024 | llvm::dyn_cast<VectorType>(Val: broadcast.getSource().getType()); |
| 5025 | unsigned srcRank = srcVecType ? srcVecType.getRank() : 0; |
| 5026 | auto dstVecType = llvm::cast<VectorType>(Val: op.getType()); |
| 5027 | unsigned dstRank = dstVecType.getRank(); |
| 5028 | unsigned rankDiff = dstRank - srcRank; |
| 5029 | // Source dimensions can be broadcasted (1 -> n with n > 1) or sliced |
| 5030 | // (n -> m with n > m). If they are originally both broadcasted *and* |
| 5031 | // sliced, this can be simplified to just broadcasting. |
| 5032 | bool needsSlice = false; |
| 5033 | for (unsigned i = 0; i < srcRank; i++) { |
| 5034 | if (srcVecType.getDimSize(idx: i) != 1 && |
| 5035 | srcVecType.getDimSize(idx: i) != dstVecType.getDimSize(idx: i + rankDiff)) { |
| 5036 | needsSlice = true; |
| 5037 | break; |
| 5038 | } |
| 5039 | } |
| 5040 | Value source = broadcast.getSource(); |
| 5041 | if (needsSlice) { |
| 5042 | SmallVector<int64_t> offsets = |
| 5043 | getI64SubArray(arrayAttr: op.getOffsets(), /*dropFront=*/rankDiff); |
| 5044 | SmallVector<int64_t> sizes = |
| 5045 | getI64SubArray(arrayAttr: op.getSizes(), /*dropFront=*/rankDiff); |
| 5046 | // Slice attributes (i.e. offsets, sizes, etc) are only required for the |
| 5047 | // leading/prefix source dimensions. This is why we use |
| 5048 | // `offset.size()` instead of `srcRank` below. |
| 5049 | for (unsigned i = 0; i < offsets.size(); i++) { |
| 5050 | if (srcVecType.getDimSize(idx: i) == 1) { |
| 5051 | // In case this dimension was broadcasted *and* sliced, the offset |
| 5052 | // and size need to be updated now that there is no broadcast before |
| 5053 | // the slice. |
| 5054 | offsets[i] = 0; |
| 5055 | sizes[i] = 1; |
| 5056 | } |
| 5057 | } |
| 5058 | source = ExtractStridedSliceOp::create( |
| 5059 | builder&: rewriter, location: op->getLoc(), source, offsets, sizes, |
| 5060 | strides: getI64SubArray(arrayAttr: op.getStrides(), /*dropFront=*/rankDiff)); |
| 5061 | } |
| 5062 | rewriter.replaceOpWithNewOp<BroadcastOp>(op, args: op.getType(), args&: source); |
| 5063 | return success(); |
| 5064 | } |
| 5065 | }; |
| 5066 | |
| 5067 | /// Rewrite extract_strided_slice(splat-like(v)) with broadcast(v). |
| 5068 | class StridedSliceSplat final : public OpRewritePattern<ExtractStridedSliceOp> { |
| 5069 | public: |
| 5070 | using Base::Base; |
| 5071 | |
| 5072 | LogicalResult matchAndRewrite(ExtractStridedSliceOp op, |
| 5073 | PatternRewriter &rewriter) const override { |
| 5074 | |
| 5075 | Value splat = getScalarSplatSource(value: op.getSource()); |
| 5076 | if (!splat) |
| 5077 | return failure(); |
| 5078 | rewriter.replaceOpWithNewOp<BroadcastOp>(op, args: op.getType(), args&: splat); |
| 5079 | return success(); |
| 5080 | } |
| 5081 | }; |
| 5082 | |
| 5083 | /// Pattern to rewrite simple cases of N-D extract_strided_slice, where the |
| 5084 | /// slice is contiguous, into extract and shape_cast. |
| 5085 | /// |
| 5086 | /// Example: |
| 5087 | /// Before: |
| 5088 | /// %1 = vector.extract_strided_slice %arg0 { |
| 5089 | /// offsets = [0, 0, 0, 0, 0], |
| 5090 | /// sizes = [1, 1, 1, 1, 8], |
| 5091 | /// strides = [1, 1, 1, 1, 1] |
| 5092 | /// } : vector<8x1x1x2x8xi8> to vector<1x1x1x1x8xi8> |
| 5093 | /// After: |
| 5094 | /// %0 = vector.extract %arg0[0, 0, 0, 0] |
| 5095 | /// : vector<8xi8> from vector<8x1x1x2x8xi8> |
| 5096 | /// %1 = vector.shape_cast %0 |
| 5097 | /// : vector<8xi8> to vector<1x1x1x1x8xi8> |
| 5098 | /// |
| 5099 | class final |
| 5100 | : public OpRewritePattern<ExtractStridedSliceOp> { |
| 5101 | public: |
| 5102 | using Base::Base; |
| 5103 | |
| 5104 | LogicalResult matchAndRewrite(ExtractStridedSliceOp op, |
| 5105 | PatternRewriter &rewriter) const override { |
| 5106 | if (op.hasNonUnitStrides()) |
| 5107 | return failure(); |
| 5108 | Value source = op.getOperand(); |
| 5109 | auto sourceType = cast<VectorType>(Val: source.getType()); |
| 5110 | if (sourceType.isScalable() || sourceType.getRank() == 0) |
| 5111 | return failure(); |
| 5112 | |
| 5113 | // Compute the number of offsets to pass to ExtractOp::build. That is the |
| 5114 | // difference between the source rank and the desired slice rank. We walk |
| 5115 | // the dimensions from innermost out, and stop when the next slice dimension |
| 5116 | // is not full-size. |
| 5117 | SmallVector<int64_t> sizes = getI64SubArray(arrayAttr: op.getSizes()); |
| 5118 | int numOffsets; |
| 5119 | for (numOffsets = sizes.size(); numOffsets > 0; --numOffsets) { |
| 5120 | if (sizes[numOffsets - 1] != sourceType.getDimSize(idx: numOffsets - 1)) |
| 5121 | break; |
| 5122 | } |
| 5123 | |
| 5124 | // If the created extract op would have no offsets, then this whole |
| 5125 | // extract_strided_slice is the identity and should have been handled by |
| 5126 | // other canonicalizations. |
| 5127 | if (numOffsets == 0) |
| 5128 | return failure(); |
| 5129 | |
| 5130 | // If not even the inner-most dimension is full-size, this op can't be |
| 5131 | // rewritten as an ExtractOp. |
| 5132 | if (numOffsets == sourceType.getRank() && |
| 5133 | static_cast<int>(sizes.size()) == sourceType.getRank()) |
| 5134 | return failure(); |
| 5135 | |
| 5136 | // The outer dimensions must have unit size. |
| 5137 | for (int i = 0; i < numOffsets; ++i) { |
| 5138 | if (sizes[i] != 1) |
| 5139 | return failure(); |
| 5140 | } |
| 5141 | |
| 5142 | // Avoid generating slices that have leading unit dimensions. The shape_cast |
| 5143 | // op that we create below would take bad generic fallback patterns |
| 5144 | // (ShapeCastOpRewritePattern). |
| 5145 | while (numOffsets < static_cast<int>(sizes.size()) - 1 && |
| 5146 | sizes[numOffsets] == 1) { |
| 5147 | ++numOffsets; |
| 5148 | } |
| 5149 | |
| 5150 | SmallVector<int64_t> offsets = getI64SubArray(arrayAttr: op.getOffsets()); |
| 5151 | auto = ArrayRef(offsets).take_front(N: numOffsets); |
| 5152 | Value = vector::ExtractOp::create(builder&: rewriter, location: op->getLoc(), source, |
| 5153 | position: extractOffsets); |
| 5154 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op, args: op.getType(), args&: extract); |
| 5155 | return success(); |
| 5156 | } |
| 5157 | }; |
| 5158 | |
| 5159 | } // namespace |
| 5160 | |
| 5161 | void ExtractStridedSliceOp::( |
| 5162 | RewritePatternSet &results, MLIRContext *context) { |
| 5163 | // Pattern to rewrite a ExtractStridedSliceOp(ConstantMaskOp) -> |
| 5164 | // ConstantMaskOp and ExtractStridedSliceOp(ConstantOp) -> ConstantOp. |
| 5165 | results.add<StridedSliceFolder, StridedSliceCreateMaskFolder, |
| 5166 | StridedSliceConstantMaskFolder, StridedSliceBroadcast, |
| 5167 | StridedSliceSplat, ContiguousExtractStridedSliceToExtract>( |
| 5168 | arg&: context); |
| 5169 | } |
| 5170 | |
| 5171 | //===----------------------------------------------------------------------===// |
| 5172 | // TransferReadOp |
| 5173 | //===----------------------------------------------------------------------===// |
| 5174 | |
| 5175 | /// 1. Builder that sets padding to zero and an empty mask (variant with attrs). |
| 5176 | /// If `padding` is null, a poison value is used. |
| 5177 | void TransferReadOp::build(OpBuilder &builder, OperationState &result, |
| 5178 | VectorType vectorType, Value source, |
| 5179 | ValueRange indices, std::optional<Value> padding, |
| 5180 | AffineMapAttr permutationMapAttr, |
| 5181 | /*optional*/ ArrayAttr inBoundsAttr) { |
| 5182 | |
| 5183 | Type elemType = llvm::cast<ShapedType>(Val: source.getType()).getElementType(); |
| 5184 | if (!padding) |
| 5185 | padding = ub::PoisonOp::create(builder, location: result.location, result: elemType); |
| 5186 | // Delegate to the most general builder (see |
| 5187 | // `mlir/Dialect/Vector/IR/VectorOps.cpp.inc`) |
| 5188 | build(odsBuilder&: builder, odsState&: result, result: vectorType, base: source, indices, permutation_map: permutationMapAttr, |
| 5189 | padding: *padding, /*mask=*/Value(), in_bounds: inBoundsAttr); |
| 5190 | } |
| 5191 | |
| 5192 | /// 2. Builder that sets padding to zero and an empty mask (variant without |
| 5193 | /// attrs). |
| 5194 | /// If `padding` is null, a poison value is used. |
| 5195 | /// If `permutationMap` is null, a minor identity map is used. |
| 5196 | /// If `inBounds` is null, an empty mask is used. |
| 5197 | void TransferReadOp::build(OpBuilder &builder, OperationState &result, |
| 5198 | VectorType vectorType, Value source, |
| 5199 | ValueRange indices, std::optional<Value> padding, |
| 5200 | AffineMap permutationMap, |
| 5201 | std::optional<ArrayRef<bool>> inBounds) { |
| 5202 | if (!permutationMap) |
| 5203 | permutationMap = getTransferMinorIdentityMap( |
| 5204 | shapedType: llvm::cast<ShapedType>(Val: source.getType()), vectorType); |
| 5205 | auto permutationMapAttr = AffineMapAttr::get(value: permutationMap); |
| 5206 | auto inBoundsAttr = (inBounds && !inBounds.value().empty()) |
| 5207 | ? builder.getBoolArrayAttr(values: inBounds.value()) |
| 5208 | : builder.getBoolArrayAttr( |
| 5209 | values: SmallVector<bool>(vectorType.getRank(), false)); |
| 5210 | // Delegate to Builder 1 |
| 5211 | build(builder, result, vectorType, source, indices, padding, |
| 5212 | permutationMapAttr, inBoundsAttr); |
| 5213 | } |
| 5214 | |
| 5215 | /// 3. Builder that sets permutation map to 'getMinorIdentityMap'. |
| 5216 | /// If `padding` is null, a poison value is used. |
| 5217 | /// If `inBounds` is null, an empty mask is used. |
| 5218 | void TransferReadOp::build(OpBuilder &builder, OperationState &result, |
| 5219 | VectorType vectorType, Value source, |
| 5220 | ValueRange indices, std::optional<Value> padding, |
| 5221 | std::optional<ArrayRef<bool>> inBounds) { |
| 5222 | // Delegate to Builder 2 |
| 5223 | build(builder, result, vectorType, source, indices, padding, |
| 5224 | /*permutationMap=*/AffineMap(), inBounds); |
| 5225 | } |
| 5226 | |
| 5227 | template <typename EmitFun> |
| 5228 | static LogicalResult verifyPermutationMap(AffineMap permutationMap, |
| 5229 | EmitFun emitOpError) { |
| 5230 | SmallVector<bool, 8> seen(permutationMap.getNumInputs(), false); |
| 5231 | for (auto expr : permutationMap.getResults()) { |
| 5232 | auto dim = dyn_cast<AffineDimExpr>(Val&: expr); |
| 5233 | auto zero = dyn_cast<AffineConstantExpr>(Val&: expr); |
| 5234 | if (zero) { |
| 5235 | if (zero.getValue() != 0) { |
| 5236 | return emitOpError( |
| 5237 | "requires a projected permutation_map (at most one dim or the zero " |
| 5238 | "constant can appear in each result)" ); |
| 5239 | } |
| 5240 | continue; |
| 5241 | } |
| 5242 | if (!dim) { |
| 5243 | return emitOpError("requires a projected permutation_map (at most one " |
| 5244 | "dim or the zero constant can appear in each result)" ); |
| 5245 | } |
| 5246 | if (seen[dim.getPosition()]) { |
| 5247 | return emitOpError( |
| 5248 | "requires a permutation_map that is a permutation (found one dim " |
| 5249 | "used more than once)" ); |
| 5250 | } |
| 5251 | seen[dim.getPosition()] = true; |
| 5252 | } |
| 5253 | return success(); |
| 5254 | } |
| 5255 | |
| 5256 | static LogicalResult |
| 5257 | verifyTransferOp(VectorTransferOpInterface op, ShapedType shapedType, |
| 5258 | VectorType vectorType, VectorType maskType, |
| 5259 | VectorType inferredMaskType, AffineMap permutationMap, |
| 5260 | ArrayAttr inBounds) { |
| 5261 | if (op->hasDiscardableAttr(name: "masked" )) { |
| 5262 | return op->emitOpError(message: "masked attribute has been removed. " |
| 5263 | "Use in_bounds instead." ); |
| 5264 | } |
| 5265 | |
| 5266 | if (!llvm::isa<MemRefType, RankedTensorType>(Val: shapedType)) |
| 5267 | return op->emitOpError( |
| 5268 | message: "requires source to be a memref or ranked tensor type" ); |
| 5269 | |
| 5270 | auto elementType = shapedType.getElementType(); |
| 5271 | DataLayout dataLayout = DataLayout::closest(op); |
| 5272 | if (auto vectorElementType = llvm::dyn_cast<VectorType>(Val&: elementType)) { |
| 5273 | // Memref or tensor has vector element type. |
| 5274 | unsigned sourceVecSize = |
| 5275 | dataLayout.getTypeSizeInBits(t: vectorElementType.getElementType()) * |
| 5276 | vectorElementType.getShape().back(); |
| 5277 | unsigned resultVecSize = |
| 5278 | dataLayout.getTypeSizeInBits(t: vectorType.getElementType()) * |
| 5279 | vectorType.getShape().back(); |
| 5280 | if (resultVecSize % sourceVecSize != 0) |
| 5281 | return op->emitOpError( |
| 5282 | message: "requires the bitwidth of the minor 1-D vector to be an integral " |
| 5283 | "multiple of the bitwidth of the minor 1-D vector of the source" ); |
| 5284 | |
| 5285 | unsigned sourceVecEltRank = vectorElementType.getRank(); |
| 5286 | unsigned resultVecRank = vectorType.getRank(); |
| 5287 | if (sourceVecEltRank > resultVecRank) |
| 5288 | return op->emitOpError( |
| 5289 | message: "requires source vector element and vector result ranks to match." ); |
| 5290 | unsigned rankOffset = resultVecRank - sourceVecEltRank; |
| 5291 | // Check that permutation map results match 'rankOffset' of vector type. |
| 5292 | if (permutationMap.getNumResults() != rankOffset) |
| 5293 | return op->emitOpError(message: "requires a permutation_map with result dims of " |
| 5294 | "the same rank as the vector type" ); |
| 5295 | |
| 5296 | if (maskType) |
| 5297 | return op->emitOpError(message: "does not support masks with vector element type" ); |
| 5298 | } else { |
| 5299 | // Memref or tensor has scalar element type. |
| 5300 | unsigned minorSize = |
| 5301 | vectorType.getRank() == 0 ? 1 : vectorType.getShape().back(); |
| 5302 | unsigned resultVecSize = |
| 5303 | dataLayout.getTypeSizeInBits(t: vectorType.getElementType()) * minorSize; |
| 5304 | if (resultVecSize % dataLayout.getTypeSizeInBits(t: elementType) != 0) |
| 5305 | return op->emitOpError( |
| 5306 | message: "requires the bitwidth of the minor 1-D vector to be an integral " |
| 5307 | "multiple of the bitwidth of the source element type" ); |
| 5308 | |
| 5309 | // Check that permutation map results match rank of vector type. |
| 5310 | if (permutationMap.getNumResults() != vectorType.getRank()) |
| 5311 | return op->emitOpError(message: "requires a permutation_map with result dims of " |
| 5312 | "the same rank as the vector type" ); |
| 5313 | } |
| 5314 | |
| 5315 | if (permutationMap.getNumSymbols() != 0) |
| 5316 | return op->emitOpError(message: "requires permutation_map without symbols" ); |
| 5317 | |
| 5318 | if (permutationMap.getNumInputs() != shapedType.getRank()) |
| 5319 | return op->emitOpError(message: "requires a permutation_map with input dims of the " |
| 5320 | "same rank as the source type" ); |
| 5321 | |
| 5322 | if (maskType && maskType != inferredMaskType) |
| 5323 | return op->emitOpError(message: "inferred mask type (" ) |
| 5324 | << inferredMaskType << ") and mask operand type (" << maskType |
| 5325 | << ") don't match" ; |
| 5326 | |
| 5327 | if (permutationMap.getNumResults() != static_cast<int64_t>(inBounds.size())) |
| 5328 | return op->emitOpError(message: "expects the in_bounds attr of same rank " |
| 5329 | "as permutation_map results: " ) |
| 5330 | << AffineMapAttr::get(value: permutationMap) |
| 5331 | << " vs inBounds of size: " << inBounds.size(); |
| 5332 | |
| 5333 | return success(); |
| 5334 | } |
| 5335 | |
| 5336 | static void printTransferAttrs(OpAsmPrinter &p, VectorTransferOpInterface op) { |
| 5337 | SmallVector<NamedAttribute> attrs(op->getDiscardableAttrs()); |
| 5338 | // Elide in_bounds attribute if all dims are out-of-bounds. |
| 5339 | if (llvm::any_of(Range: op.getInBoundsValues(), P: [](bool b) { return b; })) |
| 5340 | attrs.emplace_back(Args: op.getInBoundsAttrName(), Args: op.getInBounds()); |
| 5341 | if (!op.getPermutationMap().isMinorIdentity()) |
| 5342 | attrs.emplace_back(Args: op.getPermutationMapAttrName(), |
| 5343 | Args: AffineMapAttr::get(value: op.getPermutationMap())); |
| 5344 | llvm::sort(C&: attrs); |
| 5345 | p.printOptionalAttrDict(attrs); |
| 5346 | } |
| 5347 | |
| 5348 | void TransferReadOp::print(OpAsmPrinter &p) { |
| 5349 | p << " " << getBase() << "[" << getIndices() << "], " << getPadding(); |
| 5350 | if (getMask()) |
| 5351 | p << ", " << getMask(); |
| 5352 | printTransferAttrs(p, op: *this); |
| 5353 | p << " : " << getShapedType() << ", " << getVectorType(); |
| 5354 | } |
| 5355 | |
| 5356 | VectorType mlir::vector::inferTransferOpMaskType(VectorType vecType, |
| 5357 | AffineMap permMap) { |
| 5358 | auto i1Type = IntegerType::get(context: permMap.getContext(), width: 1); |
| 5359 | AffineMap invPermMap = inversePermutation(map: compressUnusedDims(map: permMap)); |
| 5360 | assert(invPermMap && "Inversed permutation map couldn't be computed" ); |
| 5361 | SmallVector<int64_t, 8> maskShape = invPermMap.compose(values: vecType.getShape()); |
| 5362 | |
| 5363 | // The MaskOp specification doesn't support 0-D vectors at the moment. Turn a |
| 5364 | // 0-D mask into a single-element 1-D mask. |
| 5365 | if (maskShape.empty()) |
| 5366 | maskShape.push_back(Elt: 1); |
| 5367 | |
| 5368 | SmallVector<bool> scalableDims = |
| 5369 | applyPermutationMap(map: invPermMap, source: vecType.getScalableDims()); |
| 5370 | |
| 5371 | return VectorType::get(shape: maskShape, elementType: i1Type, scalableDims); |
| 5372 | } |
| 5373 | |
| 5374 | ParseResult TransferReadOp::parse(OpAsmParser &parser, OperationState &result) { |
| 5375 | auto &builder = parser.getBuilder(); |
| 5376 | SMLoc typesLoc; |
| 5377 | OpAsmParser::UnresolvedOperand sourceInfo; |
| 5378 | SmallVector<OpAsmParser::UnresolvedOperand, 8> indexInfo; |
| 5379 | OpAsmParser::UnresolvedOperand paddingInfo; |
| 5380 | SmallVector<Type, 2> types; |
| 5381 | OpAsmParser::UnresolvedOperand maskInfo; |
| 5382 | // Parsing with support for paddingValue. |
| 5383 | if (parser.parseOperand(result&: sourceInfo) || |
| 5384 | parser.parseOperandList(result&: indexInfo, delimiter: OpAsmParser::Delimiter::Square) || |
| 5385 | parser.parseComma() || parser.parseOperand(result&: paddingInfo)) |
| 5386 | return failure(); |
| 5387 | ParseResult hasMask = parser.parseOptionalComma(); |
| 5388 | if (hasMask.succeeded()) { |
| 5389 | if (parser.parseOperand(result&: maskInfo)) |
| 5390 | return failure(); |
| 5391 | } |
| 5392 | if (parser.parseOptionalAttrDict(result&: result.attributes) || |
| 5393 | parser.getCurrentLocation(loc: &typesLoc) || parser.parseColonTypeList(result&: types)) |
| 5394 | return failure(); |
| 5395 | if (types.size() != 2) |
| 5396 | return parser.emitError(loc: typesLoc, message: "requires two types" ); |
| 5397 | auto indexType = builder.getIndexType(); |
| 5398 | auto shapedType = llvm::dyn_cast<ShapedType>(Val&: types[0]); |
| 5399 | if (!shapedType || !llvm::isa<MemRefType, RankedTensorType>(Val: shapedType)) |
| 5400 | return parser.emitError(loc: typesLoc, message: "requires memref or ranked tensor type" ); |
| 5401 | VectorType vectorType = llvm::dyn_cast<VectorType>(Val&: types[1]); |
| 5402 | if (!vectorType) |
| 5403 | return parser.emitError(loc: typesLoc, message: "requires vector type" ); |
| 5404 | auto permMapAttrName = TransferReadOp::getPermutationMapAttrName(name: result.name); |
| 5405 | Attribute permMapAttr = result.attributes.get(name: permMapAttrName); |
| 5406 | AffineMap permMap; |
| 5407 | if (!permMapAttr) { |
| 5408 | if (shapedType.getRank() < |
| 5409 | getEffectiveVectorRankForXferOp(shapedType, vectorType)) |
| 5410 | return parser.emitError(loc: typesLoc, |
| 5411 | message: "expected a custom permutation_map when " |
| 5412 | "rank(source) != rank(destination)" ); |
| 5413 | permMap = getTransferMinorIdentityMap(shapedType, vectorType); |
| 5414 | result.attributes.set(name: permMapAttrName, value: AffineMapAttr::get(value: permMap)); |
| 5415 | } else { |
| 5416 | permMap = llvm::cast<AffineMapAttr>(Val&: permMapAttr).getValue(); |
| 5417 | } |
| 5418 | auto inBoundsAttrName = TransferReadOp::getInBoundsAttrName(name: result.name); |
| 5419 | Attribute inBoundsAttr = result.attributes.get(name: inBoundsAttrName); |
| 5420 | if (!inBoundsAttr) { |
| 5421 | result.addAttribute(name: inBoundsAttrName, |
| 5422 | attr: builder.getBoolArrayAttr( |
| 5423 | values: SmallVector<bool>(permMap.getNumResults(), false))); |
| 5424 | } |
| 5425 | if (parser.resolveOperand(operand: sourceInfo, type: shapedType, result&: result.operands) || |
| 5426 | parser.resolveOperands(operands&: indexInfo, type: indexType, result&: result.operands) || |
| 5427 | parser.resolveOperand(operand: paddingInfo, type: shapedType.getElementType(), |
| 5428 | result&: result.operands)) |
| 5429 | return failure(); |
| 5430 | if (hasMask.succeeded()) { |
| 5431 | if (llvm::dyn_cast<VectorType>(Val: shapedType.getElementType())) |
| 5432 | return parser.emitError( |
| 5433 | loc: maskInfo.location, message: "does not support masks with vector element type" ); |
| 5434 | if (vectorType.getRank() != permMap.getNumResults()) { |
| 5435 | return parser.emitError(loc: typesLoc, |
| 5436 | message: "expected the same rank for the vector and the " |
| 5437 | "results of the permutation map" ); |
| 5438 | } |
| 5439 | // Instead of adding the mask type as an op type, compute it based on the |
| 5440 | // vector type and the permutation map (to keep the type signature small). |
| 5441 | auto maskType = inferTransferOpMaskType(vecType: vectorType, permMap); |
| 5442 | if (parser.resolveOperand(operand: maskInfo, type: maskType, result&: result.operands)) |
| 5443 | return failure(); |
| 5444 | } |
| 5445 | result.addAttribute(name: TransferReadOp::getOperandSegmentSizeAttr(), |
| 5446 | attr: builder.getDenseI32ArrayAttr( |
| 5447 | values: {1, static_cast<int32_t>(indexInfo.size()), 1, |
| 5448 | static_cast<int32_t>(hasMask.succeeded())})); |
| 5449 | return parser.addTypeToList(type: vectorType, result&: result.types); |
| 5450 | } |
| 5451 | |
| 5452 | LogicalResult TransferReadOp::verify() { |
| 5453 | // Consistency of elemental types in source and vector. |
| 5454 | ShapedType shapedType = getShapedType(); |
| 5455 | VectorType vectorType = getVectorType(); |
| 5456 | VectorType maskType = getMaskType(); |
| 5457 | auto paddingType = getPadding().getType(); |
| 5458 | auto permutationMap = getPermutationMap(); |
| 5459 | VectorType inferredMaskType = |
| 5460 | maskType ? inferTransferOpMaskType(vecType: vectorType, permMap: permutationMap) |
| 5461 | : VectorType(); |
| 5462 | auto sourceElementType = shapedType.getElementType(); |
| 5463 | |
| 5464 | if (static_cast<int64_t>(getIndices().size()) != shapedType.getRank()) |
| 5465 | return emitOpError(message: "requires " ) << shapedType.getRank() << " indices" ; |
| 5466 | |
| 5467 | if (failed(Result: verifyTransferOp(op: cast<VectorTransferOpInterface>(Val: getOperation()), |
| 5468 | shapedType, vectorType, maskType, |
| 5469 | inferredMaskType, permutationMap, inBounds: getInBounds()))) |
| 5470 | return failure(); |
| 5471 | |
| 5472 | if (auto sourceVectorElementType = |
| 5473 | llvm::dyn_cast<VectorType>(Val&: sourceElementType)) { |
| 5474 | // Source has vector element type. |
| 5475 | // Check that 'sourceVectorElementType' and 'paddingType' types match. |
| 5476 | if (sourceVectorElementType != paddingType) |
| 5477 | return emitOpError( |
| 5478 | message: "requires source element type and padding type to match." ); |
| 5479 | |
| 5480 | } else { |
| 5481 | // Check that 'paddingType' is valid to store in a vector type. |
| 5482 | if (!VectorType::isValidElementType(t: paddingType)) |
| 5483 | return emitOpError(message: "requires valid padding vector elemental type" ); |
| 5484 | |
| 5485 | // Check that padding type and vector element types match. |
| 5486 | if (paddingType != sourceElementType) |
| 5487 | return emitOpError( |
| 5488 | message: "requires formal padding and source of the same elemental type" ); |
| 5489 | } |
| 5490 | |
| 5491 | return verifyPermutationMap(permutationMap, |
| 5492 | emitOpError: [&](Twine t) { return emitOpError(message: t); }); |
| 5493 | } |
| 5494 | |
| 5495 | // MaskableOpInterface methods. |
| 5496 | |
| 5497 | /// Returns the mask type expected by this operation. Mostly used for |
| 5498 | /// verification purposes. It requires the operation to be vectorized." |
| 5499 | Type TransferReadOp::getExpectedMaskType() { |
| 5500 | return inferTransferOpMaskType(vecType: getVectorType(), permMap: getPermutationMap()); |
| 5501 | } |
| 5502 | |
| 5503 | //===----------------------------------------------------------------------===// |
| 5504 | // TransferReadOp: VectorTransferOpInterface methods. |
| 5505 | //===----------------------------------------------------------------------===// |
| 5506 | VectorType TransferReadOp::getVectorType() { return getResult().getType(); } |
| 5507 | |
| 5508 | Value TransferReadOp::getVector() { return getResult(); } |
| 5509 | |
| 5510 | template <typename TransferOp> |
| 5511 | static bool isInBounds(TransferOp op, int64_t resultIdx, int64_t indicesIdx) { |
| 5512 | // TODO: support more aggressive createOrFold on: |
| 5513 | // op.getIndices()[indicesIdx] + vectorType < dim(op.getSource(), indicesIdx) |
| 5514 | if (op.getShapedType().isDynamicDim(indicesIdx)) |
| 5515 | return false; |
| 5516 | // Scalable dimensions are `vscale` times larger at runtime, so the static |
| 5517 | // size is only a lower bound and cannot prove that the transfer fits. |
| 5518 | if (op.getVectorType().getScalableDims()[resultIdx]) |
| 5519 | return false; |
| 5520 | Value index = op.getIndices()[indicesIdx]; |
| 5521 | std::optional<int64_t> cstOp = getConstantIntValue(ofr: index); |
| 5522 | if (!cstOp.has_value()) |
| 5523 | return false; |
| 5524 | |
| 5525 | int64_t sourceSize = op.getShapedType().getDimSize(indicesIdx); |
| 5526 | int64_t vectorSize = op.getVectorType().getDimSize(resultIdx); |
| 5527 | // Largest index at which a full vector still fits. Computed as a subtraction |
| 5528 | // rather than adding to the index, which could overflow. The subtraction is |
| 5529 | // safe only because of the `isDynamicDim` early return above: `sourceSize` |
| 5530 | // is a real static extent here, never `ShapedType::kDynamic`, which is |
| 5531 | // `INT64_MIN` and would make this signed overflow. |
| 5532 | int64_t maxStart = sourceSize - vectorSize; |
| 5533 | |
| 5534 | // `in_bounds` guarantees that the transfer stays within the source *including |
| 5535 | // its starting point*, so a negative index is not in bounds. |
| 5536 | return *cstOp >= 0 && *cstOp <= maxStart; |
| 5537 | } |
| 5538 | |
| 5539 | template <typename TransferOp> |
| 5540 | static LogicalResult foldTransferInBoundsAttribute(TransferOp op) { |
| 5541 | // TODO: support 0-d corner case. |
| 5542 | // TODO: Be less conservative. |
| 5543 | if (op.getTransferRank() == 0) |
| 5544 | return failure(); |
| 5545 | AffineMap permutationMap = op.getPermutationMap(); |
| 5546 | bool changed = false; |
| 5547 | SmallVector<bool, 4> newInBounds; |
| 5548 | newInBounds.reserve(N: op.getTransferRank()); |
| 5549 | // Idxs of non-bcast dims - used when analysing bcast dims. |
| 5550 | SmallVector<unsigned> nonBcastDims; |
| 5551 | |
| 5552 | // 1. Process non-broadcast dims |
| 5553 | for (unsigned i = 0; i < op.getTransferRank(); ++i) { |
| 5554 | // 1.1. Already marked as in-bounds, nothing to see here. |
| 5555 | if (op.isDimInBounds(i)) { |
| 5556 | newInBounds.push_back(Elt: true); |
| 5557 | continue; |
| 5558 | } |
| 5559 | // 1.2. Currently out-of-bounds, check whether we can statically determine |
| 5560 | // it is inBounds. |
| 5561 | bool inBounds = false; |
| 5562 | auto dimExpr = dyn_cast<AffineDimExpr>(Val: permutationMap.getResult(idx: i)); |
| 5563 | if (dimExpr) { |
| 5564 | inBounds = isInBounds(op, /*resultIdx=*/i, |
| 5565 | /*indicesIdx=*/dimExpr.getPosition()); |
| 5566 | nonBcastDims.push_back(Elt: i); |
| 5567 | } |
| 5568 | |
| 5569 | newInBounds.push_back(Elt: inBounds); |
| 5570 | // We commit the pattern if it is "more inbounds". |
| 5571 | changed |= inBounds; |
| 5572 | } |
| 5573 | |
| 5574 | // 2. Handle broadcast dims |
| 5575 | // If all non-broadcast dims are "in bounds", then all bcast dims should be |
| 5576 | // "in bounds" as well. |
| 5577 | bool allNonBcastDimsInBounds = llvm::all_of( |
| 5578 | nonBcastDims, [&newInBounds](unsigned idx) { return newInBounds[idx]; }); |
| 5579 | if (allNonBcastDimsInBounds) { |
| 5580 | for (size_t idx : permutationMap.getBroadcastDims()) { |
| 5581 | changed |= !newInBounds[idx]; |
| 5582 | newInBounds[idx] = true; |
| 5583 | } |
| 5584 | } |
| 5585 | |
| 5586 | if (!changed) |
| 5587 | return failure(); |
| 5588 | // OpBuilder is only used as a helper to build an I64ArrayAttr. |
| 5589 | OpBuilder b(op.getContext()); |
| 5590 | op.setInBoundsAttr(b.getBoolArrayAttr(values: newInBounds)); |
| 5591 | return success(); |
| 5592 | } |
| 5593 | |
| 5594 | template <typename TransferOp> |
| 5595 | static LogicalResult foldTransferFullMask(TransferOp op) { |
| 5596 | auto mask = op.getMask(); |
| 5597 | if (!mask) |
| 5598 | return failure(); |
| 5599 | |
| 5600 | if (getMaskFormat(mask) != MaskFormat::AllTrue) |
| 5601 | return failure(); |
| 5602 | |
| 5603 | op.getMaskMutable().clear(); |
| 5604 | return success(); |
| 5605 | } |
| 5606 | |
| 5607 | /// When the vector type is `vector<1xT>`, the permutation map is irrelevant: |
| 5608 | /// the single vector lane always has iteration offset 0, so the element is at |
| 5609 | /// `indices` regardless of which source dimension the map points at. Replace |
| 5610 | /// with the minor identity to unblock lowering to vector.load / vector.store. |
| 5611 | template <typename TransferOp> |
| 5612 | static LogicalResult foldSize1TransferPermutationMap(TransferOp op) { |
| 5613 | VectorType vecType = op.getVectorType(); |
| 5614 | if (vecType.getRank() != 1 || vecType.getShape()[0] != 1 || |
| 5615 | vecType.isScalable()) |
| 5616 | return failure(); |
| 5617 | |
| 5618 | AffineMap map = op.getPermutationMap(); |
| 5619 | if (map.isMinorIdentity()) |
| 5620 | return failure(); |
| 5621 | |
| 5622 | int64_t srcRank = op.getShapedType().getRank(); |
| 5623 | if (srcRank < 1) |
| 5624 | return failure(); |
| 5625 | |
| 5626 | AffineMap minorIdentity = |
| 5627 | AffineMap::getMinorIdentityMap(dims: srcRank, results: 1, context: op.getContext()); |
| 5628 | op.setPermutationMapAttr(AffineMapAttr::get(value: minorIdentity)); |
| 5629 | return success(); |
| 5630 | } |
| 5631 | |
| 5632 | /// ``` |
| 5633 | /// %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]} |
| 5634 | /// : vector<1x4xf32>, tensor<4x4xf32> |
| 5635 | /// %0 = vector.transfer_read %w0[%c1, %c0], %cf0 {in_bounds = [true, true]} |
| 5636 | /// : tensor<4x4xf32>, vector<1x4xf32> |
| 5637 | /// ``` |
| 5638 | /// -> Folds into |
| 5639 | /// ``` |
| 5640 | /// %v0 |
| 5641 | /// ``` |
| 5642 | static Value foldRAW(TransferReadOp readOp) { |
| 5643 | if (!llvm::isa<RankedTensorType>(Val: readOp.getShapedType())) |
| 5644 | return {}; |
| 5645 | auto defWrite = readOp.getBase().getDefiningOp<vector::TransferWriteOp>(); |
| 5646 | while (defWrite) { |
| 5647 | if (checkSameValueRAW(defWrite, read: readOp)) |
| 5648 | return defWrite.getVector(); |
| 5649 | if (!isDisjointTransferIndices( |
| 5650 | transferA: cast<VectorTransferOpInterface>(Val: defWrite.getOperation()), |
| 5651 | transferB: cast<VectorTransferOpInterface>(Val: readOp.getOperation()))) |
| 5652 | break; |
| 5653 | defWrite = defWrite.getBase().getDefiningOp<vector::TransferWriteOp>(); |
| 5654 | } |
| 5655 | return {}; |
| 5656 | } |
| 5657 | |
| 5658 | OpFoldResult TransferReadOp::fold(FoldAdaptor) { |
| 5659 | if (Value vec = foldRAW(readOp: *this)) |
| 5660 | return vec; |
| 5661 | /// transfer_read(memrefcast) -> transfer_read |
| 5662 | if (succeeded(Result: foldTransferInBoundsAttribute(op: *this))) |
| 5663 | return getResult(); |
| 5664 | if (succeeded(Result: foldTransferFullMask(op: *this))) |
| 5665 | return getResult(); |
| 5666 | if (succeeded(Result: foldSize1TransferPermutationMap(op: *this))) |
| 5667 | return getResult(); |
| 5668 | if (succeeded(Result: memref::foldMemRefCast(op: *this))) |
| 5669 | return getResult(); |
| 5670 | if (succeeded(Result: tensor::foldTensorCast(op: *this))) |
| 5671 | return getResult(); |
| 5672 | return OpFoldResult(); |
| 5673 | } |
| 5674 | |
| 5675 | //----------------------------------------------------------------------------- |
| 5676 | // VectorUnrollOpInterface methods |
| 5677 | //----------------------------------------------------------------------------- |
| 5678 | std::optional<SmallVector<int64_t, 4>> |
| 5679 | TransferReadOp::getIterationShapeToUnroll() { |
| 5680 | return llvm::to_vector<4>(Range: getVectorType().getShape()); |
| 5681 | } |
| 5682 | |
| 5683 | void TransferReadOp::getEffects( |
| 5684 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 5685 | &effects) { |
| 5686 | if (llvm::isa<MemRefType>(Val: getShapedType())) |
| 5687 | effects.emplace_back(Args: MemoryEffects::Read::get(), Args: &getBaseMutable(), |
| 5688 | Args: SideEffects::DefaultResource::get()); |
| 5689 | } |
| 5690 | |
| 5691 | Speculation::Speculatability TransferReadOp::getSpeculatability() { |
| 5692 | if (hasPureTensorSemantics()) |
| 5693 | return Speculation::Speculatable; |
| 5694 | return Speculation::NotSpeculatable; |
| 5695 | } |
| 5696 | |
| 5697 | /// Given a projected permutation, inverse an affine map, making the unused dims |
| 5698 | /// 0 in the result. |
| 5699 | static AffineMap inverseWithUnusedDims(AffineMap map) { |
| 5700 | assert(map.isProjectedPermutation() && |
| 5701 | "expected a projected permutation map" ); |
| 5702 | SmallVector<AffineExpr> results(map.getNumInputs(), |
| 5703 | getAffineConstantExpr(constant: 0, context: map.getContext())); |
| 5704 | for (auto [idx, result] : llvm::enumerate(First: map.getResults())) { |
| 5705 | // We should only have dim exprs because this is a projected permutation. |
| 5706 | int64_t pos = cast<AffineDimExpr>(Val: result).getPosition(); |
| 5707 | results[pos] = getAffineDimExpr(position: idx, context: map.getContext()); |
| 5708 | } |
| 5709 | return AffineMap::get(/*dimCount=*/map.getNumResults(), /*symbolCount=*/0, |
| 5710 | results, context: map.getContext()); |
| 5711 | } |
| 5712 | |
| 5713 | namespace { |
| 5714 | /// Store to load forwarding for transfer operations with permuation maps. |
| 5715 | /// Even if the permutation maps are different we can still propagate the store |
| 5716 | /// into the load if the size of the dimensions read and written match. Then we |
| 5717 | /// can replace the transfer_read + transfer_write by vector.broadcast and |
| 5718 | /// vector.transpose. |
| 5719 | /// Example: |
| 5720 | /// ``` |
| 5721 | /// %w0 = vector.transfer_write %v0, %arg0[%c0, %c0, %c0] |
| 5722 | /// {in_bounds = [true, true], |
| 5723 | /// permutation_map = affine_map<(d0, d1, d2) -> (d2, d1)>} : |
| 5724 | /// vector<4x1xf32>, tensor<4x4x4xf32> |
| 5725 | /// %r = vector.transfer_read %w0[%c0, %c0, %c0], %cf0 |
| 5726 | /// {in_bounds = [true, true, true, true], |
| 5727 | /// permutation_map = affine_map<(d0, d1, d2) -> (d1, 0, d2, 0)>} : |
| 5728 | /// tensor<4x4x4xf32>, vector<1x100x4x5xf32> |
| 5729 | /// ``` |
| 5730 | /// To: |
| 5731 | /// ``` |
| 5732 | /// %0 = vector.broadcast %arg1 : vector<4x1xf32> to vector<100x5x4x1xf32> |
| 5733 | /// %r = vector.transpose %0, [3, 0, 2, 1] : |
| 5734 | /// vector<100x5x4x1xf32> to vector<1x100x4x5xf32> |
| 5735 | /// ``` |
| 5736 | struct TransferReadAfterWriteToBroadcast |
| 5737 | : public OpRewritePattern<TransferReadOp> { |
| 5738 | using Base::Base; |
| 5739 | |
| 5740 | LogicalResult matchAndRewrite(TransferReadOp readOp, |
| 5741 | PatternRewriter &rewriter) const override { |
| 5742 | auto defWrite = readOp.getBase().getDefiningOp<vector::TransferWriteOp>(); |
| 5743 | if (!defWrite) |
| 5744 | return failure(); |
| 5745 | // Bail if we need an alias analysis. |
| 5746 | if (!readOp.hasPureTensorSemantics() || !defWrite.hasPureTensorSemantics()) |
| 5747 | return failure(); |
| 5748 | // Bail in the masked case (too complex atm and needed to properly account |
| 5749 | // for padding). This includes an enclosing vector.mask. |
| 5750 | // TODO: Support mask operands and reads nested in vector.mask while |
| 5751 | // properly accounting for padding. |
| 5752 | if (readOp.getMask() || defWrite.getMask() || readOp.isMasked()) |
| 5753 | return failure(); |
| 5754 | // If indices are not the same a shift may be required, bail. |
| 5755 | if (readOp.getIndices() != defWrite.getIndices()) |
| 5756 | return failure(); |
| 5757 | // Bail if we need a bounds analysis. |
| 5758 | if (readOp.hasOutOfBoundsDim() || defWrite.hasOutOfBoundsDim()) |
| 5759 | return failure(); |
| 5760 | // TODO: If the written transfer chunk is a superset of the read transfer |
| 5761 | // chunk we could do an extract_strided_slice. |
| 5762 | if (readOp.getTransferChunkAccessed() != |
| 5763 | defWrite.getTransferChunkAccessed()) |
| 5764 | return failure(); |
| 5765 | // WriteMap: tensor -> w_vec |
| 5766 | // ReadMap: tensor -> r_vec |
| 5767 | // |
| 5768 | // inv(WriteMap): w_vec -> tensor |
| 5769 | // inv(WriteMap) o ReadMap: w_vec -> r_vec |
| 5770 | AffineMap readMap = readOp.getPermutationMap(); |
| 5771 | AffineMap writeMap = defWrite.getPermutationMap(); |
| 5772 | AffineMap invWriteMap = inverseWithUnusedDims(map: writeMap); |
| 5773 | AffineMap composedMap = readMap.compose(map: invWriteMap); |
| 5774 | // If there are any unused dims in the composedMap, we have to drop some |
| 5775 | // unit dims from the written vector before we can do transpose(broadcast). |
| 5776 | // TODO: Support this case. |
| 5777 | if (getUnusedDimsBitVector(maps: composedMap).any()) |
| 5778 | return failure(); |
| 5779 | // readVec = transpose(broadcast(writeVec)) |
| 5780 | // |
| 5781 | // Build a transpose permutation for the above transpose operation. |
| 5782 | // |
| 5783 | // Treat the composed map as having extra leading dimensions which are |
| 5784 | // the broadcasted dimensions, and treat the zeros as these new broadcasted |
| 5785 | // dimensions. |
| 5786 | SmallVector<unsigned> broadcastedDims = composedMap.getBroadcastDims(); |
| 5787 | int64_t numBroadcastedDims = broadcastedDims.size(); |
| 5788 | auto invPerm = llvm::to_vector_of<int64_t>(Range&: broadcastedDims); |
| 5789 | invPerm.resize(N: composedMap.getNumResults()); |
| 5790 | for (auto [idx, expr] : llvm::enumerate(First: composedMap.getResults())) { |
| 5791 | if (auto dim = dyn_cast<AffineDimExpr>(Val: expr)) { |
| 5792 | int64_t effectiveDim = dim.getPosition() + numBroadcastedDims; |
| 5793 | invPerm[effectiveDim] = idx; |
| 5794 | } |
| 5795 | } |
| 5796 | // Applying the inverse permutation on the readVecTy will give us the |
| 5797 | // broadcast result type. |
| 5798 | VectorType readVecTy = readOp.getVectorType(); |
| 5799 | SmallVector<int64_t> permutation = invertPermutationVector(permutation: invPerm); |
| 5800 | auto broadcastedVecTy = |
| 5801 | VectorType::get(shape: applyPermutation(input: readVecTy.getShape(), permutation: invPerm), |
| 5802 | elementType: readVecTy.getElementType(), |
| 5803 | scalableDims: applyPermutation(input: readVecTy.getScalableDims(), permutation: invPerm)); |
| 5804 | // Build the transpose(broadcast) transformation. |
| 5805 | Value vec = defWrite.getVector(); |
| 5806 | Location loc = readOp.getLoc(); |
| 5807 | vec = vector::BroadcastOp::create(builder&: rewriter, location: loc, result: broadcastedVecTy, source: vec); |
| 5808 | rewriter.replaceOpWithNewOp<vector::TransposeOp>(op: readOp, args&: vec, args&: permutation); |
| 5809 | return success(); |
| 5810 | } |
| 5811 | }; |
| 5812 | } // namespace |
| 5813 | |
| 5814 | void TransferReadOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 5815 | MLIRContext *context) { |
| 5816 | results.add<TransferReadAfterWriteToBroadcast>(arg&: context); |
| 5817 | } |
| 5818 | |
| 5819 | FailureOr<std::optional<SmallVector<Value>>> |
| 5820 | TransferReadOp::bubbleDownCasts(OpBuilder &builder) { |
| 5821 | if (!hasPureBufferSemantics()) |
| 5822 | return failure(); |
| 5823 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 5824 | results: getResult()); |
| 5825 | } |
| 5826 | |
| 5827 | //===----------------------------------------------------------------------===// |
| 5828 | // TransferWriteOp |
| 5829 | //===----------------------------------------------------------------------===// |
| 5830 | |
| 5831 | /// 1. Builder with type inference. |
| 5832 | void TransferWriteOp::build(OpBuilder &builder, OperationState &result, |
| 5833 | Value vector, Value dest, ValueRange indices, |
| 5834 | AffineMapAttr permutationMapAttr, |
| 5835 | /*optional*/ Value mask, |
| 5836 | /*optional*/ ArrayAttr inBoundsAttr) { |
| 5837 | Type resultType = llvm::dyn_cast<RankedTensorType>(Val: dest.getType()); |
| 5838 | build(odsBuilder&: builder, odsState&: result, result: resultType, valueToStore: vector, base: dest, indices, permutation_map: permutationMapAttr, |
| 5839 | mask, in_bounds: inBoundsAttr); |
| 5840 | } |
| 5841 | |
| 5842 | /// 2. Builder with type inference that sets an empty mask (variant with attrs). |
| 5843 | void TransferWriteOp::build(OpBuilder &builder, OperationState &result, |
| 5844 | Value vector, Value dest, ValueRange indices, |
| 5845 | AffineMapAttr permutationMapAttr, |
| 5846 | /*optional*/ ArrayAttr inBoundsAttr) { |
| 5847 | build(builder, result, vector, dest, indices, permutationMapAttr, |
| 5848 | /*mask=*/Value(), inBoundsAttr); |
| 5849 | } |
| 5850 | |
| 5851 | /// 3. Builder with type inference that sets an empty mask (variant without |
| 5852 | /// attrs). If `permutationMap` is null, a minor identity map is used. |
| 5853 | void TransferWriteOp::build(OpBuilder &builder, OperationState &result, |
| 5854 | Value vector, Value dest, ValueRange indices, |
| 5855 | AffineMap permutationMap, |
| 5856 | std::optional<ArrayRef<bool>> inBounds) { |
| 5857 | if (!permutationMap) |
| 5858 | permutationMap = |
| 5859 | getTransferMinorIdentityMap(shapedType: llvm::cast<ShapedType>(Val: dest.getType()), |
| 5860 | vectorType: llvm::cast<VectorType>(Val: vector.getType())); |
| 5861 | auto permutationMapAttr = AffineMapAttr::get(value: permutationMap); |
| 5862 | auto inBoundsAttr = |
| 5863 | (inBounds && !inBounds.value().empty()) |
| 5864 | ? builder.getBoolArrayAttr(values: inBounds.value()) |
| 5865 | : builder.getBoolArrayAttr(values: SmallVector<bool>( |
| 5866 | llvm::cast<VectorType>(Val: vector.getType()).getRank(), false)); |
| 5867 | build(builder, result, vector, dest, indices, permutationMapAttr, |
| 5868 | /*mask=*/Value(), inBoundsAttr); |
| 5869 | } |
| 5870 | |
| 5871 | /// 4. Builder with type inference that sets an empty mask and sets permutation |
| 5872 | /// map to 'getMinorIdentityMap'. |
| 5873 | void TransferWriteOp::build(OpBuilder &builder, OperationState &result, |
| 5874 | Value vector, Value dest, ValueRange indices, |
| 5875 | std::optional<ArrayRef<bool>> inBounds) { |
| 5876 | build(builder, result, vector, dest, indices, /*permutationMap=*/AffineMap(), |
| 5877 | inBounds); |
| 5878 | } |
| 5879 | |
| 5880 | ParseResult TransferWriteOp::parse(OpAsmParser &parser, |
| 5881 | OperationState &result) { |
| 5882 | auto &builder = parser.getBuilder(); |
| 5883 | SMLoc typesLoc; |
| 5884 | OpAsmParser::UnresolvedOperand vectorInfo, sourceInfo; |
| 5885 | SmallVector<OpAsmParser::UnresolvedOperand, 8> indexInfo; |
| 5886 | SmallVector<Type, 2> types; |
| 5887 | OpAsmParser::UnresolvedOperand maskInfo; |
| 5888 | if (parser.parseOperand(result&: vectorInfo) || parser.parseComma() || |
| 5889 | parser.parseOperand(result&: sourceInfo) || |
| 5890 | parser.parseOperandList(result&: indexInfo, delimiter: OpAsmParser::Delimiter::Square)) |
| 5891 | return failure(); |
| 5892 | ParseResult hasMask = parser.parseOptionalComma(); |
| 5893 | if (hasMask.succeeded() && parser.parseOperand(result&: maskInfo)) |
| 5894 | return failure(); |
| 5895 | if (parser.parseOptionalAttrDict(result&: result.attributes) || |
| 5896 | parser.getCurrentLocation(loc: &typesLoc) || parser.parseColonTypeList(result&: types)) |
| 5897 | return failure(); |
| 5898 | if (types.size() != 2) |
| 5899 | return parser.emitError(loc: typesLoc, message: "requires two types" ); |
| 5900 | auto indexType = builder.getIndexType(); |
| 5901 | VectorType vectorType = llvm::dyn_cast<VectorType>(Val&: types[0]); |
| 5902 | if (!vectorType) |
| 5903 | return parser.emitError(loc: typesLoc, message: "requires vector type" ); |
| 5904 | ShapedType shapedType = llvm::dyn_cast<ShapedType>(Val&: types[1]); |
| 5905 | if (!shapedType || !llvm::isa<MemRefType, RankedTensorType>(Val: shapedType)) |
| 5906 | return parser.emitError(loc: typesLoc, message: "requires memref or ranked tensor type" ); |
| 5907 | auto permMapAttrName = |
| 5908 | TransferWriteOp::getPermutationMapAttrName(name: result.name); |
| 5909 | auto permMapAttr = result.attributes.get(name: permMapAttrName); |
| 5910 | AffineMap permMap; |
| 5911 | if (!permMapAttr) { |
| 5912 | if (shapedType.getRank() < |
| 5913 | getEffectiveVectorRankForXferOp(shapedType, vectorType)) |
| 5914 | return parser.emitError(loc: typesLoc, |
| 5915 | message: "expected a custom permutation_map when " |
| 5916 | "rank(source) != rank(destination)" ); |
| 5917 | permMap = getTransferMinorIdentityMap(shapedType, vectorType); |
| 5918 | result.attributes.set(name: permMapAttrName, value: AffineMapAttr::get(value: permMap)); |
| 5919 | } else { |
| 5920 | permMap = llvm::cast<AffineMapAttr>(Val&: permMapAttr).getValue(); |
| 5921 | } |
| 5922 | auto inBoundsAttrName = TransferWriteOp::getInBoundsAttrName(name: result.name); |
| 5923 | Attribute inBoundsAttr = result.attributes.get(name: inBoundsAttrName); |
| 5924 | if (!inBoundsAttr) { |
| 5925 | result.addAttribute(name: inBoundsAttrName, |
| 5926 | attr: builder.getBoolArrayAttr( |
| 5927 | values: SmallVector<bool>(permMap.getNumResults(), false))); |
| 5928 | } |
| 5929 | if (parser.resolveOperand(operand: vectorInfo, type: vectorType, result&: result.operands) || |
| 5930 | parser.resolveOperand(operand: sourceInfo, type: shapedType, result&: result.operands) || |
| 5931 | parser.resolveOperands(operands&: indexInfo, type: indexType, result&: result.operands)) |
| 5932 | return failure(); |
| 5933 | if (hasMask.succeeded()) { |
| 5934 | if (llvm::dyn_cast<VectorType>(Val: shapedType.getElementType())) |
| 5935 | return parser.emitError( |
| 5936 | loc: maskInfo.location, message: "does not support masks with vector element type" ); |
| 5937 | if (vectorType.getRank() != permMap.getNumResults()) { |
| 5938 | return parser.emitError(loc: typesLoc, |
| 5939 | message: "expected the same rank for the vector and the " |
| 5940 | "results of the permutation map" ); |
| 5941 | } |
| 5942 | auto maskType = inferTransferOpMaskType(vecType: vectorType, permMap); |
| 5943 | if (parser.resolveOperand(operand: maskInfo, type: maskType, result&: result.operands)) |
| 5944 | return failure(); |
| 5945 | } |
| 5946 | result.addAttribute(name: TransferWriteOp::getOperandSegmentSizeAttr(), |
| 5947 | attr: builder.getDenseI32ArrayAttr( |
| 5948 | values: {1, 1, static_cast<int32_t>(indexInfo.size()), |
| 5949 | static_cast<int32_t>(hasMask.succeeded())})); |
| 5950 | return failure(IsFailure: llvm::isa<RankedTensorType>(Val: shapedType) && |
| 5951 | parser.addTypeToList(type: shapedType, result&: result.types)); |
| 5952 | } |
| 5953 | |
| 5954 | void TransferWriteOp::print(OpAsmPrinter &p) { |
| 5955 | p << " " << getVector() << ", " << getBase() << "[" << getIndices() << "]" ; |
| 5956 | if (getMask()) |
| 5957 | p << ", " << getMask(); |
| 5958 | printTransferAttrs(p, op: *this); |
| 5959 | p << " : " << getVectorType() << ", " << getShapedType(); |
| 5960 | } |
| 5961 | |
| 5962 | LogicalResult TransferWriteOp::verify() { |
| 5963 | // Consistency of elemental types in shape and vector. |
| 5964 | ShapedType shapedType = getShapedType(); |
| 5965 | VectorType vectorType = getVectorType(); |
| 5966 | VectorType maskType = getMaskType(); |
| 5967 | auto permutationMap = getPermutationMap(); |
| 5968 | VectorType inferredMaskType = |
| 5969 | maskType ? inferTransferOpMaskType(vecType: vectorType, permMap: permutationMap) |
| 5970 | : VectorType(); |
| 5971 | |
| 5972 | if (llvm::size(Range: getIndices()) != shapedType.getRank()) |
| 5973 | return emitOpError(message: "requires " ) << shapedType.getRank() << " indices" ; |
| 5974 | |
| 5975 | // We do not allow broadcast dimensions on TransferWriteOps for the moment, |
| 5976 | // as the semantics is unclear. This can be revisited later if necessary. |
| 5977 | if (hasBroadcastDim()) |
| 5978 | return emitOpError(message: "should not have broadcast dimensions" ); |
| 5979 | |
| 5980 | if (failed(Result: verifyTransferOp(op: cast<VectorTransferOpInterface>(Val: getOperation()), |
| 5981 | shapedType, vectorType, maskType, |
| 5982 | inferredMaskType, permutationMap, inBounds: getInBounds()))) |
| 5983 | return failure(); |
| 5984 | |
| 5985 | return verifyPermutationMap(permutationMap, |
| 5986 | emitOpError: [&](Twine t) { return emitOpError(message: t); }); |
| 5987 | } |
| 5988 | |
| 5989 | //===----------------------------------------------------------------------===// |
| 5990 | // TransferWriteOp: MaskableOpInterface methods. |
| 5991 | //===----------------------------------------------------------------------===// |
| 5992 | |
| 5993 | /// Returns the mask type expected by this operation. Mostly used for |
| 5994 | /// verification purposes. |
| 5995 | Type TransferWriteOp::getExpectedMaskType() { |
| 5996 | return inferTransferOpMaskType(vecType: getVectorType(), permMap: getPermutationMap()); |
| 5997 | } |
| 5998 | |
| 5999 | //===----------------------------------------------------------------------===// |
| 6000 | // TransferWriteOp: VectorTransferOpInterface methods. |
| 6001 | //===----------------------------------------------------------------------===// |
| 6002 | Value TransferWriteOp::getVector() { return getOperand(i: 0); } |
| 6003 | VectorType TransferWriteOp::getVectorType() { |
| 6004 | return cast<VectorType>(Val: getValueToStore().getType()); |
| 6005 | } |
| 6006 | |
| 6007 | //===----------------------------------------------------------------------===// |
| 6008 | // TransferWriteOp: fold methods. |
| 6009 | //===----------------------------------------------------------------------===// |
| 6010 | /// Fold: |
| 6011 | /// ``` |
| 6012 | /// %t1 = ... |
| 6013 | /// %v = vector.transfer_read %t0[%c0...], {in_bounds = [true...]} : |
| 6014 | /// tensor<static_sizesxf32>, vector<static_sizesxf32> |
| 6015 | /// %t2 = vector.transfer_write %v, %t1[%c0...] {in_bounds = [true...]} : |
| 6016 | /// vector<static_sizesxf32>, tensor<static_sizesxf32> |
| 6017 | /// ``` |
| 6018 | /// |
| 6019 | /// into: |
| 6020 | /// |
| 6021 | /// ``` |
| 6022 | /// %t0 |
| 6023 | /// ``` |
| 6024 | /// |
| 6025 | /// The producer of t1 may or may not be DCE'd depending on whether it is a |
| 6026 | /// block argument or has side effects. |
| 6027 | static LogicalResult foldReadInitWrite(TransferWriteOp write, |
| 6028 | ArrayRef<Attribute>, |
| 6029 | SmallVectorImpl<OpFoldResult> &results) { |
| 6030 | // TODO: support 0-d corner case. |
| 6031 | if (write.getTransferRank() == 0) |
| 6032 | return failure(); |
| 6033 | auto rankedTensorType = |
| 6034 | llvm::dyn_cast<RankedTensorType>(Val: write.getBase().getType()); |
| 6035 | // If not operating on tensors, bail. |
| 6036 | if (!rankedTensorType) |
| 6037 | return failure(); |
| 6038 | // If no read, bail. |
| 6039 | auto read = write.getVector().getDefiningOp<vector::TransferReadOp>(); |
| 6040 | if (!read) |
| 6041 | return failure(); |
| 6042 | // TODO: support 0-d corner case. |
| 6043 | if (read.getTransferRank() == 0) |
| 6044 | return failure(); |
| 6045 | // For now, only accept minor identity. Future: composition is minor identity. |
| 6046 | if (!read.getPermutationMap().isMinorIdentity() || |
| 6047 | !write.getPermutationMap().isMinorIdentity()) |
| 6048 | return failure(); |
| 6049 | // Bail on mismatching ranks. |
| 6050 | if (read.getTransferRank() != write.getTransferRank()) |
| 6051 | return failure(); |
| 6052 | // Bail on potential out-of-bounds accesses. |
| 6053 | if (read.hasOutOfBoundsDim() || write.hasOutOfBoundsDim()) |
| 6054 | return failure(); |
| 6055 | // Masked transfers have padding/select semantics and are not identity folds. |
| 6056 | if (read.getMask() || write.getMask()) |
| 6057 | return failure(); |
| 6058 | // Tensor types must be the same. |
| 6059 | if (read.getBase().getType() != rankedTensorType) |
| 6060 | return failure(); |
| 6061 | // Vector types must be the same. |
| 6062 | if (read.getVectorType() != write.getVectorType()) |
| 6063 | return failure(); |
| 6064 | // Vector and Tensor shapes must match. |
| 6065 | if (read.getVectorType().getShape() != rankedTensorType.getShape()) |
| 6066 | return failure(); |
| 6067 | // If any index is nonzero. |
| 6068 | auto isNotConstantZero = [](Value v) { |
| 6069 | auto cstOp = getConstantIntValue(ofr: v); |
| 6070 | return !cstOp.has_value() || cstOp.value() != 0; |
| 6071 | }; |
| 6072 | if (llvm::any_of(Range: read.getIndices(), P: isNotConstantZero) || |
| 6073 | llvm::any_of(Range: write.getIndices(), P: isNotConstantZero)) |
| 6074 | return failure(); |
| 6075 | // Success. |
| 6076 | results.push_back(Elt: read.getBase()); |
| 6077 | return success(); |
| 6078 | } |
| 6079 | |
| 6080 | static bool checkSameValueWAR(vector::TransferReadOp read, |
| 6081 | vector::TransferWriteOp write) { |
| 6082 | return read.getBase() == write.getBase() && |
| 6083 | read.getIndices() == write.getIndices() && |
| 6084 | read.getPermutationMap() == write.getPermutationMap() && |
| 6085 | read.getVectorType() == write.getVectorType() && !read.getMask() && |
| 6086 | !write.getMask(); |
| 6087 | } |
| 6088 | /// Fold transfer_write write after read: |
| 6089 | /// ``` |
| 6090 | /// %t0 = ... |
| 6091 | /// %v = vector.transfer_read %t0[%c0...] : |
| 6092 | /// tensor<static_sizesxf32>, vector<static_sizesxf32> |
| 6093 | /// %t1 = vector.transfer_write %v, %t0[%c0...] : |
| 6094 | /// vector<static_sizesxf32>, tensor<static_sizesxf32> |
| 6095 | /// ``` |
| 6096 | /// |
| 6097 | /// into: |
| 6098 | /// |
| 6099 | /// ``` |
| 6100 | /// %t0 |
| 6101 | /// ``` |
| 6102 | static LogicalResult foldWAR(TransferWriteOp write, |
| 6103 | SmallVectorImpl<OpFoldResult> &results) { |
| 6104 | if (!llvm::isa<RankedTensorType>(Val: write.getBase().getType())) |
| 6105 | return failure(); |
| 6106 | auto read = write.getVector().getDefiningOp<vector::TransferReadOp>(); |
| 6107 | if (!read) |
| 6108 | return failure(); |
| 6109 | |
| 6110 | if (!checkSameValueWAR(read, write)) |
| 6111 | return failure(); |
| 6112 | results.push_back(Elt: read.getBase()); |
| 6113 | return success(); |
| 6114 | } |
| 6115 | |
| 6116 | LogicalResult TransferWriteOp::fold(FoldAdaptor adaptor, |
| 6117 | SmallVectorImpl<OpFoldResult> &results) { |
| 6118 | if (succeeded(Result: foldReadInitWrite(write: *this, adaptor.getOperands(), results))) |
| 6119 | return success(); |
| 6120 | if (succeeded(Result: foldWAR(write: *this, results))) |
| 6121 | return success(); |
| 6122 | if (succeeded(Result: foldTransferInBoundsAttribute(op: *this))) |
| 6123 | return success(); |
| 6124 | if (succeeded(Result: foldTransferFullMask(op: *this))) |
| 6125 | return success(); |
| 6126 | if (succeeded(Result: foldSize1TransferPermutationMap(op: *this))) |
| 6127 | return success(); |
| 6128 | return memref::foldMemRefCast(op: *this); |
| 6129 | } |
| 6130 | |
| 6131 | //===----------------------------------------------------------------------===// |
| 6132 | // TransferWriteOp: other methods. |
| 6133 | //===----------------------------------------------------------------------===// |
| 6134 | std::optional<SmallVector<int64_t, 4>> |
| 6135 | TransferWriteOp::getIterationShapeToUnroll() { |
| 6136 | return llvm::to_vector<4>(Range: getVectorType().getShape()); |
| 6137 | } |
| 6138 | |
| 6139 | void TransferWriteOp::getEffects( |
| 6140 | SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>> |
| 6141 | &effects) { |
| 6142 | if (llvm::isa<MemRefType>(Val: getShapedType())) |
| 6143 | effects.emplace_back(Args: MemoryEffects::Write::get(), Args: &getBaseMutable(), |
| 6144 | Args: SideEffects::DefaultResource::get()); |
| 6145 | } |
| 6146 | |
| 6147 | Speculation::Speculatability TransferWriteOp::getSpeculatability() { |
| 6148 | if (hasPureTensorSemantics()) |
| 6149 | return Speculation::Speculatable; |
| 6150 | return Speculation::NotSpeculatable; |
| 6151 | } |
| 6152 | |
| 6153 | namespace { |
| 6154 | /// Remove dead transfer write from the SSA chain so that it an be eliminated by |
| 6155 | /// DCE |
| 6156 | /// ``` |
| 6157 | /// %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]} |
| 6158 | /// : vector<1x4xf32>, tensor<4x4xf32> |
| 6159 | /// %w1 = vector.transfer_write %v0, %w0[%c2, %c0] {in_bounds = [true, true]} |
| 6160 | /// : vector<1x4xf32>, tensor<4x4xf32> |
| 6161 | /// %w2 = vector.transfer_write %v1, %w1[%c1, %c0] {in_bounds = [true, true]} |
| 6162 | /// : vector<1x4xf32>, tensor<4x4xf32> |
| 6163 | /// ``` |
| 6164 | /// |
| 6165 | /// into: |
| 6166 | /// |
| 6167 | /// ``` |
| 6168 | /// %w0 = vector.transfer_write %v0, %arg0[%c1, %c0] {in_bounds = [true, true]} |
| 6169 | /// : vector<1x4xf32>, tensor<4x4xf32> |
| 6170 | /// %w1 = vector.transfer_write %v0, %arg0[%c2, %c0] {in_bounds = [true, true]} |
| 6171 | /// : vector<1x4xf32>, tensor<4x4xf32> |
| 6172 | /// %w2 = vector.transfer_write %v1, %w1[%c1, %c0] {in_bounds = [true, true]} |
| 6173 | /// : vector<1x4xf32>, tensor<4x4xf32> |
| 6174 | /// ``` |
| 6175 | /// |
| 6176 | /// `%w0 = vector.transfer_write` op will be removed by DCE if it doesn't have |
| 6177 | /// any other uses. |
| 6178 | class FoldWaw final : public OpRewritePattern<TransferWriteOp> { |
| 6179 | public: |
| 6180 | using Base::Base; |
| 6181 | LogicalResult matchAndRewrite(TransferWriteOp writeOp, |
| 6182 | PatternRewriter &rewriter) const override { |
| 6183 | if (!llvm::isa<RankedTensorType>(Val: writeOp.getShapedType())) |
| 6184 | return failure(); |
| 6185 | vector::TransferWriteOp writeToModify = writeOp; |
| 6186 | |
| 6187 | auto defWrite = writeOp.getBase().getDefiningOp<vector::TransferWriteOp>(); |
| 6188 | while (defWrite) { |
| 6189 | if (checkSameValueWAW(write: writeOp, priorWrite: defWrite)) { |
| 6190 | rewriter.modifyOpInPlace(root: writeToModify, callable: [&]() { |
| 6191 | writeToModify.getBaseMutable().assign(value: defWrite.getBase()); |
| 6192 | }); |
| 6193 | return success(); |
| 6194 | } |
| 6195 | if (!isDisjointTransferIndices( |
| 6196 | transferA: cast<VectorTransferOpInterface>(Val: defWrite.getOperation()), |
| 6197 | transferB: cast<VectorTransferOpInterface>(Val: writeOp.getOperation()))) |
| 6198 | break; |
| 6199 | // If the previous write op doesn't have any other use we an safely look |
| 6200 | // at the previous store to see if it can be removed. |
| 6201 | if (!defWrite->hasOneUse()) |
| 6202 | break; |
| 6203 | writeToModify = defWrite; |
| 6204 | defWrite = defWrite.getBase().getDefiningOp<vector::TransferWriteOp>(); |
| 6205 | } |
| 6206 | return failure(); |
| 6207 | } |
| 6208 | }; |
| 6209 | |
| 6210 | /// Rewrite tensor::ExtractSliceOp(vector::TransferWriteOp) to |
| 6211 | /// vector::TransferWriteOp(tensor::ExtractSliceOp) if the full slice is |
| 6212 | /// overwritten and inserted into another tensor. After this rewrite, the |
| 6213 | /// operations bufferize in-place since all of them work on the same slice. |
| 6214 | /// |
| 6215 | /// For example: |
| 6216 | /// ```mlir |
| 6217 | /// %0 = vector.transfer_write %vec, %init_tensor[%c0, %c0] |
| 6218 | /// : vector<8x16xf32>, tensor<8x16xf32> |
| 6219 | /// %1 = tensor.extract_slice %0[0, 0] [%sz0, %sz1] [1, 1] |
| 6220 | /// : tensor<8x16xf32> to tensor<?x?xf32> |
| 6221 | /// %r = tensor.insert_slice %1 into %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1] |
| 6222 | /// : tensor<?x?xf32> into tensor<27x37xf32> |
| 6223 | /// ``` |
| 6224 | /// folds to |
| 6225 | /// ```mlir |
| 6226 | /// %0 = tensor.extract_slice %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1] |
| 6227 | /// : tensor<27x37xf32> to tensor<?x?xf32> |
| 6228 | /// %1 = vector.transfer_write %vec, %0[%c0, %c0] |
| 6229 | /// : vector<8x16xf32>, tensor<?x?xf32> |
| 6230 | /// %r = tensor.insert_slice %1 into %iter_arg[%iv0, %iv1] [%sz0, %sz1] [1, 1] |
| 6231 | /// : tensor<?x?xf32> into tensor<27x37xf32> |
| 6232 | /// ``` |
| 6233 | struct |
| 6234 | : public OpRewritePattern<tensor::InsertSliceOp> { |
| 6235 | public: |
| 6236 | using Base::Base; |
| 6237 | |
| 6238 | LogicalResult matchAndRewrite(tensor::InsertSliceOp insertOp, |
| 6239 | PatternRewriter &rewriter) const override { |
| 6240 | if (!insertOp.hasUnitStride()) |
| 6241 | return failure(); |
| 6242 | auto = |
| 6243 | insertOp.getSource().getDefiningOp<tensor::ExtractSliceOp>(); |
| 6244 | if (!extractOp || !extractOp.hasUnitStride() || !extractOp->hasOneUse()) |
| 6245 | return failure(); |
| 6246 | auto transferOp = extractOp.getSource().getDefiningOp<TransferWriteOp>(); |
| 6247 | if (!transferOp || !transferOp->hasOneUse()) |
| 6248 | return failure(); |
| 6249 | |
| 6250 | // Fail if vector::TransferWriteOp or tensor::ExtractSliceOp is |
| 6251 | // rank-reducing. |
| 6252 | if (insertOp.getSourceType().getRank() != transferOp.getTransferRank()) { |
| 6253 | return rewriter.notifyMatchFailure(arg&: insertOp, |
| 6254 | msg: "use-def chain is rank-reducing" ); |
| 6255 | } |
| 6256 | |
| 6257 | // Fail if tensor::ExtractSliceOp has non-zero offset. |
| 6258 | if (!extractOp.hasZeroOffset()) { |
| 6259 | return rewriter.notifyMatchFailure(arg&: insertOp, |
| 6260 | msg: "ExtractSliceOp has non-zero offset" ); |
| 6261 | } |
| 6262 | |
| 6263 | // Fail if tensor::TransferWriteOp has non-zero offset. |
| 6264 | if (!llvm::all_of(Range: transferOp.getIndices(), P: [](Value value) { |
| 6265 | return getConstantIntValue(ofr: value) == static_cast<int64_t>(0); |
| 6266 | })) { |
| 6267 | return rewriter.notifyMatchFailure(arg&: insertOp, |
| 6268 | msg: "TranferWriteOp has non-zero offset" ); |
| 6269 | } |
| 6270 | |
| 6271 | // Fail if tensor::ExtractSliceOp and tensor::InsertSliceOp sizes differ. |
| 6272 | if (insertOp.getMixedSizes().size() != extractOp.getMixedSizes().size()) { |
| 6273 | return rewriter.notifyMatchFailure( |
| 6274 | arg&: insertOp, msg: "InsertSliceOp and ExtractSliceOp ranks differ" ); |
| 6275 | } |
| 6276 | |
| 6277 | for (auto [insertSize, extractSize] : |
| 6278 | llvm::zip_equal(t: insertOp.getMixedSizes(), u: extractOp.getMixedSizes())) { |
| 6279 | if (!isEqualConstantIntOrValue(ofr1: insertSize, ofr2: extractSize)) { |
| 6280 | return rewriter.notifyMatchFailure( |
| 6281 | arg&: insertOp, msg: "InsertSliceOp and ExtractSliceOp sizes differ" ); |
| 6282 | } |
| 6283 | } |
| 6284 | |
| 6285 | // Fail if the vector::TransferWriteOp may not overwrite the full tensor. |
| 6286 | assert(transferOp.getVectorType().hasStaticShape() && |
| 6287 | "expected vector to have a static shape" ); |
| 6288 | ArrayRef<int64_t> vectorShape = transferOp.getVectorType().getShape(); |
| 6289 | SmallVector<int64_t> resultShape = applyPermutationMap( |
| 6290 | map: transferOp.getPermutationMap(), source: transferOp.getShapedType().getShape()); |
| 6291 | if (transferOp.getMask() || !vectorShape.equals(RHS: resultShape)) { |
| 6292 | return rewriter.notifyMatchFailure( |
| 6293 | arg&: insertOp, msg: "TransferWriteOp may not write the full tensor." ); |
| 6294 | } |
| 6295 | |
| 6296 | // Swap the tensor::ExtractSliceOp in front of the vector::TransferWriteOp. |
| 6297 | // Set all in_bounds to false and let the folder infer them. |
| 6298 | SmallVector<bool> newInBounds(vectorShape.size(), false); |
| 6299 | auto = tensor::ExtractSliceOp::create( |
| 6300 | builder&: rewriter, location: extractOp.getLoc(), resultType: insertOp.getSourceType(), |
| 6301 | source: insertOp.getDest(), offsets: insertOp.getMixedOffsets(), |
| 6302 | sizes: insertOp.getMixedSizes(), strides: insertOp.getMixedStrides()); |
| 6303 | auto newTransferWriteOp = TransferWriteOp::create( |
| 6304 | builder&: rewriter, location: transferOp.getLoc(), vector: transferOp.getVector(), |
| 6305 | dest: newExtractOp.getResult(), indices: transferOp.getIndices(), |
| 6306 | permutationMapAttr: transferOp.getPermutationMapAttr(), |
| 6307 | inBoundsAttr: rewriter.getBoolArrayAttr(values: newInBounds)); |
| 6308 | rewriter.modifyOpInPlace(root: insertOp, callable: [&]() { |
| 6309 | insertOp.getSourceMutable().assign(value: newTransferWriteOp.getResult()); |
| 6310 | }); |
| 6311 | return success(); |
| 6312 | } |
| 6313 | }; |
| 6314 | |
| 6315 | } // namespace |
| 6316 | |
| 6317 | void TransferWriteOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 6318 | MLIRContext *context) { |
| 6319 | results.add<FoldWaw, SwapExtractSliceOfTransferWrite>(arg&: context); |
| 6320 | } |
| 6321 | |
| 6322 | FailureOr<std::optional<SmallVector<Value>>> |
| 6323 | TransferWriteOp::bubbleDownCasts(OpBuilder &builder) { |
| 6324 | if (!hasPureBufferSemantics()) |
| 6325 | return failure(); |
| 6326 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6327 | results: ValueRange()); |
| 6328 | } |
| 6329 | |
| 6330 | //===----------------------------------------------------------------------===// |
| 6331 | // LoadOp |
| 6332 | //===----------------------------------------------------------------------===// |
| 6333 | |
| 6334 | static ParseResult parseBoolAttr(OpAsmParser &parser, BoolAttr &result) { |
| 6335 | Attribute attr; |
| 6336 | if (parser.parseAttribute(result&: attr)) |
| 6337 | return failure(); |
| 6338 | result = dyn_cast<BoolAttr>(Val&: attr); |
| 6339 | if (!result) |
| 6340 | return parser.emitError(loc: parser.getCurrentLocation(), |
| 6341 | message: "expected boolean attribute" ); |
| 6342 | return success(); |
| 6343 | } |
| 6344 | |
| 6345 | static void printBoolAttr(OpAsmPrinter &printer, Operation *, BoolAttr attr) { |
| 6346 | printer.printAttribute(attr); |
| 6347 | } |
| 6348 | |
| 6349 | static LogicalResult verifyLoadStoreMemRefLayout(Operation *op, |
| 6350 | VectorType vecTy, |
| 6351 | MemRefType memRefTy) { |
| 6352 | // If rank==0 or size==1 it's equivalent to scalar load/store, so we don't |
| 6353 | // need any strides limitations. |
| 6354 | if (!vecTy.isScalable() && |
| 6355 | (vecTy.getRank() == 0 || vecTy.getNumElements() == 1)) |
| 6356 | return success(); |
| 6357 | |
| 6358 | if (!memRefTy.isLastDimUnitStride()) |
| 6359 | return op->emitOpError(message: "most minor memref dim must have unit stride" ); |
| 6360 | return success(); |
| 6361 | } |
| 6362 | |
| 6363 | LogicalResult vector::LoadOp::verify() { |
| 6364 | VectorType resVecTy = getVectorType(); |
| 6365 | MemRefType memRefTy = getMemRefType(); |
| 6366 | |
| 6367 | if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: resVecTy, memRefTy))) |
| 6368 | return failure(); |
| 6369 | |
| 6370 | // Negative strides are not supported on vector.load. The lowering to LLVM |
| 6371 | // emits arithmetic operations (e.g., GEP, mul) with nuw flags that assume |
| 6372 | // non-negative strides to avoid undefined behavior. |
| 6373 | if (memref::hasNegativeStaticStride(memRefTy)) |
| 6374 | return emitOpError(message: "memref strides must be non-negative" ); |
| 6375 | |
| 6376 | if (memRefTy.getRank() < resVecTy.getRank()) |
| 6377 | return emitOpError( |
| 6378 | message: "destination memref has lower rank than the result vector" ); |
| 6379 | |
| 6380 | // Checks for vector memrefs. |
| 6381 | Type memElemTy = memRefTy.getElementType(); |
| 6382 | if (auto memVecTy = llvm::dyn_cast<VectorType>(Val&: memElemTy)) { |
| 6383 | if (memVecTy != resVecTy) |
| 6384 | return emitOpError(message: "base memref and result vector types should match" ); |
| 6385 | memElemTy = memVecTy.getElementType(); |
| 6386 | } |
| 6387 | |
| 6388 | if (resVecTy.getElementType() != memElemTy) |
| 6389 | return emitOpError(message: "base and result element types should match" ); |
| 6390 | if (llvm::size(Range: getIndices()) != memRefTy.getRank()) |
| 6391 | return emitOpError(message: "requires " ) << memRefTy.getRank() << " indices" ; |
| 6392 | return success(); |
| 6393 | } |
| 6394 | |
| 6395 | OpFoldResult LoadOp::fold(FoldAdaptor) { |
| 6396 | if (succeeded(Result: memref::foldMemRefCast(op: *this))) |
| 6397 | return getResult(); |
| 6398 | return OpFoldResult(); |
| 6399 | } |
| 6400 | |
| 6401 | FailureOr<std::optional<SmallVector<Value>>> |
| 6402 | LoadOp::bubbleDownCasts(OpBuilder &builder) { |
| 6403 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6404 | results: getResult()); |
| 6405 | } |
| 6406 | |
| 6407 | //----------------------------------------------------------------------------- |
| 6408 | // VectorUnrollOpInterface methods |
| 6409 | //----------------------------------------------------------------------------- |
| 6410 | std::optional<SmallVector<int64_t, 4>> LoadOp::getIterationShapeToUnroll() { |
| 6411 | return llvm::to_vector<4>(Range: getVectorType().getShape()); |
| 6412 | } |
| 6413 | |
| 6414 | //===----------------------------------------------------------------------===// |
| 6415 | // StoreOp |
| 6416 | //===----------------------------------------------------------------------===// |
| 6417 | |
| 6418 | LogicalResult vector::StoreOp::verify() { |
| 6419 | VectorType valueVecTy = getVectorType(); |
| 6420 | MemRefType memRefTy = getMemRefType(); |
| 6421 | |
| 6422 | if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: valueVecTy, memRefTy))) |
| 6423 | return failure(); |
| 6424 | |
| 6425 | // Negative strides are not supported on vector.store. The lowering to LLVM |
| 6426 | // emits arithmetic operations (e.g., GEP, mul) with nuw flags that assume |
| 6427 | // non-negative strides to avoid undefined behavior. |
| 6428 | if (memref::hasNegativeStaticStride(memRefTy)) |
| 6429 | return emitOpError(message: "memref strides must be non-negative" ); |
| 6430 | |
| 6431 | if (memRefTy.getRank() < valueVecTy.getRank()) |
| 6432 | return emitOpError(message: "source memref has lower rank than the vector to store" ); |
| 6433 | |
| 6434 | // Checks for vector memrefs. |
| 6435 | Type memElemTy = memRefTy.getElementType(); |
| 6436 | if (auto memVecTy = llvm::dyn_cast<VectorType>(Val&: memElemTy)) { |
| 6437 | if (memVecTy != valueVecTy) |
| 6438 | return emitOpError( |
| 6439 | message: "base memref and valueToStore vector types should match" ); |
| 6440 | memElemTy = memVecTy.getElementType(); |
| 6441 | } |
| 6442 | |
| 6443 | if (valueVecTy.getElementType() != memElemTy) |
| 6444 | return emitOpError(message: "base and valueToStore element type should match" ); |
| 6445 | if (llvm::size(Range: getIndices()) != memRefTy.getRank()) |
| 6446 | return emitOpError(message: "requires " ) << memRefTy.getRank() << " indices" ; |
| 6447 | return success(); |
| 6448 | } |
| 6449 | |
| 6450 | LogicalResult StoreOp::fold(FoldAdaptor adaptor, |
| 6451 | SmallVectorImpl<OpFoldResult> &results) { |
| 6452 | return memref::foldMemRefCast(op: *this); |
| 6453 | } |
| 6454 | |
| 6455 | FailureOr<std::optional<SmallVector<Value>>> |
| 6456 | StoreOp::bubbleDownCasts(OpBuilder &builder) { |
| 6457 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6458 | results: ValueRange()); |
| 6459 | } |
| 6460 | |
| 6461 | //----------------------------------------------------------------------------- |
| 6462 | // VectorUnrollOpInterface methods |
| 6463 | //----------------------------------------------------------------------------- |
| 6464 | std::optional<SmallVector<int64_t, 4>> StoreOp::getIterationShapeToUnroll() { |
| 6465 | return llvm::to_vector<4>(Range: getVectorType().getShape()); |
| 6466 | } |
| 6467 | |
| 6468 | //===----------------------------------------------------------------------===// |
| 6469 | // MaskedLoadOp |
| 6470 | //===----------------------------------------------------------------------===// |
| 6471 | |
| 6472 | LogicalResult MaskedLoadOp::verify() { |
| 6473 | VectorType maskVType = getMaskVectorType(); |
| 6474 | VectorType passVType = getPassThruVectorType(); |
| 6475 | VectorType resVType = getVectorType(); |
| 6476 | MemRefType memType = getMemRefType(); |
| 6477 | |
| 6478 | if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: resVType, memRefTy: memType))) |
| 6479 | return failure(); |
| 6480 | |
| 6481 | // Negative strides are not supported on vector.maskedload. The lowering to |
| 6482 | // LLVM emits arithmetic operations (e.g., GEP, mul) with nuw flags that |
| 6483 | // assume non-negative strides to avoid undefined behavior. |
| 6484 | if (memref::hasNegativeStaticStride(memRefTy: memType)) |
| 6485 | return emitOpError(message: "memref strides must be non-negative" ); |
| 6486 | |
| 6487 | if (failed( |
| 6488 | Result: verifyElementTypesMatch(op: *this, lhs: memType, rhs: resVType, lhsName: "base" , rhsName: "result" ))) |
| 6489 | return failure(); |
| 6490 | if (llvm::size(Range: getIndices()) != memType.getRank()) |
| 6491 | return emitOpError(message: "requires " ) << memType.getRank() << " indices" ; |
| 6492 | if (resVType.getShape() != maskVType.getShape()) |
| 6493 | return emitOpError(message: "expected result shape to match mask shape" ); |
| 6494 | if (resVType != passVType) |
| 6495 | return emitOpError(message: "expected pass_thru of same type as result type" ); |
| 6496 | return success(); |
| 6497 | } |
| 6498 | |
| 6499 | namespace { |
| 6500 | class MaskedLoadFolder final : public OpRewritePattern<MaskedLoadOp> { |
| 6501 | public: |
| 6502 | using Base::Base; |
| 6503 | LogicalResult matchAndRewrite(MaskedLoadOp load, |
| 6504 | PatternRewriter &rewriter) const override { |
| 6505 | switch (getMaskFormat(mask: load.getMask())) { |
| 6506 | case MaskFormat::AllTrue: |
| 6507 | rewriter.replaceOpWithNewOp<vector::LoadOp>( |
| 6508 | op: load, args: load.getType(), args: load.getBase(), args: load.getIndices()); |
| 6509 | return success(); |
| 6510 | case MaskFormat::AllFalse: |
| 6511 | rewriter.replaceOp(op: load, newValues: load.getPassThru()); |
| 6512 | return success(); |
| 6513 | case MaskFormat::Unknown: |
| 6514 | return failure(); |
| 6515 | } |
| 6516 | llvm_unreachable("Unexpected 1DMaskFormat on MaskedLoad" ); |
| 6517 | } |
| 6518 | }; |
| 6519 | } // namespace |
| 6520 | |
| 6521 | void MaskedLoadOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 6522 | MLIRContext *context) { |
| 6523 | results.add<MaskedLoadFolder>(arg&: context); |
| 6524 | } |
| 6525 | |
| 6526 | OpFoldResult MaskedLoadOp::fold(FoldAdaptor) { |
| 6527 | if (succeeded(Result: memref::foldMemRefCast(op: *this))) |
| 6528 | return getResult(); |
| 6529 | return OpFoldResult(); |
| 6530 | } |
| 6531 | |
| 6532 | FailureOr<std::optional<SmallVector<Value>>> |
| 6533 | MaskedLoadOp::bubbleDownCasts(OpBuilder &builder) { |
| 6534 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6535 | results: getResult()); |
| 6536 | } |
| 6537 | |
| 6538 | //===----------------------------------------------------------------------===// |
| 6539 | // MaskedStoreOp |
| 6540 | //===----------------------------------------------------------------------===// |
| 6541 | |
| 6542 | LogicalResult MaskedStoreOp::verify() { |
| 6543 | VectorType maskVType = getMaskVectorType(); |
| 6544 | VectorType valueVType = getVectorType(); |
| 6545 | MemRefType memType = getMemRefType(); |
| 6546 | |
| 6547 | if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: valueVType, memRefTy: memType))) |
| 6548 | return failure(); |
| 6549 | |
| 6550 | // Negative strides are not supported on vector.maskedstore. The lowering to |
| 6551 | // LLVM emits arithmetic operations (e.g., GEP, mul) with nuw flags that |
| 6552 | // assume non-negative strides to avoid undefined behavior. |
| 6553 | if (memref::hasNegativeStaticStride(memRefTy: memType)) |
| 6554 | return emitOpError(message: "memref strides must be non-negative" ); |
| 6555 | |
| 6556 | if (failed(Result: verifyElementTypesMatch(op: *this, lhs: memType, rhs: valueVType, lhsName: "base" , |
| 6557 | rhsName: "valueToStore" ))) |
| 6558 | return failure(); |
| 6559 | if (llvm::size(Range: getIndices()) != memType.getRank()) |
| 6560 | return emitOpError(message: "requires " ) << memType.getRank() << " indices" ; |
| 6561 | if (valueVType.getShape() != maskVType.getShape()) |
| 6562 | return emitOpError(message: "expected valueToStore shape to match mask shape" ); |
| 6563 | return success(); |
| 6564 | } |
| 6565 | |
| 6566 | namespace { |
| 6567 | class MaskedStoreFolder final : public OpRewritePattern<MaskedStoreOp> { |
| 6568 | public: |
| 6569 | using Base::Base; |
| 6570 | LogicalResult matchAndRewrite(MaskedStoreOp store, |
| 6571 | PatternRewriter &rewriter) const override { |
| 6572 | switch (getMaskFormat(mask: store.getMask())) { |
| 6573 | case MaskFormat::AllTrue: |
| 6574 | rewriter.replaceOpWithNewOp<vector::StoreOp>( |
| 6575 | op: store, args: store.getValueToStore(), args: store.getBase(), args: store.getIndices()); |
| 6576 | return success(); |
| 6577 | case MaskFormat::AllFalse: |
| 6578 | rewriter.eraseOp(op: store); |
| 6579 | return success(); |
| 6580 | case MaskFormat::Unknown: |
| 6581 | return failure(); |
| 6582 | } |
| 6583 | llvm_unreachable("Unexpected 1DMaskFormat on MaskedStore" ); |
| 6584 | } |
| 6585 | }; |
| 6586 | } // namespace |
| 6587 | |
| 6588 | void MaskedStoreOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 6589 | MLIRContext *context) { |
| 6590 | results.add<MaskedStoreFolder>(arg&: context); |
| 6591 | } |
| 6592 | |
| 6593 | LogicalResult MaskedStoreOp::fold(FoldAdaptor adaptor, |
| 6594 | SmallVectorImpl<OpFoldResult> &results) { |
| 6595 | return memref::foldMemRefCast(op: *this); |
| 6596 | } |
| 6597 | |
| 6598 | FailureOr<std::optional<SmallVector<Value>>> |
| 6599 | MaskedStoreOp::bubbleDownCasts(OpBuilder &builder) { |
| 6600 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6601 | results: ValueRange()); |
| 6602 | } |
| 6603 | |
| 6604 | //===----------------------------------------------------------------------===// |
| 6605 | // GatherOp |
| 6606 | //===----------------------------------------------------------------------===// |
| 6607 | |
| 6608 | LogicalResult GatherOp::verify() { |
| 6609 | VectorType indVType = getIndexVectorType(); |
| 6610 | VectorType maskVType = getMaskVectorType(); |
| 6611 | VectorType resVType = getVectorType(); |
| 6612 | ShapedType baseType = getBaseType(); |
| 6613 | |
| 6614 | if (!llvm::isa<MemRefType, RankedTensorType>(Val: baseType)) |
| 6615 | return emitOpError(message: "requires base to be a memref or ranked tensor type" ); |
| 6616 | |
| 6617 | // Negative strides are not supported on vector.gather. |
| 6618 | // The lowering to LLVM emits arithmetic operations (e.g., GEP, mul) with nuw |
| 6619 | // flags that assume non-negative strides to avoid undefined behavior. |
| 6620 | if (auto memRefType = dyn_cast<MemRefType>(Val&: baseType)) |
| 6621 | if (memref::hasNegativeStaticStride(memRefTy: memRefType)) |
| 6622 | return emitOpError(message: "memref strides must be non-negative" ); |
| 6623 | |
| 6624 | if (failed( |
| 6625 | Result: verifyElementTypesMatch(op: *this, lhs: baseType, rhs: resVType, lhsName: "base" , rhsName: "result" ))) |
| 6626 | return failure(); |
| 6627 | if (llvm::size(Range: getOffsets()) != baseType.getRank()) |
| 6628 | return emitOpError(message: "requires " ) << baseType.getRank() << " indices" ; |
| 6629 | if (resVType.getShape() != indVType.getShape()) |
| 6630 | return emitOpError(message: "expected result dim to match indices dim" ); |
| 6631 | if (resVType.getShape() != maskVType.getShape()) |
| 6632 | return emitOpError(message: "expected result dim to match mask dim" ); |
| 6633 | if (resVType != getPassThruVectorType()) |
| 6634 | return emitOpError(message: "expected pass_thru of same type as result type" ); |
| 6635 | if (getAlignmentAttr() && !isa<MemRefType>(Val: baseType)) { |
| 6636 | return emitOpError( |
| 6637 | message: "alignment is only supported for memref bases, not tensor bases" ); |
| 6638 | } |
| 6639 | return success(); |
| 6640 | } |
| 6641 | |
| 6642 | //----------------------------------------------------------------------------- |
| 6643 | // MaskableOpInterface methods. |
| 6644 | //----------------------------------------------------------------------------- |
| 6645 | /// Cheeck if `indexVec` is constant 1D vec of consecutive values [0, 1, 2, ...] |
| 6646 | static LogicalResult isZeroBasedContiguousSeq(Value indexVec) { |
| 6647 | auto vecType = dyn_cast<VectorType>(Val: indexVec.getType()); |
| 6648 | if (!vecType || vecType.getRank() != 1 || vecType.isScalable()) |
| 6649 | return failure(); |
| 6650 | |
| 6651 | if (indexVec.getDefiningOp<StepOp>()) |
| 6652 | return success(); |
| 6653 | |
| 6654 | DenseIntElementsAttr elements; |
| 6655 | if (!matchPattern(value: indexVec, pattern: m_Constant(bind_value: &elements))) |
| 6656 | return failure(); |
| 6657 | |
| 6658 | return success( |
| 6659 | IsSuccess: llvm::equal(LRange&: elements, RRange: llvm::seq<int64_t>(Begin: 0, End: vecType.getNumElements()))); |
| 6660 | } |
| 6661 | |
| 6662 | //----------------------------------------------------------------------------- |
| 6663 | // VectorUnrollOpInterface methods |
| 6664 | //----------------------------------------------------------------------------- |
| 6665 | std::optional<SmallVector<int64_t, 4>> GatherOp::getIterationShapeToUnroll() { |
| 6666 | return llvm::to_vector<4>(Range: getVectorType().getShape()); |
| 6667 | } |
| 6668 | |
| 6669 | namespace { |
| 6670 | class GatherFolder final : public OpRewritePattern<GatherOp> { |
| 6671 | public: |
| 6672 | using Base::Base; |
| 6673 | LogicalResult matchAndRewrite(GatherOp gather, |
| 6674 | PatternRewriter &rewriter) const override { |
| 6675 | switch (getMaskFormat(mask: gather.getMask())) { |
| 6676 | case MaskFormat::AllTrue: |
| 6677 | return failure(); // no unmasked equivalent |
| 6678 | case MaskFormat::AllFalse: |
| 6679 | rewriter.replaceOp(op: gather, newValues: gather.getPassThru()); |
| 6680 | return success(); |
| 6681 | case MaskFormat::Unknown: |
| 6682 | return failure(); |
| 6683 | } |
| 6684 | llvm_unreachable("Unexpected 1DMaskFormat on GatherFolder" ); |
| 6685 | } |
| 6686 | }; |
| 6687 | |
| 6688 | /// Fold gathers with consecutive offsets [0, 1, 2, ...] into contiguous |
| 6689 | /// maskedload. Only 1D fixed vectors are supported for now. |
| 6690 | class FoldContiguousGather final : public OpRewritePattern<GatherOp> { |
| 6691 | public: |
| 6692 | using Base::Base; |
| 6693 | LogicalResult matchAndRewrite(GatherOp op, |
| 6694 | PatternRewriter &rewriter) const override { |
| 6695 | if (!isa<MemRefType>(Val: op.getBase().getType())) |
| 6696 | return rewriter.notifyMatchFailure(arg&: op, msg: "base must be of memref type" ); |
| 6697 | |
| 6698 | if (failed(Result: isZeroBasedContiguousSeq(indexVec: op.getIndices()))) |
| 6699 | return failure(); |
| 6700 | |
| 6701 | rewriter.replaceOpWithNewOp<MaskedLoadOp>(op, args: op.getType(), args: op.getBase(), |
| 6702 | args: op.getOffsets(), args: op.getMask(), |
| 6703 | args: op.getPassThru()); |
| 6704 | return success(); |
| 6705 | } |
| 6706 | }; |
| 6707 | } // namespace |
| 6708 | |
| 6709 | void GatherOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 6710 | MLIRContext *context) { |
| 6711 | results.add<GatherFolder, FoldContiguousGather>(arg&: context); |
| 6712 | } |
| 6713 | |
| 6714 | FailureOr<std::optional<SmallVector<Value>>> |
| 6715 | GatherOp::bubbleDownCasts(OpBuilder &builder) { |
| 6716 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6717 | results: getResult()); |
| 6718 | } |
| 6719 | |
| 6720 | //===----------------------------------------------------------------------===// |
| 6721 | // ScatterOp |
| 6722 | //===----------------------------------------------------------------------===// |
| 6723 | |
| 6724 | LogicalResult ScatterOp::verify() { |
| 6725 | VectorType indVType = getIndexVectorType(); |
| 6726 | VectorType maskVType = getMaskVectorType(); |
| 6727 | VectorType valueVType = getVectorType(); |
| 6728 | ShapedType baseType = getBaseType(); |
| 6729 | |
| 6730 | if (!llvm::isa<MemRefType, RankedTensorType>(Val: baseType)) |
| 6731 | return emitOpError(message: "requires base to be a memref or ranked tensor type" ); |
| 6732 | |
| 6733 | // Negative strides are not supported on vector.scatter. |
| 6734 | // The lowering to LLVM emits arithmetic operations (e.g., GEP, mul) with nuw |
| 6735 | // flags that assume non-negative strides to avoid undefined behavior. |
| 6736 | if (auto memRefType = dyn_cast<MemRefType>(Val&: baseType)) |
| 6737 | if (memref::hasNegativeStaticStride(memRefTy: memRefType)) |
| 6738 | return emitOpError(message: "memref strides must be non-negative" ); |
| 6739 | |
| 6740 | if (failed(Result: verifyElementTypesMatch(op: *this, lhs: baseType, rhs: valueVType, lhsName: "base" , |
| 6741 | rhsName: "valueToStore" ))) |
| 6742 | return failure(); |
| 6743 | if (llvm::size(Range: getOffsets()) != baseType.getRank()) |
| 6744 | return emitOpError(message: "requires " ) << baseType.getRank() << " indices" ; |
| 6745 | if (valueVType.getShape() != indVType.getShape()) |
| 6746 | return emitOpError(message: "expected valueToStore dim to match indices dim" ); |
| 6747 | if (valueVType.getShape() != maskVType.getShape()) |
| 6748 | return emitOpError(message: "expected valueToStore dim to match mask dim" ); |
| 6749 | if (getAlignmentAttr() && !isa<MemRefType>(Val: baseType)) { |
| 6750 | return emitOpError( |
| 6751 | message: "alignment is only supported for memref bases, not tensor bases" ); |
| 6752 | } |
| 6753 | return success(); |
| 6754 | } |
| 6755 | namespace { |
| 6756 | class ScatterFolder final : public OpRewritePattern<ScatterOp> { |
| 6757 | public: |
| 6758 | using Base::Base; |
| 6759 | LogicalResult matchAndRewrite(ScatterOp scatter, |
| 6760 | PatternRewriter &rewriter) const override { |
| 6761 | ShapedType baseType = scatter.getBaseType(); |
| 6762 | bool isMemRef = isa<MemRefType>(Val: baseType); |
| 6763 | if (!isMemRef && !isa<RankedTensorType>(Val: baseType)) |
| 6764 | return failure(); |
| 6765 | |
| 6766 | // Memrefs have no result, so an all-false mask can simply erase the op. |
| 6767 | // Tensors carry the updated value, so we must replace uses with the |
| 6768 | // original base tensor instead of erasing. |
| 6769 | switch (getMaskFormat(mask: scatter.getMask())) { |
| 6770 | case MaskFormat::AllTrue: |
| 6771 | return failure(); // no unmasked equivalent |
| 6772 | case MaskFormat::AllFalse: |
| 6773 | if (isMemRef) |
| 6774 | rewriter.eraseOp(op: scatter); |
| 6775 | else |
| 6776 | rewriter.replaceOp(op: scatter, newValues: scatter.getBase()); |
| 6777 | return success(); |
| 6778 | case MaskFormat::Unknown: |
| 6779 | return failure(); |
| 6780 | } |
| 6781 | llvm_unreachable("Unexpected 1DMaskFormat on ScatterFolder" ); |
| 6782 | } |
| 6783 | }; |
| 6784 | |
| 6785 | /// Fold scatters with consecutive offsets [0, 1, 2, ...] into contiguous |
| 6786 | /// maskedstore. Only 1D fixed vectors are supported for now. |
| 6787 | class FoldContiguousScatter final : public OpRewritePattern<ScatterOp> { |
| 6788 | public: |
| 6789 | using Base::Base; |
| 6790 | LogicalResult matchAndRewrite(ScatterOp op, |
| 6791 | PatternRewriter &rewriter) const override { |
| 6792 | // Fold only for memrefs: the replacement uses maskedstore, which does not |
| 6793 | // support tensor bases. Tensor cases intentionally bail out. |
| 6794 | if (!isa<MemRefType>(Val: op.getBase().getType())) |
| 6795 | return failure(); |
| 6796 | |
| 6797 | if (failed(Result: isZeroBasedContiguousSeq(indexVec: op.getIndices()))) |
| 6798 | return failure(); |
| 6799 | |
| 6800 | rewriter.replaceOpWithNewOp<MaskedStoreOp>( |
| 6801 | op, args: op.getBase(), args: op.getOffsets(), args: op.getMask(), args: op.getValueToStore()); |
| 6802 | return success(); |
| 6803 | } |
| 6804 | }; |
| 6805 | } // namespace |
| 6806 | |
| 6807 | void ScatterOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 6808 | MLIRContext *context) { |
| 6809 | results.add<ScatterFolder, FoldContiguousScatter>(arg&: context); |
| 6810 | } |
| 6811 | |
| 6812 | FailureOr<std::optional<SmallVector<Value>>> |
| 6813 | ScatterOp::bubbleDownCasts(OpBuilder &builder) { |
| 6814 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6815 | results: ValueRange()); |
| 6816 | } |
| 6817 | |
| 6818 | //===----------------------------------------------------------------------===// |
| 6819 | // ExpandLoadOp |
| 6820 | //===----------------------------------------------------------------------===// |
| 6821 | |
| 6822 | LogicalResult ExpandLoadOp::verify() { |
| 6823 | VectorType maskVType = getMaskVectorType(); |
| 6824 | VectorType passVType = getPassThruVectorType(); |
| 6825 | VectorType resVType = getVectorType(); |
| 6826 | MemRefType memType = getMemRefType(); |
| 6827 | |
| 6828 | if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: resVType, memRefTy: memType))) |
| 6829 | return failure(); |
| 6830 | |
| 6831 | // Negative strides are not supported on vector.expandload. The lowering to |
| 6832 | // LLVM emits arithmetic operations (e.g., GEP, mul) with nuw flags that |
| 6833 | // assume non-negative strides to avoid undefined behavior. |
| 6834 | if (memref::hasNegativeStaticStride(memRefTy: memType)) |
| 6835 | return emitOpError(message: "memref strides must be non-negative" ); |
| 6836 | |
| 6837 | if (failed( |
| 6838 | Result: verifyElementTypesMatch(op: *this, lhs: memType, rhs: resVType, lhsName: "base" , rhsName: "result" ))) |
| 6839 | return failure(); |
| 6840 | if (llvm::size(Range: getIndices()) != memType.getRank()) |
| 6841 | return emitOpError(message: "requires " ) << memType.getRank() << " indices" ; |
| 6842 | if (resVType.getShape() != maskVType.getShape()) |
| 6843 | return emitOpError(message: "expected result shape to match mask shape" ); |
| 6844 | if (resVType.getScalableDims() != maskVType.getScalableDims()) |
| 6845 | return emitOpError( |
| 6846 | message: "expected result scalable dims to match mask scalable dims" ); |
| 6847 | if (resVType != passVType) |
| 6848 | return emitOpError(message: "expected pass_thru of same type as result type" ); |
| 6849 | return success(); |
| 6850 | } |
| 6851 | |
| 6852 | namespace { |
| 6853 | class ExpandLoadFolder final : public OpRewritePattern<ExpandLoadOp> { |
| 6854 | public: |
| 6855 | using Base::Base; |
| 6856 | LogicalResult matchAndRewrite(ExpandLoadOp expand, |
| 6857 | PatternRewriter &rewriter) const override { |
| 6858 | switch (getMaskFormat(mask: expand.getMask())) { |
| 6859 | case MaskFormat::AllTrue: |
| 6860 | rewriter.replaceOpWithNewOp<vector::LoadOp>( |
| 6861 | op: expand, args: expand.getType(), args: expand.getBase(), args: expand.getIndices()); |
| 6862 | return success(); |
| 6863 | case MaskFormat::AllFalse: |
| 6864 | rewriter.replaceOp(op: expand, newValues: expand.getPassThru()); |
| 6865 | return success(); |
| 6866 | case MaskFormat::Unknown: |
| 6867 | return failure(); |
| 6868 | } |
| 6869 | llvm_unreachable("Unexpected 1DMaskFormat on ExpandLoadFolder" ); |
| 6870 | } |
| 6871 | }; |
| 6872 | } // namespace |
| 6873 | |
| 6874 | void ExpandLoadOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 6875 | MLIRContext *context) { |
| 6876 | results.add<ExpandLoadFolder>(arg&: context); |
| 6877 | } |
| 6878 | |
| 6879 | FailureOr<std::optional<SmallVector<Value>>> |
| 6880 | ExpandLoadOp::bubbleDownCasts(OpBuilder &builder) { |
| 6881 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6882 | results: getResult()); |
| 6883 | } |
| 6884 | |
| 6885 | //===----------------------------------------------------------------------===// |
| 6886 | // CompressStoreOp |
| 6887 | //===----------------------------------------------------------------------===// |
| 6888 | |
| 6889 | LogicalResult CompressStoreOp::verify() { |
| 6890 | VectorType maskVType = getMaskVectorType(); |
| 6891 | VectorType valueVType = getVectorType(); |
| 6892 | MemRefType memType = getMemRefType(); |
| 6893 | |
| 6894 | if (failed(Result: verifyLoadStoreMemRefLayout(op: *this, vecTy: valueVType, memRefTy: memType))) |
| 6895 | return failure(); |
| 6896 | |
| 6897 | // Negative strides are not supported on vector.compressstore. The lowering |
| 6898 | // to LLVM emits arithmetic operations (e.g., GEP, mul) with nuw flags that |
| 6899 | // assume non-negative strides to avoid undefined behavior. |
| 6900 | if (memref::hasNegativeStaticStride(memRefTy: memType)) |
| 6901 | return emitOpError(message: "memref strides must be non-negative" ); |
| 6902 | |
| 6903 | if (failed(Result: verifyElementTypesMatch(op: *this, lhs: memType, rhs: valueVType, lhsName: "base" , |
| 6904 | rhsName: "valueToStore" ))) |
| 6905 | return failure(); |
| 6906 | if (llvm::size(Range: getIndices()) != memType.getRank()) |
| 6907 | return emitOpError(message: "requires " ) << memType.getRank() << " indices" ; |
| 6908 | if (valueVType.getShape() != maskVType.getShape()) |
| 6909 | return emitOpError(message: "expected valueToStore shape to match mask shape" ); |
| 6910 | if (valueVType.getScalableDims() != maskVType.getScalableDims()) |
| 6911 | return emitOpError( |
| 6912 | message: "expected valueToStore scalable dims to match mask scalable dims" ); |
| 6913 | return success(); |
| 6914 | } |
| 6915 | |
| 6916 | namespace { |
| 6917 | class CompressStoreFolder final : public OpRewritePattern<CompressStoreOp> { |
| 6918 | public: |
| 6919 | using Base::Base; |
| 6920 | LogicalResult matchAndRewrite(CompressStoreOp compress, |
| 6921 | PatternRewriter &rewriter) const override { |
| 6922 | switch (getMaskFormat(mask: compress.getMask())) { |
| 6923 | case MaskFormat::AllTrue: |
| 6924 | rewriter.replaceOpWithNewOp<vector::StoreOp>( |
| 6925 | op: compress, args: compress.getValueToStore(), args: compress.getBase(), |
| 6926 | args: compress.getIndices()); |
| 6927 | return success(); |
| 6928 | case MaskFormat::AllFalse: |
| 6929 | rewriter.eraseOp(op: compress); |
| 6930 | return success(); |
| 6931 | case MaskFormat::Unknown: |
| 6932 | return failure(); |
| 6933 | } |
| 6934 | llvm_unreachable("Unexpected 1DMaskFormat on CompressStoreFolder" ); |
| 6935 | } |
| 6936 | }; |
| 6937 | } // namespace |
| 6938 | |
| 6939 | void CompressStoreOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 6940 | MLIRContext *context) { |
| 6941 | results.add<CompressStoreFolder>(arg&: context); |
| 6942 | } |
| 6943 | |
| 6944 | FailureOr<std::optional<SmallVector<Value>>> |
| 6945 | CompressStoreOp::bubbleDownCasts(OpBuilder &builder) { |
| 6946 | return mlir::detail::bubbleDownInPlaceMemorySpaceCastImpl(operand&: getBaseMutable(), |
| 6947 | results: ValueRange()); |
| 6948 | } |
| 6949 | |
| 6950 | //===----------------------------------------------------------------------===// |
| 6951 | // ShapeCastOp |
| 6952 | //===----------------------------------------------------------------------===// |
| 6953 | |
| 6954 | void ShapeCastOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges, |
| 6955 | SetIntRangeFn setResultRanges) { |
| 6956 | setResultRanges(getResult(), argRanges.front()); |
| 6957 | } |
| 6958 | |
| 6959 | LogicalResult ShapeCastOp::verify() { |
| 6960 | |
| 6961 | VectorType sourceType = getSourceVectorType(); |
| 6962 | VectorType resultType = getResultVectorType(); |
| 6963 | |
| 6964 | // Check that element type is preserved |
| 6965 | if (failed(Result: verifyElementTypesMatch(op: *this, lhs: sourceType, rhs: resultType, lhsName: "source" , |
| 6966 | rhsName: "result" ))) |
| 6967 | return failure(); |
| 6968 | |
| 6969 | // Check that number of elements is preserved |
| 6970 | int64_t sourceNElms = sourceType.getNumElements(); |
| 6971 | int64_t resultNElms = resultType.getNumElements(); |
| 6972 | if (sourceNElms != resultNElms) { |
| 6973 | return emitOpError() << "has different number of elements at source (" |
| 6974 | << sourceNElms << ") and result (" << resultNElms |
| 6975 | << ")" ; |
| 6976 | } |
| 6977 | |
| 6978 | // Check that (non-)scalability is preserved |
| 6979 | int64_t sourceNScalableDims = sourceType.getNumScalableDims(); |
| 6980 | int64_t resultNScalableDims = resultType.getNumScalableDims(); |
| 6981 | if (sourceNScalableDims != resultNScalableDims) |
| 6982 | return emitOpError() << "has different number of scalable dims at source (" |
| 6983 | << sourceNScalableDims << ") and result (" |
| 6984 | << resultNScalableDims << ")" ; |
| 6985 | |
| 6986 | return success(); |
| 6987 | } |
| 6988 | |
| 6989 | /// Check whether this ShapeCastOp is effectively a BroadcastOp. |
| 6990 | /// |
| 6991 | /// The only case in which this method can return `true` is when the underlying |
| 6992 | /// op merely adds leading unit dimensions, e.g.: |
| 6993 | /// %res = vector.shape_cast %src : vector<8x4xi32> to vector<1x8x4xi32> |
| 6994 | /// |
| 6995 | bool ShapeCastOp::isBroadcastLike() { |
| 6996 | auto srcType = getSourceVectorType(); |
| 6997 | auto resType = getResultVectorType(); |
| 6998 | |
| 6999 | // Is srcType broadcastable to resType? |
| 7000 | std::pair<VectorDim, VectorDim> mismatchingDims; |
| 7001 | if (isBroadcastableTo(srcType, dstVectorType: resType, mismatchingDims: &mismatchingDims) != |
| 7002 | BroadcastableToResult::Success) |
| 7003 | return false; |
| 7004 | |
| 7005 | // Do ranks mismatch? |
| 7006 | // |
| 7007 | // The only case where ranks match and this ShapeCastOp is also a broadcast, |
| 7008 | // is when it's effectively a NOp, but that's an uninteresting edge case. |
| 7009 | size_t rankDiff = resType.getRank() - srcType.getRank(); |
| 7010 | if (rankDiff == 0) |
| 7011 | return false; |
| 7012 | |
| 7013 | // Are all newly added leading dims unit? |
| 7014 | if (!llvm::all_of(Range: resType.getShape().take_front(N: rankDiff), |
| 7015 | P: [](int64_t dim) { return dim == 1; })) |
| 7016 | return false; |
| 7017 | |
| 7018 | // Do all trailing dims match? |
| 7019 | return resType.getShape().take_back(N: srcType.getRank()) == srcType.getShape(); |
| 7020 | } |
| 7021 | |
| 7022 | /// Return true if `transpose` does not permute a pair of non-unit dims. |
| 7023 | /// By `order preserving` we mean that the flattened versions of the input and |
| 7024 | /// output vectors are (numerically) identical. In other words `transpose` is |
| 7025 | /// effectively a shape cast. |
| 7026 | static bool isOrderPreserving(TransposeOp transpose) { |
| 7027 | ArrayRef<int64_t> permutation = transpose.getPermutation(); |
| 7028 | VectorType sourceType = transpose.getSourceVectorType(); |
| 7029 | ArrayRef<int64_t> inShape = sourceType.getShape(); |
| 7030 | ArrayRef<bool> inDimIsScalable = sourceType.getScalableDims(); |
| 7031 | auto isNonScalableUnitDim = [&](int64_t dim) { |
| 7032 | return inShape[dim] == 1 && !inDimIsScalable[dim]; |
| 7033 | }; |
| 7034 | int64_t current = 0; |
| 7035 | for (auto p : permutation) { |
| 7036 | if (!isNonScalableUnitDim(p)) { |
| 7037 | if (p < current) { |
| 7038 | return false; |
| 7039 | } |
| 7040 | current = p; |
| 7041 | } |
| 7042 | } |
| 7043 | return true; |
| 7044 | } |
| 7045 | |
| 7046 | OpFoldResult ShapeCastOp::fold(FoldAdaptor adaptor) { |
| 7047 | |
| 7048 | VectorType resultType = getType(); |
| 7049 | |
| 7050 | // No-op shape cast. |
| 7051 | if (getSource().getType() == resultType) |
| 7052 | return getSource(); |
| 7053 | |
| 7054 | // shape_cast(shape_cast(x)) -> shape_cast(x) |
| 7055 | if (auto precedingShapeCast = getSource().getDefiningOp<ShapeCastOp>()) { |
| 7056 | setOperand(precedingShapeCast.getSource()); |
| 7057 | return getResult(); |
| 7058 | } |
| 7059 | |
| 7060 | // shape_cast(transpose(x)) -> shape_cast(x) |
| 7061 | if (auto transpose = getSource().getDefiningOp<TransposeOp>()) { |
| 7062 | if (isOrderPreserving(transpose)) { |
| 7063 | setOperand(transpose.getVector()); |
| 7064 | return getResult(); |
| 7065 | } |
| 7066 | return {}; |
| 7067 | } |
| 7068 | |
| 7069 | // Y = shape_cast(broadcast(X)) |
| 7070 | // -> X, if X and Y have same type |
| 7071 | if (auto bcastOp = getSource().getDefiningOp<BroadcastOp>()) { |
| 7072 | if (bcastOp.getSourceType() == resultType) |
| 7073 | return bcastOp.getSource(); |
| 7074 | } |
| 7075 | |
| 7076 | // shape_cast(constant) -> constant |
| 7077 | if (auto denseAttr = |
| 7078 | dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getSource())) |
| 7079 | return denseAttr.reshape(newType: getType()); |
| 7080 | |
| 7081 | // shape_cast(poison) -> poison |
| 7082 | if (matchPattern(attr: adaptor.getSource(), pattern: ub::m_Poison())) |
| 7083 | return ub::PoisonAttr::get(ctx: getContext()); |
| 7084 | |
| 7085 | return {}; |
| 7086 | } |
| 7087 | |
| 7088 | //----------------------------------------------------------------------------- |
| 7089 | // VectorUnrollOpInterface methods |
| 7090 | //----------------------------------------------------------------------------- |
| 7091 | std::optional<SmallVector<int64_t, 4>> |
| 7092 | ShapeCastOp::getIterationShapeToUnroll() { |
| 7093 | return llvm::to_vector<4>(Range: getResultVectorType().getShape()); |
| 7094 | } |
| 7095 | |
| 7096 | std::optional<SmallVector<int64_t>> ShapeCastOp::getDefaultUnrollTargetShape() { |
| 7097 | auto resultShape = this->getResultVectorType().getShape(); |
| 7098 | // Special case with leading unit dims and different inner dim |
| 7099 | // for result and target shape. |
| 7100 | if (resultShape.size() == 2 && resultShape[0] == 1 && resultShape[1] == 32) { |
| 7101 | return SmallVector<int64_t>{1, 16}; |
| 7102 | } |
| 7103 | if (resultShape.size() == 2 && resultShape[0] == 2 && resultShape[1] == 1) { |
| 7104 | return SmallVector<int64_t>{1, 1}; |
| 7105 | } |
| 7106 | // Multi-group cases: tile contiguous per reassociation group |
| 7107 | // but strided in the whole result. |
| 7108 | auto sourceShape = this->getSourceVectorType().getShape(); |
| 7109 | if (resultShape.size() == 3 && resultShape[0] == 8 && resultShape[1] == 1 && |
| 7110 | resultShape[2] == 32) { |
| 7111 | return SmallVector<int64_t>{8, 1, 4}; |
| 7112 | } |
| 7113 | if (sourceShape.size() == 3 && resultShape.size() == 2 && |
| 7114 | resultShape[0] == 4 && resultShape[1] == 4) { |
| 7115 | return SmallVector<int64_t>{2, 2}; |
| 7116 | } |
| 7117 | if (resultShape.size() == 3 && resultShape[0] == 2 && resultShape[1] == 8 && |
| 7118 | resultShape[2] == 4) { |
| 7119 | return SmallVector<int64_t>{2, 2, 2}; |
| 7120 | } |
| 7121 | // Default case: [2,4] for all tests. |
| 7122 | return SmallVector<int64_t>{2, 4}; |
| 7123 | } |
| 7124 | |
| 7125 | namespace { |
| 7126 | |
| 7127 | /// Helper function that computes a new vector type based on the input vector |
| 7128 | /// type by removing the _trailing_ unit dims: |
| 7129 | /// |
| 7130 | /// vector<4x1x1xi1> --> vector<4x1xi1> |
| 7131 | /// |
| 7132 | static VectorType trimTrailingUnitDims(VectorType oldType) { |
| 7133 | ArrayRef<int64_t> oldShape = oldType.getShape(); |
| 7134 | ArrayRef<int64_t> newShape = oldShape; |
| 7135 | |
| 7136 | ArrayRef<bool> oldScalableDims = oldType.getScalableDims(); |
| 7137 | ArrayRef<bool> newScalableDims = oldScalableDims; |
| 7138 | |
| 7139 | while (!newShape.empty() && newShape.back() == 1 && !newScalableDims.back()) { |
| 7140 | newShape = newShape.drop_back(N: 1); |
| 7141 | newScalableDims = newScalableDims.drop_back(N: 1); |
| 7142 | } |
| 7143 | |
| 7144 | // Make sure we have at least 1 dimension. |
| 7145 | // TODO: Add support for 0-D vectors. |
| 7146 | if (newShape.empty()) { |
| 7147 | newShape = oldShape.take_back(); |
| 7148 | newScalableDims = oldScalableDims.take_back(); |
| 7149 | } |
| 7150 | |
| 7151 | return VectorType::get(shape: newShape, elementType: oldType.getElementType(), scalableDims: newScalableDims); |
| 7152 | } |
| 7153 | |
| 7154 | /// Helper function that computes a new vector type based on the input vector |
| 7155 | /// type by removing the _leading_ unit dims: |
| 7156 | /// |
| 7157 | /// vector<1x1x4xi1> --> vector<4x1xi1> |
| 7158 | /// |
| 7159 | static VectorType trimLeadingUnitDims(VectorType oldType) { |
| 7160 | ArrayRef<int64_t> oldShape = oldType.getShape(); |
| 7161 | ArrayRef<int64_t> newShape = oldShape; |
| 7162 | |
| 7163 | ArrayRef<bool> oldScalableDims = oldType.getScalableDims(); |
| 7164 | ArrayRef<bool> newScalableDims = oldScalableDims; |
| 7165 | |
| 7166 | while (!newShape.empty() && newShape.front() == 1 && |
| 7167 | !newScalableDims.front()) { |
| 7168 | newShape = newShape.drop_front(N: 1); |
| 7169 | newScalableDims = newScalableDims.drop_front(N: 1); |
| 7170 | } |
| 7171 | |
| 7172 | // Make sure we have at least 1 dimension. |
| 7173 | // TODO: Add support for 0-D vectors. |
| 7174 | if (newShape.empty()) { |
| 7175 | newShape = oldShape.take_back(); |
| 7176 | newScalableDims = oldScalableDims.take_back(); |
| 7177 | } |
| 7178 | |
| 7179 | return VectorType::get(shape: newShape, elementType: oldType.getElementType(), scalableDims: newScalableDims); |
| 7180 | } |
| 7181 | |
| 7182 | enum class UnitDimSide { Leading, Trailing }; |
| 7183 | |
| 7184 | /// Folds qualifying shape_cast(create_mask) into a new create_mask |
| 7185 | /// |
| 7186 | /// Looks at `vector.shape_cast` Ops that simply "drop" either the _trailing_ or |
| 7187 | /// the _leading_unit dimension (configured with the template parameter). If the |
| 7188 | /// input vector comes from `vector.create_mask` for which the corresponding |
| 7189 | /// mask input value is 1 (e.g. `%c1` below), then it is safe to fold shape_cast |
| 7190 | /// into create_mask. |
| 7191 | /// |
| 7192 | /// EX 1 (trailing unit dims) |
| 7193 | /// BEFORE: |
| 7194 | /// %1 = vector.create_mask %c1, %dim, %c1, %c1 : vector<1x[4]x1x1xi1> |
| 7195 | /// %2 = vector.shape_cast %1 : vector<1x[4]x1x1xi1> to vector<1x[4]xi1> |
| 7196 | /// AFTER: |
| 7197 | /// %0 = vector.create_mask %c1, %dim : vector<1x[4]xi1> |
| 7198 | /// |
| 7199 | /// EX 2 (leading unit dims) |
| 7200 | /// BEFORE: |
| 7201 | /// %1 = vector.create_mask %c1, %c1, %dim, %c1 : vector<1x1x[4]x1xi1> |
| 7202 | /// %2 = vector.shape_cast %1 : vector<1x1x[4]x1xi1> to vector<[4]x1xi1> |
| 7203 | /// AFTER: |
| 7204 | /// %0 = vector.create_mask %c1, %dim : vector<[4]xi1> |
| 7205 | template <UnitDimSide Side> |
| 7206 | class ShapeCastCreateMaskFolderBoundaryUnitDim final |
| 7207 | : public OpRewritePattern<ShapeCastOp> { |
| 7208 | public: |
| 7209 | using Base::Base; |
| 7210 | |
| 7211 | LogicalResult matchAndRewrite(ShapeCastOp shapeOp, |
| 7212 | PatternRewriter &rewriter) const override { |
| 7213 | Value shapeOpSrc = shapeOp->getOperand(idx: 0); |
| 7214 | auto createMaskOp = shapeOpSrc.getDefiningOp<vector::CreateMaskOp>(); |
| 7215 | auto constantMaskOp = shapeOpSrc.getDefiningOp<vector::ConstantMaskOp>(); |
| 7216 | if (!createMaskOp && !constantMaskOp) |
| 7217 | return failure(); |
| 7218 | |
| 7219 | VectorType shapeOpResTy = shapeOp.getResultVectorType(); |
| 7220 | VectorType shapeOpSrcTy = shapeOp.getSourceVectorType(); |
| 7221 | |
| 7222 | VectorType newVecType = (Side == UnitDimSide::Trailing) |
| 7223 | ? trimTrailingUnitDims(oldType: shapeOpSrcTy) |
| 7224 | : trimLeadingUnitDims(oldType: shapeOpSrcTy); |
| 7225 | |
| 7226 | if (newVecType != shapeOpResTy) { |
| 7227 | return (Side == UnitDimSide::Trailing) |
| 7228 | ? rewriter.notifyMatchFailure( |
| 7229 | arg&: shapeOp, msg: "Non-trailing-unit-dim dropping shape_cast Op" ) |
| 7230 | : rewriter.notifyMatchFailure( |
| 7231 | arg&: shapeOp, msg: "Non-leading-unit-dim dropping shape_cast Op" ); |
| 7232 | } |
| 7233 | |
| 7234 | auto numDimsToDrop = shapeOpSrcTy.getRank() - shapeOpResTy.getRank(); |
| 7235 | |
| 7236 | // No unit dims to drop |
| 7237 | if (!numDimsToDrop) { |
| 7238 | return (Side == UnitDimSide::Trailing) |
| 7239 | ? rewriter.notifyMatchFailure( |
| 7240 | arg&: shapeOp, msg: "Non-trailing-unit-dim dropping shape_cast Op" ) |
| 7241 | : rewriter.notifyMatchFailure( |
| 7242 | arg&: shapeOp, msg: "Non-leading-unit-dim dropping shape_cast Op" ); |
| 7243 | } |
| 7244 | |
| 7245 | if (createMaskOp) { |
| 7246 | auto maskOperands = createMaskOp.getOperands(); |
| 7247 | |
| 7248 | auto maskOperandsToDrop = (Side == UnitDimSide::Trailing) |
| 7249 | ? maskOperands.take_back(n: numDimsToDrop) |
| 7250 | : maskOperands.take_front(n: numDimsToDrop); |
| 7251 | |
| 7252 | // Check that every mask-dim-size to-be-dropped is constant and == 1. We |
| 7253 | // could also check for == 0, but that's left as a TODO. |
| 7254 | if (llvm::any_of(maskOperandsToDrop, [](Value maskDim) { |
| 7255 | auto cst = maskDim.getDefiningOp<arith::ConstantIndexOp>(); |
| 7256 | return !cst || (cst.value() != 1); |
| 7257 | })) |
| 7258 | return failure(); |
| 7259 | |
| 7260 | auto newMaskOperands = (Side == UnitDimSide::Trailing) |
| 7261 | ? maskOperands.drop_back(n: numDimsToDrop) |
| 7262 | : maskOperands.drop_front(n: numDimsToDrop); |
| 7263 | rewriter.replaceOpWithNewOp<vector::CreateMaskOp>(op: shapeOp, args&: shapeOpResTy, |
| 7264 | args&: newMaskOperands); |
| 7265 | return success(); |
| 7266 | } |
| 7267 | |
| 7268 | if (constantMaskOp) { |
| 7269 | auto maskDimSizes = constantMaskOp.getMaskDimSizes(); |
| 7270 | |
| 7271 | ArrayRef<int64_t> maskDimSizesToDrop = |
| 7272 | (Side == UnitDimSide::Trailing) |
| 7273 | ? maskDimSizes.take_back(N: numDimsToDrop) |
| 7274 | : maskDimSizes.take_front(N: numDimsToDrop); |
| 7275 | |
| 7276 | // Check that every mask-dim-size to-be-dropped is constant and == 1. We |
| 7277 | // could also check for == 0, but that's left as a TODO. |
| 7278 | if (llvm::any_of(maskDimSizesToDrop, |
| 7279 | [](int64_t dim) { return dim != 1; })) |
| 7280 | return failure(); |
| 7281 | |
| 7282 | ArrayRef<int64_t> newMaskDimSizes = |
| 7283 | (Side == UnitDimSide::Trailing) |
| 7284 | ? maskDimSizes.drop_back(N: numDimsToDrop) |
| 7285 | : maskDimSizes.drop_front(N: numDimsToDrop); |
| 7286 | rewriter.replaceOpWithNewOp<vector::ConstantMaskOp>(op: shapeOp, args&: shapeOpResTy, |
| 7287 | args&: newMaskDimSizes); |
| 7288 | return success(); |
| 7289 | } |
| 7290 | |
| 7291 | return failure(); |
| 7292 | } |
| 7293 | }; |
| 7294 | |
| 7295 | /// Pattern to rewrite Y = ShapeCast(Broadcast(X)) as Y = Broadcast(X) |
| 7296 | class ShapeCastBroadcastFolder final : public OpRewritePattern<ShapeCastOp> { |
| 7297 | public: |
| 7298 | using Base::Base; |
| 7299 | |
| 7300 | LogicalResult matchAndRewrite(ShapeCastOp shapeCastOp, |
| 7301 | PatternRewriter &rewriter) const override { |
| 7302 | auto oldBcOp = shapeCastOp.getSource().getDefiningOp<vector::BroadcastOp>(); |
| 7303 | if (!oldBcOp) |
| 7304 | return failure(); |
| 7305 | |
| 7306 | // Replace Y = ShapeCast(Broadcast(X)) with Y = Broadcast(X) |
| 7307 | // Example |
| 7308 | // %oldBc = vector.broadcast %src : vector<3xf32> to vector<2x4x3xf32> |
| 7309 | // %res = vector.shape_cast %oldBc : vector<2x4x3xf32> to vector<8x3xf32> |
| 7310 | // to |
| 7311 | // %newBc = vector.broadcast %src : vector<3xf32> to vector<8x3xf32> |
| 7312 | VectorType resVectorType = shapeCastOp.getResultVectorType(); |
| 7313 | |
| 7314 | // Limit this pattern to vectors with at most one scalable dimension for |
| 7315 | // now. |
| 7316 | if (resVectorType.getNumScalableDims() > 1) |
| 7317 | return failure(); |
| 7318 | |
| 7319 | auto srcVectorType = dyn_cast<VectorType>(Val: oldBcOp.getSourceType()); |
| 7320 | bool srcIsScalar = !srcVectorType; |
| 7321 | |
| 7322 | if (!srcIsScalar) { |
| 7323 | if (isBroadcastableTo(srcType: srcVectorType, dstVectorType: resVectorType) != |
| 7324 | BroadcastableToResult::Success) { |
| 7325 | return failure(); |
| 7326 | } |
| 7327 | // For a fixed source shape, the result strides determine which source |
| 7328 | // element supplies each linear position in the broadcast result (see the |
| 7329 | // `BroadcastOp` definition for the closed-form mapping). Since |
| 7330 | // `shape_cast` preserves the flattened 1-D (row-major) element order, |
| 7331 | // folding into `%newBc` is valid when every non-unit `%src` dimension has |
| 7332 | // the same stride (both static product and scalability) in `%oldBc` and |
| 7333 | // `%res`. Note that a scalable `[1]` has runtime size `vscale` and is not |
| 7334 | // a unit dimension. |
| 7335 | VectorType oldBcVectorType = oldBcOp.getResultVectorType(); |
| 7336 | int64_t srcRank = srcVectorType.getRank(); |
| 7337 | int64_t oldBcRank = oldBcVectorType.getRank(); |
| 7338 | int64_t resRank = resVectorType.getRank(); |
| 7339 | ArrayRef<int64_t> srcShape = srcVectorType.getShape(); |
| 7340 | ArrayRef<int64_t> oldBcShape = oldBcVectorType.getShape(); |
| 7341 | ArrayRef<int64_t> resShape = resVectorType.getShape(); |
| 7342 | ArrayRef<bool> srcScalable = srcVectorType.getScalableDims(); |
| 7343 | ArrayRef<bool> oldBcScalable = oldBcVectorType.getScalableDims(); |
| 7344 | ArrayRef<bool> resScalable = resVectorType.getScalableDims(); |
| 7345 | |
| 7346 | // Iterate from the innermost (trailing) dimension outward, aligning each |
| 7347 | // `%src` dimension with the corresponding trailing dimension of `%oldBc` |
| 7348 | // and `%res`. |
| 7349 | int64_t oldBcStride = 1, resStride = 1; |
| 7350 | bool isOldBcStrideScalable = false, isResStrideScalable = false; |
| 7351 | for (int64_t i = 0; i < srcRank; ++i) { |
| 7352 | int64_t srcDim = srcRank - 1 - i; |
| 7353 | int64_t oldBcDim = oldBcRank - 1 - i; |
| 7354 | int64_t resDim = resRank - 1 - i; |
| 7355 | |
| 7356 | bool isUnitDim = srcShape[srcDim] == 1 && !srcScalable[srcDim]; |
| 7357 | if (!isUnitDim && (oldBcStride != resStride || |
| 7358 | isOldBcStrideScalable != isResStrideScalable)) { |
| 7359 | return failure(); |
| 7360 | } |
| 7361 | |
| 7362 | oldBcStride *= oldBcShape[oldBcDim]; |
| 7363 | resStride *= resShape[resDim]; |
| 7364 | isOldBcStrideScalable |= oldBcScalable[oldBcDim]; |
| 7365 | isResStrideScalable |= resScalable[resDim]; |
| 7366 | } |
| 7367 | } |
| 7368 | |
| 7369 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: shapeCastOp, args&: resVectorType, |
| 7370 | args: oldBcOp.getSource()); |
| 7371 | return success(); |
| 7372 | } |
| 7373 | }; |
| 7374 | |
| 7375 | /// Pattern to rewrite Y = ShapeCast(FromElements(X)) as Y = FromElements(X) |
| 7376 | /// |
| 7377 | /// BEFORE: |
| 7378 | /// %1 = vector.from_elements %c1, %c2, %c3 : vector<3xf32> |
| 7379 | /// %2 = vector.shape_cast %1 : vector<3xf32> to vector<1x3xf32> |
| 7380 | /// AFTER: |
| 7381 | /// %2 = vector.from_elements %c1, %c2, %c3 : vector<1x3xf32> |
| 7382 | /// |
| 7383 | /// Note: this transformation is implemented as an OpRewritePattern, not as a |
| 7384 | /// fold, because we have to create new op FromElementsOp with updated result |
| 7385 | /// type. This cannot be done with a fold, because fold cannot create new ops |
| 7386 | /// and the existing FromElementsOp result type differs from the ShapeCastOp |
| 7387 | /// result type. Mutating the FromElementsOp (not root op) would violate the |
| 7388 | /// fold contract and break other users. |
| 7389 | class FoldShapeCastOfFromElements final : public OpRewritePattern<ShapeCastOp> { |
| 7390 | public: |
| 7391 | using Base::Base; |
| 7392 | |
| 7393 | LogicalResult matchAndRewrite(ShapeCastOp shapeCastOp, |
| 7394 | PatternRewriter &rewriter) const override { |
| 7395 | auto fromElements = shapeCastOp.getSource().getDefiningOp<FromElementsOp>(); |
| 7396 | if (!fromElements) |
| 7397 | return failure(); |
| 7398 | |
| 7399 | rewriter.replaceOpWithNewOp<FromElementsOp>( |
| 7400 | op: shapeCastOp, args: shapeCastOp.getResultVectorType(), |
| 7401 | args: fromElements.getElements()); |
| 7402 | return success(); |
| 7403 | } |
| 7404 | }; |
| 7405 | |
| 7406 | } // namespace |
| 7407 | |
| 7408 | void ShapeCastOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 7409 | MLIRContext *context) { |
| 7410 | results.add<ShapeCastCreateMaskFolderBoundaryUnitDim<UnitDimSide::Leading>, |
| 7411 | ShapeCastCreateMaskFolderBoundaryUnitDim<UnitDimSide::Trailing>, |
| 7412 | ShapeCastBroadcastFolder, FoldShapeCastOfFromElements>(arg&: context); |
| 7413 | } |
| 7414 | |
| 7415 | //===----------------------------------------------------------------------===// |
| 7416 | // VectorBitCastOp |
| 7417 | //===----------------------------------------------------------------------===// |
| 7418 | |
| 7419 | LogicalResult BitCastOp::verify() { |
| 7420 | auto sourceVectorType = getSourceVectorType(); |
| 7421 | auto resultVectorType = getResultVectorType(); |
| 7422 | |
| 7423 | for (int64_t i = 0, e = sourceVectorType.getRank() - 1; i < e; i++) { |
| 7424 | if (sourceVectorType.getDimSize(idx: i) != resultVectorType.getDimSize(idx: i)) |
| 7425 | return emitOpError(message: "dimension size mismatch at: " ) << i; |
| 7426 | } |
| 7427 | |
| 7428 | DataLayout dataLayout = DataLayout::closest(op: *this); |
| 7429 | auto sourceElementBits = |
| 7430 | dataLayout.getTypeSizeInBits(t: sourceVectorType.getElementType()); |
| 7431 | auto resultElementBits = |
| 7432 | dataLayout.getTypeSizeInBits(t: resultVectorType.getElementType()); |
| 7433 | |
| 7434 | if (sourceVectorType.getRank() == 0) { |
| 7435 | if (sourceElementBits != resultElementBits) |
| 7436 | return emitOpError(message: "source/result bitwidth of the 0-D vector element " |
| 7437 | "types must be equal" ); |
| 7438 | } else if (sourceElementBits * sourceVectorType.getShape().back() != |
| 7439 | resultElementBits * resultVectorType.getShape().back()) { |
| 7440 | return emitOpError( |
| 7441 | message: "source/result bitwidth of the minor 1-D vectors must be equal" ); |
| 7442 | } |
| 7443 | |
| 7444 | return success(); |
| 7445 | } |
| 7446 | |
| 7447 | OpFoldResult BitCastOp::fold(FoldAdaptor adaptor) { |
| 7448 | // Nop cast. |
| 7449 | if (getSource().getType() == getResult().getType()) |
| 7450 | return getSource(); |
| 7451 | |
| 7452 | // Canceling bitcasts. |
| 7453 | if (auto otherOp = getSource().getDefiningOp<BitCastOp>()) { |
| 7454 | if (getResult().getType() == otherOp.getSource().getType()) |
| 7455 | return otherOp.getSource(); |
| 7456 | |
| 7457 | setOperand(otherOp.getSource()); |
| 7458 | return getResult(); |
| 7459 | } |
| 7460 | |
| 7461 | Attribute sourceConstant = adaptor.getSource(); |
| 7462 | if (!sourceConstant) |
| 7463 | return {}; |
| 7464 | |
| 7465 | Type srcElemType = getSourceVectorType().getElementType(); |
| 7466 | Type dstElemType = getResultVectorType().getElementType(); |
| 7467 | |
| 7468 | if (auto floatPack = llvm::dyn_cast<DenseFPElementsAttr>(Val&: sourceConstant)) { |
| 7469 | if (floatPack.isSplat()) { |
| 7470 | auto splat = floatPack.getSplatValue<FloatAttr>(); |
| 7471 | |
| 7472 | // Casting fp16 into fp32. |
| 7473 | if (srcElemType.isF16() && dstElemType.isF32()) { |
| 7474 | uint32_t bits = static_cast<uint32_t>( |
| 7475 | splat.getValue().bitcastToAPInt().getZExtValue()); |
| 7476 | // Duplicate the 16-bit pattern. |
| 7477 | bits = (bits << 16) | (bits & 0xffff); |
| 7478 | APInt intBits(32, bits); |
| 7479 | APFloat floatBits(llvm::APFloat::IEEEsingle(), intBits); |
| 7480 | return DenseElementsAttr::get(type: getResultVectorType(), values: floatBits); |
| 7481 | } |
| 7482 | } |
| 7483 | } |
| 7484 | |
| 7485 | if (auto intPack = llvm::dyn_cast<DenseIntElementsAttr>(Val&: sourceConstant)) { |
| 7486 | if (intPack.isSplat()) { |
| 7487 | auto splat = intPack.getSplatValue<IntegerAttr>(); |
| 7488 | |
| 7489 | if (llvm::isa<IntegerType>(Val: dstElemType) && srcElemType.isIntOrFloat()) { |
| 7490 | uint64_t srcBitWidth = srcElemType.getIntOrFloatBitWidth(); |
| 7491 | uint64_t dstBitWidth = dstElemType.getIntOrFloatBitWidth(); |
| 7492 | |
| 7493 | // Casting to a larger integer bit width. |
| 7494 | if (dstBitWidth > srcBitWidth && dstBitWidth % srcBitWidth == 0) { |
| 7495 | APInt intBits = splat.getValue().zext(width: dstBitWidth); |
| 7496 | |
| 7497 | // Duplicate the lower width element. |
| 7498 | for (uint64_t i = 0; i < dstBitWidth / srcBitWidth - 1; i++) |
| 7499 | intBits = (intBits << srcBitWidth) | intBits; |
| 7500 | return DenseElementsAttr::get(type: getResultVectorType(), values: intBits); |
| 7501 | } |
| 7502 | } |
| 7503 | } |
| 7504 | } |
| 7505 | |
| 7506 | return {}; |
| 7507 | } |
| 7508 | |
| 7509 | //----------------------------------------------------------------------------- |
| 7510 | // VectorUnrollOpInterface methods |
| 7511 | //----------------------------------------------------------------------------- |
| 7512 | std::optional<SmallVector<int64_t, 4>> BitCastOp::getIterationShapeToUnroll() { |
| 7513 | return llvm::to_vector<4>(Range: getResultVectorType().getShape()); |
| 7514 | } |
| 7515 | |
| 7516 | std::optional<SmallVector<int64_t>> BitCastOp::getDefaultUnrollTargetShape() { |
| 7517 | return SmallVector<int64_t>{4, 4}; |
| 7518 | } |
| 7519 | |
| 7520 | //===----------------------------------------------------------------------===// |
| 7521 | // TypeCastOp |
| 7522 | //===----------------------------------------------------------------------===// |
| 7523 | |
| 7524 | static SmallVector<int64_t, 8> (MemRefType memRefType) { |
| 7525 | auto vectorType = llvm::dyn_cast<VectorType>(Val: memRefType.getElementType()); |
| 7526 | SmallVector<int64_t, 8> res(memRefType.getShape()); |
| 7527 | if (vectorType) |
| 7528 | res.append(in_start: vectorType.getShape().begin(), in_end: vectorType.getShape().end()); |
| 7529 | return res; |
| 7530 | } |
| 7531 | |
| 7532 | /// Build the canonical memRefType with a single vector. |
| 7533 | /// E.g. memref<4 x 5 x vector<6 x f32>> -> memref<vector<4 x 5 x 6 x f32>>. |
| 7534 | void TypeCastOp::build(OpBuilder &builder, OperationState &result, |
| 7535 | Value source) { |
| 7536 | result.addOperands(newOperands: source); |
| 7537 | MemRefType memRefType = llvm::cast<MemRefType>(Val: source.getType()); |
| 7538 | VectorType vectorType = |
| 7539 | VectorType::get(shape: extractShape(memRefType), |
| 7540 | elementType: getElementTypeOrSelf(type: getElementTypeOrSelf(type: memRefType))); |
| 7541 | result.addTypes(newTypes: MemRefType::get(shape: {}, elementType: vectorType, layout: MemRefLayoutAttrInterface(), |
| 7542 | memorySpace: memRefType.getMemorySpace())); |
| 7543 | } |
| 7544 | |
| 7545 | LogicalResult TypeCastOp::verify() { |
| 7546 | MemRefType canonicalType = getMemRefType().canonicalizeStridedLayout(); |
| 7547 | if (!canonicalType.getLayout().isIdentity()) |
| 7548 | return emitOpError(message: "expects operand to be a memref with identity layout" ); |
| 7549 | if (!getResultMemRefType().getLayout().isIdentity()) |
| 7550 | return emitOpError(message: "expects result to be a memref with identity layout" ); |
| 7551 | if (getResultMemRefType().getMemorySpace() != |
| 7552 | getMemRefType().getMemorySpace()) |
| 7553 | return emitOpError(message: "expects result in same memory space" ); |
| 7554 | |
| 7555 | auto sourceType = getMemRefType(); |
| 7556 | auto resultType = getResultMemRefType(); |
| 7557 | if (getElementTypeOrSelf(type: getElementTypeOrSelf(type: sourceType)) != |
| 7558 | getElementTypeOrSelf(type: getElementTypeOrSelf(type: resultType))) |
| 7559 | return emitOpError( |
| 7560 | message: "expects result and operand with same underlying scalar type: " ) |
| 7561 | << resultType; |
| 7562 | if (extractShape(memRefType: sourceType) != extractShape(memRefType: resultType)) |
| 7563 | return emitOpError( |
| 7564 | message: "expects concatenated result and operand shapes to be equal: " ) |
| 7565 | << resultType; |
| 7566 | return success(); |
| 7567 | } |
| 7568 | |
| 7569 | //===----------------------------------------------------------------------===// |
| 7570 | // TransposeOp |
| 7571 | //===----------------------------------------------------------------------===// |
| 7572 | |
| 7573 | void vector::TransposeOp::build(OpBuilder &builder, OperationState &result, |
| 7574 | Value vector, ArrayRef<int64_t> permutation) { |
| 7575 | VectorType vt = llvm::cast<VectorType>(Val: vector.getType()); |
| 7576 | SmallVector<int64_t, 4> transposedShape(vt.getRank()); |
| 7577 | SmallVector<bool, 4> transposedScalableDims(vt.getRank()); |
| 7578 | for (unsigned i = 0; i < permutation.size(); ++i) { |
| 7579 | transposedShape[i] = vt.getShape()[permutation[i]]; |
| 7580 | transposedScalableDims[i] = vt.getScalableDims()[permutation[i]]; |
| 7581 | } |
| 7582 | |
| 7583 | result.addOperands(newOperands: vector); |
| 7584 | result.addTypes(newTypes: VectorType::get(shape: transposedShape, elementType: vt.getElementType(), |
| 7585 | scalableDims: transposedScalableDims)); |
| 7586 | result.addAttribute(name: TransposeOp::getPermutationAttrName(name: result.name), |
| 7587 | attr: builder.getDenseI64ArrayAttr(values: permutation)); |
| 7588 | } |
| 7589 | |
| 7590 | OpFoldResult vector::TransposeOp::fold(FoldAdaptor adaptor) { |
| 7591 | // Eliminate splat constant transpose ops. |
| 7592 | if (auto splat = |
| 7593 | llvm::dyn_cast_if_present<SplatElementsAttr>(Val: adaptor.getVector())) |
| 7594 | return splat.reshape(newType: getResultVectorType()); |
| 7595 | |
| 7596 | // Eliminate poison transpose ops. |
| 7597 | if (matchPattern(attr: adaptor.getVector(), pattern: ub::m_Poison())) |
| 7598 | return ub::PoisonAttr::get(ctx: getContext()); |
| 7599 | |
| 7600 | // Eliminate identity transposes, and more generally any transposes that |
| 7601 | // preserves the shape without permuting elements. |
| 7602 | // |
| 7603 | // Examples of what to fold: |
| 7604 | // %0 = vector.transpose %arg, [0, 1] : vector<1x1xi8> to vector<1x1xi8> |
| 7605 | // %0 = vector.transpose %arg, [0, 1] : vector<2x2xi8> to vector<2x2xi8> |
| 7606 | // %0 = vector.transpose %arg, [1, 0] : vector<1x1xi8> to vector<1x1xi8> |
| 7607 | // |
| 7608 | // Example of what NOT to fold: |
| 7609 | // %0 = vector.transpose %arg, [1, 0] : vector<2x2xi8> to vector<2x2xi8> |
| 7610 | // |
| 7611 | if (getSourceVectorType() == getResultVectorType() && |
| 7612 | isOrderPreserving(transpose: *this)) |
| 7613 | return getVector(); |
| 7614 | |
| 7615 | return {}; |
| 7616 | } |
| 7617 | |
| 7618 | LogicalResult vector::TransposeOp::verify() { |
| 7619 | VectorType vectorType = getSourceVectorType(); |
| 7620 | VectorType resultType = getResultVectorType(); |
| 7621 | int64_t rank = resultType.getRank(); |
| 7622 | if (vectorType.getRank() != rank) |
| 7623 | return emitOpError(message: "vector result rank mismatch: " ) << rank; |
| 7624 | // Verify transposition array. |
| 7625 | ArrayRef<int64_t> perm = getPermutation(); |
| 7626 | int64_t size = perm.size(); |
| 7627 | if (rank != size) |
| 7628 | return emitOpError(message: "transposition length mismatch: " ) << size; |
| 7629 | SmallVector<bool, 8> seen(rank, false); |
| 7630 | for (const auto &ta : llvm::enumerate(First&: perm)) { |
| 7631 | if (ta.value() < 0 || ta.value() >= rank) |
| 7632 | return emitOpError(message: "transposition index out of range: " ) << ta.value(); |
| 7633 | if (seen[ta.value()]) |
| 7634 | return emitOpError(message: "duplicate position index: " ) << ta.value(); |
| 7635 | seen[ta.value()] = true; |
| 7636 | if (resultType.getDimSize(idx: ta.index()) != vectorType.getDimSize(idx: ta.value())) |
| 7637 | return emitOpError(message: "dimension size mismatch at: " ) << ta.value(); |
| 7638 | } |
| 7639 | return success(); |
| 7640 | } |
| 7641 | |
| 7642 | void TransposeOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges, |
| 7643 | SetIntRangeFn setResultRanges) { |
| 7644 | setResultRanges(getResult(), argRanges.front()); |
| 7645 | } |
| 7646 | |
| 7647 | //----------------------------------------------------------------------------- |
| 7648 | // VectorUnrollOpInterface methods |
| 7649 | //----------------------------------------------------------------------------- |
| 7650 | std::optional<SmallVector<int64_t, 4>> |
| 7651 | TransposeOp::getIterationShapeToUnroll() { |
| 7652 | return llvm::to_vector<4>(Range: getResultVectorType().getShape()); |
| 7653 | } |
| 7654 | |
| 7655 | std::optional<SmallVector<int64_t>> TransposeOp::getDefaultUnrollTargetShape() { |
| 7656 | return SmallVector<int64_t>{1, 3, 4, 2}; |
| 7657 | } |
| 7658 | |
| 7659 | namespace { |
| 7660 | |
| 7661 | // Rewrites two back-to-back TransposeOp operations into a single TransposeOp. |
| 7662 | class TransposeFolder final : public OpRewritePattern<vector::TransposeOp> { |
| 7663 | public: |
| 7664 | using Base::Base; |
| 7665 | |
| 7666 | LogicalResult matchAndRewrite(vector::TransposeOp transposeOp, |
| 7667 | PatternRewriter &rewriter) const override { |
| 7668 | // Composes two permutations: result[i] = permutation1[permutation2[i]]. |
| 7669 | auto composePermutations = [](ArrayRef<int64_t> permutation1, |
| 7670 | ArrayRef<int64_t> permutation2) { |
| 7671 | SmallVector<int64_t, 4> result; |
| 7672 | for (auto index : permutation2) |
| 7673 | result.push_back(Elt: permutation1[index]); |
| 7674 | return result; |
| 7675 | }; |
| 7676 | |
| 7677 | // Return if the input of 'transposeOp' is not defined by another transpose. |
| 7678 | vector::TransposeOp parentTransposeOp = |
| 7679 | transposeOp.getVector().getDefiningOp<vector::TransposeOp>(); |
| 7680 | if (!parentTransposeOp) |
| 7681 | return failure(); |
| 7682 | |
| 7683 | SmallVector<int64_t, 4> permutation = composePermutations( |
| 7684 | parentTransposeOp.getPermutation(), transposeOp.getPermutation()); |
| 7685 | // Replace 'transposeOp' with a new transpose operation. |
| 7686 | rewriter.replaceOpWithNewOp<vector::TransposeOp>( |
| 7687 | op: transposeOp, args: transposeOp.getResult().getType(), |
| 7688 | args: parentTransposeOp.getVector(), args&: permutation); |
| 7689 | return success(); |
| 7690 | } |
| 7691 | }; |
| 7692 | |
| 7693 | /// Replace transpose(splat-like(v)) with broadcast(v) |
| 7694 | class FoldTransposeSplat final : public OpRewritePattern<TransposeOp> { |
| 7695 | public: |
| 7696 | using Base::Base; |
| 7697 | |
| 7698 | LogicalResult matchAndRewrite(TransposeOp transposeOp, |
| 7699 | PatternRewriter &rewriter) const override { |
| 7700 | Value splat = getScalarSplatSource(value: transposeOp.getVector()); |
| 7701 | if (!splat) |
| 7702 | return failure(); |
| 7703 | |
| 7704 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>( |
| 7705 | op: transposeOp, args: transposeOp.getResultVectorType(), args&: splat); |
| 7706 | return success(); |
| 7707 | } |
| 7708 | }; |
| 7709 | |
| 7710 | /// Folds transpose(create_mask) into a new transposed create_mask. |
| 7711 | class FoldTransposeCreateMask final : public OpRewritePattern<TransposeOp> { |
| 7712 | public: |
| 7713 | using Base::Base; |
| 7714 | |
| 7715 | LogicalResult matchAndRewrite(TransposeOp transpOp, |
| 7716 | PatternRewriter &rewriter) const override { |
| 7717 | Value transposeSrc = transpOp.getVector(); |
| 7718 | auto createMaskOp = transposeSrc.getDefiningOp<vector::CreateMaskOp>(); |
| 7719 | auto constantMaskOp = transposeSrc.getDefiningOp<vector::ConstantMaskOp>(); |
| 7720 | if (!createMaskOp && !constantMaskOp) |
| 7721 | return failure(); |
| 7722 | |
| 7723 | // Get the transpose permutation and apply it to the vector.create_mask or |
| 7724 | // vector.constant_mask operands. |
| 7725 | ArrayRef<int64_t> permutation = transpOp.getPermutation(); |
| 7726 | |
| 7727 | if (createMaskOp) { |
| 7728 | auto maskOperands = createMaskOp.getOperands(); |
| 7729 | SmallVector<Value> newOperands(maskOperands.begin(), maskOperands.end()); |
| 7730 | applyPermutationToVector(inVec&: newOperands, permutation); |
| 7731 | |
| 7732 | rewriter.replaceOpWithNewOp<vector::CreateMaskOp>( |
| 7733 | op: transpOp, args: transpOp.getResultVectorType(), args&: newOperands); |
| 7734 | return success(); |
| 7735 | } |
| 7736 | |
| 7737 | // ConstantMaskOp case. |
| 7738 | auto maskDimSizes = constantMaskOp.getMaskDimSizes(); |
| 7739 | auto newMaskDimSizes = applyPermutation(input: maskDimSizes, permutation); |
| 7740 | |
| 7741 | rewriter.replaceOpWithNewOp<vector::ConstantMaskOp>( |
| 7742 | op: transpOp, args: transpOp.getResultVectorType(), args&: newMaskDimSizes); |
| 7743 | return success(); |
| 7744 | } |
| 7745 | }; |
| 7746 | |
| 7747 | /// Folds transpose(shape_cast) into a new shape_cast. |
| 7748 | class FoldTransposeShapeCast final : public OpRewritePattern<TransposeOp> { |
| 7749 | public: |
| 7750 | using Base::Base; |
| 7751 | |
| 7752 | LogicalResult matchAndRewrite(TransposeOp transposeOp, |
| 7753 | PatternRewriter &rewriter) const override { |
| 7754 | auto shapeCastOp = |
| 7755 | transposeOp.getVector().getDefiningOp<vector::ShapeCastOp>(); |
| 7756 | if (!shapeCastOp) |
| 7757 | return failure(); |
| 7758 | if (!isOrderPreserving(transpose: transposeOp)) |
| 7759 | return failure(); |
| 7760 | |
| 7761 | VectorType resultType = transposeOp.getType(); |
| 7762 | |
| 7763 | // We don't need to check isValidShapeCast at this point, because it is |
| 7764 | // guaranteed that merging the transpose into the the shape_cast is a valid |
| 7765 | // shape_cast, because the transpose just inserts/removes ones. |
| 7766 | |
| 7767 | rewriter.replaceOpWithNewOp<vector::ShapeCastOp>(op: transposeOp, args&: resultType, |
| 7768 | args: shapeCastOp.getSource()); |
| 7769 | return success(); |
| 7770 | } |
| 7771 | }; |
| 7772 | |
| 7773 | /// Folds transpose(from_elements(...)) into a new from_elements with permuted |
| 7774 | /// operands matching the transposed shape. |
| 7775 | /// |
| 7776 | /// Example: |
| 7777 | /// |
| 7778 | /// %v = vector.from_elements %a00, %a01, %a02, %a10, %a11, %a12 : |
| 7779 | /// vector<2x3xi32> %t = vector.transpose %v, [1, 0] : vector<2x3xi32> to |
| 7780 | /// vector<3x2xi32> |
| 7781 | /// |
| 7782 | /// becomes -> |
| 7783 | /// |
| 7784 | /// %r = vector.from_elements %a00, %a10, %a01, %a11, %a02, %a12 : |
| 7785 | /// vector<3x2xi32> |
| 7786 | /// |
| 7787 | class FoldTransposeFromElements final : public OpRewritePattern<TransposeOp> { |
| 7788 | public: |
| 7789 | using Base::Base; |
| 7790 | LogicalResult matchAndRewrite(vector::TransposeOp transposeOp, |
| 7791 | PatternRewriter &rewriter) const override { |
| 7792 | auto fromElementsOp = |
| 7793 | transposeOp.getVector().getDefiningOp<vector::FromElementsOp>(); |
| 7794 | if (!fromElementsOp) |
| 7795 | return failure(); |
| 7796 | |
| 7797 | VectorType srcTy = fromElementsOp.getResult().getType(); |
| 7798 | VectorType dstTy = transposeOp.getType(); |
| 7799 | |
| 7800 | ArrayRef<int64_t> permutation = transposeOp.getPermutation(); |
| 7801 | int64_t rank = srcTy.getRank(); |
| 7802 | |
| 7803 | // Build inverse permutation to map destination indices back to source. |
| 7804 | SmallVector<int64_t> inversePerm(rank, 0); |
| 7805 | for (int64_t i = 0; i < rank; ++i) |
| 7806 | inversePerm[permutation[i]] = i; |
| 7807 | |
| 7808 | ArrayRef<int64_t> srcShape = srcTy.getShape(); |
| 7809 | ArrayRef<int64_t> dstShape = dstTy.getShape(); |
| 7810 | SmallVector<int64_t> srcIdx(rank, 0); |
| 7811 | SmallVector<int64_t> dstIdx(rank, 0); |
| 7812 | SmallVector<int64_t> srcStrides = computeStrides(sizes: srcShape); |
| 7813 | SmallVector<int64_t> dstStrides = computeStrides(sizes: dstShape); |
| 7814 | |
| 7815 | auto elementsOld = fromElementsOp.getElements(); |
| 7816 | SmallVector<Value> elementsNew; |
| 7817 | int64_t dstNumElements = dstTy.getNumElements(); |
| 7818 | elementsNew.reserve(N: dstNumElements); |
| 7819 | |
| 7820 | // For each element in destination row-major order, pick the corresponding |
| 7821 | // source element. |
| 7822 | for (int64_t linearIdx = 0; linearIdx < dstNumElements; ++linearIdx) { |
| 7823 | // Pick the destination element index. |
| 7824 | dstIdx = delinearize(linearIndex: linearIdx, strides: dstStrides); |
| 7825 | // Map the destination element index to the source element index. |
| 7826 | for (int64_t j = 0; j < rank; ++j) |
| 7827 | srcIdx[j] = dstIdx[inversePerm[j]]; |
| 7828 | // Linearize the source element index. |
| 7829 | int64_t srcLin = linearize(offsets: srcIdx, basis: srcStrides); |
| 7830 | // Add the source element to the new elements. |
| 7831 | elementsNew.push_back(Elt: elementsOld[srcLin]); |
| 7832 | } |
| 7833 | |
| 7834 | rewriter.replaceOpWithNewOp<FromElementsOp>(op: transposeOp, args&: dstTy, |
| 7835 | args&: elementsNew); |
| 7836 | return success(); |
| 7837 | } |
| 7838 | }; |
| 7839 | |
| 7840 | /// Folds transpose(broadcast(x)) to broadcast(x) if the transpose is |
| 7841 | /// 'order preserving', where 'order preserving' means the flattened |
| 7842 | /// inputs and outputs of the transpose have identical (numerical) values. |
| 7843 | /// |
| 7844 | /// Example: |
| 7845 | /// ``` |
| 7846 | /// %0 = vector.broadcast %input : vector<1x1xi32> to vector<1x8xi32> |
| 7847 | /// %1 = vector.transpose %0, [1, 0] : vector<1x8xi32> |
| 7848 | /// to vector<8x1xi32> |
| 7849 | /// ``` |
| 7850 | /// can be rewritten as the equivalent |
| 7851 | /// ``` |
| 7852 | /// %0 = vector.broadcast %input : vector<1x1xi32> to vector<8x1xi32>. |
| 7853 | /// ``` |
| 7854 | /// The algorithm works by partitioning dimensions into groups that can be |
| 7855 | /// locally permuted while preserving order, and checks that the transpose |
| 7856 | /// only permutes within these groups. |
| 7857 | /// |
| 7858 | /// Groups are either contiguous sequences of 1s, or non-1s (1-element groups). |
| 7859 | /// Consider broadcasting 4x1x1x7 to 2x3x4x5x6x7. This is equivalent to |
| 7860 | /// broadcasting from 1x1x4x1x1x7. |
| 7861 | /// ^^^ ^ ^^^ ^ |
| 7862 | /// groups: 0 1 2 3 |
| 7863 | /// Order preserving permutations for this example are ones that only permute |
| 7864 | /// within the groups [0,1] and [3,4], like (1 0 2 4 3 5 6). |
| 7865 | class FoldTransposeBroadcast : public OpRewritePattern<vector::TransposeOp> { |
| 7866 | public: |
| 7867 | using Base::Base; |
| 7868 | FoldTransposeBroadcast(MLIRContext *context, PatternBenefit benefit = 1) |
| 7869 | : OpRewritePattern<vector::TransposeOp>(context, benefit) {} |
| 7870 | |
| 7871 | LogicalResult matchAndRewrite(vector::TransposeOp transpose, |
| 7872 | PatternRewriter &rewriter) const override { |
| 7873 | |
| 7874 | vector::BroadcastOp broadcast = |
| 7875 | transpose.getVector().getDefiningOp<vector::BroadcastOp>(); |
| 7876 | if (!broadcast) { |
| 7877 | return rewriter.notifyMatchFailure(arg&: transpose, |
| 7878 | msg: "not preceded by a broadcast" ); |
| 7879 | } |
| 7880 | |
| 7881 | auto inputType = dyn_cast<VectorType>(Val: broadcast.getSourceType()); |
| 7882 | VectorType outputType = transpose.getResultVectorType(); |
| 7883 | |
| 7884 | // transpose(broadcast(scalar)) -> broadcast(scalar) is always valid |
| 7885 | bool inputIsScalar = !inputType; |
| 7886 | if (inputIsScalar) { |
| 7887 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: transpose, args&: outputType, |
| 7888 | args: broadcast.getSource()); |
| 7889 | return success(); |
| 7890 | } |
| 7891 | |
| 7892 | ArrayRef<int64_t> permutation = transpose.getPermutation(); |
| 7893 | ArrayRef<int64_t> inputShape = inputType.getShape(); |
| 7894 | int64_t inputRank = inputType.getRank(); |
| 7895 | int64_t outputRank = transpose.getType().getRank(); |
| 7896 | int64_t deltaRank = outputRank - inputRank; |
| 7897 | |
| 7898 | int low = 0; |
| 7899 | for (int inputIndex = 0; inputIndex < inputRank; ++inputIndex) { |
| 7900 | bool notOne = inputShape[inputIndex] != 1; |
| 7901 | bool prevNotOne = (inputIndex != 0 && inputShape[inputIndex - 1] != 1); |
| 7902 | bool groupEndFound = notOne || prevNotOne; |
| 7903 | if (groupEndFound) { |
| 7904 | int high = inputIndex + deltaRank; |
| 7905 | // Return failure if not all permutation destinations for indices in |
| 7906 | // [low, high) are in [low, high), i.e. the permutation is not local to |
| 7907 | // the group. |
| 7908 | for (int i = low; i < high; ++i) { |
| 7909 | if (permutation[i] < low || permutation[i] >= high) { |
| 7910 | return rewriter.notifyMatchFailure( |
| 7911 | arg&: transpose, msg: "permutation not local to group" ); |
| 7912 | } |
| 7913 | } |
| 7914 | low = high; |
| 7915 | } |
| 7916 | } |
| 7917 | |
| 7918 | // We don't need to check the final group [low, outputRank) because if it is |
| 7919 | // not locally bound, there must be a preceding group that already failed |
| 7920 | // the check (impossible to have just 1 non-locally bound group). |
| 7921 | |
| 7922 | // The preceding logic also ensures that at this point, the output of the |
| 7923 | // transpose is definitely broadcastable from the input shape, assert so: |
| 7924 | assert(vector::isBroadcastableTo(inputType, outputType) == |
| 7925 | vector::BroadcastableToResult::Success && |
| 7926 | "not broadcastable directly to transpose output" ); |
| 7927 | |
| 7928 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: transpose, args&: outputType, |
| 7929 | args: broadcast.getSource()); |
| 7930 | |
| 7931 | return success(); |
| 7932 | } |
| 7933 | }; |
| 7934 | |
| 7935 | } // namespace |
| 7936 | |
| 7937 | void vector::TransposeOp::getCanonicalizationPatterns( |
| 7938 | RewritePatternSet &results, MLIRContext *context) { |
| 7939 | results.add<FoldTransposeCreateMask, FoldTransposeShapeCast, TransposeFolder, |
| 7940 | FoldTransposeSplat, FoldTransposeFromElements, |
| 7941 | FoldTransposeBroadcast>(arg&: context); |
| 7942 | } |
| 7943 | |
| 7944 | //===----------------------------------------------------------------------===// |
| 7945 | // ConstantMaskOp |
| 7946 | //===----------------------------------------------------------------------===// |
| 7947 | |
| 7948 | void ConstantMaskOp::build(OpBuilder &builder, OperationState &result, |
| 7949 | VectorType type, ConstantMaskKind kind) { |
| 7950 | assert(kind == ConstantMaskKind::AllTrue || |
| 7951 | kind == ConstantMaskKind::AllFalse); |
| 7952 | auto maskDimSizes = llvm::to_vector(Range: type.getShape()); |
| 7953 | if (maskDimSizes.empty()) |
| 7954 | maskDimSizes.push_back(Elt: 1); |
| 7955 | if (kind == ConstantMaskKind::AllFalse) |
| 7956 | maskDimSizes.assign(NumElts: maskDimSizes.size(), Elt: 0); |
| 7957 | build(odsBuilder&: builder, odsState&: result, resultType0: type, mask_dim_sizes: maskDimSizes); |
| 7958 | } |
| 7959 | |
| 7960 | LogicalResult ConstantMaskOp::verify() { |
| 7961 | auto resultType = llvm::cast<VectorType>(Val: getResult().getType()); |
| 7962 | // Check the corner case of 0-D vectors first. |
| 7963 | if (resultType.getRank() == 0) { |
| 7964 | if (getMaskDimSizes().size() != 1) |
| 7965 | return emitError(message: "array attr must have length 1 for 0-D vectors" ); |
| 7966 | auto dim = getMaskDimSizes()[0]; |
| 7967 | if (dim != 0 && dim != 1) |
| 7968 | return emitError(message: "mask dim size must be either 0 or 1 for 0-D vectors" ); |
| 7969 | return success(); |
| 7970 | } |
| 7971 | |
| 7972 | // Verify that array attr size matches the rank of the vector result. |
| 7973 | if (static_cast<int64_t>(getMaskDimSizes().size()) != resultType.getRank()) |
| 7974 | return emitOpError( |
| 7975 | message: "must specify array attr of size equal vector result rank" ); |
| 7976 | // Verify that each array attr element is in bounds of corresponding vector |
| 7977 | // result dimension size. |
| 7978 | auto resultShape = resultType.getShape(); |
| 7979 | auto resultScalableDims = resultType.getScalableDims(); |
| 7980 | ArrayRef<int64_t> maskDimSizes = getMaskDimSizes(); |
| 7981 | for (const auto [index, maskDimSize] : llvm::enumerate(First&: maskDimSizes)) { |
| 7982 | if (maskDimSize < 0 || maskDimSize > resultShape[index]) |
| 7983 | return emitOpError( |
| 7984 | message: "array attr of size out of bounds of vector result dimension size" ); |
| 7985 | if (resultScalableDims[index] && maskDimSize != 0 && |
| 7986 | maskDimSize != resultShape[index]) |
| 7987 | return emitOpError( |
| 7988 | message: "only supports 'none set' or 'all set' scalable dimensions" ); |
| 7989 | } |
| 7990 | // Verify that if one mask dim size is zero, they all should be zero (because |
| 7991 | // the mask region is a conjunction of each mask dimension interval). |
| 7992 | bool anyZeros = llvm::is_contained(Range&: maskDimSizes, Element: 0); |
| 7993 | bool allZeros = llvm::all_of(Range&: maskDimSizes, P: [](int64_t s) { return s == 0; }); |
| 7994 | if (anyZeros && !allZeros) |
| 7995 | return emitOpError(message: "expected all mask dim sizes to be zeros, " |
| 7996 | "as a result of conjunction with zero mask dim" ); |
| 7997 | return success(); |
| 7998 | } |
| 7999 | |
| 8000 | bool ConstantMaskOp::isAllOnesMask() { |
| 8001 | auto resultType = getVectorType(); |
| 8002 | // Check the corner case of 0-D vectors first. |
| 8003 | if (resultType.getRank() == 0) { |
| 8004 | assert(getMaskDimSizes().size() == 1 && "invalid sizes for zero rank mask" ); |
| 8005 | return getMaskDimSizes()[0] == 1; |
| 8006 | } |
| 8007 | for (const auto [resultSize, maskDimSize] : |
| 8008 | llvm::zip_equal(t: resultType.getShape(), u: getMaskDimSizes())) { |
| 8009 | if (maskDimSize < resultSize) |
| 8010 | return false; |
| 8011 | } |
| 8012 | return true; |
| 8013 | } |
| 8014 | |
| 8015 | OpFoldResult ConstantMaskOp::fold(FoldAdaptor adaptor) { |
| 8016 | ArrayRef<int64_t> bounds = getMaskDimSizes(); |
| 8017 | ArrayRef<int64_t> vectorSizes = getVectorType().getShape(); |
| 8018 | |
| 8019 | auto createBoolSplat = [&](bool x) { |
| 8020 | return SplatElementsAttr::get(type: getVectorType(), |
| 8021 | values: BoolAttr::get(context: getContext(), value: x)); |
| 8022 | }; |
| 8023 | |
| 8024 | // Check the corner case of 0-D vectors first. |
| 8025 | if (vectorSizes.empty()) { |
| 8026 | assert(bounds.size() == 1 && "invalid sizes for zero rank mask" ); |
| 8027 | return createBoolSplat(bounds[0] == 1); |
| 8028 | } |
| 8029 | // Fold vector.constant_mask to splat if possible. |
| 8030 | if (bounds == vectorSizes) |
| 8031 | return createBoolSplat(true); |
| 8032 | if (llvm::all_of(Range&: bounds, P: [](int64_t x) { return x == 0; })) |
| 8033 | return createBoolSplat(false); |
| 8034 | return OpFoldResult(); |
| 8035 | } |
| 8036 | |
| 8037 | std::optional<SmallVector<int64_t>> |
| 8038 | ConstantMaskOp::getDefaultUnrollTargetShape() { |
| 8039 | return getDefaultTargetMaskShapeForUnroll(); |
| 8040 | } |
| 8041 | |
| 8042 | //===----------------------------------------------------------------------===// |
| 8043 | // CreateMaskOp |
| 8044 | //===----------------------------------------------------------------------===// |
| 8045 | |
| 8046 | void CreateMaskOp::build(OpBuilder &builder, OperationState &result, |
| 8047 | VectorType type, |
| 8048 | ArrayRef<OpFoldResult> mixedOperands) { |
| 8049 | SmallVector<Value> operands = |
| 8050 | getValueOrCreateConstantIndexOp(b&: builder, loc: result.location, valueOrAttrVec: mixedOperands); |
| 8051 | build(odsBuilder&: builder, odsState&: result, resultType0: type, mask_dim_sizes: operands); |
| 8052 | } |
| 8053 | |
| 8054 | LogicalResult CreateMaskOp::verify() { |
| 8055 | auto vectorType = llvm::cast<VectorType>(Val: getResult().getType()); |
| 8056 | // Verify that an operand was specified for each result vector each dimension. |
| 8057 | if (vectorType.getRank() == 0) { |
| 8058 | if (getNumOperands() != 1) |
| 8059 | return emitOpError( |
| 8060 | message: "must specify exactly one operand for 0-D create_mask" ); |
| 8061 | } else if (getNumOperands() != |
| 8062 | llvm::cast<VectorType>(Val: getResult().getType()).getRank()) { |
| 8063 | return emitOpError( |
| 8064 | message: "must specify an operand for each result vector dimension" ); |
| 8065 | } |
| 8066 | return success(); |
| 8067 | } |
| 8068 | |
| 8069 | std::optional<SmallVector<int64_t>> |
| 8070 | CreateMaskOp::getDefaultUnrollTargetShape() { |
| 8071 | return getDefaultTargetMaskShapeForUnroll(); |
| 8072 | } |
| 8073 | |
| 8074 | namespace { |
| 8075 | |
| 8076 | /// Pattern to rewrite a CreateMaskOp with a ConstantMaskOp. |
| 8077 | /// |
| 8078 | /// Ex 1: |
| 8079 | /// %c2 = arith.constant 2 : index |
| 8080 | /// %c3 = arith.constant 3 : index |
| 8081 | /// %0 = vector.create_mask %c3, %c2 : vector<4x3xi1> |
| 8082 | /// Becomes: |
| 8083 | /// vector.constant_mask [3, 2] : vector<4x3xi1> |
| 8084 | /// |
| 8085 | /// Ex 2: |
| 8086 | /// %c_neg_1 = arith.constant -1 : index |
| 8087 | /// %0 = vector.create_mask %c_neg_1 : vector<[8]xi1> |
| 8088 | /// becomes: |
| 8089 | /// vector.constant_mask [0] : vector<[8]xi1> |
| 8090 | /// |
| 8091 | /// Ex 3: |
| 8092 | /// %c8 = arith.constant 8 : index |
| 8093 | /// %c16 = arith.constant 16 : index |
| 8094 | /// %0 = vector.vscale |
| 8095 | /// %1 = arith.muli %0, %c16 : index |
| 8096 | /// %10 = vector.create_mask %c8, %1 : vector<8x[16]xi1> |
| 8097 | /// becomes: |
| 8098 | /// %0 = vector.constant_mask [8, 16] : vector<8x[16]xi1> |
| 8099 | class CreateMaskFolder final : public OpRewritePattern<CreateMaskOp> { |
| 8100 | public: |
| 8101 | using Base::Base; |
| 8102 | |
| 8103 | LogicalResult matchAndRewrite(CreateMaskOp createMaskOp, |
| 8104 | PatternRewriter &rewriter) const override { |
| 8105 | VectorType maskType = createMaskOp.getVectorType(); |
| 8106 | ArrayRef<int64_t> maskTypeDimSizes = maskType.getShape(); |
| 8107 | ArrayRef<bool> maskTypeDimScalableFlags = maskType.getScalableDims(); |
| 8108 | |
| 8109 | // Special case: Rank zero shape. |
| 8110 | constexpr std::array<int64_t, 1> rankZeroShape{1}; |
| 8111 | constexpr std::array<bool, 1> rankZeroScalableDims{false}; |
| 8112 | if (maskType.getRank() == 0) { |
| 8113 | maskTypeDimSizes = rankZeroShape; |
| 8114 | maskTypeDimScalableFlags = rankZeroScalableDims; |
| 8115 | } |
| 8116 | |
| 8117 | // Determine if this CreateMaskOp can be folded to a ConstantMaskOp and |
| 8118 | // collect the `constantDims` (for the ConstantMaskOp). |
| 8119 | SmallVector<int64_t, 4> constantDims; |
| 8120 | for (auto [i, dimSize] : llvm::enumerate(First: createMaskOp.getOperands())) { |
| 8121 | if (auto intSize = getConstantIntValue(ofr: dimSize)) { |
| 8122 | // Constant value. |
| 8123 | // If the mask dim is non-scalable this can be any value. |
| 8124 | // If the mask dim is scalable only zero (all-false) is supported. |
| 8125 | if (maskTypeDimScalableFlags[i] && intSize >= 0) |
| 8126 | return failure(); |
| 8127 | constantDims.push_back(Elt: *intSize); |
| 8128 | } else if (auto vscaleMultiplier = getConstantVscaleMultiplier(value: dimSize)) { |
| 8129 | // Constant vscale multiple (e.g. 4 x vscale). |
| 8130 | // Must be all-true to fold to a ConstantMask. |
| 8131 | if (vscaleMultiplier < maskTypeDimSizes[i]) |
| 8132 | return failure(); |
| 8133 | constantDims.push_back(Elt: *vscaleMultiplier); |
| 8134 | } else { |
| 8135 | return failure(); |
| 8136 | } |
| 8137 | } |
| 8138 | |
| 8139 | // Clamp values to constant_mask bounds. |
| 8140 | for (auto [value, maskDimSize] : llvm::zip(t&: constantDims, u&: maskTypeDimSizes)) |
| 8141 | value = std::clamp<int64_t>(val: value, lo: 0, hi: maskDimSize); |
| 8142 | |
| 8143 | // If one of dim sizes is zero, set all dims to zero. |
| 8144 | if (llvm::is_contained(Range&: constantDims, Element: 0)) |
| 8145 | constantDims.assign(NumElts: constantDims.size(), Elt: 0); |
| 8146 | |
| 8147 | // Replace 'createMaskOp' with ConstantMaskOp. |
| 8148 | rewriter.replaceOpWithNewOp<ConstantMaskOp>(op: createMaskOp, args&: maskType, |
| 8149 | args&: constantDims); |
| 8150 | return success(); |
| 8151 | } |
| 8152 | }; |
| 8153 | |
| 8154 | } // namespace |
| 8155 | |
| 8156 | void CreateMaskOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 8157 | MLIRContext *context) { |
| 8158 | results.add<CreateMaskFolder>(arg&: context); |
| 8159 | } |
| 8160 | |
| 8161 | //===----------------------------------------------------------------------===// |
| 8162 | // MaskOp |
| 8163 | //===----------------------------------------------------------------------===// |
| 8164 | |
| 8165 | void MaskOp::build( |
| 8166 | OpBuilder &builder, OperationState &result, Value mask, |
| 8167 | Operation *maskableOp, |
| 8168 | function_ref<void(OpBuilder &, Operation *)> maskRegionBuilder) { |
| 8169 | assert(maskRegionBuilder && |
| 8170 | "builder callback for 'maskRegion' must be present" ); |
| 8171 | |
| 8172 | result.addOperands(newOperands: mask); |
| 8173 | OpBuilder::InsertionGuard guard(builder); |
| 8174 | Region *maskRegion = result.addRegion(); |
| 8175 | builder.createBlock(parent: maskRegion); |
| 8176 | maskRegionBuilder(builder, maskableOp); |
| 8177 | } |
| 8178 | |
| 8179 | void MaskOp::build( |
| 8180 | OpBuilder &builder, OperationState &result, TypeRange resultTypes, |
| 8181 | Value mask, Operation *maskableOp, |
| 8182 | function_ref<void(OpBuilder &, Operation *)> maskRegionBuilder) { |
| 8183 | build(odsBuilder&: builder, odsState&: result, resultTypes, mask, /*passthru=*/Value(), maskableOp, |
| 8184 | maskRegion: maskRegionBuilder); |
| 8185 | } |
| 8186 | |
| 8187 | void MaskOp::build( |
| 8188 | OpBuilder &builder, OperationState &result, TypeRange resultTypes, |
| 8189 | Value mask, Value passthru, Operation *maskableOp, |
| 8190 | function_ref<void(OpBuilder &, Operation *)> maskRegionBuilder) { |
| 8191 | build(builder, result, mask, maskableOp, maskRegionBuilder); |
| 8192 | if (passthru) |
| 8193 | result.addOperands(newOperands: passthru); |
| 8194 | result.addTypes(newTypes&: resultTypes); |
| 8195 | } |
| 8196 | |
| 8197 | ParseResult MaskOp::parse(OpAsmParser &parser, OperationState &result) { |
| 8198 | // Create the op region. |
| 8199 | result.regions.reserve(N: 1); |
| 8200 | Region &maskRegion = *result.addRegion(); |
| 8201 | |
| 8202 | auto &builder = parser.getBuilder(); |
| 8203 | |
| 8204 | // Parse all the operands. |
| 8205 | OpAsmParser::UnresolvedOperand mask; |
| 8206 | if (parser.parseOperand(result&: mask)) |
| 8207 | return failure(); |
| 8208 | |
| 8209 | // Optional passthru operand. |
| 8210 | OpAsmParser::UnresolvedOperand passthru; |
| 8211 | ParseResult parsePassthru = parser.parseOptionalComma(); |
| 8212 | if (parsePassthru.succeeded() && parser.parseOperand(result&: passthru)) |
| 8213 | return failure(); |
| 8214 | |
| 8215 | // Parse op region. |
| 8216 | if (parser.parseRegion(region&: maskRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {})) |
| 8217 | return failure(); |
| 8218 | |
| 8219 | MaskOp::ensureTerminator(region&: maskRegion, builder, loc: result.location); |
| 8220 | |
| 8221 | // Parse the optional attribute list. |
| 8222 | if (parser.parseOptionalAttrDict(result&: result.attributes)) |
| 8223 | return failure(); |
| 8224 | |
| 8225 | // Parse all the types. |
| 8226 | Type maskType; |
| 8227 | if (parser.parseColonType(result&: maskType)) |
| 8228 | return failure(); |
| 8229 | |
| 8230 | SmallVector<Type> resultTypes; |
| 8231 | if (parser.parseOptionalArrowTypeList(result&: resultTypes)) |
| 8232 | return failure(); |
| 8233 | result.types.append(RHS: resultTypes); |
| 8234 | |
| 8235 | // Resolve operands. |
| 8236 | if (parser.resolveOperand(operand: mask, type: maskType, result&: result.operands)) |
| 8237 | return failure(); |
| 8238 | |
| 8239 | if (parsePassthru.succeeded()) { |
| 8240 | if (resultTypes.empty()) |
| 8241 | return parser.emitError( |
| 8242 | loc: parser.getNameLoc(), |
| 8243 | message: "expects a result if passthru operand is provided" ); |
| 8244 | |
| 8245 | if (parser.resolveOperand(operand: passthru, type: resultTypes[0], result&: result.operands)) |
| 8246 | return failure(); |
| 8247 | } |
| 8248 | |
| 8249 | return success(); |
| 8250 | } |
| 8251 | |
| 8252 | void mlir::vector::MaskOp::print(OpAsmPrinter &p) { |
| 8253 | p << " " << getMask(); |
| 8254 | if (getPassthru()) |
| 8255 | p << ", " << getPassthru(); |
| 8256 | |
| 8257 | // Print single masked operation and skip terminator. |
| 8258 | p << " { " ; |
| 8259 | Block *singleBlock = &getMaskRegion().getBlocks().front(); |
| 8260 | if (singleBlock && !singleBlock->getOperations().empty()) |
| 8261 | p.printCustomOrGenericOp(op: &singleBlock->front()); |
| 8262 | p << " }" ; |
| 8263 | |
| 8264 | p.printOptionalAttrDict( |
| 8265 | attrs: getOperation()->getDiscardableAttrDictionary().getValue()); |
| 8266 | |
| 8267 | p << " : " << getMask().getType(); |
| 8268 | if (getNumResults() > 0) |
| 8269 | p << " -> " << getResultTypes(); |
| 8270 | } |
| 8271 | |
| 8272 | void MaskOp::ensureTerminator(Region ®ion, Builder &builder, Location loc) { |
| 8273 | // 1. For an empty `vector.mask`, create a default terminator. |
| 8274 | if (region.empty() || region.front().empty()) { |
| 8275 | OpTrait::SingleBlockImplicitTerminator<vector::YieldOp>::Impl< |
| 8276 | MaskOp>::ensureTerminator(region, builder, loc); |
| 8277 | return; |
| 8278 | } |
| 8279 | |
| 8280 | // 2. For a non-empty `vector.mask` with an explicit terminator, do nothing. |
| 8281 | Block &block = region.front(); |
| 8282 | if (isa<vector::YieldOp>(Val: block.back())) |
| 8283 | return; |
| 8284 | |
| 8285 | // 3. For a non-empty `vector.mask` without an explicit terminator: |
| 8286 | |
| 8287 | // Create default terminator if the number of masked operations is not |
| 8288 | // one. This case will trigger a verification failure. |
| 8289 | if (block.getOperations().size() != 1) { |
| 8290 | OpTrait::SingleBlockImplicitTerminator<vector::YieldOp>::Impl< |
| 8291 | MaskOp>::ensureTerminator(region, builder, loc); |
| 8292 | return; |
| 8293 | } |
| 8294 | |
| 8295 | // Create a terminator that yields the results from the masked operation. |
| 8296 | OpBuilder opBuilder(builder.getContext()); |
| 8297 | Operation *maskedOp = &block.front(); |
| 8298 | opBuilder.setInsertionPointToEnd(&block); |
| 8299 | vector::YieldOp::create(builder&: opBuilder, location: loc, operands: maskedOp->getResults()); |
| 8300 | } |
| 8301 | |
| 8302 | LogicalResult MaskOp::verify() { |
| 8303 | // Structural checks. |
| 8304 | Block &block = getMaskRegion().getBlocks().front(); |
| 8305 | if (block.getOperations().empty()) |
| 8306 | return emitOpError(message: "expects a terminator within the mask region" ); |
| 8307 | |
| 8308 | unsigned numMaskRegionOps = block.getOperations().size(); |
| 8309 | if (numMaskRegionOps > 2) |
| 8310 | return emitOpError(message: "expects only one operation to mask" ); |
| 8311 | |
| 8312 | // Terminator checks. |
| 8313 | auto terminator = dyn_cast<vector::YieldOp>(Val&: block.back()); |
| 8314 | if (!terminator) |
| 8315 | return emitOpError(message: "expects a terminator within the mask region" ); |
| 8316 | |
| 8317 | if (terminator->getNumOperands() != getNumResults()) |
| 8318 | return emitOpError( |
| 8319 | message: "expects number of results to match mask region yielded values" ); |
| 8320 | |
| 8321 | // Empty vector.mask. Nothing else to check. |
| 8322 | if (numMaskRegionOps == 1) |
| 8323 | return success(); |
| 8324 | |
| 8325 | auto maskableOp = dyn_cast<MaskableOpInterface>(Val&: block.front()); |
| 8326 | if (!maskableOp) |
| 8327 | return emitOpError(message: "expects a MaskableOpInterface within the mask region" ); |
| 8328 | |
| 8329 | // Result checks. |
| 8330 | if (maskableOp->getNumResults() != getNumResults()) |
| 8331 | return emitOpError(message: "expects number of results to match maskable operation " |
| 8332 | "number of results" ); |
| 8333 | |
| 8334 | if (!llvm::equal(LRange: maskableOp->getResults(), RRange: terminator.getOperands())) |
| 8335 | return emitOpError(message: "expects all the results from the MaskableOpInterface " |
| 8336 | "to match all the values returned by the terminator" ); |
| 8337 | |
| 8338 | if (!llvm::equal(LRange: maskableOp->getResultTypes(), RRange: getResultTypes())) |
| 8339 | return emitOpError( |
| 8340 | message: "expects result type to match maskable operation result type" ); |
| 8341 | |
| 8342 | if (llvm::count_if(Range: maskableOp->getResultTypes(), |
| 8343 | P: [](Type t) { return llvm::isa<VectorType>(Val: t); }) > 1) |
| 8344 | return emitOpError(message: "multiple vector results not supported" ); |
| 8345 | |
| 8346 | // Mask checks. |
| 8347 | Type expectedMaskType = maskableOp.getExpectedMaskType(); |
| 8348 | if (getMask().getType() != expectedMaskType) |
| 8349 | return emitOpError(message: "expects a " ) |
| 8350 | << expectedMaskType << " mask for the maskable operation" ; |
| 8351 | |
| 8352 | auto transferOp = |
| 8353 | dyn_cast<VectorTransferOpInterface>(Val: maskableOp.getOperation()); |
| 8354 | if (transferOp && transferOp.getMask()) |
| 8355 | return emitOpError( |
| 8356 | message: "doesn't expect a mask operand for the masked transfer operation" ); |
| 8357 | |
| 8358 | // Passthru checks. |
| 8359 | Value passthru = getPassthru(); |
| 8360 | if (passthru) { |
| 8361 | if (!maskableOp.supportsPassthru()) |
| 8362 | return emitOpError( |
| 8363 | message: "doesn't expect a passthru argument for this maskable operation" ); |
| 8364 | |
| 8365 | if (maskableOp->getNumResults() != 1) |
| 8366 | return emitOpError(message: "expects result when passthru argument is provided" ); |
| 8367 | |
| 8368 | if (passthru.getType() != maskableOp->getResultTypes()[0]) |
| 8369 | return emitOpError(message: "expects passthru type to match result type" ); |
| 8370 | } |
| 8371 | |
| 8372 | return success(); |
| 8373 | } |
| 8374 | |
| 8375 | /// Folds empty `vector.mask` with no passthru operand and with or without |
| 8376 | /// return values. For example: |
| 8377 | /// |
| 8378 | /// %0 = vector.mask %mask { vector.yield %a : vector<8xf32> } : |
| 8379 | /// vector<8xi1> -> vector<8xf32> |
| 8380 | /// %1 = user_op %0 : vector<8xf32> |
| 8381 | /// |
| 8382 | /// becomes: |
| 8383 | /// |
| 8384 | /// %0 = user_op %a : vector<8xf32> |
| 8385 | /// |
| 8386 | /// Empty `vector.mask` with passthru operand are handled by the canonicalizer |
| 8387 | /// as it requires creating new operations. |
| 8388 | |
| 8389 | static LogicalResult foldEmptyMaskOp(MaskOp maskOp, MaskOp::FoldAdaptor adaptor, |
| 8390 | SmallVectorImpl<OpFoldResult> &results) { |
| 8391 | if (!maskOp.isEmpty() || maskOp.hasPassthru()) |
| 8392 | return failure(); |
| 8393 | |
| 8394 | Block *block = maskOp.getMaskBlock(); |
| 8395 | auto terminator = cast<vector::YieldOp>(Val&: block->front()); |
| 8396 | if (terminator.getNumOperands() == 0) |
| 8397 | return failure(); |
| 8398 | |
| 8399 | // `vector.mask` has results, propagate the results. |
| 8400 | llvm::append_range(C&: results, R: terminator.getOperands()); |
| 8401 | return success(); |
| 8402 | } |
| 8403 | |
| 8404 | LogicalResult MaskOp::fold(FoldAdaptor adaptor, |
| 8405 | SmallVectorImpl<OpFoldResult> &results) { |
| 8406 | if (succeeded(Result: foldEmptyMaskOp(maskOp: *this, adaptor, results))) |
| 8407 | return success(); |
| 8408 | |
| 8409 | MaskFormat maskFormat = getMaskFormat(mask: getMask()); |
| 8410 | if (maskFormat != MaskFormat::AllTrue) |
| 8411 | return failure(); |
| 8412 | |
| 8413 | // Move maskable operation outside of the `vector.mask` region. |
| 8414 | // If there is no maskable op (empty body), the fold cannot proceed; the |
| 8415 | // canonicalizer handles this case instead. |
| 8416 | Operation *maskableOp = getMaskableOp(); |
| 8417 | if (!maskableOp) |
| 8418 | return failure(); |
| 8419 | maskableOp->dropAllUses(); |
| 8420 | maskableOp->moveBefore(existingOp: getOperation()); |
| 8421 | |
| 8422 | llvm::append_range(C&: results, R: maskableOp->getResults()); |
| 8423 | return success(); |
| 8424 | } |
| 8425 | |
| 8426 | /// Canonialize empty `vector.mask` operations that can't be handled in |
| 8427 | /// `VectorMask::fold` as they require creating new operations. |
| 8428 | /// |
| 8429 | /// Example 1: Empty `vector.mask` with passthru operand. |
| 8430 | /// |
| 8431 | /// %0 = vector.mask %mask, %passthru { vector.yield %a : vector<8xf32> } : |
| 8432 | /// vector<8xi1> -> vector<8xf32> |
| 8433 | /// |
| 8434 | /// becomes: |
| 8435 | /// |
| 8436 | /// %0 = arith.select %mask, %a, %passthru : vector<8xf32> |
| 8437 | /// |
| 8438 | class CanonializeEmptyMaskOp : public OpRewritePattern<MaskOp> { |
| 8439 | using Base::Base; |
| 8440 | |
| 8441 | LogicalResult matchAndRewrite(MaskOp maskOp, |
| 8442 | PatternRewriter &rewriter) const override { |
| 8443 | if (!maskOp.isEmpty()) |
| 8444 | return failure(); |
| 8445 | |
| 8446 | if (!maskOp.hasPassthru()) |
| 8447 | return failure(); |
| 8448 | |
| 8449 | // arith.select with a vector condition requires the value types to be |
| 8450 | // vectors of the same shape. Since vector.mask always has a vector mask |
| 8451 | // type, bail out when any result type doesn't match the mask shape to |
| 8452 | // avoid creating invalid IR. |
| 8453 | VectorType maskType = maskOp.getMask().getType(); |
| 8454 | for (Type resultType : maskOp.getResultTypes()) { |
| 8455 | auto vecResultType = dyn_cast<VectorType>(Val&: resultType); |
| 8456 | if (!vecResultType || vecResultType.getShape() != maskType.getShape()) |
| 8457 | return failure(); |
| 8458 | } |
| 8459 | |
| 8460 | Block *block = maskOp.getMaskBlock(); |
| 8461 | auto terminator = cast<vector::YieldOp>(Val&: block->front()); |
| 8462 | assert(terminator.getNumOperands() == 1 && |
| 8463 | "expected one result when passthru is provided" ); |
| 8464 | |
| 8465 | rewriter.replaceOpWithNewOp<arith::SelectOp>( |
| 8466 | op: maskOp, args: maskOp.getResultTypes(), args: maskOp.getMask(), |
| 8467 | args: terminator.getOperand(i: 0), args: maskOp.getPassthru()); |
| 8468 | |
| 8469 | return success(); |
| 8470 | } |
| 8471 | }; |
| 8472 | |
| 8473 | void MaskOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 8474 | MLIRContext *context) { |
| 8475 | results.add<CanonializeEmptyMaskOp>(arg&: context); |
| 8476 | } |
| 8477 | |
| 8478 | // MaskingOpInterface definitions. |
| 8479 | |
| 8480 | /// Returns the operation masked by this 'vector.mask'. |
| 8481 | Operation *MaskOp::getMaskableOp() { |
| 8482 | Block *block = getMaskBlock(); |
| 8483 | if (block->getOperations().size() < 2) |
| 8484 | return nullptr; |
| 8485 | |
| 8486 | return &block->front(); |
| 8487 | } |
| 8488 | |
| 8489 | /// Returns true if 'vector.mask' has a passthru value. |
| 8490 | bool MaskOp::hasPassthru() { return getPassthru() != Value(); } |
| 8491 | |
| 8492 | //===----------------------------------------------------------------------===// |
| 8493 | // ScanOp |
| 8494 | //===----------------------------------------------------------------------===// |
| 8495 | |
| 8496 | LogicalResult ScanOp::verify() { |
| 8497 | VectorType srcType = getSourceType(); |
| 8498 | VectorType initialType = getInitialValueType(); |
| 8499 | // Check reduction dimension < rank. |
| 8500 | int64_t srcRank = srcType.getRank(); |
| 8501 | int64_t reductionDim = getReductionDim(); |
| 8502 | if (reductionDim >= srcRank) |
| 8503 | return emitOpError(message: "reduction dimension " ) |
| 8504 | << reductionDim << " has to be less than " << srcRank; |
| 8505 | |
| 8506 | // Check that rank(initial_value) = rank(src) - 1. |
| 8507 | int64_t initialValueRank = initialType.getRank(); |
| 8508 | if (initialValueRank != srcRank - 1) |
| 8509 | return emitOpError(message: "initial value rank " ) |
| 8510 | << initialValueRank << " has to be equal to " << srcRank - 1; |
| 8511 | |
| 8512 | // Check shapes of initial value and src. |
| 8513 | ArrayRef<int64_t> srcShape = srcType.getShape(); |
| 8514 | ArrayRef<int64_t> initialValueShapes = initialType.getShape(); |
| 8515 | SmallVector<int64_t> expectedShape; |
| 8516 | for (int i = 0; i < srcRank; i++) { |
| 8517 | if (i != reductionDim) |
| 8518 | expectedShape.push_back(Elt: srcShape[i]); |
| 8519 | } |
| 8520 | if (!llvm::equal(LRange&: initialValueShapes, RRange&: expectedShape)) { |
| 8521 | return emitOpError(message: "incompatible input/initial value shapes" ); |
| 8522 | } |
| 8523 | |
| 8524 | // Verify supported reduction kind. |
| 8525 | Type eltType = getDestType().getElementType(); |
| 8526 | if (!isSupportedCombiningKind(combiningKind: getKind(), elementType: eltType)) |
| 8527 | return emitOpError(message: "unsupported reduction type " ) |
| 8528 | << eltType << " for kind '" << stringifyCombiningKind(val: getKind()) |
| 8529 | << "'" ; |
| 8530 | |
| 8531 | return success(); |
| 8532 | } |
| 8533 | |
| 8534 | void mlir::vector::populateVectorToVectorCanonicalizationPatterns( |
| 8535 | RewritePatternSet &patterns, PatternBenefit benefit) { |
| 8536 | patterns |
| 8537 | .add<CreateMaskFolder, MaskedLoadFolder, MaskedStoreFolder, GatherFolder, |
| 8538 | ScatterFolder, ExpandLoadFolder, CompressStoreFolder, |
| 8539 | StridedSliceConstantMaskFolder, TransposeFolder>( |
| 8540 | arg: patterns.getContext(), args&: benefit); |
| 8541 | } |
| 8542 | |
| 8543 | Value mlir::vector::makeArithReduction(OpBuilder &b, Location loc, |
| 8544 | CombiningKind kind, Value v1, Value acc, |
| 8545 | arith::FastMathFlagsAttr fastmath, |
| 8546 | Value mask) { |
| 8547 | Type t1 = getElementTypeOrSelf(type: v1.getType()); |
| 8548 | Type tAcc = getElementTypeOrSelf(type: acc.getType()); |
| 8549 | Value result; |
| 8550 | |
| 8551 | switch (kind) { |
| 8552 | case CombiningKind::ADD: |
| 8553 | if (t1.isIntOrIndex() && tAcc.isIntOrIndex()) |
| 8554 | result = b.createOrFold<arith::AddIOp>(location: loc, args&: v1, args&: acc); |
| 8555 | else if (llvm::isa<FloatType>(Val: t1) && llvm::isa<FloatType>(Val: tAcc)) |
| 8556 | result = b.createOrFold<arith::AddFOp>(location: loc, args&: v1, args&: acc, args&: fastmath); |
| 8557 | else |
| 8558 | llvm_unreachable("invalid value types for ADD reduction" ); |
| 8559 | break; |
| 8560 | case CombiningKind::AND: |
| 8561 | assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values" ); |
| 8562 | result = b.createOrFold<arith::AndIOp>(location: loc, args&: v1, args&: acc); |
| 8563 | break; |
| 8564 | case CombiningKind::MAXNUMF: |
| 8565 | assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) && |
| 8566 | "expected float values" ); |
| 8567 | result = b.createOrFold<arith::MaxNumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath); |
| 8568 | break; |
| 8569 | case CombiningKind::MAXIMUMNUMF: |
| 8570 | assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) && |
| 8571 | "expected float values" ); |
| 8572 | result = b.createOrFold<arith::MaximumNumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath); |
| 8573 | break; |
| 8574 | case CombiningKind::MAXIMUMF: |
| 8575 | assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) && |
| 8576 | "expected float values" ); |
| 8577 | result = b.createOrFold<arith::MaximumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath); |
| 8578 | break; |
| 8579 | case CombiningKind::MINNUMF: |
| 8580 | assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) && |
| 8581 | "expected float values" ); |
| 8582 | result = b.createOrFold<arith::MinNumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath); |
| 8583 | break; |
| 8584 | case CombiningKind::MINIMUMNUMF: |
| 8585 | assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) && |
| 8586 | "expected float values" ); |
| 8587 | result = b.createOrFold<arith::MinimumNumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath); |
| 8588 | break; |
| 8589 | case CombiningKind::MINIMUMF: |
| 8590 | assert(llvm::isa<FloatType>(t1) && llvm::isa<FloatType>(tAcc) && |
| 8591 | "expected float values" ); |
| 8592 | result = b.createOrFold<arith::MinimumFOp>(location: loc, args&: v1, args&: acc, args&: fastmath); |
| 8593 | break; |
| 8594 | case CombiningKind::MAXSI: |
| 8595 | assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values" ); |
| 8596 | result = b.createOrFold<arith::MaxSIOp>(location: loc, args&: v1, args&: acc); |
| 8597 | break; |
| 8598 | case CombiningKind::MINSI: |
| 8599 | assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values" ); |
| 8600 | result = b.createOrFold<arith::MinSIOp>(location: loc, args&: v1, args&: acc); |
| 8601 | break; |
| 8602 | case CombiningKind::MAXUI: |
| 8603 | assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values" ); |
| 8604 | result = b.createOrFold<arith::MaxUIOp>(location: loc, args&: v1, args&: acc); |
| 8605 | break; |
| 8606 | case CombiningKind::MINUI: |
| 8607 | assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values" ); |
| 8608 | result = b.createOrFold<arith::MinUIOp>(location: loc, args&: v1, args&: acc); |
| 8609 | break; |
| 8610 | case CombiningKind::MUL: |
| 8611 | if (t1.isIntOrIndex() && tAcc.isIntOrIndex()) |
| 8612 | result = b.createOrFold<arith::MulIOp>(location: loc, args&: v1, args&: acc); |
| 8613 | else if (llvm::isa<FloatType>(Val: t1) && llvm::isa<FloatType>(Val: tAcc)) |
| 8614 | result = b.createOrFold<arith::MulFOp>(location: loc, args&: v1, args&: acc, args&: fastmath); |
| 8615 | else |
| 8616 | llvm_unreachable("invalid value types for MUL reduction" ); |
| 8617 | break; |
| 8618 | case CombiningKind::OR: |
| 8619 | assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values" ); |
| 8620 | result = b.createOrFold<arith::OrIOp>(location: loc, args&: v1, args&: acc); |
| 8621 | break; |
| 8622 | case CombiningKind::XOR: |
| 8623 | assert(t1.isIntOrIndex() && tAcc.isIntOrIndex() && "expected int values" ); |
| 8624 | result = b.createOrFold<arith::XOrIOp>(location: loc, args&: v1, args&: acc); |
| 8625 | break; |
| 8626 | }; |
| 8627 | |
| 8628 | assert(result && "unknown CombiningKind" ); |
| 8629 | return selectPassthru(builder&: b, mask, newValue: result, passthru: acc); |
| 8630 | } |
| 8631 | |
| 8632 | //===----------------------------------------------------------------------===// |
| 8633 | // StepOp |
| 8634 | //===----------------------------------------------------------------------===// |
| 8635 | |
| 8636 | void StepOp::inferResultRanges(ArrayRef<ConstantIntRanges> argRanges, |
| 8637 | SetIntRangeFn setResultRanges) { |
| 8638 | auto resultType = cast<VectorType>(Val: getType()); |
| 8639 | if (resultType.isScalable()) { |
| 8640 | return; |
| 8641 | } |
| 8642 | unsigned bitwidth = ConstantIntRanges::getStorageBitwidth(type: resultType); |
| 8643 | // The result holds the sequence [0, 1, ..., N-1], with each value truncated |
| 8644 | // to the result element type. |
| 8645 | uint64_t maxIndex = resultType.getDimSize(idx: 0) - 1; |
| 8646 | APInt umin = APInt::getZero(numBits: bitwidth); |
| 8647 | APInt umax = APInt::getMaxValue(numBits: bitwidth).ugt(RHS: maxIndex) |
| 8648 | ? APInt(bitwidth, maxIndex) |
| 8649 | : APInt::getMaxValue(numBits: bitwidth); |
| 8650 | setResultRanges(getResult(), ConstantIntRanges::fromUnsigned(umin, umax)); |
| 8651 | } |
| 8652 | |
| 8653 | std::optional<SmallVector<int64_t>> StepOp::getDefaultUnrollTargetShape() { |
| 8654 | return SmallVector<int64_t>{8}; |
| 8655 | } |
| 8656 | |
| 8657 | namespace { |
| 8658 | |
| 8659 | /// Fold `vector.step -> arith.cmpi` when the step value is compared to a |
| 8660 | /// constant large enough such that the result is the same at all indices. |
| 8661 | /// |
| 8662 | /// For example, rewrite the 'greater than' comparison below, |
| 8663 | /// |
| 8664 | /// ```mlir |
| 8665 | /// %cst = arith.constant dense<7> : vector<3xindex> |
| 8666 | /// %stp = vector.step : vector<3xindex> |
| 8667 | /// %out = arith.cmpi ugt, %stp, %cst : vector<3xindex> |
| 8668 | /// ``` |
| 8669 | /// |
| 8670 | /// as, |
| 8671 | /// |
| 8672 | /// ```mlir |
| 8673 | /// %out = arith.constant dense<false> : vector<3xi1>. |
| 8674 | /// ``` |
| 8675 | /// |
| 8676 | /// Above `[0, 1, 2] > [7, 7, 7]` => `[false, false, false]`. Because the result |
| 8677 | /// is false at ALL indices we fold. If the constant was 1, then |
| 8678 | /// `[0, 1, 2] > [1, 1, 1]` => `[false, false, true]` and we do fold, |
| 8679 | /// conservatively preferring the 'compact' vector.step representation. |
| 8680 | /// |
| 8681 | /// Note: this folder only works for the case where the constant (`%cst` above) |
| 8682 | /// is the second operand of the comparison. The arith.cmpi canonicalizer will |
| 8683 | /// ensure that constants are always second (on the right). |
| 8684 | struct StepCompareFolder : public OpRewritePattern<StepOp> { |
| 8685 | using Base::Base; |
| 8686 | |
| 8687 | LogicalResult matchAndRewrite(StepOp stepOp, |
| 8688 | PatternRewriter &rewriter) const override { |
| 8689 | const int64_t stepSize = stepOp.getResult().getType().getNumElements(); |
| 8690 | |
| 8691 | for (OpOperand &use : stepOp.getResult().getUses()) { |
| 8692 | auto cmpiOp = dyn_cast<arith::CmpIOp>(Val: use.getOwner()); |
| 8693 | if (!cmpiOp) |
| 8694 | continue; |
| 8695 | |
| 8696 | // arith.cmpi canonicalizer makes constants final operands. |
| 8697 | const unsigned stepOperandNumber = use.getOperandNumber(); |
| 8698 | if (stepOperandNumber != 0) |
| 8699 | continue; |
| 8700 | |
| 8701 | // Check that operand 1 is a constant. |
| 8702 | unsigned constOperandNumber = 1; |
| 8703 | Value otherOperand = cmpiOp.getOperand(i: constOperandNumber); |
| 8704 | std::optional<int64_t> maybeConstValue = |
| 8705 | getConstantIntValue(ofr: otherOperand); |
| 8706 | if (!maybeConstValue.has_value()) |
| 8707 | continue; |
| 8708 | |
| 8709 | int64_t constValue = maybeConstValue.value(); |
| 8710 | arith::CmpIPredicate pred = cmpiOp.getPredicate(); |
| 8711 | |
| 8712 | auto maybeSplat = [&]() -> std::optional<bool> { |
| 8713 | // Handle ult (unsigned less than) and uge (unsigned greater equal). |
| 8714 | if ((pred == arith::CmpIPredicate::ult || |
| 8715 | pred == arith::CmpIPredicate::uge) && |
| 8716 | stepSize <= constValue) |
| 8717 | return pred == arith::CmpIPredicate::ult; |
| 8718 | |
| 8719 | // Handle ule and ugt. |
| 8720 | if ((pred == arith::CmpIPredicate::ule || |
| 8721 | pred == arith::CmpIPredicate::ugt) && |
| 8722 | stepSize - 1 <= constValue) { |
| 8723 | return pred == arith::CmpIPredicate::ule; |
| 8724 | } |
| 8725 | |
| 8726 | // Handle eq and ne. |
| 8727 | if ((pred == arith::CmpIPredicate::eq || |
| 8728 | pred == arith::CmpIPredicate::ne) && |
| 8729 | stepSize <= constValue) |
| 8730 | return pred == arith::CmpIPredicate::ne; |
| 8731 | |
| 8732 | return std::nullopt; |
| 8733 | }(); |
| 8734 | |
| 8735 | if (!maybeSplat.has_value()) |
| 8736 | continue; |
| 8737 | |
| 8738 | rewriter.setInsertionPointAfter(cmpiOp); |
| 8739 | |
| 8740 | auto type = dyn_cast<VectorType>(Val: cmpiOp.getResult().getType()); |
| 8741 | if (!type) |
| 8742 | continue; |
| 8743 | |
| 8744 | auto boolAttr = DenseElementsAttr::get(type, value: maybeSplat.value()); |
| 8745 | Value splat = mlir::arith::ConstantOp::create(builder&: rewriter, location: cmpiOp.getLoc(), |
| 8746 | result: type, value: boolAttr); |
| 8747 | |
| 8748 | rewriter.replaceOp(op: cmpiOp, newValues: splat); |
| 8749 | return success(); |
| 8750 | } |
| 8751 | |
| 8752 | return failure(); |
| 8753 | } |
| 8754 | }; |
| 8755 | } // namespace |
| 8756 | |
| 8757 | void StepOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 8758 | MLIRContext *context) { |
| 8759 | results.add<StepCompareFolder>(arg&: context); |
| 8760 | } |
| 8761 | |
| 8762 | //===----------------------------------------------------------------------===// |
| 8763 | // Vector Masking Utilities |
| 8764 | //===----------------------------------------------------------------------===// |
| 8765 | |
| 8766 | /// Create the vector.yield-ended region of a vector.mask op with `maskableOp` |
| 8767 | /// as masked operation. |
| 8768 | void mlir::vector::createMaskOpRegion(OpBuilder &builder, |
| 8769 | Operation *maskableOp) { |
| 8770 | assert(maskableOp->getBlock() && "MaskableOp must be inserted into a block" ); |
| 8771 | Block *insBlock = builder.getInsertionBlock(); |
| 8772 | // Create a block and move the op to that block. |
| 8773 | insBlock->getOperations().splice( |
| 8774 | where: insBlock->begin(), L2&: maskableOp->getBlock()->getOperations(), N: maskableOp); |
| 8775 | YieldOp::create(builder, location: maskableOp->getLoc(), operands: maskableOp->getResults()); |
| 8776 | } |
| 8777 | |
| 8778 | /// Creates a vector.mask operation around a maskable operation. Returns the |
| 8779 | /// vector.mask operation if the mask provided is valid. Otherwise, returns |
| 8780 | /// the maskable operation itself. |
| 8781 | Operation *mlir::vector::maskOperation(OpBuilder &builder, |
| 8782 | Operation *maskableOp, Value mask, |
| 8783 | Value passthru) { |
| 8784 | if (!mask) |
| 8785 | return maskableOp; |
| 8786 | if (passthru) |
| 8787 | return MaskOp::create(builder, location: maskableOp->getLoc(), |
| 8788 | resultTypes: maskableOp->getResultTypes(), mask, passthru, |
| 8789 | maskableOp, maskRegion: createMaskOpRegion); |
| 8790 | return MaskOp::create(builder, location: maskableOp->getLoc(), |
| 8791 | resultTypes: maskableOp->getResultTypes(), mask, maskableOp, |
| 8792 | maskRegion: createMaskOpRegion); |
| 8793 | } |
| 8794 | |
| 8795 | /// Creates a vector select operation that picks values from `newValue` or |
| 8796 | /// `passthru` for each result vector lane based on `mask`. This utility is used |
| 8797 | /// to propagate the pass-thru value of vector.mask or for cases where only the |
| 8798 | /// pass-thru value propagation is needed. VP intrinsics do not support |
| 8799 | /// pass-thru values and every mask-out lane is set to poison. LLVM backends are |
| 8800 | /// usually able to match op + select patterns and fold them into a native |
| 8801 | /// target instructions. |
| 8802 | Value mlir::vector::selectPassthru(OpBuilder &builder, Value mask, |
| 8803 | Value newValue, Value passthru) { |
| 8804 | if (!mask) |
| 8805 | return newValue; |
| 8806 | |
| 8807 | return arith::SelectOp::create(builder, location: newValue.getLoc(), result: newValue.getType(), |
| 8808 | condition: mask, true_value: newValue, false_value: passthru); |
| 8809 | } |
| 8810 | |
| 8811 | //===----------------------------------------------------------------------===// |
| 8812 | // InterleaveOp |
| 8813 | //===----------------------------------------------------------------------===// |
| 8814 | |
| 8815 | namespace { |
| 8816 | |
| 8817 | /// This folder works on the following round-trip identity: |
| 8818 | /// interleave(deinterleave(x).even, deinterleave(x).odd) -> x |
| 8819 | struct InterleaveDeinterleaveFolder : public OpRewritePattern<InterleaveOp> { |
| 8820 | using Base::Base; |
| 8821 | |
| 8822 | LogicalResult matchAndRewrite(InterleaveOp interleaveOp, |
| 8823 | PatternRewriter &rewriter) const override { |
| 8824 | auto lhsDefOp = interleaveOp.getLhs().getDefiningOp<DeinterleaveOp>(); |
| 8825 | auto rhsDefOp = interleaveOp.getRhs().getDefiningOp<DeinterleaveOp>(); |
| 8826 | if (!lhsDefOp || !rhsDefOp || lhsDefOp != rhsDefOp) |
| 8827 | return failure(); |
| 8828 | for (auto [idx, operand] : llvm::enumerate(First: interleaveOp.getOperands())) { |
| 8829 | if (cast<OpResult>(Val&: operand).getResultNumber() != idx) |
| 8830 | return failure(); |
| 8831 | } |
| 8832 | rewriter.replaceOp(op: interleaveOp, newValues: lhsDefOp.getSource()); |
| 8833 | return success(); |
| 8834 | } |
| 8835 | }; |
| 8836 | } // namespace |
| 8837 | |
| 8838 | void InterleaveOp::getCanonicalizationPatterns(RewritePatternSet &results, |
| 8839 | MLIRContext *context) { |
| 8840 | results.add<InterleaveDeinterleaveFolder>(arg&: context); |
| 8841 | } |
| 8842 | |
| 8843 | OpFoldResult InterleaveOp::fold(FoldAdaptor adaptor) { |
| 8844 | // interleave(splat(x), splat(x)) -> widened splat(x) |
| 8845 | auto splat = dyn_cast_if_present<SplatElementsAttr>(Val: adaptor.getLhs()); |
| 8846 | if (!splat || adaptor.getLhs() != adaptor.getRhs()) |
| 8847 | return {}; |
| 8848 | return SplatElementsAttr::get(type: getResultVectorType(), |
| 8849 | values: splat.getSplatValue<Attribute>()); |
| 8850 | } |
| 8851 | |
| 8852 | //----------------------------------------------------------------------------- |
| 8853 | // VectorUnrollOpInterface methods |
| 8854 | //----------------------------------------------------------------------------- |
| 8855 | std::optional<SmallVector<int64_t, 4>> |
| 8856 | InterleaveOp::getIterationShapeToUnroll() { |
| 8857 | return llvm::to_vector<4>(Range: getResultVectorType().getShape()); |
| 8858 | } |
| 8859 | |
| 8860 | std::optional<SmallVector<int64_t>> |
| 8861 | InterleaveOp::getDefaultUnrollTargetShape() { |
| 8862 | return getDefaultTargetInterleaveShapeForUnroll(); |
| 8863 | } |
| 8864 | |
| 8865 | //===----------------------------------------------------------------------===// |
| 8866 | // DeinterleaveOp |
| 8867 | //===----------------------------------------------------------------------===// |
| 8868 | //----------------------------------------------------------------------------- |
| 8869 | // VectorUnrollOpInterface methods |
| 8870 | //----------------------------------------------------------------------------- |
| 8871 | |
| 8872 | std::optional<SmallVector<int64_t, 4>> |
| 8873 | DeinterleaveOp::getIterationShapeToUnroll() { |
| 8874 | return llvm::to_vector<4>(Range: getResultVectorType().getShape()); |
| 8875 | } |
| 8876 | std::optional<SmallVector<int64_t>> |
| 8877 | DeinterleaveOp::getDefaultUnrollTargetShape() { |
| 8878 | return getDefaultTargetInterleaveShapeForUnroll(); |
| 8879 | } |
| 8880 | |
| 8881 | //===----------------------------------------------------------------------===// |
| 8882 | // VectorDialect canonicalization |
| 8883 | //===----------------------------------------------------------------------===// |
| 8884 | |
| 8885 | namespace { |
| 8886 | /// A broadcast of a scalar condition selects the same arm in every lane. |
| 8887 | struct SelectBroadcastCondition : OpRewritePattern<arith::SelectOp> { |
| 8888 | using Base::Base; |
| 8889 | |
| 8890 | LogicalResult matchAndRewrite(arith::SelectOp select, |
| 8891 | PatternRewriter &rewriter) const override { |
| 8892 | auto broadcast = select.getCondition().getDefiningOp<BroadcastOp>(); |
| 8893 | if (!broadcast || !broadcast.getSourceType().isSignlessInteger(width: 1)) |
| 8894 | return failure(); |
| 8895 | rewriter.modifyOpInPlace(root: select, callable: [&] { |
| 8896 | select.getConditionMutable().assign(value: broadcast.getSource()); |
| 8897 | }); |
| 8898 | return success(); |
| 8899 | } |
| 8900 | }; |
| 8901 | } // namespace |
| 8902 | |
| 8903 | void VectorDialect::getCanonicalizationPatterns( |
| 8904 | RewritePatternSet &results) const { |
| 8905 | results.add<SelectBroadcastCondition>(arg: getContext()); |
| 8906 | } |
| 8907 | |
| 8908 | //===----------------------------------------------------------------------===// |
| 8909 | // TableGen'd op method definitions |
| 8910 | //===----------------------------------------------------------------------===// |
| 8911 | |
| 8912 | #define GET_ATTRDEF_CLASSES |
| 8913 | #include "mlir/Dialect/Vector/IR/VectorAttributes.cpp.inc" |
| 8914 | |
| 8915 | #define GET_OP_CLASSES |
| 8916 | #include "mlir/Dialect/Vector/IR/VectorOps.cpp.inc" |
| 8917 | |