| 1 | //===- VectorLinearize.cpp - vector linearization transforms --------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements patterns and pass for linearizing ND vectors into 1D. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/UB/IR/UBOps.h" |
| 14 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 15 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
| 16 | #include "mlir/IR/Attributes.h" |
| 17 | #include "mlir/IR/BuiltinAttributes.h" |
| 18 | #include "mlir/IR/Matchers.h" |
| 19 | #include "mlir/IR/Operation.h" |
| 20 | #include "mlir/IR/PatternMatch.h" |
| 21 | #include "mlir/IR/TypeUtilities.h" |
| 22 | #include "mlir/Transforms/DialectConversion.h" |
| 23 | #include "llvm/ADT/ArrayRef.h" |
| 24 | #include <cstdint> |
| 25 | #include <limits> |
| 26 | #include <numeric> |
| 27 | #include <optional> |
| 28 | |
| 29 | using namespace mlir; |
| 30 | |
| 31 | static FailureOr<Attribute> |
| 32 | linearizeConstAttr(Location loc, ConversionPatternRewriter &rewriter, |
| 33 | VectorType resType, Attribute value) { |
| 34 | |
| 35 | if (auto dstElementsAttr = dyn_cast<DenseElementsAttr>(Val&: value)) { |
| 36 | if (resType.isScalable() && !isa<SplatElementsAttr>(Val: value)) |
| 37 | return rewriter.notifyMatchFailure( |
| 38 | arg&: loc, |
| 39 | msg: "Cannot linearize a constant scalable vector that's not a splat" ); |
| 40 | |
| 41 | return dstElementsAttr.reshape(newType: resType); |
| 42 | } |
| 43 | |
| 44 | if (auto poisonAttr = dyn_cast<ub::PoisonAttr>(Val&: value)) |
| 45 | return poisonAttr; |
| 46 | |
| 47 | return rewriter.notifyMatchFailure(arg&: loc, msg: "unsupported attr type" ); |
| 48 | } |
| 49 | |
| 50 | namespace { |
| 51 | |
| 52 | struct LinearizeConstantLike final |
| 53 | : OpTraitConversionPattern<OpTrait::ConstantLike> { |
| 54 | using Base::Base; |
| 55 | |
| 56 | LinearizeConstantLike(const TypeConverter &typeConverter, |
| 57 | MLIRContext *context, PatternBenefit benefit = 1) |
| 58 | : OpTraitConversionPattern(typeConverter, context, benefit) {} |
| 59 | LogicalResult |
| 60 | matchAndRewrite(Operation *op, ArrayRef<Value> operands, |
| 61 | ConversionPatternRewriter &rewriter) const override { |
| 62 | Location loc = op->getLoc(); |
| 63 | if (op->getNumResults() != 1) |
| 64 | return rewriter.notifyMatchFailure(arg&: loc, msg: "expected 1 result" ); |
| 65 | |
| 66 | const TypeConverter &typeConverter = *getTypeConverter(); |
| 67 | auto resType = |
| 68 | typeConverter.convertType<VectorType>(t: op->getResult(idx: 0).getType()); |
| 69 | assert(resType && "expected 1-D vector type" ); |
| 70 | |
| 71 | Attribute value; |
| 72 | if (!matchPattern(op, pattern: m_Constant(bind_value: &value))) |
| 73 | return rewriter.notifyMatchFailure(arg&: loc, |
| 74 | msg: "could not fold constant-like op" ); |
| 75 | |
| 76 | FailureOr<Attribute> newValue = |
| 77 | linearizeConstAttr(loc, rewriter, resType, value); |
| 78 | if (failed(Result: newValue)) |
| 79 | return failure(); |
| 80 | |
| 81 | Operation *newOp = op->getDialect()->materializeConstant( |
| 82 | builder&: rewriter, value: *newValue, type: resType, loc); |
| 83 | if (!newOp) |
| 84 | return rewriter.notifyMatchFailure( |
| 85 | arg&: loc, msg: "could not materialize linearized constant" ); |
| 86 | newOp->setDiscardableAttrs(op->getDiscardableAttrDictionary()); |
| 87 | rewriter.replaceOp(op, newOp); |
| 88 | return success(); |
| 89 | } |
| 90 | }; |
| 91 | |
| 92 | struct LinearizeVectorizable final |
| 93 | : OpTraitConversionPattern<OpTrait::Vectorizable> { |
| 94 | using Base::Base; |
| 95 | |
| 96 | public: |
| 97 | LinearizeVectorizable(const TypeConverter &typeConverter, |
| 98 | MLIRContext *context, PatternBenefit benefit = 1) |
| 99 | : OpTraitConversionPattern(typeConverter, context, benefit) {} |
| 100 | LogicalResult |
| 101 | matchAndRewrite(Operation *op, ArrayRef<Value> operands, |
| 102 | ConversionPatternRewriter &rewriter) const override { |
| 103 | FailureOr<Operation *> newOp = |
| 104 | convertOpResultTypes(op, operands, converter: *getTypeConverter(), rewriter); |
| 105 | if (failed(Result: newOp)) |
| 106 | return failure(); |
| 107 | |
| 108 | rewriter.replaceOp(op, newValues: (*newOp)->getResults()); |
| 109 | return success(); |
| 110 | } |
| 111 | }; |
| 112 | |
| 113 | template <typename TOp> |
| 114 | static bool stridesAllOne(TOp op) { |
| 115 | static_assert( |
| 116 | std::is_same_v<TOp, vector::ExtractStridedSliceOp> || |
| 117 | std::is_same_v<TOp, vector::InsertStridedSliceOp>, |
| 118 | "expected vector.extract_strided_slice or vector.insert_strided_slice" ); |
| 119 | ArrayAttr strides = op.getStrides(); |
| 120 | return llvm::all_of(Range&: strides, P: isOneInteger); |
| 121 | } |
| 122 | |
| 123 | /// Convert an array of attributes into a vector of integers, if possible. |
| 124 | static FailureOr<SmallVector<int64_t>> intsFromArrayAttr(ArrayAttr attrs) { |
| 125 | if (!attrs) |
| 126 | return failure(); |
| 127 | SmallVector<int64_t> ints; |
| 128 | ints.reserve(N: attrs.size()); |
| 129 | for (auto attr : attrs) { |
| 130 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: attr)) { |
| 131 | ints.push_back(Elt: intAttr.getInt()); |
| 132 | } else { |
| 133 | return failure(); |
| 134 | } |
| 135 | } |
| 136 | return ints; |
| 137 | } |
| 138 | |
| 139 | /// Consider inserting a vector of shape `small` into a vector of shape `large`, |
| 140 | /// at position `offsets`: this function enumeratates all the indices in `large` |
| 141 | /// that are written to. The enumeration is with row-major ordering. |
| 142 | /// |
| 143 | /// Example: insert a 1x2 vector into a 4x5 vector at position (1,3). The 2 |
| 144 | /// positions written to are (1,3) and (1,4), which have linearized indices 8 |
| 145 | /// and 9. So [8,9] is returned. |
| 146 | /// |
| 147 | /// The length of the returned vector is equal to the number of elements in |
| 148 | /// the shape `small` (i.e. the product of dimensions of `small`). |
| 149 | SmallVector<int64_t> static getStridedSliceInsertionIndices( |
| 150 | ArrayRef<int64_t> small, ArrayRef<int64_t> large, |
| 151 | ArrayRef<int64_t> offsets) { |
| 152 | |
| 153 | // Example of alignment between, `large`, `small` and `offsets`: |
| 154 | // large = 4, 5, 6, 7, 8 |
| 155 | // small = 1, 6, 7, 8 |
| 156 | // offsets = 2, 3, 0 |
| 157 | // |
| 158 | // `offsets` has implicit trailing 0s, `small` has implicit leading 1s. |
| 159 | assert((large.size() >= small.size()) && |
| 160 | "rank of 'large' cannot be lower than rank of 'small'" ); |
| 161 | assert((large.size() >= offsets.size()) && |
| 162 | "rank of 'large' cannot be lower than the number of offsets" ); |
| 163 | unsigned delta = large.size() - small.size(); |
| 164 | unsigned nOffsets = offsets.size(); |
| 165 | auto getSmall = [&](int64_t i) -> int64_t { |
| 166 | return i >= delta ? small[i - delta] : 1; |
| 167 | }; |
| 168 | auto getOffset = [&](int64_t i) -> int64_t { |
| 169 | return i < nOffsets ? offsets[i] : 0; |
| 170 | }; |
| 171 | |
| 172 | // Using 2 vectors of indices, at each iteration populate the updated set of |
| 173 | // indices based on the old set of indices, and the size of the small vector |
| 174 | // in the current iteration. |
| 175 | SmallVector<int64_t> indices{0}; |
| 176 | int64_t stride = 1; |
| 177 | for (int i = large.size() - 1; i >= 0; --i) { |
| 178 | int64_t currentSize = indices.size(); |
| 179 | int64_t smallSize = getSmall(i); |
| 180 | int64_t nextSize = currentSize * smallSize; |
| 181 | SmallVector<int64_t> nextIndices(nextSize); |
| 182 | int64_t *base = nextIndices.begin(); |
| 183 | int64_t offset = getOffset(i) * stride; |
| 184 | for (int j = 0; j < smallSize; ++j) { |
| 185 | for (int k = 0; k < currentSize; ++k) { |
| 186 | base[k] = indices[k] + offset; |
| 187 | } |
| 188 | offset += stride; |
| 189 | base += currentSize; |
| 190 | } |
| 191 | stride *= large[i]; |
| 192 | indices = std::move(nextIndices); |
| 193 | } |
| 194 | return indices; |
| 195 | } |
| 196 | |
| 197 | /// This pattern converts a vector.extract_strided_slice operation into a |
| 198 | /// vector.shuffle operation that has a rank-1 (linearized) operand and result. |
| 199 | /// |
| 200 | /// For example, the following: |
| 201 | /// |
| 202 | /// ``` |
| 203 | /// vector.extract_strided_slice %source |
| 204 | /// { offsets = [..], strides = [..], sizes = [..] } |
| 205 | /// ``` |
| 206 | /// |
| 207 | /// is converted to : |
| 208 | /// ``` |
| 209 | /// %source_1d = vector.shape_cast %source |
| 210 | /// %out_1d = vector.shuffle %source_1d, %source_1d [ shuffle_indices_1d ] |
| 211 | /// %out_nd = vector.shape_cast %out_1d |
| 212 | /// ``` |
| 213 | /// |
| 214 | /// `shuffle_indices_1d` is computed using the offsets and sizes of the original |
| 215 | /// vector.extract_strided_slice operation. |
| 216 | struct final |
| 217 | : public mlir::OpConversionPattern<mlir::vector::ExtractStridedSliceOp> { |
| 218 | using Base::Base; |
| 219 | (const TypeConverter &typeConverter, |
| 220 | MLIRContext *context, |
| 221 | PatternBenefit benefit = 1) |
| 222 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 223 | |
| 224 | LogicalResult |
| 225 | matchAndRewrite(vector::ExtractStridedSliceOp , |
| 226 | OpAdaptor adaptor, |
| 227 | ConversionPatternRewriter &rewriter) const override { |
| 228 | |
| 229 | VectorType flatOutputType = getTypeConverter()->convertType<VectorType>( |
| 230 | t: extractStridedSliceOp.getType()); |
| 231 | assert(flatOutputType && "vector type expected" ); |
| 232 | |
| 233 | // Expect a legalization failure if the strides are not all 1 (if ever the |
| 234 | // verifier for extract_strided_slice allows non-1 strides). |
| 235 | if (!stridesAllOne(op: extractStridedSliceOp)) { |
| 236 | return rewriter.notifyMatchFailure( |
| 237 | arg&: extractStridedSliceOp, |
| 238 | msg: "extract_strided_slice with strides != 1 not supported" ); |
| 239 | } |
| 240 | |
| 241 | FailureOr<SmallVector<int64_t>> offsets = |
| 242 | intsFromArrayAttr(attrs: extractStridedSliceOp.getOffsets()); |
| 243 | if (failed(Result: offsets)) { |
| 244 | return rewriter.notifyMatchFailure(arg&: extractStridedSliceOp, |
| 245 | msg: "failed to get integer offsets" ); |
| 246 | } |
| 247 | |
| 248 | ArrayRef<int64_t> inputShape = |
| 249 | extractStridedSliceOp.getSourceVectorType().getShape(); |
| 250 | |
| 251 | ArrayRef<int64_t> outputShape = extractStridedSliceOp.getType().getShape(); |
| 252 | |
| 253 | SmallVector<int64_t> indices = getStridedSliceInsertionIndices( |
| 254 | small: outputShape, large: inputShape, offsets: offsets.value()); |
| 255 | |
| 256 | Value srcVector = adaptor.getSource(); |
| 257 | rewriter.replaceOpWithNewOp<vector::ShuffleOp>( |
| 258 | op: extractStridedSliceOp, args&: flatOutputType, args&: srcVector, args&: srcVector, args&: indices); |
| 259 | return success(); |
| 260 | } |
| 261 | }; |
| 262 | |
| 263 | /// This pattern converts a vector.insert_strided_slice operation into a |
| 264 | /// vector.shuffle operation that has rank-1 (linearized) operands and result. |
| 265 | /// |
| 266 | /// For example, the following: |
| 267 | /// ``` |
| 268 | /// %0 = vector.insert_strided_slice %to_store, %into |
| 269 | /// {offsets = [1, 0, 0, 0], strides = [1, 1]} |
| 270 | /// : vector<2x2xi8> into vector<2x1x3x2xi8> |
| 271 | /// ``` |
| 272 | /// |
| 273 | /// is converted to |
| 274 | /// ``` |
| 275 | /// %to_store_1d |
| 276 | /// = vector.shape_cast %to_store : vector<2x2xi8> to vector<4xi8> |
| 277 | /// %into_1d = vector.shape_cast %into : vector<2x1x3x2xi8> to vector<12xi8> |
| 278 | /// %out_1d = vector.shuffle %into_1d, %to_store_1d [ shuffle_indices_1d ] |
| 279 | /// %out_nd = vector.shape_cast %out_1d : vector<12xi8> to vector<2x1x3x2xi8> |
| 280 | /// ``` |
| 281 | /// |
| 282 | /// where shuffle_indices_1d in this case is |
| 283 | /// [0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 10, 11]. |
| 284 | /// ^^^^^^^^^^^^^^ |
| 285 | /// to_store_1d |
| 286 | /// |
| 287 | struct LinearizeVectorInsertStridedSlice final |
| 288 | : public mlir::OpConversionPattern<mlir::vector::InsertStridedSliceOp> { |
| 289 | using Base::Base; |
| 290 | LinearizeVectorInsertStridedSlice(const TypeConverter &typeConverter, |
| 291 | MLIRContext *context, |
| 292 | PatternBenefit benefit = 1) |
| 293 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 294 | |
| 295 | LogicalResult |
| 296 | matchAndRewrite(vector::InsertStridedSliceOp insertStridedSliceOp, |
| 297 | OpAdaptor adaptor, |
| 298 | ConversionPatternRewriter &rewriter) const override { |
| 299 | |
| 300 | // Expect a legalization failure if the strides are not all 1 (if ever the |
| 301 | // verifier for insert_strided_slice allows non-1 strides). |
| 302 | if (!stridesAllOne(op: insertStridedSliceOp)) { |
| 303 | return rewriter.notifyMatchFailure( |
| 304 | arg&: insertStridedSliceOp, |
| 305 | msg: "insert_strided_slice with strides != 1 not supported" ); |
| 306 | } |
| 307 | |
| 308 | VectorType inputType = insertStridedSliceOp.getValueToStore().getType(); |
| 309 | ArrayRef<int64_t> inputShape = inputType.getShape(); |
| 310 | |
| 311 | VectorType outputType = insertStridedSliceOp.getType(); |
| 312 | ArrayRef<int64_t> outputShape = outputType.getShape(); |
| 313 | int64_t nOutputElements = outputType.getNumElements(); |
| 314 | |
| 315 | FailureOr<SmallVector<int64_t>> offsets = |
| 316 | intsFromArrayAttr(attrs: insertStridedSliceOp.getOffsets()); |
| 317 | if (failed(Result: offsets)) { |
| 318 | return rewriter.notifyMatchFailure(arg&: insertStridedSliceOp, |
| 319 | msg: "failed to get integer offsets" ); |
| 320 | } |
| 321 | SmallVector<int64_t> sliceIndices = getStridedSliceInsertionIndices( |
| 322 | small: inputShape, large: outputShape, offsets: offsets.value()); |
| 323 | |
| 324 | SmallVector<int64_t> indices(nOutputElements); |
| 325 | std::iota(first: indices.begin(), last: indices.end(), value: 0); |
| 326 | for (auto [index, sliceIndex] : llvm::enumerate(First&: sliceIndices)) { |
| 327 | indices[sliceIndex] = index + nOutputElements; |
| 328 | } |
| 329 | |
| 330 | Value flatToStore = adaptor.getValueToStore(); |
| 331 | Value flatDest = adaptor.getDest(); |
| 332 | rewriter.replaceOpWithNewOp<vector::ShuffleOp>(op: insertStridedSliceOp, |
| 333 | args: flatDest.getType(), args&: flatDest, |
| 334 | args&: flatToStore, args&: indices); |
| 335 | return success(); |
| 336 | } |
| 337 | }; |
| 338 | |
| 339 | /// This pattern converts the ShuffleOp that works on nD (n > 1) |
| 340 | /// vectors to a ShuffleOp that works on linearized vectors. |
| 341 | /// Following, |
| 342 | /// vector.shuffle %v1, %v2 [ shuffle_indices ] |
| 343 | /// is converted to : |
| 344 | /// %v1_1d = vector.shape_cast %v1 |
| 345 | /// %v2_1d = vector.shape_cast %v2 |
| 346 | /// %out_1d = vector.shuffle %v1_1d, %v2_1d [ shuffle_indices_1d ] |
| 347 | /// %out_nd = vector.shape_cast %out_1d |
| 348 | // `shuffle_indices_1d` is computed using the sizes and `shuffle_indices` |
| 349 | /// of the original shuffle operation. |
| 350 | struct LinearizeVectorShuffle final |
| 351 | : public OpConversionPattern<vector::ShuffleOp> { |
| 352 | using Base::Base; |
| 353 | LinearizeVectorShuffle(const TypeConverter &typeConverter, |
| 354 | MLIRContext *context, PatternBenefit benefit = 1) |
| 355 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 356 | |
| 357 | LogicalResult |
| 358 | matchAndRewrite(vector::ShuffleOp shuffleOp, OpAdaptor adaptor, |
| 359 | ConversionPatternRewriter &rewriter) const override { |
| 360 | VectorType dstType = |
| 361 | getTypeConverter()->convertType<VectorType>(t: shuffleOp.getType()); |
| 362 | assert(dstType && "vector type destination expected." ); |
| 363 | |
| 364 | Value vec1 = adaptor.getV1(); |
| 365 | Value vec2 = adaptor.getV2(); |
| 366 | int shuffleSliceLen = 1; |
| 367 | int rank = shuffleOp.getV1().getType().getRank(); |
| 368 | |
| 369 | // If rank > 1, we need to do the shuffle in the granularity of slices |
| 370 | // instead of scalars. Size of the slice is equal to the rank-1 innermost |
| 371 | // dims. Mask of the shuffle op specifies which slice to take from the |
| 372 | // outermost dim. |
| 373 | if (rank > 1) { |
| 374 | llvm::ArrayRef<int64_t> shape = shuffleOp.getV1().getType().getShape(); |
| 375 | for (unsigned i = 1; i < shape.size(); ++i) { |
| 376 | shuffleSliceLen *= shape[i]; |
| 377 | } |
| 378 | } |
| 379 | |
| 380 | // For each value in the mask, we generate the indices of the source vectors |
| 381 | // that need to be shuffled to the destination vector. If shuffleSliceLen > |
| 382 | // 1 we need to shuffle the slices (consecutive shuffleSliceLen number of |
| 383 | // elements) instead of scalars. |
| 384 | ArrayRef<int64_t> mask = shuffleOp.getMask(); |
| 385 | int64_t totalSizeOfShuffledElmnts = mask.size() * shuffleSliceLen; |
| 386 | llvm::SmallVector<int64_t, 2> indices(totalSizeOfShuffledElmnts); |
| 387 | for (auto [i, value] : llvm::enumerate(First&: mask)) { |
| 388 | std::iota(first: indices.begin() + shuffleSliceLen * i, |
| 389 | last: indices.begin() + shuffleSliceLen * (i + 1), |
| 390 | value: shuffleSliceLen * value); |
| 391 | } |
| 392 | |
| 393 | rewriter.replaceOpWithNewOp<vector::ShuffleOp>(op: shuffleOp, args&: dstType, args&: vec1, |
| 394 | args&: vec2, args&: indices); |
| 395 | return success(); |
| 396 | } |
| 397 | }; |
| 398 | |
| 399 | /// This pattern linearizes `vector.extract` operations. It generates a 1-D |
| 400 | /// version of the `vector.extract` operation when extracting a scalar from a |
| 401 | /// vector. It generates a 1-D `vector.shuffle` operation when extracting a |
| 402 | /// subvector from a larger vector. |
| 403 | /// |
| 404 | /// Example #1: |
| 405 | /// |
| 406 | /// %0 = vector.extract %arg0[1]: vector<8x2xf32> from vector<2x8x2xf32> |
| 407 | /// |
| 408 | /// is converted to: |
| 409 | /// |
| 410 | /// %0 = vector.shape_cast %arg0 : vector<2x8x2xf32> to vector<32xf32> |
| 411 | /// %1 = vector.shuffle %0, %0 [16, 17, 18, 19, 20, 21, 22, 23, |
| 412 | /// 24, 25, 26, 27, 28, 29, 30, 31] : |
| 413 | /// vector<32xf32>, vector<32xf32> |
| 414 | /// %2 = vector.shape_cast %1 : vector<16xf32> to vector<8x2xf32> |
| 415 | /// |
| 416 | /// Example #2: |
| 417 | /// |
| 418 | /// %0 = vector.extract %arg0[1, 2] : i32 from vector<2x4xi32> |
| 419 | /// |
| 420 | /// is converted to: |
| 421 | /// |
| 422 | /// %0 = vector.shape_cast %arg0 : vector<2x4xi32> to vector<8xi32> |
| 423 | /// %1 = vector.extract %0[6] : i32 from vector<8xi32> |
| 424 | /// |
| 425 | struct final |
| 426 | : public OpConversionPattern<vector::ExtractOp> { |
| 427 | using Base::Base; |
| 428 | (const TypeConverter &typeConverter, |
| 429 | MLIRContext *context, PatternBenefit benefit = 1) |
| 430 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 431 | LogicalResult |
| 432 | matchAndRewrite(vector::ExtractOp , OpAdaptor adaptor, |
| 433 | ConversionPatternRewriter &rewriter) const override { |
| 434 | Type dstTy = getTypeConverter()->convertType(t: extractOp.getType()); |
| 435 | assert(dstTy && "expected 1-D vector type" ); |
| 436 | |
| 437 | // Dynamic position is not supported. |
| 438 | if (extractOp.hasDynamicPosition()) |
| 439 | return rewriter.notifyMatchFailure(arg&: extractOp, |
| 440 | msg: "dynamic position is not supported." ); |
| 441 | |
| 442 | llvm::ArrayRef<int64_t> shape = extractOp.getSource().getType().getShape(); |
| 443 | int64_t size = extractOp.getSource().getType().getNumElements(); |
| 444 | |
| 445 | // Compute linearized offset. |
| 446 | int64_t linearizedOffset = 0; |
| 447 | llvm::ArrayRef<int64_t> offsets = extractOp.getStaticPosition(); |
| 448 | for (auto [i, off] : llvm::enumerate(First&: offsets)) { |
| 449 | size /= shape[i]; |
| 450 | linearizedOffset += offsets[i] * size; |
| 451 | } |
| 452 | |
| 453 | Value srcVector = adaptor.getSource(); |
| 454 | if (!isa<VectorType>(Val: extractOp.getType())) { |
| 455 | // Scalar case: generate a 1-D extract. |
| 456 | Value result = rewriter.createOrFold<vector::ExtractOp>( |
| 457 | location: extractOp.getLoc(), args&: srcVector, args&: linearizedOffset); |
| 458 | rewriter.replaceOp(op: extractOp, newValues: result); |
| 459 | return success(); |
| 460 | } |
| 461 | |
| 462 | // Vector case: generate a shuffle. |
| 463 | |
| 464 | llvm::SmallVector<int64_t, 2> indices(size); |
| 465 | std::iota(first: indices.begin(), last: indices.end(), value: linearizedOffset); |
| 466 | rewriter.replaceOpWithNewOp<vector::ShuffleOp>(op: extractOp, args&: dstTy, args&: srcVector, |
| 467 | args&: srcVector, args&: indices); |
| 468 | |
| 469 | return success(); |
| 470 | } |
| 471 | }; |
| 472 | |
| 473 | /// This pattern linearizes `vector.insert` operations. It generates a 1-D |
| 474 | /// version of the `vector.insert` operation when inserting a scalar into a |
| 475 | /// vector. It generates a 1-D `vector.shuffle` operation when inserting a |
| 476 | /// vector into another vector. |
| 477 | /// |
| 478 | /// Example #1: |
| 479 | /// |
| 480 | /// %0 = vector.insert %source, %destination[0] : |
| 481 | /// vector<2x4xf32> into vector<2x2x4xf32> |
| 482 | /// |
| 483 | /// is converted to: |
| 484 | /// |
| 485 | /// %0 = vector.shape_cast %source : vector<2x4xf32> to vector<8xf32> |
| 486 | /// %1 = vector.shape_cast %destination : |
| 487 | /// vector<2x2x4xf32> to vector<16xf32> |
| 488 | /// %2 = vector.shuffle %1, %0 [16, 17, 18, 19, 20, 21, 22, 23 |
| 489 | /// 8, 9, 10, 11, 12, 13, 14, 15] : |
| 490 | /// vector<16xf32>, vector<8xf32> |
| 491 | /// %3 = vector.shape_cast %2 : vector<16xf32> to vector<2x2x4xf32> |
| 492 | /// |
| 493 | /// Example #2: |
| 494 | /// |
| 495 | /// %0 = vector.insert %source, %destination[1, 2]: f32 into vector<2x4xf32> |
| 496 | /// |
| 497 | /// is converted to: |
| 498 | /// |
| 499 | /// %0 = vector.shape_cast %destination : vector<2x4xf32> to vector<8xf32> |
| 500 | /// %1 = vector.insert %source, %0[6]: f32 into vector<8xf32> |
| 501 | /// %2 = vector.shape_cast %1 : vector<8xf32> to vector<2x4xf32> |
| 502 | /// |
| 503 | struct LinearizeVectorInsert final |
| 504 | : public OpConversionPattern<vector::InsertOp> { |
| 505 | using Base::Base; |
| 506 | LinearizeVectorInsert(const TypeConverter &typeConverter, |
| 507 | MLIRContext *context, PatternBenefit benefit = 1) |
| 508 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 509 | LogicalResult |
| 510 | matchAndRewrite(vector::InsertOp insertOp, OpAdaptor adaptor, |
| 511 | ConversionPatternRewriter &rewriter) const override { |
| 512 | VectorType dstTy = getTypeConverter()->convertType<VectorType>( |
| 513 | t: insertOp.getDestVectorType()); |
| 514 | assert(dstTy && "vector type destination expected." ); |
| 515 | |
| 516 | // Dynamic position is not supported. |
| 517 | if (insertOp.hasDynamicPosition()) |
| 518 | return rewriter.notifyMatchFailure(arg&: insertOp, |
| 519 | msg: "dynamic position is not supported." ); |
| 520 | auto srcTy = insertOp.getValueToStoreType(); |
| 521 | auto srcAsVec = dyn_cast<VectorType>(Val&: srcTy); |
| 522 | uint64_t srcSize = srcAsVec ? srcAsVec.getNumElements() : 1; |
| 523 | |
| 524 | auto dstShape = insertOp.getDestVectorType().getShape(); |
| 525 | const auto dstSize = insertOp.getDestVectorType().getNumElements(); |
| 526 | auto dstSizeForOffsets = dstSize; |
| 527 | |
| 528 | // Compute linearized offset. |
| 529 | int64_t linearizedOffset = 0; |
| 530 | auto offsetsNd = insertOp.getStaticPosition(); |
| 531 | for (auto [dim, offset] : llvm::enumerate(First&: offsetsNd)) { |
| 532 | dstSizeForOffsets /= dstShape[dim]; |
| 533 | linearizedOffset += offset * dstSizeForOffsets; |
| 534 | } |
| 535 | |
| 536 | Location loc = insertOp.getLoc(); |
| 537 | Value valueToStore = adaptor.getValueToStore(); |
| 538 | |
| 539 | if (!isa<VectorType>(Val: valueToStore.getType())) { |
| 540 | // Scalar case: generate a 1-D insert. |
| 541 | Value result = rewriter.createOrFold<vector::InsertOp>( |
| 542 | location: loc, args&: valueToStore, args: adaptor.getDest(), args&: linearizedOffset); |
| 543 | rewriter.replaceOp(op: insertOp, newValues: result); |
| 544 | return success(); |
| 545 | } |
| 546 | |
| 547 | // Vector case: generate a shuffle. |
| 548 | llvm::SmallVector<int64_t, 2> indices(dstSize); |
| 549 | auto *origValsUntil = indices.begin(); |
| 550 | std::advance(i&: origValsUntil, n: linearizedOffset); |
| 551 | |
| 552 | // Original values that remain [0, offset). |
| 553 | std::iota(first: indices.begin(), last: origValsUntil, value: 0); |
| 554 | auto *newValsUntil = origValsUntil; |
| 555 | std::advance(i&: newValsUntil, n: srcSize); |
| 556 | // New values [offset, offset+srcNumElements). |
| 557 | std::iota(first: origValsUntil, last: newValsUntil, value: dstSize); |
| 558 | // The rest of original values [offset+srcNumElements, end); |
| 559 | std::iota(first: newValsUntil, last: indices.end(), value: linearizedOffset + srcSize); |
| 560 | |
| 561 | Value result = rewriter.createOrFold<vector::ShuffleOp>( |
| 562 | location: loc, args&: dstTy, args: adaptor.getDest(), args&: valueToStore, args&: indices); |
| 563 | |
| 564 | rewriter.replaceOp(op: insertOp, newValues: result); |
| 565 | return success(); |
| 566 | } |
| 567 | }; |
| 568 | |
| 569 | /// This pattern converts the BitCastOp that works on nD (n > 1) |
| 570 | /// vectors to a BitCastOp that works on linearized vectors. |
| 571 | /// Following, |
| 572 | /// vector.bitcast %v1: vector<4x2xf32> to vector<4x4xf16> |
| 573 | /// is converted to : |
| 574 | /// %v1_1d = vector.shape_cast %v1: vector<4x2xf32> to vector<8xf32> |
| 575 | /// %out_1d = vector.bitcast %v1_1d: vector<8xf32> to vector<16xf16> |
| 576 | /// %out_nd = vector.shape_cast %out_1d: vector<16xf16> to vector<4x4xf16> |
| 577 | struct LinearizeVectorBitCast final |
| 578 | : public OpConversionPattern<vector::BitCastOp> { |
| 579 | using Base::Base; |
| 580 | LinearizeVectorBitCast(const TypeConverter &typeConverter, |
| 581 | MLIRContext *context, PatternBenefit benefit = 1) |
| 582 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 583 | LogicalResult |
| 584 | matchAndRewrite(vector::BitCastOp castOp, OpAdaptor adaptor, |
| 585 | ConversionPatternRewriter &rewriter) const override { |
| 586 | auto resType = getTypeConverter()->convertType(t: castOp.getType()); |
| 587 | assert(resType && "expected 1-D vector type" ); |
| 588 | rewriter.replaceOpWithNewOp<vector::BitCastOp>(op: castOp, args&: resType, |
| 589 | args: adaptor.getSource()); |
| 590 | return mlir::success(); |
| 591 | } |
| 592 | }; |
| 593 | |
| 594 | /// This pattern converts the CreateMaskOp to work on a linearized vector. |
| 595 | /// It currently supports only 2D masks with a unit outer dimension. |
| 596 | /// Following, |
| 597 | /// vector.create_mask %arg0, %arg1 : vector<1x4xi1> |
| 598 | /// is converted to: |
| 599 | /// %zero = arith.constant 0 : index |
| 600 | /// %cmpi = arith.cmpi sgt, %arg0, %zero : index |
| 601 | /// %index = arith.index_cast %cmpi : i1 to index |
| 602 | /// %mul = arith.andi %index, %arg1 : index |
| 603 | /// %mask = vector.create_mask %mul : vector<4xi1> |
| 604 | /// %shape_cast = vector.shape_cast %mask : vector<4xi1> to vector<1x4xi1> |
| 605 | struct LinearizeVectorCreateMask final |
| 606 | : OpConversionPattern<vector::CreateMaskOp> { |
| 607 | using Base::Base; |
| 608 | |
| 609 | LinearizeVectorCreateMask(const TypeConverter &typeConverter, |
| 610 | MLIRContext *context, PatternBenefit benefit = 1) |
| 611 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 612 | |
| 613 | LogicalResult |
| 614 | matchAndRewrite(vector::CreateMaskOp createMaskOp, OpAdaptor adaptor, |
| 615 | ConversionPatternRewriter &rewriter) const override { |
| 616 | Location loc = createMaskOp.getLoc(); |
| 617 | VectorType srcTy = createMaskOp.getType(); |
| 618 | auto srcShape = srcTy.getShape(); |
| 619 | if (srcShape.size() != 2) |
| 620 | return rewriter.notifyMatchFailure(arg&: createMaskOp, |
| 621 | msg: "only 2D mask is supported." ); |
| 622 | |
| 623 | if (srcShape[0] != 1) |
| 624 | return rewriter.notifyMatchFailure( |
| 625 | arg&: createMaskOp, msg: "only unit outer dimension is supported." ); |
| 626 | |
| 627 | auto dstTy = getTypeConverter()->convertType(t: srcTy); |
| 628 | if (!dstTy) |
| 629 | return rewriter.notifyMatchFailure(arg&: createMaskOp, msg: "cannot convert type." ); |
| 630 | |
| 631 | // Compare the first operand with 0. If it is greater than 0, the |
| 632 | // corresponding mask element is set to true, otherwise false. |
| 633 | // The result of the comparison is then multiplied with |
| 634 | // the second operand of create_mask to get the 1D mask. |
| 635 | auto firstOperand = adaptor.getOperands().front(); |
| 636 | auto zero = mlir::arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0); |
| 637 | auto isNonZero = rewriter.createOrFold<mlir::arith::CmpIOp>( |
| 638 | location: loc, args: mlir::arith::CmpIPredicate::sgt, args&: firstOperand, args&: zero); |
| 639 | auto isNonZeroIndex = rewriter.createOrFold<mlir::arith::IndexCastOp>( |
| 640 | location: loc, args: rewriter.getIndexType(), args&: isNonZero); |
| 641 | auto secondOperand = adaptor.getOperands().back(); |
| 642 | auto maskSize = rewriter.createOrFold<mlir::arith::AndIOp>( |
| 643 | location: loc, args: rewriter.getIndexType(), args&: isNonZeroIndex, args&: secondOperand); |
| 644 | |
| 645 | auto newMask = |
| 646 | mlir::vector::CreateMaskOp::create(builder&: rewriter, location: loc, resultType0: dstTy, mask_dim_sizes: maskSize); |
| 647 | rewriter.replaceOp(op: createMaskOp, newOp: newMask); |
| 648 | return success(); |
| 649 | } |
| 650 | }; |
| 651 | |
| 652 | /// This pattern linearizes vector.load from vector<1x1x...xN> to vector<N> |
| 653 | /// It currently supports linearization where all but the last dimension are 1 |
| 654 | /// The following, |
| 655 | /// vector.load %arg0[%c0, %c0] : memref<1x4xf32>, vector<1x4xf32> |
| 656 | /// is converted to: |
| 657 | /// vector.load %arg0[%c0, %c0] : memref<1x4xf32>, vector<4xf32> |
| 658 | /// vector.shape_cast %load_result : vector<4xf32> to vector<1x4xf32> |
| 659 | /// For generic cases, the vector unroll pass should be used to unroll the load |
| 660 | /// to vector<1x1x...xN> form and then linearized |
| 661 | struct LinearizeVectorLoad final : public OpConversionPattern<vector::LoadOp> { |
| 662 | using Base::Base; |
| 663 | LinearizeVectorLoad(const TypeConverter &typeConverter, MLIRContext *context, |
| 664 | PatternBenefit benefit = 1) |
| 665 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 666 | |
| 667 | LogicalResult |
| 668 | matchAndRewrite(vector::LoadOp loadOp, OpAdaptor adaptor, |
| 669 | ConversionPatternRewriter &rewriter) const override { |
| 670 | VectorType vecTy = loadOp.getType(); |
| 671 | if (!vecTy) |
| 672 | return rewriter.notifyMatchFailure(arg&: loadOp, msg: "expected vector type" ); |
| 673 | |
| 674 | auto shape = vecTy.getShape(); |
| 675 | auto scalableDims = vecTy.getScalableDims(); |
| 676 | // All but the last dim must be 1, and only the last dim may be scalable (if |
| 677 | // any). |
| 678 | if (!llvm::all_of(Range: shape.drop_back(N: 1), P: [](auto d) { return d == 1; })) |
| 679 | return rewriter.notifyMatchFailure(arg&: loadOp, |
| 680 | msg: "only vector<1x1x...xN> supported" ); |
| 681 | |
| 682 | if (llvm::any_of(Range: scalableDims.drop_back(N: 1), P: [](bool s) { return s; })) |
| 683 | return rewriter.notifyMatchFailure(arg&: loadOp, |
| 684 | msg: "only innermost dim may be scalable" ); |
| 685 | |
| 686 | auto linearTy = typeConverter->convertType<VectorType>(t: vecTy); |
| 687 | |
| 688 | auto newLoad = |
| 689 | vector::LoadOp::create(builder&: rewriter, location: loadOp.getLoc(), resultType: linearTy, |
| 690 | base: adaptor.getBase(), indices: adaptor.getIndices()); |
| 691 | rewriter.replaceOp(op: loadOp, newValues: newLoad.getResult()); |
| 692 | return success(); |
| 693 | } |
| 694 | }; |
| 695 | |
| 696 | /// This pattern linearizes vector.store from vector<1x1x...xN> to vector<N> |
| 697 | /// It currently supports linearization where all but the last dimension are 1 |
| 698 | /// The following, |
| 699 | /// vector.store %arg0, %arg1[%c0, %c0]s |
| 700 | /// : vector<1x4xf32>, memref<1x4xf32> |
| 701 | /// is converted to: |
| 702 | /// vector.shape_cast %arg0 : vector<1x4xf32> to vector<4xf32> |
| 703 | /// vector.store %arg0, %arg1[%c0, %c0] |
| 704 | /// : vector<4xf32>, memref<1x4xf32> |
| 705 | /// For generic cases, the vector unroll pass should be used to unroll the store |
| 706 | /// to vector<1x1x...xN> form and then linearized |
| 707 | struct LinearizeVectorStore final |
| 708 | : public OpConversionPattern<vector::StoreOp> { |
| 709 | using Base::Base; |
| 710 | LinearizeVectorStore(const TypeConverter &typeConverter, MLIRContext *context, |
| 711 | PatternBenefit benefit = 1) |
| 712 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 713 | |
| 714 | LogicalResult |
| 715 | matchAndRewrite(vector::StoreOp storeOp, OpAdaptor adaptor, |
| 716 | ConversionPatternRewriter &rewriter) const override { |
| 717 | VectorType vecTy = storeOp.getValueToStore().getType(); |
| 718 | if (!vecTy) |
| 719 | return rewriter.notifyMatchFailure(arg&: storeOp, msg: "expected vector type" ); |
| 720 | |
| 721 | auto shape = vecTy.getShape(); |
| 722 | auto scalableDims = vecTy.getScalableDims(); |
| 723 | // All but the last dim must be 1, and only the last dim may be scalable (if |
| 724 | // any). |
| 725 | if (!llvm::all_of(Range: shape.drop_back(N: 1), P: [](auto d) { return d == 1; })) |
| 726 | return rewriter.notifyMatchFailure(arg&: storeOp, |
| 727 | msg: "only vector<1x1x...xN> supported" ); |
| 728 | |
| 729 | if (llvm::any_of(Range: scalableDims.drop_back(N: 1), P: [](bool s) { return s; })) |
| 730 | return rewriter.notifyMatchFailure(arg&: storeOp, |
| 731 | msg: "only innermost dim may be scalable" ); |
| 732 | |
| 733 | rewriter.replaceOpWithNewOp<vector::StoreOp>( |
| 734 | op: storeOp, args: adaptor.getValueToStore(), args: adaptor.getBase(), |
| 735 | args: adaptor.getIndices()); |
| 736 | return success(); |
| 737 | } |
| 738 | }; |
| 739 | |
| 740 | /// This pattern linearizes `vector.from_elements` operations by converting |
| 741 | /// the result type to a 1-D vector while preserving all element values. |
| 742 | /// The transformation creates a linearized `vector.from_elements` followed by |
| 743 | /// a `vector.shape_cast` to restore the original multidimensional shape. |
| 744 | /// |
| 745 | /// Example: |
| 746 | /// |
| 747 | /// %0 = vector.from_elements %a, %b, %c, %d : vector<2x2xf32> |
| 748 | /// |
| 749 | /// is converted to: |
| 750 | /// |
| 751 | /// %0 = vector.from_elements %a, %b, %c, %d : vector<4xf32> |
| 752 | /// %1 = vector.shape_cast %0 : vector<4xf32> to vector<2x2xf32> |
| 753 | /// |
| 754 | struct LinearizeVectorFromElements final |
| 755 | : public OpConversionPattern<vector::FromElementsOp> { |
| 756 | using Base::Base; |
| 757 | LinearizeVectorFromElements(const TypeConverter &typeConverter, |
| 758 | MLIRContext *context, PatternBenefit benefit = 1) |
| 759 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 760 | LogicalResult |
| 761 | matchAndRewrite(vector::FromElementsOp fromElementsOp, OpAdaptor adaptor, |
| 762 | ConversionPatternRewriter &rewriter) const override { |
| 763 | VectorType dstTy = |
| 764 | getTypeConverter()->convertType<VectorType>(t: fromElementsOp.getType()); |
| 765 | assert(dstTy && "vector type destination expected." ); |
| 766 | |
| 767 | OperandRange elements = fromElementsOp.getElements(); |
| 768 | assert(elements.size() == static_cast<size_t>(dstTy.getNumElements()) && |
| 769 | "expected same number of elements" ); |
| 770 | rewriter.replaceOpWithNewOp<vector::FromElementsOp>(op: fromElementsOp, args&: dstTy, |
| 771 | args&: elements); |
| 772 | return success(); |
| 773 | } |
| 774 | }; |
| 775 | |
| 776 | /// This pattern linearizes the operand in `vector.to_elements` operations |
| 777 | /// by converting the source type to a 1-D vector while preserving all element |
| 778 | /// values. The transformation creates a linearized `vector.shape_cast` |
| 779 | /// followed by a `vector.to_elements`. |
| 780 | /// |
| 781 | /// Example: |
| 782 | /// |
| 783 | /// %0:4 = vector.to_elements %v : vector<2x2xf32> |
| 784 | /// |
| 785 | /// is converted to: |
| 786 | /// |
| 787 | /// %vector_cast = vector.shape_cast %v : vector<2x2xf32> to vector<4xf32> |
| 788 | /// %0:4 = vector.to_elements %vector_cast : vector<4xf32> |
| 789 | /// |
| 790 | struct LinearizeVectorToElements final |
| 791 | : public OpConversionPattern<vector::ToElementsOp> { |
| 792 | using Base::Base; |
| 793 | |
| 794 | LinearizeVectorToElements(const TypeConverter &typeConverter, |
| 795 | MLIRContext *context, PatternBenefit benefit = 1) |
| 796 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 797 | |
| 798 | LogicalResult |
| 799 | matchAndRewrite(vector::ToElementsOp toElementsOp, OpAdaptor adaptor, |
| 800 | ConversionPatternRewriter &rewriter) const override { |
| 801 | |
| 802 | VectorType vecType = toElementsOp.getSource().getType(); |
| 803 | if (vecType.getRank() <= 1) |
| 804 | return rewriter.notifyMatchFailure( |
| 805 | arg&: toElementsOp, msg: "the rank is already less than or equal to 1" ); |
| 806 | |
| 807 | assert(vecType.getNumScalableDims() == 0 && |
| 808 | "to_elements does not support scalable vectors" ); |
| 809 | auto vec1DType = |
| 810 | VectorType::get(shape: {vecType.getNumElements()}, elementType: vecType.getElementType()); |
| 811 | Value shapeCast = vector::ShapeCastOp::create( |
| 812 | builder&: rewriter, location: toElementsOp.getLoc(), result: vec1DType, source: toElementsOp.getSource()); |
| 813 | auto newToElementsOp = |
| 814 | vector::ToElementsOp::create(builder&: rewriter, location: toElementsOp.getLoc(), |
| 815 | elements: toElementsOp.getResultTypes(), source: shapeCast); |
| 816 | rewriter.replaceOp(op: toElementsOp, newOp: newToElementsOp); |
| 817 | return success(); |
| 818 | } |
| 819 | }; |
| 820 | |
| 821 | /// Convert broadcasts from scalars or 1-element vectors, such as |
| 822 | /// |
| 823 | /// ```mlir |
| 824 | /// vector.broadcast %value : f32 to vector<4x4xf32> |
| 825 | /// ``` |
| 826 | /// |
| 827 | /// to broadcasts to rank-1 vectors, with shape_casts before/after as needed. |
| 828 | /// The above becomes, |
| 829 | /// |
| 830 | /// ```mlir |
| 831 | /// %out_1d = vector.broadcast %value : f32 to vector<16xf32> |
| 832 | /// %out_nd = vector.shape_cast %out_1d : vector<16xf32> to vector<4x4xf32> |
| 833 | /// ``` |
| 834 | struct LinearizeVectorBroadcast final |
| 835 | : public OpConversionPattern<vector::BroadcastOp> { |
| 836 | using Base::Base; |
| 837 | |
| 838 | LinearizeVectorBroadcast(const TypeConverter &typeConverter, |
| 839 | MLIRContext *context, PatternBenefit benefit = 1) |
| 840 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 841 | |
| 842 | LogicalResult |
| 843 | matchAndRewrite(vector::BroadcastOp broadcastOp, OpAdaptor adaptor, |
| 844 | ConversionPatternRewriter &rewriter) const override { |
| 845 | |
| 846 | int numElements = 1; |
| 847 | Type sourceType = broadcastOp.getSourceType(); |
| 848 | if (auto vecType = dyn_cast<VectorType>(Val&: sourceType)) { |
| 849 | numElements = vecType.getNumElements(); |
| 850 | } |
| 851 | |
| 852 | if (numElements != 1) { |
| 853 | return rewriter.notifyMatchFailure( |
| 854 | arg&: broadcastOp, msg: "only broadcasts of single elements can be linearized." ); |
| 855 | } |
| 856 | |
| 857 | auto dstTy = getTypeConverter()->convertType(t: broadcastOp.getType()); |
| 858 | rewriter.replaceOpWithNewOp<vector::BroadcastOp>(op: broadcastOp, args&: dstTy, |
| 859 | args: adaptor.getSource()); |
| 860 | |
| 861 | return success(); |
| 862 | } |
| 863 | }; |
| 864 | |
| 865 | /// Linearize `vector.interleave` to operate on flattened 1D operands and |
| 866 | /// result. The flattening is commutative here, the order of flatten and |
| 867 | /// interleave ops does not matter, so this transform is valid for |
| 868 | /// any ND shape. |
| 869 | /// |
| 870 | /// Example: |
| 871 | /// vector.interleave %a, %b : vector<4x1xT> -> vector<4x2xT> |
| 872 | /// becomes: |
| 873 | /// vector.interleave %a_flat, %b_flat : vector<4xT> -> vector<8xT> |
| 874 | struct LinearizeVectorInterleave final |
| 875 | : public OpConversionPattern<vector::InterleaveOp> { |
| 876 | using Base::Base; |
| 877 | LinearizeVectorInterleave(const TypeConverter &typeConverter, |
| 878 | MLIRContext *context, PatternBenefit benefit = 1) |
| 879 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 880 | |
| 881 | LogicalResult |
| 882 | matchAndRewrite(vector::InterleaveOp op, OpAdaptor adaptor, |
| 883 | ConversionPatternRewriter &rewriter) const override { |
| 884 | VectorType flatResultTy = |
| 885 | getTypeConverter()->convertType<VectorType>(t: op.getResultVectorType()); |
| 886 | if (!flatResultTy) |
| 887 | return rewriter.notifyMatchFailure(arg&: op, msg: "failed to linearize result type" ); |
| 888 | rewriter.replaceOpWithNewOp<vector::InterleaveOp>( |
| 889 | op, args&: flatResultTy, args: adaptor.getLhs(), args: adaptor.getRhs()); |
| 890 | return success(); |
| 891 | } |
| 892 | }; |
| 893 | |
| 894 | /// Linearize `vector.deinterleave` to operate on a flattened 1D source. |
| 895 | /// The flattening is commutative here, the order of flatten and deinterleave |
| 896 | /// ops does not matter, so this transform is valid for any ND shape. |
| 897 | /// |
| 898 | /// Example: |
| 899 | /// %even, %odd = vector.deinterleave %src : vector<4x2xT> -> vector<4x1xT> |
| 900 | /// becomes: |
| 901 | /// %even, %odd = vector.deinterleave %src_flat : vector<8xT> -> vector<4xT> |
| 902 | /// This linearization only works if the innermost dimension is even. |
| 903 | /// Consider vector<2x5xT>: |
| 904 | /// [[0,1,2,3,4], |
| 905 | /// [5,6,7,8,9]] |
| 906 | /// Non-linearized deinterleave returns the odd part: [[1, 3], [6, 8]] |
| 907 | /// Linearized deinterleave would the odd part: [1, 3, 5, 7, 9] |
| 908 | struct LinearizeVectorDeinterleave final |
| 909 | : public OpConversionPattern<vector::DeinterleaveOp> { |
| 910 | using Base::Base; |
| 911 | LinearizeVectorDeinterleave(const TypeConverter &typeConverter, |
| 912 | MLIRContext *context, PatternBenefit benefit = 1) |
| 913 | : OpConversionPattern(typeConverter, context, benefit) {} |
| 914 | |
| 915 | LogicalResult |
| 916 | matchAndRewrite(vector::DeinterleaveOp op, OpAdaptor adaptor, |
| 917 | ConversionPatternRewriter &rewriter) const override { |
| 918 | if (op.getSource().getType().getShape().back() % 2) |
| 919 | return failure(); |
| 920 | auto newOp = vector::DeinterleaveOp::create(builder&: rewriter, location: op.getLoc(), |
| 921 | source: adaptor.getSource()); |
| 922 | rewriter.replaceOp(op, newValues: newOp.getResults()); |
| 923 | return success(); |
| 924 | } |
| 925 | }; |
| 926 | |
| 927 | } // namespace |
| 928 | |
| 929 | /// This method defines the set of operations that are linearizable, and hence |
| 930 | /// that are considered illegal for the conversion target. |
| 931 | static bool isLinearizable(Operation *op) { |
| 932 | |
| 933 | // Only ops that are in the vector dialect, are ConstantLike, or |
| 934 | // are Vectorizable might be linearized currently. |
| 935 | StringLiteral vectorDialect = vector::VectorDialect::getDialectNamespace(); |
| 936 | StringRef opDialect = op->getDialect()->getNamespace(); |
| 937 | bool supported = (opDialect == vectorDialect) || |
| 938 | op->hasTrait<OpTrait::ConstantLike>() || |
| 939 | op->hasTrait<OpTrait::Vectorizable>(); |
| 940 | if (!supported) |
| 941 | return false; |
| 942 | |
| 943 | return TypeSwitch<Operation *, bool>(op) |
| 944 | // As type legalization is done with vector.shape_cast, shape_cast |
| 945 | // itself cannot be linearized (will create new shape_casts to linearize |
| 946 | // ad infinitum). |
| 947 | .Case(caseFn: [&](vector::ShapeCastOp) { return false; }) |
| 948 | // The operations |
| 949 | // - vector.extract_strided_slice |
| 950 | // - vector.extract |
| 951 | // - vector.insert_strided_slice |
| 952 | // - vector.insert |
| 953 | // are linearized to a rank-1 vector.shuffle by the current patterns. |
| 954 | // vector.shuffle only supports fixed size vectors, so it is impossible to |
| 955 | // use this approach to linearize these ops if they operate on scalable |
| 956 | // vectors. |
| 957 | .Case(caseFn: [&](vector::ExtractStridedSliceOp ) { |
| 958 | return !extractOp.getType().isScalable(); |
| 959 | }) |
| 960 | .Case(caseFn: [&](vector::InsertStridedSliceOp insertOp) { |
| 961 | return !insertOp.getType().isScalable(); |
| 962 | }) |
| 963 | .Case(caseFn: [&](vector::InsertOp insertOp) { |
| 964 | return !insertOp.getType().isScalable(); |
| 965 | }) |
| 966 | .Case(caseFn: [&](vector::ExtractOp ) { |
| 967 | return !extractOp.getSourceVectorType().isScalable(); |
| 968 | }) |
| 969 | .Default(defaultFn: [&](auto) { return true; }); |
| 970 | } |
| 971 | |
| 972 | void mlir::vector::populateForVectorLinearize(TypeConverter &typeConverter, |
| 973 | ConversionTarget &target) { |
| 974 | |
| 975 | auto convertType = [](Type type) -> std::optional<Type> { |
| 976 | VectorType vectorType = dyn_cast<VectorType>(Val&: type); |
| 977 | if (!vectorType || !isLinearizableVector(type: vectorType)) |
| 978 | return type; |
| 979 | |
| 980 | VectorType linearizedType = |
| 981 | VectorType::get(shape: vectorType.getNumElements(), |
| 982 | elementType: vectorType.getElementType(), scalableDims: vectorType.isScalable()); |
| 983 | return linearizedType; |
| 984 | }; |
| 985 | typeConverter.addConversion(callback&: convertType); |
| 986 | |
| 987 | auto materializeCast = [](OpBuilder &builder, Type type, ValueRange inputs, |
| 988 | Location loc) -> Value { |
| 989 | if (inputs.size() != 1) |
| 990 | return nullptr; |
| 991 | |
| 992 | Value value = inputs.front(); |
| 993 | if (!isa<VectorType>(Val: type) || !isa<VectorType>(Val: value.getType())) |
| 994 | return nullptr; |
| 995 | |
| 996 | return vector::ShapeCastOp::create(builder, location: loc, result: type, source: value); |
| 997 | }; |
| 998 | typeConverter.addSourceMaterialization(callback&: materializeCast); |
| 999 | typeConverter.addTargetMaterialization(callback&: materializeCast); |
| 1000 | |
| 1001 | target.markUnknownOpDynamicallyLegal( |
| 1002 | fn: [=](Operation *op) -> std::optional<bool> { |
| 1003 | if (!isLinearizable(op)) |
| 1004 | return true; |
| 1005 | // This will return true if, for all operand and result types `t`, |
| 1006 | // convertType(t) = t. This is true if there are no rank>=2 vectors. |
| 1007 | return typeConverter.isLegal(op); |
| 1008 | }); |
| 1009 | } |
| 1010 | |
| 1011 | /// Get the set of operand/result types to check for sufficiently |
| 1012 | /// small inner-most dimension size. |
| 1013 | static SmallVector<std::pair<Type, unsigned>> |
| 1014 | getTypeBitWidthBoundPairs(Operation *op, unsigned targetBitWidth) { |
| 1015 | |
| 1016 | if (auto insertOp = dyn_cast<vector::InsertOp>(Val: op)) { |
| 1017 | unsigned w = targetBitWidth < std::numeric_limits<unsigned>::max() |
| 1018 | ? targetBitWidth + 1 |
| 1019 | : targetBitWidth; |
| 1020 | return {{insertOp.getValueToStoreType(), w}}; |
| 1021 | } |
| 1022 | |
| 1023 | auto resultTypes = op->getResultTypes(); |
| 1024 | SmallVector<std::pair<Type, unsigned>> resultsWithBitWidth; |
| 1025 | resultsWithBitWidth.reserve(N: resultTypes.size()); |
| 1026 | for (Type type : resultTypes) { |
| 1027 | resultsWithBitWidth.push_back(Elt: {type, targetBitWidth}); |
| 1028 | } |
| 1029 | return resultsWithBitWidth; |
| 1030 | } |
| 1031 | |
| 1032 | /// Returns true if any checked vector type of op's inner-most dimension |
| 1033 | /// size is equal to or exceeds target bit width. |
| 1034 | static bool hasWideInnerDim(Operation *op, unsigned targetBitWidth) { |
| 1035 | // Check on bitwidths. |
| 1036 | SmallVector<std::pair<Type, unsigned>> toCheck = |
| 1037 | getTypeBitWidthBoundPairs(op, targetBitWidth); |
| 1038 | |
| 1039 | // If `type` is VectorType with trailing dimension of (bit) size greater than |
| 1040 | // or equal to `targetBitWidth`, its defining op is considered legal. |
| 1041 | return llvm::any_of(Range&: toCheck, P: [&](std::pair<Type, unsigned> typeWidth) { |
| 1042 | VectorType vecType = dyn_cast<VectorType>(Val&: typeWidth.first); |
| 1043 | |
| 1044 | // Not linearizable for reasons other than what this function checks. |
| 1045 | if (!vecType || vecType.getRank() == 0) |
| 1046 | return false; |
| 1047 | |
| 1048 | // The width of the type 'index' is unbounded (and therefore potentially |
| 1049 | // above the target width). |
| 1050 | if (vecType.getElementType().isIndex()) |
| 1051 | return true; |
| 1052 | |
| 1053 | unsigned finalDimSize = vecType.getShape().back(); |
| 1054 | unsigned nbBitsPerElm = vecType.getElementTypeBitWidth(); |
| 1055 | unsigned trailingVecDimBitWidth = finalDimSize * nbBitsPerElm; |
| 1056 | return trailingVecDimBitWidth >= typeWidth.second; |
| 1057 | }); |
| 1058 | } |
| 1059 | |
| 1060 | void mlir::vector::populateVectorLinearizeBitWidthTargetRules( |
| 1061 | ConversionTarget &target, unsigned targetBitWidth) { |
| 1062 | target.markUnknownOpDynamicallyLegal( |
| 1063 | fn: [=](Operation *op) -> std::optional<bool> { |
| 1064 | if (hasWideInnerDim(op, targetBitWidth)) |
| 1065 | return true; |
| 1066 | return std::nullopt; |
| 1067 | }); |
| 1068 | } |
| 1069 | |
| 1070 | void mlir::vector::populateVectorLinearizeBasePatterns( |
| 1071 | const TypeConverter &typeConverter, const ConversionTarget &target, |
| 1072 | RewritePatternSet &patterns) { |
| 1073 | patterns |
| 1074 | .add<LinearizeConstantLike, LinearizeVectorizable, LinearizeVectorBitCast, |
| 1075 | LinearizeVectorCreateMask, LinearizeVectorLoad, LinearizeVectorStore, |
| 1076 | LinearizeVectorBroadcast, LinearizeVectorFromElements, |
| 1077 | LinearizeVectorToElements, LinearizeVectorInterleave, |
| 1078 | LinearizeVectorDeinterleave>(arg: typeConverter, args: patterns.getContext()); |
| 1079 | } |
| 1080 | |
| 1081 | void mlir::vector::populateVectorLinearizeShuffleLikeOpsPatterns( |
| 1082 | const TypeConverter &typeConverter, const ConversionTarget &target, |
| 1083 | RewritePatternSet &patterns) { |
| 1084 | patterns.add<LinearizeVectorShuffle, LinearizeVectorExtract, |
| 1085 | LinearizeVectorInsert, LinearizeVectorExtractStridedSlice, |
| 1086 | LinearizeVectorInsertStridedSlice>(arg: typeConverter, |
| 1087 | args: patterns.getContext()); |
| 1088 | } |
| 1089 | |