| 1 | //===- VectorRewritePatterns.h - Vector rewrite patterns --------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORREWRITEPATTERNS_H |
| 10 | #define MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORREWRITEPATTERNS_H |
| 11 | |
| 12 | #include <optional> |
| 13 | #include <utility> |
| 14 | |
| 15 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 16 | #include "mlir/Dialect/Vector/Utils/VectorUtils.h" |
| 17 | #include "mlir/IR/PatternMatch.h" |
| 18 | |
| 19 | #include "mlir/Dialect/Vector/Transforms/VectorTransformsEnums.h.inc" |
| 20 | |
| 21 | namespace mlir { |
| 22 | class ConversionTarget; |
| 23 | class RewritePatternSet; |
| 24 | class TypeConverter; |
| 25 | |
| 26 | namespace arith { |
| 27 | class AndIOp; |
| 28 | class NarrowTypeEmulationConverter; |
| 29 | class TruncIOp; |
| 30 | } // namespace arith |
| 31 | |
| 32 | namespace vector { |
| 33 | struct VectorTransformsOptions; |
| 34 | |
| 35 | /// Options that control the vector unrolling. |
| 36 | struct UnrollVectorOptions { |
| 37 | using FilterConstraintFnType = std::function<LogicalResult(Operation *op)>; |
| 38 | /// Callback function that indicates whether vector unrolling should be |
| 39 | /// attempted on the operation. |
| 40 | FilterConstraintFnType filterConstraint = nullptr; |
| 41 | UnrollVectorOptions &setFilterConstraint(FilterConstraintFnType constraint) { |
| 42 | filterConstraint = std::move(constraint); |
| 43 | return *this; |
| 44 | } |
| 45 | |
| 46 | using NativeShapeFnType = |
| 47 | std::function<std::optional<SmallVector<int64_t>>(Operation *op)>; |
| 48 | /// Function that returns the shape of the vector to unroll to for a given |
| 49 | /// operation. The unrolling is aborted if the function returns |
| 50 | /// `std::nullopt`. |
| 51 | NativeShapeFnType nativeShape = nullptr; |
| 52 | UnrollVectorOptions &setNativeShapeFn(NativeShapeFnType fn) { |
| 53 | nativeShape = std::move(fn); |
| 54 | return *this; |
| 55 | } |
| 56 | |
| 57 | /// Set the native shape to use for unrolling. |
| 58 | UnrollVectorOptions &setNativeShape(ArrayRef<int64_t> shape) { |
| 59 | SmallVector<int64_t> tsShape(shape); |
| 60 | nativeShape = [=](Operation *) -> std::optional<SmallVector<int64_t>> { |
| 61 | return tsShape; |
| 62 | }; |
| 63 | return *this; |
| 64 | } |
| 65 | |
| 66 | /// Function that returns the traversal order (in terms of "for loop order", |
| 67 | /// i.e. slowest varying dimension to fastest varying dimension) that should |
| 68 | /// be used when unrolling the given operation into units of the native vector |
| 69 | /// size. |
| 70 | using UnrollTraversalOrderFnType = |
| 71 | std::function<std::optional<SmallVector<int64_t>>(Operation *op)>; |
| 72 | UnrollTraversalOrderFnType traversalOrderCallback = nullptr; |
| 73 | UnrollVectorOptions & |
| 74 | setUnrollTraversalOrderFn(UnrollTraversalOrderFnType traversalOrderFn) { |
| 75 | traversalOrderCallback = std::move(traversalOrderFn); |
| 76 | return *this; |
| 77 | } |
| 78 | }; |
| 79 | |
| 80 | /// Canonicalization of a `vector.contract %a, %b, %c` with row-major matmul |
| 81 | /// semantics to a contraction with MMT semantics (matrix matrix multiplication |
| 82 | /// with the RHS transposed). This specific form is meant to have the vector |
| 83 | /// operands are organized such that the reduction dimension is contiguous. |
| 84 | /// Example: |
| 85 | /// ``` |
| 86 | /// vector.contract {indexing_maps = [affine_map<(m, n, k) -> (m, k)>, |
| 87 | /// affine_map<(m, n, k) -> (n, k)>, |
| 88 | /// affine_map<(m, n, k) -> (m, n)>], |
| 89 | /// iterator_types = ["parallel", "parallel", "reduction"], |
| 90 | /// kind = #vector.kind<add>} %a, %b, %c : ... |
| 91 | /// ``` |
| 92 | /// |
| 93 | /// The `constraint` predicate is used to decide which `vector.contract` ops |
| 94 | /// to filter out. |
| 95 | void populateVectorContractCanonicalizeMatmulToMMT( |
| 96 | RewritePatternSet &patterns, |
| 97 | std::function<LogicalResult(vector::ContractionOp)> constraint = |
| 98 | [](vector::ContractionOp) { return success(); }, |
| 99 | PatternBenefit = 1); |
| 100 | |
| 101 | /// Collect patterns to convert reduction op to vector.contract and fold |
| 102 | /// transpose/broadcast ops into the contract. |
| 103 | /// |
| 104 | /// The new contract keeps only the fast-math flags that both the multiply and |
| 105 | /// the reduction carry. If `allowFastMathFlagLoss` is false, a multiply and a |
| 106 | /// reduction with different flags are not converted, so no flag is lost. |
| 107 | void populateVectorReductionToContractPatterns( |
| 108 | RewritePatternSet &patterns, PatternBenefit benefit = 1, |
| 109 | bool allowFastMathFlagLoss = true); |
| 110 | |
| 111 | /// Populate `patterns` with the following patterns. |
| 112 | /// |
| 113 | /// - VectorTransferFullPartialRewriter |
| 114 | /// |
| 115 | /// Split a vector.transfer operation into an in-bounds (i.e., no out-of-bounds |
| 116 | /// masking) fast path and a slow path. |
| 117 | /// |
| 118 | /// Example (a 2-D vector.transfer_read): |
| 119 | /// ``` |
| 120 | /// %1 = vector.transfer_read %0[...], %pad : memref<A...>, vector<...> |
| 121 | /// ``` |
| 122 | /// is transformed into: |
| 123 | /// ``` |
| 124 | /// %1:3 = scf.if (%inBounds) { |
| 125 | /// // fast path, direct cast |
| 126 | /// memref.cast %A: memref<A...> to compatibleMemRefType |
| 127 | /// scf.yield %view : compatibleMemRefType, index, index |
| 128 | /// } else { |
| 129 | /// // slow path, not in-bounds vector.transfer or memref.copy. |
| 130 | /// memref.cast %alloc: memref<B...> to compatibleMemRefType |
| 131 | /// scf.yield %4 : compatibleMemRefType, index, index |
| 132 | // } |
| 133 | /// %0 = vector.transfer_read %1#0[%1#1, %1#2] {in_bounds = [true ... true]} |
| 134 | /// ``` |
| 135 | /// where `alloc` is a top of the function alloca'ed buffer of one vector. |
| 136 | /// |
| 137 | /// Preconditions: |
| 138 | /// 1. `xferOp.permutation_map()` must be a minor identity map |
| 139 | /// 2. the rank of the `xferOp.memref()` and the rank of the `xferOp.vector()` |
| 140 | /// must be equal. This will be relaxed in the future but requires |
| 141 | /// rank-reducing subviews. |
| 142 | void populateVectorTransferFullPartialPatterns( |
| 143 | RewritePatternSet &patterns, const VectorTransformsOptions &options); |
| 144 | |
| 145 | /// Collect a set of patterns to collapse the most inner unit dims in xfer Ops |
| 146 | /// |
| 147 | /// These patters reduce the rank of the operands of vector transfer ops to |
| 148 | /// operate on vectors without trailing unit dims. This helps reduce the rank of |
| 149 | /// the operands, which can be helpful when lowering to dialects that only |
| 150 | /// support 1D vector type such as LLVM. |
| 151 | void populateDropInnerMostUnitDimsXferOpPatterns(RewritePatternSet &patterns, |
| 152 | PatternBenefit benefit = 1); |
| 153 | |
| 154 | /// Patterns that remove redundant Vector Ops by re-ordering them with |
| 155 | /// e.g. elementwise Ops: |
| 156 | /// ``` |
| 157 | /// %at = vector.transpose %a, [1, 0]: vector<4x2xf32> to vector<2x4xf32> |
| 158 | /// %bt = vector.transpose %b, [1, 0]: vector<4x2xf32> to vector<2x4xf32> |
| 159 | /// %r = arith.addf %at, %bt : vector<2x4xf32> |
| 160 | /// ``` |
| 161 | /// gets converted to: |
| 162 | /// ``` |
| 163 | /// %0 = arith.addf %a, %b : vector<4x2xf32> |
| 164 | /// %r = vector.transpose %0, [1, 0] : vector<2x4xf32> |
| 165 | /// ``` |
| 166 | /// At the moment, these patterns are limited to vector.broadcast and |
| 167 | /// vector.transpose. |
| 168 | void populateSinkVectorOpsPatterns(RewritePatternSet &patterns, |
| 169 | PatternBenefit benefit = 1); |
| 170 | |
| 171 | /// Patterns that remove redundant Vector Ops by merging them with load/store |
| 172 | /// ops |
| 173 | /// ``` |
| 174 | /// vector.load %arg0[%arg1] : memref<?xf32>, vector<4xf32> |
| 175 | /// vector.extract %0[1] : f32 from vector<4xf32> |
| 176 | /// ``` |
| 177 | /// Gets converted to: |
| 178 | /// ``` |
| 179 | /// %c1 = arith.constant 1 : index |
| 180 | /// %0 = arith.addi %arg1, %c1 overflow<nsw> : index |
| 181 | /// %1 = memref.load %arg0[%0] : memref<?xf32> |
| 182 | void populateSinkVectorMemOpsPatterns(RewritePatternSet &patterns, |
| 183 | PatternBenefit benefit = 1); |
| 184 | |
| 185 | /// Patterns that fold chained vector reductions. These patterns assume that |
| 186 | /// elementwise operations (e.g., `arith.addf` with vector operands) are |
| 187 | /// cheaper than vector reduction. |
| 188 | /// Note that these patterns change the order of reduction which may not always |
| 189 | /// produce bit-identical results on some floating point inputs. |
| 190 | /// |
| 191 | /// Example: |
| 192 | /// ``` |
| 193 | /// %a = vector.reduction <add> %x, %acc |
| 194 | /// %b = vector.reduction <add> %y, %a |
| 195 | /// ``` |
| 196 | /// is transformed into: |
| 197 | /// ``` |
| 198 | /// %a = arith.addf %x, %y |
| 199 | /// %b = vector.reduction <add> %a, %acc |
| 200 | /// ``` |
| 201 | void populateChainedVectorReductionFoldingPatterns(RewritePatternSet &patterns, |
| 202 | PatternBenefit benefit = 1); |
| 203 | |
| 204 | /// Patterns to break down vector reductions into a series of arith reductions |
| 205 | /// over vector elements. This is intended to be simplify code with reductions |
| 206 | /// over small vector types and avoid more specialized reduction lowering when |
| 207 | /// possible. |
| 208 | /// |
| 209 | /// Example: |
| 210 | /// ``` |
| 211 | /// %a = vector.reduction <add> %x : vector<2xf32> into f32 |
| 212 | /// ``` |
| 213 | /// is transformed into: |
| 214 | /// ``` |
| 215 | /// %y = vector.extract %x[0] : f32 from vector<2xf32> |
| 216 | /// %z = vector.extract %x[1] : f32 from vector<2xf32> |
| 217 | /// %a = arith.addf %y, %z : f32 |
| 218 | /// ``` |
| 219 | void populateBreakDownVectorReductionPatterns( |
| 220 | RewritePatternSet &patterns, unsigned = 2, |
| 221 | PatternBenefit benefit = 1); |
| 222 | |
| 223 | /// Populate `patterns` with the following patterns. |
| 224 | /// |
| 225 | /// [DecomposeDifferentRankInsertStridedSlice] |
| 226 | /// ========================================== |
| 227 | /// RewritePattern for InsertStridedSliceOp where source and destination vectors |
| 228 | /// have different ranks. |
| 229 | /// |
| 230 | /// When ranks are different, InsertStridedSlice needs to extract a properly |
| 231 | /// ranked vector from the destination vector into which to insert. This pattern |
| 232 | /// only takes care of this extraction part and forwards the rest to |
| 233 | /// [VectorInsertStridedSliceOpSameRankRewritePattern]. |
| 234 | /// |
| 235 | /// For a k-D source and n-D destination vector (k < n), we emit: |
| 236 | /// 1. ExtractOp to extract the (unique) (n-1)-D subvector into which to |
| 237 | /// insert the k-D source. |
| 238 | /// 2. k-D -> (n-1)-D InsertStridedSlice op |
| 239 | /// 3. InsertOp that is the reverse of 1. |
| 240 | /// |
| 241 | /// [DecomposeNDExtractStridedSlice] |
| 242 | /// ================================ |
| 243 | /// For such cases, we can rewrite it to ExtractOp + lower rank |
| 244 | /// ExtractStridedSliceOp + InsertOp for the n-D case. |
| 245 | void ( |
| 246 | RewritePatternSet &patterns, PatternBenefit benefit = 1); |
| 247 | |
| 248 | /// Populate `patterns` with a pattern to breaks down 1-D extract_strided_slice |
| 249 | /// ops into a chain of Extract ops to extract each element from the source, and |
| 250 | /// then a chain of Insert ops to insert to the target vector. |
| 251 | /// |
| 252 | /// If `controlFn` is not nullptr, the pattern will only be invoked on ops that |
| 253 | /// `controlFn` returns true. Otherwise runs on ops. |
| 254 | void ( |
| 255 | RewritePatternSet &patterns, |
| 256 | std::function<bool(ExtractStridedSliceOp)> controlFn = nullptr, |
| 257 | PatternBenefit benefit = 1); |
| 258 | |
| 259 | /// Populate `patterns` with a pattern to break down 1-D vector.bitcast ops |
| 260 | /// based on the destination vector shape. Bitcasts from a lower bitwidth |
| 261 | /// element type to a higher bitwidth one are extracted from the lower bitwidth |
| 262 | /// based on the native destination vector shape and inserted based on the ratio |
| 263 | /// of the bitwidths. |
| 264 | /// |
| 265 | /// This acts as a last resort way to break down vector.bitcast ops to smaller |
| 266 | /// vector sizes. Because this pattern composes until it is bitcasting to a |
| 267 | /// single element of the higher bitwidth, the is an optional control function. |
| 268 | /// If `controlFn` is not nullptr, the pattern will only apply to ops where |
| 269 | /// `controlFn` returns true, otherwise applies to all bitcast ops. |
| 270 | void populateBreakDownVectorBitCastOpPatterns( |
| 271 | RewritePatternSet &patterns, |
| 272 | std::function<bool(BitCastOp)> controlFn = nullptr, |
| 273 | PatternBenefit benefit = 1); |
| 274 | |
| 275 | /// Populate `patterns` with the following patterns. |
| 276 | /// |
| 277 | /// Patterns in populateVectorInsertExtractStridedSliceDecompositionPatterns(); |
| 278 | /// |
| 279 | /// [ConvertSameRankInsertStridedSliceIntoShuffle] |
| 280 | /// ============================================== |
| 281 | /// RewritePattern for InsertStridedSliceOp where source and destination vectors |
| 282 | /// have the same rank. For each outermost index in the slice: |
| 283 | /// begin end stride |
| 284 | /// [offset : offset+size*stride : stride] |
| 285 | /// 1. ExtractOp one (k-1)-D source subvector and one (n-1)-D dest subvector. |
| 286 | /// 2. InsertStridedSlice (k-1)-D into (n-1)-D |
| 287 | /// 3. the destination subvector is inserted back in the proper place |
| 288 | /// 3. InsertOp that is the reverse of 1. |
| 289 | /// |
| 290 | /// [Convert1DExtractStridedSliceIntoShuffle] |
| 291 | /// ========================================= |
| 292 | /// For such cases, we can lower it to a ShuffleOp. |
| 293 | void ( |
| 294 | RewritePatternSet &patterns, PatternBenefit benefit = 1); |
| 295 | |
| 296 | /// Collect a set of pattern to unroll vector operations to a smaller shapes. |
| 297 | /// `options` structure controls which operations are unrolled and the target |
| 298 | /// shape. |
| 299 | /// `op` is unrolled to the `targetShape` as follows, for each of its operands: |
| 300 | /// 1. the unrolled type `unrolledVectorType` and number of unrolled instances |
| 301 | /// `numUnrolledInstances` are computed from the `targetShape`. For now it is |
| 302 | /// assumed the unrolling factors divide the vector sizes. |
| 303 | /// 2. ExtractStridedSlice are created to break-up the vector operands. |
| 304 | /// 3. the original op is cloned `numUnrolledInstances` times, once for each |
| 305 | /// result. |
| 306 | /// 4. InsertStridedSlice are inserted to re-assemble the slices into the |
| 307 | /// original vectore shape. |
| 308 | /// |
| 309 | /// Example: |
| 310 | /// |
| 311 | /// opA(operand0, operand1) // numUnrolledInstances = 3 |
| 312 | /// |
| 313 | /// operand0 operand1 |
| 314 | /// | | |
| 315 | /// fork fork |
| 316 | /// <----------gather all fork ops ---------> |
| 317 | /// /|\ /|\ |
| 318 | /// f00 f01 f02 f10 f11 f12 |
| 319 | /// <---------- clone op 3 times ---------> |
| 320 | /// opA0(f00, f10), opA1(f01, f11), opA2(f02, f12) |
| 321 | /// \ | / |
| 322 | /// <-------------------- join -------------------------> |
| 323 | /// |
| 324 | /// Other local patterns then kick in iteratively (including DCE) and compose |
| 325 | /// to combine the ExtractStridedSlice/InsertStridedSlice. |
| 326 | void populateVectorUnrollPatterns(RewritePatternSet &patterns, |
| 327 | const UnrollVectorOptions &options, |
| 328 | PatternBenefit benefit = 1); |
| 329 | |
| 330 | /// Compute indices for a transfer op slice. |
| 331 | SmallVector<Value> sliceTransferIndices(ArrayRef<int64_t> elementOffsets, |
| 332 | ArrayRef<Value> indices, |
| 333 | AffineMap permutationMap, Location loc, |
| 334 | OpBuilder &builder); |
| 335 | |
| 336 | /// Unrolls 2 or more dimensional `vector.to_elements` ops by unrolling the |
| 337 | /// outermost dimension of the operand. |
| 338 | void populateVectorContractUnrollPatterns(RewritePatternSet &patterns, |
| 339 | const UnrollVectorOptions &options, |
| 340 | PatternBenefit benefit = 1); |
| 341 | |
| 342 | /// Unrolls 2 or more dimensional `vector.to_elements` ops by unrolling the |
| 343 | /// outermost dimension of the operand. |
| 344 | void populateVectorToElementsUnrollPatterns(RewritePatternSet &patterns, |
| 345 | PatternBenefit benefit = 1); |
| 346 | |
| 347 | /// Unrolls 2 or more dimensional `vector.from_elements` ops by unrolling the |
| 348 | /// outermost dimension. |
| 349 | void populateVectorFromElementsUnrollPatterns(RewritePatternSet &patterns, |
| 350 | PatternBenefit benefit = 1); |
| 351 | |
| 352 | /// Collect a set of leading one dimension removal patterns. |
| 353 | /// |
| 354 | /// These patterns insert vector.shape_cast to remove leading one dimensions |
| 355 | /// to expose more canonical forms of read/write/insert/extract operations. |
| 356 | /// With them, there are more chances that we can cancel out extract-insert |
| 357 | /// pairs or forward write-read pairs. |
| 358 | void populateCastAwayVectorLeadingOneDimPatterns(RewritePatternSet &patterns, |
| 359 | PatternBenefit benefit = 1); |
| 360 | |
| 361 | /// Collect a set of one dimension removal patterns. |
| 362 | /// |
| 363 | /// These patterns insert rank-reducing memref.subview ops to remove one |
| 364 | /// dimensions. With them, there are more chances that we can avoid |
| 365 | /// potentially expensive vector.shape_cast operations. |
| 366 | void populateVectorTransferDropUnitDimsPatterns(RewritePatternSet &patterns, |
| 367 | PatternBenefit benefit = 1); |
| 368 | |
| 369 | /// Collect a set of patterns that use vector.shape_cast to help fold unit dims. |
| 370 | /// |
| 371 | /// These patterns use vector.shape_cast to remove unit dims from e.g. |
| 372 | /// arithmetic operations on Vectors. The newly inserted shape_casts will either |
| 373 | /// cancel each other out or will be folded away when combined with other |
| 374 | /// patterns. |
| 375 | void populateDropUnitDimWithShapeCastPatterns(RewritePatternSet &patterns, |
| 376 | PatternBenefit benefit = 1); |
| 377 | |
| 378 | /// Collect a set of patterns to flatten n-D vector transfers on contiguous |
| 379 | /// memref. |
| 380 | /// |
| 381 | /// These patterns insert memref.collapse_shape + vector.shape_cast patterns |
| 382 | /// to transform multiple small n-D transfers into a larger 1-D transfer where |
| 383 | /// the memref contiguity properties allow it. |
| 384 | /// |
| 385 | /// Flattening is only applied if the bitwidth of the trailing vector dimension |
| 386 | /// is smaller or equal to `targetVectorBitwidth`. |
| 387 | void populateFlattenVectorTransferPatterns( |
| 388 | RewritePatternSet &patterns, |
| 389 | unsigned targetVectorBitwidth = std::numeric_limits<unsigned>::max(), |
| 390 | PatternBenefit benefit = 1); |
| 391 | |
| 392 | /// Collect a set of patterns that bubble up/down bitcast ops. |
| 393 | /// |
| 394 | /// These patterns move vector.bitcast ops to be before insert ops or after |
| 395 | /// extract ops where suitable. With them, bitcast will happen on smaller |
| 396 | /// vectors and there are more chances to share extract/insert ops. |
| 397 | void populateBubbleVectorBitCastOpPatterns(RewritePatternSet &patterns, |
| 398 | PatternBenefit benefit = 1); |
| 399 | |
| 400 | /// These patterns materialize masks for various vector ops such as transfers. |
| 401 | void populateVectorMaskMaterializationPatterns(RewritePatternSet &patterns, |
| 402 | bool force32BitVectorIndices, |
| 403 | PatternBenefit benefit = 1); |
| 404 | |
| 405 | /// Appends patterns for emulating vector operations over narrow types with ops |
| 406 | /// over wider types. The `disableAtomicRMW` indicates whether to use a normal |
| 407 | /// read-modify-write sequence instead of using `memref.generic_atomic_rmw` to |
| 408 | /// perform subbyte storing. When `assumeAligned` is true, store offsets are |
| 409 | /// assumed to be aligned to container element boundaries, so a store whose |
| 410 | /// source vector fills whole container elements is emitted as a simple |
| 411 | /// bitcast + store without checking the offset. Stores that are not divisible |
| 412 | /// in size are rejected. |
| 413 | void populateVectorNarrowTypeEmulationPatterns( |
| 414 | const arith::NarrowTypeEmulationConverter &typeConverter, |
| 415 | RewritePatternSet &patterns, bool disableAtomicRMW = false, |
| 416 | bool assumeAligned = false); |
| 417 | |
| 418 | /// Populates patterns for both MeMref flattening and Vector narrow type |
| 419 | /// emulation. |
| 420 | /// |
| 421 | /// Patterns for narrow-type-emulation require "flattened" MemRef(s), so this |
| 422 | /// composite populate* method can be used for narrow-type-emulation for Ops |
| 423 | /// operating on MemRef(s) that are rank > 2. |
| 424 | void populateMemRefFlattenAndVectorNarrowTypeEmulationPatterns( |
| 425 | arith::NarrowTypeEmulationConverter &typeConverter, |
| 426 | RewritePatternSet &patterns); |
| 427 | |
| 428 | /// Appends patterns for rewriting vector operations over narrow types with |
| 429 | /// ops over wider types. |
| 430 | /// Warning: these patterns currently only work for little endian targets. |
| 431 | void populateVectorNarrowTypeRewritePatterns(RewritePatternSet &patterns, |
| 432 | PatternBenefit benefit = 1); |
| 433 | |
| 434 | /// Appends patterns for emulating a sub-byte vector transpose. |
| 435 | void populateVectorTransposeNarrowTypeRewritePatterns( |
| 436 | RewritePatternSet &patterns, PatternBenefit benefit = 1); |
| 437 | |
| 438 | /// Initialize `typeConverter` and `conversionTarget` for vector linearization. |
| 439 | /// |
| 440 | /// Definition: here 'linearization' means converting a single operation with |
| 441 | /// 1+ vector operand/result of rank>1, into a new single operation whose |
| 442 | /// vector operands and results are all of rank<=1. |
| 443 | /// |
| 444 | /// This function registers (1) which operations are legal, and hence should not |
| 445 | /// be linearized, (2) what the converted types are (rank-1 vectors) and how to |
| 446 | /// materialze the conversion (with shape_cast) |
| 447 | /// |
| 448 | /// Note: the set of legal operations can be extended by a user if for example |
| 449 | /// certain rank>1 vectors are considered valid, by adding additional |
| 450 | /// dynamically legal ops to `conversionTarget`. |
| 451 | /// |
| 452 | /// Further note: the choice to use a dialect conversion design for |
| 453 | /// linearization is to make it easy to reuse generic structural type |
| 454 | /// conversions for linearizing scf/cf/func operations |
| 455 | void populateForVectorLinearize(TypeConverter &typeConverter, |
| 456 | ConversionTarget &conversionTarget); |
| 457 | |
| 458 | /// Extends `conversionTarget` so that ops whose inner-most vector dimension is |
| 459 | /// at least `targetBitWidth` bits wide (or has `index` elements) stay legal. |
| 460 | /// Must be called after `populateForVectorLinearize`. |
| 461 | void populateVectorLinearizeBitWidthTargetRules( |
| 462 | ConversionTarget &conversionTarget, unsigned targetBitWidth); |
| 463 | |
| 464 | /// Populates `patterns` for ND vector (N >= 2) linearization. This currently |
| 465 | /// contains patterns for converting ConstantLike, Vectorizable, and |
| 466 | /// vector::BitCast ops. |
| 467 | void populateVectorLinearizeBasePatterns(const TypeConverter &, |
| 468 | const ConversionTarget &, |
| 469 | RewritePatternSet &patterns); |
| 470 | |
| 471 | /// Populates `patterns` for linearizing ND (N >= 2) vector operations |
| 472 | /// to 1D vector shuffle operations. |
| 473 | void populateVectorLinearizeShuffleLikeOpsPatterns(const TypeConverter &, |
| 474 | const ConversionTarget &, |
| 475 | RewritePatternSet &patterns); |
| 476 | |
| 477 | } // namespace vector |
| 478 | } // namespace mlir |
| 479 | |
| 480 | #endif // MLIR_DIALECT_VECTOR_TRANSFORMS_VECTORREWRITEPATTERNS_H |
| 481 | |