| 1 | //===- LoweringPatterns.h - Vector rewrite patterns --------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef MLIR_DIALECT_VECTOR_TRANSFORMS_LOWERINGPATTERNS_H |
| 10 | #define MLIR_DIALECT_VECTOR_TRANSFORMS_LOWERINGPATTERNS_H |
| 11 | |
| 12 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
| 13 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 14 | |
| 15 | namespace mlir { |
| 16 | class RewritePatternSet; |
| 17 | |
| 18 | namespace vector { |
| 19 | |
| 20 | //===----------------------------------------------------------------------===// |
| 21 | // Lowering pattern populate functions |
| 22 | //===----------------------------------------------------------------------===// |
| 23 | |
| 24 | /// Populate the pattern set with the following patterns: |
| 25 | /// |
| 26 | /// [OuterProductOpLowering] |
| 27 | /// Progressively lower a `vector.outerproduct` to linearized |
| 28 | /// `vector.extract` + `vector.fma` + `vector.insert`. |
| 29 | /// |
| 30 | /// [ContractionOpLowering] |
| 31 | /// Progressive lowering of ContractionOp. |
| 32 | /// One: |
| 33 | /// %x = vector.contract with at least one free/batch dimension |
| 34 | /// is replaced by: |
| 35 | /// %a = vector.contract with one less free/batch dimension |
| 36 | /// %b = vector.contract with one less free/batch dimension |
| 37 | /// |
| 38 | /// [ContractionOpToMatmulOpLowering] |
| 39 | /// Progressively lower a `vector.contract` with row-major matmul semantics to |
| 40 | /// linearized `vector.shape_cast` + `vector.matmul` on the way to |
| 41 | /// `llvm.matrix.multiply`. |
| 42 | /// |
| 43 | /// [ContractionOpToDotLowering] |
| 44 | /// Progressively lower a `vector.contract` with row-major matmul semantics to |
| 45 | /// linearized `vector.extract` + `vector.reduction` + `vector.insert`. |
| 46 | /// |
| 47 | /// [ContractionOpToOuterProductOpLowering] |
| 48 | /// Progressively lower a `vector.contract` with row-major matmul semantics to |
| 49 | /// linearized `vector.extract` + `vector.outerproduct` + `vector.insert`. |
| 50 | void populateVectorContractLoweringPatterns( |
| 51 | RewritePatternSet &patterns, |
| 52 | VectorContractLowering vectorContractLoweringOption, |
| 53 | PatternBenefit benefit = 1, bool disableOuterProductLowering = false); |
| 54 | |
| 55 | /// Populate the pattern set with the following patterns: |
| 56 | /// |
| 57 | /// [OuterProductOpLowering] |
| 58 | /// Progressively lower a `vector.outerproduct` to linearized |
| 59 | /// `vector.extract` + `vector.fma` + `vector.insert`. |
| 60 | void populateVectorOuterProductLoweringPatterns(RewritePatternSet &patterns, |
| 61 | PatternBenefit benefit = 1); |
| 62 | |
| 63 | /// Populate the pattern set with the following patterns: |
| 64 | /// |
| 65 | /// [InnerOuterDimReductionConversion] |
| 66 | /// Rewrites vector.multi_reduction such that all reduction dimensions are |
| 67 | /// either innermost or outermost, by adding the proper vector.transpose |
| 68 | /// operations. |
| 69 | void populateVectorMultiReductionReorderPatterns( |
| 70 | RewritePatternSet &patterns, VectorMultiReductionLowering options, |
| 71 | PatternBenefit benefit = 1); |
| 72 | |
| 73 | /// Populate the pattern set with the following patterns: |
| 74 | /// |
| 75 | /// [ReduceMultiDimReductionRank] |
| 76 | /// Once in innermost or outermost reduction |
| 77 | /// form, rewrites n-D vector.multi_reduction into 2-D vector.multi_reduction, |
| 78 | /// by introducing vector.shape_cast ops to collapse + multi-reduce + expand |
| 79 | /// back. |
| 80 | void populateVectorMultiReductionFlatteningPatterns( |
| 81 | RewritePatternSet &patterns, VectorMultiReductionLowering options, |
| 82 | PatternBenefit benefit = 1); |
| 83 | |
| 84 | /// Populate the pattern set with the following patterns: |
| 85 | /// |
| 86 | /// [OneDimMultiReductionToReduction] |
| 87 | /// Converts 1-D vector.multi_reduction to vector.reduction. |
| 88 | /// |
| 89 | /// [TwoDimMultiReductionToElementWise] |
| 90 | /// Once in 2-D vector.multi_reduction form, with an **outermost** reduction |
| 91 | /// dimension, unroll the outer dimension to obtain a sequence of 1-D vector |
| 92 | /// ops. This also has an opportunity for tree-reduction (in the future). |
| 93 | /// |
| 94 | /// [TwoDimMultiReductionToReduction] |
| 95 | /// Once in 2-D vector.multi_reduction form, with an **innermost** reduction |
| 96 | /// dimension, unroll the outer dimension to obtain a sequence of extract + |
| 97 | /// vector.reduction + insert. This can further lower to horizontal reduction |
| 98 | /// ops. |
| 99 | void populateVectorMultiReductionUnrollingPatterns( |
| 100 | RewritePatternSet &patterns, VectorMultiReductionLowering options, |
| 101 | PatternBenefit benefit = 1); |
| 102 | |
| 103 | /// Populate the pattern set with the following patterns: |
| 104 | /// |
| 105 | /// [TransferReadToVectorLoadLowering] |
| 106 | /// Progressive lowering of BroadcastOp to ExtractOp + InsertOp + lower-D |
| 107 | /// BroadcastOp until dim 1. |
| 108 | void populateVectorBroadcastLoweringPatterns(RewritePatternSet &patterns, |
| 109 | PatternBenefit benefit = 1); |
| 110 | |
| 111 | /// Populate the pattern set with the following patterns: |
| 112 | /// |
| 113 | /// [CreateMaskOp] |
| 114 | /// Progressive lowering of CreateMaskOp to lower-D CreateMaskOp until dim 1. |
| 115 | /// |
| 116 | /// [ConstantMaskOp] |
| 117 | /// Progressive lowering of ConstantMaskOp to lower-D ConstantMaskOp until |
| 118 | /// dim 1. |
| 119 | void populateVectorMaskOpLoweringPatterns(RewritePatternSet &patterns, |
| 120 | PatternBenefit benefit = 1); |
| 121 | |
| 122 | /// Collects patterns that lower scalar vector transfer ops to memref loads and |
| 123 | /// stores when beneficial. If `allowMultipleUses` is set to true, the patterns |
| 124 | /// are applied to vector transfer reads with any number of uses. Otherwise, |
| 125 | /// only vector transfer reads with a single use will be lowered. |
| 126 | void populateScalarVectorTransferLoweringPatterns(RewritePatternSet &patterns, |
| 127 | PatternBenefit benefit, |
| 128 | bool allowMultipleUses); |
| 129 | |
| 130 | /// Populate the pattern set with the following patterns: |
| 131 | /// |
| 132 | /// [ShapeCastOp2DDownCastRewritePattern] |
| 133 | /// ShapeOp 2D -> 1D downcast serves the purpose of flattening 2-D to 1-D |
| 134 | /// vectors progressively. |
| 135 | /// |
| 136 | /// [ShapeCastOp2DUpCastRewritePattern] |
| 137 | /// ShapeOp 1D -> 2D upcast serves the purpose of unflattening 2-D from 1-D |
| 138 | /// vectors progressively. |
| 139 | /// |
| 140 | /// [ShapeCastOpRewritePattern] |
| 141 | /// Reference lowering to fully unrolled sequences of single element ExtractOp + |
| 142 | /// InsertOp. Note that applying this pattern can almost always be considered a |
| 143 | /// performance bug. |
| 144 | void populateVectorShapeCastLoweringPatterns(RewritePatternSet &patterns, |
| 145 | PatternBenefit benefit = 1); |
| 146 | |
| 147 | /// Populate the pattern set with the following patterns: |
| 148 | /// |
| 149 | /// [TransposeOpLowering] |
| 150 | /// |
| 151 | /// [TransposeOp2DToShuffleLowering] |
| 152 | /// |
| 153 | void populateVectorTransposeLoweringPatterns( |
| 154 | RewritePatternSet &patterns, |
| 155 | VectorTransposeLowering vectorTransposeLowering, |
| 156 | PatternBenefit benefit = 1); |
| 157 | |
| 158 | /// Populate the pattern set with the following patterns: |
| 159 | /// |
| 160 | /// [TransferReadToVectorLoadLowering] |
| 161 | /// Progressive lowering of transfer_read.This pattern supports lowering of |
| 162 | /// `vector.transfer_read` to a combination of `vector.load` and |
| 163 | /// `vector.broadcast` |
| 164 | /// |
| 165 | /// [TransferWriteToVectorStoreLowering] |
| 166 | /// Progressive lowering of transfer_write. This pattern supports lowering of |
| 167 | /// `vector.transfer_write` to `vector.store` |
| 168 | /// |
| 169 | /// These patterns lower transfer ops to simpler ops like `vector.load`, |
| 170 | /// `vector.store` and `vector.broadcast`. Only transfers with a transfer rank |
| 171 | /// of a most `maxTransferRank` are lowered. This is useful when combined with |
| 172 | /// VectorToSCF, which reduces the rank of vector transfer ops. |
| 173 | void populateVectorTransferLoweringPatterns( |
| 174 | RewritePatternSet &patterns, |
| 175 | std::optional<unsigned> maxTransferRank = std::nullopt, |
| 176 | PatternBenefit benefit = 1); |
| 177 | |
| 178 | /// Collect a set of transfer read/write lowering patterns that simplify the |
| 179 | /// permutation map (e.g., converting it to a minor identity map) by inserting |
| 180 | /// broadcasts and transposes. More specifically: |
| 181 | /// |
| 182 | /// [TransferReadPermutationLowering] |
| 183 | /// Lower transfer_read op with permutation into a transfer_read with a |
| 184 | /// permutation map composed of leading zeros followed by a minor identity + |
| 185 | /// vector.transpose op. |
| 186 | /// Ex: |
| 187 | /// vector.transfer_read ... |
| 188 | /// permutation_map: (d0, d1, d2) -> (0, d1) |
| 189 | /// into: |
| 190 | /// %v = vector.transfer_read ... |
| 191 | /// permutation_map: (d0, d1, d2) -> (d1, 0) |
| 192 | /// vector.transpose %v, [1, 0] |
| 193 | /// |
| 194 | /// vector.transfer_read ... |
| 195 | /// permutation_map: (d0, d1, d2, d3) -> (0, 0, 0, d1, d3) |
| 196 | /// into: |
| 197 | /// %v = vector.transfer_read ... |
| 198 | /// permutation_map: (d0, d1, d2, d3) -> (0, 0, d1, 0, d3) |
| 199 | /// vector.transpose %v, [0, 1, 3, 2, 4] |
| 200 | /// Note that an alternative is to transform it to linalg.transpose + |
| 201 | /// vector.transfer_read to do the transpose in memory instead. |
| 202 | /// |
| 203 | /// [TransferWritePermutationLowering] |
| 204 | /// Lower transfer_write op with permutation into a transfer_write with a |
| 205 | /// minor identity permutation map. (transfer_write ops cannot have broadcasts.) |
| 206 | /// Ex: |
| 207 | /// vector.transfer_write %v ... |
| 208 | /// permutation_map: (d0, d1, d2) -> (d2, d0, d1) |
| 209 | /// into: |
| 210 | /// %tmp = vector.transpose %v, [2, 0, 1] |
| 211 | /// vector.transfer_write %tmp ... |
| 212 | /// permutation_map: (d0, d1, d2) -> (d0, d1, d2) |
| 213 | /// |
| 214 | /// vector.transfer_write %v ... |
| 215 | /// permutation_map: (d0, d1, d2, d3) -> (d3, d2) |
| 216 | /// into: |
| 217 | /// %tmp = vector.transpose %v, [1, 0] |
| 218 | /// %v = vector.transfer_write %tmp ... |
| 219 | /// permutation_map: (d0, d1, d2, d3) -> (d2, d3) |
| 220 | /// |
| 221 | /// [TransferOpReduceRank] |
| 222 | /// Lower transfer_read op with broadcast in the leading dimensions into |
| 223 | /// transfer_read of lower rank + vector.broadcast. |
| 224 | /// Ex: vector.transfer_read ... |
| 225 | /// permutation_map: (d0, d1, d2, d3) -> (0, d1, 0, d3) |
| 226 | /// into: |
| 227 | /// %v = vector.transfer_read ... |
| 228 | /// permutation_map: (d0, d1, d2, d3) -> (d1, 0, d3) |
| 229 | /// vector.broadcast %v |
| 230 | void populateVectorTransferPermutationMapLoweringPatterns( |
| 231 | RewritePatternSet &patterns, PatternBenefit benefit = 1); |
| 232 | |
| 233 | /// Populate the pattern set with the following patterns: |
| 234 | /// |
| 235 | /// [ScanToArithOps] |
| 236 | /// Convert vector.scan op into arith ops and vector.insert_strided_slice / |
| 237 | /// vector.extract_strided_slice. |
| 238 | void populateVectorScanLoweringPatterns(RewritePatternSet &patterns, |
| 239 | PatternBenefit benefit = 1); |
| 240 | |
| 241 | /// Populate the pattern set with the following patterns: |
| 242 | /// |
| 243 | /// [StepToArithConstantOp] |
| 244 | /// Convert a non-scalable `vector.step` into an `arith.constant`. `index`-typed |
| 245 | /// steps are materialized using `indexBitwidth` as the index bitwidth; |
| 246 | /// an `indexBitwidth` of 0 leaves them untouched. `indexBitwidth` must not |
| 247 | /// exceed `IndexType::kInternalStorageBitWidth`. |
| 248 | /// |
| 249 | /// NOTE: `indexBitwidth` defaults to 64 for backwards compatibility - this |
| 250 | /// coincides with the index type bitwidth that this pattern used to assume |
| 251 | /// unconditionally before the parameter was introduced. The index type is not |
| 252 | /// always 64-bit wide though (e.g. it can be lowered to 32-bit on some |
| 253 | /// targets), so this default is merely a safe historical choice. In the future |
| 254 | /// this default should be removed and callers should pass the appropriate index |
| 255 | /// bitwidth explicitly. |
| 256 | void populateVectorStepLoweringPatterns(RewritePatternSet &patterns, |
| 257 | unsigned indexBitwidth = 64, |
| 258 | PatternBenefit benefit = 1); |
| 259 | |
| 260 | /// Populate the pattern set with the following patterns: |
| 261 | /// |
| 262 | /// [UnrollGather] |
| 263 | /// Unrolls 2 or more dimensional `vector.gather` ops by unrolling the |
| 264 | /// outermost dimension. |
| 265 | void populateVectorGatherLoweringPatterns(RewritePatternSet &patterns, |
| 266 | PatternBenefit benefit = 1); |
| 267 | |
| 268 | /// Populate the pattern set with the following patterns: |
| 269 | /// |
| 270 | /// [Gather1DToConditionalLoads] |
| 271 | /// Turns 1-d `vector.gather` into a scalarized sequence of `vector.loads` or |
| 272 | /// `tensor.extract`s. To avoid out-of-bounds memory accesses, these |
| 273 | /// loads/extracts are made conditional using `scf.if` ops. |
| 274 | void populateVectorGatherToConditionalLoadPatterns(RewritePatternSet &patterns, |
| 275 | PatternBenefit benefit = 1); |
| 276 | |
| 277 | /// Populates instances of `MaskOpRewritePattern` to lower masked operations |
| 278 | /// with `vector.mask`. Patterns should rewrite the `vector.mask` operation and |
| 279 | /// not its nested `MaskableOpInterface`. |
| 280 | void populateVectorMaskLoweringPatternsForSideEffectingOps( |
| 281 | RewritePatternSet &patterns); |
| 282 | |
| 283 | /// Populate the pattern set with the following patterns: |
| 284 | /// |
| 285 | /// [VectorMaskedLoadOpConverter] |
| 286 | /// Turns vector.maskedload to scf.if + memref.load |
| 287 | /// |
| 288 | /// [VectorMaskedStoreOpConverter] |
| 289 | /// Turns vector.maskedstore to scf.if + memref.store |
| 290 | void populateVectorMaskedLoadStoreEmulationPatterns(RewritePatternSet &patterns, |
| 291 | PatternBenefit benefit = 1); |
| 292 | |
| 293 | /// Populate the pattern set with the following patterns: |
| 294 | /// |
| 295 | /// [UnrollInterleaveOp] |
| 296 | /// A one-shot unrolling of InterleaveOp to (one or more) ExtractOp + |
| 297 | /// InterleaveOp (of `targetRank`) + InsertOp. |
| 298 | void populateVectorInterleaveLoweringPatterns(RewritePatternSet &patterns, |
| 299 | int64_t targetRank = 1, |
| 300 | PatternBenefit benefit = 1); |
| 301 | |
| 302 | void populateVectorInterleaveToShufflePatterns(RewritePatternSet &patterns, |
| 303 | PatternBenefit benefit = 1); |
| 304 | |
| 305 | void populateVectorDeinterleaveToShufflePatterns(RewritePatternSet &patterns, |
| 306 | PatternBenefit benefit = 1); |
| 307 | |
| 308 | /// Populates the pattern set with the following patterns: |
| 309 | /// |
| 310 | /// [UnrollBitCastOp] |
| 311 | /// A one-shot unrolling of BitCastOp to (one or more) ExtractOp + |
| 312 | /// BitCastOp (of `targetRank`) + InsertOp. |
| 313 | void populateVectorBitCastLoweringPatterns(RewritePatternSet &patterns, |
| 314 | int64_t targetRank = 1, |
| 315 | PatternBenefit benefit = 1); |
| 316 | |
| 317 | void populateVectorShuffleLoweringPatterns(RewritePatternSet &patterns, |
| 318 | PatternBenefit benefit = 1); |
| 319 | |
| 320 | /// Populates a pattern that rank-reduces n-D FMAs into (n-1)-D FMAs where |
| 321 | /// n > 1. |
| 322 | void populateVectorRankReducingFMAPattern(RewritePatternSet &patterns); |
| 323 | |
| 324 | /// Populate patterns to rewrite sequences of `vector.to_elements` + |
| 325 | /// `vector.from_elements` operations into a tree of `vector.shuffle` |
| 326 | /// operations. |
| 327 | void populateVectorToFromElementsToShuffleTreePatterns( |
| 328 | RewritePatternSet &patterns, PatternBenefit benefit = 1); |
| 329 | |
| 330 | /// Populate the pattern set with the following patterns: |
| 331 | /// |
| 332 | /// [ContractionOpToMatmulOpLowering] |
| 333 | /// Lowers `vector.contract` to `llvm.intr.matrix.multiply`. |
| 334 | /// |
| 335 | /// Given the high benefit, this will be prioriotised over other |
| 336 | /// contract-lowering patterns. As such, the convert-vector-to-llvm pass will |
| 337 | /// only run this registration conditionally. |
| 338 | void populateVectorContractToMatrixMultiply(RewritePatternSet &patterns, |
| 339 | PatternBenefit benefit = 100); |
| 340 | |
| 341 | /// Populate the pattern set with the following patterns: |
| 342 | /// |
| 343 | /// [TransposeOpLowering] |
| 344 | /// Lowers `vector.transpose` to `llvm.intr.matrix.flat_transpose`. |
| 345 | /// |
| 346 | /// Given the high benefit, this will be prioriotised over other |
| 347 | /// transpose-lowering patterns. As such, the convert-vector-to-llvm pass will |
| 348 | /// only run this registration conditionally. |
| 349 | void populateVectorTransposeToFlatTranspose(RewritePatternSet &patterns, |
| 350 | PatternBenefit benefit = 100); |
| 351 | |
| 352 | } // namespace vector |
| 353 | } // namespace mlir |
| 354 | |
| 355 | #endif // MLIR_DIALECT_VECTOR_TRANSFORMS_LOWERINGPATTERNS_H |
| 356 | |