| 1 | //===- VectorTransformOps.cpp - Implementation of Vector transform ops ----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.h" |
| 10 | |
| 11 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 12 | #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h" |
| 13 | #include "mlir/Conversion/VectorToSCF/VectorToSCF.h" |
| 14 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 15 | #include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h" |
| 16 | #include "mlir/Dialect/Transform/IR/TransformDialect.h" |
| 17 | #include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h" |
| 18 | #include "mlir/Dialect/Vector/IR/VectorOps.h" |
| 19 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
| 20 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
| 21 | #include "mlir/Dialect/Vector/Transforms/VectorTransforms.h" |
| 22 | #include "mlir/Dialect/X86/Transforms.h" |
| 23 | |
| 24 | using namespace mlir; |
| 25 | using namespace mlir::vector; |
| 26 | using namespace mlir::transform; |
| 27 | |
| 28 | //===----------------------------------------------------------------------===// |
| 29 | // Apply...ConversionPatternsOp |
| 30 | //===----------------------------------------------------------------------===// |
| 31 | |
| 32 | void transform::ApplyVectorToLLVMConversionPatternsOp::populatePatterns( |
| 33 | TypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 34 | populateVectorToLLVMConversionPatterns( |
| 35 | converter: static_cast<LLVMTypeConverter &>(typeConverter), patterns, |
| 36 | reassociateFPReductions: getReassociateFpReductions(), force32BitVectorIndices: getForce_32bitVectorIndices(), |
| 37 | useVectorAlignment: getUseVectorAlignment()); |
| 38 | } |
| 39 | |
| 40 | LogicalResult |
| 41 | transform::ApplyVectorToLLVMConversionPatternsOp::verifyTypeConverter( |
| 42 | transform::TypeConverterBuilderOpInterface builder) { |
| 43 | if (builder.getTypeConverterType() != "LLVMTypeConverter" ) |
| 44 | return emitOpError(message: "expected LLVMTypeConverter" ); |
| 45 | return success(); |
| 46 | } |
| 47 | |
| 48 | void transform::ApplyVectorLinearizeConversionPatternsOp::populatePatterns( |
| 49 | TypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 50 | ConversionTarget target(*getContext()); |
| 51 | vector::populateVectorLinearizeBasePatterns(typeConverter, target, patterns); |
| 52 | if (getShuffleLikeOps()) |
| 53 | vector::populateVectorLinearizeShuffleLikeOpsPatterns(typeConverter, target, |
| 54 | patterns); |
| 55 | } |
| 56 | |
| 57 | void transform::ApplyVectorLinearizeConversionPatternsOp:: |
| 58 | populateConversionTargetRules(const TypeConverter &typeConverter, |
| 59 | ConversionTarget &conversionTarget) { |
| 60 | // The legality rules need a mutable converter; they keep their own copy. |
| 61 | TypeConverter linearizer; |
| 62 | vector::populateForVectorLinearize(typeConverter&: linearizer, conversionTarget); |
| 63 | if (std::optional<int64_t> bitwidth = getTargetVectorBitwidth()) |
| 64 | vector::populateVectorLinearizeBitWidthTargetRules(conversionTarget, |
| 65 | targetBitWidth: *bitwidth); |
| 66 | } |
| 67 | |
| 68 | LogicalResult |
| 69 | transform::ApplyVectorLinearizeConversionPatternsOp::verifyTypeConverter( |
| 70 | transform::TypeConverterBuilderOpInterface builder) { |
| 71 | if (builder.getTypeConverterType() != |
| 72 | VectorLinearizeTypeConverterOp::getTypeConverterType()) |
| 73 | return emitOpError(message: "expected VectorLinearizeTypeConverter" ); |
| 74 | return success(); |
| 75 | } |
| 76 | |
| 77 | std::unique_ptr<TypeConverter> |
| 78 | transform::VectorLinearizeTypeConverterOp::getTypeConverter() { |
| 79 | auto typeConverter = std::make_unique<TypeConverter>(); |
| 80 | ConversionTarget target(*getContext()); |
| 81 | vector::populateForVectorLinearize(typeConverter&: *typeConverter, conversionTarget&: target); |
| 82 | return typeConverter; |
| 83 | } |
| 84 | |
| 85 | StringRef transform::VectorLinearizeTypeConverterOp::getTypeConverterType() { |
| 86 | return "VectorLinearizeTypeConverter" ; |
| 87 | } |
| 88 | |
| 89 | //===----------------------------------------------------------------------===// |
| 90 | // Apply...PatternsOp |
| 91 | //===----------------------------------------------------------------------===// |
| 92 | |
| 93 | void transform::ApplyCastAwayVectorLeadingOneDimPatternsOp::populatePatterns( |
| 94 | RewritePatternSet &patterns) { |
| 95 | vector::populateCastAwayVectorLeadingOneDimPatterns(patterns); |
| 96 | } |
| 97 | |
| 98 | void transform::ApplyFoldArithExtensionPatternsOp::populatePatterns( |
| 99 | RewritePatternSet &patterns) { |
| 100 | vector::populateFoldArithExtensionPatterns(patterns); |
| 101 | } |
| 102 | |
| 103 | void transform::ApplyFoldElementwiseToVectorPatternsOp::populatePatterns( |
| 104 | RewritePatternSet &patterns) { |
| 105 | vector::populateElementwiseToVectorOpsPatterns(patterns); |
| 106 | } |
| 107 | |
| 108 | void transform::ApplyVectorReductionToContractPatternsOp::populatePatterns( |
| 109 | RewritePatternSet &patterns) { |
| 110 | vector::populateVectorReductionToContractPatterns(patterns); |
| 111 | } |
| 112 | |
| 113 | void transform::ApplyLowerCreateMaskPatternsOp::populatePatterns( |
| 114 | RewritePatternSet &patterns) { |
| 115 | vector::populateVectorMaskOpLoweringPatterns(patterns); |
| 116 | } |
| 117 | |
| 118 | void transform::ApplyRankReducingSubviewPatternsOp::populatePatterns( |
| 119 | RewritePatternSet &patterns) { |
| 120 | vector::populateVectorTransferDropUnitDimsPatterns(patterns); |
| 121 | } |
| 122 | |
| 123 | void transform::ApplyTransferPermutationPatternsOp::populatePatterns( |
| 124 | RewritePatternSet &patterns) { |
| 125 | vector::populateVectorTransferPermutationMapLoweringPatterns(patterns); |
| 126 | } |
| 127 | |
| 128 | void transform::ApplyDropUnitDimWithShapeCastPatternsOp::populatePatterns( |
| 129 | RewritePatternSet &patterns) { |
| 130 | vector::populateDropUnitDimWithShapeCastPatterns(patterns); |
| 131 | } |
| 132 | |
| 133 | void transform::ApplyDropInnerMostUnitDimsFromXferOpsPatternsOp:: |
| 134 | populatePatterns(RewritePatternSet &patterns) { |
| 135 | vector::populateDropInnerMostUnitDimsXferOpPatterns(patterns); |
| 136 | } |
| 137 | |
| 138 | void transform::ApplyLowerBitCastPatternsOp::populatePatterns( |
| 139 | RewritePatternSet &patterns) { |
| 140 | vector::populateVectorBitCastLoweringPatterns(patterns); |
| 141 | } |
| 142 | |
| 143 | void transform::ApplyLowerBroadcastPatternsOp::populatePatterns( |
| 144 | RewritePatternSet &patterns) { |
| 145 | populateVectorBroadcastLoweringPatterns(patterns); |
| 146 | } |
| 147 | |
| 148 | void transform::ApplyLowerContractionPatternsOp::populatePatterns( |
| 149 | RewritePatternSet &patterns) { |
| 150 | populateVectorContractLoweringPatterns(patterns, vectorContractLoweringOption: getLoweringStrategy(), |
| 151 | /*benefit=*/1, |
| 152 | /*disableOuterProductLowering=*/true); |
| 153 | } |
| 154 | |
| 155 | void transform::ApplyLowerMasksPatternsOp::populatePatterns( |
| 156 | RewritePatternSet &patterns) { |
| 157 | populateVectorMaskOpLoweringPatterns(patterns); |
| 158 | } |
| 159 | |
| 160 | void transform::ApplyLowerMaskedTransfersPatternsOp::populatePatterns( |
| 161 | RewritePatternSet &patterns) { |
| 162 | populateVectorMaskLoweringPatternsForSideEffectingOps(patterns); |
| 163 | } |
| 164 | |
| 165 | void transform::ApplyMaterializeMasksPatternsOp::populatePatterns( |
| 166 | RewritePatternSet &patterns) { |
| 167 | populateVectorMaskMaterializationPatterns(patterns, |
| 168 | /*force32BitVectorIndices=*/false); |
| 169 | } |
| 170 | |
| 171 | //===----------------------------------------------------------------------===// |
| 172 | // Multi-reduction patterns |
| 173 | //===----------------------------------------------------------------------===// |
| 174 | void transform::ApplyReorderMultiReductionPatternsOp::populatePatterns( |
| 175 | RewritePatternSet &patterns) { |
| 176 | vector::VectorTransformsOptions vectorTransformOptions; |
| 177 | vectorTransformOptions.setVectorMultiReductionLowering(getLoweringStrategy()); |
| 178 | vector::populateVectorMultiReductionReorderPatterns( |
| 179 | patterns, options: vectorTransformOptions.vectorMultiReductionLowering); |
| 180 | } |
| 181 | |
| 182 | void transform::ApplyMultiReductionFlatteningPatternsOp::populatePatterns( |
| 183 | RewritePatternSet &patterns) { |
| 184 | vector::VectorTransformsOptions vectorTransformOptions; |
| 185 | vectorTransformOptions.setVectorMultiReductionLowering(getLoweringStrategy()); |
| 186 | vector::populateVectorMultiReductionFlatteningPatterns( |
| 187 | patterns, options: vectorTransformOptions.vectorMultiReductionLowering); |
| 188 | } |
| 189 | |
| 190 | void transform::ApplyMultiReductionUnrollingPatternsOp::populatePatterns( |
| 191 | RewritePatternSet &patterns) { |
| 192 | vector::VectorTransformsOptions vectorTransformOptions; |
| 193 | vectorTransformOptions.setVectorMultiReductionLowering(getLoweringStrategy()); |
| 194 | vector::populateVectorMultiReductionUnrollingPatterns( |
| 195 | patterns, options: vectorTransformOptions.vectorMultiReductionLowering); |
| 196 | } |
| 197 | |
| 198 | void transform::ApplyLowerOuterProductPatternsOp::populatePatterns( |
| 199 | RewritePatternSet &patterns) { |
| 200 | populateVectorOuterProductLoweringPatterns(patterns); |
| 201 | } |
| 202 | |
| 203 | void transform::ApplyLowerGatherPatternsOp::populatePatterns( |
| 204 | RewritePatternSet &patterns) { |
| 205 | vector::populateVectorGatherLoweringPatterns(patterns); |
| 206 | } |
| 207 | |
| 208 | void transform::ApplyUnrollFromElementsPatternsOp::populatePatterns( |
| 209 | RewritePatternSet &patterns) { |
| 210 | vector::populateVectorFromElementsUnrollPatterns(patterns); |
| 211 | } |
| 212 | |
| 213 | void transform::ApplyUnrollContractPatternsOp::populatePatterns( |
| 214 | RewritePatternSet &patterns) { |
| 215 | int64_t m = getM(); |
| 216 | int64_t n = getN(); |
| 217 | int64_t k = getK(); |
| 218 | |
| 219 | auto nativeShapeFn = |
| 220 | [m, n, k](Operation *op) -> std::optional<SmallVector<int64_t>> { |
| 221 | return SmallVector<int64_t>({m, n, k}); |
| 222 | }; |
| 223 | |
| 224 | auto constraint = [](Operation *op) { |
| 225 | auto contract = dyn_cast<vector::ContractionOp>(Val: op); |
| 226 | if (!contract) |
| 227 | return failure(); |
| 228 | |
| 229 | auto iterators = contract.getIteratorTypesArray(); |
| 230 | if (iterators.size() != 3 || |
| 231 | iterators[0] != vector::IteratorType::parallel || |
| 232 | iterators[1] != vector::IteratorType::parallel || |
| 233 | iterators[2] != vector::IteratorType::reduction) |
| 234 | return failure(); |
| 235 | |
| 236 | auto type = dyn_cast<VectorType>(Val: contract.getType()); |
| 237 | if (!type || type.getRank() != 2) |
| 238 | return failure(); |
| 239 | return success(); |
| 240 | }; |
| 241 | |
| 242 | vector::populateVectorContractUnrollPatterns( |
| 243 | patterns, options: vector::UnrollVectorOptions() |
| 244 | .setNativeShapeFn(nativeShapeFn) |
| 245 | .setFilterConstraint(constraint)); |
| 246 | } |
| 247 | |
| 248 | void transform::ApplyUnrollToElementsPatternsOp::populatePatterns( |
| 249 | RewritePatternSet &patterns) { |
| 250 | vector::populateVectorToElementsUnrollPatterns(patterns); |
| 251 | } |
| 252 | |
| 253 | void transform::ApplyLowerScanPatternsOp::populatePatterns( |
| 254 | RewritePatternSet &patterns) { |
| 255 | vector::populateVectorScanLoweringPatterns(patterns); |
| 256 | } |
| 257 | |
| 258 | void transform::ApplyLowerShapeCastPatternsOp::populatePatterns( |
| 259 | RewritePatternSet &patterns) { |
| 260 | vector::populateVectorShapeCastLoweringPatterns(patterns); |
| 261 | } |
| 262 | |
| 263 | void transform::ApplyLowerStepPatternsOp::populatePatterns( |
| 264 | RewritePatternSet &patterns) { |
| 265 | vector::populateVectorStepLoweringPatterns(patterns, indexBitwidth: getIndexBitwidth()); |
| 266 | } |
| 267 | |
| 268 | void transform::ApplyLowerTransferPatternsOp::populatePatterns( |
| 269 | RewritePatternSet &patterns) { |
| 270 | vector::populateVectorTransferLoweringPatterns(patterns, |
| 271 | maxTransferRank: getMaxTransferRank()); |
| 272 | } |
| 273 | |
| 274 | void transform::ApplyLowerTransposePatternsOp::populatePatterns( |
| 275 | RewritePatternSet &patterns) { |
| 276 | vector::populateVectorTransposeLoweringPatterns(patterns, |
| 277 | vectorTransposeLowering: getLoweringStrategy()); |
| 278 | if (getAvx2LoweringStrategy()) { |
| 279 | auto avx2LoweringOptions = x86::avx2::LoweringOptions().setTransposeOptions( |
| 280 | x86::avx2::TransposeLoweringOptions().lower4x8xf32(lower: true).lower8x8xf32( |
| 281 | lower: true)); |
| 282 | x86::avx2::populateSpecializedTransposeLoweringPatterns( |
| 283 | patterns, options: avx2LoweringOptions, /*benefit=*/10); |
| 284 | } |
| 285 | } |
| 286 | |
| 287 | void transform::ApplyLowerInterleavePatternsOp::populatePatterns( |
| 288 | RewritePatternSet &patterns) { |
| 289 | vector::populateVectorInterleaveLoweringPatterns(patterns); |
| 290 | } |
| 291 | |
| 292 | void transform::ApplyInterleaveAndDeinterleaveToShufflePatternsOp:: |
| 293 | populatePatterns(RewritePatternSet &patterns) { |
| 294 | vector::populateVectorInterleaveToShufflePatterns(patterns); |
| 295 | vector::populateVectorDeinterleaveToShufflePatterns(patterns); |
| 296 | } |
| 297 | |
| 298 | void transform::ApplyRewriteNarrowTypePatternsOp::populatePatterns( |
| 299 | RewritePatternSet &patterns) { |
| 300 | populateVectorNarrowTypeRewritePatterns(patterns); |
| 301 | populateVectorTransposeNarrowTypeRewritePatterns(patterns); |
| 302 | } |
| 303 | |
| 304 | void transform::ApplySplitTransferFullPartialPatternsOp::populatePatterns( |
| 305 | RewritePatternSet &patterns) { |
| 306 | vector::VectorTransformsOptions vectorTransformOptions; |
| 307 | vectorTransformOptions.setVectorTransferSplit(getSplitTransferStrategy()); |
| 308 | populateVectorTransferFullPartialPatterns(patterns, options: vectorTransformOptions); |
| 309 | } |
| 310 | |
| 311 | void transform::ApplyTransferToScfPatternsOp::populatePatterns( |
| 312 | RewritePatternSet &patterns) { |
| 313 | VectorTransferToSCFOptions vectorTransferToSCFOptions = |
| 314 | VectorTransferToSCFOptions() |
| 315 | .enableFullUnroll(u: getFullUnroll()) |
| 316 | .setTargetRank(getMaxTransferRank()); |
| 317 | populateVectorToSCFConversionPatterns(patterns, options: vectorTransferToSCFOptions); |
| 318 | } |
| 319 | |
| 320 | void transform::ApplySinkVectorPatternsOp::populatePatterns( |
| 321 | RewritePatternSet &patterns) { |
| 322 | vector::populateSinkVectorOpsPatterns(patterns); |
| 323 | } |
| 324 | |
| 325 | void transform::ApplySinkVectorMemPatternsOp::populatePatterns( |
| 326 | RewritePatternSet &patterns) { |
| 327 | vector::populateSinkVectorMemOpsPatterns(patterns); |
| 328 | } |
| 329 | |
| 330 | void transform::ApplyFlattenVectorTransferOpsPatternsOp::populatePatterns( |
| 331 | RewritePatternSet &patterns) { |
| 332 | vector::populateFlattenVectorTransferPatterns(patterns, |
| 333 | targetVectorBitwidth: getTargetVectorBitwidth()); |
| 334 | } |
| 335 | |
| 336 | //===----------------------------------------------------------------------===// |
| 337 | // Transform op registration |
| 338 | //===----------------------------------------------------------------------===// |
| 339 | |
| 340 | namespace { |
| 341 | /// Registers new ops and declares PDL as dependent dialect since the additional |
| 342 | /// ops are using PDL types for operands and results. |
| 343 | class VectorTransformDialectExtension |
| 344 | : public transform::TransformDialectExtension< |
| 345 | VectorTransformDialectExtension> { |
| 346 | public: |
| 347 | MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(VectorTransformDialectExtension) |
| 348 | |
| 349 | VectorTransformDialectExtension() { |
| 350 | declareGeneratedDialect<vector::VectorDialect>(); |
| 351 | declareGeneratedDialect<arith::ArithDialect>(); |
| 352 | declareGeneratedDialect<LLVM::LLVMDialect>(); |
| 353 | registerTransformOps< |
| 354 | #define GET_OP_LIST |
| 355 | #include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.cpp.inc" |
| 356 | >(); |
| 357 | } |
| 358 | }; |
| 359 | } // namespace |
| 360 | |
| 361 | #define GET_OP_CLASSES |
| 362 | #include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.cpp.inc" |
| 363 | |
| 364 | void mlir::vector::registerTransformDialectExtension( |
| 365 | DialectRegistry ®istry) { |
| 366 | registry.addExtensions<VectorTransformDialectExtension>(); |
| 367 | } |
| 368 | |