1//===- VectorTransformOps.cpp - Implementation of Vector transform ops ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.h"
10
11#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
12#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
13#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
16#include "mlir/Dialect/Transform/IR/TransformDialect.h"
17#include "mlir/Dialect/Transform/Interfaces/TransformInterfaces.h"
18#include "mlir/Dialect/Vector/IR/VectorOps.h"
19#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
20#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
21#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
22#include "mlir/Dialect/X86/Transforms.h"
23
24using namespace mlir;
25using namespace mlir::vector;
26using namespace mlir::transform;
27
28//===----------------------------------------------------------------------===//
29// Apply...ConversionPatternsOp
30//===----------------------------------------------------------------------===//
31
32void transform::ApplyVectorToLLVMConversionPatternsOp::populatePatterns(
33 TypeConverter &typeConverter, RewritePatternSet &patterns) {
34 populateVectorToLLVMConversionPatterns(
35 converter: static_cast<LLVMTypeConverter &>(typeConverter), patterns,
36 reassociateFPReductions: getReassociateFpReductions(), force32BitVectorIndices: getForce_32bitVectorIndices(),
37 useVectorAlignment: getUseVectorAlignment());
38}
39
40LogicalResult
41transform::ApplyVectorToLLVMConversionPatternsOp::verifyTypeConverter(
42 transform::TypeConverterBuilderOpInterface builder) {
43 if (builder.getTypeConverterType() != "LLVMTypeConverter")
44 return emitOpError(message: "expected LLVMTypeConverter");
45 return success();
46}
47
48void transform::ApplyVectorLinearizeConversionPatternsOp::populatePatterns(
49 TypeConverter &typeConverter, RewritePatternSet &patterns) {
50 ConversionTarget target(*getContext());
51 vector::populateVectorLinearizeBasePatterns(typeConverter, target, patterns);
52 if (getShuffleLikeOps())
53 vector::populateVectorLinearizeShuffleLikeOpsPatterns(typeConverter, target,
54 patterns);
55}
56
57void transform::ApplyVectorLinearizeConversionPatternsOp::
58 populateConversionTargetRules(const TypeConverter &typeConverter,
59 ConversionTarget &conversionTarget) {
60 // The legality rules need a mutable converter; they keep their own copy.
61 TypeConverter linearizer;
62 vector::populateForVectorLinearize(typeConverter&: linearizer, conversionTarget);
63 if (std::optional<int64_t> bitwidth = getTargetVectorBitwidth())
64 vector::populateVectorLinearizeBitWidthTargetRules(conversionTarget,
65 targetBitWidth: *bitwidth);
66}
67
68LogicalResult
69transform::ApplyVectorLinearizeConversionPatternsOp::verifyTypeConverter(
70 transform::TypeConverterBuilderOpInterface builder) {
71 if (builder.getTypeConverterType() !=
72 VectorLinearizeTypeConverterOp::getTypeConverterType())
73 return emitOpError(message: "expected VectorLinearizeTypeConverter");
74 return success();
75}
76
77std::unique_ptr<TypeConverter>
78transform::VectorLinearizeTypeConverterOp::getTypeConverter() {
79 auto typeConverter = std::make_unique<TypeConverter>();
80 ConversionTarget target(*getContext());
81 vector::populateForVectorLinearize(typeConverter&: *typeConverter, conversionTarget&: target);
82 return typeConverter;
83}
84
85StringRef transform::VectorLinearizeTypeConverterOp::getTypeConverterType() {
86 return "VectorLinearizeTypeConverter";
87}
88
89//===----------------------------------------------------------------------===//
90// Apply...PatternsOp
91//===----------------------------------------------------------------------===//
92
93void transform::ApplyCastAwayVectorLeadingOneDimPatternsOp::populatePatterns(
94 RewritePatternSet &patterns) {
95 vector::populateCastAwayVectorLeadingOneDimPatterns(patterns);
96}
97
98void transform::ApplyFoldArithExtensionPatternsOp::populatePatterns(
99 RewritePatternSet &patterns) {
100 vector::populateFoldArithExtensionPatterns(patterns);
101}
102
103void transform::ApplyFoldElementwiseToVectorPatternsOp::populatePatterns(
104 RewritePatternSet &patterns) {
105 vector::populateElementwiseToVectorOpsPatterns(patterns);
106}
107
108void transform::ApplyVectorReductionToContractPatternsOp::populatePatterns(
109 RewritePatternSet &patterns) {
110 vector::populateVectorReductionToContractPatterns(patterns);
111}
112
113void transform::ApplyLowerCreateMaskPatternsOp::populatePatterns(
114 RewritePatternSet &patterns) {
115 vector::populateVectorMaskOpLoweringPatterns(patterns);
116}
117
118void transform::ApplyRankReducingSubviewPatternsOp::populatePatterns(
119 RewritePatternSet &patterns) {
120 vector::populateVectorTransferDropUnitDimsPatterns(patterns);
121}
122
123void transform::ApplyTransferPermutationPatternsOp::populatePatterns(
124 RewritePatternSet &patterns) {
125 vector::populateVectorTransferPermutationMapLoweringPatterns(patterns);
126}
127
128void transform::ApplyDropUnitDimWithShapeCastPatternsOp::populatePatterns(
129 RewritePatternSet &patterns) {
130 vector::populateDropUnitDimWithShapeCastPatterns(patterns);
131}
132
133void transform::ApplyDropInnerMostUnitDimsFromXferOpsPatternsOp::
134 populatePatterns(RewritePatternSet &patterns) {
135 vector::populateDropInnerMostUnitDimsXferOpPatterns(patterns);
136}
137
138void transform::ApplyLowerBitCastPatternsOp::populatePatterns(
139 RewritePatternSet &patterns) {
140 vector::populateVectorBitCastLoweringPatterns(patterns);
141}
142
143void transform::ApplyLowerBroadcastPatternsOp::populatePatterns(
144 RewritePatternSet &patterns) {
145 populateVectorBroadcastLoweringPatterns(patterns);
146}
147
148void transform::ApplyLowerContractionPatternsOp::populatePatterns(
149 RewritePatternSet &patterns) {
150 populateVectorContractLoweringPatterns(patterns, vectorContractLoweringOption: getLoweringStrategy(),
151 /*benefit=*/1,
152 /*disableOuterProductLowering=*/true);
153}
154
155void transform::ApplyLowerMasksPatternsOp::populatePatterns(
156 RewritePatternSet &patterns) {
157 populateVectorMaskOpLoweringPatterns(patterns);
158}
159
160void transform::ApplyLowerMaskedTransfersPatternsOp::populatePatterns(
161 RewritePatternSet &patterns) {
162 populateVectorMaskLoweringPatternsForSideEffectingOps(patterns);
163}
164
165void transform::ApplyMaterializeMasksPatternsOp::populatePatterns(
166 RewritePatternSet &patterns) {
167 populateVectorMaskMaterializationPatterns(patterns,
168 /*force32BitVectorIndices=*/false);
169}
170
171//===----------------------------------------------------------------------===//
172// Multi-reduction patterns
173//===----------------------------------------------------------------------===//
174void transform::ApplyReorderMultiReductionPatternsOp::populatePatterns(
175 RewritePatternSet &patterns) {
176 vector::VectorTransformsOptions vectorTransformOptions;
177 vectorTransformOptions.setVectorMultiReductionLowering(getLoweringStrategy());
178 vector::populateVectorMultiReductionReorderPatterns(
179 patterns, options: vectorTransformOptions.vectorMultiReductionLowering);
180}
181
182void transform::ApplyMultiReductionFlatteningPatternsOp::populatePatterns(
183 RewritePatternSet &patterns) {
184 vector::VectorTransformsOptions vectorTransformOptions;
185 vectorTransformOptions.setVectorMultiReductionLowering(getLoweringStrategy());
186 vector::populateVectorMultiReductionFlatteningPatterns(
187 patterns, options: vectorTransformOptions.vectorMultiReductionLowering);
188}
189
190void transform::ApplyMultiReductionUnrollingPatternsOp::populatePatterns(
191 RewritePatternSet &patterns) {
192 vector::VectorTransformsOptions vectorTransformOptions;
193 vectorTransformOptions.setVectorMultiReductionLowering(getLoweringStrategy());
194 vector::populateVectorMultiReductionUnrollingPatterns(
195 patterns, options: vectorTransformOptions.vectorMultiReductionLowering);
196}
197
198void transform::ApplyLowerOuterProductPatternsOp::populatePatterns(
199 RewritePatternSet &patterns) {
200 populateVectorOuterProductLoweringPatterns(patterns);
201}
202
203void transform::ApplyLowerGatherPatternsOp::populatePatterns(
204 RewritePatternSet &patterns) {
205 vector::populateVectorGatherLoweringPatterns(patterns);
206}
207
208void transform::ApplyUnrollFromElementsPatternsOp::populatePatterns(
209 RewritePatternSet &patterns) {
210 vector::populateVectorFromElementsUnrollPatterns(patterns);
211}
212
213void transform::ApplyUnrollContractPatternsOp::populatePatterns(
214 RewritePatternSet &patterns) {
215 int64_t m = getM();
216 int64_t n = getN();
217 int64_t k = getK();
218
219 auto nativeShapeFn =
220 [m, n, k](Operation *op) -> std::optional<SmallVector<int64_t>> {
221 return SmallVector<int64_t>({m, n, k});
222 };
223
224 auto constraint = [](Operation *op) {
225 auto contract = dyn_cast<vector::ContractionOp>(Val: op);
226 if (!contract)
227 return failure();
228
229 auto iterators = contract.getIteratorTypesArray();
230 if (iterators.size() != 3 ||
231 iterators[0] != vector::IteratorType::parallel ||
232 iterators[1] != vector::IteratorType::parallel ||
233 iterators[2] != vector::IteratorType::reduction)
234 return failure();
235
236 auto type = dyn_cast<VectorType>(Val: contract.getType());
237 if (!type || type.getRank() != 2)
238 return failure();
239 return success();
240 };
241
242 vector::populateVectorContractUnrollPatterns(
243 patterns, options: vector::UnrollVectorOptions()
244 .setNativeShapeFn(nativeShapeFn)
245 .setFilterConstraint(constraint));
246}
247
248void transform::ApplyUnrollToElementsPatternsOp::populatePatterns(
249 RewritePatternSet &patterns) {
250 vector::populateVectorToElementsUnrollPatterns(patterns);
251}
252
253void transform::ApplyLowerScanPatternsOp::populatePatterns(
254 RewritePatternSet &patterns) {
255 vector::populateVectorScanLoweringPatterns(patterns);
256}
257
258void transform::ApplyLowerShapeCastPatternsOp::populatePatterns(
259 RewritePatternSet &patterns) {
260 vector::populateVectorShapeCastLoweringPatterns(patterns);
261}
262
263void transform::ApplyLowerStepPatternsOp::populatePatterns(
264 RewritePatternSet &patterns) {
265 vector::populateVectorStepLoweringPatterns(patterns, indexBitwidth: getIndexBitwidth());
266}
267
268void transform::ApplyLowerTransferPatternsOp::populatePatterns(
269 RewritePatternSet &patterns) {
270 vector::populateVectorTransferLoweringPatterns(patterns,
271 maxTransferRank: getMaxTransferRank());
272}
273
274void transform::ApplyLowerTransposePatternsOp::populatePatterns(
275 RewritePatternSet &patterns) {
276 vector::populateVectorTransposeLoweringPatterns(patterns,
277 vectorTransposeLowering: getLoweringStrategy());
278 if (getAvx2LoweringStrategy()) {
279 auto avx2LoweringOptions = x86::avx2::LoweringOptions().setTransposeOptions(
280 x86::avx2::TransposeLoweringOptions().lower4x8xf32(lower: true).lower8x8xf32(
281 lower: true));
282 x86::avx2::populateSpecializedTransposeLoweringPatterns(
283 patterns, options: avx2LoweringOptions, /*benefit=*/10);
284 }
285}
286
287void transform::ApplyLowerInterleavePatternsOp::populatePatterns(
288 RewritePatternSet &patterns) {
289 vector::populateVectorInterleaveLoweringPatterns(patterns);
290}
291
292void transform::ApplyInterleaveAndDeinterleaveToShufflePatternsOp::
293 populatePatterns(RewritePatternSet &patterns) {
294 vector::populateVectorInterleaveToShufflePatterns(patterns);
295 vector::populateVectorDeinterleaveToShufflePatterns(patterns);
296}
297
298void transform::ApplyRewriteNarrowTypePatternsOp::populatePatterns(
299 RewritePatternSet &patterns) {
300 populateVectorNarrowTypeRewritePatterns(patterns);
301 populateVectorTransposeNarrowTypeRewritePatterns(patterns);
302}
303
304void transform::ApplySplitTransferFullPartialPatternsOp::populatePatterns(
305 RewritePatternSet &patterns) {
306 vector::VectorTransformsOptions vectorTransformOptions;
307 vectorTransformOptions.setVectorTransferSplit(getSplitTransferStrategy());
308 populateVectorTransferFullPartialPatterns(patterns, options: vectorTransformOptions);
309}
310
311void transform::ApplyTransferToScfPatternsOp::populatePatterns(
312 RewritePatternSet &patterns) {
313 VectorTransferToSCFOptions vectorTransferToSCFOptions =
314 VectorTransferToSCFOptions()
315 .enableFullUnroll(u: getFullUnroll())
316 .setTargetRank(getMaxTransferRank());
317 populateVectorToSCFConversionPatterns(patterns, options: vectorTransferToSCFOptions);
318}
319
320void transform::ApplySinkVectorPatternsOp::populatePatterns(
321 RewritePatternSet &patterns) {
322 vector::populateSinkVectorOpsPatterns(patterns);
323}
324
325void transform::ApplySinkVectorMemPatternsOp::populatePatterns(
326 RewritePatternSet &patterns) {
327 vector::populateSinkVectorMemOpsPatterns(patterns);
328}
329
330void transform::ApplyFlattenVectorTransferOpsPatternsOp::populatePatterns(
331 RewritePatternSet &patterns) {
332 vector::populateFlattenVectorTransferPatterns(patterns,
333 targetVectorBitwidth: getTargetVectorBitwidth());
334}
335
336//===----------------------------------------------------------------------===//
337// Transform op registration
338//===----------------------------------------------------------------------===//
339
340namespace {
341/// Registers new ops and declares PDL as dependent dialect since the additional
342/// ops are using PDL types for operands and results.
343class VectorTransformDialectExtension
344 : public transform::TransformDialectExtension<
345 VectorTransformDialectExtension> {
346public:
347 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(VectorTransformDialectExtension)
348
349 VectorTransformDialectExtension() {
350 declareGeneratedDialect<vector::VectorDialect>();
351 declareGeneratedDialect<arith::ArithDialect>();
352 declareGeneratedDialect<LLVM::LLVMDialect>();
353 registerTransformOps<
354#define GET_OP_LIST
355#include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.cpp.inc"
356 >();
357 }
358};
359} // namespace
360
361#define GET_OP_CLASSES
362#include "mlir/Dialect/Vector/TransformOps/VectorTransformOps.cpp.inc"
363
364void mlir::vector::registerTransformDialectExtension(
365 DialectRegistry &registry) {
366 registry.addExtensions<VectorTransformDialectExtension>();
367}
368