1//===- TestVectorTransforms.cpp - Test Vector transforms and lowerings ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include <optional>
10
11#include "mlir/Analysis/SliceAnalysis.h"
12#include "mlir/Dialect/Affine/IR/AffineDialect.h"
13#include "mlir/Dialect/Arith/IR/Arith.h"
14#include "mlir/Dialect/Func/IR/FuncOps.h"
15#include "mlir/Dialect/GPU/IR/GPUDialect.h"
16#include "mlir/Dialect/Linalg/Passes.h"
17#include "mlir/Dialect/MemRef/IR/MemRef.h"
18#include "mlir/Dialect/NVGPU/IR/NVGPUDialectDecl.h"
19#include "mlir/Dialect/SCF/IR/SCF.h"
20#include "mlir/Dialect/Tensor/IR/Tensor.h"
21#include "mlir/Dialect/Vector/IR/VectorOps.h"
22#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
23#include "mlir/Dialect/Vector/Transforms/VectorDistribution.h"
24#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
25#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
26#include "mlir/Pass/Pass.h"
27#include "mlir/Pass/PassManager.h"
28#include "mlir/Support/LLVM.h"
29#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
30
31using namespace mlir;
32using namespace mlir::linalg;
33using namespace mlir::vector;
34
35namespace {
36
37struct TestVectorToVectorLowering
38 : public PassWrapper<TestVectorToVectorLowering,
39 OperationPass<func::FuncOp>> {
40 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorToVectorLowering)
41
42 TestVectorToVectorLowering() = default;
43 TestVectorToVectorLowering(const TestVectorToVectorLowering &pass)
44 : PassWrapper(pass) {}
45 StringRef getArgument() const final {
46 return "test-vector-to-vector-lowering";
47 }
48 StringRef getDescription() const final {
49 return "Test lowering patterns between ops in the vector dialect";
50 }
51
52 void getDependentDialects(DialectRegistry &registry) const override {
53 registry.insert<affine::AffineDialect>();
54 registry.insert<vector::VectorDialect>();
55 }
56
57 Option<bool> unroll{*this, "unroll", llvm::cl::desc("Include unrolling"),
58 llvm::cl::init(Val: false)};
59
60 void runOnOperation() override {
61 auto *ctx = &getContext();
62 RewritePatternSet patterns(ctx);
63 if (unroll) {
64 populateVectorUnrollPatterns(
65 patterns,
66 options: UnrollVectorOptions().setNativeShapeFn(getShape).setFilterConstraint(
67 filter));
68 }
69 populateVectorToVectorCanonicalizationPatterns(patterns);
70 populateBubbleVectorBitCastOpPatterns(patterns);
71 populateCastAwayVectorLeadingOneDimPatterns(patterns);
72 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
73 }
74
75private:
76 // Return the target shape based on op type.
77 static std::optional<SmallVector<int64_t>> getShape(Operation *op) {
78 if (isa<arith::AddFOp, arith::SelectOp, arith::CmpFOp>(Val: op))
79 return SmallVector<int64_t>(2, 2);
80 if (isa<vector::ContractionOp>(Val: op))
81 return SmallVector<int64_t>(3, 2);
82 // For transfer ops, just propagate the shape coming from
83 // InsertStridedSlices/ExtractStridedSlices.
84 if (auto readOp = dyn_cast<vector::TransferReadOp>(Val: op)) {
85 VectorType dstVec;
86 for (Operation *users : readOp->getUsers()) {
87 auto extract = dyn_cast<ExtractStridedSliceOp>(Val: users);
88 if (!extract)
89 return std::nullopt;
90 auto vecType = cast<VectorType>(Val: extract.getResult().getType());
91 if (dstVec && dstVec != vecType)
92 return std::nullopt;
93 dstVec = vecType;
94 }
95 return SmallVector<int64_t>(dstVec.getShape());
96 }
97 if (auto writeOp = dyn_cast<vector::TransferWriteOp>(Val: op)) {
98 auto insert = writeOp.getVector().getDefiningOp<InsertStridedSliceOp>();
99 if (!insert)
100 return std::nullopt;
101 ArrayRef<int64_t> shape = insert.getSourceVectorType().getShape();
102 return SmallVector<int64_t>(shape);
103 }
104 return std::nullopt;
105 }
106
107 static LogicalResult filter(Operation *op) {
108 return success(IsSuccess: isa<arith::AddFOp, arith::SelectOp, arith::CmpFOp,
109 ContractionOp, TransferReadOp, TransferWriteOp>(Val: op));
110 }
111};
112
113struct TestVectorContractionPrepareForMMTLowering
114 : public PassWrapper<TestVectorContractionPrepareForMMTLowering,
115 OperationPass<func::FuncOp>> {
116 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
117 TestVectorContractionPrepareForMMTLowering)
118
119 StringRef getArgument() const final {
120 return "test-vector-contraction-prepare-for-mmt-lowering";
121 }
122 StringRef getDescription() const final {
123 return "Test vector.contract matmul canonicalization for MMT lowering.";
124 }
125 TestVectorContractionPrepareForMMTLowering() = default;
126
127 void getDependentDialects(DialectRegistry &registry) const override {
128 registry.insert<affine::AffineDialect, arith::ArithDialect,
129 vector::VectorDialect>();
130 }
131
132 void runOnOperation() override {
133 MLIRContext *ctx = &getContext();
134 RewritePatternSet patterns(ctx);
135 vector::populateVectorContractCanonicalizeMatmulToMMT(patterns);
136 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
137 }
138};
139
140struct TestVectorUnrollingPatterns
141 : public PassWrapper<TestVectorUnrollingPatterns,
142 OperationPass<func::FuncOp>> {
143 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorUnrollingPatterns)
144
145 StringRef getArgument() const final {
146 return "test-vector-unrolling-patterns";
147 }
148 StringRef getDescription() const final {
149 return "Test lowering patterns to unroll contract ops in the vector "
150 "dialect";
151 }
152
153 void getDependentDialects(DialectRegistry &registry) const override {
154 registry.insert<vector::VectorDialect>();
155 }
156
157 TestVectorUnrollingPatterns() = default;
158 TestVectorUnrollingPatterns(const TestVectorUnrollingPatterns &pass)
159 : PassWrapper(pass) {}
160 void runOnOperation() override {
161 MLIRContext *ctx = &getContext();
162 RewritePatternSet patterns(ctx);
163 populateVectorUnrollPatterns(
164 patterns,
165 options: UnrollVectorOptions()
166 .setFilterConstraint([](Operation *op) {
167 return success(
168 IsSuccess: isa<arith::AddFOp, vector::FMAOp, vector::MultiDimReductionOp,
169 vector::BroadcastOp, vector::LoadOp, vector::StoreOp>(
170 Val: op));
171 }));
172 populateVectorUnrollPatterns(
173 patterns, options: UnrollVectorOptions()
174 .setFilterConstraint([](Operation *op) {
175 return success(IsSuccess: isa<vector::ReductionOp>(Val: op));
176 }));
177 populateVectorUnrollPatterns(patterns,
178 options: UnrollVectorOptions()
179 .setFilterConstraint([](Operation *op) {
180 return success(IsSuccess: isa<vector::StepOp>(Val: op));
181 }));
182 populateVectorUnrollPatterns(
183 patterns,
184 options: UnrollVectorOptions()
185 .setFilterConstraint([](Operation *op) {
186 return success(
187 IsSuccess: isa<vector::CreateMaskOp, vector::ConstantMaskOp>(Val: op));
188 }));
189 populateVectorUnrollPatterns(
190 patterns,
191 options: UnrollVectorOptions()
192 .setFilterConstraint([](Operation *op) {
193 return success(IsSuccess: isa<vector::ShapeCastOp>(Val: op));
194 }));
195 populateVectorUnrollPatterns(
196 patterns, options: UnrollVectorOptions()
197 .setFilterConstraint([](Operation *op) {
198 return success(IsSuccess: isa<vector::TransposeOp>(Val: op));
199 }));
200 populateVectorUnrollPatterns(
201 patterns, options: UnrollVectorOptions()
202 .setFilterConstraint([](Operation *op) {
203 return success(IsSuccess: isa<vector::BitCastOp>(Val: op));
204 }));
205 populateVectorUnrollPatterns(
206 patterns, options: UnrollVectorOptions()
207 .setFilterConstraint([](Operation *op) {
208 return success(IsSuccess: isa<vector::InterleaveOp>(Val: op));
209 }));
210 populateVectorUnrollPatterns(
211 patterns, options: UnrollVectorOptions()
212 .setFilterConstraint([](Operation *op) {
213 return success(IsSuccess: isa<vector::DeinterleaveOp>(Val: op));
214 }));
215
216 if (unrollBasedOnType) {
217 UnrollVectorOptions opts;
218 opts.setFilterConstraint(
219 [](Operation *op) { return success(IsSuccess: isa<ContractionOp>(Val: op)); });
220
221 if (!unrollOrder.empty()) {
222 opts.setUnrollTraversalOrderFn(
223 [this](Operation *op) -> std::optional<SmallVector<int64_t>> {
224 vector::ContractionOp contractOp =
225 cast<vector::ContractionOp>(Val: op);
226 if (contractOp.getIteratorTypes().size() == unrollOrder.size())
227 return SmallVector<int64_t>(unrollOrder.begin(),
228 unrollOrder.end());
229 return std::nullopt;
230 });
231 }
232 populateVectorUnrollPatterns(patterns, options: opts);
233 } else {
234 populateVectorUnrollPatterns(patterns,
235 options: UnrollVectorOptions()
236 .setFilterConstraint([](Operation *op) {
237 return success(IsSuccess: isa<ContractionOp>(Val: op));
238 }));
239 }
240 populateVectorToVectorCanonicalizationPatterns(patterns);
241 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
242 }
243
244 ListOption<int64_t> unrollOrder{*this, "unroll-order",
245 llvm::cl::desc("set the unroll order")};
246
247 Option<bool> unrollBasedOnType{
248 *this, "unroll-based-on-type",
249 llvm::cl::desc("Set the unroll factor based on type of the operation"),
250 llvm::cl::init(Val: false)};
251};
252
253struct TestVectorTransferUnrollingPatterns
254 : public PassWrapper<TestVectorTransferUnrollingPatterns,
255 OperationPass<func::FuncOp>> {
256 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
257 TestVectorTransferUnrollingPatterns)
258
259 TestVectorTransferUnrollingPatterns() = default;
260 TestVectorTransferUnrollingPatterns(
261 const TestVectorTransferUnrollingPatterns &pass)
262 : PassWrapper(pass) {}
263
264 void getDependentDialects(DialectRegistry &registry) const override {
265 registry.insert<affine::AffineDialect>();
266 }
267 StringRef getArgument() const final {
268 return "test-vector-transfer-unrolling-patterns";
269 }
270 StringRef getDescription() const final {
271 return "Test lowering patterns to unroll transfer ops in the vector "
272 "dialect";
273 }
274 void runOnOperation() override {
275 MLIRContext *ctx = &getContext();
276 RewritePatternSet patterns(ctx);
277 UnrollVectorOptions opts;
278 opts.setNativeShape(ArrayRef<int64_t>{2, 2})
279 .setFilterConstraint([](Operation *op) {
280 return success(IsSuccess: isa<vector::TransferReadOp, vector::TransferWriteOp,
281 vector::GatherOp>(Val: op));
282 });
283 if (reverseUnrollOrder.getValue()) {
284 opts.setUnrollTraversalOrderFn(
285 [](Operation *op) -> std::optional<SmallVector<int64_t>> {
286 int64_t numLoops = 0;
287 if (auto readOp = dyn_cast<vector::TransferReadOp>(Val: op))
288 numLoops = readOp.getVectorType().getRank();
289 else if (auto writeOp = dyn_cast<vector::TransferWriteOp>(Val: op))
290 numLoops = writeOp.getVectorType().getRank();
291 else if (auto gatherOp = dyn_cast<vector::GatherOp>(Val: op))
292 numLoops = gatherOp.getVectorType().getRank();
293 else
294 return std::nullopt;
295 auto order = llvm::reverse(C: llvm::seq<int64_t>(Begin: 0, End: numLoops));
296 return llvm::to_vector(Range&: order);
297 });
298 }
299 populateVectorUnrollPatterns(patterns, options: opts);
300 populateVectorToVectorCanonicalizationPatterns(patterns);
301 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
302 }
303
304 Option<bool> reverseUnrollOrder{
305 *this, "reverse-unroll-order",
306 llvm::cl::desc(
307 "reverse the order of unrolling of vector transfer operations"),
308 llvm::cl::init(Val: false)};
309};
310
311struct TestScalarVectorTransferLoweringPatterns
312 : public PassWrapper<TestScalarVectorTransferLoweringPatterns,
313 OperationPass<func::FuncOp>> {
314 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
315 TestScalarVectorTransferLoweringPatterns)
316
317 TestScalarVectorTransferLoweringPatterns() = default;
318 TestScalarVectorTransferLoweringPatterns(
319 const TestScalarVectorTransferLoweringPatterns &pass)
320 : PassWrapper(pass) {}
321
322 StringRef getArgument() const final {
323 return "test-scalar-vector-transfer-lowering";
324 }
325 StringRef getDescription() const final {
326 return "Test lowering of scalar vector transfers to memref loads/stores.";
327 }
328
329 void getDependentDialects(DialectRegistry &registry) const override {
330 registry.insert<affine::AffineDialect, memref::MemRefDialect,
331 tensor::TensorDialect, vector::VectorDialect>();
332 }
333
334 Option<bool> allowMultipleUses{
335 *this, "allow-multiple-uses",
336 llvm::cl::desc("Fold transfer operations with multiple uses"),
337 llvm::cl::init(Val: false)};
338
339 void runOnOperation() override {
340 MLIRContext *ctx = &getContext();
341 RewritePatternSet patterns(ctx);
342 vector::populateScalarVectorTransferLoweringPatterns(
343 patterns, /*benefit=*/1, allowMultipleUses: allowMultipleUses.getValue());
344 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
345 }
346};
347
348struct TestVectorTransferOpt
349 : public PassWrapper<TestVectorTransferOpt, OperationPass<func::FuncOp>> {
350 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorTransferOpt)
351
352 StringRef getArgument() const final { return "test-vector-transferop-opt"; }
353 StringRef getDescription() const final {
354 return "Test optimization transformations for transfer ops";
355 }
356 void runOnOperation() override {
357 IRRewriter rewriter(&getContext());
358 transferOpflowOpt(rewriter, rootOp: getOperation());
359 }
360};
361
362struct TestVectorSinkPatterns
363 : public PassWrapper<TestVectorSinkPatterns, OperationPass<func::FuncOp>> {
364 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorSinkPatterns)
365
366 TestVectorSinkPatterns() = default;
367 TestVectorSinkPatterns(const TestVectorSinkPatterns &pass) = default;
368
369 void getDependentDialects(DialectRegistry &registry) const override {
370 registry.insert<memref::MemRefDialect, affine::AffineDialect>();
371 }
372
373 StringRef getArgument() const final { return "test-vector-sink-patterns"; }
374
375 StringRef getDescription() const final {
376 return "Test lowering patterns that eliminate redundant broadcast "
377 "and transpose operations.";
378 }
379
380 void runOnOperation() override {
381 RewritePatternSet patterns(&getContext());
382 populateSinkVectorOpsPatterns(patterns);
383 populateSinkVectorMemOpsPatterns(patterns);
384 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
385 }
386};
387
388struct TestVectorReduceToContractPatternsPatterns
389 : public PassWrapper<TestVectorReduceToContractPatternsPatterns,
390 OperationPass<func::FuncOp>> {
391 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
392 TestVectorReduceToContractPatternsPatterns)
393
394 TestVectorReduceToContractPatternsPatterns() = default;
395 TestVectorReduceToContractPatternsPatterns(
396 const TestVectorReduceToContractPatternsPatterns &pass)
397 : PassWrapper(pass) {}
398 StringRef getArgument() const final {
399 return "test-vector-reduction-to-contract-patterns";
400 }
401 StringRef getDescription() const final {
402 return "Test patterns to convert multireduce op to contract and combine "
403 "broadcast/transpose to contract";
404 }
405
406 Option<bool> allowFastMathFlagLoss{
407 *this, "allow-fast-math-flag-loss",
408 llvm::cl::desc("Convert even if the contract would drop fast-math "
409 "flags of the multiply or the reduction"),
410 llvm::cl::init(Val: true)};
411
412 void runOnOperation() override {
413 RewritePatternSet patterns(&getContext());
414 populateVectorReductionToContractPatterns(patterns, /*benefit=*/1,
415 allowFastMathFlagLoss);
416 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
417 }
418};
419
420struct TestVectorChainedReductionFoldingPatterns
421 : public PassWrapper<TestVectorChainedReductionFoldingPatterns,
422 OperationPass<func::FuncOp>> {
423 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
424 TestVectorChainedReductionFoldingPatterns)
425
426 StringRef getArgument() const final {
427 return "test-vector-chained-reduction-folding-patterns";
428 }
429 StringRef getDescription() const final {
430 return "Test patterns to fold chained vector reductions";
431 }
432 void runOnOperation() override {
433 RewritePatternSet patterns(&getContext());
434 populateChainedVectorReductionFoldingPatterns(patterns);
435 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
436 }
437};
438
439struct TestVectorBreakDownReductionPatterns
440 : public PassWrapper<TestVectorBreakDownReductionPatterns,
441 OperationPass<func::FuncOp>> {
442 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
443 TestVectorBreakDownReductionPatterns)
444
445 StringRef getArgument() const final {
446 return "test-vector-break-down-reduction-patterns";
447 }
448 StringRef getDescription() const final {
449 return "Test patterns to break down vector reductions into arith "
450 "reductions";
451 }
452 void runOnOperation() override {
453 RewritePatternSet patterns(&getContext());
454 populateBreakDownVectorReductionPatterns(patterns,
455 /*maxNumElementsToExtract=*/2);
456 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
457 }
458};
459
460struct TestFlattenVectorTransferPatterns
461 : public PassWrapper<TestFlattenVectorTransferPatterns,
462 OperationPass<func::FuncOp>> {
463 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
464 TestFlattenVectorTransferPatterns)
465
466 TestFlattenVectorTransferPatterns() = default;
467 TestFlattenVectorTransferPatterns(
468 const TestFlattenVectorTransferPatterns &pass)
469 : PassWrapper(pass) {}
470
471 StringRef getArgument() const final {
472 return "test-vector-transfer-flatten-patterns";
473 }
474
475 StringRef getDescription() const final {
476 return "Test patterns to rewrite contiguous row-major N-dimensional "
477 "vector.transfer_{read,write} ops into 1D transfers";
478 }
479
480 void getDependentDialects(DialectRegistry &registry) const override {
481 registry.insert<memref::MemRefDialect>();
482 registry.insert<affine::AffineDialect>();
483 registry.insert<vector::VectorDialect>();
484 }
485
486 Option<unsigned> targetVectorBitwidth{
487 *this, "target-vector-bitwidth",
488 llvm::cl::desc(
489 "Minimum vector bitwidth to enable the flattening transformation. "
490 "For scalable vectors this is the base size, i.e. the size "
491 "corresponding to vscale=1."),
492 llvm::cl::init(Val: std::numeric_limits<unsigned>::max())};
493
494 void runOnOperation() override {
495 RewritePatternSet patterns(&getContext());
496 populateFlattenVectorTransferPatterns(patterns, targetVectorBitwidth);
497 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
498 }
499};
500
501struct TestVectorScanLowering
502 : public PassWrapper<TestVectorScanLowering, OperationPass<func::FuncOp>> {
503 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorScanLowering)
504
505 StringRef getArgument() const final { return "test-vector-scan-lowering"; }
506 StringRef getDescription() const final {
507 return "Test lowering patterns that lower the scan op in the vector "
508 "dialect";
509 }
510 void runOnOperation() override {
511 RewritePatternSet patterns(&getContext());
512 populateVectorScanLoweringPatterns(patterns);
513 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
514 }
515};
516
517/// Allocate shared memory for a single warp to test lowering of
518/// WarpExecuteOnLane0Op.
519static Value allocateGlobalSharedMemory(Location loc, OpBuilder &builder,
520 gpu::WarpExecuteOnLane0Op warpOp,
521 Type type) {
522 static constexpr int64_t kSharedMemorySpace = 3;
523 // Compute type of shared memory buffer.
524 MemRefType memrefType;
525 if (auto vectorType = dyn_cast<VectorType>(Val&: type)) {
526 memrefType =
527 MemRefType::get(shape: vectorType.getShape(), elementType: vectorType.getElementType(), map: {},
528 memorySpaceInd: kSharedMemorySpace);
529 } else {
530 memrefType = MemRefType::get(shape: {1}, elementType: type, map: {}, memorySpaceInd: kSharedMemorySpace);
531 }
532
533 // Get symbol table holding all shared memory globals.
534 ModuleOp moduleOp = warpOp->getParentOfType<ModuleOp>();
535 SymbolTable symbolTable(moduleOp);
536
537 // Create a pretty name.
538 SmallString<64> buf;
539 llvm::raw_svector_ostream os(buf);
540 interleave(c: memrefType.getShape(), os, separator: "x");
541 os << "x" << memrefType.getElementType();
542 std::string symbolName = (Twine("__shared_") + os.str()).str();
543
544 auto ip = builder.saveInsertionPoint();
545 builder.setInsertionPoint(moduleOp);
546 auto global = memref::GlobalOp::create(
547 builder, location: loc,
548 /*sym_name=*/symbolName,
549 /*sym_visibility=*/builder.getStringAttr(bytes: "private"),
550 /*type=*/memrefType,
551 /*initial_value=*/Attribute(),
552 /*constant=*/false,
553 /*alignment=*/IntegerAttr());
554 symbolTable.insert(symbol: global);
555 // The symbol table inserts at the end of the module, but globals are a bit
556 // nicer if they are at the beginning.
557 global->moveBefore(existingOp: &moduleOp.front());
558
559 builder.restoreInsertionPoint(ip);
560 return memref::GetGlobalOp::create(builder, location: loc, result: memrefType, name: symbolName);
561}
562
563static Value warpReduction(Location loc, OpBuilder &builder, Value input,
564 CombiningKind kind, uint32_t size) {
565 // First reduce on a single thread to get per lane reduction value.
566 Value laneVal = vector::ReductionOp::create(builder, location: loc, kind, vector: input);
567 // Parallel reduction using butterfly shuffles.
568 for (uint64_t i = 1; i < size; i <<= 1) {
569 Value shuffled = gpu::ShuffleOp::create(builder, location: loc, value: laneVal, offset: i,
570 /*width=*/size,
571 /*mode=*/gpu::ShuffleMode::XOR)
572 .getShuffleResult();
573 laneVal = makeArithReduction(b&: builder, loc, kind, v1: laneVal, acc: shuffled);
574 }
575 return laneVal;
576}
577
578struct TestVectorDistribution
579 : public PassWrapper<TestVectorDistribution, OperationPass<func::FuncOp>> {
580 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorDistribution)
581
582 void getDependentDialects(DialectRegistry &registry) const override {
583 registry
584 .insert<vector::VectorDialect, scf::SCFDialect, memref::MemRefDialect,
585 gpu::GPUDialect, affine::AffineDialect>();
586 }
587
588 StringRef getArgument() const final { return "test-vector-warp-distribute"; }
589 StringRef getDescription() const final {
590 return "Test vector warp distribute transformation and lowering patterns";
591 }
592 TestVectorDistribution() = default;
593 TestVectorDistribution(const TestVectorDistribution &pass)
594 : PassWrapper(pass) {}
595
596 Option<bool> warpOpToSCF{
597 *this, "rewrite-warp-ops-to-scf-if",
598 llvm::cl::desc("Lower vector.warp_execute_on_lane0 to scf.if op"),
599 llvm::cl::init(Val: false)};
600
601 Option<bool> distributeTransferWriteOps{
602 *this, "distribute-transfer-write",
603 llvm::cl::desc("Test distribution of transfer write"),
604 llvm::cl::init(Val: false)};
605
606 Option<unsigned> maxTransferWriteElements{
607 *this, "max-transfer-write-elements",
608 llvm::cl::desc("Maximum number of transfer write elements to distribute"),
609 llvm::cl::init(Val: 1)};
610
611 Option<bool> hoistUniform{*this, "hoist-uniform",
612 llvm::cl::desc("Test hoist uniform"),
613 llvm::cl::init(Val: false)};
614
615 Option<bool> propagateDistribution{
616 *this, "propagate-distribution",
617 llvm::cl::desc("Test distribution propagation"), llvm::cl::init(Val: false)};
618
619 void runOnOperation() override {
620 RewritePatternSet patterns(&getContext());
621
622 getOperation().walk(callback: [&](Operation *op) {
623 if (auto warpOp = dyn_cast<gpu::WarpExecuteOnLane0Op>(Val: op)) {
624 if (hoistUniform) {
625 moveScalarUniformCode(op: warpOp);
626 }
627 WalkResult::interrupt();
628 }
629 });
630 MLIRContext *ctx = &getContext();
631 auto distributionFn = [](Value val) {
632 // Create an identity dim map of the same rank as the vector.
633 VectorType vecType = dyn_cast<VectorType>(Val: val.getType());
634 int64_t vecRank = vecType ? vecType.getRank() : 0;
635 OpBuilder builder(val.getContext());
636 if (vecRank == 0)
637 return AffineMap::get(context: val.getContext());
638 return AffineMap::getMultiDimIdentityMap(numDims: vecRank, context: val.getContext());
639 };
640 auto shuffleFn = [](Location loc, OpBuilder &builder, Value val,
641 Value srcIdx, int64_t warpSz) {
642 assert((val.getType().isF32() || val.getType().isInteger(32)) &&
643 "unsupported shuffle type");
644 Type i32Type = builder.getIntegerType(width: 32);
645 Value srcIdxI32 =
646 arith::IndexCastOp::create(builder, location: loc, out: i32Type, in: srcIdx);
647 Value warpSzI32 = arith::ConstantOp::create(
648 builder, location: loc, value: builder.getIntegerAttr(type: i32Type, value: warpSz));
649 Value result = gpu::ShuffleOp::create(builder, location: loc, value: val, offset: srcIdxI32,
650 width: warpSzI32, mode: gpu::ShuffleMode::IDX)
651 .getResult(i: 0);
652 return result;
653 };
654 if (distributeTransferWriteOps && propagateDistribution) {
655 RewritePatternSet patterns(ctx);
656 vector::populatePropagateWarpVectorDistributionPatterns(
657 pattern&: patterns, distributionMapFn: distributionFn, warpShuffleFromIdxFn: shuffleFn, /*benefit=*/1,
658 /*readBenefit=*/0);
659 vector::populateDistributeReduction(pattern&: patterns, distributedReductionFn: warpReduction, benefit: 1);
660 populateDistributeTransferWriteOpPatterns(patterns, distributionMapFn: distributionFn, maxNumElementsToExtract: 2);
661 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
662 } else if (distributeTransferWriteOps) {
663 RewritePatternSet patterns(ctx);
664 populateDistributeTransferWriteOpPatterns(patterns, distributionMapFn: distributionFn,
665 maxNumElementsToExtract: maxTransferWriteElements);
666 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
667 } else if (propagateDistribution) {
668 RewritePatternSet patterns(ctx);
669 vector::populatePropagateWarpVectorDistributionPatterns(
670 pattern&: patterns, distributionMapFn: distributionFn, warpShuffleFromIdxFn: shuffleFn);
671 vector::populateDistributeReduction(pattern&: patterns, distributedReductionFn: warpReduction);
672 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
673 }
674 WarpExecuteOnLane0LoweringOptions options;
675 options.warpAllocationFn = allocateGlobalSharedMemory;
676 options.warpSynchronizationFn = [](Location loc, OpBuilder &builder,
677 gpu::WarpExecuteOnLane0Op warpOp) {
678 gpu::BarrierOp::create(builder, location: loc, addressSpace: gpu::AddressSpace::Workgroup);
679 };
680 // Test on one pattern in isolation.
681 if (warpOpToSCF) {
682 populateWarpExecuteOnLane0OpToScfForPattern(patterns, options);
683 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
684 return;
685 }
686 }
687};
688
689struct TestVectorExtractStridedSliceLowering
690 : public PassWrapper<TestVectorExtractStridedSliceLowering,
691 OperationPass<func::FuncOp>> {
692 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
693 TestVectorExtractStridedSliceLowering)
694
695 StringRef getArgument() const final {
696 return "test-vector-extract-strided-slice-lowering";
697 }
698 StringRef getDescription() const final {
699 return "Test lowering patterns that converts vector.extract_strided_slice "
700 "into a chain of vector.extract and vector.insert ops";
701 }
702 void runOnOperation() override {
703 RewritePatternSet patterns(&getContext());
704 populateVectorExtractStridedSliceToExtractInsertChainPatterns(patterns);
705 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
706 }
707};
708
709struct TestVectorBreakDownBitCast
710 : public PassWrapper<TestVectorBreakDownBitCast,
711 OperationPass<func::FuncOp>> {
712 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorBreakDownBitCast)
713
714 StringRef getArgument() const final {
715 return "test-vector-break-down-bitcast";
716 }
717 StringRef getDescription() const final {
718 return "Test pattern that breaks down vector.bitcast ops ";
719 }
720 void runOnOperation() override {
721 RewritePatternSet patterns(&getContext());
722 populateBreakDownVectorBitCastOpPatterns(patterns, controlFn: [](BitCastOp op) {
723 return op.getSourceVectorType().getShape().back() > 4;
724 });
725 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
726 }
727};
728
729struct TestCreateVectorBroadcast
730 : public PassWrapper<TestCreateVectorBroadcast,
731 OperationPass<func::FuncOp>> {
732 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestCreateVectorBroadcast)
733
734 StringRef getArgument() const final { return "test-create-vector-broadcast"; }
735 StringRef getDescription() const final {
736 return "Test optimization transformations for transfer ops";
737 }
738 void getDependentDialects(DialectRegistry &registry) const override {
739 registry.insert<vector::VectorDialect>();
740 }
741
742 void runOnOperation() override {
743 getOperation()->walk(callback: [](Operation *op) {
744 if (op->getName().getStringRef() != "test.create_broadcast")
745 return;
746 auto targetShape =
747 cast<VectorType>(Val: op->getResult(idx: 0).getType()).getShape();
748 auto arrayAttr =
749 cast<DenseI64ArrayAttr>(Val: op->getDiscardableAttr(name: "broadcast_dims"))
750 .asArrayRef();
751 llvm::SetVector<int64_t> broadcastedDims;
752 broadcastedDims.insert_range(R&: arrayAttr);
753 OpBuilder b(op);
754 Value bcast = vector::BroadcastOp::createOrFoldBroadcastOp(
755 b, value: op->getOperand(idx: 0), dstShape: targetShape, broadcastedDims);
756 op->getResult(idx: 0).replaceAllUsesWith(newValue: bcast);
757 op->erase();
758 });
759 }
760};
761
762struct TestVectorGatherLowering
763 : public PassWrapper<TestVectorGatherLowering,
764 OperationPass<func::FuncOp>> {
765 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorGatherLowering)
766
767 StringRef getArgument() const final { return "test-vector-gather-lowering"; }
768 StringRef getDescription() const final {
769 return "Test patterns that lower the gather op in the vector conditional "
770 "loads";
771 }
772 void getDependentDialects(DialectRegistry &registry) const override {
773 registry.insert<affine::AffineDialect, arith::ArithDialect,
774 func::FuncDialect, memref::MemRefDialect, scf::SCFDialect,
775 tensor::TensorDialect, vector::VectorDialect>();
776 }
777
778 void runOnOperation() override {
779 RewritePatternSet patterns(&getContext());
780 populateVectorGatherLoweringPatterns(patterns);
781 populateVectorGatherToConditionalLoadPatterns(patterns);
782 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
783 }
784};
785
786struct TestFoldArithExtensionIntoVectorContractPatterns
787 : public PassWrapper<TestFoldArithExtensionIntoVectorContractPatterns,
788 OperationPass<func::FuncOp>> {
789 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(
790 TestFoldArithExtensionIntoVectorContractPatterns)
791
792 StringRef getArgument() const final {
793 return "test-fold-arith-extf-into-vector-contract-patterns";
794 }
795 StringRef getDescription() const final {
796 return "Test patterns that fold arithmetic extension ops into vector "
797 "contract ops";
798 }
799
800 void getDependentDialects(DialectRegistry &registry) const override {
801 registry.insert<arith::ArithDialect, func::FuncDialect, nvgpu::NVGPUDialect,
802 memref::MemRefDialect, scf::SCFDialect,
803 tensor::TensorDialect, vector::VectorDialect>();
804 }
805
806 void runOnOperation() override {
807 RewritePatternSet patterns(&getContext());
808 populateFoldArithExtensionPatterns(patterns);
809 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
810 }
811};
812
813struct TestVectorEmulateMaskedLoadStore final
814 : public PassWrapper<TestVectorEmulateMaskedLoadStore,
815 OperationPass<func::FuncOp>> {
816 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorEmulateMaskedLoadStore)
817
818 StringRef getArgument() const override {
819 return "test-vector-emulate-masked-load-store";
820 }
821 StringRef getDescription() const override {
822 return "Test patterns that emulate the maskedload/maskedstore op by "
823 " memref.load/store and scf.if";
824 }
825 void getDependentDialects(DialectRegistry &registry) const override {
826 registry
827 .insert<arith::ArithDialect, func::FuncDialect, memref::MemRefDialect,
828 scf::SCFDialect, vector::VectorDialect>();
829 }
830
831 void runOnOperation() override {
832 RewritePatternSet patterns(&getContext());
833 populateVectorMaskedLoadStoreEmulationPatterns(patterns);
834 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
835 }
836};
837
838struct TestVectorShuffleLowering
839 : public PassWrapper<TestVectorShuffleLowering,
840 OperationPass<func::FuncOp>> {
841 MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(TestVectorShuffleLowering)
842
843 StringRef getArgument() const final { return "test-vector-shuffle-lowering"; }
844 StringRef getDescription() const final {
845 return "Test lowering patterns for vector.shuffle with mixed-size inputs";
846 }
847 void runOnOperation() override {
848 RewritePatternSet patterns(&getContext());
849 populateVectorShuffleLoweringPatterns(patterns);
850 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
851 }
852};
853} // namespace
854
855namespace mlir {
856namespace test {
857void registerTestVectorLowerings() {
858 PassRegistration<TestVectorToVectorLowering>();
859
860 PassRegistration<TestVectorContractionPrepareForMMTLowering>();
861
862 PassRegistration<TestVectorUnrollingPatterns>();
863
864 PassRegistration<TestVectorTransferUnrollingPatterns>();
865
866 PassRegistration<TestScalarVectorTransferLoweringPatterns>();
867
868 PassRegistration<TestVectorTransferOpt>();
869
870 PassRegistration<TestVectorSinkPatterns>();
871
872 PassRegistration<TestVectorReduceToContractPatternsPatterns>();
873
874 PassRegistration<TestVectorChainedReductionFoldingPatterns>();
875
876 PassRegistration<TestVectorBreakDownReductionPatterns>();
877
878 PassRegistration<TestFlattenVectorTransferPatterns>();
879
880 PassRegistration<TestVectorScanLowering>();
881
882 PassRegistration<TestVectorShuffleLowering>();
883
884 PassRegistration<TestVectorDistribution>();
885
886 PassRegistration<TestVectorExtractStridedSliceLowering>();
887
888 PassRegistration<TestVectorBreakDownBitCast>();
889
890 PassRegistration<TestCreateVectorBroadcast>();
891
892 PassRegistration<TestVectorGatherLowering>();
893
894 PassRegistration<TestFoldArithExtensionIntoVectorContractPatterns>();
895
896 PassRegistration<TestVectorEmulateMaskedLoadStore>();
897}
898} // namespace test
899} // namespace mlir
900