1//===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h"
10
11#include "mlir/Analysis/DataLayoutAnalysis.h"
12#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
13#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
14#include "mlir/Dialect/Arith/IR/Arith.h"
15#include "mlir/Dialect/ArmNeon/ArmNeonDialect.h"
16#include "mlir/Dialect/ArmNeon/Transforms.h"
17#include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h"
18#include "mlir/Dialect/ArmSVE/Transforms/Transforms.h"
19#include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h"
20#include "mlir/Dialect/MemRef/IR/MemRef.h"
21#include "mlir/Dialect/Tensor/IR/Tensor.h"
22#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
23#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
24#include "mlir/Dialect/X86/Transforms.h"
25#include "mlir/Dialect/X86/X86Dialect.h"
26#include "mlir/Pass/Pass.h"
27#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
28
29namespace mlir {
30#define GEN_PASS_DEF_CONVERTVECTORTOLLVMPASS
31#include "mlir/Conversion/Passes.h.inc"
32} // namespace mlir
33
34using namespace mlir;
35using namespace mlir::vector;
36
37namespace {
38struct ConvertVectorToLLVMPass
39 : public impl::ConvertVectorToLLVMPassBase<ConvertVectorToLLVMPass> {
40
41 using Base::Base;
42
43 // Override explicitly to allow conditional dialect dependence.
44 void getDependentDialects(DialectRegistry &registry) const override {
45 registry.insert<LLVM::LLVMDialect>();
46 registry.insert<arith::ArithDialect>();
47 registry.insert<memref::MemRefDialect>();
48 registry.insert<tensor::TensorDialect>();
49 if (armNeon)
50 registry.insert<arm_neon::ArmNeonDialect>();
51 if (armSVE)
52 registry.insert<arm_sve::ArmSVEDialect>();
53 if (x86)
54 registry.insert<x86::X86Dialect>();
55 }
56 void runOnOperation() override;
57};
58} // namespace
59
60void ConvertVectorToLLVMPass::runOnOperation() {
61 // Perform progressive lowering of operations on slices and all contraction
62 // operations. Also materializes masks, lowers vector.step, rank-reduces FMA,
63 // applies folding and DCE.
64 {
65 RewritePatternSet patterns(&getContext());
66 populateVectorToVectorCanonicalizationPatterns(patterns);
67 populateVectorBitCastLoweringPatterns(patterns);
68 populateVectorBroadcastLoweringPatterns(patterns);
69 populateVectorContractLoweringPatterns(patterns, vectorContractLoweringOption: vectorContractLowering);
70 if (vectorContractLowering == vector::VectorContractLowering::LLVMIntr) {
71 // This pattern creates a dependency on the LLVM dialect, hence we don't
72 // include it in `populateVectorContractLoweringPatterns` that is part of
73 // the Vector dialect (and should not depend on LLVM).
74 populateVectorContractToMatrixMultiply(patterns);
75 }
76 populateVectorMaskOpLoweringPatterns(patterns);
77 populateVectorShapeCastLoweringPatterns(patterns);
78 populateVectorInterleaveLoweringPatterns(patterns);
79 populateVectorTransposeLoweringPatterns(patterns, vectorTransposeLowering);
80 if (vectorTransposeLowering == vector::VectorTransposeLowering::LLVMIntr) {
81 // This pattern creates a dependency on the LLVM dialect, hence we don't
82 // include it in `populateVectorTransposeLoweringPatterns` that is part of
83 // the Vector dialect (and should not depend on LLVM).
84 populateVectorTransposeToFlatTranspose(patterns);
85 }
86 // Vector transfer ops with rank > 1 should be lowered with VectorToSCF.
87 populateVectorTransferLoweringPatterns(patterns, /*maxTransferRank=*/1);
88 populateVectorMaskMaterializationPatterns(patterns,
89 force32BitVectorIndices);
90 populateVectorInsertExtractStridedSliceTransforms(patterns);
91 populateVectorRankReducingFMAPattern(patterns);
92 populateVectorGatherLoweringPatterns(patterns);
93 populateVectorFromElementsUnrollPatterns(patterns);
94 populateVectorToElementsUnrollPatterns(patterns);
95 if (armI8MM) {
96 if (armNeon)
97 arm_neon::populateLowerContractionToNeonI8MMPatterns(patterns);
98 if (armSVE)
99 arm_sve::populateLowerContractionToSVEI8MMPatterns(patterns);
100 }
101 if (armBF16) {
102 if (armNeon)
103 arm_neon::populateLowerContractionToNeonBFMMLAPatterns(patterns);
104 if (armSVE)
105 arm_sve::populateLowerContractionToSVEBFMMLAPatterns(patterns);
106 }
107 (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns));
108 }
109
110 // Convert to the LLVM IR dialect.
111 const auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
112 LowerToLLVMOptions options(&getContext(),
113 dataLayoutAnalysis.getAtOrAbove(operation: getOperation()));
114 LLVMTypeConverter converter(&getContext(), options, &dataLayoutAnalysis);
115 RewritePatternSet patterns(&getContext());
116 populateVectorTransferLoweringPatterns(patterns);
117 populateVectorToLLVMConversionPatterns(
118 converter, patterns, reassociateFPReductions, force32BitVectorIndices,
119 useVectorAlignment, enableGEPInboundsNuw);
120
121 // Architecture specific augmentations.
122 LLVMConversionTarget target(getContext());
123 target.addLegalDialect<arith::ArithDialect>();
124 target.addLegalDialect<memref::MemRefDialect>();
125 target.addLegalOp<UnrealizedConversionCastOp>();
126
127 if (armNeon) {
128 // TODO: we may or may not want to include in-dialect lowering to
129 // LLVM-compatible operations here. So far, all operations in the dialect
130 // can be translated to LLVM IR so there is no conversion necessary.
131 target.addLegalDialect<arm_neon::ArmNeonDialect>();
132 }
133 if (armSVE) {
134 configureArmSVELegalizeForExportTarget(target);
135 populateArmSVELegalizeForLLVMExportPatterns(converter, patterns);
136 }
137 if (x86) {
138 configureX86LegalizeForExportTarget(target);
139 populateX86LegalizeForLLVMExportPatterns(converter, patterns);
140 }
141
142 if (failed(
143 Result: applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns))))
144 signalPassFailure();
145}
146