| 1 | //===- VectorToLLVM.cpp - Conversion from Vector to the LLVM dialect ------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h" |
| 10 | |
| 11 | #include "mlir/Analysis/DataLayoutAnalysis.h" |
| 12 | #include "mlir/Conversion/LLVMCommon/ConversionTarget.h" |
| 13 | #include "mlir/Conversion/LLVMCommon/TypeConverter.h" |
| 14 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 15 | #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" |
| 16 | #include "mlir/Dialect/ArmNeon/Transforms.h" |
| 17 | #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h" |
| 18 | #include "mlir/Dialect/ArmSVE/Transforms/Transforms.h" |
| 19 | #include "mlir/Dialect/LLVMIR/LLVMDialectDecl.h" |
| 20 | #include "mlir/Dialect/MemRef/IR/MemRef.h" |
| 21 | #include "mlir/Dialect/Tensor/IR/Tensor.h" |
| 22 | #include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h" |
| 23 | #include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h" |
| 24 | #include "mlir/Dialect/X86/Transforms.h" |
| 25 | #include "mlir/Dialect/X86/X86Dialect.h" |
| 26 | #include "mlir/Pass/Pass.h" |
| 27 | #include "mlir/Transforms/GreedyPatternRewriteDriver.h" |
| 28 | |
| 29 | namespace mlir { |
| 30 | #define GEN_PASS_DEF_CONVERTVECTORTOLLVMPASS |
| 31 | #include "mlir/Conversion/Passes.h.inc" |
| 32 | } // namespace mlir |
| 33 | |
| 34 | using namespace mlir; |
| 35 | using namespace mlir::vector; |
| 36 | |
| 37 | namespace { |
| 38 | struct ConvertVectorToLLVMPass |
| 39 | : public impl::ConvertVectorToLLVMPassBase<ConvertVectorToLLVMPass> { |
| 40 | |
| 41 | using Base::Base; |
| 42 | |
| 43 | // Override explicitly to allow conditional dialect dependence. |
| 44 | void getDependentDialects(DialectRegistry ®istry) const override { |
| 45 | registry.insert<LLVM::LLVMDialect>(); |
| 46 | registry.insert<arith::ArithDialect>(); |
| 47 | registry.insert<memref::MemRefDialect>(); |
| 48 | registry.insert<tensor::TensorDialect>(); |
| 49 | if (armNeon) |
| 50 | registry.insert<arm_neon::ArmNeonDialect>(); |
| 51 | if (armSVE) |
| 52 | registry.insert<arm_sve::ArmSVEDialect>(); |
| 53 | if (x86) |
| 54 | registry.insert<x86::X86Dialect>(); |
| 55 | } |
| 56 | void runOnOperation() override; |
| 57 | }; |
| 58 | } // namespace |
| 59 | |
| 60 | void ConvertVectorToLLVMPass::runOnOperation() { |
| 61 | // Perform progressive lowering of operations on slices and all contraction |
| 62 | // operations. Also materializes masks, lowers vector.step, rank-reduces FMA, |
| 63 | // applies folding and DCE. |
| 64 | { |
| 65 | RewritePatternSet patterns(&getContext()); |
| 66 | populateVectorToVectorCanonicalizationPatterns(patterns); |
| 67 | populateVectorBitCastLoweringPatterns(patterns); |
| 68 | populateVectorBroadcastLoweringPatterns(patterns); |
| 69 | populateVectorContractLoweringPatterns(patterns, vectorContractLoweringOption: vectorContractLowering); |
| 70 | if (vectorContractLowering == vector::VectorContractLowering::LLVMIntr) { |
| 71 | // This pattern creates a dependency on the LLVM dialect, hence we don't |
| 72 | // include it in `populateVectorContractLoweringPatterns` that is part of |
| 73 | // the Vector dialect (and should not depend on LLVM). |
| 74 | populateVectorContractToMatrixMultiply(patterns); |
| 75 | } |
| 76 | populateVectorMaskOpLoweringPatterns(patterns); |
| 77 | populateVectorShapeCastLoweringPatterns(patterns); |
| 78 | populateVectorInterleaveLoweringPatterns(patterns); |
| 79 | populateVectorTransposeLoweringPatterns(patterns, vectorTransposeLowering); |
| 80 | if (vectorTransposeLowering == vector::VectorTransposeLowering::LLVMIntr) { |
| 81 | // This pattern creates a dependency on the LLVM dialect, hence we don't |
| 82 | // include it in `populateVectorTransposeLoweringPatterns` that is part of |
| 83 | // the Vector dialect (and should not depend on LLVM). |
| 84 | populateVectorTransposeToFlatTranspose(patterns); |
| 85 | } |
| 86 | // Vector transfer ops with rank > 1 should be lowered with VectorToSCF. |
| 87 | populateVectorTransferLoweringPatterns(patterns, /*maxTransferRank=*/1); |
| 88 | populateVectorMaskMaterializationPatterns(patterns, |
| 89 | force32BitVectorIndices); |
| 90 | populateVectorInsertExtractStridedSliceTransforms(patterns); |
| 91 | populateVectorRankReducingFMAPattern(patterns); |
| 92 | populateVectorGatherLoweringPatterns(patterns); |
| 93 | populateVectorFromElementsUnrollPatterns(patterns); |
| 94 | populateVectorToElementsUnrollPatterns(patterns); |
| 95 | if (armI8MM) { |
| 96 | if (armNeon) |
| 97 | arm_neon::populateLowerContractionToNeonI8MMPatterns(patterns); |
| 98 | if (armSVE) |
| 99 | arm_sve::populateLowerContractionToSVEI8MMPatterns(patterns); |
| 100 | } |
| 101 | if (armBF16) { |
| 102 | if (armNeon) |
| 103 | arm_neon::populateLowerContractionToNeonBFMMLAPatterns(patterns); |
| 104 | if (armSVE) |
| 105 | arm_sve::populateLowerContractionToSVEBFMMLAPatterns(patterns); |
| 106 | } |
| 107 | (void)applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns)); |
| 108 | } |
| 109 | |
| 110 | // Convert to the LLVM IR dialect. |
| 111 | const auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>(); |
| 112 | LowerToLLVMOptions options(&getContext(), |
| 113 | dataLayoutAnalysis.getAtOrAbove(operation: getOperation())); |
| 114 | LLVMTypeConverter converter(&getContext(), options, &dataLayoutAnalysis); |
| 115 | RewritePatternSet patterns(&getContext()); |
| 116 | populateVectorTransferLoweringPatterns(patterns); |
| 117 | populateVectorToLLVMConversionPatterns( |
| 118 | converter, patterns, reassociateFPReductions, force32BitVectorIndices, |
| 119 | useVectorAlignment, enableGEPInboundsNuw); |
| 120 | |
| 121 | // Architecture specific augmentations. |
| 122 | LLVMConversionTarget target(getContext()); |
| 123 | target.addLegalDialect<arith::ArithDialect>(); |
| 124 | target.addLegalDialect<memref::MemRefDialect>(); |
| 125 | target.addLegalOp<UnrealizedConversionCastOp>(); |
| 126 | |
| 127 | if (armNeon) { |
| 128 | // TODO: we may or may not want to include in-dialect lowering to |
| 129 | // LLVM-compatible operations here. So far, all operations in the dialect |
| 130 | // can be translated to LLVM IR so there is no conversion necessary. |
| 131 | target.addLegalDialect<arm_neon::ArmNeonDialect>(); |
| 132 | } |
| 133 | if (armSVE) { |
| 134 | configureArmSVELegalizeForExportTarget(target); |
| 135 | populateArmSVELegalizeForLLVMExportPatterns(converter, patterns); |
| 136 | } |
| 137 | if (x86) { |
| 138 | configureX86LegalizeForExportTarget(target); |
| 139 | populateX86LegalizeForLLVMExportPatterns(converter, patterns); |
| 140 | } |
| 141 | |
| 142 | if (failed( |
| 143 | Result: applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns)))) |
| 144 | signalPassFailure(); |
| 145 | } |
| 146 | |