| 1 | //===- SparseTensorPipelines.cpp - Pipelines for sparse tensor code -------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/AffineToStandard/AffineToStandard.h" |
| 10 | #include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h" |
| 11 | #include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h" |
| 12 | #include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h" |
| 13 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 14 | #include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h" |
| 15 | #include "mlir/Conversion/MathToLLVM/MathToLLVM.h" |
| 16 | #include "mlir/Conversion/MathToLibm/MathToLibm.h" |
| 17 | #include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h" |
| 18 | #include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h" |
| 19 | #include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h" |
| 20 | #include "mlir/Conversion/VectorToSCF/VectorToSCF.h" |
| 21 | #include "mlir/Dialect/Arith/Transforms/Passes.h" |
| 22 | #include "mlir/Dialect/Bufferization/Transforms/Passes.h" |
| 23 | #include "mlir/Dialect/Func/IR/FuncOps.h" |
| 24 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 25 | #include "mlir/Dialect/GPU/Transforms/NVVM/Passes.h" |
| 26 | #include "mlir/Dialect/GPU/Transforms/Passes.h" |
| 27 | #include "mlir/Dialect/Linalg/Passes.h" |
| 28 | #include "mlir/Dialect/MemRef/Transforms/Passes.h" |
| 29 | #include "mlir/Dialect/SparseTensor/Pipelines/Passes.h" |
| 30 | #include "mlir/Dialect/SparseTensor/Transforms/Passes.h" |
| 31 | #include "mlir/Pass/PassManager.h" |
| 32 | #include "mlir/Transforms/Passes.h" |
| 33 | |
| 34 | //===----------------------------------------------------------------------===// |
| 35 | // Pipeline implementation. |
| 36 | //===----------------------------------------------------------------------===// |
| 37 | |
| 38 | void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm, |
| 39 | const SparsifierOptions &options) { |
| 40 | // Rewrite named linalg ops into generic ops and apply fusion. |
| 41 | pm.addNestedPass<func::FuncOp>(pass: createLinalgGeneralizeNamedOpsPass()); |
| 42 | pm.addNestedPass<func::FuncOp>(pass: createLinalgElementwiseOpFusionPass()); |
| 43 | |
| 44 | // Sparsification and bufferization mini-pipeline. |
| 45 | pm.addPass(pass: createSparsificationAndBufferizationPass( |
| 46 | bufferizationOptions: getBufferizationOptionsForSparsification( |
| 47 | analysisOnly: options.testBufferizationAnalysisOnly), |
| 48 | sparsificationOptions: options.sparsificationOptions(), createSparseDeallocs: options.createSparseDeallocs, |
| 49 | enableRuntimeLibrary: options.enableRuntimeLibrary, enableBufferInitialization: options.enableBufferInitialization, |
| 50 | vectorLength: options.vectorLength, |
| 51 | /*enableVLAVectorization=*/options.armSVE, |
| 52 | /*enableSIMDIndex32=*/options.force32BitVectorIndices, |
| 53 | enableGPULibgen: options.enableGPULibgen, |
| 54 | emitStrategy: options.sparsificationOptions().sparseEmitStrategy, |
| 55 | parallelizationStrategy: options.sparsificationOptions().parallelizationStrategy)); |
| 56 | |
| 57 | // Bail-early for test setup. |
| 58 | if (options.testBufferizationAnalysisOnly) |
| 59 | return; |
| 60 | |
| 61 | // Storage specifier lowering and bufferization wrap-up. |
| 62 | pm.addPass(pass: createStorageSpecifierToLLVMPass()); |
| 63 | pm.addNestedPass<func::FuncOp>(pass: createCanonicalizerPass()); |
| 64 | |
| 65 | // GPU code generation. |
| 66 | const bool gpuCodegen = options.gpuTriple.hasValue(); |
| 67 | if (gpuCodegen) { |
| 68 | pm.addPass(pass: createSparseGPUCodegenPass(numThreads: options.gpuNumThreads, |
| 69 | enableRT: options.enableRuntimeLibrary)); |
| 70 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createStripDebugInfoPass()); |
| 71 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createSCFToControlFlowPass()); |
| 72 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertGpuOpsToNVVMOps()); |
| 73 | } |
| 74 | |
| 75 | // Progressively lower to LLVM. Note that the convert-vector-to-llvm |
| 76 | // pass is repeated on purpose. |
| 77 | // TODO(springerm): Add sparse support to the BufferDeallocation pass and add |
| 78 | // it to this pipeline. |
| 79 | pm.addNestedPass<func::FuncOp>(pass: createConvertLinalgToLoopsPass()); |
| 80 | pm.addNestedPass<func::FuncOp>(pass: createConvertVectorToSCFPass()); |
| 81 | pm.addNestedPass<func::FuncOp>(pass: memref::createExpandReallocPass()); |
| 82 | pm.addPass(pass: memref::createExpandStridedMetadataPass()); |
| 83 | pm.addPass(pass: createLowerAffinePass()); |
| 84 | pm.addNestedPass<func::FuncOp>(pass: createSCFToControlFlowPass()); |
| 85 | pm.addPass( |
| 86 | pass: createConvertVectorToLLVMPass(options: options.convertVectorToLLVMOptions())); |
| 87 | pm.addNestedPass<func::FuncOp>(pass: createConvertComplexToStandardPass()); |
| 88 | pm.addNestedPass<func::FuncOp>(pass: arith::createArithExpandOpsPass()); |
| 89 | pm.addNestedPass<func::FuncOp>(pass: createConvertMathToLLVMPass()); |
| 90 | pm.addPass(pass: createConvertMathToLibmPass()); |
| 91 | pm.addPass(pass: createConvertComplexToLibm()); |
| 92 | pm.addPass( |
| 93 | pass: createConvertVectorToLLVMPass(options: options.convertVectorToLLVMOptions())); |
| 94 | |
| 95 | // Finalize GPU code generation. |
| 96 | if (gpuCodegen) { |
| 97 | GpuNVVMAttachTargetOptions nvvmTargetOptions; |
| 98 | nvvmTargetOptions.triple = options.gpuTriple; |
| 99 | // `gpu-chip` has no default here, so forwarding it unset would overwrite |
| 100 | // the pass default with the empty string, which `#nvvm.target` rejects. |
| 101 | if (options.gpuChip.hasValue()) |
| 102 | nvvmTargetOptions.chip = options.gpuChip; |
| 103 | nvvmTargetOptions.features = options.gpuFeatures; |
| 104 | pm.addPass(pass: createGpuNVVMAttachTarget(options: nvvmTargetOptions)); |
| 105 | pm.addPass(pass: createGpuToLLVMConversionPass()); |
| 106 | GpuModuleToBinaryPassOptions gpuModuleToBinaryPassOptions; |
| 107 | gpuModuleToBinaryPassOptions.compilationTarget = options.gpuFormat; |
| 108 | pm.addPass(pass: createGpuModuleToBinaryPass(options: gpuModuleToBinaryPassOptions)); |
| 109 | } |
| 110 | |
| 111 | // Convert to LLVM. |
| 112 | pm.addPass(pass: createConvertToLLVMPass()); |
| 113 | |
| 114 | // Ensure all casts are realized. |
| 115 | pm.addPass(pass: createReconcileUnrealizedCastsPass()); |
| 116 | } |
| 117 | |
| 118 | //===----------------------------------------------------------------------===// |
| 119 | // Pipeline registration. |
| 120 | //===----------------------------------------------------------------------===// |
| 121 | |
| 122 | void mlir::sparse_tensor::registerSparseTensorPipelines() { |
| 123 | PassPipelineRegistration<SparsifierOptions>( |
| 124 | "sparsifier" , |
| 125 | "The standard pipeline for taking sparsity-agnostic IR using the" |
| 126 | " sparse-tensor type, and lowering it to LLVM IR with concrete" |
| 127 | " representations and algorithms for sparse tensors." , |
| 128 | buildSparsifier); |
| 129 | } |
| 130 | |