1//===- SparseTensorPipelines.cpp - Pipelines for sparse tensor code -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
10#include "mlir/Conversion/ComplexToLibm/ComplexToLibm.h"
11#include "mlir/Conversion/ComplexToStandard/ComplexToStandard.h"
12#include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
13#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
14#include "mlir/Conversion/GPUToNVVM/GPUToNVVMPass.h"
15#include "mlir/Conversion/MathToLLVM/MathToLLVM.h"
16#include "mlir/Conversion/MathToLibm/MathToLibm.h"
17#include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h"
18#include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
19#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVMPass.h"
20#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
21#include "mlir/Dialect/Arith/Transforms/Passes.h"
22#include "mlir/Dialect/Bufferization/Transforms/Passes.h"
23#include "mlir/Dialect/Func/IR/FuncOps.h"
24#include "mlir/Dialect/GPU/IR/GPUDialect.h"
25#include "mlir/Dialect/GPU/Transforms/NVVM/Passes.h"
26#include "mlir/Dialect/GPU/Transforms/Passes.h"
27#include "mlir/Dialect/Linalg/Passes.h"
28#include "mlir/Dialect/MemRef/Transforms/Passes.h"
29#include "mlir/Dialect/SparseTensor/Pipelines/Passes.h"
30#include "mlir/Dialect/SparseTensor/Transforms/Passes.h"
31#include "mlir/Pass/PassManager.h"
32#include "mlir/Transforms/Passes.h"
33
34//===----------------------------------------------------------------------===//
35// Pipeline implementation.
36//===----------------------------------------------------------------------===//
37
38void mlir::sparse_tensor::buildSparsifier(OpPassManager &pm,
39 const SparsifierOptions &options) {
40 // Rewrite named linalg ops into generic ops and apply fusion.
41 pm.addNestedPass<func::FuncOp>(pass: createLinalgGeneralizeNamedOpsPass());
42 pm.addNestedPass<func::FuncOp>(pass: createLinalgElementwiseOpFusionPass());
43
44 // Sparsification and bufferization mini-pipeline.
45 pm.addPass(pass: createSparsificationAndBufferizationPass(
46 bufferizationOptions: getBufferizationOptionsForSparsification(
47 analysisOnly: options.testBufferizationAnalysisOnly),
48 sparsificationOptions: options.sparsificationOptions(), createSparseDeallocs: options.createSparseDeallocs,
49 enableRuntimeLibrary: options.enableRuntimeLibrary, enableBufferInitialization: options.enableBufferInitialization,
50 vectorLength: options.vectorLength,
51 /*enableVLAVectorization=*/options.armSVE,
52 /*enableSIMDIndex32=*/options.force32BitVectorIndices,
53 enableGPULibgen: options.enableGPULibgen,
54 emitStrategy: options.sparsificationOptions().sparseEmitStrategy,
55 parallelizationStrategy: options.sparsificationOptions().parallelizationStrategy));
56
57 // Bail-early for test setup.
58 if (options.testBufferizationAnalysisOnly)
59 return;
60
61 // Storage specifier lowering and bufferization wrap-up.
62 pm.addPass(pass: createStorageSpecifierToLLVMPass());
63 pm.addNestedPass<func::FuncOp>(pass: createCanonicalizerPass());
64
65 // GPU code generation.
66 const bool gpuCodegen = options.gpuTriple.hasValue();
67 if (gpuCodegen) {
68 pm.addPass(pass: createSparseGPUCodegenPass(numThreads: options.gpuNumThreads,
69 enableRT: options.enableRuntimeLibrary));
70 pm.addNestedPass<gpu::GPUModuleOp>(pass: createStripDebugInfoPass());
71 pm.addNestedPass<gpu::GPUModuleOp>(pass: createSCFToControlFlowPass());
72 pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertGpuOpsToNVVMOps());
73 }
74
75 // Progressively lower to LLVM. Note that the convert-vector-to-llvm
76 // pass is repeated on purpose.
77 // TODO(springerm): Add sparse support to the BufferDeallocation pass and add
78 // it to this pipeline.
79 pm.addNestedPass<func::FuncOp>(pass: createConvertLinalgToLoopsPass());
80 pm.addNestedPass<func::FuncOp>(pass: createConvertVectorToSCFPass());
81 pm.addNestedPass<func::FuncOp>(pass: memref::createExpandReallocPass());
82 pm.addPass(pass: memref::createExpandStridedMetadataPass());
83 pm.addPass(pass: createLowerAffinePass());
84 pm.addNestedPass<func::FuncOp>(pass: createSCFToControlFlowPass());
85 pm.addPass(
86 pass: createConvertVectorToLLVMPass(options: options.convertVectorToLLVMOptions()));
87 pm.addNestedPass<func::FuncOp>(pass: createConvertComplexToStandardPass());
88 pm.addNestedPass<func::FuncOp>(pass: arith::createArithExpandOpsPass());
89 pm.addNestedPass<func::FuncOp>(pass: createConvertMathToLLVMPass());
90 pm.addPass(pass: createConvertMathToLibmPass());
91 pm.addPass(pass: createConvertComplexToLibm());
92 pm.addPass(
93 pass: createConvertVectorToLLVMPass(options: options.convertVectorToLLVMOptions()));
94
95 // Finalize GPU code generation.
96 if (gpuCodegen) {
97 GpuNVVMAttachTargetOptions nvvmTargetOptions;
98 nvvmTargetOptions.triple = options.gpuTriple;
99 // `gpu-chip` has no default here, so forwarding it unset would overwrite
100 // the pass default with the empty string, which `#nvvm.target` rejects.
101 if (options.gpuChip.hasValue())
102 nvvmTargetOptions.chip = options.gpuChip;
103 nvvmTargetOptions.features = options.gpuFeatures;
104 pm.addPass(pass: createGpuNVVMAttachTarget(options: nvvmTargetOptions));
105 pm.addPass(pass: createGpuToLLVMConversionPass());
106 GpuModuleToBinaryPassOptions gpuModuleToBinaryPassOptions;
107 gpuModuleToBinaryPassOptions.compilationTarget = options.gpuFormat;
108 pm.addPass(pass: createGpuModuleToBinaryPass(options: gpuModuleToBinaryPassOptions));
109 }
110
111 // Convert to LLVM.
112 pm.addPass(pass: createConvertToLLVMPass());
113
114 // Ensure all casts are realized.
115 pm.addPass(pass: createReconcileUnrealizedCastsPass());
116}
117
118//===----------------------------------------------------------------------===//
119// Pipeline registration.
120//===----------------------------------------------------------------------===//
121
122void mlir::sparse_tensor::registerSparseTensorPipelines() {
123 PassPipelineRegistration<SparsifierOptions>(
124 "sparsifier",
125 "The standard pipeline for taking sparsity-agnostic IR using the"
126 " sparse-tensor type, and lowering it to LLVM IR with concrete"
127 " representations and algorithms for sparse tensors.",
128 buildSparsifier);
129}
130