| 1 | //===- GPUToROCDLPipeline.cpp - Lowering pipeline to ROCDL/AMDGPU --------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file implements a sink pipeline that lowers a payload containing |
| 10 | // `gpu.launch` / `gpu.module` ops to AMDGPU/ROCDL and emits an AMDGCN binary |
| 11 | // blob via `gpu-module-to-binary`. It is the AMD counterpart of |
| 12 | // `gpu-lower-to-nvvm-pipeline` and `gpu-lower-to-xevm-pipeline`. |
| 13 | // |
| 14 | //===----------------------------------------------------------------------===// |
| 15 | |
| 16 | #include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h" |
| 17 | #include "mlir/Conversion/AffineToStandard/AffineToStandard.h" |
| 18 | #include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h" |
| 19 | #include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h" |
| 20 | #include "mlir/Conversion/GPUCommon/GPUCommonPass.h" |
| 21 | #include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h" |
| 22 | #include "mlir/Conversion/GPUToROCDL/Runtimes.h" |
| 23 | #include "mlir/Conversion/IndexToLLVM/IndexToLLVM.h" |
| 24 | #include "mlir/Conversion/MathToLLVM/MathToLLVM.h" |
| 25 | #include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h" |
| 26 | #include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h" |
| 27 | #include "mlir/Conversion/VectorToSCF/VectorToSCF.h" |
| 28 | #include "mlir/Dialect/GPU/IR/GPUDialect.h" |
| 29 | #include "mlir/Dialect/GPU/Pipelines/Passes.h" |
| 30 | #include "mlir/Dialect/GPU/Transforms/Passes.h" |
| 31 | #include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h" |
| 32 | #include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h" |
| 33 | #include "mlir/Dialect/MemRef/Transforms/Passes.h" |
| 34 | #include "mlir/Pass/PassManager.h" |
| 35 | #include "mlir/Pass/PassOptions.h" |
| 36 | #include "mlir/Transforms/Passes.h" |
| 37 | |
| 38 | using namespace mlir; |
| 39 | |
| 40 | namespace { |
| 41 | |
| 42 | //===----------------------------------------------------------------------===// |
| 43 | // Common pipeline |
| 44 | //===----------------------------------------------------------------------===// |
| 45 | void buildCommonPassPipeline( |
| 46 | OpPassManager &pm, const mlir::gpu::GPUToROCDLPipelineOptions &options) { |
| 47 | std::string arch = ROCDL::resolveArchOption(arch: options.arch, deprecatedAlias: options.chip).str(); |
| 48 | // Lower AMDGPU dialect ops (e.g. amdgpu.lds_barrier, amdgpu.dpp, |
| 49 | // amdgpu.mfma, amdgpu.dot, ...) to ROCDL intrinsics first, while they may |
| 50 | // still live in unout-lined `gpu.launch` bodies. Mirrors the way NVVM's |
| 51 | // pipeline runs `convert-nvgpu-to-nvvm` before kernel outlining. |
| 52 | ConvertAMDGPUToROCDLPassOptions amdgpuToROCDLOpt; |
| 53 | amdgpuToROCDLOpt.arch = arch; |
| 54 | pm.addPass(pass: createConvertAMDGPUToROCDLPass(options: amdgpuToROCDLOpt)); |
| 55 | |
| 56 | pm.addPass(pass: createGpuKernelOutliningPass()); |
| 57 | pm.addPass(pass: createConvertVectorToSCFPass()); |
| 58 | pm.addPass(pass: createSCFToControlFlowPass()); |
| 59 | pm.addPass(pass: createConvertFuncToLLVMPass()); |
| 60 | pm.addPass(pass: memref::createExpandStridedMetadataPass()); |
| 61 | |
| 62 | GpuROCDLAttachTargetOptions rocdlTargetOptions; |
| 63 | rocdlTargetOptions.arch = arch; |
| 64 | rocdlTargetOptions.abiVersion = options.abiVersion; |
| 65 | rocdlTargetOptions.optLevel = options.optLevel; |
| 66 | rocdlTargetOptions.waveSize = options.waveSize; |
| 67 | pm.addPass(pass: createGpuROCDLAttachTarget(options: rocdlTargetOptions)); |
| 68 | |
| 69 | pm.addPass(pass: createLowerAffinePass()); |
| 70 | pm.addPass(pass: createArithToLLVMConversionPass()); |
| 71 | ConvertIndexToLLVMPassOptions convertIndexToLLVMPassOpt; |
| 72 | convertIndexToLLVMPassOpt.indexBitwidth = options.indexBitWidth; |
| 73 | pm.addPass(pass: createConvertIndexToLLVMPass(options: convertIndexToLLVMPassOpt)); |
| 74 | pm.addPass(pass: createCanonicalizerPass()); |
| 75 | pm.addPass(pass: createCSEPass()); |
| 76 | } |
| 77 | |
| 78 | //===----------------------------------------------------------------------===// |
| 79 | // GPUModule-specific stuff. |
| 80 | //===----------------------------------------------------------------------===// |
| 81 | void buildGpuPassPipeline(OpPassManager &pm, |
| 82 | const mlir::gpu::GPUToROCDLPipelineOptions &options) { |
| 83 | std::string arch = ROCDL::resolveArchOption(arch: options.arch, deprecatedAlias: options.chip).str(); |
| 84 | ConvertGpuOpsToROCDLOpsOptions opt; |
| 85 | opt.arch = arch; |
| 86 | opt.waveSize = options.waveSize; |
| 87 | opt.useBarePtrCallConv = options.kernelUseBarePtrCallConv; |
| 88 | opt.indexBitwidth = options.indexBitWidth; |
| 89 | // Always declare HIP as the runtime so that gpu.printf etc. lower to the |
| 90 | // matching runtime entry points exposed by `libmlir_rocm_runtime.so`. |
| 91 | opt.runtime = mlir::gpu::amd::Runtime::HIP; |
| 92 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertGpuOpsToROCDLOps(options: opt)); |
| 93 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCanonicalizerPass()); |
| 94 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass()); |
| 95 | pm.addNestedPass<gpu::GPUModuleOp>(pass: createReconcileUnrealizedCastsPass()); |
| 96 | } |
| 97 | |
| 98 | //===----------------------------------------------------------------------===// |
| 99 | // Host Post-GPU pipeline |
| 100 | //===----------------------------------------------------------------------===// |
| 101 | void buildHostPostPipeline( |
| 102 | OpPassManager &pm, const mlir::gpu::GPUToROCDLPipelineOptions &options) { |
| 103 | GpuToLLVMConversionPassOptions opt; |
| 104 | opt.hostBarePtrCallConv = options.hostUseBarePtrCallConv; |
| 105 | opt.kernelBarePtrCallConv = options.kernelUseBarePtrCallConv; |
| 106 | pm.addPass(pass: createGpuToLLVMConversionPass(options: opt)); |
| 107 | |
| 108 | GpuModuleToBinaryPassOptions gpuModuleToBinaryPassOptions; |
| 109 | gpuModuleToBinaryPassOptions.compilationTarget = options.binaryFormat; |
| 110 | gpuModuleToBinaryPassOptions.cmdOptions = options.cmdOptions; |
| 111 | pm.addPass(pass: createGpuModuleToBinaryPass(options: gpuModuleToBinaryPassOptions)); |
| 112 | pm.addPass(pass: createConvertMathToLLVMPass()); |
| 113 | pm.addPass(pass: createCanonicalizerPass()); |
| 114 | pm.addPass(pass: createCSEPass()); |
| 115 | pm.addPass(pass: createReconcileUnrealizedCastsPass()); |
| 116 | } |
| 117 | |
| 118 | } // namespace |
| 119 | |
| 120 | void mlir::gpu::buildLowerToROCDLPassPipeline( |
| 121 | OpPassManager &pm, const GPUToROCDLPipelineOptions &options) { |
| 122 | // Common pipelines |
| 123 | buildCommonPassPipeline(pm, options); |
| 124 | |
| 125 | // GPUModule-specific stuff |
| 126 | buildGpuPassPipeline(pm, options); |
| 127 | |
| 128 | // Host post-GPUModule-specific stuff |
| 129 | buildHostPostPipeline(pm, options); |
| 130 | } |
| 131 | |
| 132 | void mlir::gpu::registerGPUToROCDLPipeline() { |
| 133 | PassPipelineRegistration<GPUToROCDLPipelineOptions>( |
| 134 | "gpu-lower-to-rocdl-pipeline" , |
| 135 | "The default pipeline lowers main dialects (arith, memref, scf, vector, " |
| 136 | "gpu) to ROCDL. It starts by lowering GPU code to the specified " |
| 137 | "compilation target (default is fatbin) then lowers the host code." , |
| 138 | buildLowerToROCDLPassPipeline); |
| 139 | } |
| 140 | |