1//===- GPUToROCDLPipeline.cpp - Lowering pipeline to ROCDL/AMDGPU --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a sink pipeline that lowers a payload containing
10// `gpu.launch` / `gpu.module` ops to AMDGPU/ROCDL and emits an AMDGCN binary
11// blob via `gpu-module-to-binary`. It is the AMD counterpart of
12// `gpu-lower-to-nvvm-pipeline` and `gpu-lower-to-xevm-pipeline`.
13//
14//===----------------------------------------------------------------------===//
15
16#include "mlir/Conversion/AMDGPUToROCDL/AMDGPUToROCDL.h"
17#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
18#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
19#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVMPass.h"
20#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
21#include "mlir/Conversion/GPUToROCDL/GPUToROCDLPass.h"
22#include "mlir/Conversion/GPUToROCDL/Runtimes.h"
23#include "mlir/Conversion/IndexToLLVM/IndexToLLVM.h"
24#include "mlir/Conversion/MathToLLVM/MathToLLVM.h"
25#include "mlir/Conversion/ReconcileUnrealizedCasts/ReconcileUnrealizedCasts.h"
26#include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
27#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
28#include "mlir/Dialect/GPU/IR/GPUDialect.h"
29#include "mlir/Dialect/GPU/Pipelines/Passes.h"
30#include "mlir/Dialect/GPU/Transforms/Passes.h"
31#include "mlir/Dialect/GPU/Transforms/ROCDL/Passes.h"
32#include "mlir/Dialect/LLVMIR/ROCDLTargetInfo.h"
33#include "mlir/Dialect/MemRef/Transforms/Passes.h"
34#include "mlir/Pass/PassManager.h"
35#include "mlir/Pass/PassOptions.h"
36#include "mlir/Transforms/Passes.h"
37
38using namespace mlir;
39
40namespace {
41
42//===----------------------------------------------------------------------===//
43// Common pipeline
44//===----------------------------------------------------------------------===//
45void buildCommonPassPipeline(
46 OpPassManager &pm, const mlir::gpu::GPUToROCDLPipelineOptions &options) {
47 std::string arch = ROCDL::resolveArchOption(arch: options.arch, deprecatedAlias: options.chip).str();
48 // Lower AMDGPU dialect ops (e.g. amdgpu.lds_barrier, amdgpu.dpp,
49 // amdgpu.mfma, amdgpu.dot, ...) to ROCDL intrinsics first, while they may
50 // still live in unout-lined `gpu.launch` bodies. Mirrors the way NVVM's
51 // pipeline runs `convert-nvgpu-to-nvvm` before kernel outlining.
52 ConvertAMDGPUToROCDLPassOptions amdgpuToROCDLOpt;
53 amdgpuToROCDLOpt.arch = arch;
54 pm.addPass(pass: createConvertAMDGPUToROCDLPass(options: amdgpuToROCDLOpt));
55
56 pm.addPass(pass: createGpuKernelOutliningPass());
57 pm.addPass(pass: createConvertVectorToSCFPass());
58 pm.addPass(pass: createSCFToControlFlowPass());
59 pm.addPass(pass: createConvertFuncToLLVMPass());
60 pm.addPass(pass: memref::createExpandStridedMetadataPass());
61
62 GpuROCDLAttachTargetOptions rocdlTargetOptions;
63 rocdlTargetOptions.arch = arch;
64 rocdlTargetOptions.abiVersion = options.abiVersion;
65 rocdlTargetOptions.optLevel = options.optLevel;
66 rocdlTargetOptions.waveSize = options.waveSize;
67 pm.addPass(pass: createGpuROCDLAttachTarget(options: rocdlTargetOptions));
68
69 pm.addPass(pass: createLowerAffinePass());
70 pm.addPass(pass: createArithToLLVMConversionPass());
71 ConvertIndexToLLVMPassOptions convertIndexToLLVMPassOpt;
72 convertIndexToLLVMPassOpt.indexBitwidth = options.indexBitWidth;
73 pm.addPass(pass: createConvertIndexToLLVMPass(options: convertIndexToLLVMPassOpt));
74 pm.addPass(pass: createCanonicalizerPass());
75 pm.addPass(pass: createCSEPass());
76}
77
78//===----------------------------------------------------------------------===//
79// GPUModule-specific stuff.
80//===----------------------------------------------------------------------===//
81void buildGpuPassPipeline(OpPassManager &pm,
82 const mlir::gpu::GPUToROCDLPipelineOptions &options) {
83 std::string arch = ROCDL::resolveArchOption(arch: options.arch, deprecatedAlias: options.chip).str();
84 ConvertGpuOpsToROCDLOpsOptions opt;
85 opt.arch = arch;
86 opt.waveSize = options.waveSize;
87 opt.useBarePtrCallConv = options.kernelUseBarePtrCallConv;
88 opt.indexBitwidth = options.indexBitWidth;
89 // Always declare HIP as the runtime so that gpu.printf etc. lower to the
90 // matching runtime entry points exposed by `libmlir_rocm_runtime.so`.
91 opt.runtime = mlir::gpu::amd::Runtime::HIP;
92 pm.addNestedPass<gpu::GPUModuleOp>(pass: createConvertGpuOpsToROCDLOps(options: opt));
93 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCanonicalizerPass());
94 pm.addNestedPass<gpu::GPUModuleOp>(pass: createCSEPass());
95 pm.addNestedPass<gpu::GPUModuleOp>(pass: createReconcileUnrealizedCastsPass());
96}
97
98//===----------------------------------------------------------------------===//
99// Host Post-GPU pipeline
100//===----------------------------------------------------------------------===//
101void buildHostPostPipeline(
102 OpPassManager &pm, const mlir::gpu::GPUToROCDLPipelineOptions &options) {
103 GpuToLLVMConversionPassOptions opt;
104 opt.hostBarePtrCallConv = options.hostUseBarePtrCallConv;
105 opt.kernelBarePtrCallConv = options.kernelUseBarePtrCallConv;
106 pm.addPass(pass: createGpuToLLVMConversionPass(options: opt));
107
108 GpuModuleToBinaryPassOptions gpuModuleToBinaryPassOptions;
109 gpuModuleToBinaryPassOptions.compilationTarget = options.binaryFormat;
110 gpuModuleToBinaryPassOptions.cmdOptions = options.cmdOptions;
111 pm.addPass(pass: createGpuModuleToBinaryPass(options: gpuModuleToBinaryPassOptions));
112 pm.addPass(pass: createConvertMathToLLVMPass());
113 pm.addPass(pass: createCanonicalizerPass());
114 pm.addPass(pass: createCSEPass());
115 pm.addPass(pass: createReconcileUnrealizedCastsPass());
116}
117
118} // namespace
119
120void mlir::gpu::buildLowerToROCDLPassPipeline(
121 OpPassManager &pm, const GPUToROCDLPipelineOptions &options) {
122 // Common pipelines
123 buildCommonPassPipeline(pm, options);
124
125 // GPUModule-specific stuff
126 buildGpuPassPipeline(pm, options);
127
128 // Host post-GPUModule-specific stuff
129 buildHostPostPipeline(pm, options);
130}
131
132void mlir::gpu::registerGPUToROCDLPipeline() {
133 PassPipelineRegistration<GPUToROCDLPipelineOptions>(
134 "gpu-lower-to-rocdl-pipeline",
135 "The default pipeline lowers main dialects (arith, memref, scf, vector, "
136 "gpu) to ROCDL. It starts by lowering GPU code to the specified "
137 "compilation target (default is fatbin) then lowers the host code.",
138 buildLowerToROCDLPassPipeline);
139}
140