1//===- GPUToSPIRVPass.cpp - GPU to SPIR-V Passes --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass to convert a kernel function in the GPU Dialect
10// into a spirv.module operation.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRVPass.h"
15
16#include "mlir/Conversion/ArithToSPIRV/ArithToSPIRV.h"
17#include "mlir/Conversion/FuncToSPIRV/FuncToSPIRV.h"
18#include "mlir/Conversion/GPUToSPIRV/GPUToSPIRV.h"
19#include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h"
20#include "mlir/Conversion/SCFToSPIRV/SCFToSPIRV.h"
21#include "mlir/Conversion/VectorToSPIRV/VectorToSPIRV.h"
22#include "mlir/Dialect/Func/IR/FuncOps.h"
23#include "mlir/Dialect/GPU/IR/GPUDialect.h"
24#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
25#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
26#include "mlir/IR/PatternMatch.h"
27
28namespace mlir {
29#define GEN_PASS_DEF_CONVERTGPUTOSPIRV
30#include "mlir/Conversion/Passes.h.inc"
31} // namespace mlir
32
33using namespace mlir;
34
35namespace {
36/// Pass to lower GPU Dialect to SPIR-V. The pass only converts the gpu.func ops
37/// inside gpu.module ops. i.e., the function that are referenced in
38/// gpu.launch_func ops. For each such function
39///
40/// 1) Create a spirv::ModuleOp, and clone the function into spirv::ModuleOp
41/// (the original function is still needed by the gpu::LaunchKernelOp, so cannot
42/// replace it).
43///
44/// 2) Lower the body of the spirv::ModuleOp.
45struct GPUToSPIRVPass final : impl::ConvertGPUToSPIRVBase<GPUToSPIRVPass> {
46 explicit GPUToSPIRVPass(bool mapMemorySpace)
47 : mapMemorySpace(mapMemorySpace) {}
48 void runOnOperation() override;
49
50private:
51 /// Queries the target environment from 'targets' attribute of the given
52 /// `moduleOp`.
53 spirv::TargetEnvAttr lookupTargetEnvInTargets(gpu::GPUModuleOp moduleOp);
54
55 /// Queries the target environment from 'targets' attribute of the given
56 /// `moduleOp` or returns target environment as returned by
57 /// `spirv::lookupTargetEnvOrDefault` if not provided by 'targets'.
58 spirv::TargetEnvAttr lookupTargetEnvOrDefault(gpu::GPUModuleOp moduleOp);
59 bool mapMemorySpace;
60};
61
62spirv::TargetEnvAttr
63GPUToSPIRVPass::lookupTargetEnvInTargets(gpu::GPUModuleOp moduleOp) {
64 if (ArrayAttr targets = moduleOp.getTargetsAttr()) {
65 for (Attribute targetAttr : targets)
66 if (auto spirvTargetEnvAttr = dyn_cast<spirv::TargetEnvAttr>(Val&: targetAttr))
67 return spirvTargetEnvAttr;
68 }
69
70 return {};
71}
72
73spirv::TargetEnvAttr
74GPUToSPIRVPass::lookupTargetEnvOrDefault(gpu::GPUModuleOp moduleOp) {
75 if (spirv::TargetEnvAttr targetEnvAttr = lookupTargetEnvInTargets(moduleOp))
76 return targetEnvAttr;
77
78 return spirv::lookupTargetEnvOrDefault(op: moduleOp);
79}
80
81void GPUToSPIRVPass::runOnOperation() {
82 MLIRContext *context = &getContext();
83 ModuleOp module = getOperation();
84
85 SmallVector<Operation *, 1> gpuModules;
86 OpBuilder builder(context);
87
88 auto targetEnvSupportsKernelCapability = [this](gpu::GPUModuleOp moduleOp) {
89 auto targetAttr = lookupTargetEnvOrDefault(moduleOp);
90 spirv::TargetEnv targetEnv(targetAttr);
91 return targetEnv.allows(spirv::Capability::Kernel);
92 };
93
94 module.walk(callback: [&](gpu::GPUModuleOp moduleOp) {
95 // Clone each GPU kernel module for conversion, given that the GPU
96 // launch op still needs the original GPU kernel module.
97 // For Vulkan Shader capabilities, we insert the newly converted SPIR-V
98 // module right after the original GPU module, as that's the expectation of
99 // the in-tree SPIR-V CPU runner (the Vulkan runner does not use this pass).
100 // For OpenCL Kernel capabilities, we insert the newly converted SPIR-V
101 // module inside the original GPU module, as that's the expectaion of the
102 // normal GPU compilation pipeline.
103 if (targetEnvSupportsKernelCapability(moduleOp)) {
104 builder.setInsertionPointToStart(moduleOp.getBody());
105 } else {
106 builder.setInsertionPoint(moduleOp.getOperation());
107 }
108 gpuModules.push_back(Elt: builder.clone(op&: *moduleOp.getOperation()));
109 });
110
111 // Run conversion for each module independently as they can have different
112 // TargetEnv attributes.
113 for (Operation *gpuModule : gpuModules) {
114 spirv::TargetEnvAttr targetAttr =
115 lookupTargetEnvOrDefault(moduleOp: cast<gpu::GPUModuleOp>(Val: gpuModule));
116
117 // Map MemRef memory space to SPIR-V storage class first if requested.
118 if (mapMemorySpace) {
119 spirv::MemorySpaceToStorageClassMap memorySpaceMap =
120 targetEnvSupportsKernelCapability(
121 dyn_cast<gpu::GPUModuleOp>(Val: gpuModule))
122 ? spirv::mapMemorySpaceToOpenCLStorageClass
123 : spirv::mapMemorySpaceToVulkanStorageClass;
124 spirv::MemorySpaceToStorageClassConverter converter(memorySpaceMap);
125 spirv::convertMemRefTypesAndAttrs(op: gpuModule, typeConverter&: converter);
126
127 // Check if there are any illegal ops remaining.
128 std::unique_ptr<ConversionTarget> target =
129 spirv::getMemorySpaceToStorageClassTarget(*context);
130 gpuModule->walk(callback: [&target, this](Operation *childOp) {
131 if (target->isIllegal(op: childOp)) {
132 childOp->emitOpError(message: "failed to legalize memory space");
133 signalPassFailure();
134 return WalkResult::interrupt();
135 }
136 return WalkResult::advance();
137 });
138 }
139
140 std::unique_ptr<ConversionTarget> target =
141 SPIRVConversionTarget::get(targetAttr);
142
143 SPIRVConversionOptions options;
144 options.use64bitIndex = this->use64bitIndex;
145 SPIRVTypeConverter typeConverter(targetAttr, options);
146 populateMMAToSPIRVCoopMatrixTypeConversion(typeConverter);
147 populateGPUNamedBarrierToSPIRVTypeConversion(typeConverter);
148
149 RewritePatternSet patterns(context);
150 populateGPUToSPIRVPatterns(typeConverter, patterns);
151 populateGpuWMMAToSPIRVCoopMatrixKHRConversionPatterns(typeConverter,
152 patterns);
153
154 // TODO: Change SPIR-V conversion to be progressive and remove the following
155 // patterns.
156 ScfToSPIRVContext scfContext;
157 populateSCFToSPIRVPatterns(typeConverter, scfToSPIRVContext&: scfContext, patterns);
158 mlir::arith::populateArithToSPIRVPatterns(typeConverter, patterns);
159 populateMemRefToSPIRVPatterns(typeConverter, patterns);
160 populateFuncToSPIRVPatterns(typeConverter, patterns);
161 populateVectorToSPIRVPatterns(typeConverter, patterns);
162
163 if (failed(Result: applyFullConversion(op: gpuModule, target: *target, patterns: std::move(patterns))))
164 return signalPassFailure();
165 }
166
167 // For OpenCL, the gpu.func op in the original gpu.module op needs to be
168 // replaced with an empty func.func op with the same arguments as the gpu.func
169 // op. The func.func op needs gpu.kernel attribute set.
170 module.walk(callback: [&](gpu::GPUModuleOp moduleOp) {
171 if (targetEnvSupportsKernelCapability(moduleOp)) {
172 moduleOp.walk(callback: [&](gpu::GPUFuncOp funcOp) {
173 builder.setInsertionPoint(funcOp);
174 auto newFuncOp =
175 func::FuncOp::create(builder, location: funcOp.getLoc(), name: funcOp.getName(),
176 type: funcOp.getFunctionType());
177 auto entryBlock = newFuncOp.addEntryBlock();
178 builder.setInsertionPointToEnd(entryBlock);
179 func::ReturnOp::create(builder, location: funcOp.getLoc());
180 newFuncOp->setDiscardableAttr(name: gpu::GPUDialect::getKernelFuncAttrName(),
181 value: builder.getUnitAttr());
182 funcOp.erase();
183 });
184 }
185 });
186}
187
188} // namespace
189
190std::unique_ptr<OperationPass<ModuleOp>>
191mlir::createConvertGPUToSPIRVPass(bool mapMemorySpace) {
192 return std::make_unique<GPUToSPIRVPass>(args&: mapMemorySpace);
193}
194