1//===- ConvertLaunchFuncToLLVMCalls.cpp - MLIR GPU launch to LLVM pass ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements passes to convert `gpu.launch_func` op into a sequence
10// of LLVM calls that emulate the host and device sides.
11//
12//===----------------------------------------------------------------------===//
13
14#include "../SPIRVCommon/Pattern.h"
15#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
16#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
17#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
18#include "mlir/Conversion/LLVMCommon/Pattern.h"
19#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
20#include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
21#include "mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h"
22#include "mlir/Dialect/Func/IR/FuncOps.h"
23#include "mlir/Dialect/GPU/IR/GPUDialect.h"
24#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
25#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
26#include "mlir/Dialect/LLVMIR/LLVMMemoryIntrinsicOps.h"
27#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
28#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
29#include "mlir/IR/BuiltinOps.h"
30#include "mlir/IR/SymbolTable.h"
31#include "mlir/Pass/Pass.h"
32#include "mlir/Transforms/DialectConversion.h"
33#include "llvm/ADT/DenseMap.h"
34#include "llvm/ADT/StringExtras.h"
35#include "llvm/Support/FormatVariadic.h"
36
37namespace mlir {
38#define GEN_PASS_DEF_LOWERHOSTCODETOLLVMPASS
39#include "mlir/Conversion/Passes.h.inc"
40} // namespace mlir
41
42using namespace mlir;
43
44static constexpr const char kSPIRVModule[] = "__spv__";
45
46//===----------------------------------------------------------------------===//
47// Utility functions
48//===----------------------------------------------------------------------===//
49
50/// Calculates the index of the kernel's operand that is represented by the
51/// given global variable with the `bind` attribute. We assume that the index of
52/// each kernel's operand is mapped to (descriptorSet, binding) by the map:
53/// i -> (0, i)
54/// which is implemented under `LowerABIAttributesPass`.
55static unsigned calculateGlobalIndex(spirv::GlobalVariableOp op) {
56 return *op.getBinding();
57}
58
59/// Copies the given number of bytes from src to dst pointers.
60static void copy(Location loc, Value dst, Value src, Value size,
61 OpBuilder &builder) {
62 LLVM::MemcpyOp::create(builder, location: loc, dst, src, len: size, /*isVolatile=*/false);
63}
64
65/// Encodes the binding and descriptor set numbers into a new symbolic name.
66/// The name is specified by
67/// {kernel_module_name}_{variable_name}_descriptor_set{ds}_binding{b}
68/// to avoid symbolic conflicts, where 'ds' and 'b' are descriptor set and
69/// binding numbers.
70static std::string
71createGlobalVariableWithBindName(spirv::GlobalVariableOp op,
72 StringRef kernelModuleName) {
73 return llvm::formatv(Fmt: "{0}_{1}_descriptor_set{2}_binding{3}",
74 Vals: kernelModuleName.str(), Vals: op.getSymName().str(),
75 Vals: std::to_string(val: *op.getDescriptorSet()),
76 Vals: std::to_string(val: *op.getBinding()));
77}
78
79/// Returns true if the given global variable has both a descriptor set number
80/// and a binding number.
81static bool hasDescriptorSetAndBinding(spirv::GlobalVariableOp op) {
82 return op.getDescriptorSetAttr() && op.getBindingAttr();
83}
84
85/// Fills `globalVariableMap` with SPIR-V global variables that represent kernel
86/// arguments from the given SPIR-V module. We assume that the module contains a
87/// single entry point function. Hence, all `spirv.GlobalVariable`s with a bind
88/// attribute are kernel arguments.
89static LogicalResult getKernelGlobalVariables(
90 spirv::ModuleOp module,
91 DenseMap<uint32_t, spirv::GlobalVariableOp> &globalVariableMap) {
92 auto entryPoints = module.getOps<spirv::EntryPointOp>();
93 if (!llvm::hasSingleElement(C&: entryPoints)) {
94 return module.emitError(
95 message: "The module must contain exactly one entry point function");
96 }
97 auto globalVariables = module.getOps<spirv::GlobalVariableOp>();
98 for (auto globalOp : globalVariables) {
99 if (hasDescriptorSetAndBinding(op: globalOp))
100 globalVariableMap[calculateGlobalIndex(op: globalOp)] = globalOp;
101 }
102 return success();
103}
104
105/// Encodes the SPIR-V module's symbolic name into the name of the entry point
106/// function.
107static LogicalResult encodeKernelName(spirv::ModuleOp module) {
108 StringRef spvModuleName = module.getSymName().value_or(u: kSPIRVModule);
109 // We already know that the module contains exactly one entry point function
110 // based on `getKernelGlobalVariables()` call. Update this function's name
111 // to:
112 // {spv_module_name}_{function_name}
113 auto entryPoints = module.getOps<spirv::EntryPointOp>();
114 if (!llvm::hasSingleElement(C&: entryPoints)) {
115 return module.emitError(
116 message: "The module must contain exactly one entry point function");
117 }
118 spirv::EntryPointOp entryPoint = *entryPoints.begin();
119 StringRef funcName = entryPoint.getFn();
120 auto funcOp = module.lookupSymbol<spirv::FuncOp>(symbol: entryPoint.getFnAttr());
121 StringAttr newFuncName =
122 StringAttr::get(context: module->getContext(), bytes: spvModuleName + "_" + funcName);
123 if (failed(Result: SymbolTable::replaceAllSymbolUses(oldSymbol: funcOp, newSymbolName: newFuncName, from: module)))
124 return failure();
125 SymbolTable::setSymbolName(symbol: funcOp, name: newFuncName);
126 return success();
127}
128
129//===----------------------------------------------------------------------===//
130// Conversion patterns
131//===----------------------------------------------------------------------===//
132
133namespace {
134
135/// Structure to group information about the variables being copied.
136struct CopyInfo {
137 Value dst;
138 Value src;
139 Value size;
140};
141
142/// This pattern emulates a call to the kernel in LLVM dialect. For that, we
143/// copy the data to the global variable (emulating device side), call the
144/// kernel as a normal void LLVM function, and copy the data back (emulating the
145/// host side).
146class GPULaunchLowering : public ConvertOpToLLVMPattern<gpu::LaunchFuncOp> {
147 using ConvertOpToLLVMPattern<gpu::LaunchFuncOp>::ConvertOpToLLVMPattern;
148
149 LogicalResult
150 matchAndRewrite(gpu::LaunchFuncOp launchOp, OpAdaptor adaptor,
151 ConversionPatternRewriter &rewriter) const override {
152 auto *op = launchOp.getOperation();
153 MLIRContext *context = rewriter.getContext();
154 auto module = launchOp->getParentOfType<ModuleOp>();
155
156 // Get the SPIR-V module that represents the gpu kernel module. The module
157 // is named:
158 // __spv__{kernel_module_name}
159 // based on GPU to SPIR-V conversion.
160 StringRef kernelModuleName = launchOp.getKernelModuleName().getValue();
161 std::string spvModuleName = kSPIRVModule + kernelModuleName.str();
162 auto spvModule = module.lookupSymbol<spirv::ModuleOp>(
163 name: StringAttr::get(context, bytes: spvModuleName));
164 if (!spvModule) {
165 return launchOp.emitOpError(message: "SPIR-V kernel module '")
166 << spvModuleName << "' is not found";
167 }
168
169 // Declare kernel function in the main module so that it later can be linked
170 // with its definition from the kernel module. We know that the kernel
171 // function would have no arguments and the data is passed via global
172 // variables. The name of the kernel will be
173 // {spv_module_name}_{kernel_function_name}
174 // to avoid symbolic name conflicts.
175 StringRef kernelFuncName = launchOp.getKernelName().getValue();
176 std::string newKernelFuncName = spvModuleName + "_" + kernelFuncName.str();
177 auto kernelFunc = module.lookupSymbol<LLVM::LLVMFuncOp>(
178 name: StringAttr::get(context, bytes: newKernelFuncName));
179 if (!kernelFunc) {
180 OpBuilder::InsertionGuard guard(rewriter);
181 rewriter.setInsertionPointToStart(module.getBody());
182 kernelFunc = LLVM::LLVMFuncOp::create(
183 builder&: rewriter, location: rewriter.getUnknownLoc(), name: newKernelFuncName,
184 type: LLVM::LLVMFunctionType::get(result: LLVM::LLVMVoidType::get(ctx: context),
185 arguments: ArrayRef<Type>()));
186 rewriter.setInsertionPoint(launchOp);
187 }
188
189 // Get all global variables associated with the kernel operands.
190 DenseMap<uint32_t, spirv::GlobalVariableOp> globalVariableMap;
191 if (failed(Result: getKernelGlobalVariables(module: spvModule, globalVariableMap)))
192 return failure();
193
194 // Traverse kernel operands that were converted to MemRefDescriptors. For
195 // each operand, create a global variable and copy data from operand to it.
196 Location loc = launchOp.getLoc();
197 SmallVector<CopyInfo, 4> copyInfo;
198 auto numKernelOperands = launchOp.getNumKernelOperands();
199 auto kernelOperands = adaptor.getOperands().take_back(n: numKernelOperands);
200 for (const auto &operand : llvm::enumerate(First&: kernelOperands)) {
201 // Check if the kernel's operand is a ranked memref.
202 auto memRefType = dyn_cast<MemRefType>(
203 Val: launchOp.getKernelOperand(i: operand.index()).getType());
204 if (!memRefType)
205 return failure();
206
207 // Calculate the size of the memref and get the pointer to the allocated
208 // buffer.
209 SmallVector<Value, 4> sizes;
210 SmallVector<Value, 4> strides;
211 Value sizeBytes;
212 getMemRefDescriptorSizes(loc, memRefType, dynamicSizes: {}, rewriter, sizes, strides,
213 size&: sizeBytes);
214 MemRefDescriptor descriptor(operand.value());
215 Value src = descriptor.allocatedPtr(builder&: rewriter, loc);
216
217 // Get the global variable in the SPIR-V module that is associated with
218 // the kernel operand. Construct its new name and create a corresponding
219 // LLVM dialect global variable.
220 spirv::GlobalVariableOp spirvGlobal = globalVariableMap[operand.index()];
221 auto pointeeType =
222 cast<spirv::PointerType>(Val: spirvGlobal.getType()).getPointeeType();
223 auto dstGlobalType = typeConverter->convertType(t: pointeeType);
224 if (!dstGlobalType)
225 return failure();
226 std::string name =
227 createGlobalVariableWithBindName(op: spirvGlobal, kernelModuleName: spvModuleName);
228 // Check if this variable has already been created.
229 auto dstGlobal = module.lookupSymbol<LLVM::GlobalOp>(name);
230 if (!dstGlobal) {
231 OpBuilder::InsertionGuard guard(rewriter);
232 rewriter.setInsertionPointToStart(module.getBody());
233 dstGlobal = LLVM::GlobalOp::create(
234 builder&: rewriter, location: loc, type: dstGlobalType,
235 /*isConstant=*/false, linkage: LLVM::Linkage::Linkonce, name, value: Attribute(),
236 /*alignment=*/0);
237 rewriter.setInsertionPoint(launchOp);
238 }
239
240 // Copy the data from src operand pointer to dst global variable. Save
241 // src, dst and size so that we can copy data back after emulating the
242 // kernel call.
243 Value dst = LLVM::AddressOfOp::create(
244 builder&: rewriter, location: loc, res: typeConverter->convertType(t: spirvGlobal.getType()),
245 global_name: dstGlobal.getSymName());
246 copy(loc, dst, src, size: sizeBytes, builder&: rewriter);
247
248 CopyInfo info;
249 info.dst = dst;
250 info.src = src;
251 info.size = sizeBytes;
252 copyInfo.push_back(Elt: info);
253 }
254 // Create a call to the kernel and copy the data back.
255 rewriter.replaceOpWithNewOp<LLVM::CallOp>(op, args&: kernelFunc,
256 args: ArrayRef<Value>());
257 for (CopyInfo info : copyInfo)
258 copy(loc, dst: info.src, src: info.dst, size: info.size, builder&: rewriter);
259 return success();
260 }
261};
262
263class LowerHostCodeToLLVM
264 : public impl::LowerHostCodeToLLVMPassBase<LowerHostCodeToLLVM> {
265public:
266 using Base::Base;
267
268 void runOnOperation() override {
269 ModuleOp module = getOperation();
270
271 // Erase the GPU module.
272 for (auto gpuModule :
273 llvm::make_early_inc_range(Range: module.getOps<gpu::GPUModuleOp>()))
274 gpuModule.erase();
275
276 // Request C wrapper emission.
277 for (auto func : module.getOps<func::FuncOp>()) {
278 func->setDiscardableAttr(name: LLVM::LLVMDialect::getEmitCWrapperAttrName(),
279 value: UnitAttr::get(context: &getContext()));
280 }
281
282 // Specify options to lower to LLVM and pull in the conversion patterns.
283 LowerToLLVMOptions options(module.getContext());
284
285 auto *context = module.getContext();
286 RewritePatternSet patterns(context);
287 LLVMTypeConverter typeConverter(context, options);
288 mlir::arith::populateArithToLLVMConversionPatterns(converter: typeConverter, patterns);
289 populateFinalizeMemRefToLLVMConversionPatterns(converter: typeConverter, patterns);
290 populateFuncToLLVMConversionPatterns(converter: typeConverter, patterns);
291 patterns.add<GPULaunchLowering>(arg&: typeConverter);
292
293 // Pull in SPIR-V type conversion patterns to convert SPIR-V global
294 // variable's type to LLVM dialect type.
295 populateSPIRVToLLVMTypeConversion(typeConverter);
296
297 ConversionTarget target(*context);
298 target.addLegalDialect<LLVM::LLVMDialect>();
299 if (failed(Result: applyPartialConversion(op: module, target, patterns: std::move(patterns))))
300 signalPassFailure();
301
302 // Finally, modify the kernel function in SPIR-V modules to avoid symbolic
303 // conflicts.
304 for (auto spvModule : module.getOps<spirv::ModuleOp>()) {
305 if (failed(Result: encodeKernelName(module: spvModule))) {
306 signalPassFailure();
307 return;
308 }
309 }
310 }
311};
312} // namespace
313