1//===- ConvertLaunchFuncToGpuRuntimeCalls.cpp - MLIR GPU lowering passes --===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements a pass to convert gpu.launch_func op into a sequence of
10// GPU runtime calls. As most of GPU runtimes does not have a stable published
11// ABI, this pass uses a slim runtime layer that builds on top of the public
12// API from GPU runtime headers.
13//
14//===----------------------------------------------------------------------===//
15
16#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
17
18#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
19#include "mlir/Conversion/AsyncToLLVM/AsyncToLLVM.h"
20#include "mlir/Conversion/ControlFlowToLLVM/ControlFlowToLLVM.h"
21#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
22#include "mlir/Conversion/ConvertToLLVM/ToLLVMPass.h"
23#include "mlir/Conversion/FuncToLLVM/ConvertFuncToLLVM.h"
24#include "mlir/Conversion/GPUCommon/GPUToLLVM.h"
25#include "mlir/Conversion/LLVMCommon/Pattern.h"
26#include "mlir/Conversion/MemRefToLLVM/MemRefToLLVM.h"
27#include "mlir/Conversion/VectorToLLVM/ConvertVectorToLLVM.h"
28#include "mlir/Dialect/Async/IR/Async.h"
29#include "mlir/Dialect/GPU/IR/GPUDialect.h"
30#include "mlir/Dialect/GPU/Transforms/Passes.h"
31#include "mlir/Dialect/LLVMIR/LLVMArithOps.h"
32#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
33#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
34#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
35#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.h"
36#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
37#include "mlir/Dialect/MemRef/IR/MemRef.h"
38#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
39#include "mlir/IR/Attributes.h"
40#include "mlir/IR/Builders.h"
41#include "mlir/IR/BuiltinOps.h"
42#include "mlir/IR/BuiltinTypes.h"
43#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
44
45#include "llvm/ADT/STLExtras.h"
46
47#define DEBUG_TYPE "gpu-to-llvm"
48
49namespace mlir {
50#define GEN_PASS_DEF_GPUTOLLVMCONVERSIONPASS
51#include "mlir/Conversion/Passes.h.inc"
52} // namespace mlir
53
54using namespace mlir;
55
56namespace {
57class GpuToLLVMConversionPass
58 : public impl::GpuToLLVMConversionPassBase<GpuToLLVMConversionPass> {
59public:
60 using Base::Base;
61 void getDependentDialects(DialectRegistry &registry) const final {
62 Base::getDependentDialects(registry);
63 registerConvertToLLVMDependentDialectLoading(registry);
64 }
65 // Run the dialect converter on the module.
66 void runOnOperation() override;
67};
68
69template <typename OpTy>
70class ConvertOpToGpuRuntimeCallPattern : public ConvertOpToLLVMPattern<OpTy> {
71public:
72 explicit ConvertOpToGpuRuntimeCallPattern(
73 const LLVMTypeConverter &typeConverter, PatternBenefit benefit = 1)
74 : ConvertOpToLLVMPattern<OpTy>(typeConverter, benefit) {}
75
76protected:
77 Value getNumElements(ConversionPatternRewriter &rewriter, Location loc,
78 MemRefType type, MemRefDescriptor desc) const {
79 Type indexType = ConvertToLLVMPattern::getIndexType();
80 if (type.hasStaticShape())
81 return ConvertToLLVMPattern::createIndexAttrConstant(
82 builder&: rewriter, loc, resultType: indexType, value: type.getNumElements());
83 // Compute the number of elements by multiplying all the dim sizes.
84 uint64_t rank = type.getRank();
85 Value numElements = desc.size(builder&: rewriter, loc, /*pos=*/0);
86 for (unsigned i = 1; i < rank; i++)
87 numElements = LLVM::MulOp::create(builder&: rewriter, location: loc, lhs: numElements,
88 rhs: desc.size(builder&: rewriter, loc, /*pos=*/i));
89 return numElements;
90 }
91
92 MLIRContext *context = &this->getTypeConverter()->getContext();
93
94 Type llvmVoidType = LLVM::LLVMVoidType::get(ctx: context);
95 LLVM::LLVMPointerType llvmPointerType = LLVM::LLVMPointerType::get(context);
96 Type llvmInt8Type = IntegerType::get(context, width: 8);
97 Type llvmInt16Type = IntegerType::get(context, width: 16);
98 Type llvmInt32Type = IntegerType::get(context, width: 32);
99 Type llvmInt64Type = IntegerType::get(context, width: 64);
100 Type llvmFloat32Type = Float32Type::get(context);
101 Type llvmIntPtrType = IntegerType::get(
102 context, width: this->getTypeConverter()->getPointerBitwidth(0));
103
104 FunctionCallBuilder streamCreateCallBuilder = {
105 "mgpuStreamCreate", llvmPointerType /* void *stream */, {}};
106 FunctionCallBuilder streamDestroyCallBuilder = {
107 "mgpuStreamDestroy", llvmVoidType, {llvmPointerType /* void *stream */}};
108 FunctionCallBuilder streamSynchronizeCallBuilder = {
109 "mgpuStreamSynchronize",
110 llvmVoidType,
111 {llvmPointerType /* void *stream */}};
112 FunctionCallBuilder streamWaitEventCallBuilder = {
113 "mgpuStreamWaitEvent",
114 llvmVoidType,
115 {llvmPointerType /* void *stream */, llvmPointerType /* void *event */}};
116 FunctionCallBuilder eventCreateCallBuilder = {
117 "mgpuEventCreate", llvmPointerType /* void *event */, {}};
118 FunctionCallBuilder eventDestroyCallBuilder = {
119 "mgpuEventDestroy", llvmVoidType, {llvmPointerType /* void *event */}};
120 FunctionCallBuilder eventSynchronizeCallBuilder = {
121 "mgpuEventSynchronize",
122 llvmVoidType,
123 {llvmPointerType /* void *event */}};
124 FunctionCallBuilder eventRecordCallBuilder = {
125 "mgpuEventRecord",
126 llvmVoidType,
127 {llvmPointerType /* void *event */, llvmPointerType /* void *stream */}};
128 FunctionCallBuilder hostRegisterCallBuilder = {
129 "mgpuMemHostRegisterMemRef",
130 llvmVoidType,
131 {llvmIntPtrType /* intptr_t rank */,
132 llvmPointerType /* void *memrefDesc */,
133 llvmIntPtrType /* intptr_t elementSizeBytes */}};
134 FunctionCallBuilder hostUnregisterCallBuilder = {
135 "mgpuMemHostUnregisterMemRef",
136 llvmVoidType,
137 {llvmIntPtrType /* intptr_t rank */,
138 llvmPointerType /* void *memrefDesc */,
139 llvmIntPtrType /* intptr_t elementSizeBytes */}};
140 FunctionCallBuilder allocCallBuilder = {
141 "mgpuMemAlloc",
142 llvmPointerType /* void * */,
143 {llvmIntPtrType /* intptr_t sizeBytes */,
144 llvmPointerType /* void *stream */,
145 llvmInt8Type /* bool isHostShared */}};
146 FunctionCallBuilder deallocCallBuilder = {
147 "mgpuMemFree",
148 llvmVoidType,
149 {llvmPointerType /* void *ptr */, llvmPointerType /* void *stream */}};
150 FunctionCallBuilder memcpyCallBuilder = {
151 "mgpuMemcpy",
152 llvmVoidType,
153 {llvmPointerType /* void *dst */, llvmPointerType /* void *src */,
154 llvmIntPtrType /* intptr_t sizeBytes */,
155 llvmPointerType /* void *stream */}};
156 FunctionCallBuilder memset16CallBuilder = {
157 "mgpuMemset16",
158 llvmVoidType,
159 {llvmPointerType /* void *dst */,
160 llvmInt16Type /* unsigned short value */,
161 llvmIntPtrType /* intptr_t sizeBytes */,
162 llvmPointerType /* void *stream */}};
163 FunctionCallBuilder memset32CallBuilder = {
164 "mgpuMemset32",
165 llvmVoidType,
166 {llvmPointerType /* void *dst */, llvmInt32Type /* unsigned int value */,
167 llvmIntPtrType /* intptr_t sizeBytes */,
168 llvmPointerType /* void *stream */}};
169 FunctionCallBuilder setDefaultDeviceCallBuilder = {
170 "mgpuSetDefaultDevice",
171 llvmVoidType,
172 {llvmInt32Type /* uint32_t devIndex */}};
173 FunctionCallBuilder createDnVecCallBuilder = {
174 "mgpuCreateDnVec",
175 llvmPointerType,
176 {llvmIntPtrType, llvmPointerType, llvmInt32Type,
177 llvmPointerType /* void *stream */}};
178 FunctionCallBuilder destroyDnVecCallBuilder = {
179 "mgpuDestroyDnVec",
180 llvmVoidType,
181 {llvmPointerType, llvmPointerType /* void *stream */}};
182 FunctionCallBuilder createDnMatCallBuilder = {
183 "mgpuCreateDnMat",
184 llvmPointerType,
185 {llvmIntPtrType, llvmIntPtrType, llvmPointerType, llvmInt32Type,
186 llvmPointerType /* void *stream */}};
187 FunctionCallBuilder destroyDnMatCallBuilder = {
188 "mgpuDestroyDnMat",
189 llvmVoidType,
190 {llvmPointerType, llvmPointerType /* void *stream */}};
191 FunctionCallBuilder createCooCallBuilder = {
192 "mgpuCreateCoo",
193 llvmPointerType,
194 {llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
195 llvmPointerType, llvmPointerType, llvmInt32Type, llvmInt32Type,
196 llvmPointerType /* void *stream */}};
197 FunctionCallBuilder createCooAoSCallBuilder = {
198 "mgpuCreateCooAoS", // deprecated in cuSPARSE 11.2
199 llvmPointerType,
200 {llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
201 llvmPointerType, llvmInt32Type, llvmInt32Type,
202 llvmPointerType /* void *stream */}};
203 FunctionCallBuilder createCsrCallBuilder = {
204 "mgpuCreateCsr",
205 llvmPointerType,
206 {llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
207 llvmPointerType, llvmPointerType, llvmInt32Type, llvmInt32Type,
208 llvmInt32Type, llvmPointerType /* void *stream */}};
209 FunctionCallBuilder createCscCallBuilder = {
210 "mgpuCreateCsc",
211 llvmPointerType,
212 {llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
213 llvmPointerType, llvmPointerType, llvmInt32Type, llvmInt32Type,
214 llvmInt32Type, llvmPointerType /* void *stream */}};
215 FunctionCallBuilder createBsrCallBuilder = {
216 "mgpuCreateBsr",
217 llvmPointerType,
218 {llvmIntPtrType, llvmIntPtrType, llvmIntPtrType, llvmIntPtrType,
219 llvmIntPtrType, llvmPointerType, llvmPointerType, llvmPointerType,
220 llvmInt32Type, llvmInt32Type, llvmInt32Type,
221 llvmPointerType /* void *stream */}};
222 FunctionCallBuilder destroySpMatCallBuilder = {
223 "mgpuDestroySpMat",
224 llvmVoidType,
225 {llvmPointerType, llvmPointerType /* void *stream */}};
226 FunctionCallBuilder spMVBufferSizeCallBuilder = {
227 "mgpuSpMVBufferSize",
228 llvmIntPtrType,
229 {llvmInt32Type, llvmPointerType, llvmPointerType, llvmPointerType,
230 llvmInt32Type, llvmPointerType /* void *stream */}};
231 FunctionCallBuilder spMVCallBuilder = {
232 "mgpuSpMV",
233 llvmVoidType,
234 {llvmInt32Type, llvmPointerType, llvmPointerType, llvmPointerType,
235 llvmInt32Type, llvmPointerType, llvmPointerType /* void *stream */}};
236 FunctionCallBuilder createSpMMBufferSizeCallBuilder = {
237 "mgpuSpMMBufferSize",
238 llvmIntPtrType,
239 {llvmInt32Type, llvmInt32Type, llvmPointerType, llvmPointerType,
240 llvmPointerType, llvmInt32Type, llvmPointerType /* void *stream */}};
241 FunctionCallBuilder createSpMMCallBuilder = {
242 "mgpuSpMM",
243 llvmVoidType,
244 {llvmInt32Type, llvmInt32Type, llvmPointerType, llvmPointerType,
245 llvmPointerType, llvmInt32Type, llvmPointerType,
246 llvmPointerType /* void *stream */}};
247 FunctionCallBuilder createSDDMMBufferSizeCallBuilder = {
248 "mgpuSDDMMBufferSize",
249 llvmIntPtrType,
250 {llvmInt32Type, llvmInt32Type, llvmPointerType, llvmPointerType,
251 llvmPointerType, llvmInt32Type, llvmPointerType /* void *stream */}};
252 FunctionCallBuilder createSDDMMCallBuilder = {
253 "mgpuSDDMM",
254 llvmVoidType,
255 {llvmInt32Type, llvmInt32Type, llvmPointerType, llvmPointerType,
256 llvmPointerType, llvmInt32Type, llvmPointerType,
257 llvmPointerType /* void *stream */}};
258 FunctionCallBuilder createLtDnMatCallBuilder = {
259 "mgpuCreateCuSparseLtDnMat",
260 llvmVoidType,
261 {llvmPointerType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
262 llvmInt32Type, llvmPointerType /* void *stream */}};
263 FunctionCallBuilder destroyCuSparseLtSpMatBuilder = {
264 "mgpuDestroyCuSparseLtSpMat",
265 llvmVoidType,
266 {llvmPointerType, llvmPointerType /* void *stream */}};
267 FunctionCallBuilder destroyCuSparseLtDnMatBuilder = {
268 "mgpuDestroyCuSparseLtDnMat",
269 llvmVoidType,
270 {llvmPointerType, llvmPointerType /* void *stream */}};
271 FunctionCallBuilder create2To4SpMatCallBuilder = {
272 "mgpuCusparseLtCreate2To4SpMat",
273 llvmVoidType,
274 {llvmPointerType, llvmIntPtrType, llvmIntPtrType, llvmPointerType,
275 llvmInt32Type, llvmPointerType /* void *stream */}};
276 FunctionCallBuilder createCuSparseLtSpMMBufferSizeBuilder = {
277 "mgpuCuSparseLtSpMMBufferSize",
278 llvmVoidType,
279 {llvmPointerType, llvmInt32Type, llvmInt32Type, llvmPointerType,
280 llvmPointerType, llvmPointerType, llvmInt32Type, llvmInt32Type,
281 llvmPointerType /*void *stream*/}};
282 FunctionCallBuilder createCuSparseLtSpMMBuilder = {
283 "mgpuCuSparseLtSpMM",
284 llvmVoidType,
285 {llvmPointerType, llvmPointerType, llvmPointerType, llvmPointerType,
286 llvmPointerType, llvmPointerType, llvmPointerType /*void *stream*/}};
287 FunctionCallBuilder createSpGEMMCreateDescrBuilder = {
288 "mgpuSpGEMMCreateDescr",
289 llvmPointerType,
290 {llvmPointerType /*void *stream*/}};
291 FunctionCallBuilder createSpGEMMDestroyDescrBuilder = {
292 "mgpuSpGEMMDestroyDescr",
293 llvmVoidType,
294 {llvmPointerType /*s*/, llvmPointerType /*void *stream*/}};
295 FunctionCallBuilder createSpGEMMWorkEstimationBuilder = {
296 "mgpuSpGEMMWorkEstimation",
297 llvmIntPtrType,
298 {llvmPointerType /*s*/, llvmInt32Type /*ma*/, llvmInt32Type /*mb*/,
299 llvmPointerType /*a*/, llvmPointerType /*b*/, llvmPointerType /*c*/,
300 llvmInt32Type /*ctp*/, llvmIntPtrType /*bs*/, llvmPointerType /*buf*/,
301 llvmPointerType /*void *stream*/}};
302 FunctionCallBuilder createSpGEMMComputeBuilder = {
303 "mgpuSpGEMMCompute",
304 llvmIntPtrType,
305 {llvmPointerType /*s*/, llvmInt32Type /*ma*/, llvmInt32Type /*mb*/,
306 llvmPointerType /*a*/, llvmPointerType /*b*/, llvmPointerType /*c*/,
307 llvmInt32Type /*ctp*/, llvmIntPtrType /*bs*/, llvmPointerType /*buf*/,
308 llvmPointerType /*void *stream*/}};
309 FunctionCallBuilder createSpGEMMCopyBuilder = {
310 "mgpuSpGEMMCopy",
311 llvmVoidType,
312 {llvmPointerType /*s*/, llvmInt32Type /*ma*/, llvmInt32Type /*mb*/,
313 llvmPointerType /*a*/, llvmPointerType /*b*/, llvmPointerType /*c*/,
314 llvmInt32Type /*ctp*/, llvmPointerType /*void *stream*/}};
315 FunctionCallBuilder createSpMatGetSizeBuilder = {
316 "mgpuSpMatGetSize",
317 llvmVoidType,
318 {llvmPointerType /*mc*/, llvmPointerType /*rc*/, llvmPointerType /*cc*/,
319 llvmPointerType /*nc*/, llvmPointerType /*void *stream*/}};
320 FunctionCallBuilder createSetCsrPointersBuilder = {
321 "mgpuSetCsrPointers",
322 llvmVoidType,
323 {llvmPointerType /*spmat*/, llvmPointerType /*pos*/,
324 llvmPointerType /*crd*/, llvmPointerType /*val*/,
325 llvmPointerType /*void *stream*/}};
326};
327
328/// A rewrite pattern to convert gpu.host_register operations into a GPU runtime
329/// call. Currently it supports CUDA and ROCm (HIP).
330class ConvertHostRegisterOpToGpuRuntimeCallPattern
331 : public ConvertOpToGpuRuntimeCallPattern<gpu::HostRegisterOp> {
332public:
333 ConvertHostRegisterOpToGpuRuntimeCallPattern(
334 const LLVMTypeConverter &typeConverter)
335 : ConvertOpToGpuRuntimeCallPattern<gpu::HostRegisterOp>(typeConverter) {}
336
337private:
338 LogicalResult
339 matchAndRewrite(gpu::HostRegisterOp hostRegisterOp, OpAdaptor adaptor,
340 ConversionPatternRewriter &rewriter) const override;
341};
342
343class ConvertHostUnregisterOpToGpuRuntimeCallPattern
344 : public ConvertOpToGpuRuntimeCallPattern<gpu::HostUnregisterOp> {
345public:
346 ConvertHostUnregisterOpToGpuRuntimeCallPattern(
347 const LLVMTypeConverter &typeConverter)
348 : ConvertOpToGpuRuntimeCallPattern<gpu::HostUnregisterOp>(typeConverter) {
349 }
350
351private:
352 LogicalResult
353 matchAndRewrite(gpu::HostUnregisterOp hostUnregisterOp, OpAdaptor adaptor,
354 ConversionPatternRewriter &rewriter) const override;
355};
356
357/// A rewrite pattern to convert gpu.alloc operations into a GPU runtime
358/// call. Currently it supports CUDA and ROCm (HIP).
359class ConvertAllocOpToGpuRuntimeCallPattern
360 : public ConvertOpToGpuRuntimeCallPattern<gpu::AllocOp> {
361public:
362 ConvertAllocOpToGpuRuntimeCallPattern(const LLVMTypeConverter &typeConverter)
363 : ConvertOpToGpuRuntimeCallPattern<gpu::AllocOp>(typeConverter) {}
364
365private:
366 LogicalResult
367 matchAndRewrite(gpu::AllocOp allocOp, OpAdaptor adaptor,
368 ConversionPatternRewriter &rewriter) const override;
369};
370
371/// A rewrite pattern to convert gpu.dealloc operations into a GPU runtime
372/// call. Currently it supports CUDA and ROCm (HIP).
373class ConvertDeallocOpToGpuRuntimeCallPattern
374 : public ConvertOpToGpuRuntimeCallPattern<gpu::DeallocOp> {
375public:
376 ConvertDeallocOpToGpuRuntimeCallPattern(
377 const LLVMTypeConverter &typeConverter)
378 : ConvertOpToGpuRuntimeCallPattern<gpu::DeallocOp>(typeConverter) {}
379
380private:
381 LogicalResult
382 matchAndRewrite(gpu::DeallocOp deallocOp, OpAdaptor adaptor,
383 ConversionPatternRewriter &rewriter) const override;
384};
385
386class ConvertAsyncYieldToGpuRuntimeCallPattern
387 : public ConvertOpToGpuRuntimeCallPattern<async::YieldOp> {
388public:
389 ConvertAsyncYieldToGpuRuntimeCallPattern(
390 const LLVMTypeConverter &typeConverter, PatternBenefit benefit = 1)
391 : ConvertOpToGpuRuntimeCallPattern<async::YieldOp>(typeConverter,
392 benefit) {}
393
394private:
395 LogicalResult
396 matchAndRewrite(async::YieldOp yieldOp, OpAdaptor adaptor,
397 ConversionPatternRewriter &rewriter) const override;
398};
399
400/// A rewrite pattern to convert gpu.wait operations into a GPU runtime
401/// call. Currently it supports CUDA and ROCm (HIP).
402class ConvertWaitOpToGpuRuntimeCallPattern
403 : public ConvertOpToGpuRuntimeCallPattern<gpu::WaitOp> {
404public:
405 ConvertWaitOpToGpuRuntimeCallPattern(const LLVMTypeConverter &typeConverter)
406 : ConvertOpToGpuRuntimeCallPattern<gpu::WaitOp>(typeConverter) {}
407
408private:
409 LogicalResult
410 matchAndRewrite(gpu::WaitOp waitOp, OpAdaptor adaptor,
411 ConversionPatternRewriter &rewriter) const override;
412};
413
414/// A rewrite pattern to convert gpu.wait async operations into a GPU runtime
415/// call. Currently it supports CUDA and ROCm (HIP).
416class ConvertWaitAsyncOpToGpuRuntimeCallPattern
417 : public ConvertOpToGpuRuntimeCallPattern<gpu::WaitOp> {
418public:
419 ConvertWaitAsyncOpToGpuRuntimeCallPattern(
420 const LLVMTypeConverter &typeConverter)
421 : ConvertOpToGpuRuntimeCallPattern<gpu::WaitOp>(typeConverter) {}
422
423private:
424 LogicalResult
425 matchAndRewrite(gpu::WaitOp waitOp, OpAdaptor adaptor,
426 ConversionPatternRewriter &rewriter) const override;
427};
428
429/// A rewrite patter to legalize gpu.launch_func with LLVM types.
430class LegalizeLaunchFuncOpPattern
431 : public ConvertOpToGpuRuntimeCallPattern<gpu::LaunchFuncOp> {
432public:
433 LegalizeLaunchFuncOpPattern(const LLVMTypeConverter &typeConverter,
434 bool kernelBarePtrCallConv,
435 bool kernelIntersperseSizeCallConv)
436 : ConvertOpToGpuRuntimeCallPattern<gpu::LaunchFuncOp>(typeConverter),
437 kernelBarePtrCallConv(kernelBarePtrCallConv),
438 kernelIntersperseSizeCallConv(kernelIntersperseSizeCallConv) {}
439
440private:
441 LogicalResult
442 matchAndRewrite(gpu::LaunchFuncOp launchOp, OpAdaptor adaptor,
443 ConversionPatternRewriter &rewriter) const override;
444
445 bool kernelBarePtrCallConv;
446 bool kernelIntersperseSizeCallConv;
447};
448
449/// A rewrite pattern to convert gpu.memcpy operations into a GPU runtime
450/// call. Currently it supports CUDA and ROCm (HIP).
451class ConvertMemcpyOpToGpuRuntimeCallPattern
452 : public ConvertOpToGpuRuntimeCallPattern<gpu::MemcpyOp> {
453public:
454 ConvertMemcpyOpToGpuRuntimeCallPattern(const LLVMTypeConverter &typeConverter)
455 : ConvertOpToGpuRuntimeCallPattern<gpu::MemcpyOp>(typeConverter) {}
456
457private:
458 LogicalResult
459 matchAndRewrite(gpu::MemcpyOp memcpyOp, OpAdaptor adaptor,
460 ConversionPatternRewriter &rewriter) const override;
461};
462
463/// A rewrite pattern to convert gpu.memset operations into a GPU runtime
464/// call. Currently it supports CUDA and ROCm (HIP).
465class ConvertMemsetOpToGpuRuntimeCallPattern
466 : public ConvertOpToGpuRuntimeCallPattern<gpu::MemsetOp> {
467public:
468 ConvertMemsetOpToGpuRuntimeCallPattern(const LLVMTypeConverter &typeConverter)
469 : ConvertOpToGpuRuntimeCallPattern<gpu::MemsetOp>(typeConverter) {}
470
471private:
472 LogicalResult
473 matchAndRewrite(gpu::MemsetOp memsetOp, OpAdaptor adaptor,
474 ConversionPatternRewriter &rewriter) const override;
475};
476
477/// A rewrite pattern to convert gpu.set_default_device to a GPU runtime call.
478/// Currently supports CUDA and ROCm (HIP)
479class ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern
480 : public ConvertOpToGpuRuntimeCallPattern<gpu::SetDefaultDeviceOp> {
481public:
482 ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern(
483 const LLVMTypeConverter &typeConverter)
484 : ConvertOpToGpuRuntimeCallPattern<gpu::SetDefaultDeviceOp>(
485 typeConverter) {}
486
487 LogicalResult
488 matchAndRewrite(gpu::SetDefaultDeviceOp op, OpAdaptor adaptor,
489 ConversionPatternRewriter &rewriter) const override;
490};
491
492/// Generic rewriting rule for operation on sparse matrices.
493/// Currently supports CUDA (by means of cuSparse and cuSparseLt).
494#define DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(op_name) \
495 class Convert##op_name##ToGpuRuntimeCallPattern \
496 : public ConvertOpToGpuRuntimeCallPattern<gpu::op_name> { \
497 public: \
498 Convert##op_name##ToGpuRuntimeCallPattern( \
499 const LLVMTypeConverter &typeConverter) \
500 : ConvertOpToGpuRuntimeCallPattern<gpu::op_name>(typeConverter) {} \
501 \
502 private: \
503 LogicalResult \
504 matchAndRewrite(gpu::op_name op, OpAdaptor adaptor, \
505 ConversionPatternRewriter &rewriter) const override; \
506 };
507
508DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(CreateDnTensorOp)
509DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(DestroyDnTensorOp)
510DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(CreateCooOp)
511DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(CreateCooAoSOp)
512DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(CreateCsrOp)
513DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(CreateCscOp)
514DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(CreateBsrOp)
515DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(Create2To4SpMatOp)
516DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(DestroySpMatOp)
517DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpMVBufferSizeOp)
518DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpMVOp)
519DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpMMBufferSizeOp)
520DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SDDMMBufferSizeOp)
521DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpMMOp)
522DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SDDMMOp)
523DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpGEMMCreateDescrOp)
524DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpGEMMDestroyDescrOp)
525DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpGEMMWorkEstimationOrComputeOp)
526DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpGEMMCopyOp)
527DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SpMatGetSizeOp)
528DECLARE_CONVERT_OP_TO_GPU_RUNTIME_CALL_PATTERN(SetCsrPointersOp)
529
530} // namespace
531
532void GpuToLLVMConversionPass::runOnOperation() {
533 MLIRContext *context = &getContext();
534
535 // Perform progressive lowering of vector transfer operations.
536 {
537 RewritePatternSet patterns(&getContext());
538 // Vector transfer ops with rank > 1 should be lowered with VectorToSCF.
539 vector::populateVectorTransferLoweringPatterns(patterns,
540 /*maxTransferRank=*/1);
541 // Transform N-D vector.from_elements to 1-D vector.from_elements before
542 // conversion.
543 vector::populateVectorFromElementsUnrollPatterns(patterns);
544 if (failed(Result: applyPatternsGreedily(op: getOperation(), patterns: std::move(patterns))))
545 return signalPassFailure();
546 }
547
548 LowerToLLVMOptions options(context);
549 options.useBarePtrCallConv = hostBarePtrCallConv;
550 RewritePatternSet patterns(context);
551 ConversionTarget target(*context);
552 target.addLegalDialect<LLVM::LLVMDialect>();
553 LLVMTypeConverter converter(context, options);
554
555 // Populate all patterns from all dialects that implement the
556 // `ConvertToLLVMPatternInterface` interface.
557 std::vector<Dialect *> dialects = context->getLoadedDialects();
558 for (auto *iface :
559 llvm::make_isa_range<ConvertToLLVMPatternInterface>(Range&: dialects))
560 iface->populateConvertToLLVMConversionPatterns(target, typeConverter&: converter, patterns);
561
562 // Preserve GPU modules and binaries. Modules are preserved as they can be
563 // converted later by `gpu-module-to-binary`.
564 target.addLegalOp<gpu::GPUModuleOp, gpu::BinaryOp>();
565 // Accept as legal LaunchFuncOps if the operands have been lowered.
566 target.addDynamicallyLegalOp<gpu::LaunchFuncOp>(
567 callback: [&](gpu::LaunchFuncOp op) -> bool { return converter.isLegal(op); });
568
569 // These aren't covered by the ConvertToLLVMPatternInterface right now.
570 populateVectorToLLVMConversionPatterns(converter, patterns);
571 populateFinalizeMemRefToLLVMConversionPatterns(converter, patterns);
572 populateAsyncStructuralTypeConversionsAndLegality(typeConverter&: converter, patterns,
573 target);
574 populateGpuToLLVMConversionPatterns(converter, patterns,
575 kernelBarePtrCallConv,
576 kernelIntersperseSizeCallConv);
577
578 if (failed(
579 Result: applyPartialConversion(op: getOperation(), target, patterns: std::move(patterns))))
580 signalPassFailure();
581}
582
583LLVM::CallOp FunctionCallBuilder::create(Location loc, OpBuilder &builder,
584 ArrayRef<Value> arguments) const {
585 auto module = builder.getBlock()->getParent()->getParentOfType<ModuleOp>();
586 auto function = [&] {
587 if (auto function = module.lookupSymbol<LLVM::LLVMFuncOp>(name: functionName))
588 return function;
589 auto builder = OpBuilder::atBlockEnd(block: module.getBody());
590 return LLVM::LLVMFuncOp::create(builder, location: loc, name: functionName, type: functionType);
591 }();
592 return LLVM::CallOp::create(builder, location: loc, func: function, args: arguments);
593}
594
595// Corresponding to cusparseIndexType_t defined in cusparse.h.
596static int32_t getCuSparseIndexTypeFrom(Type type) {
597 if (type.isInteger(width: 16))
598 return 1; // CUSPARSE_INDEX_16U
599 if (type.isInteger(width: 32))
600 return 2; // CUSPARSE_INDEX_32I
601 return 3; // CUSPARSE_INDEX_64I
602}
603
604static int32_t getCuSparseLtDataTypeFrom(Type type) {
605 if (type.isF16())
606 return 0; // CUSPARSE_COMPUTE_16F,
607 if (type.isInteger(width: 32))
608 return 1; // CUSPARSE_COMPUTE_32I
609 llvm_unreachable("unsupported type");
610 // TODO: add support to TF32
611}
612
613// Corresponding to cudaDataType_t defined in CUDA library_types.h.
614static int32_t getCuSparseDataTypeFrom(Type type) {
615 if (llvm::isa<ComplexType>(Val: type)) {
616 // get the element type
617 auto elementType = cast<ComplexType>(Val&: type).getElementType();
618 if (elementType.isBF16())
619 return 15; // CUDA_C_16BF
620 if (elementType.isF16())
621 return 6; // CUDA_C_16F
622 if (elementType.isF32())
623 return 4; // CUDA_C_32F
624 if (elementType.isF64())
625 return 5; // CUDA_C_64F
626 if (elementType.isInteger(width: 8))
627 return 7; // CUDA_C_8I
628 if (elementType.isInteger(width: 16))
629 return 21; // CUDA_C_16I
630 if (elementType.isInteger(width: 32))
631 return 11; // CUDA_C_32I
632 }
633 if (type.isBF16())
634 return 14; // CUDA_R_16BF
635 if (type.isF16())
636 return 2; // CUDA_R_16F
637 if (type.isF32())
638 return 0; // CUDA_R_32F
639 if (type.isF64())
640 return 1; // CUDA_R_64F
641 if (type.isInteger(width: 8))
642 return 3; // CUDA_R_8I
643 if (type.isInteger(width: 16))
644 return 20; // CUDA_R_16I
645 if (type.isInteger(width: 32))
646 return 10; // CUDA_R_32I
647
648 llvm_unreachable("unsupported element type");
649}
650
651static gpu::Prune2To4SpMatFlag get2To4PruneFlag(Value spMat) {
652 return spMat.getDefiningOp<gpu::Create2To4SpMatOp>().getPruneFlag();
653}
654
655// TODO: We may want a run-time (of the mlir compiler) disablement/warning:
656// cusparseLt currently won't work for cuda architecture <8.0 and will trigger a
657// runtime (of the CUDA program) error , but it might be great if we could at
658// least output a warning when we found the target architecture is <8.0 and the
659// user still wants to use cusparseLt. to make sure when lowering gpu sparse
660// dialect to llvm calls, the cusparselt calls are disabled for cuda
661// architecture <8.0
662static bool is2To4Sparsity(Value spMat) {
663 if (auto op = spMat.getDefiningOp<gpu::Create2To4SpMatOp>())
664 return true;
665 if (auto op = spMat.getDefiningOp<gpu::CreateCooOp>())
666 return false;
667 if (auto op = spMat.getDefiningOp<gpu::CreateCooAoSOp>())
668 return false;
669 if (auto op = spMat.getDefiningOp<gpu::CreateCsrOp>())
670 return false;
671 if (auto op = spMat.getDefiningOp<gpu::CreateCscOp>())
672 return false;
673 if (auto op = spMat.getDefiningOp<gpu::CreateBsrOp>())
674 return false;
675 // Print the spMat defining op
676 spMat.getDefiningOp()->print(os&: llvm::errs());
677 llvm_unreachable("cannot find spmat def");
678}
679
680static bool isSpMMCusparseLtOp(Value op) {
681 for (Operation *user : op.getUsers()) {
682 auto spmmOp = dyn_cast<gpu::SpMMOp>(Val: user);
683 // If the other operator is 50% sparsity then we should use cusparseLt
684 if (!spmmOp)
685 continue;
686 if (is2To4Sparsity(spMat: spmmOp.getSpmatA()))
687 return true;
688 }
689 return false;
690}
691
692// Returns whether all operands are of LLVM type.
693static LogicalResult areAllLLVMTypes(Operation *op, ValueRange operands,
694 ConversionPatternRewriter &rewriter) {
695 if (!llvm::all_of(Range&: operands, P: [](Value value) {
696 return LLVM::isCompatibleType(type: value.getType());
697 }))
698 return rewriter.notifyMatchFailure(
699 arg&: op, msg: "Cannot convert if operands aren't of LLVM type.");
700 return success();
701}
702
703static LogicalResult
704isAsyncWithOneDependency(ConversionPatternRewriter &rewriter,
705 gpu::AsyncOpInterface op) {
706 if (op.getAsyncDependencies().size() != 1)
707 return rewriter.notifyMatchFailure(
708 arg&: op, msg: "Can only convert with exactly one async dependency.");
709
710 if (!op.getAsyncToken())
711 return rewriter.notifyMatchFailure(arg&: op, msg: "Can convert only async version.");
712
713 return success();
714}
715
716LogicalResult ConvertHostRegisterOpToGpuRuntimeCallPattern::matchAndRewrite(
717 gpu::HostRegisterOp hostRegisterOp, OpAdaptor adaptor,
718 ConversionPatternRewriter &rewriter) const {
719 auto *op = hostRegisterOp.getOperation();
720 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)))
721 return failure();
722
723 Location loc = op->getLoc();
724
725 auto memRefType = hostRegisterOp.getValue().getType();
726 auto elementType = cast<UnrankedMemRefType>(Val&: memRefType).getElementType();
727 auto elementSize = getSizeInBytes(loc, type: elementType, rewriter);
728
729 auto arguments = getTypeConverter()->promoteOperands(
730 loc, opOperands: op->getOperands(), adaptorOperands: adaptor.getOperands(), builder&: rewriter);
731 arguments.push_back(Elt: elementSize);
732 hostRegisterCallBuilder.create(loc, builder&: rewriter, arguments);
733
734 rewriter.eraseOp(op);
735 return success();
736}
737
738LogicalResult ConvertHostUnregisterOpToGpuRuntimeCallPattern::matchAndRewrite(
739 gpu::HostUnregisterOp hostUnregisterOp, OpAdaptor adaptor,
740 ConversionPatternRewriter &rewriter) const {
741 Operation *op = hostUnregisterOp.getOperation();
742 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)))
743 return failure();
744
745 Location loc = op->getLoc();
746
747 auto memRefType = hostUnregisterOp.getValue().getType();
748 auto elementType = cast<UnrankedMemRefType>(Val&: memRefType).getElementType();
749 auto elementSize = getSizeInBytes(loc, type: elementType, rewriter);
750
751 auto arguments = getTypeConverter()->promoteOperands(
752 loc, opOperands: op->getOperands(), adaptorOperands: adaptor.getOperands(), builder&: rewriter);
753 arguments.push_back(Elt: elementSize);
754 hostUnregisterCallBuilder.create(loc, builder&: rewriter, arguments);
755
756 rewriter.eraseOp(op);
757 return success();
758}
759
760LogicalResult ConvertAllocOpToGpuRuntimeCallPattern::matchAndRewrite(
761 gpu::AllocOp allocOp, OpAdaptor adaptor,
762 ConversionPatternRewriter &rewriter) const {
763
764 MemRefType memRefType = allocOp.getType();
765
766 if (failed(Result: areAllLLVMTypes(op: allocOp, operands: adaptor.getOperands(), rewriter)) ||
767 !isConvertibleAndHasIdentityMaps(type: memRefType))
768 return failure();
769
770 auto loc = allocOp.getLoc();
771
772 bool isShared = allocOp.getHostShared();
773
774 if (isShared && allocOp.getAsyncToken())
775 return rewriter.notifyMatchFailure(
776 arg&: allocOp, msg: "Host Shared allocation cannot be done async");
777 if (adaptor.getAsyncDependencies().size() > 1)
778 return rewriter.notifyMatchFailure(
779 arg&: allocOp, msg: "Can convert with at most one async dependency.");
780
781 // Get shape of the memref as values: static sizes are constant
782 // values and dynamic sizes are passed to 'alloc' as operands.
783 SmallVector<Value, 4> shape;
784 SmallVector<Value, 4> strides;
785 Value sizeBytes;
786 getMemRefDescriptorSizes(loc, memRefType, dynamicSizes: adaptor.getDynamicSizes(), rewriter,
787 sizes&: shape, strides, size&: sizeBytes);
788
789 // Allocate the underlying buffer and store a pointer to it in the MemRef
790 // descriptor.
791 auto nullPtr = mlir::LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: llvmPointerType);
792 Value stream = adaptor.getAsyncDependencies().empty()
793 ? nullPtr
794 : adaptor.getAsyncDependencies().front();
795
796 auto isHostShared = mlir::LLVM::ConstantOp::create(
797 builder&: rewriter, location: loc, res: llvmInt8Type, value: rewriter.getI8IntegerAttr(value: isShared));
798
799 Value allocatedPtr =
800 allocCallBuilder.create(loc, builder&: rewriter, arguments: {sizeBytes, stream, isHostShared})
801 .getResult();
802
803 // Cast the runtime-returned pointer to the memref's address space if it
804 // isn't there already, so it fits the descriptor's pointer slots.
805 unsigned dstAddrSpace =
806 *getTypeConverter()->getMemRefAddressSpace(type: memRefType);
807 unsigned srcAddrSpace =
808 cast<LLVM::LLVMPointerType>(Val: allocatedPtr.getType()).getAddressSpace();
809 if (dstAddrSpace != srcAddrSpace) {
810 auto targetPtrTy =
811 LLVM::LLVMPointerType::get(context: rewriter.getContext(), addressSpace: dstAddrSpace);
812 allocatedPtr =
813 LLVM::AddrSpaceCastOp::create(builder&: rewriter, location: loc, res: targetPtrTy, arg: allocatedPtr);
814 }
815
816 // No alignment.
817 Value alignedPtr = allocatedPtr;
818
819 // Create the MemRef descriptor.
820 auto memRefDescriptor = this->createMemRefDescriptor(
821 loc, memRefType, allocatedPtr, alignedPtr, sizes: shape, strides, rewriter);
822
823 if (allocOp.getAsyncToken()) {
824 // Async alloc: make dependent ops use the same stream.
825 rewriter.replaceOp(op: allocOp, newValues: {memRefDescriptor, stream});
826 } else {
827 rewriter.replaceOp(op: allocOp, newValues: {memRefDescriptor});
828 }
829
830 return success();
831}
832
833LogicalResult ConvertDeallocOpToGpuRuntimeCallPattern::matchAndRewrite(
834 gpu::DeallocOp deallocOp, OpAdaptor adaptor,
835 ConversionPatternRewriter &rewriter) const {
836 if (failed(Result: areAllLLVMTypes(op: deallocOp, operands: adaptor.getOperands(), rewriter)))
837 return failure();
838 if (adaptor.getAsyncDependencies().size() > 1)
839 return rewriter.notifyMatchFailure(
840 arg&: deallocOp, msg: "Can convert with at most one async dependency.");
841
842 Location loc = deallocOp.getLoc();
843
844 Value pointer =
845 MemRefDescriptor(adaptor.getMemref()).allocatedPtr(builder&: rewriter, loc);
846 // The runtime frees a pointer in the default address space.
847 if (pointer.getType() != llvmPointerType)
848 pointer =
849 LLVM::AddrSpaceCastOp::create(builder&: rewriter, location: loc, res: llvmPointerType, arg: pointer);
850 auto nullPtr = mlir::LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: llvmPointerType);
851 Value stream = adaptor.getAsyncDependencies().empty()
852 ? nullPtr
853 : adaptor.getAsyncDependencies().front();
854 deallocCallBuilder.create(loc, builder&: rewriter, arguments: {pointer, stream});
855
856 if (deallocOp.getAsyncToken()) {
857 // Async dealloc: propagate the stream as the async token replacement.
858 rewriter.replaceOp(op: deallocOp, newValues: {stream});
859 } else {
860 // Sync dealloc: no results to replace, just remove the op.
861 rewriter.eraseOp(op: deallocOp);
862 }
863 return success();
864}
865
866static bool isGpuAsyncTokenType(Value value) {
867 return isa<gpu::AsyncTokenType>(Val: value.getType());
868}
869
870// Converts !gpu.async.token operands of `async.yield` to runtime calls. The
871// !gpu.async.token are lowered to stream within the async.execute region, but
872// are passed as events between them. For each !gpu.async.token operand, we
873// create an event and record it on the stream.
874//
875// This pattern is registered with a higher benefit than the structural
876// async.yield rewriter from populateAsyncStructuralTypeConversionsAndLegality
877// so it wins when both match. Without that benefit override, the structural
878// pattern can win and silently retype gpu.async.token operands without
879// recording an event, leaving the host await to call cuEventSynchronize on
880// a stream pointer (a no-op that returns an error), racing the host against
881// the GPU.
882LogicalResult ConvertAsyncYieldToGpuRuntimeCallPattern::matchAndRewrite(
883 async::YieldOp yieldOp, OpAdaptor adaptor,
884 ConversionPatternRewriter &rewriter) const {
885 if (llvm::none_of(Range: yieldOp.getOperands(), P: isGpuAsyncTokenType))
886 return rewriter.notifyMatchFailure(arg&: yieldOp, msg: "no gpu async token operand");
887
888 Location loc = yieldOp.getLoc();
889 SmallVector<Value, 4> newOperands(adaptor.getOperands());
890 llvm::SmallDenseSet<Value> streams;
891 for (auto &operand : yieldOp->getOpOperands()) {
892 if (!isGpuAsyncTokenType(value: operand.get()))
893 continue;
894 auto idx = operand.getOperandNumber();
895 auto stream = adaptor.getOperands()[idx];
896 auto event = eventCreateCallBuilder.create(loc, builder&: rewriter, arguments: {}).getResult();
897 eventRecordCallBuilder.create(loc, builder&: rewriter, arguments: {event, stream});
898 newOperands[idx] = event;
899 streams.insert(V: stream);
900 }
901 for (auto stream : streams)
902 streamDestroyCallBuilder.create(loc, builder&: rewriter, arguments: {stream});
903
904 rewriter.modifyOpInPlace(root: yieldOp, callable: [&] { yieldOp->setOperands(newOperands); });
905 return success();
906}
907
908// Returns whether `value` is the result of an LLVM::CallOp to `functionName`.
909static bool isDefinedByCallTo(Value value, StringRef functionName) {
910 assert(isa<LLVM::LLVMPointerType>(value.getType()));
911 if (auto defOp = value.getDefiningOp<LLVM::CallOp>())
912 return *defOp.getCallee() == functionName;
913 return false;
914}
915
916// Converts `gpu.wait` to runtime calls. The converted op synchronizes the host
917// with the stream/event operands. The operands are destroyed. That is, it
918// assumes that it is not used afterwards or elsewhere. Otherwise we will get a
919// runtime error. Eventually, we should guarantee this property.
920LogicalResult ConvertWaitOpToGpuRuntimeCallPattern::matchAndRewrite(
921 gpu::WaitOp waitOp, OpAdaptor adaptor,
922 ConversionPatternRewriter &rewriter) const {
923 if (waitOp.getAsyncToken())
924 return rewriter.notifyMatchFailure(arg&: waitOp, msg: "Cannot convert async op.");
925
926 Location loc = waitOp.getLoc();
927
928 for (auto operand : adaptor.getOperands()) {
929 if (isDefinedByCallTo(value: operand, functionName: streamCreateCallBuilder.functionName)) {
930 // The converted operand's definition created a stream.
931 streamSynchronizeCallBuilder.create(loc, builder&: rewriter, arguments: {operand});
932 streamDestroyCallBuilder.create(loc, builder&: rewriter, arguments: {operand});
933 } else {
934 // Otherwise the converted operand is an event. This assumes that we use
935 // events in control flow code as well.
936 eventSynchronizeCallBuilder.create(loc, builder&: rewriter, arguments: {operand});
937 eventDestroyCallBuilder.create(loc, builder&: rewriter, arguments: {operand});
938 }
939 }
940
941 rewriter.eraseOp(op: waitOp);
942 return success();
943}
944
945// Converts `gpu.wait async` to runtime calls. The converted op creates a new
946// stream that is synchronized with stream/event operands. The operands are
947// destroyed. That is, it assumes that it is not used afterwards or elsewhere.
948// Otherwise we will get a runtime error. Eventually, we should guarantee this
949// property.
950LogicalResult ConvertWaitAsyncOpToGpuRuntimeCallPattern::matchAndRewrite(
951 gpu::WaitOp waitOp, OpAdaptor adaptor,
952 ConversionPatternRewriter &rewriter) const {
953 if (!waitOp.getAsyncToken())
954 return rewriter.notifyMatchFailure(arg&: waitOp, msg: "Can only convert async op.");
955
956 Location loc = waitOp.getLoc();
957
958 auto insertionPoint = rewriter.saveInsertionPoint();
959 SmallVector<Value, 1> events;
960 for (auto pair :
961 llvm::zip(t: waitOp.getAsyncDependencies(), u: adaptor.getOperands())) {
962 auto operand = std::get<1>(t&: pair);
963 if (isDefinedByCallTo(value: operand, functionName: streamCreateCallBuilder.functionName)) {
964 // The converted operand's definition created a stream. Insert an event
965 // into the stream just after the last use of the original token operand.
966 auto *defOp = std::get<0>(t&: pair).getDefiningOp();
967 rewriter.setInsertionPointAfter(defOp);
968 auto event = eventCreateCallBuilder.create(loc, builder&: rewriter, arguments: {}).getResult();
969 eventRecordCallBuilder.create(loc, builder&: rewriter, arguments: {event, operand});
970 events.push_back(Elt: event);
971 } else {
972 // Otherwise the converted operand is an event. This assumes that we use
973 // events in control flow code as well.
974 events.push_back(Elt: operand);
975 }
976 }
977 rewriter.restoreInsertionPoint(ip: insertionPoint);
978 auto stream = streamCreateCallBuilder.create(loc, builder&: rewriter, arguments: {}).getResult();
979 for (auto event : events)
980 streamWaitEventCallBuilder.create(loc, builder&: rewriter, arguments: {stream, event});
981 for (auto event : events)
982 eventDestroyCallBuilder.create(loc, builder&: rewriter, arguments: {event});
983 rewriter.replaceOp(op: waitOp, newValues: {stream});
984
985 return success();
986}
987
988// Legalize the op's operands.
989LogicalResult LegalizeLaunchFuncOpPattern::matchAndRewrite(
990 gpu::LaunchFuncOp launchOp, OpAdaptor adaptor,
991 ConversionPatternRewriter &rewriter) const {
992 if (failed(Result: areAllLLVMTypes(op: launchOp, operands: adaptor.getOperands(), rewriter)))
993 return failure();
994
995 // Fail when the synchronous version of the op has async dependencies. The
996 // lowering destroys the stream, and we do not want to check that there is no
997 // use of the stream after this op.
998 if (!launchOp.getAsyncToken() && !launchOp.getAsyncDependencies().empty())
999 return rewriter.notifyMatchFailure(
1000 arg&: launchOp, msg: "Cannot convert non-async op with async dependencies.");
1001
1002 Location loc = launchOp.getLoc();
1003
1004 Value stream = Value();
1005 if (!adaptor.getAsyncDependencies().empty()) {
1006 stream = adaptor.getAsyncDependencies().front();
1007 // Synchronize additional async dependencies onto the primary stream using
1008 // events, following the same approach as gpu.wait async lowering.
1009 if (adaptor.getAsyncDependencies().size() > 1) {
1010 auto insertionPoint = rewriter.saveInsertionPoint();
1011 SmallVector<Value, 4> events;
1012 for (auto [origDep, convertedDep] :
1013 llvm::zip(t: launchOp.getAsyncDependencies().drop_front(),
1014 u: adaptor.getAsyncDependencies().drop_front())) {
1015 if (!isDefinedByCallTo(value: convertedDep,
1016 functionName: streamCreateCallBuilder.functionName)) {
1017 events.push_back(Elt: convertedDep);
1018 continue;
1019 }
1020 Operation *defOp = origDep.getDefiningOp();
1021 rewriter.setInsertionPointAfter(defOp);
1022 Value event =
1023 eventCreateCallBuilder.create(loc, builder&: rewriter, arguments: {}).getResult();
1024 eventRecordCallBuilder.create(loc, builder&: rewriter, arguments: {event, convertedDep});
1025 events.push_back(Elt: event);
1026 }
1027 rewriter.restoreInsertionPoint(ip: insertionPoint);
1028 for (Value event : events)
1029 streamWaitEventCallBuilder.create(loc, builder&: rewriter, arguments: {stream, event});
1030 for (Value event : events)
1031 eventDestroyCallBuilder.create(loc, builder&: rewriter, arguments: {event});
1032 }
1033 }
1034 // If the async keyword is present and there are no dependencies, then a
1035 // stream must be created to pass to subsequent operations.
1036 else if (launchOp.getAsyncToken())
1037 stream = streamCreateCallBuilder.create(loc, builder&: rewriter, arguments: {}).getResult();
1038
1039 // Lower the kernel operands to match kernel parameters.
1040 // Note: If `useBarePtrCallConv` is set in the type converter's options,
1041 // the value of `kernelBarePtrCallConv` will be ignored.
1042 OperandRange origArguments = launchOp.getKernelOperands();
1043 bool effectiveBarePtr = kernelBarePtrCallConv ||
1044 getTypeConverter()->getOptions().useBarePtrCallConv;
1045 if (effectiveBarePtr) {
1046 for (Value arg : origArguments) {
1047 if (isa<UnrankedMemRefType>(Val: arg.getType()))
1048 return rewriter.notifyMatchFailure(
1049 arg&: loc, msg: "unranked memref kernel argument is not supported with "
1050 "the bare-pointer calling convention");
1051 }
1052 }
1053 SmallVector<Value, 8> llvmArguments = getTypeConverter()->promoteOperands(
1054 loc, opOperands: origArguments, adaptorOperands: adaptor.getKernelOperands(), builder&: rewriter,
1055 /*useBarePtrCallConv=*/kernelBarePtrCallConv);
1056 SmallVector<Value, 8> llvmArgumentsWithSizes;
1057
1058 // Intersperse size information if requested.
1059 if (kernelIntersperseSizeCallConv) {
1060 if (origArguments.size() != llvmArguments.size()) {
1061 // This shouldn't happen if the bare-pointer calling convention is used.
1062 return rewriter.notifyMatchFailure(
1063 arg&: launchOp,
1064 msg: "Cannot add sizes to arguments with one-to-many LLVM IR expansion.");
1065 }
1066
1067 llvmArgumentsWithSizes.reserve(N: llvmArguments.size() * 2);
1068 for (auto [llvmArg, origArg] : zip_equal(t&: llvmArguments, u&: origArguments)) {
1069 auto memrefTy = dyn_cast<MemRefType>(Val: origArg.getType());
1070 if (!memrefTy) {
1071 return rewriter.notifyMatchFailure(
1072 arg&: launchOp, msg: "Operand to launch op is not a memref.");
1073 }
1074
1075 if (!memrefTy.hasStaticShape() ||
1076 !memrefTy.getElementType().isIntOrFloat()) {
1077 return rewriter.notifyMatchFailure(
1078 arg&: launchOp, msg: "Operand to launch op is not a memref with a static "
1079 "shape and an integer or float element type.");
1080 }
1081
1082 unsigned bitwidth = memrefTy.getElementTypeBitWidth();
1083 if (bitwidth % 8 != 0) {
1084 return rewriter.notifyMatchFailure(
1085 arg&: launchOp, msg: "Operand to launch op is not a memref with a "
1086 "byte-aligned element type.");
1087 }
1088
1089 uint64_t staticSize = static_cast<uint64_t>(bitwidth / 8) *
1090 static_cast<uint64_t>(memrefTy.getNumElements());
1091
1092 Value sizeArg =
1093 createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: staticSize);
1094 llvmArgumentsWithSizes.push_back(Elt: llvmArg); // Presumably a bare pointer.
1095 llvmArgumentsWithSizes.push_back(Elt: sizeArg);
1096 }
1097 }
1098
1099 std::optional<gpu::KernelDim3> clusterSize = std::nullopt;
1100 if (launchOp.hasClusterSize()) {
1101 clusterSize =
1102 gpu::KernelDim3{.x: adaptor.getClusterSizeX(), .y: adaptor.getClusterSizeY(),
1103 .z: adaptor.getClusterSizeZ()};
1104 }
1105 auto newLaunchOp = gpu::LaunchFuncOp::create(
1106 builder&: rewriter, location: launchOp.getLoc(), kernel: launchOp.getKernelAttr(),
1107 gridSize: gpu::KernelDim3{.x: adaptor.getGridSizeX(), .y: adaptor.getGridSizeY(),
1108 .z: adaptor.getGridSizeZ()},
1109 blockSize: gpu::KernelDim3{.x: adaptor.getBlockSizeX(), .y: adaptor.getBlockSizeY(),
1110 .z: adaptor.getBlockSizeZ()},
1111 dynamicSharedMemorySize: adaptor.getDynamicSharedMemorySize(),
1112 kernelOperands: llvmArgumentsWithSizes.empty() ? llvmArguments : llvmArgumentsWithSizes,
1113 asyncTokenType: nullptr, asyncDependencies: {}, asyncObject: stream, clusterSize);
1114 if (launchOp.getCooperative())
1115 newLaunchOp.setCooperative(true);
1116 if (launchOp.getAsyncToken())
1117 rewriter.replaceOp(op: launchOp, newValues: {stream});
1118 else
1119 rewriter.eraseOp(op: launchOp);
1120 return success();
1121}
1122
1123static Value bitAndAddrspaceCast(Location loc,
1124 ConversionPatternRewriter &rewriter,
1125 LLVM::LLVMPointerType destinationType,
1126 Value sourcePtr,
1127 const LLVMTypeConverter &typeConverter) {
1128 auto sourceTy = cast<LLVM::LLVMPointerType>(Val: sourcePtr.getType());
1129 if (destinationType.getAddressSpace() != sourceTy.getAddressSpace())
1130 sourcePtr = LLVM::AddrSpaceCastOp::create(
1131 builder&: rewriter, location: loc,
1132 res: LLVM::LLVMPointerType::get(context: rewriter.getContext(),
1133 addressSpace: destinationType.getAddressSpace()),
1134 arg: sourcePtr);
1135 return sourcePtr;
1136}
1137
1138LogicalResult ConvertMemcpyOpToGpuRuntimeCallPattern::matchAndRewrite(
1139 gpu::MemcpyOp memcpyOp, OpAdaptor adaptor,
1140 ConversionPatternRewriter &rewriter) const {
1141 auto memRefType = cast<MemRefType>(Val: memcpyOp.getSrc().getType());
1142
1143 if (failed(Result: areAllLLVMTypes(op: memcpyOp, operands: adaptor.getOperands(), rewriter)) ||
1144 !isConvertibleAndHasIdentityMaps(type: memRefType) ||
1145 failed(Result: isAsyncWithOneDependency(rewriter, op: memcpyOp)))
1146 return failure();
1147
1148 auto loc = memcpyOp.getLoc();
1149
1150 MemRefDescriptor srcDesc(adaptor.getSrc());
1151 Value numElements = getNumElements(rewriter, loc, type: memRefType, desc: srcDesc);
1152
1153 Type elementPtrType = getElementPtrType(type: memRefType);
1154 Value nullPtr = LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: elementPtrType);
1155 Value gepPtr = LLVM::GEPOp::create(
1156 builder&: rewriter, location: loc, resultType: elementPtrType,
1157 elementType: typeConverter->convertType(t: memRefType.getElementType()), basePtr: nullPtr,
1158 indices: numElements);
1159 auto sizeBytes =
1160 LLVM::PtrToIntOp::create(builder&: rewriter, location: loc, res: getIndexType(), arg: gepPtr);
1161
1162 auto src = bitAndAddrspaceCast(loc, rewriter, destinationType: llvmPointerType,
1163 sourcePtr: srcDesc.alignedPtr(builder&: rewriter, loc),
1164 typeConverter: *getTypeConverter());
1165 auto dst = bitAndAddrspaceCast(
1166 loc, rewriter, destinationType: llvmPointerType,
1167 sourcePtr: MemRefDescriptor(adaptor.getDst()).alignedPtr(builder&: rewriter, loc),
1168 typeConverter: *getTypeConverter());
1169
1170 auto stream = adaptor.getAsyncDependencies().front();
1171 memcpyCallBuilder.create(loc, builder&: rewriter, arguments: {dst, src, sizeBytes, stream});
1172
1173 rewriter.replaceOp(op: memcpyOp, newValues: {stream});
1174
1175 return success();
1176}
1177
1178LogicalResult ConvertMemsetOpToGpuRuntimeCallPattern::matchAndRewrite(
1179 gpu::MemsetOp memsetOp, OpAdaptor adaptor,
1180 ConversionPatternRewriter &rewriter) const {
1181 auto memRefType = cast<MemRefType>(Val: memsetOp.getDst().getType());
1182
1183 if (failed(Result: areAllLLVMTypes(op: memsetOp, operands: adaptor.getOperands(), rewriter)) ||
1184 !isConvertibleAndHasIdentityMaps(type: memRefType) ||
1185 failed(Result: isAsyncWithOneDependency(rewriter, op: memsetOp)))
1186 return failure();
1187
1188 auto loc = memsetOp.getLoc();
1189
1190 Type valueType = adaptor.getValue().getType();
1191 unsigned bitWidth = valueType.getIntOrFloatBitWidth();
1192 // Ints and floats of 16 or 32 bit width are allowed.
1193 if (!valueType.isIntOrFloat() || (bitWidth != 16 && bitWidth != 32)) {
1194 return rewriter.notifyMatchFailure(
1195 arg&: memsetOp, msg: "value must be a 16 or 32 bit int or float");
1196 }
1197
1198 unsigned valueTypeWidth = valueType.getIntOrFloatBitWidth();
1199 Type bitCastType = valueTypeWidth == 32 ? llvmInt32Type : llvmInt16Type;
1200
1201 MemRefDescriptor dstDesc(adaptor.getDst());
1202 Value numElements = getNumElements(rewriter, loc, type: memRefType, desc: dstDesc);
1203
1204 auto value =
1205 LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: bitCastType, arg: adaptor.getValue());
1206 auto dst = bitAndAddrspaceCast(loc, rewriter, destinationType: llvmPointerType,
1207 sourcePtr: dstDesc.alignedPtr(builder&: rewriter, loc),
1208 typeConverter: *getTypeConverter());
1209
1210 auto stream = adaptor.getAsyncDependencies().front();
1211 FunctionCallBuilder builder =
1212 valueTypeWidth == 32 ? memset32CallBuilder : memset16CallBuilder;
1213 builder.create(loc, builder&: rewriter, arguments: {dst, value, numElements, stream});
1214
1215 rewriter.replaceOp(op: memsetOp, newValues: {stream});
1216 return success();
1217}
1218
1219LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite(
1220 gpu::SetDefaultDeviceOp op, OpAdaptor adaptor,
1221 ConversionPatternRewriter &rewriter) const {
1222 Location loc = op.getLoc();
1223 auto call = setDefaultDeviceCallBuilder.create(loc, builder&: rewriter,
1224 arguments: {adaptor.getDevIndex()});
1225 rewriter.replaceOp(op, newOp: call);
1226 return success();
1227}
1228
1229template <typename T>
1230static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) {
1231 Type llvmInt32Type = builder.getIntegerType(width: 32);
1232 return LLVM::ConstantOp::create(builder, location: loc, type: llvmInt32Type,
1233 value: static_cast<int32_t>(tValue));
1234}
1235
1236LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite(
1237 gpu::CreateDnTensorOp op, OpAdaptor adaptor,
1238 ConversionPatternRewriter &rewriter) const {
1239 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1240 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1241 return failure();
1242 Location loc = op.getLoc();
1243 auto stream = adaptor.getAsyncDependencies().front();
1244 Value pTensor =
1245 MemRefDescriptor(adaptor.getMemref()).allocatedPtr(builder&: rewriter, loc);
1246 Type dType = op.getMemref().getType().getElementType();
1247 auto dtp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: dType));
1248
1249 SmallVector<Value, 4> dims;
1250 for (Value dim : adaptor.getDims()) {
1251 dims.push_back(Elt: dim);
1252 }
1253
1254 Value handle;
1255 // TODO: For now, we track the use of the handle and lower it to cusparse /
1256 // cusparseLt accordingly. If in a block, both cusparse and cusparseLt are
1257 // used, we require two separate Creation ops to be the correct logic. In
1258 // future, we may add support to using one handle in sparse tensor / GPU
1259 // dialect in both cusparse and cusparseLt. use the cusparseLt create call if
1260 // the dnmat is used with spmat with 2:4 sparsity
1261 if (dims.size() == 2) {
1262 if (isSpMMCusparseLtOp(op: op.getDnTensor())) {
1263 auto handleSz =
1264 createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 11032);
1265 handle = LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: llvmPointerType,
1266 elementType: llvmInt8Type, arraySize: handleSz, /*alignment=*/16);
1267 handle = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: llvmPointerType, arg: handle);
1268
1269 createLtDnMatCallBuilder
1270 .create(loc, builder&: rewriter,
1271 arguments: {handle, dims[0], dims[1], pTensor, dtp, stream})
1272 .getResult();
1273 } else {
1274 handle =
1275 createDnMatCallBuilder
1276 .create(loc, builder&: rewriter, arguments: {dims[0], dims[1], pTensor, dtp, stream})
1277 .getResult();
1278 }
1279 } else {
1280 assert(dims.size() == 1 && "Only 1D and 2D tensors are supported");
1281 handle = createDnVecCallBuilder
1282 .create(loc, builder&: rewriter, arguments: {dims[0], pTensor, dtp, stream})
1283 .getResult();
1284 }
1285 rewriter.replaceOp(op, newValues: {handle, stream});
1286 return success();
1287}
1288
1289LogicalResult ConvertDestroyDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite(
1290 gpu::DestroyDnTensorOp op, OpAdaptor adaptor,
1291 ConversionPatternRewriter &rewriter) const {
1292 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1293 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1294 return failure();
1295 Location loc = op.getLoc();
1296 auto stream = adaptor.getAsyncDependencies().front();
1297 auto definingOp = op.getDnTensor().getDefiningOp<gpu::CreateDnTensorOp>();
1298 SmallVector<Value, 4> dims;
1299 for (Value dim : definingOp.getDims()) {
1300 dims.push_back(Elt: dim);
1301 }
1302 if (dims.size() == 2) {
1303 // Use the cusparseLt destroy call if the dnmat is used with spmat with
1304 // 2:4 sparsity
1305 if (isSpMMCusparseLtOp(op: op.getDnTensor())) {
1306 destroyCuSparseLtDnMatBuilder.create(loc, builder&: rewriter,
1307 arguments: {adaptor.getDnTensor(), stream});
1308 } else {
1309 destroyDnMatCallBuilder.create(loc, builder&: rewriter,
1310 arguments: {adaptor.getDnTensor(), stream});
1311 }
1312 } else {
1313 assert(dims.size() == 1 && "Only 1D and 2D tensors are supported");
1314 destroyDnVecCallBuilder.create(loc, builder&: rewriter,
1315 arguments: {adaptor.getDnTensor(), stream});
1316 }
1317 rewriter.replaceOp(op, newValues: {stream});
1318 return success();
1319}
1320
1321LogicalResult ConvertCreateCooOpToGpuRuntimeCallPattern::matchAndRewrite(
1322 gpu::CreateCooOp op, OpAdaptor adaptor,
1323 ConversionPatternRewriter &rewriter) const {
1324 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1325 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1326 return failure();
1327 Location loc = op.getLoc();
1328 auto stream = adaptor.getAsyncDependencies().front();
1329 Value pRowIdxs =
1330 MemRefDescriptor(adaptor.getRowIdxs()).allocatedPtr(builder&: rewriter, loc);
1331 Value pColIdxs =
1332 MemRefDescriptor(adaptor.getColIdxs()).allocatedPtr(builder&: rewriter, loc);
1333 Value pValues =
1334 MemRefDescriptor(adaptor.getValues()).allocatedPtr(builder&: rewriter, loc);
1335 Type iType =
1336 llvm::cast<MemRefType>(Val: op.getColIdxs().getType()).getElementType();
1337 Type dType =
1338 llvm::cast<MemRefType>(Val: op.getValues().getType()).getElementType();
1339 auto itp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseIndexTypeFrom(type: iType));
1340 auto dtp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: dType));
1341 auto handle =
1342 createCooCallBuilder
1343 .create(loc, builder&: rewriter,
1344 arguments: {adaptor.getRows(), adaptor.getCols(), adaptor.getNnz(),
1345 pRowIdxs, pColIdxs, pValues, itp, dtp, stream})
1346 .getResult();
1347 rewriter.replaceOp(op, newValues: {handle, stream});
1348 return success();
1349}
1350
1351LogicalResult ConvertCreateCooAoSOpToGpuRuntimeCallPattern::matchAndRewrite(
1352 gpu::CreateCooAoSOp op, OpAdaptor adaptor,
1353 ConversionPatternRewriter &rewriter) const {
1354 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1355 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1356 return failure();
1357 Location loc = op.getLoc();
1358 auto stream = adaptor.getAsyncDependencies().front();
1359 Value pIdxs = MemRefDescriptor(adaptor.getIdxs()).allocatedPtr(builder&: rewriter, loc);
1360 Value pValues =
1361 MemRefDescriptor(adaptor.getValues()).allocatedPtr(builder&: rewriter, loc);
1362 Type iType = llvm::cast<MemRefType>(Val: op.getIdxs().getType()).getElementType();
1363 Type dType =
1364 llvm::cast<MemRefType>(Val: op.getValues().getType()).getElementType();
1365 auto itp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseIndexTypeFrom(type: iType));
1366 auto dtp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: dType));
1367 auto handle =
1368 createCooAoSCallBuilder
1369 .create(loc, builder&: rewriter,
1370 arguments: {adaptor.getRows(), adaptor.getCols(), adaptor.getNnz(),
1371 pIdxs, pValues, itp, dtp, stream})
1372 .getResult();
1373 rewriter.replaceOp(op, newValues: {handle, stream});
1374 return success();
1375}
1376
1377LogicalResult ConvertCreateCsrOpToGpuRuntimeCallPattern::matchAndRewrite(
1378 gpu::CreateCsrOp op, OpAdaptor adaptor,
1379 ConversionPatternRewriter &rewriter) const {
1380 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1381 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1382 return failure();
1383 Location loc = op.getLoc();
1384 auto stream = adaptor.getAsyncDependencies().front();
1385 Value pRowPos =
1386 MemRefDescriptor(adaptor.getRowPos()).allocatedPtr(builder&: rewriter, loc);
1387 Value pColIdxs =
1388 MemRefDescriptor(adaptor.getColIdxs()).allocatedPtr(builder&: rewriter, loc);
1389 Value pValues =
1390 MemRefDescriptor(adaptor.getValues()).allocatedPtr(builder&: rewriter, loc);
1391 Type pType =
1392 llvm::cast<MemRefType>(Val: op.getRowPos().getType()).getElementType();
1393 Type iType =
1394 llvm::cast<MemRefType>(Val: op.getColIdxs().getType()).getElementType();
1395 Type dType =
1396 llvm::cast<MemRefType>(Val: op.getValues().getType()).getElementType();
1397 auto ptp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseIndexTypeFrom(type: pType));
1398 auto itp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseIndexTypeFrom(type: iType));
1399 auto dtp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: dType));
1400 auto handle =
1401 createCsrCallBuilder
1402 .create(loc, builder&: rewriter,
1403 arguments: {adaptor.getRows(), adaptor.getCols(), adaptor.getNnz(),
1404 pRowPos, pColIdxs, pValues, ptp, itp, dtp, stream})
1405 .getResult();
1406 rewriter.replaceOp(op, newValues: {handle, stream});
1407 return success();
1408}
1409
1410LogicalResult ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
1411 gpu::Create2To4SpMatOp op, OpAdaptor adaptor,
1412 ConversionPatternRewriter &rewriter) const {
1413 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1414 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1415 return failure();
1416 Location loc = op.getLoc();
1417 auto stream = adaptor.getAsyncDependencies().front();
1418 Value pMat =
1419 MemRefDescriptor(adaptor.getMemref()).allocatedPtr(builder&: rewriter, loc);
1420 Type dType =
1421 llvm::cast<MemRefType>(Val: op.getMemref().getType()).getElementType();
1422 auto dtp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: dType));
1423
1424 // CUDA runner asserts the size is 44104 bytes.
1425 auto handleSz = createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 44104);
1426 Value handle = LLVM::AllocaOp::create(
1427 builder&: rewriter, location: loc, resultType: llvmPointerType, elementType: llvmInt8Type, arraySize: handleSz, /*alignment=*/16);
1428 handle = LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: llvmPointerType, arg: handle);
1429
1430 create2To4SpMatCallBuilder
1431 .create(loc, builder&: rewriter,
1432 arguments: {handle, adaptor.getRows(), adaptor.getCols(), pMat, dtp, stream})
1433 .getResult();
1434 rewriter.replaceOp(op, newValues: {handle, stream});
1435 return success();
1436}
1437
1438LogicalResult ConvertDestroySpMatOpToGpuRuntimeCallPattern::matchAndRewrite(
1439 gpu::DestroySpMatOp op, OpAdaptor adaptor,
1440 ConversionPatternRewriter &rewriter) const {
1441 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1442 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1443 return failure();
1444 Location loc = op.getLoc();
1445 auto stream = adaptor.getAsyncDependencies().front();
1446 // Use the cusparseLt destroy call if the spmat is 2:4 sparsity
1447 if (is2To4Sparsity(spMat: op.getSpmat())) {
1448 destroyCuSparseLtSpMatBuilder.create(loc, builder&: rewriter,
1449 arguments: {adaptor.getSpmat(), stream});
1450
1451 } else {
1452 destroySpMatCallBuilder.create(loc, builder&: rewriter, arguments: {adaptor.getSpmat(), stream});
1453 }
1454 rewriter.replaceOp(op, newValues: {stream});
1455 return success();
1456}
1457
1458LogicalResult ConvertSpMVBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
1459 gpu::SpMVBufferSizeOp op, OpAdaptor adaptor,
1460 ConversionPatternRewriter &rewriter) const {
1461 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1462 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1463 return failure();
1464 Location loc = op.getLoc();
1465 auto modeA = genConstInt32From(builder&: rewriter, loc, tValue: op.getModeA());
1466 auto computeType = genConstInt32From(
1467 builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: adaptor.getComputeType()));
1468 auto stream = adaptor.getAsyncDependencies().front();
1469 auto bufferSize = spMVBufferSizeCallBuilder
1470 .create(loc, builder&: rewriter,
1471 arguments: {modeA, adaptor.getSpmatA(), adaptor.getDnX(),
1472 adaptor.getDnY(), computeType, stream})
1473 .getResult();
1474 rewriter.replaceOp(op, newValues: {bufferSize, stream});
1475 return success();
1476}
1477
1478LogicalResult ConvertSpMVOpToGpuRuntimeCallPattern::matchAndRewrite(
1479 gpu::SpMVOp op, OpAdaptor adaptor,
1480 ConversionPatternRewriter &rewriter) const {
1481 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1482 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1483 return failure();
1484 Location loc = op.getLoc();
1485 auto modeA = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeA());
1486 auto computeType = genConstInt32From(
1487 builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: adaptor.getComputeType()));
1488 auto stream = adaptor.getAsyncDependencies().front();
1489 Value pBuf =
1490 MemRefDescriptor(adaptor.getBuffer()).allocatedPtr(builder&: rewriter, loc);
1491 spMVCallBuilder.create(loc, builder&: rewriter,
1492 arguments: {modeA, adaptor.getSpmatA(), adaptor.getDnX(),
1493 adaptor.getDnY(), computeType, pBuf, stream});
1494 rewriter.replaceOp(op, newValues: {stream});
1495 return success();
1496}
1497
1498LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
1499 gpu::SpMMBufferSizeOp op, OpAdaptor adaptor,
1500 ConversionPatternRewriter &rewriter) const {
1501 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1502 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1503 return failure();
1504 Location loc = op.getLoc();
1505 auto modeA = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeA());
1506 auto modeB = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeB());
1507 auto stream = adaptor.getAsyncDependencies().front();
1508 Value bufferSize;
1509 if (is2To4Sparsity(spMat: op.getSpmatA())) {
1510 auto pruneFlag =
1511 genConstInt32From(builder&: rewriter, loc, tValue: get2To4PruneFlag(spMat: op.getSpmatA()));
1512 auto computeType = genConstInt32From(
1513 builder&: rewriter, loc, tValue: getCuSparseLtDataTypeFrom(type: adaptor.getComputeType()));
1514 auto three = createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 3);
1515 auto bufferSize =
1516 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: llvmPointerType, elementType: llvmPointerType,
1517 arraySize: three, /*alignment=*/16);
1518 createCuSparseLtSpMMBufferSizeBuilder
1519 .create(loc, builder&: rewriter,
1520 arguments: {bufferSize, modeA, modeB, adaptor.getSpmatA(),
1521 adaptor.getDnmatB(), adaptor.getDnmatC(), computeType,
1522 pruneFlag, stream})
1523 .getResult();
1524
1525 auto bufferSizePtr1 = LLVM::GEPOp::create(
1526 builder&: rewriter, location: loc, resultType: llvmPointerType, elementType: llvmPointerType, basePtr: bufferSize,
1527 indices: ValueRange{createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 1)});
1528 auto bufferSizePtr2 = LLVM::GEPOp::create(
1529 builder&: rewriter, location: loc, resultType: llvmPointerType, elementType: llvmPointerType, basePtr: bufferSize,
1530 indices: ValueRange{createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 2)});
1531 auto bufferSize0 =
1532 LLVM::LoadOp::create(builder&: rewriter, location: loc, type: llvmInt64Type, addr: bufferSize);
1533 auto bufferSize1 =
1534 LLVM::LoadOp::create(builder&: rewriter, location: loc, type: llvmInt64Type, addr: bufferSizePtr1);
1535 auto bufferSize2 =
1536 LLVM::LoadOp::create(builder&: rewriter, location: loc, type: llvmInt64Type, addr: bufferSizePtr2);
1537
1538 rewriter.replaceOp(op, newValues: {bufferSize0, bufferSize1, bufferSize2, stream});
1539 } else {
1540 auto computeType = genConstInt32From(
1541 builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: adaptor.getComputeType()));
1542 bufferSize =
1543 createSpMMBufferSizeCallBuilder
1544 .create(loc, builder&: rewriter,
1545 arguments: {modeA, modeB, adaptor.getSpmatA(), adaptor.getDnmatB(),
1546 adaptor.getDnmatC(), computeType, stream})
1547 .getResult();
1548 rewriter.replaceOp(op, newValues: {bufferSize, stream});
1549 }
1550 return success();
1551}
1552
1553LogicalResult ConvertSDDMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
1554 gpu::SDDMMBufferSizeOp op, OpAdaptor adaptor,
1555 ConversionPatternRewriter &rewriter) const {
1556 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1557 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1558 return failure();
1559 Location loc = op.getLoc();
1560 auto modeA = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeA());
1561 auto modeB = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeB());
1562 auto computeType = genConstInt32From(
1563 builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: adaptor.getComputeType()));
1564 auto stream = adaptor.getAsyncDependencies().front();
1565 auto bufferSize =
1566 createSDDMMBufferSizeCallBuilder
1567 .create(loc, builder&: rewriter,
1568 arguments: {modeA, modeB, adaptor.getDnmatA(), adaptor.getDnmatB(),
1569 adaptor.getSpmatC(), computeType, stream})
1570 .getResult();
1571 rewriter.replaceOp(op, newValues: {bufferSize, stream});
1572 return success();
1573}
1574
1575LogicalResult ConvertSpMMOpToGpuRuntimeCallPattern::matchAndRewrite(
1576 gpu::SpMMOp op, OpAdaptor adaptor,
1577 ConversionPatternRewriter &rewriter) const {
1578 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1579 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1580 return failure();
1581 Location loc = op.getLoc();
1582 auto modeA = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeA());
1583 auto modeB = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeB());
1584 auto computeType = genConstInt32From(
1585 builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: adaptor.getComputeType()));
1586
1587 auto stream = adaptor.getAsyncDependencies().front();
1588
1589 // Lower to cusparseLt if applicable
1590 if (is2To4Sparsity(spMat: op.getSpmatA())) {
1591 SmallVector<Value> pBufs;
1592 for (Value buffer : adaptor.getBuffers()) {
1593 Value pBuf = MemRefDescriptor(buffer).allocatedPtr(builder&: rewriter, loc);
1594 pBufs.push_back(Elt: pBuf);
1595 }
1596 createCuSparseLtSpMMBuilder.create(
1597 loc, builder&: rewriter,
1598 arguments: {adaptor.getSpmatA(), adaptor.getDnmatB(), adaptor.getDnmatC(),
1599 pBufs[0], pBufs[1], pBufs[2], stream});
1600 } else {
1601 Value pBuf = MemRefDescriptor(adaptor.getBuffers().front())
1602 .allocatedPtr(builder&: rewriter, loc);
1603 createSpMMCallBuilder.create(loc, builder&: rewriter,
1604 arguments: {modeA, modeB, adaptor.getSpmatA(),
1605 adaptor.getDnmatB(), adaptor.getDnmatC(),
1606 computeType, pBuf, stream});
1607 }
1608 rewriter.replaceOp(op, newValues: {stream});
1609 return success();
1610}
1611
1612template <typename T>
1613static void addOpaquePointerConversion(LLVMTypeConverter &converter) {
1614 converter.addConversion([&converter](T) -> Type {
1615 return LLVM::LLVMPointerType::get(context: &converter.getContext());
1616 });
1617}
1618
1619LogicalResult ConvertSDDMMOpToGpuRuntimeCallPattern::matchAndRewrite(
1620 gpu::SDDMMOp op, OpAdaptor adaptor,
1621 ConversionPatternRewriter &rewriter) const {
1622 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1623 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1624 return failure();
1625 Location loc = op.getLoc();
1626 auto computeType = genConstInt32From(
1627 builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: adaptor.getComputeType()));
1628 auto modeA = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeA());
1629 auto modeB = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeB());
1630 auto stream = adaptor.getAsyncDependencies().front();
1631 Value pBuf =
1632 MemRefDescriptor(adaptor.getBuffer()).allocatedPtr(builder&: rewriter, loc);
1633 createSDDMMCallBuilder.create(loc, builder&: rewriter,
1634 arguments: {modeA, modeB, adaptor.getDnmatA(),
1635 adaptor.getDnmatB(), adaptor.getSpmatC(),
1636 computeType, pBuf, stream});
1637 rewriter.replaceOp(op, newValues: {stream});
1638 return success();
1639}
1640
1641LogicalResult
1642ConvertSpGEMMCreateDescrOpToGpuRuntimeCallPattern::matchAndRewrite(
1643 gpu::SpGEMMCreateDescrOp op, OpAdaptor adaptor,
1644 ConversionPatternRewriter &rewriter) const {
1645 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1646 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1647 return failure();
1648 Location loc = op.getLoc();
1649 auto stream = adaptor.getAsyncDependencies().front();
1650 Value descr = createSpGEMMCreateDescrBuilder.create(loc, builder&: rewriter, arguments: {stream})
1651 .getResult();
1652 rewriter.replaceOp(op, newValues: {descr, stream});
1653 return success();
1654}
1655
1656LogicalResult
1657ConvertSpGEMMDestroyDescrOpToGpuRuntimeCallPattern::matchAndRewrite(
1658 gpu::SpGEMMDestroyDescrOp op, OpAdaptor adaptor,
1659 ConversionPatternRewriter &rewriter) const {
1660 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1661 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1662 return failure();
1663 Location loc = op.getLoc();
1664 auto stream = adaptor.getAsyncDependencies().front();
1665 createSpGEMMDestroyDescrBuilder.create(loc, builder&: rewriter,
1666 arguments: {adaptor.getDesc(), stream});
1667 rewriter.replaceOp(op, newValues: {stream});
1668 return success();
1669}
1670
1671LogicalResult
1672ConvertSpGEMMWorkEstimationOrComputeOpToGpuRuntimeCallPattern::matchAndRewrite(
1673 gpu::SpGEMMWorkEstimationOrComputeOp op, OpAdaptor adaptor,
1674 ConversionPatternRewriter &rewriter) const {
1675 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1676 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1677 return failure();
1678 Location loc = op.getLoc();
1679 auto computeType = genConstInt32From(
1680 builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: adaptor.getComputeType()));
1681 auto modeA = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeA());
1682 auto modeB = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeB());
1683 auto stream = adaptor.getAsyncDependencies().front();
1684
1685 Value pBuf =
1686 MemRefDescriptor(adaptor.getBuffer()).allocatedPtr(builder&: rewriter, loc);
1687 Value bufferSizeNew;
1688
1689 if (adaptor.getKind() ==
1690 gpu::SpGEMMWorkEstimationOrComputeKind::WORK_ESTIMATION) {
1691 bufferSizeNew =
1692 createSpGEMMWorkEstimationBuilder
1693 .create(loc, builder&: rewriter,
1694 arguments: {adaptor.getDesc(), modeA, modeB, adaptor.getSpmatA(),
1695 adaptor.getSpmatB(), adaptor.getSpmatC(), computeType,
1696 adaptor.getBufferSz(), pBuf, stream})
1697 .getResult();
1698 } else {
1699 bufferSizeNew =
1700 createSpGEMMComputeBuilder
1701 .create(loc, builder&: rewriter,
1702 arguments: {adaptor.getDesc(), modeA, modeB, adaptor.getSpmatA(),
1703 adaptor.getSpmatB(), adaptor.getSpmatC(), computeType,
1704 adaptor.getBufferSz(), pBuf, stream})
1705 .getResult();
1706 }
1707 rewriter.replaceOp(op, newValues: {bufferSizeNew, stream});
1708 return success();
1709}
1710
1711LogicalResult ConvertSpGEMMCopyOpToGpuRuntimeCallPattern::matchAndRewrite(
1712 gpu::SpGEMMCopyOp op, OpAdaptor adaptor,
1713 ConversionPatternRewriter &rewriter) const {
1714 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1715 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1716 return failure();
1717 Location loc = op.getLoc();
1718 auto computeType = genConstInt32From(
1719 builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: adaptor.getComputeType()));
1720 auto modeA = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeA());
1721 auto modeB = genConstInt32From(builder&: rewriter, loc, tValue: adaptor.getModeB());
1722 auto stream = adaptor.getAsyncDependencies().front();
1723 createSpGEMMCopyBuilder.create(loc, builder&: rewriter,
1724 arguments: {adaptor.getDesc(), modeA, modeB,
1725 adaptor.getSpmatA(), adaptor.getSpmatB(),
1726 adaptor.getSpmatC(), computeType, stream});
1727 rewriter.replaceOp(op, newValues: {stream});
1728 return success();
1729}
1730
1731LogicalResult ConvertSpMatGetSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
1732 gpu::SpMatGetSizeOp op, OpAdaptor adaptor,
1733 ConversionPatternRewriter &rewriter) const {
1734 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1735 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1736 return failure();
1737 Location loc = op.getLoc();
1738 auto stream = adaptor.getAsyncDependencies().front();
1739
1740 auto three = createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 3);
1741 auto buffer = LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: llvmPointerType,
1742 elementType: llvmInt64Type, arraySize: three, /*alignment=*/16);
1743
1744 auto rowsPtr = LLVM::GEPOp::create(
1745 builder&: rewriter, location: loc, resultType: llvmPointerType, elementType: llvmPointerType, basePtr: buffer,
1746 indices: ValueRange{createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 0)});
1747 auto colsPtr = LLVM::GEPOp::create(
1748 builder&: rewriter, location: loc, resultType: llvmPointerType, elementType: llvmPointerType, basePtr: buffer,
1749 indices: ValueRange{createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 1)});
1750 auto nnzsPtr = LLVM::GEPOp::create(
1751 builder&: rewriter, location: loc, resultType: llvmPointerType, elementType: llvmPointerType, basePtr: buffer,
1752 indices: ValueRange{createIndexAttrConstant(builder&: rewriter, loc, resultType: getIndexType(), value: 2)});
1753 createSpMatGetSizeBuilder.create(
1754 loc, builder&: rewriter, arguments: {adaptor.getSpmat(), rowsPtr, colsPtr, nnzsPtr, stream});
1755 auto rows = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: llvmInt64Type, addr: rowsPtr);
1756 auto cols = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: llvmInt64Type, addr: colsPtr);
1757 auto nnzs = LLVM::LoadOp::create(builder&: rewriter, location: loc, type: llvmInt64Type, addr: nnzsPtr);
1758
1759 rewriter.replaceOp(op, newValues: {rows, cols, nnzs, stream});
1760 return success();
1761}
1762
1763LogicalResult ConvertSetCsrPointersOpToGpuRuntimeCallPattern::matchAndRewrite(
1764 gpu::SetCsrPointersOp op, OpAdaptor adaptor,
1765 ConversionPatternRewriter &rewriter) const {
1766 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1767 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1768 return failure();
1769 Location loc = op.getLoc();
1770 auto stream = adaptor.getAsyncDependencies().front();
1771 Value pPos =
1772 MemRefDescriptor(adaptor.getPositions()).allocatedPtr(builder&: rewriter, loc);
1773 Value pCrd =
1774 MemRefDescriptor(adaptor.getCoordinates()).allocatedPtr(builder&: rewriter, loc);
1775 Value pVal =
1776 MemRefDescriptor(adaptor.getValues()).allocatedPtr(builder&: rewriter, loc);
1777 createSetCsrPointersBuilder.create(
1778 loc, builder&: rewriter, arguments: {adaptor.getSpmat(), pPos, pCrd, pVal, stream});
1779 rewriter.replaceOp(op, newValues: {stream});
1780 return success();
1781}
1782
1783LogicalResult ConvertCreateCscOpToGpuRuntimeCallPattern::matchAndRewrite(
1784 gpu::CreateCscOp op, OpAdaptor adaptor,
1785 ConversionPatternRewriter &rewriter) const {
1786 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1787 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1788 return failure();
1789 Location loc = op.getLoc();
1790 auto stream = adaptor.getAsyncDependencies().front();
1791 Value pColPos =
1792 MemRefDescriptor(adaptor.getColPos()).allocatedPtr(builder&: rewriter, loc);
1793 Value pRowIdxs =
1794 MemRefDescriptor(adaptor.getRowIdxs()).allocatedPtr(builder&: rewriter, loc);
1795 Value pValues =
1796 MemRefDescriptor(adaptor.getValues()).allocatedPtr(builder&: rewriter, loc);
1797 Type pType =
1798 llvm::cast<MemRefType>(Val: op.getColPos().getType()).getElementType();
1799 Type iType =
1800 llvm::cast<MemRefType>(Val: op.getRowIdxs().getType()).getElementType();
1801 Type dType =
1802 llvm::cast<MemRefType>(Val: op.getValues().getType()).getElementType();
1803 auto ptp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseIndexTypeFrom(type: pType));
1804 auto itp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseIndexTypeFrom(type: iType));
1805 auto dtp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: dType));
1806 auto handle =
1807 createCscCallBuilder
1808 .create(loc, builder&: rewriter,
1809 arguments: {adaptor.getRows(), adaptor.getCols(), adaptor.getNnz(),
1810 pColPos, pRowIdxs, pValues, ptp, itp, dtp, stream})
1811 .getResult();
1812 rewriter.replaceOp(op, newValues: {handle, stream});
1813 return success();
1814}
1815
1816LogicalResult ConvertCreateBsrOpToGpuRuntimeCallPattern::matchAndRewrite(
1817 gpu::CreateBsrOp op, OpAdaptor adaptor,
1818 ConversionPatternRewriter &rewriter) const {
1819 if (failed(Result: areAllLLVMTypes(op, operands: adaptor.getOperands(), rewriter)) ||
1820 failed(Result: isAsyncWithOneDependency(rewriter, op)))
1821 return failure();
1822 Location loc = op.getLoc();
1823 auto stream = adaptor.getAsyncDependencies().front();
1824 Value pRowPos =
1825 MemRefDescriptor(adaptor.getBRowPos()).allocatedPtr(builder&: rewriter, loc);
1826 Value pColIdxs =
1827 MemRefDescriptor(adaptor.getBColIdxs()).allocatedPtr(builder&: rewriter, loc);
1828 Value pValues =
1829 MemRefDescriptor(adaptor.getValues()).allocatedPtr(builder&: rewriter, loc);
1830 Type pType =
1831 llvm::cast<MemRefType>(Val: op.getBRowPos().getType()).getElementType();
1832 Type iType =
1833 llvm::cast<MemRefType>(Val: op.getBColIdxs().getType()).getElementType();
1834 Type dType =
1835 llvm::cast<MemRefType>(Val: op.getValues().getType()).getElementType();
1836 auto ptp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseIndexTypeFrom(type: pType));
1837 auto itp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseIndexTypeFrom(type: iType));
1838 auto dtp = genConstInt32From(builder&: rewriter, loc, tValue: getCuSparseDataTypeFrom(type: dType));
1839 auto handle =
1840 createBsrCallBuilder
1841 .create(loc, builder&: rewriter,
1842 arguments: {adaptor.getBrows(), adaptor.getBcols(), adaptor.getBnnz(),
1843 adaptor.getRBlockSize(), adaptor.getCBlockSize(), pRowPos,
1844 pColIdxs, pValues, ptp, itp, dtp, stream})
1845 .getResult();
1846 rewriter.replaceOp(op, newValues: {handle, stream});
1847 return success();
1848}
1849
1850void mlir::populateGpuToLLVMConversionPatterns(
1851 LLVMTypeConverter &converter, RewritePatternSet &patterns,
1852 bool kernelBarePtrCallConv, bool kernelIntersperseSizeCallConv) {
1853 addOpaquePointerConversion<gpu::AsyncTokenType>(converter);
1854 addOpaquePointerConversion<gpu::SparseDnTensorHandleType>(converter);
1855 addOpaquePointerConversion<gpu::SparseSpMatHandleType>(converter);
1856 addOpaquePointerConversion<gpu::SparseSpGEMMOpHandleType>(converter);
1857
1858 // Higher benefit so this pattern wins over the structural async.yield
1859 // rewriter from populateAsyncStructuralTypeConversionsAndLegality on yields
1860 // with gpu.async.token operands. The structural rewriter would silently
1861 // retype operands without recording an event on the underlying stream.
1862 patterns.add<ConvertAsyncYieldToGpuRuntimeCallPattern>(arg&: converter,
1863 /*benefit=*/args: 2);
1864
1865 patterns.add<ConvertAllocOpToGpuRuntimeCallPattern,
1866 ConvertDeallocOpToGpuRuntimeCallPattern,
1867 ConvertHostRegisterOpToGpuRuntimeCallPattern,
1868 ConvertHostUnregisterOpToGpuRuntimeCallPattern,
1869 ConvertMemcpyOpToGpuRuntimeCallPattern,
1870 ConvertMemsetOpToGpuRuntimeCallPattern,
1871 ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern,
1872 ConvertWaitAsyncOpToGpuRuntimeCallPattern,
1873 ConvertWaitOpToGpuRuntimeCallPattern,
1874 ConvertCreateDnTensorOpToGpuRuntimeCallPattern,
1875 ConvertDestroyDnTensorOpToGpuRuntimeCallPattern,
1876 ConvertCreateCooOpToGpuRuntimeCallPattern,
1877 ConvertCreateCooAoSOpToGpuRuntimeCallPattern,
1878 ConvertCreateCsrOpToGpuRuntimeCallPattern,
1879 ConvertCreateCscOpToGpuRuntimeCallPattern,
1880 ConvertCreateBsrOpToGpuRuntimeCallPattern,
1881 ConvertCreate2To4SpMatOpToGpuRuntimeCallPattern,
1882 ConvertDestroySpMatOpToGpuRuntimeCallPattern,
1883 ConvertSpMVBufferSizeOpToGpuRuntimeCallPattern,
1884 ConvertSpMVOpToGpuRuntimeCallPattern,
1885 ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern,
1886 ConvertSDDMMBufferSizeOpToGpuRuntimeCallPattern,
1887 ConvertSpMMOpToGpuRuntimeCallPattern,
1888 ConvertSDDMMOpToGpuRuntimeCallPattern,
1889 ConvertSpGEMMCreateDescrOpToGpuRuntimeCallPattern,
1890 ConvertSpGEMMDestroyDescrOpToGpuRuntimeCallPattern,
1891 ConvertSpGEMMWorkEstimationOrComputeOpToGpuRuntimeCallPattern,
1892 ConvertSpGEMMCopyOpToGpuRuntimeCallPattern,
1893 ConvertSpMatGetSizeOpToGpuRuntimeCallPattern,
1894 ConvertSetCsrPointersOpToGpuRuntimeCallPattern>(arg&: converter);
1895 patterns.add<LegalizeLaunchFuncOpPattern>(arg&: converter, args&: kernelBarePtrCallConv,
1896 args&: kernelIntersperseSizeCallConv);
1897}
1898
1899//===----------------------------------------------------------------------===//
1900// GPUModuleOp convert to LLVM op interface
1901//===----------------------------------------------------------------------===//
1902
1903namespace {
1904struct GPUModuleOpConvertToLLVMInterface
1905 : public ConvertToLLVMOpInterface::ExternalModel<
1906 GPUModuleOpConvertToLLVMInterface, gpu::GPUModuleOp> {
1907 /// Get the conversion patterns from the target attribute.
1908 void getConvertToLLVMConversionAttrs(
1909 Operation *op, SmallVectorImpl<ConvertToLLVMAttrInterface> &attrs) const;
1910};
1911} // namespace
1912
1913void GPUModuleOpConvertToLLVMInterface::getConvertToLLVMConversionAttrs(
1914 Operation *op, SmallVectorImpl<ConvertToLLVMAttrInterface> &attrs) const {
1915 auto module = cast<gpu::GPUModuleOp>(Val: op);
1916 ArrayAttr targetsAttr = module.getTargetsAttr();
1917 // Fail if there are no target attributes or there is more than one target.
1918 if (!targetsAttr || targetsAttr.size() != 1)
1919 return;
1920 if (auto patternAttr = dyn_cast<ConvertToLLVMAttrInterface>(Val: targetsAttr[0]))
1921 attrs.push_back(Elt: patternAttr);
1922}
1923
1924void mlir::gpu::registerConvertGpuToLLVMInterface(DialectRegistry &registry) {
1925 registry.addExtension(extensionFn: +[](MLIRContext *ctx, gpu::GPUDialect *dialect) {
1926 gpu::GPUModuleOp::attachInterface<GPUModuleOpConvertToLLVMInterface>(context&: *ctx);
1927 });
1928}
1929