1//===- GPUToLLVMSPV.cpp - Convert GPU operations to LLVM dialect ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/GPUToLLVMSPV/GPUToLLVMSPVPass.h"
10
11#include "../GPUCommon/GPUOpsLowering.h"
12#include "mlir/Conversion/GPUCommon/AttrToSPIRVConverter.h"
13#include "mlir/Conversion/GPUCommon/GPUCommonPass.h"
14#include "mlir/Conversion/LLVMCommon/ConversionTarget.h"
15#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
16#include "mlir/Conversion/LLVMCommon/Pattern.h"
17#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
18#include "mlir/Conversion/SPIRVCommon/AttrToLLVMConverter.h"
19#include "mlir/Dialect/GPU/IR/GPUDialect.h"
20#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
21#include "mlir/Dialect/LLVMIR/LLVMCallOps.h"
22#include "mlir/Dialect/LLVMIR/LLVMConstantOps.h"
23#include "mlir/Dialect/LLVMIR/LLVMConversionOps.h"
24#include "mlir/Dialect/LLVMIR/LLVMModuleOps.h"
25#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
26#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
27#include "mlir/IR/BuiltinTypes.h"
28#include "mlir/IR/Matchers.h"
29#include "mlir/IR/PatternMatch.h"
30#include "mlir/IR/SymbolTable.h"
31#include "mlir/Pass/Pass.h"
32#include "mlir/Support/LLVM.h"
33#include "mlir/Transforms/DialectConversion.h"
34
35#include "llvm/ADT/TypeSwitch.h"
36#include "llvm/Support/FormatVariadic.h"
37
38#define DEBUG_TYPE "gpu-to-llvm-spv"
39
40using namespace mlir;
41
42namespace mlir {
43#define GEN_PASS_DEF_CONVERTGPUOPSTOLLVMSPVOPS
44#include "mlir/Conversion/Passes.h.inc"
45} // namespace mlir
46
47//===----------------------------------------------------------------------===//
48// Helper Functions
49//===----------------------------------------------------------------------===//
50
51static LLVM::LLVMFuncOp lookupOrCreateSPIRVFn(Operation *symbolTable,
52 StringRef name,
53 ArrayRef<Type> paramTypes,
54 Type resultType, bool isMemNone,
55 bool isConvergent) {
56 auto func = dyn_cast_or_null<LLVM::LLVMFuncOp>(
57 Val: SymbolTable::lookupSymbolIn(op: symbolTable, symbol: name));
58 if (!func) {
59 OpBuilder b(symbolTable->getRegion(index: 0));
60 func = LLVM::LLVMFuncOp::create(
61 builder&: b, location: symbolTable->getLoc(), name,
62 type: LLVM::LLVMFunctionType::get(result: resultType, arguments: paramTypes));
63 func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
64 func.setNoUnwind(true);
65 func.setWillReturn(true);
66
67 if (isMemNone) {
68 // no externally observable effects
69 constexpr auto noModRef = mlir::LLVM::ModRefInfo::NoModRef;
70 auto memAttr = b.getAttr<LLVM::MemoryEffectsAttr>(
71 /*other=*/args: noModRef,
72 /*argMem=*/args: noModRef, /*inaccessibleMem=*/args: noModRef,
73 /*errnoMem=*/args: noModRef,
74 /*targetMem0=*/args: noModRef,
75 /*targetMem1=*/args: noModRef);
76 func.setMemoryEffectsAttr(memAttr);
77 }
78
79 func.setConvergent(isConvergent);
80 }
81 return func;
82}
83
84static LLVM::CallOp createSPIRVBuiltinCall(Location loc,
85 ConversionPatternRewriter &rewriter,
86 LLVM::LLVMFuncOp func,
87 ValueRange args) {
88 auto call = LLVM::CallOp::create(builder&: rewriter, location: loc, func, args);
89 call.setCConv(func.getCConv());
90 call.setConvergentAttr(func.getConvergentAttr());
91 call.setNoUnwindAttr(func.getNoUnwindAttr());
92 call.setWillReturnAttr(func.getWillReturnAttr());
93 call.setMemoryEffectsAttr(func.getMemoryEffectsAttr());
94 return call;
95}
96
97namespace {
98//===----------------------------------------------------------------------===//
99// Barriers
100//===----------------------------------------------------------------------===//
101
102/// Replace `gpu.barrier` with an `llvm.call` to `barrier` using
103/// `CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE`, ensuring that all memory
104/// accesses are visible to all work-items in the work-group.
105/// ```
106/// // gpu.barrier
107/// // 3 = CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE
108/// %c3 = llvm.mlir.constant(3: i32) : i32
109/// llvm.call spir_funccc @_Z7barrierj(%c3) : (i32) -> ()
110/// ```
111struct GPUBarrierConversion final : ConvertOpToLLVMPattern<gpu::BarrierOp> {
112 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
113
114 LogicalResult
115 matchAndRewrite(gpu::BarrierOp op, OpAdaptor adaptor,
116 ConversionPatternRewriter &rewriter) const final {
117 constexpr StringLiteral funcName = "_Z7barrierj";
118
119 Operation *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>();
120 assert(moduleOp && "Expecting module");
121 Type flagTy = rewriter.getI32Type();
122 Type voidTy = rewriter.getType<LLVM::LLVMVoidType>();
123 LLVM::LLVMFuncOp func =
124 lookupOrCreateSPIRVFn(symbolTable: moduleOp, name: funcName, paramTypes: flagTy, resultType: voidTy,
125 /*isMemNone=*/false, /*isConvergent=*/true);
126
127 // Values used by SPIR-V backend to represent `CLK_LOCAL_MEM_FENCE` and
128 // `CLK_GLOBAL_MEM_FENCE`. See `llvm/lib/Target/SPIRV/SPIRVBuiltins.td`.
129 constexpr int64_t localMemFenceFlag = 1;
130 constexpr int64_t globalMemFenceFlag = 2;
131 int64_t memFenceFlag = 0;
132 std::optional<ArrayAttr> addressSpaces = adaptor.getAddressSpaces();
133 if (addressSpaces) {
134 for (Attribute attr : addressSpaces.value()) {
135 auto addressSpace = cast<gpu::AddressSpaceAttr>(Val&: attr).getValue();
136 switch (addressSpace) {
137 case gpu::AddressSpace::Global:
138 memFenceFlag = memFenceFlag | globalMemFenceFlag;
139 break;
140 case gpu::AddressSpace::Workgroup:
141 memFenceFlag = memFenceFlag | localMemFenceFlag;
142 break;
143 case gpu::AddressSpace::Private:
144 case gpu::AddressSpace::Constant:
145 // Private is thread-local, constant is read-only; no fencing needed.
146 break;
147 }
148 }
149 } else {
150 memFenceFlag = localMemFenceFlag | globalMemFenceFlag;
151 }
152 Location loc = op->getLoc();
153 Value flag = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: flagTy, value: memFenceFlag);
154 rewriter.replaceOp(op, newOp: createSPIRVBuiltinCall(loc, rewriter, func, args: flag));
155 return success();
156 }
157};
158
159//===----------------------------------------------------------------------===//
160// SPIR-V Builtins
161//===----------------------------------------------------------------------===//
162
163/// Replace `gpu.*` with an `llvm.call` to the corresponding SPIR-V builtin with
164/// a constant argument for the `dimension` attribute. Return type will depend
165/// on index width option:
166/// ```
167/// // %thread_id_y = gpu.thread_id y
168/// %c1 = llvm.mlir.constant(1: i32) : i32
169/// %0 = llvm.call spir_funccc @_Z12get_local_idj(%c1) : (i32) -> i64
170/// ```
171struct LaunchConfigConversion : ConvertToLLVMPattern {
172 LaunchConfigConversion(StringRef funcName, StringRef rootOpName,
173 MLIRContext *context,
174 const LLVMTypeConverter &typeConverter,
175 PatternBenefit benefit)
176 : ConvertToLLVMPattern(rootOpName, context, typeConverter, benefit),
177 funcName(funcName) {}
178
179 virtual gpu::Dimension getDimension(Operation *op) const = 0;
180
181 LogicalResult
182 matchAndRewrite(Operation *op, ArrayRef<Value> operands,
183 ConversionPatternRewriter &rewriter) const final {
184 Operation *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>();
185 assert(moduleOp && "Expecting module");
186 Type dimTy = rewriter.getI32Type();
187 Type indexTy = getTypeConverter()->getIndexType();
188 LLVM::LLVMFuncOp func = lookupOrCreateSPIRVFn(symbolTable: moduleOp, name: funcName, paramTypes: dimTy,
189 resultType: indexTy, /*isMemNone=*/true,
190 /*isConvergent=*/false);
191
192 Location loc = op->getLoc();
193 gpu::Dimension dim = getDimension(op);
194 Value dimVal = LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: dimTy,
195 value: static_cast<int64_t>(dim));
196 rewriter.replaceOp(op, newOp: createSPIRVBuiltinCall(loc, rewriter, func, args: dimVal));
197 return success();
198 }
199
200 StringRef funcName;
201};
202
203template <typename SourceOp>
204struct LaunchConfigOpConversion final : LaunchConfigConversion {
205 static StringRef getFuncName();
206
207 explicit LaunchConfigOpConversion(const LLVMTypeConverter &typeConverter,
208 PatternBenefit benefit = 1)
209 : LaunchConfigConversion(getFuncName(), SourceOp::getOperationName(),
210 &typeConverter.getContext(), typeConverter,
211 benefit) {}
212
213 gpu::Dimension getDimension(Operation *op) const final {
214 return cast<SourceOp>(op).getDimension();
215 }
216};
217
218template <>
219StringRef LaunchConfigOpConversion<gpu::BlockIdOp>::getFuncName() {
220 return "_Z12get_group_idj";
221}
222
223template <>
224StringRef LaunchConfigOpConversion<gpu::GridDimOp>::getFuncName() {
225 return "_Z14get_num_groupsj";
226}
227
228template <>
229StringRef LaunchConfigOpConversion<gpu::BlockDimOp>::getFuncName() {
230 return "_Z14get_local_sizej";
231}
232
233template <>
234StringRef LaunchConfigOpConversion<gpu::ThreadIdOp>::getFuncName() {
235 return "_Z12get_local_idj";
236}
237
238template <>
239StringRef LaunchConfigOpConversion<gpu::GlobalIdOp>::getFuncName() {
240 return "_Z13get_global_idj";
241}
242
243//===----------------------------------------------------------------------===//
244// Shuffles
245//===----------------------------------------------------------------------===//
246
247/// Replace `gpu.shuffle` with an `llvm.call` to the corresponding SPIR-V
248/// builtin for `shuffleResult`, keeping `value` and `offset` arguments, and a
249/// `true` constant for the `valid` result type. Conversion will only take place
250/// if `width` is constant and equal to the `subgroup` pass option:
251/// ```
252/// // %0 = gpu.shuffle idx %value, %offset, %width : f64
253/// %0 = llvm.call spir_funccc @_Z17sub_group_shuffledj(%value, %offset)
254/// : (f64, i32) -> f64
255/// ```
256struct GPUShuffleConversion final : ConvertOpToLLVMPattern<gpu::ShuffleOp> {
257 using ConvertOpToLLVMPattern::ConvertOpToLLVMPattern;
258
259 static StringRef getBaseName(gpu::ShuffleMode mode) {
260 switch (mode) {
261 case gpu::ShuffleMode::IDX:
262 return "sub_group_shuffle";
263 case gpu::ShuffleMode::XOR:
264 return "sub_group_shuffle_xor";
265 case gpu::ShuffleMode::UP:
266 return "sub_group_shuffle_up";
267 case gpu::ShuffleMode::DOWN:
268 return "sub_group_shuffle_down";
269 }
270 llvm_unreachable("Unhandled shuffle mode");
271 }
272
273 static std::optional<StringRef> getTypeMangling(Type type) {
274 return TypeSwitch<Type, std::optional<StringRef>>(type)
275 .Case(caseFn: [](Float16Type) { return "Dhj"; })
276 .Case(caseFn: [](Float32Type) { return "fj"; })
277 .Case(caseFn: [](Float64Type) { return "dj"; })
278 .Case(caseFn: [](IntegerType intTy) -> std::optional<StringRef> {
279 switch (intTy.getWidth()) {
280 case 8:
281 return "cj";
282 case 16:
283 return "sj";
284 case 32:
285 return "ij";
286 case 64:
287 return "lj";
288 }
289 return std::nullopt;
290 })
291 .Default(std::nullopt);
292 }
293
294 static std::optional<std::string> getFuncName(gpu::ShuffleMode mode,
295 Type type) {
296 StringRef baseName = getBaseName(mode);
297 std::optional<StringRef> typeMangling = getTypeMangling(type);
298 if (!typeMangling)
299 return std::nullopt;
300 return llvm::formatv(Fmt: "_Z{}{}{}", Vals: baseName.size(), Vals&: baseName,
301 Vals&: typeMangling.value());
302 }
303
304 /// Get the subgroup size from the target or return a default.
305 static std::optional<int> getSubgroupSize(Operation *op) {
306 auto parentFunc = op->getParentOfType<LLVM::LLVMFuncOp>();
307 if (!parentFunc)
308 return std::nullopt;
309 return parentFunc.getIntelReqdSubGroupSize();
310 }
311
312 static bool hasValidWidth(gpu::ShuffleOp op, int subgroupSize) {
313 llvm::APInt val;
314 Value width = op.getWidth();
315 return matchPattern(value: width, pattern: m_ConstantInt(bind_value: &val)) && val == subgroupSize;
316 }
317
318 static Value bitcastOrExtBeforeShuffle(Value oldVal, Location loc,
319 ConversionPatternRewriter &rewriter) {
320 return TypeSwitch<Type, Value>(oldVal.getType())
321 .Case(caseFn: [&](BFloat16Type) {
322 return LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: rewriter.getI16Type(),
323 arg: oldVal);
324 })
325 .Case(caseFn: [&](IntegerType intTy) -> Value {
326 if (intTy.getWidth() == 1)
327 return LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: rewriter.getI8Type(),
328 arg: oldVal);
329 return oldVal;
330 })
331 .Default(defaultResult: oldVal);
332 }
333
334 static Value bitcastOrTruncAfterShuffle(Value oldVal, Type newTy,
335 Location loc,
336 ConversionPatternRewriter &rewriter) {
337 return TypeSwitch<Type, Value>(newTy)
338 .Case(caseFn: [&](BFloat16Type) {
339 return LLVM::BitcastOp::create(builder&: rewriter, location: loc, res: newTy, arg: oldVal);
340 })
341 .Case(caseFn: [&](IntegerType intTy) -> Value {
342 if (intTy.getWidth() == 1)
343 return LLVM::TruncOp::create(builder&: rewriter, location: loc, res: newTy, arg: oldVal);
344 return oldVal;
345 })
346 .Default(defaultResult: oldVal);
347 }
348
349 LogicalResult
350 matchAndRewrite(gpu::ShuffleOp op, OpAdaptor adaptor,
351 ConversionPatternRewriter &rewriter) const final {
352 auto maybeSubgroupSize = getSubgroupSize(op);
353 if (maybeSubgroupSize && !hasValidWidth(op, subgroupSize: maybeSubgroupSize.value()))
354 return rewriter.notifyMatchFailure(
355 arg&: op, msg: "shuffle width and subgroup size mismatch");
356
357 Location loc = op->getLoc();
358 Value inValue =
359 bitcastOrExtBeforeShuffle(oldVal: adaptor.getValue(), loc, rewriter);
360 std::optional<std::string> funcName =
361 getFuncName(mode: op.getMode(), type: inValue.getType());
362 if (!funcName)
363 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported value type");
364
365 Operation *moduleOp = op->getParentWithTrait<OpTrait::SymbolTable>();
366 assert(moduleOp && "Expecting module");
367 Type valueType = inValue.getType();
368 Type offsetType = adaptor.getOffset().getType();
369 Type resultType = valueType;
370 LLVM::LLVMFuncOp func = lookupOrCreateSPIRVFn(
371 symbolTable: moduleOp, name: funcName.value(), paramTypes: {valueType, offsetType}, resultType,
372 /*isMemNone=*/false, /*isConvergent=*/true);
373
374 std::array<Value, 2> args{inValue, adaptor.getOffset()};
375 Value result =
376 createSPIRVBuiltinCall(loc, rewriter, func, args).getResult();
377 Value resultOrConversion =
378 bitcastOrTruncAfterShuffle(oldVal: result, newTy: op.getType(i: 0), loc, rewriter);
379
380 Value trueVal =
381 LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: rewriter.getI1Type(), value: true);
382 rewriter.replaceOp(op, newValues: {resultOrConversion, trueVal});
383 return success();
384 }
385};
386
387class MemorySpaceToOpenCLMemorySpaceConverter final : public TypeConverter {
388public:
389 MemorySpaceToOpenCLMemorySpaceConverter(MLIRContext *ctx) {
390 addConversion(callback: [](Type t) { return t; });
391 addConversion(callback: [ctx](BaseMemRefType memRefType) -> std::optional<Type> {
392 // Attach global addr space attribute to memrefs with no addr space attr
393 Attribute memSpaceAttr = memRefType.getMemorySpace();
394 if (memSpaceAttr)
395 return std::nullopt;
396
397 unsigned globalAddrspace = storageClassToAddressSpace(
398 clientAPI: spirv::ClientAPI::OpenCL, storageClass: spirv::StorageClass::CrossWorkgroup);
399 Attribute addrSpaceAttr =
400 IntegerAttr::get(type: IntegerType::get(context: ctx, width: 64), value: globalAddrspace);
401 if (auto rankedType = dyn_cast<MemRefType>(Val&: memRefType)) {
402 return MemRefType::get(shape: memRefType.getShape(),
403 elementType: memRefType.getElementType(),
404 layout: rankedType.getLayout(), memorySpace: addrSpaceAttr);
405 }
406 return UnrankedMemRefType::get(elementType: memRefType.getElementType(),
407 memorySpace: addrSpaceAttr);
408 });
409 addConversion(callback: [this](FunctionType type) {
410 auto inputs = llvm::map_to_vector(
411 C: type.getInputs(), F: [this](Type ty) { return convertType(t: ty); });
412 auto results = llvm::map_to_vector(
413 C: type.getResults(), F: [this](Type ty) { return convertType(t: ty); });
414 return FunctionType::get(context: type.getContext(), inputs, results);
415 });
416 }
417};
418
419//===----------------------------------------------------------------------===//
420// Subgroup query ops.
421//===----------------------------------------------------------------------===//
422
423template <typename SubgroupOp>
424struct GPUSubgroupOpConversion final : ConvertOpToLLVMPattern<SubgroupOp> {
425 using ConvertOpToLLVMPattern<SubgroupOp>::ConvertOpToLLVMPattern;
426 using ConvertToLLVMPattern::getTypeConverter;
427
428 LogicalResult
429 matchAndRewrite(SubgroupOp op, typename SubgroupOp::Adaptor adaptor,
430 ConversionPatternRewriter &rewriter) const final {
431 constexpr StringRef funcName = [] {
432 if constexpr (std::is_same_v<SubgroupOp, gpu::SubgroupIdOp>) {
433 return "_Z16get_sub_group_id";
434 } else if constexpr (std::is_same_v<SubgroupOp, gpu::LaneIdOp>) {
435 return "_Z22get_sub_group_local_id";
436 } else if constexpr (std::is_same_v<SubgroupOp, gpu::NumSubgroupsOp>) {
437 return "_Z18get_num_sub_groups";
438 } else if constexpr (std::is_same_v<SubgroupOp, gpu::SubgroupSizeOp>) {
439 return "_Z18get_sub_group_size";
440 }
441 }();
442
443 Operation *moduleOp =
444 op->template getParentWithTrait<OpTrait::SymbolTable>();
445 Type resultTy = rewriter.getI32Type();
446 LLVM::LLVMFuncOp func =
447 lookupOrCreateSPIRVFn(symbolTable: moduleOp, name: funcName, paramTypes: {}, resultType: resultTy,
448 /*isMemNone=*/false, /*isConvergent=*/false);
449
450 Location loc = op->getLoc();
451 Value result = createSPIRVBuiltinCall(loc, rewriter, func, args: {}).getResult();
452
453 Type indexTy = getTypeConverter()->getIndexType();
454 if (resultTy != indexTy) {
455 if (indexTy.getIntOrFloatBitWidth() < resultTy.getIntOrFloatBitWidth()) {
456 return failure();
457 }
458 result = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: indexTy, arg: result);
459 }
460
461 rewriter.replaceOp(op, result);
462 return success();
463 }
464};
465
466//===----------------------------------------------------------------------===//
467// GPU To LLVM-SPV Pass.
468//===----------------------------------------------------------------------===//
469
470struct GPUToLLVMSPVConversionPass final
471 : impl::ConvertGpuOpsToLLVMSPVOpsBase<GPUToLLVMSPVConversionPass> {
472 using Base::Base;
473
474 void runOnOperation() final {
475 MLIRContext *context = &getContext();
476 RewritePatternSet patterns(context);
477
478 LowerToLLVMOptions options(context);
479 options.overrideIndexBitwidth(bitwidth: this->use64bitIndex ? 64 : 32);
480 LLVMTypeConverter converter(context, options);
481 LLVMConversionTarget target(*context);
482
483 // Force OpenCL address spaces when they are not present
484 {
485 MemorySpaceToOpenCLMemorySpaceConverter converter(context);
486 AttrTypeReplacer replacer;
487 replacer.addReplacement(callback: [&converter](BaseMemRefType origType)
488 -> std::optional<BaseMemRefType> {
489 return converter.convertType<BaseMemRefType>(t: origType);
490 });
491
492 replacer.recursivelyReplaceElementsIn(op: getOperation(),
493 /*replaceAttrs=*/true,
494 /*replaceLocs=*/false,
495 /*replaceTypes=*/true);
496 }
497
498 target.addIllegalOp<gpu::BarrierOp, gpu::BlockDimOp, gpu::BlockIdOp,
499 gpu::GPUFuncOp, gpu::GlobalIdOp, gpu::GridDimOp,
500 gpu::LaneIdOp, gpu::NumSubgroupsOp, gpu::ReturnOp,
501 gpu::ShuffleOp, gpu::SubgroupIdOp, gpu::SubgroupSizeOp,
502 gpu::ThreadIdOp, gpu::PrintfOp>();
503
504 populateGpuToLLVMSPVConversionPatterns(converter, patterns);
505 populateGpuMemorySpaceAttributeConversions(typeConverter&: converter);
506 patterns.add<GPUPrintfOpToLLVMCallLowering>(arg&: converter, /*addressSpace=*/args: 2,
507 args: LLVM::cconv::CConv::SPIR_FUNC,
508 args: "_Z6printfPU3AS2Kcz");
509
510 if (failed(Result: applyPartialConversion(op: getOperation(), target,
511 patterns: std::move(patterns))))
512 signalPassFailure();
513 }
514};
515} // namespace
516
517//===----------------------------------------------------------------------===//
518// GPU To LLVM-SPV Patterns.
519//===----------------------------------------------------------------------===//
520
521namespace mlir {
522namespace {
523static unsigned
524gpuAddressSpaceToOCLAddressSpace(gpu::AddressSpace addressSpace) {
525 constexpr spirv::ClientAPI clientAPI = spirv::ClientAPI::OpenCL;
526 return storageClassToAddressSpace(clientAPI,
527 storageClass: addressSpaceToStorageClass(addressSpace));
528}
529} // namespace
530
531void populateGpuToLLVMSPVConversionPatterns(
532 const LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) {
533 patterns.add<GPUBarrierConversion, GPUReturnOpLowering, GPUShuffleConversion,
534 GPUSubgroupOpConversion<gpu::LaneIdOp>,
535 GPUSubgroupOpConversion<gpu::NumSubgroupsOp>,
536 GPUSubgroupOpConversion<gpu::SubgroupIdOp>,
537 GPUSubgroupOpConversion<gpu::SubgroupSizeOp>,
538 LaunchConfigOpConversion<gpu::BlockDimOp>,
539 LaunchConfigOpConversion<gpu::BlockIdOp>,
540 LaunchConfigOpConversion<gpu::GlobalIdOp>,
541 LaunchConfigOpConversion<gpu::GridDimOp>,
542 LaunchConfigOpConversion<gpu::ThreadIdOp>>(arg: typeConverter);
543 MLIRContext *context = &typeConverter.getContext();
544 unsigned privateAddressSpace =
545 gpuAddressSpaceToOCLAddressSpace(addressSpace: gpu::AddressSpace::Private);
546 unsigned localAddressSpace =
547 gpuAddressSpaceToOCLAddressSpace(addressSpace: gpu::AddressSpace::Workgroup);
548 OperationName llvmFuncOpName(LLVM::LLVMFuncOp::getOperationName(), context);
549 StringAttr kernelBlockSizeAttributeName =
550 LLVM::LLVMFuncOp::getReqdWorkGroupSizeAttrName(name: llvmFuncOpName);
551 patterns.add<GPUFuncOpLowering>(
552 arg: typeConverter,
553 args: GPUFuncOpLoweringOptions{
554 .allocaAddrSpace: privateAddressSpace, .workgroupAddrSpace: localAddressSpace,
555 /*kernelAttributeName=*/{}, .kernelBlockSizeAttributeName: kernelBlockSizeAttributeName,
556 /*kernelClusterSizeAttributeName=*/{}, .kernelCallingConvention: LLVM::CConv::SPIR_KERNEL,
557 .nonKernelCallingConvention: LLVM::CConv::SPIR_FUNC,
558 /*encodeWorkgroupAttributionsAsArguments=*/true});
559}
560
561void populateGpuMemorySpaceAttributeConversions(TypeConverter &typeConverter) {
562 populateGpuMemorySpaceAttributeConversions(typeConverter,
563 mapping: gpuAddressSpaceToOCLAddressSpace);
564}
565} // namespace mlir
566