1//===- SPIRVToLLVM.cpp - SPIR-V to LLVM Patterns --------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements patterns to convert SPIR-V dialect to LLVM dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Conversion/SPIRVToLLVM/SPIRVToLLVM.h"
14#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
15#include "mlir/Conversion/SPIRVCommon/AttrToLLVMConverter.h"
16#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
17#include "mlir/Dialect/SPIRV/IR/SPIRVEnums.h"
18#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
19#include "mlir/Dialect/SPIRV/Utils/LayoutUtils.h"
20#include "mlir/IR/BuiltinOps.h"
21#include "mlir/IR/PatternMatch.h"
22#include "mlir/Interfaces/DataLayoutInterfaces.h"
23#include "mlir/Transforms/DialectConversion.h"
24#include "llvm/ADT/FloatingPointMode.h"
25#include "llvm/ADT/TypeSwitch.h"
26#include "llvm/Support/FormatVariadic.h"
27
28#define DEBUG_TYPE "spirv-to-llvm-pattern"
29
30using namespace mlir;
31
32//===----------------------------------------------------------------------===//
33// Utility functions
34//===----------------------------------------------------------------------===//
35
36/// Returns true if the given type is a signed integer or vector type.
37static bool isSignedIntegerOrVector(Type type) {
38 if (type.isSignedInteger())
39 return true;
40 if (auto vecType = dyn_cast<VectorType>(Val&: type))
41 return vecType.getElementType().isSignedInteger();
42 return false;
43}
44
45/// Returns true if the given type is an unsigned integer or vector type
46static bool isUnsignedIntegerOrVector(Type type) {
47 if (type.isUnsignedInteger())
48 return true;
49 if (auto vecType = dyn_cast<VectorType>(Val&: type))
50 return vecType.getElementType().isUnsignedInteger();
51 return false;
52}
53
54/// Returns the width of an integer or of the element type of an integer vector,
55/// if applicable.
56static std::optional<uint64_t> getIntegerOrVectorElementWidth(Type type) {
57 if (auto intType = dyn_cast<IntegerType>(Val&: type))
58 return intType.getWidth();
59 if (auto vecType = dyn_cast<VectorType>(Val&: type))
60 if (auto intType = dyn_cast<IntegerType>(Val: vecType.getElementType()))
61 return intType.getWidth();
62 return std::nullopt;
63}
64
65/// Returns the bit width of integer, float or vector of float or integer values
66static unsigned getBitWidth(Type type) {
67 assert((type.isIntOrFloat() || isa<VectorType>(type)) &&
68 "bitwidth is not supported for this type");
69 if (type.isIntOrFloat())
70 return type.getIntOrFloatBitWidth();
71 auto vecType = dyn_cast<VectorType>(Val&: type);
72 auto elementType = vecType.getElementType();
73 assert(elementType.isIntOrFloat() &&
74 "only integers and floats have a bitwidth");
75 return elementType.getIntOrFloatBitWidth();
76}
77
78/// Returns the bit width of LLVMType integer or vector.
79static unsigned getLLVMTypeBitWidth(Type type) {
80 if (auto vecTy = dyn_cast<VectorType>(Val&: type))
81 type = vecTy.getElementType();
82 return cast<IntegerType>(Val&: type).getWidth();
83}
84
85/// Creates `llvm.mlir.constant` with a scalar or vector integer value,
86/// broadcasting `scalarAttr` across the vector if `srcType` is a vector.
87static Value createIntegerConstant(Location loc, Type srcType, Type dstType,
88 PatternRewriter &rewriter,
89 IntegerAttr scalarAttr) {
90 if (auto vecType = dyn_cast<VectorType>(Val&: srcType))
91 return LLVM::ConstantOp::create(
92 builder&: rewriter, location: loc, res: dstType, value: SplatElementsAttr::get(type: vecType, values: scalarAttr));
93 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: dstType, value: scalarAttr);
94}
95
96/// Creates `llvm.mlir.constant` with all bits set for the given type.
97static Value createConstantAllBitsSet(Location loc, Type srcType, Type dstType,
98 PatternRewriter &rewriter) {
99 auto integerType = cast<IntegerType>(
100 Val: isa<VectorType>(Val: srcType) ? cast<VectorType>(Val&: srcType).getElementType()
101 : srcType);
102 return createIntegerConstant(loc, srcType, dstType, rewriter,
103 scalarAttr: rewriter.getIntegerAttr(type: integerType, value: -1));
104}
105
106/// Creates `llvm.mlir.constant` with a floating-point scalar or vector value.
107static Value createFPConstant(Location loc, Type srcType, Type dstType,
108 PatternRewriter &rewriter, double value) {
109 if (auto vecType = dyn_cast<VectorType>(Val&: srcType)) {
110 auto floatType = cast<FloatType>(Val: vecType.getElementType());
111 return LLVM::ConstantOp::create(
112 builder&: rewriter, location: loc, res: dstType,
113 value: SplatElementsAttr::get(type: vecType,
114 values: rewriter.getFloatAttr(type: floatType, value)));
115 }
116 auto floatType = cast<FloatType>(Val&: srcType);
117 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: dstType,
118 value: rewriter.getFloatAttr(type: floatType, value));
119}
120
121/// Utility function for bitfield ops:
122/// - `BitFieldInsert`
123/// - `BitFieldSExtract`
124/// - `BitFieldUExtract`
125/// Truncates or extends the value. If the bitwidth of the value is the same as
126/// `llvmType` bitwidth, the value remains unchanged.
127static Value optionallyTruncateOrExtend(Location loc, Value value,
128 Type llvmType,
129 PatternRewriter &rewriter) {
130 auto srcType = value.getType();
131 unsigned targetBitWidth = getLLVMTypeBitWidth(type: llvmType);
132 unsigned valueBitWidth = LLVM::isCompatibleType(type: srcType)
133 ? getLLVMTypeBitWidth(type: srcType)
134 : getBitWidth(type: srcType);
135
136 if (valueBitWidth < targetBitWidth)
137 return LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: llvmType, arg: value);
138 // If the bit widths of `Count` and `Offset` are greater than the bit width
139 // of the target type, they are truncated. Truncation is safe since `Count`
140 // and `Offset` must be no more than 64 for op behaviour to be defined. Hence,
141 // both values can be expressed in 8 bits.
142 if (valueBitWidth > targetBitWidth)
143 return LLVM::TruncOp::create(builder&: rewriter, location: loc, res: llvmType, arg: value);
144 return value;
145}
146
147/// Broadcasts the value to vector with `numElements` number of elements.
148static Value broadcast(Location loc, Value toBroadcast, unsigned numElements,
149 const TypeConverter &typeConverter,
150 ConversionPatternRewriter &rewriter) {
151 auto vectorType = VectorType::get(shape: numElements, elementType: toBroadcast.getType());
152 auto llvmVectorType = typeConverter.convertType(t: vectorType);
153 auto llvmI32Type = typeConverter.convertType(t: rewriter.getIntegerType(width: 32));
154 Value broadcasted = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: llvmVectorType);
155 for (unsigned i = 0; i < numElements; ++i) {
156 auto index = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmI32Type,
157 value: rewriter.getI32IntegerAttr(value: i));
158 broadcasted = LLVM::InsertElementOp::create(
159 builder&: rewriter, location: loc, res: llvmVectorType, vector: broadcasted, value: toBroadcast, position: index);
160 }
161 return broadcasted;
162}
163
164/// Broadcasts the value. If `srcType` is a scalar, the value remains unchanged.
165static Value optionallyBroadcast(Location loc, Value value, Type srcType,
166 const TypeConverter &typeConverter,
167 ConversionPatternRewriter &rewriter) {
168 if (auto vectorType = dyn_cast<VectorType>(Val&: srcType)) {
169 unsigned numElements = vectorType.getNumElements();
170 return broadcast(loc, toBroadcast: value, numElements, typeConverter, rewriter);
171 }
172 return value;
173}
174
175/// Utility function for bitfield ops: `BitFieldInsert`, `BitFieldSExtract` and
176/// `BitFieldUExtract`.
177/// Broadcast `Offset` and `Count` to match the type of `Base`. If `Base` is of
178/// a vector type, construct a vector that has:
179/// - same number of elements as `Base`
180/// - each element has the type that is the same as the type of `Offset` or
181/// `Count`
182/// - each element has the same value as `Offset` or `Count`
183/// Then cast `Offset` and `Count` if their bit width is different
184/// from `Base` bit width.
185static Value processCountOrOffset(Location loc, Value value, Type srcType,
186 Type dstType, const TypeConverter &converter,
187 ConversionPatternRewriter &rewriter) {
188 Value broadcasted =
189 optionallyBroadcast(loc, value, srcType, typeConverter: converter, rewriter);
190 return optionallyTruncateOrExtend(loc, value: broadcasted, llvmType: dstType, rewriter);
191}
192
193/// Converts SPIR-V struct with a regular (according to `VulkanLayoutUtils`)
194/// offset to LLVM struct. Otherwise, the conversion is not supported.
195static Type convertStructTypeWithOffset(spirv::StructType type,
196 const TypeConverter &converter) {
197 if (type != VulkanLayoutUtils::decorateType(structType: type))
198 return nullptr;
199
200 SmallVector<Type> elementsVector;
201 if (failed(Result: converter.convertTypes(types: type.getElementTypes(), results&: elementsVector)))
202 return nullptr;
203 return LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: elementsVector,
204 /*isPacked=*/false);
205}
206
207/// Converts SPIR-V struct with no offset to packed LLVM struct.
208static Type convertStructTypePacked(spirv::StructType type,
209 const TypeConverter &converter) {
210 SmallVector<Type> elementsVector;
211 if (failed(Result: converter.convertTypes(types: type.getElementTypes(), results&: elementsVector)))
212 return nullptr;
213 return LLVM::LLVMStructType::getLiteral(context: type.getContext(), types: elementsVector,
214 /*isPacked=*/true);
215}
216
217/// Creates LLVM dialect constant with the given value.
218static Value createI32ConstantOf(Location loc, PatternRewriter &rewriter,
219 unsigned value) {
220 return LLVM::ConstantOp::create(
221 builder&: rewriter, location: loc, res: IntegerType::get(context: rewriter.getContext(), width: 32),
222 value: rewriter.getIntegerAttr(type: rewriter.getI32Type(), value));
223}
224
225/// Utility for `spirv.Load` and `spirv.Store` conversion.
226static LogicalResult replaceWithLoadOrStore(Operation *op, ValueRange operands,
227 ConversionPatternRewriter &rewriter,
228 const TypeConverter &typeConverter,
229 unsigned alignment, bool isVolatile,
230 bool isNonTemporal) {
231 if (auto loadOp = dyn_cast<spirv::LoadOp>(Val: op)) {
232 auto dstType = typeConverter.convertType(t: loadOp.getType());
233 if (!dstType)
234 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
235 rewriter.replaceOpWithNewOp<LLVM::LoadOp>(
236 op: loadOp, args&: dstType, args: spirv::LoadOpAdaptor(operands).getPtr(), args&: alignment,
237 args&: isVolatile, args&: isNonTemporal);
238 return success();
239 }
240 auto storeOp = cast<spirv::StoreOp>(Val: op);
241 spirv::StoreOpAdaptor adaptor(operands);
242 rewriter.replaceOpWithNewOp<LLVM::StoreOp>(op: storeOp, args: adaptor.getValue(),
243 args: adaptor.getPtr(), args&: alignment,
244 args&: isVolatile, args&: isNonTemporal);
245 return success();
246}
247
248//===----------------------------------------------------------------------===//
249// Type conversion
250//===----------------------------------------------------------------------===//
251
252/// Converts SPIR-V array type to LLVM array. Natural stride (according to
253/// `VulkanLayoutUtils`) is also mapped to LLVM array. This has to be respected
254/// when converting ops that manipulate array types.
255static std::optional<Type> convertArrayType(spirv::ArrayType type,
256 TypeConverter &converter) {
257 unsigned stride = type.getArrayStride();
258 Type elementType = type.getElementType();
259 auto sizeInBytes = cast<spirv::SPIRVType>(Val&: elementType).getSizeInBytes();
260 if (stride != 0 && (!sizeInBytes || *sizeInBytes != stride))
261 return std::nullopt;
262
263 auto llvmElementType = converter.convertType(t: elementType);
264 unsigned numElements = type.getNumElements();
265 return LLVM::LLVMArrayType::get(elementType: llvmElementType, numElements);
266}
267
268/// Converts SPIR-V pointer type to LLVM pointer. Pointer's storage class is not
269/// modelled at the moment.
270static Type convertPointerType(spirv::PointerType type,
271 const TypeConverter &converter,
272 spirv::ClientAPI clientAPI) {
273 unsigned addressSpace =
274 storageClassToAddressSpace(clientAPI, storageClass: type.getStorageClass());
275 return LLVM::LLVMPointerType::get(context: type.getContext(), addressSpace);
276}
277
278/// Converts SPIR-V runtime array to LLVM array. Since LLVM allows indexing over
279/// the bounds, the runtime array is converted to a 0-sized LLVM array. There is
280/// no modelling of array stride at the moment.
281static std::optional<Type> convertRuntimeArrayType(spirv::RuntimeArrayType type,
282 TypeConverter &converter) {
283 if (type.getArrayStride() != 0)
284 return std::nullopt;
285 auto elementType = converter.convertType(t: type.getElementType());
286 return LLVM::LLVMArrayType::get(elementType, numElements: 0);
287}
288
289/// Converts SPIR-V struct to LLVM struct. There is no support of structs with
290/// member decorations. Also, only natural offset is supported.
291static Type convertStructType(spirv::StructType type,
292 const TypeConverter &converter) {
293 SmallVector<spirv::StructType::MemberDecorationInfo, 4> memberDecorations;
294 type.getMemberDecorations(memberDecorations);
295 if (!memberDecorations.empty())
296 return nullptr;
297 if (type.hasOffset())
298 return convertStructTypeWithOffset(type, converter);
299 return convertStructTypePacked(type, converter);
300}
301
302//===----------------------------------------------------------------------===//
303// Operation conversion
304//===----------------------------------------------------------------------===//
305
306namespace {
307
308template <typename TargetOp, typename SourceOp>
309static LogicalResult
310collectAttrsForConversion(SourceOp op,
311 typename TargetOp::Properties &properties,
312 SmallVectorImpl<NamedAttribute> &discardableAttrs) {
313 NamedAttrList attrs(op->getDiscardableAttrDictionary());
314 if (auto sourceProperties =
315 dyn_cast_or_null<DictionaryAttr>(op->getPropertiesAsAttribute()))
316 attrs.append(sourceProperties.getValue());
317
318 if constexpr (std::is_same_v<typename TargetOp::Properties,
319 EmptyProperties>) {
320 llvm::append_range(C&: discardableAttrs, R&: attrs);
321 return success();
322 }
323
324 TargetOp::populateDefaultProperties(
325 OperationName(TargetOp::getOperationName(), op->getContext()),
326 properties);
327 if (failed(TargetOp::setPropertiesFromAttr(
328 properties, attrs.getDictionary(context: op->getContext()),
329 [&]() { return op.emitError("failed to convert properties"); })))
330 return failure();
331
332 auto convertedProperties = dyn_cast_or_null<DictionaryAttr>(
333 TargetOp::getPropertiesAsAttr(op->getContext(), properties));
334 for (NamedAttribute attr : attrs) {
335 StringRef name = attr.getName().getValue();
336 bool isProperty = convertedProperties && convertedProperties.contains(name);
337 if (name == "operand_segment_sizes")
338 isProperty |= convertedProperties &&
339 convertedProperties.contains("operandSegmentSizes");
340 if (name == "result_segment_sizes")
341 isProperty |= convertedProperties &&
342 convertedProperties.contains("resultSegmentSizes");
343 if (!isProperty)
344 discardableAttrs.push_back(Elt: attr);
345 }
346 return success();
347}
348
349template <typename SPIRVOp, bool InBounds>
350class AccessChainPattern : public SPIRVToLLVMConversion<SPIRVOp> {
351public:
352 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
353
354 LogicalResult
355 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
356 ConversionPatternRewriter &rewriter) const override {
357 auto dstType =
358 this->getTypeConverter()->convertType(op.getComponentPtr().getType());
359 if (!dstType)
360 return rewriter.notifyMatchFailure(op, "type conversion failed");
361 if (op.getIndices().empty()) {
362 rewriter.replaceOp(op, adaptor.getBasePtr());
363 return success();
364 }
365 auto elementType = this->getTypeConverter()->convertType(
366 cast<spirv::PointerType>(op.getBasePtr().getType()).getPointeeType());
367 if (!elementType)
368 return rewriter.notifyMatchFailure(op, "type conversion failed");
369 // To use GEP we need to add a first 0 index to go through the pointer.
370 auto indices = llvm::to_vector<4>(adaptor.getIndices());
371 Type indexType = op.getIndices().front().getType();
372 auto llvmIndexType = this->getTypeConverter()->convertType(indexType);
373 if (!llvmIndexType)
374 return rewriter.notifyMatchFailure(op, "type conversion failed");
375 Value zero =
376 LLVM::ConstantOp::create(rewriter, op.getLoc(), llvmIndexType,
377 rewriter.getIntegerAttr(llvmIndexType, 0));
378 indices.insert(indices.begin(), zero);
379
380 rewriter.replaceOpWithNewOp<LLVM::GEPOp>(
381 op, dstType, elementType, adaptor.getBasePtr(), indices,
382 InBounds ? LLVM::GEPNoWrapFlags::inbounds : LLVM::GEPNoWrapFlags::none);
383 return success();
384 }
385};
386
387/// Converts `spirv.PtrAccessChain` and `spirv.InBoundsPtrAccessChain` to
388/// `llvm.getelementptr`, using `element` as the first index and preserving
389/// `inbounds` for the latter.
390template <typename SPIRVOp, bool InBounds>
391class PtrAccessChainPattern : public SPIRVToLLVMConversion<SPIRVOp> {
392public:
393 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
394
395 LogicalResult
396 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
397 ConversionPatternRewriter &rewriter) const override {
398 auto baseType = cast<spirv::PointerType>(op.getBasePtr().getType());
399 // These storage classes may use a pointer ArrayStride decoration, which
400 // the SPIR-V dialect cannot represent.
401 switch (baseType.getStorageClass()) {
402 case spirv::StorageClass::Uniform:
403 case spirv::StorageClass::StorageBuffer:
404 case spirv::StorageClass::PushConstant:
405 case spirv::StorageClass::PhysicalStorageBuffer:
406 return rewriter.notifyMatchFailure(
407 op, "pointer ArrayStride is not supported for this storage class");
408 default:
409 break;
410 }
411
412 auto dstType = this->getTypeConverter()->convertType(op.getType());
413 if (!dstType)
414 return rewriter.notifyMatchFailure(op, "type conversion failed");
415
416 auto elementType =
417 this->getTypeConverter()->convertType(baseType.getPointeeType());
418 if (!elementType)
419 return rewriter.notifyMatchFailure(op, "type conversion failed");
420
421 SmallVector<Value, 4> indices;
422 indices.push_back(Elt: adaptor.getElement());
423 llvm::append_range(indices, adaptor.getIndices());
424
425 rewriter.replaceOpWithNewOp<LLVM::GEPOp>(
426 op, dstType, elementType, adaptor.getBasePtr(), indices,
427 InBounds ? LLVM::GEPNoWrapFlags::inbounds : LLVM::GEPNoWrapFlags::none);
428 return success();
429 }
430};
431
432class AddressOfPattern : public SPIRVToLLVMConversion<spirv::AddressOfOp> {
433public:
434 using SPIRVToLLVMConversion<spirv::AddressOfOp>::SPIRVToLLVMConversion;
435
436 LogicalResult
437 matchAndRewrite(spirv::AddressOfOp op, OpAdaptor adaptor,
438 ConversionPatternRewriter &rewriter) const override {
439 auto dstType = getTypeConverter()->convertType(t: op.getPointer().getType());
440 if (!dstType)
441 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
442 rewriter.replaceOpWithNewOp<LLVM::AddressOfOp>(op, args&: dstType,
443 args: op.getVariable());
444 return success();
445 }
446};
447
448class BitFieldInsertPattern
449 : public SPIRVToLLVMConversion<spirv::BitFieldInsertOp> {
450public:
451 using SPIRVToLLVMConversion<spirv::BitFieldInsertOp>::SPIRVToLLVMConversion;
452
453 LogicalResult
454 matchAndRewrite(spirv::BitFieldInsertOp op, OpAdaptor adaptor,
455 ConversionPatternRewriter &rewriter) const override {
456 auto srcType = op.getType();
457 auto dstType = getTypeConverter()->convertType(t: srcType);
458 if (!dstType)
459 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
460 Location loc = op.getLoc();
461
462 // Process `Offset` and `Count`: broadcast and extend/truncate if needed.
463 Value offset = processCountOrOffset(loc, value: op.getOffset(), srcType, dstType,
464 converter: *getTypeConverter(), rewriter);
465 Value count = processCountOrOffset(loc, value: op.getCount(), srcType, dstType,
466 converter: *getTypeConverter(), rewriter);
467
468 // Create a mask with bits set outside [Offset, Offset + Count - 1].
469 Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
470 Value maskShiftedByCount =
471 LLVM::ShlOp::create(builder&: rewriter, location: loc, res: dstType, lhs: minusOne, rhs: count);
472 Value negated = LLVM::XOrOp::create(builder&: rewriter, location: loc, res: dstType,
473 lhs: maskShiftedByCount, rhs: minusOne);
474 Value maskShiftedByCountAndOffset =
475 LLVM::ShlOp::create(builder&: rewriter, location: loc, res: dstType, lhs: negated, rhs: offset);
476 Value mask = LLVM::XOrOp::create(builder&: rewriter, location: loc, res: dstType,
477 lhs: maskShiftedByCountAndOffset, rhs: minusOne);
478
479 // Extract unchanged bits from the `Base` that are outside of
480 // [Offset, Offset + Count - 1]. Then `or` with shifted `Insert`.
481 Value baseAndMask =
482 LLVM::AndOp::create(builder&: rewriter, location: loc, res: dstType, lhs: op.getBase(), rhs: mask);
483 Value insertShiftedByOffset =
484 LLVM::ShlOp::create(builder&: rewriter, location: loc, res: dstType, lhs: op.getInsert(), rhs: offset);
485 rewriter.replaceOpWithNewOp<LLVM::OrOp>(op, args&: dstType, args&: baseAndMask,
486 args&: insertShiftedByOffset);
487 return success();
488 }
489};
490
491/// Converts SPIR-V ConstantOp with scalar or vector type.
492class ConstantScalarAndVectorPattern
493 : public SPIRVToLLVMConversion<spirv::ConstantOp> {
494public:
495 using SPIRVToLLVMConversion<spirv::ConstantOp>::SPIRVToLLVMConversion;
496
497 LogicalResult
498 matchAndRewrite(spirv::ConstantOp constOp, OpAdaptor adaptor,
499 ConversionPatternRewriter &rewriter) const override {
500 auto srcType = constOp.getType();
501 if (!isa<VectorType>(Val: srcType) && !srcType.isIntOrFloat())
502 return failure();
503
504 auto dstType = getTypeConverter()->convertType(t: srcType);
505 if (!dstType)
506 return rewriter.notifyMatchFailure(arg&: constOp, msg: "type conversion failed");
507
508 // SPIR-V constant can be a signed/unsigned integer, which has to be
509 // casted to signless integer when converting to LLVM dialect. Removing the
510 // sign bit may have unexpected behaviour. However, it is better to handle
511 // it case-by-case, given that the purpose of the conversion is not to
512 // cover all possible corner cases.
513 if (isSignedIntegerOrVector(type: srcType) ||
514 isUnsignedIntegerOrVector(type: srcType)) {
515 auto signlessType = rewriter.getIntegerType(width: getBitWidth(type: srcType));
516
517 if (isa<VectorType>(Val: srcType)) {
518 auto dstElementsAttr = cast<DenseIntElementsAttr>(Val: constOp.getValue());
519 rewriter.replaceOpWithNewOp<LLVM::ConstantOp>(
520 op: constOp, args&: dstType,
521 args: dstElementsAttr.mapValues(
522 newElementType: signlessType, mapping: [&](const APInt &value) { return value; }));
523 return success();
524 }
525 auto srcAttr = cast<IntegerAttr>(Val: constOp.getValue());
526 auto dstAttr = rewriter.getIntegerAttr(type: signlessType, value: srcAttr.getValue());
527 rewriter.replaceOpWithNewOp<LLVM::ConstantOp>(op: constOp, args&: dstType, args&: dstAttr);
528 return success();
529 }
530 LLVM::ConstantOp::Properties properties{};
531 SmallVector<NamedAttribute> discardableAttrs;
532 if (failed(Result: collectAttrsForConversion<LLVM::ConstantOp>(op: constOp, properties,
533 discardableAttrs)))
534 return failure();
535 rewriter.replaceOpWithNewOp<LLVM::ConstantOp>(
536 op: constOp, args&: dstType, args: adaptor.getOperands(), args&: properties, args&: discardableAttrs);
537 return success();
538 }
539};
540
541/// Returns true if `createCompositeConstant` can materialize `value`.
542static bool isSupportedCompositeValue(Attribute value) {
543 if (auto arrayAttr = dyn_cast<ArrayAttr>(Val&: value))
544 return llvm::all_of(Range&: arrayAttr, P: isSupportedCompositeValue);
545 return isa<IntegerAttr, FloatAttr, DenseElementsAttr, spirv::NullAttr>(Val: value);
546}
547
548/// Requires `srcType` to convert to an LLVM type.
549static Value createCompositeConstant(Location loc, Attribute value,
550 Type srcType,
551 const TypeConverter &converter,
552 PatternRewriter &rewriter) {
553 Type dstType = converter.convertType(t: srcType);
554 auto arrayAttr = dyn_cast<ArrayAttr>(Val&: value);
555 // Poison would leave the only value of an empty struct undefined.
556 if (isa<spirv::NullAttr>(Val: value) || (arrayAttr && arrayAttr.empty()))
557 return LLVM::ZeroOp::create(builder&: rewriter, location: loc, res: dstType);
558 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: value))
559 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, type: dstType, value: intAttr.getValue());
560 if (isa<FloatAttr>(Val: value))
561 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: dstType, value);
562
563 auto arrayType = dyn_cast<spirv::ArrayType>(Val&: srcType);
564 if (arrayAttr && arrayType && arrayType.getElementType().isIntOrFloat())
565 value = DenseElementsAttr::get(
566 type: RankedTensorType::get(shape: {arrayType.getNumElements()},
567 elementType: arrayType.getElementType()),
568 values: arrayAttr.getValue());
569
570 if (auto denseAttr = dyn_cast<DenseElementsAttr>(Val&: value)) {
571 // LLVM expects signless integers and the shape of nested LLVM arrays.
572 // Floats keep their type even if it lowers to an integer, e.g. f8.
573 ShapedType shapedType = dyn_cast<VectorType>(Val&: dstType);
574 if (!shapedType) {
575 SmallVector<int64_t> shape;
576 Type elementType = dstType;
577 while (auto llvmArrayType = dyn_cast<LLVM::LLVMArrayType>(Val&: elementType)) {
578 shape.push_back(Elt: llvmArrayType.getNumElements());
579 elementType = llvmArrayType.getElementType();
580 }
581 shapedType = RankedTensorType::get(shape, elementType);
582 }
583 if (isa<FloatType>(Val: denseAttr.getElementType()))
584 shapedType = shapedType.clone(elementType: denseAttr.getElementType());
585 else if (denseAttr.getElementType() != shapedType.getElementType())
586 denseAttr = cast<DenseIntElementsAttr>(Val&: denseAttr).mapValues(
587 newElementType: shapedType.getElementType(), mapping: [](const APInt &v) { return v; });
588 return LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: dstType,
589 value: denseAttr.reshape(newType: shapedType));
590 }
591
592 auto compositeType = cast<spirv::CompositeType>(Val&: srcType);
593 Value result = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: dstType);
594 for (auto [idx, element] : llvm::enumerate(First&: arrayAttr)) {
595 Value elementValue = createCompositeConstant(
596 loc, value: element, srcType: compositeType.getElementType(idx), converter, rewriter);
597 result =
598 LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: elementValue, position: idx);
599 }
600 return result;
601}
602
603/// Converts SPIR-V ConstantOp with array or struct type.
604class ConstantCompositePattern
605 : public SPIRVToLLVMConversion<spirv::ConstantOp> {
606public:
607 using SPIRVToLLVMConversion<spirv::ConstantOp>::SPIRVToLLVMConversion;
608
609 LogicalResult
610 matchAndRewrite(spirv::ConstantOp constOp, OpAdaptor adaptor,
611 ConversionPatternRewriter &rewriter) const override {
612 Type srcType = constOp.getType();
613 Attribute value = constOp.getValue();
614 if (!isa<spirv::ArrayType, spirv::StructType>(Val: srcType) ||
615 isa<spirv::NullAttr>(Val: value))
616 return failure();
617 if (!getTypeConverter()->convertType(t: srcType))
618 return rewriter.notifyMatchFailure(arg&: constOp, msg: "type conversion failed");
619 if (!isSupportedCompositeValue(value))
620 return rewriter.notifyMatchFailure(arg&: constOp, msg: "unsupported value");
621
622 rewriter.replaceOp(op: constOp,
623 newValues: createCompositeConstant(loc: constOp.getLoc(), value, srcType,
624 converter: *getTypeConverter(), rewriter));
625 return success();
626 }
627};
628
629class BitFieldSExtractPattern
630 : public SPIRVToLLVMConversion<spirv::BitFieldSExtractOp> {
631public:
632 using SPIRVToLLVMConversion<spirv::BitFieldSExtractOp>::SPIRVToLLVMConversion;
633
634 LogicalResult
635 matchAndRewrite(spirv::BitFieldSExtractOp op, OpAdaptor adaptor,
636 ConversionPatternRewriter &rewriter) const override {
637 auto srcType = op.getType();
638 auto dstType = getTypeConverter()->convertType(t: srcType);
639 if (!dstType)
640 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
641 Location loc = op.getLoc();
642
643 // Process `Offset` and `Count`: broadcast and extend/truncate if needed.
644 Value offset = processCountOrOffset(loc, value: op.getOffset(), srcType, dstType,
645 converter: *getTypeConverter(), rewriter);
646 Value count = processCountOrOffset(loc, value: op.getCount(), srcType, dstType,
647 converter: *getTypeConverter(), rewriter);
648
649 // Create a constant that holds the size of the `Base`.
650 IntegerType integerType;
651 if (auto vecType = dyn_cast<VectorType>(Val&: srcType))
652 integerType = cast<IntegerType>(Val: vecType.getElementType());
653 else
654 integerType = cast<IntegerType>(Val&: srcType);
655
656 auto baseSize = rewriter.getIntegerAttr(type: integerType, value: getBitWidth(type: srcType));
657 Value size =
658 isa<VectorType>(Val: srcType)
659 ? LLVM::ConstantOp::create(
660 builder&: rewriter, location: loc, res: dstType,
661 value: SplatElementsAttr::get(type: cast<ShapedType>(Val&: srcType), values: baseSize))
662 : LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: dstType, value: baseSize);
663
664 // Shift `Base` left by [sizeof(Base) - (Count + Offset)], so that the bit
665 // at Offset + Count - 1 is the most significant bit now.
666 Value countPlusOffset =
667 LLVM::AddOp::create(builder&: rewriter, location: loc, res: dstType, lhs: count, rhs: offset);
668 Value amountToShiftLeft =
669 LLVM::SubOp::create(builder&: rewriter, location: loc, res: dstType, lhs: size, rhs: countPlusOffset);
670 Value baseShiftedLeft = LLVM::ShlOp::create(
671 builder&: rewriter, location: loc, res: dstType, lhs: op.getBase(), rhs: amountToShiftLeft);
672
673 // Shift the result right, filling the bits with the sign bit.
674 Value amountToShiftRight =
675 LLVM::AddOp::create(builder&: rewriter, location: loc, res: dstType, lhs: offset, rhs: amountToShiftLeft);
676 rewriter.replaceOpWithNewOp<LLVM::AShrOp>(op, args&: dstType, args&: baseShiftedLeft,
677 args&: amountToShiftRight);
678 return success();
679 }
680};
681
682class BitFieldUExtractPattern
683 : public SPIRVToLLVMConversion<spirv::BitFieldUExtractOp> {
684public:
685 using SPIRVToLLVMConversion<spirv::BitFieldUExtractOp>::SPIRVToLLVMConversion;
686
687 LogicalResult
688 matchAndRewrite(spirv::BitFieldUExtractOp op, OpAdaptor adaptor,
689 ConversionPatternRewriter &rewriter) const override {
690 auto srcType = op.getType();
691 auto dstType = getTypeConverter()->convertType(t: srcType);
692 if (!dstType)
693 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
694 Location loc = op.getLoc();
695
696 // Process `Offset` and `Count`: broadcast and extend/truncate if needed.
697 Value offset = processCountOrOffset(loc, value: op.getOffset(), srcType, dstType,
698 converter: *getTypeConverter(), rewriter);
699 Value count = processCountOrOffset(loc, value: op.getCount(), srcType, dstType,
700 converter: *getTypeConverter(), rewriter);
701
702 // Create a mask with bits set at [0, Count - 1].
703 Value minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
704 Value maskShiftedByCount =
705 LLVM::ShlOp::create(builder&: rewriter, location: loc, res: dstType, lhs: minusOne, rhs: count);
706 Value mask = LLVM::XOrOp::create(builder&: rewriter, location: loc, res: dstType, lhs: maskShiftedByCount,
707 rhs: minusOne);
708
709 // Shift `Base` by `Offset` and apply the mask on it.
710 Value shiftedBase =
711 LLVM::LShrOp::create(builder&: rewriter, location: loc, res: dstType, lhs: op.getBase(), rhs: offset);
712 rewriter.replaceOpWithNewOp<LLVM::AndOp>(op, args&: dstType, args&: shiftedBase, args&: mask);
713 return success();
714 }
715};
716
717class BranchConversionPattern : public SPIRVToLLVMConversion<spirv::BranchOp> {
718public:
719 using SPIRVToLLVMConversion<spirv::BranchOp>::SPIRVToLLVMConversion;
720
721 LogicalResult
722 matchAndRewrite(spirv::BranchOp branchOp, OpAdaptor adaptor,
723 ConversionPatternRewriter &rewriter) const override {
724 rewriter.replaceOpWithNewOp<LLVM::BrOp>(op: branchOp, args: adaptor.getOperands(),
725 args: branchOp.getTarget());
726 return success();
727 }
728};
729
730class BranchConditionalConversionPattern
731 : public SPIRVToLLVMConversion<spirv::BranchConditionalOp> {
732public:
733 using SPIRVToLLVMConversion<
734 spirv::BranchConditionalOp>::SPIRVToLLVMConversion;
735
736 LogicalResult
737 matchAndRewrite(spirv::BranchConditionalOp op, OpAdaptor adaptor,
738 ConversionPatternRewriter &rewriter) const override {
739 // If branch weights exist, map them to 32-bit integer vector.
740 DenseI32ArrayAttr branchWeights = nullptr;
741 if (auto weights = op.getBranchWeights()) {
742 SmallVector<int32_t> weightValues;
743 for (auto weight : weights->getAsRange<IntegerAttr>())
744 weightValues.push_back(Elt: weight.getInt());
745 branchWeights = DenseI32ArrayAttr::get(context: getContext(), content: weightValues);
746 }
747
748 rewriter.replaceOpWithNewOp<LLVM::CondBrOp>(
749 op, args: op.getCondition(), args: op.getTrueBlockArguments(),
750 args: op.getFalseBlockArguments(), args&: branchWeights, args: op.getTrueBlock(),
751 args: op.getFalseBlock());
752 return success();
753 }
754};
755
756class SwitchConversionPattern : public SPIRVToLLVMConversion<spirv::SwitchOp> {
757public:
758 using SPIRVToLLVMConversion<spirv::SwitchOp>::SPIRVToLLVMConversion;
759
760 LogicalResult
761 matchAndRewrite(spirv::SwitchOp op, OpAdaptor adaptor,
762 ConversionPatternRewriter &rewriter) const override {
763 // Literals have the selector type, which may be signed or unsigned. Retype
764 // them to the signless converted selector type.
765 DenseIntElementsAttr caseValues;
766 if (std::optional<DenseIntElementsAttr> literals = op.getLiterals())
767 caseValues = cast<DenseIntElementsAttr>(
768 Val: literals->bitcast(newElType: adaptor.getSelector().getType()));
769 if (!caseValues && !op.getTargets().empty())
770 return rewriter.notifyMatchFailure(arg&: op, msg: "targets without literals");
771
772 rewriter.replaceOpWithNewOp<LLVM::SwitchOp>(
773 op, args: adaptor.getSelector(), args: op.getDefaultTarget(),
774 args: adaptor.getDefaultOperands(), args&: caseValues, args: op.getTargets(),
775 args: adaptor.getTargetOperands());
776 return success();
777 }
778};
779
780/// Converts `spirv.getCompositeExtract` to `llvm.extractvalue` if the container
781/// type is an aggregate type (struct or array). Otherwise, converts to
782/// `llvm.extractelement` that operates on vectors.
783class CompositeExtractPattern
784 : public SPIRVToLLVMConversion<spirv::CompositeExtractOp> {
785public:
786 using SPIRVToLLVMConversion<spirv::CompositeExtractOp>::SPIRVToLLVMConversion;
787
788 LogicalResult
789 matchAndRewrite(spirv::CompositeExtractOp op, OpAdaptor adaptor,
790 ConversionPatternRewriter &rewriter) const override {
791 auto dstType = this->getTypeConverter()->convertType(t: op.getType());
792 if (!dstType)
793 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
794
795 Type containerType = op.getComposite().getType();
796 if (isa<VectorType>(Val: containerType)) {
797 Location loc = op.getLoc();
798 IntegerAttr value = cast<IntegerAttr>(Val: op.getIndices()[0]);
799 Value index = createI32ConstantOf(loc, rewriter, value: value.getInt());
800 rewriter.replaceOpWithNewOp<LLVM::ExtractElementOp>(
801 op, args&: dstType, args: adaptor.getComposite(), args&: index);
802 return success();
803 }
804
805 rewriter.replaceOpWithNewOp<LLVM::ExtractValueOp>(
806 op, args: adaptor.getComposite(),
807 args: LLVM::convertArrayToIndices(attrs: op.getIndices()));
808 return success();
809 }
810};
811
812/// Converts `spirv.getCompositeInsert` to `llvm.insertvalue` if the container
813/// type is an aggregate type (struct or array). Otherwise, converts to
814/// `llvm.insertelement` that operates on vectors.
815class CompositeInsertPattern
816 : public SPIRVToLLVMConversion<spirv::CompositeInsertOp> {
817public:
818 using SPIRVToLLVMConversion<spirv::CompositeInsertOp>::SPIRVToLLVMConversion;
819
820 LogicalResult
821 matchAndRewrite(spirv::CompositeInsertOp op, OpAdaptor adaptor,
822 ConversionPatternRewriter &rewriter) const override {
823 auto dstType = this->getTypeConverter()->convertType(t: op.getType());
824 if (!dstType)
825 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
826
827 Type containerType = op.getComposite().getType();
828 if (isa<VectorType>(Val: containerType)) {
829 Location loc = op.getLoc();
830 IntegerAttr value = cast<IntegerAttr>(Val: op.getIndices()[0]);
831 Value index = createI32ConstantOf(loc, rewriter, value: value.getInt());
832 rewriter.replaceOpWithNewOp<LLVM::InsertElementOp>(
833 op, args&: dstType, args: adaptor.getComposite(), args: adaptor.getObject(), args&: index);
834 return success();
835 }
836
837 rewriter.replaceOpWithNewOp<LLVM::InsertValueOp>(
838 op, args: adaptor.getComposite(), args: adaptor.getObject(),
839 args: LLVM::convertArrayToIndices(attrs: op.getIndices()));
840 return success();
841 }
842};
843
844/// Converts `spirv.CopyObject` by forwarding its operand.
845class CopyObjectPattern : public SPIRVToLLVMConversion<spirv::CopyObjectOp> {
846public:
847 using SPIRVToLLVMConversion<spirv::CopyObjectOp>::SPIRVToLLVMConversion;
848
849 LogicalResult
850 matchAndRewrite(spirv::CopyObjectOp op, OpAdaptor adaptor,
851 ConversionPatternRewriter &rewriter) const override {
852 rewriter.replaceOp(op, newValues: adaptor.getOperand());
853 return success();
854 }
855};
856
857/// Converts SPIR-V operations that have straightforward LLVM equivalent
858/// into LLVM dialect operations.
859template <typename SPIRVOp, typename LLVMOp>
860class DirectConversionPattern : public SPIRVToLLVMConversion<SPIRVOp> {
861public:
862 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
863
864 LogicalResult
865 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
866 ConversionPatternRewriter &rewriter) const override {
867 auto dstType = this->getTypeConverter()->convertType(op.getType());
868 if (!dstType)
869 return rewriter.notifyMatchFailure(op, "type conversion failed");
870 typename LLVMOp::Properties properties{};
871 SmallVector<NamedAttribute> discardableAttrs;
872 if (failed(collectAttrsForConversion<LLVMOp>(op, properties,
873 discardableAttrs)))
874 return failure();
875 rewriter.template replaceOpWithNewOp<LLVMOp>(
876 op, dstType, adaptor.getOperands(), properties, discardableAttrs);
877 return success();
878 }
879};
880
881/// Converts SPIR-V extended arithmetic ops (`spirv.IAddCarry`,
882/// `spirv.ISubBorrow`) that produce a two-member struct of {low-order bits,
883/// carry/borrow} into the matching LLVM `*.with.overflow` intrinsic. The
884/// intrinsic yields an `i1` carry/borrow, which is zero-extended to the full
885/// component width and re-packed into the SPIR-V result struct.
886template <typename SPIRVOp, typename LLVMOp>
887class ArithmeticWithOverflowPattern : public SPIRVToLLVMConversion<SPIRVOp> {
888public:
889 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
890
891 LogicalResult
892 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
893 ConversionPatternRewriter &rewriter) const override {
894 Type dstType = this->getTypeConverter()->convertType(op.getType());
895 if (!dstType)
896 return rewriter.notifyMatchFailure(op, "type conversion failed");
897
898 Location loc = op.getLoc();
899 Type operandType = adaptor.getOperand1().getType();
900 Type overflowType = rewriter.getI1Type();
901 if (auto vecType = dyn_cast<VectorType>(Val&: operandType))
902 overflowType = VectorType::get(shape: vecType.getShape(), elementType: overflowType);
903
904 Type intrType = LLVM::LLVMStructType::getLiteral(
905 context: rewriter.getContext(), types: {operandType, overflowType});
906 Value intrResult = LLVMOp::create(
907 rewriter, loc, intrType, adaptor.getOperand1(), adaptor.getOperand2());
908 Value lowBits = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: intrResult, position: 0);
909 Value overflow = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: intrResult, position: 1);
910 overflow = LLVM::ZExtOp::create(builder&: rewriter, location: loc, res: operandType, arg: overflow);
911
912 Value result = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: dstType);
913 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: lowBits,
914 position: ArrayRef<int64_t>{0});
915 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: overflow,
916 position: ArrayRef<int64_t>{1});
917 rewriter.replaceOp(op, result);
918 return success();
919 }
920};
921
922/// Converts `spirv.SMulExtended` and `spirv.UMulExtended` by multiplying
923/// operands extended to twice the width and packing the low and high halves
924/// into the SPIR-V result struct.
925template <typename SPIRVOp, typename LLVMExtOp>
926class MulExtendedPattern : public SPIRVToLLVMConversion<SPIRVOp> {
927public:
928 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
929
930 LogicalResult
931 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
932 ConversionPatternRewriter &rewriter) const override {
933 Type dstType = this->getTypeConverter()->convertType(op.getType());
934 if (!dstType)
935 return rewriter.notifyMatchFailure(op, "type conversion failed");
936
937 Location loc = op.getLoc();
938 Type operandType = adaptor.getOperand1().getType();
939 unsigned bitwidth = getLLVMTypeBitWidth(type: operandType);
940 IntegerType wideElemType = rewriter.getIntegerType(width: bitwidth * 2);
941 Type wideType = wideElemType;
942 if (auto vecType = dyn_cast<VectorType>(Val&: operandType))
943 wideType = VectorType::get(shape: vecType.getShape(), elementType: wideElemType);
944
945 Value lhs =
946 LLVMExtOp::create(rewriter, loc, wideType, adaptor.getOperand1());
947 Value rhs =
948 LLVMExtOp::create(rewriter, loc, wideType, adaptor.getOperand2());
949 Value mul = LLVM::MulOp::create(builder&: rewriter, location: loc, res: wideType, lhs, rhs);
950 Value low = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: operandType, arg: mul);
951 Value shift =
952 createIntegerConstant(loc, srcType: wideType, dstType: wideType, rewriter,
953 scalarAttr: rewriter.getIntegerAttr(type: wideElemType, value: bitwidth));
954 Value high = LLVM::LShrOp::create(builder&: rewriter, location: loc, lhs: mul, rhs: shift);
955 high = LLVM::TruncOp::create(builder&: rewriter, location: loc, res: operandType, arg: high);
956
957 Value result = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: dstType);
958 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: low,
959 position: ArrayRef<int64_t>{0});
960 result = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: result, value: high,
961 position: ArrayRef<int64_t>{1});
962 rewriter.replaceOp(op, result);
963 return success();
964 }
965};
966
967/// Converts `spirv.ExecutionMode` into a global struct constant that holds
968/// execution mode information.
969class ExecutionModePattern
970 : public SPIRVToLLVMConversion<spirv::ExecutionModeOp> {
971public:
972 using SPIRVToLLVMConversion<spirv::ExecutionModeOp>::SPIRVToLLVMConversion;
973
974 LogicalResult
975 matchAndRewrite(spirv::ExecutionModeOp op, OpAdaptor adaptor,
976 ConversionPatternRewriter &rewriter) const override {
977 // First, create the global struct's name that would be associated with
978 // this entry point's execution mode. We set it to be:
979 // __spv__{SPIR-V module name}_{function name}_execution_mode_info_{mode}
980 ModuleOp module = op->getParentOfType<ModuleOp>();
981 spirv::ExecutionModeAttr executionModeAttr = op.getExecutionModeAttr();
982 std::string moduleName;
983 if (module.getName().has_value())
984 moduleName = "_" + module.getName()->str();
985 else
986 moduleName = "";
987 std::string executionModeInfoName = llvm::formatv(
988 Fmt: "__spv_{0}_{1}_execution_mode_info_{2}", Vals&: moduleName, Vals: op.getFn().str(),
989 Vals: static_cast<uint32_t>(executionModeAttr.getValue()));
990
991 MLIRContext *context = rewriter.getContext();
992 OpBuilder::InsertionGuard guard(rewriter);
993 rewriter.setInsertionPointToStart(module.getBody());
994
995 // Create a struct type, corresponding to the C struct below.
996 // struct {
997 // int32_t executionMode;
998 // int32_t values[]; // optional values
999 // };
1000 auto llvmI32Type = IntegerType::get(context, width: 32);
1001 SmallVector<Type, 2> fields;
1002 fields.push_back(Elt: llvmI32Type);
1003 ArrayAttr values = op.getValues();
1004 if (!values.empty()) {
1005 auto arrayType = LLVM::LLVMArrayType::get(elementType: llvmI32Type, numElements: values.size());
1006 fields.push_back(Elt: arrayType);
1007 }
1008 auto structType = LLVM::LLVMStructType::getLiteral(context, types: fields);
1009
1010 // Create `llvm.mlir.global` with initializer region containing one block.
1011 auto global = LLVM::GlobalOp::create(
1012 builder&: rewriter, location: UnknownLoc::get(context), type: structType, /*isConstant=*/true,
1013 linkage: LLVM::Linkage::External, name: executionModeInfoName, value: Attribute(),
1014 /*alignment=*/0);
1015 Location loc = global.getLoc();
1016 Region &region = global.getInitializerRegion();
1017 Block *block = rewriter.createBlock(parent: &region);
1018
1019 // Initialize the struct and set the execution mode value.
1020 rewriter.setInsertionPointToStart(block);
1021 Value structValue = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: structType);
1022 Value executionMode = LLVM::ConstantOp::create(
1023 builder&: rewriter, location: loc, res: llvmI32Type,
1024 value: rewriter.getI32IntegerAttr(
1025 value: static_cast<uint32_t>(executionModeAttr.getValue())));
1026 SmallVector<int64_t> position{0};
1027 structValue = LLVM::InsertValueOp::create(builder&: rewriter, location: loc, container: structValue,
1028 value: executionMode, position);
1029
1030 // Insert extra operands if they exist into execution mode info struct.
1031 for (unsigned i = 0, e = values.size(); i < e; ++i) {
1032 auto attr = values.getValue()[i];
1033 Value entry = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: llvmI32Type, value: attr);
1034 structValue = LLVM::InsertValueOp::create(
1035 builder&: rewriter, location: loc, container: structValue, value: entry, position: ArrayRef<int64_t>({1, i}));
1036 }
1037 LLVM::ReturnOp::create(builder&: rewriter, location: loc, args: ArrayRef<Value>({structValue}));
1038 rewriter.eraseOp(op);
1039 return success();
1040 }
1041};
1042
1043/// Converts `spirv.GlobalVariable` to `llvm.mlir.global`. Note that SPIR-V
1044/// global returns a pointer, whereas in LLVM dialect the global holds an actual
1045/// value. This difference is handled by `spirv.mlir.addressof` and
1046/// `llvm.mlir.addressof`ops that both return a pointer.
1047class GlobalVariablePattern
1048 : public SPIRVToLLVMConversion<spirv::GlobalVariableOp> {
1049public:
1050 template <typename... Args>
1051 GlobalVariablePattern(spirv::ClientAPI clientAPI, Args &&...args)
1052 : SPIRVToLLVMConversion<spirv::GlobalVariableOp>(
1053 std::forward<Args>(args)...),
1054 clientAPI(clientAPI) {}
1055
1056 LogicalResult
1057 matchAndRewrite(spirv::GlobalVariableOp op, OpAdaptor adaptor,
1058 ConversionPatternRewriter &rewriter) const override {
1059 // Currently, there is no support of initialization with a constant value in
1060 // SPIR-V dialect. Specialization constants are not considered as well.
1061 if (op.getInitializer())
1062 return failure();
1063
1064 auto srcType = cast<spirv::PointerType>(Val: op.getType());
1065 auto dstType = getTypeConverter()->convertType(t: srcType.getPointeeType());
1066 if (!dstType)
1067 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1068
1069 // Limit conversion to the current invocation only or `StorageBuffer`
1070 // required by SPIR-V runner.
1071 // This is okay because multiple invocations are not supported yet.
1072 auto storageClass = srcType.getStorageClass();
1073 switch (storageClass) {
1074 case spirv::StorageClass::Input:
1075 case spirv::StorageClass::Private:
1076 case spirv::StorageClass::Output:
1077 case spirv::StorageClass::StorageBuffer:
1078 case spirv::StorageClass::UniformConstant:
1079 break;
1080 default:
1081 return failure();
1082 }
1083
1084 // LLVM dialect spec: "If the global value is a constant, storing into it is
1085 // not allowed.". This corresponds to SPIR-V 'Input' and 'UniformConstant'
1086 // storage class that is read-only.
1087 bool isConstant = (storageClass == spirv::StorageClass::Input) ||
1088 (storageClass == spirv::StorageClass::UniformConstant);
1089 // SPIR-V spec: "By default, functions and global variables are private to a
1090 // module and cannot be accessed by other modules. However, a module may be
1091 // written to export or import functions and global (module scope)
1092 // variables.". Therefore, map 'Private' storage class to private linkage,
1093 // 'Input' and 'Output' to external linkage.
1094 auto linkage = storageClass == spirv::StorageClass::Private
1095 ? LLVM::Linkage::Private
1096 : LLVM::Linkage::External;
1097 StringAttr locationAttrName = op.getLocationAttrName();
1098 IntegerAttr locationAttr = op.getLocationAttr();
1099 auto newGlobalOp = rewriter.replaceOpWithNewOp<LLVM::GlobalOp>(
1100 op, args&: dstType, args&: isConstant, args&: linkage, args: op.getSymName(), args: Attribute(),
1101 /*alignment=*/args: 0, args: storageClassToAddressSpace(clientAPI, storageClass));
1102
1103 // Attach location attribute if applicable
1104 if (locationAttr)
1105 newGlobalOp->setDiscardableAttr(name: locationAttrName, value: locationAttr);
1106
1107 return success();
1108 }
1109
1110private:
1111 spirv::ClientAPI clientAPI;
1112};
1113
1114/// Converts SPIR-V cast ops that do not have straightforward LLVM
1115/// equivalent in LLVM dialect.
1116template <typename SPIRVOp, typename LLVMExtOp, typename LLVMTruncOp>
1117class IndirectCastPattern : public SPIRVToLLVMConversion<SPIRVOp> {
1118public:
1119 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1120
1121 LogicalResult
1122 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
1123 ConversionPatternRewriter &rewriter) const override {
1124
1125 Type fromType = op.getOperand().getType();
1126 Type toType = op.getType();
1127
1128 auto dstType = this->getTypeConverter()->convertType(toType);
1129 if (!dstType)
1130 return rewriter.notifyMatchFailure(op, "type conversion failed");
1131
1132 if (getBitWidth(type: fromType) < getBitWidth(type: toType)) {
1133 rewriter.template replaceOpWithNewOp<LLVMExtOp>(op, dstType,
1134 adaptor.getOperands());
1135 return success();
1136 }
1137 if (getBitWidth(type: fromType) > getBitWidth(type: toType)) {
1138 rewriter.template replaceOpWithNewOp<LLVMTruncOp>(op, dstType,
1139 adaptor.getOperands());
1140 return success();
1141 }
1142 return failure();
1143 }
1144};
1145
1146class FunctionCallPattern
1147 : public SPIRVToLLVMConversion<spirv::FunctionCallOp> {
1148public:
1149 using SPIRVToLLVMConversion<spirv::FunctionCallOp>::SPIRVToLLVMConversion;
1150
1151 LogicalResult
1152 matchAndRewrite(spirv::FunctionCallOp callOp, OpAdaptor adaptor,
1153 ConversionPatternRewriter &rewriter) const override {
1154 LLVM::CallOp::Properties properties{};
1155 SmallVector<NamedAttribute> discardableAttrs;
1156 if (failed(Result: collectAttrsForConversion<LLVM::CallOp>(op: callOp, properties,
1157 discardableAttrs)))
1158 return failure();
1159 properties.operandSegmentSizes = {
1160 static_cast<int32_t>(adaptor.getOperands().size()), 0};
1161 properties.op_bundle_sizes = rewriter.getDenseI32ArrayAttr(values: {});
1162
1163 if (callOp.getNumResults() == 0) {
1164 rewriter.replaceOpWithNewOp<LLVM::CallOp>(op: callOp, args: TypeRange(),
1165 args: adaptor.getOperands(),
1166 args&: properties, args&: discardableAttrs);
1167 return success();
1168 }
1169
1170 // Function returns a single result.
1171 auto dstType = getTypeConverter()->convertType(t: callOp.getType(i: 0));
1172 if (!dstType)
1173 return rewriter.notifyMatchFailure(arg&: callOp, msg: "type conversion failed");
1174 rewriter.replaceOpWithNewOp<LLVM::CallOp>(
1175 op: callOp, args&: dstType, args: adaptor.getOperands(), args&: properties, args&: discardableAttrs);
1176 return success();
1177 }
1178};
1179
1180/// Converts SPIR-V floating-point comparisons to llvm.fcmp "predicate"
1181template <typename SPIRVOp, LLVM::FCmpPredicate predicate>
1182class FComparePattern : public SPIRVToLLVMConversion<SPIRVOp> {
1183public:
1184 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1185
1186 LogicalResult
1187 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
1188 ConversionPatternRewriter &rewriter) const override {
1189
1190 auto dstType = this->getTypeConverter()->convertType(op.getType());
1191 if (!dstType)
1192 return rewriter.notifyMatchFailure(op, "type conversion failed");
1193
1194 rewriter.template replaceOpWithNewOp<LLVM::FCmpOp>(
1195 op, dstType, predicate, op.getOperand1(), op.getOperand2());
1196 return success();
1197 }
1198};
1199
1200/// Converts SPIR-V integer comparisons to llvm.icmp "predicate"
1201template <typename SPIRVOp, LLVM::ICmpPredicate predicate>
1202class IComparePattern : public SPIRVToLLVMConversion<SPIRVOp> {
1203public:
1204 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1205
1206 LogicalResult
1207 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
1208 ConversionPatternRewriter &rewriter) const override {
1209
1210 auto dstType = this->getTypeConverter()->convertType(op.getType());
1211 if (!dstType)
1212 return rewriter.notifyMatchFailure(op, "type conversion failed");
1213
1214 rewriter.template replaceOpWithNewOp<LLVM::ICmpOp>(
1215 op, dstType, predicate, op.getOperand1(), op.getOperand2());
1216 return success();
1217 }
1218};
1219
1220/// Converts SPIR-V floating-point classification ops (`spirv.IsInf`,
1221/// `spirv.IsNan`, `spirv.IsFinite`, `spirv.IsNormal`) to
1222/// `llvm.intr.is.fpclass` with the matching `llvm::FPClassTest` mask. The
1223/// intrinsic tests the operand per component, like the SPIR-V ops, and yields
1224/// `i1` or a vector of `i1`.
1225template <typename SPIRVOp, llvm::FPClassTest classMask>
1226class FPClassPattern : public SPIRVToLLVMConversion<SPIRVOp> {
1227public:
1228 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1229
1230 LogicalResult
1231 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
1232 ConversionPatternRewriter &rewriter) const override {
1233 auto dstType = this->getTypeConverter()->convertType(op.getType());
1234 if (!dstType)
1235 return rewriter.notifyMatchFailure(op, "type conversion failed");
1236
1237 rewriter.template replaceOpWithNewOp<LLVM::IsFPClass>(
1238 op, dstType, adaptor.getOperand(), classMask);
1239 return success();
1240 }
1241};
1242
1243template <typename SPIRVOp>
1244class InverseSqrtPattern : public SPIRVToLLVMConversion<SPIRVOp> {
1245public:
1246 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1247
1248 LogicalResult
1249 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
1250 ConversionPatternRewriter &rewriter) const override {
1251 auto srcType = op.getType();
1252 auto dstType = this->getTypeConverter()->convertType(srcType);
1253 if (!dstType)
1254 return rewriter.notifyMatchFailure(op, "type conversion failed");
1255
1256 Location loc = op.getLoc();
1257 Value one = createFPConstant(loc, srcType, dstType, rewriter, 1.0);
1258 Value sqrt = LLVM::SqrtOp::create(rewriter, loc, dstType, op.getOperand());
1259 rewriter.template replaceOpWithNewOp<LLVM::FDivOp>(op, dstType, one, sqrt);
1260 return success();
1261 }
1262};
1263
1264/// Converts `spirv.VectorTimesScalar` to a broadcast of the scalar followed by
1265/// an `llvm.fmul`.
1266class VectorTimesScalarPattern
1267 : public SPIRVToLLVMConversion<spirv::VectorTimesScalarOp> {
1268public:
1269 using SPIRVToLLVMConversion<
1270 spirv::VectorTimesScalarOp>::SPIRVToLLVMConversion;
1271
1272 LogicalResult
1273 matchAndRewrite(spirv::VectorTimesScalarOp op, OpAdaptor adaptor,
1274 ConversionPatternRewriter &rewriter) const override {
1275 Type srcType = op.getType();
1276 Type dstType = getTypeConverter()->convertType(t: srcType);
1277 if (!dstType)
1278 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1279
1280 unsigned numElements = op.getVector().getType().getNumElements();
1281 Value broadcasted = broadcast(loc: op.getLoc(), toBroadcast: adaptor.getScalar(), numElements,
1282 typeConverter: *getTypeConverter(), rewriter);
1283 rewriter.replaceOpWithNewOp<LLVM::FMulOp>(op, args&: dstType, args: adaptor.getVector(),
1284 args&: broadcasted);
1285 return success();
1286 }
1287};
1288
1289/// Converts `spirv.SNegate` to `0 - x`.
1290class SNegatePattern : public SPIRVToLLVMConversion<spirv::SNegateOp> {
1291public:
1292 using SPIRVToLLVMConversion<spirv::SNegateOp>::SPIRVToLLVMConversion;
1293
1294 LogicalResult
1295 matchAndRewrite(spirv::SNegateOp op, OpAdaptor adaptor,
1296 ConversionPatternRewriter &rewriter) const override {
1297 Type srcType = op.getType();
1298 Type dstType = getTypeConverter()->convertType(t: srcType);
1299 if (!dstType)
1300 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1301
1302 Location loc = op.getLoc();
1303 IntegerAttr zeroAttr = rewriter.getIntegerAttr(
1304 type: cast<IntegerType>(Val: getElementTypeOrSelf(type: srcType)), value: 0);
1305 Value zero =
1306 createIntegerConstant(loc, srcType, dstType, rewriter, scalarAttr: zeroAttr);
1307 rewriter.replaceOpWithNewOp<LLVM::SubOp>(op, args&: dstType, args&: zero,
1308 args: adaptor.getOperand());
1309 return success();
1310 }
1311};
1312
1313/// Converts the GL and CL clamp ops into a nested min/max sequence, following
1314/// the op semantics `min(max(x, minVal), maxVal)`.
1315template <typename SPIRVOp, typename LLVMMinOp, typename LLVMMaxOp>
1316class ClampPattern : public SPIRVToLLVMConversion<SPIRVOp> {
1317public:
1318 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1319
1320 LogicalResult
1321 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
1322 ConversionPatternRewriter &rewriter) const override {
1323 Type dstType = this->getTypeConverter()->convertType(op.getType());
1324 if (!dstType)
1325 return rewriter.notifyMatchFailure(op, "type conversion failed");
1326
1327 Location loc = op.getLoc();
1328 Value max = LLVMMaxOp::create(rewriter, loc, dstType, adaptor.getX(),
1329 adaptor.getY());
1330 rewriter.template replaceOpWithNewOp<LLVMMinOp>(op, dstType, max,
1331 adaptor.getZ());
1332 return success();
1333 }
1334};
1335
1336/// Converts `spirv.FMod` to `x - y * floor(x / y)`. The SPIR-V op requires the
1337/// result to take the sign of the divisor, whereas `llvm.frem` keeps the sign
1338/// of the dividend, so `frem` cannot be used directly.
1339class FModPattern : public SPIRVToLLVMConversion<spirv::FModOp> {
1340public:
1341 using SPIRVToLLVMConversion<spirv::FModOp>::SPIRVToLLVMConversion;
1342
1343 LogicalResult
1344 matchAndRewrite(spirv::FModOp op, OpAdaptor adaptor,
1345 ConversionPatternRewriter &rewriter) const override {
1346 Type dstType = getTypeConverter()->convertType(t: op.getType());
1347 if (!dstType)
1348 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1349
1350 Location loc = op.getLoc();
1351 Value lhs = adaptor.getOperand1();
1352 Value rhs = adaptor.getOperand2();
1353 Value div = LLVM::FDivOp::create(builder&: rewriter, location: loc, res: dstType, lhs, rhs);
1354 Value floored = LLVM::FFloorOp::create(builder&: rewriter, location: loc, res: dstType, in: div);
1355 Value scaled = LLVM::FMulOp::create(builder&: rewriter, location: loc, res: dstType, lhs: rhs, rhs: floored);
1356 rewriter.replaceOpWithNewOp<LLVM::FSubOp>(op, args&: dstType, args&: lhs, args&: scaled);
1357 return success();
1358 }
1359};
1360
1361/// Converts `spirv.SMod` to a signed remainder corrected to take the sign of
1362/// the divisor. `llvm.srem` keeps the sign of the dividend, so the result is
1363/// adjusted by adding the divisor when the remainder is non-zero and its sign
1364/// differs from the divisor's.
1365class SModPattern : public SPIRVToLLVMConversion<spirv::SModOp> {
1366public:
1367 using SPIRVToLLVMConversion<spirv::SModOp>::SPIRVToLLVMConversion;
1368
1369 LogicalResult
1370 matchAndRewrite(spirv::SModOp op, OpAdaptor adaptor,
1371 ConversionPatternRewriter &rewriter) const override {
1372 Type srcType = op.getType();
1373 Type dstType = getTypeConverter()->convertType(t: srcType);
1374 if (!dstType)
1375 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1376
1377 Location loc = op.getLoc();
1378 Value lhs = adaptor.getOperand1();
1379 Value rhs = adaptor.getOperand2();
1380 Type i1Type = rewriter.getI1Type();
1381 auto vecSrcType = dyn_cast<VectorType>(Val&: srcType);
1382 Type cmpType =
1383 vecSrcType ? VectorType::get(shape: vecSrcType.getShape(), elementType: i1Type) : i1Type;
1384
1385 Value rem = LLVM::SRemOp::create(builder&: rewriter, location: loc, res: dstType, lhs, rhs);
1386 IntegerAttr zeroAttr = rewriter.getIntegerAttr(
1387 type: cast<IntegerType>(Val: getElementTypeOrSelf(type: srcType)), value: 0);
1388 Value zero =
1389 createIntegerConstant(loc, srcType, dstType, rewriter, scalarAttr: zeroAttr);
1390
1391 Value remNonZero = LLVM::ICmpOp::create(builder&: rewriter, location: loc, res: cmpType,
1392 predicate: LLVM::ICmpPredicate::ne, lhs: rem, rhs: zero);
1393 Value remNeg = LLVM::ICmpOp::create(builder&: rewriter, location: loc, res: cmpType,
1394 predicate: LLVM::ICmpPredicate::slt, lhs: rem, rhs: zero);
1395 Value rhsNeg = LLVM::ICmpOp::create(builder&: rewriter, location: loc, res: cmpType,
1396 predicate: LLVM::ICmpPredicate::slt, lhs: rhs, rhs: zero);
1397 Value signMismatch =
1398 LLVM::XOrOp::create(builder&: rewriter, location: loc, res: cmpType, lhs: remNeg, rhs: rhsNeg);
1399 Value needsAdjust =
1400 LLVM::AndOp::create(builder&: rewriter, location: loc, res: cmpType, lhs: remNonZero, rhs: signMismatch);
1401
1402 Value adjusted = LLVM::AddOp::create(builder&: rewriter, location: loc, res: dstType, lhs: rem, rhs);
1403 rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, args&: dstType, args&: needsAdjust,
1404 args&: adjusted, args&: rem);
1405 return success();
1406 }
1407};
1408
1409/// Converts `spirv.Load` and `spirv.Store` to LLVM dialect.
1410template <typename SPIRVOp>
1411class LoadStorePattern : public SPIRVToLLVMConversion<SPIRVOp> {
1412public:
1413 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1414
1415 LogicalResult
1416 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
1417 ConversionPatternRewriter &rewriter) const override {
1418 if (!op.getMemoryAccess()) {
1419 return replaceWithLoadOrStore(op, adaptor.getOperands(), rewriter,
1420 *this->getTypeConverter(), /*alignment=*/0,
1421 /*isVolatile=*/false,
1422 /*isNonTemporal=*/false);
1423 }
1424 auto memoryAccess = *op.getMemoryAccess();
1425 switch (memoryAccess) {
1426 case spirv::MemoryAccess::Aligned:
1427 case spirv::MemoryAccess::None:
1428 case spirv::MemoryAccess::Nontemporal:
1429 case spirv::MemoryAccess::Volatile: {
1430 unsigned alignment =
1431 memoryAccess == spirv::MemoryAccess::Aligned ? *op.getAlignment() : 0;
1432 bool isNonTemporal = memoryAccess == spirv::MemoryAccess::Nontemporal;
1433 bool isVolatile = memoryAccess == spirv::MemoryAccess::Volatile;
1434 return replaceWithLoadOrStore(op, adaptor.getOperands(), rewriter,
1435 *this->getTypeConverter(), alignment,
1436 isVolatile, isNonTemporal);
1437 }
1438 default:
1439 // There is no support of other memory access attributes.
1440 return failure();
1441 }
1442 }
1443};
1444
1445/// Returns true if `type` is, or (transitively) includes, a runtime array.
1446/// Such types have no fixed size, so they cannot be copied with a static byte
1447/// count; the SPIR-V spec also forbids them as the copied type of
1448/// `OpCopyMemory`.
1449static bool containsRuntimeArray(Type type) {
1450 if (isa<spirv::RuntimeArrayType>(Val: type))
1451 return true;
1452 if (auto arrayType = dyn_cast<spirv::ArrayType>(Val&: type))
1453 return containsRuntimeArray(type: arrayType.getElementType());
1454 if (auto structType = dyn_cast<spirv::StructType>(Val&: type))
1455 return llvm::any_of(Range: structType.getElementTypes(), P: containsRuntimeArray);
1456 return false;
1457}
1458
1459/// Converts `spirv.CopyMemory` to `llvm.intr.memcpy`.
1460class CopyMemoryPattern : public SPIRVToLLVMConversion<spirv::CopyMemoryOp> {
1461public:
1462 using SPIRVToLLVMConversion<spirv::CopyMemoryOp>::SPIRVToLLVMConversion;
1463
1464 LogicalResult
1465 matchAndRewrite(spirv::CopyMemoryOp op, OpAdaptor adaptor,
1466 ConversionPatternRewriter &rewriter) const override {
1467 Type pointeeType =
1468 cast<spirv::PointerType>(Val: op.getTarget().getType()).getPointeeType();
1469 // The op verifier does not reject runtime arrays, so check here. Note that
1470 // a struct containing a runtime array still converts to an LLVM type with
1471 // a well-defined (but meaningless here) size.
1472 if (containsRuntimeArray(type: pointeeType))
1473 return rewriter.notifyMatchFailure(arg&: op, msg: "pointee has no static size");
1474
1475 Type dstPointeeType = getTypeConverter()->convertType(t: pointeeType);
1476 if (!dstPointeeType)
1477 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
1478
1479 // SPIR-V types have no scalable components, so the size is always fixed.
1480 uint64_t byteSize =
1481 DataLayout::closest(op).getTypeSize(t: dstPointeeType).getFixedValue();
1482
1483 // In SPIR-V < 1.4, a single memory-operands mask applies to both Target
1484 // and Source; in SPIR-V >= 1.4, a second mask (if present) applies to
1485 // Source while the first applies to Target.
1486 using spirv::MemoryAccess;
1487 constexpr MemoryAccess supportedMask = MemoryAccess::Aligned |
1488 MemoryAccess::Volatile |
1489 MemoryAccess::Nontemporal;
1490 MemoryAccess targetAccess =
1491 op.getMemoryAccess().value_or(u: MemoryAccess::None);
1492 bool hasSourceAccess = op.getSourceMemoryAccess().has_value();
1493 MemoryAccess sourceAccess =
1494 hasSourceAccess ? *op.getSourceMemoryAccess() : targetAccess;
1495 if (((targetAccess | sourceAccess) & ~supportedMask) != MemoryAccess::None)
1496 return rewriter.notifyMatchFailure(
1497 arg&: op, msg: "unsupported memory access attributes");
1498
1499 bool isVolatile = spirv::bitEnumContainsAny(bits: targetAccess | sourceAccess,
1500 bit: MemoryAccess::Volatile);
1501 uint32_t targetAlign =
1502 spirv::bitEnumContainsAny(bits: targetAccess, bit: MemoryAccess::Aligned)
1503 ? op.getAlignment().value_or(u: 0)
1504 : 0;
1505 uint32_t sourceAlign =
1506 spirv::bitEnumContainsAny(bits: sourceAccess, bit: MemoryAccess::Aligned)
1507 ? (hasSourceAccess ? op.getSourceAlignment().value_or(u: 0)
1508 : targetAlign)
1509 : 0;
1510
1511 Location loc = op.getLoc();
1512 Value size = LLVM::ConstantOp::create(builder&: rewriter, location: loc, res: rewriter.getI64Type(),
1513 value: rewriter.getI64IntegerAttr(value: byteSize));
1514 auto memcpy = rewriter.replaceOpWithNewOp<LLVM::MemcpyOp>(
1515 op, args: adaptor.getTarget(), args: adaptor.getSource(), args&: size, args&: isVolatile);
1516
1517 if (targetAlign || sourceAlign) {
1518 auto makeAlignDict = [&](uint32_t align) {
1519 if (!align)
1520 return rewriter.getDictionaryAttr(value: {});
1521 return rewriter.getDictionaryAttr(
1522 value: {rewriter.getNamedAttr(name: LLVM::LLVMDialect::getAlignAttrName(),
1523 val: rewriter.getI64IntegerAttr(value: align))});
1524 };
1525 memcpy.setArgAttrsAttr(rewriter.getArrayAttr(
1526 value: {makeAlignDict(targetAlign), makeAlignDict(sourceAlign),
1527 rewriter.getDictionaryAttr(value: {})}));
1528 }
1529 return success();
1530 }
1531};
1532
1533/// Converts `spirv.Not` and `spirv.LogicalNot` into LLVM dialect.
1534template <typename SPIRVOp>
1535class NotPattern : public SPIRVToLLVMConversion<SPIRVOp> {
1536public:
1537 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1538
1539 LogicalResult
1540 matchAndRewrite(SPIRVOp notOp, typename SPIRVOp::Adaptor adaptor,
1541 ConversionPatternRewriter &rewriter) const override {
1542 auto srcType = notOp.getType();
1543 auto dstType = this->getTypeConverter()->convertType(srcType);
1544 if (!dstType)
1545 return rewriter.notifyMatchFailure(notOp, "type conversion failed");
1546
1547 Location loc = notOp.getLoc();
1548 Value mask = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
1549 rewriter.template replaceOpWithNewOp<LLVM::XOrOp>(notOp, dstType,
1550 notOp.getOperand(), mask);
1551 return success();
1552 }
1553};
1554
1555/// A template pattern that erases the given `SPIRVOp`.
1556template <typename SPIRVOp>
1557class ErasePattern : public SPIRVToLLVMConversion<SPIRVOp> {
1558public:
1559 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
1560
1561 LogicalResult
1562 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
1563 ConversionPatternRewriter &rewriter) const override {
1564 rewriter.eraseOp(op);
1565 return success();
1566 }
1567};
1568
1569class ReturnPattern : public SPIRVToLLVMConversion<spirv::ReturnOp> {
1570public:
1571 using SPIRVToLLVMConversion<spirv::ReturnOp>::SPIRVToLLVMConversion;
1572
1573 LogicalResult
1574 matchAndRewrite(spirv::ReturnOp returnOp, OpAdaptor adaptor,
1575 ConversionPatternRewriter &rewriter) const override {
1576 rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op: returnOp, args: ArrayRef<Type>(),
1577 args: ArrayRef<Value>());
1578 return success();
1579 }
1580};
1581
1582class ReturnValuePattern : public SPIRVToLLVMConversion<spirv::ReturnValueOp> {
1583public:
1584 using SPIRVToLLVMConversion<spirv::ReturnValueOp>::SPIRVToLLVMConversion;
1585
1586 LogicalResult
1587 matchAndRewrite(spirv::ReturnValueOp returnValueOp, OpAdaptor adaptor,
1588 ConversionPatternRewriter &rewriter) const override {
1589 rewriter.replaceOpWithNewOp<LLVM::ReturnOp>(op: returnValueOp, args: ArrayRef<Type>(),
1590 args: adaptor.getOperands());
1591 return success();
1592 }
1593};
1594
1595class UnreachablePattern : public SPIRVToLLVMConversion<spirv::UnreachableOp> {
1596public:
1597 using SPIRVToLLVMConversion<spirv::UnreachableOp>::SPIRVToLLVMConversion;
1598
1599 LogicalResult
1600 matchAndRewrite(spirv::UnreachableOp unreachableOp, OpAdaptor adaptor,
1601 ConversionPatternRewriter &rewriter) const override {
1602 rewriter.replaceOpWithNewOp<LLVM::UnreachableOp>(op: unreachableOp);
1603 return success();
1604 }
1605};
1606
1607static LLVM::LLVMFuncOp
1608lookupOrCreateSPIRVFn(Operation *symbolTable, StringRef name,
1609 ArrayRef<Type> paramTypes, Type resultType,
1610 bool convergent = true, bool isVarArg = false) {
1611 auto func = dyn_cast_or_null<LLVM::LLVMFuncOp>(
1612 Val: SymbolTable::lookupSymbolIn(op: symbolTable, symbol: name));
1613 if (func)
1614 return func;
1615
1616 OpBuilder b(symbolTable->getRegion(index: 0));
1617 func = LLVM::LLVMFuncOp::create(
1618 builder&: b, location: symbolTable->getLoc(), name,
1619 type: LLVM::LLVMFunctionType::get(result: resultType, arguments: paramTypes, isVarArg));
1620 func.setCConv(LLVM::cconv::CConv::SPIR_FUNC);
1621 func.setConvergent(convergent);
1622 func.setNoUnwind(true);
1623 func.setWillReturn(true);
1624 return func;
1625}
1626
1627static LLVM::CallOp createSPIRVBuiltinCall(Location loc, OpBuilder &builder,
1628 LLVM::LLVMFuncOp func,
1629 ValueRange args) {
1630 auto call = LLVM::CallOp::create(builder, location: loc, func, args);
1631 call.setCConv(func.getCConv());
1632 call.setConvergentAttr(func.getConvergentAttr());
1633 call.setNoUnwindAttr(func.getNoUnwindAttr());
1634 call.setWillReturnAttr(func.getWillReturnAttr());
1635 return call;
1636}
1637
1638template <typename BarrierOpTy>
1639class ControlBarrierPattern : public SPIRVToLLVMConversion<BarrierOpTy> {
1640public:
1641 using OpAdaptor = typename SPIRVToLLVMConversion<BarrierOpTy>::OpAdaptor;
1642
1643 using SPIRVToLLVMConversion<BarrierOpTy>::SPIRVToLLVMConversion;
1644
1645 static constexpr StringRef getFuncName();
1646
1647 LogicalResult
1648 matchAndRewrite(BarrierOpTy controlBarrierOp, OpAdaptor adaptor,
1649 ConversionPatternRewriter &rewriter) const override {
1650 constexpr StringRef funcName = getFuncName();
1651 Operation *symbolTable =
1652 controlBarrierOp->template getParentWithTrait<OpTrait::SymbolTable>();
1653
1654 Type i32 = rewriter.getI32Type();
1655
1656 Type voidTy = rewriter.getType<LLVM::LLVMVoidType>();
1657 LLVM::LLVMFuncOp func =
1658 lookupOrCreateSPIRVFn(symbolTable, name: funcName, paramTypes: {i32, i32, i32}, resultType: voidTy);
1659
1660 Location loc = controlBarrierOp->getLoc();
1661 Value execution = LLVM::ConstantOp::create(
1662 builder&: rewriter, location: loc, type: i32, value: static_cast<int32_t>(adaptor.getExecutionScope()));
1663 Value memory = LLVM::ConstantOp::create(
1664 builder&: rewriter, location: loc, type: i32, value: static_cast<int32_t>(adaptor.getMemoryScope()));
1665 Value semantics = LLVM::ConstantOp::create(
1666 builder&: rewriter, location: loc, type: i32, value: static_cast<int32_t>(adaptor.getMemorySemantics()));
1667
1668 auto call = createSPIRVBuiltinCall(loc, builder&: rewriter, func,
1669 args: {execution, memory, semantics});
1670
1671 rewriter.replaceOp(controlBarrierOp, call);
1672 return success();
1673 }
1674};
1675
1676namespace {
1677
1678StringRef getTypeMangling(Type type, bool isSigned) {
1679 return llvm::TypeSwitch<Type, StringRef>(type)
1680 .Case(caseFn: [](Float16Type) { return "Dh"; })
1681 .Case(caseFn: [](Float32Type) { return "f"; })
1682 .Case(caseFn: [](Float64Type) { return "d"; })
1683 .Case(caseFn: [isSigned](IntegerType intTy) {
1684 switch (intTy.getWidth()) {
1685 case 1:
1686 return "b";
1687 case 8:
1688 return (isSigned) ? "a" : "c";
1689 case 16:
1690 return (isSigned) ? "s" : "t";
1691 case 32:
1692 return (isSigned) ? "i" : "j";
1693 case 64:
1694 return (isSigned) ? "l" : "m";
1695 default:
1696 llvm_unreachable("Unsupported integer width");
1697 }
1698 })
1699 .DefaultUnreachable(message: "No mangling defined");
1700}
1701
1702template <typename ReduceOp>
1703constexpr StringLiteral getGroupFuncName();
1704
1705template <>
1706constexpr StringLiteral getGroupFuncName<spirv::GroupIAddOp>() {
1707 return "_Z17__spirv_GroupIAddii";
1708}
1709template <>
1710constexpr StringLiteral getGroupFuncName<spirv::GroupFAddOp>() {
1711 return "_Z17__spirv_GroupFAddii";
1712}
1713template <>
1714constexpr StringLiteral getGroupFuncName<spirv::GroupSMinOp>() {
1715 return "_Z17__spirv_GroupSMinii";
1716}
1717template <>
1718constexpr StringLiteral getGroupFuncName<spirv::GroupUMinOp>() {
1719 return "_Z17__spirv_GroupUMinii";
1720}
1721template <>
1722constexpr StringLiteral getGroupFuncName<spirv::GroupFMinOp>() {
1723 return "_Z17__spirv_GroupFMinii";
1724}
1725template <>
1726constexpr StringLiteral getGroupFuncName<spirv::GroupSMaxOp>() {
1727 return "_Z17__spirv_GroupSMaxii";
1728}
1729template <>
1730constexpr StringLiteral getGroupFuncName<spirv::GroupUMaxOp>() {
1731 return "_Z17__spirv_GroupUMaxii";
1732}
1733template <>
1734constexpr StringLiteral getGroupFuncName<spirv::GroupFMaxOp>() {
1735 return "_Z17__spirv_GroupFMaxii";
1736}
1737template <>
1738constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformIAddOp>() {
1739 return "_Z27__spirv_GroupNonUniformIAddii";
1740}
1741template <>
1742constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformFAddOp>() {
1743 return "_Z27__spirv_GroupNonUniformFAddii";
1744}
1745template <>
1746constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformIMulOp>() {
1747 return "_Z27__spirv_GroupNonUniformIMulii";
1748}
1749template <>
1750constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformFMulOp>() {
1751 return "_Z27__spirv_GroupNonUniformFMulii";
1752}
1753template <>
1754constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformSMinOp>() {
1755 return "_Z27__spirv_GroupNonUniformSMinii";
1756}
1757template <>
1758constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformUMinOp>() {
1759 return "_Z27__spirv_GroupNonUniformUMinii";
1760}
1761template <>
1762constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformFMinOp>() {
1763 return "_Z27__spirv_GroupNonUniformFMinii";
1764}
1765template <>
1766constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformSMaxOp>() {
1767 return "_Z27__spirv_GroupNonUniformSMaxii";
1768}
1769template <>
1770constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformUMaxOp>() {
1771 return "_Z27__spirv_GroupNonUniformUMaxii";
1772}
1773template <>
1774constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformFMaxOp>() {
1775 return "_Z27__spirv_GroupNonUniformFMaxii";
1776}
1777template <>
1778constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformBitwiseAndOp>() {
1779 return "_Z33__spirv_GroupNonUniformBitwiseAndii";
1780}
1781template <>
1782constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformBitwiseOrOp>() {
1783 return "_Z32__spirv_GroupNonUniformBitwiseOrii";
1784}
1785template <>
1786constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformBitwiseXorOp>() {
1787 return "_Z33__spirv_GroupNonUniformBitwiseXorii";
1788}
1789template <>
1790constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformLogicalAndOp>() {
1791 return "_Z33__spirv_GroupNonUniformLogicalAndii";
1792}
1793template <>
1794constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformLogicalOrOp>() {
1795 return "_Z32__spirv_GroupNonUniformLogicalOrii";
1796}
1797template <>
1798constexpr StringLiteral getGroupFuncName<spirv::GroupNonUniformLogicalXorOp>() {
1799 return "_Z33__spirv_GroupNonUniformLogicalXorii";
1800}
1801} // namespace
1802
1803template <typename ReduceOp, bool Signed = false, bool NonUniform = false>
1804class GroupReducePattern : public SPIRVToLLVMConversion<ReduceOp> {
1805public:
1806 using SPIRVToLLVMConversion<ReduceOp>::SPIRVToLLVMConversion;
1807
1808 LogicalResult
1809 matchAndRewrite(ReduceOp op, typename ReduceOp::Adaptor adaptor,
1810 ConversionPatternRewriter &rewriter) const override {
1811
1812 Type retTy = op.getResult().getType();
1813 if (!retTy.isIntOrFloat()) {
1814 return failure();
1815 }
1816 SmallString<36> funcName = getGroupFuncName<ReduceOp>();
1817 funcName += getTypeMangling(type: retTy, isSigned: false);
1818
1819 Type i32Ty = rewriter.getI32Type();
1820 SmallVector<Type> paramTypes{i32Ty, i32Ty, retTy};
1821 if constexpr (NonUniform) {
1822 if (adaptor.getClusterSize()) {
1823 funcName += "j";
1824 paramTypes.push_back(Elt: i32Ty);
1825 }
1826 }
1827
1828 Operation *symbolTable =
1829 op->template getParentWithTrait<OpTrait::SymbolTable>();
1830
1831 LLVM::LLVMFuncOp func =
1832 lookupOrCreateSPIRVFn(symbolTable, name: funcName, paramTypes, resultType: retTy);
1833
1834 Location loc = op.getLoc();
1835 Value scope = LLVM::ConstantOp::create(
1836 builder&: rewriter, location: loc, type: i32Ty,
1837 value: static_cast<int32_t>(adaptor.getExecutionScope()));
1838 Value groupOp = LLVM::ConstantOp::create(
1839 builder&: rewriter, location: loc, type: i32Ty,
1840 value: static_cast<int32_t>(adaptor.getGroupOperation()));
1841 SmallVector<Value> operands{scope, groupOp};
1842 operands.append(adaptor.getOperands().begin(), adaptor.getOperands().end());
1843
1844 auto call = createSPIRVBuiltinCall(loc, builder&: rewriter, func, args: operands);
1845 rewriter.replaceOp(op, call);
1846 return success();
1847 }
1848};
1849
1850template <>
1851constexpr StringRef
1852ControlBarrierPattern<spirv::ControlBarrierOp>::getFuncName() {
1853 return "_Z22__spirv_ControlBarrieriii";
1854}
1855
1856template <>
1857constexpr StringRef
1858ControlBarrierPattern<spirv::INTELControlBarrierArriveOp>::getFuncName() {
1859 return "_Z33__spirv_ControlBarrierArriveINTELiii";
1860}
1861
1862template <>
1863constexpr StringRef
1864ControlBarrierPattern<spirv::INTELControlBarrierWaitOp>::getFuncName() {
1865 return "_Z31__spirv_ControlBarrierWaitINTELiii";
1866}
1867
1868/// Converts `spirv.mlir.loop` to LLVM dialect. All blocks within selection
1869/// should be reachable for conversion to succeed. The structure of the loop in
1870/// LLVM dialect will be the following:
1871///
1872/// +------------------------------------+
1873/// | <code before spirv.mlir.loop> |
1874/// | llvm.br ^header |
1875/// +------------------------------------+
1876/// |
1877/// +----------------+ |
1878/// | | |
1879/// | V V
1880/// | +------------------------------------+
1881/// | | ^header: |
1882/// | | <header code> |
1883/// | | llvm.cond_br %cond, ^body, ^exit |
1884/// | +------------------------------------+
1885/// | |
1886/// | |----------------------+
1887/// | | |
1888/// | V |
1889/// | +------------------------------------+ |
1890/// | | ^body: | |
1891/// | | <body code> | |
1892/// | | llvm.br ^continue | |
1893/// | +------------------------------------+ |
1894/// | | |
1895/// | V |
1896/// | +------------------------------------+ |
1897/// | | ^continue: | |
1898/// | | <continue code> | |
1899/// | | llvm.br ^header | |
1900/// | +------------------------------------+ |
1901/// | | |
1902/// +---------------+ +----------------------+
1903/// |
1904/// V
1905/// +------------------------------------+
1906/// | ^exit: |
1907/// | llvm.br ^remaining |
1908/// +------------------------------------+
1909/// |
1910/// V
1911/// +------------------------------------+
1912/// | ^remaining: |
1913/// | <code after spirv.mlir.loop> |
1914/// +------------------------------------+
1915///
1916class LoopPattern : public SPIRVToLLVMConversion<spirv::LoopOp> {
1917public:
1918 using SPIRVToLLVMConversion<spirv::LoopOp>::SPIRVToLLVMConversion;
1919
1920 LogicalResult
1921 matchAndRewrite(spirv::LoopOp loopOp, OpAdaptor adaptor,
1922 ConversionPatternRewriter &rewriter) const override {
1923 // There is no support of loop control at the moment.
1924 if (loopOp.getLoopControl() != spirv::LoopControl::None)
1925 return failure();
1926
1927 // `spirv.mlir.loop` with empty region is redundant and should be erased.
1928 if (loopOp.getBody().empty()) {
1929 rewriter.eraseOp(op: loopOp);
1930 return success();
1931 }
1932
1933 Location loc = loopOp.getLoc();
1934
1935 SmallVector<Type> resultTypes;
1936 if (failed(Result: getTypeConverter()->convertTypes(types: loopOp.getResultTypes(),
1937 results&: resultTypes)))
1938 return rewriter.notifyMatchFailure(arg&: loopOp, msg: "failed to convert type");
1939
1940 // Split the current block after `spirv.mlir.loop`. The remaining ops will
1941 // be used in `endBlock`.
1942 Block *currentBlock = rewriter.getBlock();
1943 auto position = Block::iterator(loopOp);
1944 Block *endBlock = rewriter.splitBlock(block: currentBlock, before: position);
1945 for (Type ty : resultTypes)
1946 endBlock->addArgument(type: ty, loc);
1947
1948 // Remove entry block and create a branch in the current block going to the
1949 // header block.
1950 Block *entryBlock = loopOp.getEntryBlock();
1951 assert(entryBlock->getOperations().size() == 1);
1952 auto brOp = dyn_cast<spirv::BranchOp>(Val&: entryBlock->getOperations().front());
1953 if (!brOp)
1954 return failure();
1955 Block *headerBlock = loopOp.getHeaderBlock();
1956 rewriter.setInsertionPointToEnd(currentBlock);
1957 LLVM::BrOp::create(builder&: rewriter, location: loc, operands: brOp.getBlockArguments(), dest: headerBlock);
1958 rewriter.eraseBlock(block: entryBlock);
1959
1960 // Branch from merge block to end block.
1961 Block *mergeBlock = loopOp.getMergeBlock();
1962 Operation *terminator = mergeBlock->getTerminator();
1963 ValueRange terminatorOperands = terminator->getOperands();
1964 rewriter.setInsertionPointToEnd(mergeBlock);
1965 LLVM::BrOp::create(builder&: rewriter, location: loc, operands: terminatorOperands, dest: endBlock);
1966
1967 rewriter.inlineRegionBefore(region&: loopOp.getBody(), before: endBlock);
1968 rewriter.replaceOp(op: loopOp, newValues: endBlock->getArguments());
1969 return success();
1970 }
1971};
1972
1973/// Converts `spirv.mlir.selection`. All blocks within selection should be
1974/// reachable for conversion to succeed.
1975class SelectionPattern : public SPIRVToLLVMConversion<spirv::SelectionOp> {
1976public:
1977 using SPIRVToLLVMConversion<spirv::SelectionOp>::SPIRVToLLVMConversion;
1978
1979 LogicalResult
1980 matchAndRewrite(spirv::SelectionOp op, OpAdaptor adaptor,
1981 ConversionPatternRewriter &rewriter) const override {
1982 // There is no support for `Flatten` or `DontFlatten` selection control at
1983 // the moment. This are just compiler hints and can be performed during the
1984 // optimization passes.
1985 if (op.getSelectionControl() != spirv::SelectionControl::None)
1986 return failure();
1987
1988 // `spirv.mlir.selection` with empty region is redundant and is erased.
1989 if (op.getBody().empty()) {
1990 rewriter.eraseOp(op);
1991 return success();
1992 }
1993
1994 Location loc = op.getLoc();
1995
1996 SmallVector<Type> resultTypes;
1997 if (failed(
1998 Result: getTypeConverter()->convertTypes(types: op.getResultTypes(), results&: resultTypes)))
1999 return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert type");
2000
2001 // Split the current block after `spirv.mlir.selection`. The remaining ops
2002 // will be used in `continueBlock`.
2003 auto *currentBlock = rewriter.getInsertionBlock();
2004 rewriter.setInsertionPointAfter(op);
2005 auto position = rewriter.getInsertionPoint();
2006 auto *continueBlock = rewriter.splitBlock(block: currentBlock, before: position);
2007
2008 // Add arguments to the continue block for selections that yield values.
2009 for (Type ty : resultTypes)
2010 continueBlock->addArgument(type: ty, loc);
2011
2012 // Branch from merge block to continue block.
2013 auto *mergeBlock = op.getMergeBlock();
2014 Operation *terminator = mergeBlock->getTerminator();
2015 ValueRange terminatorOperands = terminator->getOperands();
2016 rewriter.setInsertionPointToEnd(mergeBlock);
2017 LLVM::BrOp::create(builder&: rewriter, location: loc, operands: terminatorOperands, dest: continueBlock);
2018
2019 // Header ops, including the terminator, are converted by their own
2020 // patterns.
2021 Block *headerBlock = op.getHeaderBlock();
2022 rewriter.inlineRegionBefore(region&: op.getBody(), before: continueBlock);
2023 rewriter.mergeBlocks(source: headerBlock, dest: currentBlock);
2024 rewriter.replaceOp(op, newValues: continueBlock->getArguments());
2025 return success();
2026 }
2027};
2028
2029/// Converts SPIR-V shift ops to LLVM shift ops. Since LLVM dialect
2030/// puts a restriction on `Shift` and `Base` to have the same bit width,
2031/// `Shift` is zero or sign extended to match this specification. Cases when
2032/// `Shift` bit width > `Base` bit width are considered to be illegal.
2033template <typename SPIRVOp, typename LLVMOp>
2034class ShiftPattern : public SPIRVToLLVMConversion<SPIRVOp> {
2035public:
2036 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
2037
2038 LogicalResult
2039 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
2040 ConversionPatternRewriter &rewriter) const override {
2041
2042 auto dstType = this->getTypeConverter()->convertType(op.getType());
2043 if (!dstType)
2044 return rewriter.notifyMatchFailure(op, "type conversion failed");
2045
2046 Type op1Type = op.getOperand1().getType();
2047 Type op2Type = op.getOperand2().getType();
2048
2049 if (op1Type == op2Type) {
2050 rewriter.template replaceOpWithNewOp<LLVMOp>(op, dstType,
2051 adaptor.getOperands());
2052 return success();
2053 }
2054
2055 std::optional<uint64_t> dstTypeWidth =
2056 getIntegerOrVectorElementWidth(dstType);
2057 std::optional<uint64_t> op2TypeWidth =
2058 getIntegerOrVectorElementWidth(type: op2Type);
2059
2060 if (!dstTypeWidth || !op2TypeWidth)
2061 return failure();
2062
2063 Location loc = op.getLoc();
2064 Value extended;
2065 if (op2TypeWidth < dstTypeWidth) {
2066 if (isUnsignedIntegerOrVector(type: op2Type)) {
2067 extended =
2068 LLVM::ZExtOp::create(rewriter, loc, dstType, adaptor.getOperand2());
2069 } else {
2070 extended =
2071 LLVM::SExtOp::create(rewriter, loc, dstType, adaptor.getOperand2());
2072 }
2073 } else if (op2TypeWidth == dstTypeWidth) {
2074 extended = adaptor.getOperand2();
2075 } else {
2076 return failure();
2077 }
2078
2079 Value result =
2080 LLVMOp::create(rewriter, loc, dstType, adaptor.getOperand1(), extended);
2081 rewriter.replaceOp(op, result);
2082 return success();
2083 }
2084};
2085
2086// `llvm.intr.abs` requires an `is_int_min_poison` immarg that `spirv.GL.SAbs`
2087// and `spirv.CL.s_abs` do not carry; default to `false` to preserve SPIR-V's
2088// well-defined behavior on INT_MIN.
2089template <typename SPIRVOp>
2090class SAbsPattern : public SPIRVToLLVMConversion<SPIRVOp> {
2091public:
2092 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
2093
2094 LogicalResult
2095 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
2096 ConversionPatternRewriter &rewriter) const override {
2097 Type dstType = this->getTypeConverter()->convertType(op.getType());
2098 if (!dstType)
2099 return rewriter.notifyMatchFailure(op, "type conversion failed");
2100
2101 rewriter.template replaceOpWithNewOp<LLVM::AbsOp>(
2102 op, dstType, adaptor.getOperand(), /*is_int_min_poison=*/false);
2103 return success();
2104 }
2105};
2106
2107/// Converts `spirv.CL.clz` and `spirv.CL.ctz` to `llvm.intr.ctlz` and
2108/// `llvm.intr.cttz`.
2109template <typename SPIRVOp, typename LLVMOp>
2110class CountZerosPattern : public SPIRVToLLVMConversion<SPIRVOp> {
2111public:
2112 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
2113
2114 LogicalResult
2115 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
2116 ConversionPatternRewriter &rewriter) const override {
2117 Type dstType = this->getTypeConverter()->convertType(op.getType());
2118 if (!dstType)
2119 return rewriter.notifyMatchFailure(op, "type conversion failed");
2120
2121 rewriter.template replaceOpWithNewOp<LLVMOp>(op, dstType,
2122 adaptor.getOperand(),
2123 /*is_zero_poison=*/false);
2124 return success();
2125 }
2126};
2127
2128/// Converts `spirv.GL.Fract` to `x - floor(x)`.
2129class FractPattern : public SPIRVToLLVMConversion<spirv::GLFractOp> {
2130public:
2131 using SPIRVToLLVMConversion<spirv::GLFractOp>::SPIRVToLLVMConversion;
2132
2133 LogicalResult
2134 matchAndRewrite(spirv::GLFractOp op, OpAdaptor adaptor,
2135 ConversionPatternRewriter &rewriter) const override {
2136 Type dstType = getTypeConverter()->convertType(t: op.getType());
2137 if (!dstType)
2138 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
2139
2140 Location loc = op.getLoc();
2141 Value operand = adaptor.getOperand();
2142 Value floored = LLVM::FFloorOp::create(builder&: rewriter, location: loc, res: dstType, in: operand);
2143 rewriter.replaceOpWithNewOp<LLVM::FSubOp>(op, args&: dstType, args&: operand, args&: floored);
2144 return success();
2145 }
2146};
2147
2148/// Converts `spirv.GL.FMix` to `x * (1 - a) + y * a` as specified by
2149/// GL.std.450.
2150class GLFMixPattern : public SPIRVToLLVMConversion<spirv::GLFMixOp> {
2151public:
2152 using SPIRVToLLVMConversion<spirv::GLFMixOp>::SPIRVToLLVMConversion;
2153
2154 LogicalResult
2155 matchAndRewrite(spirv::GLFMixOp op, OpAdaptor adaptor,
2156 ConversionPatternRewriter &rewriter) const override {
2157 Type dstType = getTypeConverter()->convertType(t: op.getType());
2158 if (!dstType)
2159 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
2160
2161 Location loc = op.getLoc();
2162 Value x = adaptor.getX();
2163 Value y = adaptor.getY();
2164 Value a = adaptor.getA();
2165 Value one = createFPConstant(loc, srcType: op.getType(), dstType, rewriter, value: 1.0);
2166 Value oneMinusA = LLVM::FSubOp::create(builder&: rewriter, location: loc, res: dstType, lhs: one, rhs: a);
2167 Value lhs = LLVM::FMulOp::create(builder&: rewriter, location: loc, res: dstType, lhs: x, rhs: oneMinusA);
2168 Value rhs = LLVM::FMulOp::create(builder&: rewriter, location: loc, res: dstType, lhs: y, rhs: a);
2169 rewriter.replaceOpWithNewOp<LLVM::FAddOp>(op, args&: dstType, args&: lhs, args&: rhs);
2170 return success();
2171 }
2172};
2173
2174/// Converts `spirv.GL.Asinh` to `log(x + sqrt(x * x + 1))` and `spirv.GL.Acosh`
2175/// to `log(x + sqrt(x * x - 1))`, the formulas Vulkan inherits their precision
2176/// from.
2177template <typename SPIRVOp>
2178class GLAsinhAcoshPattern : public SPIRVToLLVMConversion<SPIRVOp> {
2179public:
2180 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
2181
2182 LogicalResult
2183 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
2184 ConversionPatternRewriter &rewriter) const override {
2185 Type srcType = op.getType();
2186 Type dstType = this->getTypeConverter()->convertType(srcType);
2187 if (!dstType)
2188 return rewriter.notifyMatchFailure(op, "type conversion failed");
2189
2190 Location loc = op.getLoc();
2191 Value x = adaptor.getOperand();
2192 double c = std::is_same_v<SPIRVOp, spirv::GLAsinhOp> ? 1.0 : -1.0;
2193 Value cst = createFPConstant(loc, srcType, dstType, rewriter, value: c);
2194 Value sq = LLVM::FMulOp::create(builder&: rewriter, location: loc, res: dstType, lhs: x, rhs: x);
2195 Value rad = LLVM::FAddOp::create(builder&: rewriter, location: loc, res: dstType, lhs: sq, rhs: cst);
2196 Value root = LLVM::SqrtOp::create(builder&: rewriter, location: loc, res: dstType, in: rad);
2197 Value sum = LLVM::FAddOp::create(builder&: rewriter, location: loc, res: dstType, lhs: x, rhs: root);
2198 rewriter.replaceOpWithNewOp<LLVM::LogOp>(op, dstType, sum);
2199 return success();
2200 }
2201};
2202
2203/// Converts `spirv.CL.mix` to `fma(a, y - x, x)`. The OpenCL spec defines
2204/// mix as `x + (y - x) * a` and explicitly permits FMA contractions.
2205class CLMixPattern : public SPIRVToLLVMConversion<spirv::CLMixOp> {
2206public:
2207 using SPIRVToLLVMConversion<spirv::CLMixOp>::SPIRVToLLVMConversion;
2208
2209 LogicalResult
2210 matchAndRewrite(spirv::CLMixOp op, OpAdaptor adaptor,
2211 ConversionPatternRewriter &rewriter) const override {
2212 Type dstType = getTypeConverter()->convertType(t: op.getType());
2213 if (!dstType)
2214 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
2215
2216 Location loc = op.getLoc();
2217 Value x = adaptor.getX();
2218 Value y = adaptor.getY();
2219 Value a = adaptor.getZ();
2220 Value diff = LLVM::FSubOp::create(builder&: rewriter, location: loc, res: dstType, lhs: y, rhs: x);
2221 rewriter.replaceOpWithNewOp<LLVM::FMAOp>(op, args&: dstType, args&: a, args&: diff, args&: x);
2222 return success();
2223 }
2224};
2225
2226/// Converts `spirv.CL.sincos` to `llvm.intr.sincos`, storing the cosine to
2227/// `cosval` and returning the sine.
2228class CLSincosPattern : public SPIRVToLLVMConversion<spirv::CLSincosOp> {
2229public:
2230 using SPIRVToLLVMConversion<spirv::CLSincosOp>::SPIRVToLLVMConversion;
2231
2232 LogicalResult
2233 matchAndRewrite(spirv::CLSincosOp op, OpAdaptor adaptor,
2234 ConversionPatternRewriter &rewriter) const override {
2235 Type dstType = getTypeConverter()->convertType(t: op.getType());
2236 if (!dstType)
2237 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
2238
2239 Location loc = op.getLoc();
2240 auto structType = LLVM::LLVMStructType::getLiteral(context: rewriter.getContext(),
2241 types: {dstType, dstType});
2242 Value sincos =
2243 LLVM::SincosOp::create(builder&: rewriter, location: loc, res: structType, val: adaptor.getX());
2244 Value sin = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: sincos, position: 0);
2245 Value cos = LLVM::ExtractValueOp::create(builder&: rewriter, location: loc, container: sincos, position: 1);
2246 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: cos, addr: adaptor.getCosval());
2247 rewriter.replaceOp(op, newValues: sin);
2248 return success();
2249 }
2250};
2251
2252/// Converts `spirv.CL.printf` to a call to the variadic OpenCL `printf`.
2253class CLPrintfPattern : public SPIRVToLLVMConversion<spirv::CLPrintfOp> {
2254public:
2255 using SPIRVToLLVMConversion<spirv::CLPrintfOp>::SPIRVToLLVMConversion;
2256
2257 LogicalResult
2258 matchAndRewrite(spirv::CLPrintfOp op, OpAdaptor adaptor,
2259 ConversionPatternRewriter &rewriter) const override {
2260 Type dstType = getTypeConverter()->convertType(t: op.getType());
2261 if (!dstType)
2262 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
2263
2264 Operation *symbolTable = op->getParentWithTrait<OpTrait::SymbolTable>();
2265 Type formatType = adaptor.getFormat().getType();
2266 LLVM::LLVMFuncOp func =
2267 lookupOrCreateSPIRVFn(symbolTable, name: "printf", paramTypes: {formatType}, resultType: dstType,
2268 /*convergent=*/false, /*isVarArg=*/true);
2269 // The module may already declare printf with another signature. Calling
2270 // it would produce an invalid llvm.call.
2271 if (func.getFunctionType() !=
2272 LLVM::LLVMFunctionType::get(result: dstType, arguments: formatType, /*isVarArg=*/true))
2273 return rewriter.notifyMatchFailure(arg&: op, msg: "printf signature mismatch");
2274 rewriter.replaceOp(op, newOp: createSPIRVBuiltinCall(loc: op.getLoc(), builder&: rewriter, func,
2275 args: adaptor.getOperands()));
2276 return success();
2277 }
2278};
2279
2280// Converts spirv.GL.Radians (scale = pi/180) and spirv.GL.Degrees
2281// (scale = 180/pi) by multiplying the operand by a compile-time constant.
2282template <typename SPIRVOp>
2283class ScalePattern : public SPIRVToLLVMConversion<SPIRVOp> {
2284public:
2285 template <typename... Args>
2286 ScalePattern(double scale, Args &&...args)
2287 : SPIRVToLLVMConversion<SPIRVOp>(std::forward<Args>(args)...),
2288 scale(scale) {}
2289
2290 LogicalResult
2291 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
2292 ConversionPatternRewriter &rewriter) const override {
2293 Type srcType = op.getType();
2294 Type dstType = this->getTypeConverter()->convertType(srcType);
2295 if (!dstType)
2296 return rewriter.notifyMatchFailure(op, "type conversion failed");
2297
2298 Location loc = op.getLoc();
2299 Value factor = createFPConstant(loc, srcType, dstType, rewriter, value: scale);
2300 rewriter.replaceOpWithNewOp<LLVM::FMulOp>(op, dstType, adaptor.getOperand(),
2301 factor);
2302 return success();
2303 }
2304
2305private:
2306 double scale;
2307};
2308
2309/// Converts `spirv.GL.FSign`/`spirv.GL.SSign` to a sign(x) sequence that maps
2310/// the operand to -1/0/1 using two comparisons and two selects. The `isFloat`
2311/// flag selects between floating-point and integer comparisons/constants.
2312template <typename SPIRVOp, bool isFloat>
2313class SignPattern : public SPIRVToLLVMConversion<SPIRVOp> {
2314public:
2315 using SPIRVToLLVMConversion<SPIRVOp>::SPIRVToLLVMConversion;
2316
2317 LogicalResult
2318 matchAndRewrite(SPIRVOp op, typename SPIRVOp::Adaptor adaptor,
2319 ConversionPatternRewriter &rewriter) const override {
2320 Type srcType = op.getType();
2321 Type dstType = this->getTypeConverter()->convertType(srcType);
2322 if (!dstType)
2323 return rewriter.notifyMatchFailure(op, "type conversion failed");
2324
2325 Location loc = op.getLoc();
2326 Value operand = adaptor.getOperand();
2327 auto vecSrcType = dyn_cast<VectorType>(Val&: srcType);
2328 Type i1Type = rewriter.getI1Type();
2329 Type cmpType =
2330 vecSrcType ? VectorType::get(shape: vecSrcType.getShape(), elementType: i1Type) : i1Type;
2331
2332 Value zero, one, minusOne, gt, lt;
2333 if constexpr (isFloat) {
2334 zero = createFPConstant(loc, srcType, dstType, rewriter, value: 0.0);
2335 one = createFPConstant(loc, srcType, dstType, rewriter, value: 1.0);
2336 minusOne = createFPConstant(loc, srcType, dstType, rewriter, value: -1.0);
2337 gt = LLVM::FCmpOp::create(builder&: rewriter, location: loc, res: cmpType,
2338 predicate: LLVM::FCmpPredicate::ogt, lhs: operand, rhs: zero);
2339 lt = LLVM::FCmpOp::create(builder&: rewriter, location: loc, res: cmpType,
2340 predicate: LLVM::FCmpPredicate::olt, lhs: operand, rhs: zero);
2341 } else {
2342 auto intElemType = cast<IntegerType>(Val: getElementTypeOrSelf(type: srcType));
2343 zero = createIntegerConstant(loc, srcType, dstType, rewriter,
2344 scalarAttr: rewriter.getIntegerAttr(type: intElemType, value: 0));
2345 one = createIntegerConstant(loc, srcType, dstType, rewriter,
2346 scalarAttr: rewriter.getIntegerAttr(type: intElemType, value: 1));
2347 minusOne = createConstantAllBitsSet(loc, srcType, dstType, rewriter);
2348 gt = LLVM::ICmpOp::create(builder&: rewriter, location: loc, res: cmpType,
2349 predicate: LLVM::ICmpPredicate::sgt, lhs: operand, rhs: zero);
2350 lt = LLVM::ICmpOp::create(builder&: rewriter, location: loc, res: cmpType,
2351 predicate: LLVM::ICmpPredicate::slt, lhs: operand, rhs: zero);
2352 }
2353
2354 Value negOrZero =
2355 LLVM::SelectOp::create(builder&: rewriter, location: loc, res: dstType, condition: lt, trueValue: minusOne, falseValue: zero);
2356 rewriter.replaceOpWithNewOp<LLVM::SelectOp>(op, dstType, gt, one,
2357 negOrZero);
2358 return success();
2359 }
2360};
2361
2362class VariablePattern : public SPIRVToLLVMConversion<spirv::VariableOp> {
2363public:
2364 using SPIRVToLLVMConversion<spirv::VariableOp>::SPIRVToLLVMConversion;
2365
2366 LogicalResult
2367 matchAndRewrite(spirv::VariableOp varOp, OpAdaptor adaptor,
2368 ConversionPatternRewriter &rewriter) const override {
2369 auto srcType = varOp.getType();
2370 auto pointerTo = cast<spirv::PointerType>(Val&: srcType).getPointeeType();
2371 auto init = varOp.getInitializer();
2372
2373 auto dstType = getTypeConverter()->convertType(t: srcType);
2374 if (!dstType)
2375 return rewriter.notifyMatchFailure(arg&: varOp, msg: "type conversion failed");
2376
2377 Location loc = varOp.getLoc();
2378 Value size = createI32ConstantOf(loc, rewriter, value: 1);
2379 if (!init) {
2380 auto elementType = getTypeConverter()->convertType(t: pointerTo);
2381 if (!elementType)
2382 return rewriter.notifyMatchFailure(arg&: varOp, msg: "type conversion failed");
2383 rewriter.replaceOpWithNewOp<LLVM::AllocaOp>(op: varOp, args&: dstType, args&: elementType,
2384 args&: size);
2385 return success();
2386 }
2387 auto elementType = getTypeConverter()->convertType(t: pointerTo);
2388 if (!elementType)
2389 return rewriter.notifyMatchFailure(arg&: varOp, msg: "type conversion failed");
2390 Value allocated =
2391 LLVM::AllocaOp::create(builder&: rewriter, location: loc, resultType: dstType, elementType, arraySize: size);
2392 LLVM::StoreOp::create(builder&: rewriter, location: loc, value: adaptor.getInitializer(), addr: allocated);
2393 rewriter.replaceOp(op: varOp, newValues: allocated);
2394 return success();
2395 }
2396};
2397
2398//===----------------------------------------------------------------------===//
2399// BitcastOp conversion
2400//===----------------------------------------------------------------------===//
2401
2402class BitcastConversionPattern
2403 : public SPIRVToLLVMConversion<spirv::BitcastOp> {
2404public:
2405 using SPIRVToLLVMConversion<spirv::BitcastOp>::SPIRVToLLVMConversion;
2406
2407 LogicalResult
2408 matchAndRewrite(spirv::BitcastOp bitcastOp, OpAdaptor adaptor,
2409 ConversionPatternRewriter &rewriter) const override {
2410 auto dstType = getTypeConverter()->convertType(t: bitcastOp.getType());
2411 if (!dstType)
2412 return rewriter.notifyMatchFailure(arg&: bitcastOp, msg: "type conversion failed");
2413
2414 // LLVM's opaque pointers do not require bitcasts.
2415 if (isa<LLVM::LLVMPointerType>(Val: dstType)) {
2416 rewriter.replaceOp(op: bitcastOp, newValues: adaptor.getOperand());
2417 return success();
2418 }
2419
2420 LLVM::BitcastOp::Properties properties{};
2421 SmallVector<NamedAttribute> discardableAttrs;
2422 if (failed(Result: collectAttrsForConversion<LLVM::BitcastOp>(op: bitcastOp, properties,
2423 discardableAttrs)))
2424 return failure();
2425 rewriter.replaceOpWithNewOp<LLVM::BitcastOp>(op: bitcastOp, args&: dstType,
2426 args: adaptor.getOperands(),
2427 args&: properties, args&: discardableAttrs);
2428 return success();
2429 }
2430};
2431
2432//===----------------------------------------------------------------------===//
2433// FuncOp conversion
2434//===----------------------------------------------------------------------===//
2435
2436class FuncConversionPattern : public SPIRVToLLVMConversion<spirv::FuncOp> {
2437public:
2438 using SPIRVToLLVMConversion<spirv::FuncOp>::SPIRVToLLVMConversion;
2439
2440 LogicalResult
2441 matchAndRewrite(spirv::FuncOp funcOp, OpAdaptor adaptor,
2442 ConversionPatternRewriter &rewriter) const override {
2443
2444 // Convert function signature. At the moment LLVMType converter is enough
2445 // for currently supported types.
2446 auto funcType = funcOp.getFunctionType();
2447 TypeConverter::SignatureConversion signatureConverter(
2448 funcType.getNumInputs());
2449 auto llvmType = static_cast<const LLVMTypeConverter *>(getTypeConverter())
2450 ->convertFunctionSignature(
2451 funcTy: funcType, /*isVariadic=*/false,
2452 /*useBarePtrCallConv=*/false, result&: signatureConverter);
2453 if (!llvmType)
2454 return failure();
2455
2456 // Create a new `LLVMFuncOp`
2457 Location loc = funcOp.getLoc();
2458 StringRef name = funcOp.getName();
2459 auto newFuncOp = LLVM::LLVMFuncOp::create(builder&: rewriter, location: loc, name, type: llvmType);
2460
2461 // Convert SPIR-V Function Control to equivalent LLVM function attribute
2462 MLIRContext *context = funcOp.getContext();
2463 switch (funcOp.getFunctionControl()) {
2464 case spirv::FunctionControl::Inline:
2465 newFuncOp.setAlwaysInline(true);
2466 break;
2467 case spirv::FunctionControl::DontInline:
2468 newFuncOp.setNoInline(true);
2469 break;
2470
2471#define DISPATCH(functionControl, llvmAttr) \
2472 case functionControl: \
2473 newFuncOp->setDiscardableAttr("passthrough", \
2474 ArrayAttr::get(context, {llvmAttr})); \
2475 break;
2476
2477 DISPATCH(spirv::FunctionControl::Pure,
2478 StringAttr::get(context, "readonly"));
2479 DISPATCH(spirv::FunctionControl::Const,
2480 StringAttr::get(context, "readnone"));
2481
2482#undef DISPATCH
2483
2484 // Default: if `spirv::FunctionControl::None`, then no attributes are
2485 // needed.
2486 default:
2487 break;
2488 }
2489
2490 rewriter.inlineRegionBefore(region&: funcOp.getBody(), parent&: newFuncOp.getBody(),
2491 before: newFuncOp.end());
2492 if (failed(Result: rewriter.convertRegionTypes(
2493 region: &newFuncOp.getBody(), converter: *getTypeConverter(), entryConversion: &signatureConverter))) {
2494 return failure();
2495 }
2496 rewriter.eraseOp(op: funcOp);
2497 return success();
2498 }
2499};
2500
2501//===----------------------------------------------------------------------===//
2502// ModuleOp conversion
2503//===----------------------------------------------------------------------===//
2504
2505class ModuleConversionPattern : public SPIRVToLLVMConversion<spirv::ModuleOp> {
2506public:
2507 using SPIRVToLLVMConversion<spirv::ModuleOp>::SPIRVToLLVMConversion;
2508
2509 LogicalResult
2510 matchAndRewrite(spirv::ModuleOp spvModuleOp, OpAdaptor adaptor,
2511 ConversionPatternRewriter &rewriter) const override {
2512 // Create the module with an empty body region instead of using
2513 // ModuleOp::create, which adds a body block that would have to be erased.
2514 // If the conversion is rolled back, such a block is unlinked from the new
2515 // op but never deleted.
2516 OperationState state(spvModuleOp.getLoc(), ModuleOp::getOperationName());
2517 if (std::optional<StringRef> name = spvModuleOp.getName())
2518 state.addAttribute(name: SymbolTable::getSymbolAttrName(),
2519 attr: rewriter.getStringAttr(bytes: *name));
2520 state.addRegion();
2521 auto newModuleOp = cast<ModuleOp>(Val: rewriter.create(state));
2522 rewriter.inlineRegionBefore(region&: spvModuleOp.getRegion(),
2523 parent&: newModuleOp.getBodyRegion(),
2524 before: newModuleOp.getBodyRegion().end());
2525 rewriter.eraseOp(op: spvModuleOp);
2526 return success();
2527 }
2528};
2529
2530//===----------------------------------------------------------------------===//
2531// VectorShuffleOp conversion
2532//===----------------------------------------------------------------------===//
2533
2534class VectorShufflePattern
2535 : public SPIRVToLLVMConversion<spirv::VectorShuffleOp> {
2536public:
2537 using SPIRVToLLVMConversion<spirv::VectorShuffleOp>::SPIRVToLLVMConversion;
2538 LogicalResult
2539 matchAndRewrite(spirv::VectorShuffleOp op, OpAdaptor adaptor,
2540 ConversionPatternRewriter &rewriter) const override {
2541 Location loc = op.getLoc();
2542 auto components = adaptor.getComponents();
2543 auto vector1 = adaptor.getVector1();
2544 auto vector2 = adaptor.getVector2();
2545 int vector1Size = cast<VectorType>(Val: vector1.getType()).getNumElements();
2546 int vector2Size = cast<VectorType>(Val: vector2.getType()).getNumElements();
2547 if (vector1Size == vector2Size) {
2548 rewriter.replaceOpWithNewOp<LLVM::ShuffleVectorOp>(
2549 op, args&: vector1, args&: vector2,
2550 args: LLVM::convertArrayToIndices<int32_t>(attrs: components));
2551 return success();
2552 }
2553
2554 auto dstType = getTypeConverter()->convertType(t: op.getType());
2555 if (!dstType)
2556 return rewriter.notifyMatchFailure(arg&: op, msg: "type conversion failed");
2557 auto scalarType = cast<VectorType>(Val&: dstType).getElementType();
2558 auto componentsArray = components.getValue();
2559 auto *context = rewriter.getContext();
2560 auto llvmI32Type = IntegerType::get(context, width: 32);
2561 Value targetOp = LLVM::PoisonOp::create(builder&: rewriter, location: loc, res: dstType);
2562 for (unsigned i = 0; i < componentsArray.size(); i++) {
2563 if (!isa<IntegerAttr>(Val: componentsArray[i]))
2564 return op.emitError(message: "unable to support non-constant component");
2565
2566 int indexVal = cast<IntegerAttr>(Val: componentsArray[i]).getInt();
2567 if (indexVal == -1)
2568 continue;
2569
2570 int offsetVal = 0;
2571 Value baseVector = vector1;
2572 if (indexVal >= vector1Size) {
2573 offsetVal = vector1Size;
2574 baseVector = vector2;
2575 }
2576
2577 Value dstIndex = LLVM::ConstantOp::create(
2578 builder&: rewriter, location: loc, res: llvmI32Type,
2579 value: rewriter.getIntegerAttr(type: rewriter.getI32Type(), value: i));
2580 Value index = LLVM::ConstantOp::create(
2581 builder&: rewriter, location: loc, res: llvmI32Type,
2582 value: rewriter.getIntegerAttr(type: rewriter.getI32Type(), value: indexVal - offsetVal));
2583
2584 auto extractOp = LLVM::ExtractElementOp::create(builder&: rewriter, location: loc, res: scalarType,
2585 vector: baseVector, position: index);
2586 targetOp = LLVM::InsertElementOp::create(builder&: rewriter, location: loc, res: dstType, vector: targetOp,
2587 value: extractOp, position: dstIndex);
2588 }
2589 rewriter.replaceOp(op, newValues: targetOp);
2590 return success();
2591 }
2592};
2593} // namespace
2594
2595//===----------------------------------------------------------------------===//
2596// Pattern population
2597//===----------------------------------------------------------------------===//
2598
2599void mlir::populateSPIRVToLLVMTypeConversion(LLVMTypeConverter &typeConverter,
2600 spirv::ClientAPI clientAPI) {
2601 typeConverter.addConversion(callback: [&](spirv::ArrayType type) {
2602 return convertArrayType(type, converter&: typeConverter);
2603 });
2604 typeConverter.addConversion(callback: [&, clientAPI](spirv::PointerType type) {
2605 return convertPointerType(type, converter: typeConverter, clientAPI);
2606 });
2607 typeConverter.addConversion(callback: [&](spirv::RuntimeArrayType type) {
2608 return convertRuntimeArrayType(type, converter&: typeConverter);
2609 });
2610 typeConverter.addConversion(callback: [&](spirv::StructType type) {
2611 return convertStructType(type, converter: typeConverter);
2612 });
2613}
2614
2615void mlir::populateSPIRVToLLVMConversionPatterns(
2616 const LLVMTypeConverter &typeConverter, RewritePatternSet &patterns,
2617 spirv::ClientAPI clientAPI) {
2618 patterns.add<
2619 // Arithmetic ops
2620 DirectConversionPattern<spirv::IAddOp, LLVM::AddOp>,
2621 DirectConversionPattern<spirv::IMulOp, LLVM::MulOp>,
2622 DirectConversionPattern<spirv::ISubOp, LLVM::SubOp>,
2623 DirectConversionPattern<spirv::FAddOp, LLVM::FAddOp>,
2624 DirectConversionPattern<spirv::FDivOp, LLVM::FDivOp>,
2625 DirectConversionPattern<spirv::FMulOp, LLVM::FMulOp>,
2626 DirectConversionPattern<spirv::FNegateOp, LLVM::FNegOp>,
2627 DirectConversionPattern<spirv::FRemOp, LLVM::FRemOp>,
2628 DirectConversionPattern<spirv::FSubOp, LLVM::FSubOp>,
2629 DirectConversionPattern<spirv::SDivOp, LLVM::SDivOp>,
2630 DirectConversionPattern<spirv::SRemOp, LLVM::SRemOp>,
2631 DirectConversionPattern<spirv::UDivOp, LLVM::UDivOp>,
2632 DirectConversionPattern<spirv::UModOp, LLVM::URemOp>, FModPattern,
2633 SModPattern, VectorTimesScalarPattern, SNegatePattern,
2634 ArithmeticWithOverflowPattern<spirv::IAddCarryOp,
2635 LLVM::UAddWithOverflowOp>,
2636 ArithmeticWithOverflowPattern<spirv::ISubBorrowOp,
2637 LLVM::USubWithOverflowOp>,
2638 MulExtendedPattern<spirv::SMulExtendedOp, LLVM::SExtOp>,
2639 MulExtendedPattern<spirv::UMulExtendedOp, LLVM::ZExtOp>,
2640
2641 // Bitwise ops
2642 BitFieldInsertPattern, BitFieldUExtractPattern, BitFieldSExtractPattern,
2643 DirectConversionPattern<spirv::BitCountOp, LLVM::CtPopOp>,
2644 DirectConversionPattern<spirv::BitReverseOp, LLVM::BitReverseOp>,
2645 DirectConversionPattern<spirv::BitwiseAndOp, LLVM::AndOp>,
2646 DirectConversionPattern<spirv::BitwiseOrOp, LLVM::OrOp>,
2647 DirectConversionPattern<spirv::BitwiseXorOp, LLVM::XOrOp>,
2648 NotPattern<spirv::NotOp>,
2649
2650 // Cast ops
2651 BitcastConversionPattern,
2652 DirectConversionPattern<spirv::ConvertFToSOp, LLVM::FPToSIOp>,
2653 DirectConversionPattern<spirv::ConvertFToUOp, LLVM::FPToUIOp>,
2654 DirectConversionPattern<spirv::ConvertSToFOp, LLVM::SIToFPOp>,
2655 DirectConversionPattern<spirv::ConvertUToFOp, LLVM::UIToFPOp>,
2656 IndirectCastPattern<spirv::FConvertOp, LLVM::FPExtOp, LLVM::FPTruncOp>,
2657 IndirectCastPattern<spirv::SConvertOp, LLVM::SExtOp, LLVM::TruncOp>,
2658 IndirectCastPattern<spirv::UConvertOp, LLVM::ZExtOp, LLVM::TruncOp>,
2659 DirectConversionPattern<spirv::ConvertPtrToUOp, LLVM::PtrToIntOp>,
2660 DirectConversionPattern<spirv::ConvertUToPtrOp, LLVM::IntToPtrOp>,
2661 DirectConversionPattern<spirv::PtrCastToGenericOp, LLVM::AddrSpaceCastOp>,
2662 DirectConversionPattern<spirv::GenericCastToPtrOp, LLVM::AddrSpaceCastOp>,
2663 DirectConversionPattern<spirv::GenericCastToPtrExplicitOp,
2664 LLVM::AddrSpaceCastOp>,
2665
2666 // Comparison ops
2667 IComparePattern<spirv::IEqualOp, LLVM::ICmpPredicate::eq>,
2668 IComparePattern<spirv::INotEqualOp, LLVM::ICmpPredicate::ne>,
2669 FComparePattern<spirv::FOrdEqualOp, LLVM::FCmpPredicate::oeq>,
2670 FComparePattern<spirv::FOrdGreaterThanOp, LLVM::FCmpPredicate::ogt>,
2671 FComparePattern<spirv::FOrdGreaterThanEqualOp, LLVM::FCmpPredicate::oge>,
2672 FComparePattern<spirv::FOrdLessThanEqualOp, LLVM::FCmpPredicate::ole>,
2673 FComparePattern<spirv::FOrdLessThanOp, LLVM::FCmpPredicate::olt>,
2674 FComparePattern<spirv::FOrdNotEqualOp, LLVM::FCmpPredicate::one>,
2675 FComparePattern<spirv::FUnordEqualOp, LLVM::FCmpPredicate::ueq>,
2676 FComparePattern<spirv::FUnordGreaterThanOp, LLVM::FCmpPredicate::ugt>,
2677 FComparePattern<spirv::FUnordGreaterThanEqualOp,
2678 LLVM::FCmpPredicate::uge>,
2679 FComparePattern<spirv::FUnordLessThanEqualOp, LLVM::FCmpPredicate::ule>,
2680 FComparePattern<spirv::FUnordLessThanOp, LLVM::FCmpPredicate::ult>,
2681 FComparePattern<spirv::FUnordNotEqualOp, LLVM::FCmpPredicate::une>,
2682 FComparePattern<spirv::OrderedOp, LLVM::FCmpPredicate::ord>,
2683 FComparePattern<spirv::UnorderedOp, LLVM::FCmpPredicate::uno>,
2684 IComparePattern<spirv::SGreaterThanOp, LLVM::ICmpPredicate::sgt>,
2685 IComparePattern<spirv::SGreaterThanEqualOp, LLVM::ICmpPredicate::sge>,
2686 IComparePattern<spirv::SLessThanEqualOp, LLVM::ICmpPredicate::sle>,
2687 IComparePattern<spirv::SLessThanOp, LLVM::ICmpPredicate::slt>,
2688 IComparePattern<spirv::UGreaterThanOp, LLVM::ICmpPredicate::ugt>,
2689 IComparePattern<spirv::UGreaterThanEqualOp, LLVM::ICmpPredicate::uge>,
2690 IComparePattern<spirv::ULessThanEqualOp, LLVM::ICmpPredicate::ule>,
2691 IComparePattern<spirv::ULessThanOp, LLVM::ICmpPredicate::ult>,
2692
2693 // Constant op
2694 ConstantCompositePattern, ConstantScalarAndVectorPattern,
2695
2696 // Control Flow ops
2697 BranchConversionPattern, BranchConditionalConversionPattern,
2698 SwitchConversionPattern, FunctionCallPattern, LoopPattern,
2699 SelectionPattern, ErasePattern<spirv::MergeOp>,
2700
2701 // Entry points and execution mode are handled separately.
2702 ErasePattern<spirv::EntryPointOp>, ExecutionModePattern,
2703
2704 // GLSL extended instruction set ops
2705 DirectConversionPattern<spirv::GLCeilOp, LLVM::FCeilOp>,
2706 DirectConversionPattern<spirv::GLCosOp, LLVM::CosOp>,
2707 DirectConversionPattern<spirv::GLExpOp, LLVM::ExpOp>,
2708 DirectConversionPattern<spirv::GLExp2Op, LLVM::Exp2Op>,
2709 DirectConversionPattern<spirv::GLFAbsOp, LLVM::FAbsOp>,
2710 DirectConversionPattern<spirv::GLFloorOp, LLVM::FFloorOp>,
2711 DirectConversionPattern<spirv::GLFmaOp, LLVM::FMAOp>,
2712 ClampPattern<spirv::GLFClampOp, LLVM::MinNumOp, LLVM::MaxNumOp>,
2713 ClampPattern<spirv::GLSClampOp, LLVM::SMinOp, LLVM::SMaxOp>,
2714 ClampPattern<spirv::GLUClampOp, LLVM::UMinOp, LLVM::UMaxOp>,
2715 DirectConversionPattern<spirv::GLFMaxOp, LLVM::MaxNumOp>,
2716 DirectConversionPattern<spirv::GLFMinOp, LLVM::MinNumOp>,
2717 DirectConversionPattern<spirv::GLNMaxOp, LLVM::MaxNumOp>,
2718 DirectConversionPattern<spirv::GLNMinOp, LLVM::MinNumOp>,
2719 DirectConversionPattern<spirv::GLLdexpOp, LLVM::LoadExpOp>,
2720 DirectConversionPattern<spirv::GLLogOp, LLVM::LogOp>,
2721 DirectConversionPattern<spirv::GLLog2Op, LLVM::Log2Op>,
2722 DirectConversionPattern<spirv::GLPowOp, LLVM::PowOp>,
2723 DirectConversionPattern<spirv::GLRoundOp, LLVM::RoundOp>,
2724 DirectConversionPattern<spirv::GLRoundEvenOp, LLVM::RoundEvenOp>,
2725 DirectConversionPattern<spirv::GLSinOp, LLVM::SinOp>,
2726 DirectConversionPattern<spirv::GLSinhOp, LLVM::SinhOp>,
2727 DirectConversionPattern<spirv::GLCoshOp, LLVM::CoshOp>,
2728 DirectConversionPattern<spirv::GLSMaxOp, LLVM::SMaxOp>,
2729 DirectConversionPattern<spirv::GLSMinOp, LLVM::SMinOp>,
2730 DirectConversionPattern<spirv::GLSqrtOp, LLVM::SqrtOp>,
2731 DirectConversionPattern<spirv::GLUMaxOp, LLVM::UMaxOp>,
2732 DirectConversionPattern<spirv::GLUMinOp, LLVM::UMinOp>,
2733 DirectConversionPattern<spirv::GLTruncOp, LLVM::FTruncOp>,
2734 DirectConversionPattern<spirv::GLAsinOp, LLVM::ASinOp>,
2735 DirectConversionPattern<spirv::GLAcosOp, LLVM::ACosOp>,
2736 DirectConversionPattern<spirv::GLAtanOp, LLVM::ATanOp>,
2737 DirectConversionPattern<spirv::GLAtan2Op, LLVM::ATan2Op>,
2738 DirectConversionPattern<spirv::GLTanOp, LLVM::TanOp>,
2739 DirectConversionPattern<spirv::GLTanhOp, LLVM::TanhOp>,
2740 InverseSqrtPattern<spirv::GLInverseSqrtOp>, SAbsPattern<spirv::GLSAbsOp>,
2741 FractPattern, SignPattern<spirv::GLFSignOp, /*isFloat=*/true>,
2742 SignPattern<spirv::GLSSignOp, /*isFloat=*/false>, GLFMixPattern,
2743 GLAsinhAcoshPattern<spirv::GLAsinhOp>,
2744 GLAsinhAcoshPattern<spirv::GLAcoshOp>,
2745
2746 // OpenCL extended instruction set ops
2747 DirectConversionPattern<spirv::CLCeilOp, LLVM::FCeilOp>,
2748 DirectConversionPattern<spirv::CLCosOp, LLVM::CosOp>,
2749 DirectConversionPattern<spirv::CLExpOp, LLVM::ExpOp>,
2750 DirectConversionPattern<spirv::CLExp2Op, LLVM::Exp2Op>,
2751 DirectConversionPattern<spirv::CLExp10Op, LLVM::Exp10Op>,
2752 DirectConversionPattern<spirv::CLFAbsOp, LLVM::FAbsOp>,
2753 DirectConversionPattern<spirv::CLFloorOp, LLVM::FFloorOp>,
2754 DirectConversionPattern<spirv::CLFmaOp, LLVM::FMAOp>,
2755 DirectConversionPattern<spirv::CLFMaxOp, LLVM::MaxNumOp>,
2756 DirectConversionPattern<spirv::CLFMinOp, LLVM::MinNumOp>,
2757 DirectConversionPattern<spirv::CLLdexpOp, LLVM::LoadExpOp>,
2758 DirectConversionPattern<spirv::CLLogOp, LLVM::LogOp>,
2759 DirectConversionPattern<spirv::CLLog2Op, LLVM::Log2Op>,
2760 DirectConversionPattern<spirv::CLLog10Op, LLVM::Log10Op>,
2761 DirectConversionPattern<spirv::CLPowOp, LLVM::PowOp>,
2762 DirectConversionPattern<spirv::CLRintOp, LLVM::RintOp>,
2763 DirectConversionPattern<spirv::CLRoundOp, LLVM::RoundOp>,
2764 DirectConversionPattern<spirv::CLSinOp, LLVM::SinOp>,
2765 DirectConversionPattern<spirv::CLSinhOp, LLVM::SinhOp>,
2766 DirectConversionPattern<spirv::CLCoshOp, LLVM::CoshOp>,
2767 DirectConversionPattern<spirv::CLTanOp, LLVM::TanOp>,
2768 DirectConversionPattern<spirv::CLTanhOp, LLVM::TanhOp>,
2769 DirectConversionPattern<spirv::CLAsinOp, LLVM::ASinOp>,
2770 DirectConversionPattern<spirv::CLAcosOp, LLVM::ACosOp>,
2771 DirectConversionPattern<spirv::CLAtanOp, LLVM::ATanOp>,
2772 DirectConversionPattern<spirv::CLAtan2Op, LLVM::ATan2Op>,
2773 DirectConversionPattern<spirv::CLSqrtOp, LLVM::SqrtOp>,
2774 DirectConversionPattern<spirv::CLTruncOp, LLVM::FTruncOp>,
2775 DirectConversionPattern<spirv::CLCopysignOp, LLVM::CopySignOp>,
2776 DirectConversionPattern<spirv::CLFmodOp, LLVM::FRemOp>,
2777 DirectConversionPattern<spirv::CLSMaxOp, LLVM::SMaxOp>,
2778 DirectConversionPattern<spirv::CLSMinOp, LLVM::SMinOp>,
2779 DirectConversionPattern<spirv::CLUMaxOp, LLVM::UMaxOp>,
2780 DirectConversionPattern<spirv::CLUMinOp, LLVM::UMinOp>, CLMixPattern,
2781 CLSincosPattern, InverseSqrtPattern<spirv::CLRsqrtOp>,
2782 SAbsPattern<spirv::CLSAbsOp>,
2783 ClampPattern<spirv::CLFClampOp, LLVM::MinNumOp, LLVM::MaxNumOp>,
2784 ClampPattern<spirv::CLSClampOp, LLVM::SMinOp, LLVM::SMaxOp>,
2785 ClampPattern<spirv::CLUClampOp, LLVM::UMinOp, LLVM::UMaxOp>,
2786 CountZerosPattern<spirv::CLClzOp, LLVM::CountLeadingZerosOp>,
2787 CountZerosPattern<spirv::CLCtzOp, LLVM::CountTrailingZerosOp>,
2788 CLPrintfPattern,
2789
2790 // Logical ops
2791 DirectConversionPattern<spirv::LogicalAndOp, LLVM::AndOp>,
2792 DirectConversionPattern<spirv::LogicalOrOp, LLVM::OrOp>,
2793 IComparePattern<spirv::LogicalEqualOp, LLVM::ICmpPredicate::eq>,
2794 IComparePattern<spirv::LogicalNotEqualOp, LLVM::ICmpPredicate::ne>,
2795 NotPattern<spirv::LogicalNotOp>,
2796 FPClassPattern<spirv::IsInfOp, llvm::fcInf>,
2797 FPClassPattern<spirv::IsNanOp, llvm::fcNan>,
2798 FPClassPattern<spirv::IsFiniteOp, llvm::fcFinite>,
2799 FPClassPattern<spirv::IsNormalOp, llvm::fcNormal>,
2800
2801 // Memory ops
2802 AccessChainPattern<spirv::AccessChainOp, /*InBounds=*/false>,
2803 AccessChainPattern<spirv::InBoundsAccessChainOp, /*InBounds=*/true>,
2804 PtrAccessChainPattern<spirv::PtrAccessChainOp, /*InBounds=*/false>,
2805 PtrAccessChainPattern<spirv::InBoundsPtrAccessChainOp, /*InBounds=*/true>,
2806 AddressOfPattern, CopyMemoryPattern, LoadStorePattern<spirv::LoadOp>,
2807 LoadStorePattern<spirv::StoreOp>, VariablePattern,
2808
2809 // Miscellaneous ops
2810 CompositeExtractPattern, CompositeInsertPattern, CopyObjectPattern,
2811 DirectConversionPattern<spirv::SelectOp, LLVM::SelectOp>,
2812 DirectConversionPattern<spirv::UndefOp, LLVM::UndefOp>,
2813 VectorShufflePattern,
2814
2815 // Shift ops
2816 ShiftPattern<spirv::ShiftRightArithmeticOp, LLVM::AShrOp>,
2817 ShiftPattern<spirv::ShiftRightLogicalOp, LLVM::LShrOp>,
2818 ShiftPattern<spirv::ShiftLeftLogicalOp, LLVM::ShlOp>,
2819
2820 // Return ops
2821 ReturnPattern, ReturnValuePattern,
2822
2823 // Unreachable op
2824 UnreachablePattern,
2825
2826 // Barrier ops
2827 ControlBarrierPattern<spirv::ControlBarrierOp>,
2828 ControlBarrierPattern<spirv::INTELControlBarrierArriveOp>,
2829 ControlBarrierPattern<spirv::INTELControlBarrierWaitOp>,
2830
2831 // Group reduction operations
2832 GroupReducePattern<spirv::GroupIAddOp>,
2833 GroupReducePattern<spirv::GroupFAddOp>,
2834 GroupReducePattern<spirv::GroupFMinOp>,
2835 GroupReducePattern<spirv::GroupUMinOp>,
2836 GroupReducePattern<spirv::GroupSMinOp, /*Signed=*/true>,
2837 GroupReducePattern<spirv::GroupFMaxOp>,
2838 GroupReducePattern<spirv::GroupUMaxOp>,
2839 GroupReducePattern<spirv::GroupSMaxOp, /*Signed=*/true>,
2840 GroupReducePattern<spirv::GroupNonUniformIAddOp, /*Signed=*/false,
2841 /*NonUniform=*/true>,
2842 GroupReducePattern<spirv::GroupNonUniformFAddOp, /*Signed=*/false,
2843 /*NonUniform=*/true>,
2844 GroupReducePattern<spirv::GroupNonUniformIMulOp, /*Signed=*/false,
2845 /*NonUniform=*/true>,
2846 GroupReducePattern<spirv::GroupNonUniformFMulOp, /*Signed=*/false,
2847 /*NonUniform=*/true>,
2848 GroupReducePattern<spirv::GroupNonUniformSMinOp, /*Signed=*/true,
2849 /*NonUniform=*/true>,
2850 GroupReducePattern<spirv::GroupNonUniformUMinOp, /*Signed=*/false,
2851 /*NonUniform=*/true>,
2852 GroupReducePattern<spirv::GroupNonUniformFMinOp, /*Signed=*/false,
2853 /*NonUniform=*/true>,
2854 GroupReducePattern<spirv::GroupNonUniformSMaxOp, /*Signed=*/true,
2855 /*NonUniform=*/true>,
2856 GroupReducePattern<spirv::GroupNonUniformUMaxOp, /*Signed=*/false,
2857 /*NonUniform=*/true>,
2858 GroupReducePattern<spirv::GroupNonUniformFMaxOp, /*Signed=*/false,
2859 /*NonUniform=*/true>,
2860 GroupReducePattern<spirv::GroupNonUniformBitwiseAndOp, /*Signed=*/false,
2861 /*NonUniform=*/true>,
2862 GroupReducePattern<spirv::GroupNonUniformBitwiseOrOp, /*Signed=*/false,
2863 /*NonUniform=*/true>,
2864 GroupReducePattern<spirv::GroupNonUniformBitwiseXorOp, /*Signed=*/false,
2865 /*NonUniform=*/true>,
2866 GroupReducePattern<spirv::GroupNonUniformLogicalAndOp, /*Signed=*/false,
2867 /*NonUniform=*/true>,
2868 GroupReducePattern<spirv::GroupNonUniformLogicalOrOp, /*Signed=*/false,
2869 /*NonUniform=*/true>,
2870 GroupReducePattern<spirv::GroupNonUniformLogicalXorOp, /*Signed=*/false,
2871 /*NonUniform=*/true>>(arg: patterns.getContext(),
2872 args: typeConverter);
2873
2874 patterns.add<GlobalVariablePattern>(arg&: clientAPI, args: patterns.getContext(),
2875 args: typeConverter);
2876 // pi / 180
2877 patterns.add<ScalePattern<spirv::GLRadiansOp>>(
2878 arg: 0.017453292519943295, args: patterns.getContext(), args: typeConverter);
2879 // 180 / pi
2880 patterns.add<ScalePattern<spirv::GLDegreesOp>>(
2881 arg: 57.29577951308232, args: patterns.getContext(), args: typeConverter);
2882}
2883
2884void mlir::populateSPIRVToLLVMFunctionConversionPatterns(
2885 const LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) {
2886 patterns.add<FuncConversionPattern>(arg: patterns.getContext(), args: typeConverter);
2887}
2888
2889void mlir::populateSPIRVToLLVMModuleConversionPatterns(
2890 const LLVMTypeConverter &typeConverter, RewritePatternSet &patterns) {
2891 patterns.add<ModuleConversionPattern>(arg: patterns.getContext(), args: typeConverter);
2892}
2893
2894//===----------------------------------------------------------------------===//
2895// Pre-conversion hooks
2896//===----------------------------------------------------------------------===//
2897
2898/// Hook for descriptor set and binding number encoding.
2899void mlir::encodeBindAttribute(ModuleOp module) {
2900 auto spvModules = module.getOps<spirv::ModuleOp>();
2901 for (auto spvModule : spvModules) {
2902 spvModule.walk(callback: [&](spirv::GlobalVariableOp op) {
2903 IntegerAttr descriptorSet = op.getDescriptorSetAttr();
2904 IntegerAttr binding = op.getBindingAttr();
2905 // For every global variable in the module, get the ones with descriptor
2906 // set and binding numbers.
2907 if (descriptorSet && binding) {
2908 // Encode these numbers into the variable's symbolic name. If the
2909 // SPIR-V module has a name, add it at the beginning.
2910 auto moduleAndName =
2911 spvModule.getName().has_value()
2912 ? spvModule.getName()->str() + "_" + op.getSymName().str()
2913 : op.getSymName().str();
2914 std::string name =
2915 llvm::formatv(Fmt: "{0}_descriptor_set{1}_binding{2}", Vals&: moduleAndName,
2916 Vals: std::to_string(val: descriptorSet.getInt()),
2917 Vals: std::to_string(val: binding.getInt()));
2918 auto nameAttr = StringAttr::get(context: op->getContext(), bytes: name);
2919
2920 // Replace all symbol uses and set the new symbol name. Finally, remove
2921 // descriptor set and binding attributes.
2922 if (failed(Result: SymbolTable::replaceAllSymbolUses(oldSymbol: op, newSymbolName: nameAttr, from: spvModule)))
2923 op.emitError(message: "unable to replace all symbol uses for ") << name;
2924 SymbolTable::setSymbolName(symbol: op, name: nameAttr);
2925 op.removeDescriptorSetAttr();
2926 op.removeBindingAttr();
2927 }
2928 });
2929 }
2930}
2931