1//===- ArithToSPIRV.cpp - Arithmetic to SPIRV dialect conversion -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8
9#include "mlir/Conversion/ArithToSPIRV/ArithToSPIRV.h"
10
11#include "../SPIRVCommon/Pattern.h"
12#include "mlir/Dialect/Arith/IR/Arith.h"
13#include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h"
14#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
15#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
16#include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h"
17#include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h"
18#include "mlir/IR/BuiltinAttributes.h"
19#include "mlir/IR/BuiltinTypes.h"
20#include "mlir/IR/DialectResourceBlobManager.h"
21#include "llvm/ADT/APInt.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/Support/Debug.h"
25#include "llvm/Support/MathExtras.h"
26#include <cassert>
27#include <memory>
28
29namespace mlir {
30#define GEN_PASS_DEF_CONVERTARITHTOSPIRVPASS
31#include "mlir/Conversion/Passes.h.inc"
32} // namespace mlir
33
34#define DEBUG_TYPE "arith-to-spirv-pattern"
35
36using namespace mlir;
37
38//===----------------------------------------------------------------------===//
39// Conversion Helpers
40//===----------------------------------------------------------------------===//
41
42/// Converts the given `srcAttr` into a boolean attribute if it holds an
43/// integral value. Returns null attribute if conversion fails.
44static BoolAttr convertBoolAttr(Attribute srcAttr, Builder builder) {
45 if (auto boolAttr = dyn_cast<BoolAttr>(Val&: srcAttr))
46 return boolAttr;
47 if (auto intAttr = dyn_cast<IntegerAttr>(Val&: srcAttr))
48 return builder.getBoolAttr(value: intAttr.getValue().getBoolValue());
49 return {};
50}
51
52/// Converts the given `srcAttr` to a new attribute of the given `dstType`.
53/// Returns null attribute if conversion fails.
54static IntegerAttr convertIntegerAttr(IntegerAttr srcAttr, IntegerType dstType,
55 Builder builder) {
56 // If the source number uses less active bits than the target bitwidth, then
57 // it should be safe to convert.
58 if (srcAttr.getValue().isIntN(N: dstType.getWidth()))
59 return builder.getIntegerAttr(type: dstType, value: srcAttr.getInt());
60
61 // XXX: Try again by interpreting the source number as a signed value.
62 // Although integers in the standard dialect are signless, they can represent
63 // a signed number. It's the operation decides how to interpret. This is
64 // dangerous, but it seems there is no good way of handling this if we still
65 // want to change the bitwidth. Emit a message at least.
66 if (srcAttr.getValue().isSignedIntN(N: dstType.getWidth())) {
67 auto dstAttr = builder.getIntegerAttr(type: dstType, value: srcAttr.getInt());
68 LLVM_DEBUG(llvm::dbgs() << "attribute '" << srcAttr << "' converted to '"
69 << dstAttr << "' for type '" << dstType << "'\n");
70 return dstAttr;
71 }
72
73 LLVM_DEBUG(llvm::dbgs() << "attribute '" << srcAttr
74 << "' illegal: cannot fit into target type '"
75 << dstType << "'\n");
76 return {};
77}
78
79/// Converts the given `srcAttr` to a new attribute of the given `dstType`.
80/// Returns null attribute if `dstType` is not 32-bit or conversion fails.
81static FloatAttr convertFloatAttr(FloatAttr srcAttr, FloatType dstType,
82 Builder builder) {
83 // Only support converting to float for now.
84 if (!dstType.isF32())
85 return FloatAttr();
86
87 // Try to convert the source floating-point number to single precision.
88 APFloat dstVal = srcAttr.getValue();
89 bool losesInfo = false;
90 APFloat::opStatus status =
91 dstVal.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmTowardZero, losesInfo: &losesInfo);
92 if (status != APFloat::opOK || losesInfo) {
93 LLVM_DEBUG(llvm::dbgs()
94 << srcAttr << " illegal: cannot fit into converted type '"
95 << dstType << "'\n");
96 return FloatAttr();
97 }
98
99 return builder.getF32FloatAttr(value: dstVal.convertToFloat());
100}
101
102// Gets an IntegerAttr from a FloatAttr while preserving the bits. If `dstType`
103// is wider than the float (e.g. i8 emulated as i32), the bits are widened
104// like an integer constant of the float's width.
105static IntegerAttr getIntegerAttrFromFloatAttr(FloatAttr floatAttr,
106 IntegerType dstType,
107 Builder builder) {
108 APInt intVal = floatAttr.getValue().bitcastToAPInt();
109 auto srcType = IntegerType::get(context: dstType.getContext(), width: intVal.getBitWidth());
110 return convertIntegerAttr(srcAttr: builder.getIntegerAttr(type: srcType, value: intVal), dstType,
111 builder);
112}
113
114/// Returns true if the given `type` is a boolean scalar or vector type.
115static bool isBoolScalarOrVector(Type type) {
116 assert(type && "Not a valid type");
117 if (type.isInteger(width: 1))
118 return true;
119
120 if (auto vecType = dyn_cast<VectorType>(Val&: type))
121 return vecType.getElementType().isInteger(width: 1);
122
123 return false;
124}
125
126/// Creates a scalar/vector integer constant.
127static Value getScalarOrVectorConstInt(Type type, uint64_t value,
128 OpBuilder &builder, Location loc) {
129 if (auto vectorType = dyn_cast<VectorType>(Val&: type)) {
130 Attribute element = IntegerAttr::get(type: vectorType.getElementType(), value);
131 auto attr = SplatElementsAttr::get(type: vectorType, values: element);
132 return spirv::ConstantOp::create(builder, location: loc, constant: vectorType, value: attr);
133 }
134
135 if (auto intType = dyn_cast<IntegerType>(Val&: type))
136 return spirv::ConstantOp::create(builder, location: loc, constant: type,
137 value: builder.getIntegerAttr(type, value));
138
139 return nullptr;
140}
141
142/// Returns true if scalar/vector type `a` and `b` have the same number of
143/// bitwidth.
144static bool hasSameBitwidth(Type a, Type b) {
145 auto getNumBitwidth = [](Type type) {
146 unsigned bw = 0;
147 if (type.isIntOrFloat())
148 bw = type.getIntOrFloatBitWidth();
149 else if (auto vecType = dyn_cast<VectorType>(Val&: type))
150 bw = vecType.getElementTypeBitWidth() * vecType.getNumElements();
151 return bw;
152 };
153 unsigned aBW = getNumBitwidth(a);
154 unsigned bBW = getNumBitwidth(b);
155 return aBW != 0 && bBW != 0 && aBW == bBW;
156}
157
158/// Returns a source type conversion failure for `srcType` and operation `op`.
159static LogicalResult
160getTypeConversionFailure(ConversionPatternRewriter &rewriter, Operation *op,
161 Type srcType) {
162 return rewriter.notifyMatchFailure(
163 arg: op->getLoc(),
164 msg: llvm::formatv(Fmt: "failed to convert source type '{0}'", Vals&: srcType));
165}
166
167/// Returns a source type conversion failure for the result type of `op`.
168static LogicalResult
169getTypeConversionFailure(ConversionPatternRewriter &rewriter, Operation *op) {
170 assert(op->getNumResults() == 1);
171 return getTypeConversionFailure(rewriter, op, srcType: op->getResultTypes().front());
172}
173
174namespace {
175
176/// Converts elementwise unary, binary and ternary arith operations to SPIR-V
177/// operations. Op can potentially support overflow flags.
178template <typename Op, typename SPIRVOp>
179struct ElementwiseArithOpPattern final : OpConversionPattern<Op> {
180 using OpConversionPattern<Op>::OpConversionPattern;
181
182 LogicalResult
183 matchAndRewrite(Op op, typename Op::Adaptor adaptor,
184 ConversionPatternRewriter &rewriter) const override {
185 assert(adaptor.getOperands().size() <= 3);
186 // Reject boolean types to allow specialized boolean patterns to handle
187 // them (e.g., addi/subi on i1 should use LogicalNotEqual, not IAdd/ISub).
188 if (!adaptor.getOperands().empty() &&
189 isBoolScalarOrVector(adaptor.getOperands().front().getType()))
190 return failure();
191 auto converter = this->template getTypeConverter<SPIRVTypeConverter>();
192 Type dstType = converter->convertType(op.getType());
193 if (!dstType) {
194 return rewriter.notifyMatchFailure(
195 op->getLoc(),
196 llvm::formatv("failed to convert type {0} for SPIR-V", op.getType()));
197 }
198
199 if (SPIRVOp::template hasTrait<OpTrait::spirv::UnsignedOp>() &&
200 !getElementTypeOrSelf(op.getType()).isIndex() &&
201 dstType != op.getType()) {
202 return op.emitError("bitwidth emulation is not implemented yet on "
203 "unsigned op pattern version");
204 }
205
206 auto overflowFlags = arith::IntegerOverflowFlags::none;
207 if (auto overflowIface =
208 dyn_cast<arith::ArithIntegerOverflowFlagsInterface>(*op)) {
209 if (converter->getTargetEnv().allows(
210 spirv::Extension::SPV_KHR_no_integer_wrap_decoration))
211 overflowFlags = overflowIface.getOverflowAttr().getValue();
212 }
213
214 auto newOp = rewriter.template replaceOpWithNewOp<SPIRVOp>(
215 op, dstType, adaptor.getOperands());
216
217 if (bitEnumContainsAny(bits: overflowFlags, bit: arith::IntegerOverflowFlags::nsw))
218 newOp->setDiscardableAttr(
219 getDecorationString(decoration: spirv::Decoration::NoSignedWrap),
220 rewriter.getUnitAttr());
221
222 if (bitEnumContainsAny(bits: overflowFlags, bit: arith::IntegerOverflowFlags::nuw))
223 newOp->setDiscardableAttr(
224 getDecorationString(decoration: spirv::Decoration::NoUnsignedWrap),
225 rewriter.getUnitAttr());
226
227 return success();
228 }
229};
230
231//===----------------------------------------------------------------------===//
232// ConstantOp
233//===----------------------------------------------------------------------===//
234
235/// Converts composite arith.constant operation to spirv.Constant.
236struct ConstantCompositeOpPattern final
237 : public OpConversionPattern<arith::ConstantOp> {
238 using Base::Base;
239
240 LogicalResult
241 matchAndRewrite(arith::ConstantOp constOp, OpAdaptor adaptor,
242 ConversionPatternRewriter &rewriter) const override {
243 auto srcType = dyn_cast<ShapedType>(Val: constOp.getType());
244 if (!srcType || srcType.getNumElements() == 1)
245 return failure();
246
247 // arith.constant should only have vector or tensor types. This is a MLIR
248 // wide problem at the moment.
249 if (!isa<VectorType, RankedTensorType>(Val: srcType))
250 return rewriter.notifyMatchFailure(arg&: constOp, msg: "unsupported ShapedType");
251
252 Type dstType = getTypeConverter()->convertType(t: srcType);
253 if (!dstType)
254 return failure();
255
256 // Import the resource into the IR to make use of the special handling of
257 // element types later on.
258 mlir::DenseElementsAttr dstElementsAttr;
259 if (auto denseElementsAttr =
260 dyn_cast<DenseElementsAttr>(Val: constOp.getValue())) {
261 dstElementsAttr = denseElementsAttr;
262 } else if (auto resourceAttr =
263 dyn_cast<DenseResourceElementsAttr>(Val: constOp.getValue())) {
264
265 AsmResourceBlob *blob = resourceAttr.getRawHandle().getBlob();
266 if (!blob)
267 return constOp->emitError(message: "could not find resource blob");
268
269 ArrayRef<char> ptr = blob->getData();
270
271 // Check that the buffer meets the requirements to get converted to a
272 // DenseElementsAttr
273 if (!DenseElementsAttr::isValidRawBuffer(type: srcType, rawBuffer: ptr))
274 return constOp->emitError(message: "resource is not a valid buffer");
275
276 dstElementsAttr =
277 DenseElementsAttr::getFromRawBuffer(type: resourceAttr.getType(), rawBuffer: ptr);
278 } else {
279 return constOp->emitError(message: "unsupported elements attribute");
280 }
281
282 ShapedType dstAttrType = dstElementsAttr.getType();
283
284 // If the composite type has more than one dimensions, perform
285 // linearization.
286 if (srcType.getRank() > 1) {
287 if (isa<RankedTensorType>(Val: srcType)) {
288 dstAttrType = RankedTensorType::get(shape: srcType.getNumElements(),
289 elementType: srcType.getElementType());
290 dstElementsAttr = dstElementsAttr.reshape(newType: dstAttrType);
291 } else {
292 // TODO: add support for large vectors.
293 return failure();
294 }
295 }
296
297 Type srcElemType = srcType.getElementType();
298 Type dstElemType;
299 // Tensor types are converted to SPIR-V array types; vector types are
300 // converted to SPIR-V vector/array types.
301 if (auto arrayType = dyn_cast<spirv::ArrayType>(Val&: dstType))
302 dstElemType = arrayType.getElementType();
303 else
304 dstElemType = cast<VectorType>(Val&: dstType).getElementType();
305
306 // If the source and destination element types are different, perform
307 // attribute conversion.
308 if (srcElemType != dstElemType) {
309 SmallVector<Attribute, 8> elements;
310 if (isa<FloatType>(Val: srcElemType)) {
311 for (FloatAttr srcAttr : dstElementsAttr.getValues<FloatAttr>()) {
312 Attribute dstAttr = nullptr;
313 // Handle 8-bit float conversion to 8-bit integer.
314 auto *typeConverter = getTypeConverter<SPIRVTypeConverter>();
315 if (typeConverter->getOptions().emulateUnsupportedFloatTypes &&
316 srcElemType.getIntOrFloatBitWidth() == 8 &&
317 isa<IntegerType>(Val: dstElemType)) {
318 dstAttr = getIntegerAttrFromFloatAttr(
319 floatAttr: srcAttr, dstType: cast<IntegerType>(Val&: dstElemType), builder: rewriter);
320 } else {
321 dstAttr = convertFloatAttr(srcAttr, dstType: cast<FloatType>(Val&: dstElemType),
322 builder: rewriter);
323 }
324 if (!dstAttr)
325 return failure();
326 elements.push_back(Elt: dstAttr);
327 }
328 } else if (srcElemType.isInteger(width: 1)) {
329 return failure();
330 } else {
331 for (IntegerAttr srcAttr : dstElementsAttr.getValues<IntegerAttr>()) {
332 IntegerAttr dstAttr = convertIntegerAttr(
333 srcAttr, dstType: cast<IntegerType>(Val&: dstElemType), builder: rewriter);
334 if (!dstAttr)
335 return failure();
336 elements.push_back(Elt: dstAttr);
337 }
338 }
339
340 // Unfortunately, we cannot use dialect-specific types for element
341 // attributes; element attributes only works with builtin types. So we
342 // need to prepare another converted builtin types for the destination
343 // elements attribute.
344 if (isa<RankedTensorType>(Val: dstAttrType))
345 dstAttrType =
346 RankedTensorType::get(shape: dstAttrType.getShape(), elementType: dstElemType);
347 else
348 dstAttrType = VectorType::get(shape: dstAttrType.getShape(), elementType: dstElemType);
349
350 dstElementsAttr = DenseElementsAttr::get(type: dstAttrType, values: elements);
351 }
352
353 rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op: constOp, args&: dstType,
354 args&: dstElementsAttr);
355 return success();
356 }
357};
358
359/// Converts scalar arith.constant operation to spirv.Constant.
360struct ConstantScalarOpPattern final
361 : public OpConversionPattern<arith::ConstantOp> {
362 using Base::Base;
363
364 LogicalResult
365 matchAndRewrite(arith::ConstantOp constOp, OpAdaptor adaptor,
366 ConversionPatternRewriter &rewriter) const override {
367 Type srcType = constOp.getType();
368 if (auto shapedType = dyn_cast<ShapedType>(Val&: srcType)) {
369 if (shapedType.getNumElements() != 1)
370 return failure();
371 srcType = shapedType.getElementType();
372 }
373 if (!srcType.isIntOrIndexOrFloat())
374 return failure();
375
376 Attribute cstAttr = constOp.getValue();
377 if (auto elementsAttr = dyn_cast<DenseElementsAttr>(Val&: cstAttr))
378 cstAttr = elementsAttr.getSplatValue<Attribute>();
379
380 Type dstType = getTypeConverter()->convertType(t: srcType);
381 if (!dstType)
382 return failure();
383
384 // Floating-point types.
385 if (isa<FloatType>(Val: srcType)) {
386 auto srcAttr = cast<FloatAttr>(Val&: cstAttr);
387 Attribute dstAttr = srcAttr;
388
389 // Floating-point types not supported in the target environment are all
390 // converted to float type.
391 auto *typeConverter = getTypeConverter<SPIRVTypeConverter>();
392 if (typeConverter->getOptions().emulateUnsupportedFloatTypes &&
393 srcType.getIntOrFloatBitWidth() == 8 && isa<IntegerType>(Val: dstType)) {
394 // If the source is an 8-bit float, convert it to a 8-bit integer.
395 dstAttr = getIntegerAttrFromFloatAttr(
396 floatAttr: srcAttr, dstType: cast<IntegerType>(Val&: dstType), builder: rewriter);
397 if (!dstAttr)
398 return failure();
399 } else if (srcType != dstType) {
400 dstAttr = convertFloatAttr(srcAttr, dstType: cast<FloatType>(Val&: dstType), builder: rewriter);
401 if (!dstAttr)
402 return failure();
403 }
404
405 rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op: constOp, args&: dstType, args&: dstAttr);
406 return success();
407 }
408
409 // Bool type.
410 if (srcType.isInteger(width: 1)) {
411 // arith.constant can use 0/1 instead of true/false for i1 values. We need
412 // to handle that here.
413 auto dstAttr = convertBoolAttr(srcAttr: cstAttr, builder: rewriter);
414 if (!dstAttr)
415 return failure();
416 rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op: constOp, args&: dstType, args&: dstAttr);
417 return success();
418 }
419
420 // IndexType or IntegerType. Index values are converted to 32-bit integer
421 // values when converting to SPIR-V.
422 auto srcAttr = cast<IntegerAttr>(Val&: cstAttr);
423 IntegerAttr dstAttr =
424 convertIntegerAttr(srcAttr, dstType: cast<IntegerType>(Val&: dstType), builder: rewriter);
425 if (!dstAttr)
426 return failure();
427 rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op: constOp, args&: dstType, args&: dstAttr);
428 return success();
429 }
430};
431
432//===----------------------------------------------------------------------===//
433// RemSIOp
434//===----------------------------------------------------------------------===//
435
436/// Returns signed remainder for `lhs` and `rhs` and lets the result follow
437/// the sign of `signOperand`.
438///
439/// Note that this is needed for Vulkan. Per the Vulkan's SPIR-V environment
440/// spec, "for the OpSRem and OpSMod instructions, if either operand is negative
441/// the result is undefined." So we cannot directly use spirv.SRem/spirv.SMod
442/// if either operand can be negative. Emulate it via spirv.UMod.
443template <typename SignedAbsOp>
444static Value emulateSignedRemainder(Location loc, Value lhs, Value rhs,
445 Value signOperand, OpBuilder &builder) {
446 assert(lhs.getType() == rhs.getType());
447 assert(lhs == signOperand || rhs == signOperand);
448
449 Type type = lhs.getType();
450
451 // Calculate the remainder with spirv.UMod.
452 Value lhsAbs = SignedAbsOp::create(builder, loc, type, lhs);
453 Value rhsAbs = SignedAbsOp::create(builder, loc, type, rhs);
454 Value abs = spirv::UModOp::create(builder, location: loc, operand1: lhsAbs, operand2: rhsAbs);
455
456 // Fix the sign.
457 Value isPositive;
458 if (lhs == signOperand)
459 isPositive = spirv::IEqualOp::create(builder, location: loc, operand1: lhs, operand2: lhsAbs);
460 else
461 isPositive = spirv::IEqualOp::create(builder, location: loc, operand1: rhs, operand2: rhsAbs);
462 Value absNegate = spirv::SNegateOp::create(builder, location: loc, result: type, operand: abs);
463 return spirv::SelectOp::create(builder, location: loc, result: type, condition: isPositive, true_value: abs,
464 false_value: absNegate);
465}
466
467/// Converts arith.remsi to GLSL SPIR-V ops.
468///
469/// This cannot be merged into the template unary/binary pattern due to Vulkan
470/// restrictions over spirv.SRem and spirv.SMod.
471struct RemSIOpGLPattern final : public OpConversionPattern<arith::RemSIOp> {
472 using Base::Base;
473
474 LogicalResult
475 matchAndRewrite(arith::RemSIOp op, OpAdaptor adaptor,
476 ConversionPatternRewriter &rewriter) const override {
477 Value result = emulateSignedRemainder<spirv::GLSAbsOp>(
478 loc: op.getLoc(), lhs: adaptor.getOperands()[0], rhs: adaptor.getOperands()[1],
479 signOperand: adaptor.getOperands()[0], builder&: rewriter);
480 rewriter.replaceOp(op, newValues: result);
481
482 return success();
483 }
484};
485
486/// Converts arith.remsi to OpenCL SPIR-V ops.
487struct RemSIOpCLPattern final : public OpConversionPattern<arith::RemSIOp> {
488 using Base::Base;
489
490 LogicalResult
491 matchAndRewrite(arith::RemSIOp op, OpAdaptor adaptor,
492 ConversionPatternRewriter &rewriter) const override {
493 Value result = emulateSignedRemainder<spirv::CLSAbsOp>(
494 loc: op.getLoc(), lhs: adaptor.getOperands()[0], rhs: adaptor.getOperands()[1],
495 signOperand: adaptor.getOperands()[0], builder&: rewriter);
496 rewriter.replaceOp(op, newValues: result);
497
498 return success();
499 }
500};
501
502//===----------------------------------------------------------------------===//
503// CeilDivUIOp
504//===----------------------------------------------------------------------===//
505
506/// Converts arith.ceildivui to SPIR-V ops. Convert `ceildivui(n, m)` into
507/// `n == 0 ? 0 : (n-1)/m + 1`. Formula taken from the equivalent conversion
508/// in IndexToSPIRV.
509struct CeilDivUIOpPattern final : OpConversionPattern<arith::CeilDivUIOp> {
510 using Base::Base;
511
512 LogicalResult
513 matchAndRewrite(arith::CeilDivUIOp op, OpAdaptor adaptor,
514 ConversionPatternRewriter &rewriter) const override {
515 Type dstType = getTypeConverter()->convertType(t: op.getType());
516 if (!dstType)
517 return getTypeConversionFailure(rewriter, op);
518
519 if (!getElementTypeOrSelf(type: op.getType()).isIndex() &&
520 dstType != op.getType())
521 return op.emitError(message: "bitwidth emulation is not implemented yet on "
522 "unsigned op pattern version");
523
524 Location loc = op.getLoc();
525 Value n = adaptor.getLhs();
526 Value m = adaptor.getRhs();
527
528 Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter);
529 Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder&: rewriter);
530
531 Value minusOne = spirv::ISubOp::create(builder&: rewriter, location: loc, operand1: n, operand2: one);
532 Value quotient = spirv::UDivOp::create(builder&: rewriter, location: loc, operand1: minusOne, operand2: m);
533 Value plusOne = spirv::IAddOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: one);
534
535 Value cmp = spirv::IEqualOp::create(builder&: rewriter, location: loc, operand1: n, operand2: zero);
536 rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: cmp, args&: zero, args&: plusOne);
537 return success();
538 }
539};
540
541//===----------------------------------------------------------------------===//
542// CeilDivSIOp
543//===----------------------------------------------------------------------===//
544
545/// Converts arith.ceildivsi to SPIR-V ops. Convert `ceildivsi(a, b)` into
546/// `q = a/b; (q*b != a && (a<0) == (b<0)) ? q+1 : q`. Formula taken from the
547/// equivalent conversion in ExpandOps.
548struct CeilDivSIOpPattern final : OpConversionPattern<arith::CeilDivSIOp> {
549 using Base::Base;
550
551 LogicalResult
552 matchAndRewrite(arith::CeilDivSIOp op, OpAdaptor adaptor,
553 ConversionPatternRewriter &rewriter) const override {
554 Type dstType = getTypeConverter()->convertType(t: op.getType());
555 if (!dstType)
556 return getTypeConversionFailure(rewriter, op);
557
558 Location loc = op.getLoc();
559 Value a = adaptor.getLhs();
560 Value b = adaptor.getRhs();
561
562 Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter);
563 Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder&: rewriter);
564
565 Value quotient = spirv::SDivOp::create(builder&: rewriter, location: loc, operand1: a, operand2: b);
566 Value product = spirv::IMulOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: b);
567 Value notEqual = spirv::INotEqualOp::create(builder&: rewriter, location: loc, operand1: a, operand2: product);
568
569 Value aNeg = spirv::SLessThanOp::create(builder&: rewriter, location: loc, operand1: a, operand2: zero);
570 Value bNeg = spirv::SLessThanOp::create(builder&: rewriter, location: loc, operand1: b, operand2: zero);
571 Value sameSign = spirv::LogicalEqualOp::create(builder&: rewriter, location: loc, operand1: aNeg, operand2: bNeg);
572 Value cond = spirv::LogicalAndOp::create(builder&: rewriter, location: loc, operand1: notEqual, operand2: sameSign);
573
574 Value quotientPlusOne = spirv::IAddOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: one);
575 rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: cond, args&: quotientPlusOne,
576 args&: quotient);
577 return success();
578 }
579};
580
581//===----------------------------------------------------------------------===//
582// FloorDivSIOp
583//===----------------------------------------------------------------------===//
584
585/// Converts arith.floordivsi to SPIR-V ops. Convert `floordivsi(a, b)` into
586/// `q = a/b; (q*b != a && (a<0) != (b<0)) ? q-1 : q`. Formula taken from the
587/// equivalent conversion in ExpandOps.
588struct FloorDivSIOpPattern final : OpConversionPattern<arith::FloorDivSIOp> {
589 using Base::Base;
590
591 LogicalResult
592 matchAndRewrite(arith::FloorDivSIOp op, OpAdaptor adaptor,
593 ConversionPatternRewriter &rewriter) const override {
594 Type dstType = getTypeConverter()->convertType(t: op.getType());
595 if (!dstType)
596 return getTypeConversionFailure(rewriter, op);
597
598 Location loc = op.getLoc();
599 Value a = adaptor.getLhs();
600 Value b = adaptor.getRhs();
601
602 Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter);
603 Value minusOne = getScalarOrVectorConstInt(
604 type: dstType, value: static_cast<uint64_t>(-1), builder&: rewriter, loc);
605
606 Value quotient = spirv::SDivOp::create(builder&: rewriter, location: loc, operand1: a, operand2: b);
607 Value product = spirv::IMulOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: b);
608 Value notEqual = spirv::INotEqualOp::create(builder&: rewriter, location: loc, operand1: a, operand2: product);
609
610 Value aNeg = spirv::SLessThanOp::create(builder&: rewriter, location: loc, operand1: a, operand2: zero);
611 Value bNeg = spirv::SLessThanOp::create(builder&: rewriter, location: loc, operand1: b, operand2: zero);
612 Value diffSign =
613 spirv::LogicalNotEqualOp::create(builder&: rewriter, location: loc, operand1: aNeg, operand2: bNeg);
614 Value cond = spirv::LogicalAndOp::create(builder&: rewriter, location: loc, operand1: notEqual, operand2: diffSign);
615
616 Value quotientMinusOne =
617 spirv::IAddOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: minusOne);
618 rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: cond, args&: quotientMinusOne,
619 args&: quotient);
620 return success();
621 }
622};
623
624//===----------------------------------------------------------------------===//
625// BitwiseOp
626//===----------------------------------------------------------------------===//
627
628/// Converts bitwise operations to SPIR-V operations. This is a special pattern
629/// other than the BinaryOpPatternPattern because if the operands are boolean
630/// values, SPIR-V uses different operations (`SPIRVLogicalOp`). For
631/// non-boolean operands, SPIR-V should use `SPIRVBitwiseOp`.
632template <typename Op, typename SPIRVLogicalOp, typename SPIRVBitwiseOp>
633struct BitwiseOpPattern final : public OpConversionPattern<Op> {
634 using OpConversionPattern<Op>::OpConversionPattern;
635
636 LogicalResult
637 matchAndRewrite(Op op, typename Op::Adaptor adaptor,
638 ConversionPatternRewriter &rewriter) const override {
639 assert(adaptor.getOperands().size() == 2);
640 Type dstType = this->getTypeConverter()->convertType(op.getType());
641 if (!dstType)
642 return getTypeConversionFailure(rewriter, op);
643
644 if (isBoolScalarOrVector(adaptor.getOperands().front().getType())) {
645 rewriter.template replaceOpWithNewOp<SPIRVLogicalOp>(
646 op, dstType, adaptor.getOperands());
647 } else {
648 rewriter.template replaceOpWithNewOp<SPIRVBitwiseOp>(
649 op, dstType, adaptor.getOperands());
650 }
651 return success();
652 }
653};
654
655//===----------------------------------------------------------------------===//
656// XOrIOp
657//===----------------------------------------------------------------------===//
658
659/// Converts arith.xori to SPIR-V operations.
660struct XOrIOpLogicalPattern final : public OpConversionPattern<arith::XOrIOp> {
661 using Base::Base;
662
663 LogicalResult
664 matchAndRewrite(arith::XOrIOp op, OpAdaptor adaptor,
665 ConversionPatternRewriter &rewriter) const override {
666 assert(adaptor.getOperands().size() == 2);
667
668 if (isBoolScalarOrVector(type: adaptor.getOperands().front().getType()))
669 return failure();
670
671 Type dstType = getTypeConverter()->convertType(t: op.getType());
672 if (!dstType)
673 return getTypeConversionFailure(rewriter, op);
674
675 rewriter.replaceOpWithNewOp<spirv::BitwiseXorOp>(op, args&: dstType,
676 args: adaptor.getOperands());
677
678 return success();
679 }
680};
681
682/// Converts arith.xori to SPIR-V operations if the type of source is i1 or
683/// vector of i1.
684struct XOrIOpBooleanPattern final : public OpConversionPattern<arith::XOrIOp> {
685 using Base::Base;
686
687 LogicalResult
688 matchAndRewrite(arith::XOrIOp op, OpAdaptor adaptor,
689 ConversionPatternRewriter &rewriter) const override {
690 assert(adaptor.getOperands().size() == 2);
691
692 if (!isBoolScalarOrVector(type: adaptor.getOperands().front().getType()))
693 return failure();
694
695 Type dstType = getTypeConverter()->convertType(t: op.getType());
696 if (!dstType)
697 return getTypeConversionFailure(rewriter, op);
698
699 rewriter.replaceOpWithNewOp<spirv::LogicalNotEqualOp>(
700 op, args&: dstType, args: adaptor.getOperands());
701 return success();
702 }
703};
704
705/// Converts an arith integer op to the given SPIR-V boolean op if the type is
706/// i1 or vector of i1. Each mapping follows from the boolean truth table of
707/// the operation:
708/// addi(a, b) = a ^ b (add mod 2 = XOR = LogicalNotEqual)
709/// subi(a, b) = a ^ b (sub mod 2 = XOR = LogicalNotEqual)
710/// muli(a, b) = a & b (1*1=1, else 0 = LogicalAnd)
711/// divui(a, b) = a & b (a/1=a, a/0=UB; truth table matches AND)
712/// divsi(a, b) = a & b (same as divui on i1)
713/// ceildivui(a, b) = a & b (same as divui on i1)
714/// ceildivsi(a, b) = a & b (same as divsi on i1)
715/// floordivsi(a, b) = a & b (same as divsi on i1)
716/// maxsi(a, b) = a & b (signed i1: 1 represents -1, so max is 0 unless both
717/// are 1)
718/// maxui(a, b) = a | b (unsigned max on i1: 1 when either operand is 1)
719/// minsi(a, b) = a | b (signed i1: -1 < 0, so min is 1 when either operand
720/// is 1)
721/// minui(a, b) = a & b (unsigned min on i1: 1 only when both operands are
722/// 1)
723template <typename ArithOp, typename SPIRVOp>
724struct BoolIOpPattern final : public OpConversionPattern<ArithOp> {
725 BoolIOpPattern(const TypeConverter &converter, MLIRContext *context)
726 // benefit=2: takes priority over the generic ElementwiseArithOpPattern
727 // (benefit=1) when the operand type is i1.
728 : OpConversionPattern<ArithOp>(converter, context, /*benefit=*/2) {}
729
730 LogicalResult
731 matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor,
732 ConversionPatternRewriter &rewriter) const override {
733 if (!isBoolScalarOrVector(adaptor.getOperands().front().getType()))
734 return failure();
735
736 Type dstType = this->getTypeConverter()->convertType(op.getType());
737 if (!dstType)
738 return getTypeConversionFailure(rewriter, op);
739
740 rewriter.replaceOpWithNewOp<SPIRVOp>(op, dstType, adaptor.getOperands());
741 return success();
742 }
743};
744
745/// Converts an arith binary op on i1 to spirv.LogicalAnd(lhs,
746/// spirv.LogicalNot(rhs)). This covers shift-left, shift-right-unsigned, and
747/// unsigned remainder on i1:
748/// shli(a, b) = a & ~b (shift left clears the bit when b=1)
749/// shrui(a, b) = a & ~b (shift right unsigned clears the bit when b=1)
750/// remui(a, b) = a & ~b (only defined when b=1; a%1=0, and ~b=~1=0, so AND
751/// gives 0)
752/// remsi(a, b) = a & ~b (only defined when b=1; a%1=0, and ~b=~1=0, so AND
753/// gives 0)
754template <typename ArithOp>
755struct BoolIOpAndNotPattern final : public OpConversionPattern<ArithOp> {
756 BoolIOpAndNotPattern(const TypeConverter &converter, MLIRContext *context)
757 // benefit=2: takes priority over the generic ElementwiseArithOpPattern
758 // (benefit=1) when the operand type is i1.
759 : OpConversionPattern<ArithOp>(converter, context, /*benefit=*/2) {}
760
761 LogicalResult
762 matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor,
763 ConversionPatternRewriter &rewriter) const override {
764 if (!isBoolScalarOrVector(adaptor.getOperands().front().getType()))
765 return failure();
766
767 Type dstType = this->getTypeConverter()->convertType(op.getType());
768 if (!dstType)
769 return getTypeConversionFailure(rewriter, op);
770
771 Location loc = op.getLoc();
772 Value notRhs = spirv::LogicalNotOp::create(rewriter, loc, dstType,
773 adaptor.getOperands()[1]);
774 rewriter.replaceOpWithNewOp<spirv::LogicalAndOp>(
775 op, dstType, adaptor.getOperands()[0], notRhs);
776 return success();
777 }
778};
779
780/// Converts arith.shrsi on i1 to identity: arithmetic right shift of a 1-bit
781/// signed value always yields the original value (0 >> n = 0, -1 >> n = -1).
782struct ShRSIBoolPattern final : public OpConversionPattern<arith::ShRSIOp> {
783 ShRSIBoolPattern(const TypeConverter &converter, MLIRContext *context)
784 // benefit=2: takes priority over the generic spirv::ElementwiseOpPattern
785 // (benefit=1) when the operand type is i1.
786 : OpConversionPattern<arith::ShRSIOp>(converter, context,
787 /*benefit=*/2) {}
788
789 LogicalResult
790 matchAndRewrite(arith::ShRSIOp op, OpAdaptor adaptor,
791 ConversionPatternRewriter &rewriter) const override {
792 if (!isBoolScalarOrVector(type: adaptor.getOperands().front().getType()))
793 return failure();
794
795 rewriter.replaceOp(op, newValues: adaptor.getOperands().front());
796 return success();
797 }
798};
799
800//===----------------------------------------------------------------------===//
801// i1 source to value
802//===----------------------------------------------------------------------===//
803
804/// Converts an op whose i1 (or vector of i1) source selects between one and
805/// zero of the destination type, i.e. spirv.Select(src, one, zero). Shared by
806/// arith.uitofp, arith.extui, and arith.index_cast on boolean sources.
807template <typename ArithOp>
808struct BoolToValuePattern final : public OpConversionPattern<ArithOp> {
809 using OpConversionPattern<ArithOp>::OpConversionPattern;
810
811 LogicalResult
812 matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor,
813 ConversionPatternRewriter &rewriter) const override {
814 Type srcType = adaptor.getOperands().front().getType();
815 if (!isBoolScalarOrVector(type: srcType))
816 return failure();
817
818 Type dstType = this->getTypeConverter()->convertType(op.getType());
819 if (!dstType)
820 return getTypeConversionFailure(rewriter, op);
821
822 Location loc = op.getLoc();
823 Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter);
824 Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder&: rewriter);
825 rewriter.replaceOpWithNewOp<spirv::SelectOp>(
826 op, dstType, adaptor.getOperands().front(), one, zero);
827 return success();
828 }
829};
830
831//===----------------------------------------------------------------------===//
832// UIToFPOp
833//===----------------------------------------------------------------------===//
834
835/// Converts arith.uitofp/arith.sitofp to spirv.ConvertUToF/spirv.ConvertSToF.
836/// When the source integer type was widened during type conversion (e.g., i8
837/// emulated as i32), the upper bits of the widened value may contain garbage.
838/// This pattern cleans the upper bits before the conversion:
839/// - For unsigned (IsSigned=false): mask with BitwiseAnd.
840/// - For signed (IsSigned=true): sign-extend via ShiftLeftLogical +
841/// ShiftRightArithmetic.
842template <typename ArithOp, typename SPIRVOp, bool IsSigned>
843struct IntToFPPattern final : public OpConversionPattern<ArithOp> {
844 using OpConversionPattern<ArithOp>::OpConversionPattern;
845
846 LogicalResult
847 matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor,
848 ConversionPatternRewriter &rewriter) const override {
849 Type srcType = adaptor.getOperands().front().getType();
850 if (isBoolScalarOrVector(type: srcType))
851 return failure();
852
853 Type dstType = this->getTypeConverter()->convertType(op.getType());
854 if (!dstType)
855 return getTypeConversionFailure(rewriter, op);
856
857 // Check if the source integer type was widened during type conversion.
858 unsigned originalBitwidth =
859 getElementTypeOrSelf(op.getIn().getType()).getIntOrFloatBitWidth();
860
861 Type srcElemType = getElementTypeOrSelf(type: srcType);
862 if (!srcElemType.isIntOrFloat()) {
863 return rewriter.notifyMatchFailure(op,
864 "unsupported type for uitofp/sitofp");
865 }
866
867 unsigned convertedBitwidth = srcElemType.getIntOrFloatBitWidth();
868
869 if (originalBitwidth >= convertedBitwidth) {
870 rewriter.replaceOpWithNewOp<SPIRVOp>(op, dstType, adaptor.getOperands());
871 return success();
872 }
873
874 // The source was widened. Clean the upper bits before converting.
875 Location loc = op.getLoc();
876 Value cleaned;
877 if constexpr (IsSigned) {
878 // Sign-extend by shifting left then arithmetic right.
879 unsigned shiftAmount = convertedBitwidth - originalBitwidth;
880 Value shiftSize =
881 getScalarOrVectorConstInt(type: srcType, value: shiftAmount, builder&: rewriter, loc);
882 Value shifted = spirv::ShiftLeftLogicalOp::create(
883 rewriter, loc, srcType, adaptor.getIn(), shiftSize);
884 cleaned = spirv::ShiftRightArithmeticOp::create(builder&: rewriter, location: loc, result: srcType,
885 operand1: shifted, operand2: shiftSize);
886 } else {
887 // Zero-extend by masking off the upper bits.
888 Value mask = getScalarOrVectorConstInt(
889 type: srcType, value: llvm::maskTrailingOnes<uint64_t>(N: originalBitwidth), builder&: rewriter,
890 loc);
891 cleaned = spirv::BitwiseAndOp::create(rewriter, loc, srcType,
892 adaptor.getIn(), mask);
893 }
894 rewriter.replaceOpWithNewOp<SPIRVOp>(op, dstType, cleaned);
895 return success();
896 }
897};
898
899//===----------------------------------------------------------------------===//
900// IndexCastOp
901//===----------------------------------------------------------------------===//
902
903/// Converts arith.index_cast to spirv.INotEqual if the target type is i1.
904struct IndexCastIndexI1Pattern final
905 : public OpConversionPattern<arith::IndexCastOp> {
906 using Base::Base;
907
908 LogicalResult
909 matchAndRewrite(arith::IndexCastOp op, OpAdaptor adaptor,
910 ConversionPatternRewriter &rewriter) const override {
911 if (!isBoolScalarOrVector(type: op.getType()))
912 return failure();
913
914 Type dstType = getTypeConverter()->convertType(t: op.getType());
915 if (!dstType)
916 return getTypeConversionFailure(rewriter, op);
917
918 Location loc = op.getLoc();
919 Value zeroIdx =
920 spirv::ConstantOp::getZero(type: adaptor.getIn().getType(), loc, builder&: rewriter);
921 rewriter.replaceOpWithNewOp<spirv::INotEqualOp>(op, args&: dstType, args&: zeroIdx,
922 args: adaptor.getIn());
923 return success();
924 }
925};
926
927//===----------------------------------------------------------------------===//
928// ExtSIOp
929//===----------------------------------------------------------------------===//
930
931/// Converts arith.extsi to spirv.Select if the type of source is i1 or vector
932/// of i1.
933struct ExtSII1Pattern final : public OpConversionPattern<arith::ExtSIOp> {
934 using Base::Base;
935
936 LogicalResult
937 matchAndRewrite(arith::ExtSIOp op, OpAdaptor adaptor,
938 ConversionPatternRewriter &rewriter) const override {
939 Value operand = adaptor.getIn();
940 if (!isBoolScalarOrVector(type: operand.getType()))
941 return failure();
942
943 Location loc = op.getLoc();
944 Type dstType = getTypeConverter()->convertType(t: op.getType());
945 if (!dstType)
946 return getTypeConversionFailure(rewriter, op);
947
948 Value allOnes;
949 if (auto intTy = dyn_cast<IntegerType>(Val&: dstType)) {
950 unsigned componentBitwidth = intTy.getWidth();
951 allOnes = spirv::ConstantOp::create(
952 builder&: rewriter, location: loc, constant: intTy,
953 value: rewriter.getIntegerAttr(type: intTy, value: APInt::getAllOnes(numBits: componentBitwidth)));
954 } else if (auto vectorTy = dyn_cast<VectorType>(Val&: dstType)) {
955 unsigned componentBitwidth = vectorTy.getElementTypeBitWidth();
956 allOnes = spirv::ConstantOp::create(
957 builder&: rewriter, location: loc, constant: vectorTy,
958 value: SplatElementsAttr::get(type: vectorTy,
959 values: APInt::getAllOnes(numBits: componentBitwidth)));
960 } else {
961 return rewriter.notifyMatchFailure(
962 arg&: loc, msg: llvm::formatv(Fmt: "unhandled type: {0}", Vals&: dstType));
963 }
964
965 Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter);
966 rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: dstType, args&: operand, args&: allOnes,
967 args&: zero);
968 return success();
969 }
970};
971
972/// Converts arith.extsi to spirv.Select if the type of source is neither i1 nor
973/// vector of i1.
974struct ExtSIPattern final : public OpConversionPattern<arith::ExtSIOp> {
975 using Base::Base;
976
977 LogicalResult
978 matchAndRewrite(arith::ExtSIOp op, OpAdaptor adaptor,
979 ConversionPatternRewriter &rewriter) const override {
980 Type srcType = adaptor.getIn().getType();
981 if (isBoolScalarOrVector(type: srcType))
982 return failure();
983
984 Type dstType = getTypeConverter()->convertType(t: op.getType());
985 if (!dstType)
986 return getTypeConversionFailure(rewriter, op);
987
988 if (dstType == srcType) {
989 // We can have the same source and destination type due to type emulation.
990 // Perform bit shifting to make sure we have the proper leading set bits.
991
992 unsigned srcBW =
993 getElementTypeOrSelf(type: op.getIn().getType()).getIntOrFloatBitWidth();
994 unsigned dstBW =
995 getElementTypeOrSelf(type: op.getType()).getIntOrFloatBitWidth();
996 assert(srcBW < dstBW);
997 Value shiftSize = getScalarOrVectorConstInt(type: dstType, value: dstBW - srcBW,
998 builder&: rewriter, loc: op.getLoc());
999 if (!shiftSize)
1000 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported type for shift");
1001
1002 // First shift left to sequeeze out all leading bits beyond the original
1003 // bitwidth. Here we need to use the original source and result type's
1004 // bitwidth.
1005 auto shiftLOp = spirv::ShiftLeftLogicalOp::create(
1006 builder&: rewriter, location: op.getLoc(), result: dstType, operand1: adaptor.getIn(), operand2: shiftSize);
1007
1008 // Then we perform arithmetic right shift to make sure we have the right
1009 // sign bits for negative values.
1010 rewriter.replaceOpWithNewOp<spirv::ShiftRightArithmeticOp>(
1011 op, args&: dstType, args&: shiftLOp, args&: shiftSize);
1012 } else {
1013 rewriter.replaceOpWithNewOp<spirv::SConvertOp>(op, args&: dstType,
1014 args: adaptor.getOperands());
1015 }
1016
1017 return success();
1018 }
1019};
1020
1021//===----------------------------------------------------------------------===//
1022// ExtUIOp
1023//===----------------------------------------------------------------------===//
1024
1025/// Converts arith.extui for cases where the type of source is neither i1 nor
1026/// vector of i1.
1027struct ExtUIPattern final : public OpConversionPattern<arith::ExtUIOp> {
1028 using Base::Base;
1029
1030 LogicalResult
1031 matchAndRewrite(arith::ExtUIOp op, OpAdaptor adaptor,
1032 ConversionPatternRewriter &rewriter) const override {
1033 Type srcType = adaptor.getIn().getType();
1034 if (isBoolScalarOrVector(type: srcType))
1035 return failure();
1036
1037 Type dstType = getTypeConverter()->convertType(t: op.getType());
1038 if (!dstType)
1039 return getTypeConversionFailure(rewriter, op);
1040
1041 if (dstType == srcType) {
1042 // We can have the same source and destination type due to type emulation.
1043 // Perform bit masking to make sure we don't pollute downstream consumers
1044 // with unwanted bits. Here we need to use the original source type's
1045 // bitwidth.
1046 unsigned bitwidth =
1047 getElementTypeOrSelf(type: op.getIn().getType()).getIntOrFloatBitWidth();
1048 Value mask = getScalarOrVectorConstInt(
1049 type: dstType, value: llvm::maskTrailingOnes<uint64_t>(N: bitwidth), builder&: rewriter,
1050 loc: op.getLoc());
1051 if (!mask)
1052 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported type for mask");
1053 rewriter.replaceOpWithNewOp<spirv::BitwiseAndOp>(op, args&: dstType,
1054 args: adaptor.getIn(), args&: mask);
1055 } else {
1056 rewriter.replaceOpWithNewOp<spirv::UConvertOp>(op, args&: dstType,
1057 args: adaptor.getOperands());
1058 }
1059 return success();
1060 }
1061};
1062
1063//===----------------------------------------------------------------------===//
1064// TruncIOp
1065//===----------------------------------------------------------------------===//
1066
1067/// Converts arith.trunci to spirv.Select if the type of result is i1 or vector
1068/// of i1.
1069struct TruncII1Pattern final : public OpConversionPattern<arith::TruncIOp> {
1070 using Base::Base;
1071
1072 LogicalResult
1073 matchAndRewrite(arith::TruncIOp op, OpAdaptor adaptor,
1074 ConversionPatternRewriter &rewriter) const override {
1075 Type dstType = getTypeConverter()->convertType(t: op.getType());
1076 if (!dstType)
1077 return getTypeConversionFailure(rewriter, op);
1078
1079 if (!isBoolScalarOrVector(type: dstType))
1080 return failure();
1081
1082 Location loc = op.getLoc();
1083 auto srcType = adaptor.getOperands().front().getType();
1084 // Check if (x & 1) == 1.
1085 Value mask = spirv::ConstantOp::getOne(type: srcType, loc, builder&: rewriter);
1086 Value maskedSrc = spirv::BitwiseAndOp::create(
1087 builder&: rewriter, location: loc, result: srcType, operand1: adaptor.getOperands()[0], operand2: mask);
1088 Value isOne = spirv::IEqualOp::create(builder&: rewriter, location: loc, operand1: maskedSrc, operand2: mask);
1089
1090 Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter);
1091 Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder&: rewriter);
1092 rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: dstType, args&: isOne, args&: one, args&: zero);
1093 return success();
1094 }
1095};
1096
1097/// Converts arith.trunci for cases where the type of result is neither i1
1098/// nor vector of i1.
1099struct TruncIPattern final : public OpConversionPattern<arith::TruncIOp> {
1100 using Base::Base;
1101
1102 LogicalResult
1103 matchAndRewrite(arith::TruncIOp op, OpAdaptor adaptor,
1104 ConversionPatternRewriter &rewriter) const override {
1105 Type srcType = adaptor.getIn().getType();
1106 Type dstType = getTypeConverter()->convertType(t: op.getType());
1107 if (!dstType)
1108 return getTypeConversionFailure(rewriter, op);
1109
1110 if (isBoolScalarOrVector(type: dstType))
1111 return failure();
1112
1113 if (dstType == srcType) {
1114 // We can have the same source and destination type due to type emulation.
1115 // Perform bit masking to make sure we don't pollute downstream consumers
1116 // with unwanted bits. Here we need to use the original result type's
1117 // bitwidth.
1118 unsigned bw = getElementTypeOrSelf(type: op.getType()).getIntOrFloatBitWidth();
1119 Value mask = getScalarOrVectorConstInt(
1120 type: dstType, value: llvm::maskTrailingOnes<uint64_t>(N: bw), builder&: rewriter, loc: op.getLoc());
1121 if (!mask)
1122 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported type for mask");
1123 rewriter.replaceOpWithNewOp<spirv::BitwiseAndOp>(op, args&: dstType,
1124 args: adaptor.getIn(), args&: mask);
1125 } else {
1126 // Given this is truncation, either SConvertOp or UConvertOp works.
1127 rewriter.replaceOpWithNewOp<spirv::SConvertOp>(op, args&: dstType,
1128 args: adaptor.getOperands());
1129 }
1130 return success();
1131 }
1132};
1133
1134//===----------------------------------------------------------------------===//
1135// TypeCastingOp
1136//===----------------------------------------------------------------------===//
1137
1138static std::optional<spirv::FPRoundingMode>
1139convertArithRoundingModeToSPIRV(arith::RoundingMode roundingMode) {
1140 switch (roundingMode) {
1141 case arith::RoundingMode::downward:
1142 return spirv::FPRoundingMode::RTN;
1143 case arith::RoundingMode::to_nearest_even:
1144 return spirv::FPRoundingMode::RTE;
1145 case arith::RoundingMode::toward_zero:
1146 return spirv::FPRoundingMode::RTZ;
1147 case arith::RoundingMode::upward:
1148 return spirv::FPRoundingMode::RTP;
1149 case arith::RoundingMode::to_nearest_away:
1150 // SPIR-V FPRoundingMode decoration has no ties-away-from-zero mode
1151 // (as of SPIR-V 1.6)
1152 return std::nullopt;
1153 }
1154 llvm_unreachable("Unhandled rounding mode");
1155}
1156
1157/// Converts type-casting standard operations to SPIR-V operations.
1158template <typename Op, typename SPIRVOp>
1159struct TypeCastingOpPattern final : public OpConversionPattern<Op> {
1160 using OpConversionPattern<Op>::OpConversionPattern;
1161
1162 LogicalResult
1163 matchAndRewrite(Op op, typename Op::Adaptor adaptor,
1164 ConversionPatternRewriter &rewriter) const override {
1165 Type srcType = llvm::getSingleElement(adaptor.getOperands()).getType();
1166 Type dstType = this->getTypeConverter()->convertType(op.getType());
1167 if (!dstType)
1168 return getTypeConversionFailure(rewriter, op);
1169
1170 if (isBoolScalarOrVector(type: srcType) || isBoolScalarOrVector(type: dstType))
1171 return failure();
1172
1173 if (dstType == srcType) {
1174 // Due to type conversion, we are seeing the same source and target type.
1175 // Then we can just erase this operation by forwarding its operand.
1176 rewriter.replaceOp(op, adaptor.getOperands().front());
1177 } else {
1178 // Compute new rounding mode (if any).
1179 std::optional<spirv::FPRoundingMode> rm = std::nullopt;
1180 if (auto roundingModeOp =
1181 dyn_cast<arith::ArithRoundingModeInterface>(*op)) {
1182 if (arith::RoundingModeAttr roundingMode =
1183 roundingModeOp.getRoundingModeAttr()) {
1184 if (!(rm =
1185 convertArithRoundingModeToSPIRV(roundingMode: roundingMode.getValue()))) {
1186 return rewriter.notifyMatchFailure(
1187 op->getLoc(),
1188 llvm::formatv(Fmt: "unsupported rounding mode '{0}'", Vals&: roundingMode));
1189 }
1190 }
1191 }
1192 // Create replacement op and attach rounding mode attribute (if any).
1193 auto newOp = rewriter.template replaceOpWithNewOp<SPIRVOp>(
1194 op, dstType, adaptor.getOperands());
1195 if (rm) {
1196 newOp->setDiscardableAttr(
1197 getDecorationString(decoration: spirv::Decoration::FPRoundingMode),
1198 spirv::FPRoundingModeAttr::get(context: rewriter.getContext(), value: *rm));
1199 }
1200 }
1201 return success();
1202 }
1203};
1204
1205//===----------------------------------------------------------------------===//
1206// CmpIOp
1207//===----------------------------------------------------------------------===//
1208
1209/// Converts integer compare operation on i1 type operands to SPIR-V ops.
1210class CmpIOpBooleanPattern final : public OpConversionPattern<arith::CmpIOp> {
1211public:
1212 using Base::Base;
1213
1214 LogicalResult
1215 matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
1216 ConversionPatternRewriter &rewriter) const override {
1217 Type srcType = op.getLhs().getType();
1218 if (!isBoolScalarOrVector(type: srcType))
1219 return failure();
1220 Type dstType = getTypeConverter()->convertType(t: srcType);
1221 if (!dstType)
1222 return getTypeConversionFailure(rewriter, op, srcType);
1223
1224 switch (op.getPredicate()) {
1225 case arith::CmpIPredicate::eq: {
1226 rewriter.replaceOpWithNewOp<spirv::LogicalEqualOp>(op, args: adaptor.getLhs(),
1227 args: adaptor.getRhs());
1228 return success();
1229 }
1230 case arith::CmpIPredicate::ne: {
1231 rewriter.replaceOpWithNewOp<spirv::LogicalNotEqualOp>(
1232 op, args: adaptor.getLhs(), args: adaptor.getRhs());
1233 return success();
1234 }
1235 case arith::CmpIPredicate::uge:
1236 case arith::CmpIPredicate::ugt:
1237 case arith::CmpIPredicate::ule:
1238 case arith::CmpIPredicate::ult: {
1239 // There are no direct corresponding instructions in SPIR-V for such
1240 // cases. Extend them to 32-bit and do comparision then.
1241 Type type = rewriter.getI32Type();
1242 if (auto vectorType = dyn_cast<VectorType>(Val&: dstType))
1243 type = VectorType::get(shape: vectorType.getShape(), elementType: type);
1244 Value extLhs =
1245 arith::ExtUIOp::create(builder&: rewriter, location: op.getLoc(), out: type, in: adaptor.getLhs());
1246 Value extRhs =
1247 arith::ExtUIOp::create(builder&: rewriter, location: op.getLoc(), out: type, in: adaptor.getRhs());
1248
1249 rewriter.replaceOpWithNewOp<arith::CmpIOp>(op, args: op.getPredicate(), args&: extLhs,
1250 args&: extRhs);
1251 return success();
1252 }
1253 default:
1254 break;
1255 }
1256 return failure();
1257 }
1258};
1259
1260/// Converts integer compare operation to SPIR-V ops.
1261class CmpIOpPattern final : public OpConversionPattern<arith::CmpIOp> {
1262public:
1263 using Base::Base;
1264
1265 LogicalResult
1266 matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor,
1267 ConversionPatternRewriter &rewriter) const override {
1268 Type srcType = op.getLhs().getType();
1269 if (isBoolScalarOrVector(type: srcType))
1270 return failure();
1271 Type dstType = getTypeConverter()->convertType(t: srcType);
1272 if (!dstType)
1273 return getTypeConversionFailure(rewriter, op, srcType);
1274
1275 switch (op.getPredicate()) {
1276#define DISPATCH(cmpPredicate, spirvOp) \
1277 case cmpPredicate: \
1278 if (spirvOp::template hasTrait<OpTrait::spirv::UnsignedOp>() && \
1279 !getElementTypeOrSelf(srcType).isIndex() && srcType != dstType && \
1280 !hasSameBitwidth(srcType, dstType)) { \
1281 return op.emitError( \
1282 "bitwidth emulation is not implemented yet on unsigned op"); \
1283 } \
1284 rewriter.replaceOpWithNewOp<spirvOp>(op, adaptor.getLhs(), \
1285 adaptor.getRhs()); \
1286 return success();
1287
1288 DISPATCH(arith::CmpIPredicate::eq, spirv::IEqualOp);
1289 DISPATCH(arith::CmpIPredicate::ne, spirv::INotEqualOp);
1290 DISPATCH(arith::CmpIPredicate::slt, spirv::SLessThanOp);
1291 DISPATCH(arith::CmpIPredicate::sle, spirv::SLessThanEqualOp);
1292 DISPATCH(arith::CmpIPredicate::sgt, spirv::SGreaterThanOp);
1293 DISPATCH(arith::CmpIPredicate::sge, spirv::SGreaterThanEqualOp);
1294 DISPATCH(arith::CmpIPredicate::ult, spirv::ULessThanOp);
1295 DISPATCH(arith::CmpIPredicate::ule, spirv::ULessThanEqualOp);
1296 DISPATCH(arith::CmpIPredicate::ugt, spirv::UGreaterThanOp);
1297 DISPATCH(arith::CmpIPredicate::uge, spirv::UGreaterThanEqualOp);
1298
1299#undef DISPATCH
1300 }
1301 return failure();
1302 }
1303};
1304
1305//===----------------------------------------------------------------------===//
1306// CmpFOpPattern
1307//===----------------------------------------------------------------------===//
1308
1309/// Converts floating-point comparison operations to SPIR-V ops.
1310class CmpFOpPattern final : public OpConversionPattern<arith::CmpFOp> {
1311public:
1312 using Base::Base;
1313
1314 LogicalResult
1315 matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
1316 ConversionPatternRewriter &rewriter) const override {
1317 switch (op.getPredicate()) {
1318#define DISPATCH(cmpPredicate, spirvOp) \
1319 case cmpPredicate: \
1320 rewriter.replaceOpWithNewOp<spirvOp>(op, adaptor.getLhs(), \
1321 adaptor.getRhs()); \
1322 return success();
1323
1324 // Ordered.
1325 DISPATCH(arith::CmpFPredicate::OEQ, spirv::FOrdEqualOp);
1326 DISPATCH(arith::CmpFPredicate::OGT, spirv::FOrdGreaterThanOp);
1327 DISPATCH(arith::CmpFPredicate::OGE, spirv::FOrdGreaterThanEqualOp);
1328 DISPATCH(arith::CmpFPredicate::OLT, spirv::FOrdLessThanOp);
1329 DISPATCH(arith::CmpFPredicate::OLE, spirv::FOrdLessThanEqualOp);
1330 DISPATCH(arith::CmpFPredicate::ONE, spirv::FOrdNotEqualOp);
1331 // Unordered.
1332 DISPATCH(arith::CmpFPredicate::UEQ, spirv::FUnordEqualOp);
1333 DISPATCH(arith::CmpFPredicate::UGT, spirv::FUnordGreaterThanOp);
1334 DISPATCH(arith::CmpFPredicate::UGE, spirv::FUnordGreaterThanEqualOp);
1335 DISPATCH(arith::CmpFPredicate::ULT, spirv::FUnordLessThanOp);
1336 DISPATCH(arith::CmpFPredicate::ULE, spirv::FUnordLessThanEqualOp);
1337 DISPATCH(arith::CmpFPredicate::UNE, spirv::FUnordNotEqualOp);
1338
1339#undef DISPATCH
1340
1341 default:
1342 break;
1343 }
1344 return failure();
1345 }
1346};
1347
1348/// Converts floating point NaN check to SPIR-V ops. This pattern requires
1349/// Kernel capability.
1350class CmpFOpNanKernelPattern final : public OpConversionPattern<arith::CmpFOp> {
1351public:
1352 using Base::Base;
1353
1354 LogicalResult
1355 matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
1356 ConversionPatternRewriter &rewriter) const override {
1357 if (op.getPredicate() == arith::CmpFPredicate::ORD) {
1358 rewriter.replaceOpWithNewOp<spirv::OrderedOp>(op, args: adaptor.getLhs(),
1359 args: adaptor.getRhs());
1360 return success();
1361 }
1362
1363 if (op.getPredicate() == arith::CmpFPredicate::UNO) {
1364 rewriter.replaceOpWithNewOp<spirv::UnorderedOp>(op, args: adaptor.getLhs(),
1365 args: adaptor.getRhs());
1366 return success();
1367 }
1368
1369 return failure();
1370 }
1371};
1372
1373/// Converts floating point NaN check to SPIR-V ops. This pattern does not
1374/// require additional capability.
1375class CmpFOpNanNonePattern final : public OpConversionPattern<arith::CmpFOp> {
1376public:
1377 using Base::Base;
1378
1379 LogicalResult
1380 matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor,
1381 ConversionPatternRewriter &rewriter) const override {
1382 if (op.getPredicate() != arith::CmpFPredicate::ORD &&
1383 op.getPredicate() != arith::CmpFPredicate::UNO)
1384 return failure();
1385
1386 Location loc = op.getLoc();
1387
1388 Value replace;
1389 if (bitEnumContainsAll(bits: op.getFastmath(), bit: arith::FastMathFlags::nnan)) {
1390 if (op.getPredicate() == arith::CmpFPredicate::ORD) {
1391 // Ordered comparsion checks if neither operand is NaN.
1392 replace = spirv::ConstantOp::getOne(type: op.getType(), loc, builder&: rewriter);
1393 } else {
1394 // Unordered comparsion checks if either operand is NaN.
1395 replace = spirv::ConstantOp::getZero(type: op.getType(), loc, builder&: rewriter);
1396 }
1397 } else {
1398 Value lhsIsNan = spirv::IsNanOp::create(builder&: rewriter, location: loc, operand: adaptor.getLhs());
1399 Value rhsIsNan = spirv::IsNanOp::create(builder&: rewriter, location: loc, operand: adaptor.getRhs());
1400
1401 replace = spirv::LogicalOrOp::create(builder&: rewriter, location: loc, operand1: lhsIsNan, operand2: rhsIsNan);
1402 if (op.getPredicate() == arith::CmpFPredicate::ORD)
1403 replace = spirv::LogicalNotOp::create(builder&: rewriter, location: loc, operand: replace);
1404 }
1405
1406 rewriter.replaceOp(op, newValues: replace);
1407 return success();
1408 }
1409};
1410
1411//===----------------------------------------------------------------------===//
1412// AddUIExtendedOp/SubUIExtendedOp
1413//===----------------------------------------------------------------------===//
1414
1415/// Converts arith.addui_extended/arith.subui_extended to spirv.IAddCarry/
1416/// spirv.ISubBorrow.
1417template <typename ArithExtendedOp, typename SPIRVExtendedOp>
1418class BinaryExtendedOpPattern final
1419 : public OpConversionPattern<ArithExtendedOp> {
1420public:
1421 using OpConversionPattern<ArithExtendedOp>::OpConversionPattern;
1422 LogicalResult
1423 matchAndRewrite(ArithExtendedOp op, typename ArithExtendedOp::Adaptor adaptor,
1424 ConversionPatternRewriter &rewriter) const override {
1425 Type dstElemTy = adaptor.getLhs().getType();
1426 Location loc = op->getLoc();
1427 Value result = SPIRVExtendedOp::create(rewriter, loc, adaptor.getLhs(),
1428 adaptor.getRhs());
1429
1430 Value valueResult = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: result,
1431 indices: llvm::ArrayRef(0));
1432 Value flagValue = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: result,
1433 indices: llvm::ArrayRef(1));
1434
1435 // Convert the carry/borrow value to boolean.
1436 Value one = spirv::ConstantOp::getOne(type: dstElemTy, loc, builder&: rewriter);
1437 Value flagResult = spirv::IEqualOp::create(builder&: rewriter, location: loc, operand1: flagValue, operand2: one);
1438
1439 rewriter.replaceOp(op, {valueResult, flagResult});
1440 return success();
1441 }
1442};
1443
1444//===----------------------------------------------------------------------===//
1445// MulIExtendedOp
1446//===----------------------------------------------------------------------===//
1447
1448/// Converts arith.mul*i_extended to spirv.*MulExtended.
1449template <typename ArithMulOp, typename SPIRVMulOp>
1450class MulIExtendedOpPattern final : public OpConversionPattern<ArithMulOp> {
1451public:
1452 using OpConversionPattern<ArithMulOp>::OpConversionPattern;
1453 LogicalResult
1454 matchAndRewrite(ArithMulOp op, typename ArithMulOp::Adaptor adaptor,
1455 ConversionPatternRewriter &rewriter) const override {
1456 Location loc = op->getLoc();
1457 Value result =
1458 SPIRVMulOp::create(rewriter, loc, adaptor.getLhs(), adaptor.getRhs());
1459
1460 Value low = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: result,
1461 indices: llvm::ArrayRef(0));
1462 Value high = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: result,
1463 indices: llvm::ArrayRef(1));
1464
1465 rewriter.replaceOp(op, {low, high});
1466 return success();
1467 }
1468};
1469
1470//===----------------------------------------------------------------------===//
1471// SelectOp
1472//===----------------------------------------------------------------------===//
1473
1474/// Converts arith.select to spirv.Select.
1475class SelectOpPattern final : public OpConversionPattern<arith::SelectOp> {
1476public:
1477 using Base::Base;
1478 LogicalResult
1479 matchAndRewrite(arith::SelectOp op, OpAdaptor adaptor,
1480 ConversionPatternRewriter &rewriter) const override {
1481 rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args: adaptor.getCondition(),
1482 args: adaptor.getTrueValue(),
1483 args: adaptor.getFalseValue());
1484 return success();
1485 }
1486};
1487
1488//===----------------------------------------------------------------------===//
1489// MinimumFOp, MaximumFOp
1490//===----------------------------------------------------------------------===//
1491
1492/// Converts arith.maximumf/minimumf to spirv.GL.FMax/FMin or
1493/// spirv.CL.fmax/fmin.
1494template <typename Op, typename SPIRVOp>
1495class MinimumMaximumFOpPattern final : public OpConversionPattern<Op> {
1496public:
1497 using OpConversionPattern<Op>::OpConversionPattern;
1498 LogicalResult
1499 matchAndRewrite(Op op, typename Op::Adaptor adaptor,
1500 ConversionPatternRewriter &rewriter) const override {
1501 auto *converter = this->template getTypeConverter<SPIRVTypeConverter>();
1502 Type dstType = converter->convertType(op.getType());
1503 if (!dstType)
1504 return getTypeConversionFailure(rewriter, op);
1505
1506 // arith.maximumf/minimumf:
1507 // "if one of the arguments is NaN, then the result is also NaN."
1508 // spirv.GL.FMax/FMin
1509 // "which operand is the result is undefined if one of the operands
1510 // is a NaN."
1511 // spirv.CL.fmax/fmin:
1512 // "If one argument is a NaN, Fmin returns the other argument."
1513
1514 Location loc = op.getLoc();
1515 Value spirvOp =
1516 SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
1517
1518 if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
1519 rewriter.replaceOp(op, spirvOp);
1520 return success();
1521 }
1522
1523 Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
1524 Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
1525
1526 Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
1527 adaptor.getLhs(), spirvOp);
1528 Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
1529 adaptor.getRhs(), select1);
1530
1531 rewriter.replaceOp(op, select2);
1532 return success();
1533 }
1534};
1535
1536//===----------------------------------------------------------------------===//
1537// MinNumFOp, MaxNumFOp
1538//===----------------------------------------------------------------------===//
1539
1540/// Converts arith.maxnumf/minnumf to spirv.GL.NMax/NMin or
1541/// spirv.CL.fmax/fmin.
1542template <typename Op, typename SPIRVOp>
1543class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> {
1544 template <typename TargetOp>
1545 constexpr bool shouldInsertNanGuards() const {
1546 return llvm::is_one_of<TargetOp, spirv::GLFMaxOp, spirv::GLFMinOp>::value;
1547 }
1548
1549public:
1550 using OpConversionPattern<Op>::OpConversionPattern;
1551 LogicalResult
1552 matchAndRewrite(Op op, typename Op::Adaptor adaptor,
1553 ConversionPatternRewriter &rewriter) const override {
1554 auto *converter = this->template getTypeConverter<SPIRVTypeConverter>();
1555 Type dstType = converter->convertType(op.getType());
1556 if (!dstType)
1557 return getTypeConversionFailure(rewriter, op);
1558
1559 // arith.maxnumf/minnumf:
1560 // "If one of the arguments is NaN, then the result is the other
1561 // argument."
1562 // spirv.GL.NMax/NMin: NaN is treated as missing, matches arith semantics.
1563 // spirv.CL.fmax/fmin:
1564 // "If one argument is a NaN, Fmin returns the other argument."
1565 // spirv.GL.FMax/FMin: undefined when either operand is NaN, requires
1566 // select guards to implement arith.maxnumf semantics.
1567
1568 Location loc = op.getLoc();
1569 Value spirvOp =
1570 SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands());
1571
1572 if (!shouldInsertNanGuards<SPIRVOp>() ||
1573 bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) {
1574 rewriter.replaceOp(op, spirvOp);
1575 return success();
1576 }
1577
1578 Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs());
1579 Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs());
1580
1581 Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan,
1582 adaptor.getRhs(), spirvOp);
1583 Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan,
1584 adaptor.getLhs(), select1);
1585
1586 rewriter.replaceOp(op, select2);
1587 return success();
1588 }
1589};
1590
1591} // namespace
1592
1593//===----------------------------------------------------------------------===//
1594// Pattern Population
1595//===----------------------------------------------------------------------===//
1596
1597void mlir::arith::populateArithToSPIRVPatterns(
1598 const SPIRVTypeConverter &typeConverter, RewritePatternSet &patterns) {
1599 // clang-format off
1600 patterns.add<
1601 ConstantCompositeOpPattern,
1602 ConstantScalarOpPattern,
1603 BoolIOpPattern<arith::AddIOp, spirv::LogicalNotEqualOp>, // add mod 2 = XOR = not-equal
1604 ElementwiseArithOpPattern<arith::AddIOp, spirv::IAddOp>,
1605 BoolIOpPattern<arith::SubIOp, spirv::LogicalNotEqualOp>, // sub mod 2 = XOR = not-equal
1606 ElementwiseArithOpPattern<arith::SubIOp, spirv::ISubOp>,
1607 BoolIOpPattern<arith::MulIOp, spirv::LogicalAndOp>, // 1*1=1, else 0 = AND
1608 ElementwiseArithOpPattern<arith::MulIOp, spirv::IMulOp>,
1609 BoolIOpPattern<arith::DivUIOp, spirv::LogicalAndOp>, // a/1=a, a/0=UB; truth table = AND
1610 spirv::ElementwiseOpPattern<arith::DivUIOp, spirv::UDivOp>,
1611 BoolIOpPattern<arith::DivSIOp, spirv::LogicalAndOp>, // same as divui on i1
1612 spirv::ElementwiseOpPattern<arith::DivSIOp, spirv::SDivOp>,
1613 BoolIOpPattern<arith::CeilDivUIOp, spirv::LogicalAndOp>, // same as divui on i1
1614 CeilDivUIOpPattern,
1615 BoolIOpPattern<arith::CeilDivSIOp, spirv::LogicalAndOp>, // same as divsi on i1
1616 CeilDivSIOpPattern,
1617 BoolIOpPattern<arith::FloorDivSIOp, spirv::LogicalAndOp>, // same as divsi on i1
1618 FloorDivSIOpPattern,
1619 BoolIOpAndNotPattern<arith::RemUIOp>, // remui(a,b) = a & ~b (see pattern comment)
1620 spirv::ElementwiseOpPattern<arith::RemUIOp, spirv::UModOp>,
1621 BoolIOpAndNotPattern<arith::RemSIOp>, // remsi(a,b) = a & ~b (see pattern comment)
1622 RemSIOpGLPattern, RemSIOpCLPattern,
1623 BitwiseOpPattern<arith::AndIOp, spirv::LogicalAndOp, spirv::BitwiseAndOp>,
1624 BitwiseOpPattern<arith::OrIOp, spirv::LogicalOrOp, spirv::BitwiseOrOp>,
1625 XOrIOpLogicalPattern, XOrIOpBooleanPattern,
1626 BoolIOpAndNotPattern<arith::ShLIOp>, // shli(a,b) = a & ~b (see pattern comment)
1627 ElementwiseArithOpPattern<arith::ShLIOp, spirv::ShiftLeftLogicalOp>,
1628 BoolIOpAndNotPattern<arith::ShRUIOp>, // shrui(a,b) = a & ~b (see pattern comment)
1629 spirv::ElementwiseOpPattern<arith::ShRUIOp, spirv::ShiftRightLogicalOp>,
1630 ShRSIBoolPattern, // shrsi(a,b) = a (identity; see pattern comment)
1631 spirv::ElementwiseOpPattern<arith::ShRSIOp, spirv::ShiftRightArithmeticOp>,
1632 spirv::ElementwiseOpPattern<arith::NegFOp, spirv::FNegateOp>,
1633 spirv::ElementwiseOpPattern<arith::AddFOp, spirv::FAddOp>,
1634 spirv::ElementwiseOpPattern<arith::SubFOp, spirv::FSubOp>,
1635 spirv::ElementwiseOpPattern<arith::MulFOp, spirv::FMulOp>,
1636 spirv::ElementwiseOpPattern<arith::DivFOp, spirv::FDivOp>,
1637 spirv::ElementwiseOpPattern<arith::RemFOp, spirv::FRemOp>,
1638 ExtUIPattern, BoolToValuePattern<arith::ExtUIOp>,
1639 ExtSIPattern, ExtSII1Pattern,
1640 TypeCastingOpPattern<arith::ExtFOp, spirv::FConvertOp>,
1641 TruncIPattern, TruncII1Pattern,
1642 TypeCastingOpPattern<arith::TruncFOp, spirv::FConvertOp>,
1643 TypeCastingOpPattern<arith::ConvertFOp, spirv::FConvertOp>,
1644 IntToFPPattern<arith::UIToFPOp, spirv::ConvertUToFOp, false>,
1645 BoolToValuePattern<arith::UIToFPOp>,
1646 IntToFPPattern<arith::SIToFPOp, spirv::ConvertSToFOp, true>,
1647 TypeCastingOpPattern<arith::FPToUIOp, spirv::ConvertFToUOp>,
1648 TypeCastingOpPattern<arith::FPToSIOp, spirv::ConvertFToSOp>,
1649 TypeCastingOpPattern<arith::IndexCastOp, spirv::SConvertOp>,
1650 IndexCastIndexI1Pattern, BoolToValuePattern<arith::IndexCastOp>,
1651 TypeCastingOpPattern<arith::IndexCastUIOp, spirv::UConvertOp>,
1652 TypeCastingOpPattern<arith::BitcastOp, spirv::BitcastOp>,
1653 CmpIOpBooleanPattern, CmpIOpPattern,
1654 CmpFOpNanNonePattern, CmpFOpPattern,
1655 BinaryExtendedOpPattern<arith::AddUIExtendedOp, spirv::IAddCarryOp>,
1656 BinaryExtendedOpPattern<arith::SubUIExtendedOp, spirv::ISubBorrowOp>,
1657 MulIExtendedOpPattern<arith::MulSIExtendedOp, spirv::SMulExtendedOp>,
1658 MulIExtendedOpPattern<arith::MulUIExtendedOp, spirv::UMulExtendedOp>,
1659 SelectOpPattern,
1660
1661 MinimumMaximumFOpPattern<arith::MaximumFOp, spirv::GLFMaxOp>,
1662 MinimumMaximumFOpPattern<arith::MinimumFOp, spirv::GLFMinOp>,
1663 MinNumMaxNumFOpPattern<arith::MaxNumFOp, spirv::GLNMaxOp>,
1664 MinNumMaxNumFOpPattern<arith::MinNumFOp, spirv::GLNMinOp>,
1665 BoolIOpPattern<arith::MaxSIOp, spirv::LogicalAndOp>, // signed i1: 1=-1, so max=0 unless both are 1
1666 BoolIOpPattern<arith::MaxUIOp, spirv::LogicalOrOp>, // unsigned max on i1: 1 when either is 1
1667 BoolIOpPattern<arith::MinSIOp, spirv::LogicalOrOp>, // signed i1: -1<0, so min=1 when either is 1
1668 BoolIOpPattern<arith::MinUIOp, spirv::LogicalAndOp>, // unsigned min on i1: 1 only when both are 1
1669 spirv::ElementwiseOpPattern<arith::MaxSIOp, spirv::GLSMaxOp>,
1670 spirv::ElementwiseOpPattern<arith::MaxUIOp, spirv::GLUMaxOp>,
1671 spirv::ElementwiseOpPattern<arith::MinSIOp, spirv::GLSMinOp>,
1672 spirv::ElementwiseOpPattern<arith::MinUIOp, spirv::GLUMinOp>,
1673
1674 MinimumMaximumFOpPattern<arith::MaximumFOp, spirv::CLFMaxOp>,
1675 MinimumMaximumFOpPattern<arith::MinimumFOp, spirv::CLFMinOp>,
1676 MinNumMaxNumFOpPattern<arith::MaxNumFOp, spirv::CLFMaxOp>,
1677 MinNumMaxNumFOpPattern<arith::MinNumFOp, spirv::CLFMinOp>,
1678 spirv::ElementwiseOpPattern<arith::MaxSIOp, spirv::CLSMaxOp>,
1679 spirv::ElementwiseOpPattern<arith::MaxUIOp, spirv::CLUMaxOp>,
1680 spirv::ElementwiseOpPattern<arith::MinSIOp, spirv::CLSMinOp>,
1681 spirv::ElementwiseOpPattern<arith::MinUIOp, spirv::CLUMinOp>
1682 >(arg: typeConverter, args: patterns.getContext());
1683 // clang-format on
1684
1685 // Give CmpFOpNanKernelPattern a higher benefit so it can prevail when Kernel
1686 // capability is available.
1687 patterns.add<CmpFOpNanKernelPattern>(arg: typeConverter, args: patterns.getContext(),
1688 /*benefit=*/args: 2);
1689}
1690
1691//===----------------------------------------------------------------------===//
1692// Pass Definition
1693//===----------------------------------------------------------------------===//
1694
1695namespace {
1696struct ConvertArithToSPIRVPass
1697 : public impl::ConvertArithToSPIRVPassBase<ConvertArithToSPIRVPass> {
1698 using Base::Base;
1699
1700 void runOnOperation() override {
1701 Operation *op = getOperation();
1702 spirv::TargetEnvAttr targetAttr = spirv::lookupTargetEnvOrDefault(op);
1703 std::unique_ptr<SPIRVConversionTarget> target =
1704 SPIRVConversionTarget::get(targetAttr);
1705
1706 SPIRVConversionOptions options;
1707 options.emulateLT32BitScalarTypes = this->emulateLT32BitScalarTypes;
1708 options.emulateUnsupportedFloatTypes = this->emulateUnsupportedFloatTypes;
1709 SPIRVTypeConverter typeConverter(targetAttr, options);
1710
1711 // Use UnrealizedConversionCast as the bridge so that we don't need to pull
1712 // in patterns for other dialects.
1713 target->addLegalOp<UnrealizedConversionCastOp>();
1714
1715 // Fail hard when there are any remaining 'arith' ops.
1716 target->addIllegalDialect<arith::ArithDialect>();
1717
1718 RewritePatternSet patterns(&getContext());
1719 arith::populateArithToSPIRVPatterns(typeConverter, patterns);
1720
1721 if (failed(Result: applyPartialConversion(op, target: *target, patterns: std::move(patterns))))
1722 signalPassFailure();
1723 }
1724};
1725} // namespace
1726