| 1 | //===- ArithToSPIRV.cpp - Arithmetic to SPIRV dialect conversion -----===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #include "mlir/Conversion/ArithToSPIRV/ArithToSPIRV.h" |
| 10 | |
| 11 | #include "../SPIRVCommon/Pattern.h" |
| 12 | #include "mlir/Dialect/Arith/IR/Arith.h" |
| 13 | #include "mlir/Dialect/SPIRV/IR/SPIRVAttributes.h" |
| 14 | #include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h" |
| 15 | #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" |
| 16 | #include "mlir/Dialect/SPIRV/IR/SPIRVTypes.h" |
| 17 | #include "mlir/Dialect/SPIRV/Transforms/SPIRVConversion.h" |
| 18 | #include "mlir/IR/BuiltinAttributes.h" |
| 19 | #include "mlir/IR/BuiltinTypes.h" |
| 20 | #include "mlir/IR/DialectResourceBlobManager.h" |
| 21 | #include "llvm/ADT/APInt.h" |
| 22 | #include "llvm/ADT/ArrayRef.h" |
| 23 | #include "llvm/ADT/STLExtras.h" |
| 24 | #include "llvm/Support/Debug.h" |
| 25 | #include "llvm/Support/MathExtras.h" |
| 26 | #include <cassert> |
| 27 | #include <memory> |
| 28 | |
| 29 | namespace mlir { |
| 30 | #define GEN_PASS_DEF_CONVERTARITHTOSPIRVPASS |
| 31 | #include "mlir/Conversion/Passes.h.inc" |
| 32 | } // namespace mlir |
| 33 | |
| 34 | #define DEBUG_TYPE "arith-to-spirv-pattern" |
| 35 | |
| 36 | using namespace mlir; |
| 37 | |
| 38 | //===----------------------------------------------------------------------===// |
| 39 | // Conversion Helpers |
| 40 | //===----------------------------------------------------------------------===// |
| 41 | |
| 42 | /// Converts the given `srcAttr` into a boolean attribute if it holds an |
| 43 | /// integral value. Returns null attribute if conversion fails. |
| 44 | static BoolAttr convertBoolAttr(Attribute srcAttr, Builder builder) { |
| 45 | if (auto boolAttr = dyn_cast<BoolAttr>(Val&: srcAttr)) |
| 46 | return boolAttr; |
| 47 | if (auto intAttr = dyn_cast<IntegerAttr>(Val&: srcAttr)) |
| 48 | return builder.getBoolAttr(value: intAttr.getValue().getBoolValue()); |
| 49 | return {}; |
| 50 | } |
| 51 | |
| 52 | /// Converts the given `srcAttr` to a new attribute of the given `dstType`. |
| 53 | /// Returns null attribute if conversion fails. |
| 54 | static IntegerAttr convertIntegerAttr(IntegerAttr srcAttr, IntegerType dstType, |
| 55 | Builder builder) { |
| 56 | // If the source number uses less active bits than the target bitwidth, then |
| 57 | // it should be safe to convert. |
| 58 | if (srcAttr.getValue().isIntN(N: dstType.getWidth())) |
| 59 | return builder.getIntegerAttr(type: dstType, value: srcAttr.getInt()); |
| 60 | |
| 61 | // XXX: Try again by interpreting the source number as a signed value. |
| 62 | // Although integers in the standard dialect are signless, they can represent |
| 63 | // a signed number. It's the operation decides how to interpret. This is |
| 64 | // dangerous, but it seems there is no good way of handling this if we still |
| 65 | // want to change the bitwidth. Emit a message at least. |
| 66 | if (srcAttr.getValue().isSignedIntN(N: dstType.getWidth())) { |
| 67 | auto dstAttr = builder.getIntegerAttr(type: dstType, value: srcAttr.getInt()); |
| 68 | LLVM_DEBUG(llvm::dbgs() << "attribute '" << srcAttr << "' converted to '" |
| 69 | << dstAttr << "' for type '" << dstType << "'\n" ); |
| 70 | return dstAttr; |
| 71 | } |
| 72 | |
| 73 | LLVM_DEBUG(llvm::dbgs() << "attribute '" << srcAttr |
| 74 | << "' illegal: cannot fit into target type '" |
| 75 | << dstType << "'\n" ); |
| 76 | return {}; |
| 77 | } |
| 78 | |
| 79 | /// Converts the given `srcAttr` to a new attribute of the given `dstType`. |
| 80 | /// Returns null attribute if `dstType` is not 32-bit or conversion fails. |
| 81 | static FloatAttr convertFloatAttr(FloatAttr srcAttr, FloatType dstType, |
| 82 | Builder builder) { |
| 83 | // Only support converting to float for now. |
| 84 | if (!dstType.isF32()) |
| 85 | return FloatAttr(); |
| 86 | |
| 87 | // Try to convert the source floating-point number to single precision. |
| 88 | APFloat dstVal = srcAttr.getValue(); |
| 89 | bool losesInfo = false; |
| 90 | APFloat::opStatus status = |
| 91 | dstVal.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmTowardZero, losesInfo: &losesInfo); |
| 92 | if (status != APFloat::opOK || losesInfo) { |
| 93 | LLVM_DEBUG(llvm::dbgs() |
| 94 | << srcAttr << " illegal: cannot fit into converted type '" |
| 95 | << dstType << "'\n" ); |
| 96 | return FloatAttr(); |
| 97 | } |
| 98 | |
| 99 | return builder.getF32FloatAttr(value: dstVal.convertToFloat()); |
| 100 | } |
| 101 | |
| 102 | // Gets an IntegerAttr from a FloatAttr while preserving the bits. If `dstType` |
| 103 | // is wider than the float (e.g. i8 emulated as i32), the bits are widened |
| 104 | // like an integer constant of the float's width. |
| 105 | static IntegerAttr getIntegerAttrFromFloatAttr(FloatAttr floatAttr, |
| 106 | IntegerType dstType, |
| 107 | Builder builder) { |
| 108 | APInt intVal = floatAttr.getValue().bitcastToAPInt(); |
| 109 | auto srcType = IntegerType::get(context: dstType.getContext(), width: intVal.getBitWidth()); |
| 110 | return convertIntegerAttr(srcAttr: builder.getIntegerAttr(type: srcType, value: intVal), dstType, |
| 111 | builder); |
| 112 | } |
| 113 | |
| 114 | /// Returns true if the given `type` is a boolean scalar or vector type. |
| 115 | static bool isBoolScalarOrVector(Type type) { |
| 116 | assert(type && "Not a valid type" ); |
| 117 | if (type.isInteger(width: 1)) |
| 118 | return true; |
| 119 | |
| 120 | if (auto vecType = dyn_cast<VectorType>(Val&: type)) |
| 121 | return vecType.getElementType().isInteger(width: 1); |
| 122 | |
| 123 | return false; |
| 124 | } |
| 125 | |
| 126 | /// Creates a scalar/vector integer constant. |
| 127 | static Value getScalarOrVectorConstInt(Type type, uint64_t value, |
| 128 | OpBuilder &builder, Location loc) { |
| 129 | if (auto vectorType = dyn_cast<VectorType>(Val&: type)) { |
| 130 | Attribute element = IntegerAttr::get(type: vectorType.getElementType(), value); |
| 131 | auto attr = SplatElementsAttr::get(type: vectorType, values: element); |
| 132 | return spirv::ConstantOp::create(builder, location: loc, constant: vectorType, value: attr); |
| 133 | } |
| 134 | |
| 135 | if (auto intType = dyn_cast<IntegerType>(Val&: type)) |
| 136 | return spirv::ConstantOp::create(builder, location: loc, constant: type, |
| 137 | value: builder.getIntegerAttr(type, value)); |
| 138 | |
| 139 | return nullptr; |
| 140 | } |
| 141 | |
| 142 | /// Returns true if scalar/vector type `a` and `b` have the same number of |
| 143 | /// bitwidth. |
| 144 | static bool hasSameBitwidth(Type a, Type b) { |
| 145 | auto getNumBitwidth = [](Type type) { |
| 146 | unsigned bw = 0; |
| 147 | if (type.isIntOrFloat()) |
| 148 | bw = type.getIntOrFloatBitWidth(); |
| 149 | else if (auto vecType = dyn_cast<VectorType>(Val&: type)) |
| 150 | bw = vecType.getElementTypeBitWidth() * vecType.getNumElements(); |
| 151 | return bw; |
| 152 | }; |
| 153 | unsigned aBW = getNumBitwidth(a); |
| 154 | unsigned bBW = getNumBitwidth(b); |
| 155 | return aBW != 0 && bBW != 0 && aBW == bBW; |
| 156 | } |
| 157 | |
| 158 | /// Returns a source type conversion failure for `srcType` and operation `op`. |
| 159 | static LogicalResult |
| 160 | getTypeConversionFailure(ConversionPatternRewriter &rewriter, Operation *op, |
| 161 | Type srcType) { |
| 162 | return rewriter.notifyMatchFailure( |
| 163 | arg: op->getLoc(), |
| 164 | msg: llvm::formatv(Fmt: "failed to convert source type '{0}'" , Vals&: srcType)); |
| 165 | } |
| 166 | |
| 167 | /// Returns a source type conversion failure for the result type of `op`. |
| 168 | static LogicalResult |
| 169 | getTypeConversionFailure(ConversionPatternRewriter &rewriter, Operation *op) { |
| 170 | assert(op->getNumResults() == 1); |
| 171 | return getTypeConversionFailure(rewriter, op, srcType: op->getResultTypes().front()); |
| 172 | } |
| 173 | |
| 174 | namespace { |
| 175 | |
| 176 | /// Converts elementwise unary, binary and ternary arith operations to SPIR-V |
| 177 | /// operations. Op can potentially support overflow flags. |
| 178 | template <typename Op, typename SPIRVOp> |
| 179 | struct ElementwiseArithOpPattern final : OpConversionPattern<Op> { |
| 180 | using OpConversionPattern<Op>::OpConversionPattern; |
| 181 | |
| 182 | LogicalResult |
| 183 | matchAndRewrite(Op op, typename Op::Adaptor adaptor, |
| 184 | ConversionPatternRewriter &rewriter) const override { |
| 185 | assert(adaptor.getOperands().size() <= 3); |
| 186 | // Reject boolean types to allow specialized boolean patterns to handle |
| 187 | // them (e.g., addi/subi on i1 should use LogicalNotEqual, not IAdd/ISub). |
| 188 | if (!adaptor.getOperands().empty() && |
| 189 | isBoolScalarOrVector(adaptor.getOperands().front().getType())) |
| 190 | return failure(); |
| 191 | auto converter = this->template getTypeConverter<SPIRVTypeConverter>(); |
| 192 | Type dstType = converter->convertType(op.getType()); |
| 193 | if (!dstType) { |
| 194 | return rewriter.notifyMatchFailure( |
| 195 | op->getLoc(), |
| 196 | llvm::formatv("failed to convert type {0} for SPIR-V" , op.getType())); |
| 197 | } |
| 198 | |
| 199 | if (SPIRVOp::template hasTrait<OpTrait::spirv::UnsignedOp>() && |
| 200 | !getElementTypeOrSelf(op.getType()).isIndex() && |
| 201 | dstType != op.getType()) { |
| 202 | return op.emitError("bitwidth emulation is not implemented yet on " |
| 203 | "unsigned op pattern version" ); |
| 204 | } |
| 205 | |
| 206 | auto overflowFlags = arith::IntegerOverflowFlags::none; |
| 207 | if (auto overflowIface = |
| 208 | dyn_cast<arith::ArithIntegerOverflowFlagsInterface>(*op)) { |
| 209 | if (converter->getTargetEnv().allows( |
| 210 | spirv::Extension::SPV_KHR_no_integer_wrap_decoration)) |
| 211 | overflowFlags = overflowIface.getOverflowAttr().getValue(); |
| 212 | } |
| 213 | |
| 214 | auto newOp = rewriter.template replaceOpWithNewOp<SPIRVOp>( |
| 215 | op, dstType, adaptor.getOperands()); |
| 216 | |
| 217 | if (bitEnumContainsAny(bits: overflowFlags, bit: arith::IntegerOverflowFlags::nsw)) |
| 218 | newOp->setDiscardableAttr( |
| 219 | getDecorationString(decoration: spirv::Decoration::NoSignedWrap), |
| 220 | rewriter.getUnitAttr()); |
| 221 | |
| 222 | if (bitEnumContainsAny(bits: overflowFlags, bit: arith::IntegerOverflowFlags::nuw)) |
| 223 | newOp->setDiscardableAttr( |
| 224 | getDecorationString(decoration: spirv::Decoration::NoUnsignedWrap), |
| 225 | rewriter.getUnitAttr()); |
| 226 | |
| 227 | return success(); |
| 228 | } |
| 229 | }; |
| 230 | |
| 231 | //===----------------------------------------------------------------------===// |
| 232 | // ConstantOp |
| 233 | //===----------------------------------------------------------------------===// |
| 234 | |
| 235 | /// Converts composite arith.constant operation to spirv.Constant. |
| 236 | struct ConstantCompositeOpPattern final |
| 237 | : public OpConversionPattern<arith::ConstantOp> { |
| 238 | using Base::Base; |
| 239 | |
| 240 | LogicalResult |
| 241 | matchAndRewrite(arith::ConstantOp constOp, OpAdaptor adaptor, |
| 242 | ConversionPatternRewriter &rewriter) const override { |
| 243 | auto srcType = dyn_cast<ShapedType>(Val: constOp.getType()); |
| 244 | if (!srcType || srcType.getNumElements() == 1) |
| 245 | return failure(); |
| 246 | |
| 247 | // arith.constant should only have vector or tensor types. This is a MLIR |
| 248 | // wide problem at the moment. |
| 249 | if (!isa<VectorType, RankedTensorType>(Val: srcType)) |
| 250 | return rewriter.notifyMatchFailure(arg&: constOp, msg: "unsupported ShapedType" ); |
| 251 | |
| 252 | Type dstType = getTypeConverter()->convertType(t: srcType); |
| 253 | if (!dstType) |
| 254 | return failure(); |
| 255 | |
| 256 | // Import the resource into the IR to make use of the special handling of |
| 257 | // element types later on. |
| 258 | mlir::DenseElementsAttr dstElementsAttr; |
| 259 | if (auto denseElementsAttr = |
| 260 | dyn_cast<DenseElementsAttr>(Val: constOp.getValue())) { |
| 261 | dstElementsAttr = denseElementsAttr; |
| 262 | } else if (auto resourceAttr = |
| 263 | dyn_cast<DenseResourceElementsAttr>(Val: constOp.getValue())) { |
| 264 | |
| 265 | AsmResourceBlob *blob = resourceAttr.getRawHandle().getBlob(); |
| 266 | if (!blob) |
| 267 | return constOp->emitError(message: "could not find resource blob" ); |
| 268 | |
| 269 | ArrayRef<char> ptr = blob->getData(); |
| 270 | |
| 271 | // Check that the buffer meets the requirements to get converted to a |
| 272 | // DenseElementsAttr |
| 273 | if (!DenseElementsAttr::isValidRawBuffer(type: srcType, rawBuffer: ptr)) |
| 274 | return constOp->emitError(message: "resource is not a valid buffer" ); |
| 275 | |
| 276 | dstElementsAttr = |
| 277 | DenseElementsAttr::getFromRawBuffer(type: resourceAttr.getType(), rawBuffer: ptr); |
| 278 | } else { |
| 279 | return constOp->emitError(message: "unsupported elements attribute" ); |
| 280 | } |
| 281 | |
| 282 | ShapedType dstAttrType = dstElementsAttr.getType(); |
| 283 | |
| 284 | // If the composite type has more than one dimensions, perform |
| 285 | // linearization. |
| 286 | if (srcType.getRank() > 1) { |
| 287 | if (isa<RankedTensorType>(Val: srcType)) { |
| 288 | dstAttrType = RankedTensorType::get(shape: srcType.getNumElements(), |
| 289 | elementType: srcType.getElementType()); |
| 290 | dstElementsAttr = dstElementsAttr.reshape(newType: dstAttrType); |
| 291 | } else { |
| 292 | // TODO: add support for large vectors. |
| 293 | return failure(); |
| 294 | } |
| 295 | } |
| 296 | |
| 297 | Type srcElemType = srcType.getElementType(); |
| 298 | Type dstElemType; |
| 299 | // Tensor types are converted to SPIR-V array types; vector types are |
| 300 | // converted to SPIR-V vector/array types. |
| 301 | if (auto arrayType = dyn_cast<spirv::ArrayType>(Val&: dstType)) |
| 302 | dstElemType = arrayType.getElementType(); |
| 303 | else |
| 304 | dstElemType = cast<VectorType>(Val&: dstType).getElementType(); |
| 305 | |
| 306 | // If the source and destination element types are different, perform |
| 307 | // attribute conversion. |
| 308 | if (srcElemType != dstElemType) { |
| 309 | SmallVector<Attribute, 8> elements; |
| 310 | if (isa<FloatType>(Val: srcElemType)) { |
| 311 | for (FloatAttr srcAttr : dstElementsAttr.getValues<FloatAttr>()) { |
| 312 | Attribute dstAttr = nullptr; |
| 313 | // Handle 8-bit float conversion to 8-bit integer. |
| 314 | auto *typeConverter = getTypeConverter<SPIRVTypeConverter>(); |
| 315 | if (typeConverter->getOptions().emulateUnsupportedFloatTypes && |
| 316 | srcElemType.getIntOrFloatBitWidth() == 8 && |
| 317 | isa<IntegerType>(Val: dstElemType)) { |
| 318 | dstAttr = getIntegerAttrFromFloatAttr( |
| 319 | floatAttr: srcAttr, dstType: cast<IntegerType>(Val&: dstElemType), builder: rewriter); |
| 320 | } else { |
| 321 | dstAttr = convertFloatAttr(srcAttr, dstType: cast<FloatType>(Val&: dstElemType), |
| 322 | builder: rewriter); |
| 323 | } |
| 324 | if (!dstAttr) |
| 325 | return failure(); |
| 326 | elements.push_back(Elt: dstAttr); |
| 327 | } |
| 328 | } else if (srcElemType.isInteger(width: 1)) { |
| 329 | return failure(); |
| 330 | } else { |
| 331 | for (IntegerAttr srcAttr : dstElementsAttr.getValues<IntegerAttr>()) { |
| 332 | IntegerAttr dstAttr = convertIntegerAttr( |
| 333 | srcAttr, dstType: cast<IntegerType>(Val&: dstElemType), builder: rewriter); |
| 334 | if (!dstAttr) |
| 335 | return failure(); |
| 336 | elements.push_back(Elt: dstAttr); |
| 337 | } |
| 338 | } |
| 339 | |
| 340 | // Unfortunately, we cannot use dialect-specific types for element |
| 341 | // attributes; element attributes only works with builtin types. So we |
| 342 | // need to prepare another converted builtin types for the destination |
| 343 | // elements attribute. |
| 344 | if (isa<RankedTensorType>(Val: dstAttrType)) |
| 345 | dstAttrType = |
| 346 | RankedTensorType::get(shape: dstAttrType.getShape(), elementType: dstElemType); |
| 347 | else |
| 348 | dstAttrType = VectorType::get(shape: dstAttrType.getShape(), elementType: dstElemType); |
| 349 | |
| 350 | dstElementsAttr = DenseElementsAttr::get(type: dstAttrType, values: elements); |
| 351 | } |
| 352 | |
| 353 | rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op: constOp, args&: dstType, |
| 354 | args&: dstElementsAttr); |
| 355 | return success(); |
| 356 | } |
| 357 | }; |
| 358 | |
| 359 | /// Converts scalar arith.constant operation to spirv.Constant. |
| 360 | struct ConstantScalarOpPattern final |
| 361 | : public OpConversionPattern<arith::ConstantOp> { |
| 362 | using Base::Base; |
| 363 | |
| 364 | LogicalResult |
| 365 | matchAndRewrite(arith::ConstantOp constOp, OpAdaptor adaptor, |
| 366 | ConversionPatternRewriter &rewriter) const override { |
| 367 | Type srcType = constOp.getType(); |
| 368 | if (auto shapedType = dyn_cast<ShapedType>(Val&: srcType)) { |
| 369 | if (shapedType.getNumElements() != 1) |
| 370 | return failure(); |
| 371 | srcType = shapedType.getElementType(); |
| 372 | } |
| 373 | if (!srcType.isIntOrIndexOrFloat()) |
| 374 | return failure(); |
| 375 | |
| 376 | Attribute cstAttr = constOp.getValue(); |
| 377 | if (auto elementsAttr = dyn_cast<DenseElementsAttr>(Val&: cstAttr)) |
| 378 | cstAttr = elementsAttr.getSplatValue<Attribute>(); |
| 379 | |
| 380 | Type dstType = getTypeConverter()->convertType(t: srcType); |
| 381 | if (!dstType) |
| 382 | return failure(); |
| 383 | |
| 384 | // Floating-point types. |
| 385 | if (isa<FloatType>(Val: srcType)) { |
| 386 | auto srcAttr = cast<FloatAttr>(Val&: cstAttr); |
| 387 | Attribute dstAttr = srcAttr; |
| 388 | |
| 389 | // Floating-point types not supported in the target environment are all |
| 390 | // converted to float type. |
| 391 | auto *typeConverter = getTypeConverter<SPIRVTypeConverter>(); |
| 392 | if (typeConverter->getOptions().emulateUnsupportedFloatTypes && |
| 393 | srcType.getIntOrFloatBitWidth() == 8 && isa<IntegerType>(Val: dstType)) { |
| 394 | // If the source is an 8-bit float, convert it to a 8-bit integer. |
| 395 | dstAttr = getIntegerAttrFromFloatAttr( |
| 396 | floatAttr: srcAttr, dstType: cast<IntegerType>(Val&: dstType), builder: rewriter); |
| 397 | if (!dstAttr) |
| 398 | return failure(); |
| 399 | } else if (srcType != dstType) { |
| 400 | dstAttr = convertFloatAttr(srcAttr, dstType: cast<FloatType>(Val&: dstType), builder: rewriter); |
| 401 | if (!dstAttr) |
| 402 | return failure(); |
| 403 | } |
| 404 | |
| 405 | rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op: constOp, args&: dstType, args&: dstAttr); |
| 406 | return success(); |
| 407 | } |
| 408 | |
| 409 | // Bool type. |
| 410 | if (srcType.isInteger(width: 1)) { |
| 411 | // arith.constant can use 0/1 instead of true/false for i1 values. We need |
| 412 | // to handle that here. |
| 413 | auto dstAttr = convertBoolAttr(srcAttr: cstAttr, builder: rewriter); |
| 414 | if (!dstAttr) |
| 415 | return failure(); |
| 416 | rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op: constOp, args&: dstType, args&: dstAttr); |
| 417 | return success(); |
| 418 | } |
| 419 | |
| 420 | // IndexType or IntegerType. Index values are converted to 32-bit integer |
| 421 | // values when converting to SPIR-V. |
| 422 | auto srcAttr = cast<IntegerAttr>(Val&: cstAttr); |
| 423 | IntegerAttr dstAttr = |
| 424 | convertIntegerAttr(srcAttr, dstType: cast<IntegerType>(Val&: dstType), builder: rewriter); |
| 425 | if (!dstAttr) |
| 426 | return failure(); |
| 427 | rewriter.replaceOpWithNewOp<spirv::ConstantOp>(op: constOp, args&: dstType, args&: dstAttr); |
| 428 | return success(); |
| 429 | } |
| 430 | }; |
| 431 | |
| 432 | //===----------------------------------------------------------------------===// |
| 433 | // RemSIOp |
| 434 | //===----------------------------------------------------------------------===// |
| 435 | |
| 436 | /// Returns signed remainder for `lhs` and `rhs` and lets the result follow |
| 437 | /// the sign of `signOperand`. |
| 438 | /// |
| 439 | /// Note that this is needed for Vulkan. Per the Vulkan's SPIR-V environment |
| 440 | /// spec, "for the OpSRem and OpSMod instructions, if either operand is negative |
| 441 | /// the result is undefined." So we cannot directly use spirv.SRem/spirv.SMod |
| 442 | /// if either operand can be negative. Emulate it via spirv.UMod. |
| 443 | template <typename SignedAbsOp> |
| 444 | static Value emulateSignedRemainder(Location loc, Value lhs, Value rhs, |
| 445 | Value signOperand, OpBuilder &builder) { |
| 446 | assert(lhs.getType() == rhs.getType()); |
| 447 | assert(lhs == signOperand || rhs == signOperand); |
| 448 | |
| 449 | Type type = lhs.getType(); |
| 450 | |
| 451 | // Calculate the remainder with spirv.UMod. |
| 452 | Value lhsAbs = SignedAbsOp::create(builder, loc, type, lhs); |
| 453 | Value rhsAbs = SignedAbsOp::create(builder, loc, type, rhs); |
| 454 | Value abs = spirv::UModOp::create(builder, location: loc, operand1: lhsAbs, operand2: rhsAbs); |
| 455 | |
| 456 | // Fix the sign. |
| 457 | Value isPositive; |
| 458 | if (lhs == signOperand) |
| 459 | isPositive = spirv::IEqualOp::create(builder, location: loc, operand1: lhs, operand2: lhsAbs); |
| 460 | else |
| 461 | isPositive = spirv::IEqualOp::create(builder, location: loc, operand1: rhs, operand2: rhsAbs); |
| 462 | Value absNegate = spirv::SNegateOp::create(builder, location: loc, result: type, operand: abs); |
| 463 | return spirv::SelectOp::create(builder, location: loc, result: type, condition: isPositive, true_value: abs, |
| 464 | false_value: absNegate); |
| 465 | } |
| 466 | |
| 467 | /// Converts arith.remsi to GLSL SPIR-V ops. |
| 468 | /// |
| 469 | /// This cannot be merged into the template unary/binary pattern due to Vulkan |
| 470 | /// restrictions over spirv.SRem and spirv.SMod. |
| 471 | struct RemSIOpGLPattern final : public OpConversionPattern<arith::RemSIOp> { |
| 472 | using Base::Base; |
| 473 | |
| 474 | LogicalResult |
| 475 | matchAndRewrite(arith::RemSIOp op, OpAdaptor adaptor, |
| 476 | ConversionPatternRewriter &rewriter) const override { |
| 477 | Value result = emulateSignedRemainder<spirv::GLSAbsOp>( |
| 478 | loc: op.getLoc(), lhs: adaptor.getOperands()[0], rhs: adaptor.getOperands()[1], |
| 479 | signOperand: adaptor.getOperands()[0], builder&: rewriter); |
| 480 | rewriter.replaceOp(op, newValues: result); |
| 481 | |
| 482 | return success(); |
| 483 | } |
| 484 | }; |
| 485 | |
| 486 | /// Converts arith.remsi to OpenCL SPIR-V ops. |
| 487 | struct RemSIOpCLPattern final : public OpConversionPattern<arith::RemSIOp> { |
| 488 | using Base::Base; |
| 489 | |
| 490 | LogicalResult |
| 491 | matchAndRewrite(arith::RemSIOp op, OpAdaptor adaptor, |
| 492 | ConversionPatternRewriter &rewriter) const override { |
| 493 | Value result = emulateSignedRemainder<spirv::CLSAbsOp>( |
| 494 | loc: op.getLoc(), lhs: adaptor.getOperands()[0], rhs: adaptor.getOperands()[1], |
| 495 | signOperand: adaptor.getOperands()[0], builder&: rewriter); |
| 496 | rewriter.replaceOp(op, newValues: result); |
| 497 | |
| 498 | return success(); |
| 499 | } |
| 500 | }; |
| 501 | |
| 502 | //===----------------------------------------------------------------------===// |
| 503 | // CeilDivUIOp |
| 504 | //===----------------------------------------------------------------------===// |
| 505 | |
| 506 | /// Converts arith.ceildivui to SPIR-V ops. Convert `ceildivui(n, m)` into |
| 507 | /// `n == 0 ? 0 : (n-1)/m + 1`. Formula taken from the equivalent conversion |
| 508 | /// in IndexToSPIRV. |
| 509 | struct CeilDivUIOpPattern final : OpConversionPattern<arith::CeilDivUIOp> { |
| 510 | using Base::Base; |
| 511 | |
| 512 | LogicalResult |
| 513 | matchAndRewrite(arith::CeilDivUIOp op, OpAdaptor adaptor, |
| 514 | ConversionPatternRewriter &rewriter) const override { |
| 515 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 516 | if (!dstType) |
| 517 | return getTypeConversionFailure(rewriter, op); |
| 518 | |
| 519 | if (!getElementTypeOrSelf(type: op.getType()).isIndex() && |
| 520 | dstType != op.getType()) |
| 521 | return op.emitError(message: "bitwidth emulation is not implemented yet on " |
| 522 | "unsigned op pattern version" ); |
| 523 | |
| 524 | Location loc = op.getLoc(); |
| 525 | Value n = adaptor.getLhs(); |
| 526 | Value m = adaptor.getRhs(); |
| 527 | |
| 528 | Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter); |
| 529 | Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder&: rewriter); |
| 530 | |
| 531 | Value minusOne = spirv::ISubOp::create(builder&: rewriter, location: loc, operand1: n, operand2: one); |
| 532 | Value quotient = spirv::UDivOp::create(builder&: rewriter, location: loc, operand1: minusOne, operand2: m); |
| 533 | Value plusOne = spirv::IAddOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: one); |
| 534 | |
| 535 | Value cmp = spirv::IEqualOp::create(builder&: rewriter, location: loc, operand1: n, operand2: zero); |
| 536 | rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: cmp, args&: zero, args&: plusOne); |
| 537 | return success(); |
| 538 | } |
| 539 | }; |
| 540 | |
| 541 | //===----------------------------------------------------------------------===// |
| 542 | // CeilDivSIOp |
| 543 | //===----------------------------------------------------------------------===// |
| 544 | |
| 545 | /// Converts arith.ceildivsi to SPIR-V ops. Convert `ceildivsi(a, b)` into |
| 546 | /// `q = a/b; (q*b != a && (a<0) == (b<0)) ? q+1 : q`. Formula taken from the |
| 547 | /// equivalent conversion in ExpandOps. |
| 548 | struct CeilDivSIOpPattern final : OpConversionPattern<arith::CeilDivSIOp> { |
| 549 | using Base::Base; |
| 550 | |
| 551 | LogicalResult |
| 552 | matchAndRewrite(arith::CeilDivSIOp op, OpAdaptor adaptor, |
| 553 | ConversionPatternRewriter &rewriter) const override { |
| 554 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 555 | if (!dstType) |
| 556 | return getTypeConversionFailure(rewriter, op); |
| 557 | |
| 558 | Location loc = op.getLoc(); |
| 559 | Value a = adaptor.getLhs(); |
| 560 | Value b = adaptor.getRhs(); |
| 561 | |
| 562 | Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter); |
| 563 | Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder&: rewriter); |
| 564 | |
| 565 | Value quotient = spirv::SDivOp::create(builder&: rewriter, location: loc, operand1: a, operand2: b); |
| 566 | Value product = spirv::IMulOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: b); |
| 567 | Value notEqual = spirv::INotEqualOp::create(builder&: rewriter, location: loc, operand1: a, operand2: product); |
| 568 | |
| 569 | Value aNeg = spirv::SLessThanOp::create(builder&: rewriter, location: loc, operand1: a, operand2: zero); |
| 570 | Value bNeg = spirv::SLessThanOp::create(builder&: rewriter, location: loc, operand1: b, operand2: zero); |
| 571 | Value sameSign = spirv::LogicalEqualOp::create(builder&: rewriter, location: loc, operand1: aNeg, operand2: bNeg); |
| 572 | Value cond = spirv::LogicalAndOp::create(builder&: rewriter, location: loc, operand1: notEqual, operand2: sameSign); |
| 573 | |
| 574 | Value quotientPlusOne = spirv::IAddOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: one); |
| 575 | rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: cond, args&: quotientPlusOne, |
| 576 | args&: quotient); |
| 577 | return success(); |
| 578 | } |
| 579 | }; |
| 580 | |
| 581 | //===----------------------------------------------------------------------===// |
| 582 | // FloorDivSIOp |
| 583 | //===----------------------------------------------------------------------===// |
| 584 | |
| 585 | /// Converts arith.floordivsi to SPIR-V ops. Convert `floordivsi(a, b)` into |
| 586 | /// `q = a/b; (q*b != a && (a<0) != (b<0)) ? q-1 : q`. Formula taken from the |
| 587 | /// equivalent conversion in ExpandOps. |
| 588 | struct FloorDivSIOpPattern final : OpConversionPattern<arith::FloorDivSIOp> { |
| 589 | using Base::Base; |
| 590 | |
| 591 | LogicalResult |
| 592 | matchAndRewrite(arith::FloorDivSIOp op, OpAdaptor adaptor, |
| 593 | ConversionPatternRewriter &rewriter) const override { |
| 594 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 595 | if (!dstType) |
| 596 | return getTypeConversionFailure(rewriter, op); |
| 597 | |
| 598 | Location loc = op.getLoc(); |
| 599 | Value a = adaptor.getLhs(); |
| 600 | Value b = adaptor.getRhs(); |
| 601 | |
| 602 | Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter); |
| 603 | Value minusOne = getScalarOrVectorConstInt( |
| 604 | type: dstType, value: static_cast<uint64_t>(-1), builder&: rewriter, loc); |
| 605 | |
| 606 | Value quotient = spirv::SDivOp::create(builder&: rewriter, location: loc, operand1: a, operand2: b); |
| 607 | Value product = spirv::IMulOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: b); |
| 608 | Value notEqual = spirv::INotEqualOp::create(builder&: rewriter, location: loc, operand1: a, operand2: product); |
| 609 | |
| 610 | Value aNeg = spirv::SLessThanOp::create(builder&: rewriter, location: loc, operand1: a, operand2: zero); |
| 611 | Value bNeg = spirv::SLessThanOp::create(builder&: rewriter, location: loc, operand1: b, operand2: zero); |
| 612 | Value diffSign = |
| 613 | spirv::LogicalNotEqualOp::create(builder&: rewriter, location: loc, operand1: aNeg, operand2: bNeg); |
| 614 | Value cond = spirv::LogicalAndOp::create(builder&: rewriter, location: loc, operand1: notEqual, operand2: diffSign); |
| 615 | |
| 616 | Value quotientMinusOne = |
| 617 | spirv::IAddOp::create(builder&: rewriter, location: loc, operand1: quotient, operand2: minusOne); |
| 618 | rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: cond, args&: quotientMinusOne, |
| 619 | args&: quotient); |
| 620 | return success(); |
| 621 | } |
| 622 | }; |
| 623 | |
| 624 | //===----------------------------------------------------------------------===// |
| 625 | // BitwiseOp |
| 626 | //===----------------------------------------------------------------------===// |
| 627 | |
| 628 | /// Converts bitwise operations to SPIR-V operations. This is a special pattern |
| 629 | /// other than the BinaryOpPatternPattern because if the operands are boolean |
| 630 | /// values, SPIR-V uses different operations (`SPIRVLogicalOp`). For |
| 631 | /// non-boolean operands, SPIR-V should use `SPIRVBitwiseOp`. |
| 632 | template <typename Op, typename SPIRVLogicalOp, typename SPIRVBitwiseOp> |
| 633 | struct BitwiseOpPattern final : public OpConversionPattern<Op> { |
| 634 | using OpConversionPattern<Op>::OpConversionPattern; |
| 635 | |
| 636 | LogicalResult |
| 637 | matchAndRewrite(Op op, typename Op::Adaptor adaptor, |
| 638 | ConversionPatternRewriter &rewriter) const override { |
| 639 | assert(adaptor.getOperands().size() == 2); |
| 640 | Type dstType = this->getTypeConverter()->convertType(op.getType()); |
| 641 | if (!dstType) |
| 642 | return getTypeConversionFailure(rewriter, op); |
| 643 | |
| 644 | if (isBoolScalarOrVector(adaptor.getOperands().front().getType())) { |
| 645 | rewriter.template replaceOpWithNewOp<SPIRVLogicalOp>( |
| 646 | op, dstType, adaptor.getOperands()); |
| 647 | } else { |
| 648 | rewriter.template replaceOpWithNewOp<SPIRVBitwiseOp>( |
| 649 | op, dstType, adaptor.getOperands()); |
| 650 | } |
| 651 | return success(); |
| 652 | } |
| 653 | }; |
| 654 | |
| 655 | //===----------------------------------------------------------------------===// |
| 656 | // XOrIOp |
| 657 | //===----------------------------------------------------------------------===// |
| 658 | |
| 659 | /// Converts arith.xori to SPIR-V operations. |
| 660 | struct XOrIOpLogicalPattern final : public OpConversionPattern<arith::XOrIOp> { |
| 661 | using Base::Base; |
| 662 | |
| 663 | LogicalResult |
| 664 | matchAndRewrite(arith::XOrIOp op, OpAdaptor adaptor, |
| 665 | ConversionPatternRewriter &rewriter) const override { |
| 666 | assert(adaptor.getOperands().size() == 2); |
| 667 | |
| 668 | if (isBoolScalarOrVector(type: adaptor.getOperands().front().getType())) |
| 669 | return failure(); |
| 670 | |
| 671 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 672 | if (!dstType) |
| 673 | return getTypeConversionFailure(rewriter, op); |
| 674 | |
| 675 | rewriter.replaceOpWithNewOp<spirv::BitwiseXorOp>(op, args&: dstType, |
| 676 | args: adaptor.getOperands()); |
| 677 | |
| 678 | return success(); |
| 679 | } |
| 680 | }; |
| 681 | |
| 682 | /// Converts arith.xori to SPIR-V operations if the type of source is i1 or |
| 683 | /// vector of i1. |
| 684 | struct XOrIOpBooleanPattern final : public OpConversionPattern<arith::XOrIOp> { |
| 685 | using Base::Base; |
| 686 | |
| 687 | LogicalResult |
| 688 | matchAndRewrite(arith::XOrIOp op, OpAdaptor adaptor, |
| 689 | ConversionPatternRewriter &rewriter) const override { |
| 690 | assert(adaptor.getOperands().size() == 2); |
| 691 | |
| 692 | if (!isBoolScalarOrVector(type: adaptor.getOperands().front().getType())) |
| 693 | return failure(); |
| 694 | |
| 695 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 696 | if (!dstType) |
| 697 | return getTypeConversionFailure(rewriter, op); |
| 698 | |
| 699 | rewriter.replaceOpWithNewOp<spirv::LogicalNotEqualOp>( |
| 700 | op, args&: dstType, args: adaptor.getOperands()); |
| 701 | return success(); |
| 702 | } |
| 703 | }; |
| 704 | |
| 705 | /// Converts an arith integer op to the given SPIR-V boolean op if the type is |
| 706 | /// i1 or vector of i1. Each mapping follows from the boolean truth table of |
| 707 | /// the operation: |
| 708 | /// addi(a, b) = a ^ b (add mod 2 = XOR = LogicalNotEqual) |
| 709 | /// subi(a, b) = a ^ b (sub mod 2 = XOR = LogicalNotEqual) |
| 710 | /// muli(a, b) = a & b (1*1=1, else 0 = LogicalAnd) |
| 711 | /// divui(a, b) = a & b (a/1=a, a/0=UB; truth table matches AND) |
| 712 | /// divsi(a, b) = a & b (same as divui on i1) |
| 713 | /// ceildivui(a, b) = a & b (same as divui on i1) |
| 714 | /// ceildivsi(a, b) = a & b (same as divsi on i1) |
| 715 | /// floordivsi(a, b) = a & b (same as divsi on i1) |
| 716 | /// maxsi(a, b) = a & b (signed i1: 1 represents -1, so max is 0 unless both |
| 717 | /// are 1) |
| 718 | /// maxui(a, b) = a | b (unsigned max on i1: 1 when either operand is 1) |
| 719 | /// minsi(a, b) = a | b (signed i1: -1 < 0, so min is 1 when either operand |
| 720 | /// is 1) |
| 721 | /// minui(a, b) = a & b (unsigned min on i1: 1 only when both operands are |
| 722 | /// 1) |
| 723 | template <typename ArithOp, typename SPIRVOp> |
| 724 | struct BoolIOpPattern final : public OpConversionPattern<ArithOp> { |
| 725 | BoolIOpPattern(const TypeConverter &converter, MLIRContext *context) |
| 726 | // benefit=2: takes priority over the generic ElementwiseArithOpPattern |
| 727 | // (benefit=1) when the operand type is i1. |
| 728 | : OpConversionPattern<ArithOp>(converter, context, /*benefit=*/2) {} |
| 729 | |
| 730 | LogicalResult |
| 731 | matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor, |
| 732 | ConversionPatternRewriter &rewriter) const override { |
| 733 | if (!isBoolScalarOrVector(adaptor.getOperands().front().getType())) |
| 734 | return failure(); |
| 735 | |
| 736 | Type dstType = this->getTypeConverter()->convertType(op.getType()); |
| 737 | if (!dstType) |
| 738 | return getTypeConversionFailure(rewriter, op); |
| 739 | |
| 740 | rewriter.replaceOpWithNewOp<SPIRVOp>(op, dstType, adaptor.getOperands()); |
| 741 | return success(); |
| 742 | } |
| 743 | }; |
| 744 | |
| 745 | /// Converts an arith binary op on i1 to spirv.LogicalAnd(lhs, |
| 746 | /// spirv.LogicalNot(rhs)). This covers shift-left, shift-right-unsigned, and |
| 747 | /// unsigned remainder on i1: |
| 748 | /// shli(a, b) = a & ~b (shift left clears the bit when b=1) |
| 749 | /// shrui(a, b) = a & ~b (shift right unsigned clears the bit when b=1) |
| 750 | /// remui(a, b) = a & ~b (only defined when b=1; a%1=0, and ~b=~1=0, so AND |
| 751 | /// gives 0) |
| 752 | /// remsi(a, b) = a & ~b (only defined when b=1; a%1=0, and ~b=~1=0, so AND |
| 753 | /// gives 0) |
| 754 | template <typename ArithOp> |
| 755 | struct BoolIOpAndNotPattern final : public OpConversionPattern<ArithOp> { |
| 756 | BoolIOpAndNotPattern(const TypeConverter &converter, MLIRContext *context) |
| 757 | // benefit=2: takes priority over the generic ElementwiseArithOpPattern |
| 758 | // (benefit=1) when the operand type is i1. |
| 759 | : OpConversionPattern<ArithOp>(converter, context, /*benefit=*/2) {} |
| 760 | |
| 761 | LogicalResult |
| 762 | matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor, |
| 763 | ConversionPatternRewriter &rewriter) const override { |
| 764 | if (!isBoolScalarOrVector(adaptor.getOperands().front().getType())) |
| 765 | return failure(); |
| 766 | |
| 767 | Type dstType = this->getTypeConverter()->convertType(op.getType()); |
| 768 | if (!dstType) |
| 769 | return getTypeConversionFailure(rewriter, op); |
| 770 | |
| 771 | Location loc = op.getLoc(); |
| 772 | Value notRhs = spirv::LogicalNotOp::create(rewriter, loc, dstType, |
| 773 | adaptor.getOperands()[1]); |
| 774 | rewriter.replaceOpWithNewOp<spirv::LogicalAndOp>( |
| 775 | op, dstType, adaptor.getOperands()[0], notRhs); |
| 776 | return success(); |
| 777 | } |
| 778 | }; |
| 779 | |
| 780 | /// Converts arith.shrsi on i1 to identity: arithmetic right shift of a 1-bit |
| 781 | /// signed value always yields the original value (0 >> n = 0, -1 >> n = -1). |
| 782 | struct ShRSIBoolPattern final : public OpConversionPattern<arith::ShRSIOp> { |
| 783 | ShRSIBoolPattern(const TypeConverter &converter, MLIRContext *context) |
| 784 | // benefit=2: takes priority over the generic spirv::ElementwiseOpPattern |
| 785 | // (benefit=1) when the operand type is i1. |
| 786 | : OpConversionPattern<arith::ShRSIOp>(converter, context, |
| 787 | /*benefit=*/2) {} |
| 788 | |
| 789 | LogicalResult |
| 790 | matchAndRewrite(arith::ShRSIOp op, OpAdaptor adaptor, |
| 791 | ConversionPatternRewriter &rewriter) const override { |
| 792 | if (!isBoolScalarOrVector(type: adaptor.getOperands().front().getType())) |
| 793 | return failure(); |
| 794 | |
| 795 | rewriter.replaceOp(op, newValues: adaptor.getOperands().front()); |
| 796 | return success(); |
| 797 | } |
| 798 | }; |
| 799 | |
| 800 | //===----------------------------------------------------------------------===// |
| 801 | // i1 source to value |
| 802 | //===----------------------------------------------------------------------===// |
| 803 | |
| 804 | /// Converts an op whose i1 (or vector of i1) source selects between one and |
| 805 | /// zero of the destination type, i.e. spirv.Select(src, one, zero). Shared by |
| 806 | /// arith.uitofp, arith.extui, and arith.index_cast on boolean sources. |
| 807 | template <typename ArithOp> |
| 808 | struct BoolToValuePattern final : public OpConversionPattern<ArithOp> { |
| 809 | using OpConversionPattern<ArithOp>::OpConversionPattern; |
| 810 | |
| 811 | LogicalResult |
| 812 | matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor, |
| 813 | ConversionPatternRewriter &rewriter) const override { |
| 814 | Type srcType = adaptor.getOperands().front().getType(); |
| 815 | if (!isBoolScalarOrVector(type: srcType)) |
| 816 | return failure(); |
| 817 | |
| 818 | Type dstType = this->getTypeConverter()->convertType(op.getType()); |
| 819 | if (!dstType) |
| 820 | return getTypeConversionFailure(rewriter, op); |
| 821 | |
| 822 | Location loc = op.getLoc(); |
| 823 | Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter); |
| 824 | Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder&: rewriter); |
| 825 | rewriter.replaceOpWithNewOp<spirv::SelectOp>( |
| 826 | op, dstType, adaptor.getOperands().front(), one, zero); |
| 827 | return success(); |
| 828 | } |
| 829 | }; |
| 830 | |
| 831 | //===----------------------------------------------------------------------===// |
| 832 | // UIToFPOp |
| 833 | //===----------------------------------------------------------------------===// |
| 834 | |
| 835 | /// Converts arith.uitofp/arith.sitofp to spirv.ConvertUToF/spirv.ConvertSToF. |
| 836 | /// When the source integer type was widened during type conversion (e.g., i8 |
| 837 | /// emulated as i32), the upper bits of the widened value may contain garbage. |
| 838 | /// This pattern cleans the upper bits before the conversion: |
| 839 | /// - For unsigned (IsSigned=false): mask with BitwiseAnd. |
| 840 | /// - For signed (IsSigned=true): sign-extend via ShiftLeftLogical + |
| 841 | /// ShiftRightArithmetic. |
| 842 | template <typename ArithOp, typename SPIRVOp, bool IsSigned> |
| 843 | struct IntToFPPattern final : public OpConversionPattern<ArithOp> { |
| 844 | using OpConversionPattern<ArithOp>::OpConversionPattern; |
| 845 | |
| 846 | LogicalResult |
| 847 | matchAndRewrite(ArithOp op, typename ArithOp::Adaptor adaptor, |
| 848 | ConversionPatternRewriter &rewriter) const override { |
| 849 | Type srcType = adaptor.getOperands().front().getType(); |
| 850 | if (isBoolScalarOrVector(type: srcType)) |
| 851 | return failure(); |
| 852 | |
| 853 | Type dstType = this->getTypeConverter()->convertType(op.getType()); |
| 854 | if (!dstType) |
| 855 | return getTypeConversionFailure(rewriter, op); |
| 856 | |
| 857 | // Check if the source integer type was widened during type conversion. |
| 858 | unsigned originalBitwidth = |
| 859 | getElementTypeOrSelf(op.getIn().getType()).getIntOrFloatBitWidth(); |
| 860 | |
| 861 | Type srcElemType = getElementTypeOrSelf(type: srcType); |
| 862 | if (!srcElemType.isIntOrFloat()) { |
| 863 | return rewriter.notifyMatchFailure(op, |
| 864 | "unsupported type for uitofp/sitofp" ); |
| 865 | } |
| 866 | |
| 867 | unsigned convertedBitwidth = srcElemType.getIntOrFloatBitWidth(); |
| 868 | |
| 869 | if (originalBitwidth >= convertedBitwidth) { |
| 870 | rewriter.replaceOpWithNewOp<SPIRVOp>(op, dstType, adaptor.getOperands()); |
| 871 | return success(); |
| 872 | } |
| 873 | |
| 874 | // The source was widened. Clean the upper bits before converting. |
| 875 | Location loc = op.getLoc(); |
| 876 | Value cleaned; |
| 877 | if constexpr (IsSigned) { |
| 878 | // Sign-extend by shifting left then arithmetic right. |
| 879 | unsigned shiftAmount = convertedBitwidth - originalBitwidth; |
| 880 | Value shiftSize = |
| 881 | getScalarOrVectorConstInt(type: srcType, value: shiftAmount, builder&: rewriter, loc); |
| 882 | Value shifted = spirv::ShiftLeftLogicalOp::create( |
| 883 | rewriter, loc, srcType, adaptor.getIn(), shiftSize); |
| 884 | cleaned = spirv::ShiftRightArithmeticOp::create(builder&: rewriter, location: loc, result: srcType, |
| 885 | operand1: shifted, operand2: shiftSize); |
| 886 | } else { |
| 887 | // Zero-extend by masking off the upper bits. |
| 888 | Value mask = getScalarOrVectorConstInt( |
| 889 | type: srcType, value: llvm::maskTrailingOnes<uint64_t>(N: originalBitwidth), builder&: rewriter, |
| 890 | loc); |
| 891 | cleaned = spirv::BitwiseAndOp::create(rewriter, loc, srcType, |
| 892 | adaptor.getIn(), mask); |
| 893 | } |
| 894 | rewriter.replaceOpWithNewOp<SPIRVOp>(op, dstType, cleaned); |
| 895 | return success(); |
| 896 | } |
| 897 | }; |
| 898 | |
| 899 | //===----------------------------------------------------------------------===// |
| 900 | // IndexCastOp |
| 901 | //===----------------------------------------------------------------------===// |
| 902 | |
| 903 | /// Converts arith.index_cast to spirv.INotEqual if the target type is i1. |
| 904 | struct IndexCastIndexI1Pattern final |
| 905 | : public OpConversionPattern<arith::IndexCastOp> { |
| 906 | using Base::Base; |
| 907 | |
| 908 | LogicalResult |
| 909 | matchAndRewrite(arith::IndexCastOp op, OpAdaptor adaptor, |
| 910 | ConversionPatternRewriter &rewriter) const override { |
| 911 | if (!isBoolScalarOrVector(type: op.getType())) |
| 912 | return failure(); |
| 913 | |
| 914 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 915 | if (!dstType) |
| 916 | return getTypeConversionFailure(rewriter, op); |
| 917 | |
| 918 | Location loc = op.getLoc(); |
| 919 | Value zeroIdx = |
| 920 | spirv::ConstantOp::getZero(type: adaptor.getIn().getType(), loc, builder&: rewriter); |
| 921 | rewriter.replaceOpWithNewOp<spirv::INotEqualOp>(op, args&: dstType, args&: zeroIdx, |
| 922 | args: adaptor.getIn()); |
| 923 | return success(); |
| 924 | } |
| 925 | }; |
| 926 | |
| 927 | //===----------------------------------------------------------------------===// |
| 928 | // ExtSIOp |
| 929 | //===----------------------------------------------------------------------===// |
| 930 | |
| 931 | /// Converts arith.extsi to spirv.Select if the type of source is i1 or vector |
| 932 | /// of i1. |
| 933 | struct ExtSII1Pattern final : public OpConversionPattern<arith::ExtSIOp> { |
| 934 | using Base::Base; |
| 935 | |
| 936 | LogicalResult |
| 937 | matchAndRewrite(arith::ExtSIOp op, OpAdaptor adaptor, |
| 938 | ConversionPatternRewriter &rewriter) const override { |
| 939 | Value operand = adaptor.getIn(); |
| 940 | if (!isBoolScalarOrVector(type: operand.getType())) |
| 941 | return failure(); |
| 942 | |
| 943 | Location loc = op.getLoc(); |
| 944 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 945 | if (!dstType) |
| 946 | return getTypeConversionFailure(rewriter, op); |
| 947 | |
| 948 | Value allOnes; |
| 949 | if (auto intTy = dyn_cast<IntegerType>(Val&: dstType)) { |
| 950 | unsigned componentBitwidth = intTy.getWidth(); |
| 951 | allOnes = spirv::ConstantOp::create( |
| 952 | builder&: rewriter, location: loc, constant: intTy, |
| 953 | value: rewriter.getIntegerAttr(type: intTy, value: APInt::getAllOnes(numBits: componentBitwidth))); |
| 954 | } else if (auto vectorTy = dyn_cast<VectorType>(Val&: dstType)) { |
| 955 | unsigned componentBitwidth = vectorTy.getElementTypeBitWidth(); |
| 956 | allOnes = spirv::ConstantOp::create( |
| 957 | builder&: rewriter, location: loc, constant: vectorTy, |
| 958 | value: SplatElementsAttr::get(type: vectorTy, |
| 959 | values: APInt::getAllOnes(numBits: componentBitwidth))); |
| 960 | } else { |
| 961 | return rewriter.notifyMatchFailure( |
| 962 | arg&: loc, msg: llvm::formatv(Fmt: "unhandled type: {0}" , Vals&: dstType)); |
| 963 | } |
| 964 | |
| 965 | Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter); |
| 966 | rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: dstType, args&: operand, args&: allOnes, |
| 967 | args&: zero); |
| 968 | return success(); |
| 969 | } |
| 970 | }; |
| 971 | |
| 972 | /// Converts arith.extsi to spirv.Select if the type of source is neither i1 nor |
| 973 | /// vector of i1. |
| 974 | struct ExtSIPattern final : public OpConversionPattern<arith::ExtSIOp> { |
| 975 | using Base::Base; |
| 976 | |
| 977 | LogicalResult |
| 978 | matchAndRewrite(arith::ExtSIOp op, OpAdaptor adaptor, |
| 979 | ConversionPatternRewriter &rewriter) const override { |
| 980 | Type srcType = adaptor.getIn().getType(); |
| 981 | if (isBoolScalarOrVector(type: srcType)) |
| 982 | return failure(); |
| 983 | |
| 984 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 985 | if (!dstType) |
| 986 | return getTypeConversionFailure(rewriter, op); |
| 987 | |
| 988 | if (dstType == srcType) { |
| 989 | // We can have the same source and destination type due to type emulation. |
| 990 | // Perform bit shifting to make sure we have the proper leading set bits. |
| 991 | |
| 992 | unsigned srcBW = |
| 993 | getElementTypeOrSelf(type: op.getIn().getType()).getIntOrFloatBitWidth(); |
| 994 | unsigned dstBW = |
| 995 | getElementTypeOrSelf(type: op.getType()).getIntOrFloatBitWidth(); |
| 996 | assert(srcBW < dstBW); |
| 997 | Value shiftSize = getScalarOrVectorConstInt(type: dstType, value: dstBW - srcBW, |
| 998 | builder&: rewriter, loc: op.getLoc()); |
| 999 | if (!shiftSize) |
| 1000 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported type for shift" ); |
| 1001 | |
| 1002 | // First shift left to sequeeze out all leading bits beyond the original |
| 1003 | // bitwidth. Here we need to use the original source and result type's |
| 1004 | // bitwidth. |
| 1005 | auto shiftLOp = spirv::ShiftLeftLogicalOp::create( |
| 1006 | builder&: rewriter, location: op.getLoc(), result: dstType, operand1: adaptor.getIn(), operand2: shiftSize); |
| 1007 | |
| 1008 | // Then we perform arithmetic right shift to make sure we have the right |
| 1009 | // sign bits for negative values. |
| 1010 | rewriter.replaceOpWithNewOp<spirv::ShiftRightArithmeticOp>( |
| 1011 | op, args&: dstType, args&: shiftLOp, args&: shiftSize); |
| 1012 | } else { |
| 1013 | rewriter.replaceOpWithNewOp<spirv::SConvertOp>(op, args&: dstType, |
| 1014 | args: adaptor.getOperands()); |
| 1015 | } |
| 1016 | |
| 1017 | return success(); |
| 1018 | } |
| 1019 | }; |
| 1020 | |
| 1021 | //===----------------------------------------------------------------------===// |
| 1022 | // ExtUIOp |
| 1023 | //===----------------------------------------------------------------------===// |
| 1024 | |
| 1025 | /// Converts arith.extui for cases where the type of source is neither i1 nor |
| 1026 | /// vector of i1. |
| 1027 | struct ExtUIPattern final : public OpConversionPattern<arith::ExtUIOp> { |
| 1028 | using Base::Base; |
| 1029 | |
| 1030 | LogicalResult |
| 1031 | matchAndRewrite(arith::ExtUIOp op, OpAdaptor adaptor, |
| 1032 | ConversionPatternRewriter &rewriter) const override { |
| 1033 | Type srcType = adaptor.getIn().getType(); |
| 1034 | if (isBoolScalarOrVector(type: srcType)) |
| 1035 | return failure(); |
| 1036 | |
| 1037 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 1038 | if (!dstType) |
| 1039 | return getTypeConversionFailure(rewriter, op); |
| 1040 | |
| 1041 | if (dstType == srcType) { |
| 1042 | // We can have the same source and destination type due to type emulation. |
| 1043 | // Perform bit masking to make sure we don't pollute downstream consumers |
| 1044 | // with unwanted bits. Here we need to use the original source type's |
| 1045 | // bitwidth. |
| 1046 | unsigned bitwidth = |
| 1047 | getElementTypeOrSelf(type: op.getIn().getType()).getIntOrFloatBitWidth(); |
| 1048 | Value mask = getScalarOrVectorConstInt( |
| 1049 | type: dstType, value: llvm::maskTrailingOnes<uint64_t>(N: bitwidth), builder&: rewriter, |
| 1050 | loc: op.getLoc()); |
| 1051 | if (!mask) |
| 1052 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported type for mask" ); |
| 1053 | rewriter.replaceOpWithNewOp<spirv::BitwiseAndOp>(op, args&: dstType, |
| 1054 | args: adaptor.getIn(), args&: mask); |
| 1055 | } else { |
| 1056 | rewriter.replaceOpWithNewOp<spirv::UConvertOp>(op, args&: dstType, |
| 1057 | args: adaptor.getOperands()); |
| 1058 | } |
| 1059 | return success(); |
| 1060 | } |
| 1061 | }; |
| 1062 | |
| 1063 | //===----------------------------------------------------------------------===// |
| 1064 | // TruncIOp |
| 1065 | //===----------------------------------------------------------------------===// |
| 1066 | |
| 1067 | /// Converts arith.trunci to spirv.Select if the type of result is i1 or vector |
| 1068 | /// of i1. |
| 1069 | struct TruncII1Pattern final : public OpConversionPattern<arith::TruncIOp> { |
| 1070 | using Base::Base; |
| 1071 | |
| 1072 | LogicalResult |
| 1073 | matchAndRewrite(arith::TruncIOp op, OpAdaptor adaptor, |
| 1074 | ConversionPatternRewriter &rewriter) const override { |
| 1075 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 1076 | if (!dstType) |
| 1077 | return getTypeConversionFailure(rewriter, op); |
| 1078 | |
| 1079 | if (!isBoolScalarOrVector(type: dstType)) |
| 1080 | return failure(); |
| 1081 | |
| 1082 | Location loc = op.getLoc(); |
| 1083 | auto srcType = adaptor.getOperands().front().getType(); |
| 1084 | // Check if (x & 1) == 1. |
| 1085 | Value mask = spirv::ConstantOp::getOne(type: srcType, loc, builder&: rewriter); |
| 1086 | Value maskedSrc = spirv::BitwiseAndOp::create( |
| 1087 | builder&: rewriter, location: loc, result: srcType, operand1: adaptor.getOperands()[0], operand2: mask); |
| 1088 | Value isOne = spirv::IEqualOp::create(builder&: rewriter, location: loc, operand1: maskedSrc, operand2: mask); |
| 1089 | |
| 1090 | Value zero = spirv::ConstantOp::getZero(type: dstType, loc, builder&: rewriter); |
| 1091 | Value one = spirv::ConstantOp::getOne(type: dstType, loc, builder&: rewriter); |
| 1092 | rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args&: dstType, args&: isOne, args&: one, args&: zero); |
| 1093 | return success(); |
| 1094 | } |
| 1095 | }; |
| 1096 | |
| 1097 | /// Converts arith.trunci for cases where the type of result is neither i1 |
| 1098 | /// nor vector of i1. |
| 1099 | struct TruncIPattern final : public OpConversionPattern<arith::TruncIOp> { |
| 1100 | using Base::Base; |
| 1101 | |
| 1102 | LogicalResult |
| 1103 | matchAndRewrite(arith::TruncIOp op, OpAdaptor adaptor, |
| 1104 | ConversionPatternRewriter &rewriter) const override { |
| 1105 | Type srcType = adaptor.getIn().getType(); |
| 1106 | Type dstType = getTypeConverter()->convertType(t: op.getType()); |
| 1107 | if (!dstType) |
| 1108 | return getTypeConversionFailure(rewriter, op); |
| 1109 | |
| 1110 | if (isBoolScalarOrVector(type: dstType)) |
| 1111 | return failure(); |
| 1112 | |
| 1113 | if (dstType == srcType) { |
| 1114 | // We can have the same source and destination type due to type emulation. |
| 1115 | // Perform bit masking to make sure we don't pollute downstream consumers |
| 1116 | // with unwanted bits. Here we need to use the original result type's |
| 1117 | // bitwidth. |
| 1118 | unsigned bw = getElementTypeOrSelf(type: op.getType()).getIntOrFloatBitWidth(); |
| 1119 | Value mask = getScalarOrVectorConstInt( |
| 1120 | type: dstType, value: llvm::maskTrailingOnes<uint64_t>(N: bw), builder&: rewriter, loc: op.getLoc()); |
| 1121 | if (!mask) |
| 1122 | return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported type for mask" ); |
| 1123 | rewriter.replaceOpWithNewOp<spirv::BitwiseAndOp>(op, args&: dstType, |
| 1124 | args: adaptor.getIn(), args&: mask); |
| 1125 | } else { |
| 1126 | // Given this is truncation, either SConvertOp or UConvertOp works. |
| 1127 | rewriter.replaceOpWithNewOp<spirv::SConvertOp>(op, args&: dstType, |
| 1128 | args: adaptor.getOperands()); |
| 1129 | } |
| 1130 | return success(); |
| 1131 | } |
| 1132 | }; |
| 1133 | |
| 1134 | //===----------------------------------------------------------------------===// |
| 1135 | // TypeCastingOp |
| 1136 | //===----------------------------------------------------------------------===// |
| 1137 | |
| 1138 | static std::optional<spirv::FPRoundingMode> |
| 1139 | convertArithRoundingModeToSPIRV(arith::RoundingMode roundingMode) { |
| 1140 | switch (roundingMode) { |
| 1141 | case arith::RoundingMode::downward: |
| 1142 | return spirv::FPRoundingMode::RTN; |
| 1143 | case arith::RoundingMode::to_nearest_even: |
| 1144 | return spirv::FPRoundingMode::RTE; |
| 1145 | case arith::RoundingMode::toward_zero: |
| 1146 | return spirv::FPRoundingMode::RTZ; |
| 1147 | case arith::RoundingMode::upward: |
| 1148 | return spirv::FPRoundingMode::RTP; |
| 1149 | case arith::RoundingMode::to_nearest_away: |
| 1150 | // SPIR-V FPRoundingMode decoration has no ties-away-from-zero mode |
| 1151 | // (as of SPIR-V 1.6) |
| 1152 | return std::nullopt; |
| 1153 | } |
| 1154 | llvm_unreachable("Unhandled rounding mode" ); |
| 1155 | } |
| 1156 | |
| 1157 | /// Converts type-casting standard operations to SPIR-V operations. |
| 1158 | template <typename Op, typename SPIRVOp> |
| 1159 | struct TypeCastingOpPattern final : public OpConversionPattern<Op> { |
| 1160 | using OpConversionPattern<Op>::OpConversionPattern; |
| 1161 | |
| 1162 | LogicalResult |
| 1163 | matchAndRewrite(Op op, typename Op::Adaptor adaptor, |
| 1164 | ConversionPatternRewriter &rewriter) const override { |
| 1165 | Type srcType = llvm::getSingleElement(adaptor.getOperands()).getType(); |
| 1166 | Type dstType = this->getTypeConverter()->convertType(op.getType()); |
| 1167 | if (!dstType) |
| 1168 | return getTypeConversionFailure(rewriter, op); |
| 1169 | |
| 1170 | if (isBoolScalarOrVector(type: srcType) || isBoolScalarOrVector(type: dstType)) |
| 1171 | return failure(); |
| 1172 | |
| 1173 | if (dstType == srcType) { |
| 1174 | // Due to type conversion, we are seeing the same source and target type. |
| 1175 | // Then we can just erase this operation by forwarding its operand. |
| 1176 | rewriter.replaceOp(op, adaptor.getOperands().front()); |
| 1177 | } else { |
| 1178 | // Compute new rounding mode (if any). |
| 1179 | std::optional<spirv::FPRoundingMode> rm = std::nullopt; |
| 1180 | if (auto roundingModeOp = |
| 1181 | dyn_cast<arith::ArithRoundingModeInterface>(*op)) { |
| 1182 | if (arith::RoundingModeAttr roundingMode = |
| 1183 | roundingModeOp.getRoundingModeAttr()) { |
| 1184 | if (!(rm = |
| 1185 | convertArithRoundingModeToSPIRV(roundingMode: roundingMode.getValue()))) { |
| 1186 | return rewriter.notifyMatchFailure( |
| 1187 | op->getLoc(), |
| 1188 | llvm::formatv(Fmt: "unsupported rounding mode '{0}'" , Vals&: roundingMode)); |
| 1189 | } |
| 1190 | } |
| 1191 | } |
| 1192 | // Create replacement op and attach rounding mode attribute (if any). |
| 1193 | auto newOp = rewriter.template replaceOpWithNewOp<SPIRVOp>( |
| 1194 | op, dstType, adaptor.getOperands()); |
| 1195 | if (rm) { |
| 1196 | newOp->setDiscardableAttr( |
| 1197 | getDecorationString(decoration: spirv::Decoration::FPRoundingMode), |
| 1198 | spirv::FPRoundingModeAttr::get(context: rewriter.getContext(), value: *rm)); |
| 1199 | } |
| 1200 | } |
| 1201 | return success(); |
| 1202 | } |
| 1203 | }; |
| 1204 | |
| 1205 | //===----------------------------------------------------------------------===// |
| 1206 | // CmpIOp |
| 1207 | //===----------------------------------------------------------------------===// |
| 1208 | |
| 1209 | /// Converts integer compare operation on i1 type operands to SPIR-V ops. |
| 1210 | class CmpIOpBooleanPattern final : public OpConversionPattern<arith::CmpIOp> { |
| 1211 | public: |
| 1212 | using Base::Base; |
| 1213 | |
| 1214 | LogicalResult |
| 1215 | matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor, |
| 1216 | ConversionPatternRewriter &rewriter) const override { |
| 1217 | Type srcType = op.getLhs().getType(); |
| 1218 | if (!isBoolScalarOrVector(type: srcType)) |
| 1219 | return failure(); |
| 1220 | Type dstType = getTypeConverter()->convertType(t: srcType); |
| 1221 | if (!dstType) |
| 1222 | return getTypeConversionFailure(rewriter, op, srcType); |
| 1223 | |
| 1224 | switch (op.getPredicate()) { |
| 1225 | case arith::CmpIPredicate::eq: { |
| 1226 | rewriter.replaceOpWithNewOp<spirv::LogicalEqualOp>(op, args: adaptor.getLhs(), |
| 1227 | args: adaptor.getRhs()); |
| 1228 | return success(); |
| 1229 | } |
| 1230 | case arith::CmpIPredicate::ne: { |
| 1231 | rewriter.replaceOpWithNewOp<spirv::LogicalNotEqualOp>( |
| 1232 | op, args: adaptor.getLhs(), args: adaptor.getRhs()); |
| 1233 | return success(); |
| 1234 | } |
| 1235 | case arith::CmpIPredicate::uge: |
| 1236 | case arith::CmpIPredicate::ugt: |
| 1237 | case arith::CmpIPredicate::ule: |
| 1238 | case arith::CmpIPredicate::ult: { |
| 1239 | // There are no direct corresponding instructions in SPIR-V for such |
| 1240 | // cases. Extend them to 32-bit and do comparision then. |
| 1241 | Type type = rewriter.getI32Type(); |
| 1242 | if (auto vectorType = dyn_cast<VectorType>(Val&: dstType)) |
| 1243 | type = VectorType::get(shape: vectorType.getShape(), elementType: type); |
| 1244 | Value extLhs = |
| 1245 | arith::ExtUIOp::create(builder&: rewriter, location: op.getLoc(), out: type, in: adaptor.getLhs()); |
| 1246 | Value extRhs = |
| 1247 | arith::ExtUIOp::create(builder&: rewriter, location: op.getLoc(), out: type, in: adaptor.getRhs()); |
| 1248 | |
| 1249 | rewriter.replaceOpWithNewOp<arith::CmpIOp>(op, args: op.getPredicate(), args&: extLhs, |
| 1250 | args&: extRhs); |
| 1251 | return success(); |
| 1252 | } |
| 1253 | default: |
| 1254 | break; |
| 1255 | } |
| 1256 | return failure(); |
| 1257 | } |
| 1258 | }; |
| 1259 | |
| 1260 | /// Converts integer compare operation to SPIR-V ops. |
| 1261 | class CmpIOpPattern final : public OpConversionPattern<arith::CmpIOp> { |
| 1262 | public: |
| 1263 | using Base::Base; |
| 1264 | |
| 1265 | LogicalResult |
| 1266 | matchAndRewrite(arith::CmpIOp op, OpAdaptor adaptor, |
| 1267 | ConversionPatternRewriter &rewriter) const override { |
| 1268 | Type srcType = op.getLhs().getType(); |
| 1269 | if (isBoolScalarOrVector(type: srcType)) |
| 1270 | return failure(); |
| 1271 | Type dstType = getTypeConverter()->convertType(t: srcType); |
| 1272 | if (!dstType) |
| 1273 | return getTypeConversionFailure(rewriter, op, srcType); |
| 1274 | |
| 1275 | switch (op.getPredicate()) { |
| 1276 | #define DISPATCH(cmpPredicate, spirvOp) \ |
| 1277 | case cmpPredicate: \ |
| 1278 | if (spirvOp::template hasTrait<OpTrait::spirv::UnsignedOp>() && \ |
| 1279 | !getElementTypeOrSelf(srcType).isIndex() && srcType != dstType && \ |
| 1280 | !hasSameBitwidth(srcType, dstType)) { \ |
| 1281 | return op.emitError( \ |
| 1282 | "bitwidth emulation is not implemented yet on unsigned op"); \ |
| 1283 | } \ |
| 1284 | rewriter.replaceOpWithNewOp<spirvOp>(op, adaptor.getLhs(), \ |
| 1285 | adaptor.getRhs()); \ |
| 1286 | return success(); |
| 1287 | |
| 1288 | DISPATCH(arith::CmpIPredicate::eq, spirv::IEqualOp); |
| 1289 | DISPATCH(arith::CmpIPredicate::ne, spirv::INotEqualOp); |
| 1290 | DISPATCH(arith::CmpIPredicate::slt, spirv::SLessThanOp); |
| 1291 | DISPATCH(arith::CmpIPredicate::sle, spirv::SLessThanEqualOp); |
| 1292 | DISPATCH(arith::CmpIPredicate::sgt, spirv::SGreaterThanOp); |
| 1293 | DISPATCH(arith::CmpIPredicate::sge, spirv::SGreaterThanEqualOp); |
| 1294 | DISPATCH(arith::CmpIPredicate::ult, spirv::ULessThanOp); |
| 1295 | DISPATCH(arith::CmpIPredicate::ule, spirv::ULessThanEqualOp); |
| 1296 | DISPATCH(arith::CmpIPredicate::ugt, spirv::UGreaterThanOp); |
| 1297 | DISPATCH(arith::CmpIPredicate::uge, spirv::UGreaterThanEqualOp); |
| 1298 | |
| 1299 | #undef DISPATCH |
| 1300 | } |
| 1301 | return failure(); |
| 1302 | } |
| 1303 | }; |
| 1304 | |
| 1305 | //===----------------------------------------------------------------------===// |
| 1306 | // CmpFOpPattern |
| 1307 | //===----------------------------------------------------------------------===// |
| 1308 | |
| 1309 | /// Converts floating-point comparison operations to SPIR-V ops. |
| 1310 | class CmpFOpPattern final : public OpConversionPattern<arith::CmpFOp> { |
| 1311 | public: |
| 1312 | using Base::Base; |
| 1313 | |
| 1314 | LogicalResult |
| 1315 | matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor, |
| 1316 | ConversionPatternRewriter &rewriter) const override { |
| 1317 | switch (op.getPredicate()) { |
| 1318 | #define DISPATCH(cmpPredicate, spirvOp) \ |
| 1319 | case cmpPredicate: \ |
| 1320 | rewriter.replaceOpWithNewOp<spirvOp>(op, adaptor.getLhs(), \ |
| 1321 | adaptor.getRhs()); \ |
| 1322 | return success(); |
| 1323 | |
| 1324 | // Ordered. |
| 1325 | DISPATCH(arith::CmpFPredicate::OEQ, spirv::FOrdEqualOp); |
| 1326 | DISPATCH(arith::CmpFPredicate::OGT, spirv::FOrdGreaterThanOp); |
| 1327 | DISPATCH(arith::CmpFPredicate::OGE, spirv::FOrdGreaterThanEqualOp); |
| 1328 | DISPATCH(arith::CmpFPredicate::OLT, spirv::FOrdLessThanOp); |
| 1329 | DISPATCH(arith::CmpFPredicate::OLE, spirv::FOrdLessThanEqualOp); |
| 1330 | DISPATCH(arith::CmpFPredicate::ONE, spirv::FOrdNotEqualOp); |
| 1331 | // Unordered. |
| 1332 | DISPATCH(arith::CmpFPredicate::UEQ, spirv::FUnordEqualOp); |
| 1333 | DISPATCH(arith::CmpFPredicate::UGT, spirv::FUnordGreaterThanOp); |
| 1334 | DISPATCH(arith::CmpFPredicate::UGE, spirv::FUnordGreaterThanEqualOp); |
| 1335 | DISPATCH(arith::CmpFPredicate::ULT, spirv::FUnordLessThanOp); |
| 1336 | DISPATCH(arith::CmpFPredicate::ULE, spirv::FUnordLessThanEqualOp); |
| 1337 | DISPATCH(arith::CmpFPredicate::UNE, spirv::FUnordNotEqualOp); |
| 1338 | |
| 1339 | #undef DISPATCH |
| 1340 | |
| 1341 | default: |
| 1342 | break; |
| 1343 | } |
| 1344 | return failure(); |
| 1345 | } |
| 1346 | }; |
| 1347 | |
| 1348 | /// Converts floating point NaN check to SPIR-V ops. This pattern requires |
| 1349 | /// Kernel capability. |
| 1350 | class CmpFOpNanKernelPattern final : public OpConversionPattern<arith::CmpFOp> { |
| 1351 | public: |
| 1352 | using Base::Base; |
| 1353 | |
| 1354 | LogicalResult |
| 1355 | matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor, |
| 1356 | ConversionPatternRewriter &rewriter) const override { |
| 1357 | if (op.getPredicate() == arith::CmpFPredicate::ORD) { |
| 1358 | rewriter.replaceOpWithNewOp<spirv::OrderedOp>(op, args: adaptor.getLhs(), |
| 1359 | args: adaptor.getRhs()); |
| 1360 | return success(); |
| 1361 | } |
| 1362 | |
| 1363 | if (op.getPredicate() == arith::CmpFPredicate::UNO) { |
| 1364 | rewriter.replaceOpWithNewOp<spirv::UnorderedOp>(op, args: adaptor.getLhs(), |
| 1365 | args: adaptor.getRhs()); |
| 1366 | return success(); |
| 1367 | } |
| 1368 | |
| 1369 | return failure(); |
| 1370 | } |
| 1371 | }; |
| 1372 | |
| 1373 | /// Converts floating point NaN check to SPIR-V ops. This pattern does not |
| 1374 | /// require additional capability. |
| 1375 | class CmpFOpNanNonePattern final : public OpConversionPattern<arith::CmpFOp> { |
| 1376 | public: |
| 1377 | using Base::Base; |
| 1378 | |
| 1379 | LogicalResult |
| 1380 | matchAndRewrite(arith::CmpFOp op, OpAdaptor adaptor, |
| 1381 | ConversionPatternRewriter &rewriter) const override { |
| 1382 | if (op.getPredicate() != arith::CmpFPredicate::ORD && |
| 1383 | op.getPredicate() != arith::CmpFPredicate::UNO) |
| 1384 | return failure(); |
| 1385 | |
| 1386 | Location loc = op.getLoc(); |
| 1387 | |
| 1388 | Value replace; |
| 1389 | if (bitEnumContainsAll(bits: op.getFastmath(), bit: arith::FastMathFlags::nnan)) { |
| 1390 | if (op.getPredicate() == arith::CmpFPredicate::ORD) { |
| 1391 | // Ordered comparsion checks if neither operand is NaN. |
| 1392 | replace = spirv::ConstantOp::getOne(type: op.getType(), loc, builder&: rewriter); |
| 1393 | } else { |
| 1394 | // Unordered comparsion checks if either operand is NaN. |
| 1395 | replace = spirv::ConstantOp::getZero(type: op.getType(), loc, builder&: rewriter); |
| 1396 | } |
| 1397 | } else { |
| 1398 | Value lhsIsNan = spirv::IsNanOp::create(builder&: rewriter, location: loc, operand: adaptor.getLhs()); |
| 1399 | Value rhsIsNan = spirv::IsNanOp::create(builder&: rewriter, location: loc, operand: adaptor.getRhs()); |
| 1400 | |
| 1401 | replace = spirv::LogicalOrOp::create(builder&: rewriter, location: loc, operand1: lhsIsNan, operand2: rhsIsNan); |
| 1402 | if (op.getPredicate() == arith::CmpFPredicate::ORD) |
| 1403 | replace = spirv::LogicalNotOp::create(builder&: rewriter, location: loc, operand: replace); |
| 1404 | } |
| 1405 | |
| 1406 | rewriter.replaceOp(op, newValues: replace); |
| 1407 | return success(); |
| 1408 | } |
| 1409 | }; |
| 1410 | |
| 1411 | //===----------------------------------------------------------------------===// |
| 1412 | // AddUIExtendedOp/SubUIExtendedOp |
| 1413 | //===----------------------------------------------------------------------===// |
| 1414 | |
| 1415 | /// Converts arith.addui_extended/arith.subui_extended to spirv.IAddCarry/ |
| 1416 | /// spirv.ISubBorrow. |
| 1417 | template <typename ArithExtendedOp, typename SPIRVExtendedOp> |
| 1418 | class BinaryExtendedOpPattern final |
| 1419 | : public OpConversionPattern<ArithExtendedOp> { |
| 1420 | public: |
| 1421 | using OpConversionPattern<ArithExtendedOp>::OpConversionPattern; |
| 1422 | LogicalResult |
| 1423 | matchAndRewrite(ArithExtendedOp op, typename ArithExtendedOp::Adaptor adaptor, |
| 1424 | ConversionPatternRewriter &rewriter) const override { |
| 1425 | Type dstElemTy = adaptor.getLhs().getType(); |
| 1426 | Location loc = op->getLoc(); |
| 1427 | Value result = SPIRVExtendedOp::create(rewriter, loc, adaptor.getLhs(), |
| 1428 | adaptor.getRhs()); |
| 1429 | |
| 1430 | Value valueResult = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: result, |
| 1431 | indices: llvm::ArrayRef(0)); |
| 1432 | Value flagValue = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: result, |
| 1433 | indices: llvm::ArrayRef(1)); |
| 1434 | |
| 1435 | // Convert the carry/borrow value to boolean. |
| 1436 | Value one = spirv::ConstantOp::getOne(type: dstElemTy, loc, builder&: rewriter); |
| 1437 | Value flagResult = spirv::IEqualOp::create(builder&: rewriter, location: loc, operand1: flagValue, operand2: one); |
| 1438 | |
| 1439 | rewriter.replaceOp(op, {valueResult, flagResult}); |
| 1440 | return success(); |
| 1441 | } |
| 1442 | }; |
| 1443 | |
| 1444 | //===----------------------------------------------------------------------===// |
| 1445 | // MulIExtendedOp |
| 1446 | //===----------------------------------------------------------------------===// |
| 1447 | |
| 1448 | /// Converts arith.mul*i_extended to spirv.*MulExtended. |
| 1449 | template <typename ArithMulOp, typename SPIRVMulOp> |
| 1450 | class MulIExtendedOpPattern final : public OpConversionPattern<ArithMulOp> { |
| 1451 | public: |
| 1452 | using OpConversionPattern<ArithMulOp>::OpConversionPattern; |
| 1453 | LogicalResult |
| 1454 | matchAndRewrite(ArithMulOp op, typename ArithMulOp::Adaptor adaptor, |
| 1455 | ConversionPatternRewriter &rewriter) const override { |
| 1456 | Location loc = op->getLoc(); |
| 1457 | Value result = |
| 1458 | SPIRVMulOp::create(rewriter, loc, adaptor.getLhs(), adaptor.getRhs()); |
| 1459 | |
| 1460 | Value low = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: result, |
| 1461 | indices: llvm::ArrayRef(0)); |
| 1462 | Value high = spirv::CompositeExtractOp::create(builder&: rewriter, location: loc, composite: result, |
| 1463 | indices: llvm::ArrayRef(1)); |
| 1464 | |
| 1465 | rewriter.replaceOp(op, {low, high}); |
| 1466 | return success(); |
| 1467 | } |
| 1468 | }; |
| 1469 | |
| 1470 | //===----------------------------------------------------------------------===// |
| 1471 | // SelectOp |
| 1472 | //===----------------------------------------------------------------------===// |
| 1473 | |
| 1474 | /// Converts arith.select to spirv.Select. |
| 1475 | class SelectOpPattern final : public OpConversionPattern<arith::SelectOp> { |
| 1476 | public: |
| 1477 | using Base::Base; |
| 1478 | LogicalResult |
| 1479 | matchAndRewrite(arith::SelectOp op, OpAdaptor adaptor, |
| 1480 | ConversionPatternRewriter &rewriter) const override { |
| 1481 | rewriter.replaceOpWithNewOp<spirv::SelectOp>(op, args: adaptor.getCondition(), |
| 1482 | args: adaptor.getTrueValue(), |
| 1483 | args: adaptor.getFalseValue()); |
| 1484 | return success(); |
| 1485 | } |
| 1486 | }; |
| 1487 | |
| 1488 | //===----------------------------------------------------------------------===// |
| 1489 | // MinimumFOp, MaximumFOp |
| 1490 | //===----------------------------------------------------------------------===// |
| 1491 | |
| 1492 | /// Converts arith.maximumf/minimumf to spirv.GL.FMax/FMin or |
| 1493 | /// spirv.CL.fmax/fmin. |
| 1494 | template <typename Op, typename SPIRVOp> |
| 1495 | class MinimumMaximumFOpPattern final : public OpConversionPattern<Op> { |
| 1496 | public: |
| 1497 | using OpConversionPattern<Op>::OpConversionPattern; |
| 1498 | LogicalResult |
| 1499 | matchAndRewrite(Op op, typename Op::Adaptor adaptor, |
| 1500 | ConversionPatternRewriter &rewriter) const override { |
| 1501 | auto *converter = this->template getTypeConverter<SPIRVTypeConverter>(); |
| 1502 | Type dstType = converter->convertType(op.getType()); |
| 1503 | if (!dstType) |
| 1504 | return getTypeConversionFailure(rewriter, op); |
| 1505 | |
| 1506 | // arith.maximumf/minimumf: |
| 1507 | // "if one of the arguments is NaN, then the result is also NaN." |
| 1508 | // spirv.GL.FMax/FMin |
| 1509 | // "which operand is the result is undefined if one of the operands |
| 1510 | // is a NaN." |
| 1511 | // spirv.CL.fmax/fmin: |
| 1512 | // "If one argument is a NaN, Fmin returns the other argument." |
| 1513 | |
| 1514 | Location loc = op.getLoc(); |
| 1515 | Value spirvOp = |
| 1516 | SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands()); |
| 1517 | |
| 1518 | if (bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) { |
| 1519 | rewriter.replaceOp(op, spirvOp); |
| 1520 | return success(); |
| 1521 | } |
| 1522 | |
| 1523 | Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs()); |
| 1524 | Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs()); |
| 1525 | |
| 1526 | Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan, |
| 1527 | adaptor.getLhs(), spirvOp); |
| 1528 | Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan, |
| 1529 | adaptor.getRhs(), select1); |
| 1530 | |
| 1531 | rewriter.replaceOp(op, select2); |
| 1532 | return success(); |
| 1533 | } |
| 1534 | }; |
| 1535 | |
| 1536 | //===----------------------------------------------------------------------===// |
| 1537 | // MinNumFOp, MaxNumFOp |
| 1538 | //===----------------------------------------------------------------------===// |
| 1539 | |
| 1540 | /// Converts arith.maxnumf/minnumf to spirv.GL.NMax/NMin or |
| 1541 | /// spirv.CL.fmax/fmin. |
| 1542 | template <typename Op, typename SPIRVOp> |
| 1543 | class MinNumMaxNumFOpPattern final : public OpConversionPattern<Op> { |
| 1544 | template <typename TargetOp> |
| 1545 | constexpr bool shouldInsertNanGuards() const { |
| 1546 | return llvm::is_one_of<TargetOp, spirv::GLFMaxOp, spirv::GLFMinOp>::value; |
| 1547 | } |
| 1548 | |
| 1549 | public: |
| 1550 | using OpConversionPattern<Op>::OpConversionPattern; |
| 1551 | LogicalResult |
| 1552 | matchAndRewrite(Op op, typename Op::Adaptor adaptor, |
| 1553 | ConversionPatternRewriter &rewriter) const override { |
| 1554 | auto *converter = this->template getTypeConverter<SPIRVTypeConverter>(); |
| 1555 | Type dstType = converter->convertType(op.getType()); |
| 1556 | if (!dstType) |
| 1557 | return getTypeConversionFailure(rewriter, op); |
| 1558 | |
| 1559 | // arith.maxnumf/minnumf: |
| 1560 | // "If one of the arguments is NaN, then the result is the other |
| 1561 | // argument." |
| 1562 | // spirv.GL.NMax/NMin: NaN is treated as missing, matches arith semantics. |
| 1563 | // spirv.CL.fmax/fmin: |
| 1564 | // "If one argument is a NaN, Fmin returns the other argument." |
| 1565 | // spirv.GL.FMax/FMin: undefined when either operand is NaN, requires |
| 1566 | // select guards to implement arith.maxnumf semantics. |
| 1567 | |
| 1568 | Location loc = op.getLoc(); |
| 1569 | Value spirvOp = |
| 1570 | SPIRVOp::create(rewriter, loc, dstType, adaptor.getOperands()); |
| 1571 | |
| 1572 | if (!shouldInsertNanGuards<SPIRVOp>() || |
| 1573 | bitEnumContainsAll(op.getFastmath(), arith::FastMathFlags::nnan)) { |
| 1574 | rewriter.replaceOp(op, spirvOp); |
| 1575 | return success(); |
| 1576 | } |
| 1577 | |
| 1578 | Value lhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getLhs()); |
| 1579 | Value rhsIsNan = spirv::IsNanOp::create(rewriter, loc, adaptor.getRhs()); |
| 1580 | |
| 1581 | Value select1 = spirv::SelectOp::create(rewriter, loc, dstType, lhsIsNan, |
| 1582 | adaptor.getRhs(), spirvOp); |
| 1583 | Value select2 = spirv::SelectOp::create(rewriter, loc, dstType, rhsIsNan, |
| 1584 | adaptor.getLhs(), select1); |
| 1585 | |
| 1586 | rewriter.replaceOp(op, select2); |
| 1587 | return success(); |
| 1588 | } |
| 1589 | }; |
| 1590 | |
| 1591 | } // namespace |
| 1592 | |
| 1593 | //===----------------------------------------------------------------------===// |
| 1594 | // Pattern Population |
| 1595 | //===----------------------------------------------------------------------===// |
| 1596 | |
| 1597 | void mlir::arith::populateArithToSPIRVPatterns( |
| 1598 | const SPIRVTypeConverter &typeConverter, RewritePatternSet &patterns) { |
| 1599 | // clang-format off |
| 1600 | patterns.add< |
| 1601 | ConstantCompositeOpPattern, |
| 1602 | ConstantScalarOpPattern, |
| 1603 | BoolIOpPattern<arith::AddIOp, spirv::LogicalNotEqualOp>, // add mod 2 = XOR = not-equal |
| 1604 | ElementwiseArithOpPattern<arith::AddIOp, spirv::IAddOp>, |
| 1605 | BoolIOpPattern<arith::SubIOp, spirv::LogicalNotEqualOp>, // sub mod 2 = XOR = not-equal |
| 1606 | ElementwiseArithOpPattern<arith::SubIOp, spirv::ISubOp>, |
| 1607 | BoolIOpPattern<arith::MulIOp, spirv::LogicalAndOp>, // 1*1=1, else 0 = AND |
| 1608 | ElementwiseArithOpPattern<arith::MulIOp, spirv::IMulOp>, |
| 1609 | BoolIOpPattern<arith::DivUIOp, spirv::LogicalAndOp>, // a/1=a, a/0=UB; truth table = AND |
| 1610 | spirv::ElementwiseOpPattern<arith::DivUIOp, spirv::UDivOp>, |
| 1611 | BoolIOpPattern<arith::DivSIOp, spirv::LogicalAndOp>, // same as divui on i1 |
| 1612 | spirv::ElementwiseOpPattern<arith::DivSIOp, spirv::SDivOp>, |
| 1613 | BoolIOpPattern<arith::CeilDivUIOp, spirv::LogicalAndOp>, // same as divui on i1 |
| 1614 | CeilDivUIOpPattern, |
| 1615 | BoolIOpPattern<arith::CeilDivSIOp, spirv::LogicalAndOp>, // same as divsi on i1 |
| 1616 | CeilDivSIOpPattern, |
| 1617 | BoolIOpPattern<arith::FloorDivSIOp, spirv::LogicalAndOp>, // same as divsi on i1 |
| 1618 | FloorDivSIOpPattern, |
| 1619 | BoolIOpAndNotPattern<arith::RemUIOp>, // remui(a,b) = a & ~b (see pattern comment) |
| 1620 | spirv::ElementwiseOpPattern<arith::RemUIOp, spirv::UModOp>, |
| 1621 | BoolIOpAndNotPattern<arith::RemSIOp>, // remsi(a,b) = a & ~b (see pattern comment) |
| 1622 | RemSIOpGLPattern, RemSIOpCLPattern, |
| 1623 | BitwiseOpPattern<arith::AndIOp, spirv::LogicalAndOp, spirv::BitwiseAndOp>, |
| 1624 | BitwiseOpPattern<arith::OrIOp, spirv::LogicalOrOp, spirv::BitwiseOrOp>, |
| 1625 | XOrIOpLogicalPattern, XOrIOpBooleanPattern, |
| 1626 | BoolIOpAndNotPattern<arith::ShLIOp>, // shli(a,b) = a & ~b (see pattern comment) |
| 1627 | ElementwiseArithOpPattern<arith::ShLIOp, spirv::ShiftLeftLogicalOp>, |
| 1628 | BoolIOpAndNotPattern<arith::ShRUIOp>, // shrui(a,b) = a & ~b (see pattern comment) |
| 1629 | spirv::ElementwiseOpPattern<arith::ShRUIOp, spirv::ShiftRightLogicalOp>, |
| 1630 | ShRSIBoolPattern, // shrsi(a,b) = a (identity; see pattern comment) |
| 1631 | spirv::ElementwiseOpPattern<arith::ShRSIOp, spirv::ShiftRightArithmeticOp>, |
| 1632 | spirv::ElementwiseOpPattern<arith::NegFOp, spirv::FNegateOp>, |
| 1633 | spirv::ElementwiseOpPattern<arith::AddFOp, spirv::FAddOp>, |
| 1634 | spirv::ElementwiseOpPattern<arith::SubFOp, spirv::FSubOp>, |
| 1635 | spirv::ElementwiseOpPattern<arith::MulFOp, spirv::FMulOp>, |
| 1636 | spirv::ElementwiseOpPattern<arith::DivFOp, spirv::FDivOp>, |
| 1637 | spirv::ElementwiseOpPattern<arith::RemFOp, spirv::FRemOp>, |
| 1638 | ExtUIPattern, BoolToValuePattern<arith::ExtUIOp>, |
| 1639 | ExtSIPattern, ExtSII1Pattern, |
| 1640 | TypeCastingOpPattern<arith::ExtFOp, spirv::FConvertOp>, |
| 1641 | TruncIPattern, TruncII1Pattern, |
| 1642 | TypeCastingOpPattern<arith::TruncFOp, spirv::FConvertOp>, |
| 1643 | TypeCastingOpPattern<arith::ConvertFOp, spirv::FConvertOp>, |
| 1644 | IntToFPPattern<arith::UIToFPOp, spirv::ConvertUToFOp, false>, |
| 1645 | BoolToValuePattern<arith::UIToFPOp>, |
| 1646 | IntToFPPattern<arith::SIToFPOp, spirv::ConvertSToFOp, true>, |
| 1647 | TypeCastingOpPattern<arith::FPToUIOp, spirv::ConvertFToUOp>, |
| 1648 | TypeCastingOpPattern<arith::FPToSIOp, spirv::ConvertFToSOp>, |
| 1649 | TypeCastingOpPattern<arith::IndexCastOp, spirv::SConvertOp>, |
| 1650 | IndexCastIndexI1Pattern, BoolToValuePattern<arith::IndexCastOp>, |
| 1651 | TypeCastingOpPattern<arith::IndexCastUIOp, spirv::UConvertOp>, |
| 1652 | TypeCastingOpPattern<arith::BitcastOp, spirv::BitcastOp>, |
| 1653 | CmpIOpBooleanPattern, CmpIOpPattern, |
| 1654 | CmpFOpNanNonePattern, CmpFOpPattern, |
| 1655 | BinaryExtendedOpPattern<arith::AddUIExtendedOp, spirv::IAddCarryOp>, |
| 1656 | BinaryExtendedOpPattern<arith::SubUIExtendedOp, spirv::ISubBorrowOp>, |
| 1657 | MulIExtendedOpPattern<arith::MulSIExtendedOp, spirv::SMulExtendedOp>, |
| 1658 | MulIExtendedOpPattern<arith::MulUIExtendedOp, spirv::UMulExtendedOp>, |
| 1659 | SelectOpPattern, |
| 1660 | |
| 1661 | MinimumMaximumFOpPattern<arith::MaximumFOp, spirv::GLFMaxOp>, |
| 1662 | MinimumMaximumFOpPattern<arith::MinimumFOp, spirv::GLFMinOp>, |
| 1663 | MinNumMaxNumFOpPattern<arith::MaxNumFOp, spirv::GLNMaxOp>, |
| 1664 | MinNumMaxNumFOpPattern<arith::MinNumFOp, spirv::GLNMinOp>, |
| 1665 | BoolIOpPattern<arith::MaxSIOp, spirv::LogicalAndOp>, // signed i1: 1=-1, so max=0 unless both are 1 |
| 1666 | BoolIOpPattern<arith::MaxUIOp, spirv::LogicalOrOp>, // unsigned max on i1: 1 when either is 1 |
| 1667 | BoolIOpPattern<arith::MinSIOp, spirv::LogicalOrOp>, // signed i1: -1<0, so min=1 when either is 1 |
| 1668 | BoolIOpPattern<arith::MinUIOp, spirv::LogicalAndOp>, // unsigned min on i1: 1 only when both are 1 |
| 1669 | spirv::ElementwiseOpPattern<arith::MaxSIOp, spirv::GLSMaxOp>, |
| 1670 | spirv::ElementwiseOpPattern<arith::MaxUIOp, spirv::GLUMaxOp>, |
| 1671 | spirv::ElementwiseOpPattern<arith::MinSIOp, spirv::GLSMinOp>, |
| 1672 | spirv::ElementwiseOpPattern<arith::MinUIOp, spirv::GLUMinOp>, |
| 1673 | |
| 1674 | MinimumMaximumFOpPattern<arith::MaximumFOp, spirv::CLFMaxOp>, |
| 1675 | MinimumMaximumFOpPattern<arith::MinimumFOp, spirv::CLFMinOp>, |
| 1676 | MinNumMaxNumFOpPattern<arith::MaxNumFOp, spirv::CLFMaxOp>, |
| 1677 | MinNumMaxNumFOpPattern<arith::MinNumFOp, spirv::CLFMinOp>, |
| 1678 | spirv::ElementwiseOpPattern<arith::MaxSIOp, spirv::CLSMaxOp>, |
| 1679 | spirv::ElementwiseOpPattern<arith::MaxUIOp, spirv::CLUMaxOp>, |
| 1680 | spirv::ElementwiseOpPattern<arith::MinSIOp, spirv::CLSMinOp>, |
| 1681 | spirv::ElementwiseOpPattern<arith::MinUIOp, spirv::CLUMinOp> |
| 1682 | >(arg: typeConverter, args: patterns.getContext()); |
| 1683 | // clang-format on |
| 1684 | |
| 1685 | // Give CmpFOpNanKernelPattern a higher benefit so it can prevail when Kernel |
| 1686 | // capability is available. |
| 1687 | patterns.add<CmpFOpNanKernelPattern>(arg: typeConverter, args: patterns.getContext(), |
| 1688 | /*benefit=*/args: 2); |
| 1689 | } |
| 1690 | |
| 1691 | //===----------------------------------------------------------------------===// |
| 1692 | // Pass Definition |
| 1693 | //===----------------------------------------------------------------------===// |
| 1694 | |
| 1695 | namespace { |
| 1696 | struct ConvertArithToSPIRVPass |
| 1697 | : public impl::ConvertArithToSPIRVPassBase<ConvertArithToSPIRVPass> { |
| 1698 | using Base::Base; |
| 1699 | |
| 1700 | void runOnOperation() override { |
| 1701 | Operation *op = getOperation(); |
| 1702 | spirv::TargetEnvAttr targetAttr = spirv::lookupTargetEnvOrDefault(op); |
| 1703 | std::unique_ptr<SPIRVConversionTarget> target = |
| 1704 | SPIRVConversionTarget::get(targetAttr); |
| 1705 | |
| 1706 | SPIRVConversionOptions options; |
| 1707 | options.emulateLT32BitScalarTypes = this->emulateLT32BitScalarTypes; |
| 1708 | options.emulateUnsupportedFloatTypes = this->emulateUnsupportedFloatTypes; |
| 1709 | SPIRVTypeConverter typeConverter(targetAttr, options); |
| 1710 | |
| 1711 | // Use UnrealizedConversionCast as the bridge so that we don't need to pull |
| 1712 | // in patterns for other dialects. |
| 1713 | target->addLegalOp<UnrealizedConversionCastOp>(); |
| 1714 | |
| 1715 | // Fail hard when there are any remaining 'arith' ops. |
| 1716 | target->addIllegalDialect<arith::ArithDialect>(); |
| 1717 | |
| 1718 | RewritePatternSet patterns(&getContext()); |
| 1719 | arith::populateArithToSPIRVPatterns(typeConverter, patterns); |
| 1720 | |
| 1721 | if (failed(Result: applyPartialConversion(op, target: *target, patterns: std::move(patterns)))) |
| 1722 | signalPassFailure(); |
| 1723 | } |
| 1724 | }; |
| 1725 | } // namespace |
| 1726 | |