| 1 | //===- DotProductOps.cpp - MLIR SPIR-V Dot Product Ops -------------------===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // Defines the Dot Product operations in the SPIR-V dialect. |
| 10 | // |
| 11 | //===----------------------------------------------------------------------===// |
| 12 | |
| 13 | #include "mlir/Dialect/SPIRV/IR/SPIRVOps.h" |
| 14 | |
| 15 | #include "SPIRVOpUtils.h" |
| 16 | #include "SPIRVParsingUtils.h" |
| 17 | |
| 18 | #include "llvm/Support/FormatVariadic.h" |
| 19 | |
| 20 | using namespace mlir::spirv::AttrNames; |
| 21 | |
| 22 | namespace mlir::spirv { |
| 23 | |
| 24 | //===----------------------------------------------------------------------===// |
| 25 | // Dot Product ops |
| 26 | //===----------------------------------------------------------------------===// |
| 27 | |
| 28 | static std::optional<spirv::Version> getDotProductMinVersion() { |
| 29 | return spirv::Version::V_1_0; // Available in SPIR-V >= 1.0. |
| 30 | } |
| 31 | |
| 32 | static std::optional<spirv::Version> getDotProductMaxVersion() { |
| 33 | return spirv::Version::V_1_6; // Available in SPIR-V <= 1.6. |
| 34 | } |
| 35 | |
| 36 | SmallVector<ArrayRef<spirv::Extension>, 1> DotOp::getExtensions() { |
| 37 | if (isa<BFloat16Type>(Val: getType())) { |
| 38 | static const auto extension = spirv::Extension::SPV_KHR_bfloat16; |
| 39 | return {extension}; |
| 40 | } |
| 41 | |
| 42 | return {}; |
| 43 | } |
| 44 | |
| 45 | SmallVector<ArrayRef<spirv::Capability>, 1> DotOp::getCapabilities() { |
| 46 | if (isa<BFloat16Type>(Val: getType())) { |
| 47 | static const auto capability = spirv::Capability::BFloat16DotProductKHR; |
| 48 | return {capability}; |
| 49 | } |
| 50 | |
| 51 | return {}; |
| 52 | } |
| 53 | |
| 54 | std::optional<spirv::Version> DotOp::getMinVersion() { |
| 55 | return getDotProductMinVersion(); |
| 56 | } |
| 57 | |
| 58 | std::optional<spirv::Version> DotOp::getMaxVersion() { |
| 59 | return getDotProductMaxVersion(); |
| 60 | } |
| 61 | |
| 62 | //===----------------------------------------------------------------------===// |
| 63 | // Integer Dot Product ops |
| 64 | //===----------------------------------------------------------------------===// |
| 65 | |
| 66 | template <typename IntegerDotProductOpTy> |
| 67 | static LogicalResult verifyIntegerDotProduct(Operation *op) { |
| 68 | assert(llvm::is_contained({2u, 3u}, op->getNumOperands()) && |
| 69 | "Not an integer dot product op?" ); |
| 70 | assert(op->getNumResults() == 1 && "Expected a single result" ); |
| 71 | |
| 72 | // ODS enforces that vector 1 and vector 2, and result and the accumulator |
| 73 | // have the same types. |
| 74 | Type factorTy = op->getOperand(idx: 0).getType(); |
| 75 | auto dotOp = cast<IntegerDotProductOpTy>(op); |
| 76 | if (auto intTy = dyn_cast<IntegerType>(Val&: factorTy)) { |
| 77 | spirv::PackedVectorFormatAttr packedVectorFormat = dotOp.getFormatAttr(); |
| 78 | if (!packedVectorFormat) |
| 79 | return op->emitOpError(message: "requires Packed Vector Format attribute for " |
| 80 | "integer vector operands" ); |
| 81 | |
| 82 | assert(packedVectorFormat.getValue() == |
| 83 | spirv::PackedVectorFormat::PackedVectorFormat4x8Bit && |
| 84 | "Unknown Packed Vector Format" ); |
| 85 | if (intTy.getWidth() != 32) |
| 86 | return op->emitOpError( |
| 87 | message: llvm::formatv(Fmt: "with specified Packed Vector Format ({0}) requires " |
| 88 | "integer vector operands to be 32-bits wide" , |
| 89 | Vals: packedVectorFormat.getValue())); |
| 90 | } else { |
| 91 | if (dotOp.getFormatAttr()) |
| 92 | return op->emitOpError(message: llvm::formatv( |
| 93 | Fmt: "with invalid format attribute for vector operands of type '{0}'" , |
| 94 | Vals&: factorTy)); |
| 95 | } |
| 96 | |
| 97 | Type resultTy = op->getResultTypes().front(); |
| 98 | unsigned factorBitWidth = getBitWidth(type: factorTy); |
| 99 | unsigned resultBitWidth = getBitWidth(type: resultTy); |
| 100 | if (factorBitWidth > resultBitWidth) |
| 101 | return op->emitOpError( |
| 102 | message: llvm::formatv(Fmt: "result type has insufficient bit-width ({0} bits) " |
| 103 | "for the specified vector operand type ({1} bits)" , |
| 104 | Vals&: resultBitWidth, Vals&: factorBitWidth)); |
| 105 | |
| 106 | return success(); |
| 107 | } |
| 108 | |
| 109 | static SmallVector<ArrayRef<spirv::Extension>, 1> |
| 110 | getIntegerDotProductExtensions() { |
| 111 | // Requires the SPV_KHR_integer_dot_product extension, specified either |
| 112 | // explicitly or implied by target env's SPIR-V version >= 1.6. |
| 113 | static const auto extension = spirv::Extension::SPV_KHR_integer_dot_product; |
| 114 | return {extension}; |
| 115 | } |
| 116 | |
| 117 | template <typename IntegerDotProductOpTy> |
| 118 | static SmallVector<ArrayRef<spirv::Capability>, 1> |
| 119 | getIntegerDotProductCapabilities(Operation *op) { |
| 120 | // Requires the the DotProduct capability and capabilities that depend on |
| 121 | // exact op types. |
| 122 | static const auto dotProductCap = spirv::Capability::DotProduct; |
| 123 | static const auto dotProductInput4x8BitPackedCap = |
| 124 | spirv::Capability::DotProductInput4x8BitPacked; |
| 125 | static const auto dotProductInput4x8BitCap = |
| 126 | spirv::Capability::DotProductInput4x8Bit; |
| 127 | static const auto dotProductInputAllCap = |
| 128 | spirv::Capability::DotProductInputAll; |
| 129 | |
| 130 | SmallVector<ArrayRef<spirv::Capability>, 1> capabilities = {dotProductCap}; |
| 131 | |
| 132 | Type factorTy = op->getOperand(idx: 0).getType(); |
| 133 | auto dotOp = cast<IntegerDotProductOpTy>(op); |
| 134 | if (auto intTy = dyn_cast<IntegerType>(Val&: factorTy)) { |
| 135 | spirv::PackedVectorFormatAttr formatAttr = dotOp.getFormatAttr(); |
| 136 | if (formatAttr.getValue() == |
| 137 | spirv::PackedVectorFormat::PackedVectorFormat4x8Bit) |
| 138 | capabilities.push_back(Elt: dotProductInput4x8BitPackedCap); |
| 139 | |
| 140 | return capabilities; |
| 141 | } |
| 142 | |
| 143 | auto vecTy = cast<VectorType>(Val&: factorTy); |
| 144 | if (vecTy.getElementTypeBitWidth() == 8 && vecTy.getNumElements() == 4) { |
| 145 | capabilities.push_back(Elt: dotProductInput4x8BitCap); |
| 146 | return capabilities; |
| 147 | } |
| 148 | |
| 149 | capabilities.push_back(Elt: dotProductInputAllCap); |
| 150 | return capabilities; |
| 151 | } |
| 152 | |
| 153 | #define SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(OpName) \ |
| 154 | LogicalResult OpName::verify() { \ |
| 155 | return verifyIntegerDotProduct<OpName>(*this); \ |
| 156 | } \ |
| 157 | SmallVector<ArrayRef<spirv::Extension>, 1> OpName::getExtensions() { \ |
| 158 | return getIntegerDotProductExtensions(); \ |
| 159 | } \ |
| 160 | SmallVector<ArrayRef<spirv::Capability>, 1> OpName::getCapabilities() { \ |
| 161 | return getIntegerDotProductCapabilities<OpName>(*this); \ |
| 162 | } \ |
| 163 | std::optional<spirv::Version> OpName::getMinVersion() { \ |
| 164 | return getDotProductMinVersion(); \ |
| 165 | } \ |
| 166 | std::optional<spirv::Version> OpName::getMaxVersion() { \ |
| 167 | return getDotProductMaxVersion(); \ |
| 168 | } |
| 169 | |
| 170 | SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(SDotOp) |
| 171 | SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(SUDotOp) |
| 172 | SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(UDotOp) |
| 173 | SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(SDotAccSatOp) |
| 174 | SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(SUDotAccSatOp) |
| 175 | SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(UDotAccSatOp) |
| 176 | |
| 177 | #undef SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP |
| 178 | |
| 179 | } // namespace mlir::spirv |
| 180 | |