1//===- DotProductOps.cpp - MLIR SPIR-V Dot Product Ops -------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Defines the Dot Product operations in the SPIR-V dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
14
15#include "SPIRVOpUtils.h"
16#include "SPIRVParsingUtils.h"
17
18#include "llvm/Support/FormatVariadic.h"
19
20using namespace mlir::spirv::AttrNames;
21
22namespace mlir::spirv {
23
24//===----------------------------------------------------------------------===//
25// Dot Product ops
26//===----------------------------------------------------------------------===//
27
28static std::optional<spirv::Version> getDotProductMinVersion() {
29 return spirv::Version::V_1_0; // Available in SPIR-V >= 1.0.
30}
31
32static std::optional<spirv::Version> getDotProductMaxVersion() {
33 return spirv::Version::V_1_6; // Available in SPIR-V <= 1.6.
34}
35
36SmallVector<ArrayRef<spirv::Extension>, 1> DotOp::getExtensions() {
37 if (isa<BFloat16Type>(Val: getType())) {
38 static const auto extension = spirv::Extension::SPV_KHR_bfloat16;
39 return {extension};
40 }
41
42 return {};
43}
44
45SmallVector<ArrayRef<spirv::Capability>, 1> DotOp::getCapabilities() {
46 if (isa<BFloat16Type>(Val: getType())) {
47 static const auto capability = spirv::Capability::BFloat16DotProductKHR;
48 return {capability};
49 }
50
51 return {};
52}
53
54std::optional<spirv::Version> DotOp::getMinVersion() {
55 return getDotProductMinVersion();
56}
57
58std::optional<spirv::Version> DotOp::getMaxVersion() {
59 return getDotProductMaxVersion();
60}
61
62//===----------------------------------------------------------------------===//
63// Integer Dot Product ops
64//===----------------------------------------------------------------------===//
65
66template <typename IntegerDotProductOpTy>
67static LogicalResult verifyIntegerDotProduct(Operation *op) {
68 assert(llvm::is_contained({2u, 3u}, op->getNumOperands()) &&
69 "Not an integer dot product op?");
70 assert(op->getNumResults() == 1 && "Expected a single result");
71
72 // ODS enforces that vector 1 and vector 2, and result and the accumulator
73 // have the same types.
74 Type factorTy = op->getOperand(idx: 0).getType();
75 auto dotOp = cast<IntegerDotProductOpTy>(op);
76 if (auto intTy = dyn_cast<IntegerType>(Val&: factorTy)) {
77 spirv::PackedVectorFormatAttr packedVectorFormat = dotOp.getFormatAttr();
78 if (!packedVectorFormat)
79 return op->emitOpError(message: "requires Packed Vector Format attribute for "
80 "integer vector operands");
81
82 assert(packedVectorFormat.getValue() ==
83 spirv::PackedVectorFormat::PackedVectorFormat4x8Bit &&
84 "Unknown Packed Vector Format");
85 if (intTy.getWidth() != 32)
86 return op->emitOpError(
87 message: llvm::formatv(Fmt: "with specified Packed Vector Format ({0}) requires "
88 "integer vector operands to be 32-bits wide",
89 Vals: packedVectorFormat.getValue()));
90 } else {
91 if (dotOp.getFormatAttr())
92 return op->emitOpError(message: llvm::formatv(
93 Fmt: "with invalid format attribute for vector operands of type '{0}'",
94 Vals&: factorTy));
95 }
96
97 Type resultTy = op->getResultTypes().front();
98 unsigned factorBitWidth = getBitWidth(type: factorTy);
99 unsigned resultBitWidth = getBitWidth(type: resultTy);
100 if (factorBitWidth > resultBitWidth)
101 return op->emitOpError(
102 message: llvm::formatv(Fmt: "result type has insufficient bit-width ({0} bits) "
103 "for the specified vector operand type ({1} bits)",
104 Vals&: resultBitWidth, Vals&: factorBitWidth));
105
106 return success();
107}
108
109static SmallVector<ArrayRef<spirv::Extension>, 1>
110getIntegerDotProductExtensions() {
111 // Requires the SPV_KHR_integer_dot_product extension, specified either
112 // explicitly or implied by target env's SPIR-V version >= 1.6.
113 static const auto extension = spirv::Extension::SPV_KHR_integer_dot_product;
114 return {extension};
115}
116
117template <typename IntegerDotProductOpTy>
118static SmallVector<ArrayRef<spirv::Capability>, 1>
119getIntegerDotProductCapabilities(Operation *op) {
120 // Requires the the DotProduct capability and capabilities that depend on
121 // exact op types.
122 static const auto dotProductCap = spirv::Capability::DotProduct;
123 static const auto dotProductInput4x8BitPackedCap =
124 spirv::Capability::DotProductInput4x8BitPacked;
125 static const auto dotProductInput4x8BitCap =
126 spirv::Capability::DotProductInput4x8Bit;
127 static const auto dotProductInputAllCap =
128 spirv::Capability::DotProductInputAll;
129
130 SmallVector<ArrayRef<spirv::Capability>, 1> capabilities = {dotProductCap};
131
132 Type factorTy = op->getOperand(idx: 0).getType();
133 auto dotOp = cast<IntegerDotProductOpTy>(op);
134 if (auto intTy = dyn_cast<IntegerType>(Val&: factorTy)) {
135 spirv::PackedVectorFormatAttr formatAttr = dotOp.getFormatAttr();
136 if (formatAttr.getValue() ==
137 spirv::PackedVectorFormat::PackedVectorFormat4x8Bit)
138 capabilities.push_back(Elt: dotProductInput4x8BitPackedCap);
139
140 return capabilities;
141 }
142
143 auto vecTy = cast<VectorType>(Val&: factorTy);
144 if (vecTy.getElementTypeBitWidth() == 8 && vecTy.getNumElements() == 4) {
145 capabilities.push_back(Elt: dotProductInput4x8BitCap);
146 return capabilities;
147 }
148
149 capabilities.push_back(Elt: dotProductInputAllCap);
150 return capabilities;
151}
152
153#define SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(OpName) \
154 LogicalResult OpName::verify() { \
155 return verifyIntegerDotProduct<OpName>(*this); \
156 } \
157 SmallVector<ArrayRef<spirv::Extension>, 1> OpName::getExtensions() { \
158 return getIntegerDotProductExtensions(); \
159 } \
160 SmallVector<ArrayRef<spirv::Capability>, 1> OpName::getCapabilities() { \
161 return getIntegerDotProductCapabilities<OpName>(*this); \
162 } \
163 std::optional<spirv::Version> OpName::getMinVersion() { \
164 return getDotProductMinVersion(); \
165 } \
166 std::optional<spirv::Version> OpName::getMaxVersion() { \
167 return getDotProductMaxVersion(); \
168 }
169
170SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(SDotOp)
171SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(SUDotOp)
172SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(UDotOp)
173SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(SDotAccSatOp)
174SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(SUDotAccSatOp)
175SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP(UDotAccSatOp)
176
177#undef SPIRV_IMPL_INTEGER_DOT_PRODUCT_OP
178
179} // namespace mlir::spirv
180