1//===- SPIRVTosaOps.cpp - MLIR SPIR-V Tosa operations ---------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the Tosa operations in the SPIR-V dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Dialect/SPIRV/IR/SPIRVOps.h"
14#include "llvm/ADT/STLExtras.h"
15#include "llvm/Support/InterleavedRange.h"
16#include <algorithm>
17
18namespace mlir::spirv {
19
20//===----------------------------------------------------------------------===//
21// SPIRV Tosa Custom formatters
22//===----------------------------------------------------------------------===//
23
24ParseResult parseSPIRV_I32_1DArmTensor(OpAsmParser &parser,
25 DenseIntElementsAttr &attr) {
26 SmallVector<int32_t, 6> elements;
27 auto f = [&]() {
28 int32_t value;
29 ParseResult r = parser.parseInteger(result&: value);
30 elements.push_back(Elt: value);
31 return r;
32 };
33 if (parser.parseCommaSeparatedList(
34 delimiter: OpAsmParser::Delimiter::Square, parseElementFn: f,
35 contextMessage: "parsing values in integer list attribute")) {
36 return failure();
37 }
38
39 auto i32Type = IntegerType::get(context: parser.getContext(), width: 32);
40 auto type = TensorArmType::get(
41 shape: ArrayRef{static_cast<int64_t>(elements.size())}, elementType: i32Type);
42 attr = DenseIntElementsAttr::get(type, arg&: elements);
43 return success();
44}
45
46void printSPIRV_I32_1DArmTensor(OpAsmPrinter &printer, Operation *,
47 DenseIntElementsAttr attr) {
48 printer << llvm::interleaved_array(
49 R: llvm::map_range(C: attr.getValues<APInt>(),
50 F: [](const APInt &a) { return a.getSExtValue(); }));
51}
52
53//===----------------------------------------------------------------------===//
54// SPIRV Tosa Custom verifiers
55//===----------------------------------------------------------------------===//
56
57namespace {
58
59int64_t getIntValue(DenseIntElementsAttr attr, size_t idx) {
60 return attr.getValues<APInt>()[idx].getSExtValue();
61}
62
63LogicalResult verifyPool2DOutputDim(Operation *op, int64_t inputSize,
64 int64_t outputSize, int64_t kernelSize,
65 int64_t strideSize, int64_t padBefore,
66 int64_t padAfter, StringRef dimName,
67 StringRef dimAxis, StringRef padBeforeName,
68 StringRef padAfterName) {
69 if (ShapedType::isDynamic(dValue: inputSize))
70 return success();
71
72 const int64_t numerator = inputSize + padBefore + padAfter - kernelSize;
73 if (numerator % strideSize != 0)
74 return op->emitOpError(message: "expected input_")
75 << dimName << " + pad_" << padBeforeName << " + pad_" << padAfterName
76 << " - kernel_" << dimAxis << " to be wholly divisible by stride_"
77 << dimAxis << ", got (" << inputSize << " + " << padBefore << " + "
78 << padAfter << " - " << kernelSize << ") / " << strideSize;
79
80 const int64_t calculatedOutput = numerator / strideSize + 1;
81 if (!ShapedType::isDynamic(dValue: outputSize) && outputSize != calculatedOutput)
82 return op->emitOpError(message: "failed to verify that shapes of input and output "
83 "must satisfy [N,IH,IW,C] and [N,OH,OW,C], with "
84 "OH = ((IH + pad_top + pad_bottom - kernel_y) / "
85 "stride_y) + 1 and OW = ((IW + pad_left + "
86 "pad_right - kernel_x) / stride_x) + 1");
87
88 return success();
89}
90
91LogicalResult verifyPool2DOp(Operation *op, DenseIntElementsAttr kernel,
92 DenseIntElementsAttr stride,
93 DenseIntElementsAttr pad, TensorArmType inputType,
94 TensorArmType outputType) {
95
96 if (!inputType.hasRank() || !outputType.hasRank())
97 return success();
98
99 if (failed(Result: verifyPool2DOutputDim(
100 op, inputSize: inputType.getDimSize(idx: 1), outputSize: outputType.getDimSize(idx: 1),
101 kernelSize: getIntValue(attr: kernel, idx: 0), strideSize: getIntValue(attr: stride, idx: 0), padBefore: getIntValue(attr: pad, idx: 0),
102 padAfter: getIntValue(attr: pad, idx: 1), dimName: "height", dimAxis: "y", padBeforeName: "top", padAfterName: "bottom")))
103 return failure();
104
105 if (failed(Result: verifyPool2DOutputDim(
106 op, inputSize: inputType.getDimSize(idx: 2), outputSize: outputType.getDimSize(idx: 2),
107 kernelSize: getIntValue(attr: kernel, idx: 1), strideSize: getIntValue(attr: stride, idx: 1), padBefore: getIntValue(attr: pad, idx: 2),
108 padAfter: getIntValue(attr: pad, idx: 3), dimName: "width", dimAxis: "x", padBeforeName: "left", padAfterName: "right")))
109 return failure();
110
111 return success();
112}
113
114LogicalResult verifyConvolutionOutputDim(int64_t inputSize, int64_t kernelSize,
115 int64_t outputSize, int64_t padBefore,
116 int64_t padAfter, int64_t strideSize,
117 int64_t dilationSize) {
118 if (ShapedType::isDynamic(dValue: inputSize) || ShapedType::isDynamic(dValue: kernelSize))
119 return success();
120
121 const int64_t numerator =
122 inputSize - 1 + padBefore + padAfter - (kernelSize - 1) * dilationSize;
123 if (numerator % strideSize != 0)
124 return failure();
125
126 const int64_t calculatedOutput = numerator / strideSize + 1;
127 if (!ShapedType::isDynamic(dValue: outputSize) && outputSize != calculatedOutput)
128 return failure();
129
130 return success();
131}
132
133LogicalResult
134verifyTransposeConvolutionOutputDim(int64_t inputSize, int64_t kernelSize,
135 int64_t outputSize, int64_t padBefore,
136 int64_t padAfter, int64_t strideSize) {
137 if (ShapedType::isDynamic(dValue: inputSize) || ShapedType::isDynamic(dValue: kernelSize))
138 return success();
139
140 const int64_t calculatedOutput =
141 (inputSize - 1) * strideSize + padBefore + padAfter + kernelSize;
142 if (!ShapedType::isDynamic(dValue: outputSize) && outputSize != calculatedOutput)
143 return failure();
144
145 return success();
146}
147
148LogicalResult verifyConv2DOutputShape(Operation *op, DenseIntElementsAttr pad,
149 DenseIntElementsAttr stride,
150 DenseIntElementsAttr dilation,
151 TensorArmType inputType,
152 TensorArmType weightType,
153 TensorArmType outputType) {
154 constexpr StringLiteral errorMessage =
155 "failed to verify that shapes of input, weight, and output must satisfy "
156 "[N,IH,IW,*], [*,KH,KW,*], [N,OH,OW,*], with OH = ((IH - 1 + pad_top + "
157 "pad_bottom - (KH - 1) * dilation_y) / stride_y) + 1 and OW = ((IW - 1 "
158 "+ pad_left + pad_right - (KW - 1) * dilation_x) / stride_x) + 1";
159 if (!inputType.hasRank() || !weightType.hasRank() || !outputType.hasRank())
160 return success();
161
162 if (failed(Result: verifyConvolutionOutputDim(
163 inputSize: inputType.getDimSize(idx: 1), kernelSize: weightType.getDimSize(idx: 1),
164 outputSize: outputType.getDimSize(idx: 1), padBefore: getIntValue(attr: pad, idx: 0), padAfter: getIntValue(attr: pad, idx: 1),
165 strideSize: getIntValue(attr: stride, idx: 0), dilationSize: getIntValue(attr: dilation, idx: 0))))
166 return op->emitOpError(message: errorMessage);
167
168 if (failed(Result: verifyConvolutionOutputDim(
169 inputSize: inputType.getDimSize(idx: 2), kernelSize: weightType.getDimSize(idx: 2),
170 outputSize: outputType.getDimSize(idx: 2), padBefore: getIntValue(attr: pad, idx: 2), padAfter: getIntValue(attr: pad, idx: 3),
171 strideSize: getIntValue(attr: stride, idx: 1), dilationSize: getIntValue(attr: dilation, idx: 1))))
172 return op->emitOpError(message: errorMessage);
173
174 return success();
175}
176
177LogicalResult verifyConv3DOutputShape(Operation *op, DenseIntElementsAttr pad,
178 DenseIntElementsAttr stride,
179 DenseIntElementsAttr dilation,
180 TensorArmType inputType,
181 TensorArmType weightType,
182 TensorArmType outputType) {
183 constexpr StringLiteral errorMessage =
184 "failed to verify that shapes of input, weight, and output must satisfy "
185 "[N,ID,IH,IW,*], [*,KD,KH,KW,*], [N,OD,OH,OW,*], with OD = ((ID - 1 + "
186 "pad_front + pad_back - (KD - 1) * dilation_d) / stride_d) + 1, OH = "
187 "((IH - 1 + pad_top + pad_bottom - (KH - 1) * dilation_y) / stride_y) "
188 "+ 1 and OW = ((IW - 1 + pad_left + pad_right - (KW - 1) * dilation_x) "
189 "/ stride_x) + 1";
190 if (!inputType.hasRank() || !weightType.hasRank() || !outputType.hasRank())
191 return success();
192
193 if (failed(Result: verifyConvolutionOutputDim(
194 inputSize: inputType.getDimSize(idx: 1), kernelSize: weightType.getDimSize(idx: 1),
195 outputSize: outputType.getDimSize(idx: 1), padBefore: getIntValue(attr: pad, idx: 0), padAfter: getIntValue(attr: pad, idx: 1),
196 strideSize: getIntValue(attr: stride, idx: 0), dilationSize: getIntValue(attr: dilation, idx: 0))))
197 return op->emitOpError(message: errorMessage);
198
199 if (failed(Result: verifyConvolutionOutputDim(
200 inputSize: inputType.getDimSize(idx: 2), kernelSize: weightType.getDimSize(idx: 2),
201 outputSize: outputType.getDimSize(idx: 2), padBefore: getIntValue(attr: pad, idx: 2), padAfter: getIntValue(attr: pad, idx: 3),
202 strideSize: getIntValue(attr: stride, idx: 1), dilationSize: getIntValue(attr: dilation, idx: 1))))
203 return op->emitOpError(message: errorMessage);
204
205 if (failed(Result: verifyConvolutionOutputDim(
206 inputSize: inputType.getDimSize(idx: 3), kernelSize: weightType.getDimSize(idx: 3),
207 outputSize: outputType.getDimSize(idx: 3), padBefore: getIntValue(attr: pad, idx: 4), padAfter: getIntValue(attr: pad, idx: 5),
208 strideSize: getIntValue(attr: stride, idx: 2), dilationSize: getIntValue(attr: dilation, idx: 2))))
209 return op->emitOpError(message: errorMessage);
210
211 return success();
212}
213
214LogicalResult verifyDepthwiseConv2DOutputShape(
215 Operation *op, DenseIntElementsAttr pad, DenseIntElementsAttr stride,
216 DenseIntElementsAttr dilation, TensorArmType inputType,
217 TensorArmType weightType, TensorArmType outputType) {
218 constexpr StringLiteral errorMessage =
219 "failed to verify that shapes of input, weight, and output must satisfy "
220 "[N,IH,IW,*], [KH,KW,*,*], [N,OH,OW,*], with OH = ((IH - 1 + pad_top + "
221 "pad_bottom - (KH - 1) * dilation_y) / stride_y) + 1 and OW = ((IW - 1 "
222 "+ pad_left + pad_right - (KW - 1) * dilation_x) / stride_x) + 1";
223 if (!inputType.hasRank() || !weightType.hasRank() || !outputType.hasRank())
224 return success();
225
226 if (failed(Result: verifyConvolutionOutputDim(
227 inputSize: inputType.getDimSize(idx: 1), kernelSize: weightType.getDimSize(idx: 0),
228 outputSize: outputType.getDimSize(idx: 1), padBefore: getIntValue(attr: pad, idx: 0), padAfter: getIntValue(attr: pad, idx: 1),
229 strideSize: getIntValue(attr: stride, idx: 0), dilationSize: getIntValue(attr: dilation, idx: 0))))
230 return op->emitOpError(message: errorMessage);
231
232 if (failed(Result: verifyConvolutionOutputDim(
233 inputSize: inputType.getDimSize(idx: 2), kernelSize: weightType.getDimSize(idx: 1),
234 outputSize: outputType.getDimSize(idx: 2), padBefore: getIntValue(attr: pad, idx: 2), padAfter: getIntValue(attr: pad, idx: 3),
235 strideSize: getIntValue(attr: stride, idx: 1), dilationSize: getIntValue(attr: dilation, idx: 1))))
236 return op->emitOpError(message: errorMessage);
237
238 return success();
239}
240
241LogicalResult verifyTransposeConv2DOutputShape(Operation *op,
242 DenseIntElementsAttr outPad,
243 DenseIntElementsAttr stride,
244 TensorArmType inputType,
245 TensorArmType weightType,
246 TensorArmType outputType) {
247 constexpr StringLiteral errorMessage =
248 "failed to verify that shapes of input, weight, and output must satisfy "
249 "[N,IH,IW,*], [*,KH,KW,*], [N,OH,OW,*], with OH = (IH - 1) * stride_y + "
250 "out_pad_top + out_pad_bottom + KH and OW = (IW - 1) * stride_x + "
251 "out_pad_left + out_pad_right + KW";
252 if (!inputType.hasRank() || !weightType.hasRank() || !outputType.hasRank())
253 return success();
254
255 const int64_t kernelHeight = weightType.getDimSize(idx: 1);
256 if (ShapedType::isStatic(dValue: kernelHeight) &&
257 (getIntValue(attr: outPad, idx: 0) <= -kernelHeight ||
258 getIntValue(attr: outPad, idx: 1) <= -kernelHeight))
259 return op->emitOpError(message: "expected out_pad_top and out_pad_bottom to be > "
260 "-KH");
261
262 const int64_t kernelWidth = weightType.getDimSize(idx: 2);
263 if (ShapedType::isStatic(dValue: kernelWidth) &&
264 (getIntValue(attr: outPad, idx: 2) <= -kernelWidth ||
265 getIntValue(attr: outPad, idx: 3) <= -kernelWidth))
266 return op->emitOpError(message: "expected out_pad_left and out_pad_right to be > "
267 "-KW");
268
269 if (failed(Result: verifyTransposeConvolutionOutputDim(
270 inputSize: inputType.getDimSize(idx: 1), kernelSize: kernelHeight, outputSize: outputType.getDimSize(idx: 1),
271 padBefore: getIntValue(attr: outPad, idx: 0), padAfter: getIntValue(attr: outPad, idx: 1),
272 strideSize: getIntValue(attr: stride, idx: 0))))
273 return op->emitOpError(message: errorMessage);
274
275 if (failed(Result: verifyTransposeConvolutionOutputDim(
276 inputSize: inputType.getDimSize(idx: 2), kernelSize: kernelWidth, outputSize: outputType.getDimSize(idx: 2),
277 padBefore: getIntValue(attr: outPad, idx: 2), padAfter: getIntValue(attr: outPad, idx: 3),
278 strideSize: getIntValue(attr: stride, idx: 1))))
279 return op->emitOpError(message: errorMessage);
280
281 return success();
282}
283
284LogicalResult verifyConcatOutputShape(Operation *op, TypeRange inputTypes,
285 TensorArmType outputType, int32_t axis) {
286 constexpr StringLiteral errorMessage =
287 "failed to verify that shape of output must match the concatenation of "
288 "input1 along axis";
289 if (!outputType.hasRank())
290 return success();
291
292 if (llvm::any_of(Range&: inputTypes, P: [](Type type) {
293 return !cast<TensorArmType>(Val&: type).hasRank();
294 }))
295 return success();
296
297 for (int64_t dim = 0, rank = outputType.getRank(); dim < rank; ++dim) {
298 int64_t outputDim = outputType.getDimSize(idx: dim);
299 if (ShapedType::isDynamic(dValue: outputDim))
300 continue;
301
302 if (dim != axis) {
303 for (Type type : inputTypes) {
304 int64_t inputDim = cast<TensorArmType>(Val&: type).getDimSize(idx: dim);
305 if (ShapedType::isStatic(dValue: inputDim) && inputDim != outputDim)
306 return op->emitOpError(message: errorMessage);
307 }
308 continue;
309 }
310
311 int64_t concatDim = 0;
312 for (Type type : inputTypes) {
313 int64_t inputDim = cast<TensorArmType>(Val&: type).getDimSize(idx: dim);
314 if (ShapedType::isDynamic(dValue: inputDim)) {
315 concatDim = ShapedType::kDynamic;
316 break;
317 }
318 concatDim += inputDim;
319 }
320
321 if (ShapedType::isStatic(dValue: concatDim) && concatDim != outputDim)
322 return op->emitOpError(message: errorMessage);
323 }
324
325 return success();
326}
327
328} // namespace
329
330LogicalResult TosaAvgPool2DOp::verify() {
331 return verifyPool2DOp(op: getOperation(), kernel: getKernel(), stride: getStride(), pad: getPad(),
332 inputType: getInputType(), outputType: getResultType());
333}
334
335LogicalResult TosaConv2DOp::verify() {
336 return verifyConv2DOutputShape(op: getOperation(), pad: getPad(), stride: getStride(),
337 dilation: getDilation(), inputType: getInputType(), weightType: getWeightType(),
338 outputType: getResultType());
339}
340
341LogicalResult TosaConv3DOp::verify() {
342 return verifyConv3DOutputShape(op: getOperation(), pad: getPad(), stride: getStride(),
343 dilation: getDilation(), inputType: getInputType(), weightType: getWeightType(),
344 outputType: getResultType());
345}
346
347LogicalResult TosaDepthwiseConv2DOp::verify() {
348 return verifyDepthwiseConv2DOutputShape(op: getOperation(), pad: getPad(), stride: getStride(),
349 dilation: getDilation(), inputType: getInputType(),
350 weightType: getWeightType(), outputType: getResultType());
351}
352
353LogicalResult TosaMaxPool2DOp::verify() {
354 return verifyPool2DOp(op: getOperation(), kernel: getKernel(), stride: getStride(), pad: getPad(),
355 inputType: getInputType(), outputType: getResultType());
356}
357
358LogicalResult TosaTransposeConv2DOp::verify() {
359 return verifyTransposeConv2DOutputShape(op: getOperation(), outPad: getOutPad(),
360 stride: getStride(), inputType: getInputType(),
361 weightType: getWeightType(), outputType: getResultType());
362}
363
364LogicalResult TosaConcatOp::verify() {
365 return verifyConcatOutputShape(op: getOperation(), inputTypes: getInput1Types(),
366 outputType: getResultType(), axis: getAxis());
367}
368
369LogicalResult TosaSelectOp::verify() {
370 TensorArmType condType = getConditionType();
371 TensorArmType trueValType = getTrueValueType();
372 TensorArmType falseValType = getFalseValueType();
373 TensorArmType resultType = getResultType();
374
375 if (llvm::any_of(Range: ArrayRef<TensorArmType>{condType, trueValType, falseValType,
376 resultType},
377 P: [](TensorArmType type) { return !type.hasRank(); }))
378 return success();
379
380 ArrayRef<int64_t> condShape = condType.getShape();
381 ArrayRef<int64_t> trueValShape = trueValType.getShape();
382 ArrayRef<int64_t> falseValShape = falseValType.getShape();
383 ArrayRef<int64_t> resultShape = resultType.getShape();
384
385 if (!llvm::all_equal(Values: {condShape.size(), trueValShape.size(),
386 falseValShape.size(), resultShape.size()})) {
387 // The AllRanksMatch predicate enforces that all ranks are equal.
388 // This is just an extra safe guard for the code coming after that
389 // assumes that all ranks are equal.
390 return failure();
391 }
392
393 for (auto dims :
394 llvm::zip_equal(t&: condShape, u&: trueValShape, args&: falseValShape, args&: resultShape)) {
395 auto [condDim, trueValDim, falseValDim, resultDim] = dims;
396
397 if (llvm::any_of(
398 Range: ArrayRef<int64_t>{condDim, trueValDim, falseValDim, resultDim},
399 P: [](int64_t dim) { return ShapedType::isDynamic(dValue: dim); })) {
400 continue;
401 }
402
403 auto isPairBroadcastable = [](int64_t lhs, int64_t rhs) {
404 return lhs == rhs || lhs == 1 || rhs == 1;
405 };
406
407 if (!isPairBroadcastable(condDim, trueValDim) ||
408 !isPairBroadcastable(condDim, falseValDim) ||
409 !isPairBroadcastable(trueValDim, falseValDim)) {
410 return emitOpError(
411 message: "failed to verify that the shape of inputs: condition, "
412 "true_value, and false_value are compatible for "
413 "broadcasting");
414 }
415
416 int64_t bradcastedInputDim =
417 std::max(a: condDim, b: std::max(a: trueValDim, b: falseValDim));
418 if (bradcastedInputDim != resultDim) {
419 return emitOpError(
420 message: "failed to verify that the broadcast shape of inputs: condition, "
421 "true_value, and false_value is equal to "
422 "the output shape");
423 }
424 }
425 return success();
426}
427
428} // namespace mlir::spirv
429