1//===- TosaOps.cpp - MLIR Dialect for TOSA --------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// This file implements the TOSA Specification:
11// https://www.mlplatform.org/tosa/tosa_spec.html
12//
13//===----------------------------------------------------------------------===//
14
15#include "mlir/Dialect/Tosa/IR/TosaOps.h"
16#include "mlir/Dialect/Quant/IR/Quant.h"
17#include "mlir/Dialect/Shard/Interfaces/ShardingInterface.h"
18#include "mlir/Dialect/Tensor/IR/Tensor.h"
19#include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
20#include "mlir/Dialect/Tosa/Utils/ShapeUtils.h"
21#include "mlir/Dialect/Utils/IndexingUtils.h"
22#include "mlir/Dialect/Utils/VerificationUtils.h"
23#include "mlir/IR/BuiltinTypeInterfaces.h"
24#include "mlir/IR/BuiltinTypes.h"
25#include "mlir/IR/Matchers.h"
26#include "mlir/IR/TypeUtilities.h"
27#include "mlir/Interfaces/InferTypeOpInterface.h"
28#include "mlir/Transforms/InliningUtils.h"
29#include "llvm/ADT/APFloat.h"
30#include "llvm/ADT/SmallVectorExtras.h"
31
32#include <numeric>
33#include <type_traits>
34
35using namespace mlir;
36using namespace mlir::tosa;
37
38#include "mlir/Dialect/Tosa/IR/TosaOpsDialect.cpp.inc"
39#include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
40
41//===----------------------------------------------------------------------===//
42// Tosa dialect interface includes.
43//===----------------------------------------------------------------------===//
44
45#include "mlir/Dialect/Tosa/IR/TosaDialectBytecode.cpp.inc"
46#include "mlir/Dialect/Tosa/IR/TosaEnums.cpp.inc"
47#include "mlir/Dialect/Tosa/IR/TosaInterfaces.cpp.inc"
48
49namespace {
50//===----------------------------------------------------------------------===//
51// Dialect Function Inliner Interface.
52//===----------------------------------------------------------------------===//
53struct TosaInlinerInterface : public DialectInlinerInterface {
54 using DialectInlinerInterface::DialectInlinerInterface;
55
56 //===--------------------------------------------------------------------===//
57 // Analysis Hooks.
58 //===--------------------------------------------------------------------===//
59
60 /// All operations can be inlined by default.
61 bool isLegalToInline(Operation *op, Region *region, bool wouldBeCloned,
62 IRMapping &map) const final {
63 return true;
64 }
65
66 /// All regions with If and While parent operators can be inlined.
67 bool isLegalToInline(Region *dest, Region *src, bool wouldBeCloned,
68 IRMapping &map) const final {
69 return (isa<tosa::IfOp>(Val: dest->getParentOp()) ||
70 isa<tosa::WhileOp>(Val: dest->getParentOp()));
71 }
72};
73
74/// This class implements the bytecode interface for the Tosa dialect.
75struct TosaDialectBytecodeInterface : public BytecodeDialectInterface {
76 TosaDialectBytecodeInterface(Dialect *dialect)
77 : BytecodeDialectInterface(dialect) {}
78
79 //===--------------------------------------------------------------------===//
80 // Attributes
81
82 Attribute readAttribute(DialectBytecodeReader &reader) const override {
83 return ::readAttribute(context: getContext(), reader);
84 }
85
86 LogicalResult writeAttribute(Attribute attr,
87 DialectBytecodeWriter &writer) const override {
88 return ::writeAttribute(attribute: attr, writer);
89 }
90
91 //===--------------------------------------------------------------------===//
92 // Types
93
94 Type readType(DialectBytecodeReader &reader) const override {
95 return ::readType(context: getContext(), reader);
96 }
97
98 LogicalResult writeType(Type type,
99 DialectBytecodeWriter &writer) const override {
100 return ::writeType(type, writer);
101 }
102
103 void writeVersion(DialectBytecodeWriter &writer) const final {
104 // TODO: Populate.
105 }
106
107 std::unique_ptr<DialectVersion>
108 readVersion(DialectBytecodeReader &reader) const final {
109 // TODO: Populate
110 reader.emitError(msg: "Dialect does not support versioning");
111 return nullptr;
112 }
113
114 LogicalResult upgradeFromVersion(Operation *topLevelOp,
115 const DialectVersion &version) const final {
116 return success();
117 }
118};
119} // namespace
120
121//===----------------------------------------------------------------------===//
122// TOSA control flow support.
123//===----------------------------------------------------------------------===//
124
125/// Returns the while loop body.
126SmallVector<Region *> tosa::WhileOp::getLoopRegions() {
127 return {&getBodyGraph()};
128}
129
130//===----------------------------------------------------------------------===//
131// TOSA variable operator support.
132//===----------------------------------------------------------------------===//
133
134static SmallVector<int64_t> convertToMlirShape(ArrayRef<int64_t> shape) {
135 return map_to_vector(C&: shape, F: [](int64_t dim) {
136 return dim == -1 ? ShapedType::kDynamic : dim;
137 });
138}
139
140// returns type of variable op
141RankedTensorType mlir::tosa::getVariableType(tosa::VariableOp variableOp) {
142 Type elementType = variableOp.getType();
143 DenseIntElementsAttr varShapeAttr = variableOp.getVarShape();
144 auto shape = convertToMlirShape(shape: to_vector(Range: varShapeAttr.getValues<int64_t>()));
145 return RankedTensorType::get(shape, elementType);
146}
147
148//===----------------------------------------------------------------------===//
149// Tosa dialect initialization.
150//===----------------------------------------------------------------------===//
151
152void TosaDialect::initialize() {
153 addTypes<
154#define GET_TYPEDEF_LIST
155#include "mlir/Dialect/Tosa/IR/TosaOpsTypesBase.cpp.inc"
156 >();
157 addOperations<
158#define GET_OP_LIST
159#include "mlir/Dialect/Tosa/IR/TosaOps.cpp.inc"
160 >();
161 addAttributes<
162#define GET_ATTRDEF_LIST
163#include "mlir/Dialect/Tosa/IR/TosaAttributes.cpp.inc"
164 >();
165 addInterfaces<TosaDialectBytecodeInterface, TosaInlinerInterface>();
166 declarePromisedInterfaces<
167 shard::ShardingInterface, ClampOp, SigmoidOp, TanhOp, AddOp,
168 ArithmeticRightShiftOp, BitwiseAndOp, BitwiseOrOp, BitwiseXorOp, IntDivOp,
169 LogicalAndOp, LogicalLeftShiftOp, LogicalRightShiftOp, LogicalOrOp,
170 LogicalXorOp, MaximumOp, MinimumOp, MulOp, PowOp, SubOp, AbsOp,
171 BitwiseNotOp, CeilOp, ClzOp, ExpOp, FloorOp, LogOp, LogicalNotOp,
172 NegateOp, ReciprocalOp, RsqrtOp, SelectOp, EqualOp, GreaterOp,
173 GreaterEqualOp, MatMulOp>();
174}
175
176Operation *TosaDialect::materializeConstant(OpBuilder &builder, Attribute value,
177 Type type, Location loc) {
178 // Tosa dialect constants only support ElementsAttr unlike standard dialect
179 // constant which supports all attributes.
180 if (llvm::isa<shapeType>(Val: type) && llvm::isa<DenseIntElementsAttr>(Val: value)) {
181 return tosa::ConstShapeOp::create(builder, location: loc, output: type,
182 values: llvm::cast<DenseIntElementsAttr>(Val&: value));
183 }
184 if (llvm::isa<ElementsAttr>(Val: value))
185 return tosa::ConstOp::create(builder, location: loc, output: type,
186 values: llvm::cast<ElementsAttr>(Val&: value));
187 return nullptr;
188}
189
190//===----------------------------------------------------------------------===//
191// Parsers and printers
192//===----------------------------------------------------------------------===//
193
194namespace {
195
196ParseResult getShapeAndElementType(OpAsmParser &parser, Type parsedType,
197 DenseElementsAttr &varShapeAttr,
198 TypeAttr &typeAttr) {
199 if (auto shapedType = dyn_cast<ShapedType>(Val&: parsedType)) {
200 if (!shapedType.hasRank())
201 return parser.emitError(loc: parser.getCurrentLocation())
202 << "expected ranked type";
203
204 auto elementType = shapedType.getElementType();
205 typeAttr = TypeAttr::get(type: elementType);
206 ArrayRef<int64_t> shape = shapedType.getShape();
207 Builder builder(parser.getContext());
208 varShapeAttr = builder.getIndexTensorAttr(values: convertFromMlirShape(shape));
209 return success();
210 }
211 return parser.emitError(loc: parser.getCurrentLocation())
212 << "expected shaped type";
213}
214
215} // namespace
216
217// parses the optional initial value or type for a tosa variable
218// with initial value:
219// tosa.variable @name = dense<0.0> : tensor<1x8xf32>
220//
221// without initial value:
222// tosa.variable @name : tensor<1x8xf32>
223ParseResult mlir::tosa::parseVariableOpTypeOrInitialValue(
224 OpAsmParser &parser, DenseElementsAttr &varShapeAttr, TypeAttr &typeAttr,
225 Attribute &initialValueAttr) {
226 if (succeeded(Result: parser.parseOptionalEqual())) {
227 if (failed(Result: parser.parseAttribute(result&: initialValueAttr))) {
228 return parser.emitError(loc: parser.getCurrentLocation())
229 << "expected attribute";
230 }
231 if (auto typedAttr = dyn_cast<TypedAttr>(Val&: initialValueAttr)) {
232 return getShapeAndElementType(parser, parsedType: typedAttr.getType(), varShapeAttr,
233 typeAttr);
234 }
235 return parser.emitError(loc: parser.getCurrentLocation())
236 << "expected Typed attr";
237 }
238
239 initialValueAttr = nullptr;
240 Type parsedType;
241 if (failed(Result: parser.parseColonType(result&: parsedType))) {
242 return parser.emitError(loc: parser.getCurrentLocation())
243 << "expected type after colon";
244 }
245 return getShapeAndElementType(parser, parsedType, varShapeAttr, typeAttr);
246}
247
248void mlir::tosa::printVariableOpTypeOrInitialValue(
249 OpAsmPrinter &p, Operation *op, DenseElementsAttr varShapeAttr,
250 TypeAttr typeAttr, Attribute initialValueAttr) {
251 bool needsSpace = false;
252 if (!dyn_cast_or_null<TypedAttr>(Val&: initialValueAttr)) {
253 auto shape =
254 convertToMlirShape(shape: to_vector(Range: varShapeAttr.getValues<int64_t>()));
255 Type elementType = typeAttr.getValue();
256 RankedTensorType tensorType =
257 RankedTensorType::get(shape: ArrayRef<int64_t>(shape), elementType);
258 auto tensorTypeAttr = TypeAttr::get(type: tensorType);
259 p << ": ";
260 p.printAttribute(attr: tensorTypeAttr);
261 needsSpace = true; // subsequent attr value needs a space separator
262 }
263 if (initialValueAttr) {
264 if (needsSpace)
265 p << ' ';
266 p << "= ";
267 p.printAttribute(attr: initialValueAttr);
268 }
269}
270
271//===----------------------------------------------------------------------===//
272// Tosa utilities.
273//===----------------------------------------------------------------------===//
274
275static std::optional<int64_t> idivCheck(const int64_t lhs, const int64_t rhs) {
276 if (lhs % rhs != 0)
277 return std::nullopt;
278 return lhs / rhs;
279}
280
281Type mlir::tosa::getStorageElementTypeOrSelf(Type type) {
282 auto srcType = getElementTypeOrSelf(type);
283 if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(Val&: srcType))
284 srcType = getStorageElementTypeFromQuantized(quantizedType: quantType);
285 return srcType;
286}
287
288Type mlir::tosa::getStorageElementTypeOrSelf(Value value) {
289 return getStorageElementTypeOrSelf(type: value.getType());
290}
291
292static LogicalResult verifyRescaleValueAndZpTypes(Operation *op, Value val,
293 Value valZp, StringRef name) {
294 Type eType = getStorageElementTypeOrSelf(type: val.getType());
295 Type eZpType = getStorageElementTypeOrSelf(type: valZp.getType());
296
297 bool bothInts =
298 mlir::isa<IntegerType>(Val: eType) && mlir::isa<IntegerType>(Val: eZpType);
299 bool sameBitWidth =
300 (eType.getIntOrFloatBitWidth() == eZpType.getIntOrFloatBitWidth());
301
302 if (!bothInts || !sameBitWidth) {
303 return op->emitOpError()
304 << "expected " << name << " and " << name
305 << "_zp to both be integer of the same bitwidth, but got " << eType
306 << " vs. " << eZpType;
307 }
308 return success();
309}
310
311// Create a pad-const const tensor with value of `val` of required data-type
312Value mlir::tosa::createPadConstTensor(OpBuilder &builder, Location loc,
313 Value src, int32_t val) {
314 const auto srcType = getElementTypeOrSelf(val: src);
315 const auto srcElemType = getStorageElementTypeOrSelf(value: src);
316 const auto padConstType = mlir::RankedTensorType::get(shape: {1}, elementType: srcType);
317 const auto padConstEType = mlir::RankedTensorType::get(shape: {1}, elementType: srcElemType);
318 const auto padConstAttr{
319 llvm::isa<FloatType>(Val: srcElemType)
320 ? DenseElementsAttr::get(type: padConstEType,
321 values: builder.getFloatAttr(type: srcElemType, value: val))
322 : DenseElementsAttr::get(type: padConstEType,
323 values: builder.getIntegerAttr(type: srcElemType, value: val))};
324 return tosa::ConstOp::create(builder, location: loc, output: padConstType, values: padConstAttr);
325}
326
327unsigned mlir::tosa::getBitWidth(Type type) {
328 if (auto blockScaledTy = dyn_cast<tosa::BlockScaledType>(Val&: type))
329 return getBitWidth(type: blockScaledTy.getValueType());
330 if (isa<MXI8Type, tosa::mxint8Type>(Val: type))
331 return 8;
332 return type.getIntOrFloatBitWidth();
333}
334
335// Update dim size if current dim is dynamic, otherwise raise an error if sizes
336// do not match
337LogicalResult tryUpdateDimOrFailure(Operation *op, int64_t &currDim,
338 const int64_t newDim,
339 const StringRef operandName,
340 const StringRef dimName) {
341 if (ShapedType::isDynamic(dValue: currDim)) {
342 currDim = newDim;
343 return success();
344 } else if (ShapedType::isStatic(dValue: newDim) && currDim != newDim) {
345 return op->emitOpError(message: "expected ")
346 << dimName << " of " << operandName << " to match size " << currDim
347 << ", got " << newDim;
348 }
349 return success();
350}
351
352static void printShapeToDiagnostic(InFlightDiagnostic &diag,
353 ArrayRef<int64_t> shape) {
354 auto printDim = [&](int64_t dim) {
355 if (ShapedType::isDynamic(dValue: dim))
356 diag << "?";
357 else
358 diag << dim;
359 };
360
361 llvm::interleaveComma(c: shape, os&: diag, each_fn: printDim);
362}
363
364static LogicalResult
365verifyOutputShapeCompatibleWithExpected(Operation *op, ShapedType outputType,
366 ArrayRef<int64_t> expectedShape,
367 StringRef outputName = "output") {
368 assert(outputType.hasRank() && "expected output type to be ranked");
369
370 if (succeeded(Result: verifyCompatibleShape(shape1: outputType.getShape(), shape2: expectedShape)))
371 return success();
372
373 InFlightDiagnostic diag = op->emitOpError(message: "expected ");
374 diag << outputName << " shape ";
375 printShapeToDiagnostic(diag, shape: outputType.getShape());
376 diag << " to be compatible with inferred shape ";
377 printShapeToDiagnostic(diag, shape: expectedShape);
378 return diag;
379}
380
381LogicalResult verifyConvOutputSize(
382 Operation *op, const int64_t inputSize, const int64_t kernelSize,
383 const int64_t outputSize, const int64_t padBefore, const int64_t padAfter,
384 const int64_t stride, const int64_t dilation, const llvm::StringRef dimName,
385 const llvm::StringRef dimAxis, const llvm::StringRef padBeforeName,
386 const llvm::StringRef padAfterName) {
387 if (inputSize == ShapedType::kDynamic || kernelSize == ShapedType::kDynamic)
388 return success();
389
390 // ERROR_IF: O != idiv_check(I - 1 + pa + pb - (K - 1) * d, s) + 1
391
392 const std::optional<int64_t> calculatedOutSizeMinusOne = idivCheck(
393 lhs: inputSize - 1 + padBefore + padAfter - (kernelSize - 1) * dilation,
394 rhs: stride);
395 if (!calculatedOutSizeMinusOne.has_value())
396 return op->emitOpError(message: "expected input_")
397 << dimName << " - 1 + pad_" << padBeforeName << " + pad_"
398 << padAfterName << " - (kernel_" << dimName << " - 1) * dilation_"
399 << dimAxis << " to be wholly divisible by stride_" << dimAxis
400 << ", got (" << inputSize << " - 1 + " << padBefore << " + "
401 << padAfter << " - (" << kernelSize << " - 1) * " << dilation
402 << ") / " << stride;
403
404 const int64_t calculatedOutSize = calculatedOutSizeMinusOne.value() + 1;
405 if (outputSize != ShapedType::kDynamic && calculatedOutSize != outputSize)
406 return op->emitOpError(message: "calculated output ")
407 << dimName << " did not match expected: "
408 << "calculated=" << calculatedOutSize << ", expected=" << outputSize;
409
410 return success();
411}
412
413//===----------------------------------------------------------------------===//
414// mxint8Type DenseElementTypeInterface implementation.
415//===----------------------------------------------------------------------===//
416size_t mlir::tosa::mxint8Type::getDenseElementBitSize() const { return 8; }
417
418Attribute
419mlir::tosa::mxint8Type::convertToAttribute(ArrayRef<char> rawData) const {
420 assert(rawData.size() == 1 && "expected 1 byte for tosa.mxint8 element");
421 const auto intType = IntegerType::get(context: getContext(), width: 8);
422 return intType.convertToAttribute(rawData);
423}
424
425LogicalResult mlir::tosa::mxint8Type::convertFromAttribute(
426 Attribute attr, SmallVectorImpl<char> &result) const {
427 const auto intAttr = dyn_cast<IntegerAttr>(Val&: attr);
428 if (!intAttr)
429 return failure();
430 const Type attrType = intAttr.getType();
431 if (!attrType.isSignlessInteger(width: 8))
432 return failure();
433 return cast<IntegerType>(Val: attrType).convertFromAttribute(attr, result);
434}
435
436//===----------------------------------------------------------------------===//
437// TOSA block scaling utilities.
438//===----------------------------------------------------------------------===//
439
440LogicalResult mlir::tosa::verifyBlockScaledTensorType(
441 mlir::Type type, llvm::function_ref<mlir::InFlightDiagnostic()> emitError,
442 bool allowScaleValues) {
443 if (!llvm::isa<ShapedType>(Val: type)) {
444 if (emitError)
445 emitError() << "non tensor type " << type << " is not allowed";
446 return failure();
447 }
448
449 const auto tensorType = llvm::cast<ShapedType>(Val&: type);
450 const BlockScaledType elemType =
451 llvm::dyn_cast<BlockScaledType>(Val: tensorType.getElementType());
452 if (!elemType)
453 return success();
454
455 if (!allowScaleValues && elemType.hasScaleValues()) {
456 if (emitError)
457 emitError()
458 << "block scaled tensor type with scale values is not allowed";
459 return failure();
460 }
461
462 if (!tensorType.hasRank())
463 return success();
464
465 if (tensorType.getRank() == 0) {
466 if (emitError)
467 emitError() << "block scaled tensor type must have rank greater than "
468 "zero";
469 return failure();
470 }
471
472 const ArrayRef<int64_t> tensorShape = tensorType.getShape();
473 const uint32_t blockSize =
474 BlockShapeAttr::getBlockShapeValue(blockShape: elemType.getBlockShape());
475
476 if (allowScaleValues && elemType.hasScaleValues() &&
477 tensorType.hasStaticShape()) {
478 const size_t numBlocks = tensorType.getNumElements() / blockSize;
479 if (elemType.getScaleValues().size() != numBlocks) {
480 if (emitError)
481 emitError() << "block scaled tensor type with scale values must have "
482 "scale values for each block, expected "
483 << numBlocks << ", got "
484 << elemType.getScaleValues().size();
485 return failure();
486 }
487 }
488
489 const int64_t blockedDimension = tensorShape.back();
490 if (ShapedType::isDynamic(dValue: blockedDimension))
491 return success();
492
493 if (blockedDimension % blockSize != 0) {
494 if (emitError)
495 emitError() << "last dimension of block scaled tensor type ("
496 << blockedDimension << ") must be divisible by block size ("
497 << blockSize << ")";
498
499 return failure();
500 }
501
502 return success();
503}
504
505std::string mlir::tosa::getTosaTensorTypeErrorMessage(mlir::Type type) {
506 MLIRContext *ctx = type.getContext();
507 std::string message;
508 ScopedDiagnosticHandler handler(
509 ctx, [&](Diagnostic &diag) { message = diag.str(); });
510
511 if (failed(Result: verifyBlockScaledTensorType(
512 type, emitError: [ctx] { return emitError(loc: UnknownLoc::get(context: ctx)); })) &&
513 !message.empty()) {
514 return ": " + message;
515 }
516
517 return "";
518}
519
520static ParseResult parseScaleValues(AsmParser &parser,
521 SmallVector<Attribute> &scaleValues,
522 Type scaleType) {
523 const auto parseScaleValue = [&]() -> ParseResult {
524 const SMLoc loc = parser.getCurrentLocation();
525
526 double floatValue;
527 if (parser.parseFloat(result&: floatValue))
528 return failure();
529
530 if (floatValue < 0.0)
531 return parser.emitError(loc, message: "scale value must be non-negative, got ")
532 << floatValue;
533
534 Type attrType = scaleType;
535 if (succeeded(Result: parser.parseOptionalColon()) && parser.parseType(result&: attrType))
536 return failure();
537
538 if (attrType != scaleType)
539 return parser.emitError(loc, message: "parsed attribute type ")
540 << attrType << " does not match expected scale type " << scaleType;
541
542 scaleValues.push_back(Elt: FloatAttr::get(type: attrType, value: floatValue));
543 return success();
544 };
545
546 return parser.parseCommaSeparatedList(parseElementFn: parseScaleValue);
547}
548
549static void printScaleValues(AsmPrinter &printer,
550 ArrayRef<Attribute> scaleValues, Type) {
551 llvm::interleaveComma(c: scaleValues, os&: printer, each_fn: [&](Attribute scaleValue) {
552 printer.printAttributeWithoutType(attr: scaleValue);
553 });
554}
555
556size_t mlir::tosa::BlockScaledType::getDenseElementBitSize() const {
557 const Type valueType = getValueType();
558 if (isa<MXI8Type, tosa::mxint8Type>(Val: valueType))
559 return 8;
560 return valueType.getIntOrFloatBitWidth();
561}
562
563Attribute
564mlir::tosa::BlockScaledType::convertToAttribute(ArrayRef<char> rawData) const {
565 // Block scaled values are stored as a single byte. This is because possible
566 // value data types are either 8-bit or sub-byte. Sub-byte types are aligned
567 // to 8-bits.
568 assert(rawData.size() == 1 && "expected 1 byte for block_scaled element");
569 const Type valueType = getValueType();
570 if (const auto mxi8Value = dyn_cast<MXI8Type>(Val: valueType))
571 return mxi8Value.convertToAttribute(rawData);
572 if (const auto mxint8Value = dyn_cast<tosa::mxint8Type>(Val: valueType))
573 return mxint8Value.convertToAttribute(rawData);
574 if (!isa<FloatType>(Val: valueType))
575 return {};
576 return mlir::detail::convertFloatTypeToAttribute(type: valueType, rawData);
577}
578
579LogicalResult mlir::tosa::BlockScaledType::convertFromAttribute(
580 Attribute attr, SmallVectorImpl<char> &result) const {
581 const Type valueType = getValueType();
582 if (const auto mxi8Value = dyn_cast<MXI8Type>(Val: valueType))
583 return mxi8Value.convertFromAttribute(attr, result);
584 if (const auto mxint8Value = dyn_cast<tosa::mxint8Type>(Val: valueType))
585 return mxint8Value.convertFromAttribute(attr, result);
586
587 const auto floatAttr = dyn_cast<FloatAttr>(Val&: attr);
588 if (!floatAttr || floatAttr.getType() != valueType)
589 return failure();
590 // const APFloat value = floatAttr.getValue();
591 return mlir::detail::convertFloatTypeFromAttribute(type: valueType, attr: floatAttr,
592 result);
593}
594
595//===----------------------------------------------------------------------===//
596// TOSA Operator shape inference
597//===----------------------------------------------------------------------===//
598
599template <typename A, std::enable_if_t<std::is_same_v<A, ArgMaxOp::Adaptor> ||
600 std::is_same_v<A, ArgMinOp::Adaptor>,
601 int> = 0>
602LogicalResult inferArgMaxMinReturnTypeComponents(
603 MLIRContext *context, ::std::optional<Location> location, A adaptor,
604 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
605 ShapeAdaptor inputShape(adaptor.getInput().getType());
606 IntegerAttr axis = adaptor.getProperties().axis;
607 int32_t axisVal = axis.getValue().getSExtValue();
608
609 if (!inputShape.hasRank()) {
610 inferredReturnShapes.push_back(Elt: ShapedTypeComponents());
611 return success();
612 }
613
614 const auto inputRank = inputShape.getRank();
615 SmallVector<int64_t> outShape;
616 outShape.reserve(N: inputRank - 1);
617 for (int i = 0, s = inputRank; i < s; i++) {
618 if (i == axisVal)
619 continue;
620 outShape.push_back(Elt: inputShape.getDimSize(index: i));
621 }
622
623 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outShape));
624 return success();
625}
626
627//===----------------------------------------------------------------------===//
628// TOSA Operator Verifiers.
629//===----------------------------------------------------------------------===//
630template <typename T>
631LogicalResult argMaxMinVerify(T op) {
632 const ShapedType resultType = llvm::cast<ShapedType>(op.getType());
633
634 if (const auto resultETy = resultType.getElementType();
635 !resultETy.isIntOrIndex())
636 return op.emitOpError("result tensor is not of integer type");
637
638 const auto inputType = llvm::cast<ShapedType>(op.getInput().getType());
639 if (!inputType.hasRank())
640 return success();
641
642 // Ensure axis is within the tensor rank
643 const int64_t axis = op.getAxisAttr().getInt();
644 if (((axis < 0) || axis >= inputType.getRank()))
645 return op.emitOpError("specified axis is outside the rank of the tensor");
646
647 if (!resultType.hasRank())
648 return success();
649
650 const ArrayRef<int64_t> inputShape = inputType.getShape();
651 const ArrayRef<int64_t> outputShape = resultType.getShape();
652 llvm::SmallVector<int64_t> expectedOutputShape(inputShape);
653 expectedOutputShape.erase(CI: expectedOutputShape.begin() + axis);
654 if (failed(Result: verifyCompatibleShape(shape1: expectedOutputShape, shape2: outputShape)))
655 return op.emitOpError("expected output shape '")
656 << expectedOutputShape << "', got '" << outputShape << "'";
657
658 return success();
659}
660
661template <typename T>
662static LogicalResult verifyConvOp(T op) {
663 const auto inputType = llvm::dyn_cast<TensorType>(op.getInput().getType());
664 const auto weightType = llvm::dyn_cast<TensorType>(op.getWeight().getType());
665
666 auto inputEType = inputType.getElementType();
667 auto weightEType = weightType.getElementType();
668 auto biasEType =
669 llvm::cast<ShapedType>(op.getBias().getType()).getElementType();
670 auto resultEType =
671 llvm::cast<ShapedType>(op.getResult().getType()).getElementType();
672 bool biasIsFloat = llvm::isa<FloatType>(biasEType);
673 bool resultIsFloat = llvm::isa<FloatType>(resultEType);
674
675 if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(inputEType))
676 inputEType = getStorageElementTypeFromQuantized(quantType);
677
678 if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(weightEType))
679 weightEType = getStorageElementTypeFromQuantized(quantType);
680
681 if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(biasEType))
682 biasEType = getStorageElementTypeFromQuantized(quantType);
683
684 if (auto quantType = llvm::dyn_cast<mlir::quant::QuantizedType>(resultEType))
685 resultEType = getStorageElementTypeFromQuantized(quantType);
686
687 if (biasIsFloat && resultIsFloat && (biasEType != resultEType)) {
688 // for now, only enforce bias element type == result element type for
689 // float types.
690 op.emitOpError(
691 "expect both bias and result to have same element type, got ")
692 << biasEType << " and " << resultEType;
693 return failure();
694 }
695
696 const bool isInputBlockScaled = llvm::isa<BlockScaledType>(inputEType);
697 const bool isWeightBlockScaled = llvm::isa<BlockScaledType>(weightEType);
698 const bool isInputFloat = llvm::isa<FloatType>(inputEType);
699 const bool isWeightFloat = llvm::isa<FloatType>(weightEType);
700
701 const bool isInputBSorFloat = isInputBlockScaled || isInputFloat;
702 const bool isWeightBSorFloat = isWeightBlockScaled || isWeightFloat;
703
704 // Either both must be float or both non-float.
705 if (isInputBSorFloat != isWeightBSorFloat) {
706 op.emitOpError(
707 "expect both input and weight to be float or not together, got ")
708 << inputEType << " and " << weightEType;
709 return failure();
710 }
711
712 auto inputZpEType = getStorageElementTypeOrSelf(op.getInputZp().getType());
713 if (!isInputBlockScaled && inputEType != inputZpEType) {
714 return op.emitOpError("expect both input and its zero point are the same "
715 "element type, got ")
716 << inputEType << " and " << inputZpEType;
717 }
718 if (isInputBlockScaled && !llvm::isa<Float32Type>(inputZpEType)) {
719 return op.emitOpError(
720 "expect block scaled input to have fp32 zero point, got ")
721 << inputEType << " and " << inputZpEType;
722 }
723
724 auto weightZpEType = getStorageElementTypeOrSelf(op.getWeightZp().getType());
725 if (!isWeightBlockScaled && weightEType != weightZpEType) {
726 return op.emitOpError("expect both weight and its zero point are the same "
727 "element type, got ")
728 << weightEType << " and " << weightZpEType;
729 }
730 if (isWeightBlockScaled && !llvm::isa<Float32Type>(weightZpEType)) {
731 return op.emitOpError(
732 "expect block scaled weight to have fp32 zero point, got ")
733 << weightEType << " and " << weightZpEType;
734 }
735
736 FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
737 if (succeeded(Result: maybeIZp) && op.verifyInputZeroPoint(*maybeIZp).failed())
738 return failure();
739
740 FailureOr<int64_t> maybeWZp = op.getWeightZeroPoint();
741 if (succeeded(Result: maybeWZp) && op.verifyWeightZeroPoint(*maybeWZp).failed())
742 return failure();
743
744 return success();
745}
746
747LogicalResult tosa::ConstOp::verify() {
748 Operation &op = *getOperation();
749 auto attrType = llvm::dyn_cast<TensorType>(Val: getValuesAttr().getType());
750 auto outputType = llvm::dyn_cast<TensorType>(Val: getOutput().getType());
751
752 if (!attrType || !outputType) {
753 emitOpError(message: "expected tensors for attr/result type");
754 return failure();
755 }
756
757 const Type attrElemType = attrType.getElementType();
758 const Type resultElemType = outputType.getElementType();
759
760 if (auto result =
761 llvm::dyn_cast<mlir::quant::QuantizedType>(Val: resultElemType)) {
762 if (getStorageElementTypeFromQuantized(quantizedType: result) == attrElemType)
763 return success();
764 }
765
766 if (auto attrBlockScaledType =
767 llvm::dyn_cast<mlir::tosa::BlockScaledType>(Val: attrElemType)) {
768 if (!attrBlockScaledType.hasScaleValues())
769 return op.emitOpError(
770 message: "attribute block scaled type must have scale values");
771
772 const auto emitAttributeError = [&op]() {
773 return op.emitOpError(message: "attribute block scaled type is invalid: ");
774 };
775
776 if (failed(Result: verifyBlockScaledTensorType(type: attrType, emitError: emitAttributeError, allowScaleValues: true)))
777 return failure();
778
779 const BlockScaledType resultBlockScaledType =
780 llvm::dyn_cast<mlir::tosa::BlockScaledType>(Val: resultElemType);
781 if (!resultBlockScaledType)
782 return op.emitOpError(
783 message: "result type must be block scaled type if attribute is block "
784 "scaled type");
785
786 if (attrBlockScaledType.getValueType() !=
787 resultBlockScaledType.getValueType() ||
788 attrBlockScaledType.getScaleType() !=
789 resultBlockScaledType.getScaleType() ||
790 attrBlockScaledType.getBlockShape() !=
791 resultBlockScaledType.getBlockShape())
792 return op.emitOpError(
793 message: "expected block scaled element type to be compatible "
794 "between attr and result, got ")
795 << attrBlockScaledType << " vs. " << resultBlockScaledType;
796
797 return success();
798 }
799
800 if (attrElemType != resultElemType)
801 return emitOpError(message: "expected same attr/result element types");
802
803 return success();
804}
805
806//===----------------------------------------------------------------------===//
807// ERROR_IF functions.
808// ERROR_IF is a predicate that must set an error if the condition holds.
809//===----------------------------------------------------------------------===//
810
811template <typename T>
812static LogicalResult verifyConvOpErrorIf(T op) {
813 llvm::ArrayRef<int64_t> padding = op.getPad();
814 if (llvm::any_of(padding, [](int64_t p) { return p < 0; }))
815 return op.emitOpError("expect all padding values to be >= 0, got ")
816 << padding;
817
818 llvm::ArrayRef<int64_t> strides = op.getStride();
819 if (llvm::any_of(strides, [](int64_t s) { return s < 1; }))
820 return op.emitOpError("expect all stride values to be >= 1, got ")
821 << strides;
822
823 llvm::ArrayRef<int64_t> dilations = op.getDilation();
824 if (llvm::any_of(dilations, [](int64_t d) { return d < 1; }))
825 return op.emitOpError("expect all dilation values to be >= 1, got ")
826 << dilations;
827
828 const RankedTensorType outputType =
829 llvm::dyn_cast<RankedTensorType>(op.getOutput().getType());
830 if (!outputType)
831 // Skip following checks if output is not ranked
832 return success();
833
834 const RankedTensorType inputType =
835 llvm::dyn_cast<RankedTensorType>(op.getInput().getType());
836 const RankedTensorType weightType =
837 llvm::dyn_cast<RankedTensorType>(op.getWeight().getType());
838
839 if (inputType && weightType) {
840 // input = [_,IH,IW,_], weight = [_,KH,KW,_], output = [_,OH,OW,_]
841 if constexpr (std::is_same<T, tosa::Conv2DOp>::value) {
842 if (failed(verifyConvOutputSize(
843 op, inputType.getDimSize(idx: 1), weightType.getDimSize(idx: 1),
844 outputType.getDimSize(idx: 1), padding[0], padding[1], strides[0],
845 dilations[0], "height", "y", "top", "bottom")))
846 return failure();
847
848 if (failed(verifyConvOutputSize(
849 op, inputType.getDimSize(idx: 2), weightType.getDimSize(idx: 2),
850 outputType.getDimSize(idx: 2), padding[2], padding[3], strides[1],
851 dilations[1], "width", "x", "left", "right")))
852 return failure();
853 }
854
855 // input = [_,IH,IW,_], weight = [KH,KW,_,_], output = [_,OH,OW,_]
856 if constexpr (std::is_same<T, tosa::DepthwiseConv2DOp>::value) {
857 if (failed(verifyConvOutputSize(
858 op, inputType.getDimSize(idx: 1), weightType.getDimSize(idx: 0),
859 outputType.getDimSize(idx: 1), padding[0], padding[1], strides[0],
860 dilations[0], "height", "y", "top", "bottom")))
861 return failure();
862
863 if (failed(verifyConvOutputSize(
864 op, inputType.getDimSize(idx: 2), weightType.getDimSize(idx: 1),
865 outputType.getDimSize(idx: 2), padding[2], padding[3], strides[1],
866 dilations[1], "width", "x", "left", "right")))
867 return failure();
868 }
869
870 // input = [_,ID,IH,IW,_], weight = [_,KD,KH,KW,_], output = [_,OD,OH,OW,_]
871 if constexpr (std::is_same<T, tosa::Conv3DOp>::value) {
872 if (failed(verifyConvOutputSize(
873 op, inputType.getDimSize(idx: 1), weightType.getDimSize(idx: 1),
874 outputType.getDimSize(idx: 1), padding[0], padding[1], strides[0],
875 dilations[0], "depth", "d", "front", "back")))
876 return failure();
877
878 if (failed(verifyConvOutputSize(
879 op, inputType.getDimSize(idx: 2), weightType.getDimSize(idx: 2),
880 outputType.getDimSize(idx: 2), padding[2], padding[3], strides[1],
881 dilations[1], "height", "y", "top", "bottom")))
882 return failure();
883
884 if (failed(verifyConvOutputSize(
885 op, inputType.getDimSize(idx: 3), weightType.getDimSize(idx: 3),
886 outputType.getDimSize(idx: 3), padding[4], padding[5], strides[2],
887 dilations[2], "width", "x", "left", "right")))
888 return failure();
889 }
890 }
891
892 const RankedTensorType biasType =
893 llvm::dyn_cast<RankedTensorType>(op.getBias().getType());
894 if (!biasType)
895 // Skip following checks if bias is not ranked
896 return success();
897
898 const int64_t biasChannels = biasType.getDimSize(idx: 0);
899 const int64_t outputChannels =
900 outputType.getDimSize(idx: outputType.getRank() - 1);
901 if (biasChannels == ShapedType::kDynamic ||
902 outputChannels == ShapedType::kDynamic)
903 // Skip following checks if biasChannels or outputChannels is dynamic dim
904 return success();
905
906 if (biasChannels != outputChannels && biasChannels != 1)
907 return op.emitOpError(
908 "bias channels expected to be equal to output channels (")
909 << outputChannels << ") or 1, got " << biasChannels;
910
911 return success();
912}
913
914// Verify whether same type and shape of the given two types.
915static LogicalResult errorIfTypeOrShapeMismatch(Operation *op, Type type1,
916 StringRef name1, Type type2,
917 StringRef name2) {
918 auto shapeType1 = dyn_cast<ShapedType>(Val&: type1);
919 auto shapeType2 = dyn_cast<ShapedType>(Val&: type2);
920 if (!shapeType1 || !shapeType2)
921 return failure();
922
923 auto elemType1 = shapeType1.getElementType();
924 auto elemType2 = shapeType2.getElementType();
925 if (elemType1 != elemType2)
926 return op->emitOpError()
927 << "require same element type for " << name1 << " (" << elemType1
928 << ") and " << name2 << " (" << elemType2 << ")";
929
930 if (failed(Result: verifyCompatibleShape(type1, type2)))
931 return op->emitOpError()
932 << "require same shapes for " << name1 << " (" << type1 << ") and "
933 << name2 << " (" << type2 << ")";
934
935 return success();
936}
937
938// Verify whether same length, type, and shape of the given two tensor lists.
939static LogicalResult errorIfTypeOrShapeMismatch(Operation *op, ValueRange list1,
940 StringRef name1,
941 ValueRange list2,
942 StringRef name2) {
943 if (list1.size() != list2.size())
944 return op->emitOpError()
945 << "require same number of values in " << name1 << " ("
946 << list1.size() << ") and " << name2 << " (" << list2.size() << ")";
947
948 for (auto [type1, type2] :
949 llvm::zip_equal(t: list1.getTypes(), u: list2.getTypes())) {
950 if (errorIfTypeOrShapeMismatch(op, type1, name1, type2, name2).failed())
951 return failure();
952 }
953
954 return success();
955}
956
957static inline LogicalResult errorIfShapeNotSizeOne(Operation *op, Type type) {
958 ShapeAdaptor shapeAdaptor(type);
959 if (!shapeAdaptor.hasRank() || !shapeAdaptor.hasStaticShape())
960 return success();
961
962 return shapeAdaptor.getNumElements() == 1 ? success() : failure();
963}
964
965template <typename T>
966static LogicalResult verifyVariableOpErrorIf(T op, Type type, StringRef name) {
967 Operation *symTableOp =
968 op->template getParentWithTrait<OpTrait::SymbolTable>();
969 if (!symTableOp)
970 // If the operation is not the scope of a symbol table, we cannot
971 // verify it against it's declaration.
972 return success();
973
974 SymbolTable symTable(symTableOp);
975 const auto varOp = symTable.lookup<tosa::VariableOp>(op.getName());
976
977 // Verify prior declaration
978 if (!varOp)
979 return op->emitOpError("'")
980 << op.getName() << "' has not been declared by 'tosa.variable'";
981
982 // Verify type and shape
983 auto variableType = getVariableType(varOp);
984 if (errorIfTypeOrShapeMismatch(op, type, name, variableType,
985 "the input tensor")
986 .failed())
987 return failure();
988 return success();
989}
990
991// verify that inType and outType have same element types
992static LogicalResult verifySameElementTypes(Operation *op, Type aType,
993 Type bType,
994 StringRef aName = "input",
995 StringRef bName = "output") {
996 auto aTType = llvm::dyn_cast<TensorType>(Val&: aType);
997 auto bTType = llvm::dyn_cast<TensorType>(Val&: bType);
998 if (!aTType) {
999 op->emitOpError(message: "expect shaped tensor for") << aName << ", got " << aType;
1000 return failure();
1001 }
1002 if (!bTType) {
1003 op->emitOpError(message: "expect shaped tensor for") << bName << ", got" << bType;
1004 return failure();
1005 }
1006 auto aElementType = aTType.getElementType();
1007 auto bElementType = bTType.getElementType();
1008 auto aQuantType =
1009 llvm::dyn_cast<mlir::quant::UniformQuantizedType>(Val&: aElementType);
1010 auto bQuantType =
1011 llvm::dyn_cast<mlir::quant::UniformQuantizedType>(Val&: bElementType);
1012 if ((aElementType.isIntOrIndexOrFloat() || aQuantType) &&
1013 (bElementType.isIntOrIndexOrFloat() || bQuantType) &&
1014 aElementType != bElementType) {
1015 // only check if both element types are int/index/float/UniformQuantized
1016 // eg, not sure how to check quant::QuantizedType
1017 // this happens in test_conv2d_q_grouped_convolution in
1018 // tfl-to-tosa-pipeline.mlir
1019 op->emitOpError(message: "expect ")
1020 << aName << " and " << bName << " to have same element type, got "
1021 << aElementType << " and " << bElementType;
1022 return failure();
1023 }
1024 return success();
1025}
1026
1027LogicalResult tosa::ArgMaxOp::verify() { return argMaxMinVerify(op: *this); }
1028
1029LogicalResult tosa::ArgMinOp::verify() { return argMaxMinVerify(op: *this); }
1030
1031static LogicalResult verifyPoolingOpImpl(Operation *op,
1032 ArrayRef<int64_t> kernel,
1033 ArrayRef<int64_t> strides,
1034 ArrayRef<int64_t> padding, Value input,
1035 Value output) {
1036 if (failed(Result: verifySameElementTypes(op, aType: input.getType(), bType: output.getType())))
1037 return failure();
1038
1039 const bool hasKernel = kernel.size() > 0;
1040 const bool hasStrides = strides.size() > 0;
1041 const bool hasPad = padding.size() > 0;
1042
1043 if (hasKernel && llvm::any_of(Range&: kernel, P: [](int64_t s) { return s < 1; }))
1044 return op->emitOpError(message: "expect all kernel values to be >= 1, got ")
1045 << kernel;
1046
1047 if (hasStrides && llvm::any_of(Range&: strides, P: [](int64_t s) { return s < 1; }))
1048 return op->emitOpError(message: "expect all stride values to be >= 1, got ")
1049 << strides;
1050
1051 if (hasPad && llvm::any_of(Range&: padding, P: [](int64_t p) { return p < 0; }))
1052 return op->emitOpError(message: "expect all padding values to be >= 0, got ")
1053 << padding;
1054
1055 if (hasKernel && hasPad) {
1056 // Padding must be less than kernel size to avoid a divide-by-zero
1057 const int64_t kernelX = kernel[1];
1058 const int64_t padLeft = padding[2];
1059 const int64_t padRight = padding[3];
1060 if (padRight >= kernelX || padLeft >= kernelX)
1061 return op->emitOpError(message: "expected left/right padding to be less than the "
1062 "width of the kernel, got pad_left=")
1063 << padLeft << ", pad_right=" << padRight
1064 << ", kernel_x=" << kernelX;
1065
1066 const int64_t kernelY = kernel[0];
1067 const int64_t padTop = padding[0];
1068 const int64_t padBottom = padding[1];
1069 if (padTop >= kernelY || padBottom >= kernelY)
1070 return op->emitOpError(message: "expected top/bottom padding to be less than the "
1071 "height of the kernel, got pad_top=")
1072 << padTop << ", pad_bottom=" << padBottom
1073 << ", kernel_y=" << kernelY;
1074 }
1075
1076 const auto inputType = llvm::dyn_cast<RankedTensorType>(Val: input.getType());
1077 const auto outputType = llvm::dyn_cast<RankedTensorType>(Val: output.getType());
1078 if (!inputType || !outputType)
1079 return success();
1080
1081 if (hasKernel && hasStrides && hasPad) {
1082 const auto verifyOutputSize =
1083 [op](const int64_t inputSize, const int64_t outputSize,
1084 const int64_t kernelSize, const int64_t strideSize,
1085 const int64_t padBefore, const int64_t padAfter,
1086 const llvm::StringRef dimName, const llvm::StringRef dimAxis,
1087 const llvm::StringRef padBeforeName,
1088 const llvm::StringRef padAfterName) -> LogicalResult {
1089 if (ShapedType::isDynamic(dValue: inputSize))
1090 return success();
1091
1092 const std::optional<int64_t> calculatedOutSizeMinusOne =
1093 idivCheck(lhs: inputSize + padBefore + padAfter - kernelSize, rhs: strideSize);
1094 if (!calculatedOutSizeMinusOne.has_value())
1095 return op->emitOpError(message: "expected input_")
1096 << dimName << " + pad_" << padBeforeName << " + pad_"
1097 << padAfterName << " - kernel_" << dimAxis
1098 << " to be wholly divisible by stride_" << dimAxis << ", got ("
1099 << inputSize << " + " << padBefore << " + " << padAfter << " - "
1100 << kernelSize << ") / " << strideSize;
1101
1102 const int64_t calculatedOutSize = calculatedOutSizeMinusOne.value() + 1;
1103 if (ShapedType::isStatic(dValue: outputSize) && calculatedOutSize != outputSize)
1104 return op->emitOpError(message: "calculated output ")
1105 << dimName << " did not match expected: " << "calculated="
1106 << calculatedOutSize << ", expected=" << outputSize;
1107
1108 return success();
1109 };
1110
1111 if (failed(Result: verifyOutputSize(inputType.getDimSize(idx: 1),
1112 outputType.getDimSize(idx: 1), kernel[0], strides[0],
1113 padding[0], padding[1], "height", "y", "top",
1114 "bottom")))
1115 return failure();
1116
1117 if (failed(Result: verifyOutputSize(
1118 inputType.getDimSize(idx: 2), outputType.getDimSize(idx: 2), kernel[1],
1119 strides[1], padding[2], padding[3], "width", "x", "left", "right")))
1120 return failure();
1121 }
1122 return success();
1123}
1124
1125template <typename T>
1126static LogicalResult verifyPoolingOp(T op) {
1127 return verifyPoolingOpImpl(op.getOperation(), op.getKernel(), op.getStride(),
1128 op.getPad(), op.getInput(), op.getOutput());
1129}
1130
1131template <typename T>
1132static LogicalResult verifyAvgPoolCommonTypeAndZpChecks(T op) {
1133 const Type inputETy = getStorageElementTypeOrSelf(op.getInput().getType());
1134 const Type resultETy = getStorageElementTypeOrSelf(op.getOutput().getType());
1135 const Type inputZpETy =
1136 getStorageElementTypeOrSelf(op.getInputZp().getType());
1137 const Type outputZpETy =
1138 getStorageElementTypeOrSelf(op.getOutputZp().getType());
1139
1140 auto accType = op.getAccType();
1141 if (llvm::isa<IntegerType>(Val: inputETy) && !accType.isInteger(32))
1142 return op.emitOpError("accumulator type for integer tensor is not i32");
1143
1144 if (inputETy.isF16() && !(accType.isF16() || accType.isF32()))
1145 return op.emitOpError("accumulator type for f16 tensor is not f16/f32");
1146
1147 if (inputETy.isBF16() && !accType.isF32())
1148 return op.emitOpError("accumulator type for bf16 tensor is not f32");
1149
1150 if (inputETy.isF32() && !accType.isF32())
1151 return op.emitOpError("accumulator type for f32 tensor is not f32");
1152
1153 if (inputETy != inputZpETy)
1154 return op.emitOpError("expect both input and its zero point are the same "
1155 "element type, got ")
1156 << inputETy << " and " << inputZpETy;
1157
1158 if (resultETy != outputZpETy)
1159 return op.emitOpError("expect both output and its zero point are the same "
1160 "element type, got ")
1161 << resultETy << " and " << outputZpETy;
1162
1163 FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
1164 if (succeeded(Result: maybeIZp) && op.verifyInputZeroPoint(*maybeIZp).failed())
1165 return failure();
1166
1167 FailureOr<int64_t> maybeOZp = op.getOutputZeroPoint();
1168 if (succeeded(Result: maybeOZp) && op.verifyOutputZeroPoint(*maybeOZp).failed())
1169 return failure();
1170
1171 return success();
1172}
1173
1174namespace {
1175struct AdaptivePoolingConstShapeValues {
1176 llvm::SmallVector<int64_t> kernel;
1177 llvm::SmallVector<int64_t> stride;
1178 llvm::SmallVector<int64_t> pad;
1179};
1180} // namespace
1181
1182template <typename T>
1183static constexpr bool IsSupportedAdaptivePoolConstShapeVerifyOp =
1184 std::is_same_v<T, tosa::AvgPool2dAdaptiveOp> ||
1185 std::is_same_v<T, tosa::MaxPool2dAdaptiveOp>;
1186
1187template <typename T,
1188 typename std::enable_if<IsSupportedAdaptivePoolConstShapeVerifyOp<T>,
1189 int>::type = 0>
1190static void extractAdaptivePoolingConstShapeOperands(
1191 T op, AdaptivePoolingConstShapeValues &values) {
1192 tosa::getConstShapeValues(op: op.getKernel().getDefiningOp(), result_shape&: values.kernel);
1193 tosa::getConstShapeValues(op: op.getStride().getDefiningOp(), result_shape&: values.stride);
1194 tosa::getConstShapeValues(op: op.getPad().getDefiningOp(), result_shape&: values.pad);
1195}
1196
1197LogicalResult tosa::AvgPool2dOp::verify() {
1198 if (failed(Result: verifyPoolingOp(op: *this)))
1199 return failure();
1200 if (failed(Result: verifyAvgPoolCommonTypeAndZpChecks(op: *this)))
1201 return failure();
1202 return success();
1203}
1204
1205LogicalResult tosa::AvgPool2dAdaptiveOp::verify() {
1206 AdaptivePoolingConstShapeValues values;
1207 extractAdaptivePoolingConstShapeOperands(op: *this, values);
1208
1209 // If pad/stride/kernel are not constant, this is okay, we just can't check
1210 // their values. extractAdaptivePoolingConstShapeOperands will return an empty
1211 // list for each non CTC input. verifyPoolingOpImpl will need to handle values
1212 // not being present, and return success if they cannot be checked.
1213
1214 if (failed(Result: verifyPoolingOpImpl(op: getOperation(), kernel: values.kernel, strides: values.stride,
1215 padding: values.pad, input: getInput(), output: getOutput())))
1216 return failure();
1217
1218 if (failed(Result: verifyAvgPoolCommonTypeAndZpChecks(op: *this)))
1219 return failure();
1220
1221 return success();
1222}
1223
1224LogicalResult tosa::ClampOp::verify() {
1225 mlir::Type inputETy =
1226 llvm::cast<ShapedType>(Val: getInput().getType()).getElementType();
1227 if (auto quantType =
1228 llvm::dyn_cast<mlir::quant::UniformQuantizedType>(Val&: inputETy)) {
1229 inputETy = getStorageElementTypeFromQuantized(quantizedType: quantType);
1230 }
1231 mlir::Type outputETy =
1232 llvm::cast<ShapedType>(Val: getOutput().getType()).getElementType();
1233 if (auto quantType =
1234 llvm::dyn_cast<mlir::quant::UniformQuantizedType>(Val&: outputETy)) {
1235 outputETy = getStorageElementTypeFromQuantized(quantizedType: quantType);
1236 }
1237 if (inputETy != outputETy)
1238 return emitOpError(message: "input/output element types are incompatible.");
1239
1240 auto maxValAttr = getMaxValAttr();
1241 auto minValAttr = getMinValAttr();
1242
1243 unsigned dataTypeBitWidth = inputETy.getIntOrFloatBitWidth();
1244
1245 if (inputETy.isInteger(width: dataTypeBitWidth)) {
1246 // if input datatype is integer, check that the min_val/max_val attributes
1247 // are integer attributes, and that their type is the same as the input's
1248 // datatype
1249 auto intMaxValAttr = mlir::dyn_cast<mlir::IntegerAttr>(Val&: maxValAttr);
1250 auto intMinValAttr = mlir::dyn_cast<mlir::IntegerAttr>(Val&: minValAttr);
1251 if (!intMaxValAttr || !intMinValAttr ||
1252 (intMaxValAttr.getType() != intMinValAttr.getType()) ||
1253 (intMaxValAttr.getType() != inputETy))
1254 return emitOpError(message: "min/max attributes types are incompatible with "
1255 "input/output element types.");
1256
1257 const bool isUnsigned = inputETy.isUnsignedInteger();
1258 const bool isBoolean = inputETy.isInteger(width: 1);
1259 const APInt minVal = intMinValAttr.getValue();
1260 const APInt maxVal = intMaxValAttr.getValue();
1261 if ((isUnsigned || isBoolean) ? maxVal.ult(RHS: minVal) : maxVal.slt(RHS: minVal))
1262 return emitOpError(message: "expected min_val <= max_val, got min_val=")
1263 << minValAttr << ", max_val=" << maxValAttr;
1264 } else {
1265 // otherwise, input datatype is float, check that the min_val/max_val
1266 // attributes share the same type and that their type is the same as the
1267 // input's datatype
1268 auto floatMaxValAttr = mlir::dyn_cast<mlir::FloatAttr>(Val&: maxValAttr);
1269 auto floatMinValAttr = mlir::dyn_cast<mlir::FloatAttr>(Val&: minValAttr);
1270 if (!floatMaxValAttr || !floatMinValAttr ||
1271 (floatMaxValAttr.getType() != floatMinValAttr.getType()) ||
1272 (floatMaxValAttr.getType() != inputETy))
1273 return emitOpError(message: "min/max attributes types are incompatible with "
1274 "input/output element types.");
1275
1276 const APFloat minVal = floatMinValAttr.getValue();
1277 const APFloat maxVal = floatMaxValAttr.getValue();
1278 if (minVal.isNaN() || maxVal.isNaN())
1279 return emitOpError(message: "min/max attributes should not be 'NaN', got min_val=")
1280 << minValAttr << ", max_val=" << maxValAttr;
1281
1282 if (maxVal < minVal)
1283 return emitOpError(message: "expected min_val <= max_val, got min_val=")
1284 << minValAttr << ", max_val=" << maxValAttr;
1285 }
1286
1287 return success();
1288}
1289
1290//===----------------------------------------------------------------------===//
1291// TOSA Operator Quantization Builders.
1292//===----------------------------------------------------------------------===//
1293
1294/// This builder is called on all convolution operators except TransposeConv,
1295/// which has specialized output shape semantics. The builder also defines the
1296/// bitwidth of the output given the bit width of the input & weight content.
1297static void buildConvOpWithQuantInfo(OpBuilder &builder, OperationState &result,
1298 Type outputType, Value input, Value weight,
1299 Value bias, DenseI64ArrayAttr pad,
1300 DenseI64ArrayAttr stride,
1301 DenseI64ArrayAttr dilation,
1302 TypeAttr accType) {
1303 auto zps = createZPsAsConst(builder, input, weight);
1304 result.addOperands(newOperands: {input, weight, bias, zps.first, zps.second});
1305 result.addAttribute(name: "pad", attr: pad);
1306 result.addAttribute(name: "stride", attr: stride);
1307 result.addAttribute(name: "dilation", attr: dilation);
1308 result.addAttribute(name: "acc_type", attr: accType);
1309 Type finalOutputType = outputType;
1310 auto quantAttr = buildConvOpQuantizationAttr(builder, input, weight);
1311 if (quantAttr) {
1312 finalOutputType =
1313 buildConvOpResultTypeInfo(builder, outputType, input, weight);
1314 }
1315 result.addTypes(newTypes: finalOutputType);
1316}
1317
1318/// Handles tosa.transpose_conv2d which has outpad and output shape
1319/// attributes.
1320static void
1321buildTransConvOpWithQuantInfo(OpBuilder &builder, OperationState &result,
1322 Type outputType, Value input, Value weight,
1323 Value bias, DenseI64ArrayAttr outpad,
1324 DenseI64ArrayAttr stride, TypeAttr accType) {
1325 auto zps = createZPsAsConst(builder, input, weight);
1326 result.addOperands(newOperands: {input, weight, bias, zps.first, zps.second});
1327 result.addAttribute(name: "out_pad", attr: outpad);
1328 result.addAttribute(name: "stride", attr: stride);
1329 result.addAttribute(name: "acc_type", attr: accType);
1330 Type finalOutputType = outputType;
1331 auto quantAttr = buildConvOpQuantizationAttr(builder, input, weight);
1332 if (quantAttr) {
1333 finalOutputType =
1334 buildConvOpResultTypeInfo(builder, outputType, input, weight);
1335 }
1336 result.addTypes(newTypes: finalOutputType);
1337}
1338
1339static void buildMatMulLikeOpWithQuantInfo(OpBuilder &builder,
1340 OperationState &result,
1341 Type outputType, Value a, Value b) {
1342 const std::pair<Value, Value> zps = createZPsAsConst(builder, input: a, weight: b);
1343 result.addOperands(newOperands: {a, b, zps.first, zps.second});
1344
1345 Type finalOutputType{outputType};
1346 if (buildMatMulOpQuantizationAttr(builder, a, b)) {
1347 auto eType = getStorageElementTypeOrSelf(type: a.getType());
1348 auto inputBits = eType.getIntOrFloatBitWidth();
1349
1350 auto outputShapedType = llvm::dyn_cast<ShapedType>(Val&: outputType);
1351 assert(outputShapedType && "Output must be a shaped type");
1352
1353 IntegerType accElementType;
1354 if (inputBits == 16)
1355 accElementType = builder.getIntegerType(width: 48);
1356 else
1357 accElementType = builder.getI32Type();
1358
1359 finalOutputType = outputShapedType.clone(elementType: accElementType);
1360 }
1361 result.addTypes(newTypes: finalOutputType);
1362}
1363
1364static void buildMatMulOpWithQuantInfo(OpBuilder &builder,
1365 OperationState &result, Type outputType,
1366 Value a, Value b) {
1367 buildMatMulLikeOpWithQuantInfo(builder, result, outputType, a, b);
1368}
1369
1370static void buildMatMulTOpWithQuantInfo(OpBuilder &builder,
1371 OperationState &result, Type outputType,
1372 Value a, Value b) {
1373 buildMatMulLikeOpWithQuantInfo(builder, result, outputType, a, b);
1374}
1375
1376/// Both the tosa.avg_pool2d and unary ops use the same
1377/// UnaryOpQuantizationAttr but avg_pool operator has its own builder as it
1378/// has additional parameters not part of the unary ops.
1379static void
1380buildAvgPool2dOpWithQuantInfo(OpBuilder &builder, OperationState &result,
1381 Type outputType, Value input,
1382 DenseArrayAttr kernel, DenseArrayAttr stride,
1383 DenseArrayAttr pad, TypeAttr accType) {
1384 const Location loc{result.location};
1385 int64_t inputZp{0};
1386 int64_t outputZp{0};
1387
1388 if (auto quantAttr =
1389 buildUnaryOpQuantizationAttr(builder, input, outputRawType: outputType)) {
1390 inputZp = quantAttr.getInputZp();
1391 outputZp = quantAttr.getOutputZp();
1392 }
1393 const std::optional<Value> inputZpOp =
1394 createZeroPointTensor(builder, loc, srcElemType: input.getType(), zp: inputZp);
1395 if (!inputZpOp) {
1396 (void)emitError(
1397 loc,
1398 message: "Failed to create input zero point tensor for quantized AVG_POOL2D op");
1399 }
1400 const std::optional<Value> outputZpOp =
1401 createZeroPointTensor(builder, loc, srcElemType: outputType, zp: outputZp);
1402 if (!outputZpOp) {
1403 (void)emitError(loc, message: "Failed to create output zero point tensor for "
1404 "quantized AVG_POOL2D op");
1405 }
1406
1407 if (inputZpOp && outputZpOp) {
1408 result.addOperands(newOperands: {input, inputZpOp.value(), outputZpOp.value()});
1409 } else {
1410 // failed to create one or more zero points above: just add input as
1411 // operands this will trigger error in building the op because of missing
1412 // zero points
1413 result.addOperands(newOperands: {input});
1414 }
1415 result.addAttribute(name: "kernel", attr: kernel);
1416 result.addAttribute(name: "stride", attr: stride);
1417 result.addAttribute(name: "pad", attr: pad);
1418 result.addAttribute(name: "acc_type", attr: accType);
1419 result.types.push_back(Elt: outputType);
1420}
1421
1422/// This builder mirrors avg_pool2d quant-info handling and materializes
1423/// kernel/stride/pad as const_shape operands for avg_pool2d_adaptive.
1424static void buildAvgPool2dAdaptiveOpWithQuantInfo(
1425 OpBuilder &builder, OperationState &result, Type outputType, Value input,
1426 DenseI64ArrayAttr kernel, DenseI64ArrayAttr stride, DenseI64ArrayAttr pad,
1427 TypeAttr accType) {
1428 const Location loc{result.location};
1429 int64_t inputZp{0};
1430 int64_t outputZp{0};
1431
1432 if (auto quantAttr =
1433 buildUnaryOpQuantizationAttr(builder, input, outputRawType: outputType)) {
1434 inputZp = quantAttr.getInputZp();
1435 outputZp = quantAttr.getOutputZp();
1436 }
1437 const std::optional<Value> inputZpOp =
1438 createZeroPointTensor(builder, loc, srcElemType: input.getType(), zp: inputZp);
1439 if (!inputZpOp) {
1440 (void)emitError(loc,
1441 message: "Failed to create input zero point tensor for quantized "
1442 "AVG_POOL2D_ADAPTIVE op");
1443 }
1444 const std::optional<Value> outputZpOp =
1445 createZeroPointTensor(builder, loc, srcElemType: outputType, zp: outputZp);
1446 if (!outputZpOp) {
1447 (void)emitError(loc, message: "Failed to create output zero point tensor for "
1448 "quantized AVG_POOL2D_ADAPTIVE op");
1449 }
1450
1451 if (inputZpOp && outputZpOp) {
1452 ImplicitLocOpBuilder b(loc, builder);
1453 Value kernelShape = getTosaConstShape(builder&: b, shape: kernel.asArrayRef());
1454 Value strideShape = getTosaConstShape(builder&: b, shape: stride.asArrayRef());
1455 Value padShape = getTosaConstShape(builder&: b, shape: pad.asArrayRef());
1456 result.addOperands(newOperands: {input, inputZpOp.value(), outputZpOp.value(),
1457 kernelShape, strideShape, padShape});
1458 } else {
1459 // Failed to create one or more zero points above: just add input as
1460 // operands. This will trigger error in building the op because of missing
1461 // operands.
1462 result.addOperands(newOperands: {input});
1463 }
1464 result.addAttribute(name: "acc_type", attr: accType);
1465 result.types.push_back(Elt: outputType);
1466}
1467
1468/// This builder is called on single-parameter negate operator
1469/// to construct input and output zero points based on their
1470/// types.
1471static void buildNegateOpWithQuantInfo(OpBuilder &builder,
1472 OperationState &result, Type outputType,
1473 Value input) {
1474 const Location loc{result.location};
1475 int64_t input1Zp{0};
1476 int64_t outputZp{0};
1477 auto quantAttr = buildUnaryOpQuantizationAttr(builder, input, outputRawType: outputType);
1478 if (quantAttr) {
1479 input1Zp = quantAttr.getInputZp();
1480 outputZp = quantAttr.getOutputZp();
1481 }
1482 const std::optional<Value> input1ZpOp =
1483 createZeroPointTensor(builder, loc, srcElemType: input.getType(), zp: input1Zp);
1484 if (!input1ZpOp) {
1485 (void)emitError(
1486 loc, message: "Failed to create input1 zero point for quantized NEGATE op");
1487 }
1488
1489 const std::optional<Value> outputZpOp =
1490 createZeroPointTensor(builder, loc, srcElemType: input.getType(), zp: outputZp);
1491 if (!outputZpOp) {
1492 (void)emitError(
1493 loc, message: "Failed to create output zero point for quantized NEGATE op");
1494 }
1495
1496 if (input1ZpOp && outputZpOp) {
1497 result.addOperands(newOperands: {input, input1ZpOp.value(), outputZpOp.value()});
1498 } else {
1499 // failed to create one or more zero points above: just add input as
1500 // operands. This will trigger error in building the op because of
1501 // missing zero points
1502 result.addOperands(newOperands: {input});
1503 }
1504
1505 result.types.push_back(Elt: outputType);
1506}
1507
1508/// This builder is called on TOSA pad operator that needs to create its own
1509/// OptionalAttr quantization_attr parameter to scale the padding values
1510/// correctly. No pad_const is interpreted as zero-padding.
1511static void buildPadOpWithQuantInfo(OpBuilder &builder, OperationState &result,
1512 Type outputType, Value input,
1513 Value paddings) {
1514 const Location loc{result.location};
1515 int32_t zp{0};
1516 const auto quantAttr = buildPadOpQuantizationAttr(builder, input);
1517 if (quantAttr) {
1518 zp = static_cast<int32_t>(quantAttr.getInputZp());
1519 }
1520 const auto padConstOp{createPadConstTensor(builder, loc, src: input, val: zp)};
1521 result.addOperands(newOperands: {input, paddings, padConstOp});
1522 result.types.push_back(Elt: outputType);
1523}
1524
1525static void buildVariableOp(OpBuilder &builder, OperationState &result,
1526 StringRef name, Type variableType,
1527 Attribute initialValue) {
1528 const Location loc{result.location};
1529 auto nameAttr = builder.getStringAttr(bytes: name);
1530
1531 auto shapedType = dyn_cast<ShapedType>(Val&: variableType);
1532 if (!shapedType) {
1533 (void)emitError(loc, message: "variable type must be a shaped type");
1534 return;
1535 }
1536 if (!shapedType.hasRank()) {
1537 (void)emitError(loc, message: "variable type must be a ranked type");
1538 return;
1539 }
1540
1541 auto elementType = shapedType.getElementType();
1542 auto elementTypeAttr = TypeAttr::get(type: elementType);
1543 ArrayRef<int64_t> shape = shapedType.getShape();
1544 auto varShapeAttr = builder.getIndexTensorAttr(values: convertFromMlirShape(shape));
1545
1546 result.addAttribute(name: "sym_name", attr: nameAttr);
1547 result.addAttribute(name: "var_shape", attr: varShapeAttr);
1548 result.addAttribute(name: "type", attr: elementTypeAttr);
1549 result.addAttribute(name: "initial_value", attr: initialValue);
1550}
1551
1552//===----------------------------------------------------------------------===//
1553// TOSA Operator Return Type Inference.
1554//===----------------------------------------------------------------------===//
1555static FailureOr<int64_t> resolveBroadcastDim(const int64_t dim1,
1556 const int64_t dim2) {
1557 if (dim1 == 1)
1558 return dim2;
1559 if (dim2 == 1)
1560 return dim1;
1561
1562 if (ShapedType::isStatic(dValue: dim1) && ShapedType::isStatic(dValue: dim2) && dim1 != dim2)
1563 return failure();
1564
1565 // Prefer static dimension over dynamic
1566 return ShapedType::isDynamic(dValue: dim1) ? dim2 : dim1;
1567}
1568
1569static LogicalResult resolveBroadcastShape(const ValueShapeRange &operands,
1570 SmallVector<int64_t> &outShape) {
1571 int64_t outRank = 0;
1572 for (int i = 0, e = operands.size(); i != e; ++i) {
1573 auto shape = operands.getShape(index: i);
1574 if (!shape.hasRank()) {
1575 // TODO(jennik): Update function to have better case handling for
1576 // invalid operands and for ranked tensors.
1577 return failure();
1578 }
1579 outRank = std::max<int64_t>(a: outRank, b: shape.getRank());
1580 }
1581
1582 outShape.resize(N: outRank, NV: 1);
1583
1584 for (int i = 0, e = operands.size(); i != e; ++i) {
1585 auto shape = operands.getShape(index: i);
1586 auto rankDiff = outShape.size() - shape.getRank();
1587
1588 for (size_t i = 0, e = shape.getRank(); i < e; ++i) {
1589 auto dim1 = outShape[i + rankDiff];
1590 auto dim2 = shape.getDimSize(index: i);
1591
1592 const FailureOr<int64_t> maybeResolvedDim =
1593 resolveBroadcastDim(dim1, dim2);
1594 if (failed(Result: maybeResolvedDim))
1595 return failure();
1596 const int64_t resolvedDim = *maybeResolvedDim;
1597 outShape[i + rankDiff] = resolvedDim;
1598 }
1599 }
1600
1601 return success();
1602}
1603
1604LogicalResult tosa::ArgMaxOp::inferReturnTypeComponents(
1605 MLIRContext *context, ::std::optional<Location> location,
1606 ArgMaxOp::Adaptor adaptor,
1607 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
1608 return inferArgMaxMinReturnTypeComponents(context, location, adaptor,
1609 inferredReturnShapes);
1610}
1611
1612LogicalResult tosa::ArgMinOp::inferReturnTypeComponents(
1613 MLIRContext *context, ::std::optional<Location> location,
1614 ArgMinOp::Adaptor adaptor,
1615 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
1616 return inferArgMaxMinReturnTypeComponents(context, location, adaptor,
1617 inferredReturnShapes);
1618}
1619
1620LogicalResult tosa::RFFT2dOp::inferReturnTypeComponents(
1621 MLIRContext *context, ::std::optional<Location> location,
1622 RFFT2dOp::Adaptor adaptor,
1623 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
1624 ShapeAdaptor inputShape(adaptor.getInputReal().getType());
1625
1626 if (!inputShape.hasRank())
1627 return failure();
1628
1629 llvm::SmallVector<int64_t> outputShape;
1630 outputShape.resize(N: 3, NV: ShapedType::kDynamic);
1631 outputShape[0] = inputShape.getDimSize(index: 0);
1632 outputShape[1] = inputShape.getDimSize(index: 1);
1633 int64_t inWidth = inputShape.getDimSize(index: 2);
1634
1635 // Note that we can support this calculation symbolically
1636 // in the future e.g. [x, y, z] -> [x, y, z / 2 + 1]
1637 if (inWidth != ShapedType::kDynamic)
1638 outputShape[2] = inWidth / 2 + 1;
1639
1640 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
1641 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
1642
1643 return success();
1644}
1645
1646static LogicalResult verifyDimIsPowerOfTwo(Operation *op, const int64_t dimSize,
1647 const llvm::StringRef dimName) {
1648 const bool isPowerOfTwo = (dimSize & (dimSize - 1)) == 0 && dimSize > 0;
1649 if (!isPowerOfTwo)
1650 return op->emitOpError(message: "expected ")
1651 << dimName << " to be a power of two, got " << dimSize;
1652
1653 return success();
1654}
1655
1656LogicalResult tosa::RFFT2dOp::verify() {
1657 const auto outputTypes = getResultTypes();
1658 if (failed(Result: verifyCompatibleShapes(types: outputTypes)))
1659 return emitOpError(message: "expected output shapes to match, got ") << outputTypes;
1660
1661 const auto inputType =
1662 llvm::dyn_cast<RankedTensorType>(Val: getInputReal().getType());
1663 if (!inputType)
1664 return success();
1665
1666 const int64_t height = inputType.getDimSize(idx: 1);
1667 if (ShapedType::isStatic(dValue: height) &&
1668 failed(Result: verifyDimIsPowerOfTwo(op: *this, dimSize: height, dimName: "height")))
1669 return failure();
1670
1671 const int64_t width = inputType.getDimSize(idx: 2);
1672 if (ShapedType::isStatic(dValue: width) &&
1673 failed(Result: verifyDimIsPowerOfTwo(op: *this, dimSize: width, dimName: "width")))
1674 return failure();
1675
1676 const auto outputType = llvm::dyn_cast<RankedTensorType>(Val: outputTypes[0]);
1677 if (!outputType)
1678 return success();
1679
1680 // Batch and height input/output dimensions should match
1681 if (failed(Result: verifyCompatibleShape(shape1: inputType.getShape().drop_back(),
1682 shape2: outputType.getShape().drop_back())))
1683 return emitOpError(message: "expected batch and height dimensions of input/output "
1684 "to match, got input=")
1685 << inputType << " output=" << outputType;
1686
1687 // Output width dimension expected to be input_width / 2 + 1
1688 const int64_t outputWidth = outputType.getDimSize(idx: 2);
1689 if (ShapedType::isStatic(dValue: width) && ShapedType::isStatic(dValue: outputWidth) &&
1690 (outputWidth != (width / 2) + 1))
1691 return emitOpError(
1692 message: "expected output width to be equal to input_width / 2 + 1, got ")
1693 << outputWidth;
1694
1695 return success();
1696}
1697
1698LogicalResult tosa::FFT2dOp::inferReturnTypeComponents(
1699 MLIRContext *context, ::std::optional<Location> location,
1700 FFT2dOp::Adaptor adaptor,
1701 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
1702 inferredReturnShapes.push_back(
1703 Elt: ShapedTypeComponents(ShapeAdaptor(adaptor.getInputReal().getType())));
1704 inferredReturnShapes.push_back(
1705 Elt: ShapedTypeComponents(ShapeAdaptor(adaptor.getInputImag().getType())));
1706 return success();
1707}
1708
1709LogicalResult tosa::FFT2dOp::verify() {
1710 const auto inputRealType =
1711 llvm::dyn_cast<RankedTensorType>(Val: getInputReal().getType());
1712 const auto inputImagType =
1713 llvm::dyn_cast<RankedTensorType>(Val: getInputImag().getType());
1714 if (!inputRealType || !inputImagType)
1715 return success();
1716
1717 const auto trySelectStaticDim = [](const int64_t a, const int64_t b) {
1718 return ShapedType::isDynamic(dValue: a) ? a : b;
1719 };
1720
1721 const int64_t height = trySelectStaticDim(inputRealType.getDimSize(idx: 1),
1722 inputImagType.getDimSize(idx: 1));
1723 if (ShapedType::isStatic(dValue: height) &&
1724 failed(Result: verifyDimIsPowerOfTwo(op: *this, dimSize: height, dimName: "height")))
1725 return failure();
1726
1727 const int64_t width = trySelectStaticDim(inputRealType.getDimSize(idx: 2),
1728 inputImagType.getDimSize(idx: 2));
1729 if (ShapedType::isStatic(dValue: width) &&
1730 failed(Result: verifyDimIsPowerOfTwo(op: *this, dimSize: width, dimName: "width")))
1731 return failure();
1732
1733 return success();
1734}
1735
1736LogicalResult tosa::ConcatOp::inferReturnTypeComponents(
1737 MLIRContext *context, ::std::optional<Location> location,
1738 ConcatOp::Adaptor adaptor,
1739 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
1740 // Infer all dimension sizes by reducing based on inputs.
1741 const Properties &prop = adaptor.getProperties();
1742 int32_t axis = prop.axis.getValue().getSExtValue();
1743 llvm::SmallVector<int64_t> outputShape;
1744 bool hasRankedInput = false;
1745 for (auto operand : adaptor.getOperands()) {
1746 ShapeAdaptor operandShape(operand.getType());
1747 if (!operandShape.hasRank())
1748 continue;
1749
1750 // Copy the Operand's rank.
1751 if (!hasRankedInput)
1752 outputShape.resize(N: operandShape.getRank(), NV: ShapedType::kDynamic);
1753
1754 // Copy shapes until the dim is non-dynamic.
1755 for (int i = 0, s = operandShape.getRank(); i < s; i++) {
1756 if (i == axis || operandShape.isDynamicDim(index: i))
1757 continue;
1758 if (outputShape[i] == ShapedType::kDynamic)
1759 outputShape[i] = operandShape.getDimSize(index: i);
1760 if (outputShape[i] != operandShape.getDimSize(index: i))
1761 return emitOptionalError(loc: location,
1762 args: "Cannot concat tensors with different sizes"
1763 " on the non-axis dimension ",
1764 args&: i);
1765 }
1766
1767 hasRankedInput = true;
1768 }
1769
1770 if (adaptor.getInput1().empty())
1771 return failure();
1772
1773 Type inputType =
1774 llvm::cast<TensorType>(Val: adaptor.getInput1().getType()[0]).getElementType();
1775 if (!hasRankedInput) {
1776 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(inputType));
1777 return success();
1778 }
1779
1780 // Determine the dimension size along the concatenation axis.
1781 int64_t concatDimSize = 0;
1782 for (auto operand : adaptor.getOperands()) {
1783 ShapeAdaptor operandShape(operand.getType());
1784
1785 // We need to know the length of the concatenation axis of all inputs to
1786 // determine the dimension size of the output shape.
1787 if (!operandShape.hasRank() || operandShape.isDynamicDim(index: axis)) {
1788 concatDimSize = ShapedType::kDynamic;
1789 break;
1790 }
1791
1792 concatDimSize += operandShape.getDimSize(index: axis);
1793 }
1794
1795 outputShape[axis] = concatDimSize;
1796
1797 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape, inputType));
1798 return success();
1799}
1800
1801LogicalResult tosa::ConcatOp::verify() {
1802 // check that each input has same element type as output
1803 auto outType = getOutput().getType();
1804 const Operation::operand_range inputList = getInput1();
1805
1806 // Check there is at least one input
1807 if (inputList.empty())
1808 return emitOpError(message: "expect at least one input");
1809
1810 if (!llvm::all_of(Range: inputList, P: [&](auto input) {
1811 return succeeded(verifySameElementTypes(
1812 *this, /* inType = */ input.getType(), outType));
1813 })) {
1814 return failure();
1815 }
1816
1817 const int32_t axis = getAxis();
1818 ShapeAdaptor firstRankedInputShape = nullptr;
1819 for (const auto &input : inputList) {
1820 const Type inputType = input.getType();
1821 ShapeAdaptor currShape(inputType);
1822 if (currShape.hasRank()) {
1823 firstRankedInputShape = currShape;
1824 // Check axis is in expected range
1825 if (axis < 0 || axis >= firstRankedInputShape.getRank())
1826 return emitOpError(message: "expect axis to be within range 0 < axis < "
1827 "rank(input1[firstRankedTensorIdx]), got ")
1828 << axis;
1829 break;
1830 }
1831 }
1832
1833 const auto allOperandsHasRank = [](const Value input) {
1834 return ShapeAdaptor(input.getType()).hasRank();
1835 };
1836 if (llvm::all_of(Range: inputList, P: allOperandsHasRank)) {
1837 const int64_t firstInputRank = firstRankedInputShape.getRank();
1838
1839 for (const auto &[index, input] : llvm::enumerate(First: inputList.drop_front())) {
1840 const ShapeAdaptor inputShape(input.getType());
1841 const int64_t inputRank = inputShape.getRank();
1842 const size_t operandNum = index + 1;
1843
1844 // Check that each operand has the same rank
1845 if (inputRank != firstInputRank)
1846 return emitOpError(
1847 message: "expect all operands to have the same rank, but got ")
1848 << firstInputRank << " vs " << inputRank << " on operands 0 and "
1849 << operandNum;
1850
1851 // Check non-axis dims match
1852 for (int i = 0; i < inputRank; i++) {
1853 const int64_t inputDim = inputShape.getDimSize(index: i);
1854 const int64_t firstInputDim = firstRankedInputShape.getDimSize(index: i);
1855 if (i == axis || firstRankedInputShape.isDynamicDim(index: i) ||
1856 inputShape.isDynamicDim(index: i))
1857 continue;
1858 if (inputDim != firstInputDim)
1859 return emitOpError(message: "expect all operand shapes to have the same sizes "
1860 "on non-axis dimensions, but got ")
1861 << inputDim << " vs " << firstInputDim << " at index " << i
1862 << " on operands 0 and " << operandNum;
1863 }
1864 }
1865
1866 const ShapeAdaptor outputShape(outType);
1867 if (outputShape.hasRank() && outputShape.getRank() != firstInputRank)
1868 return emitOpError(message: "expect output rank to match inputs rank, got ")
1869 << outputShape.getRank() << " vs " << firstInputRank;
1870
1871 // ERROR_IF(axis_sum != shape[axis]);
1872 int64_t axisSum = 0;
1873 for (const auto &input : inputList) {
1874 const ShapeAdaptor inputShape(input.getType());
1875 if (inputShape.isDynamicDim(index: axis)) {
1876 // make axisSum negative to indicate invalid value
1877 axisSum = -1;
1878 break;
1879 }
1880 axisSum += inputShape.getDimSize(index: axis);
1881 }
1882
1883 if (axisSum >= 0 && outputShape.hasRank() &&
1884 !outputShape.isDynamicDim(index: axis) &&
1885 axisSum != outputShape.getDimSize(index: axis))
1886 return emitOpError(message: "requires sum of axis dimensions of input1 "
1887 "equal to output axis dimension, got ")
1888 << axisSum << " and " << outputShape.getDimSize(index: axis);
1889 }
1890
1891 return success();
1892}
1893
1894LogicalResult tosa::EqualOp::inferReturnTypeComponents(
1895 MLIRContext *context, ::std::optional<Location> location,
1896 ValueShapeRange operands, DictionaryAttr attributes, PropertyRef properties,
1897 RegionRange regions,
1898 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
1899 auto elementType = IntegerType::get(context, /*width=*/1);
1900
1901 llvm::SmallVector<int64_t> outShape;
1902 if (resolveBroadcastShape(operands, outShape).failed()) {
1903 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(elementType));
1904 return success();
1905 }
1906
1907 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outShape, elementType));
1908 return success();
1909}
1910
1911bool tosa::EqualOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
1912 if (l.size() != r.size() || l.size() != 1)
1913 return false;
1914 return succeeded(Result: verifyCompatibleShape(type1: l[0], type2: r[0]));
1915}
1916
1917// MATMUL batch shapes are right aligned and may have different ranks. Missing
1918// leading dimensions are treated as one, while an unranked input remains
1919// unknown because it may contain any number of batch dimensions.
1920static int64_t getMatMulBatchDim(const ShapeAdaptor &shape, int64_t outputRank,
1921 int64_t axis) {
1922 if (!shape.hasRank())
1923 return ShapedType::kDynamic;
1924 const int64_t inputAxis = axis - (outputRank - shape.getRank());
1925 return inputAxis < 0 ? 1 : shape.getDimSize(index: inputAxis);
1926}
1927
1928static FailureOr<SmallVector<int64_t>>
1929resolveMatMulOutputShape(const ShapeAdaptor &aShape, const ShapeAdaptor &bShape,
1930 int64_t outputRank, bool transposeB) {
1931 if (outputRank < 2 ||
1932 (aShape.hasRank() &&
1933 (aShape.getRank() < 2 || aShape.getRank() > outputRank)) ||
1934 (bShape.hasRank() &&
1935 (bShape.getRank() < 2 || bShape.getRank() > outputRank)))
1936 return failure();
1937
1938 SmallVector<int64_t> outputShape(outputRank, ShapedType::kDynamic);
1939 for (int64_t axis = 0; axis < outputRank - 2; ++axis) {
1940 const int64_t aDim = getMatMulBatchDim(shape: aShape, outputRank, axis);
1941 const int64_t bDim = getMatMulBatchDim(shape: bShape, outputRank, axis);
1942 FailureOr<int64_t> resolvedDim = resolveBroadcastDim(dim1: aDim, dim2: bDim);
1943 if (failed(Result: resolvedDim))
1944 return failure();
1945 outputShape[axis] = *resolvedDim;
1946 }
1947
1948 if (aShape.hasRank())
1949 outputShape[outputRank - 2] = aShape.getDimSize(index: aShape.getRank() - 2);
1950 if (bShape.hasRank())
1951 outputShape[outputRank - 1] =
1952 bShape.getDimSize(index: bShape.getRank() - (transposeB ? 2 : 1));
1953 return outputShape;
1954}
1955
1956static LogicalResult inferMatMulReturnTypeComponents(
1957 const ShapeAdaptor &aShape, const ShapeAdaptor &bShape, bool transposeB,
1958 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
1959 if ((aShape.hasRank() && aShape.getRank() < 2) ||
1960 (bShape.hasRank() && bShape.getRank() < 2))
1961 return failure();
1962
1963 // An unranked input can have an arbitrary batch prefix, so the output rank
1964 // cannot yet be inferred.
1965 if (!aShape.hasRank() || !bShape.hasRank()) {
1966 inferredReturnShapes.emplace_back();
1967 return success();
1968 }
1969
1970 const int64_t aChannels = aShape.getDimSize(index: aShape.getRank() - 1);
1971 const int64_t bChannels =
1972 bShape.getDimSize(index: bShape.getRank() - (transposeB ? 1 : 2));
1973 if (ShapedType::isStatic(dValue: aChannels) && ShapedType::isStatic(dValue: bChannels) &&
1974 aChannels != bChannels)
1975 return failure();
1976
1977 const int64_t outputRank = std::max(a: aShape.getRank(), b: bShape.getRank());
1978 FailureOr<SmallVector<int64_t>> outputShape =
1979 resolveMatMulOutputShape(aShape, bShape, outputRank, transposeB);
1980 if (failed(Result: outputShape))
1981 return failure();
1982
1983 inferredReturnShapes.emplace_back(Args&: *outputShape);
1984 return success();
1985}
1986
1987LogicalResult tosa::MatMulOp::inferReturnTypeComponents(
1988 MLIRContext *context, ::std::optional<Location> location,
1989 MatMulOp::Adaptor adaptor,
1990 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
1991 return inferMatMulReturnTypeComponents(aShape: ShapeAdaptor(adaptor.getA().getType()),
1992 bShape: ShapeAdaptor(adaptor.getB().getType()),
1993 /*transposeB=*/false,
1994 inferredReturnShapes);
1995}
1996
1997template <typename T>
1998static LogicalResult verifyMatMulQuantizedOperandsType(T op, Type aElementType,
1999 Type bElementType) {
2000 const auto aQuantizedEType =
2001 llvm::dyn_cast<quant::UniformQuantizedType>(Val&: aElementType);
2002 const auto bQuantizedEType =
2003 llvm::dyn_cast<quant::UniformQuantizedType>(Val&: bElementType);
2004
2005 if (aQuantizedEType || bQuantizedEType) {
2006 if (!aQuantizedEType || !bQuantizedEType) {
2007 return op.emitOpError("expect operands to be both quantized or both not "
2008 "quantized, got ")
2009 << aElementType << " and " << bElementType;
2010 }
2011 // both a and b have quantized element types
2012 auto aQuantWidth = aQuantizedEType.getStorageTypeIntegralWidth();
2013 auto bQuantWidth = bQuantizedEType.getStorageTypeIntegralWidth();
2014 if (aQuantWidth != bQuantWidth) {
2015 return op.emitOpError("expect quantized operands to have same widths, "
2016 "got ")
2017 << aQuantWidth << " and " << bQuantWidth;
2018 }
2019 }
2020
2021 return success();
2022}
2023
2024template <typename T>
2025static LogicalResult verifyMatMulZeroPointType(T op, Value input, Value zp,
2026 StringRef inputName,
2027 StringRef zpName) {
2028 const Type inputElementType = getElementTypeOrSelf(type: input.getType());
2029 const Type inputStorageElementType = getStorageElementTypeOrSelf(value: input);
2030 const Type zpElementType = getStorageElementTypeOrSelf(value: zp);
2031 Type expectedElementType = inputStorageElementType;
2032
2033 if (isa<BlockScaledType>(Val: inputElementType))
2034 expectedElementType = Float32Type::get(context: op.getContext());
2035
2036 if (expectedElementType == zpElementType)
2037 return success();
2038
2039 InFlightDiagnostic diag = op.emitOpError("expect input ");
2040 diag << inputName << " and " << zpName;
2041 if (isa<BlockScaledType>(Val: inputElementType))
2042 diag << " have compatible element types, got " << inputElementType
2043 << " and " << zpElementType;
2044 else
2045 diag << " have the same element type, got " << inputStorageElementType
2046 << " and " << zpElementType;
2047 return diag;
2048}
2049
2050static SmallVector<int64_t> getMatMulBatchShape(const ShapeAdaptor &shape) {
2051 SmallVector<int64_t> batchShape;
2052 if (!shape.hasRank() || shape.getRank() < 2)
2053 return batchShape;
2054 batchShape.reserve(N: shape.getRank() - 2);
2055 for (int64_t i = 0, e = shape.getRank() - 2; i < e; ++i)
2056 batchShape.push_back(Elt: shape.getDimSize(index: i));
2057 return batchShape;
2058}
2059
2060template <typename T>
2061static LogicalResult verifyMatMulShapes(T op, bool transposeB) {
2062 const ShapeAdaptor aShape(op.getA().getType());
2063 const ShapeAdaptor bShape(op.getB().getType());
2064 const auto outputType = cast<ShapedType>(op.getResult().getType());
2065
2066 int64_t channels = aShape.hasRank() ? aShape.getDimSize(index: aShape.getRank() - 1)
2067 : ShapedType::kDynamic;
2068 if (bShape.hasRank() &&
2069 failed(tryUpdateDimOrFailure(
2070 op, channels,
2071 bShape.getDimSize(index: bShape.getRank() - (transposeB ? 1 : 2)), "b",
2072 "channels")))
2073 return failure();
2074
2075 const int64_t minimumOutputRank =
2076 std::max(a: aShape.hasRank() ? aShape.getRank() : 2,
2077 b: bShape.hasRank() ? bShape.getRank() : 2);
2078 if (outputType.hasRank() && outputType.getRank() < minimumOutputRank)
2079 return op.emitOpError("expected output rank of at least ")
2080 << minimumOutputRank << ", got " << outputType.getRank();
2081
2082 const bool bothInputsRanked = aShape.hasRank() && bShape.hasRank();
2083 if (!bothInputsRanked && !outputType.hasRank())
2084 return success();
2085
2086 // When an input is unranked, use the declared output rank to validate every
2087 // result dimension constrained by the ranked input.
2088 const int64_t expectedOutputRank =
2089 bothInputsRanked ? minimumOutputRank : outputType.getRank();
2090 FailureOr<SmallVector<int64_t>> expectedOutputShape =
2091 resolveMatMulOutputShape(aShape, bShape, outputRank: expectedOutputRank, transposeB);
2092 if (failed(Result: expectedOutputShape)) {
2093 InFlightDiagnostic diag = op.emitOpError(
2094 "expected batch dimensions of a and b to be broadcast compatible, "
2095 "got a=[");
2096 printShapeToDiagnostic(diag, shape: getMatMulBatchShape(shape: aShape));
2097 diag << "] and b=[";
2098 printShapeToDiagnostic(diag, shape: getMatMulBatchShape(shape: bShape));
2099 diag << "]";
2100 return diag;
2101 }
2102
2103 if (outputType.hasRank())
2104 return verifyOutputShapeCompatibleWithExpected(
2105 op.getOperation(), outputType, *expectedOutputShape);
2106 return success();
2107}
2108
2109LogicalResult MatMulOp::verify() {
2110 const Type aElementType = getElementTypeOrSelf(val: getA());
2111 const Type bElementType = getElementTypeOrSelf(val: getB());
2112
2113 if (failed(
2114 Result: verifyMatMulQuantizedOperandsType(op: *this, aElementType, bElementType)))
2115 return failure();
2116
2117 if (failed(Result: verifyMatMulZeroPointType(op: *this, input: getA(), zp: getAZp(), inputName: "a", zpName: "a_zp")) ||
2118 failed(Result: verifyMatMulZeroPointType(op: *this, input: getB(), zp: getBZp(), inputName: "b", zpName: "b_zp")))
2119 return failure();
2120
2121 FailureOr<int64_t> maybeAZp = getAZeroPoint();
2122 if (succeeded(Result: maybeAZp) && verifyAZeroPoint(zp: *maybeAZp).failed())
2123 return failure();
2124
2125 FailureOr<int64_t> maybeBZp = getBZeroPoint();
2126 if (succeeded(Result: maybeBZp) && verifyBZeroPoint(zp: *maybeBZp).failed())
2127 return failure();
2128
2129 return verifyMatMulShapes(op: *this, /*transposeB=*/false);
2130}
2131
2132LogicalResult tosa::MatMulTOp::inferReturnTypeComponents(
2133 MLIRContext *context, ::std::optional<Location> location,
2134 MatMulTOp::Adaptor adaptor,
2135 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2136 return inferMatMulReturnTypeComponents(aShape: ShapeAdaptor(adaptor.getA().getType()),
2137 bShape: ShapeAdaptor(adaptor.getB().getType()),
2138 /*transposeB=*/true,
2139 inferredReturnShapes);
2140}
2141
2142LogicalResult MatMulTOp::verify() {
2143 const Type aElementType = getElementTypeOrSelf(val: getA());
2144 const Type bElementType = getElementTypeOrSelf(val: getB());
2145
2146 if (failed(
2147 Result: verifyMatMulQuantizedOperandsType(op: *this, aElementType, bElementType)))
2148 return failure();
2149
2150 if (failed(Result: verifyMatMulZeroPointType(op: *this, input: getA(), zp: getAZp(), inputName: "a", zpName: "a_zp")) ||
2151 failed(Result: verifyMatMulZeroPointType(op: *this, input: getB(), zp: getBZp(), inputName: "b", zpName: "b_zp")))
2152 return failure();
2153
2154 FailureOr<int64_t> maybeAZp = getAZeroPoint();
2155 if (succeeded(Result: maybeAZp) && verifyAZeroPoint(zp: *maybeAZp).failed())
2156 return failure();
2157
2158 FailureOr<int64_t> maybeBZp = getBZeroPoint();
2159 if (succeeded(Result: maybeBZp) && verifyBZeroPoint(zp: *maybeBZp).failed())
2160 return failure();
2161
2162 return verifyMatMulShapes(op: *this, /*transposeB=*/true);
2163}
2164
2165LogicalResult tosa::MatmulTBlockScaledOp::inferReturnTypeComponents(
2166 MLIRContext *context, ::std::optional<Location> location,
2167 MatmulTBlockScaledOp::Adaptor adaptor,
2168 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2169 SmallVector<int64_t, 3> outShape(3, ShapedType::kDynamic);
2170
2171 const auto aDataShape = cast<ShapedType>(Val: adaptor.getAData().getType());
2172 if (aDataShape.hasRank()) {
2173 outShape[0] = aDataShape.getDimSize(idx: 0);
2174 outShape[1] = aDataShape.getDimSize(idx: 1);
2175 }
2176
2177 const auto aScaleShape = cast<ShapedType>(Val: adaptor.getAScale().getType());
2178 if (aScaleShape.hasRank()) {
2179 outShape[0] = ShapedType::isDynamic(dValue: outShape[0]) ? aScaleShape.getDimSize(idx: 0)
2180 : outShape[0];
2181 outShape[1] = ShapedType::isDynamic(dValue: outShape[1]) ? aScaleShape.getDimSize(idx: 1)
2182 : outShape[1];
2183 }
2184
2185 // If B batch size is 1, it is broadcast across A's batch size
2186 const auto bDataShape = cast<ShapedType>(Val: adaptor.getBData().getType());
2187 if (bDataShape.hasRank()) {
2188 const int64_t bDataBatchSize = bDataShape.getDimSize(idx: 0);
2189 if (bDataBatchSize != 1)
2190 outShape[0] =
2191 ShapedType::isDynamic(dValue: outShape[0]) ? bDataBatchSize : outShape[0];
2192 outShape[2] = bDataShape.getDimSize(idx: 1);
2193 }
2194
2195 const auto bScaleShape = cast<ShapedType>(Val: adaptor.getBScale().getType());
2196 if (bScaleShape.hasRank()) {
2197 const int64_t bScaleBatchSize = bScaleShape.getDimSize(idx: 0);
2198 if (bScaleBatchSize != 1)
2199 outShape[0] =
2200 ShapedType::isDynamic(dValue: outShape[0]) ? bScaleBatchSize : outShape[0];
2201 outShape[2] = ShapedType::isDynamic(dValue: outShape[2]) ? bScaleShape.getDimSize(idx: 1)
2202 : outShape[2];
2203 }
2204
2205 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outShape));
2206 return success();
2207}
2208
2209LogicalResult MatmulTBlockScaledOp::verify() {
2210 // Verify same input data types
2211 const Type aDataType = getAData().getType();
2212 const Type bDataType = getBData().getType();
2213 if (failed(Result: verifySameElementTypes(op: *this, aType: aDataType, bType: bDataType, aName: "A_data",
2214 bName: "B_data")))
2215 return failure();
2216
2217 // Verify input shape compatibility
2218 int64_t N = ShapedType::kDynamic;
2219 int64_t D = ShapedType::kDynamic;
2220 int64_t H = ShapedType::kDynamic;
2221 int64_t W = ShapedType::kDynamic;
2222 int64_t C = ShapedType::kDynamic;
2223 int64_t multiplesOfC = ShapedType::kDynamic;
2224
2225 const ShapeAdaptor aDataShape = ShapeAdaptor(aDataType);
2226 if (aDataShape.hasRank()) {
2227 N = aDataShape.getDimSize(index: 0);
2228 H = aDataShape.getDimSize(index: 1);
2229 C = aDataShape.getDimSize(index: 2);
2230 }
2231
2232 const ShapeAdaptor aScaleShape = ShapeAdaptor(getAScale().getType());
2233 if (aScaleShape.hasRank()) {
2234 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: N, newDim: aScaleShape.getDimSize(index: 0),
2235 operandName: "a_scale", dimName: "batch")) ||
2236 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: H, newDim: aScaleShape.getDimSize(index: 1),
2237 operandName: "a_scale", dimName: "height")))
2238 return failure();
2239 multiplesOfC = aScaleShape.getDimSize(index: 2);
2240 }
2241
2242 const ShapeAdaptor bDataShape = ShapeAdaptor(bDataType);
2243 if (bDataShape.hasRank()) {
2244 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: D, newDim: bDataShape.getDimSize(index: 0),
2245 operandName: "b_data", dimName: "batch")) ||
2246 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: C, newDim: bDataShape.getDimSize(index: 2),
2247 operandName: "b_data", dimName: "channels")))
2248 return failure();
2249 W = bDataShape.getDimSize(index: 1);
2250 }
2251
2252 const ShapeAdaptor bScaleShape = ShapeAdaptor(getBScale().getType());
2253 if (bScaleShape.hasRank()) {
2254 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: D, newDim: bScaleShape.getDimSize(index: 0),
2255 operandName: "b_scale", dimName: "batch")) ||
2256 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: W, newDim: bScaleShape.getDimSize(index: 1),
2257 operandName: "b_scale", dimName: "width")) ||
2258 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: multiplesOfC,
2259 newDim: bScaleShape.getDimSize(index: 2), operandName: "b_scale",
2260 dimName: "C/block_size")))
2261 return failure();
2262 }
2263
2264 // Verify batch size is broadcast compatible
2265 if (ShapedType::isStatic(dValue: N) && ShapedType::isStatic(dValue: D) && N != D && D != 1)
2266 return emitOpError(message: "expect B matrix batch size to be broadcast compatible "
2267 "with A, got D=")
2268 << D << " vs N=" << N;
2269
2270 // Verify C is a multiple of block size
2271 const uint32_t blockSize = BlockSizeAttr::getBlockSizeValue(blockSize: getBlockSize());
2272 if (blockSize != BlockSizeAttr::getBlockSizeValue(blockSize: BlockSize::BLOCK_SIZE_32))
2273 return emitOpError(message: "expect block size to be 32, got ") << blockSize;
2274 if (ShapedType::isStatic(dValue: C) && C % blockSize != 0)
2275 return emitOpError(message: "expect C to be a multiple of block size, got C=")
2276 << C << ", block_size=" << blockSize;
2277
2278 // Verify multiplesOfC is C / block size
2279 if (ShapedType::isStatic(dValue: C) && ShapedType::isStatic(dValue: multiplesOfC) &&
2280 multiplesOfC != C / blockSize)
2281 return emitOpError(
2282 message: "expect scale operands dimension 2 to equal C/block_size (")
2283 << C << "/" << blockSize << ")" << ", got " << multiplesOfC;
2284
2285 // Verify output shape
2286 N = ShapedType::isDynamic(dValue: N) ? D : N;
2287 const SmallVector<int64_t, 3> expectedOutputShape = {N, H, W};
2288 const auto outputType = cast<ShapedType>(Val: getResult().getType());
2289 if (outputType.hasRank() &&
2290 failed(
2291 Result: verifyCompatibleShape(shape1: outputType.getShape(), shape2: expectedOutputShape))) {
2292 InFlightDiagnostic opError = emitOpError(message: "expected output shape ");
2293 printShapeToDiagnostic(diag&: opError, shape: outputType.getShape());
2294 opError << " to be compatible with expected output shape ";
2295 printShapeToDiagnostic(diag&: opError, shape: expectedOutputShape);
2296 return opError;
2297 }
2298
2299 return success();
2300}
2301
2302LogicalResult tosa::PadOp::inferReturnTypeComponents(
2303 MLIRContext *context, ::std::optional<Location> location,
2304 PadOp::Adaptor adaptor,
2305 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2306 ShapeAdaptor inputShape(adaptor.getInput1().getType());
2307 auto paddingRank =
2308 cast<tosa::shapeType>(Val: adaptor.getPadding().getType()).getRank();
2309 SmallVector<int64_t> outputShape;
2310
2311 // If the input rank is unknown, we can infer the output rank using the
2312 // padding shape's rank divided by 2.
2313 if (!inputShape.hasRank()) {
2314 outputShape.resize(N: paddingRank / 2, NV: ShapedType::kDynamic);
2315 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
2316 return success();
2317 }
2318
2319 SmallVector<int64_t> paddingValues;
2320 // If the paddings value is not a constant, all dimensions must be dynamic.
2321 if (!tosa::getConstShapeValues(op: adaptor.getPadding().getDefiningOp(),
2322 result_shape&: paddingValues)) {
2323 outputShape.resize(N: inputShape.getRank(), NV: ShapedType::kDynamic);
2324 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
2325 return success();
2326 }
2327
2328 outputShape.reserve(N: inputShape.getRank());
2329 for (int i = 0, s = inputShape.getRank(); i < s; i++) {
2330 if (inputShape.isDynamicDim(index: i)) {
2331 outputShape.push_back(Elt: ShapedType::kDynamic);
2332 continue;
2333 }
2334 auto padFront = paddingValues[i * 2];
2335 auto padBack = paddingValues[i * 2 + 1];
2336 outputShape.push_back(Elt: inputShape.getDimSize(index: i) + padFront + padBack);
2337 }
2338
2339 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
2340 return success();
2341}
2342
2343LogicalResult tosa::PadOp::verify() {
2344 if (verifySameElementTypes(op: *this, /* inType = */ aType: getInput1().getType(),
2345 /* outType = */ bType: getOutput().getType())
2346 .failed()) {
2347 return failure();
2348 }
2349
2350 if (auto padConst = getPadConst()) {
2351 if (verifySameElementTypes(op: *this, /* inType = */ aType: padConst.getType(),
2352 /* outType = */ bType: getOutput().getType())
2353 .failed()) {
2354 return failure();
2355 }
2356 }
2357
2358 RankedTensorType inputType =
2359 llvm::dyn_cast<RankedTensorType>(Val: getInput1().getType());
2360 RankedTensorType outputType =
2361 llvm::dyn_cast<RankedTensorType>(Val: getOutput().getType());
2362 if (!inputType || !outputType)
2363 return success();
2364
2365 if (failed(Result: verifyRanksMatch(op: getOperation(), lhs: inputType, rhs: outputType, lhsName: "input",
2366 rhsName: "output")))
2367 return failure();
2368
2369 auto inputRank = inputType.getRank();
2370 auto paddingRank = cast<tosa::shapeType>(Val: getPadding().getType()).getRank();
2371 if (paddingRank != inputRank * 2)
2372 return emitOpError() << "padding tensor must have " << inputRank
2373 << " * 2 = " << inputRank * 2 << " elements, but got "
2374 << paddingRank;
2375
2376 return success();
2377}
2378
2379LogicalResult tosa::SliceOp::inferReturnTypeComponents(
2380 MLIRContext *context, ::std::optional<Location> location,
2381 SliceOp::Adaptor adaptor,
2382 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2383
2384 Type inputType = getElementTypeOrSelf(type: adaptor.getInput1().getType());
2385 SmallVector<int64_t> start;
2386 SmallVector<int64_t> size;
2387
2388 if (!tosa::getConstShapeValues(op: adaptor.getStart().getDefiningOp(), result_shape&: start) ||
2389 !tosa::getConstShapeValues(op: adaptor.getSize().getDefiningOp(), result_shape&: size)) {
2390 auto rank = cast<tosa::shapeType>(Val: adaptor.getSize().getType()).getRank();
2391 SmallVector<int64_t> fallback(rank, ShapedType::kDynamic);
2392 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(fallback, inputType));
2393 return success();
2394 }
2395
2396 // if size[i] is -1, all remaining elements in dimension i are included
2397 // in the slice, similar to TF.
2398 ShapeAdaptor inputShape(adaptor.getInput1().getType());
2399 // initialize outputShape to all unknown
2400 SmallVector<int64_t> outputShape(size.size(), ShapedType::kDynamic);
2401 if (inputShape.hasRank()) {
2402 for (size_t i = 0; i < size.size(); i++) {
2403 if (size[i] != 0 && size[i] >= -1 && start[i] >= 0 &&
2404 (ShapedType::isDynamic(dValue: inputShape.getDimSize(index: i)) ||
2405 start[i] < inputShape.getDimSize(index: i))) {
2406 // size[i] is not 0 and not < -1, and start[i] is in valid range
2407 if (ShapedType::isDynamic(dValue: inputShape.getDimSize(index: i))) {
2408 // input shape has unknown dim[i] - only valid if size[i] > 0
2409 if (size[i] > 0) {
2410 outputShape[i] = size[i];
2411 }
2412 } else {
2413 // input shape has known dim[i]
2414 if (size[i] == -1) {
2415 outputShape[i] = inputShape.getDimSize(index: i) - start[i];
2416 } else if (start[i] + size[i] <= inputShape.getDimSize(index: i)) {
2417 // start[i] + size[i] is within bound of input shape's dim[i]
2418 outputShape[i] = size[i];
2419 }
2420 }
2421 }
2422 }
2423 } else {
2424 outputShape = convertToMlirShape(shape: size);
2425 }
2426 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
2427 return success();
2428}
2429
2430LogicalResult tosa::SliceOp::verify() {
2431 const Value input = getInput1();
2432 const Value output = getOutput();
2433 if (verifySameElementTypes(op: *this, /* inType = */ aType: input.getType(),
2434 /* outType = */ bType: output.getType())
2435 .failed())
2436 return failure();
2437
2438 const Value start = getStart();
2439 const Value size = getSize();
2440 const ShapeAdaptor inputShape(input.getType());
2441 const ShapeAdaptor outputShape(output.getType());
2442
2443 if (inputShape.hasRank()) {
2444 const auto inputRank = inputShape.getRank();
2445 if (outputShape.hasRank() && inputRank != outputShape.getRank())
2446 return emitOpError(
2447 message: "expect input1 and output to have the same ranks, got ")
2448 << inputRank << " and " << outputShape.getRank();
2449
2450 const auto startShapeRank =
2451 llvm::cast<tosa::shapeType>(Val: start.getType()).getRank();
2452 if (inputRank != startShapeRank)
2453 return emitOpError(message: "length of start is not equal to rank of input shape");
2454
2455 const auto sizeShapeRank =
2456 llvm::cast<tosa::shapeType>(Val: size.getType()).getRank();
2457 if (inputRank != sizeShapeRank)
2458 return emitOpError(message: "length of size is not equal to rank of input shape");
2459 }
2460
2461 return success();
2462}
2463
2464LogicalResult tosa::MulOp::inferReturnTypeComponents(
2465 MLIRContext *context, ::std::optional<Location> location,
2466 ValueShapeRange operands, DictionaryAttr attributes, PropertyRef properties,
2467 RegionRange regions,
2468 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2469 // mul op's output shape only depend on input1 and input2, not on shift
2470 ValueShapeRange twoInputs = operands.drop_back();
2471 llvm::SmallVector<int64_t> outShape;
2472 if (resolveBroadcastShape(operands: twoInputs, outShape).failed()) {
2473 inferredReturnShapes.push_back(Elt: ShapedTypeComponents());
2474 } else {
2475 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outShape));
2476 }
2477 return success();
2478}
2479
2480LogicalResult tosa::MulOp::verify() {
2481 const Value output = getOutput();
2482 auto resElemType = getElementTypeOrSelf(val: output);
2483
2484 // Verify if the element type among operands and result match tosa
2485 // specification.
2486 if (auto resIntType = dyn_cast<IntegerType>(Val&: resElemType)) {
2487 IntegerType lhsIntType =
2488 dyn_cast<IntegerType>(Val: getElementTypeOrSelf(val: getInput1()));
2489 IntegerType rhsIntType =
2490 dyn_cast<IntegerType>(Val: getElementTypeOrSelf(val: getInput2()));
2491 if (!lhsIntType || !rhsIntType || lhsIntType != rhsIntType)
2492 return emitOpError(message: "requires the same element type for all operands");
2493
2494 // Though the spec requires the element type of result to be i32, a more
2495 // relaxed way is provided at dialect level for easier cooperating with
2496 // other dialects.
2497 if (lhsIntType.getWidth() > resIntType.getWidth())
2498 return emitOpError(message: "invalid data type size for operands or result");
2499
2500 } else {
2501 // For other supported type, the spec requires requires the same element
2502 // type for all operands (excludes `shift` operand) and results.
2503 for (int i = 0; i < 2; ++i) {
2504 if (getElementTypeOrSelf(val: getOperand(i)) != resElemType)
2505 return emitOpError(
2506 message: "requires the same element type for all operands and results");
2507 }
2508 }
2509
2510 // Verify the op has same ranks for all main operands (excludes extra operands
2511 // such as shift of mul op, so this is the only difference with the built-in
2512 // `SameOperandsAndResultRank` trait) and results types, if known.
2513 TypeRange operandTypes = getOperandTypes();
2514 ShapedType aType = cast<ShapedType>(Val: operandTypes[0]);
2515 ShapedType bType = cast<ShapedType>(Val: operandTypes[1]);
2516
2517 const bool aHasRank = aType.hasRank();
2518 const bool bHasRank = bType.hasRank();
2519
2520 bool hasExpectedOutputShape = false;
2521 SmallVector<int64_t> expectedOutputShape;
2522
2523 if (aHasRank && bHasRank) {
2524 const int64_t aRank = aType.getRank();
2525 const int64_t bRank = bType.getRank();
2526 if (aRank != bRank)
2527 return emitOpError(message: "a and b operands don't have matching ranks, got ")
2528 << aRank << " and " << bRank;
2529
2530 // check for broadcast compatible shapes
2531 if (!mlir::OpTrait::util::getBroadcastedShape(
2532 shape1: aType.getShape(), shape2: bType.getShape(), resultShape&: expectedOutputShape))
2533 return emitOpError(message: "a and b operands don't have broadcast-compatible "
2534 "shapes, got ")
2535 << aType << " and " << bType;
2536 hasExpectedOutputShape = true;
2537 }
2538
2539 ShapedType resultType = cast<ShapedType>(Val: output.getType());
2540 if (!resultType.hasRank())
2541 return success();
2542
2543 const int64_t resultRank = resultType.getRank();
2544 if (aHasRank && resultRank != aType.getRank())
2545 return emitOpError(message: "result type has different rank than a, got ")
2546 << resultRank << " vs " << aType.getRank();
2547 if (bHasRank && resultRank != bType.getRank())
2548 return emitOpError(message: "result type has different rank than b, got ")
2549 << resultRank << " vs " << bType.getRank();
2550
2551 if (hasExpectedOutputShape &&
2552 failed(Result: verifyOutputShapeCompatibleWithExpected(op: getOperation(), outputType: resultType,
2553 expectedShape: expectedOutputShape)))
2554 return failure();
2555
2556 return success();
2557}
2558
2559LogicalResult tosa::TableOp::inferReturnTypeComponents(
2560 MLIRContext *context, ::std::optional<Location> location,
2561 TableOp::Adaptor adaptor,
2562 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2563 ShapeAdaptor inputShape(adaptor.getInput1().getType());
2564
2565 if (!inputShape.hasRank()) {
2566 inferredReturnShapes.push_back(Elt: ShapedTypeComponents());
2567 return success();
2568 }
2569
2570 inferredReturnShapes.resize(N: 1);
2571 inputShape.getDims(res&: inferredReturnShapes[0]);
2572 return success();
2573}
2574
2575LogicalResult tosa::TableOp::verify() {
2576 const TensorType inputType = getInput1().getType();
2577 const TensorType outputType = getOutput().getType();
2578
2579 if (!inputType.hasRank() || !outputType.hasRank())
2580 return success();
2581
2582 if (failed(Result: verifyRanksMatch(op: getOperation(), lhs: inputType, rhs: outputType, lhsName: "input",
2583 rhsName: "result")))
2584 return failure();
2585
2586 auto inputDims = inputType.getShape();
2587 auto outputDims = outputType.getShape();
2588 for (auto it : llvm::enumerate(First: llvm::zip(t&: inputDims, u&: outputDims))) {
2589 int64_t dim = it.index();
2590 auto [inputDim, outputDim] = it.value();
2591 if (ShapedType::isStatic(dValue: outputDim) && outputDim != inputDim) {
2592 return emitOpError() << "dim(result, " << dim << ") = " << outputDim
2593 << " doesn't match dim(input, " << dim
2594 << ") = " << inputDim;
2595 }
2596 }
2597 return success();
2598}
2599
2600LogicalResult
2601tosa::TileOp::getConstantMultiples(SmallVector<int64_t> &multiples) {
2602 // Multiples must be constants.
2603 DenseIntElementsAttr multiplesAttr;
2604 if (!matchPattern(value: getMultiples(), pattern: m_Constant(bind_value: &multiplesAttr)))
2605 return failure();
2606 multiples =
2607 llvm::map_to_vector(C: multiplesAttr.getValues<APInt>(),
2608 F: [](const APInt &val) { return val.getSExtValue(); });
2609 return success();
2610}
2611
2612LogicalResult tosa::TileOp::inferReturnTypeComponents(
2613 MLIRContext *context, ::std::optional<Location> location,
2614 TileOp::Adaptor adaptor,
2615 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2616 Type inputType = getElementTypeOrSelf(type: adaptor.getInput1().getType());
2617 SmallVector<int64_t> multiples;
2618 if (!tosa::getConstShapeValues(op: adaptor.getMultiples().getDefiningOp(),
2619 result_shape&: multiples)) {
2620 auto rank =
2621 cast<tosa::shapeType>(Val: adaptor.getMultiples().getType()).getRank();
2622 SmallVector<int64_t> fallback(rank, ShapedType::kDynamic);
2623 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(fallback, inputType));
2624 return success();
2625 }
2626 ShapeAdaptor inputShape(adaptor.getInput1().getType());
2627 SmallVector<int64_t> outputShape;
2628 if (!inputShape.hasRank()) {
2629 outputShape.resize(N: multiples.size(), NV: ShapedType::kDynamic);
2630 inferredReturnShapes.push_back(
2631 Elt: ShapedTypeComponents(outputShape, inputType));
2632 return success();
2633 }
2634 if (static_cast<size_t>(inputShape.getRank()) != multiples.size())
2635 return failure();
2636
2637 // Any non dynamic dimension can be multiplied to a known size.
2638 outputShape.reserve(N: multiples.size());
2639 for (int i = 0, s = inputShape.getRank(); i < s; i++) {
2640 // Conservatively infer a dynamic dimension for non-positive multiples.
2641 // Leave their validation to tosa-validate.
2642 if (multiples[i] <= 0) {
2643 outputShape.push_back(Elt: ShapedType::kDynamic);
2644 } else {
2645 int64_t dim = inputShape.getDimSize(index: i);
2646 if (dim != ShapedType::kDynamic)
2647 dim *= multiples[i];
2648 outputShape.push_back(Elt: dim);
2649 }
2650 }
2651
2652 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape, inputType));
2653 return success();
2654}
2655
2656LogicalResult tosa::TileOp::verify() {
2657 if (verifySameElementTypes(op: *this, /* intype = */ aType: getInput1().getType(),
2658 /* outType = */ bType: getOutput().getType())
2659 .failed()) {
2660 return failure();
2661 }
2662 ShapedType inputType = llvm::cast<ShapedType>(Val: getInput1().getType());
2663 ShapedType outputType = llvm::cast<ShapedType>(Val: getType());
2664
2665 shapeType multiplesType =
2666 llvm::cast<tosa::shapeType>(Val: getMultiples().getType());
2667
2668 auto multiplesRank = multiplesType.getRank();
2669
2670 if (inputType.hasRank()) {
2671 if (inputType.getRank() != multiplesRank)
2672 return emitOpError(message: "expect 'multiples' to have rank ")
2673 << inputType.getRank() << " but got " << multiplesRank << ".";
2674 if (outputType.hasRank() &&
2675 failed(Result: verifyRanksMatch(op: getOperation(), lhs: inputType, rhs: outputType, lhsName: "input",
2676 rhsName: "output")))
2677 return failure();
2678 } else if (outputType.hasRank() && outputType.getRank() != multiplesRank)
2679 return emitOpError(message: "expect 'multiples' array to have length ")
2680 << outputType.getRank() << " but got " << multiplesRank << ".";
2681
2682 return success();
2683}
2684
2685bool tosa::ReshapeOp::isCompatibleReturnTypes(TypeRange l, TypeRange r) {
2686 if (l.size() != r.size() || l.size() != 1)
2687 return false;
2688 return getElementTypeOrSelf(type: l[0]) == getElementTypeOrSelf(type: r[0]);
2689}
2690
2691LogicalResult tosa::ReshapeOp::inferReturnTypeComponents(
2692 MLIRContext *context, ::std::optional<Location> location,
2693 ReshapeOp::Adaptor adaptor,
2694 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2695 ShapeAdaptor inputShape(adaptor.getInput1().getType());
2696 Type inputType = getElementTypeOrSelf(type: adaptor.getInput1().getType());
2697 llvm::SmallVector<int64_t> newShapeValue;
2698 // A shape that is not a compile-time constant, or that contains a dimension
2699 // which cannot describe a TOSA tensor (a negative value other than the
2700 // inferable -1, or zero), cannot be used to infer the result type. Fall back
2701 // to dynamic dimensions and let tosa-validate diagnose the shape. Such a
2702 // dimension must not reach RankedTensorType::get().
2703 if (!tosa::getConstShapeValues(op: adaptor.getShape().getDefiningOp(),
2704 result_shape&: newShapeValue) ||
2705 llvm::any_of(Range&: newShapeValue, P: [](int64_t dim) {
2706 return dim == 0 || dim < kInferableDimSize;
2707 })) {
2708 auto rank = cast<tosa::shapeType>(Val: adaptor.getShape().getType()).getRank();
2709 SmallVector<int64_t> fallback(rank, ShapedType::kDynamic);
2710 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(fallback, inputType));
2711 return success();
2712 }
2713
2714 newShapeValue = convertToMlirShape(shape: newShapeValue);
2715
2716 // We cannot infer from the total number of elements so we must take the
2717 // shape attribute as exact.
2718 if (!inputShape.hasRank() || !inputShape.hasStaticShape()) {
2719 inferredReturnShapes.push_back(
2720 Elt: ShapedTypeComponents(newShapeValue, inputType));
2721 return success();
2722 }
2723
2724 // Determine the number of elements covered by the slice of all static
2725 // dimensions. This allows us to infer the length of the remaining dynamic
2726 // dimension.
2727 int64_t numElements = inputShape.getNumElements();
2728 int64_t staticMul = 1;
2729 for (auto val : newShapeValue) {
2730 if (ShapedType::isStatic(dValue: val)) {
2731 staticMul *= val;
2732 }
2733 }
2734
2735 // Determine the length of the dynamic dimension.
2736 for (auto &val : newShapeValue) {
2737 if (ShapedType::isDynamic(dValue: val))
2738 val = numElements / staticMul;
2739 }
2740
2741 inferredReturnShapes.push_back(
2742 Elt: ShapedTypeComponents(newShapeValue, inputType));
2743 return success();
2744}
2745
2746llvm::LogicalResult tosa::ReshapeOp::verify() {
2747 TensorType outputType = getOutput().getType();
2748 if (verifySameElementTypes(op: *this, /* inType = */ aType: getInput1().getType(),
2749 bType: outputType)
2750 .failed()) {
2751 return failure();
2752 }
2753
2754 // The rank of the shape operand is encoded in its type (`!tosa.shape<N>`),
2755 // so it can be compared without inspecting the constant values.
2756 if (auto rankedOutputType = dyn_cast<RankedTensorType>(Val&: outputType)) {
2757 if (cast<tosa::shapeType>(Val: getShape().getType()).getRank() !=
2758 rankedOutputType.getRank())
2759 return emitOpError() << "new shape does not match result rank";
2760 }
2761
2762 return mlir::success();
2763}
2764
2765bool tosa::ReshapeBlockScaledOp::isCompatibleReturnTypes(TypeRange l,
2766 TypeRange r) {
2767 if (l.size() != r.size() || l.size() < 1 || l.size() > 2)
2768 return false;
2769 bool ok = (getElementTypeOrSelf(type: l[0]) == getElementTypeOrSelf(type: r[0]));
2770 if (l.size() == 2)
2771 ok = ok && (getElementTypeOrSelf(type: l[1]) == getElementTypeOrSelf(type: r[1]));
2772 return ok;
2773}
2774
2775LogicalResult tosa::ReshapeBlockScaledOp::inferReturnTypeComponents(
2776 MLIRContext *context, ::std::optional<Location> location,
2777 ReshapeBlockScaledOp::Adaptor adaptor,
2778 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
2779
2780 const auto numInputs = adaptor.getInput().size();
2781 ShapeAdaptor inputShape(adaptor.getInput()[0].getType());
2782 Type inputType = getElementTypeOrSelf(type: adaptor.getInput()[0].getType());
2783 llvm::SmallVector<int64_t> newShapeValue;
2784 const auto newShape = adaptor.getNewValueShape();
2785 if (!tosa::getConstShapeValues(op: newShape.getDefiningOp(), result_shape&: newShapeValue)) {
2786 auto rank = cast<tosa::shapeType>(Val: newShape.getType()).getRank();
2787 SmallVector<int64_t> fallback(rank, ShapedType::kDynamic);
2788 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(fallback, inputType));
2789 if (numInputs == 2)
2790 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(
2791 fallback, getElementTypeOrSelf(type: adaptor.getInput()[1].getType())));
2792 return success();
2793 }
2794
2795 const uint32_t blockSize =
2796 BlockSizeAttr::getBlockSizeValue(blockSize: adaptor.getBlockSize());
2797
2798 llvm::SmallVector<int64_t> newScaleShapeValue;
2799 if (numInputs == 2) {
2800 newScaleShapeValue.assign(in_start: newShapeValue.begin(), in_end: newShapeValue.end());
2801 if (!newScaleShapeValue.empty() &&
2802 ShapedType::isStatic(dValue: newScaleShapeValue.back()))
2803 newScaleShapeValue.back() /= blockSize;
2804 }
2805
2806 inferredReturnShapes.push_back(
2807 Elt: ShapedTypeComponents(newShapeValue, inputType));
2808 if (numInputs == 2) {
2809 // Fix up scale shape - with special case for last dimension
2810 for (size_t idx = 0; idx < newShapeValue.size(); idx++) {
2811 if (ShapedType::isDynamic(dValue: newScaleShapeValue[idx])) {
2812 newScaleShapeValue[idx] = newShapeValue[idx];
2813 if (idx + 1 == newShapeValue.size())
2814 newScaleShapeValue[idx] /= blockSize;
2815 }
2816 }
2817
2818 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(
2819 newScaleShapeValue,
2820 getElementTypeOrSelf(type: adaptor.getInput()[1].getType())));
2821 }
2822 return success();
2823}
2824
2825llvm::LogicalResult tosa::ReshapeBlockScaledOp::verify() {
2826 const Operation::operand_range inputList = getInput();
2827 const Operation::result_range outputList = getResults();
2828
2829 if (inputList.size() == 0)
2830 return emitOpError(message: "requires at least one input");
2831
2832 if (inputList.size() > 2)
2833 return emitOpError(message: "requires at most two inputs");
2834
2835 if (inputList.size() != outputList.size())
2836 return emitOpError(message: "requires number of results to match inputs");
2837
2838 if (verifySameElementTypes(op: *this, /* inType = */ aType: inputList[0].getType(),
2839 /* outType = */ bType: outputList[0].getType())
2840 .failed()) {
2841 return failure();
2842 }
2843
2844 if (inputList.size() == 2 &&
2845 cast<tosa::shapeType>(Val: getNewValueShape().getType()).getRank() == 0)
2846 return emitOpError(message: "requires new shape to have a rank greater than 0");
2847
2848 const auto inputType = llvm::cast<ShapedType>(Val: inputList[0].getType());
2849 if (!inputType.hasRank())
2850 return success();
2851 const uint32_t blockSize = BlockSizeAttr::getBlockSizeValue(blockSize: getBlockSize());
2852
2853 if (inputList.size() == 2) {
2854 if (blockSize != BlockSizeAttr::getBlockSizeValue(blockSize: BlockSize::BLOCK_SIZE_32))
2855 return emitOpError(message: "expect block size to be 32, got ") << blockSize;
2856 if (llvm::any_of(Range: inputList, P: [](Value v) {
2857 const auto input = cast<ShapedType>(Val: v.getType());
2858 return input.hasRank() && input.getRank() == 0;
2859 }))
2860 return emitOpError(
2861 message: "requires all input shapes have a rank greater than 0");
2862 if (llvm::any_of(Range: outputList, P: [](Value v) {
2863 const auto output = cast<ShapedType>(Val: v.getType());
2864 return output.hasRank() && output.getRank() == 0;
2865 }))
2866 return emitOpError(
2867 message: "requires all result shapes have a rank greater than 0");
2868
2869 if (verifySameElementTypes(op: *this, /* inType = */ aType: inputList[1].getType(),
2870 /* outType = */ bType: outputList[1].getType())
2871 .failed()) {
2872 return failure();
2873 }
2874
2875 const auto inputScaleType = llvm::cast<ShapedType>(Val: inputList[1].getType());
2876 if (inputScaleType.hasRank()) {
2877 if (inputType.getRank() != inputScaleType.getRank())
2878 return emitOpError(message: "input shapes do not have same rank");
2879
2880 // Check all but the last dimension that the input shape dimensions match
2881 for (auto dimIdx = 0; dimIdx < inputType.getRank() - 1; dimIdx++) {
2882 const int64_t inputValueDim = inputType.getDimSize(idx: dimIdx);
2883 const int64_t inputScaleDim = inputScaleType.getShape()[dimIdx];
2884 if (ShapedType::isStatic(dValue: inputValueDim) &&
2885 ShapedType::isStatic(dValue: inputScaleDim) &&
2886 inputValueDim != inputScaleDim)
2887 return emitOpError(message: "input shapes for data and scale do not match on "
2888 "dimension ")
2889 << dimIdx;
2890 }
2891
2892 // Verify last dimension of input is a multiple of block size
2893 const int64_t lastValueDim =
2894 inputType.getDimSize(idx: inputType.getRank() - 1);
2895 if (ShapedType::isStatic(dValue: lastValueDim)) {
2896 if (lastValueDim % blockSize != 0)
2897 return emitOpError(message: "expect last dimension of input_data (")
2898 << lastValueDim << ") to be divisible by block_size ("
2899 << blockSize << ")";
2900
2901 const int64_t lastScaleDim =
2902 inputScaleType.getDimSize(idx: inputScaleType.getRank() - 1);
2903 // Verify last dimension of scale is lastValueDim / block size
2904 if (ShapedType::isStatic(dValue: lastScaleDim) &&
2905 lastScaleDim != lastValueDim / blockSize)
2906 return emitOpError(message: "expect last dimension of scale_data (")
2907 << lastScaleDim << ") to be " << lastValueDim << "/"
2908 << blockSize;
2909 }
2910 }
2911 } else {
2912 if (blockSize != BlockSizeAttr::getBlockSizeValue(blockSize: BlockSize::BLOCK_SIZE_1))
2913 return emitOpError(message: "expect block size to be 1, got ") << blockSize;
2914 }
2915
2916 // Get the new value shape dimension values.
2917 SmallVector<int64_t> shapeValues;
2918 if (!tosa::getConstShapeValues(op: getNewValueShape().getDefiningOp(),
2919 result_shape&: shapeValues)) {
2920 // skip following checks if shape is not constant
2921 return mlir::success();
2922 }
2923
2924 if (inputList.size() == 2) {
2925 const int64_t lastShapeDim = shapeValues.back();
2926 if (ShapedType::isStatic(dValue: lastShapeDim) && lastShapeDim % blockSize != 0)
2927 return emitOpError(message: "expect last dimension of new shape (")
2928 << lastShapeDim << ") to be divisible by block_size (" << blockSize
2929 << ")";
2930 }
2931
2932 const auto outputType = llvm::cast<ShapedType>(Val: outputList[0].getType());
2933 if (!outputType.hasRank())
2934 return success();
2935
2936 if (static_cast<int64_t>(shapeValues.size()) != outputType.getRank())
2937 return emitOpError() << "result does not match new shape rank";
2938
2939 for (auto [newShapeDim, outputShapeDim] :
2940 zip(t&: shapeValues, u: outputType.getShape())) {
2941 if (ShapedType::isStatic(dValue: newShapeDim) &&
2942 ShapedType::isStatic(dValue: outputShapeDim) && newShapeDim != outputShapeDim)
2943 return emitOpError() << "result shape is inconsistent with new shape";
2944 }
2945
2946 if (outputList.size() == 2) {
2947 // Set up scale shape from new shape given
2948 SmallVector<int64_t> scaleShapeValues(shapeValues.begin(),
2949 shapeValues.end());
2950 scaleShapeValues.back() /= blockSize;
2951
2952 const auto outputScaleType =
2953 llvm::cast<ShapedType>(Val: outputList[1].getType());
2954 if (outputScaleType.hasRank()) {
2955 if ((int64_t)scaleShapeValues.size() != outputScaleType.getRank())
2956 return emitOpError() << "result scale does not match new shape rank";
2957
2958 for (auto [newScaleShapeDim, outputScaleShapeDim] :
2959 zip(t&: scaleShapeValues, u: outputScaleType.getShape())) {
2960 if (ShapedType::isStatic(dValue: newScaleShapeDim) &&
2961 ShapedType::isStatic(dValue: outputScaleShapeDim) &&
2962 newScaleShapeDim != outputScaleShapeDim)
2963 return emitOpError()
2964 << "result scale shape is inconsistent with new shape";
2965 }
2966 }
2967 }
2968
2969 if (inputType.hasStaticShape()) {
2970 int64_t inputElementsNum = inputType.getNumElements();
2971 if (outputType.hasStaticShape()) {
2972 int64_t outputElementsNum = outputType.getNumElements();
2973 if (inputElementsNum != outputElementsNum) {
2974 return emitOpError() << "cannot reshape " << inputElementsNum
2975 << " elements into " << outputElementsNum;
2976 }
2977 }
2978
2979 int64_t newShapeElementsNum =
2980 llvm::accumulate(Range&: shapeValues, Init: int64_t(1), Op: [](int64_t acc, int64_t dim) {
2981 return (dim > 0) ? acc * dim : acc;
2982 });
2983 bool isStaticNewShape =
2984 llvm::all_of(Range&: shapeValues, P: [](int64_t s) { return s > 0; });
2985 if ((isStaticNewShape && inputElementsNum != newShapeElementsNum) ||
2986 (!isStaticNewShape && newShapeElementsNum > inputElementsNum)) {
2987 return emitOpError() << "cannot reshape " << inputElementsNum
2988 << " elements into " << newShapeElementsNum;
2989 }
2990 }
2991
2992 return mlir::success();
2993}
2994
2995// return failure if val is not a constant
2996// set zp to -1 if val is non-zero float or val is not integer nor float
2997// otherwise set zp to val's constant value
2998static FailureOr<int64_t> getZeroPoint(Value val, bool signExtend) {
2999 ElementsAttr zpAttr;
3000 if (!matchPattern(value: val, pattern: m_Constant(bind_value: &zpAttr))) {
3001 return failure();
3002 }
3003
3004 Type zpElemType = zpAttr.getElementType();
3005
3006 if (llvm::isa<FloatType>(Val: zpElemType)) {
3007 if (zpAttr.getValues<APFloat>()[0].isZero()) {
3008 return 0;
3009 }
3010 // return non-zero value to trigger error check
3011 return -1;
3012 }
3013
3014 if (llvm::isa<IntegerType>(Val: zpElemType)) {
3015 if (signExtend)
3016 return zpAttr.getValues<APInt>()[0].getSExtValue();
3017 return zpAttr.getValues<APInt>()[0].getZExtValue();
3018 }
3019
3020 // return non-zero value to trigger error check
3021 return -1;
3022}
3023
3024template <typename T>
3025static LogicalResult verifyZeroPoint(T op, Value val, const int64_t &zp,
3026 const std::string &operand) {
3027 Type zpElemType = getElementTypeOrSelf(val);
3028
3029 if (!zpElemType.isInteger(width: 8) && zp != 0) {
3030 // convert operand to lower case for error message
3031 std::string lower = operand;
3032 llvm::transform(Range&: lower, d_first: lower.begin(), F: ::tolower);
3033 return op.emitOpError()
3034 << lower << " zero point must be zero for non-int8 integer types";
3035 }
3036
3037 return success();
3038}
3039
3040static LogicalResult verifyZeroPoint(tosa::RescaleOp op, Value zpVal,
3041 const int64_t &zp,
3042 const std::string &operand) {
3043 bool isInputZp = (operand == "Input");
3044
3045 bool tensorUnsigned =
3046 isInputZp ? op.getInputUnsigned() : op.getOutputUnsigned();
3047 StringRef tensorName = isInputZp ? "input" : "output";
3048
3049 Type zpElemType = getElementTypeOrSelf(val: zpVal);
3050
3051 if (zp != 0) {
3052 if (!zpElemType.isInteger(width: 8) &&
3053 !(zpElemType.isInteger(width: 16) && tensorUnsigned)) {
3054 return op.emitOpError()
3055 << "expect " << tensorName << "_zp of 0, got " << zp;
3056 }
3057 if (zpElemType.isInteger(width: 16) && tensorUnsigned && zp != 32768) {
3058 return op.emitOpError() << "expect " << tensorName
3059 << "_zp of 0 or 32768 for unsigned int16 "
3060 << tensorName << ", got " << zp;
3061 }
3062 }
3063
3064 return success();
3065}
3066
3067#define ZERO_POINT_HELPER(OP, OPERAND_NAME, SIGN_EXTEND) \
3068 FailureOr<int64_t> tosa::OP::get##OPERAND_NAME##ZeroPoint() { \
3069 return getZeroPoint(get##OPERAND_NAME##Zp(), SIGN_EXTEND); \
3070 } \
3071 LogicalResult tosa::OP::verify##OPERAND_NAME##ZeroPoint(int64_t zp) { \
3072 return verifyZeroPoint(*this, get##OPERAND_NAME##Zp(), zp, #OPERAND_NAME); \
3073 }
3074
3075ZERO_POINT_HELPER(Conv2DOp, Input, true)
3076ZERO_POINT_HELPER(Conv2DOp, Weight, true)
3077ZERO_POINT_HELPER(Conv3DOp, Input, true)
3078ZERO_POINT_HELPER(Conv3DOp, Weight, true)
3079ZERO_POINT_HELPER(DepthwiseConv2DOp, Input, true)
3080ZERO_POINT_HELPER(DepthwiseConv2DOp, Weight, true)
3081ZERO_POINT_HELPER(TransposeConv2DOp, Input, true)
3082ZERO_POINT_HELPER(TransposeConv2DOp, Weight, true)
3083ZERO_POINT_HELPER(AvgPool2dOp, Input, true)
3084ZERO_POINT_HELPER(AvgPool2dOp, Output, true)
3085ZERO_POINT_HELPER(AvgPool2dAdaptiveOp, Input, true)
3086ZERO_POINT_HELPER(AvgPool2dAdaptiveOp, Output, true)
3087ZERO_POINT_HELPER(MatMulOp, A, true)
3088ZERO_POINT_HELPER(MatMulOp, B, true)
3089ZERO_POINT_HELPER(MatMulTOp, A, true)
3090ZERO_POINT_HELPER(MatMulTOp, B, true)
3091ZERO_POINT_HELPER(NegateOp, Input1, true)
3092ZERO_POINT_HELPER(NegateOp, Output, true)
3093ZERO_POINT_HELPER(RescaleOp, Input, !getInputUnsigned())
3094ZERO_POINT_HELPER(RescaleOp, Output, !getOutputUnsigned())
3095#undef ZERO_POINT_HELPER
3096
3097LogicalResult tosa::TransposeOp::inferReturnTypeComponents(
3098 MLIRContext *context, ::std::optional<Location> location,
3099 TransposeOp::Adaptor adaptor,
3100 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3101 ShapeAdaptor inputShape(adaptor.getInput1().getType());
3102
3103 // If input rank and permutation length is unknown, the output rank is
3104 // unknown.
3105 if (!inputShape.hasRank()) {
3106 inferredReturnShapes.push_back(Elt: ShapedTypeComponents());
3107 return success();
3108 }
3109
3110 const auto inputRank = inputShape.getRank();
3111
3112 // This would imply the number of permutations does not match the rank of
3113 // the input which is illegal.
3114 if (adaptor.getPerms().size() != static_cast<size_t>(inputRank)) {
3115 return failure();
3116 }
3117
3118 SmallVector<int64_t> outputShape;
3119 // Rank-0 means no permutations matter.
3120 if (inputRank == 0) {
3121 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
3122 return success();
3123 }
3124
3125 // Check whether the input dimensions are all the same.
3126 bool allTheSame = true;
3127 for (int i = 1, s = inputRank; i < s; i++) {
3128 if (inputShape.getDimSize(index: 0) != inputShape.getDimSize(index: i)) {
3129 allTheSame = false;
3130 break;
3131 }
3132 }
3133
3134 // If all of the input dimensions are the same we don't care about the
3135 // permutation.
3136 if (allTheSame) {
3137 outputShape.resize(N: inputRank, NV: inputShape.getDimSize(index: 0));
3138 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
3139 return success();
3140 }
3141
3142 outputShape.resize(N: inputRank, NV: ShapedType::kDynamic);
3143
3144 // Constant permutation values must be within the input rank.
3145 if (llvm::any_of(Range: adaptor.getPerms(),
3146 P: [inputRank](const auto i) { return i >= inputRank; }))
3147 return failure();
3148
3149 outputShape.reserve(N: inputRank);
3150 for (int i = 0, s = inputRank; i < s; i++) {
3151 outputShape[i] = inputShape.getDimSize(index: adaptor.getPerms()[i]);
3152 }
3153
3154 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
3155 return success();
3156}
3157
3158LogicalResult tosa::TransposeOp::verify() {
3159 if (verifySameElementTypes(op: *this, /* inType = */ aType: getInput1().getType(),
3160 /* outType = */ bType: getOutput().getType())
3161 .failed()) {
3162 return failure();
3163 }
3164
3165 const ShapeAdaptor inputShape(getInput1().getType());
3166 const ShapeAdaptor outputShape(getOutput().getType());
3167
3168 const llvm::ArrayRef<int32_t> constantPerms = getPerms();
3169
3170 if (inputShape.hasRank() &&
3171 constantPerms.size() != static_cast<size_t>(inputShape.getRank()))
3172 return emitOpError() << "expected perms attribute to have size "
3173 << inputShape.getRank()
3174 << " (input rank) but got size "
3175 << constantPerms.size();
3176
3177 if (inputShape.hasRank() && outputShape.hasRank() &&
3178 inputShape.getRank() != outputShape.getRank())
3179 return emitOpError()
3180 << "expected input tensor rank to equal result tensor rank";
3181
3182 if (outputShape.hasRank() &&
3183 constantPerms.size() != static_cast<size_t>(outputShape.getRank()))
3184 return emitOpError() << "expected perms attribute to have size "
3185 << outputShape.getRank()
3186 << " (output rank) but got size "
3187 << constantPerms.size();
3188
3189 if (!llvm::all_of(Range: constantPerms,
3190 P: [&constantPerms](int32_t s) {
3191 return s >= 0 &&
3192 static_cast<size_t>(s) < constantPerms.size();
3193 }) ||
3194 !isPermutationVector(interchange: llvm::map_to_vector(
3195 C: constantPerms, F: [](int32_t v) -> int64_t { return v; })))
3196 return emitOpError() << "expected valid permutation indices";
3197
3198 if (isa<BlockScaledType>(Val: getInput1().getType().getElementType()) &&
3199 constantPerms.back() != static_cast<int32_t>(constantPerms.size()) - 1) {
3200 return emitOpError() << "expected no-op permutation on innermost dimension "
3201 "for block scaled input";
3202 }
3203
3204 // ERROR_IF(tensor_size(shape1) != tensor_size(shape))
3205 if (inputShape.hasStaticShape() && outputShape.hasStaticShape() &&
3206 inputShape.getNumElements() != outputShape.getNumElements())
3207 return emitOpError() << "expected input1 and output to have same numbers "
3208 "of elements, got "
3209 << inputShape.getNumElements() << " and "
3210 << outputShape.getNumElements();
3211
3212 // Verify that the types of the input and output tensors are properly
3213 // permuted.
3214 if (inputShape.hasRank() && outputShape.hasRank()) {
3215 for (auto i = 0; i < outputShape.getRank(); i++) {
3216 if (inputShape.isDynamicDim(index: constantPerms[i]) ||
3217 outputShape.isDynamicDim(index: i))
3218 continue;
3219
3220 if (inputShape.getDimSize(index: constantPerms[i]) != outputShape.getDimSize(index: i))
3221 return emitOpError()
3222 << "expected output tensor dim " << i << " to match "
3223 << "input dim " << constantPerms[i] << " with value of "
3224 << inputShape.getDimSize(index: constantPerms[i]);
3225 }
3226 }
3227
3228 return success();
3229}
3230
3231LogicalResult TransposeOp::reifyResultShapes(
3232 OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
3233
3234 const llvm::ArrayRef<int32_t> transposePerms = getPerms();
3235
3236 Value input = getInput1();
3237 auto inputType = cast<TensorType>(Val: input.getType());
3238
3239 SmallVector<OpFoldResult> returnedDims(inputType.getRank());
3240 for (auto dim : transposePerms) {
3241 int32_t dimInInput = transposePerms[dim];
3242 if (inputType.isDynamicDim(idx: dimInInput))
3243 returnedDims[dim] =
3244 tensor::DimOp::create(builder, location: getLoc(), source: input, index: dimInInput)
3245 .getResult();
3246 else
3247 returnedDims[dim] =
3248 builder.getIndexAttr(value: inputType.getDimSize(idx: dimInInput));
3249 }
3250
3251 reifiedReturnShapes.emplace_back(Args: std::move(returnedDims));
3252 return success();
3253}
3254
3255LogicalResult tosa::GatherOp::inferReturnTypeComponents(
3256 MLIRContext *context, ::std::optional<Location> location,
3257 GatherOp::Adaptor adaptor,
3258 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3259 llvm::SmallVector<int64_t> outputShape;
3260 outputShape.resize(N: 3, NV: ShapedType::kDynamic);
3261
3262 ShapeAdaptor valuesShape(adaptor.getValues().getType());
3263 if (valuesShape.hasRank()) {
3264 outputShape[0] = valuesShape.getDimSize(index: 0);
3265 outputShape[2] = valuesShape.getDimSize(index: 2);
3266 }
3267
3268 ShapeAdaptor indicesShape(adaptor.getIndices().getType());
3269 if (indicesShape.hasRank()) {
3270 if (outputShape[0] == ShapedType::kDynamic)
3271 outputShape[0] = indicesShape.getDimSize(index: 0);
3272 if (outputShape[1] == ShapedType::kDynamic)
3273 outputShape[1] = indicesShape.getDimSize(index: 1);
3274 }
3275
3276 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
3277 return success();
3278}
3279
3280LogicalResult tosa::RowGatherOp::inferReturnTypeComponents(
3281 MLIRContext *context, ::std::optional<Location> location,
3282 RowGatherOp::Adaptor adaptor,
3283 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3284 llvm::SmallVector<int64_t> outputShape;
3285 outputShape.resize(N: 3, NV: ShapedType::kDynamic);
3286
3287 const ShapeAdaptor valuesShape(adaptor.getValues().getType());
3288 if (valuesShape.hasRank()) {
3289 outputShape[0] = valuesShape.getDimSize(index: 0);
3290 outputShape[2] = valuesShape.getDimSize(index: 2);
3291 }
3292
3293 const ShapeAdaptor indicesShape(adaptor.getIndices().getType());
3294 if (indicesShape.hasRank()) {
3295 if (outputShape[0] == ShapedType::kDynamic)
3296 outputShape[0] = indicesShape.getDimSize(index: 0);
3297
3298 const FailureOr<int32_t> maybeRowCount =
3299 getConstantScalarIntValue<int32_t>(val: adaptor.getRowCount());
3300 if (succeeded(Result: maybeRowCount)) {
3301 const int64_t indicesW = indicesShape.getDimSize(index: 1);
3302 if (ShapedType::isStatic(dValue: indicesW))
3303 outputShape[1] = indicesW * maybeRowCount.value();
3304 }
3305 }
3306
3307 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
3308 return success();
3309}
3310
3311LogicalResult tosa::RowGatherBlockScaledOp::inferReturnTypeComponents(
3312 MLIRContext *context, ::std::optional<Location> location,
3313 RowGatherBlockScaledOp::Adaptor adaptor,
3314 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3315 const auto values = adaptor.getValues();
3316 if (values.empty())
3317 return failure();
3318
3319 SmallVector<int64_t> dataShape(3, ShapedType::kDynamic);
3320 const ShapeAdaptor valuesShape(values.front().getType());
3321 if (valuesShape.hasRank()) {
3322 dataShape[0] = valuesShape.getDimSize(index: 0);
3323 dataShape[2] = valuesShape.getDimSize(index: 2);
3324 }
3325
3326 const ShapeAdaptor indicesShape(adaptor.getIndices().getType());
3327 if (indicesShape.hasRank()) {
3328 if (dataShape[0] == ShapedType::kDynamic)
3329 dataShape[0] = indicesShape.getDimSize(index: 0);
3330
3331 if (auto rowCount =
3332 getConstantScalarIntValue<int32_t>(val: adaptor.getRowCount());
3333 succeeded(Result: rowCount) && rowCount.value() > 0) {
3334 const int64_t indicesW = indicesShape.getDimSize(index: 1);
3335 if (ShapedType::isStatic(dValue: indicesW))
3336 dataShape[1] = indicesW * rowCount.value();
3337 }
3338 }
3339
3340 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(dataShape));
3341 if (values.size() == 1)
3342 return success();
3343
3344 SmallVector<int64_t> scaleShape = dataShape;
3345 const uint32_t blockSize =
3346 BlockSizeAttr::getBlockSizeValue(blockSize: adaptor.getBlockSize());
3347 if (ShapedType::isStatic(dValue: dataShape[2]))
3348 scaleShape[2] = dataShape[2] / blockSize;
3349
3350 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(scaleShape));
3351 return success();
3352}
3353
3354LogicalResult tosa::GatherOp::verify() {
3355 if (verifySameElementTypes(op: *this, /* inType = */ aType: getValues().getType(),
3356 /* outType = */ bType: getOutput().getType())
3357 .failed()) {
3358 return failure();
3359 }
3360
3361 const ShapeAdaptor valuesShape(getValues().getType());
3362 const ShapeAdaptor indicesShape(getIndices().getType());
3363 const ShapeAdaptor outputShape(getOutput().getType());
3364
3365 int64_t n = ShapedType::kDynamic;
3366 int64_t w = ShapedType::kDynamic;
3367 int64_t c = ShapedType::kDynamic;
3368
3369 if (valuesShape.hasRank()) {
3370 n = valuesShape.getDimSize(index: 0);
3371 c = valuesShape.getDimSize(index: 2);
3372 }
3373 if (indicesShape.hasRank()) {
3374 const int64_t indicesN = indicesShape.getDimSize(index: 0);
3375 w = indicesShape.getDimSize(index: 1);
3376 if (n == ShapedType::kDynamic)
3377 n = indicesN;
3378 else if (indicesN != ShapedType::kDynamic && n != indicesN)
3379 return emitOpError() << "requires indices dimension 0 to have size " << n
3380 << ", got " << indicesN;
3381 }
3382 if (outputShape.hasRank()) {
3383 const int64_t outputN = outputShape.getDimSize(index: 0);
3384 const int64_t outputW = outputShape.getDimSize(index: 1);
3385 const int64_t outputC = outputShape.getDimSize(index: 2);
3386 if (n != ShapedType::kDynamic && outputN != ShapedType::kDynamic &&
3387 n != outputN)
3388 return emitOpError() << "requires output dimension 0 to have size " << n
3389 << ", got " << outputN;
3390
3391 if (w != ShapedType::kDynamic && outputW != ShapedType::kDynamic &&
3392 w != outputW)
3393 return emitOpError() << "requires output dimension 1 to have size " << w
3394 << ", got " << outputW;
3395 if (c != ShapedType::kDynamic && outputC != ShapedType::kDynamic &&
3396 c != outputC)
3397 return emitOpError() << "requires output dimension 2 to have size " << c
3398 << ", got " << outputC;
3399 }
3400 return success();
3401}
3402
3403LogicalResult tosa::RowGatherOp::verify() {
3404 if (failed(Result: verifySameElementTypes(op: *this, /* inType = */ aType: getValues().getType(),
3405 /* outType = */ bType: getOutput().getType())))
3406 return failure();
3407
3408 const FailureOr<int32_t> maybeRowCount =
3409 getConstantScalarIntValue<int32_t>(val: getRowCount());
3410 if (succeeded(Result: maybeRowCount) && maybeRowCount.value() <= 0)
3411 return emitOpError() << "requires row_count to be > 0, got "
3412 << maybeRowCount.value();
3413
3414 int64_t n = ShapedType::kDynamic;
3415 int64_t c = ShapedType::kDynamic;
3416 int64_t w = ShapedType::kDynamic;
3417
3418 const ShapeAdaptor valuesShape(getValues().getType());
3419 if (valuesShape.hasRank()) {
3420 n = valuesShape.getDimSize(index: 0);
3421 c = valuesShape.getDimSize(index: 2);
3422 }
3423
3424 const ShapeAdaptor indicesShape(getIndices().getType());
3425 if (indicesShape.hasRank()) {
3426 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: n, newDim: indicesShape.getDimSize(index: 0),
3427 operandName: "indices", dimName: "batch")))
3428 return failure();
3429 w = indicesShape.getDimSize(index: 1);
3430 }
3431
3432 const ShapeAdaptor outputShape(getOutput().getType());
3433 if (outputShape.hasRank()) {
3434 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: n, newDim: outputShape.getDimSize(index: 0),
3435 operandName: "output", dimName: "batch")) ||
3436 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: c, newDim: outputShape.getDimSize(index: 2),
3437 operandName: "output", dimName: "channels")))
3438 return failure();
3439
3440 if (succeeded(Result: maybeRowCount) && maybeRowCount.value() > 0 &&
3441 ShapedType::isStatic(dValue: w)) {
3442 const int64_t expectedOutputRows = w * maybeRowCount.value();
3443 if (ShapedType::isStatic(dValue: outputShape.getDimSize(index: 1)) &&
3444 outputShape.getDimSize(index: 1) != expectedOutputRows)
3445 return emitOpError()
3446 << "requires output dimension to be equal to "
3447 "indices[1]*row_count ("
3448 << expectedOutputRows << "), got " << outputShape.getDimSize(index: 1);
3449 }
3450 }
3451
3452 return success();
3453}
3454
3455LogicalResult tosa::RowGatherBlockScaledOp::verify() {
3456 const OperandRange values = getValues();
3457 const ResultRange output = getOutput();
3458 if (values.empty() || values.size() > 2)
3459 return emitOpError()
3460 << "expects values tensor list length to be 1 or 2, got "
3461 << values.size();
3462 if (output.size() != values.size())
3463 return emitOpError()
3464 << "expects output tensor list length to match values tensor list "
3465 "length, got "
3466 << output.size() << " results for " << values.size()
3467 << " input tensors";
3468
3469 const uint32_t blockSize = BlockSizeAttr::getBlockSizeValue(blockSize: getBlockSize());
3470 if (values.size() == 1 && blockSize != 1)
3471 return emitOpError()
3472 << "requires block_size to be BLOCK_SIZE_1 when values tensor list "
3473 "length is 1";
3474 if (values.size() == 2 && blockSize == 1)
3475 return emitOpError()
3476 << "requires block_size to not be BLOCK_SIZE_1 when values tensor "
3477 "list length is 2";
3478
3479 if (failed(Result: verifySameElementTypes(op: *this, aType: values[0].getType(),
3480 bType: output[0].getType(), aName: "values[0]",
3481 bName: "output[0]")))
3482 return failure();
3483 if (values.size() == 2 && failed(Result: verifySameElementTypes(
3484 op: *this, aType: values[1].getType(), bType: output[1].getType(),
3485 aName: "values[1]", bName: "output[1]")))
3486 return failure();
3487
3488 if (auto rowCount = getConstantScalarIntValue<int32_t>(val: getRowCount());
3489 succeeded(Result: rowCount) && rowCount.value() <= 0)
3490 return emitOpError() << "requires row_count to be > 0, got "
3491 << rowCount.value();
3492
3493 int64_t n = ShapedType::kDynamic;
3494 int64_t k = ShapedType::kDynamic;
3495 int64_t c = ShapedType::kDynamic;
3496 int64_t w = ShapedType::kDynamic;
3497 int64_t multiplesOfC = ShapedType::kDynamic;
3498
3499 const ShapeAdaptor valuesDataShape(values[0].getType());
3500 if (valuesDataShape.hasRank()) {
3501 n = valuesDataShape.getDimSize(index: 0);
3502 k = valuesDataShape.getDimSize(index: 1);
3503 c = valuesDataShape.getDimSize(index: 2);
3504 }
3505
3506 if (ShapedType::isStatic(dValue: c) && c % blockSize != 0)
3507 return emitOpError() << "expects channels of values[0] (" << c
3508 << ") to be divisible by block_size (" << blockSize
3509 << ")";
3510
3511 const ShapeAdaptor indicesShape(getIndices().getType());
3512 if (indicesShape.hasRank()) {
3513 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: n, newDim: indicesShape.getDimSize(index: 0),
3514 operandName: "indices", dimName: "batch")))
3515 return failure();
3516 w = indicesShape.getDimSize(index: 1);
3517 }
3518
3519 const ShapeAdaptor outputDataShape(output[0].getType());
3520 if (outputDataShape.hasRank()) {
3521 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: n, newDim: outputDataShape.getDimSize(index: 0),
3522 operandName: "output[0]", dimName: "batch")) ||
3523 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: c, newDim: outputDataShape.getDimSize(index: 2),
3524 operandName: "output[0]", dimName: "channels")))
3525 return failure();
3526
3527 if (auto rowCount = getConstantScalarIntValue<int32_t>(val: getRowCount());
3528 succeeded(Result: rowCount) && rowCount.value() > 0 &&
3529 ShapedType::isStatic(dValue: w)) {
3530 const int64_t expectedOutputRows = w * rowCount.value();
3531 if (ShapedType::isStatic(dValue: outputDataShape.getDimSize(index: 1)) &&
3532 outputDataShape.getDimSize(index: 1) != expectedOutputRows)
3533 return emitOpError() << "requires output[0] dimension 1 to have size "
3534 << expectedOutputRows << ", got "
3535 << outputDataShape.getDimSize(index: 1);
3536 }
3537 }
3538
3539 if (values.size() == 2) {
3540 const ShapeAdaptor valuesScaleShape(values[1].getType());
3541 if (valuesScaleShape.hasRank()) {
3542 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: n, newDim: valuesScaleShape.getDimSize(index: 0),
3543 operandName: "values[1]", dimName: "batch")) ||
3544 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: k, newDim: valuesScaleShape.getDimSize(index: 1),
3545 operandName: "values[1]", dimName: "rows")))
3546 return failure();
3547 multiplesOfC = valuesScaleShape.getDimSize(index: 2);
3548 }
3549
3550 const ShapeAdaptor outputScaleShape(output[1].getType());
3551 if (outputScaleShape.hasRank()) {
3552 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: n, newDim: outputScaleShape.getDimSize(index: 0),
3553 operandName: "output[1]", dimName: "batch")))
3554 return failure();
3555
3556 if (auto rowCount = getConstantScalarIntValue<int32_t>(val: getRowCount());
3557 succeeded(Result: rowCount) && rowCount.value() > 0 &&
3558 ShapedType::isStatic(dValue: w)) {
3559 const int64_t expectedOutputRows = w * rowCount.value();
3560 if (ShapedType::isStatic(dValue: outputScaleShape.getDimSize(index: 1)) &&
3561 outputScaleShape.getDimSize(index: 1) != expectedOutputRows)
3562 return emitOpError() << "requires output[1] dimension 1 to have size "
3563 << expectedOutputRows << ", got "
3564 << outputScaleShape.getDimSize(index: 1);
3565 }
3566
3567 if (ShapedType::isDynamic(dValue: multiplesOfC))
3568 multiplesOfC = outputScaleShape.getDimSize(index: 2);
3569 else if (ShapedType::isStatic(dValue: outputScaleShape.getDimSize(index: 2)) &&
3570 multiplesOfC != outputScaleShape.getDimSize(index: 2))
3571 return emitOpError()
3572 << "expected channels of output[1] to match size "
3573 << multiplesOfC << ", got " << outputScaleShape.getDimSize(index: 2);
3574 }
3575
3576 if (ShapedType::isStatic(dValue: c) && ShapedType::isStatic(dValue: multiplesOfC) &&
3577 multiplesOfC != c / blockSize)
3578 return emitOpError()
3579 << "expects channels of scale tensors to equal C/block_size (" << c
3580 << "/" << blockSize << "), got " << multiplesOfC;
3581 }
3582
3583 return success();
3584}
3585
3586LogicalResult tosa::ResizeOp::inferReturnTypeComponents(
3587 MLIRContext *context, ::std::optional<Location> location,
3588 ResizeOp::Adaptor adaptor,
3589 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3590 llvm::SmallVector<int64_t, 4> outputShape;
3591 outputShape.resize(N: 4, NV: ShapedType::kDynamic);
3592
3593 ShapeAdaptor inputShape(adaptor.getInput().getType());
3594 if (!inputShape.hasRank())
3595 return failure();
3596
3597 outputShape[0] = inputShape.getDimSize(index: 0);
3598 outputShape[3] = inputShape.getDimSize(index: 3);
3599 int64_t inputHeight = inputShape.getDimSize(index: 1);
3600 int64_t inputWidth = inputShape.getDimSize(index: 2);
3601
3602 if ((inputHeight == ShapedType::kDynamic) ||
3603 (inputWidth == ShapedType::kDynamic))
3604 return failure();
3605
3606 SmallVector<int64_t> scaleInt, offsetInt, borderInt;
3607 if (!tosa::getConstShapeValues(op: adaptor.getScale().getDefiningOp(),
3608 result_shape&: scaleInt) ||
3609 !tosa::getConstShapeValues(op: adaptor.getOffset().getDefiningOp(),
3610 result_shape&: offsetInt) ||
3611 !tosa::getConstShapeValues(op: adaptor.getBorder().getDefiningOp(),
3612 result_shape&: borderInt)) {
3613 return failure();
3614 }
3615
3616 // Non-positive scale values are invalid, so the output shape cannot be
3617 // inferred here. They are reported by the ERROR_IF check in tosa-validate.
3618 if (llvm::any_of(Range&: scaleInt, P: [](int64_t s) { return s <= 0; }))
3619 return failure();
3620
3621 // Compute the output shape based on attributes: scale, offset, and border.
3622 const int64_t outputHeight =
3623 (((inputHeight - 1) * scaleInt[0] - offsetInt[0] + borderInt[0]) /
3624 scaleInt[1]) +
3625 1;
3626
3627 const int64_t outputWidth =
3628 (((inputWidth - 1) * scaleInt[2] - offsetInt[1] + borderInt[1]) /
3629 scaleInt[3]) +
3630 1;
3631
3632 if (outputHeight < 0 || outputWidth < 0) {
3633 return emitOptionalError(
3634 loc: location,
3635 args: "calculated output height and width must be non-negative, "
3636 "got height = ",
3637 args: outputHeight, args: ", width = ", args: outputWidth);
3638 }
3639
3640 outputShape[1] = outputHeight;
3641 outputShape[2] = outputWidth;
3642 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
3643 return success();
3644}
3645
3646LogicalResult tosa::ResizeOp::verify() {
3647 const Type inputElementType = getElementTypeOrSelf(type: getInput().getType());
3648
3649 if (isa<BlockScaledType>(Val: inputElementType) &&
3650 getMode() != ResizeMode::NEAREST_NEIGHBOR)
3651 return emitOpError(message: "requires NEAREST_NEIGHBOR mode for block scaled input");
3652
3653 return success();
3654}
3655
3656LogicalResult tosa::ScatterOp::inferReturnTypeComponents(
3657 MLIRContext *context, ::std::optional<Location> location,
3658 ScatterOp::Adaptor adaptor,
3659 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3660 llvm::SmallVector<int64_t> outputShape;
3661 outputShape.resize(N: 3, NV: ShapedType::kDynamic);
3662
3663 ShapeAdaptor valuesInShape(adaptor.getValuesIn().getType());
3664 if (valuesInShape.hasRank()) {
3665 outputShape[0] = valuesInShape.getDimSize(index: 0);
3666 outputShape[1] = valuesInShape.getDimSize(index: 1);
3667 outputShape[2] = valuesInShape.getDimSize(index: 2);
3668 }
3669
3670 ShapeAdaptor indicesShape(adaptor.getIndices().getType());
3671 if (indicesShape.hasRank()) {
3672 if (outputShape[0] == ShapedType::kDynamic)
3673 outputShape[0] = indicesShape.getDimSize(index: 0);
3674 }
3675
3676 ShapeAdaptor inputShape(adaptor.getInput().getType());
3677 if (inputShape.hasRank()) {
3678 if (outputShape[0] == ShapedType::kDynamic)
3679 outputShape[0] = inputShape.getDimSize(index: 0);
3680 if (outputShape[2] == ShapedType::kDynamic)
3681 outputShape[2] = inputShape.getDimSize(index: 2);
3682 }
3683
3684 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
3685 return success();
3686}
3687
3688LogicalResult tosa::ScatterOp::verify() {
3689 if (verifySameElementTypes(op: *this, /* inType = */ aType: getValuesIn().getType(),
3690 /* outType = */ bType: getValuesOut().getType())
3691 .failed() ||
3692 verifySameElementTypes(op: *this, /* inType = */ aType: getInput().getType(),
3693 /* outType = */ bType: getValuesOut().getType())
3694 .failed()) {
3695 return failure();
3696 }
3697
3698 const ShapeAdaptor valuesInShape(getValuesIn().getType());
3699 const ShapeAdaptor indicesShape(getIndices().getType());
3700 const ShapeAdaptor inputShape(getInput().getType());
3701 const ShapeAdaptor outputShape(getValuesOut().getType());
3702
3703 int64_t n = ShapedType::kDynamic;
3704 int64_t k = ShapedType::kDynamic;
3705 int64_t w = ShapedType::kDynamic;
3706 int64_t c = ShapedType::kDynamic;
3707 if (valuesInShape.hasRank()) {
3708 n = valuesInShape.getDimSize(index: 0);
3709 k = valuesInShape.getDimSize(index: 1);
3710 c = valuesInShape.getDimSize(index: 2);
3711 }
3712 if (indicesShape.hasRank()) {
3713 const int64_t indicesN = indicesShape.getDimSize(index: 0);
3714 w = indicesShape.getDimSize(index: 1);
3715 if (n == ShapedType::kDynamic)
3716 n = indicesN;
3717 else if (indicesN != ShapedType::kDynamic && n != indicesN)
3718 return emitOpError() << "requires indices dimension 0 to have size " << n
3719 << ", got " << indicesN;
3720 }
3721 if (inputShape.hasRank()) {
3722 const int64_t inputN = inputShape.getDimSize(index: 0);
3723 const int64_t inputW = inputShape.getDimSize(index: 1);
3724 const int64_t inputC = inputShape.getDimSize(index: 2);
3725 if (n == ShapedType::kDynamic)
3726 n = inputN;
3727 else if (inputN != ShapedType::kDynamic && n != inputN)
3728 return emitOpError() << "requires input dimension 0 to have size " << n
3729 << ", got " << inputN;
3730 if (w == ShapedType::kDynamic)
3731 w = inputW;
3732 else if (inputW != ShapedType::kDynamic && w != inputW)
3733 return emitOpError() << "requires input dimension 1 to have size " << w
3734 << ", got " << inputW;
3735
3736 if (c == ShapedType::kDynamic)
3737 c = inputC;
3738 else if (inputC != ShapedType::kDynamic && c != inputC)
3739 return emitOpError() << "requires input dimension 2 to have size " << c
3740 << ", got " << inputC;
3741 }
3742 if (outputShape.hasRank()) {
3743 const int64_t outputN = outputShape.getDimSize(index: 0);
3744 const int64_t outputK = outputShape.getDimSize(index: 1);
3745 const int64_t outputC = outputShape.getDimSize(index: 2);
3746 if (n != ShapedType::kDynamic && outputN != ShapedType::kDynamic &&
3747 n != outputN)
3748 return emitOpError() << "requires values_out dimension 0 to have size "
3749 << n << ", got " << outputN;
3750 if (k == ShapedType::kDynamic)
3751 k = outputK;
3752 else if (outputK != ShapedType::kDynamic && k != outputK)
3753 return emitOpError() << "requires values_out dimension 1 to have size "
3754 << k << ", got " << outputK;
3755 if (c != ShapedType::kDynamic && outputC != ShapedType::kDynamic &&
3756 c != outputC)
3757 return emitOpError() << "requires values_out dimension 2 to have size "
3758 << c << ", got " << outputC;
3759 }
3760 if (k != ShapedType::kDynamic && w != ShapedType::kDynamic && !(k >= w))
3761 return emitOpError() << "requires dimensions K >= W, got K=" << k
3762 << " and W=" << w;
3763
3764 return success();
3765}
3766
3767static LogicalResult ReduceInferReturnTypes(
3768 ShapeAdaptor operandShape, Type inputType, IntegerAttr axis,
3769 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3770 int64_t axisVal = axis.getValue().getSExtValue();
3771 if (!operandShape.hasRank() || operandShape.getRank() <= axisVal) {
3772 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(inputType));
3773 return success();
3774 }
3775
3776 SmallVector<int64_t> outputShape;
3777 operandShape.getDims(res&: outputShape);
3778 outputShape[axisVal] = 1;
3779 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape, inputType));
3780 return success();
3781}
3782
3783#define COMPATIBLE_RETURN_TYPES(OP) \
3784 bool OP::isCompatibleReturnTypes(TypeRange l, TypeRange r) { \
3785 if (l.size() != r.size() || l.size() != 1) \
3786 return false; \
3787 if (getElementTypeOrSelf(l[0]) != getElementTypeOrSelf(r[0])) \
3788 return false; \
3789 return succeeded(verifyCompatibleShape(l[0], r[0])); \
3790 }
3791
3792#define REDUCE_SHAPE_INFER(OP) \
3793 LogicalResult OP::inferReturnTypeComponents( \
3794 MLIRContext *context, ::std::optional<Location> location, \
3795 OP::Adaptor adaptor, \
3796 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) { \
3797 Type inputType = \
3798 llvm::cast<TensorType>(adaptor.getInput().getType()).getElementType(); \
3799 ShapeAdaptor inputShape(adaptor.getInput().getType()); \
3800 const Properties &prop = adaptor.getProperties(); \
3801 return ReduceInferReturnTypes(inputShape, inputType, prop.axis, \
3802 inferredReturnShapes); \
3803 } \
3804 COMPATIBLE_RETURN_TYPES(OP)
3805
3806REDUCE_SHAPE_INFER(tosa::ReduceAllOp)
3807REDUCE_SHAPE_INFER(tosa::ReduceAnyOp)
3808REDUCE_SHAPE_INFER(tosa::ReduceMaxOp)
3809REDUCE_SHAPE_INFER(tosa::ReduceMinOp)
3810REDUCE_SHAPE_INFER(tosa::ReduceProductOp)
3811REDUCE_SHAPE_INFER(tosa::ReduceSumOp)
3812#undef REDUCE_SHAPE_INFER
3813COMPATIBLE_RETURN_TYPES(tosa::ConcatOp)
3814#undef COMPATIBLE_RETURN_TYPES
3815
3816template <typename T>
3817static LogicalResult verifyReduceOp(T op) {
3818 // All TOSA reduce Ops have input, output and axis.
3819 TensorType inputType = op.getInput().getType();
3820 TensorType outputType = op.getOutput().getType();
3821 int32_t reduceAxis = op.getAxis();
3822
3823 if (reduceAxis < 0) {
3824 op.emitOpError("reduce axis must not be negative");
3825 return failure();
3826 }
3827 if (inputType.hasRank()) {
3828 int64_t inputRank = inputType.getRank();
3829 // We allow for a special case where the input/output shape has rank 0 and
3830 // axis is also 0.
3831 if (reduceAxis >= inputRank && (reduceAxis != 0 || inputRank != 0)) {
3832 op.emitOpError("expect input tensor rank (")
3833 << inputRank << ") to be larger than reduce axis (" << reduceAxis
3834 << ")";
3835 return failure();
3836 }
3837 }
3838 if (outputType.hasRank()) {
3839 int64_t outputRank = outputType.getRank();
3840 if (inputType.hasRank() && outputRank != inputType.getRank()) {
3841 op.emitOpError(
3842 "expect output tensor rank to be equal to input tensor rank");
3843 return failure();
3844 }
3845 if (reduceAxis >= outputRank && (reduceAxis != 0 || outputRank != 0)) {
3846 op.emitOpError("expect output tensor rank (")
3847 << outputRank << ") to be larger than reduce axis (" << reduceAxis
3848 << ")";
3849 return failure();
3850 }
3851 // We can only verify the reduced dimension size to be 1 if this is not
3852 // the special case of output rank == 0.
3853 if (outputRank != 0) {
3854 auto outputShape = outputType.getShape();
3855 if (!outputType.isDynamicDim(idx: reduceAxis) &&
3856 outputShape[reduceAxis] != 1) {
3857 op.emitOpError("expect reduced dimension size to be 1, got ")
3858 << outputShape[reduceAxis];
3859 return failure();
3860 }
3861 }
3862 }
3863 return success();
3864}
3865
3866LogicalResult tosa::ReduceAllOp::verify() { return verifyReduceOp(op: *this); }
3867LogicalResult tosa::ReduceAnyOp::verify() { return verifyReduceOp(op: *this); }
3868LogicalResult tosa::ReduceMaxOp::verify() { return verifyReduceOp(op: *this); }
3869LogicalResult tosa::ReduceMinOp::verify() { return verifyReduceOp(op: *this); }
3870LogicalResult tosa::ReduceProductOp::verify() { return verifyReduceOp(op: *this); }
3871LogicalResult tosa::ReduceSumOp::verify() { return verifyReduceOp(op: *this); }
3872
3873static LogicalResult NAryInferReturnTypes(
3874 const ValueShapeRange &operands,
3875 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3876 llvm::SmallVector<int64_t> outShape;
3877 if (resolveBroadcastShape(operands, outShape).failed()) {
3878 inferredReturnShapes.push_back(Elt: ShapedTypeComponents());
3879 } else {
3880 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outShape));
3881 }
3882 return success();
3883}
3884
3885#define NARY_SHAPE_INFER(OP) \
3886 LogicalResult OP::inferReturnTypeComponents( \
3887 MLIRContext *context, ::std::optional<Location> location, \
3888 ValueShapeRange operands, DictionaryAttr attributes, \
3889 PropertyRef properties, RegionRange regions, \
3890 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) { \
3891 return NAryInferReturnTypes(operands, inferredReturnShapes); \
3892 }
3893
3894NARY_SHAPE_INFER(tosa::AbsOp)
3895NARY_SHAPE_INFER(tosa::AddOp)
3896NARY_SHAPE_INFER(tosa::ArithmeticRightShiftOp)
3897NARY_SHAPE_INFER(tosa::BitwiseAndOp)
3898NARY_SHAPE_INFER(tosa::BitwiseOrOp)
3899NARY_SHAPE_INFER(tosa::BitwiseXorOp)
3900NARY_SHAPE_INFER(tosa::BitwiseNotOp)
3901NARY_SHAPE_INFER(tosa::CastOp)
3902NARY_SHAPE_INFER(tosa::CeilOp)
3903NARY_SHAPE_INFER(tosa::ClampOp)
3904NARY_SHAPE_INFER(tosa::ClzOp)
3905NARY_SHAPE_INFER(tosa::CosOp)
3906NARY_SHAPE_INFER(tosa::ExpOp)
3907NARY_SHAPE_INFER(tosa::FloorOp)
3908NARY_SHAPE_INFER(tosa::GreaterEqualOp)
3909NARY_SHAPE_INFER(tosa::GreaterOp)
3910NARY_SHAPE_INFER(tosa::IdentityOp)
3911NARY_SHAPE_INFER(tosa::IntDivOp)
3912NARY_SHAPE_INFER(tosa::LogOp)
3913NARY_SHAPE_INFER(tosa::LogicalAndOp)
3914NARY_SHAPE_INFER(tosa::LogicalLeftShiftOp)
3915NARY_SHAPE_INFER(tosa::LogicalNotOp)
3916NARY_SHAPE_INFER(tosa::LogicalOrOp)
3917NARY_SHAPE_INFER(tosa::LogicalRightShiftOp)
3918NARY_SHAPE_INFER(tosa::LogicalXorOp)
3919NARY_SHAPE_INFER(tosa::MaximumOp)
3920NARY_SHAPE_INFER(tosa::MinimumOp)
3921NARY_SHAPE_INFER(tosa::PowOp)
3922NARY_SHAPE_INFER(tosa::ReciprocalOp)
3923NARY_SHAPE_INFER(tosa::ReverseOp)
3924NARY_SHAPE_INFER(tosa::RsqrtOp)
3925NARY_SHAPE_INFER(tosa::SinOp)
3926NARY_SHAPE_INFER(tosa::SelectOp)
3927NARY_SHAPE_INFER(tosa::SubOp)
3928NARY_SHAPE_INFER(tosa::TanhOp)
3929NARY_SHAPE_INFER(tosa::ErfOp)
3930NARY_SHAPE_INFER(tosa::SigmoidOp)
3931#undef PRED_SHAPE_INFER
3932
3933LogicalResult tosa::NegateOp::inferReturnTypeComponents(
3934 MLIRContext *context, ::std::optional<Location> location,
3935 NegateOp::Adaptor adaptor,
3936 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3937 ShapeAdaptor inputShape(adaptor.getInput1().getType());
3938 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(inputShape));
3939 return success();
3940}
3941
3942LogicalResult tosa::NegateOp::verify() {
3943 // Verify same element type
3944 const Type input1Type = getInput1().getType();
3945 const Type outputType = getOutput().getType();
3946 if (verifySameElementTypes(op: *this, aType: input1Type, bType: outputType).failed())
3947 return failure();
3948
3949 // Verify same shape
3950 const SmallVector<Type, 2> types = {input1Type, outputType};
3951 if (failed(Result: verifyCompatibleShapes(types)))
3952 return emitOpError() << "requires the same shape for input1 and output";
3953
3954 const Type input1EType = getStorageElementTypeOrSelf(type: getInput1().getType());
3955 const Type input1ZpEType =
3956 getStorageElementTypeOrSelf(type: getInput1Zp().getType());
3957 if (input1EType != input1ZpEType) {
3958 return emitOpError(message: "expect both input1 and its zero point are the same "
3959 "element type, got ")
3960 << input1EType << " and " << input1ZpEType;
3961 }
3962 const Type outputEType = getStorageElementTypeOrSelf(type: getOutput().getType());
3963 const Type outputZpEType =
3964 getStorageElementTypeOrSelf(type: getOutputZp().getType());
3965 if (outputEType != outputZpEType) {
3966 return emitOpError(message: "expect both output and its zero point are the same "
3967 "element type, got ")
3968 << outputEType << " and " << outputZpEType;
3969 }
3970
3971 FailureOr<int64_t> maybeIZp = getInput1ZeroPoint();
3972 if (succeeded(Result: maybeIZp) && verifyInput1ZeroPoint(zp: *maybeIZp).failed())
3973 return failure();
3974
3975 FailureOr<int64_t> maybeOZp = getOutputZeroPoint();
3976 if (succeeded(Result: maybeOZp) && verifyOutputZeroPoint(zp: *maybeOZp).failed())
3977 return failure();
3978
3979 return success();
3980}
3981
3982static LogicalResult poolingInferReturnTypes(
3983 ShapeAdaptor inputShape, ArrayRef<int64_t> kernel, ArrayRef<int64_t> stride,
3984 ArrayRef<int64_t> pad,
3985 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
3986 llvm::SmallVector<int64_t> outputShape;
3987 outputShape.resize(N: 4, NV: ShapedType::kDynamic);
3988
3989 // We only know the rank if the input type is unranked.
3990 if (!inputShape.hasRank()) {
3991 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
3992 return success();
3993 }
3994
3995 // Batch and number of channels are identical for pooling layer.
3996 outputShape[0] = inputShape.getDimSize(index: 0);
3997 outputShape[3] = inputShape.getDimSize(index: 3);
3998
3999 int64_t height = inputShape.getDimSize(index: 1);
4000 int64_t width = inputShape.getDimSize(index: 2);
4001
4002 if (ShapedType::isStatic(dValue: height)) {
4003 int64_t padded = height + pad[0] + pad[1] - kernel[0];
4004 outputShape[1] = padded / stride[0] + 1;
4005 }
4006
4007 if (ShapedType::isStatic(dValue: width)) {
4008 int64_t padded = width + pad[2] + pad[3] - kernel[1];
4009 outputShape[2] = padded / stride[1] + 1;
4010 }
4011
4012 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
4013 return success();
4014}
4015
4016template <typename AdaptorT>
4017class ConvInferShapeAdaptor;
4018
4019class ConvInferShapeAdaptorBase {
4020protected:
4021 static void updateIfDynamic(int64_t &current, int64_t candidate) {
4022 if (ShapedType::isDynamic(dValue: current))
4023 current = candidate;
4024 }
4025};
4026
4027template <>
4028class ConvInferShapeAdaptor<Conv2DOp::Adaptor>
4029 : public ConvInferShapeAdaptorBase {
4030public:
4031 explicit ConvInferShapeAdaptor(Conv2DOp::Adaptor adaptor)
4032 : adaptor(adaptor) {}
4033
4034 void inferInputShape(SmallVectorImpl<int64_t> &outputShape,
4035 SmallVectorImpl<int64_t> &inputSpatial) {
4036 const ShapeAdaptor inputShape(adaptor.getInput().getType());
4037 if (!inputShape.hasRank())
4038 return;
4039
4040 const int64_t outputBatch = inputShape.getDimSize(index: 0);
4041 const int64_t inputHeight = inputShape.getDimSize(index: 1);
4042 const int64_t inputWidth = inputShape.getDimSize(index: 2);
4043
4044 outputShape[0] = outputBatch;
4045 inputSpatial[0] = inputHeight;
4046 inputSpatial[1] = inputWidth;
4047 }
4048
4049 void inferWeightShape(SmallVectorImpl<int64_t> &outputShape,
4050 SmallVectorImpl<int64_t> &weightSpatial) {
4051 const ShapeAdaptor weightShape(adaptor.getWeight().getType());
4052 if (!weightShape.hasRank())
4053 return;
4054
4055 const int64_t outputChannels = weightShape.getDimSize(index: 0);
4056 const int64_t kernelHeight = weightShape.getDimSize(index: 1);
4057 const int64_t kernelWidth = weightShape.getDimSize(index: 2);
4058
4059 outputShape[3] = outputChannels;
4060 weightSpatial[0] = kernelHeight;
4061 weightSpatial[1] = kernelWidth;
4062 }
4063
4064 int64_t getNumSpatialDims() const { return 2; }
4065 int64_t getOutputRank() const { return 4; }
4066
4067 LogicalResult getSpatialParameters(SmallVector<int64_t> &padValues,
4068 SmallVector<int64_t> &strideValues,
4069 SmallVector<int64_t> &dilationValues) {
4070 padValues.assign(in_start: adaptor.getPad().begin(), in_end: adaptor.getPad().end());
4071 strideValues.assign(in_start: adaptor.getStride().begin(), in_end: adaptor.getStride().end());
4072 dilationValues.assign(in_start: adaptor.getDilation().begin(),
4073 in_end: adaptor.getDilation().end());
4074 return success();
4075 }
4076
4077private:
4078 Conv2DOp::Adaptor adaptor;
4079};
4080
4081template <>
4082class ConvInferShapeAdaptor<Conv2DBlockScaledOp::Adaptor>
4083 : public ConvInferShapeAdaptorBase {
4084public:
4085 explicit ConvInferShapeAdaptor(Conv2DBlockScaledOp::Adaptor adaptor)
4086 : adaptor(adaptor) {}
4087
4088 void inferInputShape(SmallVectorImpl<int64_t> &outputShape,
4089 SmallVectorImpl<int64_t> &inputSpatial) {
4090 const ShapeAdaptor inputDataShape(adaptor.getInputData().getType());
4091 if (inputDataShape.hasRank()) {
4092 const int64_t outputBatch = inputDataShape.getDimSize(index: 0);
4093 const int64_t inputHeight = inputDataShape.getDimSize(index: 1);
4094 const int64_t inputWidth = inputDataShape.getDimSize(index: 2);
4095
4096 outputShape[0] = outputBatch;
4097 inputSpatial[0] = inputHeight;
4098 inputSpatial[1] = inputWidth;
4099 }
4100
4101 const ShapeAdaptor inputScaleShape(adaptor.getInputScale().getType());
4102 if (!inputScaleShape.hasRank())
4103 return;
4104
4105 const int64_t scaleBatch = inputScaleShape.getDimSize(index: 0);
4106 const int64_t scaleHeight = inputScaleShape.getDimSize(index: 1);
4107 const int64_t scaleWidth = inputScaleShape.getDimSize(index: 2);
4108
4109 updateIfDynamic(current&: outputShape[0], candidate: scaleBatch);
4110 updateIfDynamic(current&: inputSpatial[0], candidate: scaleHeight);
4111 updateIfDynamic(current&: inputSpatial[1], candidate: scaleWidth);
4112 }
4113
4114 void inferWeightShape(SmallVectorImpl<int64_t> &outputShape,
4115 SmallVectorImpl<int64_t> &weightSpatial) {
4116 const ShapeAdaptor weightDataShape(adaptor.getWeightData().getType());
4117 if (weightDataShape.hasRank()) {
4118 const int64_t outputChannels = weightDataShape.getDimSize(index: 0);
4119 const int64_t kernelHeight = weightDataShape.getDimSize(index: 1);
4120 const int64_t kernelWidth = weightDataShape.getDimSize(index: 2);
4121
4122 outputShape[3] = outputChannels;
4123 weightSpatial[0] = kernelHeight;
4124 weightSpatial[1] = kernelWidth;
4125 }
4126
4127 const ShapeAdaptor weightScaleShape(adaptor.getWeightScale().getType());
4128 if (!weightScaleShape.hasRank())
4129 return;
4130
4131 const int64_t scaleOutputChannels = weightScaleShape.getDimSize(index: 0);
4132 const int64_t scaleKernelHeight = weightScaleShape.getDimSize(index: 1);
4133 const int64_t scaleKernelWidth = weightScaleShape.getDimSize(index: 2);
4134
4135 updateIfDynamic(current&: outputShape[3], candidate: scaleOutputChannels);
4136 updateIfDynamic(current&: weightSpatial[0], candidate: scaleKernelHeight);
4137 updateIfDynamic(current&: weightSpatial[1], candidate: scaleKernelWidth);
4138 }
4139
4140 int64_t getNumSpatialDims() const { return 2; }
4141 int64_t getOutputRank() const { return 4; }
4142
4143 LogicalResult getSpatialParameters(SmallVector<int64_t> &padValues,
4144 SmallVector<int64_t> &strideValues,
4145 SmallVector<int64_t> &dilationValues) {
4146 if (!tosa::getConstShapeValues(op: adaptor.getPad().getDefiningOp(),
4147 result_shape&: padValues) ||
4148 !tosa::getConstShapeValues(op: adaptor.getStride().getDefiningOp(),
4149 result_shape&: strideValues) ||
4150 !tosa::getConstShapeValues(op: adaptor.getDilation().getDefiningOp(),
4151 result_shape&: dilationValues))
4152 return failure();
4153 return success();
4154 }
4155
4156private:
4157 Conv2DBlockScaledOp::Adaptor adaptor;
4158};
4159
4160template <>
4161class ConvInferShapeAdaptor<Conv3DOp::Adaptor>
4162 : public ConvInferShapeAdaptorBase {
4163public:
4164 explicit ConvInferShapeAdaptor(Conv3DOp::Adaptor adaptor)
4165 : adaptor(adaptor) {}
4166
4167 void inferInputShape(SmallVectorImpl<int64_t> &outputShape,
4168 SmallVectorImpl<int64_t> &inputSpatial) {
4169 const ShapeAdaptor inputShape(adaptor.getInput().getType());
4170 if (!inputShape.hasRank())
4171 return;
4172
4173 const int64_t outputBatch = inputShape.getDimSize(index: 0);
4174 const int64_t inputDepth = inputShape.getDimSize(index: 1);
4175 const int64_t inputHeight = inputShape.getDimSize(index: 2);
4176 const int64_t inputWidth = inputShape.getDimSize(index: 3);
4177
4178 outputShape[0] = outputBatch;
4179 inputSpatial[0] = inputDepth;
4180 inputSpatial[1] = inputHeight;
4181 inputSpatial[2] = inputWidth;
4182 }
4183
4184 void inferWeightShape(SmallVectorImpl<int64_t> &outputShape,
4185 SmallVectorImpl<int64_t> &weightSpatial) {
4186 const ShapeAdaptor weightShape(adaptor.getWeight().getType());
4187 if (!weightShape.hasRank())
4188 return;
4189
4190 const int64_t outputChannels = weightShape.getDimSize(index: 0);
4191 const int64_t kernelDepth = weightShape.getDimSize(index: 1);
4192 const int64_t kernelHeight = weightShape.getDimSize(index: 2);
4193 const int64_t kernelWidth = weightShape.getDimSize(index: 3);
4194
4195 outputShape[4] = outputChannels;
4196 weightSpatial[0] = kernelDepth;
4197 weightSpatial[1] = kernelHeight;
4198 weightSpatial[2] = kernelWidth;
4199 }
4200
4201 int64_t getNumSpatialDims() const { return 3; }
4202 int64_t getOutputRank() const { return 5; }
4203
4204 LogicalResult getSpatialParameters(SmallVector<int64_t> &padValues,
4205 SmallVector<int64_t> &strideValues,
4206 SmallVector<int64_t> &dilationValues) {
4207 padValues.assign(in_start: adaptor.getPad().begin(), in_end: adaptor.getPad().end());
4208 strideValues.assign(in_start: adaptor.getStride().begin(), in_end: adaptor.getStride().end());
4209 dilationValues.assign(in_start: adaptor.getDilation().begin(),
4210 in_end: adaptor.getDilation().end());
4211 return success();
4212 }
4213
4214private:
4215 Conv3DOp::Adaptor adaptor;
4216};
4217
4218template <typename AdaptorT>
4219LogicalResult inferConvReturnTypeComponents(
4220 AdaptorT adaptor,
4221 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4222 ConvInferShapeAdaptor<AdaptorT> convShapeAdaptor(adaptor);
4223 llvm::SmallVector<int64_t> outputShape(convShapeAdaptor.getOutputRank(),
4224 ShapedType::kDynamic);
4225 llvm::SmallVector<int64_t> inputSpatial(convShapeAdaptor.getNumSpatialDims(),
4226 ShapedType::kDynamic);
4227 llvm::SmallVector<int64_t> weightSpatial(convShapeAdaptor.getNumSpatialDims(),
4228 ShapedType::kDynamic);
4229
4230 convShapeAdaptor.inferInputShape(outputShape, inputSpatial);
4231 convShapeAdaptor.inferWeightShape(outputShape, weightSpatial);
4232
4233 const ShapeAdaptor biasShape = adaptor.getBias().getType();
4234 if (biasShape.hasRank()) {
4235 const int64_t biasSize = biasShape.getDimSize(index: 0);
4236 if (biasSize != 1) {
4237 const size_t outputChannelDim = convShapeAdaptor.getOutputRank() - 1;
4238 outputShape[outputChannelDim] =
4239 ShapedType::isDynamic(dValue: outputShape[outputChannelDim])
4240 ? biasSize
4241 : outputShape[outputChannelDim];
4242 }
4243 }
4244
4245 SmallVector<int64_t> padValues;
4246 SmallVector<int64_t> strideValues;
4247 SmallVector<int64_t> dilationValues;
4248 if (failed(convShapeAdaptor.getSpatialParameters(padValues, strideValues,
4249 dilationValues))) {
4250 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
4251 return success();
4252 }
4253
4254 for (int64_t dim = 0; dim < convShapeAdaptor.getNumSpatialDims(); ++dim) {
4255 if (!ShapedType::isStatic(dValue: inputSpatial[dim]) ||
4256 !ShapedType::isStatic(dValue: weightSpatial[dim]))
4257 continue;
4258 const int64_t inputSize =
4259 inputSpatial[dim] + padValues[2 * dim] + padValues[2 * dim + 1];
4260 const int64_t filterSize =
4261 (weightSpatial[dim] - 1) * dilationValues[dim] + 1;
4262 const int64_t unstridedResult = inputSize - filterSize + 1;
4263 outputShape[dim + 1] = (unstridedResult - 1) / strideValues[dim] + 1;
4264 }
4265
4266 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
4267 return success();
4268}
4269
4270LogicalResult Conv2DOp::inferReturnTypeComponents(
4271 MLIRContext *context, ::std::optional<Location> location,
4272 Conv2DOp::Adaptor adaptor,
4273 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4274 return inferConvReturnTypeComponents(adaptor, inferredReturnShapes);
4275}
4276
4277LogicalResult Conv2DOp::verify() {
4278 if (verifyConvOp(op: *this).failed() || verifyConvOpErrorIf(op: *this).failed())
4279 return failure();
4280 return success();
4281}
4282
4283LogicalResult Conv2DBlockScaledOp::inferReturnTypeComponents(
4284 MLIRContext *context, ::std::optional<Location> location,
4285 Conv2DBlockScaledOp::Adaptor adaptor,
4286 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4287 return inferConvReturnTypeComponents(adaptor, inferredReturnShapes);
4288}
4289
4290LogicalResult Conv2DBlockScaledOp::verify() {
4291 if (failed(Result: verifySameElementTypes(op: *this, aType: getInputData().getType(),
4292 bType: getWeightData().getType(), aName: "input_data",
4293 bName: "weight_data")) ||
4294 failed(Result: verifySameElementTypes(op: *this, aType: getInputScale().getType(),
4295 bType: getWeightScale().getType(), aName: "input_scale",
4296 bName: "weight_scale")) ||
4297 failed(Result: verifySameElementTypes(op: *this, aType: getBias().getType(),
4298 bType: getOutput().getType(), aName: "bias", bName: "output")))
4299 return failure();
4300
4301 // Verify input shape compatibility
4302 int64_t N = ShapedType::kDynamic;
4303 int64_t IH = ShapedType::kDynamic;
4304 int64_t IW = ShapedType::kDynamic;
4305 int64_t IC = ShapedType::kDynamic;
4306 int64_t multiplesOfIC = ShapedType::kDynamic;
4307 int64_t OC = ShapedType::kDynamic;
4308 int64_t KH = ShapedType::kDynamic;
4309 int64_t KW = ShapedType::kDynamic;
4310
4311 const ShapeAdaptor inputDataShape(getInputData().getType());
4312 if (inputDataShape.hasRank()) {
4313 N = inputDataShape.getDimSize(index: 0);
4314 IH = inputDataShape.getDimSize(index: 1);
4315 IW = inputDataShape.getDimSize(index: 2);
4316 IC = inputDataShape.getDimSize(index: 3);
4317 }
4318
4319 const ShapeAdaptor inputScaleShape(getInputScale().getType());
4320 if (inputScaleShape.hasRank()) {
4321 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: N, newDim: inputScaleShape.getDimSize(index: 0),
4322 operandName: "input_scale", dimName: "batch size")) ||
4323 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: IH, newDim: inputScaleShape.getDimSize(index: 1),
4324 operandName: "input_scale", dimName: "input height")) ||
4325 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: IW, newDim: inputScaleShape.getDimSize(index: 2),
4326 operandName: "input_scale", dimName: "input width")))
4327 return failure();
4328 multiplesOfIC = inputScaleShape.getDimSize(index: 3);
4329 }
4330
4331 const ShapeAdaptor weightDataShape(getWeightData().getType());
4332 if (weightDataShape.hasRank()) {
4333 OC = weightDataShape.getDimSize(index: 0);
4334 KH = weightDataShape.getDimSize(index: 1);
4335 KW = weightDataShape.getDimSize(index: 2);
4336 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: IC, newDim: weightDataShape.getDimSize(index: 3),
4337 operandName: "weight_data", dimName: "input channels")))
4338 return failure();
4339 }
4340
4341 const ShapeAdaptor weightScaleShape(getWeightScale().getType());
4342 if (weightScaleShape.hasRank()) {
4343 if (failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: OC, newDim: weightScaleShape.getDimSize(index: 0),
4344 operandName: "weight_scale", dimName: "output channels")) ||
4345 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: KH, newDim: weightScaleShape.getDimSize(index: 1),
4346 operandName: "weight_scale", dimName: "kernel height")) ||
4347 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: KW, newDim: weightScaleShape.getDimSize(index: 2),
4348 operandName: "weight_scale", dimName: "kernel width")) ||
4349 failed(Result: tryUpdateDimOrFailure(op: *this, currDim&: multiplesOfIC,
4350 newDim: weightScaleShape.getDimSize(index: 3),
4351 operandName: "weight_scale", dimName: "input channel blocks")))
4352 return failure();
4353 }
4354
4355 const uint32_t blockSize = BlockSizeAttr::getBlockSizeValue(blockSize: getBlockSize());
4356 if (blockSize != BlockSizeAttr::getBlockSizeValue(blockSize: BlockSize::BLOCK_SIZE_32))
4357 return emitOpError(message: "expect block size to be 32, got ") << blockSize;
4358 // Verify IC is a multiple of block size
4359 if (ShapedType::isStatic(dValue: IC) && IC % blockSize != 0)
4360 return emitOpError(message: "expect IC to be a multiple of block size, got IC=")
4361 << IC << ", block_size=" << blockSize;
4362
4363 // Verify multiplesOfIC is IC / block size
4364 if (ShapedType::isStatic(dValue: IC) && ShapedType::isStatic(dValue: multiplesOfIC) &&
4365 multiplesOfIC != IC / blockSize)
4366 return emitOpError(
4367 message: "expect scale operands dimension 2 to equal IC/block_size (")
4368 << IC << "/" << blockSize << ")"
4369 << ", got " << multiplesOfIC;
4370
4371 // Verify pad/stride/dilation values
4372 SmallVector<int64_t> padValues;
4373 if (tosa::getConstShapeValues(op: getPad().getDefiningOp(), result_shape&: padValues)) {
4374 if (llvm::any_of(Range&: padValues, P: [](int64_t p) { return p < 0; }))
4375 return emitOpError(message: "expect all padding values to be >= 0, got ")
4376 << padValues;
4377 }
4378
4379 SmallVector<int64_t> strideValues;
4380 if (tosa::getConstShapeValues(op: getStride().getDefiningOp(), result_shape&: strideValues)) {
4381 if (llvm::any_of(Range&: strideValues, P: [](int64_t s) { return s < 1; }))
4382 return emitOpError(message: "expect all stride values to be >= 1, got ")
4383 << strideValues;
4384 }
4385
4386 SmallVector<int64_t> dilationValues;
4387 if (tosa::getConstShapeValues(op: getDilation().getDefiningOp(),
4388 result_shape&: dilationValues)) {
4389 if (llvm::any_of(Range&: dilationValues, P: [](int64_t d) { return d < 1; }))
4390 return emitOpError(message: "expect all dilation values to be >= 1, got ")
4391 << dilationValues;
4392 }
4393
4394 // Verify output shape compatibility
4395 const ShapeAdaptor outputShape(getOutput().getType());
4396 if (!padValues.empty() && !strideValues.empty() && !dilationValues.empty() &&
4397 outputShape.hasRank()) {
4398 if (failed(Result: verifyConvOutputSize(op: *this, inputSize: IH, kernelSize: KH, outputSize: outputShape.getDimSize(index: 1),
4399 padBefore: padValues[0], padAfter: padValues[1], stride: strideValues[0],
4400 dilation: dilationValues[0], dimName: "height", dimAxis: "y", padBeforeName: "top",
4401 padAfterName: "bottom")) ||
4402 failed(Result: verifyConvOutputSize(op: *this, inputSize: IW, kernelSize: KW, outputSize: outputShape.getDimSize(index: 2),
4403 padBefore: padValues[2], padAfter: padValues[3], stride: strideValues[1],
4404 dilation: dilationValues[1], dimName: "width", dimAxis: "x", padBeforeName: "left",
4405 padAfterName: "right")))
4406 return failure();
4407 }
4408
4409 // Verify bias
4410 const ShapeAdaptor biasShape(getBias().getType());
4411 if (biasShape.hasRank() && outputShape.hasRank()) {
4412 const int64_t biasChannels = biasShape.getDimSize(index: 0);
4413 const int64_t outputChannels =
4414 outputShape.getDimSize(index: outputShape.getRank() - 1);
4415 if (biasChannels == ShapedType::kDynamic ||
4416 outputChannels == ShapedType::kDynamic)
4417 // Skip following checks if biasChannels or outputChannels is dynamic dim
4418 return success();
4419
4420 if (biasChannels != outputChannels && biasChannels != 1)
4421 return emitOpError(
4422 message: "bias channels expected to be equal to output channels (")
4423 << outputChannels << ") or 1, got " << biasChannels;
4424 }
4425
4426 return success();
4427}
4428
4429LogicalResult Conv3DOp::inferReturnTypeComponents(
4430 MLIRContext *context, ::std::optional<Location> location,
4431 Conv3DOp::Adaptor adaptor,
4432 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4433 return inferConvReturnTypeComponents(adaptor, inferredReturnShapes);
4434}
4435
4436LogicalResult Conv3DOp::verify() {
4437 if (verifyConvOp(op: *this).failed() || verifyConvOpErrorIf(op: *this).failed())
4438 return failure();
4439 return success();
4440}
4441
4442LogicalResult AvgPool2dOp::inferReturnTypeComponents(
4443 MLIRContext *context, ::std::optional<Location> location,
4444 AvgPool2dOp::Adaptor adaptor,
4445 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4446 ShapeAdaptor inputShape(adaptor.getInput().getType());
4447 const Properties &prop = adaptor.getProperties();
4448 return poolingInferReturnTypes(inputShape, kernel: prop.kernel, stride: prop.stride, pad: prop.pad,
4449 inferredReturnShapes);
4450}
4451
4452LogicalResult AvgPool2dAdaptiveOp::inferReturnTypeComponents(
4453 MLIRContext *context, ::std::optional<Location> location,
4454 AvgPool2dAdaptiveOp::Adaptor adaptor,
4455 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4456 ShapeAdaptor inputShape(adaptor.getInput().getType());
4457
4458 llvm::SmallVector<int64_t> kernelValues;
4459 llvm::SmallVector<int64_t> strideValues;
4460 llvm::SmallVector<int64_t> padValues;
4461 if (tosa::getConstShapeValues(op: adaptor.getKernel().getDefiningOp(),
4462 result_shape&: kernelValues) &&
4463 tosa::getConstShapeValues(op: adaptor.getStride().getDefiningOp(),
4464 result_shape&: strideValues) &&
4465 tosa::getConstShapeValues(op: adaptor.getPad().getDefiningOp(), result_shape&: padValues)) {
4466 return poolingInferReturnTypes(inputShape, kernel: kernelValues, stride: strideValues,
4467 pad: padValues, inferredReturnShapes);
4468 }
4469
4470 llvm::SmallVector<int64_t> outputShape(4, ShapedType::kDynamic);
4471 if (inputShape.hasRank()) {
4472 // Keep N & C as pooling only changes H & W.
4473 outputShape[0] = inputShape.getDimSize(index: 0);
4474 outputShape[3] = inputShape.getDimSize(index: 3);
4475 }
4476
4477 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
4478 return success();
4479}
4480
4481LogicalResult MaxPool2dOp::inferReturnTypeComponents(
4482 MLIRContext *context, ::std::optional<Location> location,
4483 MaxPool2dOp::Adaptor adaptor,
4484 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4485 ShapeAdaptor inputShape(adaptor.getInput().getType());
4486 const Properties &prop = adaptor.getProperties();
4487 return poolingInferReturnTypes(inputShape, kernel: prop.kernel, stride: prop.stride, pad: prop.pad,
4488 inferredReturnShapes);
4489}
4490
4491LogicalResult MaxPool2dAdaptiveOp::inferReturnTypeComponents(
4492 MLIRContext *context, ::std::optional<Location> location,
4493 MaxPool2dAdaptiveOp::Adaptor adaptor,
4494 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4495 ShapeAdaptor inputShape(adaptor.getInput().getType());
4496
4497 llvm::SmallVector<int64_t> kernelValues;
4498 llvm::SmallVector<int64_t> strideValues;
4499 llvm::SmallVector<int64_t> padValues;
4500 if (tosa::getConstShapeValues(op: adaptor.getKernel().getDefiningOp(),
4501 result_shape&: kernelValues) &&
4502 tosa::getConstShapeValues(op: adaptor.getStride().getDefiningOp(),
4503 result_shape&: strideValues) &&
4504 tosa::getConstShapeValues(op: adaptor.getPad().getDefiningOp(), result_shape&: padValues)) {
4505 return poolingInferReturnTypes(inputShape, kernel: kernelValues, stride: strideValues,
4506 pad: padValues, inferredReturnShapes);
4507 }
4508
4509 llvm::SmallVector<int64_t> outputShape(4, ShapedType::kDynamic);
4510 if (inputShape.hasRank()) {
4511 outputShape[0] = inputShape.getDimSize(index: 0);
4512 outputShape[3] = inputShape.getDimSize(index: 3);
4513 }
4514 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
4515 return success();
4516}
4517
4518LogicalResult MaxPool2dOp::verify() {
4519 if (failed(Result: verifySameElementTypes(op: *this, /* intype = */ aType: getInput().getType(),
4520 /* outType = */ bType: getOutput().getType())))
4521 return failure();
4522
4523 if (failed(Result: verifyPoolingOp(op: *this)))
4524 return failure();
4525
4526 return success();
4527}
4528
4529LogicalResult MaxPool2dAdaptiveOp::verify() {
4530 if (failed(Result: verifySameElementTypes(op: *this, /* intype = */ aType: getInput().getType(),
4531 /* outType = */ bType: getOutput().getType())))
4532 return failure();
4533
4534 AdaptivePoolingConstShapeValues values;
4535 extractAdaptivePoolingConstShapeOperands(op: *this, values);
4536
4537 if (failed(Result: verifyPoolingOpImpl(op: getOperation(), kernel: values.kernel, strides: values.stride,
4538 padding: values.pad, input: getInput(), output: getOutput())))
4539 return failure();
4540
4541 return success();
4542}
4543
4544LogicalResult DepthwiseConv2DOp::inferReturnTypeComponents(
4545 MLIRContext *context, ::std::optional<Location> location,
4546 DepthwiseConv2DOp::Adaptor adaptor,
4547 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4548 llvm::SmallVector<int64_t> outputShape(4, ShapedType::kDynamic);
4549
4550 int64_t inputWidth = ShapedType::kDynamic;
4551 int64_t inputHeight = ShapedType::kDynamic;
4552 int64_t inputChannels = ShapedType::kDynamic;
4553
4554 int64_t weightWidth = ShapedType::kDynamic;
4555 int64_t weightHeight = ShapedType::kDynamic;
4556 int64_t depthChannels = ShapedType::kDynamic;
4557
4558 // Input shape describes input width/height and batch.
4559 ShapeAdaptor inputShape(adaptor.getInput().getType());
4560 if (inputShape.hasRank()) {
4561 outputShape[0] = inputShape.getDimSize(index: 0);
4562 inputHeight = inputShape.getDimSize(index: 1);
4563 inputWidth = inputShape.getDimSize(index: 2);
4564 inputChannels = inputShape.getDimSize(index: 3);
4565 }
4566
4567 // Weight shapes describes the filter width/height and the output channels.
4568 ShapeAdaptor weightShape(adaptor.getWeight().getType());
4569 if (weightShape.hasRank()) {
4570 weightHeight = weightShape.getDimSize(index: 0);
4571 weightWidth = weightShape.getDimSize(index: 1);
4572 inputChannels = ShapedType::isDynamic(dValue: inputChannels)
4573 ? weightShape.getDimSize(index: 2)
4574 : inputChannels;
4575 depthChannels = weightShape.getDimSize(index: 3);
4576 }
4577
4578 // If both inputChannels and depthChannels are available we can determine
4579 // the output channels.
4580 if (ShapedType::isStatic(dValue: inputChannels) &&
4581 ShapedType::isStatic(dValue: depthChannels)) {
4582 outputShape[3] = inputChannels * depthChannels;
4583 }
4584
4585 // Bias shape can describe the output channels.
4586 ShapeAdaptor biasShape(adaptor.getBias().getType());
4587 if (biasShape.hasRank() && ShapedType::isDynamic(dValue: outputShape[3])) {
4588 int64_t bc = biasShape.getDimSize(index: 0);
4589 if (bc != ShapedType::kDynamic && bc != 1)
4590 outputShape[3] = bc;
4591 }
4592
4593 llvm::ArrayRef<int64_t> dilation = adaptor.getDilation();
4594 llvm::ArrayRef<int64_t> padding = adaptor.getPad();
4595 llvm::ArrayRef<int64_t> stride = adaptor.getStride();
4596
4597 if (ShapedType::isStatic(dValue: inputHeight) && ShapedType::isStatic(dValue: weightHeight)) {
4598 int64_t inputSize = inputHeight + padding[0] + padding[1];
4599 int64_t filterSize = (weightHeight - 1) * dilation[0] + 1;
4600 int64_t unstridedResult = inputSize - filterSize + 1;
4601 outputShape[1] = (unstridedResult - 1) / stride[0] + 1;
4602 }
4603
4604 if (ShapedType::isStatic(dValue: inputWidth) && ShapedType::isStatic(dValue: weightWidth)) {
4605 int64_t inputSize = inputWidth + padding[2] + padding[3];
4606 int64_t filterSize = (weightWidth - 1) * dilation[1] + 1;
4607 int64_t unstridedResult = inputSize - filterSize + 1;
4608 outputShape[2] = (unstridedResult - 1) / stride[1] + 1;
4609 }
4610
4611 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
4612 return success();
4613}
4614
4615LogicalResult DepthwiseConv2DOp::verify() {
4616 if (verifyConvOp(op: *this).failed() || verifyConvOpErrorIf(op: *this).failed())
4617 return failure();
4618 return success();
4619}
4620
4621LogicalResult TransposeConv2DOp::inferReturnTypeComponents(
4622 MLIRContext *context, ::std::optional<Location> location,
4623 TransposeConv2DOp::Adaptor adaptor,
4624 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4625 llvm::SmallVector<int64_t> outputShape(4, ShapedType::kDynamic);
4626
4627 int64_t inputWidth = ShapedType::kDynamic;
4628 int64_t inputHeight = ShapedType::kDynamic;
4629 int64_t weightWidth = ShapedType::kDynamic;
4630 int64_t weightHeight = ShapedType::kDynamic;
4631
4632 // Input shape describes input width/height and batch.
4633 ShapeAdaptor inputShape(adaptor.getInput().getType());
4634 if (inputShape.hasRank()) {
4635 outputShape[0] = ShapedType::isDynamic(dValue: outputShape[0])
4636 ? inputShape.getDimSize(index: 0)
4637 : outputShape[0];
4638 inputHeight = inputShape.getDimSize(index: 1);
4639 inputWidth = inputShape.getDimSize(index: 2);
4640 }
4641
4642 // Weight shapes describes the filter width/height and the output channels.
4643 ShapeAdaptor weightShape(adaptor.getWeight().getType());
4644 if (weightShape.hasRank()) {
4645 outputShape[3] = ShapedType::isDynamic(dValue: outputShape[3])
4646 ? weightShape.getDimSize(index: 0)
4647 : outputShape[3];
4648 weightHeight = weightShape.getDimSize(index: 1);
4649 weightWidth = weightShape.getDimSize(index: 2);
4650 }
4651
4652 // Bias shape can describe the output channels.
4653 ShapeAdaptor biasShape(adaptor.getBias().getType());
4654 if (biasShape.hasRank() && ShapedType::isDynamic(dValue: outputShape[3])) {
4655 int64_t bc = biasShape.getDimSize(index: 0);
4656 if (bc != ShapedType::kDynamic && bc != 1)
4657 outputShape[3] = bc;
4658 }
4659
4660 llvm::ArrayRef<int64_t> padding = adaptor.getOutPad();
4661 llvm::ArrayRef<int64_t> stride = adaptor.getStride();
4662
4663 if (ShapedType::isStatic(dValue: inputHeight) && ShapedType::isStatic(dValue: weightHeight)) {
4664 int64_t calculateSize =
4665 (inputHeight - 1) * stride[0] + padding[0] + padding[1] + weightHeight;
4666 outputShape[1] =
4667 ShapedType::isDynamic(dValue: outputShape[1]) ? calculateSize : outputShape[1];
4668 }
4669
4670 if (ShapedType::isStatic(dValue: inputWidth) && ShapedType::isStatic(dValue: weightWidth)) {
4671 int64_t calculateSize =
4672 (inputWidth - 1) * stride[1] + padding[2] + padding[3] + weightWidth;
4673 outputShape[2] =
4674 ShapedType::isDynamic(dValue: outputShape[2]) ? calculateSize : outputShape[2];
4675 }
4676
4677 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputShape));
4678 return success();
4679}
4680
4681LogicalResult TransposeConv2DOp::verify() {
4682 if (verifyConvOp(op: *this).failed())
4683 return failure();
4684
4685 const llvm::ArrayRef<int64_t> strides = getStride();
4686 const int64_t strideY = strides[0];
4687 const int64_t strideX = strides[1];
4688
4689 if (strideY < 1 || strideX < 1)
4690 return emitOpError(message: "expect all stride values to be >= 1, got [")
4691 << strides << "]";
4692
4693 const auto checkPadAgainstKernelDim =
4694 [this](int64_t padValue, int64_t kernelDimSize, llvm::StringRef padName,
4695 llvm::StringRef kernelDimName) -> LogicalResult {
4696 if (padValue <= -kernelDimSize)
4697 return emitOpError(message: "expected ")
4698 << padName << " > -" << kernelDimName << ", but got: " << padName
4699 << "=" << padValue << " and " << kernelDimName << "="
4700 << kernelDimSize;
4701 return success();
4702 };
4703
4704 const llvm::ArrayRef<int64_t> padding = getOutPad();
4705 const int64_t outPadTop = padding[0];
4706 const int64_t outPadBottom = padding[1];
4707 const int64_t outPadLeft = padding[2];
4708 const int64_t outPadRight = padding[3];
4709
4710 const auto weightType =
4711 llvm::dyn_cast<RankedTensorType>(Val: getWeight().getType());
4712
4713 if (weightType) {
4714 const int64_t kernelHeight = weightType.getDimSize(idx: 1);
4715 if (ShapedType::isStatic(dValue: kernelHeight)) {
4716 if (failed(Result: checkPadAgainstKernelDim(outPadTop, kernelHeight,
4717 "out_pad_top", "KH")))
4718 return failure();
4719
4720 if (failed(Result: checkPadAgainstKernelDim(outPadBottom, kernelHeight,
4721 "out_pad_bottom", "KH")))
4722 return failure();
4723 }
4724
4725 const int64_t kernelWidth = weightType.getDimSize(idx: 2);
4726 if (ShapedType::isStatic(dValue: kernelWidth)) {
4727 if (failed(Result: checkPadAgainstKernelDim(outPadLeft, kernelWidth,
4728 "out_pad_left", "KW")))
4729 return failure();
4730
4731 if (failed(Result: checkPadAgainstKernelDim(outPadRight, kernelWidth,
4732 "out_pad_right", "KW")))
4733 return failure();
4734 }
4735 }
4736
4737 // Rest of the checks depend on the output type being a RankedTensorType
4738 const auto outputType =
4739 llvm::dyn_cast<RankedTensorType>(Val: getOutput().getType());
4740 if (!outputType)
4741 return success();
4742
4743 const auto inputType = llvm::dyn_cast<RankedTensorType>(Val: getInput().getType());
4744 if (inputType && weightType) {
4745 const int64_t inputHeight = inputType.getDimSize(idx: 1);
4746 const int64_t kernelHeight = weightType.getDimSize(idx: 1);
4747 const int64_t outputHeight = outputType.getDimSize(idx: 1);
4748
4749 if (ShapedType::isStatic(dValue: inputHeight) &&
4750 ShapedType::isStatic(dValue: outputHeight)) {
4751 if (outputHeight !=
4752 (inputHeight - 1) * strideY + outPadTop + outPadBottom + kernelHeight)
4753 return emitOpError(
4754 message: "dimension mismatch: expected OH == (IH - 1) * stride_y "
4755 "+ out_pad_top + out_pad_bottom + KH, but got ")
4756 << outputHeight << " != (" << inputHeight << " - 1) * "
4757 << strideY << " + " << outPadTop << " + " << outPadBottom
4758 << " + " << kernelHeight;
4759 }
4760
4761 const int64_t inputWidth = inputType.getDimSize(idx: 2);
4762 const int64_t kernelWidth = weightType.getDimSize(idx: 2);
4763 const int64_t outputWidth = outputType.getDimSize(idx: 2);
4764
4765 if (ShapedType::isStatic(dValue: inputWidth) && ShapedType::isStatic(dValue: outputWidth)) {
4766 if (outputWidth !=
4767 (inputWidth - 1) * strideX + outPadLeft + outPadRight + kernelWidth)
4768 return emitOpError(
4769 message: "dimension mismatch: expected OW == (IW - 1) * stride_x "
4770 "+ out_pad_left + out_pad_right + KW, but got ")
4771 << outputWidth << " != (" << inputWidth << " - 1) * " << strideX
4772 << " + " << outPadLeft << " + " << outPadRight << " + "
4773 << kernelWidth;
4774 }
4775 }
4776
4777 const auto biasType = llvm::dyn_cast<RankedTensorType>(Val: getBias().getType());
4778
4779 if (!biasType)
4780 return success();
4781
4782 const int64_t biasChannels = biasType.getDimSize(idx: 0);
4783
4784 // Skip further checks if bias is dynamic
4785 if (biasChannels == ShapedType::kDynamic)
4786 return success();
4787
4788 const int64_t outputChannels = outputType.getDimSize(idx: 3);
4789 if (!ShapedType::isDynamic(dValue: outputChannels) &&
4790 biasChannels != outputChannels && biasChannels != 1)
4791 return emitOpError(
4792 message: "bias channels expected to be equal to output channels (")
4793 << outputChannels << ") or 1, got " << biasChannels;
4794
4795 return success();
4796}
4797
4798LogicalResult RescaleOp::verify() {
4799 const auto inputType = llvm::cast<ShapedType>(Val: getInput().getType());
4800 auto inputElementType =
4801 getStorageElementTypeOrSelf(type: inputType.getElementType());
4802 if (!mlir::isa<IntegerType>(Val: inputElementType)) {
4803 emitOpError(message: "expect input to have integer element type, got ")
4804 << inputElementType;
4805 return failure();
4806 }
4807
4808 const auto outputType = llvm::cast<ShapedType>(Val: getOutput().getType());
4809 auto outputElementType =
4810 getStorageElementTypeOrSelf(type: outputType.getElementType());
4811 if (!mlir::isa<IntegerType>(Val: outputElementType)) {
4812 emitOpError(message: "expect output to have integer element type, got ")
4813 << outputElementType;
4814 return failure();
4815 }
4816
4817 if (verifyRescaleValueAndZpTypes(op: *this, val: getInput(), valZp: getInputZp(), name: "input")
4818 .failed())
4819 return failure();
4820
4821 if (verifyRescaleValueAndZpTypes(op: *this, val: getOutput(), valZp: getOutputZp(), name: "output")
4822 .failed())
4823 return failure();
4824
4825 const auto multiplierType = llvm::cast<ShapedType>(Val: getMultiplier().getType());
4826 // multiplier element type must be i32 for scale32 = true
4827 if (getScale32() && !multiplierType.getElementType().isInteger(width: 32)) {
4828 emitOpError(message: "expect i32 element type for multiplier for scale32=true, got ")
4829 << multiplierType.getElementType();
4830 return failure();
4831 }
4832
4833 // multiplier element type must be i16 for scale32 = false
4834 if (!getScale32() && !multiplierType.getElementType().isInteger(width: 16)) {
4835 emitOpError(
4836 message: "expect i16 element type for multiplier for scale32=false, got ")
4837 << multiplierType.getElementType();
4838 return failure();
4839 }
4840
4841 if (!inputType.hasRank())
4842 return success();
4843
4844 // multiplier/shift must have shape = {numChannels},
4845 // where numChannel is 1 if per_channel = false
4846 // otherwise numChannel is dimension in input shape's last axis
4847 int64_t numChannels = 1;
4848 if (getPerChannel()) {
4849 if (inputType.getRank() < 1) {
4850 emitOpError(message: "requires input to be at least rank 1 when per_channel is "
4851 "true, but got rank ")
4852 << inputType.getRank();
4853 return failure();
4854 }
4855 numChannels = inputType.getDimSize(idx: inputType.getRank() - 1);
4856 }
4857
4858 if (outputType.hasRank()) {
4859 if (failed(Result: verifyOutputShapeCompatibleWithExpected(
4860 op: getOperation(), outputType, expectedShape: inputType.getShape())))
4861 return failure();
4862 }
4863
4864 if (multiplierType.hasRank()) {
4865 ArrayRef<int64_t> multiplierShape = multiplierType.getShape();
4866 // multiplier input has rank 1 by dialect definition
4867 if (multiplierShape[0] != ShapedType::kDynamic &&
4868 multiplierShape[0] != numChannels) {
4869 emitOpError(message: "expect shape of { ")
4870 << numChannels << " } for multiplier input, got { "
4871 << multiplierShape[0] << " }";
4872 return failure();
4873 }
4874 }
4875
4876 const auto shiftType = llvm::cast<ShapedType>(Val: getShift().getType());
4877 if (shiftType.hasRank()) {
4878 ArrayRef<int64_t> shiftShape = shiftType.getShape();
4879 // shift input has rank 1 by dialect definition
4880 if (shiftShape[0] != ShapedType::kDynamic && shiftShape[0] != numChannels) {
4881 emitOpError(message: "expect shape of { ")
4882 << numChannels << " } for shift input, got { " << shiftShape[0]
4883 << " }";
4884 return failure();
4885 }
4886 }
4887
4888 return success();
4889}
4890
4891LogicalResult RescaleOp::inferReturnTypeComponents(
4892 MLIRContext *context, ::std::optional<Location> location,
4893 RescaleOp::Adaptor adaptor,
4894 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4895 ShapeAdaptor inputShape(adaptor.getInput().getType());
4896 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(inputShape));
4897 return success();
4898}
4899
4900LogicalResult CastOp::verify() {
4901 const ShapedType inputType = llvm::cast<ShapedType>(Val: getInput().getType());
4902 const ShapedType outputType = llvm::cast<ShapedType>(Val: getType());
4903 const Type inputElementType = inputType.getElementType();
4904 const Type outputElementType = outputType.getElementType();
4905
4906 const bool inputIsBlockScaled = llvm::isa<BlockScaledType>(Val: inputElementType);
4907 const bool outputIsBlockScaled =
4908 llvm::isa<BlockScaledType>(Val: outputElementType);
4909
4910 const bool isUnsigned = this->getInputUnsigned();
4911 const Type inputDataType = getStorageElementTypeOrSelf(type: inputType);
4912
4913 if (isUnsigned)
4914 if (!inputDataType.isInteger() || inputDataType.isInteger(width: 1))
4915 return emitOpError()
4916 << "attribute input_unsigned requires integer type inputs. Got: "
4917 << inputDataType;
4918
4919 if (!inputIsBlockScaled && !outputIsBlockScaled)
4920 return success();
4921
4922 if (inputIsBlockScaled && outputIsBlockScaled)
4923 return emitOpError()
4924 << "requires exactly one of input or output to have block scaled "
4925 "element type";
4926
4927 const Type scalarElementType =
4928 inputIsBlockScaled ? outputElementType : inputElementType;
4929 if (!llvm::isa<FloatType>(Val: scalarElementType))
4930 return emitOpError()
4931 << "requires non-block-scaled element type to be floating-point "
4932 "when casting to or from block scaled element type, got "
4933 << scalarElementType;
4934
4935 return success();
4936}
4937
4938LogicalResult CastFromBlockScaledOp::inferReturnTypeComponents(
4939 MLIRContext *context, ::std::optional<Location> location,
4940 CastFromBlockScaledOp::Adaptor adaptor,
4941 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
4942 const ShapeAdaptor inputShape(adaptor.getInputData().getType());
4943 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(inputShape));
4944 return success();
4945}
4946
4947LogicalResult CastFromBlockScaledOp::verify() {
4948 const Type inputDataType = getInputData().getType();
4949 const Type outputDataType = getResult().getType();
4950 if (failed(Result: verifyCompatibleShape(type1: inputDataType, type2: outputDataType)))
4951 return emitOpError() << "require compatible shapes for input_data ("
4952 << inputDataType << ") and " << "output_data ("
4953 << outputDataType << ")";
4954
4955 const ShapeAdaptor inputDataShape = ShapeAdaptor(inputDataType);
4956
4957 if (inputDataShape.hasRank()) {
4958 const unsigned int blockSize =
4959 BlockSizeAttr::getBlockSizeValue(blockSize: getBlockSize());
4960 if (blockSize != BlockSizeAttr::getBlockSizeValue(blockSize: BlockSize::BLOCK_SIZE_32))
4961 return emitOpError(message: "expect block size to be 32, got ") << blockSize;
4962 const int64_t inputDataLastDim =
4963 inputDataShape.getDimSize(index: inputDataShape.getRank() - 1);
4964 if (inputDataLastDim % blockSize != 0)
4965 return emitOpError() << "expect last dimension of input_data ("
4966 << inputDataLastDim
4967 << ") to be divisible by block_size (" << blockSize
4968 << ")";
4969
4970 const Type inputScaleType = getInputScale().getType();
4971 const ShapeAdaptor inputScaleShape = ShapeAdaptor(inputScaleType);
4972
4973 if (inputScaleShape.hasRank()) {
4974 SmallVector<int64_t> inputDataDims, inputScaleDims;
4975 inputDataShape.getDims(res&: inputDataDims);
4976 inputScaleShape.getDims(res&: inputScaleDims);
4977
4978 if (inputDataDims.size() != inputScaleDims.size() ||
4979 failed(Result: verifyCompatibleShape(
4980 shape1: ArrayRef<int64_t>(inputDataDims).drop_back(N: 1),
4981 shape2: ArrayRef<int64_t>(inputScaleDims).drop_back(N: 1))))
4982 return emitOpError()
4983 << "require compatible shapes for input_data (" << inputDataType
4984 << ") and " << "input_scale (" << inputScaleType
4985 << ") except for the last dimension";
4986
4987 const SmallVector<int64_t, 2> dimsToCheck{inputDataLastDim / blockSize,
4988 inputScaleDims.back()};
4989 if (ShapedType::isStatic(dValue: inputDataLastDim) &&
4990 failed(Result: verifyCompatibleDims(dims: dimsToCheck)))
4991 return emitOpError()
4992 << "expect last dimension of input_scale ("
4993 << inputScaleDims.back()
4994 << ") to be equal to last dimension of input_data / block_size ("
4995 << inputDataDims.back() / blockSize << ")";
4996 }
4997 }
4998
4999 return success();
5000}
5001
5002LogicalResult CastToBlockScaledOp::inferReturnTypeComponents(
5003 MLIRContext *context, ::std::optional<Location> location,
5004 CastToBlockScaledOp::Adaptor adaptor,
5005 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
5006 const ShapeAdaptor inputShape(adaptor.getInputData().getType());
5007 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(inputShape));
5008 if (!inputShape.hasRank())
5009 return success();
5010
5011 // Calculate output_scale shape if ranked input provided
5012 SmallVector<int64_t> outputScaleShape;
5013 inputShape.getDims(res&: outputScaleShape);
5014 const int64_t lastDimLoc = inputShape.getRank() - 1;
5015 const int64_t lastDimSize = inputShape.getDimSize(index: lastDimLoc);
5016 if (ShapedType::isStatic(dValue: lastDimSize)) {
5017 const unsigned int blockSize =
5018 BlockSizeAttr::getBlockSizeValue(blockSize: adaptor.getBlockSize());
5019 outputScaleShape[lastDimLoc] = lastDimSize / blockSize;
5020 }
5021 inferredReturnShapes.push_back(Elt: ShapedTypeComponents(outputScaleShape));
5022 return success();
5023}
5024
5025LogicalResult CastToBlockScaledOp::verify() {
5026 const Type inputDataType = getInputData().getType();
5027 const Type outputDataType = getResult(i: 0).getType();
5028 if (failed(Result: verifyCompatibleShape(type1: inputDataType, type2: outputDataType)))
5029 return emitOpError() << "require compatible shapes for input_data ("
5030 << inputDataType << ") and " << "output_data ("
5031 << outputDataType << ")";
5032
5033 const unsigned int blockSize =
5034 BlockSizeAttr::getBlockSizeValue(blockSize: getBlockSize());
5035 if (blockSize != BlockSizeAttr::getBlockSizeValue(blockSize: BlockSize::BLOCK_SIZE_32))
5036 return emitOpError(message: "expect block size to be 32, got ") << blockSize;
5037 const ShapeAdaptor inputDataShape = ShapeAdaptor(inputDataType);
5038 if (inputDataShape.hasRank()) {
5039 const int64_t inputDataLastDim =
5040 inputDataShape.getDimSize(index: inputDataShape.getRank() - 1);
5041 if (ShapedType::isStatic(dValue: inputDataLastDim) &&
5042 inputDataLastDim % blockSize != 0)
5043 return emitOpError() << "expect last dimension of input_data ("
5044 << inputDataLastDim
5045 << ") to be divisible by block_size (" << blockSize
5046 << ")";
5047 }
5048
5049 const ShapeAdaptor outputDataShape = ShapeAdaptor(outputDataType);
5050 const Type outputScaleType = getResult(i: 1).getType();
5051 const ShapeAdaptor outputScaleShape = ShapeAdaptor(outputScaleType);
5052 if (outputDataShape.hasRank() && outputScaleShape.hasRank()) {
5053 SmallVector<int64_t> outputDataDims, outputScaleDims;
5054 outputDataShape.getDims(res&: outputDataDims);
5055 outputScaleShape.getDims(res&: outputScaleDims);
5056
5057 if (outputDataDims.size() != outputScaleDims.size() ||
5058 failed(Result: verifyCompatibleShape(
5059 shape1: ArrayRef<int64_t>(outputDataDims).drop_back(N: 1),
5060 shape2: ArrayRef<int64_t>(outputScaleDims).drop_back(N: 1))))
5061 return emitOpError() << "require compatible shapes for output_data ("
5062 << outputDataType << ") and " << "output_scale ("
5063 << outputScaleType
5064 << ") except for the last dimension";
5065
5066 const int64_t outputDataLastDim = outputDataDims.back();
5067 const SmallVector<int64_t, 2> dimsToCheck{outputDataLastDim / blockSize,
5068 outputScaleDims.back()};
5069 if (ShapedType::isStatic(dValue: outputDataLastDim) &&
5070 failed(Result: verifyCompatibleDims(dims: dimsToCheck)))
5071 return emitOpError()
5072 << "expect last dimension of output_scale ("
5073 << outputScaleDims.back()
5074 << ") to be equal to last dimension of output_data / block_size ("
5075 << outputDataDims.back() / blockSize << ")";
5076 }
5077
5078 return success();
5079}
5080
5081LogicalResult IfOp::inferReturnTypeComponents(
5082 MLIRContext *context, ::std::optional<Location> location,
5083 IfOp::Adaptor adaptor,
5084 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
5085 llvm::SmallVector<tosa::YieldOp> yieldOps;
5086 for (Region *region : adaptor.getRegions()) {
5087 for (auto &block : *region)
5088 if (auto returnOp = dyn_cast<tosa::YieldOp>(Val: block.getTerminator()))
5089 yieldOps.push_back(Elt: returnOp);
5090 }
5091
5092 if (yieldOps.empty())
5093 return failure();
5094
5095 // Get the initial type information for the yield op.
5096 llvm::SmallVector<ValueKnowledge> resultKnowledge;
5097 resultKnowledge.reserve(N: yieldOps.front().getNumOperands());
5098 for (auto operand : yieldOps.front().getOperands()) {
5099 resultKnowledge.push_back(
5100 Elt: ValueKnowledge::getKnowledgeFromType(type: operand.getType()));
5101 }
5102
5103 for (auto yieldOp : yieldOps) {
5104 if (resultKnowledge.size() != yieldOp.getNumOperands())
5105 return failure();
5106
5107 for (const auto &it : llvm::enumerate(First: yieldOp.getOperands())) {
5108 int32_t index = it.index();
5109 auto meet = ValueKnowledge::meet(
5110 lhs: resultKnowledge[index],
5111 rhs: ValueKnowledge::getKnowledgeFromType(type: it.value().getType()));
5112 if (!meet)
5113 continue;
5114 resultKnowledge[index] = meet;
5115 }
5116 }
5117
5118 for (const ValueKnowledge &result : resultKnowledge) {
5119 inferredReturnShapes.push_back(Elt: result.getShapedTypeComponents());
5120 }
5121
5122 return success();
5123}
5124
5125LogicalResult WhileOp::inferReturnTypeComponents(
5126 MLIRContext *context, ::std::optional<Location> location,
5127 WhileOp::Adaptor adaptor,
5128 SmallVectorImpl<ShapedTypeComponents> &inferredReturnShapes) {
5129 llvm::SmallVector<tosa::YieldOp> yieldOps;
5130 for (auto &block : adaptor.getBodyGraph())
5131 if (auto returnOp = dyn_cast<tosa::YieldOp>(Val: block.getTerminator()))
5132 yieldOps.push_back(Elt: returnOp);
5133
5134 // TOSA's while must have a tosa.yield as its terminator. If not found this
5135 // tosa.while is invalid.
5136 if (yieldOps.empty())
5137 return failure();
5138
5139 // Get the initial type information from the operand types.
5140 llvm::SmallVector<ValueKnowledge> resultKnowledge;
5141 resultKnowledge.reserve(N: yieldOps.front().getNumOperands());
5142 for (auto operand : yieldOps.front().getOperands()) {
5143 resultKnowledge.push_back(
5144 Elt: ValueKnowledge::getKnowledgeFromType(type: operand.getType()));
5145 }
5146
5147 for (auto yieldOp : yieldOps) {
5148 if (resultKnowledge.size() != yieldOp.getNumOperands())
5149 return failure();
5150
5151 for (const auto &it : llvm::enumerate(First: yieldOp.getOperands())) {
5152 int32_t index = it.index();
5153 if (auto meet = ValueKnowledge::meet(
5154 lhs: resultKnowledge[index],
5155 rhs: ValueKnowledge::getKnowledgeFromType(type: it.value().getType()))) {
5156 resultKnowledge[index] = meet;
5157 }
5158 }
5159 }
5160
5161 for (const ValueKnowledge &result : resultKnowledge) {
5162 inferredReturnShapes.push_back(Elt: result.getShapedTypeComponents());
5163 }
5164
5165 return success();
5166}
5167
5168static void printInitializationList(OpAsmPrinter &parser,
5169 Block::BlockArgListType blocksArgs,
5170 ValueRange initializers,
5171 StringRef prefix = "") {
5172 assert(blocksArgs.size() == initializers.size() &&
5173 "expected same length of arguments and initializers");
5174 if (initializers.empty())
5175 return;
5176
5177 parser << prefix << '(';
5178 llvm::interleaveComma(
5179 c: llvm::zip(t&: blocksArgs, u&: initializers), os&: parser,
5180 each_fn: [&](auto it) { parser << std::get<0>(it) << " = " << std::get<1>(it); });
5181 parser << ")";
5182}
5183
5184// parse and print of IfOp refer to the implementation of SCF dialect.
5185ParseResult IfOp::parse(OpAsmParser &parser, OperationState &result) {
5186 // Create the regions for 'then'.
5187 result.regions.reserve(N: 2);
5188 Region *thenRegion = result.addRegion();
5189 Region *elseRegion = result.addRegion();
5190
5191 OpAsmParser::UnresolvedOperand cond;
5192
5193 if (parser.parseOperand(result&: cond))
5194 return failure();
5195
5196 SmallVector<OpAsmParser::Argument, 4> regionArgs;
5197 SmallVector<OpAsmParser::UnresolvedOperand, 4> operands;
5198
5199 // Parse the optional block arguments
5200 OptionalParseResult listResult =
5201 parser.parseOptionalAssignmentList(lhs&: regionArgs, rhs&: operands);
5202 if (listResult.has_value() && failed(Result: listResult.value()))
5203 return failure();
5204
5205 // Parse a colon.
5206 if (failed(Result: parser.parseColon()))
5207 return parser.emitError(loc: parser.getCurrentLocation(),
5208 message: "expected type for condition operand");
5209
5210 // Parse the type of the condition operand
5211 Type condType;
5212 if (failed(Result: parser.parseType(result&: condType)))
5213 return parser.emitError(loc: parser.getCurrentLocation(),
5214 message: "expected type for condition operand");
5215
5216 // Resolve operand with provided type
5217 if (failed(Result: parser.resolveOperand(operand: cond, type: condType, result&: result.operands)))
5218 return failure();
5219
5220 // Parse optional block arg types
5221 if (listResult.has_value()) {
5222 FunctionType functionType;
5223
5224 if (failed(Result: parser.parseType(result&: functionType)))
5225 return parser.emitError(loc: parser.getCurrentLocation())
5226 << "expected list of types for block arguments "
5227 << "followed by arrow type and list of return types";
5228
5229 result.addTypes(newTypes: functionType.getResults());
5230
5231 if (functionType.getNumInputs() != operands.size()) {
5232 return parser.emitError(loc: parser.getCurrentLocation())
5233 << "expected as many input types as operands " << "(expected "
5234 << operands.size() << " got " << functionType.getNumInputs()
5235 << ")";
5236 }
5237
5238 // Resolve input operands.
5239 if (failed(Result: parser.resolveOperands(operands, types: functionType.getInputs(),
5240 loc: parser.getCurrentLocation(),
5241 result&: result.operands)))
5242 return failure();
5243 } else {
5244 // Parse optional results type list.
5245 if (parser.parseOptionalArrowTypeList(result&: result.types))
5246 return failure();
5247 }
5248
5249 // Parse the 'then' region.
5250 if (parser.parseRegion(region&: *thenRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {}))
5251 return failure();
5252
5253 // If we find an 'else' keyword then parse the 'else' region.
5254 if (!parser.parseOptionalKeyword(keyword: "else")) {
5255 if (parser.parseRegion(region&: *elseRegion, /*arguments=*/{}, /*argTypes=*/enableNameShadowing: {}))
5256 return failure();
5257 }
5258
5259 // Parse the optional attribute list.
5260 if (parser.parseOptionalAttrDict(result&: result.attributes))
5261 return failure();
5262 return success();
5263}
5264
5265void IfOp::print(OpAsmPrinter &p) {
5266 p << " " << getCondition();
5267
5268 printInitializationList(parser&: p, blocksArgs: getThenGraph().front().getArguments(),
5269 initializers: getInputList(), prefix: " ");
5270 p << " : ";
5271 p << getCondition().getType();
5272
5273 if (!getInputList().empty()) {
5274 p << " (";
5275 llvm::interleaveComma(c: getInputList().getTypes(), os&: p);
5276 p << ")";
5277 }
5278 p.printArrowTypeList(types: getResultTypes());
5279 p << " ";
5280
5281 p.printRegion(blocks&: getThenGraph());
5282
5283 // Print the 'else' regions if it exists and has a block.
5284 auto &elseRegion = getElseGraph();
5285 if (!elseRegion.empty()) {
5286 p << " else ";
5287 p.printRegion(blocks&: elseRegion);
5288 }
5289
5290 p.printOptionalAttrDict(attrs: (*this)->getDiscardableAttrDictionary().getValue());
5291}
5292
5293LogicalResult IfOp::verify() {
5294 if (errorIfTypeOrShapeMismatch(op: *this, list1: getThenGraph().front().getArguments(),
5295 name1: "'then_graph' arguments", list2: getInputList(),
5296 name2: "'input_list'")
5297 .failed())
5298 return failure();
5299
5300 if (errorIfTypeOrShapeMismatch(op: *this, list1: getElseGraph().front().getArguments(),
5301 name1: "'else_graph' arguments", list2: getInputList(),
5302 name2: "'input_list'")
5303 .failed())
5304 return failure();
5305
5306 // MLIR will verify the absence of the terminator for us if otherwise.
5307 if (getThenGraph().front().mightHaveTerminator()) {
5308 auto thenYield =
5309 dyn_cast<tosa::YieldOp>(Val: getThenGraph().front().getTerminator());
5310 if (thenYield && errorIfTypeOrShapeMismatch(
5311 op: *this, list1: thenYield.getInputs(), name1: "'then_graph' results",
5312 list2: getOutputList(), name2: "'output_list'")
5313 .failed())
5314 return failure();
5315 }
5316
5317 // MLIR will verify the absence of the terminator for us if otherwise.
5318 if (getElseGraph().front().mightHaveTerminator()) {
5319 auto elseYield =
5320 dyn_cast<tosa::YieldOp>(Val: getElseGraph().front().getTerminator());
5321 if (elseYield && errorIfTypeOrShapeMismatch(
5322 op: *this, list1: elseYield.getInputs(), name1: "'else_graph' results",
5323 list2: getOutputList(), name2: "'output_list'")
5324 .failed())
5325 return failure();
5326 }
5327
5328 auto condType = getCondition().getType();
5329 if (errorIfShapeNotSizeOne(op: *this, type: condType).failed())
5330 return emitOpError() << "'condition' must be a size 1 tensor, got "
5331 << condType;
5332
5333 return success();
5334}
5335
5336LogicalResult WhileOp::verify() {
5337 if (errorIfTypeOrShapeMismatch(op: *this, list1: getInputList(), name1: "'input_list'",
5338 list2: getOutputList(), name2: "'output_list'")
5339 .failed())
5340 return failure();
5341
5342 if (errorIfTypeOrShapeMismatch(op: *this, list1: getCondGraph().front().getArguments(),
5343 name1: "'cond_graph' arguments", list2: getInputList(),
5344 name2: "'input_list'")
5345 .failed())
5346 return failure();
5347
5348 if (errorIfTypeOrShapeMismatch(op: *this, list1: getBodyGraph().front().getArguments(),
5349 name1: "'body_graph' arguments", list2: getInputList(),
5350 name2: "'input_list'")
5351 .failed())
5352 return failure();
5353
5354 if (getBodyGraph().front().mightHaveTerminator()) {
5355 auto bodyYield =
5356 dyn_cast<tosa::YieldOp>(Val: getBodyGraph().front().getTerminator());
5357 if (bodyYield && errorIfTypeOrShapeMismatch(op: *this, list1: bodyYield.getInputs(),
5358 name1: "'body_graph' results",
5359 list2: getInputList(), name2: "'input_list'")
5360 .failed())
5361 return failure();
5362 }
5363
5364 // Condition block output must be a single element tensor with a single bool
5365 // value.
5366 if (!getCondGraph().front().mightHaveTerminator())
5367 return success();
5368
5369 auto condYield =
5370 dyn_cast<tosa::YieldOp>(Val: getCondGraph().front().getTerminator());
5371 if (!condYield)
5372 return success();
5373
5374 if (condYield.getInputs().size() != 1)
5375 return emitOpError() << "require 'cond_graph' only have one result";
5376
5377 auto condOutType = condYield.getInputs()[0].getType();
5378 if (errorIfShapeNotSizeOne(op: *this, type: condOutType).failed())
5379 return emitOpError() << "'cond_graph' result must be a size 1 tensor, got "
5380 << condOutType;
5381
5382 if (!getElementTypeOrSelf(type: condOutType).isInteger(width: 1))
5383 return emitOpError() << "'cond_graph' result must be a boolean tensor, got "
5384 << condOutType;
5385
5386 return success();
5387}
5388
5389LogicalResult ReverseOp::verify() {
5390 TensorType inputType = getInput1().getType();
5391 int32_t reverseAxis = getAxis();
5392
5393 if (reverseAxis < 0)
5394 return emitOpError(message: "expected non-negative reverse axis");
5395 if (inputType.hasRank()) {
5396 int64_t inputRank = inputType.getRank();
5397 // We allow for a special case where the input/output shape has rank 0 and
5398 // axis is also 0.
5399 if (reverseAxis >= inputRank && (reverseAxis != 0 || inputRank != 0))
5400 return emitOpError(message: "expect input tensor rank (")
5401 << inputRank << ") to be larger than reverse axis (" << reverseAxis
5402 << ")";
5403 }
5404
5405 return success();
5406}
5407
5408LogicalResult tosa::SelectOp::verify() {
5409 // verify input2 and input3 have same element type as output
5410 if (verifySameElementTypes(op: *this, /* inType = */ aType: getOnTrue().getType(),
5411 /* outType = */ bType: getOutput().getType())
5412 .failed() ||
5413 verifySameElementTypes(op: *this, /* inType = */ aType: getOnFalse().getType(),
5414 /* outType = */ bType: getOutput().getType())
5415 .failed()) {
5416 return failure();
5417 }
5418 // verify input1 has element type of bool
5419 auto predicateType = llvm::dyn_cast<ShapedType>(Val: getPred().getType());
5420 if (!predicateType) {
5421 return emitOpError(message: "expect shaped tensor for input1, got ")
5422 << getInput1().getType();
5423 }
5424 auto predicateElementType = predicateType.getElementType();
5425 if (!predicateElementType.isInteger(width: 1)) {
5426 return emitOpError(message: "expect element type of bool for input1, got ")
5427 << predicateElementType;
5428 }
5429
5430 return success();
5431}
5432
5433LogicalResult tosa::VariableReadOp::verify() {
5434 if (verifyVariableOpErrorIf(op: *this, type: getOutput1().getType(), name: "'output1'")
5435 .failed())
5436 return failure();
5437
5438 return success();
5439}
5440
5441LogicalResult tosa::VariableWriteOp::verify() {
5442 if (verifyVariableOpErrorIf(op: *this, type: getInput1().getType(), name: "'input1'")
5443 .failed())
5444 return failure();
5445
5446 return success();
5447}
5448
5449// parse and print of WhileOp refer to the implementation of SCF dialect.
5450ParseResult WhileOp::parse(OpAsmParser &parser, OperationState &result) {
5451 SmallVector<OpAsmParser::Argument, 4> regionArgs;
5452 SmallVector<OpAsmParser::UnresolvedOperand, 4> operands;
5453 Region *cond = result.addRegion();
5454 Region *body = result.addRegion();
5455
5456 OptionalParseResult listResult =
5457 parser.parseOptionalAssignmentList(lhs&: regionArgs, rhs&: operands);
5458 if (listResult.has_value() && failed(Result: listResult.value()))
5459 return failure();
5460
5461 FunctionType functionType;
5462 SMLoc typeLoc = parser.getCurrentLocation();
5463 if (failed(Result: parser.parseColonType(result&: functionType)))
5464 return failure();
5465
5466 result.addTypes(newTypes: functionType.getResults());
5467
5468 if (functionType.getNumInputs() != operands.size()) {
5469 return parser.emitError(loc: typeLoc)
5470 << "expected as many input types as operands " << "(expected "
5471 << operands.size() << " got " << functionType.getNumInputs() << ")";
5472 }
5473
5474 // Resolve input operands.
5475 if (failed(Result: parser.resolveOperands(operands, types: functionType.getInputs(),
5476 loc: parser.getCurrentLocation(),
5477 result&: result.operands)))
5478 return failure();
5479
5480 // Propagate the types into the region arguments.
5481 for (size_t i = 0, e = regionArgs.size(); i != e; ++i)
5482 regionArgs[i].type = functionType.getInput(i);
5483
5484 return failure(IsFailure: parser.parseRegion(region&: *cond, arguments: regionArgs) ||
5485 parser.parseKeyword(keyword: "do") || parser.parseRegion(region&: *body) ||
5486 parser.parseOptionalAttrDictWithKeyword(result&: result.attributes));
5487}
5488
5489void WhileOp::print(OpAsmPrinter &parser) {
5490 printInitializationList(parser, blocksArgs: getCondGraph().front().getArguments(),
5491 initializers: getInputList(), prefix: " ");
5492 parser << " : ";
5493 parser.printFunctionalType(inputs: getInputList().getTypes(),
5494 results: getResults().getTypes());
5495 parser << ' ';
5496 parser.printRegion(blocks&: getCondGraph(), /*printEntryBlockArgs=*/false);
5497 parser << " do ";
5498 parser.printRegion(blocks&: getBodyGraph());
5499 parser.printOptionalAttrDictWithKeyword(
5500 attrs: (*this)->getDiscardableAttrDictionary().getValue());
5501}
5502
5503// Create a rank-1 const tensor for zero point of the source tensor.
5504std::optional<Value> mlir::tosa::createZeroPointTensor(OpBuilder &builder,
5505 Location loc,
5506 Type srcElemType,
5507 int64_t zp) {
5508 srcElemType = getStorageElementTypeOrSelf(type: srcElemType);
5509 auto zpType = mlir::RankedTensorType::get(shape: {1}, elementType: srcElemType);
5510 if (llvm::isa<FloatType>(Val: srcElemType)) {
5511 auto zpAttr = DenseElementsAttr::get(
5512 type: zpType, values: builder.getFloatAttr(type: srcElemType, value: static_cast<double>(zp)));
5513 return tosa::ConstOp::create(builder, location: loc, output: zpType, values: zpAttr);
5514 }
5515 if (llvm::isa<IntegerType>(Val: srcElemType)) {
5516 auto zpAttr =
5517 DenseElementsAttr::get(type: zpType, values: builder.getIntegerAttr(type: srcElemType, value: zp));
5518 return tosa::ConstOp::create(builder, location: loc, output: zpType, values: zpAttr);
5519 }
5520 llvm::errs() << "zero point is not allowed for unsupported data types\n";
5521 return std::nullopt;
5522}
5523
5524//===----------------------------------------------------------------------===//
5525// TOSA Shape and Shape Operators Helper functions.
5526//===----------------------------------------------------------------------===//
5527
5528bool mlir::tosa::isa_tosa_shape_type(mlir::Type t) {
5529 return mlir::isa<tosa::shapeType>(Val: t);
5530}
5531
5532LogicalResult
5533mlir::tosa::shapeType::verify(function_ref<InFlightDiagnostic()> emitError,
5534 int rank) {
5535 if (rank < 0)
5536 return emitError() << "invalid rank (must be >= 0): " << rank;
5537 return success();
5538}
5539
5540LogicalResult OpTrait::tosa::verifyTosaResolvableShapeOperands(Operation *op) {
5541 for (auto v : op->getOperands()) {
5542 if (mlir::isa<::mlir::tosa::shapeType>(Val: v.getType())) {
5543 Operation *definingOp = v.getDefiningOp();
5544 if (!definingOp || !definingOp->hasTrait<TosaShapeOperator>()) {
5545 return op->emitOpError(message: "shape operand is not compile time resolvable");
5546 }
5547 }
5548 }
5549 return success();
5550}
5551
5552LogicalResult
5553OpTrait::tosa::verifyTosaShapeOperatorWithSameRanks(Operation *op) {
5554 if (failed(Result: OpTrait::impl::verifyAtLeastNOperands(op, numOperands: 1)))
5555 return failure();
5556
5557 // delegate function that returns rank of shape type
5558 auto getRank = [](const Type type) {
5559 return mlir::cast<mlir::tosa::shapeType>(Val: type).getRank();
5560 };
5561 auto operandTypes = op->getOperandTypes();
5562 auto resultTypes = op->getResultTypes();
5563
5564 auto rank = getRank(*op->getOperandTypes().begin());
5565 for (auto type : operandTypes) {
5566 if (getRank(type) != rank) {
5567 return op->emitOpError(message: "operands don't have matching ranks");
5568 }
5569 }
5570 for (auto type : resultTypes) {
5571 if (getRank(type) != rank) {
5572 return op->emitOpError(message: "result shape has different rank than operands");
5573 }
5574 }
5575 return success();
5576}
5577
5578//===----------------------------------------------------------------------===//
5579// TOSA Shape Operators verify functions.
5580//===----------------------------------------------------------------------===//
5581
5582LogicalResult tosa::ConstShapeOp::verify() {
5583 // check one dimensional rank
5584 auto valuesRank = getValues().getType().getRank();
5585 if (valuesRank != 1)
5586 return emitOpError(message: "expect elements in attribute values with rank 1");
5587 // check that number of elements in values attr equal to rank of result shape
5588 auto count = getValues().getNumElements();
5589 auto rank = (cast<tosa::shapeType>(Val: getResult().getType())).getRank();
5590 if (count != rank && (count != 1 || rank != 0)) {
5591 return emitOpError(message: "expect number of elements in attribute values (")
5592 << count << ") to be equal to the rank (" << rank
5593 << ") for the result shape type";
5594 }
5595 return success();
5596}
5597
5598LogicalResult tosa::DimOp::verify() {
5599 const tosa::shapeType outShapeType =
5600 cast<tosa::shapeType>(Val: getResult().getType());
5601 if (outShapeType.getRank() != 1)
5602 return emitOpError(message: "expect output shape type to contain one element, got ")
5603 << outShapeType;
5604
5605 const ShapeAdaptor inputType(getInput1().getType());
5606 if (inputType.hasRank()) {
5607 const int64_t inputRank = inputType.getRank();
5608 const int64_t axis = getAxisAttr().getInt();
5609 if (axis < 0 || axis >= inputRank)
5610 return emitOpError(message: "expect axis to be in the range [0, ")
5611 << inputRank << "), got " << axis;
5612 }
5613 return success();
5614}
5615
5616LogicalResult tosa::ConcatShapeOp::verify() {
5617 const tosa::shapeType outShapeType =
5618 cast<tosa::shapeType>(Val: getResult().getType());
5619 const int64_t outputRank = outShapeType.getRank();
5620 const Operation::operand_range inputList = getInput();
5621
5622 if (inputList.size() == 0)
5623 return emitOpError(message: "requires at least one input shape");
5624
5625 if (llvm::any_of(Range: inputList, P: [](Value v) {
5626 return cast<tosa::shapeType>(Val: v.getType()).getRank() == 0;
5627 }))
5628 return emitOpError(message: "requires all inputs shapes have a rank greater than 0");
5629
5630 const int64_t inputsRank =
5631 llvm::accumulate(Range: inputList, Init: 0, Op: [](int64_t acc, const Value &input) {
5632 const tosa::shapeType inShapeType =
5633 cast<tosa::shapeType>(Val: input.getType());
5634 return acc + inShapeType.getRank();
5635 });
5636 if (outputRank != inputsRank)
5637 return emitOpError(message: "requires output shape rank to be equal to the sum of "
5638 "the input shape ranks (")
5639 << inputsRank << "), got " << outputRank;
5640
5641 return success();
5642}
5643
5644LogicalResult tosa::SliceShapeOp::verify() {
5645 std::optional<int32_t> start;
5646 DenseIntElementsAttr startAttr;
5647 if (matchPattern(value: getStart(), pattern: m_Constant(bind_value: &startAttr)))
5648 start = startAttr.getValues<int32_t>()[0];
5649 if (start && start.value() < 0)
5650 return emitOpError(message: "expected non-negative start index, got ")
5651 << start.value();
5652
5653 std::optional<int32_t> size;
5654 DenseIntElementsAttr sizeAttr;
5655 if (matchPattern(value: getSize(), pattern: m_Constant(bind_value: &sizeAttr)))
5656 size = sizeAttr.getValues<int32_t>()[0];
5657 if (size && size.value() <= 0)
5658 return emitOpError(message: "expected positive size, got ") << size.value();
5659
5660 if (!size)
5661 return success();
5662
5663 const tosa::shapeType outShapeType =
5664 cast<tosa::shapeType>(Val: getResult().getType());
5665 const int64_t outputRank = outShapeType.getRank();
5666 if (outputRank != size)
5667 return emitOpError(
5668 message: "expected output type size to be equal to size attribute, got ")
5669 << outputRank << " vs " << size.value();
5670
5671 if (!start)
5672 return success();
5673
5674 const tosa::shapeType inShapeType =
5675 cast<tosa::shapeType>(Val: getInput().getType());
5676 const int64_t inputRank = inShapeType.getRank();
5677 const int64_t sliceSize = start.value() + size.value();
5678 if (sliceSize > inputRank)
5679 return emitOpError(message: "expected start + size to be less than or equal to "
5680 "input shape rank (")
5681 << inputRank << "), got " << sliceSize;
5682
5683 return success();
5684}
5685
5686//===----------------------------------------------------------------------===//
5687// TOSA Attribute Definitions.
5688//===----------------------------------------------------------------------===//
5689
5690#define GET_ATTRDEF_CLASSES
5691#include "mlir/Dialect/Tosa/IR/TosaAttributes.cpp.inc"
5692
5693//===----------------------------------------------------------------------===//
5694// TOSA Type Definitions.
5695//===----------------------------------------------------------------------===//
5696#define GET_TYPEDEF_CLASSES
5697#include "mlir/Dialect/Tosa/IR/TosaOpsTypesBase.cpp.inc"
5698
5699//===----------------------------------------------------------------------===//
5700// TOSA Operator Definitions.
5701//===----------------------------------------------------------------------===//
5702
5703static ParseResult parseOptionalBoolClause(OpAsmParser &parser,
5704 StringRef keyword,
5705 BoolAttr &result) {
5706 if (failed(Result: parser.parseOptionalKeyword(keyword)))
5707 return success();
5708 if (parser.parseLParen() || parser.parseAttribute(result) ||
5709 parser.parseRParen())
5710 return failure();
5711 return success();
5712}
5713
5714static void printOptionalBoolClause(OpAsmPrinter &printer, StringRef keyword,
5715 BoolAttr attr) {
5716 if (!attr)
5717 return;
5718 printer << keyword << '(';
5719 printer.printAttribute(attr);
5720 printer << ')';
5721}
5722
5723static ParseResult parseLocalBound(OpAsmParser &parser, BoolAttr &result) {
5724 return parseOptionalBoolClause(parser, keyword: "local_bound", result);
5725}
5726
5727static void printLocalBound(OpAsmPrinter &printer, Operation *, BoolAttr attr) {
5728 printOptionalBoolClause(printer, keyword: "local_bound", attr);
5729}
5730
5731static ParseResult parseInputUnsigned(OpAsmParser &parser, BoolAttr &result) {
5732 return parseOptionalBoolClause(parser, keyword: "input_unsigned", result);
5733}
5734
5735static void printInputUnsigned(OpAsmPrinter &printer, Operation *,
5736 BoolAttr attr) {
5737 printOptionalBoolClause(printer, keyword: "input_unsigned", attr);
5738}
5739
5740#define GET_OP_CLASSES
5741#include "mlir/Dialect/Tosa/IR/TosaOps.cpp.inc"
5742