1//===- TosaToLinalgNamed.cpp - Lowering Tosa to Linalg Named Ops ----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These rewriters lower from the Tosa to the Linalg named ops.
10//
11//===----------------------------------------------------------------------===//
12
13#include "TosaToLinalgUtils.h"
14#include "mlir/Conversion/TosaToLinalg/TosaToLinalg.h"
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/Arith/Utils/Utils.h"
17#include "mlir/Dialect/Linalg/IR/Linalg.h"
18#include "mlir/Dialect/Math/IR/Math.h"
19#include "mlir/Dialect/Tensor/IR/Tensor.h"
20#include "mlir/Dialect/Tosa/IR/TosaOps.h"
21#include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
22#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
23#include "mlir/IR/Matchers.h"
24#include "mlir/IR/PatternMatch.h"
25#include "mlir/Transforms/DialectConversion.h"
26#include "llvm/ADT/SmallVectorExtras.h"
27#include "llvm/ADT/TypeSwitch.h"
28
29#include <optional>
30#include <type_traits>
31
32using namespace mlir;
33using namespace mlir::tosa;
34
35// Return the linalg.elementwise kind that is semantically equivalent to the
36// given TOSA elementwise op for the provided element type, or std::nullopt if
37// the op has no direct linalg.elementwise counterpart. Only ops whose TOSA
38// lowering maps to a single linalg.elementwise function are handled; the more
39// involved lowerings (e.g. quantized negate, mul with shift, casts) are left to
40// the generic TosaToLinalg conversion.
41static std::optional<linalg::ElementwiseKind>
42getElementwiseKind(Operation *op, Type elementTy) {
43 using linalg::ElementwiseKind;
44 const bool isFloat = isa<FloatType>(Val: elementTy);
45 const bool isSignlessInt = elementTy.isSignlessInteger();
46 const bool isBoolean = elementTy.isInteger(width: 1);
47
48 // Unary ops. linalg.elementwise only defines these on floating point types.
49 if (isFloat) {
50 if (isa<tosa::AbsOp>(Val: op))
51 return ElementwiseKind::abs;
52 if (isa<tosa::CeilOp>(Val: op))
53 return ElementwiseKind::ceil;
54 if (isa<tosa::FloorOp>(Val: op))
55 return ElementwiseKind::floor;
56 if (isa<tosa::ExpOp>(Val: op))
57 return ElementwiseKind::exp;
58 if (isa<tosa::LogOp>(Val: op))
59 return ElementwiseKind::log;
60 if (isa<tosa::RsqrtOp>(Val: op))
61 return ElementwiseKind::rsqrt;
62 if (isa<tosa::SinOp>(Val: op))
63 return ElementwiseKind::sin;
64 if (isa<tosa::CosOp>(Val: op))
65 return ElementwiseKind::cos;
66 if (isa<tosa::TanhOp>(Val: op))
67 return ElementwiseKind::tanh;
68 if (isa<tosa::ErfOp>(Val: op))
69 return ElementwiseKind::erf;
70 if (isa<tosa::ReciprocalOp>(Val: op))
71 return ElementwiseKind::reciprocal;
72 if (isa<tosa::PowOp>(Val: op))
73 return ElementwiseKind::powf;
74 }
75
76 // Binary ops defined on both floating point and (non-boolean) integers.
77 if (isFloat || (isSignlessInt && !isBoolean)) {
78 if (isa<tosa::AddOp>(Val: op))
79 return ElementwiseKind::add;
80 if (isa<tosa::SubOp>(Val: op))
81 return ElementwiseKind::sub;
82 }
83
84 // Signed integer division.
85 if (isSignlessInt && !isBoolean && isa<tosa::IntDivOp>(Val: op))
86 return ElementwiseKind::div;
87
88 // Multiply maps to a plain elementwise multiply only when no rescaling shift
89 // is requested (the shift must be a constant zero) and the inputs already
90 // share the result element type. The shifted/rescaled integer variant is left
91 // to the generic conversion.
92 if (auto mulOp = dyn_cast<tosa::MulOp>(Val: op)) {
93 DenseElementsAttr shiftAttr;
94 const bool zeroShift =
95 matchPattern(value: mulOp.getShift(), pattern: m_Constant(bind_value: &shiftAttr)) &&
96 shiftAttr.getValues<APInt>()[0].isZero();
97 const bool matchingTypes =
98 cast<ShapedType>(Val: mulOp.getInput1().getType()).getElementType() ==
99 elementTy &&
100 cast<ShapedType>(Val: mulOp.getInput2().getType()).getElementType() ==
101 elementTy;
102 if ((isFloat || (isSignlessInt && !isBoolean)) && zeroShift &&
103 matchingTypes)
104 return ElementwiseKind::mul;
105 }
106
107 // Negate maps to floating-point negation only when the zero points are zero.
108 // The quantized integer variant is left to the generic conversion.
109 if (auto negateOp = dyn_cast<tosa::NegateOp>(Val: op)) {
110 if (isFloat) {
111 FailureOr<int64_t> inZp = negateOp.getInput1ZeroPoint();
112 FailureOr<int64_t> outZp = negateOp.getOutputZeroPoint();
113 if (succeeded(Result: inZp) && succeeded(Result: outZp) && *inZp == 0 && *outZp == 0)
114 return ElementwiseKind::negf;
115 }
116 }
117
118 // Maximum/minimum. The NaN "IGNORE" mode requires an additional compare and
119 // select that linalg.elementwise cannot express, so only the default
120 // "PROPAGATE" mode is handled for floating point types.
121 auto nanModeIsPropagate = [](auto tosaOp) {
122 return tosaOp.getNanMode() == NanPropagationMode::PROPAGATE;
123 };
124 if (isFloat || (isSignlessInt && !isBoolean)) {
125 if (auto maxOp = dyn_cast<tosa::MaximumOp>(Val: op)) {
126 if (!isFloat || nanModeIsPropagate(maxOp))
127 return ElementwiseKind::max_signed;
128 }
129 if (auto minOp = dyn_cast<tosa::MinimumOp>(Val: op)) {
130 if (!isFloat || nanModeIsPropagate(minOp))
131 return ElementwiseKind::min_signed;
132 }
133 }
134
135 // Select (ternary) for valid element types.
136 if ((isFloat || isSignlessInt || isBoolean) && isa<tosa::SelectOp>(Val: op))
137 return ElementwiseKind::select;
138
139 return std::nullopt;
140}
141
142std::optional<linalg::ElementwiseKind>
143mlir::tosa::isConvertibleToLinalgElementwise(Operation *op) {
144 if (op->getNumResults() != 1)
145 return std::nullopt;
146
147 // Make sure all operands and result have static sizes
148 for (Value operand : op->getOperands()) {
149 auto operandTy = dyn_cast<RankedTensorType>(Val: operand.getType());
150 if (!operandTy || !operandTy.hasStaticShape())
151 return std::nullopt;
152 }
153 auto resultTy = dyn_cast<RankedTensorType>(Val: op->getResult(idx: 0).getType());
154 if (!resultTy || !resultTy.hasStaticShape())
155 return std::nullopt;
156
157 return getElementwiseKind(op, elementTy: resultTy.getElementType());
158}
159
160static mlir::Value applyPad(Location loc, Value input, ArrayRef<int64_t> pad,
161 TypedAttr padAttr, OpBuilder &rewriter) {
162 // Input should be padded only if necessary.
163 if (llvm::all_of(Range&: pad, P: [](int64_t p) { return p == 0; }))
164 return input;
165
166 ShapedType inputTy = cast<ShapedType>(Val: input.getType());
167 Type inputETy = inputTy.getElementType();
168 auto inputShape = inputTy.getShape();
169
170 assert((inputShape.size() * 2) == pad.size());
171
172 SmallVector<int64_t, 4> paddedShape;
173 SmallVector<OpFoldResult, 8> lowIndices;
174 SmallVector<OpFoldResult, 8> highIndices;
175 for (size_t i : llvm::seq(Size: inputShape.size())) {
176 auto lowPad = pad[i * 2];
177 auto highPad = pad[i * 2 + 1];
178 if (ShapedType::isDynamic(dValue: inputShape[i]))
179 paddedShape.push_back(Elt: inputShape[i]);
180 else
181 paddedShape.push_back(Elt: inputShape[i] + highPad + lowPad);
182 lowIndices.push_back(Elt: rewriter.getIndexAttr(value: lowPad));
183 highIndices.push_back(Elt: rewriter.getIndexAttr(value: highPad));
184 }
185
186 Value padValue = arith::ConstantOp::create(builder&: rewriter, location: loc, value: padAttr);
187
188 return tensor::PadOp::create(builder&: rewriter, location: loc,
189 resultType: RankedTensorType::get(shape: paddedShape, elementType: inputETy),
190 source: input, low: lowIndices, high: highIndices, constantPadValue: padValue);
191}
192
193static mlir::Value
194linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias,
195 Value conv, Value result,
196 ArrayRef<AffineMap> indexingMaps) {
197 ShapedType resultTy = cast<ShapedType>(Val: conv.getType());
198 return linalg::GenericOp::create(
199 builder&: rewriter, location: loc, resultTensorTypes: resultTy, inputs: ValueRange({bias, conv}), outputs: result,
200 indexingMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()),
201 odsArg5: [](OpBuilder &builder, Location loc, ValueRange args) {
202 Value biasVal = args[0];
203 Type resType = args[1].getType();
204 if (resType != biasVal.getType()) {
205 biasVal =
206 arith::ExtSIOp::create(builder, location: loc, out: resType, in: biasVal);
207 }
208 Value added =
209 arith::AddIOp::create(builder, location: loc, lhs: biasVal, rhs: args[1]);
210 linalg::YieldOp::create(builder, location: loc, values: added);
211 })
212 .getResult(i: 0);
213}
214
215// Construct the affine map that a linalg generic would use to broadcast the
216// source tensor into the shape of the result tensor.
217static AffineMap getBroadcastingMap(PatternRewriter &rewriter, Value source,
218 Value result) {
219 ShapedType resultTy = cast<ShapedType>(Val: result.getType());
220 ShapedType sourceTy = cast<ShapedType>(Val: source.getType());
221 const int64_t resultRank = resultTy.getRank();
222 const int64_t sourceRank = sourceTy.getRank();
223
224 // The source tensor is broadcast to all the outer dimensions of the
225 // result tensor.
226 SmallVector<AffineExpr> sourceDims;
227 // In the case of a rank one source tensor with a single element TOSA
228 // specifies that the value be broadcast meaning we need an edge case for a
229 // constant map.
230 assert(sourceTy.hasStaticShape() &&
231 "Dynamic broadcasting shapes not supported!");
232 if (sourceRank == 1 && sourceTy.getDimSize(idx: 0) == 1) {
233 sourceDims.push_back(Elt: rewriter.getAffineConstantExpr(constant: 0));
234 } else {
235 for (auto dim : llvm::seq<int64_t>(Begin: 0, End: sourceRank)) {
236 auto expr = rewriter.getAffineDimExpr(position: dim + resultRank - sourceRank);
237 sourceDims.push_back(Elt: expr);
238 }
239 }
240
241 return AffineMap::get(/*dimCount=*/resultRank,
242 /*symbolCount=*/0, results: sourceDims, context: rewriter.getContext());
243}
244
245// Broadcast the source value to all the outer dimensions of the result value.
246// If required, the element type is expanded using an arith.extsi or arith.extf
247// operation as appropriate.
248static mlir::Value linalgBroadcastAndMaybeExt(PatternRewriter &rewriter,
249 Location loc, Value source,
250 Value result) {
251 ShapedType resultTy = cast<ShapedType>(Val: result.getType());
252 const int64_t resultRank = resultTy.getRank();
253 // Creating maps for the input and output of the broacast-like generic op.
254 SmallVector<AffineMap, 2> indexingMaps;
255 indexingMaps.push_back(Elt: getBroadcastingMap(rewriter, source, result));
256 indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank: resultRank));
257
258 // Build the broadcast-like operation as a linalg.generic.
259 return linalg::GenericOp::create(
260 builder&: rewriter, location: loc, resultTensorTypes: resultTy, inputs: ValueRange({source}), outputs: result,
261 indexingMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()),
262 odsArg5: [&resultTy](OpBuilder &builder, Location loc, ValueRange args) {
263 Value biasVal = args[0];
264 Type resType = args[1].getType();
265 if (resType != biasVal.getType()) {
266 biasVal =
267 resultTy.getElementType().isFloat()
268 ? arith::ExtFOp::create(
269 builder, location: loc, resultTypes: TypeRange{resType},
270 operands: ValueRange{biasVal}, properties: arith::ExtFOp::Properties{})
271 .getResult()
272 : arith::ExtSIOp::create(builder, location: loc, out: resType,
273 in: biasVal)
274 .getResult();
275 }
276 linalg::YieldOp::create(builder, location: loc, values: biasVal);
277 })
278 .getResult(i: 0);
279}
280
281static mlir::Value reifyConstantDim(int64_t attr,
282 ImplicitLocOpBuilder &builder) {
283 return arith::ConstantIndexOp::create(builder, value: attr);
284}
285
286// Calculating the output width/height using the formula:
287// H = ((IH+pad_top+pad_bottom-(dilation_y*(KH-1)+1))/stride_y)+1
288// W = ((IW+pad_left+pad_right-(dilation_x*(KW-1)+1))/stride_x)+1
289
290static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim,
291 int64_t padBeforeAttr,
292 int64_t padAfterAttr, Value kernelDim,
293 int64_t strideAttr,
294 int64_t dilationAttr,
295 OpBuilder &rewriter) {
296 ImplicitLocOpBuilder builder(loc, rewriter);
297 auto one = arith::ConstantOp::create(builder&: rewriter, location: loc,
298 value: IntegerAttr::get(type: inputDim.getType(), value: 1));
299 Value padBefore = reifyConstantDim(attr: padBeforeAttr, builder);
300 Value paddedBefore = arith::AddIOp::create(builder, lhs: inputDim, rhs: padBefore);
301 Value padAfter = reifyConstantDim(attr: padAfterAttr, builder);
302 Value paddedAfter = arith::AddIOp::create(builder, lhs: paddedBefore, rhs: padAfter);
303
304 Value subOne = arith::SubIOp::create(builder, lhs: kernelDim, rhs: one);
305 Value dilation = reifyConstantDim(attr: dilationAttr, builder);
306 Value dilated = arith::MulIOp::create(builder, lhs: dilation, rhs: subOne);
307 Value addOne = arith::AddIOp::create(builder, lhs: dilated, rhs: one);
308
309 Value subtract = arith::SubIOp::create(builder, lhs: paddedAfter, rhs: addOne);
310 Value stride = reifyConstantDim(attr: strideAttr, builder);
311 Value divide = arith::DivUIOp::create(builder, lhs: subtract, rhs: stride);
312 return arith::AddIOp::create(builder, lhs: divide, rhs: one);
313}
314
315// Creates a vector of the dynamic output dims for Conv2D and Depthwise_Conv2D
316static SmallVector<Value> inferDynamicDimsForConv(
317 Location loc, Value input, Value weight, ShapedType resultTy,
318 ArrayRef<int64_t> padAttr, ArrayRef<int64_t> strideAttr,
319 ArrayRef<int64_t> dilationAttr, ArrayRef<int64_t> inputSizeDims,
320 ArrayRef<int64_t> kernelSizeDims, OpBuilder &rewriter) {
321 ShapedType inputTy = cast<ShapedType>(Val: input.getType());
322 int64_t inputRank = inputTy.getRank();
323
324 SmallVector<Value> dynDims;
325 dynDims.resize(N: resultTy.getRank());
326
327 for (uint32_t i = 0, s = inputSizeDims.size(); i < s; ++i) {
328 int64_t inputDim = inputSizeDims[i];
329 int64_t kernelDim = kernelSizeDims[i];
330 if (resultTy.isDynamicDim(idx: inputDim)) {
331 auto padTop = padAttr[i * 2];
332 auto padBottom = padAttr[i * 2 + 1];
333 auto stride = strideAttr[i];
334 auto dilation = dilationAttr[i];
335 Value initDynDim = tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: inputDim);
336 Value kernelDynDim =
337 tensor::DimOp::create(builder&: rewriter, location: loc, source: weight, index: kernelDim);
338 // H = F(IH, pad_top, pad_bottom, dilation_y, KH, stride_y)
339 dynDims[inputDim] =
340 getConvOrPoolOutputDim(loc, inputDim: initDynDim, padBeforeAttr: padTop, padAfterAttr: padBottom,
341 kernelDim: kernelDynDim, strideAttr: stride, dilationAttr: dilation, rewriter);
342 }
343 }
344
345 // Get the batch/channels dimensions.
346 for (int i = 0; i < inputRank; i++) {
347 if (resultTy.isDynamicDim(idx: i) && !dynDims[i])
348 dynDims[i] = tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: i);
349 }
350
351 SmallVector<Value> filteredDims = condenseValues(values: dynDims);
352 return filteredDims;
353}
354
355// Creates a map to collapse the last dimension of the Depthwise convolution op
356// due to a shape mismatch
357static void createDepthwiseConvCollapseMap(
358 int64_t outputRank, SmallVector<ReassociationExprs, 4> &reassociationMap,
359 OpBuilder &rewriter) {
360 reassociationMap.resize(N: outputRank);
361 for (int i = 0; i < outputRank; i++) {
362 reassociationMap[i].push_back(Elt: rewriter.getAffineDimExpr(position: i));
363 }
364 reassociationMap[outputRank - 1].push_back(
365 Elt: rewriter.getAffineDimExpr(position: outputRank));
366}
367
368namespace {
369
370template <typename TosaConvOp, typename LinalgConvOp, typename LinalgConvQOp>
371class ConvConverter : public OpConversionPattern<TosaConvOp> {
372public:
373 using OpConversionPattern<TosaConvOp>::OpConversionPattern;
374 LogicalResult
375 matchAndRewrite(TosaConvOp op, typename TosaConvOp::Adaptor adaptor,
376 ConversionPatternRewriter &rewriter) const final {
377 Location loc = op->getLoc();
378 Value input = op->getOperand(0);
379 Value weight = op->getOperand(1);
380 Value bias = op->getOperand(2);
381
382 ShapedType inputTy = cast<ShapedType>(Val: input.getType());
383 ShapedType weightTy = cast<ShapedType>(Val: weight.getType());
384 ShapedType biasTy = cast<ShapedType>(Val: bias.getType());
385 ShapedType resultTy = cast<ShapedType>(op->getResult(0).getType());
386
387 Type inputETy = inputTy.getElementType();
388
389 DenseI64ArrayAttr padAttr = op.getPadAttr();
390 DenseI64ArrayAttr strideTosaAttr = op.getStrideAttr();
391 DenseI64ArrayAttr dilationTosaAttr = op.getDilationAttr();
392
393 Type accETy = op.getAccType();
394 Type accTy = RankedTensorType::get(shape: resultTy.getShape(), elementType: accETy);
395
396 // Get and verify zero points.
397 FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
398 if (failed(Result: maybeIZp))
399 return rewriter.notifyMatchFailure(
400 op, "input zero point cannot be statically determined");
401
402 FailureOr<int64_t> maybeWZp = op.getWeightZeroPoint();
403 if (failed(Result: maybeWZp))
404 return rewriter.notifyMatchFailure(
405 op, "weight zero point cannot be statically determined");
406
407 const int64_t inputZpVal = *maybeIZp;
408 const int64_t weightZpVal = *maybeWZp;
409
410 if (op.verifyInputZeroPoint(inputZpVal).failed())
411 return rewriter.notifyMatchFailure(
412 op, "input zero point must be zero for non-int8 integer types");
413
414 if (op.verifyWeightZeroPoint(weightZpVal).failed())
415 return rewriter.notifyMatchFailure(
416 op, "weight zero point must be zero for non-int8 integer types");
417
418 bool hasZp = (inputZpVal != 0) || (weightZpVal != 0);
419
420 if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
421 return rewriter.notifyMatchFailure(
422 op, "tosa.conv ops require static shapes for weight and bias");
423
424 if (inputETy.isUnsignedInteger())
425 return rewriter.notifyMatchFailure(
426 op, "tosa.conv ops does not support unsigned integer input");
427
428 llvm::SmallVector<int64_t> inputSizeDims;
429 llvm::SmallVector<int64_t> kernelSizeDims;
430 for (int i = 1; i < resultTy.getRank() - 1; i++) {
431 inputSizeDims.push_back(Elt: i);
432 kernelSizeDims.push_back(Elt: i);
433 }
434
435 SmallVector<Value> filteredDims = inferDynamicDimsForConv(
436 loc, input, weight, resultTy, padAttr: padAttr.asArrayRef(),
437 strideAttr: strideTosaAttr.asArrayRef(), dilationAttr: dilationTosaAttr.asArrayRef(),
438 inputSizeDims, kernelSizeDims, rewriter);
439
440 auto weightShape = weightTy.getShape();
441
442 // Apply padding as necessary.
443 TypedAttr zeroAttr = rewriter.getZeroAttr(type: inputETy);
444 if (hasZp) {
445 int64_t intMin =
446 APInt::getSignedMinValue(numBits: inputETy.getIntOrFloatBitWidth())
447 .getSExtValue();
448 int64_t intMax =
449 APInt::getSignedMaxValue(numBits: inputETy.getIntOrFloatBitWidth())
450 .getSExtValue();
451
452 if (inputZpVal < intMin || inputZpVal > intMax)
453 return rewriter.notifyMatchFailure(
454 op, "tosa.conv op quantization has zp outside of input range");
455
456 zeroAttr = rewriter.getIntegerAttr(type: inputETy, value: inputZpVal);
457 }
458
459 llvm::SmallVector<int64_t> pad;
460 pad.resize(N: 2, NV: 0);
461 llvm::append_range(C&: pad, R: padAttr.asArrayRef());
462 pad.resize(N: pad.size() + 2, NV: 0);
463 input = applyPad(loc, input, pad, padAttr: zeroAttr, rewriter);
464
465 if (4 == inputTy.getRank()) {
466 // For 2D convolutions, we need to check if the target convolution op
467 // wants a HWCF kernel layout.
468 bool wantHwcf =
469 hasZp ? std::is_same_v<LinalgConvQOp, linalg::Conv2DNhwcHwcfQOp>
470 : std::is_same_v<LinalgConvOp, linalg::Conv2DNhwcHwcfOp>;
471 if (wantHwcf) {
472 // Transpose the kernel to match dimension ordering of the linalg
473 // convolution operation.
474 // TODO(suderman): See if this can be efficiently folded - check whether
475 // the input is used anywhere else, if not fold the constant.
476 SmallVector<int32_t> weightPerm;
477 for (int i = 1; i < resultTy.getRank(); i++)
478 weightPerm.push_back(Elt: i);
479 weightPerm.push_back(Elt: 0);
480
481 SmallVector<int64_t> newWeightShape;
482 for (auto dim : weightPerm)
483 newWeightShape.push_back(Elt: weightShape[dim]);
484 auto weightPermAttr = rewriter.getDenseI32ArrayAttr(values: weightPerm);
485 Type newWeightTy =
486 RankedTensorType::get(shape: newWeightShape, elementType: weightTy.getElementType());
487 weight = tosa::TransposeOp::create(builder&: rewriter, location: loc, output: newWeightTy, input1: weight,
488 perms: weightPermAttr);
489 }
490 }
491
492 // For Conv3D transpose the kernel to match dimension ordering of the linalg
493 // convolution operation. Conv2D has a 1-1 mapping in linalg so better to
494 // map directly and then transpose later if desired.
495 if (5 == inputTy.getRank()) {
496 // TODO(suderman): See if this can be efficiently folded - check whether
497 // the input is used anywhere else, if not fold the constant.
498 SmallVector<int32_t> weightPerm;
499 for (int i = 1; i < resultTy.getRank(); i++)
500 weightPerm.push_back(Elt: i);
501 weightPerm.push_back(Elt: 0);
502
503 SmallVector<int64_t> newWeightShape;
504 for (auto dim : weightPerm)
505 newWeightShape.push_back(Elt: weightShape[dim]);
506 auto weightPermAttr = rewriter.getDenseI32ArrayAttr(values: weightPerm);
507 Type newWeightTy =
508 RankedTensorType::get(shape: newWeightShape, elementType: weightTy.getElementType());
509 weight = tosa::TransposeOp::create(builder&: rewriter, location: loc, output: newWeightTy, input1: weight,
510 perms: weightPermAttr);
511 }
512
513 // Extract the attributes for convolution.
514 ArrayRef<int64_t> stride = strideTosaAttr;
515 ArrayRef<int64_t> dilation = dilationTosaAttr;
516
517 // Create the convolution op.
518 auto strideAttr = rewriter.getI64TensorAttr(values: stride);
519 auto dilationAttr = rewriter.getI64TensorAttr(values: dilation);
520
521 Value biasEmptyTensor = tensor::EmptyOp::create(
522 builder&: rewriter, location: loc, staticShape: resultTy.getShape(), elementType: accETy, dynamicSizes: filteredDims);
523
524 Value broadcastBias =
525 linalgBroadcastAndMaybeExt(rewriter, loc, source: bias, result: biasEmptyTensor);
526
527 if (hasZp) {
528 auto iZp = rewriter.getI32IntegerAttr(value: inputZpVal);
529 auto kZp = rewriter.getI32IntegerAttr(value: weightZpVal);
530
531 auto iZpVal = arith::ConstantOp::create(builder&: rewriter, location: loc, value: iZp);
532 auto kZpVal = arith::ConstantOp::create(builder&: rewriter, location: loc, value: kZp);
533
534 Value conv = LinalgConvQOp::create(
535 rewriter, loc, resultTy,
536 ValueRange{input, weight, iZpVal, kZpVal},
537 ValueRange{broadcastBias}, strideAttr, dilationAttr)
538 ->getResult(0);
539
540 rewriter.replaceOp(op, conv);
541 return success();
542 }
543
544 Value conv = LinalgConvOp::create(
545 rewriter, loc, accTy, ValueRange{input, weight},
546 ValueRange{broadcastBias}, strideAttr, dilationAttr)
547 ->getResult(0);
548
549 // We may need to truncate back to the result type if the accumulator was
550 // wider than the result.
551 if (resultTy != accTy)
552 conv = tosa::CastOp::create(builder&: rewriter, location: loc, output: resultTy, input: conv);
553
554 rewriter.replaceOp(op, conv);
555 return success();
556 }
557};
558
559class DepthwiseConvConverter
560 : public OpConversionPattern<tosa::DepthwiseConv2DOp> {
561public:
562 using OpConversionPattern<tosa::DepthwiseConv2DOp>::OpConversionPattern;
563 LogicalResult
564 matchAndRewrite(tosa::DepthwiseConv2DOp op, OpAdaptor adaptor,
565 ConversionPatternRewriter &rewriter) const final {
566 Location loc = op->getLoc();
567 Value input = op->getOperand(idx: 0);
568 Value weight = op->getOperand(idx: 1);
569 Value bias = op->getOperand(idx: 2);
570
571 ShapedType inputTy = cast<ShapedType>(Val: input.getType());
572 ShapedType weightTy = cast<ShapedType>(Val: weight.getType());
573 ShapedType biasTy = cast<ShapedType>(Val: bias.getType());
574 ShapedType resultTy = cast<ShapedType>(Val: op->getResult(idx: 0).getType());
575 int64_t resultRank = resultTy.getRank();
576
577 Type inputETy = inputTy.getElementType();
578 Type resultETy = resultTy.getElementType();
579
580 auto padAttr = op.getPadAttr();
581 auto strideTosaAttr = op.getStrideAttr();
582 auto dilationTosaAttr = op.getDilationAttr();
583
584 Type accETy = op.getAccType();
585
586 if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
587 return rewriter.notifyMatchFailure(
588 arg&: op, msg: "tosa.depthwise_conv ops require static shapes");
589
590 // Compute output dynamic dims
591 SmallVector<Value> filteredDims = inferDynamicDimsForConv(
592 loc, input, weight, resultTy, padAttr: padAttr.asArrayRef(),
593 strideAttr: strideTosaAttr.asArrayRef(), dilationAttr: dilationTosaAttr.asArrayRef(),
594 /*inputSizeDims=*/{1, 2},
595 /*kernelSizeDims=*/{0, 1}, rewriter);
596
597 // Get and verify zero points.
598
599 FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
600 FailureOr<int64_t> maybeWZp = op.getWeightZeroPoint();
601 if (failed(Result: maybeIZp))
602 return rewriter.notifyMatchFailure(
603 arg&: op, msg: "input zero point cannot be statically determined");
604 if (failed(Result: maybeWZp))
605 return rewriter.notifyMatchFailure(
606 arg&: op, msg: "weight zero point cannot be statically determined");
607
608 const int64_t inputZpVal = *maybeIZp;
609 const int64_t weightZpVal = *maybeWZp;
610
611 if (op.verifyInputZeroPoint(zp: inputZpVal).failed())
612 return rewriter.notifyMatchFailure(
613 arg&: op, msg: "input zero point must be zero for non-int8 integer types");
614
615 if (op.verifyWeightZeroPoint(zp: weightZpVal).failed())
616 return rewriter.notifyMatchFailure(
617 arg&: op, msg: "weight zero point must be zero for non-int8 integer types");
618
619 bool hasNullZps = (inputZpVal == 0) && (weightZpVal == 0);
620 auto weightShape = weightTy.getShape();
621 auto resultShape = resultTy.getShape();
622
623 // Apply padding as necessary.
624 TypedAttr zeroAttr = rewriter.getZeroAttr(type: inputETy);
625 if (!hasNullZps) {
626 int64_t intMin =
627 APInt::getSignedMinValue(numBits: inputETy.getIntOrFloatBitWidth())
628 .getSExtValue();
629 int64_t intMax =
630 APInt::getSignedMaxValue(numBits: inputETy.getIntOrFloatBitWidth())
631 .getSExtValue();
632
633 if (inputZpVal < intMin || inputZpVal > intMax)
634 return rewriter.notifyMatchFailure(
635 arg&: op, msg: "tosa.depthwise_conv op quantization has zp outside of input "
636 "range");
637
638 zeroAttr = rewriter.getIntegerAttr(type: inputETy, value: inputZpVal);
639 }
640
641 llvm::SmallVector<int64_t> pad;
642 pad.resize(N: 2, NV: 0);
643 llvm::append_range(C&: pad, R: padAttr.asArrayRef());
644 pad.resize(N: pad.size() + 2, NV: 0);
645
646 input = applyPad(loc, input, pad, padAttr: zeroAttr, rewriter);
647
648 // Extract the attributes for convolution.
649 ArrayRef<int64_t> stride = strideTosaAttr;
650 ArrayRef<int64_t> dilation = dilationTosaAttr;
651
652 // Create the convolution op.
653 auto strideAttr = rewriter.getI64TensorAttr(values: stride);
654 auto dilationAttr = rewriter.getI64TensorAttr(values: dilation);
655 ShapedType linalgConvTy =
656 RankedTensorType::get(shape: {resultShape[0], resultShape[1], resultShape[2],
657 weightShape[2], weightShape[3]},
658 elementType: accETy);
659
660 auto resultZeroAttr = rewriter.getZeroAttr(type: accETy);
661 Value emptyTensor = tensor::EmptyOp::create(
662 builder&: rewriter, location: loc, staticShape: linalgConvTy.getShape(), elementType: accETy, dynamicSizes: filteredDims);
663 Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc, value: resultZeroAttr);
664 Value zeroTensor = linalg::FillOp::create(builder&: rewriter, location: loc, inputs: ValueRange{zero},
665 outputs: ValueRange{emptyTensor})
666 .result();
667
668 Value biasEmptyTensor = tensor::EmptyOp::create(
669 builder&: rewriter, location: loc, staticShape: resultTy.getShape(), elementType: resultETy, dynamicSizes: filteredDims);
670
671 // Broadcast the initial value to the output tensor before convolving.
672 SmallVector<AffineMap, 4> indexingMaps;
673 indexingMaps.push_back(Elt: getBroadcastingMap(rewriter, source: bias, result: biasEmptyTensor));
674 indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank: resultRank));
675 indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank: resultRank));
676
677 if (hasNullZps) {
678 Value conv = linalg::DepthwiseConv2DNhwcHwcmOp::create(
679 builder&: rewriter, location: loc, resultTensorTypes: linalgConvTy, inputs: ValueRange{input, weight},
680 outputs: ValueRange{zeroTensor}, strides: strideAttr, dilations: dilationAttr)
681 .getResult(i: 0);
682
683 // We may need to truncate back to the result type if the accumulator was
684 // wider than the result.
685 if (accETy != resultETy)
686 conv = tosa::CastOp::create(
687 builder&: rewriter, location: loc,
688 output: RankedTensorType::get(shape: cast<ShapedType>(Val: conv.getType()).getShape(),
689 elementType: resultETy),
690 input: conv);
691
692 SmallVector<ReassociationExprs, 4> reassociationMap;
693 createDepthwiseConvCollapseMap(outputRank: resultRank, reassociationMap, rewriter);
694 Value convReshape = tensor::CollapseShapeOp::create(
695 builder&: rewriter, location: loc, resultType: resultTy, src: conv, reassociation: reassociationMap);
696
697 Value result =
698 linalg::GenericOp::create(
699 builder&: rewriter, location: loc, resultTensorTypes: resultTy, inputs: ValueRange({bias, convReshape}),
700 outputs: biasEmptyTensor, indexingMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultRank),
701 odsArg5: [&](OpBuilder &nestedBuilder, Location nestedLoc,
702 ValueRange args) {
703 Value added;
704 if (llvm::isa<FloatType>(Val: inputETy))
705 added = arith::AddFOp::create(builder&: nestedBuilder, location: loc, lhs: args[0],
706 rhs: args[1]);
707 else
708 added = arith::AddIOp::create(builder&: nestedBuilder, location: loc, lhs: args[0],
709 rhs: args[1]);
710 linalg::YieldOp::create(builder&: nestedBuilder, location: nestedLoc, values: added);
711 })
712 .getResult(i: 0);
713 rewriter.replaceOp(op, newValues: result);
714 } else {
715 IntegerAttr iZp = rewriter.getI32IntegerAttr(value: inputZpVal);
716 IntegerAttr wZp = rewriter.getI32IntegerAttr(value: weightZpVal);
717 auto iZpVal = arith::ConstantOp::create(builder&: rewriter, location: loc, value: iZp);
718 auto kZpVal = arith::ConstantOp::create(builder&: rewriter, location: loc, value: wZp);
719 Value conv = linalg::DepthwiseConv2DNhwcHwcmQOp::create(
720 builder&: rewriter, location: loc, resultTensorTypes: linalgConvTy,
721 inputs: ValueRange{input, weight, iZpVal, kZpVal},
722 outputs: ValueRange{zeroTensor}, strides: strideAttr, dilations: dilationAttr)
723 .getResult(i: 0);
724 SmallVector<ReassociationExprs, 4> reassociationMap;
725 createDepthwiseConvCollapseMap(outputRank: resultRank, reassociationMap, rewriter);
726 Value convReshape = tensor::CollapseShapeOp::create(
727 builder&: rewriter, location: loc, resultType: resultTy, src: conv, reassociation: reassociationMap);
728 Value result = linalgIntBroadcastExtSIAdd(
729 rewriter, loc, bias, conv: convReshape, result: biasEmptyTensor, indexingMaps);
730 rewriter.replaceOp(op, newValues: result);
731 }
732 return success();
733 }
734};
735
736// Lower TOSA elementwise ops that have a direct linalg.elementwise named op
737// counterpart. Unlike linalg.generic, linalg.elementwise dispatches on a single
738// function kind and infers the body, so the conversion is a handful of checks
739// (kind, ranks, static shapes) followed by construction of the indexing maps
740// and the named op.
741template <typename TosaOp>
742class ElementwiseConverter : public OpConversionPattern<TosaOp> {
743public:
744 using OpConversionPattern<TosaOp>::OpConversionPattern;
745 using typename OpConversionPattern<TosaOp>::OpAdaptor;
746
747 LogicalResult
748 matchAndRewrite(TosaOp op, OpAdaptor adaptor,
749 ConversionPatternRewriter &rewriter) const final {
750 std::optional<linalg::ElementwiseKind> kind =
751 isConvertibleToLinalgElementwise(op);
752 if (!kind)
753 return rewriter.notifyMatchFailure(
754 op, "op has no linalg.elementwise counterpart");
755
756 Location loc = op.getLoc();
757 auto resultTy = cast<RankedTensorType>(op.getType());
758 const int64_t rank = resultTy.getRank();
759
760 const unsigned arity =
761 llvm::to_underlying(E: linalg::getArityGroupAndKind(kind: *kind).arityGroup);
762
763 // Trailing metadata operands (e.g. MulOp's shift, NegateOp's zero points)
764 // are not forwarded to linalg.elementwise.
765 ValueRange inputs = adaptor.getOperands().take_front(arity);
766
767 // Build one indexing map per input, broadcasting static size-1 dimensions,
768 // followed by the identity map for the output.
769 SmallVector<AffineMap> indexingMaps;
770 for (Value operand : inputs) {
771 ArrayRef<int64_t> shape = cast<ShapedType>(Val: operand.getType()).getShape();
772 SmallVector<AffineExpr> exprs;
773 for (const auto &dim : llvm::enumerate(First&: shape)) {
774 bool requiresBroadcast =
775 dim.value() == 1 && resultTy.getDimSize(dim.index()) != 1;
776 exprs.push_back(Elt: requiresBroadcast
777 ? rewriter.getAffineConstantExpr(constant: 0)
778 : rewriter.getAffineDimExpr(position: dim.index()));
779 }
780 indexingMaps.push_back(Elt: AffineMap::get(dimCount: rank, /*symbolCount=*/0, results: exprs,
781 context: rewriter.getContext()));
782 }
783 indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank));
784
785 // Shapes are guaranteed static by isConvertibleToLinalgElementwise.
786 Value emptyTensor = tensor::EmptyOp::create(
787 rewriter, loc, resultTy.getShape(), resultTy.getElementType());
788
789 rewriter.replaceOpWithNewOp<linalg::ElementwiseOp>(
790 op, inputs, ValueRange{emptyTensor}, *kind,
791 rewriter.getAffineMapArrayAttr(values: indexingMaps));
792 return success();
793 }
794};
795
796static SmallVector<ReassociationIndices>
797getMatMulBatchCollapseReassociation(int64_t rank) {
798 SmallVector<ReassociationIndices> reassociation(3);
799 for (int64_t dim = 0; dim < rank - 2; ++dim)
800 reassociation[0].push_back(Elt: dim);
801 reassociation[1].push_back(Elt: rank - 2);
802 reassociation[2].push_back(Elt: rank - 1);
803 return reassociation;
804}
805
806class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
807public:
808 using OpConversionPattern<tosa::MatMulOp>::OpConversionPattern;
809 LogicalResult
810 matchAndRewrite(tosa::MatMulOp op, OpAdaptor adaptor,
811 ConversionPatternRewriter &rewriter) const final {
812 Location loc = op.getLoc();
813 auto aType = dyn_cast<RankedTensorType>(Val: adaptor.getA().getType());
814 auto bType = dyn_cast<RankedTensorType>(Val: adaptor.getB().getType());
815 auto outputType = dyn_cast<RankedTensorType>(Val: op.getType());
816 if (!aType || !bType || !outputType)
817 return rewriter.notifyMatchFailure(arg&: op, msg: "only supports ranked tensors");
818
819 FailureOr<int64_t> maybeAZp = op.getAZeroPoint();
820 FailureOr<int64_t> maybeBZp = op.getBZeroPoint();
821 if (failed(Result: maybeAZp))
822 return rewriter.notifyMatchFailure(
823 arg&: op, msg: "input A zero point cannot be statically determined");
824 if (failed(Result: maybeBZp))
825 return rewriter.notifyMatchFailure(
826 arg&: op, msg: "input B zero point cannot be statically determined");
827
828 const int64_t aZpVal = *maybeAZp;
829 const int64_t bZpVal = *maybeBZp;
830
831 if (op.verifyAZeroPoint(zp: aZpVal).failed())
832 return rewriter.notifyMatchFailure(
833 arg&: op, msg: "input A zero point must be zero for non-int8 integer types");
834
835 if (op.verifyBZeroPoint(zp: bZpVal).failed())
836 return rewriter.notifyMatchFailure(
837 arg&: op, msg: "input B zero point must be zero for non-int8 integer types");
838
839 const int64_t outputRank = outputType.getRank();
840 auto [a, b, zeroTensor] =
841 prepareMatMulOperands(rewriter, loc, a: adaptor.getA(), b: adaptor.getB(),
842 outputType, /*transposeB=*/false);
843
844 Value aZp, bZp;
845 const bool hasNonzeroZeroPoint = aZpVal != 0 || bZpVal != 0;
846 if (hasNonzeroZeroPoint) {
847 aZp = arith::ConstantOp::create(builder&: rewriter, location: loc,
848 value: rewriter.getI32IntegerAttr(value: aZpVal));
849 bZp = arith::ConstantOp::create(builder&: rewriter, location: loc,
850 value: rewriter.getI32IntegerAttr(value: bZpVal));
851 }
852
853 if (outputRank == 2) {
854 if (!hasNonzeroZeroPoint) {
855 rewriter.replaceOpWithNewOp<linalg::MatmulOp>(op, args: TypeRange{outputType},
856 args: ValueRange{a, b},
857 args: ValueRange{zeroTensor});
858 } else {
859 rewriter.replaceOpWithNewOp<linalg::QuantizedMatmulOp>(
860 op, args: TypeRange{outputType}, args: ValueRange{a, b, aZp, bZp},
861 args: ValueRange{zeroTensor});
862 }
863 return success();
864 }
865
866 if (outputRank == 3) {
867 if (!hasNonzeroZeroPoint) {
868 rewriter.replaceOpWithNewOp<linalg::BatchMatmulOp>(
869 op, args: TypeRange{outputType}, args: ValueRange{a, b},
870 args: ValueRange{zeroTensor});
871 } else {
872 rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
873 op, args: TypeRange{outputType}, args: ValueRange{a, b, aZp, bZp},
874 args: ValueRange{zeroTensor});
875 }
876 return success();
877 }
878
879 SmallVector<ReassociationIndices> reassociation =
880 getMatMulBatchCollapseReassociation(rank: outputRank);
881 auto collapse = [&](Value value) {
882 auto type = cast<RankedTensorType>(Val: value.getType());
883 auto collapsedType =
884 tensor::CollapseShapeOp::inferCollapsedType(type, reassociation);
885 return tensor::CollapseShapeOp::create(builder&: rewriter, location: loc, resultType: collapsedType,
886 src: value, reassociation);
887 };
888
889 Value collapsedA = collapse(a);
890 Value collapsedB = collapse(b);
891 Value collapsedInit = collapse(zeroTensor);
892 auto collapsedOutputType = cast<RankedTensorType>(Val: collapsedInit.getType());
893 Value collapsedResult;
894
895 if (!hasNonzeroZeroPoint) {
896 collapsedResult =
897 linalg::BatchMatmulOp::create(
898 builder&: rewriter, location: loc, resultTensorTypes: TypeRange{collapsedOutputType},
899 inputs: ValueRange{collapsedA, collapsedB}, outputs: ValueRange{collapsedInit})
900 .getResult(i: 0);
901 } else {
902 collapsedResult = linalg::QuantizedBatchMatmulOp::create(
903 builder&: rewriter, location: loc, resultTensorTypes: TypeRange{collapsedOutputType},
904 inputs: ValueRange{collapsedA, collapsedB, aZp, bZp},
905 outputs: ValueRange{collapsedInit})
906 .getResult(i: 0);
907 }
908
909 Value result = tensor::ExpandShapeOp::create(
910 builder&: rewriter, location: loc, resultType: outputType, src: collapsedResult, reassociation,
911 outputShape: tensor::getMixedSizes(builder&: rewriter, loc, value: zeroTensor));
912 rewriter.replaceOp(op, newValues: result);
913 return success();
914 }
915};
916
917class MatMulTConverter : public OpConversionPattern<tosa::MatMulTOp> {
918public:
919 using OpConversionPattern<tosa::MatMulTOp>::OpConversionPattern;
920 LogicalResult
921 matchAndRewrite(tosa::MatMulTOp op, OpAdaptor adaptor,
922 ConversionPatternRewriter &rewriter) const final {
923 Location loc = op.getLoc();
924 auto aType = dyn_cast<RankedTensorType>(Val: adaptor.getA().getType());
925 auto bType = dyn_cast<RankedTensorType>(Val: adaptor.getB().getType());
926 auto outputType = dyn_cast<RankedTensorType>(Val: op.getType());
927 if (!aType || !bType || !outputType)
928 return rewriter.notifyMatchFailure(arg&: op, msg: "only supports ranked tensors");
929
930 FailureOr<int64_t> maybeAZp = op.getAZeroPoint();
931 FailureOr<int64_t> maybeBZp = op.getBZeroPoint();
932 if (failed(Result: maybeAZp))
933 return rewriter.notifyMatchFailure(
934 arg&: op, msg: "input A zero point cannot be statically determined");
935 if (failed(Result: maybeBZp))
936 return rewriter.notifyMatchFailure(
937 arg&: op, msg: "input B zero point cannot be statically determined");
938
939 const int64_t aZpVal = *maybeAZp;
940 const int64_t bZpVal = *maybeBZp;
941
942 if (aZpVal != 0 || bZpVal != 0)
943 return rewriter.notifyMatchFailure(
944 arg&: op, msg: "nonzero zero points require the generic MATMUL_T lowering");
945
946 const int64_t outputRank = outputType.getRank();
947 auto [a, b, zeroTensor] =
948 prepareMatMulOperands(rewriter, loc, a: adaptor.getA(), b: adaptor.getB(),
949 outputType, /*transposeB=*/true);
950
951 if (outputRank == 2) {
952 rewriter.replaceOpWithNewOp<linalg::MatmulTransposeBOp>(
953 op, args: TypeRange{outputType}, args: ValueRange{a, b}, args: ValueRange{zeroTensor});
954 return success();
955 }
956
957 if (outputRank == 3) {
958 rewriter.replaceOpWithNewOp<linalg::BatchMatmulTransposeBOp>(
959 op, args: TypeRange{outputType}, args: ValueRange{a, b}, args: ValueRange{zeroTensor});
960 return success();
961 }
962
963 SmallVector<ReassociationIndices> reassociation =
964 getMatMulBatchCollapseReassociation(rank: outputRank);
965 auto collapse = [&](Value value) {
966 auto type = cast<RankedTensorType>(Val: value.getType());
967 auto collapsedType =
968 tensor::CollapseShapeOp::inferCollapsedType(type, reassociation);
969 return tensor::CollapseShapeOp::create(builder&: rewriter, location: loc, resultType: collapsedType,
970 src: value, reassociation);
971 };
972 Value collapsedA = collapse(a);
973 Value collapsedB = collapse(b);
974 Value collapsedInit = collapse(zeroTensor);
975 auto collapsedOutputType = cast<RankedTensorType>(Val: collapsedInit.getType());
976 Value collapsedResult =
977 linalg::BatchMatmulTransposeBOp::create(
978 builder&: rewriter, location: loc, resultTensorTypes: TypeRange{collapsedOutputType},
979 inputs: ValueRange{collapsedA, collapsedB}, outputs: ValueRange{collapsedInit})
980 .getResult(i: 0);
981 Value result = tensor::ExpandShapeOp::create(
982 builder&: rewriter, location: loc, resultType: outputType, src: collapsedResult, reassociation,
983 outputShape: tensor::getMixedSizes(builder&: rewriter, loc, value: zeroTensor));
984 rewriter.replaceOp(op, newValues: result);
985 return success();
986 }
987};
988
989class MaxPool2dConverter : public OpConversionPattern<tosa::MaxPool2dOp> {
990public:
991 MaxPool2dConverter(const TypeConverter &typeConverter, MLIRContext *context,
992 bool allowNonFinites)
993 : OpConversionPattern(typeConverter, context),
994 allowNonFinites(allowNonFinites) {}
995
996 // Compute the dynamic output sizes of the maxpool operation.
997 static SmallVector<Value>
998 computeDynamicOutputSizes(tosa::MaxPool2dOp op, OpAdaptor adaptor,
999 ConversionPatternRewriter &rewriter) {
1000 TensorType resultTy = op.getType();
1001 Location loc = op.getLoc();
1002
1003 Value input = adaptor.getInput();
1004 ArrayRef<int64_t> kernel = op.getKernel();
1005 ArrayRef<int64_t> pad = op.getPad();
1006 ArrayRef<int64_t> stride = op.getStride();
1007
1008 SmallVector<Value> dynamicDims;
1009
1010 // Batch dimension
1011 if (resultTy.isDynamicDim(idx: 0))
1012 dynamicDims.push_back(Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: 0));
1013
1014 // Height/width dimensions
1015 for (int64_t dim : {1, 2}) {
1016 if (!resultTy.isDynamicDim(idx: dim))
1017 continue;
1018
1019 // Index into the attribute arrays
1020 int64_t index = dim - 1;
1021
1022 // Input height/width
1023 Value ihw = tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: dim);
1024
1025 // Kernel height/width
1026 Value khw = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: kernel[index]);
1027
1028 // Output height/width
1029 Value ohw = getConvOrPoolOutputDim(loc, inputDim: ihw, padBeforeAttr: pad[index * 2],
1030 padAfterAttr: pad[index * 2 + 1], kernelDim: khw, strideAttr: stride[index],
1031 /*dilationAttr=*/1, rewriter);
1032 dynamicDims.push_back(Elt: ohw);
1033 }
1034
1035 // Channel dimension
1036 if (resultTy.isDynamicDim(idx: 3))
1037 dynamicDims.push_back(Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: 3));
1038
1039 return dynamicDims;
1040 }
1041
1042 LogicalResult
1043 matchAndRewrite(tosa::MaxPool2dOp op, OpAdaptor adaptor,
1044 ConversionPatternRewriter &rewriter) const final {
1045 Location loc = op.getLoc();
1046 Value input = adaptor.getInput();
1047 ShapedType inputTy = cast<ShapedType>(Val: input.getType());
1048
1049 bool isUnsigned = op.getType().getElementType().isUnsignedInteger();
1050 ShapedType resultTy =
1051 getTypeConverter()->convertType<ShapedType>(t: op.getType());
1052 if (!resultTy)
1053 return rewriter.notifyMatchFailure(arg&: op, msg: "failed to convert type");
1054 Type resultETy = inputTy.getElementType();
1055 NanPropagationMode nanMode = op.getNanMode();
1056
1057 SmallVector<Value> dynamicDims =
1058 computeDynamicOutputSizes(op, adaptor, rewriter);
1059
1060 // Determine what the initial value needs to be for the max pool op.
1061 TypedAttr initialAttr;
1062 if (resultETy.isF32() || resultETy.isBF16() || resultETy.isF16()) {
1063 const llvm::fltSemantics &semantics =
1064 cast<FloatType>(Val&: resultETy).getFloatSemantics();
1065 if (llvm::APFloat::semanticsHasNaN(semantics))
1066 initialAttr = rewriter.getFloatAttr(
1067 type: resultETy,
1068 value: nanMode == NanPropagationMode::IGNORE
1069 ? APFloat::getNaN(Sem: semantics)
1070 : getFloatMinMaxIdentity(semantics,
1071 /*negative=*/true, allowNonFinites));
1072 }
1073
1074 else if (isUnsigned)
1075 initialAttr = rewriter.getIntegerAttr(
1076 type: resultETy, value: APInt::getZero(numBits: resultETy.getIntOrFloatBitWidth()));
1077 else if (isa<IntegerType>(Val: resultETy))
1078 initialAttr = rewriter.getIntegerAttr(
1079 type: resultETy,
1080 value: APInt::getSignedMinValue(numBits: resultETy.getIntOrFloatBitWidth()));
1081
1082 if (!initialAttr)
1083 return rewriter.notifyMatchFailure(
1084 arg&: op, msg: "Unsupported initial value for tosa.maxpool_2d op");
1085
1086 // Apply padding as necessary.
1087 llvm::SmallVector<int64_t> pad;
1088 pad.resize(N: 2, NV: 0);
1089 llvm::append_range(C&: pad, R: op.getPad());
1090 pad.resize(N: pad.size() + 2, NV: 0);
1091
1092 // The initial value doubles as the padding value. This is safe for every
1093 // seed above because the verifier rejects a pad as wide as the kernel on
1094 // any side, so no window consists purely of padding.
1095 Value paddedInput = applyPad(loc, input, pad, padAttr: initialAttr, rewriter);
1096
1097 Value initialValue = arith::ConstantOp::create(builder&: rewriter, location: loc, value: initialAttr);
1098
1099 ArrayRef<int64_t> kernel = op.getKernel();
1100 ArrayRef<int64_t> stride = op.getStride();
1101
1102 Attribute strideAttr = rewriter.getI64VectorAttr(values: stride);
1103 Attribute dilationAttr = rewriter.getI64VectorAttr(values: {1, 1});
1104
1105 // Create the linalg op that performs pooling.
1106 Value emptyTensor =
1107 tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: resultTy.getShape(),
1108 elementType: resultTy.getElementType(), dynamicSizes: dynamicDims);
1109
1110 Value filledEmptyTensor =
1111 linalg::FillOp::create(builder&: rewriter, location: loc, inputs: initialValue, outputs: emptyTensor)
1112 .result();
1113
1114 Value fakeWindowDims =
1115 tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: kernel, elementType: resultETy);
1116
1117 if (isUnsigned) {
1118 rewriter.replaceOpWithNewOp<linalg::PoolingNhwcMaxUnsignedOp>(
1119 op, args: ArrayRef<Type>{resultTy}, args: ValueRange{paddedInput, fakeWindowDims},
1120 args&: filledEmptyTensor, args&: strideAttr, args&: dilationAttr);
1121 return llvm::success();
1122 }
1123
1124 auto resultOp = linalg::PoolingNhwcMaxOp::create(
1125 builder&: rewriter, location: op->getLoc(), resultTensorTypes: ArrayRef<Type>{resultTy},
1126 inputs: ValueRange{paddedInput, fakeWindowDims}, outputs: filledEmptyTensor, strides: strideAttr,
1127 dilations: dilationAttr);
1128
1129 rewriter.replaceOp(op, newOp: resultOp);
1130
1131 // NaN propagation has no meaning for non floating point types.
1132 if (!isa<FloatType>(Val: getElementTypeOrSelf(type: inputTy)))
1133 return success();
1134
1135 // "PROPAGATE" mode matches the behaviour of the LinAlg named op, so no
1136 // compare and select materialization is required.
1137 //
1138 // In the case of "IGNORE" we need to insert a compare and select. Since
1139 // we've already produced a named op we will just take its body and modify
1140 // it to include the appropriate checks. A NaN accumulator represents a
1141 // window with no finite values yet. The first finite input replaces it;
1142 // input NaNs are ignored. An all-NaN window therefore remains NaN.
1143 if (nanMode == NanPropagationMode::IGNORE) {
1144 auto genericOp = linalg::GenericOp::create(
1145 builder&: rewriter, location: loc, resultTensorTypes: resultOp.getType(i: 0), inputs: resultOp.getInputs(),
1146 outputs: resultOp.getOutputs(), indexingMaps: resultOp.getIndexingMapsArray(),
1147 iteratorTypes: resultOp.getIteratorTypesArray(),
1148 odsArg5: [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
1149 IRMapping map;
1150 auto oldBlock = resultOp.getRegion().begin();
1151 auto oldArgs = oldBlock->getArguments();
1152 auto &oldMaxOp = *resultOp.getBlock()->begin();
1153 map.map(from&: oldArgs, to&: blockArgs);
1154 auto *newOp = opBuilder.clone(op&: oldMaxOp, mapper&: map);
1155 Value inputIsNaN =
1156 arith::CmpFOp::create(builder&: opBuilder, location: loc, predicate: arith::CmpFPredicate::UNO,
1157 lhs: blockArgs.front(), rhs: blockArgs.front());
1158 Value accumulatorIsNaN =
1159 arith::CmpFOp::create(builder&: opBuilder, location: loc, predicate: arith::CmpFPredicate::UNO,
1160 lhs: blockArgs.back(), rhs: blockArgs.back());
1161 Value finiteMaximum =
1162 arith::SelectOp::create(builder&: opBuilder, location: loc, condition: accumulatorIsNaN,
1163 true_value: blockArgs.front(), false_value: newOp->getResult(idx: 0));
1164 Value result = arith::SelectOp::create(
1165 builder&: opBuilder, location: loc, condition: inputIsNaN, true_value: blockArgs.back(), false_value: finiteMaximum);
1166 linalg::YieldOp::create(builder&: opBuilder, location: loc, values: result);
1167 });
1168 rewriter.replaceOp(op: resultOp, newOp: genericOp);
1169 }
1170
1171 return success();
1172 }
1173
1174private:
1175 bool allowNonFinites;
1176};
1177
1178class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
1179public:
1180 using OpRewritePattern<tosa::AvgPool2dOp>::OpRewritePattern;
1181
1182 LogicalResult matchAndRewrite(tosa::AvgPool2dOp op,
1183 PatternRewriter &rewriter) const final {
1184 Location loc = op.getLoc();
1185 Value input = op.getInput();
1186 ShapedType inputTy = cast<ShapedType>(Val: input.getType());
1187 Type inElementTy = inputTy.getElementType();
1188
1189 ShapedType resultTy = cast<ShapedType>(Val: op.getType());
1190 Type resultETy = cast<ShapedType>(Val: op.getType()).getElementType();
1191
1192 Type accETy = op.getAccType();
1193 ShapedType accTy = resultTy.clone(elementType: accETy);
1194
1195 auto dynamicDimsOr =
1196 checkHasDynamicBatchDims(rewriter, op, params: {input, op.getOutput()});
1197 if (!dynamicDimsOr.has_value())
1198 return failure();
1199 SmallVector<Value> dynamicDims = *dynamicDimsOr;
1200
1201 FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
1202 FailureOr<int64_t> maybeOZp = op.getOutputZeroPoint();
1203 if (failed(Result: maybeIZp))
1204 return rewriter.notifyMatchFailure(
1205 arg&: op, msg: "input zero point could not be statically determined");
1206 if (failed(Result: maybeOZp))
1207 return rewriter.notifyMatchFailure(
1208 arg&: op, msg: "output zero point could not be statically determined");
1209
1210 const int64_t inputZpVal = *maybeIZp;
1211 const int64_t outputZpVal = *maybeOZp;
1212
1213 // Apply padding as necessary.
1214 llvm::SmallVector<int64_t> pad;
1215 pad.resize(N: 2, NV: 0);
1216 llvm::append_range(C&: pad, R: op.getPad());
1217 pad.resize(N: pad.size() + 2, NV: 0);
1218 TypedAttr padAttr = rewriter.getZeroAttr(type: inElementTy);
1219 // Unsupported element type
1220 if (!padAttr)
1221 return failure();
1222 Value paddedInput = applyPad(loc, input, pad, padAttr, rewriter);
1223
1224 auto initialAttr = rewriter.getZeroAttr(type: accETy);
1225 Value initialValue = arith::ConstantOp::create(builder&: rewriter, location: loc, value: initialAttr);
1226
1227 ArrayRef<int64_t> kernel = op.getKernel();
1228 ArrayRef<int64_t> stride = op.getStride();
1229
1230 Attribute strideAttr = rewriter.getI64VectorAttr(values: stride);
1231 Attribute dilationAttr = rewriter.getI64VectorAttr(values: {1, 1});
1232
1233 // Create the linalg op that performs pooling.
1234 Value poolEmptyTensor = tensor::EmptyOp::create(
1235 builder&: rewriter, location: loc, staticShape: accTy.getShape(), elementType: accETy, dynamicSizes: dynamicDims);
1236
1237 Value filledEmptyTensor =
1238 linalg::FillOp::create(builder&: rewriter, location: loc, inputs: ValueRange{initialValue},
1239 outputs: ValueRange{poolEmptyTensor})
1240 .result();
1241
1242 Value fakeWindowDims =
1243 tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: kernel, elementType: accETy);
1244
1245 // Sum across the pooled region.
1246 Value poolingOp = linalg::PoolingNhwcSumOp::create(
1247 builder&: rewriter, location: loc, resultTensorTypes: ArrayRef<Type>{accTy},
1248 inputs: ValueRange{paddedInput, fakeWindowDims},
1249 outputs: filledEmptyTensor, strides: strideAttr, dilations: dilationAttr)
1250 .getResult(i: 0);
1251
1252 // Normalize the summed value by the number of elements grouped in each
1253 // pool.
1254 Value iH = tensor::DimOp::create(builder&: rewriter, location: loc, source: poolingOp, index: 1);
1255 Value iW = tensor::DimOp::create(builder&: rewriter, location: loc, source: poolingOp, index: 2);
1256
1257 auto one = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 1);
1258 iH = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: iH, rhs: one);
1259 iW = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: iW, rhs: one);
1260
1261 Value genericEmptyTensor = tensor::EmptyOp::create(
1262 builder&: rewriter, location: loc, staticShape: resultTy.getShape(), elementType: resultETy, dynamicSizes: dynamicDims);
1263
1264 auto affineMap = rewriter.getMultiDimIdentityMap(rank: resultTy.getRank());
1265 auto genericOp = linalg::GenericOp::create(
1266 builder&: rewriter, location: loc, resultTensorTypes: ArrayRef<Type>({resultTy}), inputs: ValueRange{poolingOp},
1267 outputs: ValueRange{genericEmptyTensor},
1268 indexingMaps: ArrayRef<AffineMap>({affineMap, affineMap}),
1269 iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()),
1270 odsArg5: [&](OpBuilder &b, Location loc, ValueRange args) {
1271 auto zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
1272
1273 // Determines what the portion of valid input is covered by the
1274 // kernel.
1275 auto padFn = [&](Value valid, Value pos, int64_t pad) -> Value {
1276 if (pad == 0)
1277 return valid;
1278
1279 auto padVal = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: pad);
1280 Value dpos = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: pos, rhs: padVal);
1281
1282 Value offset = arith::MinSIOp::create(builder&: rewriter, location: loc, lhs: dpos, rhs: zero);
1283 return arith::AddIOp::create(builder&: rewriter, location: loc, lhs: valid, rhs: offset)
1284 ->getResult(idx: 0);
1285 };
1286
1287 auto coverageFn = [&](int64_t i, Value isize) -> Value {
1288 Value strideVal =
1289 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: stride[i - 1]);
1290 Value val =
1291 arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: kernel[i - 1]);
1292
1293 // Find the position relative to the input tensor's ends.
1294 Value left = linalg::IndexOp::create(builder&: rewriter, location: loc, dim: i);
1295 Value right = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: isize, rhs: left);
1296 left = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: left, rhs: strideVal);
1297 right = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: right, rhs: strideVal);
1298
1299 // Determine how much padding was included.
1300 val = padFn(val, left, pad[i * 2]);
1301 val = padFn(val, right, pad[i * 2 + 1]);
1302 return arith::MaxSIOp::create(builder&: rewriter, location: loc, lhs: one, rhs: val);
1303 };
1304
1305 // Compute the indices from either end.
1306 Value kH3 = coverageFn(1, iH);
1307 Value kW3 = coverageFn(2, iW);
1308
1309 // Compute the total number of elements and normalize.
1310 auto count = arith::IndexCastOp::create(
1311 builder&: rewriter, location: loc, out: rewriter.getI32Type(),
1312 in: arith::MulIOp::create(builder&: rewriter, location: loc, lhs: kH3, rhs: kW3));
1313
1314 // Divide by the number of summed values. For floats this is just
1315 // a div however for quantized values input normalization had
1316 // to be applied.
1317 Value poolVal = args[0];
1318 if (isa<FloatType>(Val: accETy)) {
1319 auto countF = arith::SIToFPOp::create(builder&: rewriter, location: loc, out: accETy, in: count);
1320 poolVal = arith::DivFOp::create(builder&: rewriter, location: loc, lhs: poolVal, rhs: countF)
1321 ->getResult(idx: 0);
1322 if (accETy.getIntOrFloatBitWidth() >
1323 resultETy.getIntOrFloatBitWidth())
1324 poolVal =
1325 arith::TruncFOp::create(builder&: rewriter, location: loc, out: resultETy, in: poolVal);
1326 } else {
1327
1328 // If we have quantization information we need to apply an offset
1329 // for the input zp value.
1330 if (inputZpVal != 0) {
1331 auto inputZp = arith::ConstantOp::create(
1332 builder&: rewriter, location: loc, value: b.getIntegerAttr(type: accETy, value: inputZpVal));
1333 Value offset =
1334 arith::MulIOp::create(builder&: rewriter, location: loc, result: accETy, lhs: count, rhs: inputZp);
1335 poolVal =
1336 arith::SubIOp::create(builder&: rewriter, location: loc, result: accETy, lhs: poolVal, rhs: offset);
1337 }
1338
1339 // Compute: k = 32 - count_leading_zeros(value - 1)
1340 Value one32 = arith::ConstantOp::create(
1341 builder&: rewriter, location: loc, value: rewriter.getI32IntegerAttr(value: 1));
1342 Value thirtyTwo32 = arith::ConstantOp::create(
1343 builder&: rewriter, location: loc, value: rewriter.getI32IntegerAttr(value: 32));
1344
1345 Value countSubOne =
1346 arith::SubIOp::create(builder&: rewriter, location: loc, lhs: count, rhs: one32);
1347 Value leadingZeros =
1348 math::CountLeadingZerosOp::create(builder&: rewriter, location: loc, operand: countSubOne);
1349 Value k =
1350 arith::SubIOp::create(builder&: rewriter, location: loc, lhs: thirtyTwo32, rhs: leadingZeros);
1351
1352 // Compute: numerator = ((1 << 30) + 1) << k
1353 Value k64 =
1354 arith::ExtUIOp::create(builder&: rewriter, location: loc, out: rewriter.getI64Type(), in: k);
1355 Value thirtyShiftPlusOne = arith::ConstantOp::create(
1356 builder&: rewriter, location: loc, value: rewriter.getI64IntegerAttr(value: (1 << 30) + 1));
1357 Value numerator =
1358 arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: thirtyShiftPlusOne, rhs: k64);
1359
1360 // Compute: scale.multiplier = numerator / value;
1361 Value count64 = arith::ExtUIOp::create(
1362 builder&: rewriter, location: loc, out: rewriter.getI64Type(), in: count);
1363 Value multiplier =
1364 arith::DivUIOp::create(builder&: rewriter, location: loc, lhs: numerator, rhs: count64);
1365 multiplier = arith::TruncIOp::create(
1366 builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: multiplier);
1367
1368 // Compute: scale.shift = 30 + k
1369 Value k8 =
1370 arith::TruncIOp::create(builder&: rewriter, location: loc, out: rewriter.getI8Type(), in: k);
1371 Value thirty8 = arith::ConstantOp::create(
1372 builder&: rewriter, location: loc, value: rewriter.getI8IntegerAttr(value: 30));
1373 Value shift = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: k8, rhs: thirty8);
1374
1375 auto roundingAttr = RoundingModeAttr::get(
1376 context: rewriter.getContext(), value: RoundingMode::SINGLE_ROUND);
1377
1378 auto scaled = tosa::ApplyScaleOp::create(
1379 builder&: rewriter, location: loc, output: rewriter.getI32Type(), value: poolVal,
1380 multiplier, shift, rounding_mode: roundingAttr)
1381 .getResult();
1382
1383 // If we have quantization information we need to apply output
1384 // zeropoint.
1385 if (outputZpVal != 0) {
1386 auto outputZp = arith::ConstantOp::create(
1387 builder&: rewriter, location: loc,
1388 value: b.getIntegerAttr(type: scaled.getType(), value: outputZpVal));
1389 scaled = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: scaled, rhs: outputZp)
1390 .getResult();
1391 }
1392
1393 // Apply Clip.
1394 int64_t outBitwidth = resultETy.getIntOrFloatBitWidth();
1395
1396 auto min = arith::ConstantIntOp::create(
1397 builder&: rewriter, location: loc, type: accETy,
1398 value: APInt::getSignedMinValue(numBits: outBitwidth).getSExtValue());
1399 auto max = arith::ConstantIntOp::create(
1400 builder&: rewriter, location: loc, type: accETy,
1401 value: APInt::getSignedMaxValue(numBits: outBitwidth).getSExtValue());
1402 auto clamp = clampIntHelper(loc, arg: scaled, min, max, rewriter,
1403 /*isUnsigned=*/false);
1404
1405 poolVal = clamp;
1406 // Convert type.
1407 if (resultETy != clamp.getType()) {
1408 poolVal =
1409 arith::TruncIOp::create(builder&: rewriter, location: loc, out: resultETy, in: poolVal);
1410 }
1411 }
1412
1413 linalg::YieldOp::create(builder&: rewriter, location: loc, values: poolVal);
1414 });
1415
1416 rewriter.replaceOp(op, newValues: genericOp.getResult(i: 0));
1417 return success();
1418 }
1419};
1420
1421class TransposeConverter : public OpRewritePattern<tosa::TransposeOp> {
1422public:
1423 using OpRewritePattern<tosa::TransposeOp>::OpRewritePattern;
1424
1425 LogicalResult matchAndRewrite(tosa::TransposeOp op,
1426 PatternRewriter &rewriter) const final {
1427 const llvm::ArrayRef<int32_t> constantPerms = op.getPerms();
1428
1429 Location loc = op.getLoc();
1430 // The verifier should have made sure we have a valid TOSA permutation
1431 // tensor. isPermutationVector doesn't actually check the TOSA perms we
1432 // expect.
1433 SmallVector<OpFoldResult> inputSizes =
1434 tensor::getMixedSizes(builder&: rewriter, loc, value: op.getInput1());
1435 auto permutedSizes =
1436 applyTOSAPermutation<OpFoldResult>(input: inputSizes, perms: constantPerms);
1437
1438 auto permutedInit =
1439 tensor::EmptyOp::create(builder&: rewriter, location: loc, sizes: permutedSizes,
1440 elementType: op.getInput1().getType().getElementType());
1441 rewriter.replaceOpWithNewOp<linalg::TransposeOp>(
1442 op, args: op.getInput1(), args&: permutedInit,
1443 args: llvm::map_to_vector(C: constantPerms,
1444 F: [](int32_t v) -> int64_t { return v; }));
1445 return success();
1446 }
1447};
1448} // namespace
1449
1450void mlir::tosa::populateTosaToLinalgNamedConversionPatterns(
1451 const TypeConverter &converter, RewritePatternSet *patterns,
1452 const TosaToLinalgNamedOptions &options) {
1453 if (options.preferConv2DKernelLayoutHWCF) {
1454 patterns->add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcHwcfOp,
1455 linalg::Conv2DNhwcHwcfQOp>>(
1456 arg: patterns->getContext());
1457 } else {
1458 patterns->add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcFhwcOp,
1459 linalg::Conv2DNhwcFhwcQOp>>(
1460 arg: patterns->getContext());
1461 }
1462 patterns->add<
1463 // clang-format off
1464 ConvConverter<tosa::Conv3DOp, linalg::Conv3DNdhwcDhwcfOp, linalg::Conv3DNdhwcDhwcfQOp>,
1465 DepthwiseConvConverter,
1466 MatMulConverter,
1467 MatMulTConverter,
1468 AvgPool2dConverter,
1469 TransposeConverter
1470 >(arg: patterns->getContext());
1471
1472 patterns->add<
1473 ElementwiseConverter<tosa::AbsOp>,
1474 ElementwiseConverter<tosa::CeilOp>,
1475 ElementwiseConverter<tosa::FloorOp>,
1476 ElementwiseConverter<tosa::ExpOp>,
1477 ElementwiseConverter<tosa::LogOp>,
1478 ElementwiseConverter<tosa::RsqrtOp>,
1479 ElementwiseConverter<tosa::SinOp>,
1480 ElementwiseConverter<tosa::CosOp>,
1481 ElementwiseConverter<tosa::TanhOp>,
1482 ElementwiseConverter<tosa::ErfOp>,
1483 ElementwiseConverter<tosa::ReciprocalOp>,
1484 ElementwiseConverter<tosa::PowOp>,
1485 ElementwiseConverter<tosa::AddOp>,
1486 ElementwiseConverter<tosa::SubOp>,
1487 ElementwiseConverter<tosa::IntDivOp>,
1488 ElementwiseConverter<tosa::MulOp>,
1489 ElementwiseConverter<tosa::NegateOp>,
1490 ElementwiseConverter<tosa::MaximumOp>,
1491 ElementwiseConverter<tosa::MinimumOp>,
1492 ElementwiseConverter<tosa::SelectOp>
1493 >(arg: patterns->getContext());
1494
1495 patterns->add<
1496 MaxPool2dConverter
1497 >(arg: converter, args: patterns->getContext(), args: options.allowNonFinites);
1498 // clang-format on
1499}
1500