1//===- TosaToLinalg.cpp - Lowering Tosa to Linalg Dialect -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// These rewriters lower from the Tosa to the Linalg dialect.
10//
11//===----------------------------------------------------------------------===//
12
13#include "mlir/Conversion/TosaToLinalg/TosaToLinalg.h"
14#include "TosaToLinalgUtils.h"
15#include "mlir/Dialect/Arith/IR/Arith.h"
16#include "mlir/Dialect/Arith/Utils/Utils.h"
17#include "mlir/Dialect/Index/IR/IndexOps.h"
18#include "mlir/Dialect/Linalg/IR/Linalg.h"
19#include "mlir/Dialect/Math/IR/Math.h"
20#include "mlir/Dialect/SCF/IR/SCF.h"
21#include "mlir/Dialect/Tensor/IR/Tensor.h"
22#include "mlir/Dialect/Tosa/IR/TosaOps.h"
23#include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
24#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
25#include "mlir/Dialect/Utils/StaticValueUtils.h"
26#include "mlir/IR/Matchers.h"
27#include "mlir/IR/OpDefinition.h"
28#include "mlir/IR/PatternMatch.h"
29#include "mlir/Transforms/DialectConversion.h"
30#include "llvm/ADT/STLExtras.h"
31#include "llvm/ADT/Sequence.h"
32#include "llvm/ADT/SmallVectorExtras.h"
33
34#include <type_traits>
35
36using namespace mlir;
37using namespace mlir::tosa;
38
39template <typename OpTy>
40static OpTy createWithDefaultProperties(OpBuilder &builder, Location loc,
41 TypeRange resultTypes,
42 ValueRange operands) {
43 typename OpTy::Properties properties{};
44 OpTy::populateDefaultProperties(
45 OperationName(OpTy::getOperationName(), builder.getContext()),
46 properties);
47 return OpTy::create(builder, loc, resultTypes, operands, properties,
48 /*discardableAttributes=*/{});
49}
50
51// Helper function to materialize the semantically correct compare and select
52// operations given a binary operation with a specific NaN propagation mode.
53//
54// In the case of "PROPAGATE" semantics no compare and selection is required and
55// this function does nothing.
56//
57// In the case of "IGNORE" semantics this function materializes a comparison of
58// the current operands to the op which will return true for any NaN
59// argument and then selects between the non-NaN operation argument and the
60// calculated result based on whether the lhs or rhs is NaN or not. In pseudo
61// code:
62//
63// In the case that the op is operating on non floating point types we ignore
64// the attribute completely, this is consistent with the TOSA spec which has
65// the following wording: "This attribute is ignored by non floating-point
66// types."
67//
68// binary<op>(lhs, rhs):
69// result = op(lhs, rhs)
70// if lhs == NaN return rhs
71// if rhs == NaN return lhs
72// return result
73template <typename OpTy>
74static Value
75materializeBinaryNanCheckIfRequired(OpTy op, PatternRewriter &rewriter,
76 Value lhs, Value rhs, Value result) {
77 // NaN propagation has no meaning for non floating point types.
78 if (!isa<FloatType>(Val: getElementTypeOrSelf(val: lhs)))
79 return result;
80
81 auto nanMode = op.getNanMode();
82 if (nanMode == NanPropagationMode::PROPAGATE)
83 return result;
84
85 // Unordered comparison of NaN against itself will always return true.
86 Value lhsIsNaN = arith::CmpFOp::create(rewriter, op.getLoc(),
87 arith::CmpFPredicate::UNO, lhs, lhs);
88 Value rhsIsNaN = arith::CmpFOp::create(rewriter, op.getLoc(),
89 arith::CmpFPredicate::UNO, rhs, rhs);
90 Value rhsOrResult =
91 arith::SelectOp::create(rewriter, op.getLoc(), lhsIsNaN, rhs, result);
92 return arith::SelectOp::create(rewriter, op.getLoc(), rhsIsNaN, lhs,
93 rhsOrResult);
94}
95
96static Value createLinalgBodyCalculationForElementwiseOp(
97 Operation *op, ValueRange args, ArrayRef<Type> resultTypes,
98 ConversionPatternRewriter &rewriter) {
99 Location loc = op->getLoc();
100 auto elementTy =
101 cast<ShapedType>(Val: op->getOperand(idx: 0).getType()).getElementType();
102
103 // tosa::AbsOp
104 if (isa<tosa::AbsOp>(Val: op) && isa<FloatType>(Val: elementTy))
105 return createWithDefaultProperties<math::AbsFOp>(builder&: rewriter, loc, resultTypes,
106 operands: args);
107
108 if (isa<tosa::AbsOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
109 auto zero = arith::ConstantOp::create(builder&: rewriter, location: loc,
110 value: rewriter.getZeroAttr(type: elementTy));
111 auto neg = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: zero, rhs: args[0]);
112 return arith::MaxSIOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: neg);
113 }
114
115 // tosa::AddOp
116 if (isa<tosa::AddOp>(Val: op) && isa<FloatType>(Val: elementTy))
117 return createWithDefaultProperties<arith::AddFOp>(builder&: rewriter, loc,
118 resultTypes, operands: args);
119
120 if (isa<tosa::AddOp>(Val: op) && isa<IntegerType>(Val: elementTy))
121 return createWithDefaultProperties<arith::AddIOp>(builder&: rewriter, loc,
122 resultTypes, operands: args);
123
124 // tosa::SubOp
125 if (isa<tosa::SubOp>(Val: op) && isa<FloatType>(Val: elementTy))
126 return createWithDefaultProperties<arith::SubFOp>(builder&: rewriter, loc,
127 resultTypes, operands: args);
128
129 if (isa<tosa::SubOp>(Val: op) && isa<IntegerType>(Val: elementTy))
130 return createWithDefaultProperties<arith::SubIOp>(builder&: rewriter, loc,
131 resultTypes, operands: args);
132
133 // tosa::IntDivOp
134 if (isa<tosa::IntDivOp>(Val: op) && isa<IntegerType>(Val: elementTy))
135 return createWithDefaultProperties<arith::DivSIOp>(builder&: rewriter, loc,
136 resultTypes, operands: args);
137
138 // tosa::ReciprocalOp
139 if (isa<tosa::ReciprocalOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
140 auto one =
141 arith::ConstantOp::create(builder&: rewriter, location: loc, value: FloatAttr::get(type: elementTy, value: 1));
142 return arith::DivFOp::create(builder&: rewriter, location: loc, lhs: one, rhs: args[0]);
143 }
144
145 // tosa::MulOp
146 if (isa<tosa::MulOp>(Val: op)) {
147 auto shiftVal = cast<tosa::MulOp>(Val: op).getShift();
148 DenseElementsAttr shiftElem;
149 bool shiftIsConstant = true;
150 int32_t shift = 0;
151 if (matchPattern(value: shiftVal, pattern: m_Constant(bind_value: &shiftElem)))
152 shift = shiftElem.getValues<IntegerAttr>()[0].getInt();
153 else
154 shiftIsConstant = false;
155
156 if (isa<FloatType>(Val: elementTy)) {
157 if (shift != 0) {
158 (void)rewriter.notifyMatchFailure(arg&: op,
159 msg: "Cannot have shift value for float");
160 return nullptr;
161 }
162 return arith::MulFOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
163 }
164
165 if (isa<IntegerType>(Val: elementTy)) {
166 Value a = args[0];
167 Value b = args[1];
168
169 if (shift > 0 || !shiftIsConstant) {
170 Value shiftConst;
171 if (shiftIsConstant)
172 shiftConst = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: shift,
173 /*bitwidth=*/width: 8);
174
175 if (!a.getType().isInteger(width: 32))
176 a = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: a);
177
178 if (!b.getType().isInteger(width: 32))
179 b = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: b);
180
181 auto shiftAmount = shiftIsConstant ? shiftConst : args[2];
182 auto roundingAttr = RoundingModeAttr::get(context: rewriter.getContext(),
183 value: RoundingMode::SINGLE_ROUND);
184 auto result =
185 tosa::ApplyScaleOp::create(builder&: rewriter, location: loc, output: rewriter.getI32Type(), value: a,
186 multiplier: b, shift: shiftAmount, rounding_mode: roundingAttr);
187
188 return result;
189 }
190
191 int aWidth = a.getType().getIntOrFloatBitWidth();
192 int bWidth = b.getType().getIntOrFloatBitWidth();
193 int cWidth = resultTypes[0].getIntOrFloatBitWidth();
194
195 if (aWidth < cWidth)
196 a = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: resultTypes[0], in: a);
197 if (bWidth < cWidth)
198 b = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: resultTypes[0], in: b);
199
200 return arith::MulIOp::create(builder&: rewriter, location: loc, resultTypes, lhs: a, rhs: b);
201 }
202 }
203
204 // tosa::NegateOp
205 if (isa<tosa::NegateOp>(Val: op)) {
206 auto negate = cast<tosa::NegateOp>(Val: op);
207
208 int64_t inZp = 0, outZp = 0;
209 FailureOr<int64_t> maybeInZp = negate.getInput1ZeroPoint();
210 FailureOr<int64_t> maybeOutZp = negate.getOutputZeroPoint();
211 bool hasInZp = !failed(Result: maybeInZp);
212 bool hasOutZp = !failed(Result: maybeOutZp);
213 if (hasInZp)
214 inZp = *maybeInZp;
215 if (hasOutZp)
216 outZp = *maybeOutZp;
217
218 if (isa<FloatType>(Val: elementTy))
219 return arith::NegFOp::create(builder&: rewriter, location: loc, resultTypes, operand: args[0]);
220
221 if (isa<IntegerType>(Val: elementTy)) {
222 Value zpAddValue;
223 Type intermediateType;
224 // Compute the maximum value that can occur in the intermediate buffer.
225 const int32_t inputBitWidth = elementTy.getIntOrFloatBitWidth();
226 int intermediateBitWidth = 64;
227
228 if (hasInZp && hasOutZp) {
229 // Compute the maximum value that can occur in the intermediate buffer.
230 // Zero points can only be non-zero for int8, so their sum fits in
231 // int64_t. The bound is computed with 128 bits so that it cannot
232 // overflow for 64-bit inputs.
233 const int64_t zpAdd = inZp + outZp;
234 const APInt maxValue =
235 APInt::getSignedMaxValue(numBits: inputBitWidth).sext(width: 128) +
236 std::abs(i: zpAdd) + 1;
237
238 // Convert that maximum value into the maximum bitwidth needed to
239 // represent it.
240 if (maxValue.sle(RHS: APInt::getSignedMaxValue(numBits: 16).getSExtValue())) {
241 intermediateBitWidth = 16;
242 } else if (maxValue.sle(RHS: APInt::getSignedMaxValue(numBits: 32).getSExtValue())) {
243 intermediateBitWidth = 32;
244 }
245
246 intermediateType = rewriter.getIntegerType(width: intermediateBitWidth);
247 zpAddValue = arith::ConstantOp::create(
248 builder&: rewriter, location: loc, value: rewriter.getIntegerAttr(type: intermediateType, value: zpAdd));
249 } else {
250 intermediateType = rewriter.getIntegerType(width: intermediateBitWidth);
251 Value arg1 = args[1];
252 Value arg2 = args[2];
253 // Avoid verifier-invalid no-op sign-extends; only widen when needed.
254 if (arg1.getType() != intermediateType)
255 arg1 = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: intermediateType, in: arg1);
256 if (arg2.getType() != intermediateType)
257 arg2 = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: intermediateType, in: arg2);
258 zpAddValue =
259 arith::AddIOp::create(builder&: rewriter, location: loc, result: intermediateType, lhs: arg1, rhs: arg2);
260 }
261
262 // The negation can be applied by doing:
263 // outputValue = inZp + outZp - inputValue
264 Value ext = args[0];
265 if (ext.getType() != intermediateType)
266 ext = arith::ExtSIOp::create(builder&: rewriter, location: loc, out: intermediateType, in: ext);
267 auto sub = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: zpAddValue, rhs: ext);
268
269 // Clamp to the negation range.
270 Value min = arith::ConstantIntOp::create(
271 builder&: rewriter, location: loc, type: intermediateType,
272 value: APInt::getSignedMinValue(numBits: inputBitWidth).getSExtValue());
273 Value max = arith::ConstantIntOp::create(
274 builder&: rewriter, location: loc, type: intermediateType,
275 value: APInt::getSignedMaxValue(numBits: inputBitWidth).getSExtValue());
276 auto clamp = clampIntHelper(loc, arg: sub, min, max, rewriter, isUnsigned: false);
277
278 // Truncate to the final value, skipping no-op trunci when widths match.
279 if (clamp.getType() == elementTy)
280 return clamp;
281 return arith::TruncIOp::create(builder&: rewriter, location: loc, out: elementTy, in: clamp);
282 }
283 }
284
285 // tosa::BitwiseAndOp
286 if (isa<tosa::BitwiseAndOp>(Val: op) && isa<IntegerType>(Val: elementTy))
287 return arith::AndIOp::create(builder&: rewriter, location: loc, resultTypes, operands: args);
288
289 // tosa::BitwiseOrOp
290 if (isa<tosa::BitwiseOrOp>(Val: op) && isa<IntegerType>(Val: elementTy))
291 return arith::OrIOp::create(builder&: rewriter, location: loc, resultTypes, operands: args);
292
293 // tosa::BitwiseNotOp
294 if (isa<tosa::BitwiseNotOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
295 auto allOnesAttr = rewriter.getIntegerAttr(
296 type: elementTy, value: APInt::getAllOnes(numBits: elementTy.getIntOrFloatBitWidth()));
297 auto allOnes = arith::ConstantOp::create(builder&: rewriter, location: loc, value: allOnesAttr);
298 return arith::XOrIOp::create(builder&: rewriter, location: loc, resultTypes, lhs: args[0], rhs: allOnes);
299 }
300
301 // tosa::BitwiseXOrOp
302 if (isa<tosa::BitwiseXorOp>(Val: op) && isa<IntegerType>(Val: elementTy))
303 return arith::XOrIOp::create(builder&: rewriter, location: loc, resultTypes, operands: args);
304
305 // tosa::LogicalLeftShiftOp
306 if (isa<tosa::LogicalLeftShiftOp>(Val: op) && isa<IntegerType>(Val: elementTy))
307 return createWithDefaultProperties<arith::ShLIOp>(builder&: rewriter, loc,
308 resultTypes, operands: args);
309
310 // tosa::LogicalRightShiftOp
311 if (isa<tosa::LogicalRightShiftOp>(Val: op) && isa<IntegerType>(Val: elementTy))
312 return createWithDefaultProperties<arith::ShRUIOp>(builder&: rewriter, loc,
313 resultTypes, operands: args);
314
315 // tosa::ArithmeticRightShiftOp
316 if (isa<tosa::ArithmeticRightShiftOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
317 auto result = createWithDefaultProperties<arith::ShRSIOp>(
318 builder&: rewriter, loc, resultTypes, operands: args);
319 bool round = cast<tosa::ArithmeticRightShiftOp>(Val: op).getRound();
320 if (!round) {
321 return result;
322 }
323
324 Type i1Ty = IntegerType::get(context: rewriter.getContext(), /*width=*/1);
325 auto one = arith::ConstantOp::create(builder&: rewriter, location: loc,
326 value: IntegerAttr::get(type: elementTy, value: 1));
327 auto zero = arith::ConstantOp::create(builder&: rewriter, location: loc,
328 value: IntegerAttr::get(type: elementTy, value: 0));
329 auto i1zero =
330 arith::ConstantOp::create(builder&: rewriter, location: loc, value: IntegerAttr::get(type: i1Ty, value: 0));
331 auto i1one =
332 arith::ConstantOp::create(builder&: rewriter, location: loc, value: IntegerAttr::get(type: i1Ty, value: 1));
333
334 // Checking that input2 != 0
335 auto shiftValueGreaterThanZero = arith::CmpIOp::create(
336 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::sgt, lhs: args[1], rhs: zero);
337
338 // Checking for the last bit of input1 to be 1
339 auto subtract =
340 arith::SubIOp::create(builder&: rewriter, location: loc, resultTypes, lhs: args[1], rhs: one);
341 auto shifted =
342 arith::ShRSIOp::create(builder&: rewriter, location: loc, resultTypes, lhs: args[0], rhs: subtract)
343 ->getResults();
344 auto truncated = createWithDefaultProperties<arith::TruncIOp>(
345 builder&: rewriter, loc, resultTypes: TypeRange{i1Ty}, operands: shifted);
346 auto isInputOdd =
347 arith::AndIOp::create(builder&: rewriter, location: loc, result: i1Ty, lhs: truncated, rhs: i1one);
348 // shifted, truncated, isInputOdd can be poison when input2 is 0.
349 auto shouldRound = arith::SelectOp::create(
350 builder&: rewriter, location: loc, result: i1Ty, condition: shiftValueGreaterThanZero, true_value: isInputOdd, false_value: i1zero);
351 auto extended =
352 arith::ExtUIOp::create(builder&: rewriter, location: loc, resultTypes, in: shouldRound);
353 return arith::AddIOp::create(builder&: rewriter, location: loc, resultTypes, lhs: result, rhs: extended);
354 }
355
356 // tosa::ClzOp
357 if (isa<tosa::ClzOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
358 return math::CountLeadingZerosOp::create(builder&: rewriter, location: loc, result: elementTy, operand: args[0]);
359 }
360
361 // tosa::LogicalAnd
362 if (isa<tosa::LogicalAndOp>(Val: op) && elementTy.isInteger(width: 1))
363 return arith::AndIOp::create(builder&: rewriter, location: loc, resultTypes, operands: args);
364
365 // tosa::LogicalNot
366 if (isa<tosa::LogicalNotOp>(Val: op) && elementTy.isInteger(width: 1)) {
367 auto one = arith::ConstantOp::create(builder&: rewriter, location: loc,
368 value: rewriter.getIntegerAttr(type: elementTy, value: 1));
369 return arith::XOrIOp::create(builder&: rewriter, location: loc, resultTypes, lhs: args[0], rhs: one);
370 }
371
372 // tosa::LogicalOr
373 if (isa<tosa::LogicalOrOp>(Val: op) && elementTy.isInteger(width: 1))
374 return arith::OrIOp::create(builder&: rewriter, location: loc, resultTypes, operands: args);
375
376 // tosa::LogicalXor
377 if (isa<tosa::LogicalXorOp>(Val: op) && elementTy.isInteger(width: 1))
378 return arith::XOrIOp::create(builder&: rewriter, location: loc, resultTypes, operands: args);
379
380 // tosa::PowOp
381 if (isa<tosa::PowOp>(Val: op) && isa<FloatType>(Val: elementTy))
382 return createWithDefaultProperties<mlir::math::PowFOp>(builder&: rewriter, loc,
383 resultTypes, operands: args);
384
385 // tosa::RsqrtOp
386 if (isa<tosa::RsqrtOp>(Val: op) && isa<FloatType>(Val: elementTy))
387 return createWithDefaultProperties<mlir::math::RsqrtOp>(builder&: rewriter, loc,
388 resultTypes, operands: args);
389
390 // tosa::LogOp
391 if (isa<tosa::LogOp>(Val: op) && isa<FloatType>(Val: elementTy))
392 return createWithDefaultProperties<mlir::math::LogOp>(builder&: rewriter, loc,
393 resultTypes, operands: args);
394
395 // tosa::ExpOp
396 if (isa<tosa::ExpOp>(Val: op) && isa<FloatType>(Val: elementTy))
397 return createWithDefaultProperties<mlir::math::ExpOp>(builder&: rewriter, loc,
398 resultTypes, operands: args);
399
400 // tosa::SinOp
401 if (isa<tosa::SinOp>(Val: op) && isa<FloatType>(Val: elementTy))
402 return createWithDefaultProperties<mlir::math::SinOp>(builder&: rewriter, loc,
403 resultTypes, operands: args);
404
405 // tosa::CosOp
406 if (isa<tosa::CosOp>(Val: op) && isa<FloatType>(Val: elementTy))
407 return createWithDefaultProperties<mlir::math::CosOp>(builder&: rewriter, loc,
408 resultTypes, operands: args);
409
410 // tosa::TanhOp
411 if (isa<tosa::TanhOp>(Val: op) && isa<FloatType>(Val: elementTy))
412 return createWithDefaultProperties<mlir::math::TanhOp>(builder&: rewriter, loc,
413 resultTypes, operands: args);
414
415 // tosa::ErfOp
416 if (isa<tosa::ErfOp>(Val: op) && llvm::isa<FloatType>(Val: elementTy))
417 return createWithDefaultProperties<mlir::math::ErfOp>(builder&: rewriter, loc,
418 resultTypes, operands: args);
419
420 // tosa::GreaterOp
421 if (isa<tosa::GreaterOp>(Val: op) && isa<FloatType>(Val: elementTy))
422 return arith::CmpFOp::create(builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::OGT,
423 lhs: args[0], rhs: args[1]);
424
425 if (isa<tosa::GreaterOp>(Val: op) && elementTy.isSignlessInteger())
426 return arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::sgt,
427 lhs: args[0], rhs: args[1]);
428
429 // tosa::GreaterEqualOp
430 if (isa<tosa::GreaterEqualOp>(Val: op) && isa<FloatType>(Val: elementTy))
431 return arith::CmpFOp::create(builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::OGE,
432 lhs: args[0], rhs: args[1]);
433
434 if (isa<tosa::GreaterEqualOp>(Val: op) && elementTy.isSignlessInteger())
435 return arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::sge,
436 lhs: args[0], rhs: args[1]);
437
438 // tosa::EqualOp
439 if (isa<tosa::EqualOp>(Val: op) && isa<FloatType>(Val: elementTy))
440 return arith::CmpFOp::create(builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::OEQ,
441 lhs: args[0], rhs: args[1]);
442
443 if (isa<tosa::EqualOp>(Val: op) && elementTy.isSignlessInteger())
444 return arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq,
445 lhs: args[0], rhs: args[1]);
446
447 // tosa::SelectOp
448 if (isa<tosa::SelectOp>(Val: op)) {
449 elementTy = cast<ShapedType>(Val: op->getOperand(idx: 1).getType()).getElementType();
450 if (isa<FloatType>(Val: elementTy) || isa<IntegerType>(Val: elementTy))
451 return arith::SelectOp::create(builder&: rewriter, location: loc, condition: args[0], true_value: args[1], false_value: args[2]);
452 }
453
454 // tosa::MaximumOp
455 if (isa<tosa::MaximumOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
456 auto max = arith::MaximumFOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
457 return materializeBinaryNanCheckIfRequired(op: llvm::cast<tosa::MaximumOp>(Val: op),
458 rewriter, lhs: args[0], rhs: args[1], result: max);
459 }
460
461 if (isa<tosa::MaximumOp>(Val: op) && elementTy.isSignlessInteger()) {
462 return arith::MaxSIOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
463 }
464
465 // tosa::MinimumOp
466 if (isa<tosa::MinimumOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
467 auto min = arith::MinimumFOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
468 return materializeBinaryNanCheckIfRequired(op: llvm::cast<tosa::MinimumOp>(Val: op),
469 rewriter, lhs: args[0], rhs: args[1], result: min);
470 }
471
472 if (isa<tosa::MinimumOp>(Val: op) && elementTy.isSignlessInteger()) {
473 return arith::MinSIOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
474 }
475
476 // tosa::CeilOp
477 if (isa<tosa::CeilOp>(Val: op) && isa<FloatType>(Val: elementTy))
478 return createWithDefaultProperties<math::CeilOp>(builder&: rewriter, loc, resultTypes,
479 operands: args);
480
481 // tosa::FloorOp
482 if (isa<tosa::FloorOp>(Val: op) && isa<FloatType>(Val: elementTy))
483 return createWithDefaultProperties<math::FloorOp>(builder&: rewriter, loc,
484 resultTypes, operands: args);
485
486 // tosa::ClampOp
487 if (isa<tosa::ClampOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
488 bool losesInfo = false;
489 auto clampOp = cast<tosa::ClampOp>(Val: op);
490 APFloat minApf = cast<FloatAttr>(Val: clampOp.getMinValAttr()).getValue();
491 APFloat maxApf = cast<FloatAttr>(Val: clampOp.getMaxValAttr()).getValue();
492 minApf.convert(ToSemantics: cast<FloatType>(Val&: elementTy).getFloatSemantics(),
493 RM: APFloat::rmNearestTiesToEven, losesInfo: &losesInfo);
494 maxApf.convert(ToSemantics: cast<FloatType>(Val&: elementTy).getFloatSemantics(),
495 RM: APFloat::rmNearestTiesToEven, losesInfo: &losesInfo);
496 auto min = arith::ConstantOp::create(
497 builder&: rewriter, location: loc, result: elementTy, value: rewriter.getFloatAttr(type: elementTy, value: minApf));
498 auto max = arith::ConstantOp::create(
499 builder&: rewriter, location: loc, result: elementTy, value: rewriter.getFloatAttr(type: elementTy, value: maxApf));
500 auto result = clampFloatHelper(loc, arg: args[0], min, max, rewriter);
501
502 const auto nanMode = clampOp.getNanMode();
503
504 // NaN propagation has no meaning for non floating point types.
505 if (!isa<FloatType>(Val: elementTy))
506 return result;
507
508 // In the case of "PROPAGATE" semantics no compare and selection is
509 // required.
510 if (nanMode == NanPropagationMode::PROPAGATE)
511 return result;
512
513 // In the case of "IGNORE" semantics materialize a comparison
514 // of the current operand to the reduction which will return true for a NaN
515 // argument and then selects between the initial reduction value and the
516 // calculated result based on whether the argument is NaN or not. In pseudo
517 // code:
518 //
519 // reduce<op>(x, init):
520 // result = op(init, x)
521 // return init if x == NaN else result
522
523 // Unordered comparison of NaN against itself will always return true.
524 Value isNaN = arith::CmpFOp::create(
525 builder&: rewriter, location: op->getLoc(), predicate: arith::CmpFPredicate::UNO, lhs: args[0], rhs: args[0]);
526 // TOSA specifies that in "ignore" NaN mode the result is "min" if the input
527 // is NaN.
528 return arith::SelectOp::create(builder&: rewriter, location: op->getLoc(), condition: isNaN, true_value: min, false_value: result);
529 }
530
531 if (isa<tosa::ClampOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
532 auto intTy = cast<IntegerType>(Val&: elementTy);
533 auto clampOp = cast<tosa::ClampOp>(Val: op);
534 int64_t min =
535 cast<IntegerAttr>(Val: clampOp.getMinValAttr()).getValue().getSExtValue();
536 int64_t max =
537 cast<IntegerAttr>(Val: clampOp.getMaxValAttr()).getValue().getSExtValue();
538
539 int64_t minRepresentable = std::numeric_limits<int64_t>::min();
540 int64_t maxRepresentable = std::numeric_limits<int64_t>::max();
541 if (intTy.isUnsignedInteger()) {
542 minRepresentable = 0;
543 if (intTy.getIntOrFloatBitWidth() <= 63) {
544 maxRepresentable =
545 (int64_t)APInt::getMaxValue(numBits: intTy.getIntOrFloatBitWidth())
546 .getZExtValue();
547 }
548 } else if (intTy.getIntOrFloatBitWidth() <= 64) {
549 // Ensure that min & max fit into signed n-bit constants.
550 minRepresentable = APInt::getSignedMinValue(numBits: intTy.getIntOrFloatBitWidth())
551 .getSExtValue();
552 maxRepresentable = APInt::getSignedMaxValue(numBits: intTy.getIntOrFloatBitWidth())
553 .getSExtValue();
554 }
555 // Ensure that the bounds are representable as n-bit signed/unsigned
556 // integers.
557 min = std::max(a: min, b: minRepresentable);
558 max = std::max(a: max, b: minRepresentable);
559 min = std::min(a: min, b: maxRepresentable);
560 max = std::min(a: max, b: maxRepresentable);
561
562 auto minVal = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: min,
563 width: intTy.getIntOrFloatBitWidth());
564 auto maxVal = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: max,
565 width: intTy.getIntOrFloatBitWidth());
566 return clampIntHelper(loc, arg: args[0], min: minVal, max: maxVal, rewriter,
567 isUnsigned: intTy.isUnsignedInteger());
568 }
569
570 // tosa::SigmoidOp
571 if (isa<tosa::SigmoidOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
572 auto one =
573 arith::ConstantOp::create(builder&: rewriter, location: loc, value: FloatAttr::get(type: elementTy, value: 1));
574 auto negate = arith::NegFOp::create(builder&: rewriter, location: loc, resultTypes, operand: args[0]);
575 auto exp = mlir::math::ExpOp::create(builder&: rewriter, location: loc, resultTypes, operand: negate);
576 auto added = arith::AddFOp::create(builder&: rewriter, location: loc, lhs: exp, rhs: one);
577 return arith::DivFOp::create(builder&: rewriter, location: loc, lhs: one, rhs: added);
578 }
579
580 // tosa::CastOp
581 if (isa<tosa::CastOp>(Val: op)) {
582 Type srcTy = elementTy;
583 Type dstTy = resultTypes.front();
584 if (!srcTy.isIntOrFloat() || !dstTy.isIntOrFloat()) {
585 (void)rewriter.notifyMatchFailure(arg&: op, msg: "unsupported type");
586 return nullptr;
587 }
588
589 bool bitExtend =
590 srcTy.getIntOrFloatBitWidth() < dstTy.getIntOrFloatBitWidth();
591
592 // With `input_unsigned`, the integer input is read as unsigned.
593 bool inputUnsigned = cast<tosa::CastOp>(Val: op).getInputUnsigned();
594
595 if (srcTy == dstTy)
596 return args.front();
597
598 if (isa<FloatType>(Val: srcTy) && isa<FloatType>(Val: dstTy) && bitExtend)
599 return createWithDefaultProperties<arith::ExtFOp>(builder&: rewriter, loc,
600 resultTypes, operands: args);
601
602 if (isa<FloatType>(Val: srcTy) && isa<FloatType>(Val: dstTy) && !bitExtend)
603 return createWithDefaultProperties<arith::TruncFOp>(builder&: rewriter, loc,
604 resultTypes, operands: args);
605
606 // 1-bit integers need to be treated as signless.
607 if (srcTy.isInteger(width: 1) && arith::UIToFPOp::areCastCompatible(inputs: srcTy, outputs: dstTy))
608 return createWithDefaultProperties<arith::UIToFPOp>(builder&: rewriter, loc,
609 resultTypes, operands: args);
610
611 if (srcTy.isInteger(width: 1) && isa<IntegerType>(Val: dstTy) && bitExtend)
612 return createWithDefaultProperties<arith::ExtUIOp>(builder&: rewriter, loc,
613 resultTypes, operands: args);
614
615 // Unsigned integers need an unrealized cast so that they can be passed
616 // to UIToFP.
617 if (srcTy.isUnsignedInteger() && isa<FloatType>(Val: dstTy)) {
618 auto unrealizedCast =
619 UnrealizedConversionCastOp::create(
620 builder&: rewriter, location: loc,
621 resultTypes: rewriter.getIntegerType(width: srcTy.getIntOrFloatBitWidth()), operands: args[0])
622 .getResult(i: 0);
623 return arith::UIToFPOp::create(builder&: rewriter, location: loc, out: resultTypes[0],
624 in: unrealizedCast);
625 }
626
627 // Unsigned inputs are converted with UIToFP.
628 if (inputUnsigned && arith::UIToFPOp::areCastCompatible(inputs: srcTy, outputs: dstTy))
629 return createWithDefaultProperties<arith::UIToFPOp>(builder&: rewriter, loc,
630 resultTypes, operands: args);
631
632 // All other si-to-fp conversions should be handled by SIToFP.
633 if (arith::SIToFPOp::areCastCompatible(inputs: srcTy, outputs: dstTy))
634 return arith::SIToFPOp::create(builder&: rewriter, location: loc, resultTypes, operands: args,
635 attributes: ArrayRef<NamedAttribute>());
636
637 // Casting to boolean, floats need to only be checked as not-equal to zero.
638 if (isa<FloatType>(Val: srcTy) && dstTy.isInteger(width: 1)) {
639 Value zero = arith::ConstantOp::create(builder&: rewriter, location: loc,
640 value: rewriter.getFloatAttr(type: srcTy, value: 0.0));
641 return arith::CmpFOp::create(builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::UNE,
642 lhs: args.front(), rhs: zero);
643 }
644
645 if (arith::FPToSIOp::areCastCompatible(inputs: srcTy, outputs: dstTy)) {
646 auto rounded = math::RoundEvenOp::create(builder&: rewriter, location: loc, operand: args[0]);
647
648 const auto &fltSemantics = cast<FloatType>(Val&: srcTy).getFloatSemantics();
649 // Check whether neither int min nor int max can be represented in the
650 // input floating-point type due to too short exponent range.
651 if (static_cast<int>(dstTy.getIntOrFloatBitWidth()) - 1 >
652 APFloat::semanticsMaxExponent(fltSemantics)) {
653 // Use cmp + select to replace infinites by int min / int max. Other
654 // integral values can be represented in the integer space.
655 auto conv = arith::FPToSIOp::create(builder&: rewriter, location: loc, out: dstTy, in: rounded);
656 auto posInf = arith::ConstantOp::create(
657 builder&: rewriter, location: loc,
658 value: rewriter.getFloatAttr(type: getElementTypeOrSelf(type: srcTy),
659 value: APFloat::getInf(Sem: fltSemantics)));
660 auto negInf = arith::ConstantOp::create(
661 builder&: rewriter, location: loc,
662 value: rewriter.getFloatAttr(
663 type: getElementTypeOrSelf(type: srcTy),
664 value: APFloat::getInf(Sem: fltSemantics, /*Negative=*/true)));
665 auto overflow = arith::CmpFOp::create(
666 builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::UEQ, lhs: rounded, rhs: posInf);
667 auto underflow = arith::CmpFOp::create(
668 builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::UEQ, lhs: rounded, rhs: negInf);
669 auto intMin = arith::ConstantOp::create(
670 builder&: rewriter, location: loc,
671 value: rewriter.getIntegerAttr(
672 type: getElementTypeOrSelf(type: dstTy),
673 value: APInt::getSignedMinValue(numBits: dstTy.getIntOrFloatBitWidth())));
674 auto intMax = arith::ConstantOp::create(
675 builder&: rewriter, location: loc,
676 value: rewriter.getIntegerAttr(
677 type: getElementTypeOrSelf(type: dstTy),
678 value: APInt::getSignedMaxValue(numBits: dstTy.getIntOrFloatBitWidth())));
679 auto maxClamped =
680 arith::SelectOp::create(builder&: rewriter, location: loc, condition: overflow, true_value: intMax, false_value: conv);
681 return arith::SelectOp::create(builder&: rewriter, location: loc, condition: underflow, true_value: intMin,
682 false_value: maxClamped);
683 }
684
685 auto intMinFP = arith::ConstantOp::create(
686 builder&: rewriter, location: loc,
687 value: rewriter.getFloatAttr(
688 type: getElementTypeOrSelf(type: srcTy),
689 value: APInt::getSignedMinValue(numBits: dstTy.getIntOrFloatBitWidth())
690 .getSExtValue()));
691
692 // Check whether the mantissa has enough bits to represent int max.
693 if (cast<FloatType>(Val&: srcTy).getFPMantissaWidth() >=
694 dstTy.getIntOrFloatBitWidth() - 1) {
695 // Int min can also be represented since it is a power of two and thus
696 // consists of a single leading bit. Therefore we can clamp the input
697 // in the floating-point domain.
698
699 auto intMaxFP = arith::ConstantOp::create(
700 builder&: rewriter, location: loc,
701 value: rewriter.getFloatAttr(
702 type: getElementTypeOrSelf(type: srcTy),
703 value: APInt::getSignedMaxValue(numBits: dstTy.getIntOrFloatBitWidth())
704 .getSExtValue()));
705
706 Value clamped =
707 clampFloatHelper(loc, arg: rounded, min: intMinFP, max: intMaxFP, rewriter);
708 return arith::FPToSIOp::create(builder&: rewriter, location: loc, out: dstTy, in: clamped);
709 }
710
711 // Due to earlier check we know exponant range is big enough to represent
712 // int min. We can therefore rely on int max + 1 being representable as
713 // well because it's just int min with a positive sign. So clamp the min
714 // value and compare against that to select the max int value if needed.
715 auto intMaxPlusOneFP = arith::ConstantOp::create(
716 builder&: rewriter, location: loc,
717 value: rewriter.getFloatAttr(
718 type: getElementTypeOrSelf(type: srcTy),
719 value: static_cast<double>(
720 APInt::getSignedMaxValue(numBits: dstTy.getIntOrFloatBitWidth())
721 .getSExtValue()) +
722 1.0f));
723
724 auto intMax = arith::ConstantOp::create(
725 builder&: rewriter, location: loc,
726 value: rewriter.getIntegerAttr(
727 type: getElementTypeOrSelf(type: dstTy),
728 value: APInt::getSignedMaxValue(numBits: dstTy.getIntOrFloatBitWidth())));
729 auto minClampedFP =
730 arith::MaximumFOp::create(builder&: rewriter, location: loc, lhs: rounded, rhs: intMinFP);
731 auto minClamped =
732 arith::FPToSIOp::create(builder&: rewriter, location: loc, out: dstTy, in: minClampedFP);
733 auto overflow = arith::CmpFOp::create(
734 builder&: rewriter, location: loc, predicate: arith::CmpFPredicate::UGE, lhs: rounded, rhs: intMaxPlusOneFP);
735 return arith::SelectOp::create(builder&: rewriter, location: loc, condition: overflow, true_value: intMax,
736 false_value: minClamped);
737 }
738
739 // Casting to boolean, integers need to only be checked as not-equal to
740 // zero.
741 if (isa<IntegerType>(Val: srcTy) && dstTy.isInteger(width: 1)) {
742 Value zero = arith::ConstantIntOp::create(builder&: rewriter, location: loc, value: 0,
743 width: srcTy.getIntOrFloatBitWidth());
744 return arith::CmpIOp::create(builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::ne,
745 lhs: args.front(), rhs: zero);
746 }
747
748 // Unsigned inputs are zero-extended.
749 if (inputUnsigned && isa<IntegerType>(Val: srcTy) && isa<IntegerType>(Val: dstTy) &&
750 bitExtend)
751 return createWithDefaultProperties<arith::ExtUIOp>(builder&: rewriter, loc,
752 resultTypes, operands: args);
753
754 if (isa<IntegerType>(Val: srcTy) && isa<IntegerType>(Val: dstTy) && bitExtend)
755 return arith::ExtSIOp::create(builder&: rewriter, location: loc, resultTypes, operands: args,
756 attributes: ArrayRef<NamedAttribute>());
757
758 if (isa<IntegerType>(Val: srcTy) && isa<IntegerType>(Val: dstTy) && !bitExtend) {
759 return arith::TruncIOp::create(builder&: rewriter, location: loc, out: dstTy, in: args[0]);
760 }
761 }
762
763 (void)rewriter.notifyMatchFailure(
764 arg&: op, msg: "unhandled op for linalg body calculation for elementwise op");
765 return nullptr;
766}
767
768// Emit an 'arith.constant' op for the given index if it has not been created
769// yet, or return an existing constant. This will prevent an excessive creation
770// of redundant constants, easing readability of emitted code for unit tests.
771static Value createIndex(PatternRewriter &rewriter, Location loc,
772 IndexPool &indexPool, int64_t index) {
773 auto [it, inserted] = indexPool.try_emplace(Key: index);
774 if (inserted)
775 it->second =
776 arith::ConstantOp::create(builder&: rewriter, location: loc, value: rewriter.getIndexAttr(value: index));
777 return it->second;
778}
779
780static Value getTensorDim(PatternRewriter &rewriter, Location loc,
781 IndexPool &indexPool, Value tensor, int64_t index) {
782 auto indexValue = createIndex(rewriter, loc, indexPool, index);
783 return tensor::DimOp::create(builder&: rewriter, location: loc, source: tensor, index: indexValue).getResult();
784}
785
786static OpFoldResult getOrFoldTensorDim(PatternRewriter &rewriter, Location loc,
787 IndexPool &indexPool, Value tensor,
788 int64_t index) {
789 auto shapedType = dyn_cast<ShapedType>(Val: tensor.getType());
790 assert(shapedType && shapedType.hasRank() && "expected a ranked shaped type");
791 assert(index >= 0 && index < shapedType.getRank() && "index out of bounds");
792 if (shapedType.isDynamicDim(idx: index))
793 return getTensorDim(rewriter, loc, indexPool, tensor, index);
794 return rewriter.getIndexAttr(value: shapedType.getDimSize(idx: index));
795}
796
797static bool operandsAndResultsRanked(Operation *operation) {
798 auto isRanked = [](Value value) {
799 return isa<RankedTensorType>(Val: value.getType());
800 };
801 return llvm::all_of(Range: operation->getOperands(), P: isRanked) &&
802 llvm::all_of(Range: operation->getResults(), P: isRanked);
803}
804
805// Compute the runtime dimension size for dimension 'dim' of the output by
806// inspecting input 'operands', all of which are expected to have the same rank.
807// This function returns a pair {targetSize, masterOperand}.
808//
809// The runtime size of the output dimension is returned either as a statically
810// computed attribute or as a runtime SSA value.
811//
812// If the target size was inferred directly from one dominating operand, that
813// operand is returned in 'masterOperand'. If the target size is inferred from
814// multiple operands, 'masterOperand' is set to nullptr.
815std::pair<OpFoldResult, Value>
816mlir::tosa::computeTargetSize(PatternRewriter &rewriter, Location loc,
817 IndexPool &indexPool, ValueRange operands,
818 int64_t dim) {
819 // If any input operand contains a static size greater than 1 for this
820 // dimension, that is the target size. An occurrence of an additional static
821 // dimension greater than 1 with a different value is undefined behavior.
822 for (auto operand : operands) {
823 auto size = cast<RankedTensorType>(Val: operand.getType()).getDimSize(idx: dim);
824 if (ShapedType::isStatic(dValue: size) && size > 1)
825 return {rewriter.getIndexAttr(value: size), operand};
826 }
827
828 // Filter operands with dynamic dimension
829 auto operandsWithDynamicDim =
830 llvm::filter_to_vector(C&: operands, Pred: [&](Value operand) {
831 return cast<RankedTensorType>(Val: operand.getType()).isDynamicDim(idx: dim);
832 });
833
834 // If no operand has a dynamic dimension, it means all sizes were 1
835 if (operandsWithDynamicDim.empty())
836 return {rewriter.getIndexAttr(value: 1), operands.front()};
837
838 // Emit code that computes the runtime size for this dimension. If there is
839 // only one operand with a dynamic dimension, it is considered the master
840 // operand that determines the runtime size of the output dimension.
841 auto targetSize =
842 getTensorDim(rewriter, loc, indexPool, tensor: operandsWithDynamicDim[0], index: dim);
843 if (operandsWithDynamicDim.size() == 1)
844 return {targetSize, operandsWithDynamicDim[0]};
845
846 // Calculate maximum size among all dynamic dimensions
847 for (size_t i = 1; i < operandsWithDynamicDim.size(); i++) {
848 auto nextSize =
849 getTensorDim(rewriter, loc, indexPool, tensor: operandsWithDynamicDim[i], index: dim);
850 targetSize = arith::MaxUIOp::create(builder&: rewriter, location: loc, lhs: targetSize, rhs: nextSize);
851 }
852 return {targetSize, nullptr};
853}
854
855// Compute the runtime output size for all dimensions. This function returns
856// a pair {targetShape, masterOperands}.
857static std::pair<SmallVector<OpFoldResult>, SmallVector<Value>>
858computeTargetShape(PatternRewriter &rewriter, Location loc,
859 IndexPool &indexPool, ValueRange operands) {
860 assert(!operands.empty());
861 auto rank = cast<RankedTensorType>(Val: operands.front().getType()).getRank();
862 SmallVector<OpFoldResult> targetShape;
863 SmallVector<Value> masterOperands;
864 for (auto dim : llvm::seq<int64_t>(Begin: 0, End: rank)) {
865 auto [targetSize, masterOperand] =
866 computeTargetSize(rewriter, loc, indexPool, operands, dim);
867 targetShape.push_back(Elt: targetSize);
868 masterOperands.push_back(Elt: masterOperand);
869 }
870 return {targetShape, masterOperands};
871}
872
873static Value broadcastDynamicDimension(PatternRewriter &rewriter, Location loc,
874 IndexPool &indexPool, Value operand,
875 int64_t dim, OpFoldResult targetSize,
876 Value masterOperand) {
877 // Nothing to do if this is a static dimension
878 auto rankedTensorType = cast<RankedTensorType>(Val: operand.getType());
879 if (!rankedTensorType.isDynamicDim(idx: dim))
880 return operand;
881
882 // If the target size for this dimension was directly inferred by only taking
883 // this operand into account, there is no need to broadcast. This is an
884 // optimization that will prevent redundant control flow, and constitutes the
885 // main motivation for tracking "master operands".
886 if (operand == masterOperand)
887 return operand;
888
889 // Affine maps for 'linalg.generic' op
890 auto rank = rankedTensorType.getRank();
891 SmallVector<AffineExpr> affineExprs;
892 for (auto index : llvm::seq<int64_t>(Begin: 0, End: rank)) {
893 auto affineExpr = index == dim ? rewriter.getAffineConstantExpr(constant: 0)
894 : rewriter.getAffineDimExpr(position: index);
895 affineExprs.push_back(Elt: affineExpr);
896 }
897 auto broadcastAffineMap =
898 AffineMap::get(dimCount: rank, symbolCount: 0, results: affineExprs, context: rewriter.getContext());
899 auto identityAffineMap = rewriter.getMultiDimIdentityMap(rank);
900 SmallVector<AffineMap> affineMaps = {broadcastAffineMap, identityAffineMap};
901
902 // Check if broadcast is necessary
903 auto one = createIndex(rewriter, loc, indexPool, index: 1);
904 auto runtimeSize = getTensorDim(rewriter, loc, indexPool, tensor: operand, index: dim);
905 auto broadcastNecessary = arith::CmpIOp::create(
906 builder&: rewriter, location: loc, predicate: arith::CmpIPredicate::eq, lhs: runtimeSize, rhs: one);
907
908 // Emit 'then' region of 'scf.if'
909 auto emitThenRegion = [&](OpBuilder &opBuilder, Location loc) {
910 // It is not safe to cache constants across regions.
911 // New constants could potentially violate dominance requirements.
912 IndexPool localPool;
913
914 // Emit 'tensor.empty' op
915 SmallVector<OpFoldResult> outputTensorShape;
916 for (auto index : llvm::seq<int64_t>(Begin: 0, End: rank)) {
917 auto size = index == dim ? targetSize
918 : getOrFoldTensorDim(rewriter, loc, indexPool&: localPool,
919 tensor: operand, index);
920 outputTensorShape.push_back(Elt: size);
921 }
922 Value outputTensor = tensor::EmptyOp::create(
923 builder&: opBuilder, location: loc, sizes: outputTensorShape, elementType: rankedTensorType.getElementType());
924
925 // Emit 'linalg.generic' op
926 auto resultTensor =
927 linalg::GenericOp::create(
928 builder&: opBuilder, location: loc, resultTensorTypes: outputTensor.getType(), inputs: operand, outputs: outputTensor,
929 indexingMaps: affineMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: rank),
930 odsArg5: [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
931 // Emit 'linalg.yield' op
932 linalg::YieldOp::create(builder&: opBuilder, location: loc, values: blockArgs.front());
933 })
934 .getResult(i: 0);
935
936 // Cast to original operand type if necessary
937 auto castResultTensor = rewriter.createOrFold<tensor::CastOp>(
938 location: loc, args: operand.getType(), args&: resultTensor);
939
940 // Emit 'scf.yield' op
941 scf::YieldOp::create(builder&: opBuilder, location: loc, results: castResultTensor);
942 };
943
944 // Emit 'else' region of 'scf.if'
945 auto emitElseRegion = [&](OpBuilder &opBuilder, Location loc) {
946 scf::YieldOp::create(builder&: opBuilder, location: loc, results: operand);
947 };
948
949 // Emit 'scf.if' op
950 auto ifOp = scf::IfOp::create(builder&: rewriter, location: loc, cond: broadcastNecessary,
951 thenBuilder: emitThenRegion, elseBuilder: emitElseRegion);
952 return ifOp.getResult(i: 0);
953}
954
955static Value broadcastDynamicDimensions(PatternRewriter &rewriter, Location loc,
956 IndexPool &indexPool, Value operand,
957 ArrayRef<OpFoldResult> targetShape,
958 ArrayRef<Value> masterOperands) {
959 int64_t rank = cast<RankedTensorType>(Val: operand.getType()).getRank();
960 assert((int64_t)targetShape.size() == rank);
961 assert((int64_t)masterOperands.size() == rank);
962 for (auto index : llvm::seq<int64_t>(Begin: 0, End: rank))
963 operand =
964 broadcastDynamicDimension(rewriter, loc, indexPool, operand, dim: index,
965 targetSize: targetShape[index], masterOperand: masterOperands[index]);
966 return operand;
967}
968
969static SmallVector<Value>
970broadcastDynamicDimensions(PatternRewriter &rewriter, Location loc,
971 IndexPool &indexPool, ValueRange operands,
972 ArrayRef<OpFoldResult> targetShape,
973 ArrayRef<Value> masterOperands) {
974 // No need to broadcast for unary operations
975 if (operands.size() == 1)
976 return operands;
977
978 // No need to broadcast for static shape
979 bool hasDynamic = false;
980 for (auto op : operands) {
981 const auto tType = dyn_cast<RankedTensorType>(Val: op.getType());
982 if (tType && !tType.hasStaticShape()) {
983 hasDynamic = true;
984 break;
985 }
986 }
987 if (!hasDynamic)
988 return operands;
989
990 // Broadcast dynamic dimensions operand by operand
991 return llvm::map_to_vector(C&: operands, F: [&](Value operand) {
992 return broadcastDynamicDimensions(rewriter, loc, indexPool, operand,
993 targetShape, masterOperands);
994 });
995}
996
997static LogicalResult emitElementwiseComputation(
998 ConversionPatternRewriter &rewriter, Location loc, Operation *operation,
999 ValueRange operands, ValueRange scalarOperands,
1000 ArrayRef<OpFoldResult> targetShape, const TypeConverter &converter) {
1001 // Generate output tensor
1002 auto resultType = cast_or_null<RankedTensorType>(
1003 Val: converter.convertType(t: operation->getResultTypes().front()));
1004 if (!resultType) {
1005 return rewriter.notifyMatchFailure(arg&: operation, msg: "failed to convert type");
1006 }
1007 Value outputTensor = tensor::EmptyOp::create(builder&: rewriter, location: loc, sizes: targetShape,
1008 elementType: resultType.getElementType());
1009
1010 // Create affine maps. Input affine maps broadcast static dimensions of size
1011 // 1. The output affine map is an identity map.
1012 //
1013 auto rank = resultType.getRank();
1014 auto affineMaps = llvm::map_to_vector(C&: operands, F: [&](Value operand) {
1015 auto shape = cast<ShapedType>(Val: operand.getType()).getShape();
1016 SmallVector<AffineExpr> affineExprs;
1017 for (auto it : llvm::enumerate(First&: shape)) {
1018 // Prefer producting identity maps whenever possible (i.e. no broadcasting
1019 // needed) because some transforms (like reshape folding)
1020 // do not support affine constant exprs.
1021 bool requiresBroadcast =
1022 (it.value() == 1 && resultType.getDimSize(idx: it.index()) != 1);
1023 auto affineExpr = requiresBroadcast
1024 ? rewriter.getAffineConstantExpr(constant: 0)
1025 : rewriter.getAffineDimExpr(position: it.index());
1026 affineExprs.push_back(Elt: affineExpr);
1027 }
1028 return AffineMap::get(dimCount: rank, symbolCount: 0, results: affineExprs, context: rewriter.getContext());
1029 });
1030 // Scalar operands are single-element tensors: every iteration reads their
1031 // first element.
1032 for (Value scalar : scalarOperands) {
1033 auto scalarRank = cast<ShapedType>(Val: scalar.getType()).getRank();
1034 SmallVector<AffineExpr> zeros(scalarRank,
1035 rewriter.getAffineConstantExpr(constant: 0));
1036 affineMaps.push_back(Elt: AffineMap::get(dimCount: rank, symbolCount: 0, results: zeros, context: rewriter.getContext()));
1037 }
1038 affineMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank));
1039
1040 SmallVector<Value> inputs(operands);
1041 llvm::append_range(C&: inputs, R&: scalarOperands);
1042
1043 // Emit 'linalg.generic' op
1044 bool encounteredError = false;
1045 auto linalgOp = linalg::GenericOp::create(
1046 builder&: rewriter, location: loc, resultTensorTypes: outputTensor.getType(), inputs, outputs: outputTensor, indexingMaps: affineMaps,
1047 iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: rank),
1048 odsArg5: [&](OpBuilder &opBuilder, Location loc, ValueRange blockArgs) {
1049 Value opResult = createLinalgBodyCalculationForElementwiseOp(
1050 op: operation, args: blockArgs.take_front(n: operation->getNumOperands()),
1051 resultTypes: {resultType.getElementType()}, rewriter);
1052 if (!opResult) {
1053 encounteredError = true;
1054 return;
1055 }
1056 linalg::YieldOp::create(builder&: opBuilder, location: loc, values: opResult);
1057 });
1058 if (encounteredError)
1059 return rewriter.notifyMatchFailure(
1060 arg&: operation, msg: "unable to create linalg.generic body for elementwise op");
1061
1062 // Cast 'linalg.generic' result into original result type if needed
1063 auto castResult = rewriter.createOrFold<tensor::CastOp>(
1064 location: loc, args&: resultType, args: linalgOp->getResult(idx: 0));
1065 rewriter.replaceOp(op: operation, newValues: castResult);
1066 return success();
1067}
1068
1069// The shift of tosa.mul and the zero points of tosa.negate are single-element
1070// tensors, so they never take part in broadcasting.
1071static ValueRange getBroadcastableOperands(Operation *operation,
1072 ValueRange operands) {
1073 if (isa<tosa::MulOp>(Val: operation))
1074 return operands.take_front(n: 2);
1075 if (isa<tosa::NegateOp>(Val: operation))
1076 return operands.take_front(n: 1);
1077 return operands;
1078}
1079
1080// Returns the single-element operands that the body reads as block arguments:
1081// a non-constant tosa.mul shift, or both tosa.negate zero points when either of
1082// them is non-constant. Constant ones are created in the body instead.
1083static ValueRange getScalarOperands(Operation *operation, ValueRange operands) {
1084 if (isa<tosa::MulOp>(Val: operation)) {
1085 DenseElementsAttr shiftElems;
1086 if (matchPattern(value: operation->getOperand(idx: 2), pattern: m_Constant(bind_value: &shiftElems)))
1087 return {};
1088 return operands.drop_front(n: 2);
1089 }
1090 if (auto negate = dyn_cast<tosa::NegateOp>(Val: operation)) {
1091 if (succeeded(Result: negate.getInput1ZeroPoint()) &&
1092 succeeded(Result: negate.getOutputZeroPoint()))
1093 return {};
1094 return operands.drop_front(n: 1);
1095 }
1096 return {};
1097}
1098
1099static LogicalResult
1100elementwiseMatchAndRewriteHelper(Operation *operation, ValueRange operands,
1101 ConversionPatternRewriter &rewriter,
1102 const TypeConverter &converter) {
1103
1104 // Collect op properties
1105 assert(operation->getNumResults() == 1 && "elementwise op expects 1 result");
1106 assert(operation->getNumOperands() >= 1 &&
1107 "elementwise op expects at least 1 operand");
1108 if (!operandsAndResultsRanked(operation))
1109 return rewriter.notifyMatchFailure(arg&: operation,
1110 msg: "Unranked tensors not supported");
1111
1112 // Lower operation
1113 IndexPool indexPool;
1114 auto loc = operation->getLoc();
1115 auto operandsToBroadcast = getBroadcastableOperands(operation, operands);
1116 auto scalarOperands = getScalarOperands(operation, operands);
1117 auto [targetShape, masterOperands] =
1118 computeTargetShape(rewriter, loc, indexPool, operands: operandsToBroadcast);
1119 auto broadcastOperands =
1120 broadcastDynamicDimensions(rewriter, loc, indexPool, operands: operandsToBroadcast,
1121 targetShape, masterOperands);
1122 return emitElementwiseComputation(rewriter, loc, operation, operands: broadcastOperands,
1123 scalarOperands, targetShape, converter);
1124}
1125
1126// Returns the constant initial value for a given reduction operation. The
1127// attribute type varies depending on the element type required.
1128static TypedAttr createInitialValueForReduceOp(Operation *op, Type elementTy,
1129 PatternRewriter &rewriter,
1130 bool allowNonFinites) {
1131 if (isa<tosa::ReduceSumOp>(Val: op) && isa<FloatType>(Val: elementTy))
1132 return rewriter.getFloatAttr(type: elementTy, value: 0.0);
1133
1134 if (isa<tosa::ReduceSumOp>(Val: op) && isa<IntegerType>(Val: elementTy))
1135 return rewriter.getIntegerAttr(type: elementTy, value: 0);
1136
1137 if (isa<tosa::ReduceProductOp>(Val: op) && isa<FloatType>(Val: elementTy))
1138 return rewriter.getFloatAttr(type: elementTy, value: 1.0);
1139
1140 if (isa<tosa::ReduceProductOp>(Val: op) && isa<IntegerType>(Val: elementTy))
1141 return rewriter.getIntegerAttr(type: elementTy, value: 1);
1142
1143 if (isa<tosa::ReduceMinOp>(Val: op) && isa<FloatType>(Val: elementTy))
1144 return rewriter.getFloatAttr(
1145 type: elementTy,
1146 value: getFloatMinMaxIdentity(semantics: cast<FloatType>(Val&: elementTy).getFloatSemantics(),
1147 /*negative=*/false, allowNonFinites));
1148
1149 if (isa<tosa::ReduceMinOp>(Val: op) && isa<IntegerType>(Val: elementTy))
1150 return rewriter.getIntegerAttr(
1151 type: elementTy, value: APInt::getSignedMaxValue(numBits: elementTy.getIntOrFloatBitWidth()));
1152
1153 if (isa<tosa::ReduceMaxOp>(Val: op) && isa<FloatType>(Val: elementTy))
1154 return rewriter.getFloatAttr(
1155 type: elementTy,
1156 value: getFloatMinMaxIdentity(semantics: cast<FloatType>(Val&: elementTy).getFloatSemantics(),
1157 /*negative=*/true, allowNonFinites));
1158
1159 if (isa<tosa::ReduceMaxOp>(Val: op) && isa<IntegerType>(Val: elementTy))
1160 return rewriter.getIntegerAttr(
1161 type: elementTy, value: APInt::getSignedMinValue(numBits: elementTy.getIntOrFloatBitWidth()));
1162
1163 if (isa<tosa::ReduceAllOp>(Val: op) && elementTy.isInteger(width: 1))
1164 return rewriter.getIntegerAttr(type: elementTy, value: APInt::getAllOnes(numBits: 1));
1165
1166 if (isa<tosa::ReduceAnyOp>(Val: op) && elementTy.isInteger(width: 1))
1167 return rewriter.getIntegerAttr(type: elementTy, value: APInt::getZero(numBits: 1));
1168
1169 if (isa<tosa::ArgMaxOp, tosa::ArgMinOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
1170 const bool isMax = isa<tosa::ArgMaxOp>(Val: op);
1171 const auto nanMode = isMax ? cast<tosa::ArgMaxOp>(Val: op).getNanMode()
1172 : cast<tosa::ArgMinOp>(Val: op).getNanMode();
1173
1174 auto floatTy = dyn_cast<FloatType>(Val&: elementTy);
1175 const auto &semantics = floatTy.getFloatSemantics();
1176
1177 const bool useNaN = nanMode == NanPropagationMode::IGNORE &&
1178 APFloat::semanticsHasNaN(semantics);
1179 const auto seed =
1180 useNaN ? APFloat::getQNaN(Sem: semantics)
1181 : getFloatMinMaxIdentity(semantics, /*negative=*/isMax,
1182 allowNonFinites);
1183
1184 return rewriter.getFloatAttr(type: floatTy, value: seed);
1185 }
1186
1187 if (isa<tosa::ArgMaxOp>(Val: op) && isa<IntegerType>(Val: elementTy))
1188 return rewriter.getIntegerAttr(
1189 type: elementTy, value: APInt::getSignedMinValue(numBits: elementTy.getIntOrFloatBitWidth()));
1190
1191 if (isa<tosa::ArgMinOp>(Val: op) && isa<IntegerType>(Val: elementTy))
1192 return rewriter.getIntegerAttr(
1193 type: elementTy, value: APInt::getSignedMaxValue(numBits: elementTy.getIntOrFloatBitWidth()));
1194
1195 return {};
1196}
1197
1198// Creates the body calculation for a reduction. The operations vary depending
1199// on the input type.
1200static Value createLinalgBodyCalculationForReduceOp(Operation *op,
1201 ValueRange args,
1202 Type elementTy,
1203 PatternRewriter &rewriter) {
1204 Location loc = op->getLoc();
1205 if (isa<tosa::ReduceSumOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
1206 return createWithDefaultProperties<arith::AddFOp>(
1207 builder&: rewriter, loc, resultTypes: TypeRange{elementTy}, operands: args);
1208 }
1209
1210 if (isa<tosa::ReduceSumOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
1211 return createWithDefaultProperties<arith::AddIOp>(
1212 builder&: rewriter, loc, resultTypes: TypeRange{elementTy}, operands: args);
1213 }
1214
1215 if (isa<tosa::ReduceProductOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
1216 return createWithDefaultProperties<arith::MulFOp>(
1217 builder&: rewriter, loc, resultTypes: TypeRange{elementTy}, operands: args);
1218 }
1219
1220 if (isa<tosa::ReduceProductOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
1221 return createWithDefaultProperties<arith::MulIOp>(
1222 builder&: rewriter, loc, resultTypes: TypeRange{elementTy}, operands: args);
1223 }
1224
1225 if (isa<tosa::ReduceMinOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
1226 return arith::MinimumFOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
1227 }
1228
1229 if (isa<tosa::ReduceMinOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
1230 return arith::MinSIOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
1231 }
1232
1233 if (isa<tosa::ReduceMaxOp>(Val: op) && isa<FloatType>(Val: elementTy)) {
1234 return arith::MaximumFOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
1235 }
1236
1237 if (isa<tosa::ReduceMaxOp>(Val: op) && isa<IntegerType>(Val: elementTy)) {
1238 return arith::MaxSIOp::create(builder&: rewriter, location: loc, lhs: args[0], rhs: args[1]);
1239 }
1240
1241 if (isa<tosa::ReduceAllOp>(Val: op) && elementTy.isInteger(width: 1))
1242 return arith::AndIOp::create(builder&: rewriter, location: loc, operands: args);
1243
1244 if (isa<tosa::ReduceAnyOp>(Val: op) && elementTy.isInteger(width: 1))
1245 return arith::OrIOp::create(builder&: rewriter, location: loc, operands: args);
1246
1247 return {};
1248}
1249
1250// Performs the match and rewrite for reduction operations. This includes
1251// declaring a correctly sized initial value, and the linalg.generic operation
1252// that reduces across the specified axis.
1253template <typename OpTy>
1254static LogicalResult reduceMatchAndRewriteHelper(OpTy op, uint64_t axis,
1255 PatternRewriter &rewriter,
1256 bool allowNonFinites) {
1257 auto loc = op->getLoc();
1258 auto inputTy = dyn_cast<RankedTensorType>(op->getOperand(0).getType());
1259 auto resultTy = dyn_cast<RankedTensorType>(op->getResult(0).getType());
1260 if (!inputTy || !resultTy)
1261 return rewriter.notifyMatchFailure(op, "unranked tensors not supported");
1262
1263 auto elementTy = resultTy.getElementType();
1264 Value input = op->getOperand(0);
1265
1266 // Figure out the accType if needed
1267 bool widenAccTy = std::is_same_v<OpTy, tosa::ReduceSumOp> &&
1268 isa<FloatType>(elementTy) &&
1269 cast<FloatType>(elementTy).isBF16();
1270 Type accTy = widenAccTy ? rewriter.getF32Type() : elementTy;
1271
1272 SmallVector<int64_t> reduceShape;
1273 SmallVector<Value> dynDims;
1274 for (unsigned i = 0; i < inputTy.getRank(); i++) {
1275 if (axis != i) {
1276 reduceShape.push_back(Elt: inputTy.getDimSize(i));
1277 if (inputTy.isDynamicDim(i))
1278 dynDims.push_back(Elt: tensor::DimOp::create(rewriter, loc, input, i));
1279 }
1280 }
1281
1282 SmallVector<Value> inputs, outputs;
1283 inputs.push_back(Elt: input);
1284
1285 // First fill the output buffer with the init value.
1286 auto emptyTensor =
1287 tensor::EmptyOp::create(rewriter, loc, reduceShape, accTy, dynDims)
1288 .getResult();
1289
1290 auto fillValueAttr =
1291 createInitialValueForReduceOp(op, accTy, rewriter, allowNonFinites);
1292 if (!fillValueAttr)
1293 return rewriter.notifyMatchFailure(
1294 op, "No initial value found for reduction operation");
1295
1296 auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
1297 auto filledTensor =
1298 linalg::FillOp::create(rewriter, loc, ValueRange{fillValue},
1299 ValueRange{emptyTensor})
1300 .result();
1301 outputs.push_back(Elt: filledTensor);
1302
1303 bool isNanIgnoreMode = false;
1304 if constexpr (std::is_same_v<OpTy, tosa::ReduceMinOp> ||
1305 std::is_same_v<OpTy, tosa::ReduceMaxOp>) {
1306 // NaN propagation has no meaning for non floating point types.
1307 if (isa<FloatType>(elementTy) &&
1308 op.getNanMode() == NanPropagationMode::IGNORE) {
1309 isNanIgnoreMode = true;
1310 // Because the TOSA spec requires the result be NaN iff all elements in
1311 // the reduction are NaN we can't simply perform a compare and select.
1312 // Additionally we have to keep track of whether we've seen any non-NaN
1313 // values and then do a final select based on this predicate.
1314 auto trueAttr = rewriter.getBoolAttr(value: true);
1315 auto trueValue = arith::ConstantOp::create(rewriter, loc, trueAttr);
1316 auto emptyBoolTensor =
1317 tensor::EmptyOp::create(rewriter, loc, reduceShape,
1318 trueValue.getType(), dynDims)
1319 .getResult();
1320 auto allResultsNaNTensor =
1321 linalg::FillOp::create(rewriter, loc, ValueRange{trueValue},
1322 ValueRange{emptyBoolTensor})
1323 .result();
1324 // Note that because the linalg::ReduceOp has two variadic arguments
1325 // (inputs and outputs) and it has the SameVariadicOperandSize trait we
1326 // need to have the same number of inputs and outputs.
1327 //
1328 // The second input isn't actually used anywhere since the value used to
1329 // update the NaN flag is calculated inside the body of the reduction and
1330 // then used to update an out value.
1331 // In order to satisfy type constraints we just pass another copy of the
1332 // input here.
1333 inputs.push_back(Elt: input);
1334 outputs.push_back(Elt: allResultsNaNTensor);
1335 }
1336 }
1337
1338 bool didEncounterError = false;
1339 linalg::LinalgOp linalgOp = linalg::ReduceOp::create(
1340 rewriter, loc, inputs, outputs, axis,
1341 [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange blockArgs) {
1342 std::array<Value, 2> binaryArgs{
1343 blockArgs[0], isNanIgnoreMode ? blockArgs[2] : blockArgs[1]};
1344
1345 // If reduction type differs then extend (applicable to reduce_sum)
1346 if (binaryArgs[0].getType() != accTy)
1347 binaryArgs[0] = arith::ExtFOp::create(
1348 builder&: nestedBuilder, location: nestedLoc, resultTypes: TypeRange{accTy},
1349 operands: ValueRange{binaryArgs[0]}, properties: arith::ExtFOp::Properties{});
1350
1351 auto result = createLinalgBodyCalculationForReduceOp(op, binaryArgs,
1352 accTy, rewriter);
1353 if (result)
1354 didEncounterError = true;
1355
1356 SmallVector<Value> resultsToYield;
1357 if (isNanIgnoreMode) {
1358 auto inputValue = blockArgs[0];
1359 auto initialValue = blockArgs[2];
1360 auto oldAllResultsNanFlagValue = blockArgs[3];
1361
1362 // Unordered comparison of NaN against itself will always return true.
1363 Value isNaN = arith::CmpFOp::create(nestedBuilder, op->getLoc(),
1364 arith::CmpFPredicate::UNO,
1365 inputValue, inputValue);
1366 // If we've encountered a NaN, take the non-NaN value.
1367 auto selectOp = arith::SelectOp::create(nestedBuilder, op->getLoc(),
1368 isNaN, initialValue, result);
1369 // Update the flag which keeps track of whether we have seen a non-NaN
1370 // value.
1371 auto newAllResultsNanFlagValue = arith::AndIOp::create(
1372 nestedBuilder, op->getLoc(), oldAllResultsNanFlagValue, isNaN);
1373 resultsToYield.push_back(Elt: selectOp);
1374 resultsToYield.push_back(Elt: newAllResultsNanFlagValue);
1375 } else {
1376 resultsToYield.push_back(Elt: result);
1377 }
1378 linalg::YieldOp::create(nestedBuilder, loc, resultsToYield);
1379 });
1380
1381 if (!didEncounterError)
1382 return rewriter.notifyMatchFailure(
1383 op, "unable to create linalg.generic body for reduce op");
1384
1385 if (isNanIgnoreMode) {
1386 // Materialize a check to see whether we encountered any non-NaN values, if
1387 // we didn't we need to select a tensor of NaNs since the result will just
1388 // be the initial identity value propagated through all the compares and
1389 // selects inside the reduction.
1390
1391 // Create a tensor full of NaNs.
1392 auto nanValueAttr = rewriter.getFloatAttr(
1393 accTy,
1394 APFloat::getNaN(Sem: cast<FloatType>(elementTy).getFloatSemantics(), Negative: false));
1395 auto nanValue = arith::ConstantOp::create(rewriter, loc, nanValueAttr);
1396 auto emptyNanTensor =
1397 tensor::EmptyOp::create(rewriter, loc, reduceShape, accTy, dynDims)
1398 .getResult();
1399 auto nanFilledTensor =
1400 linalg::FillOp::create(rewriter, loc, ValueRange{nanValue},
1401 ValueRange{emptyNanTensor})
1402 .result();
1403
1404 // Create an empty tensor, non need to fill this since it will be
1405 // overwritten by the select.
1406 auto finalEmptyTensor =
1407 tensor::EmptyOp::create(rewriter, loc, reduceShape, accTy, dynDims)
1408 .getResult();
1409
1410 // Do a selection between the tensors akin to:
1411 // result = NaN if "all results NaN" else result.
1412 SmallVector<Value> ins, outs;
1413 ins.push_back(Elt: linalgOp->getOpResult(idx: 1));
1414 ins.push_back(Elt: nanFilledTensor);
1415 ins.push_back(Elt: linalgOp->getResult(idx: 0));
1416 outs.push_back(Elt: finalEmptyTensor);
1417 auto linalgSelect =
1418 linalg::ElementwiseOp::create(rewriter, op->getLoc(), ins, outs,
1419 mlir::linalg::ElementwiseKind::select);
1420 linalgOp = linalgSelect;
1421 }
1422
1423 // Truncate back to resultTy if needed
1424 Value reducedRes = linalgOp->getResult(idx: 0);
1425 if (widenAccTy) {
1426 auto resEmptyOp =
1427 tensor::EmptyOp::create(rewriter, loc, reduceShape, elementTy, dynDims)
1428 .getResult();
1429
1430 const unsigned reducedRank =
1431 cast<ShapedType>(Val: reducedRes.getType()).getRank();
1432 auto identityMap = rewriter.getMultiDimIdentityMap(rank: reducedRank);
1433 reducedRes =
1434 linalg::GenericOp::create(
1435 rewriter, loc, resEmptyOp.getType(), ValueRange{reducedRes},
1436 ValueRange{resEmptyOp},
1437 ArrayRef<AffineMap>{identityMap, identityMap},
1438 getNParallelLoopsAttrs(nParallelLoops: reducedRank),
1439 [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
1440 Value truncf = arith::TruncFOp::create(nestedBuilder, nestedLoc,
1441 elementTy, args[0]);
1442 linalg::YieldOp::create(builder&: nestedBuilder, location: nestedLoc, values: truncf);
1443 })
1444 .getResults()[0];
1445 }
1446
1447 SmallVector<ReassociationExprs, 4> reassociationMap;
1448 uint64_t expandInputRank = cast<ShapedType>(Val: reducedRes.getType()).getRank();
1449 reassociationMap.resize(N: expandInputRank);
1450
1451 for (uint64_t i = 0; i < expandInputRank; i++) {
1452 int32_t dimToPush = i > axis ? i + 1 : i;
1453 reassociationMap[i].push_back(Elt: rewriter.getAffineDimExpr(position: dimToPush));
1454 }
1455
1456 if (expandInputRank != 0) {
1457 int32_t expandedDim = axis < expandInputRank ? axis : expandInputRank - 1;
1458 reassociationMap[expandedDim].push_back(
1459 Elt: rewriter.getAffineDimExpr(position: expandedDim + 1));
1460 }
1461
1462 // Lower directly to `tensor::ExpandShapeOp` instead of `tosa::ReshapeOp`,
1463 // since here we know which dimension to expand, and `tosa::ReshapeOp` would
1464 // not have access to such information. This matters when handling dynamically
1465 // sized tensors.
1466 rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(op, resultTy, reducedRes,
1467 reassociationMap);
1468 return success();
1469}
1470
1471// Insert missing leading batch dimensions, taking their sizes from `other`.
1472static Value broadcastMatMulLeadingDimensions(PatternRewriter &rewriter,
1473 Location loc, Value operand,
1474 Value other) {
1475 auto type = cast<RankedTensorType>(Val: operand.getType());
1476 int64_t missingDims =
1477 cast<RankedTensorType>(Val: other.getType()).getRank() - type.getRank();
1478 if (missingDims <= 0)
1479 return operand;
1480
1481 SmallVector<OpFoldResult> shape;
1482 SmallVector<int64_t> dimensions;
1483 for (int64_t dim = 0; dim < missingDims; ++dim) {
1484 shape.push_back(Elt: tensor::getMixedSize(builder&: rewriter, loc, value: other, dim));
1485 dimensions.push_back(Elt: dim);
1486 }
1487 llvm::append_range(C&: shape, R: tensor::getMixedSizes(builder&: rewriter, loc, value: operand));
1488 Value empty = tensor::EmptyOp::create(
1489 builder&: rewriter, location: loc, sizes: shape, elementType: type.getElementType(), encoding: type.getEncoding());
1490 return linalg::BroadcastOp::create(builder&: rewriter, location: loc, input: operand, init: empty, dimensions)
1491 .getResult()
1492 .front();
1493}
1494
1495// Expand an existing singleton batch dimension to `size`, preserving the rank.
1496// Missing dimensions are handled by broadcastMatMulLeadingDimensions.
1497// linalg.broadcast inserts dimensions, so first remove the singleton dimension
1498// with a rank-reducing slice, then reinsert it at the target size.
1499static Value broadcastMatMulSingletonDimension(PatternRewriter &rewriter,
1500 Location loc, Value operand,
1501 int64_t dim, OpFoldResult size) {
1502 auto type = cast<RankedTensorType>(Val: operand.getType());
1503 SmallVector<int64_t> sliceShape(type.getShape());
1504 sliceShape.erase(CI: sliceShape.begin() + dim);
1505 auto sliceType = RankedTensorType::get(shape: sliceShape, elementType: type.getElementType(),
1506 encoding: type.getEncoding());
1507 Value slice = tensor::createCanonicalRankReducingExtractSliceOp(
1508 b&: rewriter, loc, tensor: operand, targetType: sliceType);
1509
1510 SmallVector<OpFoldResult> shape =
1511 tensor::getMixedSizes(builder&: rewriter, loc, value: operand);
1512 shape[dim] = size;
1513 Value empty = tensor::EmptyOp::create(
1514 builder&: rewriter, location: loc, sizes: shape, elementType: type.getElementType(), encoding: type.getEncoding());
1515 return linalg::BroadcastOp::create(builder&: rewriter, location: loc, input: slice, init: empty,
1516 dimensions: ArrayRef<int64_t>{dim})
1517 .getResult()
1518 .front();
1519}
1520
1521// Align the ranks, then broadcast each shared batch dimension independently.
1522static std::pair<Value, Value>
1523normalizeMatMulBatchDimensions(PatternRewriter &rewriter, Location loc, Value a,
1524 Value b, int64_t batchRank) {
1525 int64_t aRank = cast<RankedTensorType>(Val: a.getType()).getRank();
1526 int64_t bRank = cast<RankedTensorType>(Val: b.getType()).getRank();
1527 int64_t firstSharedDim = std::max(a: aRank, b: bRank) - std::min(a: aRank, b: bRank);
1528 a = broadcastMatMulLeadingDimensions(rewriter, loc, operand: a, other: b);
1529 b = broadcastMatMulLeadingDimensions(rewriter, loc, operand: b, other: a);
1530
1531 IndexPool indexPool;
1532 // The inserted leading dimensions already match their partner's sizes.
1533 for (int64_t dim = firstSharedDim; dim < batchRank; ++dim) {
1534 auto aType = cast<RankedTensorType>(Val: a.getType());
1535 auto bType = cast<RankedTensorType>(Val: b.getType());
1536 int64_t aSize = aType.getDimSize(idx: dim);
1537 int64_t bSize = bType.getDimSize(idx: dim);
1538 if (aSize == bSize && ShapedType::isStatic(dValue: aSize))
1539 continue;
1540
1541 // Compute the shared broadcast target only for batch dimensions.
1542 // The matrix dimensions may differ between A and B.
1543 auto [size, masterOperand] =
1544 computeTargetSize(rewriter, loc, indexPool, operands: ValueRange{a, b}, dim);
1545 if (aSize == 1) {
1546 a = broadcastMatMulSingletonDimension(rewriter, loc, operand: a, dim, size);
1547 continue;
1548 }
1549 if (bSize == 1) {
1550 b = broadcastMatMulSingletonDimension(rewriter, loc, operand: b, dim, size);
1551 continue;
1552 }
1553 a = broadcastDynamicDimension(rewriter, loc, indexPool, operand: a, dim, targetSize: size,
1554 masterOperand);
1555 b = broadcastDynamicDimension(rewriter, loc, indexPool, operand: b, dim, targetSize: size,
1556 masterOperand);
1557 }
1558 return {a, b};
1559}
1560
1561MatMulTOperands mlir::tosa::prepareMatMulOperands(PatternRewriter &rewriter,
1562 Location loc, Value a,
1563 Value b,
1564 RankedTensorType outputType,
1565 bool transposeB) {
1566 const int64_t outputRank = outputType.getRank();
1567 const int64_t batchRank = outputRank - 2;
1568 auto normalizedOperands =
1569 normalizeMatMulBatchDimensions(rewriter, loc, a, b, batchRank);
1570 a = normalizedOperands.first;
1571 b = normalizedOperands.second;
1572
1573 SmallVector<OpFoldResult> outputShape;
1574 outputShape.reserve(N: outputRank);
1575 for (int64_t batchDim = 0; batchDim < batchRank; ++batchDim)
1576 outputShape.push_back(Elt: tensor::getMixedSize(builder&: rewriter, loc, value: a, dim: batchDim));
1577 outputShape.push_back(Elt: tensor::getMixedSize(builder&: rewriter, loc, value: a, dim: outputRank - 2));
1578 outputShape.push_back(
1579 Elt: tensor::getMixedSize(builder&: rewriter, loc, value: b, dim: outputRank - 1 - transposeB));
1580 SmallVector<Value> dynamicDims;
1581 for (auto [dim, size] : llvm::enumerate(First&: outputShape))
1582 if (outputType.isDynamicDim(idx: dim))
1583 dynamicDims.push_back(
1584 Elt: getValueOrCreateConstantIndexOp(b&: rewriter, loc, ofr: size));
1585
1586 Value zero = arith::ConstantOp::create(
1587 builder&: rewriter, location: loc, value: rewriter.getZeroAttr(type: outputType.getElementType()));
1588 Value emptyTensor = tensor::EmptyOp::create(
1589 builder&: rewriter, location: loc, staticShape: outputType.getShape(), elementType: outputType.getElementType(),
1590 dynamicSizes: condenseValues(values: dynamicDims));
1591 Value zeroTensor = linalg::FillOp::create(builder&: rewriter, location: loc, inputs: ValueRange{zero},
1592 outputs: ValueRange{emptyTensor})
1593 .result();
1594
1595 return {.a: a, .b: b, .init: zeroTensor};
1596}
1597
1598namespace {
1599// Linalg has quantized matmul and batch matmul operations, but neither has a
1600// transpose-B variant. Lower nonzero zero points separately to linalg.generic
1601// so the reduction can accumulate (A - aZp) * (B - bZp).
1602class QuantizedMatMulTConverter : public OpConversionPattern<tosa::MatMulTOp> {
1603public:
1604 using OpConversionPattern<tosa::MatMulTOp>::OpConversionPattern;
1605 LogicalResult
1606 matchAndRewrite(tosa::MatMulTOp op, OpAdaptor adaptor,
1607 ConversionPatternRewriter &rewriter) const final {
1608 Location loc = op.getLoc();
1609 auto aType = dyn_cast<RankedTensorType>(Val: adaptor.getA().getType());
1610 auto bType = dyn_cast<RankedTensorType>(Val: adaptor.getB().getType());
1611 auto outputType = dyn_cast<RankedTensorType>(Val: op.getType());
1612 if (!aType || !bType || !outputType)
1613 return rewriter.notifyMatchFailure(arg&: op, msg: "only supports ranked tensors");
1614
1615 FailureOr<int64_t> maybeAZp = op.getAZeroPoint();
1616 FailureOr<int64_t> maybeBZp = op.getBZeroPoint();
1617 if (failed(Result: maybeAZp))
1618 return rewriter.notifyMatchFailure(
1619 arg&: op, msg: "input A zero point cannot be statically determined");
1620 if (failed(Result: maybeBZp))
1621 return rewriter.notifyMatchFailure(
1622 arg&: op, msg: "input B zero point cannot be statically determined");
1623
1624 const int64_t aZpVal = *maybeAZp;
1625 const int64_t bZpVal = *maybeBZp;
1626
1627 if (aZpVal == 0 && bZpVal == 0)
1628 return rewriter.notifyMatchFailure(
1629 arg&: op, msg: "requires at least one nonzero zero point");
1630
1631 const int64_t outputRank = outputType.getRank();
1632 const int64_t batchRank = outputRank - 2;
1633 auto [a, b, zeroTensor] =
1634 prepareMatMulOperands(rewriter, loc, a: adaptor.getA(), b: adaptor.getB(),
1635 outputType, /*transposeB=*/true);
1636 Type outputElementType = outputType.getElementType();
1637
1638 int64_t loopRank = outputRank + 1;
1639 SmallVector<AffineExpr> dimensions =
1640 llvm::map_to_vector(C: llvm::seq<int64_t>(Begin: 0, End: loopRank), F: [&](int64_t dim) {
1641 return rewriter.getAffineDimExpr(position: dim);
1642 });
1643 SmallVector<AffineExpr> aMap(dimensions.begin(),
1644 dimensions.begin() + batchRank);
1645 aMap.push_back(Elt: dimensions[batchRank]);
1646 aMap.push_back(Elt: dimensions[batchRank + 2]);
1647 SmallVector<AffineExpr> bMap(dimensions.begin(),
1648 dimensions.begin() + batchRank);
1649 bMap.push_back(Elt: dimensions[batchRank + 1]);
1650 bMap.push_back(Elt: dimensions[batchRank + 2]);
1651 SmallVector<AffineExpr> outputMap(dimensions.begin(),
1652 dimensions.begin() + batchRank);
1653 outputMap.push_back(Elt: dimensions[batchRank]);
1654 outputMap.push_back(Elt: dimensions[batchRank + 1]);
1655 SmallVector<AffineMap> indexingMaps = {
1656 AffineMap::get(dimCount: loopRank, symbolCount: 0, results: aMap, context: rewriter.getContext()),
1657 AffineMap::get(dimCount: loopRank, symbolCount: 0, results: bMap, context: rewriter.getContext()),
1658 AffineMap::get(dimCount: loopRank, symbolCount: 0, results: outputMap, context: rewriter.getContext())};
1659 SmallVector<utils::IteratorType> iteratorTypes(
1660 loopRank, utils::IteratorType::parallel);
1661 iteratorTypes.back() = utils::IteratorType::reduction;
1662
1663 auto generic = linalg::GenericOp::create(
1664 builder&: rewriter, location: loc, resultTensorTypes: outputType, inputs: ValueRange{a, b}, outputs: zeroTensor, indexingMaps,
1665 iteratorTypes,
1666 odsArg5: [=](OpBuilder &builder, Location nestedLoc, ValueRange args) {
1667 auto extendToOutputType = [&](Value value) {
1668 if (value.getType() == outputElementType)
1669 return value;
1670 return arith::ExtSIOp::create(builder, location: nestedLoc, out: outputElementType,
1671 in: value)
1672 .getResult();
1673 };
1674
1675 Value aValue = extendToOutputType(args[0]);
1676 Value bValue = extendToOutputType(args[1]);
1677 Value aZp = arith::ConstantOp::create(
1678 builder, location: nestedLoc,
1679 value: builder.getIntegerAttr(type: outputElementType, value: aZpVal));
1680 Value bZp = arith::ConstantOp::create(
1681 builder, location: nestedLoc,
1682 value: builder.getIntegerAttr(type: outputElementType, value: bZpVal));
1683 Value adjustedA =
1684 arith::SubIOp::create(builder, location: nestedLoc, lhs: aValue, rhs: aZp);
1685 Value adjustedB =
1686 arith::SubIOp::create(builder, location: nestedLoc, lhs: bValue, rhs: bZp);
1687 Value product =
1688 arith::MulIOp::create(builder, location: nestedLoc, lhs: adjustedA, rhs: adjustedB);
1689 Value sum =
1690 arith::AddIOp::create(builder, location: nestedLoc, lhs: args[2], rhs: product);
1691 linalg::YieldOp::create(builder, location: nestedLoc, values: sum);
1692 });
1693 rewriter.replaceOp(op, newValues: generic.getResult(i: 0));
1694 return success();
1695 }
1696};
1697
1698template <typename SrcOp>
1699class PointwiseConverter : public OpConversionPattern<SrcOp> {
1700public:
1701 using OpConversionPattern<SrcOp>::OpConversionPattern;
1702 using typename OpConversionPattern<SrcOp>::OpAdaptor;
1703
1704 LogicalResult
1705 matchAndRewrite(SrcOp op, OpAdaptor operands,
1706 ConversionPatternRewriter &rewriter) const final {
1707 return elementwiseMatchAndRewriteHelper(
1708 op, operands.getOperands(), rewriter, *this->getTypeConverter());
1709 }
1710};
1711
1712// Collapse tensor<1xiN> into tensor<iN>
1713// E.g. tensor.collapse_shape %arg1 [] : tensor<1xi16> into tensor<i16>
1714static Value collapse1xNTensorToN(PatternRewriter &rewriter, Value input,
1715 Location loc) {
1716 SmallVector<ReassociationExprs, 1> reassociation;
1717 // Create the collapsed type
1718 auto inputType = cast<RankedTensorType>(Val: input.getType());
1719 auto elemType = inputType.getElementType();
1720 auto collapsedType = RankedTensorType::get(shape: {}, elementType: elemType);
1721 // Emit the collapse op
1722 return tensor::CollapseShapeOp::create(builder&: rewriter, location: loc, resultType: collapsedType, src: input,
1723 reassociation);
1724}
1725
1726static llvm::SmallVector<int8_t>
1727convertToI8(const llvm::SmallVector<int32_t> &input) {
1728 llvm::SmallVector<int8_t> output;
1729 output.reserve(N: input.size());
1730
1731 for (auto v : llvm::map_range(
1732 C: input, F: [](int32_t val) { return static_cast<int8_t>(val); })) {
1733 output.push_back(Elt: v);
1734 }
1735 return output;
1736}
1737
1738// The shift or multiplier may be either constant or non-constant, depending on
1739// whether dynamic extension is enabled.
1740// - If the shift or multiplier is non-constant, add it as an input to
1741// linalg::GenericOp by:
1742// 1. Pushing it into 'genericInputs'.
1743// 2. Appending a corresponding affine map to 'indexingMaps'.
1744// - If the shift or multiplier is constant, set 'constant' instead.
1745static void setupLinalgGenericOpInputAndIndexingMap(
1746 PatternRewriter &rewriter, llvm::SmallVector<int32_t> &values,
1747 SmallVector<Value, 4> &genericInputs, SmallVector<AffineMap> &indexingMaps,
1748 bool isConstant, tosa::RescaleOp op, Value &constant, int64_t &arg,
1749 bool isShift = false) {
1750
1751 auto loc = op.getLoc();
1752 auto inputTy = cast<ShapedType>(Val: op.getInput().getType());
1753 unsigned rank = inputTy.getRank();
1754 SmallVector<AffineExpr, 2> exprs = {rewriter.getAffineDimExpr(position: rank - 1)};
1755
1756 if (isConstant) {
1757 // If we are rescaling per-channel then we need to store the
1758 // values in a buffer.
1759 if (values.size() == 1) {
1760 IntegerAttr intAttr = isShift
1761 ? rewriter.getI8IntegerAttr(value: values.front())
1762 : rewriter.getI32IntegerAttr(value: values.front());
1763 constant = arith::ConstantOp::create(builder&: rewriter, location: loc, value: intAttr);
1764 } else {
1765 auto elementType =
1766 isShift ? rewriter.getIntegerType(width: 8) : rewriter.getI32Type();
1767 auto tensorType = RankedTensorType::get(
1768 shape: {static_cast<int64_t>(values.size())}, elementType);
1769 DenseIntElementsAttr EltAttr;
1770 if (isShift)
1771 EltAttr = DenseIntElementsAttr::get(type: tensorType, arg: convertToI8(input: values));
1772 else
1773 EltAttr = DenseIntElementsAttr::get(type: tensorType, arg&: values);
1774 genericInputs.push_back(
1775 Elt: arith::ConstantOp::create(builder&: rewriter, location: loc, value: EltAttr));
1776 indexingMaps.push_back(Elt: AffineMap::get(/*dimCount=*/rank,
1777 /*symbolCount=*/0, results: exprs,
1778 context: rewriter.getContext()));
1779 }
1780 } else {
1781 // If we are not rescaling per-channel then we need to collapse 1xN to N
1782 // and push broadcastMap.
1783 auto operand = isShift ? op.getShift() : op.getMultiplier();
1784 auto tensorType = dyn_cast<RankedTensorType>(Val: operand.getType());
1785 if (tensorType && tensorType.hasStaticShape() &&
1786 tensorType.getShape()[0] == 1) {
1787 // broadcastMap = affine_map<(d0, d1) -> ()>
1788 // It would affect as broadcast for scalar values in linalg::GenericOp.
1789 AffineMap broadcastMap =
1790 AffineMap::get(dimCount: rank, symbolCount: 0, results: {}, context: rewriter.getContext());
1791 genericInputs.push_back(Elt: collapse1xNTensorToN(rewriter, input: operand, loc));
1792 indexingMaps.push_back(Elt: broadcastMap);
1793 } else {
1794 genericInputs.push_back(Elt: operand);
1795 indexingMaps.push_back(Elt: AffineMap::get(/*dimCount=*/rank,
1796 /*symbolCount=*/0, results: exprs,
1797 context: rewriter.getContext()));
1798 }
1799 }
1800 arg = indexingMaps.size() - 1;
1801}
1802
1803// Return the extended Zp to be used in subsequent arithmetic operations.
1804static Value getExtendZp(OpBuilder &builder, Type valueTy,
1805 FailureOr<int64_t> maybeZp, Location loc,
1806 ValueRange blockArgs, int64_t zpArg,
1807 bool isOutputZp = false) {
1808 Value result;
1809 const int32_t bitwidth = valueTy.getIntOrFloatBitWidth();
1810 const uint32_t attrBitwidth =
1811 isOutputZp ? 32 : (bitwidth > 32 ? bitwidth : 32);
1812 auto extendType = builder.getIntegerType(width: attrBitwidth);
1813 // The Zp value can be either constant or non-constant, depending on
1814 // whether dynamic extension is enabled.
1815 // If 'maybeZp' fails, it indicates that Zp is non-constant and will
1816 // be passed as an input to linalg::GenericOp.
1817 if (failed(Result: maybeZp)) {
1818 result = blockArgs[zpArg];
1819 auto zpTy = result.getType();
1820 if (zpTy.getIntOrFloatBitWidth() < attrBitwidth) {
1821 // For ExtUIOp, the input must be signless.
1822 // UnrealizedConversionCastOp will cast the input to signless type.
1823 if (zpTy.isUnsignedInteger()) {
1824 result =
1825 UnrealizedConversionCastOp::create(
1826 builder, location: loc,
1827 resultTypes: builder.getIntegerType(width: zpTy.getIntOrFloatBitWidth()), operands: result)
1828 .getResult(i: 0);
1829 }
1830 if (zpTy.isUnsignedInteger()) {
1831 return arith::ExtUIOp::create(builder, location: loc, out: extendType, in: result);
1832 } else {
1833 return arith::ExtSIOp::create(builder, location: loc, out: extendType, in: result);
1834 }
1835 }
1836 } else {
1837 return arith::ConstantOp::create(builder, location: loc,
1838 value: IntegerAttr::get(type: extendType, value: *maybeZp));
1839 }
1840 return result;
1841}
1842
1843class RescaleConverter : public OpRewritePattern<tosa::RescaleOp> {
1844public:
1845 using OpRewritePattern<tosa::RescaleOp>::OpRewritePattern;
1846
1847 LogicalResult matchAndRewrite(tosa::RescaleOp op,
1848 PatternRewriter &rewriter) const final {
1849 auto loc = op.getLoc();
1850 auto input = op.getInput();
1851 auto inputTy = cast<ShapedType>(Val: op.getInput().getType());
1852 auto outputTy = cast<ShapedType>(Val: op.getOutput().getType());
1853 unsigned rank = inputTy.getRank();
1854
1855 // This is an illegal configuration. terminate and log an error
1856 if (op.getRoundingMode() == RoundingMode::INEXACT_ROUND)
1857 return rewriter.notifyMatchFailure(
1858 arg&: op, msg: "tosa.rescale with rounding mode = 'INEXACT_ROUND' is not "
1859 "currently supported");
1860 if (op.getRoundingMode() == RoundingMode::DOUBLE_ROUND && !op.getScale32())
1861 return rewriter.notifyMatchFailure(
1862 arg&: op, msg: "tosa.rescale requires scale32 for double_round to be true");
1863
1864 if (!isa<IntegerType>(Val: inputTy.getElementType()))
1865 return rewriter.notifyMatchFailure(arg&: op, msg: "only support integer type");
1866
1867 // The generic body computes in i32 and saturates to the output type.
1868 auto outputElemTy = dyn_cast<IntegerType>(Val: outputTy.getElementType());
1869 if (!outputElemTy || outputElemTy.getWidth() > 32)
1870 return rewriter.notifyMatchFailure(
1871 arg&: op, msg: "only support integer output types up to 32 bits");
1872
1873 // Zero points are already validated by tosa-validate under
1874 // strict-op-spec-alignment, but reject invalid values here as well so
1875 // that conversion without validation fails instead of silently
1876 // generating incorrect arithmetic.
1877 FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
1878 if (succeeded(Result: maybeIZp) && op.verifyInputZeroPoint(zp: *maybeIZp).failed())
1879 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported input zero point");
1880 FailureOr<int64_t> maybeOZp = op.getOutputZeroPoint();
1881 if (succeeded(Result: maybeOZp) && op.verifyOutputZeroPoint(zp: *maybeOZp).failed())
1882 return rewriter.notifyMatchFailure(arg&: op, msg: "unsupported output zero point");
1883
1884 SmallVector<Value> dynDims;
1885 for (int i = 0; i < outputTy.getRank(); i++) {
1886 if (outputTy.isDynamicDim(idx: i)) {
1887 dynDims.push_back(Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: i));
1888 }
1889 }
1890
1891 DenseElementsAttr shiftElems;
1892 bool isShiftConstant = false;
1893 if (matchPattern(value: op.getShift(), pattern: m_Constant(bind_value: &shiftElems)))
1894 isShiftConstant = true;
1895
1896 DenseElementsAttr multiplierElems;
1897 bool isMultiplierConstant = false;
1898 if (matchPattern(value: op.getMultiplier(), pattern: m_Constant(bind_value: &multiplierElems)))
1899 isMultiplierConstant = true;
1900
1901 llvm::SmallVector<int32_t> shiftValues;
1902 llvm::SmallVector<int32_t> multiplierValues;
1903 bool doubleRound;
1904
1905 if (isMultiplierConstant && isShiftConstant) {
1906 // explicit cast is required here
1907 shiftValues = llvm::map_to_vector(
1908 C: shiftElems.getValues<IntegerAttr>(), F: [](IntegerAttr attr) -> int32_t {
1909 return static_cast<int32_t>(attr.getInt());
1910 });
1911 multiplierValues =
1912 llvm::map_to_vector(C: multiplierElems.getValues<IntegerAttr>(),
1913 F: [](IntegerAttr attr) -> int32_t {
1914 return static_cast<int32_t>(attr.getInt());
1915 });
1916
1917 // If we shift by more than the bitwidth, this just sets to 0.
1918 for (int i = 0, s = multiplierValues.size(); i < s; i++) {
1919 if (shiftValues[i] > 63) {
1920 shiftValues[i] = 0;
1921 multiplierValues[i] = 0;
1922 }
1923 }
1924 // Double round only occurs if shift is greater than 31, check that this
1925 // is ever true.
1926 doubleRound = op.getRoundingMode() == RoundingMode::DOUBLE_ROUND &&
1927 llvm::any_of(Range&: shiftValues, P: [](int32_t v) { return v > 31; });
1928 } else
1929 doubleRound = op.getRoundingMode() == RoundingMode::DOUBLE_ROUND;
1930
1931 RoundingMode roundingMode =
1932 doubleRound ? RoundingMode::DOUBLE_ROUND : RoundingMode::SINGLE_ROUND;
1933
1934 SmallVector<AffineMap> indexingMaps = {
1935 rewriter.getMultiDimIdentityMap(rank)};
1936 SmallVector<Value, 4> genericInputs = {input};
1937
1938 // If we are rescaling per-channel then we need to store the multiplier
1939 // values in a buffer.
1940 Value multiplierConstant;
1941 int64_t multiplierArg = 0;
1942 setupLinalgGenericOpInputAndIndexingMap(
1943 rewriter, values&: multiplierValues, genericInputs, indexingMaps,
1944 isConstant: isMultiplierConstant, op, constant&: multiplierConstant, arg&: multiplierArg);
1945
1946 // If we are rescaling per-channel then we need to store the shift
1947 // values in a buffer.
1948 Value shiftConstant;
1949 int64_t shiftArg = 0;
1950 setupLinalgGenericOpInputAndIndexingMap(
1951 rewriter, values&: shiftValues, genericInputs, indexingMaps, isConstant: isShiftConstant, op,
1952 constant&: shiftConstant, arg&: shiftArg, isShift: true);
1953
1954 // broadcastMap = affine_map<(d0, d1) -> ()>
1955 // It would affect as broadcast for scalar values in linalg::GenericOp.
1956 AffineMap broadcastMap = AffineMap::get(dimCount: rank, symbolCount: 0, results: {}, context: rewriter.getContext());
1957 // The inputZp and outputZp may be either constant or non-constant,
1958 // depending on whether dynamic extension is enabled.
1959 // - If the zp's are non-constant, add them as an inputs to
1960 // linalg::GenericOp by:
1961 // 1. Pushing it into 'genericInputs'.
1962 // 2. Appending a corresponding affine map to 'indexingMaps'.
1963 // - If the zp's are constant, they would be generated as arith.constant.
1964 int64_t iZpArg = 0;
1965 if (failed(Result: maybeIZp)) {
1966 genericInputs.push_back(
1967 Elt: collapse1xNTensorToN(rewriter, input: op->getOperand(idx: 3), loc));
1968 indexingMaps.push_back(Elt: broadcastMap);
1969 iZpArg = indexingMaps.size() - 1;
1970 }
1971 int64_t oZpArg = 0;
1972 if (failed(Result: maybeOZp)) {
1973 genericInputs.push_back(
1974 Elt: collapse1xNTensorToN(rewriter, input: op->getOperand(idx: 4), loc));
1975 indexingMaps.push_back(Elt: broadcastMap);
1976 oZpArg = indexingMaps.size() - 1;
1977 }
1978
1979 // Indexing maps for output values.
1980 indexingMaps.push_back(Elt: rewriter.getMultiDimIdentityMap(rank));
1981
1982 // Construct the indexing maps needed for linalg.generic ops.
1983 Value emptyTensor = tensor::EmptyOp::create(
1984 builder&: rewriter, location: loc, staticShape: outputTy.getShape(), elementType: outputTy.getElementType(),
1985 dynamicSizes: ArrayRef<Value>({dynDims}));
1986
1987 auto linalgOp = linalg::GenericOp::create(
1988 builder&: rewriter, location: loc, resultTensorTypes: outputTy, inputs: genericInputs, outputs: ValueRange{emptyTensor},
1989 indexingMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: rank),
1990 odsArg5: [&](OpBuilder &nestedBuilder, Location nestedLoc,
1991 ValueRange blockArgs) {
1992 Value value = blockArgs[0];
1993 Type valueTy = value.getType();
1994
1995 FailureOr<int64_t> maybeIZp = op.getInputZeroPoint();
1996 auto inputZp = getExtendZp(builder&: nestedBuilder, valueTy, maybeZp: maybeIZp,
1997 loc: nestedLoc, blockArgs, zpArg: iZpArg);
1998
1999 FailureOr<int64_t> maybeOZp = op.getOutputZeroPoint();
2000 auto outputZp = getExtendZp(builder&: nestedBuilder, valueTy, maybeZp: maybeOZp,
2001 loc: nestedLoc, blockArgs, zpArg: oZpArg, isOutputZp: true);
2002
2003 IntegerType outIntType =
2004 cast<IntegerType>(Val: blockArgs.back().getType());
2005 unsigned outBitWidth = outIntType.getWidth();
2006 assert(outBitWidth <= 32 && "Unexpected output zeropoint bitwidth");
2007
2008 Value multiplier = multiplierConstant ? multiplierConstant
2009 : blockArgs[multiplierArg];
2010 Value shift = shiftConstant ? shiftConstant : blockArgs[shiftArg];
2011
2012 if (valueTy.isUnsignedInteger()) {
2013 value = UnrealizedConversionCastOp::create(
2014 builder&: nestedBuilder, location: nestedLoc,
2015 resultTypes: nestedBuilder.getIntegerType(
2016 width: valueTy.getIntOrFloatBitWidth()),
2017 operands: value)
2018 .getResult(i: 0);
2019 }
2020 if (valueTy.getIntOrFloatBitWidth() < 32) {
2021 if (op.getInputUnsigned()) {
2022 value = arith::ExtUIOp::create(builder&: nestedBuilder, location: nestedLoc,
2023 out: nestedBuilder.getI32Type(), in: value);
2024 } else {
2025 value = arith::ExtSIOp::create(builder&: nestedBuilder, location: nestedLoc,
2026 out: nestedBuilder.getI32Type(), in: value);
2027 }
2028 }
2029
2030 value =
2031 arith::SubIOp::create(builder&: nestedBuilder, location: nestedLoc, lhs: value, rhs: inputZp);
2032
2033 value = tosa::ApplyScaleOp::create(builder&: nestedBuilder, location: loc,
2034 output: nestedBuilder.getI32Type(), value,
2035 multiplier, shift, rounding_mode: roundingMode);
2036
2037 // Move to the new zero-point.
2038 value =
2039 arith::AddIOp::create(builder&: nestedBuilder, location: nestedLoc, lhs: value, rhs: outputZp);
2040
2041 // Saturate to the output size.
2042 int32_t intMin = APInt::getSignedMinValue(numBits: outBitWidth).getSExtValue();
2043 int32_t intMax = APInt::getSignedMaxValue(numBits: outBitWidth).getSExtValue();
2044
2045 // Unsigned integers have a difference output value.
2046 if (op.getOutputUnsigned()) {
2047 intMin = 0;
2048 intMax = APInt::getMaxValue(numBits: outBitWidth).getZExtValue();
2049 }
2050
2051 auto intMinVal = arith::ConstantOp::create(
2052 builder&: nestedBuilder, location: loc, value: nestedBuilder.getI32IntegerAttr(value: intMin));
2053 auto intMaxVal = arith::ConstantOp::create(
2054 builder&: nestedBuilder, location: loc, value: nestedBuilder.getI32IntegerAttr(value: intMax));
2055
2056 value = clampIntHelper(loc: nestedLoc, arg: value, min: intMinVal, max: intMaxVal,
2057 rewriter&: nestedBuilder, /*isUnsigned=*/false);
2058
2059 if (outIntType.getWidth() < 32) {
2060 value = arith::TruncIOp::create(
2061 builder&: nestedBuilder, location: nestedLoc,
2062 out: rewriter.getIntegerType(width: outIntType.getWidth()), in: value);
2063 }
2064
2065 if (outIntType.isUnsignedInteger()) {
2066 value = UnrealizedConversionCastOp::create(builder&: nestedBuilder, location: nestedLoc,
2067 resultTypes: outIntType, operands: value)
2068 .getResult(i: 0);
2069 }
2070 linalg::YieldOp::create(builder&: nestedBuilder, location: loc, values: value);
2071 });
2072
2073 rewriter.replaceOp(op, newValues: linalgOp->getResults());
2074 return success();
2075 }
2076};
2077
2078// Handle the resize case where the input is a 1x1 image. This case
2079// can entirely avoiding having extract operations which target much
2080// more difficult to optimize away.
2081class ResizeUnaryConverter : public OpRewritePattern<tosa::ResizeOp> {
2082public:
2083 using OpRewritePattern<tosa::ResizeOp>::OpRewritePattern;
2084
2085 LogicalResult matchAndRewrite(tosa::ResizeOp op,
2086 PatternRewriter &rewriter) const final {
2087 Location loc = op.getLoc();
2088 ImplicitLocOpBuilder builder(loc, rewriter);
2089 auto input = op.getInput();
2090 auto inputTy = cast<RankedTensorType>(Val: input.getType());
2091 auto resultTy = cast<RankedTensorType>(Val: op.getType());
2092 const bool isBilinear = op.getMode() == ResizeMode::BILINEAR;
2093
2094 auto inputH = inputTy.getDimSize(idx: 1);
2095 auto inputW = inputTy.getDimSize(idx: 2);
2096 auto outputH = resultTy.getDimSize(idx: 1);
2097 auto outputW = resultTy.getDimSize(idx: 2);
2098
2099 if (inputH != 1 || inputW != 1 || outputH != 1 || outputW != 1)
2100 return rewriter.notifyMatchFailure(
2101 arg&: op, msg: "tosa.resize is not a pure 1x1->1x1 image operation");
2102
2103 if (op.getMode() != ResizeMode::NEAREST_NEIGHBOR &&
2104 op.getMode() != ResizeMode::BILINEAR)
2105 return rewriter.notifyMatchFailure(
2106 arg&: op, msg: "tosa.resize mode should be NEAREST_NEIGHBOR or BILINEAR");
2107
2108 if (inputTy == resultTy) {
2109 rewriter.replaceOp(op, newValues: input);
2110 return success();
2111 }
2112
2113 SmallVector<int64_t> scale;
2114 if (!tosa::getConstShapeValues(op: op.getScale().getDefiningOp(), result_shape&: scale)) {
2115 return failure();
2116 }
2117
2118 // Collapse the unit width and height away.
2119 SmallVector<ReassociationExprs, 4> reassociationMap(2);
2120 reassociationMap[0].push_back(Elt: builder.getAffineDimExpr(position: 0));
2121 reassociationMap[1].push_back(Elt: builder.getAffineDimExpr(position: 1));
2122 reassociationMap[1].push_back(Elt: builder.getAffineDimExpr(position: 2));
2123 reassociationMap[1].push_back(Elt: builder.getAffineDimExpr(position: 3));
2124
2125 auto collapseTy =
2126 RankedTensorType::get(shape: {inputTy.getDimSize(idx: 0), inputTy.getDimSize(idx: 3)},
2127 elementType: inputTy.getElementType());
2128 Value collapse = tensor::CollapseShapeOp::create(builder, resultType: collapseTy, src: input,
2129 reassociation: reassociationMap);
2130
2131 // Get any dynamic shapes that appear in the input format.
2132 llvm::SmallVector<Value> outputDynSize;
2133 if (inputTy.isDynamicDim(idx: 0))
2134 outputDynSize.push_back(Elt: tensor::DimOp::create(builder, source: input, index: 0));
2135 if (inputTy.isDynamicDim(idx: 3))
2136 outputDynSize.push_back(Elt: tensor::DimOp::create(builder, source: input, index: 3));
2137
2138 // Generate the elementwise operation for casting scaling the input value.
2139 auto genericTy = collapseTy.clone(elementType: resultTy.getElementType());
2140 Value empty =
2141 tensor::EmptyOp::create(builder, staticShape: genericTy.getShape(),
2142 elementType: resultTy.getElementType(), dynamicSizes: outputDynSize);
2143 auto genericMap = rewriter.getMultiDimIdentityMap(rank: genericTy.getRank());
2144 SmallVector<utils::IteratorType> iterators(genericTy.getRank(),
2145 utils::IteratorType::parallel);
2146
2147 auto generic = linalg::GenericOp::create(
2148 builder, resultTensorTypes: genericTy, inputs: ValueRange{collapse}, outputs: ValueRange{empty},
2149 indexingMaps: ArrayRef<AffineMap>{genericMap, genericMap}, iteratorTypes: iterators,
2150 odsArg5: [=](OpBuilder &b, Location loc, ValueRange args) {
2151 Value value = args[0];
2152 // This is the quantized case.
2153 if (inputTy.getElementType() != resultTy.getElementType()) {
2154 value = arith::ExtSIOp::create(builder&: b, location: loc, out: resultTy.getElementType(),
2155 in: value);
2156
2157 if (isBilinear && scale[0] != 0) {
2158 Value scaleY = arith::ConstantOp::create(
2159 builder&: b, location: loc, value: b.getI32IntegerAttr(value: scale[0]));
2160 value = arith::MulIOp::create(builder&: b, location: loc, lhs: value, rhs: scaleY);
2161 }
2162
2163 if (isBilinear && scale[2] != 0) {
2164 Value scaleX = arith::ConstantOp::create(
2165 builder&: b, location: loc, value: b.getI32IntegerAttr(value: scale[2]));
2166 value = arith::MulIOp::create(builder&: b, location: loc, lhs: value, rhs: scaleX);
2167 }
2168 }
2169
2170 linalg::YieldOp::create(builder&: b, location: loc, values: value);
2171 });
2172
2173 rewriter.replaceOpWithNewOp<tensor::ExpandShapeOp>(
2174 op, args&: resultTy, args: generic.getResults()[0], args&: reassociationMap);
2175 return success();
2176 }
2177};
2178
2179// TOSA resize with width or height of 1 may be broadcasted to a wider
2180// dimension. This is done by materializing a new tosa.resize without
2181// the broadcasting behavior, and an explicit broadcast afterwards.
2182class MaterializeResizeBroadcast : public OpRewritePattern<tosa::ResizeOp> {
2183public:
2184 using OpRewritePattern<tosa::ResizeOp>::OpRewritePattern;
2185
2186 LogicalResult matchAndRewrite(tosa::ResizeOp op,
2187 PatternRewriter &rewriter) const final {
2188 Location loc = op.getLoc();
2189 ImplicitLocOpBuilder builder(loc, rewriter);
2190 auto input = op.getInput();
2191 auto inputTy = dyn_cast<RankedTensorType>(Val: input.getType());
2192 auto resultTy = dyn_cast<RankedTensorType>(Val: op.getType());
2193
2194 if (!inputTy || !resultTy)
2195 return rewriter.notifyMatchFailure(arg&: op,
2196 msg: "requires ranked input/output types");
2197
2198 auto batch = inputTy.getDimSize(idx: 0);
2199 auto channels = inputTy.getDimSize(idx: 3);
2200 auto inputH = inputTy.getDimSize(idx: 1);
2201 auto inputW = inputTy.getDimSize(idx: 2);
2202 auto outputH = resultTy.getDimSize(idx: 1);
2203 auto outputW = resultTy.getDimSize(idx: 2);
2204
2205 if ((inputH != 1 || outputH == 1) && (inputW != 1 || outputW == 1))
2206 return rewriter.notifyMatchFailure(
2207 arg&: op, msg: "tosa.resize has no broadcasting behavior");
2208
2209 // For any dimension that is broadcastable we generate a width of 1
2210 // on the output.
2211 llvm::SmallVector<int64_t> resizeShape;
2212 resizeShape.push_back(Elt: batch);
2213 resizeShape.push_back(Elt: inputH == 1 ? 1 : outputH);
2214 resizeShape.push_back(Elt: inputW == 1 ? 1 : outputW);
2215 resizeShape.push_back(Elt: channels);
2216
2217 auto resizeTy = resultTy.clone(shape: resizeShape);
2218 auto resize =
2219 tosa::ResizeOp::create(builder, output: resizeTy, input, scale: op.getScale(),
2220 offset: op.getOffset(), border: op.getBorder(), mode: op.getMode());
2221
2222 // Collapse an unit result dims.
2223 SmallVector<ReassociationExprs, 4> reassociationMap(2);
2224 reassociationMap[0].push_back(Elt: builder.getAffineDimExpr(position: 0));
2225 reassociationMap.back().push_back(Elt: builder.getAffineDimExpr(position: 1));
2226 if (inputH != 1)
2227 reassociationMap.push_back(Elt: {});
2228 reassociationMap.back().push_back(Elt: builder.getAffineDimExpr(position: 2));
2229 if (inputW != 1)
2230 reassociationMap.push_back(Elt: {});
2231 reassociationMap.back().push_back(Elt: builder.getAffineDimExpr(position: 3));
2232
2233 llvm::SmallVector<int64_t> collapseShape = {batch};
2234 if (inputH != 1)
2235 collapseShape.push_back(Elt: outputH);
2236 if (inputW != 1)
2237 collapseShape.push_back(Elt: outputW);
2238 collapseShape.push_back(Elt: channels);
2239
2240 auto collapseTy = resultTy.clone(shape: collapseShape);
2241 Value collapse = tensor::CollapseShapeOp::create(builder, resultType: collapseTy,
2242 src: resize, reassociation: reassociationMap);
2243
2244 // Broadcast the collapsed shape to the output result.
2245 llvm::SmallVector<Value> outputDynSize;
2246 if (inputTy.isDynamicDim(idx: 0))
2247 outputDynSize.push_back(Elt: tensor::DimOp::create(builder, source: input, index: 0));
2248 if (inputTy.isDynamicDim(idx: 3))
2249 outputDynSize.push_back(Elt: tensor::DimOp::create(builder, source: input, index: 3));
2250
2251 SmallVector<utils::IteratorType> iterators(resultTy.getRank(),
2252 utils::IteratorType::parallel);
2253 Value empty = tensor::EmptyOp::create(
2254 builder, staticShape: resultTy.getShape(), elementType: resultTy.getElementType(), dynamicSizes: outputDynSize);
2255
2256 SmallVector<AffineExpr, 4> inputExprs{rewriter.getAffineDimExpr(position: 0)};
2257 if (inputH != 1)
2258 inputExprs.push_back(Elt: rewriter.getAffineDimExpr(position: 1));
2259 if (inputW != 1)
2260 inputExprs.push_back(Elt: rewriter.getAffineDimExpr(position: 2));
2261 inputExprs.push_back(Elt: rewriter.getAffineDimExpr(position: 3));
2262
2263 auto inputMap = AffineMap::get(dimCount: resultTy.getRank(), /*symbolCount=*/0,
2264 results: inputExprs, context: rewriter.getContext());
2265
2266 auto outputMap = rewriter.getMultiDimIdentityMap(rank: resultTy.getRank());
2267 rewriter.replaceOpWithNewOp<linalg::GenericOp>(
2268 op, args&: resultTy, args: ValueRange{collapse}, args: ValueRange{empty},
2269 args: ArrayRef<AffineMap>{inputMap, outputMap}, args&: iterators,
2270 args: [=](OpBuilder &b, Location loc, ValueRange args) {
2271 Value value = args[0];
2272 linalg::YieldOp::create(builder&: b, location: loc, values: value);
2273 });
2274
2275 return success();
2276 }
2277};
2278
2279class GenericResizeConverter : public OpRewritePattern<tosa::ResizeOp> {
2280public:
2281 using OpRewritePattern<tosa::ResizeOp>::OpRewritePattern;
2282
2283 LogicalResult matchAndRewrite(tosa::ResizeOp op,
2284 PatternRewriter &rewriter) const final {
2285 Location loc = op.getLoc();
2286 ImplicitLocOpBuilder b(loc, rewriter);
2287 auto input = op.getInput();
2288 auto inputTy = cast<ShapedType>(Val: input.getType());
2289 auto resultTy = cast<ShapedType>(Val: op.getType());
2290 auto resultETy = resultTy.getElementType();
2291
2292 bool floatingPointMode = isa<FloatType>(Val: resultETy);
2293 auto floatTy = resultETy;
2294
2295 auto imageH = inputTy.getShape()[1];
2296 auto imageW = inputTy.getShape()[2];
2297
2298 auto dynamicDimsOr =
2299 checkHasDynamicBatchDims(rewriter, op, params: {input, op.getOutput()});
2300 if (!dynamicDimsOr.has_value())
2301 return rewriter.notifyMatchFailure(
2302 arg&: op, msg: "unable to get dynamic dimensions of tosa.resize");
2303
2304 if (op.getMode() != ResizeMode::NEAREST_NEIGHBOR &&
2305 op.getMode() != ResizeMode::BILINEAR)
2306 return rewriter.notifyMatchFailure(
2307 arg&: op, msg: "tosa.resize mode should be NEAREST_NEIGHBOR or BILINEAR");
2308
2309 SmallVector<AffineMap, 2> affineMaps = {
2310 rewriter.getMultiDimIdentityMap(rank: resultTy.getRank())};
2311 auto emptyTensor = tensor::EmptyOp::create(builder&: b, staticShape: resultTy.getShape(),
2312 elementType: resultETy, dynamicSizes: *dynamicDimsOr);
2313 auto genericOp = linalg::GenericOp::create(
2314 builder&: b, resultTensorTypes: resultTy, inputs: ValueRange({}), outputs: ValueRange{emptyTensor}, indexingMaps: affineMaps,
2315 iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()));
2316 Value resize = genericOp.getResult(i: 0);
2317
2318 {
2319 OpBuilder::InsertionGuard regionGuard(b);
2320 b.createBlock(parent: &genericOp.getRegion(), insertPt: genericOp.getRegion().end(),
2321 argTypes: TypeRange({resultETy}), locs: loc);
2322 Value batch = linalg::IndexOp::create(builder&: b, dim: 0);
2323 Value y = linalg::IndexOp::create(builder&: b, dim: 1);
2324 Value x = linalg::IndexOp::create(builder&: b, dim: 2);
2325 Value channel = linalg::IndexOp::create(builder&: b, dim: 3);
2326
2327 Value zeroI32 =
2328 arith::ConstantOp::create(builder&: b, value: b.getZeroAttr(type: b.getI32Type()));
2329 Value zeroFp = arith::ConstantOp::create(builder&: b, value: b.getZeroAttr(type: floatTy));
2330 Value hMax =
2331 arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: imageH - 1));
2332 Value wMax =
2333 arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: imageW - 1));
2334
2335 Value inY = arith::IndexCastOp::create(builder&: b, out: b.getI32Type(), in: y);
2336 Value inX = arith::IndexCastOp::create(builder&: b, out: b.getI32Type(), in: x);
2337
2338 SmallVector<int64_t> scale, offset, border;
2339 if (!tosa::getConstShapeValues(op: op.getScale().getDefiningOp(), result_shape&: scale) ||
2340 !tosa::getConstShapeValues(op: op.getOffset().getDefiningOp(), result_shape&: offset) ||
2341 !tosa::getConstShapeValues(op: op.getBorder().getDefiningOp(), result_shape&: border)) {
2342 return rewriter.notifyMatchFailure(
2343 arg&: op, msg: "tosa.resize scale/offset/border should have compile time "
2344 "constant values.");
2345 }
2346
2347 Value yScaleN, yScaleD, xScaleN, xScaleD;
2348 yScaleN = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: scale[0]));
2349 yScaleD = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: scale[1]));
2350 xScaleN = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: scale[2]));
2351 xScaleD = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: scale[3]));
2352
2353 Value yOffset, xOffset, yBorder, xBorder;
2354 yOffset = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: offset[0]));
2355 xOffset = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: offset[1]));
2356 yBorder = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: border[0]));
2357 xBorder = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: border[1]));
2358
2359 // Compute the ix and dx values for both the X and Y dimensions.
2360 auto getIndexAndDeltaFp = [&](Value &index, Value &delta, Value in,
2361 Value scaleN, Value scaleD, Value offset,
2362 int size, ImplicitLocOpBuilder &b) {
2363 if (size == 1) {
2364 index = zeroI32;
2365 delta = zeroFp;
2366 return;
2367 }
2368 // x = x * scale_d + offset;
2369 // ix = floor(x / scale_n)
2370 Value val = arith::MulIOp::create(builder&: b, lhs: in, rhs: scaleD);
2371 val = arith::AddIOp::create(builder&: b, lhs: val, rhs: offset);
2372 index = arith::FloorDivSIOp::create(builder&: b, lhs: val, rhs: scaleN);
2373
2374 // rx = x - ix * scale_n (x % scale_n, if values are positive)
2375 Value scaledIndex = arith::MulIOp::create(builder&: b, lhs: index, rhs: scaleN);
2376 Value r = arith::SubIOp::create(builder&: b, lhs: val, rhs: scaledIndex);
2377 Value rFp = arith::SIToFPOp::create(builder&: b, out: floatTy, in: r);
2378
2379 // dx = rx / scale_n
2380 Value scaleNfp = arith::UIToFPOp::create(builder&: b, out: floatTy, in: scaleN);
2381 delta = arith::DivFOp::create(builder&: b, lhs: rFp, rhs: scaleNfp);
2382 };
2383
2384 // Compute the ix and dx values for the X and Y dimensions - int case.
2385 auto getIndexAndDeltaInt = [&](Value &index, Value &delta, Value in,
2386 Value scaleN, Value scaleD, Value offset,
2387 int size, ImplicitLocOpBuilder &b) {
2388 if (size == 1) {
2389 index = zeroI32;
2390 delta = zeroI32;
2391 return;
2392 }
2393 // x = x * scale_d + offset;
2394 // ix = floor(x / scale_n)
2395 // dx = x - ix * scale_n;
2396 Value val = arith::MulIOp::create(builder&: b, lhs: in, rhs: scaleD);
2397 val = arith::AddIOp::create(builder&: b, lhs: val, rhs: offset);
2398 index = arith::FloorDivSIOp::create(builder&: b, lhs: val, rhs: scaleN);
2399 delta = arith::MulIOp::create(builder&: b, lhs: index, rhs: scaleN);
2400 delta = arith::SubIOp::create(builder&: b, lhs: val, rhs: delta);
2401 };
2402
2403 Value ix, iy, dx, dy;
2404 if (floatingPointMode) {
2405 getIndexAndDeltaFp(iy, dy, inY, yScaleN, yScaleD, yOffset, imageH, b);
2406 getIndexAndDeltaFp(ix, dx, inX, xScaleN, xScaleD, xOffset, imageW, b);
2407 } else {
2408 getIndexAndDeltaInt(iy, dy, inY, yScaleN, yScaleD, yOffset, imageH, b);
2409 getIndexAndDeltaInt(ix, dx, inX, xScaleN, xScaleD, xOffset, imageW, b);
2410 }
2411
2412 if (op.getMode() == ResizeMode::NEAREST_NEIGHBOR) {
2413 auto one = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: 1));
2414
2415 auto getNearestIndexAndClamp = [&](Value val, Value dval, Value scale,
2416 Value max, int size,
2417 ImplicitLocOpBuilder &b) -> Value {
2418 if (size == 1) {
2419 return arith::ConstantIndexOp::create(builder&: b, value: 0);
2420 }
2421
2422 Value pred;
2423 if (floatingPointMode) {
2424 auto h =
2425 arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: floatTy, value: 0.5f));
2426 pred = arith::CmpFOp::create(builder&: b, predicate: arith::CmpFPredicate::OGE, lhs: dval, rhs: h);
2427 } else {
2428 Value dvalDouble = arith::ShLIOp::create(builder&: b, lhs: dval, rhs: one);
2429 pred = arith::CmpIOp::create(builder&: b, predicate: arith::CmpIPredicate::sge,
2430 lhs: dvalDouble, rhs: scale);
2431 }
2432
2433 auto offset = arith::SelectOp::create(builder&: b, condition: pred, true_value: one, false_value: zeroI32);
2434 val = arith::AddIOp::create(builder&: b, lhs: val, rhs: offset);
2435 val = clampIntHelper(loc, arg: val, min: zeroI32, max, rewriter&: b, /*isUnsigned=*/false);
2436 return arith::IndexCastOp::create(builder&: b, out: b.getIndexType(), in: val);
2437 };
2438
2439 iy = getNearestIndexAndClamp(iy, dy, yScaleN, hMax, imageH, b);
2440 ix = getNearestIndexAndClamp(ix, dx, xScaleN, wMax, imageW, b);
2441
2442 Value result = tensor::ExtractOp::create(
2443 builder&: b, tensor: input, indices: ValueRange{batch, iy, ix, channel});
2444
2445 linalg::YieldOp::create(builder&: b, values: result);
2446 } else {
2447 // The mode here must be BILINEAR.
2448 assert(op.getMode() == ResizeMode::BILINEAR);
2449
2450 auto oneVal = arith::ConstantOp::create(builder&: b, value: b.getI32IntegerAttr(value: 1));
2451
2452 auto getClampedIdxs = [&](Value &val0, Value &val1, int size, Value in,
2453 Value max, ImplicitLocOpBuilder &b) {
2454 val0 = in;
2455 val1 = arith::AddIOp::create(builder&: b, lhs: val0, rhs: oneVal);
2456 val0 =
2457 clampIntHelper(loc, arg: val0, min: zeroI32, max, rewriter&: b, /*isUnsigned=*/false);
2458 val1 =
2459 clampIntHelper(loc, arg: val1, min: zeroI32, max, rewriter&: b, /*isUnsigned=*/false);
2460 val0 = arith::IndexCastOp::create(builder&: b, out: b.getIndexType(), in: val0);
2461 val1 = arith::IndexCastOp::create(builder&: b, out: b.getIndexType(), in: val1);
2462 };
2463
2464 // Linalg equivalent to the section below:
2465 // int16_t iy0 = apply_max(iy, 0);
2466 // int16_t iy1 = apply_min(iy + 1, IH - 1);
2467 // int16_t ix0 = apply_max(ix, 0);
2468 // int16_t ix1 = apply_min(ix + 1, IW - 1);
2469 Value x0, x1, y0, y1;
2470 getClampedIdxs(y0, y1, imageH, iy, hMax, b);
2471 getClampedIdxs(x0, x1, imageW, ix, wMax, b);
2472
2473 Value y0x0 = tensor::ExtractOp::create(
2474 builder&: b, tensor: input, indices: ValueRange{batch, y0, x0, channel});
2475 Value y0x1 = tensor::ExtractOp::create(
2476 builder&: b, tensor: input, indices: ValueRange{batch, y0, x1, channel});
2477 Value y1x0 = tensor::ExtractOp::create(
2478 builder&: b, tensor: input, indices: ValueRange{batch, y1, x0, channel});
2479 Value y1x1 = tensor::ExtractOp::create(
2480 builder&: b, tensor: input, indices: ValueRange{batch, y1, x1, channel});
2481
2482 if (floatingPointMode) {
2483 auto oneVal =
2484 arith::ConstantOp::create(builder&: b, value: b.getFloatAttr(type: floatTy, value: 1.0f));
2485 auto interpolate = [&](Value val0, Value val1, Value delta,
2486 int inputSize,
2487 ImplicitLocOpBuilder &b) -> Value {
2488 if (inputSize == 1)
2489 return val0;
2490 Value oneMinusDelta = arith::SubFOp::create(builder&: b, lhs: oneVal, rhs: delta);
2491 Value mul0 = arith::MulFOp::create(builder&: b, lhs: val0, rhs: oneMinusDelta);
2492 Value mul1 = arith::MulFOp::create(builder&: b, lhs: val1, rhs: delta);
2493 return arith::AddFOp::create(builder&: b, lhs: mul0, rhs: mul1);
2494 };
2495
2496 // Linalg equivalent to the section below:
2497 // topAcc = v00 * (unit_x - dx);
2498 // topAcc += v01 * dx;
2499 Value topAcc = interpolate(y0x0, y0x1, dx, imageW, b);
2500
2501 // Linalg equivalent to the section below:
2502 // bottomAcc = v10 * (unit_x - dx);
2503 // bottomAcc += v11 * dx;
2504 Value bottomAcc = interpolate(y1x0, y1x1, dx, imageW, b);
2505
2506 // Linalg equivalent to the section below:
2507 // result = topAcc * (unit_y - dy) + bottomAcc * dy
2508 Value result = interpolate(topAcc, bottomAcc, dy, imageH, b);
2509 linalg::YieldOp::create(builder&: b, values: result);
2510 } else {
2511 // Perform in quantized space.
2512 y0x0 = arith::ExtSIOp::create(builder&: b, out: resultETy, in: y0x0);
2513 y0x1 = arith::ExtSIOp::create(builder&: b, out: resultETy, in: y0x1);
2514 y1x0 = arith::ExtSIOp::create(builder&: b, out: resultETy, in: y1x0);
2515 y1x1 = arith::ExtSIOp::create(builder&: b, out: resultETy, in: y1x1);
2516
2517 const int64_t deltaBitwidth = dx.getType().getIntOrFloatBitWidth();
2518 if (resultETy.getIntOrFloatBitWidth() > deltaBitwidth) {
2519 dx = arith::ExtSIOp::create(builder&: b, out: resultETy, in: dx);
2520 dy = arith::ExtSIOp::create(builder&: b, out: resultETy, in: dy);
2521 }
2522
2523 Value yScaleNExt = yScaleN;
2524 Value xScaleNExt = xScaleN;
2525
2526 const int64_t scaleBitwidth =
2527 xScaleN.getType().getIntOrFloatBitWidth();
2528 if (resultETy.getIntOrFloatBitWidth() > scaleBitwidth) {
2529 yScaleNExt = arith::ExtSIOp::create(builder&: b, out: resultETy, in: yScaleN);
2530 xScaleNExt = arith::ExtSIOp::create(builder&: b, out: resultETy, in: xScaleN);
2531 }
2532
2533 auto interpolate = [](Value val0, Value val1, Value weight1,
2534 Value scale, int inputSize,
2535 ImplicitLocOpBuilder &b) -> Value {
2536 if (inputSize == 1)
2537 return arith::MulIOp::create(builder&: b, lhs: val0, rhs: scale);
2538 Value weight0 = arith::SubIOp::create(builder&: b, lhs: scale, rhs: weight1);
2539 Value mul0 = arith::MulIOp::create(builder&: b, lhs: val0, rhs: weight0);
2540 Value mul1 = arith::MulIOp::create(builder&: b, lhs: val1, rhs: weight1);
2541 return arith::AddIOp::create(builder&: b, lhs: mul0, rhs: mul1);
2542 };
2543
2544 Value topAcc = interpolate(y0x0, y0x1, dx, xScaleNExt, imageW, b);
2545 Value bottomAcc = interpolate(y1x0, y1x1, dx, xScaleNExt, imageW, b);
2546 Value result =
2547 interpolate(topAcc, bottomAcc, dy, yScaleNExt, imageH, b);
2548 linalg::YieldOp::create(builder&: b, values: result);
2549 }
2550 }
2551 }
2552
2553 rewriter.replaceOp(op, newValues: resize);
2554 return success();
2555 }
2556};
2557
2558// At the codegen level any identity operations should be removed. Any cases
2559// where identity is load-bearing (e.g. cross device computation) should be
2560// handled before lowering to codegen.
2561template <typename SrcOp>
2562class IdentityNConverter : public OpRewritePattern<SrcOp> {
2563public:
2564 using OpRewritePattern<SrcOp>::OpRewritePattern;
2565
2566 LogicalResult matchAndRewrite(SrcOp op,
2567 PatternRewriter &rewriter) const final {
2568 rewriter.replaceOp(op, op.getOperation()->getOperands());
2569 return success();
2570 }
2571};
2572
2573template <typename SrcOp>
2574class ReduceConverter : public OpRewritePattern<SrcOp> {
2575public:
2576 ReduceConverter(MLIRContext *context, bool allowNonFinites)
2577 : OpRewritePattern<SrcOp>(context), allowNonFinites(allowNonFinites) {}
2578
2579 LogicalResult matchAndRewrite(SrcOp reduceOp,
2580 PatternRewriter &rewriter) const final {
2581 return reduceMatchAndRewriteHelper(reduceOp, reduceOp.getAxis(), rewriter,
2582 allowNonFinites);
2583 }
2584
2585private:
2586 bool allowNonFinites;
2587};
2588
2589class ReverseConverter : public OpRewritePattern<tosa::ReverseOp> {
2590public:
2591 using OpRewritePattern<tosa::ReverseOp>::OpRewritePattern;
2592
2593 LogicalResult matchAndRewrite(tosa::ReverseOp op,
2594 PatternRewriter &rewriter) const final {
2595 auto loc = op.getLoc();
2596 Value input = op.getInput1();
2597 auto inputTy = cast<ShapedType>(Val: input.getType());
2598 auto resultTy = cast<ShapedType>(Val: op.getType());
2599 auto axis = op.getAxis();
2600
2601 SmallVector<Value> dynDims;
2602 for (int i = 0; i < inputTy.getRank(); i++) {
2603 if (inputTy.isDynamicDim(idx: i)) {
2604 dynDims.push_back(Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: i));
2605 }
2606 }
2607
2608 Value axisDimSize = tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: axis);
2609
2610 // First fill the output buffer with the init value.
2611 auto emptyTensor = tensor::EmptyOp::create(
2612 builder&: rewriter, location: loc, staticShape: inputTy.getShape(),
2613 elementType: inputTy.getElementType(), dynamicSizes: ArrayRef<Value>({dynDims}))
2614 .getResult();
2615 SmallVector<AffineMap, 2> affineMaps = {
2616 rewriter.getMultiDimIdentityMap(rank: resultTy.getRank())};
2617
2618 rewriter.replaceOpWithNewOp<linalg::GenericOp>(
2619 op, args&: resultTy, args: ArrayRef<Value>({}), args: ValueRange{emptyTensor}, args&: affineMaps,
2620 args: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()),
2621 args: [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
2622 llvm::SmallVector<Value> indices;
2623 for (unsigned int i = 0; i < inputTy.getRank(); i++) {
2624 Value index =
2625 linalg::IndexOp::create(builder&: rewriter, location: nestedLoc, dim: i).getResult();
2626 if (i == axis) {
2627 auto one = arith::ConstantIndexOp::create(builder&: rewriter, location: nestedLoc, value: 1);
2628 auto sizeMinusOne =
2629 arith::SubIOp::create(builder&: rewriter, location: nestedLoc, lhs: axisDimSize, rhs: one);
2630 index = arith::SubIOp::create(builder&: rewriter, location: nestedLoc, lhs: sizeMinusOne,
2631 rhs: index);
2632 }
2633
2634 indices.push_back(Elt: index);
2635 }
2636
2637 auto extract = tensor::ExtractOp::create(builder&: nestedBuilder, location: nestedLoc,
2638 tensor: input, indices);
2639 linalg::YieldOp::create(builder&: nestedBuilder, location: op.getLoc(),
2640 values: extract.getResult());
2641 });
2642 return success();
2643 }
2644};
2645
2646// This converter translate a tile operation to a reshape, broadcast, reshape.
2647// The first reshape minimally expands each tiled dimension to include a
2648// proceding size-1 dim. This dim is then broadcasted to the appropriate
2649// multiple.
2650struct TileConverter : public OpConversionPattern<tosa::TileOp> {
2651 using OpConversionPattern<tosa::TileOp>::OpConversionPattern;
2652
2653 LogicalResult
2654 matchAndRewrite(tosa::TileOp op, OpAdaptor adaptor,
2655 ConversionPatternRewriter &rewriter) const override {
2656 SmallVector<int64_t> multiples;
2657 if (failed(Result: op.getConstantMultiples(multiples)))
2658 return failure();
2659
2660 // Avoid constructing invalid tensor types when validation is skipped.
2661 // Preserve the existing handling of -1 below.
2662 if (llvm::any_of(Range&: multiples, P: [](int64_t v) { return v <= 0 && v != -1; }))
2663 return rewriter.notifyMatchFailure(arg&: op,
2664 msg: "unsupported non-positive multiples");
2665
2666 auto loc = op.getLoc();
2667 auto input = op.getInput1();
2668 auto inputTy = cast<ShapedType>(Val: input.getType());
2669 auto inputShape = inputTy.getShape();
2670 auto resultTy = cast<ShapedType>(Val: op.getType());
2671 auto elementTy = inputTy.getElementType();
2672 int64_t rank = inputTy.getRank();
2673
2674 // Broadcast the newly added dimensions to their appropriate multiple.
2675 SmallVector<int64_t, 2> genericShape;
2676 for (int i = 0; i < rank; i++) {
2677 int64_t dim = multiples[i];
2678 genericShape.push_back(Elt: dim == -1 ? ShapedType::kDynamic : dim);
2679 genericShape.push_back(Elt: inputShape[i]);
2680 }
2681
2682 SmallVector<Value> dynDims;
2683 for (int i = 0; i < inputTy.getRank(); i++) {
2684 if (inputTy.isDynamicDim(idx: i) || multiples[i] == -1) {
2685 dynDims.push_back(Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: input, index: i));
2686 }
2687 }
2688
2689 auto emptyTensor = tensor::EmptyOp::create(
2690 builder&: rewriter, location: op.getLoc(), staticShape: genericShape, elementType: elementTy, dynamicSizes: dynDims);
2691
2692 // We needs to map the input shape to the non-broadcasted dimensions.
2693 SmallVector<AffineExpr, 4> dimExprs;
2694 dimExprs.reserve(N: rank);
2695 for (unsigned i = 0; i < rank; ++i)
2696 dimExprs.push_back(Elt: rewriter.getAffineDimExpr(position: i * 2 + 1));
2697
2698 auto readAffineMap =
2699 AffineMap::get(/*dimCount=*/rank * 2, /*symbolCount=*/0, results: dimExprs,
2700 context: rewriter.getContext());
2701
2702 SmallVector<AffineMap, 2> affineMaps = {
2703 readAffineMap, rewriter.getMultiDimIdentityMap(rank: genericShape.size())};
2704
2705 auto genericOp = linalg::GenericOp::create(
2706 builder&: rewriter, location: loc, resultTensorTypes: RankedTensorType::get(shape: genericShape, elementType: elementTy), inputs: input,
2707 outputs: ValueRange{emptyTensor}, indexingMaps: affineMaps,
2708 iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: genericShape.size()),
2709 odsArg5: [&](OpBuilder &nestedBuilder, Location nestedLoc, ValueRange args) {
2710 linalg::YieldOp::create(builder&: nestedBuilder, location: op.getLoc(), values: *args.begin());
2711 });
2712
2713 auto shapeValue = getTosaConstShape(
2714 rewriter, loc, shape: mlir::tosa::convertFromMlirShape(shape: resultTy.getShape()));
2715 rewriter.replaceOpWithNewOp<tosa::ReshapeOp>(
2716 op, args&: resultTy, args: genericOp.getResult(i: 0), args&: shapeValue);
2717 return success();
2718 }
2719};
2720
2721// Tosa argmin/argmax lowering represents the op as a linalg.generic
2722// op, producing two output buffers.
2723//
2724// The first output buffer contains the index of the found min/max value. It is
2725// initialized to 0 and is resulting integer type.
2726//
2727// The second output buffer contains the min/max value found. It is initialized
2728// to the max/min representable value of the input element type. After being
2729// populated by generic, this buffer is discarded as only the index is
2730// requested.
2731//
2732// The generic op updates both the min/max value and index if the
2733// current value is lower than/exceeds the running min/max.
2734template <typename SrcOp>
2735class ArgMinMaxConverter : public OpRewritePattern<SrcOp> {
2736public:
2737 ArgMinMaxConverter(MLIRContext *context, bool allowNonFinites)
2738 : OpRewritePattern<SrcOp>(context), allowNonFinites(allowNonFinites) {}
2739
2740 LogicalResult matchAndRewrite(SrcOp argOp,
2741 PatternRewriter &rewriter) const final {
2742 constexpr bool isMin = std::is_same_v<SrcOp, tosa::ArgMinOp>;
2743
2744 auto loc = argOp.getLoc();
2745 Value input = argOp.getInput();
2746 auto inputTy = cast<ShapedType>(Val: input.getType());
2747 auto resultTy = cast<ShapedType>(argOp.getOutput().getType());
2748 auto inElementTy = inputTy.getElementType();
2749 auto outElementTy = resultTy.getElementType();
2750 int axis = argOp.getAxis();
2751 auto resultValueTy =
2752 RankedTensorType::get(shape: resultTy.getShape(), elementType: inElementTy);
2753
2754 if (!isa<IntegerType>(outElementTy))
2755 return rewriter.notifyMatchFailure(argOp,
2756 "requires integer-like result type");
2757
2758 SmallVector<Value> dynDims;
2759 for (int i = 0; i < inputTy.getRank(); i++) {
2760 if (inputTy.isDynamicDim(idx: i) && i != axis) {
2761 dynDims.push_back(Elt: tensor::DimOp::create(rewriter, loc, input, i));
2762 }
2763 }
2764
2765 // First fill the output buffer for the index.
2766 auto emptyTensorIdx =
2767 tensor::EmptyOp::create(rewriter, loc, resultTy.getShape(),
2768 outElementTy, dynDims)
2769 .getResult();
2770 auto fillValueIdx = arith::ConstantOp::create(
2771 rewriter, loc, rewriter.getIntegerAttr(outElementTy, 0));
2772 auto filledTensorIdx =
2773 linalg::FillOp::create(rewriter, loc, ValueRange{fillValueIdx},
2774 ValueRange{emptyTensorIdx})
2775 .result();
2776
2777 // Second fill the output buffer for the running min/max.
2778 auto emptyTensorValue =
2779 tensor::EmptyOp::create(rewriter, loc, resultTy.getShape(), inElementTy,
2780 dynDims)
2781 .getResult();
2782 auto fillValueAttr = createInitialValueForReduceOp(
2783 argOp, inElementTy, rewriter, allowNonFinites);
2784
2785 if (!fillValueAttr)
2786 return rewriter.notifyMatchFailure(argOp, "unsupported element type");
2787
2788 auto fillValue = arith::ConstantOp::create(rewriter, loc, fillValueAttr);
2789 auto filledTensorValue =
2790 linalg::FillOp::create(rewriter, loc, ValueRange{fillValue},
2791 ValueRange{emptyTensorValue})
2792 .result();
2793
2794 // We need to reduce along the specified axis, with parallel operations
2795 // along the rest.
2796 SmallVector<utils::IteratorType, 4> iteratorTypes;
2797 iteratorTypes.resize(N: inputTy.getRank(), NV: utils::IteratorType::parallel);
2798 iteratorTypes[axis] = utils::IteratorType::reduction;
2799
2800 SmallVector<AffineExpr, 2> srcExprs;
2801 SmallVector<AffineExpr, 2> dstExprs;
2802 for (int i = 0, rank = inputTy.getRank(); i != rank; ++i) {
2803 srcExprs.push_back(Elt: mlir::getAffineDimExpr(position: i, context: rewriter.getContext()));
2804 if (axis != i)
2805 dstExprs.push_back(Elt: mlir::getAffineDimExpr(position: i, context: rewriter.getContext()));
2806 }
2807
2808 bool didEncounterError = false;
2809 auto maps = AffineMap::inferFromExprList(exprsList: {srcExprs, dstExprs, dstExprs},
2810 context: rewriter.getContext());
2811 auto linalgOp = linalg::GenericOp::create(
2812 rewriter, loc, ArrayRef<Type>({resultTy, resultValueTy}), input,
2813 ValueRange({filledTensorIdx, filledTensorValue}), maps, iteratorTypes,
2814 [&](OpBuilder &nestedBuilder, Location nestedLoc,
2815 ValueRange blockArgs) {
2816 auto newValue = blockArgs[0];
2817 auto oldIndex = blockArgs[1];
2818 auto oldValue = blockArgs[2];
2819
2820 Value newIndex = arith::IndexCastOp::create(
2821 rewriter, nestedLoc, oldIndex.getType(),
2822 linalg::IndexOp::create(rewriter, loc, axis));
2823
2824 Value predicate;
2825 if (isa<FloatType>(Val: inElementTy)) {
2826 if (argOp.getNanMode() == NanPropagationMode::IGNORE) {
2827 // Only update index & value for non NaN values. If all
2828 // values are NaNs, the initial index will be return which is 0.
2829 Value comparison = arith::CmpFOp::create(
2830 builder&: rewriter, location: nestedLoc,
2831 predicate: isMin ? arith::CmpFPredicate::ULT : arith::CmpFPredicate::UGT,
2832 lhs: newValue, rhs: oldValue);
2833 Value newNonNaN = arith::CmpFOp::create(builder&: rewriter, location: nestedLoc,
2834 predicate: arith::CmpFPredicate::ORD,
2835 lhs: newValue, rhs: newValue);
2836 predicate = arith::AndIOp::create(builder&: rewriter, location: nestedLoc,
2837 result: rewriter.getI1Type(),
2838 lhs: comparison, rhs: newNonNaN);
2839 } else {
2840 // Update value if either of the following is true:
2841 // - new value is smaller/bigger
2842 // - cur value is not NaN and new value is NaN
2843 Value comparison = arith::CmpFOp::create(
2844 builder&: rewriter, location: nestedLoc,
2845 predicate: isMin ? arith::CmpFPredicate::ULT : arith::CmpFPredicate::UGT,
2846 lhs: newValue, rhs: oldValue);
2847 Value oldNonNaN = arith::CmpFOp::create(builder&: rewriter, location: nestedLoc,
2848 predicate: arith::CmpFPredicate::ORD,
2849 lhs: oldValue, rhs: oldValue);
2850 predicate = arith::AndIOp::create(builder&: rewriter, location: nestedLoc,
2851 result: rewriter.getI1Type(),
2852 lhs: comparison, rhs: oldNonNaN);
2853 }
2854 } else if (isa<IntegerType>(Val: inElementTy)) {
2855 predicate = arith::CmpIOp::create(builder&: rewriter, location: nestedLoc,
2856 predicate: isMin ? arith::CmpIPredicate::slt
2857 : arith::CmpIPredicate::sgt,
2858 lhs: newValue, rhs: oldValue);
2859 } else {
2860 didEncounterError = true;
2861 return;
2862 }
2863
2864 auto resultValue = arith::SelectOp::create(
2865 builder&: rewriter, location: nestedLoc, condition: predicate, true_value: newValue, false_value: oldValue);
2866 auto resultIndex = arith::SelectOp::create(
2867 builder&: rewriter, location: nestedLoc, condition: predicate, true_value: newIndex, false_value: oldIndex);
2868 linalg::YieldOp::create(builder&: nestedBuilder, location: nestedLoc,
2869 values: ValueRange({resultIndex, resultValue}));
2870 });
2871
2872 if (didEncounterError)
2873 return rewriter.notifyMatchFailure(argOp, "unsupported element type");
2874
2875 rewriter.replaceOp(argOp, linalgOp.getResult(0));
2876 return success();
2877 }
2878
2879private:
2880 bool allowNonFinites;
2881};
2882
2883class GatherConverter : public OpConversionPattern<tosa::GatherOp> {
2884public:
2885 using OpConversionPattern<tosa::GatherOp>::OpConversionPattern;
2886 LogicalResult
2887 matchAndRewrite(tosa::GatherOp op, OpAdaptor adaptor,
2888 ConversionPatternRewriter &rewriter) const final {
2889 auto input = adaptor.getOperands()[0];
2890 auto indices = adaptor.getOperands()[1];
2891
2892 auto valuesTy = dyn_cast<RankedTensorType>(Val: op.getValues().getType());
2893 auto resultTy = dyn_cast<RankedTensorType>(Val: op.getType());
2894 if (!valuesTy || !resultTy)
2895 return rewriter.notifyMatchFailure(arg&: op, msg: "unranked tensors not supported");
2896
2897 auto dynamicDims = inferDynamicDimsForGather(
2898 builder&: rewriter, loc: op.getLoc(), values: adaptor.getValues(), indices: adaptor.getIndices());
2899
2900 auto resultElementTy = resultTy.getElementType();
2901
2902 auto loc = op.getLoc();
2903 auto emptyTensor =
2904 tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: resultTy.getShape(),
2905 elementType: resultElementTy, dynamicSizes: dynamicDims)
2906 .getResult();
2907
2908 SmallVector<AffineMap, 2> affineMaps = {
2909 AffineMap::get(
2910 /*dimCount=*/resultTy.getRank(), /*symbolCount=*/0,
2911 results: {rewriter.getAffineDimExpr(position: 0), rewriter.getAffineDimExpr(position: 1)},
2912 context: rewriter.getContext()),
2913 rewriter.getMultiDimIdentityMap(rank: resultTy.getRank())};
2914
2915 auto genericOp = linalg::GenericOp::create(
2916 builder&: rewriter, location: loc, resultTensorTypes: ArrayRef<Type>({resultTy}), inputs: ValueRange{indices},
2917 outputs: ValueRange{emptyTensor}, indexingMaps: affineMaps,
2918 iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()),
2919 odsArg5: [&](OpBuilder &b, Location loc, ValueRange args) {
2920 auto indexValue = args[0];
2921 auto index0 = linalg::IndexOp::create(builder&: rewriter, location: loc, dim: 0);
2922 Value index1 = arith::IndexCastOp::create(
2923 builder&: rewriter, location: loc, out: rewriter.getIndexType(), in: indexValue);
2924 auto index2 = linalg::IndexOp::create(builder&: rewriter, location: loc, dim: 2);
2925 Value extract = tensor::ExtractOp::create(
2926 builder&: rewriter, location: loc, tensor: input, indices: ValueRange{index0, index1, index2});
2927 linalg::YieldOp::create(builder&: rewriter, location: loc, values: extract);
2928 });
2929 rewriter.replaceOp(op, newValues: genericOp.getResult(i: 0));
2930 return success();
2931 }
2932
2933 static llvm::SmallVector<Value> inferDynamicDimsForGather(OpBuilder &builder,
2934 Location loc,
2935 Value values,
2936 Value indices) {
2937 llvm::SmallVector<Value> results;
2938
2939 auto addDynamicDimension = [&](Value source, int64_t dim) {
2940 auto sz = tensor::getMixedSize(builder, loc, value: source, dim);
2941 if (auto dimValue = llvm::dyn_cast_if_present<Value>(Val&: sz))
2942 results.push_back(Elt: dimValue);
2943 };
2944
2945 addDynamicDimension(values, 0);
2946 addDynamicDimension(indices, 1);
2947 addDynamicDimension(values, 2);
2948 return results;
2949 }
2950};
2951
2952class RowGatherConverter : public OpConversionPattern<tosa::RowGatherOp> {
2953public:
2954 using OpConversionPattern<tosa::RowGatherOp>::OpConversionPattern;
2955 LogicalResult
2956 matchAndRewrite(tosa::RowGatherOp op, OpAdaptor adaptor,
2957 ConversionPatternRewriter &rewriter) const final {
2958 auto valuesTy = dyn_cast<RankedTensorType>(Val: adaptor.getValues().getType());
2959 auto indicesTy = dyn_cast<RankedTensorType>(Val: adaptor.getIndices().getType());
2960 auto rowCountTy =
2961 dyn_cast<RankedTensorType>(Val: adaptor.getRowCount().getType());
2962 auto resultTy = dyn_cast<RankedTensorType>(Val: op.getType());
2963 if (!valuesTy || !indicesTy || !rowCountTy || !resultTy)
2964 return rewriter.notifyMatchFailure(arg&: op, msg: "unranked tensors not supported");
2965
2966 Location loc = op.getLoc();
2967 Value zero = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
2968 Value rowCount = tensor::ExtractOp::create(
2969 builder&: rewriter, location: loc, tensor: adaptor.getRowCount(), indices: ValueRange{zero});
2970 Value rowCountIndex = arith::IndexCastOp::create(
2971 builder&: rewriter, location: loc, out: rewriter.getIndexType(), in: rowCount);
2972
2973 SmallVector<Value> dynamicDims;
2974 if (resultTy.isDynamicDim(idx: 0))
2975 dynamicDims.push_back(
2976 Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: adaptor.getValues(), index: 0));
2977 if (resultTy.isDynamicDim(idx: 1)) {
2978 Value indicesWidth =
2979 tensor::DimOp::create(builder&: rewriter, location: loc, source: adaptor.getIndices(), index: 1);
2980 dynamicDims.push_back(
2981 Elt: arith::MulIOp::create(builder&: rewriter, location: loc, lhs: indicesWidth, rhs: rowCountIndex));
2982 }
2983 if (resultTy.isDynamicDim(idx: 2))
2984 dynamicDims.push_back(
2985 Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: adaptor.getValues(), index: 2));
2986
2987 Value emptyTensor =
2988 tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: resultTy.getShape(),
2989 elementType: resultTy.getElementType(), dynamicSizes: dynamicDims);
2990 SmallVector<AffineMap> affineMaps = {
2991 rewriter.getMultiDimIdentityMap(rank: resultTy.getRank())};
2992
2993 auto genericOp = linalg::GenericOp::create(
2994 builder&: rewriter, location: loc, resultTensorTypes: ArrayRef<Type>{resultTy}, inputs: ValueRange{},
2995 outputs: ValueRange{emptyTensor}, indexingMaps: affineMaps,
2996 iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()),
2997 odsArg5: [&](OpBuilder &builder, Location nestedLoc, ValueRange) {
2998 Value batch = linalg::IndexOp::create(builder, location: nestedLoc, dim: 0);
2999 Value outputRow = linalg::IndexOp::create(builder, location: nestedLoc, dim: 1);
3000 Value channel = linalg::IndexOp::create(builder, location: nestedLoc, dim: 2);
3001 Value indexSlot = arith::DivUIOp::create(builder, location: nestedLoc,
3002 lhs: outputRow, rhs: rowCountIndex);
3003 Value rowOffset = arith::RemUIOp::create(builder, location: nestedLoc,
3004 lhs: outputRow, rhs: rowCountIndex);
3005 Value index = tensor::ExtractOp::create(builder, location: nestedLoc,
3006 tensor: adaptor.getIndices(),
3007 indices: ValueRange{batch, indexSlot});
3008 Value row = arith::IndexCastOp::create(builder, location: nestedLoc,
3009 out: builder.getIndexType(), in: index);
3010 row = arith::AddIOp::create(builder, location: nestedLoc, lhs: row, rhs: rowOffset);
3011 Value result =
3012 tensor::ExtractOp::create(builder, location: nestedLoc, tensor: adaptor.getValues(),
3013 indices: ValueRange{batch, row, channel});
3014 linalg::YieldOp::create(builder, location: nestedLoc, values: result);
3015 });
3016 rewriter.replaceOp(op, newValues: genericOp.getResult(i: 0));
3017 return success();
3018 }
3019};
3020
3021// Lowerings the TableOp to a series of gathers and numerica operations. This
3022// includes interpolation between the high/low values. For the I8 varient, this
3023// simplifies to a single gather operation.
3024class TableConverter : public OpRewritePattern<tosa::TableOp> {
3025public:
3026 using OpRewritePattern<tosa::TableOp>::OpRewritePattern;
3027
3028 LogicalResult matchAndRewrite(tosa::TableOp op,
3029 PatternRewriter &rewriter) const final {
3030 auto loc = op.getLoc();
3031 Value input = op.getInput1();
3032 Value table = op.getTable();
3033 auto inputTy = cast<ShapedType>(Val: input.getType());
3034 auto tableTy = cast<ShapedType>(Val: table.getType());
3035 auto resultTy = cast<ShapedType>(Val: op.getType());
3036
3037 auto inputElementTy = inputTy.getElementType();
3038 auto tableElementTy = tableTy.getElementType();
3039 auto resultElementTy = resultTy.getElementType();
3040
3041 SmallVector<Value> dynDims;
3042 for (int i = 0; i < resultTy.getRank(); ++i) {
3043 if (inputTy.isDynamicDim(idx: i)) {
3044 dynDims.push_back(
3045 Elt: tensor::DimOp::create(builder&: rewriter, location: loc, source: op.getOperand(i: 0), index: i));
3046 }
3047 }
3048
3049 auto emptyTensor =
3050 tensor::EmptyOp::create(builder&: rewriter, location: loc, staticShape: resultTy.getShape(),
3051 elementType: resultElementTy, dynamicSizes: dynDims)
3052 .getResult();
3053
3054 SmallVector<AffineMap, 2> affineMaps = {
3055 rewriter.getMultiDimIdentityMap(rank: resultTy.getRank()),
3056 rewriter.getMultiDimIdentityMap(rank: resultTy.getRank())};
3057
3058 auto genericOp = linalg::GenericOp::create(
3059 builder&: rewriter, location: loc, resultTensorTypes: resultTy, inputs: ValueRange({input}), outputs: ValueRange{emptyTensor},
3060 indexingMaps: affineMaps, iteratorTypes: getNParallelLoopsAttrs(nParallelLoops: resultTy.getRank()));
3061 rewriter.replaceOp(op, newValues: genericOp.getResult(i: 0));
3062
3063 {
3064 OpBuilder::InsertionGuard regionGuard(rewriter);
3065 Block *block = rewriter.createBlock(
3066 parent: &genericOp.getRegion(), insertPt: genericOp.getRegion().end(),
3067 argTypes: TypeRange({inputElementTy, resultElementTy}), locs: {loc, loc});
3068
3069 auto inputValue = block->getArgument(i: 0);
3070 rewriter.setInsertionPointToStart(block);
3071 if (inputElementTy.isInteger(width: 8) && tableElementTy.isInteger(width: 8) &&
3072 resultElementTy.isInteger(width: 8)) {
3073 Value index = arith::IndexCastOp::create(
3074 builder&: rewriter, location: loc, out: rewriter.getIndexType(), in: inputValue);
3075 Value offset = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 128);
3076 index = arith::AddIOp::create(builder&: rewriter, location: loc, result: rewriter.getIndexType(),
3077 lhs: index, rhs: offset);
3078 Value extract =
3079 tensor::ExtractOp::create(builder&: rewriter, location: loc, tensor: table, indices: ValueRange{index});
3080 linalg::YieldOp::create(builder&: rewriter, location: loc, values: extract);
3081 return success();
3082 }
3083
3084 if (inputElementTy.isInteger(width: 16) && tableElementTy.isInteger(width: 16) &&
3085 resultElementTy.isInteger(width: 32)) {
3086 Value extend = arith::ExtSIOp::create(
3087 builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: inputValue);
3088
3089 auto offset = arith::ConstantOp::create(
3090 builder&: rewriter, location: loc, value: rewriter.getI32IntegerAttr(value: 32768));
3091 auto seven = arith::ConstantOp::create(builder&: rewriter, location: loc,
3092 value: rewriter.getI32IntegerAttr(value: 7));
3093 auto one = arith::ConstantOp::create(builder&: rewriter, location: loc,
3094 value: rewriter.getI32IntegerAttr(value: 1));
3095 auto b1111111 = arith::ConstantOp::create(
3096 builder&: rewriter, location: loc, value: rewriter.getI32IntegerAttr(value: 127));
3097
3098 // Compute the index and fractional part from the input value:
3099 // value = value + 32768
3100 // index = value >> 7;
3101 // fraction = 0x01111111 & value
3102 auto extendAdd = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: extend, rhs: offset);
3103 Value index = arith::ShRUIOp::create(builder&: rewriter, location: loc, lhs: extendAdd, rhs: seven);
3104 Value fraction =
3105 arith::AndIOp::create(builder&: rewriter, location: loc, lhs: extendAdd, rhs: b1111111);
3106
3107 // Extract the base and next values from the table.
3108 // base = (int32_t) table[index];
3109 // next = (int32_t) table[index + 1];
3110 Value indexPlusOne = arith::AddIOp::create(builder&: rewriter, location: loc, lhs: index, rhs: one);
3111
3112 index = arith::IndexCastOp::create(builder&: rewriter, location: loc,
3113 out: rewriter.getIndexType(), in: index);
3114 indexPlusOne = arith::IndexCastOp::create(
3115 builder&: rewriter, location: loc, out: rewriter.getIndexType(), in: indexPlusOne);
3116
3117 Value base =
3118 tensor::ExtractOp::create(builder&: rewriter, location: loc, tensor: table, indices: ValueRange{index});
3119 Value next = tensor::ExtractOp::create(builder&: rewriter, location: loc, tensor: table,
3120 indices: ValueRange{indexPlusOne});
3121
3122 base =
3123 arith::ExtSIOp::create(builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: base);
3124 next =
3125 arith::ExtSIOp::create(builder&: rewriter, location: loc, out: rewriter.getI32Type(), in: next);
3126
3127 // Use the fractional part to interpolate between the input values:
3128 // result = (base << 7) + (next - base) * fraction
3129 Value baseScaled = arith::ShLIOp::create(builder&: rewriter, location: loc, lhs: base, rhs: seven);
3130 Value diff = arith::SubIOp::create(builder&: rewriter, location: loc, lhs: next, rhs: base);
3131 Value diffScaled = arith::MulIOp::create(builder&: rewriter, location: loc, lhs: diff, rhs: fraction);
3132 Value result =
3133 arith::AddIOp::create(builder&: rewriter, location: loc, lhs: baseScaled, rhs: diffScaled);
3134
3135 linalg::YieldOp::create(builder&: rewriter, location: loc, values: result);
3136
3137 return success();
3138 }
3139 }
3140
3141 return rewriter.notifyMatchFailure(
3142 arg&: op, msg: "unable to create body for tosa.table op");
3143 }
3144};
3145
3146struct RFFT2dConverter final : public OpRewritePattern<RFFT2dOp> {
3147 using OpRewritePattern<RFFT2dOp>::OpRewritePattern;
3148
3149 static bool isRankedTensor(Type type) { return isa<RankedTensorType>(Val: type); }
3150
3151 static OpFoldResult halfPlusOne(OpBuilder &builder, Location loc,
3152 OpFoldResult ofr) {
3153 auto one = arith::ConstantIndexOp::create(builder, location: loc, value: 1);
3154 auto two = arith::ConstantIndexOp::create(builder, location: loc, value: 2);
3155
3156 auto value = getValueOrCreateConstantIndexOp(b&: builder, loc, ofr);
3157 auto divBy2 = builder.createOrFold<arith::DivUIOp>(location: loc, args&: value, args&: two);
3158 auto plusOne = builder.createOrFold<arith::AddIOp>(location: loc, args&: divBy2, args&: one);
3159 return getAsOpFoldResult(val: plusOne);
3160 }
3161
3162 static RankedTensorType
3163 computeOutputShape(OpBuilder &builder, Location loc, Value input,
3164 llvm::SmallVectorImpl<Value> &dynamicSizes) {
3165 // Get [N, H, W]
3166 auto dims = tensor::getMixedSizes(builder, loc, value: input);
3167
3168 // Set W = (W / 2) + 1 to account for the half-sized W dimension of the
3169 // output tensors.
3170 dims[2] = halfPlusOne(builder, loc, ofr: dims[2]);
3171
3172 llvm::SmallVector<int64_t, 3> staticSizes;
3173 dispatchIndexOpFoldResults(ofrs: dims, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
3174
3175 auto elementType = cast<RankedTensorType>(Val: input.getType()).getElementType();
3176 return RankedTensorType::get(shape: staticSizes, elementType);
3177 }
3178
3179 static Value createZeroTensor(PatternRewriter &rewriter, Location loc,
3180 RankedTensorType type,
3181 llvm::ArrayRef<Value> dynamicSizes) {
3182 auto emptyTensor =
3183 tensor::EmptyOp::create(builder&: rewriter, location: loc, result: type, dynamicSizes);
3184 auto fillValueAttr = rewriter.getZeroAttr(type: type.getElementType());
3185 auto fillValue = arith::ConstantOp::create(builder&: rewriter, location: loc, value: fillValueAttr);
3186 auto filledTensor =
3187 linalg::FillOp::create(builder&: rewriter, location: loc, inputs: ValueRange{fillValue},
3188 outputs: ValueRange{emptyTensor})
3189 .result();
3190 return filledTensor;
3191 }
3192
3193 static Value castIndexToFloat(OpBuilder &builder, Location loc,
3194 FloatType type, Value value) {
3195 auto integerVal = arith::IndexCastUIOp::create(
3196 builder, location: loc,
3197 out: type.getIntOrFloatBitWidth() > 32 ? builder.getI64Type()
3198 : builder.getI32Type(),
3199 in: value);
3200
3201 return arith::UIToFPOp::create(builder, location: loc, out: type, in: integerVal);
3202 }
3203
3204 static Value createLinalgIndex(OpBuilder &builder, Location loc,
3205 FloatType type, int64_t index) {
3206 auto indexVal = linalg::IndexOp::create(builder, location: loc, dim: index);
3207 return castIndexToFloat(builder, loc, type, value: indexVal);
3208 }
3209
3210 template <typename... Args>
3211 static llvm::SmallVector<AffineExpr, 4> affineDimsExpr(OpBuilder &builder,
3212 Args... args) {
3213 return {builder.getAffineDimExpr(position: args)...};
3214 }
3215
3216 LogicalResult matchAndRewrite(RFFT2dOp rfft2d,
3217 PatternRewriter &rewriter) const override {
3218 if (!llvm::all_of(Range: rfft2d->getOperandTypes(), P: isRankedTensor) ||
3219 !llvm::all_of(Range: rfft2d->getResultTypes(), P: isRankedTensor)) {
3220 return rewriter.notifyMatchFailure(arg&: rfft2d,
3221 msg: "only supports ranked tensors");
3222 }
3223
3224 auto loc = rfft2d.getLoc();
3225 auto input = rfft2d.getInputReal();
3226 auto elementType =
3227 dyn_cast<FloatType>(Val: cast<ShapedType>(Val: input.getType()).getElementType());
3228 if (!elementType)
3229 return rewriter.notifyMatchFailure(arg&: rfft2d,
3230 msg: "only supports float element types");
3231
3232 // Compute the output type and set of dynamic sizes
3233 llvm::SmallVector<Value> dynamicSizes;
3234 auto outputType = computeOutputShape(builder&: rewriter, loc, input, dynamicSizes);
3235
3236 // Iterator types for the linalg.generic implementation
3237 llvm::SmallVector<utils::IteratorType, 5> iteratorTypes = {
3238 utils::IteratorType::parallel, utils::IteratorType::parallel,
3239 utils::IteratorType::parallel, utils::IteratorType::reduction,
3240 utils::IteratorType::reduction};
3241
3242 // Inputs/outputs to the linalg.generic implementation
3243 llvm::SmallVector<Value> genericOpInputs = {input};
3244 llvm::SmallVector<Value> genericOpOutputs = {
3245 createZeroTensor(rewriter, loc, type: outputType, dynamicSizes),
3246 createZeroTensor(rewriter, loc, type: outputType, dynamicSizes)};
3247
3248 // Indexing maps for input and output tensors
3249 auto indexingMaps = AffineMap::inferFromExprList(
3250 exprsList: llvm::ArrayRef{affineDimsExpr(builder&: rewriter, args: 0, args: 3, args: 4),
3251 affineDimsExpr(builder&: rewriter, args: 0, args: 1, args: 2),
3252 affineDimsExpr(builder&: rewriter, args: 0, args: 1, args: 2)},
3253 context: rewriter.getContext());
3254
3255 // Width and height dimensions of the original input.
3256 auto dimH = rewriter.createOrFold<tensor::DimOp>(location: loc, args&: input, args: 1);
3257 auto dimW = rewriter.createOrFold<tensor::DimOp>(location: loc, args&: input, args: 2);
3258
3259 // Constants and dimension sizes
3260 auto zeroFloat = arith::ConstantOp::create(
3261 builder&: rewriter, location: loc, value: rewriter.getZeroAttr(type: elementType));
3262 auto twoPiAttr = rewriter.getFloatAttr(type: elementType, value: 6.283185307179586);
3263 auto twoPi = arith::ConstantOp::create(builder&: rewriter, location: loc, value: twoPiAttr);
3264
3265 auto zeroIndex = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 0);
3266 auto twoIndex = arith::ConstantIndexOp::create(builder&: rewriter, location: loc, value: 2);
3267
3268 auto constH = castIndexToFloat(builder&: rewriter, loc, type: elementType, value: dimH);
3269 auto constW = castIndexToFloat(builder&: rewriter, loc, type: elementType, value: dimW);
3270 auto halfH = index::DivUOp::create(builder&: rewriter, location: loc, lhs: dimH, rhs: twoIndex);
3271 auto halfW = index::DivUOp::create(builder&: rewriter, location: loc, lhs: dimW, rhs: twoIndex);
3272
3273 auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange args) {
3274 Value valReal = args[0];
3275 Value sumReal = args[1];
3276 Value sumImag = args[2];
3277
3278 // Indices for angle computation
3279 Value oy = linalg::IndexOp::create(builder, location: loc, dim: 1);
3280 Value ox = linalg::IndexOp::create(builder, location: loc, dim: 2);
3281 Value iy = linalg::IndexOp::create(builder, location: loc, dim: 3);
3282 Value ix = linalg::IndexOp::create(builder, location: loc, dim: 4);
3283
3284 // Calculating angle without integer parts of components as sin/cos are
3285 // periodic: angle = 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix * ox) % W )
3286 // / W);
3287 auto iyXoy = index::MulOp::create(builder, location: loc, lhs: iy, rhs: oy);
3288 auto ixXox = index::MulOp::create(builder, location: loc, lhs: ix, rhs: ox);
3289
3290 auto iyRem = index::RemUOp::create(builder, location: loc, lhs: iyXoy, rhs: dimH);
3291 auto ixRem = index::RemUOp::create(builder, location: loc, lhs: ixXox, rhs: dimW);
3292
3293 auto iyRemFloat = castIndexToFloat(builder, loc, type: elementType, value: iyRem);
3294 auto ixRemFloat = castIndexToFloat(builder, loc, type: elementType, value: ixRem);
3295
3296 auto yComponent = arith::DivFOp::create(builder, location: loc, lhs: iyRemFloat, rhs: constH);
3297 auto xComponent = arith::DivFOp::create(builder, location: loc, lhs: ixRemFloat, rhs: constW);
3298 auto sumXY = arith::AddFOp::create(builder, location: loc, lhs: yComponent, rhs: xComponent);
3299 auto angle = arith::MulFOp::create(builder, location: loc, lhs: twoPi, rhs: sumXY);
3300
3301 // We will check the indices to see if this is a position that should use
3302 // a 0.0 weight for the imaginary value computation following the TOSA
3303 // specification with `tosa_extra_multiplies=true`.
3304 //
3305 // These are the relevant locations: (0,0), (0,W/2), (H/2,0), (H/2, W/2).
3306 auto iyIs0 = arith::CmpIOp::create(builder, location: loc, predicate: arith::CmpIPredicate::eq,
3307 lhs: iyRem, rhs: zeroIndex);
3308 auto iyIsHalfH = arith::CmpIOp::create(
3309 builder, location: loc, predicate: arith::CmpIPredicate::eq, lhs: iyRem, rhs: halfH);
3310 auto ixIs0 = arith::CmpIOp::create(builder, location: loc, predicate: arith::CmpIPredicate::eq,
3311 lhs: ixRem, rhs: zeroIndex);
3312 auto ixIsHalfW = arith::CmpIOp::create(
3313 builder, location: loc, predicate: arith::CmpIPredicate::eq, lhs: ixRem, rhs: halfW);
3314
3315 auto iyIsSinSkippable =
3316 arith::OrIOp::create(builder, location: loc, lhs: iyIs0, rhs: iyIsHalfH);
3317 auto ixIsSinSkippable =
3318 arith::OrIOp::create(builder, location: loc, lhs: ixIs0, rhs: ixIsHalfW);
3319 auto shouldSkipSin = arith::AndIOp::create(builder, location: loc, lhs: iyIsSinSkippable,
3320 rhs: ixIsSinSkippable);
3321
3322 // realComponent = valReal * cos(angle)
3323 // imagComponent = valReal * (shouldSkipSin ? 0.0 : sin(angle))
3324 auto cosAngle = math::CosOp::create(builder, location: loc, operand: angle);
3325 auto sinAngle = math::SinOp::create(builder, location: loc, operand: angle);
3326 auto imagWeight = arith::SelectOp::create(builder, location: loc, condition: shouldSkipSin,
3327 true_value: zeroFloat, false_value: sinAngle);
3328 auto realComponent =
3329 arith::MulFOp::create(builder, location: loc, lhs: valReal, rhs: cosAngle);
3330 auto imagComponent =
3331 arith::MulFOp::create(builder, location: loc, lhs: valReal, rhs: imagWeight);
3332
3333 // outReal = sumReal + realComponent
3334 // outImag = sumImag - imagComponent
3335 auto outReal =
3336 arith::AddFOp::create(builder, location: loc, lhs: sumReal, rhs: realComponent);
3337 auto outImag =
3338 arith::SubFOp::create(builder, location: loc, lhs: sumImag, rhs: imagComponent);
3339
3340 linalg::YieldOp::create(builder, location: loc, values: ValueRange{outReal, outImag});
3341 };
3342
3343 rewriter.replaceOpWithNewOp<linalg::GenericOp>(
3344 op: rfft2d, args: rfft2d.getResultTypes(), args&: genericOpInputs, args&: genericOpOutputs,
3345 args&: indexingMaps, args&: iteratorTypes, args&: buildBody);
3346
3347 return success();
3348 }
3349};
3350
3351struct FFT2dConverter final : OpRewritePattern<FFT2dOp> {
3352 using Base::Base;
3353
3354 LogicalResult matchAndRewrite(FFT2dOp fft2d,
3355 PatternRewriter &rewriter) const override {
3356 if (!llvm::all_of(Range: fft2d->getOperandTypes(),
3357 P: RFFT2dConverter::isRankedTensor) ||
3358 !llvm::all_of(Range: fft2d->getResultTypes(),
3359 P: RFFT2dConverter::isRankedTensor)) {
3360 return rewriter.notifyMatchFailure(arg&: fft2d, msg: "only supports ranked tensors");
3361 }
3362
3363 Location loc = fft2d.getLoc();
3364 Value input_real = fft2d.getInputReal();
3365 Value input_imag = fft2d.getInputImag();
3366 BoolAttr inverse = fft2d.getInverseAttr();
3367
3368 auto real_el_ty = cast<FloatType>(
3369 Val: cast<ShapedType>(Val: input_real.getType()).getElementType());
3370 [[maybe_unused]] auto imag_el_ty = cast<FloatType>(
3371 Val: cast<ShapedType>(Val: input_imag.getType()).getElementType());
3372
3373 assert(real_el_ty == imag_el_ty);
3374
3375 // Compute the output type and set of dynamic sizes
3376 SmallVector<Value> dynamicSizes;
3377
3378 // Get [N, H, W]
3379 auto dims = tensor::getMixedSizes(builder&: rewriter, loc, value: input_real);
3380
3381 SmallVector<int64_t, 3> staticSizes;
3382 dispatchIndexOpFoldResults(ofrs: dims, dynamicVec&: dynamicSizes, staticVec&: staticSizes);
3383
3384 auto outputType = RankedTensorType::get(shape: staticSizes, elementType: real_el_ty);
3385
3386 // Iterator types for the linalg.generic implementation
3387 SmallVector<utils::IteratorType, 5> iteratorTypes = {
3388 utils::IteratorType::parallel, utils::IteratorType::parallel,
3389 utils::IteratorType::parallel, utils::IteratorType::reduction,
3390 utils::IteratorType::reduction};
3391
3392 // Inputs/outputs to the linalg.generic implementation
3393 SmallVector<Value> genericOpInputs = {input_real, input_imag};
3394 SmallVector<Value> genericOpOutputs = {
3395 RFFT2dConverter::createZeroTensor(rewriter, loc, type: outputType,
3396 dynamicSizes),
3397 RFFT2dConverter::createZeroTensor(rewriter, loc, type: outputType,
3398 dynamicSizes)};
3399
3400 // Indexing maps for input and output tensors
3401 auto indexingMaps = AffineMap::inferFromExprList(
3402 exprsList: ArrayRef{RFFT2dConverter::affineDimsExpr(builder&: rewriter, args: 0, args: 3, args: 4),
3403 RFFT2dConverter::affineDimsExpr(builder&: rewriter, args: 0, args: 3, args: 4),
3404 RFFT2dConverter::affineDimsExpr(builder&: rewriter, args: 0, args: 1, args: 2),
3405 RFFT2dConverter::affineDimsExpr(builder&: rewriter, args: 0, args: 1, args: 2)},
3406 context: rewriter.getContext());
3407
3408 // Width and height dimensions of the original input.
3409 auto dimH = rewriter.createOrFold<tensor::DimOp>(location: loc, args&: input_real, args: 1);
3410 auto dimW = rewriter.createOrFold<tensor::DimOp>(location: loc, args&: input_real, args: 2);
3411
3412 // Constants and dimension sizes
3413 auto twoPiAttr = rewriter.getFloatAttr(type: real_el_ty, value: 6.283185307179586);
3414 auto twoPi = arith::ConstantOp::create(builder&: rewriter, location: loc, value: twoPiAttr);
3415 Value constH =
3416 RFFT2dConverter::castIndexToFloat(builder&: rewriter, loc, type: real_el_ty, value: dimH);
3417 Value constW =
3418 RFFT2dConverter::castIndexToFloat(builder&: rewriter, loc, type: real_el_ty, value: dimW);
3419
3420 auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange args) {
3421 Value valReal = args[0];
3422 Value valImag = args[1];
3423 Value sumReal = args[2];
3424 Value sumImag = args[3];
3425
3426 // Indices for angle computation
3427 Value oy = linalg::IndexOp::create(builder, location: loc, dim: 1);
3428 Value ox = linalg::IndexOp::create(builder, location: loc, dim: 2);
3429 Value iy = linalg::IndexOp::create(builder, location: loc, dim: 3);
3430 Value ix = linalg::IndexOp::create(builder, location: loc, dim: 4);
3431
3432 // float_t angle = sign_val * 2 * pi() * ( ( (iy * oy) % H) / H + ( (ix *
3433 // ox) % W ) / W);
3434 auto iyXoy = index::MulOp::create(builder, location: loc, lhs: iy, rhs: oy);
3435 auto ixXox = index::MulOp::create(builder, location: loc, lhs: ix, rhs: ox);
3436
3437 auto iyRem = index::RemUOp::create(builder, location: loc, lhs: iyXoy, rhs: dimH);
3438 auto ixRem = index::RemUOp::create(builder, location: loc, lhs: ixXox, rhs: dimW);
3439
3440 auto iyRemFloat =
3441 RFFT2dConverter::castIndexToFloat(builder, loc, type: real_el_ty, value: iyRem);
3442 auto ixRemFloat =
3443 RFFT2dConverter::castIndexToFloat(builder, loc, type: real_el_ty, value: ixRem);
3444
3445 auto yComponent = arith::DivFOp::create(builder, location: loc, lhs: iyRemFloat, rhs: constH);
3446 auto xComponent = arith::DivFOp::create(builder, location: loc, lhs: ixRemFloat, rhs: constW);
3447
3448 auto sumXY = arith::AddFOp::create(builder, location: loc, lhs: yComponent, rhs: xComponent);
3449 auto angle = arith::MulFOp::create(builder, location: loc, lhs: twoPi, rhs: sumXY);
3450
3451 if (inverse.getValue()) {
3452 angle = arith::MulFOp::create(
3453 builder, location: loc, lhs: angle,
3454 rhs: arith::ConstantOp::create(builder&: rewriter, location: loc,
3455 value: rewriter.getFloatAttr(type: real_el_ty, value: -1.0)));
3456 }
3457
3458 // realComponent = val_real * cos(a) + val_imag * sin(a);
3459 // imagComponent = -val_real * sin(a) + val_imag * cos(a);
3460 auto cosAngle = math::CosOp::create(builder, location: loc, operand: angle);
3461 auto sinAngle = math::SinOp::create(builder, location: loc, operand: angle);
3462
3463 auto rcos = arith::MulFOp::create(builder, location: loc, lhs: valReal, rhs: cosAngle);
3464 auto rsin = arith::MulFOp::create(builder, location: loc, lhs: valImag, rhs: sinAngle);
3465 auto realComponent = arith::AddFOp::create(builder, location: loc, lhs: rcos, rhs: rsin);
3466
3467 auto icos = arith::MulFOp::create(builder, location: loc, lhs: valImag, rhs: cosAngle);
3468 auto isin = arith::MulFOp::create(builder, location: loc, lhs: valReal, rhs: sinAngle);
3469
3470 auto imagComponent = arith::SubFOp::create(builder, location: loc, lhs: icos, rhs: isin);
3471
3472 // outReal = sumReal + realComponent
3473 // outImag = sumImag - imagComponent
3474 auto outReal =
3475 arith::AddFOp::create(builder, location: loc, lhs: sumReal, rhs: realComponent);
3476 auto outImag =
3477 arith::AddFOp::create(builder, location: loc, lhs: sumImag, rhs: imagComponent);
3478
3479 linalg::YieldOp::create(builder, location: loc, values: ValueRange{outReal, outImag});
3480 };
3481
3482 rewriter.replaceOpWithNewOp<linalg::GenericOp>(
3483 op: fft2d, args: fft2d.getResultTypes(), args&: genericOpInputs, args&: genericOpOutputs,
3484 args&: indexingMaps, args&: iteratorTypes, args&: buildBody);
3485
3486 return success();
3487 }
3488};
3489
3490} // namespace
3491
3492void mlir::tosa::populateTosaToLinalgConversionPatterns(
3493 const TypeConverter &converter, RewritePatternSet *patterns,
3494 const TosaToLinalgOptions &options) {
3495
3496 // We have multiple resize coverters to handle degenerate cases.
3497 patterns->add<GenericResizeConverter>(arg: patterns->getContext(),
3498 /*benefit=*/args: 100);
3499 patterns->add<ResizeUnaryConverter>(arg: patterns->getContext(),
3500 /*benefit=*/args: 200);
3501 patterns->add<MaterializeResizeBroadcast>(arg: patterns->getContext(),
3502 /*benefit=*/args: 300);
3503
3504 patterns->add<
3505 // clang-format off
3506 PointwiseConverter<tosa::AddOp>,
3507 PointwiseConverter<tosa::SubOp>,
3508 PointwiseConverter<tosa::MulOp>,
3509 PointwiseConverter<tosa::IntDivOp>,
3510 PointwiseConverter<tosa::NegateOp>,
3511 PointwiseConverter<tosa::PowOp>,
3512 PointwiseConverter<tosa::ReciprocalOp>,
3513 PointwiseConverter<tosa::RsqrtOp>,
3514 PointwiseConverter<tosa::LogOp>,
3515 PointwiseConverter<tosa::ExpOp>,
3516 PointwiseConverter<tosa::AbsOp>,
3517 PointwiseConverter<tosa::SinOp>,
3518 PointwiseConverter<tosa::CosOp>,
3519 PointwiseConverter<tosa::TanhOp>,
3520 PointwiseConverter<tosa::ErfOp>,
3521 PointwiseConverter<tosa::BitwiseAndOp>,
3522 PointwiseConverter<tosa::BitwiseOrOp>,
3523 PointwiseConverter<tosa::BitwiseNotOp>,
3524 PointwiseConverter<tosa::BitwiseXorOp>,
3525 PointwiseConverter<tosa::LogicalAndOp>,
3526 PointwiseConverter<tosa::LogicalNotOp>,
3527 PointwiseConverter<tosa::LogicalOrOp>,
3528 PointwiseConverter<tosa::LogicalXorOp>,
3529 PointwiseConverter<tosa::CastOp>,
3530 PointwiseConverter<tosa::LogicalLeftShiftOp>,
3531 PointwiseConverter<tosa::LogicalRightShiftOp>,
3532 PointwiseConverter<tosa::ArithmeticRightShiftOp>,
3533 PointwiseConverter<tosa::ClzOp>,
3534 PointwiseConverter<tosa::SelectOp>,
3535 PointwiseConverter<tosa::GreaterOp>,
3536 PointwiseConverter<tosa::GreaterEqualOp>,
3537 PointwiseConverter<tosa::EqualOp>,
3538 PointwiseConverter<tosa::MaximumOp>,
3539 PointwiseConverter<tosa::MinimumOp>,
3540 PointwiseConverter<tosa::CeilOp>,
3541 PointwiseConverter<tosa::FloorOp>,
3542 PointwiseConverter<tosa::ClampOp>,
3543 PointwiseConverter<tosa::SigmoidOp>
3544 >(arg: converter, args: patterns->getContext());
3545
3546 patterns->add<QuantizedMatMulTConverter>(arg: converter, args: patterns->getContext());
3547
3548 patterns->add<
3549 IdentityNConverter<tosa::IdentityOp>,
3550 GatherConverter,
3551 RowGatherConverter,
3552 RescaleConverter,
3553 ReverseConverter,
3554 RFFT2dConverter,
3555 FFT2dConverter,
3556 TableConverter,
3557 TileConverter>(arg: patterns->getContext());
3558
3559 // Reductions seeded with a float min/max identity need to know whether
3560 // non-finite values are available on the target.
3561 patterns->add<
3562 ReduceConverter<tosa::ReduceAllOp>,
3563 ReduceConverter<tosa::ReduceAnyOp>,
3564 ReduceConverter<tosa::ReduceMinOp>,
3565 ReduceConverter<tosa::ReduceMaxOp>,
3566 ReduceConverter<tosa::ReduceSumOp>,
3567 ReduceConverter<tosa::ReduceProductOp>,
3568 ArgMinMaxConverter<tosa::ArgMaxOp>,
3569 ArgMinMaxConverter<tosa::ArgMinOp>>(arg: patterns->getContext(), args: options.allowNonFinites);
3570 // clang-format on
3571}
3572