1//===- TosaCanonicalizations.cpp - Canonicalization patterns & folders ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// \file
10// TOSA canonicalization patterns and folders.
11//
12//===----------------------------------------------------------------------===//
13
14#include "mlir/Dialect/Quant/IR/Quant.h"
15#include "mlir/Dialect/Tensor/IR/Tensor.h"
16#include "mlir/Dialect/Tosa/IR/TargetEnv.h"
17#include "mlir/Dialect/Tosa/IR/TosaOps.h"
18#include "mlir/Dialect/Tosa/Utils/ConversionUtils.h"
19#include "mlir/Dialect/Tosa/Utils/QuantUtils.h"
20#include "mlir/Dialect/Traits.h"
21#include "mlir/IR/BuiltinTypeInterfaces.h"
22#include "mlir/IR/BuiltinTypes.h"
23#include "mlir/IR/IRMapping.h"
24#include "mlir/IR/Matchers.h"
25#include "mlir/IR/PatternMatch.h"
26#include "mlir/IR/TypeUtilities.h"
27#include "mlir/Transforms/FoldUtils.h"
28#include "mlir/Transforms/InliningUtils.h"
29#include "llvm/ADT/APFloat.h"
30#include "llvm/ADT/APInt.h"
31
32#include <functional>
33
34using namespace mlir;
35using namespace mlir::tosa;
36
37namespace {
38OpFoldResult foldToInputIfTypeMatches(Type typeRef, Value input) {
39 return input.getType() == typeRef ? OpFoldResult(input) : OpFoldResult{};
40}
41} // namespace
42
43//===----------------------------------------------------------------------===//
44// Operator Canonicalizers.
45//===----------------------------------------------------------------------===//
46
47//===----------------------------------------------------------------------===//
48// Tensor Data Engine Operators.
49//===----------------------------------------------------------------------===//
50
51// Check that the zero point of the tensor and padding operations are aligned.
52static bool checkMatchingPadConstAndZp(Value padConst, Value zp) {
53 // Check that padConst is a constant value and a scalar tensor
54 DenseElementsAttr padConstAttr;
55 if (!matchPattern(value: padConst, pattern: m_Constant(bind_value: &padConstAttr)) ||
56 (padConstAttr.size() != 1)) {
57 return false;
58 }
59
60 // Check that floating point pad is zero
61 if (auto padConstFpAttr = mlir::dyn_cast<DenseFPElementsAttr>(Val&: padConstAttr)) {
62 float padConstVal = (*padConstFpAttr.begin()).convertToFloat();
63 return padConstVal == 0.0f;
64 }
65
66 // Check that the zp and padConst align for the integer (quantized) case
67 if (auto padConstIntAttr =
68 mlir::dyn_cast<DenseIntElementsAttr>(Val&: padConstAttr)) {
69 DenseIntElementsAttr zpAttr;
70 // Check that zp is a constant value and a scalar tensor
71 if (!matchPattern(value: zp, pattern: m_Constant(bind_value: &zpAttr)) || (padConstAttr.size() != 1)) {
72 return false;
73 }
74
75 // Check equality
76 int64_t zpVal = (*zpAttr.begin()).getSExtValue();
77 int64_t padConstVal = (*padConstIntAttr.begin()).getSExtValue();
78 return zpVal == padConstVal;
79 }
80
81 // Bail-out on unsupported type
82 return false;
83}
84
85namespace {
86template <typename OpTy>
87struct PoolPadFoldAdaptor;
88
89template <>
90struct PoolPadFoldAdaptor<tosa::MaxPool2dOp> {
91 using OpTy = tosa::MaxPool2dOp;
92 static bool checkKernelCompliance(OpTy op, const ArrayRef<int64_t> newPad) {
93 const llvm::ArrayRef<int64_t> kernel = op.getKernel();
94 if (newPad[2] >= kernel[1] || newPad[3] >= kernel[1] ||
95 newPad[0] >= kernel[0] || newPad[1] >= kernel[0])
96 return false;
97 return true;
98 }
99 static bool checkPadConstCompliance(OpTy, Value padConst) {
100 // Check that padConst is a constant value and a scalar tensor
101 DenseElementsAttr padConstAttr;
102 if (!matchPattern(value: padConst, pattern: m_Constant(bind_value: &padConstAttr)) ||
103 padConstAttr.size() != 1) {
104 return false;
105 }
106
107 // Pad needs to be in the minimum value to be able to merge
108 if (auto padConstFpAttr =
109 mlir::dyn_cast<DenseFPElementsAttr>(Val&: padConstAttr)) {
110 const APFloat padConstVal = *padConstFpAttr.begin();
111 const APFloat lowestVal =
112 APFloat::getLargest(Sem: padConstVal.getSemantics(), Negative: true);
113 return padConstVal == lowestVal;
114 }
115 if (auto padConstIntAttr =
116 mlir::dyn_cast<DenseIntElementsAttr>(Val&: padConstAttr)) {
117 const APInt padConstVal = *padConstIntAttr.begin();
118 const unsigned int bitWidth = padConstVal.getBitWidth();
119 const APInt lowestVal =
120 padConstIntAttr.getElementType().isUnsignedInteger()
121 ? APInt::getZero(numBits: bitWidth)
122 : APInt::getSignedMinValue(numBits: bitWidth);
123 return padConstVal == lowestVal;
124 }
125
126 // Bail-out on unsupported type
127 return false;
128 }
129 static void replaceOpWithNewPad(PatternRewriter &rewriter, OpTy op,
130 Value padInput, ArrayRef<int64_t> newPad) {
131 rewriter.replaceOpWithNewOp<tosa::MaxPool2dOp>(
132 op, args: op.getType(), args&: padInput, args: op.getKernel(), args: op.getStride(),
133 args: rewriter.getDenseI64ArrayAttr(values: newPad), args: op.getNanMode());
134 }
135};
136
137template <typename OpTy>
138struct ConvPadFoldAdaptor {
139 static bool checkKernelCompliance(OpTy, const ArrayRef<int64_t>) {
140 return true;
141 }
142 static bool checkPadConstCompliance(OpTy op, Value padConst) {
143 return checkMatchingPadConstAndZp(padConst, op.getInputZp());
144 }
145 static void replaceOpWithNewPad(PatternRewriter &rewriter, OpTy op,
146 Value padInput, ArrayRef<int64_t> newPad) {
147 rewriter.replaceOpWithNewOp<OpTy>(
148 op, op.getResult().getType(), padInput, op.getWeight(), op.getBias(),
149 op.getInputZp(), op.getWeightZp(), newPad, op.getStrideAttr(),
150 op.getDilationAttr(), op.getAccType(), op.getLocalBound());
151 }
152};
153
154// Pattern attempts to fold a `tosa.pad` operator to a following tensor
155// operation like `tosa.conv2d` by merging the padding associated with the
156// pad operator directly to the implicit padding of the tensor operation.
157// This helps eliminate the explicit padding operator if unused.
158template <typename OpTy, typename AdaptorTy>
159struct FoldPadToTensorOp : public OpRewritePattern<OpTy> {
160 using OpRewritePattern<OpTy>::OpRewritePattern;
161
162 LogicalResult matchAndRewrite(OpTy tensorOp,
163 PatternRewriter &rewriter) const override {
164 // Check producer is a tosa::PadOp
165 auto padOp = tensorOp.getInput().template getDefiningOp<tosa::PadOp>();
166 if (!padOp)
167 return rewriter.notifyMatchFailure(tensorOp,
168 "Producer must be a tosa::PadOp.");
169
170 // Validate that tensor operation has sane padding
171 const std::vector<int64_t> &tensorOpPad = tensorOp.getPad().vec();
172 if (tensorOpPad.size() != 4) // pad_top, pad_bottom, pad_left, pad_right
173 return rewriter.notifyMatchFailure(
174 tensorOp, "Tensor operation padding shall have 4 elements.");
175
176 // Validate tosa::PadOp padding
177 DenseIntElementsAttr padOpPadding;
178 if (!matchPattern(padOp.getPadding(), m_Constant(bind_value: &padOpPadding))) {
179 return rewriter.notifyMatchFailure(
180 tensorOp,
181 "The `padding` input specified on the tosa::PadOp must be constant.");
182 }
183 // N_before, N_after, H_before, H_after, W_before, W_after, C_before,
184 // C_after
185 if (padOpPadding.size() != 8)
186 return rewriter.notifyMatchFailure(tensorOp,
187 "Pad padding should have 8 elements.");
188 int64_t padNBefore = (*(padOpPadding.begin() + 0)).getLimitedValue();
189 int64_t padNAfter = (*(padOpPadding.begin() + 1)).getLimitedValue();
190 int64_t padHBefore = (*(padOpPadding.begin() + 2)).getLimitedValue();
191 int64_t padHAfter = (*(padOpPadding.begin() + 3)).getLimitedValue();
192 int64_t padWBefore = (*(padOpPadding.begin() + 4)).getLimitedValue();
193 int64_t padWAfter = (*(padOpPadding.begin() + 5)).getLimitedValue();
194 int64_t padCBefore = (*(padOpPadding.begin() + 6)).getLimitedValue();
195 int64_t padCAfter = (*(padOpPadding.begin() + 7)).getLimitedValue();
196
197 if (padNBefore != 0 || padNAfter != 0 || padCBefore != 0 || padCAfter != 0)
198 return rewriter.notifyMatchFailure(
199 tensorOp, "Folding padding in N or C dimensions is not supported.");
200
201 // Fold padding from Pad into the tensor operation
202 // 4 elements - pad_top, pad_bottom, pad_left, pad_right
203 SmallVector<int64_t> foldedPad(tensorOpPad.size());
204 foldedPad[0] = padHBefore + tensorOpPad[0];
205 foldedPad[1] = padHAfter + tensorOpPad[1];
206 foldedPad[2] = padWBefore + tensorOpPad[2];
207 foldedPad[3] = padWAfter + tensorOpPad[3];
208
209 // Check kernel related restrictions
210 if (!AdaptorTy::checkKernelCompliance(tensorOp, foldedPad)) {
211 return rewriter.notifyMatchFailure(
212 tensorOp, "Padding size not aligned with kernel restrictions.");
213 }
214
215 // Check padding constant restrictions
216 if (!AdaptorTy::checkPadConstCompliance(tensorOp, padOp.getPadConst())) {
217 return rewriter.notifyMatchFailure(
218 tensorOp,
219 "Padding constant is not aligned with operator zero-point.");
220 }
221
222 // Check that padding doesn't grow more than 8K level (8192) for now
223 if (llvm::any_of(foldedPad, [](int64_t padVal) { return padVal > 8192; })) {
224 return rewriter.notifyMatchFailure(
225 tensorOp, "Padding size more than the 8K level limit.");
226 }
227
228 // Create operator
229 AdaptorTy::replaceOpWithNewPad(rewriter, tensorOp, padOp.getInput1(),
230 foldedPad);
231
232 return success();
233 }
234};
235} // namespace
236
237void Conv2DOp::getCanonicalizationPatterns(RewritePatternSet &results,
238 MLIRContext *context) {
239 results.add<
240 FoldPadToTensorOp<tosa::Conv2DOp, ConvPadFoldAdaptor<tosa::Conv2DOp>>>(
241 arg&: context);
242}
243
244void DepthwiseConv2DOp::getCanonicalizationPatterns(RewritePatternSet &results,
245 MLIRContext *context) {
246 results.add<FoldPadToTensorOp<tosa::DepthwiseConv2DOp,
247 ConvPadFoldAdaptor<tosa::DepthwiseConv2DOp>>>(
248 arg&: context);
249}
250
251struct AvgPool2dAdaptiveToAvgPool2d
252 : public OpRewritePattern<tosa::AvgPool2dAdaptiveOp> {
253 using Base::Base;
254
255 LogicalResult matchAndRewrite(tosa::AvgPool2dAdaptiveOp op,
256 PatternRewriter &rewriter) const override {
257 llvm::SmallVector<int64_t> kernel;
258 llvm::SmallVector<int64_t> stride;
259 llvm::SmallVector<int64_t> pad;
260 if (!tosa::getConstShapeValues(op: op.getKernel().getDefiningOp(), result_shape&: kernel) ||
261 !tosa::getConstShapeValues(op: op.getStride().getDefiningOp(), result_shape&: stride) ||
262 !tosa::getConstShapeValues(op: op.getPad().getDefiningOp(), result_shape&: pad))
263 return rewriter.notifyMatchFailure(
264 arg&: op, msg: "expected constant kernel, stride, and pad operands");
265
266 auto replacement = tosa::AvgPool2dOp::create(
267 builder&: rewriter, location: op.getLoc(), output: op.getType(), input: op.getInput(), input_zp: op.getInputZp(),
268 output_zp: op.getOutputZp(), kernel: rewriter.getDenseI64ArrayAttr(values: kernel),
269 stride: rewriter.getDenseI64ArrayAttr(values: stride),
270 pad: rewriter.getDenseI64ArrayAttr(values: pad), acc_type: op.getAccTypeAttr());
271 rewriter.replaceOp(op, newValues: replacement.getOutput());
272 return success();
273 }
274};
275
276struct AvgPool2dIsNoOp : public OpRewritePattern<tosa::AvgPool2dOp> {
277 using Base::Base;
278
279 LogicalResult matchAndRewrite(tosa::AvgPool2dOp op,
280 PatternRewriter &rewriter) const override {
281 // Prevent canonicalization if input/output shapes don't align
282 if (op.getInput().getType() != op.getOutput().getType())
283 return rewriter.notifyMatchFailure(
284 arg&: op, msg: "expected input and output types to match");
285
286 const auto inputType = llvm::cast<ShapedType>(Val: op.getInput().getType());
287 if (!llvm::isa<FloatType>(Val: inputType.getElementType()))
288 return rewriter.notifyMatchFailure(arg&: op,
289 msg: "expected floating-point input type");
290
291 // For statically known zero points, the verifier ensures zero points are
292 // zero for floating-point types
293 if (!matchPattern(value: op.getInputZp(), pattern: m_Constant()) ||
294 !matchPattern(value: op.getOutputZp(), pattern: m_Constant()))
295 return rewriter.notifyMatchFailure(
296 arg&: op,
297 msg: "expected input and output zero points to be statically verifiable");
298
299 if (!llvm::all_of(Range: op.getKernel(), P: [](int64_t val) { return val == 1; }))
300 return rewriter.notifyMatchFailure(arg&: op, msg: "expected unit kernel");
301
302 if (!llvm::all_of(Range: op.getStride(), P: [](int64_t val) { return val == 1; }))
303 return rewriter.notifyMatchFailure(arg&: op, msg: "expected unit stride");
304
305 if (!llvm::all_of(Range: op.getPad(), P: [](int64_t val) { return val == 0; }))
306 return rewriter.notifyMatchFailure(arg&: op, msg: "expected zero padding");
307
308 rewriter.replaceOp(op, newValues: op.getInput());
309 return success();
310 }
311};
312
313void AvgPool2dOp::getCanonicalizationPatterns(RewritePatternSet &results,
314 MLIRContext *context) {
315 results.add<AvgPool2dIsNoOp>(arg&: context);
316}
317
318void AvgPool2dAdaptiveOp::getCanonicalizationPatterns(
319 RewritePatternSet &results, MLIRContext *context) {
320 results.add<AvgPool2dAdaptiveToAvgPool2d>(arg&: context);
321}
322
323struct MaxPool2dIsNoOp : public OpRewritePattern<tosa::MaxPool2dOp> {
324 using Base::Base;
325
326 LogicalResult matchAndRewrite(tosa::MaxPool2dOp op,
327 PatternRewriter &rewriter) const override {
328 Value input = op.getInput();
329 Value output = op.getOutput();
330 ShapedType inputType = llvm::cast<ShapedType>(Val: input.getType());
331 ShapedType outputType = llvm::cast<ShapedType>(Val: output.getType());
332
333 if (input.getType() == output.getType() &&
334 llvm::all_of(Range: op.getKernel(), P: [](int64_t val) { return val == 1; }) &&
335 llvm::all_of(Range: op.getStride(), P: [](int64_t val) { return val == 1; }) &&
336 llvm::all_of(Range: op.getPad(), P: [](int64_t val) { return val == 0; }) &&
337 op.getNanMode() == tosa::NanPropagationMode::PROPAGATE) {
338 rewriter.replaceOp(op, newValues: input);
339 return success();
340 }
341
342 if (!inputType.hasStaticShape() || !outputType.hasStaticShape()) {
343 return failure();
344 }
345
346 // If the output and input shapes are 1x1, then this is a no op.
347 ArrayRef<int64_t> outputShape = outputType.getShape();
348 if (outputShape[1] != 1 || outputShape[2] != 1) {
349 return failure();
350 }
351
352 ArrayRef<int64_t> inputShape = inputType.getShape();
353 if (inputShape[1] != 1 || inputShape[2] != 1) {
354 return failure();
355 }
356
357 rewriter.replaceOp(op, newValues: input);
358 return success();
359 }
360};
361
362void MaxPool2dOp::getCanonicalizationPatterns(RewritePatternSet &results,
363 MLIRContext *context) {
364 results.add<MaxPool2dIsNoOp,
365 FoldPadToTensorOp<tosa::MaxPool2dOp,
366 PoolPadFoldAdaptor<tosa::MaxPool2dOp>>>(
367 arg&: context);
368}
369
370struct MaxPool2dAdaptiveToMaxPool2d
371 : public OpRewritePattern<tosa::MaxPool2dAdaptiveOp> {
372 using Base::Base;
373
374 LogicalResult matchAndRewrite(tosa::MaxPool2dAdaptiveOp op,
375 PatternRewriter &rewriter) const override {
376 llvm::SmallVector<int64_t> kernel;
377 llvm::SmallVector<int64_t> stride;
378 llvm::SmallVector<int64_t> pad;
379 if (!tosa::getConstShapeValues(op: op.getKernel().getDefiningOp(), result_shape&: kernel) ||
380 !tosa::getConstShapeValues(op: op.getStride().getDefiningOp(), result_shape&: stride) ||
381 !tosa::getConstShapeValues(op: op.getPad().getDefiningOp(), result_shape&: pad))
382 return rewriter.notifyMatchFailure(
383 arg&: op, msg: "expected constant kernel, stride, and pad operands");
384
385 auto replacement = tosa::MaxPool2dOp::create(
386 builder&: rewriter, location: op.getLoc(), output: op.getType(), input: op.getInput(),
387 kernel: rewriter.getDenseI64ArrayAttr(values: kernel),
388 stride: rewriter.getDenseI64ArrayAttr(values: stride),
389 pad: rewriter.getDenseI64ArrayAttr(values: pad), nan_mode: op.getNanModeAttr());
390 rewriter.replaceOp(op, newValues: replacement.getOutput());
391 return success();
392 }
393};
394
395void MaxPool2dAdaptiveOp::getCanonicalizationPatterns(
396 RewritePatternSet &results, MLIRContext *context) {
397 results.add<MaxPool2dAdaptiveToMaxPool2d>(arg&: context);
398}
399
400//===----------------------------------------------------------------------===//
401// Data Layout / Memory Reinterpretation.
402//===----------------------------------------------------------------------===//
403
404struct ConcatOptimization : public OpRewritePattern<tosa::ConcatOp> {
405 using OpRewritePattern<tosa::ConcatOp>::OpRewritePattern;
406
407 LogicalResult matchAndRewrite(tosa::ConcatOp op,
408 PatternRewriter &rewriter) const override {
409 if (op.getInput1().size() != 1)
410 return failure();
411 if (op.getInput1().front().getType() != op.getType()) {
412 rewriter
413 .replaceOpWithNewOp<tensor::CastOp>(op, args: op.getType(),
414 args: op.getInput1().front())
415 .getResult();
416 return success();
417 }
418
419 rewriter.replaceOp(op, newValues: op.getInput1().front());
420 return success();
421 }
422};
423
424struct ConsecutiveConcatOptimization : public OpRewritePattern<tosa::ConcatOp> {
425 using OpRewritePattern<tosa::ConcatOp>::OpRewritePattern;
426
427 LogicalResult matchAndRewrite(tosa::ConcatOp op,
428 PatternRewriter &rewriter) const override {
429 // Rewrite consecutive concats on the same axis into a single op.
430 // Keep track of the operands so we are able to construct a new concat
431 // later. Conservatively assume that we double the number of operands when
432 // canonicalizing
433 SmallVector<Value, 8> concatOperands;
434 concatOperands.reserve(N: 2 * op.getNumOperands());
435
436 int32_t maxNumOperands = 0;
437 if (auto targetEnvAttr = tosa::lookupTargetEnv(op))
438 maxNumOperands =
439 getTosaLevelFromEnum(level: targetEnvAttr.getLevel()).MAX_TENSOR_LIST_SIZE;
440
441 // Find all operands that are foldable concats
442 bool foundRewritableConcat = false;
443 for (Value operand : op.getOperands()) {
444 concatOperands.emplace_back(Args&: operand);
445
446 auto producer = operand.getDefiningOp<tosa::ConcatOp>();
447 if (!producer)
448 continue;
449
450 // Not rewritable if axes are not the same
451 if (op.getAxis() != producer.getAxis())
452 continue;
453
454 // Replace the original operand with all incoming operands
455 foundRewritableConcat = true;
456 concatOperands.pop_back();
457 llvm::append_range(C&: concatOperands, R: producer->getOperands());
458 }
459
460 if (!foundRewritableConcat)
461 return rewriter.notifyMatchFailure(arg&: op,
462 msg: "No rewritable concat operand found.");
463
464 if (maxNumOperands > 0 &&
465 concatOperands.size() > static_cast<size_t>(maxNumOperands))
466 return rewriter.notifyMatchFailure(
467 arg&: op, msg: "Rewriting would exceed the maximum number of operands for the "
468 "target environment level.");
469
470 rewriter.replaceOpWithNewOp<tosa::ConcatOp>(
471 op, args: op.getType(), args&: concatOperands, args: op.getAxisAttr());
472 return success();
473 }
474};
475
476void ConcatOp::getCanonicalizationPatterns(RewritePatternSet &results,
477 MLIRContext *context) {
478 results.add<ConcatOptimization, ConsecutiveConcatOptimization>(arg&: context);
479}
480
481LogicalResult SelectOp::canonicalize(SelectOp op, PatternRewriter &rewriter) {
482 auto notOp = op.getInput1().getDefiningOp<tosa::LogicalNotOp>();
483 if (!notOp)
484 return failure();
485 rewriter.modifyOpInPlace(root: op, callable: [&]() {
486 op.getOperation()->setOperands(
487 {notOp.getInput1(), op.getOnFalse(), op.getOnTrue()});
488 });
489 return success();
490}
491
492struct ConsolidateTransposeOptimization
493 : public OpRewritePattern<tosa::TransposeOp> {
494 using Base::Base;
495
496 LogicalResult matchAndRewrite(tosa::TransposeOp transposeOp,
497 PatternRewriter &rewriter) const override {
498 // Input is also TransposeOp - transpose(transpose(A)).
499 auto innerTranspose =
500 transposeOp.getInput1().getDefiningOp<tosa::TransposeOp>();
501 if (!innerTranspose)
502 return rewriter.notifyMatchFailure(arg&: transposeOp,
503 msg: "input must be transpose operation");
504
505 const llvm::ArrayRef<int32_t> transposePerms = transposeOp.getPerms();
506 const llvm::ArrayRef<int32_t> innerTransposePerms =
507 innerTranspose.getPerms();
508
509 if (transposePerms.size() != innerTransposePerms.size())
510 return rewriter.notifyMatchFailure(
511 arg&: transposeOp,
512 msg: "transpose and inner transpose perms sizes must be equal");
513 if (transposePerms.empty())
514 return rewriter.notifyMatchFailure(
515 arg&: transposeOp, msg: "transpose perms sizes must be positive");
516
517 // Consolidate transposes into one transpose.
518 SmallVector<int32_t> perms(transposePerms.size());
519 for (int i = 0, s = transposePerms.size(); i < s; ++i)
520 perms[i] = innerTransposePerms[transposePerms[i]];
521
522 rewriter.replaceOpWithNewOp<tosa::TransposeOp>(
523 op: transposeOp, args: transposeOp.getResult().getType(),
524 args: innerTranspose.getInput1(), args: rewriter.getDenseI32ArrayAttr(values: perms));
525
526 return success();
527 }
528};
529
530// Determines the case when tosa.transpose is a tosa.reshape operation.
531struct TransposeIsReshape : public OpRewritePattern<tosa::TransposeOp> {
532 using Base::Base;
533
534 LogicalResult matchAndRewrite(tosa::TransposeOp op,
535 PatternRewriter &rewriter) const override {
536 if (op.getInput1().getDefiningOp<tosa::TransposeOp>())
537 return rewriter.notifyMatchFailure(
538 arg&: op, msg: "Src is from transpose, can compose transposes");
539
540 Value result = op.getResult();
541 for (Operation *subop : result.getUsers()) {
542 if (isa_and_nonnull<tosa::TransposeOp>(Val: subop))
543 return rewriter.notifyMatchFailure(
544 arg&: op, msg: "Dest is used by transpose, can compose transposes");
545 }
546
547 auto input = op.getInput1();
548 auto inputTy = llvm::cast<ShapedType>(Val: input.getType());
549 if (!inputTy.hasRank())
550 return rewriter.notifyMatchFailure(arg&: op, msg: "Unranked input.");
551
552 int64_t numDynDims = 0;
553 for (int i = 0; i < inputTy.getRank(); ++i)
554 if (inputTy.isDynamicDim(idx: i))
555 numDynDims++;
556
557 if (numDynDims > 1)
558 return rewriter.notifyMatchFailure(arg&: op, msg: "Has more than one dynamic dim.");
559
560 const llvm::ArrayRef<int32_t> permValues = op.getPerms();
561
562 SmallVector<int64_t> nonZeroPerms;
563 nonZeroPerms.reserve(N: permValues.size());
564 for (auto idx : permValues) {
565 auto sz = inputTy.getDimSize(idx);
566 if (sz != 1)
567 nonZeroPerms.push_back(Elt: idx);
568 }
569
570 for (int i = 1, s = nonZeroPerms.size(); i < s; ++i)
571 if (nonZeroPerms[i - 1] > nonZeroPerms[i])
572 return rewriter.notifyMatchFailure(arg&: op,
573 msg: "Transpose changes memory layout.");
574
575 SmallVector<int64_t> newShape;
576 newShape.reserve(N: inputTy.getRank());
577 for (int i = 0, s = inputTy.getRank(); i < s; ++i)
578 newShape.push_back(Elt: inputTy.getDimSize(idx: permValues[i]));
579
580 rewriter.replaceOpWithNewOp<tosa::ReshapeOp>(
581 op, args: op.getType(), args: op.getInput1(),
582 args: getTosaConstShape(rewriter, loc: op.getLoc(), shape: newShape));
583 return success();
584 }
585};
586
587void TransposeOp::getCanonicalizationPatterns(RewritePatternSet &results,
588 MLIRContext *context) {
589 results.add<ConsolidateTransposeOptimization, TransposeIsReshape>(arg&: context);
590}
591
592struct ClampIsNoOp : public OpRewritePattern<tosa::ClampOp> {
593 using Base::Base;
594
595 LogicalResult matchAndRewrite(tosa::ClampOp op,
596 PatternRewriter &rewriter) const override {
597 Value input = op.getInput();
598 auto inputType = llvm::cast<ShapedType>(Val: op.getInput().getType());
599 auto inputElementType = inputType.getElementType();
600
601 if (isa<FloatType>(Val: inputElementType)) {
602 if (op.getNanMode() == tosa::NanPropagationMode::IGNORE)
603 return failure();
604
605 // Unlike integer types, floating point types can represent infinity.
606 const auto minClamp =
607 llvm::cast<mlir::FloatAttr>(Val: op.getMinValAttr()).getValue();
608 const auto maxClamp =
609 llvm::cast<mlir::FloatAttr>(Val: op.getMaxValAttr()).getValue();
610 const bool isMin = minClamp.isNegInfinity();
611 const bool isMax = maxClamp.isInfinity();
612
613 if (isMin && isMax) {
614 rewriter.replaceOp(op, newValues: input);
615 return success();
616 }
617 return failure();
618 }
619
620 // i1 types are boolean in TOSA
621 const bool isBoolean = inputElementType.isInteger(width: 1);
622 if (inputElementType.isUnsignedInteger() || isBoolean) {
623 const int64_t minClamp = llvm::cast<mlir::IntegerAttr>(Val: op.getMinValAttr())
624 .getValue()
625 .getZExtValue();
626 const int64_t maxClamp = llvm::cast<mlir::IntegerAttr>(Val: op.getMaxValAttr())
627 .getValue()
628 .getZExtValue();
629
630 const unsigned bitWidth = inputElementType.getIntOrFloatBitWidth();
631 const int64_t intMin = APInt::getMinValue(numBits: bitWidth).getZExtValue();
632 const int64_t intMax = APInt::getMaxValue(numBits: bitWidth).getZExtValue();
633
634 if (minClamp <= intMin && maxClamp >= intMax) {
635 rewriter.replaceOp(op, newValues: input);
636 return success();
637 }
638 return failure();
639 }
640
641 if (llvm::isa<IntegerType>(Val: inputElementType)) {
642 const int64_t minClamp =
643 llvm::cast<mlir::IntegerAttr>(Val: op.getMinValAttr()).getInt();
644 const int64_t maxClamp =
645 llvm::cast<mlir::IntegerAttr>(Val: op.getMaxValAttr()).getInt();
646
647 const unsigned bitWidth = inputElementType.getIntOrFloatBitWidth();
648 const int64_t intMin = APInt::getSignedMinValue(numBits: bitWidth).getSExtValue();
649 const int64_t intMax = APInt::getSignedMaxValue(numBits: bitWidth).getSExtValue();
650
651 if (minClamp <= intMin && maxClamp >= intMax) {
652 rewriter.replaceOp(op, newValues: input);
653 return success();
654 }
655 return failure();
656 }
657
658 return failure();
659 }
660};
661
662// Attempts the following transformation:
663//
664// For integers a, b, a', and b' such that [a, b] ∩ [a', b'] ≠ ∅ and input
665// tensor X the following identity holds:
666//
667// CLAMP(CLAMP(X, a, b), a', b') = CLAMP(X, max(a, a'), min(b, b'))
668//
669// subject to the following valid NaN propagation semantics:
670// --------------------------------------------
671// | OUTER CLAMP | INNER CLAMP | RESULT MODE |
672// |-------------|--------------|-------------|
673// | PROPAGATE | PROPAGATE | PROPAGATE |
674// | PROPAGATE | IGNORE | IGNORE |
675// | IGNORE | PROPAGATE | INVALID |
676// | IGNORE | IGNORE | IGNORE |
677// |------------------------------------------|
678
679struct ClampClampOptimization : public OpRewritePattern<tosa::ClampOp> {
680 using OpRewritePattern<tosa::ClampOp>::OpRewritePattern;
681
682 // Helper structure to describe the range of a clamp operation.
683 template <typename T>
684 struct ClampRange {
685 ClampRange(const T &start, const T &end) : start(start), end(end) {}
686 T start;
687 T end;
688
689 // Helper function to determine if two Clamp ranges intersect.
690 bool intersects(const ClampRange<T> &otherRange) {
691 return start < otherRange.end && otherRange.start < end;
692 }
693 };
694
695 LogicalResult matchAndRewrite(tosa::ClampOp op,
696 PatternRewriter &rewriter) const override {
697 Value input = op.getInput();
698
699 // Check the input to the CLAMP op is itself a CLAMP.
700 auto clampOp = input.getDefiningOp<tosa::ClampOp>();
701 if (!clampOp)
702 return failure();
703
704 // Check we have a valid NaN propagation combination.
705 const auto opNanMode = op.getNanMode();
706 const auto clampNanMode = clampOp.getNanMode();
707 if (opNanMode == NanPropagationMode::IGNORE &&
708 clampNanMode == NanPropagationMode::PROPAGATE)
709 return failure();
710
711 auto maxValAttr = op.getMaxValAttr();
712 auto minValAttr = op.getMinValAttr();
713 auto clampOpMaxValAttr = clampOp.getMaxValAttr();
714 auto clampOpMinValAttr = clampOp.getMinValAttr();
715
716 auto inputEType = llvm::cast<ShapedType>(Val: input.getType()).getElementType();
717 if (auto quantType =
718 llvm::dyn_cast<mlir::quant::UniformQuantizedType>(Val&: inputEType)) {
719 inputEType = getStorageElementTypeFromQuantized(quantizedType: quantType);
720 }
721
722 Attribute newMinValAttr, newMaxValAttr;
723 if (mlir::isa<FloatType>(Val: inputEType)) {
724 auto floatMaxValAttr = cast<mlir::FloatAttr>(Val&: maxValAttr);
725 auto floatMinValAttr = cast<mlir::FloatAttr>(Val&: minValAttr);
726 auto clampOpFloatMaxValAttr = cast<mlir::FloatAttr>(Val&: clampOpMaxValAttr);
727 auto clampOpFloatMinValAttr = cast<mlir::FloatAttr>(Val&: clampOpMinValAttr);
728
729 // Check we have intersecting ranges.
730 const auto opMinFloat = floatMinValAttr.getValue();
731 const auto opMaxFloat = floatMaxValAttr.getValue();
732 const auto clampOpMinFloat = clampOpFloatMinValAttr.getValue();
733 const auto clampOpMaxFloat = clampOpFloatMaxValAttr.getValue();
734 ClampRange<APFloat> opRangeFloatRange(opMinFloat, opMaxFloat);
735 ClampRange<APFloat> clampRangeFloatRange(clampOpMinFloat,
736 clampOpMaxFloat);
737 if (!opRangeFloatRange.intersects(otherRange: clampRangeFloatRange))
738 return failure();
739
740 // Run the transformation.
741 auto newMinVal = std::max(a: opMinFloat, b: clampOpMinFloat);
742 auto newMaxVal = std::min(a: opMaxFloat, b: clampOpMaxFloat);
743 newMinValAttr = rewriter.getFloatAttr(type: inputEType, value: newMinVal);
744 newMaxValAttr = rewriter.getFloatAttr(type: inputEType, value: newMaxVal);
745 } else {
746 assert(mlir::isa<IntegerType>(inputEType));
747 auto intMaxValAttr = cast<mlir::IntegerAttr>(Val&: maxValAttr);
748 auto intMinValAttr = cast<mlir::IntegerAttr>(Val&: minValAttr);
749 auto clampOpIntMaxValAttr = cast<mlir::IntegerAttr>(Val&: clampOpMaxValAttr);
750 auto clampOpIntMinValAttr = cast<mlir::IntegerAttr>(Val&: clampOpMinValAttr);
751
752 if (inputEType.isUnsignedInteger()) {
753 // Check we have intersecting ranges.
754 const auto opMinInt = intMinValAttr.getUInt();
755 const auto opMaxInt = intMaxValAttr.getUInt();
756 const auto clampOpMinInt = clampOpIntMinValAttr.getUInt();
757 const auto clampOpMaxInt = clampOpIntMaxValAttr.getUInt();
758 ClampRange<std::uint64_t> opRangeIntRange(opMinInt, opMaxInt);
759 ClampRange<std::uint64_t> clampRangeIntRange(clampOpMinInt,
760 clampOpMaxInt);
761 if (!opRangeIntRange.intersects(otherRange: clampRangeIntRange))
762 return failure();
763
764 // Run the transformation.
765 auto newMinVal = std::max(a: opMinInt, b: clampOpMinInt);
766 auto newMaxVal = std::min(a: opMaxInt, b: clampOpMaxInt);
767 newMinValAttr = rewriter.getIntegerAttr(type: inputEType, value: newMinVal);
768 newMaxValAttr = rewriter.getIntegerAttr(type: inputEType, value: newMaxVal);
769 } else {
770 // Check we have intersecting ranges.
771 const auto opMinInt = intMinValAttr.getInt();
772 const auto opMaxInt = intMaxValAttr.getInt();
773 const auto clampOpMinInt = clampOpIntMinValAttr.getInt();
774 const auto clampOpMaxInt = clampOpIntMaxValAttr.getInt();
775 ClampRange<std::int64_t> opRangeIntRange(opMinInt, opMaxInt);
776 ClampRange<std::int64_t> clampRangeIntRange(clampOpMinInt,
777 clampOpMaxInt);
778 if (!opRangeIntRange.intersects(otherRange: clampRangeIntRange))
779 return failure();
780
781 // Run the transformation.
782 auto newMinVal = std::max(a: opMinInt, b: clampOpMinInt);
783 auto newMaxVal = std::min(a: opMaxInt, b: clampOpMaxInt);
784 newMinValAttr = rewriter.getIntegerAttr(type: inputEType, value: newMinVal);
785 newMaxValAttr = rewriter.getIntegerAttr(type: inputEType, value: newMaxVal);
786 }
787 }
788
789 auto newMode = (opNanMode != clampNanMode)
790 ? tosa::NanPropagationMode::IGNORE
791 : opNanMode;
792
793 auto newModeAttr =
794 NanPropagationModeAttr::get(context: rewriter.getContext(), value: newMode);
795
796 rewriter.replaceOpWithNewOp<tosa::ClampOp>(
797 op, args: op.getType(), args: clampOp.getInput(), args&: newMinValAttr, args&: newMaxValAttr,
798 args&: newModeAttr);
799 return success();
800 }
801};
802
803void ClampOp::getCanonicalizationPatterns(RewritePatternSet &results,
804 MLIRContext *context) {
805 results.add<ClampIsNoOp>(arg&: context);
806 results.add<ClampClampOptimization>(arg&: context);
807}
808
809struct ConcatSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
810 using OpRewritePattern<tosa::SliceOp>::OpRewritePattern;
811
812 LogicalResult matchAndRewrite(tosa::SliceOp sliceOp,
813 PatternRewriter &rewriter) const override {
814 Value sliceInput = sliceOp.getInput1();
815 auto concatOp = sliceInput.getDefiningOp<tosa::ConcatOp>();
816 if (!concatOp)
817 return rewriter.notifyMatchFailure(
818 arg&: sliceOp, msg: "slice input must be concat operation");
819
820 OperandRange inputs = concatOp.getInput1();
821 auto concatType = dyn_cast<RankedTensorType>(Val: concatOp.getType());
822 if (!concatType || !concatType.hasStaticShape())
823 return rewriter.notifyMatchFailure(
824 arg&: sliceOp, msg: "slice input must be a static ranked tensor");
825 int32_t axis = concatOp.getAxis();
826
827 DenseElementsAttr startElems;
828 DenseElementsAttr sizeElems;
829
830 if (!matchPattern(value: sliceOp.getStart(), pattern: m_Constant(bind_value: &startElems)))
831 return rewriter.notifyMatchFailure(
832 arg&: sliceOp, msg: "start of slice must be a static ranked shape");
833
834 if (!matchPattern(value: sliceOp.getSize(), pattern: m_Constant(bind_value: &sizeElems)))
835 return rewriter.notifyMatchFailure(
836 arg&: sliceOp, msg: "size of slice must be a static ranked shape");
837
838 llvm::SmallVector<int64_t> sliceStarts =
839 llvm::to_vector(Range: startElems.getValues<int64_t>());
840 llvm::SmallVector<int64_t> sliceSizes =
841 llvm::to_vector(Range: sizeElems.getValues<int64_t>());
842
843 // Validate slice on the concatenated axis. Slicing along this
844 // axis should span only one of the inputs to the concatenate
845 // operation.
846 std::optional<Value> replaceWithSlice;
847 for (auto input : inputs) {
848 auto inputType = dyn_cast<RankedTensorType>(Val: input.getType());
849 if (!inputType || !inputType.hasStaticShape())
850 return rewriter.notifyMatchFailure(
851 arg&: sliceOp, msg: "concat input must be a static ranked tensor");
852
853 if (sliceStarts[axis] >= 0 && (sliceStarts[axis] + sliceSizes[axis]) <=
854 inputType.getDimSize(idx: axis)) {
855 auto start_op =
856 getTosaConstShape(rewriter, loc: sliceOp.getLoc(), shape: sliceStarts);
857 auto size_op =
858 getTosaConstShape(rewriter, loc: sliceOp.getLoc(), shape: sliceSizes);
859 replaceWithSlice =
860 tosa::SliceOp::create(builder&: rewriter, location: sliceOp.getLoc(), output: sliceOp.getType(),
861 input1: input, start: start_op, size: size_op)
862 .getResult();
863 break;
864 }
865 sliceStarts[axis] -= inputType.getDimSize(idx: axis);
866 }
867
868 if (!replaceWithSlice)
869 return rewriter.notifyMatchFailure(
870 arg&: sliceOp, msg: "corresponding concat input not found for slice");
871
872 rewriter.replaceOp(op: sliceOp, newValues: replaceWithSlice.value());
873 return success();
874 }
875};
876
877struct PadSliceOptimization : public OpRewritePattern<tosa::SliceOp> {
878 using OpRewritePattern<tosa::SliceOp>::OpRewritePattern;
879
880 LogicalResult matchAndRewrite(tosa::SliceOp sliceOp,
881 PatternRewriter &rewriter) const override {
882 Value sliceInput = sliceOp.getInput1();
883
884 // Check if producer is a PadOp
885 auto padOp = sliceInput.getDefiningOp<tosa::PadOp>();
886 if (!padOp)
887 return rewriter.notifyMatchFailure(arg&: sliceOp,
888 msg: "slice input must be a pad operation");
889
890 // Check PadOp has a single consumer
891 if (!padOp->hasOneUse())
892 return rewriter.notifyMatchFailure(arg&: sliceOp,
893 msg: "pad shall have a single consumer");
894
895 // Check input is statically ranked
896 auto inputTy = dyn_cast<RankedTensorType>(Val: padOp.getInput1().getType());
897 auto padTy = dyn_cast<RankedTensorType>(Val: padOp.getType());
898 if (!inputTy || !padTy || !inputTy.hasRank())
899 return rewriter.notifyMatchFailure(arg&: sliceOp,
900 msg: "slice input must be a ranked tensor");
901
902 // Validate and extract tosa::PadOp padding
903 DenseIntElementsAttr paddingElems;
904 if (!matchPattern(value: padOp.getPadding(), pattern: m_Constant(bind_value: &paddingElems))) {
905 return rewriter.notifyMatchFailure(
906 arg&: sliceOp,
907 msg: "`padding` input specified on the tosa::PadOp must be constant.");
908 }
909 llvm::SmallVector<int64_t> padPaddings =
910 llvm::to_vector(Range: paddingElems.getValues<int64_t>());
911
912 // Extract slice parameters
913 DenseElementsAttr startElems;
914 if (!matchPattern(value: sliceOp.getStart(), pattern: m_Constant(bind_value: &startElems)))
915 return rewriter.notifyMatchFailure(
916 arg&: sliceOp, msg: "start of slice must be a static ranked shape");
917 llvm::SmallVector<int64_t> sliceStarts =
918 llvm::to_vector(Range: startElems.getValues<int64_t>());
919
920 DenseElementsAttr sizeElems;
921 if (!matchPattern(value: sliceOp.getSize(), pattern: m_Constant(bind_value: &sizeElems)))
922 return rewriter.notifyMatchFailure(
923 arg&: sliceOp, msg: "size of slice must be a static ranked shape");
924 llvm::SmallVector<int64_t> sliceSizes =
925 llvm::to_vector(Range: sizeElems.getValues<int64_t>());
926
927 // Check if dynamic dimensions are sliced
928 const int64_t rank = inputTy.getRank();
929 if (llvm::any_of(Range: llvm::seq<int64_t>(Begin: 0, End: rank), P: [&](int64_t i) {
930 const bool isDimDynamic = inputTy.isDynamicDim(idx: i);
931 const bool isDimSliced =
932 (sliceStarts[i] != 0) || (sliceSizes[i] != kInferableDimSize);
933
934 return isDimDynamic && isDimSliced;
935 })) {
936 return rewriter.notifyMatchFailure(
937 arg&: sliceOp, msg: "axis that are sliced shall be statically known.");
938 }
939
940 // Update the parameters
941 llvm::SmallVector<int64_t> newSliceStarts(rank, 0);
942 llvm::SmallVector<int64_t> newPadPaddings(2 * rank, 0);
943 llvm::SmallVector<int64_t> newPadShape(rank, ShapedType::kDynamic);
944 bool updated = false;
945
946 for (int64_t i = 0; i < rank; ++i) {
947 const int64_t padLo = padPaddings[i * 2];
948 const int64_t padHi = padPaddings[i * 2 + 1];
949 const int64_t sliceStart = sliceStarts[i];
950 const int64_t sliceSize = sliceSizes[i];
951 const int64_t sliceEnd = sliceStart + sliceSize;
952
953 // If dimension is dynamic pass-through
954 if (inputTy.isDynamicDim(idx: i)) {
955 newPadPaddings[i * 2] = padLo;
956 newPadPaddings[i * 2 + 1] = padHi;
957 newSliceStarts[i] = sliceStart;
958 continue;
959 }
960
961 // Handle static dimensions
962 const int64_t dimSize = inputTy.getShape()[i];
963 const int64_t dimTotal = padLo + dimSize + padHi;
964
965 // Check slice within bounds
966 if (sliceStart < 0 || sliceEnd > dimTotal)
967 return rewriter.notifyMatchFailure(arg&: sliceOp, msg: "slice is out-of-bounds");
968
969 // Compute updated slice start parameter
970 const int64_t newSliceStart = std::max<int64_t>(a: sliceStart - padLo, b: 0);
971 newSliceStarts[i] = newSliceStart;
972 updated |= newSliceStart != sliceStart;
973
974 // Compute updated pad parameters
975 const int64_t newPadLo = std::max<int64_t>(a: padLo - sliceStart, b: 0);
976 const int64_t newPadHi =
977 std::max<int64_t>(a: sliceEnd - (padLo + dimSize), b: 0);
978 newPadPaddings[i * 2] = newPadLo;
979 newPadPaddings[i * 2 + 1] = newPadHi;
980 updated |= (newPadLo != padLo) || (newPadHi != padHi);
981
982 // Calculate new pad output shape
983 newPadShape[i] =
984 newPadPaddings[i * 2] + dimSize + newPadPaddings[i * 2 + 1];
985 }
986
987 // Check that we actually need to proceed with the rewrite
988 if (!updated)
989 return rewriter.notifyMatchFailure(
990 arg&: sliceOp, msg: "terminate condition; nothing to rewrite");
991
992 // Create a PadOp with updated padding
993 auto newPaddingsOp =
994 getTosaConstShape(rewriter, loc: sliceOp.getLoc(), shape: newPadPaddings);
995 auto newPadTy =
996 RankedTensorType::get(shape: newPadShape, elementType: inputTy.getElementType());
997 auto newPadOp = tosa::PadOp::create(builder&: rewriter, location: padOp.getLoc(), output: newPadTy,
998 input1: padOp.getInput1(), padding: newPaddingsOp,
999 pad_const: padOp.getPadConst());
1000
1001 // Update SliceOp and point to new PadOp
1002 auto newStartOp =
1003 getTosaConstShape(rewriter, loc: sliceOp.getLoc(), shape: newSliceStarts);
1004 rewriter.replaceOpWithNewOp<tosa::SliceOp>(op: sliceOp, args: sliceOp.getType(),
1005 args: newPadOp.getResult(), args&: newStartOp,
1006 args: sliceOp.getSize());
1007
1008 return success();
1009 }
1010};
1011
1012// Update size operand of tosa.slice if size has dynamic dims but corresponding
1013// output dim is static
1014struct SliceDynamicSizeCanonicalization
1015 : public OpRewritePattern<tosa::SliceOp> {
1016 using OpRewritePattern<tosa::SliceOp>::OpRewritePattern;
1017
1018 LogicalResult matchAndRewrite(tosa::SliceOp sliceOp,
1019 PatternRewriter &rewriter) const override {
1020 ShapedType resultType = cast<ShapedType>(Val: sliceOp.getType());
1021 if (!resultType.hasRank())
1022 return rewriter.notifyMatchFailure(arg&: sliceOp, msg: "output must be ranked");
1023
1024 ElementsAttr sizeElems;
1025 if (!matchPattern(value: sliceOp.getSize(), pattern: m_Constant(bind_value: &sizeElems))) {
1026 return rewriter.notifyMatchFailure(
1027 arg&: sliceOp, msg: "size of slice must be a static ranked shape");
1028 }
1029
1030 llvm::SmallVector<int64_t> sliceSizes =
1031 llvm::to_vector(Range: sizeElems.getValues<int64_t>());
1032
1033 bool replaceSliceSize{false};
1034 // if size op has kInferableDimSize indicating dynamic shape but
1035 // corresponding dim on the output is statically known, update size to match
1036 // with known output dim shape
1037 for (const auto &[index, size] : llvm::enumerate(First&: sliceSizes)) {
1038 if (size == kInferableDimSize && !resultType.isDynamicDim(idx: index)) {
1039 sliceSizes[index] = resultType.getDimSize(idx: index);
1040 replaceSliceSize = true;
1041 }
1042 }
1043
1044 if (!replaceSliceSize) {
1045 return rewriter.notifyMatchFailure(
1046 arg&: sliceOp, msg: "no dimension of size of slice is dynamic that resolves "
1047 "to static output shape");
1048 }
1049
1050 auto size_op = getTosaConstShape(rewriter, loc: sliceOp.getLoc(), shape: sliceSizes);
1051 auto newSliceOp =
1052 tosa::SliceOp::create(builder&: rewriter, location: sliceOp.getLoc(), output: sliceOp.getType(),
1053 input1: sliceOp.getInput1(), start: sliceOp.getStart(), size: size_op);
1054
1055 rewriter.replaceOp(op: sliceOp, newValues: newSliceOp.getResult());
1056 return success();
1057 }
1058};
1059
1060void SliceOp::getCanonicalizationPatterns(RewritePatternSet &results,
1061 MLIRContext *context) {
1062 results.add<ConcatSliceOptimization, PadSliceOptimization,
1063 SliceDynamicSizeCanonicalization>(arg&: context);
1064}
1065
1066struct NonNarrowingCastsOptimization : public OpRewritePattern<tosa::CastOp> {
1067 using OpRewritePattern<tosa::CastOp>::OpRewritePattern;
1068
1069 LogicalResult matchAndRewrite(tosa::CastOp castOp,
1070 PatternRewriter &rewriter) const override {
1071 const Value castInput = castOp.getInput();
1072 auto innerCastOp = castInput.getDefiningOp<tosa::CastOp>();
1073 if (!innerCastOp)
1074 return rewriter.notifyMatchFailure(arg&: castOp,
1075 msg: "input must be cast operation");
1076
1077 const Value innerCastInput = innerCastOp.getInput();
1078
1079 const ShapedType innerInputType =
1080 llvm::cast<ShapedType>(Val: innerCastInput.getType());
1081 const ShapedType innerOutputType =
1082 llvm::cast<ShapedType>(Val: innerCastOp.getType());
1083 const ShapedType outerOutputType = llvm::cast<ShapedType>(Val: castOp.getType());
1084
1085 const Type innerInputElemType = innerInputType.getElementType();
1086 const Type innerOutputElemType = innerOutputType.getElementType();
1087 const Type outerOutputElemType = outerOutputType.getElementType();
1088
1089 const SmallVector<Type, 3> types = {innerInputElemType, innerOutputElemType,
1090 outerOutputElemType};
1091
1092 if (llvm::any_of(Range: types, P: [](const Type type) {
1093 // Support a specific set of floating point types since we need to be
1094 // careful in not introducing unsupported type combinations
1095 return !(type.isInteger() ||
1096 llvm::isa<Float8E4M3FNType, Float8E5M2Type, BFloat16Type,
1097 Float16Type, Float32Type>(Val: type));
1098 }))
1099 return rewriter.notifyMatchFailure(
1100 arg&: castOp, msg: "only integer and f32, f16, bf16, f8E4M3FN, f8E5M2 types are "
1101 "supported");
1102
1103 if (llvm::isa<Float8E5M2Type>(Val: innerInputElemType) &&
1104 llvm::isa<Float8E4M3FNType>(Val: outerOutputElemType)) {
1105 return rewriter.notifyMatchFailure(
1106 arg&: castOp, msg: "avoid introducing f8E5M2 -> f8E4M3FN casts which are not "
1107 "legal in TOSA");
1108 }
1109
1110 if (llvm::isa<Float8E4M3FNType>(Val: innerInputElemType) &&
1111 llvm::isa<Float8E5M2Type>(Val: outerOutputElemType)) {
1112 return rewriter.notifyMatchFailure(
1113 arg&: castOp, msg: "avoid introducing f8E4M3FN -> f8E5M2 casts which are not "
1114 "legal in TOSA");
1115 }
1116
1117 if (llvm::isa<Float8E5M2Type, Float8E4M3FNType>(Val: innerInputElemType) &&
1118 outerOutputElemType.isInteger()) {
1119 return rewriter.notifyMatchFailure(
1120 arg&: castOp, msg: "avoid introducing fp8 -> integer casts which are not "
1121 "legal in TOSA");
1122 }
1123
1124 if (innerInputElemType.isInteger() &&
1125 llvm::isa<Float8E5M2Type, Float8E4M3FNType>(Val: outerOutputElemType)) {
1126 return rewriter.notifyMatchFailure(
1127 arg&: castOp, msg: "avoid introducing integer -> fp8 casts which are not "
1128 "legal in TOSA");
1129 }
1130
1131 if (llvm::isa<Float16Type>(Val: innerInputElemType) &&
1132 llvm::isa<BFloat16Type>(Val: outerOutputElemType)) {
1133 return rewriter.notifyMatchFailure(
1134 arg&: castOp, msg: "avoid introducing fp16 -> bf16 casts which are not "
1135 "legal in TOSA");
1136 }
1137
1138 if (llvm::isa<BFloat16Type>(Val: innerInputElemType) &&
1139 llvm::isa<Float16Type>(Val: outerOutputElemType)) {
1140 return rewriter.notifyMatchFailure(
1141 arg&: castOp, msg: "avoid introducing bf16 -> fp16 casts which are not "
1142 "legal in TOSA");
1143 }
1144
1145 const auto isIntegerOneOfWidth = [](Type type, size_t bitwidth1,
1146 size_t bitwidth2) {
1147 return type.isInteger(width: bitwidth1) || type.isInteger(width: bitwidth2);
1148 };
1149
1150 if (isIntegerOneOfWidth(innerInputElemType, 8, 16) &&
1151 outerOutputElemType.isInteger(width: 64)) {
1152 return rewriter.notifyMatchFailure(
1153 arg&: castOp, msg: "avoid introducing i8/i16 -> i64 casts which are not "
1154 "legal in TOSA");
1155 }
1156
1157 if (isIntegerOneOfWidth(innerInputElemType, 1, 64) &&
1158 !outerOutputElemType.isInteger()) {
1159 return rewriter.notifyMatchFailure(
1160 arg&: castOp, msg: "avoid introducing bool/i64 to float casts which are not "
1161 "supported in all versions of TOSA");
1162 }
1163
1164 if (!innerInputElemType.isInteger() &&
1165 isIntegerOneOfWidth(outerOutputElemType, 1, 64)) {
1166 return rewriter.notifyMatchFailure(
1167 arg&: castOp, msg: "avoid introducing float to bool/i64 casts which are not "
1168 "supported in all versions of TOSA");
1169 }
1170
1171 // Check that the cast we're considering for removal is non-narrowing
1172 if (isNarrowingCast(inType: innerInputType, outType: innerOutputType))
1173 return rewriter.notifyMatchFailure(arg&: castOp,
1174 msg: "inner cast operation is narrowing");
1175
1176 // Bail out of the canonicalization if (inner) cast(input_unsigned=false) ->
1177 // (outer) cast(input_unsigned=true)
1178 if (!innerCastOp.getInputUnsigned() && castOp.getInputUnsigned()) {
1179 return rewriter.notifyMatchFailure(
1180 arg&: castOp, msg: "avoid rewriting cast(input_unsigned=false) -> "
1181 "cast(input_unsigned=true)");
1182 }
1183
1184 rewriter.replaceOpWithNewOp<tosa::CastOp>(op: castOp, args: outerOutputType,
1185 args: innerCastInput,
1186 args: innerCastOp.getInputUnsigned());
1187
1188 return success();
1189 }
1190
1191 bool supportsNaN(const llvm::fltSemantics &semantics) const {
1192 return semantics.nonFiniteBehavior !=
1193 llvm::fltNonfiniteBehavior::FiniteOnly;
1194 }
1195
1196 bool supportsInf(const llvm::fltSemantics &semantics) const {
1197 return semantics.nonFiniteBehavior == llvm::fltNonfiniteBehavior::IEEE754;
1198 }
1199
1200 bool isNarrowingCast(const ShapedType inType,
1201 const ShapedType outType) const {
1202
1203 if (inType.getElementType().isInteger() &&
1204 outType.getElementType().isInteger()) {
1205
1206 const auto inTypeSignedness =
1207 cast<IntegerType>(Val: inType.getElementType()).getSignedness();
1208 const auto outTypeSignedness =
1209 cast<IntegerType>(Val: outType.getElementType()).getSignedness();
1210
1211 return (inTypeSignedness != outTypeSignedness ||
1212 inType.getElementTypeBitWidth() >
1213 outType.getElementTypeBitWidth());
1214 }
1215
1216 if (inType.getElementType().isFloat() &&
1217 outType.getElementType().isFloat()) {
1218
1219 FloatType inElemTy = cast<FloatType>(Val: inType.getElementType());
1220 FloatType outElemTy = cast<FloatType>(Val: outType.getElementType());
1221 llvm::fltSemantics inTypeSemantics = inElemTy.getFloatSemantics();
1222 llvm::fltSemantics outTypeSemantics = outElemTy.getFloatSemantics();
1223
1224 // If the list of supported types needs to be updated in the future, the
1225 // check down below will need to be revised, for example to account for
1226 // unsigned floating point types, or types that use negative zero as the
1227 // representation for NaN.
1228 [[maybe_unused]] const auto isSupported = [](Type elemType) {
1229 return llvm::isa<Float8E4M3FNType, Float8E5M2Type, BFloat16Type,
1230 Float16Type, Float32Type>(Val: elemType);
1231 };
1232
1233 assert(isSupported(inElemTy) &&
1234 "unsupported input element type in isNarrowingCast");
1235 assert(isSupported(outElemTy) &&
1236 "unsupported output element type in isNarrowingCast");
1237
1238 return (
1239 inTypeSemantics.maxExponent > outTypeSemantics.maxExponent ||
1240 inTypeSemantics.minExponent < outTypeSemantics.minExponent ||
1241 inTypeSemantics.precision > outTypeSemantics.precision ||
1242 (supportsNaN(semantics: inTypeSemantics) && !supportsNaN(semantics: outTypeSemantics)) ||
1243 (supportsInf(semantics: inTypeSemantics) && !supportsInf(semantics: outTypeSemantics)));
1244 }
1245
1246 // While some cases of int -> float casts can be non-narrowing, consider
1247 // them narrowing for the purposes of this optimization
1248 return true;
1249 }
1250};
1251
1252struct CancellingBlockScaledCastsOptimization
1253 : public OpRewritePattern<tosa::CastOp> {
1254 using OpRewritePattern<tosa::CastOp>::OpRewritePattern;
1255
1256 LogicalResult matchAndRewrite(tosa::CastOp castOp,
1257 PatternRewriter &rewriter) const override {
1258 const Value outerInput = castOp.getInput();
1259 auto innerCastOp = outerInput.getDefiningOp<tosa::CastOp>();
1260 if (!innerCastOp)
1261 return rewriter.notifyMatchFailure(arg&: castOp,
1262 msg: "input must be a cast operation");
1263
1264 const Value innerInput = innerCastOp.getInput();
1265 const auto innerInputTy = llvm::cast<ShapedType>(Val: innerInput.getType());
1266 const auto innerOutputTy = llvm::cast<ShapedType>(Val: innerCastOp.getType());
1267 const auto outerOutputTy = llvm::cast<ShapedType>(Val: castOp.getType());
1268
1269 if (!llvm::isa<tosa::BlockScaledType>(Val: innerInputTy.getElementType()))
1270 return rewriter.notifyMatchFailure(
1271 arg&: castOp, msg: "inner cast input must have block scaled element type");
1272
1273 if (innerInputTy != outerOutputTy)
1274 return rewriter.notifyMatchFailure(
1275 arg&: castOp, msg: "inner input type must match outer output type");
1276
1277 const Type innerOutputElemType = innerOutputTy.getElementType();
1278 const bool isLosslessCast =
1279 isa<Float32Type, BFloat16Type>(Val: innerOutputElemType);
1280 if (!isLosslessCast)
1281 return rewriter.notifyMatchFailure(
1282 arg&: castOp, msg: "avoid cancelling casts that should be lossy");
1283
1284 rewriter.replaceOp(op: castOp, newValues: innerInput);
1285
1286 return success();
1287 }
1288};
1289
1290void CastOp::getCanonicalizationPatterns(RewritePatternSet &results,
1291 MLIRContext *context) {
1292 results.add<NonNarrowingCastsOptimization,
1293 CancellingBlockScaledCastsOptimization>(arg&: context);
1294}
1295
1296struct CancellingCastToFromBlockScaledOptimization
1297 : public OpRewritePattern<tosa::CastToBlockScaledOp> {
1298 using OpRewritePattern<tosa::CastToBlockScaledOp>::OpRewritePattern;
1299
1300 LogicalResult matchAndRewrite(tosa::CastToBlockScaledOp castToBlockScaledOp,
1301 PatternRewriter &rewriter) const override {
1302 const Value castToBlockScaledInput = castToBlockScaledOp.getInputData();
1303 auto castFromBlockScaledOp =
1304 castToBlockScaledInput.getDefiningOp<tosa::CastFromBlockScaledOp>();
1305 if (!castFromBlockScaledOp)
1306 return rewriter.notifyMatchFailure(
1307 arg&: castToBlockScaledOp,
1308 msg: "input must be cast_from_block_scaled operation");
1309
1310 const Value innerData = castFromBlockScaledOp.getInputData();
1311 const Value innerScale = castFromBlockScaledOp.getInputScale();
1312 const auto innerDataTy = llvm::cast<ShapedType>(Val: innerData.getType());
1313 const auto innerScaleTy = llvm::cast<ShapedType>(Val: innerScale.getType());
1314
1315 const Value outerData = castToBlockScaledOp.getOutputData();
1316 const Value outerScale = castToBlockScaledOp.getOutputScale();
1317 const auto outerDataTy = llvm::cast<ShapedType>(Val: outerData.getType());
1318 const auto outerScaleTy = llvm::cast<ShapedType>(Val: outerScale.getType());
1319
1320 if (innerDataTy != outerDataTy || innerScaleTy != outerScaleTy) {
1321 return rewriter.notifyMatchFailure(
1322 arg&: castToBlockScaledOp,
1323 msg: "inputs types to cast_from_block_scaled operation must match output "
1324 "types to cast_to_block_scaled");
1325 }
1326
1327 if (castFromBlockScaledOp.getBlockSize() !=
1328 castToBlockScaledOp.getBlockSize()) {
1329 return rewriter.notifyMatchFailure(
1330 arg&: castToBlockScaledOp, msg: "block sizes for cast_from_block_scaled and "
1331 "cast_to_block_scaled must match");
1332 }
1333
1334 rewriter.replaceOp(op: castToBlockScaledOp, newValues: {innerData, innerScale});
1335
1336 return success();
1337 }
1338};
1339
1340void CastToBlockScaledOp::getCanonicalizationPatterns(
1341 RewritePatternSet &results, MLIRContext *context) {
1342 results.add<CancellingCastToFromBlockScaledOptimization>(arg&: context);
1343}
1344
1345struct RowGatherToGather : public OpRewritePattern<tosa::RowGatherOp> {
1346 using OpRewritePattern<tosa::RowGatherOp>::OpRewritePattern;
1347
1348 LogicalResult matchAndRewrite(tosa::RowGatherOp op,
1349 PatternRewriter &rewriter) const override {
1350 const FailureOr<int32_t> rowCount =
1351 mlir::tosa::getConstantScalarIntValue<int32_t>(val: op.getRowCount());
1352 if (failed(Result: rowCount) || rowCount.value() != 1)
1353 return failure();
1354
1355 rewriter.replaceOpWithNewOp<tosa::GatherOp>(
1356 op, args: op.getOutput().getType(), args: op.getValues(), args: op.getIndices());
1357 return success();
1358 }
1359};
1360
1361void RowGatherOp::getCanonicalizationPatterns(RewritePatternSet &results,
1362 MLIRContext *context) {
1363 results.add<RowGatherToGather>(arg&: context);
1364}
1365
1366//===----------------------------------------------------------------------===//
1367// Operator Folders.
1368//===----------------------------------------------------------------------===//
1369
1370template <typename Folder>
1371static DenseElementsAttr
1372binaryFolder(DenseElementsAttr lhs, DenseElementsAttr rhs, ShapedType returnTy,
1373 bool foldDenseValues = false) {
1374 if (!lhs || !rhs)
1375 return {};
1376
1377 if (!returnTy.hasRank() || !returnTy.hasStaticShape())
1378 return {};
1379
1380 const auto lETy = llvm::cast<ShapedType>(Val: lhs.getType()).getElementType();
1381 const auto rETy = llvm::cast<ShapedType>(Val: rhs.getType()).getElementType();
1382 if (lETy != rETy)
1383 return {};
1384
1385 if (lhs.isSplat() && rhs.isSplat()) {
1386 if (isa<FloatType>(Val: lETy)) {
1387 const APFloat l = lhs.getSplatValue<APFloat>();
1388 const APFloat r = rhs.getSplatValue<APFloat>();
1389 const auto maybeResult = Folder::fold(l, r);
1390 if (failed(maybeResult))
1391 return {};
1392 return DenseElementsAttr::get(returnTy, maybeResult.value());
1393 }
1394
1395 if (const auto lIntTy = llvm::dyn_cast<IntegerType>(Val: lETy)) {
1396 const APInt l = lhs.getSplatValue<APInt>();
1397 const APInt r = rhs.getSplatValue<APInt>();
1398 const auto maybeResult = Folder::fold(l, r, lIntTy.isUnsigned());
1399 if (failed(maybeResult))
1400 return {};
1401 return DenseElementsAttr::get(returnTy, maybeResult.value());
1402 }
1403 }
1404
1405 if (foldDenseValues) {
1406 assert(lETy.isIntOrIndex() &&
1407 "Only integer types are currently supported.");
1408 SmallVector<APInt> resultValues;
1409 for (auto [l, r] :
1410 llvm::zip(t: lhs.getValues<APInt>(), u: rhs.getValues<APInt>())) {
1411 const auto maybeResult = Folder::fold(l, r, false);
1412 if (failed(maybeResult))
1413 return {};
1414 resultValues.push_back(maybeResult.value());
1415 }
1416 return DenseElementsAttr::get(type: returnTy, values: resultValues);
1417 }
1418
1419 return {};
1420}
1421
1422template <typename Folder>
1423static DenseElementsAttr unaryFolder(DenseElementsAttr val, ShapedType returnTy,
1424 bool foldDenseValues = false) {
1425 if (!val)
1426 return {};
1427
1428 if (!returnTy.hasRank() || !returnTy.hasStaticShape())
1429 return {};
1430
1431 const auto vETy = llvm::cast<ShapedType>(Val: val.getType()).getElementType();
1432
1433 if (val.isSplat()) {
1434 if (const auto vIntTy = llvm::dyn_cast<IntegerType>(Val: vETy)) {
1435 const APInt v = val.getSplatValue<APInt>();
1436 const auto maybeResult = Folder::fold(v, vIntTy.isUnsigned());
1437 if (failed(maybeResult))
1438 return {};
1439 return DenseElementsAttr::get(returnTy, maybeResult.value());
1440 }
1441 }
1442
1443 if (foldDenseValues) {
1444 mlir::Type elemTy = val.getElementType();
1445 if (elemTy.isIntOrIndex()) {
1446 SmallVector<APInt> resultValues;
1447 for (auto const &v : val.getValues<APInt>()) {
1448 const auto maybeResult = Folder::fold(v, false);
1449 if (failed(maybeResult))
1450 return {};
1451 resultValues.push_back(maybeResult.value());
1452 }
1453 return DenseElementsAttr::get(type: returnTy, values: resultValues);
1454 }
1455 }
1456
1457 // Folding arbitrarily sized tensor operations is not supported
1458 return {};
1459}
1460
1461static FailureOr<int64_t> getSingleI64From1ElementTensor(Value v) {
1462 DenseIntElementsAttr dense{};
1463 if (!matchPattern(value: v, pattern: m_Constant(bind_value: &dense)))
1464 return failure();
1465
1466 assert(dense.isSplat());
1467 APInt a = dense.getSplatValue<APInt>();
1468 return a.getSExtValue();
1469}
1470
1471struct AddFoldAdaptor {
1472 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1473 const bool isUnsigned) {
1474 bool overflow;
1475 const APInt result =
1476 isUnsigned ? lhs.uadd_ov(RHS: rhs, Overflow&: overflow) : lhs.sadd_ov(RHS: rhs, Overflow&: overflow);
1477 if (overflow)
1478 return failure();
1479 return result;
1480 }
1481
1482 static FailureOr<APFloat> fold(const APFloat &lhs, const APFloat &rhs) {
1483 return lhs + rhs;
1484 }
1485};
1486
1487struct SubFoldAdaptor {
1488 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1489 const bool isUnsigned) {
1490 bool overflow;
1491 const APInt result =
1492 isUnsigned ? lhs.usub_ov(RHS: rhs, Overflow&: overflow) : lhs.ssub_ov(RHS: rhs, Overflow&: overflow);
1493 if (overflow)
1494 return failure();
1495 return result;
1496 }
1497
1498 static FailureOr<APFloat> fold(const APFloat &lhs, const APFloat &rhs) {
1499 return lhs - rhs;
1500 }
1501};
1502
1503struct MulFoldAdaptor {
1504 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1505 const bool isUnsigned) {
1506
1507 const unsigned originalWidth = lhs.getBitWidth();
1508
1509 // Check same type
1510 if (lhs.getBitWidth() != rhs.getBitWidth()) {
1511 return failure();
1512 }
1513
1514 // If either is `0`
1515 if (lhs == 0 || rhs == 0)
1516 return APInt::getZero(numBits: originalWidth);
1517
1518 bool overflow = false;
1519 APInt const result =
1520 isUnsigned ? lhs.umul_ov(RHS: rhs, Overflow&: overflow) : lhs.smul_ov(RHS: rhs, Overflow&: overflow);
1521
1522 if (overflow)
1523 return failure();
1524
1525 return result.trunc(width: originalWidth);
1526 }
1527
1528 static FailureOr<APFloat> fold(const APFloat &lhs, const APFloat &rhs) {
1529 return lhs * rhs;
1530 }
1531};
1532
1533static bool signsDiffer(const APInt &a, const APInt &b) {
1534 return a.isNegative() != b.isNegative();
1535}
1536
1537template <bool Ceil>
1538struct DivFoldAdaptor {
1539 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1540 bool isUnsigned) {
1541 if (lhs.getBitWidth() != rhs.getBitWidth())
1542 return failure();
1543 if (rhs.isZero())
1544 return failure();
1545
1546 if (isUnsigned) {
1547 APInt q{};
1548 APInt r{};
1549 APInt::udivrem(LHS: lhs, RHS: rhs, Quotient&: q, Remainder&: r);
1550 if (!r.isZero() && Ceil) {
1551 return q + 1;
1552 }
1553 return q;
1554 }
1555
1556 // Signed: start from trunc-toward-zero, then adjust to ceil.
1557 bool overflow{false};
1558 APInt const q = lhs.sdiv_ov(RHS: rhs, Overflow&: overflow);
1559 if (overflow)
1560 return failure();
1561 APInt const r = lhs.srem(RHS: rhs);
1562
1563 if (Ceil && !r.isZero() && !signsDiffer(a: lhs, b: rhs)) {
1564 // Same sign => exact quotient is positive; trunc is below ceil =>
1565 // increment q.
1566 return q + 1;
1567 }
1568 return q;
1569 }
1570
1571 static FailureOr<APFloat> fold(const APFloat &lhs, const APFloat &rhs) {
1572 return lhs / rhs;
1573 }
1574};
1575
1576struct ModFoldAdaptor {
1577 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1578 bool isUnsigned) {
1579 if (lhs.getBitWidth() != rhs.getBitWidth())
1580 return failure();
1581 if (lhs.isNegative() || (!rhs.isStrictlyPositive()))
1582 return failure();
1583
1584 if (isUnsigned) {
1585 return lhs.urem(RHS: rhs);
1586 }
1587
1588 return lhs.srem(RHS: rhs);
1589 }
1590
1591 static FailureOr<APFloat> fold(const APFloat &lhs, const APFloat &rhs) {
1592 auto t = lhs;
1593 auto const r = t.mod(RHS: rhs);
1594 if (llvm::APFloatBase::opStatus::opOK == r) {
1595 return t;
1596 }
1597 return failure();
1598 }
1599};
1600
1601struct MaxFoldAdaptor {
1602 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1603 bool isUnsigned) {
1604 if (lhs.getBitWidth() != rhs.getBitWidth())
1605 return failure();
1606 return lhs.getSExtValue() >= rhs.getSExtValue() ? lhs : rhs;
1607 }
1608
1609 static FailureOr<APFloat> fold(const APFloat &lhs, const APFloat &rhs) {
1610 return lhs >= rhs ? lhs : rhs;
1611 }
1612};
1613
1614struct MinFoldAdaptor {
1615 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1616 bool isUnsigned) {
1617 if (lhs.getBitWidth() != rhs.getBitWidth())
1618 return failure();
1619 return lhs.getSExtValue() <= rhs.getSExtValue() ? lhs : rhs;
1620 }
1621
1622 static FailureOr<APFloat> fold(const APFloat &lhs, const APFloat &rhs) {
1623 return lhs <= rhs ? lhs : rhs;
1624 }
1625};
1626
1627struct Exp2FoldAdaptor {
1628 static FailureOr<APInt> fold(const APInt &value, bool isUnsigned) {
1629 auto const numBits = value.getBitWidth();
1630 if (isUnsigned) {
1631 auto const zextv = value.getZExtValue();
1632 if (zextv >= numBits)
1633 return failure();
1634 return APInt::getOneBitSet(numBits, BitNo: zextv);
1635 }
1636 auto const sextv = value.getSExtValue();
1637 if (sextv < 0 || sextv >= numBits || (value.isNegative()))
1638 return failure();
1639 return APInt::getOneBitSet(numBits, BitNo: sextv);
1640 }
1641};
1642
1643// The specification requires shape div operations to have non-negative lhs and
1644// strictly positive rhs so we can only fold when these conditions are met.
1645template <bool Ceil>
1646struct ShapeDivFoldAdaptor {
1647 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1648 bool isUnsigned) {
1649 assert(!isUnsigned &&
1650 "unsigned values are not supported for shape div folders");
1651 if (lhs.isNegative() || !rhs.isStrictlyPositive())
1652 return failure();
1653 return DivFoldAdaptor<Ceil>::fold(lhs, rhs, isUnsigned);
1654 }
1655
1656 static FailureOr<APFloat> fold(const APFloat &lhs, const APFloat &rhs) {
1657 return failure();
1658 }
1659};
1660
1661struct Log2CeilFoldAdaptor {
1662 static FailureOr<APInt> fold(const APInt &value, bool isUnsigned) {
1663 if (!value.isStrictlyPositive())
1664 return failure();
1665 return APInt(/*numBits=*/value.getBitWidth(), value.ceilLogBase2());
1666 }
1667};
1668
1669struct Log2FloorFoldAdaptor {
1670 static FailureOr<APInt> fold(const APInt &value, bool isUnsigned) {
1671 if (!value.isStrictlyPositive())
1672 return failure();
1673 return APInt(/*numBits=*/value.getBitWidth(), value.logBase2());
1674 }
1675};
1676
1677struct GreaterFoldAdaptor {
1678 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1679 const bool isUnsigned) {
1680 return isUnsigned ? APInt(1, lhs.ugt(RHS: rhs)) : APInt(1, lhs.sgt(RHS: rhs));
1681 }
1682
1683 static FailureOr<APInt> fold(const APFloat &lhs, const APFloat &rhs) {
1684 return APInt(1, lhs > rhs);
1685 }
1686};
1687
1688struct GreaterEqualFoldAdaptor {
1689 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1690 const bool isUnsigned) {
1691 return isUnsigned ? APInt(1, lhs.uge(RHS: rhs)) : APInt(1, lhs.sge(RHS: rhs));
1692 }
1693
1694 static FailureOr<APInt> fold(const APFloat &lhs, const APFloat &rhs) {
1695 return APInt(1, lhs >= rhs);
1696 }
1697};
1698
1699struct EqualFoldAdaptor {
1700 static FailureOr<APInt> fold(const APInt &lhs, const APInt &rhs,
1701 const bool isUnsigned) {
1702 return APInt(1, lhs == rhs);
1703 }
1704
1705 static FailureOr<APInt> fold(const APFloat &lhs, const APFloat &rhs) {
1706 return APInt(1, lhs == rhs);
1707 }
1708};
1709
1710static bool isSplatZero(Type elemType, DenseElementsAttr val) {
1711 if (llvm::isa<FloatType>(Val: elemType))
1712 return val && val.isSplat() && val.getSplatValue<APFloat>().isZero();
1713 if (llvm::isa<IntegerType>(Val: elemType))
1714 return val && val.isSplat() && val.getSplatValue<APInt>().isZero();
1715 return false;
1716}
1717
1718static bool isSplatOne(Type elemType, DenseElementsAttr val, int64_t shift) {
1719 if (llvm::isa<FloatType>(Val: elemType))
1720 return val && val.isSplat() &&
1721 val.getSplatValue<APFloat>().isExactlyValue(V: 1.0);
1722 if (llvm::isa<IntegerType>(Val: elemType)) {
1723 const int64_t shifted = 1LL << shift;
1724 return val && val.isSplat() &&
1725 val.getSplatValue<APInt>().getSExtValue() == shifted;
1726 }
1727 return false;
1728}
1729
1730OpFoldResult AddOp::fold(FoldAdaptor adaptor) {
1731 auto lhsTy = llvm::dyn_cast<RankedTensorType>(Val: getInput1().getType());
1732 auto rhsTy = llvm::dyn_cast<RankedTensorType>(Val: getInput2().getType());
1733 auto resultTy = llvm::dyn_cast<RankedTensorType>(Val: getType());
1734 if (!lhsTy || !rhsTy || !resultTy)
1735 return {};
1736
1737 // Cannot create an ElementsAttr from non-int/float/index types
1738 if (!lhsTy.getElementType().isIntOrIndexOrFloat() ||
1739 !rhsTy.getElementType().isIntOrIndexOrFloat())
1740 return {};
1741
1742 auto resultETy = resultTy.getElementType();
1743 auto lhsAttr =
1744 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1());
1745 auto rhsAttr =
1746 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput2());
1747
1748 const bool isBroadcastable = OpTrait::util::staticallyKnownBroadcastable(
1749 shape1: lhsTy.getShape(), shape2: rhsTy.getShape());
1750 if (isBroadcastable && lhsTy == resultTy && isSplatZero(elemType: resultETy, val: rhsAttr))
1751 return getInput1();
1752 if (isBroadcastable && rhsTy == resultTy && isSplatZero(elemType: resultETy, val: lhsAttr))
1753 return getInput2();
1754
1755 if (!lhsAttr || !rhsAttr)
1756 return {};
1757
1758 return binaryFolder<AddFoldAdaptor>(lhs: lhsAttr, rhs: rhsAttr, returnTy: resultTy);
1759}
1760
1761OpFoldResult ArgMaxOp::fold(FoldAdaptor adaptor) {
1762 auto inputTy = llvm::dyn_cast<RankedTensorType>(Val: getInput().getType());
1763 auto outputTy = llvm::dyn_cast<RankedTensorType>(Val: getType());
1764 if (!inputTy || !outputTy || !inputTy.hasStaticShape() ||
1765 !outputTy.hasStaticShape())
1766 return {};
1767
1768 const Type outputElementTy = getElementTypeOrSelf(type: outputTy);
1769 if (inputTy.getDimSize(idx: getAxis()) == 1 && outputElementTy.isInteger()) {
1770 const auto outputElemIntTy = cast<IntegerType>(Val: outputElementTy);
1771 const APInt zero = APInt::getZero(numBits: outputElemIntTy.getWidth());
1772 return DenseElementsAttr::get(type: outputTy, values: zero);
1773 }
1774
1775 return {};
1776}
1777
1778OpFoldResult ArgMinOp::fold(FoldAdaptor adaptor) {
1779 auto inputTy = llvm::dyn_cast<RankedTensorType>(Val: getInput().getType());
1780 auto outputTy = llvm::dyn_cast<RankedTensorType>(Val: getType());
1781 if (!inputTy || !outputTy || !inputTy.hasStaticShape() ||
1782 !outputTy.hasStaticShape())
1783 return {};
1784
1785 const Type outputElementTy = getElementTypeOrSelf(type: outputTy);
1786 if (inputTy.getDimSize(idx: getAxis()) == 1 && outputElementTy.isInteger()) {
1787 const auto outputElemIntTy = cast<IntegerType>(Val: outputElementTy);
1788 const APInt zero = APInt::getZero(numBits: outputElemIntTy.getWidth());
1789 return DenseElementsAttr::get(type: outputTy, values: zero);
1790 }
1791
1792 return {};
1793}
1794
1795OpFoldResult IntDivOp::fold(FoldAdaptor adaptor) {
1796 auto lhsTy = llvm::dyn_cast<RankedTensorType>(Val: getInput1().getType());
1797 auto rhsTy = llvm::dyn_cast<RankedTensorType>(Val: getInput2().getType());
1798 auto resultTy = llvm::dyn_cast<RankedTensorType>(Val: getType());
1799 if (!lhsTy || !rhsTy || !resultTy)
1800 return {};
1801 if (lhsTy.getElementType() != rhsTy.getElementType())
1802 return {};
1803
1804 // IntDivOp inputs must be integer type, no need to check for quantized
1805 // type
1806 auto resultETy = resultTy.getElementType();
1807 auto lhsAttr =
1808 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1());
1809 auto rhsAttr =
1810 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput2());
1811 if (lhsAttr && lhsAttr.isSplat() && rhsAttr && rhsAttr.isSplat()) {
1812 if (llvm::isa<IntegerType>(Val: resultETy) && resultTy.hasStaticShape() &&
1813 lhsAttr.getSplatValue<APInt>().isZero() &&
1814 !rhsAttr.getSplatValue<APInt>().isZero()) {
1815 return lhsAttr.resizeSplat(newType: resultTy);
1816 }
1817 }
1818
1819 if (rhsAttr && rhsAttr.isSplat()) {
1820 const bool isBroadcastable = OpTrait::util::staticallyKnownBroadcastable(
1821 shape1: lhsTy.getShape(), shape2: rhsTy.getShape());
1822 if (isBroadcastable && lhsTy == resultTy &&
1823 llvm::isa<IntegerType>(Val: resultETy) &&
1824 rhsAttr.getSplatValue<APInt>().isOne())
1825 return getInput1();
1826 }
1827
1828 if (rhsAttr && lhsAttr && rhsAttr.isSplat() && lhsAttr.isSplat() &&
1829 llvm::isa<IntegerType>(Val: resultETy) && resultTy.hasStaticShape()) {
1830 APInt l = lhsAttr.getSplatValue<APInt>();
1831 APInt r = rhsAttr.getSplatValue<APInt>();
1832 if (!r.isZero()) {
1833 auto intTy = dyn_cast<mlir::IntegerType>(Val&: resultETy);
1834 auto const result =
1835 DivFoldAdaptor</*Ceil*/ false>::fold(lhs: l, rhs: r, isUnsigned: intTy.isUnsigned());
1836 if (failed(Result: result))
1837 return {};
1838 return DenseElementsAttr::get(type: resultTy, values: result.value());
1839 }
1840 }
1841
1842 return {};
1843}
1844
1845namespace {
1846// calculate lhs * rhs >> shift according to TOSA Spec
1847// return nullopt if result is not in range of int32_t when shift > 0
1848std::optional<APInt> mulInt(APInt lhs, APInt rhs, int32_t shift,
1849 unsigned bitwidth) {
1850 bool overflow = false;
1851 APInt result = lhs.sext(width: 64).smul_ov(RHS: rhs.sext(width: 64), Overflow&: overflow);
1852
1853 if (overflow)
1854 return std::nullopt;
1855
1856 if (shift > 0) {
1857 auto round = APInt(64, 1) << (shift - 1);
1858 result += round;
1859 result.ashrInPlace(ShiftAmt: shift);
1860 // REQUIRE(product >= minimum_s<i32_t>() && product <=
1861 // maximum_s<i32_t>())
1862 if (!(result.getSExtValue() >= INT32_MIN &&
1863 result.getSExtValue() <= INT32_MAX)) {
1864 // REQUIRE failed
1865 return std::nullopt;
1866 }
1867 }
1868
1869 return result.trunc(width: bitwidth);
1870}
1871
1872DenseElementsAttr mulBinaryFolder(DenseElementsAttr lhs, DenseElementsAttr rhs,
1873 RankedTensorType ty, int32_t shift) {
1874 // A constant result can only be built for a statically-shaped type.
1875 if (!ty.hasStaticShape())
1876 return {};
1877 if (rhs && lhs && rhs.isSplat() && lhs.isSplat()) {
1878 if (llvm::isa<IntegerType>(Val: ty.getElementType())) {
1879 APInt l = lhs.getSplatValue<APInt>();
1880 APInt r = rhs.getSplatValue<APInt>();
1881
1882 if (shift == 0) {
1883 return DenseElementsAttr::get(type: ty, values: l * r);
1884 }
1885
1886 auto bitwidth = ty.getElementType().getIntOrFloatBitWidth();
1887 const std::optional<APInt> result = mulInt(lhs: l, rhs: r, shift, bitwidth);
1888 if (!result)
1889 return {};
1890 return DenseElementsAttr::get(type: ty, values: result.value());
1891 }
1892
1893 if (llvm::isa<FloatType>(Val: ty.getElementType())) {
1894 APFloat l = lhs.getSplatValue<APFloat>();
1895 APFloat r = rhs.getSplatValue<APFloat>();
1896 APFloat result = l * r;
1897 return DenseElementsAttr::get(type: ty, values: result);
1898 }
1899 }
1900
1901 return {};
1902}
1903} // namespace
1904
1905OpFoldResult MulOp::fold(FoldAdaptor adaptor) {
1906 auto lhs = getInput1();
1907 auto rhs = getInput2();
1908 auto lhsTy = llvm::dyn_cast<RankedTensorType>(Val: lhs.getType());
1909 auto rhsTy = llvm::dyn_cast<RankedTensorType>(Val: rhs.getType());
1910 auto resultTy = llvm::dyn_cast<RankedTensorType>(Val: getType());
1911 if (!lhsTy || !rhsTy || !resultTy)
1912 return {};
1913
1914 auto resultETy = resultTy.getElementType();
1915 auto lhsAttr =
1916 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1());
1917 auto rhsAttr =
1918 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput2());
1919
1920 // Result right shift on i32_t data type only. For simplification,
1921 // synthesize a zero shift for other data type.
1922 int32_t shift = 0;
1923 if (resultETy.isInteger(width: 32)) {
1924 ElementsAttr shiftElem;
1925 if (getShift().getImpl()) {
1926 if (!matchPattern(value: getShift(), pattern: m_Constant(bind_value: &shiftElem)))
1927 // cannot be folded when the shift value is unknown.
1928 return {};
1929 shift = shiftElem.getValues<IntegerAttr>()[0].getInt();
1930 if (shift < 0 || shift > 63)
1931 return {};
1932 }
1933 }
1934
1935 if (rhsTy == resultTy && isSplatZero(elemType: resultETy, val: lhsAttr) &&
1936 resultTy.hasStaticShape())
1937 // constant values can only be resized if resulting type is static
1938 return lhsAttr.resizeSplat(newType: resultTy);
1939 if (lhsTy == resultTy && isSplatZero(elemType: resultETy, val: rhsAttr) &&
1940 resultTy.hasStaticShape())
1941 return rhsAttr.resizeSplat(newType: resultTy);
1942
1943 const bool isBroadcastable = OpTrait::util::staticallyKnownBroadcastable(
1944 shape1: lhsTy.getShape(), shape2: rhsTy.getShape());
1945 if (isBroadcastable && rhsTy == resultTy &&
1946 isSplatOne(elemType: resultETy, val: lhsAttr, shift))
1947 return rhs;
1948 if (isBroadcastable && lhsTy == resultTy &&
1949 isSplatOne(elemType: resultETy, val: rhsAttr, shift))
1950 return lhs;
1951
1952 return mulBinaryFolder(lhs: lhsAttr, rhs: rhsAttr, ty: resultTy, shift);
1953}
1954
1955OpFoldResult SubOp::fold(FoldAdaptor adaptor) {
1956 auto lhsTy = llvm::dyn_cast<RankedTensorType>(Val: getInput1().getType());
1957 auto rhsTy = llvm::dyn_cast<RankedTensorType>(Val: getInput2().getType());
1958 auto resultTy = llvm::dyn_cast<RankedTensorType>(Val: getType());
1959 if (!lhsTy || !rhsTy || !resultTy)
1960 return {};
1961
1962 // Cannot create an ElementsAttr from non-int/float/index types
1963 if (!lhsTy.getElementType().isIntOrIndexOrFloat() ||
1964 !rhsTy.getElementType().isIntOrIndexOrFloat())
1965 return {};
1966
1967 auto resultETy = resultTy.getElementType();
1968 auto lhsAttr =
1969 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1());
1970 auto rhsAttr =
1971 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput2());
1972
1973 const bool isBroadcastable = OpTrait::util::staticallyKnownBroadcastable(
1974 shape1: lhsTy.getShape(), shape2: rhsTy.getShape());
1975 if (isBroadcastable && lhsTy == resultTy && isSplatZero(elemType: resultETy, val: rhsAttr))
1976 return getInput1();
1977
1978 if (!lhsAttr || !rhsAttr)
1979 return {};
1980
1981 return binaryFolder<SubFoldAdaptor>(lhs: lhsAttr, rhs: rhsAttr, returnTy: resultTy);
1982}
1983
1984OpFoldResult GreaterOp::fold(FoldAdaptor adaptor) {
1985 auto resultTy = llvm::cast<ShapedType>(Val: getType());
1986 auto lhsAttr =
1987 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1());
1988 auto rhsAttr =
1989 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput2());
1990
1991 if (!lhsAttr || !rhsAttr)
1992 return {};
1993
1994 return binaryFolder<GreaterFoldAdaptor>(lhs: lhsAttr, rhs: rhsAttr, returnTy: resultTy);
1995}
1996
1997OpFoldResult GreaterEqualOp::fold(FoldAdaptor adaptor) {
1998 auto resultTy = llvm::cast<ShapedType>(Val: getType());
1999 auto lhsAttr =
2000 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1());
2001 auto rhsAttr =
2002 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput2());
2003
2004 if (!lhsAttr || !rhsAttr)
2005 return {};
2006
2007 return binaryFolder<GreaterEqualFoldAdaptor>(lhs: lhsAttr, rhs: rhsAttr, returnTy: resultTy);
2008}
2009
2010OpFoldResult EqualOp::fold(FoldAdaptor adaptor) {
2011 auto resultTy = llvm::cast<ShapedType>(Val: getType());
2012 auto lhsAttr =
2013 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1());
2014 auto rhsAttr =
2015 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput2());
2016 Value lhs = getInput1();
2017 Value rhs = getInput2();
2018 auto lhsTy = llvm::cast<ShapedType>(Val: lhs.getType());
2019
2020 // If we are comparing an integer value to itself it is always true. We
2021 // can not do this with float due to float values.
2022 if (llvm::isa<IntegerType>(Val: lhsTy.getElementType()) && resultTy.hasRank() &&
2023 resultTy.hasStaticShape() && lhs == rhs) {
2024 return DenseElementsAttr::get(type: resultTy, value: true);
2025 }
2026
2027 if (!lhsAttr || !rhsAttr)
2028 return {};
2029
2030 return binaryFolder<EqualFoldAdaptor>(lhs: lhsAttr, rhs: rhsAttr, returnTy: resultTy);
2031}
2032
2033OpFoldResult CastOp::fold(FoldAdaptor adaptor) {
2034 if (getInput().getType() == getType())
2035 return getInput();
2036
2037 auto operand = llvm::dyn_cast_if_present<ElementsAttr>(Val: adaptor.getInput());
2038 if (!operand)
2039 return {};
2040
2041 auto inTy = llvm::cast<ShapedType>(Val: getInput().getType());
2042 auto outTy = llvm::cast<ShapedType>(Val: getType());
2043 if (!outTy.hasRank() || !outTy.hasStaticShape())
2044 return {};
2045 auto inETy = inTy.getElementType();
2046 auto outETy = outTy.getElementType();
2047
2048 if (operand.isSplat()) {
2049 if (llvm::isa<FloatType>(Val: inETy) && llvm::isa<FloatType>(Val: outETy)) {
2050 bool overflow;
2051 auto splatVal = operand.getSplatValue<APFloat>();
2052 auto &semantics = llvm::cast<FloatType>(Val&: outETy).getFloatSemantics();
2053 splatVal.convert(ToSemantics: semantics, RM: llvm::RoundingMode::NearestTiesToEven,
2054 losesInfo: &overflow);
2055 return SplatElementsAttr::get(type: outTy, values: splatVal);
2056 }
2057
2058 if (llvm::isa<IntegerType>(Val: inETy) && llvm::isa<FloatType>(Val: outETy)) {
2059 const bool unsign = llvm::cast<IntegerType>(Val&: inETy).isUnsignedInteger() ||
2060 adaptor.getInputUnsigned();
2061 APFloat splatVal(llvm::cast<FloatType>(Val&: outETy).getFloatSemantics());
2062 splatVal.convertFromAPInt(Input: operand.getSplatValue<APInt>(), IsSigned: !unsign,
2063 RM: llvm::RoundingMode::NearestTiesToEven);
2064 return SplatElementsAttr::get(type: outTy, values: splatVal);
2065 }
2066
2067 if (llvm::isa<FloatType>(Val: inETy) && llvm::isa<IntegerType>(Val: outETy)) {
2068 auto floatVal = operand.getSplatValue<APFloat>();
2069 if (outETy.isInteger(width: 1))
2070 return SplatElementsAttr::get(type: outTy, value: !floatVal.isZero());
2071 auto unsign = llvm::cast<IntegerType>(Val&: outETy).isUnsignedInteger();
2072 auto intVal = APSInt(
2073 llvm::cast<IntegerType>(Val&: outETy).getIntOrFloatBitWidth(), unsign);
2074 bool exact;
2075 floatVal.convertToInteger(Result&: intVal, RM: llvm::RoundingMode::NearestTiesToEven,
2076 IsExact: &exact);
2077 return SplatElementsAttr::get(type: outTy, values: intVal);
2078 }
2079
2080 if (llvm::isa<IntegerType>(Val: inETy) && llvm::isa<IntegerType>(Val: outETy)) {
2081 const auto inIntType = llvm::cast<IntegerType>(Val&: inETy);
2082 const bool unsignIn =
2083 inIntType.isUnsignedInteger() || adaptor.getInputUnsigned();
2084 bool trunc =
2085 inETy.getIntOrFloatBitWidth() > outETy.getIntOrFloatBitWidth();
2086 auto intVal = operand.getSplatValue<APInt>();
2087 auto bitwidth = outETy.getIntOrFloatBitWidth();
2088
2089 // i1 types are boolean in TOSA
2090 if (outETy.isInteger(width: 1)) {
2091 intVal = APInt(bitwidth, intVal.isZero() ? 0 : 1);
2092 } else if (trunc) {
2093 intVal = intVal.trunc(width: bitwidth);
2094 } else if (unsignIn || inIntType.isInteger(width: 1)) {
2095 intVal = intVal.zext(width: bitwidth);
2096 } else {
2097 intVal = intVal.sext(width: bitwidth);
2098 }
2099
2100 return SplatElementsAttr::get(type: outTy, values: intVal);
2101 }
2102 }
2103
2104 return {};
2105}
2106
2107OpFoldResult ConstOp::fold(FoldAdaptor adaptor) { return getValuesAttr(); }
2108
2109OpFoldResult ConstShapeOp::fold(FoldAdaptor adaptor) { return getValuesAttr(); }
2110
2111#define REDUCE_FOLDER(OP) \
2112 OpFoldResult OP::fold(FoldAdaptor adaptor) { \
2113 ShapedType inputTy = llvm::cast<ShapedType>(getInput().getType()); \
2114 if (!inputTy.hasRank()) \
2115 return {}; \
2116 if (inputTy != getType()) \
2117 return {}; \
2118 if (inputTy.getRank() == 0 || inputTy.getDimSize(getAxis()) == 1) \
2119 return getInput(); \
2120 return {}; \
2121 }
2122
2123REDUCE_FOLDER(ReduceAllOp)
2124REDUCE_FOLDER(ReduceAnyOp)
2125REDUCE_FOLDER(ReduceMaxOp)
2126REDUCE_FOLDER(ReduceMinOp)
2127REDUCE_FOLDER(ReduceProductOp)
2128REDUCE_FOLDER(ReduceSumOp)
2129#undef REDUCE_FOLDER
2130
2131OpFoldResult ReshapeOp::fold(FoldAdaptor adaptor) {
2132 auto inputTy = llvm::dyn_cast<RankedTensorType>(Val: getInput1().getType());
2133 auto outputTy = llvm::dyn_cast<RankedTensorType>(Val: getType());
2134
2135 if (!inputTy || !outputTy)
2136 return {};
2137
2138 // Fold when the input and output types are the same. This is only safe
2139 // when there is at most 1 dynamic dimension. For 2 or more dynamic
2140 // dimensions, there may still be a productive reshape.
2141 if (inputTy == outputTy && inputTy.getNumDynamicDims() < 2)
2142 return getInput1();
2143
2144 // reshape(reshape(x)) -> reshape(x)
2145 if (auto reshapeOp = llvm::dyn_cast_if_present<tosa::ReshapeOp>(
2146 Val: getInput1().getDefiningOp())) {
2147 getInput1Mutable().assign(value: reshapeOp.getInput1());
2148 return getResult();
2149 }
2150
2151 // Cannot create an ElementsAttr from non-int/float/index types
2152 if (!inputTy.getElementType().isIntOrIndexOrFloat())
2153 return {};
2154
2155 // Constants must have static shape.
2156 if (!outputTy.hasStaticShape())
2157 return {};
2158
2159 // Reshaping a resource-backed constant only requires updating its type.
2160 if (auto operand = llvm::dyn_cast_if_present<DenseResourceElementsAttr>(
2161 Val: adaptor.getInput1()))
2162 return DenseResourceElementsAttr::get(type: outputTy, handle: operand.getRawHandle());
2163
2164 // reshape(const(x)) -> const(reshape-attr(x))
2165 if (auto operand =
2166 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1())) {
2167 // Okay to duplicate splat constants.
2168 if (operand.isSplat())
2169 return SplatElementsAttr::get(type: outputTy,
2170 values: operand.getSplatValue<Attribute>());
2171
2172 // Don't duplicate other constants.
2173 if (!getInput1().hasOneUse())
2174 return {};
2175
2176 llvm::SmallVector<int64_t> shapeVec;
2177 if (!tosa::getConstShapeValues(op: getShape().getDefiningOp(), result_shape&: shapeVec))
2178 return {};
2179
2180 return operand.reshape(
2181 newType: llvm::cast<ShapedType>(Val: operand.getType()).clone(shape: shapeVec));
2182 }
2183
2184 return {};
2185}
2186
2187OpFoldResult PadOp::fold(FoldAdaptor adaptor) {
2188 // If the pad is all zeros we can fold this operation away.
2189 if (adaptor.getPadding() && getInput1().getType() == getType()) {
2190 auto densePad = llvm::dyn_cast<DenseElementsAttr>(Val: adaptor.getPadding());
2191 if (densePad && densePad.isSplat() &&
2192 densePad.getSplatValue<APInt>().isZero()) {
2193 return getInput1();
2194 }
2195 }
2196
2197 return {};
2198}
2199
2200// Fold away cases where a tosa.resize operation returns a copy
2201// of the input image.
2202OpFoldResult ResizeOp::fold(FoldAdaptor adaptor) {
2203 auto scaleAttr =
2204 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getScale());
2205 auto offsetAttr =
2206 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getOffset());
2207 auto borderAttr =
2208 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getBorder());
2209 if (!scaleAttr || !offsetAttr || !borderAttr) {
2210 return {};
2211 }
2212
2213 auto scale = tosa::convertFromIntAttr(attr: scaleAttr, /* rank = */ 4);
2214 auto offset = tosa::convertFromIntAttr(attr: offsetAttr, /* rank = */ 2);
2215 auto border = tosa::convertFromIntAttr(attr: borderAttr, /* rank = */ 2);
2216 if (scale.size() != 4 || offset.size() != 2 || border.size() != 2) {
2217 return {};
2218 }
2219
2220 // Check unit scaling.
2221 if (scale[0] != scale[1] || scale[2] != scale[3]) {
2222 return {};
2223 }
2224
2225 // There should be no offset.
2226 if (offset[0] != 0 || offset[1] != 0) {
2227 return {};
2228 }
2229
2230 // There should be no border.
2231 if (border[0] != 0 || border[1] != 0) {
2232 return {};
2233 }
2234
2235 return foldToInputIfTypeMatches(typeRef: getType(), input: getInput());
2236}
2237
2238OpFoldResult ReverseOp::fold(FoldAdaptor adaptor) {
2239 auto operand = getInput1();
2240 auto operandTy = llvm::cast<ShapedType>(Val: operand.getType());
2241 auto axis = getAxis();
2242 // If the dim-length is 1, or reversing axis is unit-dim, also a no-op.
2243 // A splat of block-scaled values may still have different scales per block.
2244 const bool isSplatInput =
2245 !isa<BlockScaledType>(Val: operandTy.getElementType()) &&
2246 llvm::isa_and_nonnull<SplatElementsAttr>(Val: adaptor.getInput1());
2247 if (!operandTy.hasRank() ||
2248 (!isSplatInput && operandTy.getDimSize(idx: axis) != 1))
2249 return {};
2250 return foldToInputIfTypeMatches(typeRef: getType(), input: operand);
2251}
2252
2253OpFoldResult SliceOp::fold(FoldAdaptor adaptor) {
2254 auto inputTy = llvm::dyn_cast<RankedTensorType>(Val: getInput1().getType());
2255 auto outputTy = llvm::dyn_cast<RankedTensorType>(Val: getType());
2256
2257 if (!inputTy || !outputTy)
2258 return {};
2259
2260 if (inputTy == outputTy && inputTy.hasStaticShape())
2261 return getInput1();
2262
2263 // Check if this is a no-op slice (starts at 0 and size matches input)
2264
2265 DenseElementsAttr startElems;
2266 if (!matchPattern(value: getStart(), pattern: m_Constant(bind_value: &startElems)))
2267 return {};
2268
2269 // Check if all start values are zero
2270 bool startIsZeros =
2271 llvm::all_of(Range: startElems.getValues<APInt>(),
2272 P: [](const APInt &val) { return val.isZero(); });
2273
2274 if (startIsZeros) {
2275
2276 // Check if size matches input shape
2277 DenseElementsAttr sizeElems;
2278 if (!matchPattern(value: getSize(), pattern: m_Constant(bind_value: &sizeElems)))
2279 return {};
2280
2281 auto inputShape = inputTy.getShape();
2282 auto sizeValues = sizeElems.getValues<APInt>();
2283
2284 bool sizeMatchesInput = true;
2285 for (const auto &[i, sizeVal] : llvm::enumerate(First&: sizeValues)) {
2286 int64_t size = sizeVal.getSExtValue();
2287
2288 if (inputTy.isDynamicDim(idx: i)) {
2289 // For dynamic dimensions, check for kInferableDimSize indicating full
2290 // dimension is sliced
2291 if (size != kInferableDimSize) {
2292 sizeMatchesInput = false;
2293 break;
2294 }
2295 } else {
2296 // For static dimensions, check that size must match exactly or be
2297 // kInferableDimSize indicating full dimension is sliced
2298 if (size != kInferableDimSize && size != inputShape[i]) {
2299 sizeMatchesInput = false;
2300 break;
2301 }
2302 }
2303 }
2304
2305 if (sizeMatchesInput)
2306 return foldToInputIfTypeMatches(typeRef: getType(), input: getInput1());
2307 }
2308
2309 // The following checks require the input to be a constant
2310 if (!adaptor.getInput1())
2311 return {};
2312
2313 // Cannot create an ElementsAttr from non-int/float/index types
2314 if (!inputTy.getElementType().isIntOrIndexOrFloat() ||
2315 !outputTy.getElementType().isIntOrIndexOrFloat())
2316 return {};
2317
2318 auto operand = llvm::cast<ElementsAttr>(Val: adaptor.getInput1());
2319 if (operand.isSplat() && outputTy.hasStaticShape()) {
2320 return SplatElementsAttr::get(type: outputTy, values: operand.getSplatValue<Attribute>());
2321 }
2322
2323 if (inputTy.hasStaticShape() && outputTy.hasStaticShape() &&
2324 outputTy.getNumElements() == 1) {
2325 llvm::SmallVector<uint64_t> indices =
2326 llvm::to_vector(Range: startElems.getValues<uint64_t>());
2327 if (auto values = operand.tryGetValues<Attribute>())
2328 return SplatElementsAttr::get(type: outputTy, values: (*values)[indices]);
2329 }
2330
2331 return {};
2332}
2333
2334OpFoldResult tosa::SelectOp::fold(FoldAdaptor adaptor) {
2335 const Value pred = getPred();
2336 const Value onTrue = getOnTrue();
2337 const Value onFalse = getOnFalse();
2338
2339 const auto predTy = llvm::dyn_cast<RankedTensorType>(Val: pred.getType());
2340 const auto onTrueTy = llvm::dyn_cast<RankedTensorType>(Val: onTrue.getType());
2341 const auto onFalseTy = llvm::dyn_cast<RankedTensorType>(Val: onFalse.getType());
2342 if (!predTy || !onTrueTy || !onFalseTy)
2343 return {};
2344
2345 const Type resultTy = getType();
2346
2347 const ArrayRef<int64_t> predShape = predTy.getShape();
2348 const ArrayRef<int64_t> onTrueShape = onTrueTy.getShape();
2349
2350 if (onTrue == onFalse && onTrueTy == resultTy &&
2351 OpTrait::util::staticallyKnownBroadcastable(shape1: predShape, shape2: onTrueShape))
2352 return onTrue;
2353
2354 auto predicate =
2355 llvm::dyn_cast_if_present<DenseIntElementsAttr>(Val: adaptor.getInput1());
2356 if (!predicate)
2357 return {};
2358 if (!predicate.isSplat())
2359 return {};
2360
2361 const bool predicateValue = predicate.getSplatValue<APInt>().getBoolValue();
2362
2363 SmallVector<SmallVector<int64_t>, 3> shapes;
2364 shapes.emplace_back(Args: predShape);
2365 shapes.emplace_back(Args: onTrueShape);
2366 shapes.emplace_back(Args: onFalseTy.getShape());
2367 const bool isBroadcastable =
2368 OpTrait::util::staticallyKnownBroadcastable(shapes);
2369
2370 if (predicateValue == true && onTrueTy == resultTy && isBroadcastable)
2371 return onTrue;
2372 if (predicateValue == false && onFalseTy == resultTy && isBroadcastable)
2373 return onFalse;
2374 return {};
2375}
2376
2377static LogicalResult verifyTileIsBroadcast(tosa::TileOp tileOp) {
2378 const auto inputType =
2379 dyn_cast<RankedTensorType>(Val: tileOp.getInput1().getType());
2380 const auto outputType = dyn_cast<RankedTensorType>(Val: tileOp.getType());
2381 if (!inputType || !outputType)
2382 return failure();
2383
2384 SmallVector<int64_t> multiples;
2385 if (failed(Result: tileOp.getConstantMultiples(multiples)))
2386 return failure();
2387
2388 for (const auto [index, multiple] : llvm::enumerate(First&: multiples)) {
2389 if (multiple <= 0 && multiple != -1)
2390 return failure();
2391 if (multiple == 1)
2392 continue;
2393 if (outputType.isDynamicDim(idx: index))
2394 return failure();
2395 if (inputType.getDimSize(idx: index) != 1)
2396 return failure();
2397 }
2398
2399 return success();
2400}
2401
2402struct RemoveBroadcastTileFromBinaryElementwise
2403 : public OpRewritePattern<tosa::TileOp> {
2404 using OpRewritePattern<tosa::TileOp>::OpRewritePattern;
2405
2406 LogicalResult matchAndRewrite(tosa::TileOp tileOp,
2407 PatternRewriter &rewriter) const override {
2408 Value tileOutput = tileOp.getOutput();
2409 if (!tileOutput.hasOneUse())
2410 return rewriter.notifyMatchFailure(arg&: tileOp,
2411 msg: "tile output must have one use");
2412
2413 Operation *user = *tileOutput.user_begin();
2414 const bool isBinaryElementwise =
2415 user->getNumOperands() == 2 &&
2416 user->hasTrait<OpTrait::tosa::TosaElementwiseOperator>();
2417 if (!isBinaryElementwise && !isa<tosa::MulOp>(Val: user))
2418 return rewriter.notifyMatchFailure(
2419 arg&: tileOp, msg: "consumer must be binary broadcastable");
2420
2421 if (failed(Result: verifyTileIsBroadcast(tileOp)))
2422 return rewriter.notifyMatchFailure(
2423 arg&: tileOp, msg: "tile must only expand statically-known singleton dims");
2424
2425 Value lhsOperand = user->getOperand(idx: 0);
2426 Value rhsOperand = user->getOperand(idx: 1);
2427 Value otherOperand = lhsOperand == tileOutput ? rhsOperand : lhsOperand;
2428 Value tileInput = tileOp.getInput1();
2429
2430 const ShapedType newOtherType = cast<ShapedType>(Val: otherOperand.getType());
2431 const ShapedType newTileType = cast<ShapedType>(Val: tileInput.getType());
2432 SmallVector<int64_t> broadcastedShape;
2433 OpTrait::util::getBroadcastedShape(
2434 shape1: newOtherType.getShape(), shape2: newTileType.getShape(), resultShape&: broadcastedShape);
2435
2436 const ShapedType outputType = cast<ShapedType>(Val: user->getResultTypes()[0]);
2437 if (!llvm::equal(LRange&: broadcastedShape, RRange: outputType.getShape()))
2438 return rewriter.notifyMatchFailure(
2439 arg&: tileOp, msg: "tile output must be broadcastable to consumer operands");
2440
2441 rewriter.setInsertionPoint(user);
2442 IRMapping mapper;
2443 mapper.map(from: tileOutput, to: tileOp.getInput1());
2444 Operation *newUser = rewriter.clone(op&: *user, mapper);
2445 rewriter.replaceOp(op: user, newValues: newUser->getResults());
2446 return success();
2447 }
2448};
2449
2450void TileOp::getCanonicalizationPatterns(RewritePatternSet &results,
2451 MLIRContext *context) {
2452 results.add<RemoveBroadcastTileFromBinaryElementwise>(arg&: context);
2453}
2454
2455OpFoldResult TileOp::fold(FoldAdaptor adaptor) {
2456 if (getInput1().getType() == getType()) {
2457 if (auto multiples = llvm::dyn_cast_if_present<DenseElementsAttr>(
2458 Val: adaptor.getMultiples())) {
2459 if (multiples.isSplat() &&
2460 multiples.getSplatValue<APInt>().getSExtValue() == 1)
2461 return getInput1();
2462 if (auto int_array_attr =
2463 llvm::dyn_cast<DenseIntElementsAttr>(Val&: multiples)) {
2464 if (llvm::all_of(Range: int_array_attr.getValues<APInt>(),
2465 P: [](APInt v) { return v.getSExtValue() == 1; }))
2466 return getInput1();
2467 }
2468 }
2469 }
2470 return {};
2471}
2472
2473OpFoldResult TransposeOp::fold(FoldAdaptor adaptor) {
2474 auto resultTy = llvm::cast<ShapedType>(Val: getType());
2475
2476 // Transposing splat values just means reshaping.
2477 if (auto input =
2478 llvm::dyn_cast_if_present<DenseElementsAttr>(Val: adaptor.getInput1())) {
2479 if (input.isSplat() && resultTy.hasRank() && resultTy.hasStaticShape() &&
2480 input.getType().getElementType() == resultTy.getElementType())
2481 return input.reshape(newType: resultTy);
2482 }
2483
2484 // Transpose is not the identity transpose.
2485 const llvm::ArrayRef<int32_t> perms = getPerms();
2486
2487 if (!llvm::equal(LRange: llvm::seq<int32_t>(Begin: 0, End: perms.size()), RRange: perms))
2488 return {};
2489
2490 return foldToInputIfTypeMatches(typeRef: getType(), input: getInput1());
2491}
2492
2493OpFoldResult tosa::NegateOp::fold(FoldAdaptor adaptor) {
2494 // Element-wise negate(negate(x)) = x
2495 // iff all zero points are constant 0
2496 auto definingOp = getInput1().getDefiningOp<tosa::NegateOp>();
2497 if (!definingOp) {
2498 // defining op of input1 is not a negate, cannot fold
2499 return {};
2500 }
2501
2502 if (FailureOr<int64_t> maybeIZp = getInput1ZeroPoint();
2503 failed(Result: maybeIZp) || *maybeIZp != 0) {
2504 // input1 zero point is not constant 0, cannot fold
2505 return {};
2506 }
2507 if (FailureOr<int64_t> maybeOZp = getOutputZeroPoint();
2508 failed(Result: maybeOZp) || *maybeOZp != 0) {
2509 // output zero point is not constant 0, cannot fold
2510 return {};
2511 }
2512 if (FailureOr<int64_t> maybeIZp = definingOp.getInput1ZeroPoint();
2513 failed(Result: maybeIZp) || *maybeIZp != 0) {
2514 // definingOp's input1 zero point is not constant 0, cannot fold
2515 return {};
2516 }
2517 if (FailureOr<int64_t> maybeOZp = definingOp.getOutputZeroPoint();
2518 failed(Result: maybeOZp) || *maybeOZp != 0) {
2519 // definingOp's output zero point is not constant 0, cannot fold
2520 return {};
2521 }
2522
2523 return foldToInputIfTypeMatches(typeRef: getType(), input: definingOp.getInput1());
2524}
2525
2526OpFoldResult tosa::AbsOp::fold(FoldAdaptor adaptor) {
2527 auto input = getInput1();
2528 // Element-wise abs(abs(x)) = abs(x)
2529 if (input.getDefiningOp<tosa::AbsOp>())
2530 return foldToInputIfTypeMatches(typeRef: getType(), input);
2531
2532 return {};
2533}
2534
2535OpFoldResult tosa::ReciprocalOp::fold(FoldAdaptor adaptor) {
2536 auto input = adaptor.getInput1();
2537
2538 auto inputAttr = llvm::dyn_cast_if_present<DenseElementsAttr>(Val&: input);
2539 // Fold splat inputs only.
2540 if (!inputAttr || !inputAttr.isSplat())
2541 return {};
2542
2543 auto shapeType = llvm::cast<ShapedType>(Val: getType());
2544 if (!shapeType.hasRank() || !shapeType.hasStaticShape())
2545 return {};
2546 if (auto floatType = llvm::dyn_cast<FloatType>(Val: inputAttr.getElementType())) {
2547 auto floatVal = inputAttr.getSplatValue<APFloat>();
2548 return DenseElementsAttr::get(type: shapeType,
2549 values: ReciprocalOp::calcOneElement(operand: floatVal));
2550 }
2551
2552 return {};
2553}
2554
2555template <typename Op, typename OpFoldAdaptor>
2556OpFoldResult unaryShapeFold(Op *op) {
2557 auto input1ConstShape =
2558 dyn_cast<tosa::ConstShapeOp>(op->getInput().getDefiningOp());
2559 if (!input1ConstShape)
2560 return {};
2561
2562 const auto input1Attr = cast<DenseElementsAttr>(input1ConstShape.getValues());
2563
2564 return unaryFolder<OpFoldAdaptor>(input1Attr, input1Attr.getType(),
2565 /*foldDenseValues=*/true);
2566}
2567
2568template <typename Op, typename OpFoldAdaptor>
2569OpFoldResult binaryFold(Op *op) {
2570 auto input1ConstShape =
2571 dyn_cast<tosa::ConstShapeOp>(op->getInput1().getDefiningOp());
2572 auto input2ConstShape =
2573 dyn_cast<tosa::ConstShapeOp>(op->getInput2().getDefiningOp());
2574 if (!input1ConstShape || !input2ConstShape)
2575 return {};
2576
2577 const auto input1Attr = cast<DenseElementsAttr>(input1ConstShape.getValues());
2578 const auto input2Attr = cast<DenseElementsAttr>(input2ConstShape.getValues());
2579
2580 return binaryFolder<OpFoldAdaptor>(input1Attr, input2Attr,
2581 input1Attr.getType(),
2582 /*foldDenseValues=*/true);
2583}
2584
2585OpFoldResult tosa::DimOp::fold(FoldAdaptor adaptor) {
2586 const auto inputTy = llvm::dyn_cast<ShapedType>(Val: getInput1().getType());
2587 if (!inputTy || !inputTy.hasRank())
2588 return {};
2589 const int32_t axis = getAxis();
2590 const int64_t dimSize = inputTy.getDimSize(idx: axis);
2591 if (ShapedType::isDynamic(dValue: dimSize))
2592 return {};
2593
2594 OpBuilder builder(getContext());
2595 const auto resultAttrTy =
2596 RankedTensorType::get(/*rank=*/shape: 1, elementType: builder.getIndexType());
2597 return DenseElementsAttr::get(type: resultAttrTy, value: dimSize);
2598}
2599
2600OpFoldResult concatShapeFold(tosa::ConcatShapeOp *op) {
2601 auto const inputs = op->getInput();
2602
2603 if (inputs.empty())
2604 return {};
2605
2606 SmallVector<APInt> concatDims;
2607 concatDims.reserve(/*max elem*/ N: 64);
2608 for (auto const &v : inputs) {
2609 auto vConstShape = dyn_cast<tosa::ConstShapeOp>(Val: v.getDefiningOp());
2610 if (!vConstShape)
2611 return {};
2612
2613 const auto vAttr = cast<DenseElementsAttr>(Val: vConstShape.getValues());
2614 assert(vAttr);
2615
2616 auto const vAttrVals = vAttr.getValues<APInt>();
2617 for (auto const &v : vAttrVals) {
2618 concatDims.push_back(Elt: v);
2619 }
2620 }
2621
2622 auto *ctx = op->getContext();
2623 assert(ctx != nullptr && "ctx is nullptr");
2624 auto const rankedTy = RankedTensorType::get(
2625 shape: {static_cast<int64_t>(concatDims.size())}, elementType: IndexType::get(context: ctx));
2626
2627 return DenseElementsAttr::get(type: rankedTy, values: concatDims);
2628}
2629
2630OpFoldResult sliceShapeFold(tosa::SliceShapeOp *op) {
2631 auto const input1 = op->getInput();
2632 auto const input2 = op->getStart();
2633 auto const input3 = op->getSize();
2634
2635 auto input1ConstShape = dyn_cast<tosa::ConstShapeOp>(Val: input1.getDefiningOp());
2636
2637 if (!input1ConstShape)
2638 return {};
2639
2640 auto const input1Attr = cast<DenseElementsAttr>(Val: input1ConstShape.getValues());
2641 if (!input1Attr)
2642 return {};
2643
2644 auto const input1Vals = input1Attr.getValues<APInt>();
2645 auto const totalInput1 = input1Vals.size();
2646
2647 auto const start = getSingleI64From1ElementTensor(v: input2);
2648 auto const size = getSingleI64From1ElementTensor(v: input3);
2649
2650 if (failed(Result: start) || failed(Result: size))
2651 return {};
2652
2653 auto const startV = static_cast<int32_t>(start.value());
2654 auto const sizeV = static_cast<int32_t>(size.value());
2655
2656 if ((sizeV <= 0) || (startV < 0) ||
2657 (static_cast<size_t>(startV + sizeV) > totalInput1))
2658 return {};
2659
2660 SmallVector<APInt> sliceOfInput;
2661 sliceOfInput.reserve(N: totalInput1);
2662
2663 for (auto i = startV; i < (startV + sizeV); i++) {
2664 sliceOfInput.push_back(Elt: input1Vals[i]);
2665 }
2666
2667 auto *ctx = op->getContext();
2668 assert(ctx != nullptr && "ctx is nullptr");
2669
2670 auto const rankedTy = RankedTensorType::get(
2671 shape: {static_cast<int64_t>(sliceOfInput.size())}, elementType: IndexType::get(context: ctx));
2672
2673 return DenseElementsAttr::get(type: rankedTy, values: sliceOfInput);
2674}
2675
2676OpFoldResult tosa::AddShapeOp::fold(FoldAdaptor adaptor) {
2677 return binaryFold<AddShapeOp, AddFoldAdaptor>(op: this);
2678}
2679
2680OpFoldResult tosa::SubShapeOp::fold(FoldAdaptor adaptor) {
2681 return binaryFold<SubShapeOp, SubFoldAdaptor>(op: this);
2682}
2683
2684OpFoldResult tosa::MulShapeOp::fold(FoldAdaptor adaptor) {
2685 return binaryFold<MulShapeOp, MulFoldAdaptor>(op: this);
2686}
2687
2688OpFoldResult tosa::DivCeilShapeOp::fold(FoldAdaptor adaptor) {
2689 return binaryFold<DivCeilShapeOp, ShapeDivFoldAdaptor</*Ceil*/ true>>(op: this);
2690}
2691
2692OpFoldResult tosa::DivFloorShapeOp::fold(FoldAdaptor adaptor) {
2693 return binaryFold<DivFloorShapeOp, ShapeDivFoldAdaptor</*Ceil*/ false>>(op: this);
2694}
2695
2696OpFoldResult tosa::ModShapeOp::fold(FoldAdaptor adaptor) {
2697 return binaryFold<ModShapeOp, ModFoldAdaptor>(op: this);
2698}
2699
2700OpFoldResult tosa::MaxShapeOp::fold(FoldAdaptor adaptor) {
2701 return binaryFold<MaxShapeOp, MaxFoldAdaptor>(op: this);
2702}
2703
2704OpFoldResult tosa::MinShapeOp::fold(FoldAdaptor adaptor) {
2705 return binaryFold<MinShapeOp, MinFoldAdaptor>(op: this);
2706}
2707
2708OpFoldResult tosa::Exp2ShapeOp::fold(FoldAdaptor adaptor) {
2709 return unaryShapeFold<Exp2ShapeOp, Exp2FoldAdaptor>(op: this);
2710}
2711
2712OpFoldResult tosa::Log2CeilShapeOp::fold(FoldAdaptor adaptor) {
2713 return unaryShapeFold<Log2CeilShapeOp, Log2CeilFoldAdaptor>(op: this);
2714}
2715
2716OpFoldResult tosa::Log2FloorShapeOp::fold(FoldAdaptor adaptor) {
2717 return unaryShapeFold<Log2FloorShapeOp, Log2FloorFoldAdaptor>(op: this);
2718}
2719
2720OpFoldResult tosa::ConcatShapeOp::fold(FoldAdaptor adaptor) {
2721 return concatShapeFold(op: this);
2722}
2723
2724OpFoldResult tosa::SliceShapeOp::fold(FoldAdaptor adaptor) {
2725 return sliceShapeFold(op: this);
2726}
2727